mirror of
https://github.com/sqlite/sqlite.git
synced 2025-07-29 08:01:23 +03:00
OPFS VFS: add the opfs-unlock-asap=1 URI flag which tells the VFS to release implicit locks ASAP instead of during VFS idle time. This improves concurrency notably in the test app but brings a significant performance penalty in speedtest1 (roughly 4x slowdown). This is not the final word in OPFS concurrency, but gets us a step further.
FossilOrigin-Name: 9542f9ce9e023b489e2d93661f719fb0751c1e28f72fded9d3c2156d5777e7b1
This commit is contained in:
@ -201,9 +201,8 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
|
|||||||
*/
|
*/
|
||||||
dbCtorHelper.normalizeArgs = function(filename=':memory:',flags = 'c',vfs = null){
|
dbCtorHelper.normalizeArgs = function(filename=':memory:',flags = 'c',vfs = null){
|
||||||
const arg = {};
|
const arg = {};
|
||||||
if(1===arguments.length && 'object'===typeof arguments[0]){
|
if(1===arguments.length && arguments[0] && 'object'===typeof arguments[0]){
|
||||||
const x = arguments[0];
|
Object.assign(arg, arguments[0]);
|
||||||
Object.keys(x).forEach((k)=>arg[k] = x[k]);
|
|
||||||
if(undefined===arg.flags) arg.flags = 'c';
|
if(undefined===arg.flags) arg.flags = 'c';
|
||||||
if(undefined===arg.vfs) arg.vfs = null;
|
if(undefined===arg.vfs) arg.vfs = null;
|
||||||
if(undefined===arg.filename) arg.filename = ':memory:';
|
if(undefined===arg.filename) arg.filename = ':memory:';
|
||||||
|
@ -348,12 +348,32 @@ const installOpfsVfs = function callee(options){
|
|||||||
'SQLITE_NOTFOUND',
|
'SQLITE_NOTFOUND',
|
||||||
'SQLITE_OPEN_CREATE',
|
'SQLITE_OPEN_CREATE',
|
||||||
'SQLITE_OPEN_DELETEONCLOSE',
|
'SQLITE_OPEN_DELETEONCLOSE',
|
||||||
|
'SQLITE_OPEN_MAIN_DB',
|
||||||
'SQLITE_OPEN_READONLY'
|
'SQLITE_OPEN_READONLY'
|
||||||
].forEach((k)=>{
|
].forEach((k)=>{
|
||||||
if(undefined === (state.sq3Codes[k] = capi[k])){
|
if(undefined === (state.sq3Codes[k] = capi[k])){
|
||||||
toss("Maintenance required: not found:",k);
|
toss("Maintenance required: not found:",k);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
state.opfsFlags = Object.assign(Object.create(null),{
|
||||||
|
/**
|
||||||
|
Flag for use with xOpen(). "opfs-unlock-asap=1" enables
|
||||||
|
this. See defaultUnlockAsap, below.
|
||||||
|
*/
|
||||||
|
OPFS_UNLOCK_ASAP: 0x01,
|
||||||
|
/**
|
||||||
|
If true, any async routine which implicitly acquires a sync
|
||||||
|
access handle (i.e. an OPFS lock) will release that locks at
|
||||||
|
the end of the call which acquires it. If false, such
|
||||||
|
"autolocks" are not released until the VFS is idle for some
|
||||||
|
brief amount of time.
|
||||||
|
|
||||||
|
The benefit of enabling this is much higher concurrency. The
|
||||||
|
down-side is much-reduced performance (as much as a 4x decrease
|
||||||
|
in speedtest1).
|
||||||
|
*/
|
||||||
|
defaultUnlockAsap: false
|
||||||
|
});
|
||||||
|
|
||||||
/**
|
/**
|
||||||
Runs the given operation (by name) in the async worker
|
Runs the given operation (by name) in the async worker
|
||||||
@ -844,9 +864,15 @@ const installOpfsVfs = function callee(options){
|
|||||||
//xSleep is optionally defined below
|
//xSleep is optionally defined below
|
||||||
xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){
|
xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){
|
||||||
mTimeStart('xOpen');
|
mTimeStart('xOpen');
|
||||||
|
let opfsFlags = 0;
|
||||||
if(0===zName){
|
if(0===zName){
|
||||||
zName = randomFilename();
|
zName = randomFilename();
|
||||||
}else if('number'===typeof zName){
|
}else if('number'===typeof zName){
|
||||||
|
if(capi.sqlite3_uri_boolean(zName, "opfs-unlock-asap", 0)){
|
||||||
|
/* -----------------------^^^^^ MUST pass the untranslated
|
||||||
|
C-string here. */
|
||||||
|
opfsFlags |= state.opfsFlags.OPFS_UNLOCK_ASAP;
|
||||||
|
}
|
||||||
zName = wasm.cstringToJs(zName);
|
zName = wasm.cstringToJs(zName);
|
||||||
}
|
}
|
||||||
const fh = Object.create(null);
|
const fh = Object.create(null);
|
||||||
@ -854,7 +880,7 @@ const installOpfsVfs = function callee(options){
|
|||||||
fh.filename = zName;
|
fh.filename = zName;
|
||||||
fh.sab = new SharedArrayBuffer(state.fileBufferSize);
|
fh.sab = new SharedArrayBuffer(state.fileBufferSize);
|
||||||
fh.flags = flags;
|
fh.flags = flags;
|
||||||
const rc = opRun('xOpen', pFile, zName, flags);
|
const rc = opRun('xOpen', pFile, zName, flags, opfsFlags);
|
||||||
if(!rc){
|
if(!rc){
|
||||||
/* Recall that sqlite3_vfs::xClose() will be called, even on
|
/* Recall that sqlite3_vfs::xClose() will be called, even on
|
||||||
error, unless pFile->pMethods is NULL. */
|
error, unless pFile->pMethods is NULL. */
|
||||||
@ -1145,6 +1171,9 @@ const installOpfsVfs = function callee(options){
|
|||||||
|
|
||||||
//TODO to support fiddle and worker1 db upload:
|
//TODO to support fiddle and worker1 db upload:
|
||||||
//opfsUtil.createFile = function(absName, content=undefined){...}
|
//opfsUtil.createFile = function(absName, content=undefined){...}
|
||||||
|
//We have sqlite3.wasm.sqlite3_wasm_vfs_create_file() for this
|
||||||
|
//purpose but its interface and name are still under
|
||||||
|
//consideration.
|
||||||
|
|
||||||
if(sqlite3.oo1){
|
if(sqlite3.oo1){
|
||||||
opfsUtil.OpfsDb = function(...args){
|
opfsUtil.OpfsDb = function(...args){
|
||||||
|
@ -105,7 +105,7 @@ metrics.dump = ()=>{
|
|||||||
*/
|
*/
|
||||||
const __openFiles = Object.create(null);
|
const __openFiles = Object.create(null);
|
||||||
/**
|
/**
|
||||||
__autoLocks is a Set of sqlite3_file pointers (integers) which were
|
__implicitLocks is a Set of sqlite3_file pointers (integers) which were
|
||||||
"auto-locked". i.e. those for which we obtained a sync access
|
"auto-locked". i.e. those for which we obtained a sync access
|
||||||
handle without an explicit xLock() call. Such locks will be
|
handle without an explicit xLock() call. Such locks will be
|
||||||
released during db connection idle time, whereas a sync access
|
released during db connection idle time, whereas a sync access
|
||||||
@ -117,7 +117,7 @@ const __openFiles = Object.create(null);
|
|||||||
penalty: speedtest1 benchmarks take up to 4x as long. By delaying
|
penalty: speedtest1 benchmarks take up to 4x as long. By delaying
|
||||||
the lock release until idle time, the hit is negligible.
|
the lock release until idle time, the hit is negligible.
|
||||||
*/
|
*/
|
||||||
const __autoLocks = new Set();
|
const __implicitLocks = new Set();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
Expects an OPFS file path. It gets resolved, such that ".."
|
Expects an OPFS file path. It gets resolved, such that ".."
|
||||||
@ -166,7 +166,7 @@ const closeSyncHandle = async (fh)=>{
|
|||||||
const h = fh.syncHandle;
|
const h = fh.syncHandle;
|
||||||
delete fh.syncHandle;
|
delete fh.syncHandle;
|
||||||
delete fh.xLock;
|
delete fh.xLock;
|
||||||
__autoLocks.delete(fh.fid);
|
__implicitLocks.delete(fh.fid);
|
||||||
return h.close();
|
return h.close();
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@ -190,10 +190,10 @@ const closeSyncHandleNoThrow = async (fh)=>{
|
|||||||
};
|
};
|
||||||
|
|
||||||
/* Release all auto-locks. */
|
/* Release all auto-locks. */
|
||||||
const closeAutoLocks = async ()=>{
|
const releaseImplicitLocks = async ()=>{
|
||||||
if(__autoLocks.size){
|
if(__implicitLocks.size){
|
||||||
/* Release all auto-locks. */
|
/* Release all auto-locks. */
|
||||||
for(const fid of __autoLocks){
|
for(const fid of __implicitLocks){
|
||||||
const fh = __openFiles[fid];
|
const fh = __openFiles[fid];
|
||||||
await closeSyncHandleNoThrow(fh);
|
await closeSyncHandleNoThrow(fh);
|
||||||
log("Auto-unlocked",fid,fh.filenameAbs);
|
log("Auto-unlocked",fid,fh.filenameAbs);
|
||||||
@ -201,6 +201,20 @@ const closeAutoLocks = async ()=>{
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
An experiment in improving concurrency by freeing up implicit locks
|
||||||
|
sooner. This is known to impact performance dramatically but it has
|
||||||
|
also shown to improve concurrency considerably.
|
||||||
|
|
||||||
|
If fh.releaseImplicitLocks is truthy and fh is in __implicitLocks,
|
||||||
|
this routine returns closeSyncHandleNoThrow(), else it is a no-op.
|
||||||
|
*/
|
||||||
|
const releaseImplicitLock = async (fh)=>{
|
||||||
|
if(fh.releaseImplicitLocks && __implicitLocks.has(fh.fid)){
|
||||||
|
return closeSyncHandleNoThrow(fh);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
An error class specifically for use with getSyncHandle(), the goal
|
An error class specifically for use with getSyncHandle(), the goal
|
||||||
of which is to eventually be able to distinguish unambiguously
|
of which is to eventually be able to distinguish unambiguously
|
||||||
@ -246,7 +260,7 @@ GetSyncHandleError.convertRc = (e,rc)=>{
|
|||||||
still fails at that point it will give up and propagate the
|
still fails at that point it will give up and propagate the
|
||||||
exception.
|
exception.
|
||||||
*/
|
*/
|
||||||
const getSyncHandle = async (fh)=>{
|
const getSyncHandle = async (fh,opName)=>{
|
||||||
if(!fh.syncHandle){
|
if(!fh.syncHandle){
|
||||||
const t = performance.now();
|
const t = performance.now();
|
||||||
log("Acquiring sync handle for",fh.filenameAbs);
|
log("Acquiring sync handle for",fh.filenameAbs);
|
||||||
@ -262,20 +276,21 @@ const getSyncHandle = async (fh)=>{
|
|||||||
}catch(e){
|
}catch(e){
|
||||||
if(i === maxTries){
|
if(i === maxTries){
|
||||||
throw new GetSyncHandleError(
|
throw new GetSyncHandleError(
|
||||||
e, "Error getting sync handle.",maxTries,
|
e, "Error getting sync handle for",opName+"().",maxTries,
|
||||||
"attempts failed.",fh.filenameAbs
|
"attempts failed.",fh.filenameAbs
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
warn("Error getting sync handle. Waiting",ms,
|
warn("Error getting sync handle for",opName+"(). Waiting",ms,
|
||||||
"ms and trying again.",fh.filenameAbs,e);
|
"ms and trying again.",fh.filenameAbs,e);
|
||||||
await closeAutoLocks();
|
//await releaseImplicitLocks();
|
||||||
Atomics.wait(state.sabOPView, state.opIds.retry, 0, ms);
|
Atomics.wait(state.sabOPView, state.opIds.retry, 0, ms);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
log("Got sync handle for",fh.filenameAbs,'in',performance.now() - t,'ms');
|
log("Got",opName+"() sync handle for",fh.filenameAbs,
|
||||||
|
'in',performance.now() - t,'ms');
|
||||||
if(!fh.xLock){
|
if(!fh.xLock){
|
||||||
__autoLocks.add(fh.fid);
|
__implicitLocks.add(fh.fid);
|
||||||
log("Auto-locked",fh.fid,fh.filenameAbs);
|
log("Auto-locked for",opName+"()",fh.fid,fh.filenameAbs);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return fh.syncHandle;
|
return fh.syncHandle;
|
||||||
@ -409,7 +424,7 @@ const vfsAsyncImpls = {
|
|||||||
xClose: async function(fid/*sqlite3_file pointer*/){
|
xClose: async function(fid/*sqlite3_file pointer*/){
|
||||||
const opName = 'xClose';
|
const opName = 'xClose';
|
||||||
mTimeStart(opName);
|
mTimeStart(opName);
|
||||||
__autoLocks.delete(fid);
|
__implicitLocks.delete(fid);
|
||||||
const fh = __openFiles[fid];
|
const fh = __openFiles[fid];
|
||||||
let rc = 0;
|
let rc = 0;
|
||||||
wTimeStart(opName);
|
wTimeStart(opName);
|
||||||
@ -474,13 +489,14 @@ const vfsAsyncImpls = {
|
|||||||
wTimeStart('xFileSize');
|
wTimeStart('xFileSize');
|
||||||
try{
|
try{
|
||||||
affirmLocked('xFileSize',fh);
|
affirmLocked('xFileSize',fh);
|
||||||
const sz = await (await getSyncHandle(fh)).getSize();
|
const sz = await (await getSyncHandle(fh,'xFileSize')).getSize();
|
||||||
state.s11n.serialize(Number(sz));
|
state.s11n.serialize(Number(sz));
|
||||||
rc = 0;
|
rc = 0;
|
||||||
}catch(e){
|
}catch(e){
|
||||||
state.s11n.storeException(2,e);
|
state.s11n.storeException(2,e);
|
||||||
rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR);
|
rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR);
|
||||||
}
|
}
|
||||||
|
await releaseImplicitLock(fh);
|
||||||
wTimeEnd();
|
wTimeEnd();
|
||||||
storeAndNotify('xFileSize', rc);
|
storeAndNotify('xFileSize', rc);
|
||||||
mTimeEnd();
|
mTimeEnd();
|
||||||
@ -495,8 +511,8 @@ const vfsAsyncImpls = {
|
|||||||
if( !fh.syncHandle ){
|
if( !fh.syncHandle ){
|
||||||
wTimeStart('xLock');
|
wTimeStart('xLock');
|
||||||
try {
|
try {
|
||||||
await getSyncHandle(fh);
|
await getSyncHandle(fh,'xLock');
|
||||||
__autoLocks.delete(fid);
|
__implicitLocks.delete(fid);
|
||||||
}catch(e){
|
}catch(e){
|
||||||
state.s11n.storeException(1,e);
|
state.s11n.storeException(1,e);
|
||||||
rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_LOCK);
|
rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_LOCK);
|
||||||
@ -508,10 +524,10 @@ const vfsAsyncImpls = {
|
|||||||
mTimeEnd();
|
mTimeEnd();
|
||||||
},
|
},
|
||||||
xOpen: async function(fid/*sqlite3_file pointer*/, filename,
|
xOpen: async function(fid/*sqlite3_file pointer*/, filename,
|
||||||
flags/*SQLITE_OPEN_...*/){
|
flags/*SQLITE_OPEN_...*/,
|
||||||
|
opfsFlags/*OPFS_...*/){
|
||||||
const opName = 'xOpen';
|
const opName = 'xOpen';
|
||||||
mTimeStart(opName);
|
mTimeStart(opName);
|
||||||
const deleteOnClose = (state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags);
|
|
||||||
const create = (state.sq3Codes.SQLITE_OPEN_CREATE & flags);
|
const create = (state.sq3Codes.SQLITE_OPEN_CREATE & flags);
|
||||||
wTimeStart('xOpen');
|
wTimeStart('xOpen');
|
||||||
try{
|
try{
|
||||||
@ -526,14 +542,8 @@ const vfsAsyncImpls = {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const hFile = await hDir.getFileHandle(filenamePart, {create});
|
const hFile = await hDir.getFileHandle(filenamePart, {create});
|
||||||
/**
|
|
||||||
wa-sqlite, at this point, grabs a SyncAccessHandle and
|
|
||||||
assigns it to the syncHandle prop of the file state
|
|
||||||
object, but only for certain cases and it's unclear why it
|
|
||||||
places that limitation on it.
|
|
||||||
*/
|
|
||||||
wTimeEnd();
|
wTimeEnd();
|
||||||
__openFiles[fid] = Object.assign(Object.create(null),{
|
const fh = Object.assign(Object.create(null),{
|
||||||
fid: fid,
|
fid: fid,
|
||||||
filenameAbs: filename,
|
filenameAbs: filename,
|
||||||
filenamePart: filenamePart,
|
filenamePart: filenamePart,
|
||||||
@ -542,8 +552,26 @@ const vfsAsyncImpls = {
|
|||||||
sabView: state.sabFileBufView,
|
sabView: state.sabFileBufView,
|
||||||
readOnly: create
|
readOnly: create
|
||||||
? false : (state.sq3Codes.SQLITE_OPEN_READONLY & flags),
|
? false : (state.sq3Codes.SQLITE_OPEN_READONLY & flags),
|
||||||
deleteOnClose: deleteOnClose
|
deleteOnClose: !!(state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags)
|
||||||
});
|
});
|
||||||
|
fh.releaseImplicitLocks =
|
||||||
|
(opfsFlags & state.opfsFlags.OPFS_UNLOCK_ASAP)
|
||||||
|
|| state.opfsFlags.defaultUnlockAsap;
|
||||||
|
if(0 /* this block is modelled after something wa-sqlite
|
||||||
|
does but it leads to immediate contention on journal files. */
|
||||||
|
&& (0===(flags & state.sq3Codes.SQLITE_OPEN_MAIN_DB))){
|
||||||
|
/* sqlite does not lock these files, so go ahead and grab an OPFS
|
||||||
|
lock.
|
||||||
|
|
||||||
|
https://www.sqlite.org/uri.html
|
||||||
|
*/
|
||||||
|
fh.xLock = "xOpen"/* Truthy value to keep entry from getting
|
||||||
|
flagged as auto-locked. String value so
|
||||||
|
that we can easily distinguish is later
|
||||||
|
if needed. */;
|
||||||
|
await getSyncHandle(fh,'xOpen');
|
||||||
|
}
|
||||||
|
__openFiles[fid] = fh;
|
||||||
storeAndNotify(opName, 0);
|
storeAndNotify(opName, 0);
|
||||||
}catch(e){
|
}catch(e){
|
||||||
wTimeEnd();
|
wTimeEnd();
|
||||||
@ -560,7 +588,7 @@ const vfsAsyncImpls = {
|
|||||||
try{
|
try{
|
||||||
affirmLocked('xRead',fh);
|
affirmLocked('xRead',fh);
|
||||||
wTimeStart('xRead');
|
wTimeStart('xRead');
|
||||||
nRead = (await getSyncHandle(fh)).read(
|
nRead = (await getSyncHandle(fh,'xRead')).read(
|
||||||
fh.sabView.subarray(0, n),
|
fh.sabView.subarray(0, n),
|
||||||
{at: Number(offset64)}
|
{at: Number(offset64)}
|
||||||
);
|
);
|
||||||
@ -575,6 +603,7 @@ const vfsAsyncImpls = {
|
|||||||
state.s11n.storeException(1,e);
|
state.s11n.storeException(1,e);
|
||||||
rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_READ);
|
rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_READ);
|
||||||
}
|
}
|
||||||
|
await releaseImplicitLock(fh);
|
||||||
storeAndNotify('xRead',rc);
|
storeAndNotify('xRead',rc);
|
||||||
mTimeEnd();
|
mTimeEnd();
|
||||||
},
|
},
|
||||||
@ -603,12 +632,13 @@ const vfsAsyncImpls = {
|
|||||||
try{
|
try{
|
||||||
affirmLocked('xTruncate',fh);
|
affirmLocked('xTruncate',fh);
|
||||||
affirmNotRO('xTruncate', fh);
|
affirmNotRO('xTruncate', fh);
|
||||||
await (await getSyncHandle(fh)).truncate(size);
|
await (await getSyncHandle(fh,'xTruncate')).truncate(size);
|
||||||
}catch(e){
|
}catch(e){
|
||||||
error("xTruncate():",e,fh);
|
error("xTruncate():",e,fh);
|
||||||
state.s11n.storeException(2,e);
|
state.s11n.storeException(2,e);
|
||||||
rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_TRUNCATE);
|
rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_TRUNCATE);
|
||||||
}
|
}
|
||||||
|
await releaseImplicitLock(fh);
|
||||||
wTimeEnd();
|
wTimeEnd();
|
||||||
storeAndNotify('xTruncate',rc);
|
storeAndNotify('xTruncate',rc);
|
||||||
mTimeEnd();
|
mTimeEnd();
|
||||||
@ -640,7 +670,7 @@ const vfsAsyncImpls = {
|
|||||||
affirmLocked('xWrite',fh);
|
affirmLocked('xWrite',fh);
|
||||||
affirmNotRO('xWrite', fh);
|
affirmNotRO('xWrite', fh);
|
||||||
rc = (
|
rc = (
|
||||||
n === (await getSyncHandle(fh))
|
n === (await getSyncHandle(fh,'xWrite'))
|
||||||
.write(fh.sabView.subarray(0, n),
|
.write(fh.sabView.subarray(0, n),
|
||||||
{at: Number(offset64)})
|
{at: Number(offset64)})
|
||||||
) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE;
|
) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE;
|
||||||
@ -649,6 +679,7 @@ const vfsAsyncImpls = {
|
|||||||
state.s11n.storeException(1,e);
|
state.s11n.storeException(1,e);
|
||||||
rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_WRITE);
|
rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_WRITE);
|
||||||
}
|
}
|
||||||
|
await releaseImplicitLock(fh);
|
||||||
wTimeEnd();
|
wTimeEnd();
|
||||||
storeAndNotify('xWrite',rc);
|
storeAndNotify('xWrite',rc);
|
||||||
mTimeEnd();
|
mTimeEnd();
|
||||||
@ -777,13 +808,13 @@ const waitLoop = async function f(){
|
|||||||
to do other things. If this is too high (e.g. 500ms) then
|
to do other things. If this is too high (e.g. 500ms) then
|
||||||
even two workers/tabs can easily run into locking errors.
|
even two workers/tabs can easily run into locking errors.
|
||||||
*/
|
*/
|
||||||
const waitTime = 150;
|
const waitTime = 100;
|
||||||
while(!flagAsyncShutdown){
|
while(!flagAsyncShutdown){
|
||||||
try {
|
try {
|
||||||
if('timed-out'===Atomics.wait(
|
if('timed-out'===Atomics.wait(
|
||||||
state.sabOPView, state.opIds.whichOp, 0, waitTime
|
state.sabOPView, state.opIds.whichOp, 0, waitTime
|
||||||
)){
|
)){
|
||||||
await closeAutoLocks();
|
await releaseImplicitLocks();
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
const opId = Atomics.load(state.sabOPView, state.opIds.whichOp);
|
const opId = Atomics.load(state.sabOPView, state.opIds.whichOp);
|
||||||
@ -824,6 +855,7 @@ navigator.storage.getDirectory().then(function(d){
|
|||||||
state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize);
|
state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize);
|
||||||
state.opIds = opt.opIds;
|
state.opIds = opt.opIds;
|
||||||
state.sq3Codes = opt.sq3Codes;
|
state.sq3Codes = opt.sq3Codes;
|
||||||
|
state.opfsFlags = opt.opfsFlags;
|
||||||
Object.keys(vfsAsyncImpls).forEach((k)=>{
|
Object.keys(vfsAsyncImpls).forEach((k)=>{
|
||||||
if(!Number.isFinite(state.opIds[k])){
|
if(!Number.isFinite(state.opIds[k])){
|
||||||
toss("Maintenance required: missing state.opIds[",k,"]");
|
toss("Maintenance required: missing state.opIds[",k,"]");
|
||||||
|
@ -24,10 +24,13 @@
|
|||||||
</p>
|
</p>
|
||||||
<p>
|
<p>
|
||||||
URL flags: pass a number of workers using
|
URL flags: pass a number of workers using
|
||||||
the <code>workers=N</code> URL flag and the worker work interval
|
the <code>workers=N</code> URL flag. Set the time between each
|
||||||
as <code>interval=N</code> (milliseconds). Enable OPFS VFS
|
workload with <code>interval=N</code> (milliseconds). Set the
|
||||||
verbosity with <code>verbose=1-3</code> (output goes to the
|
number of worker iterations with <code>iterations=N</code>.
|
||||||
dev console).
|
Enable OPFS VFS verbosity with <code>verbose=1-3</code> (output
|
||||||
|
goes to the dev console). Enable/disable "unlock ASAP" mode
|
||||||
|
(higher concurrency, lower speed)
|
||||||
|
with <code>unlock-asap=0-1</code>.
|
||||||
</p>
|
</p>
|
||||||
<p>Achtung: if it does not start to do anything within a couple of
|
<p>Achtung: if it does not start to do anything within a couple of
|
||||||
seconds, check the dev console: Chrome often fails with "cannot allocate
|
seconds, check the dev console: Chrome often fails with "cannot allocate
|
||||||
|
@ -56,26 +56,42 @@
|
|||||||
options.sqlite3Dir = urlArgsJs.get('sqlite3.dir');
|
options.sqlite3Dir = urlArgsJs.get('sqlite3.dir');
|
||||||
options.workerCount = (
|
options.workerCount = (
|
||||||
urlArgsHtml.has('workers') ? +urlArgsHtml.get('workers') : 3
|
urlArgsHtml.has('workers') ? +urlArgsHtml.get('workers') : 3
|
||||||
) || 3;
|
) || 4;
|
||||||
options.opfsVerbose = (
|
options.opfsVerbose = (
|
||||||
urlArgsHtml.has('verbose') ? +urlArgsHtml.get('verbose') : 1
|
urlArgsHtml.has('verbose') ? +urlArgsHtml.get('verbose') : 1
|
||||||
) || 1;
|
) || 1;
|
||||||
options.interval = (
|
options.interval = (
|
||||||
urlArgsHtml.has('interval') ? +urlArgsHtml.get('interval') : 750
|
urlArgsHtml.has('interval') ? +urlArgsHtml.get('interval') : 750
|
||||||
) || 750;
|
) || 1000;
|
||||||
|
options.iterations = (
|
||||||
|
urlArgsHtml.has('iterations') ? +urlArgsHtml.get('iterations') : 10
|
||||||
|
) || 10;
|
||||||
|
options.unlockAsap = (
|
||||||
|
urlArgsHtml.has('unlock-asap') ? +urlArgsHtml.get('unlock-asap') : 0
|
||||||
|
) || 0;
|
||||||
const workers = [];
|
const workers = [];
|
||||||
workers.post = (type,...args)=>{
|
workers.post = (type,...args)=>{
|
||||||
for(const w of workers) w.postMessage({type, payload:args});
|
for(const w of workers) w.postMessage({type, payload:args});
|
||||||
};
|
};
|
||||||
workers.loadedCount = 0;
|
workers.counts = {loaded: 0, passed: 0, failed: 0};
|
||||||
|
const checkFinished = function(){
|
||||||
|
if(workers.counts.passed + workers.counts.failed !== workers.length){
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if(workers.counts.failed>0){
|
||||||
|
logCss('tests-fail',"Finished with",workers.counts.failed,"failure(s).");
|
||||||
|
}else{
|
||||||
|
logCss('tests-pass',"All",workers.length,"workers finished.");
|
||||||
|
}
|
||||||
|
};
|
||||||
workers.onmessage = function(msg){
|
workers.onmessage = function(msg){
|
||||||
msg = msg.data;
|
msg = msg.data;
|
||||||
const prefix = 'Worker #'+msg.worker+':';
|
const prefix = 'Worker #'+msg.worker+':';
|
||||||
switch(msg.type){
|
switch(msg.type){
|
||||||
case 'loaded':
|
case 'loaded':
|
||||||
stdout(prefix,"loaded");
|
stdout(prefix,"loaded");
|
||||||
if(++workers.loadedCount === workers.length){
|
if(++workers.counts.loaded === workers.length){
|
||||||
stdout("All workers loaded. Telling them to run...");
|
stdout("All",workers.length,"workers loaded. Telling them to run...");
|
||||||
workers.post('run');
|
workers.post('run');
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
@ -83,21 +99,27 @@
|
|||||||
case 'stderr': stderr(prefix,...msg.payload); break;
|
case 'stderr': stderr(prefix,...msg.payload); break;
|
||||||
case 'error': stderr(prefix,"ERROR:",...msg.payload); break;
|
case 'error': stderr(prefix,"ERROR:",...msg.payload); break;
|
||||||
case 'finished':
|
case 'finished':
|
||||||
|
++workers.counts.passed;
|
||||||
logCss('tests-pass',prefix,...msg.payload);
|
logCss('tests-pass',prefix,...msg.payload);
|
||||||
|
checkFinished();
|
||||||
break;
|
break;
|
||||||
case 'failed':
|
case 'failed':
|
||||||
|
++workers.counts.failed;
|
||||||
logCss('tests-fail',prefix,"FAILED:",...msg.payload);
|
logCss('tests-fail',prefix,"FAILED:",...msg.payload);
|
||||||
|
checkFinished();
|
||||||
break;
|
break;
|
||||||
default: logCss('error',"Unhandled message type:",msg); break;
|
default: logCss('error',"Unhandled message type:",msg); break;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
stdout("Launching",options.workerCount,"workers...");
|
stdout("Launching",options.workerCount,"workers. Options:",options);
|
||||||
workers.uri = (
|
workers.uri = (
|
||||||
'worker.js?'
|
'worker.js?'
|
||||||
+ 'sqlite3.dir='+options.sqlite3Dir
|
+ 'sqlite3.dir='+options.sqlite3Dir
|
||||||
+ '&interval='+options.interval
|
+ '&interval='+options.interval
|
||||||
|
+ '&iterations='+options.iterations
|
||||||
+ '&opfs-verbose='+options.opfsVerbose
|
+ '&opfs-verbose='+options.opfsVerbose
|
||||||
|
+ '&opfs-unlock-asap='+options.unlockAsap
|
||||||
);
|
);
|
||||||
for(let i = 0; i < options.workerCount; ++i){
|
for(let i = 0; i < options.workerCount; ++i){
|
||||||
stdout("Launching worker...");
|
stdout("Launching worker...");
|
||||||
|
@ -3,9 +3,12 @@ importScripts(
|
|||||||
);
|
);
|
||||||
self.sqlite3InitModule().then(async function(sqlite3){
|
self.sqlite3InitModule().then(async function(sqlite3){
|
||||||
const urlArgs = new URL(self.location.href).searchParams;
|
const urlArgs = new URL(self.location.href).searchParams;
|
||||||
const wName = urlArgs.get('workerId') || Math.round(Math.random()*10000);
|
const options = {
|
||||||
|
workerName: urlArgs.get('workerId') || Math.round(Math.random()*10000),
|
||||||
|
unlockAsap: urlArgs.get('opfs-unlock-asap') || 0 /*EXPERIMENTAL*/
|
||||||
|
};
|
||||||
const wPost = (type,...payload)=>{
|
const wPost = (type,...payload)=>{
|
||||||
postMessage({type, worker: wName, payload});
|
postMessage({type, worker: options.workerName, payload});
|
||||||
};
|
};
|
||||||
const stdout = (...args)=>wPost('stdout',...args);
|
const stdout = (...args)=>wPost('stdout',...args);
|
||||||
const stderr = (...args)=>wPost('stderr',...args);
|
const stderr = (...args)=>wPost('stderr',...args);
|
||||||
@ -43,7 +46,10 @@ self.sqlite3InitModule().then(async function(sqlite3){
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
const run = async function(){
|
const run = async function(){
|
||||||
db = new sqlite3.opfs.OpfsDb(dbName,'c');
|
db = new sqlite3.opfs.OpfsDb({
|
||||||
|
filename: 'file:'+dbName+'?opfs-unlock-asap='+options.unlockAsap,
|
||||||
|
flags: 'c'
|
||||||
|
});
|
||||||
sqlite3.capi.sqlite3_busy_timeout(db.pointer, 5000);
|
sqlite3.capi.sqlite3_busy_timeout(db.pointer, 5000);
|
||||||
db.transaction((db)=>{
|
db.transaction((db)=>{
|
||||||
db.exec([
|
db.exec([
|
||||||
@ -52,7 +58,8 @@ self.sqlite3InitModule().then(async function(sqlite3){
|
|||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
const maxIterations = 10;
|
const maxIterations =
|
||||||
|
urlArgs.has('iterations') ? (+urlArgs.get('iterations') || 10) : 10;
|
||||||
stdout("Starting interval-based db updates with delay of",interval.delay,"ms.");
|
stdout("Starting interval-based db updates with delay of",interval.delay,"ms.");
|
||||||
const doWork = async ()=>{
|
const doWork = async ()=>{
|
||||||
const tm = new Date().getTime();
|
const tm = new Date().getTime();
|
||||||
@ -62,7 +69,7 @@ self.sqlite3InitModule().then(async function(sqlite3){
|
|||||||
try{
|
try{
|
||||||
db.exec({
|
db.exec({
|
||||||
sql:"INSERT OR REPLACE INTO t1(w,v) VALUES(?,?)",
|
sql:"INSERT OR REPLACE INTO t1(w,v) VALUES(?,?)",
|
||||||
bind: [wName, new Date().getTime()]
|
bind: [options.workerName, new Date().getTime()]
|
||||||
});
|
});
|
||||||
//stdout("Set",prefix);
|
//stdout("Set",prefix);
|
||||||
}catch(e){
|
}catch(e){
|
||||||
|
25
manifest
25
manifest
@ -1,5 +1,5 @@
|
|||||||
C Relax\srestriction\s(8)\son\sthe\spush-down\soptimization\sso\sthat\sit\sonly\sapplies\nif\sone\sor\smore\scolumns\suses\sa\scollating\ssequence\sother\sthan\sBINARY.\nSee\s[forum:/forumpost/3824ced748baa808|forum\spost\s3824ced748baa808]\sand\ncheck-in\s[346a3b12b861ce7b].
|
C OPFS\sVFS:\sadd\sthe\sopfs-unlock-asap=1\sURI\sflag\swhich\stells\sthe\sVFS\sto\srelease\simplicit\slocks\sASAP\sinstead\sof\sduring\sVFS\sidle\stime.\sThis\simproves\sconcurrency\snotably\sin\sthe\stest\sapp\sbut\sbrings\sa\ssignificant\sperformance\spenalty\sin\sspeedtest1\s(roughly\s4x\sslowdown).\sThis\sis\snot\sthe\sfinal\sword\sin\sOPFS\sconcurrency,\sbut\sgets\sus\sa\sstep\sfurther.
|
||||||
D 2022-11-25T17:05:55.692
|
D 2022-11-26T15:24:58.092
|
||||||
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
|
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
|
||||||
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
|
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
|
||||||
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
|
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
|
||||||
@ -501,12 +501,12 @@ F ext/wasm/api/post-js-header.js d6ab3dfef4a06960d28a7eaa338d4e2a1a5981e9b387181
|
|||||||
F ext/wasm/api/pre-js.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f9e91c1d5f
|
F ext/wasm/api/pre-js.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f9e91c1d5f
|
||||||
F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e3b9b6e9b0491f92ed8ae34
|
F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e3b9b6e9b0491f92ed8ae34
|
||||||
F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5
|
F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5
|
||||||
F ext/wasm/api/sqlite3-api-oo1.js dec6c14994317ad0011714890426cdc211f4eab451c9766ea88c7ac4f535287e
|
F ext/wasm/api/sqlite3-api-oo1.js 06ad2079368e16cb9f182c18cd37bdc3932536856dff4f60582d0ca5f6c491a8
|
||||||
F ext/wasm/api/sqlite3-api-opfs.js 38d368e33f470f9ba196f1a2b0c9ce076c930c70df233c345a246f1ad4c26d3b
|
F ext/wasm/api/sqlite3-api-opfs.js e98a8bd67dea8c20b0ec17332698b44658a6fbc4be18dd87fab2ce05284a10d7
|
||||||
F ext/wasm/api/sqlite3-api-prologue.js 7fce4c6a138ec3d7c285b7c125cee809e6b668d2cb0d2328a1b790b7037765bd
|
F ext/wasm/api/sqlite3-api-prologue.js 7fce4c6a138ec3d7c285b7c125cee809e6b668d2cb0d2328a1b790b7037765bd
|
||||||
F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f
|
F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f
|
||||||
F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3
|
F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3
|
||||||
F ext/wasm/api/sqlite3-opfs-async-proxy.js 1ec10873f1d59d305f6f3b435c50a1b75d693d5fb739b226f3da46fcbb11261a
|
F ext/wasm/api/sqlite3-opfs-async-proxy.js d933d4a3c84e6dc614b57355fc9029645cfcdfbfde096ed42f4c979a6f60c18a
|
||||||
F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9
|
F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9
|
||||||
F ext/wasm/api/sqlite3-wasm.c 8b32787a3b6bb2990cbaba2304bd5b75a9652acbc8d29909b3279019b6cbaef5
|
F ext/wasm/api/sqlite3-wasm.c 8b32787a3b6bb2990cbaba2304bd5b75a9652acbc8d29909b3279019b6cbaef5
|
||||||
F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b
|
F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b
|
||||||
@ -552,9 +552,9 @@ F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d96
|
|||||||
F ext/wasm/tester1-worker.html 5ef353348c37cf2e4fd0b23da562d3275523e036260b510734e9a3239ba8c987
|
F ext/wasm/tester1-worker.html 5ef353348c37cf2e4fd0b23da562d3275523e036260b510734e9a3239ba8c987
|
||||||
F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399
|
F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399
|
||||||
F ext/wasm/tester1.c-pp.js 3b91f192c159088004fba6fe3441edea58421a8b88bccf3dd20978a077648d19
|
F ext/wasm/tester1.c-pp.js 3b91f192c159088004fba6fe3441edea58421a8b88bccf3dd20978a077648d19
|
||||||
F ext/wasm/tests/opfs/concurrency/index.html bb9b0f6da86df34c67fa506db9c45b7c4cf0045a211611cc6b8d2b53fa983481
|
F ext/wasm/tests/opfs/concurrency/index.html e8fec75ea6eddc600c8a382da7ea2579feece2263a2fb4417f2cf3e9d451744c
|
||||||
F ext/wasm/tests/opfs/concurrency/test.js 5993c08657d547d3a26b78ff3480122aed2b7361823bc127e96e558931093aff
|
F ext/wasm/tests/opfs/concurrency/test.js bfc3d7e27b207f0827f12568986b8d516a744529550b449314f5c21c9e9faf4a
|
||||||
F ext/wasm/tests/opfs/concurrency/worker.js afccb78082b57edb17d5aba0754c823772553395df6f1aed92f82b4d9e3c32de
|
F ext/wasm/tests/opfs/concurrency/worker.js 0eff027cbd3a495acb2ac94f57ca9e4d21125ab9fda07d45f3701b0efe82d450
|
||||||
F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5
|
F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5
|
||||||
F ext/wasm/wasmfs.make 8fea9b4f3cde06141de1fc4c586ab405bd32c3f401554f4ebb18c797401a678d
|
F ext/wasm/wasmfs.make 8fea9b4f3cde06141de1fc4c586ab405bd32c3f401554f4ebb18c797401a678d
|
||||||
F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
|
F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
|
||||||
@ -2060,8 +2060,9 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
|
|||||||
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
|
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
|
||||||
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
|
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
|
||||||
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
|
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
|
||||||
P 09e1e42e0ff26f9a71cbd128169f060a66425828d637bf8f781490ca38d99103
|
P adbca3448e2099f0d6149a073978f230ed9a92a2f384779879ef89e672231bcf c0458caca3508d5d252f9b5198bda4f51a5c1874540f014b17e409f2daab1706
|
||||||
R 051f7bebeb695c93dfdb45030ad5b532
|
R ef0be46e6ac8ee603582dd679fae7434
|
||||||
U drh
|
T +closed c0458caca3508d5d252f9b5198bda4f51a5c1874540f014b17e409f2daab1706 Closed\sby\sintegrate-merge.
|
||||||
Z 0a4a218169578e69e7b7469760b002aa
|
U stephan
|
||||||
|
Z 3b8060cf3f2afe36aab99fa36bba89db
|
||||||
# Remove this line to create a well-formed Fossil manifest.
|
# Remove this line to create a well-formed Fossil manifest.
|
||||||
|
@ -1 +1 @@
|
|||||||
adbca3448e2099f0d6149a073978f230ed9a92a2f384779879ef89e672231bcf
|
9542f9ce9e023b489e2d93661f719fb0751c1e28f72fded9d3c2156d5777e7b1
|
Reference in New Issue
Block a user