database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-07-30 19:03:16 +03:00
Code Activity

Reworked out the OPFS async proxy metrics are fetched so that they play more nicely with the tight event-polling loop.

Browse Source 
FossilOrigin-Name: ef503ced5c2ca842be9aea9ef13719a378ed3020e884032db09afee1b8eba0a1
This commit is contained in:
stephan
2022-09-24 10:12:19 +00:00
parent 60d9aa7c59
commit 56fae744d4
4 changed files with 86 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
ext/wasm/api/sqlite3-api-opfs.js
View File

@ -142,7 +142,8 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
"metrics for",self.location.href,":",metrics,
"\nTotal of",n,"op(s) for",t,
"ms (incl. "+w+" ms of waiting on the async side)");
console.log("Serialization metrics:",JSON.stringify(metrics.s11n,0,2));
console.log("Serialization metrics:",metrics.s11n);
opRun('async-metrics');
},
reset: function(){
let k;
@ -160,7 +161,17 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
//].forEach((k)=>r(metrics[k] = Object.create(null)));
}
}/*metrics*/;
const promiseReject = function(err){
opfsVfs.dispose();
return promiseReject_(err);
};
const W = new Worker(options.proxyUri);
W._originalOnError = W.onerror /* will be restored later */;
W.onerror = function(err){
// The error object doesn't contain any useful info when the
// failure is, e.g., that the remote script is 404.
promiseReject(new Error("Loading OPFS async Worker failed for unknown reasons."));
};
const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/;
const dVfs = pDVfs
? new sqlite3_vfs(pDVfs)
@ -193,18 +204,6 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
gets called at all in a wasm build, which is undefined).
*/
const promiseReject = function(err){
opfsVfs.dispose();
return promiseReject_(err);
};
const W = new Worker(options.proxyUri);
W._originalOnError = W.onerror /* will be restored later */;
W.onerror = function(err){
// The error object doesn't contain any useful info when the
// failure is, e.g., that the remote script is 404.
promiseReject(new Error("Loading OPFS async Worker failed for unknown reasons."));
};
/**
State which we send to the async-api Worker or share with it.
This object must initially contain only cloneable or sharable
@ -227,11 +226,19 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
const state = Object.create(null);
state.verbose = options.verbose;
state.littleEndian = true;
/** If true, the async counterpart should log exceptions to
/** Whether the async counterpart should log exceptions to
the serialization channel. That produces a great deal of
noise for seemingly innocuous things like xAccess() checks
for missing files. */
state.asyncS11nExceptions = false;
for missing files, so this option may have one of 3 values:
0 = no exception logging
1 = only log exceptions for "significant" ops like xOpen(),
xRead(), and xWrite().
2 = log all exceptions.
*/
state.asyncS11nExceptions = 1;
/* Size of file I/O buffer block. 64k = max sqlite3 page size. */
state.fileBufferSize =
1024 * 64;
@ -270,6 +277,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
state.opIds.xClose = i++;
state.opIds.xDelete = i++;
state.opIds.xDeleteNoWait = i++;
state.opIds.xFileControl = i++;
state.opIds.xFileSize = i++;
state.opIds.xOpen = i++;
state.opIds.xRead = i++;
@ -278,7 +286,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
state.opIds.xTruncate = i++;
state.opIds.xWrite = i++;
state.opIds.mkdir = i++;
state.opIds.xFileControl = i++;
state.opIds['async-metrics'] = i++;
state.sabOP = new SharedArrayBuffer(i * 4/*sizeof int32*/);
opfsUtil.metrics.reset();
}
@ -858,6 +866,27 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
*/
opfsUtil.randomFilename = randomFilename;
/**
Re-registers the OPFS VFS. This is intended only for odd use
cases which have to call sqlite3_shutdown() as part of their
initialization process, which will unregister the VFS
registered by installOpfsVfs(). If passed a truthy value, the
OPFS VFS is registered as the default VFS, else it is not made
the default. Returns the result of the the
sqlite3_vfs_register() call.
Design note: the problem of having to re-register things after
a shutdown/initialize pair is more general. How to best plug
that in to the library is unclear. In particular, we cannot
hook in to any C-side calls to sqlite3_initialize(), so we
cannot add an after-initialize callback mechanism.
*/
opfsUtil.reregisterVfs = (asDefault=false)=>{
return capi.wasm.exports.sqlite3_vfs_register(
opfsVfs.pointer, asDefault ? 1 : 0
);
};
if(sqlite3.oo1){
opfsUtil.OpfsDb = function(...args){
const opt = sqlite3.oo1.dbCtorHelper.normalizeArgs(...args);
@ -941,7 +970,6 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
}
}/*sanityCheck()*/;
W.onmessage = function({data}){
//log("Worker.onmessage:",data);
switch(data.type){

63
ext/wasm/sqlite3-opfs-async-proxy.js
View File

@ -84,11 +84,10 @@ metrics.dump = ()=>{
}
console.log(self.location.href,
"metrics for",self.location.href,":\n",
JSON.stringify(metrics,0,2)
/*dev console can't expand this object!*/,
metrics,
"\nTotal of",n,"op(s) for",t,"ms",
"approx",w,"ms spent waiting on OPFS APIs.");
console.log("Serialization metrics:",JSON.stringify(metrics.s11n,0,2));
console.log("Serialization metrics:",metrics.s11n);
};
warn("This file is very much experimental and under construction.",
@ -122,7 +121,7 @@ const getResolvedPath = function(filename,splitIt){
truthy then each directory element leading to the file is created
along the way. Throws if any creation or resolution fails.
*/
const getDirForPath = async function f(absFilename, createDirs = false){
const getDirForFilename = async function f(absFilename, createDirs = false){
const path = getResolvedPath(absFilename, true);
const filename = path.pop();
let dh = state.rootDir;
@ -183,14 +182,20 @@ const wTimeEnd = ()=>(
to simplify finding them.
*/
const vfsAsyncImpls = {
mkdir: async function(dirname){
'async-metrics': async ()=>{
mTimeStart('async-metrics');
metrics.dump();
storeAndNotify('async-metrics', 0);
mTimeEnd();
},
mkdir: async (dirname)=>{
mTimeStart('mkdir');
let rc = 0;
wTimeStart('mkdir');
try {
await getDirForPath(dirname+"/filepart", true);
await getDirForFilename(dirname+"/filepart", true);
}catch(e){
state.s11n.storeException(e);
state.s11n.storeException(2,e);
rc = state.sq3Codes.SQLITE_IOERR;
}finally{
wTimeEnd();
@ -198,7 +203,7 @@ const vfsAsyncImpls = {
storeAndNotify('mkdir', rc);
mTimeEnd();
},
xAccess: async function(filename){
xAccess: async (filename)=>{
mTimeStart('xAccess');
/* OPFS cannot support the full range of xAccess() queries sqlite3
calls for. We can essentially just tell if the file is
@ -214,10 +219,10 @@ const vfsAsyncImpls = {
let rc = 0;
wTimeStart('xAccess');
try{
const [dh, fn] = await getDirForPath(filename);
const [dh, fn] = await getDirForFilename(filename);
await dh.getFileHandle(fn);
}catch(e){
state.s11n.storeException(e);
state.s11n.storeException(2,e);
rc = state.sq3Codes.SQLITE_IOERR;
}finally{
wTimeEnd();
@ -269,7 +274,7 @@ const vfsAsyncImpls = {
wTimeStart('xDelete');
try {
while(filename){
const [hDir, filenamePart] = await getDirForPath(filename, false);
const [hDir, filenamePart] = await getDirForFilename(filename, false);
if(!filenamePart) break;
await hDir.removeEntry(filenamePart, {recursive});
if(0x1234 !== syncDir) break;
@ -278,7 +283,7 @@ const vfsAsyncImpls = {
filename = filename.join('/');
}
}catch(e){
state.s11n.storeException(e);
state.s11n.storeException(2,e);
rc = state.sq3Codes.SQLITE_IOERR_DELETE;
}
wTimeEnd();
@ -294,7 +299,7 @@ const vfsAsyncImpls = {
state.s11n.serialize(Number(sz));
sz = 0;
}catch(e){
state.s11n.storeException(e);
state.s11n.storeException(2,e);
sz = state.sq3Codes.SQLITE_IOERR;
}
wTimeEnd();
@ -310,7 +315,7 @@ const vfsAsyncImpls = {
try{
let hDir, filenamePart;
try {
[hDir, filenamePart] = await getDirForPath(filename, !!create);
[hDir, filenamePart] = await getDirForFilename(filename, !!create);
}catch(e){
storeAndNotify(opName, state.sql3Codes.SQLITE_NOTFOUND);
mTimeEnd();
@ -339,7 +344,7 @@ const vfsAsyncImpls = {
}catch(e){
wTimeEnd();
error(opName,e);
state.s11n.storeException(e);
state.s11n.storeException(1,e);
storeAndNotify(opName, state.sq3Codes.SQLITE_IOERR);
}
mTimeEnd();
@ -361,7 +366,7 @@ const vfsAsyncImpls = {
}
}catch(e){
error("xRead() failed",e,fh);
state.s11n.storeException(e);
state.s11n.storeException(1,e);
rc = state.sq3Codes.SQLITE_IOERR_READ;
}
storeAndNotify('xRead',rc);
@ -376,7 +381,7 @@ const vfsAsyncImpls = {
wTimeStart('xSync');
await fh.accessHandle.flush();
}catch(e){
state.s11n.storeException(e);
state.s11n.storeException(2,e);
}finally{
wTimeEnd();
}
@ -394,7 +399,7 @@ const vfsAsyncImpls = {
await fh.accessHandle.truncate(size);
}catch(e){
error("xTruncate():",e,fh);
state.s11n.storeException(e);
state.s11n.storeException(2,e);
rc = state.sq3Codes.SQLITE_IOERR_TRUNCATE;
}
wTimeEnd();
@ -414,7 +419,7 @@ const vfsAsyncImpls = {
) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE;
}catch(e){
error("xWrite():",e,fh);
state.s11n.storeException(e);
state.s11n.storeException(1,e);
rc = state.sq3Codes.SQLITE_IOERR_WRITE;
}finally{
wTimeEnd();
@ -519,7 +524,11 @@ const initS11n = ()=>{
};
state.s11n.storeException = state.asyncS11nExceptions
? ((e)=>state.s11n.serialize(e.message))
? ((priority,e)=>{
if(priority<=state.asyncS11nExceptions){
state.s11n.serialize(e.message);
}
})
: ()=>{};
return state.s11n;
@ -533,10 +542,8 @@ const waitLoop = async function f(){
const o = Object.create(null);
opHandlers[state.opIds[k]] = o;
o.key = k;
o.f = vi;// || toss("No vfsAsyncImpls[",k,"]");
o.f = vi || toss("No vfsAsyncImpls[",k,"]");
}
let metricsTimer = self.location.port>=1024 ? performance.now() : 0;
// ^^^ in dev environment, dump out these metrics one time after a delay.
while(true){
try {
if('timed-out'===Atomics.wait(state.sabOPView, state.opIds.whichOp, 0, 500)){
@ -545,7 +552,7 @@ const waitLoop = async function f(){
const opId = Atomics.load(state.sabOPView, state.opIds.whichOp);
Atomics.store(state.sabOPView, state.opIds.whichOp, 0);
const hnd = opHandlers[opId] ?? toss("No waitLoop handler for whichOp #",opId);
const args = state.s11n.deserialize();
const args = state.s11n.deserialize() || [];
state.s11n.serialize()/* clear s11n to keep the caller from
confusing this with an exception string
written by the upcoming operation */;
@ -554,14 +561,6 @@ const waitLoop = async function f(){
else error("Missing callback for opId",opId);
}catch(e){
error('in waitLoop():',e.message);
}finally{
// We can't call metrics.dump() from the dev console because this
// thread is continually tied up in Atomics.wait(), so let's
// do, for dev purposes only, a dump one time after 60 seconds.
if(metricsTimer && (performance.now() > metricsTimer + 60000)){
metrics.dump();
metricsTimer = 0;
}
}
};
};

14
manifest
View File

@ -1,5 +1,5 @@
C Refactoring\stowards\sgetting\sfiddle\sto\ssupport\sOPFS\sas\sa\sfirst-class\scitizen.\sCertain\soperations,\se.g.\simport,\sexport,\sand\sunlink,\sare\snot\sOPFS-aware.
D 2022-09-24T07:36:45.332
C Reworked\sout\sthe\sOPFS\sasync\sproxy\smetrics\sare\sfetched\sso\sthat\sthey\splay\smore\snicely\swith\sthe\stight\sevent-polling\sloop.
D 2022-09-24T10:12:19.409
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@ -484,7 +484,7 @@ F ext/wasm/api/post-js-header.js 2e5c886398013ba2af88028ecbced1e4b22dc96a86467f1
F ext/wasm/api/sqlite3-api-cleanup.js 8564a6077cdcaea9a9f428a019af8a05887f0131e6a2a1e72a7ff1145fadfe77
F ext/wasm/api/sqlite3-api-glue.js cfff894bdf98a6c579975d09dd45471b0e3399f08a6f9e44a22646e8403196ed
F ext/wasm/api/sqlite3-api-oo1.js f974e79d9af8f26bf33928c5730b0988cc706d14f59a5fe36394739b92249841
F ext/wasm/api/sqlite3-api-opfs.js d623ea3519cd81fe18e243adfdd07cd1fa4b07ff3b0fd0d2b269beb0e127acb3
F ext/wasm/api/sqlite3-api-opfs.js 5585dc80aea9df54c3d5d3a6c62771bf741f21b23706330ba62571c57ec07abf
F ext/wasm/api/sqlite3-api-prologue.js a50ba8618e81a10a4fecd70f8723a7295cfcc0babd6df1dd018e7c5db2904aac
F ext/wasm/api/sqlite3-api-worker1.js 2eeb2a24e1a90322d84a9b88a99919b806623de62792436446099c0988f2030b
F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9
@ -519,7 +519,7 @@ F ext/wasm/speedtest1.html 8ae6ece128151d01f90579de69cfa06f021acdb760735250ef745
F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x
F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0
F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5
F ext/wasm/sqlite3-opfs-async-proxy.js 483b6326e268589ed3749a4d1a60b4ec2afa8ff7c218a09d39430e3bde89862e
F ext/wasm/sqlite3-opfs-async-proxy.js fe4b8268eea9acaec633ebd1dd3f85dae7c461c5c68985ab1075d9560b1db8e8
F ext/wasm/sqlite3-worker1-promiser.js 4fd0465688a28a75f1d4ee4406540ba494f49844e3cad0670d0437a001943365
F ext/wasm/sqlite3-worker1.js 0c1e7626304543969c3846573e080c082bf43bcaa47e87d416458af84f340a9e
F ext/wasm/test-opfs-vfs.html eb69dda21eb414b8f5e3f7c1cc0f774103cc9c0f87b2d28a33419e778abfbab5
@ -2026,8 +2026,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P e1249369d5ec1c582c280b1f578b35d53637fdf1cbd97c16d5ed95b136b83e56
R bfb613620806c41d29eb36b1d39427ef
P 1b923ed6438d7fef4508936e0c4bc026a368721698b1539961e3fb3140a185cb
R 2784cada1344182947069d02e01bbe66
U stephan
Z 135731a0096fbab4e7ac0c6ece9bf8ed
Z 8970d31318ed191d3ed86820eec133d3
# Remove this line to create a well-formed Fossil manifest.

2
manifest.uuid
View File

@ -1 +1 @@
1b923ed6438d7fef4508936e0c4bc026a368721698b1539961e3fb3140a185cb
ef503ced5c2ca842be9aea9ef13719a378ed3020e884032db09afee1b8eba0a1