diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index b7e6887f65..3fd08d9aec 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -140,16 +140,19 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) */ opfsUtil.metrics = { dump: function(){ - let k, n = 0, t = 0; - for(k in metrics){ + let k, n = 0, t = 0, w = 0; + for(k in state.opIds){ const m = metrics[k]; n += m.count; t += m.time; + w += m.wait; m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0; m.avgWait = (m.count && m.wait) ? (m.wait / m.count) : 0; } - console.log("metrics for",self.location.href,":",metrics, - "\nTotal of",n,"op(s) for",t,"ms"); + console.log(self.location.href, + "metrics for",self.location.href,":",metrics, + "\nTotal of",n,"op(s) for",t, + "ms (incl. "+w+" ms of waiting on the async side)"); }, reset: function(){ let k; @@ -157,9 +160,9 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) for(k in state.opIds){ r(metrics[k] = Object.create(null)); } - [ // timed routines which are not in state.opIds - 'xFileControl' - ].forEach((k)=>r(metrics[k] = Object.create(null))); + //[ // timed routines which are not in state.opIds + // 'xFileControl' + //].forEach((k)=>r(metrics[k] = Object.create(null))); } }/*metrics*/; @@ -209,6 +212,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) state.opIds.xWrite = i++; state.opIds.mkdir = i++; state.opSAB = new SharedArrayBuffer(i * 4/*sizeof int32*/); + state.opIds.xFileControl = state.opIds.xSync /* special case */; opfsUtil.metrics.reset(); } @@ -413,18 +417,20 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) }, xFileControl: function(pFile, opId, pArg){ mTimeStart('xFileControl'); - if(capi.SQLITE_FCNTL_SYNC===opId){ - return opRun('xSync', {fid:pFile, flags:0}); - } + const rc = (capi.SQLITE_FCNTL_SYNC===opId) + ? opRun('xSync', {fid:pFile, flags:0}) + : capi.SQLITE_NOTFOUND; mTimeEnd(); - return capi.SQLITE_NOTFOUND; + return rc; }, xFileSize: function(pFile,pSz64){ mTimeStart('xFileSize'); const rc = opRun('xFileSize', pFile); if(!isWorkerErrCode(rc)){ - const f = __openFiles[pFile]; - wasm.setMemValue(pSz64, f.sabViewFileSize.getBigInt64(0,true) ,'i64'); + wasm.setMemValue( + pSz64, __openFiles[pFile].sabViewFileSize.getBigInt64(0,true), + 'i64' + ); } mTimeEnd(); return rc; @@ -454,6 +460,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) return rc; }, xSync: function(pFile,flags){ + ++metrics.xSync.count; return 0; // impl'd in xFileControl(). opRun('xSync', {fid:pFile, flags}); }, xTruncate: function(pFile,sz64){ @@ -492,8 +499,9 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) const vfsSyncWrappers = { xAccess: function(pVfs,zName,flags,pOut){ mTimeStart('xAccess'); - const rc = opRun('xAccess', wasm.cstringToJs(zName)); - wasm.setMemValue(pOut, rc ? 0 : 1, 'i32'); + wasm.setMemValue( + pOut, (opRun('xAccess', wasm.cstringToJs(zName)) ? 0 : 1), 'i32' + ); mTimeEnd(); return 0; }, diff --git a/ext/wasm/sqlite3-opfs-async-proxy.js b/ext/wasm/sqlite3-opfs-async-proxy.js index 73bfad9d87..fb472488bf 100644 --- a/ext/wasm/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/sqlite3-opfs-async-proxy.js @@ -61,6 +61,25 @@ const log = (...args)=>logImpl(2, ...args); const warn = (...args)=>logImpl(1, ...args); const error = (...args)=>logImpl(0, ...args); const metrics = Object.create(null); +metrics.reset = ()=>{ + let k; + const r = (m)=>(m.count = m.time = 0); + for(k in state.opIds){ + r(metrics[k] = Object.create(null)); + } +}; +metrics.dump = ()=>{ + let k, n = 0, t = 0, w = 0; + for(k in state.opIds){ + const m = metrics[k]; + n += m.count; + t += m.time; + m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0; + } + console.log(self.location.href, + "metrics for",self.location.href,":",metrics, + "\nTotal of",n,"op(s) for",t,"ms"); +}; warn("This file is very much experimental and under construction.", self.location.pathname); @@ -123,14 +142,39 @@ const affirmNotRO = function(opName,fh){ if(fh.readOnly) toss(opName+"(): File is read-only: "+fh.filenameAbs); }; + +const opTimer = Object.create(null); +opTimer.op = undefined; +opTimer.start = undefined; +const mTimeStart = (op)=>{ + opTimer.start = performance.now(); + opTimer.op = op; + //metrics[op] || toss("Maintenance required: missing metrics for",op); + ++metrics[op].count; +}; +const mTimeEnd = ()=>( + metrics[opTimer.op].time += performance.now() - opTimer.start +); + /** Asynchronous wrappers for sqlite3_vfs and sqlite3_io_methods methods. Maintenance reminder: members are in alphabetical order to simplify finding them. */ const vfsAsyncImpls = { + mkdir: async function(dirname){ + let rc = 0; + try { + await getDirForPath(dirname+"/filepart", true); + }catch(e){ + //error("mkdir failed",filename, e.message); + rc = state.sq3Codes.SQLITE_IOERR; + } + storeAndNotify('mkdir', rc); + }, xAccess: async function(filename){ log("xAccess(",arguments[0],")"); + mTimeStart('xAccess'); /* OPFS cannot support the full range of xAccess() queries sqlite3 calls for. We can essentially just tell if the file is accessible, but if it is it's automatically writable (unless @@ -150,9 +194,11 @@ const vfsAsyncImpls = { rc = state.sq3Codes.SQLITE_IOERR; } storeAndNotify('xAccess', rc); + mTimeEnd(); }, xClose: async function(fid){ const opName = 'xClose'; + mTimeStart(opName); log(opName+"(",arguments[0],")"); const fh = __openFiles[fid]; if(fh){ @@ -166,6 +212,7 @@ const vfsAsyncImpls = { }else{ storeAndNotify(opName, state.sq3Codes.SQLITE_NOFOUND); } + mTimeEnd(); }, xDeleteNoWait: async function({filename, syncDir, recursive = false}){ /* The syncDir flag is, for purposes of the VFS API's semantics, @@ -202,20 +249,13 @@ const vfsAsyncImpls = { return rc; }, xDelete: async function(...args){ + mTimeStart('xDelete'); const rc = await vfsAsyncImpls.xDeleteNoWait(...args); storeAndNotify('xDelete', rc); - }, - mkdir: async function(dirname){ - let rc = 0; - try { - await getDirForPath(dirname+"/filepart", true); - }catch(e){ - //error("mkdir failed",filename, e.message); - rc = state.sq3Codes.SQLITE_IOERR; - } - storeAndNotify('mkdir', rc); + mTimeEnd(); }, xFileSize: async function(fid){ + mTimeStart('xFileSize'); log("xFileSize(",arguments,")"); const fh = __openFiles[fid]; let sz; @@ -228,6 +268,7 @@ const vfsAsyncImpls = { sz = state.sq3Codes.SQLITE_IOERR; } storeAndNotify('xFileSize', sz); + mTimeEnd(); }, xOpen: async function({ fid/*sqlite3_file pointer*/, @@ -237,6 +278,7 @@ const vfsAsyncImpls = { create = false, readOnly = false, deleteOnClose = false }){ const opName = 'xOpen'; + mTimeStart(opName); try{ if(create) readOnly = false; log(opName+"(",arguments[0],")"); @@ -245,6 +287,7 @@ const vfsAsyncImpls = { [hDir, filenamePart] = await getDirForPath(filename, !!create); }catch(e){ storeAndNotify(opName, state.sql3Codes.SQLITE_NOTFOUND); + mTimeEnd(); return; } const hFile = await hDir.getFileHandle(filenamePart, {create: !!create}); @@ -256,6 +299,7 @@ const vfsAsyncImpls = { fobj.fileHandle = hFile; fobj.fileType = fileType; fobj.sab = sab; + fobj.sabView = new Uint8Array(sab,0,state.fbInt64Offset); fobj.sabViewFileSize = new DataView(sab,state.fbInt64Offset,8); fobj.create = !!create; fobj.readOnly = !!readOnly; @@ -272,16 +316,20 @@ const vfsAsyncImpls = { error(opName,e); storeAndNotify(opName, state.sq3Codes.SQLITE_IOERR); } + mTimeEnd(); }, xRead: async function({fid,n,offset}){ + mTimeStart('xRead'); log("xRead(",arguments[0],")"); let rc = 0; - const fh = __openFiles[fid]; try{ - const aRead = new Uint8Array(fh.sab, 0, n); - const nRead = fh.accessHandle.read(aRead, {at: Number(offset)}); + const fh = __openFiles[fid]; + const nRead = fh.accessHandle.read( + fh.sabView.subarray(0, n), + {at: Number(offset)} + ); if(nRead < n){/* Zero-fill remaining bytes */ - new Uint8Array(fh.sab).fill(0, nRead, n); + fh.sabView.fill(0, nRead, n); rc = state.sq3Codes.SQLITE_IOERR_SHORT_READ; } }catch(e){ @@ -289,14 +337,18 @@ const vfsAsyncImpls = { rc = state.sq3Codes.SQLITE_IOERR_READ; } storeAndNotify('xRead',rc); + mTimeEnd(); }, xSync: async function({fid,flags/*ignored*/}){ + mTimeStart('xSync'); log("xSync(",arguments[0],")"); const fh = __openFiles[fid]; if(!fh.readOnly && fh.accessHandle) await fh.accessHandle.flush(); storeAndNotify('xSync',0); + mTimeEnd(); }, xTruncate: async function({fid,size}){ + mTimeStart('xTruncate'); log("xTruncate(",arguments[0],")"); let rc = 0; const fh = __openFiles[fid]; @@ -308,21 +360,25 @@ const vfsAsyncImpls = { rc = state.sq3Codes.SQLITE_IOERR_TRUNCATE; } storeAndNotify('xTruncate',rc); + mTimeEnd(); }, - xWrite: async function({fid,src,n,offset}){ + xWrite: async function({fid,n,offset}){ + mTimeStart('xWrite'); log("xWrite(",arguments[0],")"); let rc; - const fh = __openFiles[fid]; try{ + const fh = __openFiles[fid]; affirmNotRO('xWrite', fh); - const nOut = fh.accessHandle.write(new Uint8Array(fh.sab, 0, n), - {at: Number(offset)}); - rc = (nOut===n) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE; + rc = ( + n === fh.accessHandle.write(fh.sabView.subarray(0, n), + {at: Number(offset)}) + ) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE; }catch(e){ error("xWrite():",e,fh); rc = state.sq3Codes.SQLITE_IOERR_WRITE; } storeAndNotify('xWrite',rc); + mTimeEnd(); } }; @@ -348,6 +404,7 @@ navigator.storage.getDirectory().then(function(d){ toss("Maintenance required: missing state.opIds[",k,"]"); } }); + metrics.reset(); log("init state",state); wMsg('inited'); break; diff --git a/manifest b/manifest index 5af5e35d20..c2229acede 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Cut\sthe\sspeedtest1\sruntime\sof\sthe\sOPFS\sVFS\sproxy\sby\sapproximately\s3/4ths\svia\sxRead/xWrite\sbuffer-copying\soptimizations.\sStill\sslower\sthan\sthe\sWASMFS\simpl\sby\sapprox.\s1/5th. -D 2022-09-19T17:09:09.850 +C Further\smetrics\sand\sbuffer-copy\soptimizations\sin\sthe\sOPFS\sproxy,\sbut\swith\slittle\seffect. +D 2022-09-19T18:22:29.467 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -484,7 +484,7 @@ F ext/wasm/api/post-js-header.js 0e853b78db83cb1c06b01663549e0e8b4f377f12f5a2d9a F ext/wasm/api/sqlite3-api-cleanup.js 8564a6077cdcaea9a9f428a019af8a05887f0131e6a2a1e72a7ff1145fadfe77 F ext/wasm/api/sqlite3-api-glue.js 366d580c8e5bf7fcf4c6dee6f646c31f5549bd417ea03a59a0acca00e8ecce30 F ext/wasm/api/sqlite3-api-oo1.js 2d13dddf0d2b4168a9249f124134d37924331e5b55e05dba18b6d661fbeefe48 -F ext/wasm/api/sqlite3-api-opfs.js 627b2d8fcd37b6479d7f0e8d5ba891d00c1eaf9b8cf99981081c2bf2e7ed8560 +F ext/wasm/api/sqlite3-api-opfs.js df3d085a55be11a0b7bce3d42ea1b721ff8aaba93659d0ec48c3327dde384596 F ext/wasm/api/sqlite3-api-prologue.js 0d2639387b94c30f492d4aea6e44fb7b16720808678464559458fd2ae3759655 F ext/wasm/api/sqlite3-api-worker1.js ee4cf149cbacb63d06b536674f822aa5088b7e022cdffc69f1f36cebe2f9fea0 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 @@ -519,7 +519,7 @@ F ext/wasm/speedtest1.html 512addeb3c27c94901178b7bcbde83a6f95c093f9ebe16a2959a0 F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5 -F ext/wasm/sqlite3-opfs-async-proxy.js 7139530dd8a408f2897115d3ea04dfe03f175df66870b1e8a49733a77a06f970 +F ext/wasm/sqlite3-opfs-async-proxy.js f1a270e7a8adeb80c73e0b345e472e574e694d9eaec6588ce3671438b79eb351 F ext/wasm/sqlite3-worker1-promiser.js 4fd0465688a28a75f1d4ee4406540ba494f49844e3cad0670d0437a001943365 F ext/wasm/sqlite3-worker1.js 0c1e7626304543969c3846573e080c082bf43bcaa47e87d416458af84f340a9e F ext/wasm/test-opfs-vfs.html eb69dda21eb414b8f5e3f7c1cc0f774103cc9c0f87b2d28a33419e778abfbab5 @@ -2026,8 +2026,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 976053925013bf1975f5f9222e28ba648af28e305bb6bdae600eb24d0e136bec -R 79e6bae24f2a8f4f545c24b5d2562e72 +P fb7f287310d74a3e236125ae9c49b859f9263c29ae85161c1bcf9dd0778d8a51 +R 946ef52a6b1d5b2d6cbf3b063dacd006 U stephan -Z 251c239274312fdfb8787b5b1f3a9b52 +Z f42132e655dd81eba8154f54be0c97de # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6b25039bc8..e76b3ad26d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fb7f287310d74a3e236125ae9c49b859f9263c29ae85161c1bcf9dd0778d8a51 \ No newline at end of file +d1f1fe6f1c60640f7770dfb9245c459a09b8d24ec2ddf664dff77c810bd51f96 \ No newline at end of file