1
0
mirror of https://github.com/sqlite/sqlite.git synced 2025-06-07 19:02:21 +03:00

OPFS VFS: moved i/o buffer from per-file to the VFS, and related refactoring, in prep for experimentation with a new inter-worker comms model.

FossilOrigin-Name: d4d63e4580ad8d497310608175308c03c517e051d7865cb66aa0b10356612d7d
This commit is contained in:
stephan 2022-09-20 01:28:47 +00:00
parent aec046a264
commit c4b87be3e8
4 changed files with 65 additions and 60 deletions

View File

@ -188,17 +188,22 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
const state = Object.create(null); const state = Object.create(null);
state.verbose = options.verbose; state.verbose = options.verbose;
state.fileBufferSize = state.fileBufferSize =
1024 * 64 + 8 /* size of aFileHandle.sab. 64k = max sqlite3 page 1024 * 64 /* size of aFileHandle.sab. 64k = max sqlite3 page
size. The additional bytes are space for size. */;
holding BigInt results, since we cannot store state.sabOffsetS11n = state.fileBufferSize;
those via the Atomics API (which only works on state.sabIO = new SharedArrayBuffer(
an Int32Array). */; state.fileBufferSize
+ 4096/* arg/result serialization */
+ 8 /* to be removed - porting crutch */
);
state.fbInt64Offset = state.fbInt64Offset =
state.fileBufferSize - 8 /*spot in fileHandle.sab to store an int64 result */; state.sabIO.byteLength - 8 /*spot in fileHandle.sab to store an int64 result.
to be removed. Porting crutch. */;
state.opIds = Object.create(null); state.opIds = Object.create(null);
const metrics = Object.create(null); const metrics = Object.create(null);
{ {
let i = 0; let i = 0;
state.opIds.nothing = i++;
state.opIds.xAccess = i++; state.opIds.xAccess = i++;
state.opIds.xClose = i++; state.opIds.xClose = i++;
state.opIds.xDelete = i++; state.opIds.xDelete = i++;
@ -211,7 +216,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
state.opIds.xTruncate = i++; state.opIds.xTruncate = i++;
state.opIds.xWrite = i++; state.opIds.xWrite = i++;
state.opIds.mkdir = i++; state.opIds.mkdir = i++;
state.opSAB = new SharedArrayBuffer(i * 4/*sizeof int32*/); state.sabOP = new SharedArrayBuffer(i * 4/*sizeof int32*/);
state.opIds.xFileControl = state.opIds.xSync /* special case */; state.opIds.xFileControl = state.opIds.xSync /* special case */;
opfsUtil.metrics.reset(); opfsUtil.metrics.reset();
} }
@ -225,7 +230,9 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
'SQLITE_IOERR_WRITE', 'SQLITE_IOERR_FSYNC', 'SQLITE_IOERR_WRITE', 'SQLITE_IOERR_FSYNC',
'SQLITE_IOERR_TRUNCATE', 'SQLITE_IOERR_DELETE', 'SQLITE_IOERR_TRUNCATE', 'SQLITE_IOERR_DELETE',
'SQLITE_IOERR_ACCESS', 'SQLITE_IOERR_CLOSE', 'SQLITE_IOERR_ACCESS', 'SQLITE_IOERR_CLOSE',
'SQLITE_IOERR_DELETE' 'SQLITE_IOERR_DELETE',
'SQLITE_OPEN_CREATE', 'SQLITE_OPEN_DELETEONCLOSE',
'SQLITE_OPEN_READONLY'
].forEach(function(k){ ].forEach(function(k){
state.sq3Codes[k] = capi[k] || toss("Maintenance required: not found:",k); state.sq3Codes[k] = capi[k] || toss("Maintenance required: not found:",k);
state.sq3Codes._reverse[capi[k]] = k; state.sq3Codes._reverse[capi[k]] = k;
@ -236,17 +243,17 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
/** /**
Runs the given operation in the async worker counterpart, waits Runs the given operation in the async worker counterpart, waits
for its response, and returns the result which the async worker for its response, and returns the result which the async worker
writes to the given op's index in state.opSABView. The 2nd argument writes to the given op's index in state.sabOPView. The 2nd argument
must be a single object or primitive value, depending on the must be a single object or primitive value, depending on the
given operation's signature in the async API counterpart. given operation's signature in the async API counterpart.
*/ */
const opRun = (op,args)=>{ const opRun = (op,args)=>{
const t = performance.now(); const t = performance.now();
Atomics.store(state.opSABView, state.opIds[op], -1); Atomics.store(state.sabOPView, state.opIds[op], -1);
wMsg(op, args); wMsg(op, args);
Atomics.wait(state.opSABView, state.opIds[op], -1); Atomics.wait(state.sabOPView, state.opIds[op], -1);
metrics[op].wait += performance.now() - t; metrics[op].wait += performance.now() - t;
return Atomics.load(state.opSABView, state.opIds[op]); return Atomics.load(state.sabOPView, state.opIds[op]);
}; };
/** /**
@ -574,10 +581,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
args.fid = pFile; args.fid = pFile;
args.filename = zName; args.filename = zName;
args.sab = new SharedArrayBuffer(state.fileBufferSize); args.sab = new SharedArrayBuffer(state.fileBufferSize);
args.fileType = f._.getFileType(args.filename, flags); args.flags = flags;
args.create = !!(flags & capi.SQLITE_OPEN_CREATE);
args.deleteOnClose = !!(flags & capi.SQLITE_OPEN_DELETEONCLOSE);
args.readOnly = !!(flags & capi.SQLITE_OPEN_READONLY);
const rc = opRun('xOpen', args); const rc = opRun('xOpen', args);
if(!rc){ if(!rc){
/* Recall that sqlite3_vfs::xClose() will be called, even on /* Recall that sqlite3_vfs::xClose() will be called, even on
@ -586,8 +590,8 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
wasm.setMemValue(pOutFlags, capi.SQLITE_OPEN_READONLY, 'i32'); wasm.setMemValue(pOutFlags, capi.SQLITE_OPEN_READONLY, 'i32');
} }
__openFiles[pFile] = args; __openFiles[pFile] = args;
args.sabView = new Uint8Array(args.sab); args.sabView = new Uint8Array(state.sabIO, 0, state.fileBufferSize);
args.sabViewFileSize = new DataView(args.sab, state.fbInt64Offset, 8); args.sabViewFileSize = new DataView(state.sabIO, state.fbInt64Offset, 8);
args.sq3File = new sqlite3_file(pFile); args.sq3File = new sqlite3_file(pFile);
args.sq3File.$pMethods = opfsIoMethods.pointer; args.sq3File.$pMethods = opfsIoMethods.pointer;
args.ba = new Uint8Array(args.sab); args.ba = new Uint8Array(args.sab);
@ -615,7 +619,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
assume it's sane and use it, otherwise install a JS-based assume it's sane and use it, otherwise install a JS-based
one. */ one. */
vfsSyncWrappers.xSleep = function(pVfs,ms){ vfsSyncWrappers.xSleep = function(pVfs,ms){
Atomics.wait(state.opSABView, state.opIds.xSleep, 0, ms); Atomics.wait(state.sabOPView, state.opIds.xSleep, 0, ms);
return 0; return 0;
}; };
} }
@ -702,8 +706,8 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
log("xAccess(",dbFile,") exists ?=",rc); log("xAccess(",dbFile,") exists ?=",rc);
rc = vfsSyncWrappers.xOpen(opfsVfs.pointer, zDbFile, rc = vfsSyncWrappers.xOpen(opfsVfs.pointer, zDbFile,
fid, openFlags, pOut); fid, openFlags, pOut);
log("open rc =",rc,"state.opSABView[xOpen] =", log("open rc =",rc,"state.sabOPView[xOpen] =",
state.opSABView[state.opIds.xOpen]); state.sabOPView[state.opIds.xOpen]);
if(isWorkerErrCode(rc)){ if(isWorkerErrCode(rc)){
error("open failed with code",rc); error("open failed with code",rc);
return; return;
@ -733,7 +737,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
log("waking up from xSleep()"); log("waking up from xSleep()");
} }
rc = ioSyncWrappers.xClose(fid); rc = ioSyncWrappers.xClose(fid);
log("xClose rc =",rc,"opSABView =",state.opSABView); log("xClose rc =",rc,"sabOPView =",state.sabOPView);
log("Deleting file:",dbFile); log("Deleting file:",dbFile);
vfsSyncWrappers.xDelete(opfsVfs.pointer, zDbFile, 0x1234); vfsSyncWrappers.xDelete(opfsVfs.pointer, zDbFile, 0x1234);
vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut);
@ -767,7 +771,8 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
toss("BUG: sqlite3_vfs_find() failed for just-installed OPFS VFS"); toss("BUG: sqlite3_vfs_find() failed for just-installed OPFS VFS");
} }
capi.sqlite3_vfs_register.addReference(opfsVfs, opfsIoMethods); capi.sqlite3_vfs_register.addReference(opfsVfs, opfsIoMethods);
state.opSABView = new Int32Array(state.opSAB); state.sabOPView = new Int32Array(state.sabOP);
state.sabFileBufView = new Uint8Array(state.sabFileBufView, 0, state.fileBufferSize);
if(options.sanityChecks){ if(options.sanityChecks){
warn("Running sanity checks because of opfs-sanity-check URL arg..."); warn("Running sanity checks because of opfs-sanity-check URL arg...");
sanityCheck(); sanityCheck();

View File

@ -131,8 +131,8 @@ const getDirForPath = async function f(absFilename, createDirs = false){
*/ */
const storeAndNotify = (opName, value)=>{ const storeAndNotify = (opName, value)=>{
log(opName+"() is notify()ing w/ value:",value); log(opName+"() is notify()ing w/ value:",value);
Atomics.store(state.opSABView, state.opIds[opName], value); Atomics.store(state.sabOPView, state.opIds[opName], value);
Atomics.notify(state.opSABView, state.opIds[opName]); Atomics.notify(state.sabOPView, state.opIds[opName]);
}; };
/** /**
@ -214,6 +214,12 @@ const vfsAsyncImpls = {
} }
mTimeEnd(); mTimeEnd();
}, },
xDelete: async function(...args){
mTimeStart('xDelete');
const rc = await vfsAsyncImpls.xDeleteNoWait(...args);
storeAndNotify('xDelete', rc);
mTimeEnd();
},
xDeleteNoWait: async function({filename, syncDir, recursive = false}){ xDeleteNoWait: async function({filename, syncDir, recursive = false}){
/* The syncDir flag is, for purposes of the VFS API's semantics, /* The syncDir flag is, for purposes of the VFS API's semantics,
ignored here. However, if it has the value 0x1234 then: after ignored here. However, if it has the value 0x1234 then: after
@ -248,12 +254,6 @@ const vfsAsyncImpls = {
} }
return rc; return rc;
}, },
xDelete: async function(...args){
mTimeStart('xDelete');
const rc = await vfsAsyncImpls.xDeleteNoWait(...args);
storeAndNotify('xDelete', rc);
mTimeEnd();
},
xFileSize: async function(fid){ xFileSize: async function(fid){
mTimeStart('xFileSize'); mTimeStart('xFileSize');
log("xFileSize(",arguments,")"); log("xFileSize(",arguments,")");
@ -261,6 +261,9 @@ const vfsAsyncImpls = {
let sz; let sz;
try{ try{
sz = await fh.accessHandle.getSize(); sz = await fh.accessHandle.getSize();
if(!fh.sabViewFileSize){
fh.sabViewFileSize = new DataView(state.sabIO,state.fbInt64Offset,8);
}
fh.sabViewFileSize.setBigInt64(0, BigInt(sz), true); fh.sabViewFileSize.setBigInt64(0, BigInt(sz), true);
sz = 0; sz = 0;
}catch(e){ }catch(e){
@ -272,16 +275,15 @@ const vfsAsyncImpls = {
}, },
xOpen: async function({ xOpen: async function({
fid/*sqlite3_file pointer*/, fid/*sqlite3_file pointer*/,
sab/*file-specific SharedArrayBuffer*/,
filename, filename,
fileType = undefined /*mainDb, mainJournal, etc.*/, flags
create = false, readOnly = false, deleteOnClose = false
}){ }){
const opName = 'xOpen'; const opName = 'xOpen';
mTimeStart(opName); mTimeStart(opName);
try{
if(create) readOnly = false;
log(opName+"(",arguments[0],")"); log(opName+"(",arguments[0],")");
const deleteOnClose = (state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags);
const create = (state.sq3Codes.SQLITE_OPEN_CREATE & flags);
try{
let hDir, filenamePart; let hDir, filenamePart;
try { try {
[hDir, filenamePart] = await getDirForPath(filename, !!create); [hDir, filenamePart] = await getDirForPath(filename, !!create);
@ -290,20 +292,8 @@ const vfsAsyncImpls = {
mTimeEnd(); mTimeEnd();
return; return;
} }
const hFile = await hDir.getFileHandle(filenamePart, {create: !!create}); const hFile = await hDir.getFileHandle(filenamePart, {create});
log(opName,"filenamePart =",filenamePart, 'hDir =',hDir); const fobj = Object.create(null);
const fobj = __openFiles[fid] = Object.create(null);
fobj.filenameAbs = filename;
fobj.filenamePart = filenamePart;
fobj.dirHandle = hDir;
fobj.fileHandle = hFile;
fobj.fileType = fileType;
fobj.sab = sab;
fobj.sabView = new Uint8Array(sab,0,state.fbInt64Offset);
fobj.sabViewFileSize = new DataView(sab,state.fbInt64Offset,8);
fobj.create = !!create;
fobj.readOnly = !!readOnly;
fobj.deleteOnClose = !!deleteOnClose;
/** /**
wa-sqlite, at this point, grabs a SyncAccessHandle and wa-sqlite, at this point, grabs a SyncAccessHandle and
assigns it to the accessHandle prop of the file state assigns it to the accessHandle prop of the file state
@ -311,6 +301,14 @@ const vfsAsyncImpls = {
places that limitation on it. places that limitation on it.
*/ */
fobj.accessHandle = await hFile.createSyncAccessHandle(); fobj.accessHandle = await hFile.createSyncAccessHandle();
__openFiles[fid] = fobj;
fobj.filenameAbs = filename;
fobj.filenamePart = filenamePart;
fobj.dirHandle = hDir;
fobj.fileHandle = hFile;
fobj.sabView = state.sabFileBufView;
fobj.readOnly = create ? false : (state.sq3Codes.SQLITE_OPEN_READONLY & flags);
fobj.deleteOnClose = deleteOnClose;
storeAndNotify(opName, 0); storeAndNotify(opName, 0);
}catch(e){ }catch(e){
error(opName,e); error(opName,e);
@ -395,8 +393,10 @@ navigator.storage.getDirectory().then(function(d){
state.verbose = opt.verbose ?? 2; state.verbose = opt.verbose ?? 2;
state.fileBufferSize = opt.fileBufferSize; state.fileBufferSize = opt.fileBufferSize;
state.fbInt64Offset = opt.fbInt64Offset; state.fbInt64Offset = opt.fbInt64Offset;
state.opSAB = opt.opSAB; state.sabOP = opt.sabOP;
state.opSABView = new Int32Array(state.opSAB); state.sabOPView = new Int32Array(state.sabOP);
state.sabIO = opt.sabIO;
state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize);
state.opIds = opt.opIds; state.opIds = opt.opIds;
state.sq3Codes = opt.sq3Codes; state.sq3Codes = opt.sq3Codes;
Object.keys(vfsAsyncImpls).forEach((k)=>{ Object.keys(vfsAsyncImpls).forEach((k)=>{

View File

@ -1,5 +1,5 @@
C Further\smetrics\sand\sbuffer-copy\soptimizations\sin\sthe\sOPFS\sproxy,\sbut\swith\slittle\seffect. C OPFS\sVFS:\smoved\si/o\sbuffer\sfrom\sper-file\sto\sthe\sVFS,\sand\srelated\srefactoring,\sin\sprep\sfor\sexperimentation\swith\sa\snew\sinter-worker\scomms\smodel.
D 2022-09-19T18:22:29.467 D 2022-09-20T01:28:47.162
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@ -484,7 +484,7 @@ F ext/wasm/api/post-js-header.js 0e853b78db83cb1c06b01663549e0e8b4f377f12f5a2d9a
F ext/wasm/api/sqlite3-api-cleanup.js 8564a6077cdcaea9a9f428a019af8a05887f0131e6a2a1e72a7ff1145fadfe77 F ext/wasm/api/sqlite3-api-cleanup.js 8564a6077cdcaea9a9f428a019af8a05887f0131e6a2a1e72a7ff1145fadfe77
F ext/wasm/api/sqlite3-api-glue.js 366d580c8e5bf7fcf4c6dee6f646c31f5549bd417ea03a59a0acca00e8ecce30 F ext/wasm/api/sqlite3-api-glue.js 366d580c8e5bf7fcf4c6dee6f646c31f5549bd417ea03a59a0acca00e8ecce30
F ext/wasm/api/sqlite3-api-oo1.js 2d13dddf0d2b4168a9249f124134d37924331e5b55e05dba18b6d661fbeefe48 F ext/wasm/api/sqlite3-api-oo1.js 2d13dddf0d2b4168a9249f124134d37924331e5b55e05dba18b6d661fbeefe48
F ext/wasm/api/sqlite3-api-opfs.js df3d085a55be11a0b7bce3d42ea1b721ff8aaba93659d0ec48c3327dde384596 F ext/wasm/api/sqlite3-api-opfs.js f8027fb4af1c24fcfad31889f8a5ccfa3b96d6e812d3495b13833bef57034046
F ext/wasm/api/sqlite3-api-prologue.js 0d2639387b94c30f492d4aea6e44fb7b16720808678464559458fd2ae3759655 F ext/wasm/api/sqlite3-api-prologue.js 0d2639387b94c30f492d4aea6e44fb7b16720808678464559458fd2ae3759655
F ext/wasm/api/sqlite3-api-worker1.js ee4cf149cbacb63d06b536674f822aa5088b7e022cdffc69f1f36cebe2f9fea0 F ext/wasm/api/sqlite3-api-worker1.js ee4cf149cbacb63d06b536674f822aa5088b7e022cdffc69f1f36cebe2f9fea0
F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9
@ -519,7 +519,7 @@ F ext/wasm/speedtest1.html 512addeb3c27c94901178b7bcbde83a6f95c093f9ebe16a2959a0
F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x
F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0
F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5 F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5
F ext/wasm/sqlite3-opfs-async-proxy.js f1a270e7a8adeb80c73e0b345e472e574e694d9eaec6588ce3671438b79eb351 F ext/wasm/sqlite3-opfs-async-proxy.js 7ebc36915cd61bd4f067a4823307f4d4eb2678a173aaae470c534e8fe9cda650
F ext/wasm/sqlite3-worker1-promiser.js 4fd0465688a28a75f1d4ee4406540ba494f49844e3cad0670d0437a001943365 F ext/wasm/sqlite3-worker1-promiser.js 4fd0465688a28a75f1d4ee4406540ba494f49844e3cad0670d0437a001943365
F ext/wasm/sqlite3-worker1.js 0c1e7626304543969c3846573e080c082bf43bcaa47e87d416458af84f340a9e F ext/wasm/sqlite3-worker1.js 0c1e7626304543969c3846573e080c082bf43bcaa47e87d416458af84f340a9e
F ext/wasm/test-opfs-vfs.html eb69dda21eb414b8f5e3f7c1cc0f774103cc9c0f87b2d28a33419e778abfbab5 F ext/wasm/test-opfs-vfs.html eb69dda21eb414b8f5e3f7c1cc0f774103cc9c0f87b2d28a33419e778abfbab5
@ -2026,8 +2026,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P fb7f287310d74a3e236125ae9c49b859f9263c29ae85161c1bcf9dd0778d8a51 P d1f1fe6f1c60640f7770dfb9245c459a09b8d24ec2ddf664dff77c810bd51f96
R 946ef52a6b1d5b2d6cbf3b063dacd006 R e5a1ff53ebe902985ab8e87bba2c71fb
U stephan U stephan
Z f42132e655dd81eba8154f54be0c97de Z d05962442428c5dae722ab00dca47e48
# Remove this line to create a well-formed Fossil manifest. # Remove this line to create a well-formed Fossil manifest.

View File

@ -1 +1 @@
d1f1fe6f1c60640f7770dfb9245c459a09b8d24ec2ddf664dff77c810bd51f96 d4d63e4580ad8d497310608175308c03c517e051d7865cb66aa0b10356612d7d