database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-08-01 06:27:03 +03:00
Code Activity

Add the remaining vfs/io_methods wrappers to the OPFS sync/async proxy, but most are not yet tested.

Browse Source 
FossilOrigin-Name: 44db9132145b3072488ea91db53f6c06be74544beccad5fd07efd22c0f03dc04
This commit is contained in:
stephan
2022-09-17 20:50:12 +00:00
parent 132a87baa3
commit 0731554629
5 changed files with 671 additions and 335 deletions
Download Patch File Download Diff File Expand all files Collapse all files

537
ext/wasm/sqlite3-opfs-async-proxy.js
View File

@ -20,267 +20,308 @@
https://github.com/rhashimoto/wa-sqlite/blob/master/src/examples/OriginPrivateFileSystemVFS.js https://github.com/rhashimoto/wa-sqlite/blob/master/src/examples/OriginPrivateFileSystemVFS.js
for demonstrating how to use the OPFS APIs. for demonstrating how to use the OPFS APIs.
This file is to be loaded as a Worker. It does not have any direct
access to the sqlite3 JS/WASM bits, so any bits which it needs (most
notably SQLITE_xxx integer codes) have to be imported into it via an
initialization process.
*/ */
'use strict'; 'use strict';
(function(){ const toss = function(...args){throw new Error(args.join(' '))};
const toss = function(...args){throw new Error(args.join(' '))}; if(self.window === self){
if(self.window === self){ toss("This code cannot run from the main thread.",
toss("This code cannot run from the main thread.", "Load it as a Worker from a separate Worker.");
"Load it as a Worker from a separate Worker."); }else if(!navigator.storage.getDirectory){
}else if(!navigator.storage.getDirectory){ toss("This API requires navigator.storage.getDirectory.");
toss("This API requires navigator.storage.getDirectory."); }
/**
Will hold state copied to this object from the syncronous side of
this API.
*/
const state = Object.create(null);
/**
verbose:
0 = no logging output
1 = only errors
2 = warnings and errors
3 = debug, warnings, and errors
*/
state.verbose = 2;
const __logPrefix = "OPFS asyncer:";
const log = (...args)=>{
if(state.verbose>2) console.log(__logPrefix,...args);
};
const warn = (...args)=>{
if(state.verbose>1) console.warn(__logPrefix,...args);
};
const error = (...args)=>{
if(state.verbose) console.error(__logPrefix,...args);
};
warn("This file is very much experimental and under construction.",self.location.pathname);
/**
Map of sqlite3_file pointers (integers) to metadata related to a
given OPFS file handles. The pointers are, in this side of the
interface, opaque file handle IDs provided by the synchronous
part of this constellation. Each value is an object with a structure
demonstrated in the xOpen() impl.
*/
const __openFiles = Object.create(null);
/**
Map of dir names to FileSystemDirectoryHandle objects.
*/
const __dirCache = new Map;
/**
Takes the absolute path to a filesystem element. Returns an array
of [handleOfContainingDir, filename]. If the 2nd argument is
truthy then each directory element leading to the file is created
along the way. Throws if any creation or resolution fails.
*/
const getDirForPath = async function f(absFilename, createDirs = false){
const url = new URL(
absFilename, 'file://xyz'
) /* use URL to resolve path pieces such as a/../b */;
const path = url.pathname.split('/').filter((v)=>!!v);
const filename = path.pop();
const allDirs = '/'+path.join('/');
let dh = __dirCache.get(allDirs);
if(!dh){
dh = state.rootDir;
for(const dirName of path){
if(dirName){
dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs});
}
}
__dirCache.set(allDirs, dh);
} }
const logPrefix = "OPFS worker:"; return [dh, filename];
const log = (...args)=>{ };
console.log(logPrefix,...args);
};
const warn = (...args)=>{
console.warn(logPrefix,...args);
};
const error = (...args)=>{
console.error(logPrefix,...args);
};
warn("This file is very much experimental and under construction.",self.location.pathname);
const wMsg = (type,payload)=>postMessage({type,payload});
const state = Object.create(null); /**
/*state.opSab; Stores the given value at the array index reserved for the given op
state.sabIO; and then Atomics.notify()'s it.
state.opBuf; */
state.opIds; const storeAndNotify = (opName, value)=>{
state.rootDir;*/ log(opName+"() is notify()ing w/ value:",value);
/** Atomics.store(state.opBuf, state.opIds[opName], value);
Map of sqlite3_file pointers (integers) to metadata related to a Atomics.notify(state.opBuf, state.opIds[opName]);
given OPFS file handles. The pointers are, in this side of the };
interface, opaque file handle IDs provided by the synchronous
part of this constellation. Each value is an object with a structure
demonstrated in the xOpen() impl.
*/
state.openFiles = Object.create(null);
/** const isInt32 = function(n){
Map of dir names to FileSystemDirectoryHandle objects. return ('bigint'!==typeof n /*TypeError: can't convert BigInt to number*/)
*/ && !!(n===(n|0) && n<=2147483647 && n>=-2147483648);
state.dirCache = new Map; };
const affirm32Bits = function(n){
return isInt32(n) || toss("Number is too large (>31 bits) (FIXME!):",n);
};
const __splitPath = (absFilename)=>{ /**
const a = absFilename.split('/').filter((v)=>!!v); Throws if fh is a file-holding object which is flagged as read-only.
return [a, a.pop()]; */
}; const affirmNotRO = function(opName,fh){
/** if(fh.readOnly) toss(opName+"(): File is read-only: "+fh.filenameAbs);
Takes the absolute path to a filesystem element. Returns an array };
of [handleOfContainingDir, filename]. If the 2nd argument is
truthy then each directory element leading to the file is created /**
along the way. Throws if any creation or resolution fails. Asynchronous wrappers for sqlite3_vfs and sqlite3_io_methods
*/ methods. Maintenance reminder: members are in alphabetical order
const getDirForPath = async function f(absFilename, createDirs = false){ to simplify finding them.
const url = new URL( */
absFilename, 'file://xyz' const vfsAsyncImpls = {
) /* use URL to resolve path pieces such as a/../b */; xAccess: async function({filename, exists, readWrite}){
const [path, filename] = __splitPath(url.pathname); warn("xAccess(",arguments[0],") is TODO");
const allDirs = path.join('/'); const rc = state.sq3Codes.SQLITE_IOERR;
let dh = state.dirCache.get(allDirs); storeAndNotify('xAccess', rc);
if(!dh){ },
dh = state.rootDir; xClose: async function(fid){
for(const dirName of path){ const opName = 'xClose';
if(dirName){ log(opName+"(",arguments[0],")");
dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs}); const fh = __openFiles[fid];
} if(fh){
delete __openFiles[fid];
if(fh.accessHandle) await fh.accessHandle.close();
if(fh.deleteOnClose){
try{ await fh.dirHandle.removeEntry(fh.filenamePart) }
catch(e){ warn("Ignoring dirHandle.removeEntry() failure of",fh,e) }
} }
state.dirCache.set(allDirs, dh); storeAndNotify(opName, 0);
}else{
storeAndNotify(opName, state.sq3Codes.SQLITE_NOFOUND);
} }
return [dh, filename]; },
}; xDelete: async function({filename, syncDir/*ignored*/}){
log("xDelete(",arguments[0],")");
try {
/** const [hDir, filenamePart] = await getDirForPath(filename, false);
Generates a random ASCII string len characters long, intended for await hDir.removeEntry(filenamePart);
use as a temporary file name. }catch(e){
*/ /* Ignoring: _presumably_ the file can't be found. */
const randomFilename = function f(len=16){
if(!f._chars){
f._chars = "abcdefghijklmnopqrstuvwxyz"+
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"+
"012346789";
f._n = f._chars.length;
} }
const a = []; storeAndNotify('xDelete', 0);
let i = 0; },
for( ; i < len; ++i){ xFileSize: async function(fid){
const ndx = Math.random() * (f._n * 64) % f._n | 0; log("xFileSize(",arguments,")");
a[i] = f._chars[ndx]; const fh = __openFiles[fid];
let sz;
try{
sz = await fh.accessHandle.getSize();
fh.sabViewFileSize.setBigInt64(0, BigInt(sz));
sz = 0;
}catch(e){
error("xFileSize():",e, fh);
sz = state.sq3Codes.SQLITE_IOERR;
} }
return a.join(''); storeAndNotify('xFileSize', sz);
}; },
xOpen: async function({
const storeAndNotify = (opName, value)=>{ fid/*sqlite3_file pointer*/,
log(opName+"() is notify()ing w/ value:",value); sab/*file-specific SharedArrayBuffer*/,
Atomics.store(state.opBuf, state.opIds[opName], value); filename,
Atomics.notify(state.opBuf, state.opIds[opName]); fileType = undefined /*mainDb, mainJournal, etc.*/,
}; create = false, readOnly = false, deleteOnClose = false
}){
const isInt32 = function(n){ const opName = 'xOpen';
return ('bigint'!==typeof n /*TypeError: can't convert BigInt to number*/) try{
&& !!(n===(n|0) && n<=2147483647 && n>=-2147483648); if(create) readOnly = false;
};
const affirm32Bits = function(n){
return isInt32(n) || toss("Number is too large (>31 bits):",n);
};
const ioMethods = {
xAccess: async function({filename, exists, readWrite}){
log("xAccess(",arguments,")");
const rc = 1;
storeAndNotify('xAccess', rc);
},
xClose: async function(fid){
const opName = 'xClose';
log(opName+"(",arguments[0],")"); log(opName+"(",arguments[0],")");
log("state.openFiles",state.openFiles); let hDir, filenamePart;
const fh = state.openFiles[fid]; try {
if(fh){ [hDir, filenamePart] = await getDirForPath(filename, !!create);
delete state.openFiles[fid];
//await fh.close();
if(fh.accessHandle) await fh.accessHandle.close();
if(fh.deleteOnClose){
try{
await fh.dirHandle.removeEntry(fh.filenamePart);
}
catch(e){
warn("Ignoring dirHandle.removeEntry() failure of",fh);
}
}
log("state.openFiles",state.openFiles);
storeAndNotify(opName, 0);
}else{
storeAndNotify(opName, state.errCodes.NotFound);
}
},
xDelete: async function(filename){
log("xDelete(",arguments,")");
storeAndNotify('xClose', 0);
},
xFileSize: async function(fid){
log("xFileSize(",arguments,")");
const fh = state.openFiles[fid];
const sz = await fh.getSize();
affirm32Bits(sz);
storeAndNotify('xFileSize', sz | 0);
},
xOpen: async function({
fid/*sqlite3_file pointer*/, sab/*file-specific SharedArrayBuffer*/,
filename,
fileType = undefined /*mainDb, mainJournal, etc.*/,
create = false, readOnly = false, deleteOnClose = false,
}){
const opName = 'xOpen';
try{
if(create) readOnly = false;
log(opName+"(",arguments[0],")");
let hDir, filenamePart, hFile;
try {
[hDir, filenamePart] = await getDirForPath(filename, !!create);
}catch(e){
storeAndNotify(opName, state.errCodes.NotFound);
return;
}
hFile = await hDir.getFileHandle(filenamePart, {create: !!create});
log(opName,"filenamePart =",filenamePart, 'hDir =',hDir);
const fobj = state.openFiles[fid] = Object.create(null);
fobj.filenameAbs = filename;
fobj.filenamePart = filenamePart;
fobj.dirHandle = hDir;
fobj.fileHandle = hFile;
fobj.accessHandle = undefined;
fobj.fileType = fileType;
fobj.sab = sab;
fobj.create = !!create;
fobj.readOnly = !!readOnly;
fobj.deleteOnClose = !!deleteOnClose;
/**
wa-sqlite, at this point, grabs a SyncAccessHandle and
assigns it to the accessHandle prop of the file state
object, but it's unclear why it does that.
*/
storeAndNotify(opName, 0);
}catch(e){ }catch(e){
error(opName,e); storeAndNotify(opName, state.sql3Codes.SQLITE_NOTFOUND);
storeAndNotify(opName, state.errCodes.IO); return;
} }
}, const hFile = await hDir.getFileHandle(filenamePart, {create: !!create});
xRead: async function({fid,n,offset}){ log(opName,"filenamePart =",filenamePart, 'hDir =',hDir);
log("xRead(",arguments,")"); const fobj = __openFiles[fid] = Object.create(null);
affirm32Bits(n + offset); fobj.filenameAbs = filename;
const fh = state.openFiles[fid]; fobj.filenamePart = filenamePart;
storeAndNotify('xRead',fid); fobj.dirHandle = hDir;
}, fobj.fileHandle = hFile;
xSleep: async function f({ms}){ fobj.fileType = fileType;
log("xSleep(",arguments[0],")"); fobj.sab = sab;
await new Promise((resolve)=>{ fobj.sabViewFileSize = new DataView(sab,state.fbInt64Offset,8);
setTimeout(()=>resolve(), ms); fobj.create = !!create;
}).finally(()=>storeAndNotify('xSleep',0)); fobj.readOnly = !!readOnly;
}, fobj.deleteOnClose = !!deleteOnClose;
xSync: async function({fid}){ /**
log("xSync(",arguments,")"); wa-sqlite, at this point, grabs a SyncAccessHandle and
const fh = state.openFiles[fid]; assigns it to the accessHandle prop of the file state
await fh.flush(); object, but only for certain cases and it's unclear why it
storeAndNotify('xSync',fid); places that limitation on it.
}, */
xTruncate: async function({fid,size}){ fobj.accessHandle = await hFile.createSyncAccessHandle();
log("xTruncate(",arguments,")"); storeAndNotify(opName, 0);
affirm32Bits(size); }catch(e){
const fh = state.openFiles[fid]; error(opName,e);
fh.truncate(size); storeAndNotify(opName, state.sq3Codes.SQLITE_IOERR);
storeAndNotify('xTruncate',fid); }
}, },
xWrite: async function({fid,src,n,offset}){ xRead: async function({fid,n,offset}){
log("xWrite(",arguments,")"); log("xRead(",arguments[0],")");
const fh = state.openFiles[fid]; let rc = 0;
storeAndNotify('xWrite',fid); const fh = __openFiles[fid];
try{
const aRead = new Uint8array(fh.sab, n);
const nRead = fh.accessHandle.read(aRead, {at: offset});
if(nRead < n){/* Zero-fill remaining bytes */
new Uint8array(fh.sab).fill(0, nRead, n);
rc = state.sq3Codes.SQLITE_IOERR_SHORT_READ;
}
}catch(e){
error("xRead() failed",e,fh);
rc = state.sq3Codes.SQLITE_IOERR_READ;
}
storeAndNotify('xRead',rc);
},
xSleep: async function f(ms){
log("xSleep(",ms,")");
await new Promise((resolve)=>{
setTimeout(()=>resolve(), ms);
}).finally(()=>storeAndNotify('xSleep',0));
},
xSync: async function({fid,flags/*ignored*/}){
log("xSync(",arguments[0],")");
const fh = __openFiles[fid];
if(!fh.readOnly && fh.accessHandle) await fh.accessHandle.flush();
storeAndNotify('xSync',0);
},
xTruncate: async function({fid,size}){
log("xTruncate(",arguments[0],")");
let rc = 0;
const fh = __openFiles[fid];
try{
affirmNotRO('xTruncate', fh);
await fh.accessHandle.truncate(size);
}catch(e){
error("xTruncate():",e,fh);
rc = state.sq3Codes.SQLITE_IOERR_TRUNCATE;
}
storeAndNotify('xTruncate',rc);
},
xWrite: async function({fid,src,n,offset}){
log("xWrite(",arguments[0],")");
let rc;
try{
const fh = __openFiles[fid];
affirmNotRO('xWrite', fh);
const nOut = fh.accessHandle.write(new UInt8Array(fh.sab, 0, n), {at: offset});
rc = (nOut===n) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE;
}catch(e){
error("xWrite():",e,fh);
rc = state.sq3Codes.SQLITE_IOERR_WRITE;
}
storeAndNotify('xWrite',rc);
}
};
navigator.storage.getDirectory().then(function(d){
const wMsg = (type)=>postMessage({type});
state.rootDir = d;
log("state.rootDir =",state.rootDir);
self.onmessage = async function({data}){
log("self.onmessage()",data);
switch(data.type){
case 'init':{
/* Receive shared state from synchronous partner */
const opt = data.payload;
state.verbose = opt.verbose ?? 2;
state.fileBufferSize = opt.fileBufferSize;
state.fbInt64Offset = opt.fbInt64Offset;
state.opSab = opt.opSab;
state.opBuf = new Int32Array(state.opSab);
state.opIds = opt.opIds;
state.sq3Codes = opt.sq3Codes;
Object.keys(vfsAsyncImpls).forEach((k)=>{
if(!Number.isFinite(state.opIds[k])){
toss("Maintenance required: missing state.opIds[",k,"]");
}
});
log("init state",state);
wMsg('inited');
break;
}
default:{
let err;
const m = vfsAsyncImpls[data.type] || toss("Unknown message type:",data.type);
try {
await m(data.payload).catch((e)=>err=e);
}catch(e){
err = e;
}
if(err){
error("Error handling",data.type+"():",e);
storeAndNotify(data.type, state.sq3Codes.SQLITE_ERROR);
}
break;
}
} }
}; };
wMsg('loaded');
const onReady = function(){ });
self.onmessage = async function({data}){
log("self.onmessage",data);
switch(data.type){
case 'init':{
const opt = data.payload;
state.opSab = opt.opSab;
state.opBuf = new Int32Array(state.opSab);
state.opIds = opt.opIds;
state.errCodes = opt.errCodes;
state.sq3Codes = opt.sq3Codes;
Object.keys(ioMethods).forEach((k)=>{
if(!state.opIds[k]){
toss("Maintenance required: missing state.opIds[",k,"]");
}
});
log("init state",state);
break;
}
default:{
const m = ioMethods[data.type] || toss("Unknown message type:",data.type);
try {
await m(data.payload);
}catch(e){
error("Error handling",data.type+"():",e);
storeAndNotify(data.type, -99);
}
break;
}
}
};
wMsg('ready');
};
navigator.storage.getDirectory().then(function(d){
state.rootDir = d;
log("state.rootDir =",state.rootDir);
onReady();
});
})();

10
ext/wasm/x-sync-async.html
View File

@ -13,16 +13,10 @@
<div>This is an experiment in wrapping the <div>This is an experiment in wrapping the
asynchronous OPFS APIs behind a fully synchronous proxy. It is asynchronous OPFS APIs behind a fully synchronous proxy. It is
very much incomplete, under construction, and experimental. very much incomplete, under construction, and experimental.
See the dev console for all output. <strong>See the dev console for all output.</strong>
</div> </div>
<div id='test-output'> <div id='test-output'>
</div> </div>
<!--script src="common/whwasmutil.js"></script--> <script>new Worker("x-sync-async.js");</script>
<!--script src="common/SqliteTestUtil.js"></script-->
<script>
(function(){
new Worker("x-sync-async.js");
})();
</script>
</body> </body>
</html> </html>

441
ext/wasm/x-sync-async.js
View File

@ -1,5 +1,29 @@
/*
2022-09-17
The author disclaims copyright to this source code. In place of a
legal notice, here is a blessing:
* May you do good and not evil.
* May you find forgiveness for yourself and forgive others.
* May you share freely, never taking more than you give.
***********************************************************************
A EXTREMELY INCOMPLETE and UNDER CONSTRUCTION experiment for OPFS.
This file holds the synchronous half of an sqlite3_vfs
implementation which proxies, in a synchronous fashion, the
asynchronous OPFS APIs using a second Worker.
*/
'use strict'; 'use strict';
const doAtomicsStuff = function(sqlite3){ /**
This function is a placeholder for use in development. When
working, this will be moved into a file named
api/sqlite3-api-opfs.js, or similar, and hooked in to the
sqlite-api build construct.
*/
const initOpfsVfs = function(sqlite3){
const toss = function(...args){throw new Error(args.join(' '))};
const logPrefix = "OPFS syncer:"; const logPrefix = "OPFS syncer:";
const log = (...args)=>{ const log = (...args)=>{
console.log(logPrefix,...args); console.log(logPrefix,...args);
@ -10,46 +34,73 @@ const doAtomicsStuff = function(sqlite3){
const error = (...args)=>{ const error = (...args)=>{
console.error(logPrefix,...args); console.error(logPrefix,...args);
}; };
warn("This file is very much experimental and under construction.",self.location.pathname);
const capi = sqlite3.capi;
const wasm = capi.wasm;
const sqlite3_vfs = capi.sqlite3_vfs
|| toss("Missing sqlite3.capi.sqlite3_vfs object.");
const sqlite3_file = capi.sqlite3_file
|| toss("Missing sqlite3.capi.sqlite3_file object.");
const sqlite3_io_methods = capi.sqlite3_io_methods
|| toss("Missing sqlite3.capi.sqlite3_io_methods object.");
const StructBinder = sqlite3.StructBinder || toss("Missing sqlite3.StructBinder.");
const W = new Worker("sqlite3-opfs-async-proxy.js"); const W = new Worker("sqlite3-opfs-async-proxy.js");
const wMsg = (type,payload)=>W.postMessage({type,payload}); const wMsg = (type,payload)=>W.postMessage({type,payload});
warn("This file is very much experimental and under construction.",self.location.pathname);
/** /**
State which we send to the async-api Worker or share with it. State which we send to the async-api Worker or share with it.
This object must initially contain only cloneable or sharable This object must initially contain only cloneable or sharable
objects. After the worker's "ready" message arrives, other types objects. After the worker's "inited" message arrives, other types
of data may be added to it. of data may be added to it.
*/ */
const state = Object.create(null); const state = Object.create(null);
state.verbose = 3;
state.fileBufferSize = 1024 * 64 + 8 /* size of fileHandle.sab. 64k = max sqlite3 page size */;
state.fbInt64Offset = state.fileBufferSize - 8 /*spot in fileHandle.sab to store an int64*/;
state.opIds = Object.create(null); state.opIds = Object.create(null);
state.opIds.xAccess = 1; {
state.opIds.xClose = 2; let i = 0;
state.opIds.xDelete = 3; state.opIds.xAccess = i++;
state.opIds.xFileSize = 4; state.opIds.xClose = i++;
state.opIds.xOpen = 5; state.opIds.xDelete = i++;
state.opIds.xRead = 6; state.opIds.xFileSize = i++;
state.opIds.xSync = 7; state.opIds.xOpen = i++;
state.opIds.xTruncate = 8; state.opIds.xRead = i++;
state.opIds.xWrite = 9; state.opIds.xSleep = i++;
state.opIds.xSleep = 10; state.opIds.xSync = i++;
state.opIds.xBlock = 99 /* to block worker while this code is still handling something */; state.opIds.xTruncate = i++;
state.opSab = new SharedArrayBuffer(64); state.opIds.xWrite = i++;
state.fileBufferSize = 1024 * 65 /* 64k = max sqlite3 page size */; state.opSab = new SharedArrayBuffer(i * 4);
/* TODO: use SQLITE_xxx err codes. */ }
state.errCodes = Object.create(null);
state.errCodes.Error = -100;
state.errCodes.IO = -101;
state.errCodes.NotFound = -102;
state.errCodes.Misuse = -103;
// TODO: add any SQLITE_xxx symbols we need here.
state.sq3Codes = Object.create(null); state.sq3Codes = Object.create(null);
state.sq3Codes._reverse = Object.create(null);
const isWorkerErrCode = (n)=>(n<=state.errCodes.Error); [ // SQLITE_xxx constants to export to the async worker counterpart...
'SQLITE_ERROR', 'SQLITE_IOERR',
'SQLITE_NOTFOUND', 'SQLITE_MISUSE',
'SQLITE_IOERR_READ', 'SQLITE_IOERR_SHORT_READ',
'SQLITE_IOERR_WRITE', 'SQLITE_IOERR_FSYNC',
'SQLITE_IOERR_TRUNCATE', 'SQLITE_IOERR_DELETE',
'SQLITE_IOERR_ACCESS', 'SQLITE_IOERR_CLOSE'
].forEach(function(k){
state.sq3Codes[k] = capi[k] || toss("Maintenance required: not found:",k);
state.sq3Codes._reverse[capi[k]] = k;
});
const isWorkerErrCode = (n)=>!!state.sq3Codes._reverse[n];
const opStore = (op,val=-1)=>Atomics.store(state.opBuf, state.opIds[op], val); const opStore = (op,val=-1)=>Atomics.store(state.opBuf, state.opIds[op], val);
const opWait = (op,val=-1)=>Atomics.wait(state.opBuf, state.opIds[op], val); const opWait = (op,val=-1)=>Atomics.wait(state.opBuf, state.opIds[op], val);
/**
Runs the given operation in the async worker counterpart, waits
for its response, and returns the result which the async worker
writes to the given op's index in state.opBuf. The 2nd argument
must be a single object or primitive value, depending on the
given operation's signature in the async API counterpart.
*/
const opRun = (op,args)=>{ const opRun = (op,args)=>{
opStore(op); opStore(op);
wMsg(op, args); wMsg(op, args);
@ -63,71 +114,321 @@ const doAtomicsStuff = function(sqlite3){
}); });
}; };
const vfsSyncWrappers = { /**
xOpen: function f(pFile, name, flags, outFlags = {}){ Generates a random ASCII string len characters long, intended for
if(!f._){ use as a temporary file name.
f._ = { */
// TODO: map openFlags to args.fileType names. const randomFilename = function f(len=16){
}; if(!f._chars){
} f._chars = "abcdefghijklmnopqrstuvwxyz"+
const args = Object.create(null); "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+
args.fid = pFile; "012346789";
args.filename = name; f._n = f._chars.length;
args.sab = new SharedArrayBuffer(state.fileBufferSize); }
args.fileType = undefined /*TODO: populate based on SQLITE_OPEN_xxx */; const a = [];
// TODO: populate args object based on flags: let i = 0;
// args.create, args.readOnly, args.deleteOnClose for( ; i < len; ++i){
args.create = true; const ndx = Math.random() * (f._n * 64) % f._n | 0;
args.deleteOnClose = true; a[i] = f._chars[ndx];
const rc = opRun('xOpen', args); }
if(!rc){ return a.join('');
outFlags.readOnly = args.readOnly; };
args.ba = new Uint8Array(args.sab);
state.openFiles[pFile] = args; /**
} Map of sqlite3_file pointers to objects constructed by xOpen().
return rc; */
const __openFiles = Object.create(null);
const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/;
const dVfs = pDVfs
? new sqlite3_vfs(pDVfs)
: null /* dVfs will be null when sqlite3 is built with
SQLITE_OS_OTHER. Though we cannot currently handle
that case, the hope is to eventually be able to. */;
const opfsVfs = new sqlite3_vfs();
const opfsIoMethods = new sqlite3_io_methods();
opfsVfs.$iVersion = 2/*yes, two*/;
opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof;
opfsVfs.$mxPathname = 1024/*sure, why not?*/;
opfsVfs.$zName = wasm.allocCString("opfs");
opfsVfs.ondispose = [
'$zName', opfsVfs.$zName,
'cleanup dVfs', ()=>(dVfs ? dVfs.dispose() : null)
];
if(dVfs){
opfsVfs.$xSleep = dVfs.$xSleep;
opfsVfs.$xRandomness = dVfs.$xRandomness;
}
// All C-side memory of opfsVfs is zeroed out, but just to be explicit:
opfsVfs.$xDlOpen = opfsVfs.$xDlError = opfsVfs.$xDlSym = opfsVfs.$xDlClose = null;
/**
Pedantic sidebar about opfsVfs.ondispose: the entries in that array
are items to clean up when opfsVfs.dispose() is called, but in this
environment it will never be called. The VFS instance simply
hangs around until the WASM module instance is cleaned up. We
"could" _hypothetically_ clean it up by "importing" an
sqlite3_os_end() impl into the wasm build, but the shutdown order
of the wasm engine and the JS one are undefined so there is no
guaranty that the opfsVfs instance would be available in one
environment or the other when sqlite3_os_end() is called (_if_ it
gets called at all in a wasm build, which is undefined).
*/
/**
Impls for the sqlite3_io_methods methods. Maintenance reminder:
members are in alphabetical order to simplify finding them.
*/
const ioSyncWrappers = {
xCheckReservedLock: function(pFile,pOut){
// Exclusive lock is automatically acquired when opened
//warn("xCheckReservedLock(",arguments,") is a no-op");
wasm.setMemValue(pOut,1,'i32');
return 0;
}, },
xClose: function(pFile){ xClose: function(pFile){
let rc = 0; let rc = 0;
if(state.openFiles[pFile]){ const f = __openFiles[pFile];
delete state.openFiles[pFile]; if(f){
delete __openFiles[pFile];
rc = opRun('xClose', pFile); rc = opRun('xClose', pFile);
if(f.sq3File) f.sq3File.dispose();
} }
return rc; return rc;
},
xDeviceCharacteristics: function(pFile){
//debug("xDeviceCharacteristics(",pFile,")");
return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN;
},
xFileControl: function(pFile,op,pArg){
//debug("xFileControl(",arguments,") is a no-op");
return capi.SQLITE_NOTFOUND;
},
xFileSize: function(pFile,pSz64){
const rc = opRun('xFileSize', pFile);
if(!isWorkerErrCode(rc)){
const f = __openFiles[pFile];
wasm.setMemValue(pSz64, f.sabViewFileSize.getBigInt64(0) ,'i64');
}
return rc;
},
xLock: function(pFile,lockType){
//2022-09: OPFS handles lock when opened
//warn("xLock(",arguments,") is a no-op");
return 0;
},
xRead: function(pFile,pDest,n,offset){
/* int (*xRead)(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst) */
const f = __opfsHandles[pFile];
try {
// FIXME(?): block until we finish copying the xRead result buffer. How?
let rc = opRun('xRead',{fid:pFile, n, offset});
if(0!==rc) return rc;
let i = 0;
for(; i < n; ++i) wasm.setMemValue(pDest + i, f.sabView[i]);
}catch(e){
error("xRead(",arguments,") failed:",e,f);
rc = capi.SQLITE_IOERR_READ;
}
return rc;
},
xSync: function(pFile,flags){
return opRun('xSync', {fid:pFile, flags});
},
xTruncate: function(pFile,sz64){
return opRun('xTruncate', {fid:pFile, size: sz64});
},
xUnlock: function(pFile,lockType){
//2022-09: OPFS handles lock when opened
//warn("xUnlock(",arguments,") is a no-op");
return 0;
},
xWrite: function(pFile,pSrc,n,offset){
/* int (*xWrite)(sqlite3_file*, const void*, int iAmt, sqlite3_int64 iOfst) */
const f = __opfsHandles[pFile];
try {
let i = 0;
// FIXME(?): block from here until we finish the xWrite. How?
for(; i < n; ++i) f.sabView[i] = wasm.getMemValue(pSrc+i);
return opRun('xWrite',{fid:pFile, n, offset});
}catch(e){
error("xWrite(",arguments,") failed:",e,f);
return capi.SQLITE_IOERR_WRITE;
}
} }
}; }/*ioSyncWrappers*/;
/**
Impls for the sqlite3_vfs methods. Maintenance reminder: members
are in alphabetical order to simplify finding them.
*/
const vfsSyncWrappers = {
// TODO: xAccess
xCurrentTime: function(pVfs,pOut){
/* If it turns out that we need to adjust for timezone, see:
https://stackoverflow.com/a/11760121/1458521 */
wasm.setMemValue(pOut, 2440587.5 + (new Date().getTime()/86400000),
'double');
return 0;
},
xCurrentTimeInt64: function(pVfs,pOut){
// TODO: confirm that this calculation is correct
wasm.setMemValue(pOut, (2440587.5 * 86400000) + new Date().getTime(),
'i64');
return 0;
},
xDelete: function(pVfs, zName, doSyncDir){
return opRun('xDelete', {filename: wasm.cstringToJs(zName), syncDir: doSyncDir});
},
xFullPathname: function(pVfs,zName,nOut,pOut){
/* Until/unless we have some notion of "current dir"
in OPFS, simply copy zName to pOut... */
const i = wasm.cstrncpy(pOut, zName, nOut);
return i<nOut ? 0 : capi.SQLITE_CANTOPEN
/*CANTOPEN is required by the docs but SQLITE_RANGE would be a closer match*/;
},
xGetLastError: function(pVfs,nOut,pOut){
/* TODO: store exception.message values from the async
partner in a dedicated SharedArrayBuffer, noting that we'd have
to encode them... TextEncoder can do that for us. */
warn("OPFS xGetLastError() has nothing sensible to return.");
return 0;
},
xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){
if(!f._){
f._ = {
fileTypes: {
SQLITE_OPEN_MAIN_DB: 'mainDb',
SQLITE_OPEN_MAIN_JOURNAL: 'mainJournal',
SQLITE_OPEN_TEMP_DB: 'tempDb',
SQLITE_OPEN_TEMP_JOURNAL: 'tempJournal',
SQLITE_OPEN_TRANSIENT_DB: 'transientDb',
SQLITE_OPEN_SUBJOURNAL: 'subjournal',
SQLITE_OPEN_SUPER_JOURNAL: 'superJournal',
SQLITE_OPEN_WAL: 'wal'
},
getFileType: function(filename,oflags){
const ft = f._.fileTypes;
for(let k of Object.keys(ft)){
if(oflags & capi[k]) return ft[k];
}
warn("Cannot determine fileType based on xOpen() flags for file",filename);
return '???';
}
};
}
if(0===zName){
zName = randomFilename();
}else if('number'===typeof zName){
zName = wasm.cstringToJs(zName);
}
const args = Object.create(null);
args.fid = pFile;
args.filename = zName;
args.sab = new SharedArrayBuffer(state.fileBufferSize);
args.fileType = f._.getFileType(args.filename, flags);
args.create = !!(flags & capi.SQLITE_OPEN_CREATE);
args.deleteOnClose = !!(flags & capi.SQLITE_OPEN_DELETEONCLOSE);
args.readOnly = !!(flags & capi.SQLITE_OPEN_READONLY);
const rc = opRun('xOpen', args);
if(!rc){
/* Recall that sqlite3_vfs::xClose() will be called, even on
error, unless pFile->pMethods is NULL. */
if(args.readOnly){
wasm.setMemValue(pOutFlags, capi.SQLITE_OPEN_READONLY, 'i32');
}
__openFiles[pFile] = args;
args.sabView = new Uint8Array(args.sab);
args.sabViewFileSize = new DataView(args.sab, state.fbInt64Offset, 8);
args.sq3File = new sqlite3_file(pFile);
args.sq3File.$pMethods = opfsIoMethods.pointer;
args.ba = new Uint8Array(args.sab);
}
return rc;
}/*xOpen()*/
}/*vfsSyncWrappers*/;
if(!opfsVfs.$xRandomness){
/* If the default VFS has no xRandomness(), add a basic JS impl... */
vfsSyncWrappers.xRandomness = function(pVfs, nOut, pOut){
const heap = wasm.heap8u();
let i = 0;
for(; i < nOut; ++i) heap[pOut + i] = (Math.random()*255000) & 0xFF;
return i;
};
}
if(!opfsVfs.$xSleep){
/* If we can inherit an xSleep() impl from the default VFS then
use it, otherwise install one which is certainly less accurate
because it has to go round-trip through the async worker, but
provides the only option for a synchronous sleep() in JS. */
vfsSyncWrappers.xSleep = (pVfs,ms)=>opRun('xSleep',ms);
}
const doSomething = function(){ /*
TODO: plug in the above functions in to opfsVfs and opfsIoMethods.
Code for doing so is in api/sqlite3-api-opfs.js.
*/
const sanityCheck = async function(){
//state.ioBuf = new Uint8Array(state.sabIo); //state.ioBuf = new Uint8Array(state.sabIo);
const fid = 37; const scope = wasm.scopedAllocPush();
let rc = vfsSyncWrappers.xOpen(fid, "/foo/bar/baz.sqlite3",0, {}); const sq3File = new sqlite3_file();
log("open rc =",rc,"state.opBuf[xOpen] =",state.opBuf[state.opIds.xOpen]); try{
if(isWorkerErrCode(rc)){ const fid = sq3File.pointer;
error("open failed with code",rc); const openFlags = capi.SQLITE_OPEN_CREATE
return; | capi.SQLITE_OPEN_READWRITE
} | capi.SQLITE_OPEN_DELETEONCLOSE
log("xSleep()ing before close()ing..."); | capi.SQLITE_OPEN_MAIN_DB;
opRun('xSleep',{ms: 1500}); const pOut = wasm.scopedAlloc(8);
log("wait()ing before close()ing..."); const dbFile = "/sanity/check/file";
wait(1500).then(function(){ let rc = vfsSyncWrappers.xOpen(opfsVfs.pointer, dbFile,
rc = vfsSyncWrappers.xClose(fid); fid, openFlags, pOut);
log("open rc =",rc,"state.opBuf[xOpen] =",state.opBuf[state.opIds.xOpen]);
if(isWorkerErrCode(rc)){
error("open failed with code",rc);
return;
}
rc = ioSyncWrappers.xSync(sq3File.pointer, 0);
if(rc) toss('sync failed w/ rc',rc);
rc = ioSyncWrappers.xTruncate(sq3File.pointer, 1024);
if(rc) toss('truncate failed w/ rc',rc);
wasm.setMemValue(pOut,0,'i64');
rc = ioSyncWrappers.xFileSize(sq3File.pointer, pOut);
if(rc) toss('xFileSize failed w/ rc',rc);
log("xFileSize says:",wasm.getMemValue(pOut, 'i64'));
log("xSleep()ing before close()ing...");
opRun('xSleep',1500);
rc = ioSyncWrappers.xClose(fid);
log("xClose rc =",rc,"opBuf =",state.opBuf); log("xClose rc =",rc,"opBuf =",state.opBuf);
}); log("Deleting file:",dbFile);
opRun('xDelete', dbFile);
}finally{
sq3File.dispose();
wasm.scopedAllocPop(scope);
}
}; };
W.onmessage = function({data}){ W.onmessage = function({data}){
log("Worker.onmessage:",data); log("Worker.onmessage:",data);
switch(data.type){ switch(data.type){
case 'ready': case 'loaded':
/*Pass our config and shared state on to the async worker.*/
wMsg('init',state); wMsg('init',state);
break;
case 'inited':
/*Indicates that the async partner has received the 'init',
so we now know that the state object is no longer subject to
being copied by a pending postMessage() call.*/
state.opBuf = new Int32Array(state.opSab); state.opBuf = new Int32Array(state.opSab);
state.openFiles = Object.create(null); sanityCheck();
doSomething(); break;
default:
error("Unexpected message from the async worker:",data);
break; break;
} }
}; };
}/*doAtomicsStuff*/ }/*initOpfsVfs*/
importScripts('sqlite3.js'); importScripts('sqlite3.js');
self.sqlite3InitModule().then((EmscriptenModule)=>doAtomicsStuff(EmscriptenModule.sqlite3)); self.sqlite3InitModule().then((EmscriptenModule)=>initOpfsVfs(EmscriptenModule.sqlite3));

16
manifest
View File

@ -1,5 +1,5 @@
C Add\sinitial\sbits\sof\san\sexperimental\sasync-impl-via-synchronous-interface\sproxy\sintended\sto\smarshal\sOPFS\svia\ssqlite3_vfs\sAPI. C Add\sthe\sremaining\svfs/io_methods\swrappers\sto\sthe\sOPFS\ssync/async\sproxy,\sbut\smost\sare\snot\syet\stested.
D 2022-09-17T15:08:22.642 D 2022-09-17T20:50:12.684
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@ -523,7 +523,7 @@ F ext/wasm/speedtest1.html fbb8e4d1639028443f3687a683be660beca6927920545cf6b1fdf
F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x
F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0
F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5 F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5
F ext/wasm/sqlite3-opfs-async-proxy.js c42a097dfbb96abef08554b173a47788f5bc1f58c266f859ba01c1fa3ff8327d F ext/wasm/sqlite3-opfs-async-proxy.js 62024877ad13fdff1834581ca1951ab58bda431e4d548aaaf4506ea54f0ed2de
F ext/wasm/sqlite3-worker1-promiser.js 92b8da5f38439ffec459a8215775d30fa498bc0f1ab929ff341fc3dd479660b9 F ext/wasm/sqlite3-worker1-promiser.js 92b8da5f38439ffec459a8215775d30fa498bc0f1ab929ff341fc3dd479660b9
F ext/wasm/sqlite3-worker1.js 0c1e7626304543969c3846573e080c082bf43bcaa47e87d416458af84f340a9e F ext/wasm/sqlite3-worker1.js 0c1e7626304543969c3846573e080c082bf43bcaa47e87d416458af84f340a9e
F ext/wasm/testing-worker1-promiser.html 6eaec6e04a56cf24cf4fa8ef49d78ce8905dde1354235c9125dca6885f7ce893 F ext/wasm/testing-worker1-promiser.html 6eaec6e04a56cf24cf4fa8ef49d78ce8905dde1354235c9125dca6885f7ce893
@ -533,8 +533,8 @@ F ext/wasm/testing1.js 7cd8ab255c238b030d928755ae8e91e7d90a12f2ae601b1b8f7827aaa
F ext/wasm/testing2.html a66951c38137ff1d687df79466351f3c734fa9c6d9cce71d3cf97c291b2167e3 F ext/wasm/testing2.html a66951c38137ff1d687df79466351f3c734fa9c6d9cce71d3cf97c291b2167e3
F ext/wasm/testing2.js 25584bcc30f19673ce13a6f301f89f8820a59dfe044e0c4f2913941f4097fe3c F ext/wasm/testing2.js 25584bcc30f19673ce13a6f301f89f8820a59dfe044e0c4f2913941f4097fe3c
F ext/wasm/wasmfs.make 21a5cf297954a689e0dc2a95299ae158f681cae5e90c10b99d986097815fd42d F ext/wasm/wasmfs.make 21a5cf297954a689e0dc2a95299ae158f681cae5e90c10b99d986097815fd42d
F ext/wasm/x-sync-async.html 283539e4fcca8c60fea18dbf1f1c0df168340145a19123f8fd5b70f41291b36f F ext/wasm/x-sync-async.html 717b0d3bee96e49cbd36731bead497ab27a8bf3a3b23dd11e40e61d4ac9e8b80
F ext/wasm/x-sync-async.js 42da502ea0b89bfa226c7ac7555c0c87d4ab8a10221ea6fadb4f7877c26a5137 F ext/wasm/x-sync-async.js 05c0b49adae0600c5ad12f3325e0873ab1f07b99c2bb017f32b50a4f701490f1
F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
@ -2030,8 +2030,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P afb79050e635f3c698e51f06c346cbf23b096cfda7d0f1d8e68514ea0c25b7b7 P 38da059b472415da52f57de7332fbeb8a91e3add1f4be3ff9c1924b52672f77c
R dbdee404fc63f84ef61ddb4fd79c0ad1 R 8887512b2075b067b4aa7969d08f1316
U stephan U stephan
Z 67d0abd2dfaa993776fbe8c511ee53f5 Z 1bf4c438e48bd1dc1717362060b2f357
# Remove this line to create a well-formed Fossil manifest. # Remove this line to create a well-formed Fossil manifest.

2
manifest.uuid
View File

@ -1 +1 @@
38da059b472415da52f57de7332fbeb8a91e3add1f4be3ff9c1924b52672f77c 44db9132145b3072488ea91db53f6c06be74544beccad5fd07efd22c0f03dc04