1
0
mirror of https://github.com/sqlite/sqlite.git synced 2026-10-25 09:59:22 +03:00

Get initial JS overrides of the kvvfs sqlite3_vfs and sqlite3_io_methods in place. It now tracks a distinct Storage-ish object per sqlite3_file instance.

FossilOrigin-Name: 19a3349a2031e2b7fae67847b55643e4f70f8dae863ebc1ace3b09d1f482c8eb
This commit is contained in:
stephan
2025-11-21 22:13:35 +00:00
parent 7c07260353
commit 92fa31bc5f
5 changed files with 245 additions and 124 deletions

View File

@@ -31,8 +31,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
if( !pKvvfs ) return /* nothing to do */;
const util = sqlite3.util,
wasm = sqlite3.wasm;
const util = sqlite3.util;
if( !util.isUIThread() ){
/* One test currently relies on this VFS not being visible in
@@ -43,18 +42,18 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
return;
}
const __hop = (o,k)=>Object.prototype.hasOwnProperty.call(o,k);
const wasm = sqlite3.wasm,
hop = (o,k)=>Object.prototype.hasOwnProperty.call(o,k);
/**
Implementation of JS's Storage interface for use
as backing store of the kvvfs.
*/
class ObjectStorage extends Storage {
class ObjectStorage /* extends Storage (ctor may not be legally called) */ {
#map;
#keys;
#getKeys(){return this.#keys ??= Object.keys(this.#map);}
constructor(){
super();
this.clear();
}
@@ -68,7 +67,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
}
setItem(k,v){
if( !__hop(this.#map, k) ){
if( !hop(this.#map, k) ){
this.#keys = null;
}
this.#map[k]=v;
@@ -90,19 +89,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
}
}/*ObjectStorage*/;
const kvvfsState = Object.assign(Object.create(null),{
jClassToStorage: new Map(
/* Map of JS-format KVVfsFile::zClass values to Storage
object. We can't use WeakMap because that requires an Object
or Symbol as a key, and we can't use Symbol in this
context. We don't use KVVfsFile::zClass pointers as keys only
because of some special-casing for local/session journal
files which remaps those pointers. */
)
});
kvvfsState.jClassToStorage.set('local', localStorage);
kvvfsState.jClassToStorage.set('session', sessionStorage);
/**
Internal helper for sqlite3_js_kvvfs_clear() and friends.
Its argument should be one of ('local','session',"").
@@ -111,20 +97,13 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
const rc = Object.create(null);
rc.prefix = 'kvvfs-'+which;
rc.stores = [];
let got = 0;
if( globalThis.sessionStorage
&& ('session'===which || ""===which)){
rc.stores.push(globalThis.sessionStorage);
++got;
}
if( globalThis.localStorage
&& ('local'===which || ""===which) ){
rc.stores.push(globalThis.localStorage);
++got;
}
if( !got && which ){
const x = kvvfsState.jClassToStorage.get(which);
if( x ) rc.stores.push(x);
}
return rc;
};
@@ -193,34 +172,74 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
const kvvfsMakeKey = wasm.exports.sqlite3__wasm_kvvfsMakeKeyOnPstack;
const pstack = wasm.pstack;
const cache = Object.create(null);
cache.jzClassToStorage = Object.assign(Object.create(null),{
/* Map of JS-stringified KVVfsFile::zClass names to
reference-counted Storage objects. We refcount so that xClose()
does not pull one out from another instance. */
local: {refc: 2, s: globalThis.localStorage},
session: {refc: 2, s: globalThis.sessionStorage}
});
cache.jzClassToStorage['local-journal'] =
cache.jzClassToStorage.local;
cache.jzClassToStorage['session-journal'] =
cache.jzClassToStorage.session;
const kvvfsStorage = function(zClass){
const s = wasm.cstrToJs(zClass);
if( !this.rxSession ){
this.rxSession = /^session(-journal)?$/;
this.rxLocal = /^local(-journal)?$/;
if( cache.jzClassToStorage[s] ){
return cache.jzClassToStorage[s].s;
}
if( this.rxSession.test(s) ) return sessionStorage;
if( this.rxLocal.test(s) ) return localStorage;
return kvvfsStorage.jClassToStorage.get(s);
if( !cache.rxSession ){
cache.rxSession = /^session(-journal)?$/;
cache.rxLocal = /^local(-journal)?$/;
}
if( cache.rxSession.test(s) ) return sessionStorage;
if( cache.rxLocal.test(s) ) return localStorage;
return cache.jzClassToStorage(s)?.s;
}.bind(Object.create(null));
{ /* Override native sqlite3_kvvfs_methods */
const kvvfsMethods = new sqlite3_kvvfs_methods(
/* Wraps the static sqlite3_api_methods singleton */
wasm.exports.sqlite3__wasm_kvvfs_methods()
);
try{
/**
Implementations for members of the object referred to by
sqlite3__wasm_kvvfs_methods(). We swap out the native
implementations with these, which use JS Storage for their
backing store.
const pFileHandles = new Map(
/* sqlite3_file pointers => objects, each of which has:
.s = Storage object
.f = KVVfsFile instance
.n = JS-string form of f.$zName
*/
);
The interface docs for these methods are in
src/os_kv.c:kvrecordRead(), kvrecordWrite(), and
kvrecordDelete().
*/
for(const e of Object.entries({
const debug = sqlite3.config.debug.bind(sqlite3.config);
const warn = sqlite3.config.warn.bind(sqlite3.config);
{
/**
Original WASM functions for methods we partially override.
*/
const originalMethods = {
vfs: Object.create(null),
ioDb: Object.create(null),
ioJrnl: Object.create(null)
};
/** Returns the appropriate originalMethods[X] instance for the
given a KVVfsFile instance. */
const originalIoMethods = (kvvfsFile)=>
originalMethods[kvvfsFile.$isJournal ? 'ioJrnl' : 'ioDb'];
/**
Implementations for members of the object referred to by
sqlite3__wasm_kvvfs_methods(). We swap out the native
implementations with these, which use JS Storage for their
backing store.
The interface docs for these methods are in
src/os_kv.c:kvrecordRead(), kvrecordWrite(), and
kvrecordDelete().
*/
sqlite3_kvvfs_methods.override = {
/* sqlite3_kvvfs_methods's own direct methods */
recordHandler: {
xRcrdRead: (zClass, zKey, zBuf, nBuf)=>{
const stack = pstack.pointer,
astack = wasm.scopedAllocPush();
@@ -253,6 +272,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
wasm.scopedAllocPop(astack);
}
},
xRcrdWrite: (zClass, zKey, zData)=>{
const stack = pstack.pointer;
try {
@@ -268,6 +288,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
pstack.restore(stack);
}
},
xRcrdDelete: (zClass, zKey)=>{
const stack = pstack.pointer;
try {
@@ -282,62 +303,96 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
pstack.restore(stack);
}
}
})){
kvvfsMethods[kvvfsMethods.memberKey(e[0])] =
wasm.installFunction(kvvfsMethods.memberSignature(e[0]), e[1]);
}
}finally{
kvvfsMethods.dispose();
}
}/* Override native sqlite3_api_methods */
/**
After initial refactoring to support the use of arbitrary Storage
objects (the interface from which localStorage and sessionStorage
dervie), we will apparently need to override some of the
associated sqlite3_vfs and sqlite3_io_methods members.
We can call back into the native impls when needed, but we need
to override certain operations here to bypass its strict
db-naming rules (which, funnily enough, are in place because
they're relevant (only) for what should soon be the previous
version of this browser-side implementation). Apropos: the port
to generic objects would also make non-persistent kvvfs available
in Worker threads and non-browser builds. They could optionally
be exported to/from JSON.
*/
const eventualTodo = {
vfsMethods:{
//#if nope
}/*recordHandler*/,
/**
*/
xOpen: function(pProtoVfs,zName,pProtoFile,flags,pOutFlags){
},
xDelete: function(pVfs, zName, iSyncFlag){},
xAccess:function(pProtoVfs, zPath, flags, pResOut){},
xFullPathname: function(pVfs, zPath, nOut, zOut){},
xDlOpen: function(pVfs, zFilename){},
xSleep: function(pVfs,usec){},
xCurrentTime: function(pVfs,pOutDbl){},
xCurrentTimeInt64: function(pVfs,pOutI64){},
xRandomness: function(pVfs, nOut, pOut){
const heap = wasm.heap8u();
let i = 0;
const npOut = Number(pOut);
for(; i < nOut; ++i) heap[npOut + i] = (Math.random()*255000) & 0xFF;
return i;
}
//#endif
},
After initial refactoring to support the use of arbitrary Storage
objects (the interface from which localStorage and sessionStorage
dervie), we will apparently need to override some of the
associated sqlite3_vfs and sqlite3_io_methods members.
We can call back into the native impls when needed, but we
need to override certain operations here to bypass its strict
db-naming rules (which, funnily enough, are in place because
they're relevant (only) for what should soon be the previous
version of this browser-side implementation). Apropos: the
port to generic objects would also make non-persistent kvvfs
available in Worker threads and non-browser builds. They
could optionally be exported to/from JSON.
*/
/* sqlite3_kvvfs_methods::pVfs's methods */
vfs:{
/**
*/
xOpen: function(pProtoVfs,zName,pProtoFile,flags,pOutFlags){
try{
const rc = originalMethods.vfs.xOpen(pProtoVfs, zName, pProtoFile,
flags, pOutFlags);
if( 0==rc ){
const jzName = wasm.cstrToJs(zName);
const f = new KVVfsFile(pProtoFile);
let s = cache.jzClassToStorage[jzName];
if( s ){
++s.refc;
}else{
s = cache.jzClassToStorage[jzName] = {
refc: 1,
s: new ObjectStorage
};
}
debug("kvvfs xOpen", f, jzName, s);
pFileHandles.set(pProtoFile, {s,f,n:jzName});
}
return rc;
}catch(e){
warn("kvvfs xOpen:",e);
return capi.SQLITE_ERROR;
}
},
//#if nope
xDelete: function(pVfs, zName, iSyncFlag){},
xAccess:function(pProtoVfs, zPath, flags, pResOut){},
xFullPathname: function(pVfs, zPath, nOut, zOut){},
xDlOpen: function(pVfs, zFilename){},
xSleep: function(pVfs,usec){},
xCurrentTime: function(pVfs,pOutDbl){},
xCurrentTimeInt64: function(pVfs,pOutI64){},
//#endif
xRandomness: function(pVfs, nOut, pOut){
const heap = wasm.heap8u();
let i = 0;
const npOut = Number(pOut);
for(; i < nOut; ++i) heap[npOut + i] = (Math.random()*255000) & 0xFF;
return i;
}
},
/**
kvvfs has separate sqlite3_api_methods impls for some of the
methods, depending on whether it's a db or journal file. Some
of the methods use shared impls but others are specific to
either db or journal files.
*/
ioDb:{
/* sqlite3_kvvfs_methods::pIoDb's methods */
xClose: function(pFile){
try{
const h = pFileHandles.get(pFile);
debug("kvvfs xClose", pFile, h);
pFileHandles.delete(pFile);
const s = cache.jzClassToStorage[h.n];
if( 0===--s.refc ){
delete cache.jzClassToStorage[h.n];
delete s.s;
delete s.refc;
}
originalIoMethods(h.f).xClose(pFile);
h.f.dispose();
return 0;
}catch(e){
warn("kvvfs xClose",e);
return capi.SQLITE_ERROR;
}
},
//#if nope
/**
kvvfs has separate impls for some of the I/O methods,
depending on whether it's a db or journal file.
*/
ioMethods:{
db:{
xClose: function(pFile){},
xRead: function(pFile,pTgt,n,iOff64){},
xWrite: function(pFile,pSrc,n,iOff64){},
xTruncate: function(pFile,i64){},
@@ -350,25 +405,88 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
xFileControl: function(pFile,iOp,pArg){},
xSectorSize: function(pFile){},
xDeviceCharacteristics: function(pFile){}
//#endif
},
jrnl:{
xClose: todoIOMethodsDb.xClose,
ioJrnl:{
/* sqlite3_kvvfs_methods::pIoJrnl's methods. Those set to true
are copied as-is from the ioDb objects. Others are specific
to journal files. */
xClose: true,
//#if nope
xRead: function(pFile,pTgt,n,iOff64){},
xWrite: function(pFile,pSrc,n,iOff64){},
xTruncate: function(pFile,i64){},
xSync: function(pFile,flags){},
xFileControl: function(pFile, opId, pArg){},
xFileSize: function(pFile,pi64Out){},
xLock: todoIOMethodsDb.xLock,
xUnlock: todoIOMethodsDb.xUnlock,
xCheckReservedLock: todoIOMethodsDb.xCheckReservedLock,
xLock: true,
xUnlock: true,
xCheckReservedLock: true,
xFileControl: function(pFile,iOp,pArg){},
xSectorSize: todoIOMethodsDb.xSectorSize,
xDeviceCharacteristics: todoIOMethodsDb.xDeviceCharacteristics
}
}
xSectorSize: true,
xDeviceCharacteristics: true
//#endif
}/*eventualTodo*/;
}
}/*sqlite3_kvvfs_methods.override*/;
const ov = sqlite3_kvvfs_methods.override;
const kvvfsMethods = new sqlite3_kvvfs_methods(
/* Wraps the static sqlite3_api_methods singleton */
wasm.exports.sqlite3__wasm_kvvfs_methods()
);
const pVfs = new capi.sqlite3_vfs(kvvfsMethods.$pVfs);
const pIoDb = new capi.sqlite3_io_methods(kvvfsMethods.$pIoDb);
const pIoJrnl = new capi.sqlite3_io_methods(kvvfsMethods.$pIoJrnl);
debug("pVfs and friends",pVfs, pIoDb, pIoJrnl);
try {
for(const e of Object.entries(ov.recordHandler)){
// Overwrite kvvfsMethods's callbacks
kvvfsMethods[kvvfsMethods.memberKey(e[0])] =
wasm.installFunction(kvvfsMethods.memberSignature(e[0]), e[1]);
}
for(const e of Object.entries(ov.vfs)){
// Overwrite some pVfs entries and stash the original impls
const k = e[0],
f = e[1],
km = pVfs.memberKey(k),
mbr = pVfs.structInfo.members[k] || util.toss("Missing pVfs member ",km);
originalMethods.vfs[k] = wasm.functionEntry(pVfs[km]);
pVfs[km] = wasm.installFunction(mbr.signature, f);
}
for(const e of Object.entries(ov.ioDb)){
// Similar treatment for pVfs.$pIoDb a.k.a. pIoDb...
const k = e[0],
f = e[1],
km = pIoDb.memberKey(k),
mbr = pIoDb.structInfo.members[k];
if( !mbr ){
warn("Missinog pIoDb member",k,km,pIoDb.structInfo);
util.toss("Missing pIoDb member",k,km);
}
originalMethods.ioDb[k] = wasm.functionEntry(pIoDb[km]);
pIoDb[km] = wasm.installFunction(mbr.signature, f);
}
for(const e of Object.entries(ov.ioJrnl)){
// Similar treatment for pVfs.$pIoJrnl a.k.a. pIoJrnl...
const k = e[0],
f = e[1],
km = pIoJrnl.memberKey(k);
originalMethods.ioJrnl[k] = wasm.functionEntry(pIoJrnl[km]);
if( true===f ){
/* use pIoDb's copy */
pIoJrnl[km] = pIoDb[km] || util.toss("Missing copied pIoDb member",km);
}else{
const mbr = pIoJrnl.structInfo.members[k] || util.toss("Missing pIoJrnl member",km)
pIoJrnl[km] = wasm.installFunction(mbr.signature, f);
}
}
}finally{
kvvfsMethods.dispose();
pVfs.dispose();
pIoDb.dispose();
pIoJrnl.dispose();
}
}/*method overrides*/
if(sqlite3?.oo1?.DB){
/**
@@ -393,7 +511,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...arguments);
storageName = opt.filename;
if('session'!==storageName && 'local'!==storageName){
toss3("JsStorageDb db name must be one of 'session' or 'local'.");
util.toss3("JsStorageDb db name must be one of 'session' or 'local'.");
}
opt.vfs = 'kvvfs';
sqlite3.oo1.DB.dbCtorHelper.call(this, opt);

View File

@@ -1105,6 +1105,7 @@ const char * sqlite3__wasm_enum_json(void){
StructBinder {
M(base, "p")/*sqlite3_file base*/;
M(zClass, "s");
M(zName, "s");
M(isJournal, "i");
M(nJrnl, "i")/*actually unsigned!*/;
M(aJrnl, "p");

View File

@@ -1,5 +1,5 @@
C More\swork\stowards\susing\sJS\sgeneric\sStorage\sobjects\sas\sdb\spage\sstorage\svia\skvvfs.
D 2025-11-21T18:55:33.689
C Get\sinitial\sJS\soverrides\sof\sthe\skvvfs\ssqlite3_vfs\sand\ssqlite3_io_methods\sin\splace.\sIt\snow\stracks\sa\sdistinct\sStorage-ish\sobject\sper\ssqlite3_file\sinstance.
D 2025-11-21T22:13:35.200
F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
@@ -600,11 +600,11 @@ F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667
F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89
F ext/wasm/api/sqlite3-opfs-async-proxy.js 9654b565b346dc609b75d15337f20acfa7af7d9d558da1afeb9b6d8eaa404966
F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d
F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js e84a6ec93acb2f6f2a19378ec7d3317de65d37dc75b13b70ad2b51b512268468
F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js b8e37020d69e09348e16111a989f29ffa41fc719412d0ea7ffd1e8f51db91417
F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 26cb41d5a62f46a106b6371eb00fef02de3cdbfaa51338ba087a45f53028e0d0
F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js aa330fa0e8ef35cbd92eb0d52e05fbaa07e61540c5cb164e693c82428ce1d763
F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 9097074724172e31e56ce20ccd7482259cf72a76124213cbc9469d757676da86
F ext/wasm/api/sqlite3-wasm.c dea6f331a03a49b8613d1fd5afcb606e1d06e8ab1df8dd65ee967fbfdd43ca1a
F ext/wasm/api/sqlite3-wasm.c e10ae835b1d8cb3bbc0974200a8339a96e66caed96d45ca962a5cebfd1a04e36
F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bda1c75bd674a92a0e27cc2f3d46dbbf21e422413f8046814515a0bd7409328a
F ext/wasm/api/sqlite3-worker1.c-pp.js 802d69ead8c38dc1be52c83afbfc77e757da8a91a2e159e7ed3ecda8b8dba2e7
F ext/wasm/c-pp-lite.c 943be1a36774d58385dca32de36fc18d4f432fe79f7aa35e6c85dd6a6b825bd0
@@ -717,7 +717,7 @@ F src/notify.c 57c2d1a2805d6dee32acd5d250d928ab94e02d76369ae057dee7d445fd64e878
F src/os.c 509452169d5ea739723e213b8e2481cf0e587f0e88579a912d200db5269f5f6d
F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63
F src/os_common.h 6c0eb8dd40ef3e12fe585a13e709710267a258e2c8dd1c40b1948a1d14582e06
F src/os_kv.c 26191ac1cb171ec3fceade6018ef66309cefba65e41f5ffe4c5385f2cb682ba7
F src/os_kv.c b5de11f31634c36ac5ab04b14f6da67801f331b73384f852de0d77c98009818d
F src/os_setup.h 8efc64eda6a6c2f221387eefc2e7e45fd5a3d5c8337a7a83519ba4fbd2957ae2
F src/os_unix.c 7945ede1e85b2d1b910e1b4af9ba342e964b1e30e79f4176480a60736445cb36
F src/os_win.c a89b501fc195085c7d6c9eec7f5bd782625e94bb2a96b000f4d009703df1083f
@@ -2178,8 +2178,8 @@ F tool/version-info.c 33d0390ef484b3b1cb685d59362be891ea162123cea181cb8e6d2cf6dd
F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7
F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
P 206275292217be4ff317d4c9186ecaf863ca69295e2f995ed175aa65d9ad11dc
R 113fe1254ac4b59119cf9c28e45f222e
P 90a33941c69b3581feaed271542f0238ca81ee34fe5b353ca7da48b81ac73a5f
R 831d8b062b833be06147d18462b6029d
U stephan
Z bcdce04b0f6728fd8539031f7a059a7e
Z 6ff913830f5e095f3c74b6184f98d188
# Remove this line to create a well-formed Fossil manifest.

View File

@@ -1 +1 @@
90a33941c69b3581feaed271542f0238ca81ee34fe5b353ca7da48b81ac73a5f
19a3349a2031e2b7fae67847b55643e4f70f8dae863ebc1ace3b09d1f482c8eb

View File

@@ -51,6 +51,7 @@ typedef struct KVVfsFile KVVfsFile;
struct KVVfsFile {
sqlite3_file base; /* IO methods */
const char *zClass; /* Storage class */
const char *zName; /* File name (used by the JS side) */
int isJournal; /* True if this is a journal file */
unsigned int nJrnl; /* Space allocated for aJrnl[] */
char *aJrnl; /* Journal content */
@@ -839,11 +840,11 @@ static int kvvfsOpen(
SQLITE_KV_LOG(("xOpen(\"%s\")\n", zName));
assert(!pFile->zClass);
assert(!pFile->aData);
pFile->aData = 0;
pFile->aJrnl = 0;
pFile->nJrnl = 0;
assert(!pFile->aJrnl);
assert(!pFile->nJrnl);
pFile->szPage = -1;
pFile->szDb = -1;
pFile->zName = zName;
if( 0==sqlite3_strglob("*-journal", zName) ){
pFile->isJournal = 1;
pFile->base.pMethods = &kvvfs_jrnl_io_methods;
@@ -863,6 +864,7 @@ static int kvvfsOpen(
#ifndef SQLITE_WASM
return SQLITE_CANTOPEN;
#else
/* The JS impl maps these to Storage objects */
pFile->zClass = zName;
#endif
}