database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-07-27 20:41:58 +03:00
Code Activity

Merge kv-vfs branch into fiddle-opfs. Adjust various JS APIs and apps to deal with the new method of handling kvvfs. Adjust speedtest1 JS build to include sqlite3-api.js so that it can use kvvfs and opfs VFSes. Permit passing of the vfs as a URL parameter to certain demo/test apps. Milestone: speedtest-worker.html?vfs=opfs runs with the standalone OPFS impl.

Browse Source 
FossilOrigin-Name: ec09f32f7ae2249aaf27388ad2062982afa8bbbb5f88d236d6d9068bf33ad93d
This commit is contained in:
stephan
2022-09-19 13:16:35 +00:00
parent 5ef7f04775
commit b5ae85eca2
13 changed files with 132 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
ext/wasm/GNUmakefile
View File

@ -53,7 +53,8 @@ SQLITE_OPT = \
-DSQLITE_OMIT_UTF16 \
-DSQLITE_OMIT_SHARED_CACHE \
-DSQLITE_THREADSAFE=0 \
-DSQLITE_TEMP_STORE=3
-DSQLITE_TEMP_STORE=3 \
-DSQLITE_OS_KV_OPTIONAL=1
#SQLITE_OPT += -DSQLITE_ENABLE_MEMSYS5
# ^^^ MEMSYS5 is hypothetically useful for non-Emscripten builds but
# requires adding more infrastructure and fixing one spot in the
@ -192,16 +193,19 @@ emcc.cflags += -I. -I$(dir.top) # $(SQLITE_OPT)
########################################################################
# emcc flags specific to building the final .js/.wasm file...
emcc.jsflags := -fPIC
emcc.jsflags := --minify 0
emcc.jsflags += --minify 0
emcc.jsflags += --no-entry
emcc.jsflags += -sMODULARIZE
emcc.jsflags += -sSTRICT_JS
emcc.jsflags += -sDYNAMIC_EXECUTION=0
emcc.jsflags += -sNO_POLYFILL
emcc.jsflags += -sEXPORTED_FUNCTIONS=@$(dir.wasm)/EXPORTED_FUNCTIONS.api
emcc.jsflags += -sEXPORTED_RUNTIME_METHODS=FS,wasmMemory,allocateUTF8OnStack
# wasmMemory ==> required by our code for use with -sIMPORTED_MEMORY
# allocateUTF8OnStack => for kvvp internals
emcc.exportedRuntimeMethods := \
-sEXPORTED_RUNTIME_METHODS=FS,wasmMemory,allocateUTF8OnStack
# FS ==> stdio/POSIX I/O proxies
# wasmMemory ==> required by our code for use with -sIMPORTED_MEMORY
# allocateUTF8OnStack => for kvvfs internals
emcc.jsflags += $(emcc.exportedRuntimeMethods)
emcc.jsflags += -sUSE_CLOSURE_COMPILER=0
emcc.jsflags += -sIMPORTED_MEMORY
emcc.environment := -sENVIRONMENT=web
@ -332,17 +336,23 @@ speedtest1-common.eflags := -g $(emcc_opt)
speedtest1.eflags :=
speedtest1.eflags += -sENVIRONMENT=web
speedtest1-common.eflags += -sINVOKE_RUN=0
#speedtest1-common.eflags += --no-entry
speedtest1-common.eflags += -flto
speedtest1-common.eflags += --no-entry
#speedtest1-common.eflags += -flto
speedtest1-common.eflags += -sABORTING_MALLOC
speedtest1-common.eflags += -sINITIAL_MEMORY=128450560
speedtest1-common.eflags += -sSTRICT_JS
speedtest1-common.eflags += -sMODULARIZE
speedtest1-common.eflags += -Wno-limited-postlink-optimizations
speedtest1-common.eflags += -sEXPORTED_FUNCTIONS=_main,_malloc,_free
speedtest1-common.eflags += -sEXPORTED_FUNCTIONS=@$(dir.wasm)/EXPORTED_FUNCTIONS.speedtest1
speedtest1-common.eflags += $(emcc.exportedRuntimeMethods)
speedtest1-common.eflags += -sALLOW_TABLE_GROWTH
speedtest1-common.eflags += -sDYNAMIC_EXECUTION=0
speedtest1-common.eflags += --minify 0
speedtest1-common.eflags += -sEXPORT_NAME=sqlite3Speedtest1InitModule
speedtest1-common.eflags += --post-js=$(post-js.js)
ifneq (0,$(enable_bigint))
speedtest1-common.eflags += -sWASM_BIGINT
endif
speedtest1.exit-runtime0 := -sEXIT_RUNTIME=0
speedtest1.exit-runtime1 := -sEXIT_RUNTIME=1
# Re -sEXIT_RUNTIME=1 vs 0: if it's 1 and speedtest1 crashes, we get
@ -362,25 +372,29 @@ speedtest1.exit-runtime1 := -sEXIT_RUNTIME=1
# -sEXIT_RUNTIME=1 but we need EXIT_RUNTIME=0 for the worker-based app
# which runs speedtest1 multiple times.
EXPORTED_FUNCTIONS.speedtest1: EXPORTED_FUNCTIONS.api
{ echo _wasm_main; cat EXPORTED_FUNCTIONS.api; } > $@
CLEAN_FILES += EXPORTED_FUNCTIONS.speedtest1
speedtest1.js := speedtest1.js
speedtest1.wasm := $(subst .js,.wasm,$(speedtest1.js))
speedtest1.cflags := \
-I. -I.. -I$(dir.top) \
-DSQLITE_SPEEDTEST1_WASM
speedtest1.cs := $(speedtest1.c) $(sqlite3-wasm.c) $(jaccwabyt_test.c)
$(speedtest1.js): emcc.cflags+=
# speedtest1 notes re. sqlite3-wasm.o vs sqlite3-wasm.c: building against
# the latter (predictably) results in a slightly faster binary, but we're
# close enough to the target speed requirements that the 500ms makes a
# difference.
$(speedtest1.js): $(speedtest1.c) $(sqlite3-wasm.c) $(MAKEFILE) $(sqlite3.c)
$(speedtest1.js): $(MAKEFILE) $(speedtest1.cs) $(post-js.js) \
EXPORTED_FUNCTIONS.speedtest1
@echo "Building $@ ..."
$(emcc.bin) \
$(speedtest1.eflags) $(speedtest1-common.eflags) $(speedtest1.cflags) \
$(SQLITE_OPT) \
$(speedtest1.exit-runtime0) \
'-DSQLITE_DEFAULT_UNIX_VFS="unix-none"' \
-o $@ $(speedtest1.c) $(sqlite3-wasm.c) -lm
-o $@ $(speedtest1.cs) -lm
$(maybe-wasm-strip) $(speedtest1.wasm)
ls -la $@ $(speedtest1.wasm)
@ -434,7 +448,6 @@ oz:
########################################################################
# Sub-makes...
include kvvfs.make
########################################################################
# Some platforms do not support the WASMFS build. Raspberry Pi OS is one

2
ext/wasm/api/sqlite3-api-prologue.js
View File

@ -749,6 +749,8 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
};
/**
Experimental and subject to change or removal.
Returns true if sqlite3.capi.sqlite3_web_persistent_dir() is a
non-empty string and the given name starts with (that string +
'/'), else returns false.

4
ext/wasm/kvvfs1.html → ext/wasm/demo-kvvfs1.html
View File

@ -42,8 +42,8 @@
.toolbar > * { margin: 0.25em; }
fieldset { border-radius: 0.5em; }
</style>
<script src="sqlite3-kvvfs.js"></script>
<script src="sqlite3.js"></script>
<script src="common/SqliteTestUtil.js"></script>
<script src="kvvfs1.js"></script>
<script src="demo-kvvfs1.js"></script>
</body>
</html>

9
ext/wasm/kvvfs1.js → ext/wasm/demo-kvvfs1.js
View File

@ -10,8 +10,9 @@
***********************************************************************
A basic test script for sqlite3-kvvfs.wasm. This file must be run in
main JS thread and sqlite3-kvvfs.js must have been loaded before it.
A basic test script for sqlite3.wasm with kvvfs support. This file
must be run in main JS thread and sqlite3.js must have been loaded
before it.
*/
'use strict';
(function(){
@ -39,7 +40,7 @@
const error = function(...args){
logHtml('error',...args);
};
const runTests = function(Module){
//log("Module",Module);
const sqlite3 = Module.sqlite3,
@ -49,7 +50,7 @@
log("Loaded module:",capi.sqlite3_libversion(), capi.sqlite3_sourceid());
T.assert( 0 !== capi.sqlite3_vfs_find(null) );
if(!capi.sqlite3_vfs_find('kvvfs')){
warn("This build is not kvvfs-capable.");
error("This build is not kvvfs-capable.");
return;
}

7
ext/wasm/index.html
View File

@ -51,9 +51,10 @@
<li><a href='speedtest1.html'>speedtest1</a>: a main-thread WASM build of speedtest1.</li>
<li><a href='speedtest1-worker.html'>speedtest1-worker</a>: an interactive Worker-thread variant of speedtest1.</li>
<li><a href='speedtest1-wasmfs.html'>speedtest1-wasmfs</a>: a variant of speedtest1 built solely for the wasmfs/opfs feature.</li>
<li><a href='speedtest1-kvvfs.html'>speedtest1-kvvfs</a>: a variant of speedtest1 built solely for the kv-vfs feature.</li>
<li><a href='kvvfs1.html'>kvvfs1</a>: very basic demo of using the key-value vfs for storing
a persistent db in JS localStorage or sessionStorage.</li>
<li><a href='speedtest1.html?vfs=kvvfs'>speedtest1-kvvfs</a>: speedtest1 with the kvvfs.</li>
<li><a href='demo-kvvfs1.html'>demo-kvvfs1</a>: very basic
demo of using the key-value vfs for storing a persistent db
in JS localStorage or sessionStorage.</li>
<li><a href='scratchpad-wasmfs-main.html'>scratchpad-wasmfs-main</a>:
experimenting with WASMFS/OPFS-based persistence. Maintenance
reminder: we cannot currently (2022-09-15) load WASMFS in a

112
ext/wasm/kvvfs.make
View File

@ -1,112 +0,0 @@
#!/usr/bin/make
#^^^^ help emacs select makefile mode
#
# This is a sub-make for building a standalone kvvfs-based
# sqlite3.wasm. It is intended to be "include"d from the main
# GNUMakefile.
#
# Notable potential TODOs:
#
# - Trim down a custom sqlite3-api.js for this build. We can elimate
# the jaccwabyt dependency, for example, because this build won't
# make use of the VFS bits. Similarly, we can eliminate or replace
# parts of the OO1 API, or provide a related API which manages
# singletons of the localStorage/sessionStorage instances.
#
########################################################################
MAKEFILE.kvvfs := $(lastword $(MAKEFILE_LIST))
kvvfs.js := sqlite3-kvvfs.js
kvvfs.wasm := sqlite3-kvvfs.wasm
kvvfs.wasm.c := $(dir.api)/sqlite3-wasm.c
CLEAN_FILES += $(kvvfs.js) $(kvvfs.wasm)
########################################################################
# emcc flags for .c/.o/.wasm.
kvvfs.flags =
#kvvfs.flags += -v # _very_ loud but also informative about what it's doing
########################################################################
# emcc flags for .c/.o.
kvvfs.cflags :=
kvvfs.cflags += -std=c99 -fPIC -g
kvvfs.cflags += -I. -I$(dir.top)
kvvfs.cflags += -DSQLITE_OS_KV=1 $(SQLITE_OPT)
kvvfs.extra.c :=
ifeq (1,1)
# To get testing1.js to run with $(kvvfs.js) we need...
kvvfs.extra.c += $(jaccwabyt_test.c)
endif
########################################################################
# emcc flags specific to building the final .js/.wasm file...
kvvfs.jsflags := -fPIC
kvvfs.jsflags += --no-entry
kvvfs.jsflags += --minify 0
kvvfs.jsflags += -sENVIRONMENT=web
kvvfs.jsflags += -sMODULARIZE
kvvfs.jsflags += -sSTRICT_JS
kvvfs.jsflags += -sDYNAMIC_EXECUTION=0
kvvfs.jsflags += -sNO_POLYFILL
ifeq (,$(kvvfs.extra.c))
kvvfs.jsflags += -sEXPORTED_FUNCTIONS=@$(dir.api)/EXPORTED_FUNCTIONS.sqlite3-api
else
# need more exports for jaccwabyt test code...
kvvfs.jsflags += -sEXPORTED_FUNCTIONS=@$(dir.wasm)/EXPORTED_FUNCTIONS.api
endif
kvvfs.jsflags += -sEXPORTED_RUNTIME_METHODS=FS,wasmMemory,allocateUTF8OnStack
# wasmMemory ==> for -sIMPORTED_MEMORY
# allocateUTF8OnStack ==> kvvfs internals
kvvfs.jsflags += -sUSE_CLOSURE_COMPILER=0
kvvfs.jsflags += -sIMPORTED_MEMORY
#kvvfs.jsflags += -sINITIAL_MEMORY=13107200
#kvvfs.jsflags += -sTOTAL_STACK=4194304
kvvfs.jsflags += -sEXPORT_NAME=sqlite3InitModule
kvvfs.jsflags += -sGLOBAL_BASE=4096 # HYPOTHETICALLY keep func table indexes from overlapping w/ heap addr.
kvvfs.jsflags += --post-js=$(post-js.js)
#kvvfs.jsflags += -sFILESYSTEM=0 # only for experimentation. sqlite3 needs the FS API
# Perhaps the kvvfs build doesn't?
#kvvfs.jsflags += -sABORTING_MALLOC
kvvfs.jsflags += -sALLOW_MEMORY_GROWTH
kvvfs.jsflags += -sALLOW_TABLE_GROWTH
kvvfs.jsflags += -Wno-limited-postlink-optimizations
# ^^^^^ it likes to warn when we have "limited optimizations" via the -g3 flag.
kvvfs.jsflags += -sERROR_ON_UNDEFINED_SYMBOLS=0
kvvfs.jsflags += -sLLD_REPORT_UNDEFINED
#kvvfs.jsflags += --import-undefined
kvvfs.jsflags += -sMEMORY64=0
ifneq (0,$(enable_bigint))
kvvfs.jsflags += -sWASM_BIGINT
endif
$(kvvfs.js): $(kvvfs.wasm.c) $(sqlite3.c) $(kvvfs.extra.c) \
EXPORTED_FUNCTIONS.api $(MAKEFILE) $(MAKEFILE.kvvfs) \
$(post-js.js)
@echo "Building $@ ..."
$(emcc.bin) -o $@ $(emcc_opt) $(emcc.flags) \
$(SQLITE_OPT) \
$(kvvfs.cflags) $(kvvfs.jsflags) $(kvvfs.wasm.c) $(kvvfs.extra.c)
chmod -x $(kvvfs.wasm)
$(maybe-wasm-strip) $(kvvfs.wasm)
@ls -la $@ $(kvvfs.wasm)
kvvfs: $(kvvfs.js)
all: kvvfs
########################################################################
# speedtest1-kvvfs
speedtest1-kvvfs.js := speedtest1-kvvfs.js
speedtest1-kvvfs.wasm := speedtest1-kvvfs.wasm
CLEAN_FILES += $(speedtest1-kvvfs.js) $(speedtest1-kvvfs.wasm)
$(speedtest1-kvvfs.js): $(speedtest1.c) $(sqlite3-wasm.c) $(sqlite3.c) $(MAKEFILE.kvvfs)
$(emcc.bin) \
$(speedtest1.eflags) $(speedtest1-common.eflags) $(speedtest1.cflags) \
$(SQLITE_OPT) \
$(speedtest1.exit-runtime1) \
$(kvvfs.cflags) \
-o $@ $(speedtest1.c) $(sqlite3-wasm.c) -lm
$(maybe-wasm-strip) $(speedtest1-kvvfs.wasm)
ls -la $@ $(speedtest1-kvvfs.wasm)
speedtest1: $(speedtest1-kvvfs.js)

2
ext/wasm/speedtest1-wasmfs.html
View File

@ -139,7 +139,7 @@
"Give it time...");
logList.length = 0;
setTimeout(function(){
wasm.xCall('__main_argc_argv', argv.length,
wasm.xCall('wasm_main', argv.length,
wasm.scopedAllocMainArgv(argv));
wasm.scopedAllocPop(scope);
if(pDir) unlink(dbFile);

18
ext/wasm/speedtest1-worker.html
View File

@ -30,7 +30,9 @@
<div id='toolbar'>
<div id='toolbar-select'>
<select id='select-flags' size='10' multiple></select>
<div>TODO? Options which require values are not represented here.</div>
<div>The following flags can be passed as URL parameters:
vfs=NAME, size=N
</div>
</div>
<div class='toolbar-inner-vertical'>
<div id='toolbar-selected-flags'></div>
@ -124,6 +126,7 @@
justify-content: flex-start;
}
</style>
<script src="common/SqliteTestUtil.js"></script>
<script>(function(){
'use strict';
const E = (sel)=>document.querySelector(sel);
@ -167,7 +170,7 @@
return str+a.join('&nbsp;');
};
const W = new Worker("speedtest1-worker.js");
const W = new Worker("speedtest1-worker.js"+self.location.search);
const mPost = function(msgType,payload){
W.postMessage({type: msgType, data: payload});
};
@ -177,7 +180,16 @@
const eLinkMainThread = E('#link-main-thread');
const eLinkWasmfs = E('#link-wasmfs');
const eLinkKvvfs = E('#link-kvvfs');
const getSelectedFlags = ()=>Array.prototype.map.call(eFlags.selectedOptions, (v)=>v.value);
const urlParams = new URL(self.location.href).searchParams;
const getSelectedFlags = ()=>{
const f = Array.prototype.map.call(eFlags.selectedOptions, (v)=>v.value);
[
'size', 'vfs'
].forEach(function(k){
if(urlParams.has(k)) f.push('--'+k, urlParams.get(k));
});
return f;
};
const updateSelectedFlags = function(){
eSelectedFlags.innerText = '';
const flags = getSelectedFlags();

34
ext/wasm/speedtest1-worker.js
View File

@ -6,7 +6,7 @@
returns the name of the dir on which OPFS is mounted, else it returns
an empty string.
*/
const opfsDir = function f(wasmUtil){
const wasmfsDir = function f(wasmUtil){
if(undefined !== f._) return f._;
const pdir = '/persistent';
if( !self.FileSystemHandle
@ -27,7 +27,7 @@
return f._ = "";
}
};
opfsDir._ = undefined;
wasmfsDir._ = undefined;
const mPost = function(msgType,payload){
postMessage({type: msgType, data: payload});
@ -42,7 +42,7 @@
};
const log = (...args)=>logMsg('stdout',args);
const logErr = (...args)=>logMsg('stderr',args);
const runSpeedtest = function(cliFlagsArray){
const scope = App.wasm.scopedAllocPush();
const dbFile = 0 ? "" : App.pDir+"/speedtest1.db";
@ -52,7 +52,7 @@
];
App.logBuffer.length = 0;
mPost('run-start', [...argv]);
App.wasm.xCall('__main_argc_argv', argv.length,
App.wasm.xCall('wasm_main', argv.length,
App.wasm.scopedAllocMainArgv(argv));
}catch(e){
mPost('error',e.message);
@ -79,20 +79,18 @@
printErr: logErr,
setStatus: (text)=>mPost('load-status',text)
};
self.sqlite3Speedtest1InitModule(EmscriptenModule).then(function(EmscriptenModule){
self.sqlite3Speedtest1InitModule(EmscriptenModule).then(function(EModule){
const S = EModule.sqlite3;
log("Module inited.");
App.wasm = {
exports: EmscriptenModule.asm,
alloc: (n)=>EmscriptenModule._malloc(n),
dealloc: (m)=>EmscriptenModule._free(m),
memory: EmscriptenModule.asm.memory || EmscriptenModule.wasmMemory
};
//console.debug('wasm =',wasm);
self.WhWasmUtilInstaller(App.wasm);
App.unlink = App.wasm.xWrap("sqlite3_wasm_vfs_unlink", "int", ["string"]);
App.pDir = opfsDir(App.wasm);
if(App.pDir) log("Persistent storage:",pDir);
else log("Using transient storage.");
mPost('ready',true);
return S.installOpfsVfs()
.catch((e)=>console.warn(e.message))
.then(()=>{
App.unlink = S.capi.wasm.xWrap("sqlite3_wasm_vfs_unlink", "int", ["string"]);
App.pDir = wasmfsDir(S.wasm);
App.wasm = S.capi.wasm;
//if(App.pDir) log("Persistent storage:",pDir);
//else log("Using transient storage.");
mPost('ready',true);
});
});
})();

56
ext/wasm/speedtest1.html
View File

@ -45,7 +45,7 @@
returns the name of the dir on which OPFS is mounted, else it returns
an empty string.
*/
const opfsDir = function f(wasmUtil){
const wasmfsDir = function f(wasmUtil){
if(undefined !== f._) return f._;
const pdir = '/persistent';
if( !self.FileSystemHandle
@ -66,7 +66,7 @@
return f._ = "";
}
};
opfsDir._ = undefined;
wasmfsDir._ = undefined;
const eOut = document.querySelector('#test-output');
const log2 = function(cssClass,...args){
@ -92,28 +92,41 @@
logList.push('ERROR: '+args.join(' '));
};
const runTests = function(EmscriptenModule){
console.log("Module inited.",EmscriptenModule);
const wasm = {
exports: EmscriptenModule.asm,
alloc: (n)=>EmscriptenModule._malloc(n),
dealloc: (m)=>EmscriptenModule._free(m),
memory: EmscriptenModule.asm.memory || EmscriptenModule.wasmMemory
};
//console.debug('wasm =',wasm);
self.WhWasmUtilInstaller(wasm);
const runTests = function(sqlite3){
const capi = sqlite3.capi, wasm = capi.wasm;
//console.debug('sqlite3 =',sqlite3);
const unlink = wasm.xWrap("sqlite3_wasm_vfs_unlink", "int", ["string"]);
const pDir = opfsDir(wasm);
const pDir = wasmfsDir(wasm);
if(pDir){
console.warn("Persistent storage:",pDir);
}
const scope = wasm.scopedAllocPush();
const dbFile = 0 ? "" : pDir+"/speedtest1.db";
const urlArgs = self.SqliteTestUtil.processUrlArgs();
let dbFile = pDir+"/speedtest1.db";
const urlParams = self.SqliteTestUtil.processUrlArgs();
const argv = ["speedtest1"];
if(urlArgs.flags){
if('string'===typeof urlParams.vfs){
if(!capi.sqlite3_vfs_find(urlParams.vfs)){
log2('error',"Unknown VFS:",urlParams.vfs);
return;
}
argv.push("--vfs", urlParams.vfs);
log2('',"Using VFS:",urlParams.vfs);
if('kvvfs' === urlParams.vfs){
urlParams.size = 1;
dbFile = 'session';
log2('',"kvvfs VFS: forcing --size 1 and filename 'session'");
capi.sqlite3_web_kvvfs_clear('session');
}
}
[
'size'
].forEach(function(k){
const v = urlParams[k];
if(v) argv.push('--'+k, urlParams[k]);
});
if(urlParams.flags){
// transform flags=a,b,c to ["--a", "--b", "--c"]
argv.push(...(urlArgs.flags.split(',').map((v)=>'--'+v)));
argv.push(...(urlParams.flags.split(',').map((v)=>'--'+v)));
}else{
argv.push(
"--singlethread",
@ -134,7 +147,7 @@
"Give it time...");
logList.length = 0;
setTimeout(function(){
wasm.xCall('__main_argc_argv', argv.length,
wasm.xCall('wasm_main', argv.length,
wasm.scopedAllocMainArgv(argv));
wasm.scopedAllocPop(scope);
if(pDir) unlink(dbFile);
@ -145,8 +158,11 @@
self.sqlite3TestModule.print = log;
self.sqlite3TestModule.printErr = logErr;
sqlite3Speedtest1InitModule(self.sqlite3TestModule).then(function(M){
setTimeout(()=>runTests(M), 100);
sqlite3Speedtest1InitModule(self.sqlite3TestModule)
.then((EmscriptenModule)=>{
return EmscriptenModule.sqlite3.installOpfsVfs()
.catch((e)=>{console.warn(e.message)})
.then(()=>runTests(EmscriptenModule.sqlite3));
});
})();
</script>