database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-08-08 14:02:16 +03:00
Code Activity

Merge the latest trunk enhancments into the reuse-schema branch.

Browse Source 
FossilOrigin-Name: 52262ac9205e7e4f19855a47955a5df10d53c222b4872d2ed1e5ca5230034a1d
This commit is contained in:
drh
2023-10-23 19:27:44 +00:00
parent 1d8366292e 4fecc81e20
commit 7c897ff2d6
297 changed files with 23786 additions and 10492 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
ext/wasm/GNUmakefile
View File

@@ -456,13 +456,6 @@ emcc.exportedRuntimeMethods := \
emcc.jsflags += $(emcc.exportedRuntimeMethods)
emcc.jsflags += -sUSE_CLOSURE_COMPILER=0
emcc.jsflags += -sIMPORTED_MEMORY
#emcc.jsflags += -sASYNCIFY=2
# ^^^ ASYNCIFY=2 is for experimental JSPI support
# (https://v8.dev/blog/jspi), but enabling it causes the lib-level
# init code to throw inexplicable complaints about C-level function
# signatures not matching what we expect them to be. JSPI requires, as of
# this writing, requires an experimental Chrome flag:
# chrome://flags/#enable-experimental-webassembly-stack-switching
emcc.jsflags += -sSTRICT_JS=0
# STRICT_JS disabled due to:
# https://github.com/emscripten-core/emscripten/issues/18610
@@ -845,11 +838,11 @@ sqlite3-worker1-bundler-friendly.js := $(dir.dout)/sqlite3-worker1-bundler-frien
sqlite3-worker1-promiser-bundler-friendly.js := $(dir.dout)/sqlite3-worker1-promiser-bundler-friendly.js
$(eval $(call C-PP.FILTER,$(sqlite3-worker1.js.in),$(sqlite3-worker1.js)))
$(eval $(call C-PP.FILTER,$(sqlite3-worker1.js.in),$(sqlite3-worker1-bundler-friendly.js),\
$(c-pp.D.bundler-friendly)))
$(c-pp.D.sqlite3-bundler-friendly)))
$(eval $(call C-PP.FILTER,$(sqlite3-worker1-promiser.js.in),$(sqlite3-worker1-promiser.js)))
$(eval $(call C-PP.FILTER,$(sqlite3-worker1-promiser.js.in),\
$(sqlite3-worker1-promiser-bundler-friendly.js),\
$(c-pp.D.bundler-friendly)))
$(c-pp.D.sqlite3-bundler-friendly)))
$(sqlite3-bundler-friendly.mjs): $(sqlite3-worker1-bundler-friendly.js) \
$(sqlite3-worker1-promiser-bundler-friendly.js)
$(sqlite3.js) $(sqlite3.mjs): $(sqlite3-worker1.js) $(sqlite3-worker1-promiser.js)

55
ext/wasm/SQLTester/GNUmakefile Normal file
View File

@@ -0,0 +1,55 @@
#!/this/is/make
#
# This makefile compiles SQLTester test files into something
# we can readily import into JavaScript.
all:
SHELL := $(shell which bash 2>/dev/null)
MAKEFILE := $(lastword $(MAKEFILE_LIST))
CLEAN_FILES :=
DISTCLEAN_FILES := ./--dummy-- *~
test-list.mjs := test-list.mjs
test-list.mjs.gz := $(test-list.mjs).gz
CLEAN_FILES += $(test-list.mjs)
tests.dir := $(firstword $(wildcard tests ../../jni/src/tests))
$(info test script dir=$(tests.dir))
tests.all := $(wildcard $(tests.dir)/*.test)
bin.touint8array := ./touint8array
$(bin.touint8array): $(bin.touint8array).c $(MAKEFILE)
$(CC) -o $@ $<
CLEAN_FILES += $(bin.touint8array)
ifneq (,$(tests.all))
$(test-list.mjs): $(bin.touint8array) $(tests.all) $(MAKEFILE)
@{\
echo 'export default ['; \
sep=''; \
for f in $(sort $(tests.all)); do \
echo -en $$sep'{"name": "'$${f##*/}'", "content":'; \
$(bin.touint8array) < $$f; \
echo -n '}'; \
sep=',\n'; \
done; \
echo '];'; \
} > $@
@echo "Created $@"
$(test-list.mjs.gz): $(test-list.mjs)
gzip -c $< > $@
CLEAN_FILES += $(test-list.mjs.gz)
all: $(test-list.mjs.gz)
else
@echo "Cannot build $(test-list.mjs) for lack of input test files."; \
echo "Symlink ./tests to a directory containing SQLTester-format "; \
echo "test scripts named *.test, then try again"; \
exit 1
endif
.PHONY: clean distclean
clean:
-rm -f $(CLEAN_FILES)
distclean: clean
-rm -f $(DISTCLEAN_FILES)

1339
ext/wasm/SQLTester/SQLTester.mjs Normal file
View File

File diff suppressed because it is too large Load Diff

148
ext/wasm/SQLTester/SQLTester.run.mjs Normal file
View File

@@ -0,0 +1,148 @@
/*
** 2023-08-29
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This file contains a test application for SQLTester.js.
*/
import {default as ns} from './SQLTester.mjs';
import {default as allTests} from './test-list.mjs';
globalThis.sqlite3 = ns.sqlite3;
const log = function f(...args){
console.log('SQLTester.run:',...args);
return f;
};
const out = function f(...args){ return f.outer.out(...args) };
out.outer = new ns.Outer();
out.outer.getOutputPrefix = ()=>'SQLTester.run: ';
const outln = (...args)=>{ return out.outer.outln(...args) };
const affirm = function(expr, msg){
if( !expr ){
throw new Error(arguments[1]
? ("Assertion failed: "+arguments[1])
: "Assertion failed");
}
}
let ts = new ns.TestScript('/foo.test',`
/*
** This is a comment. There are many like it but this one is mine.
**
** SCRIPT_MODULE_NAME: sanity-check-0
** xMIXED_MODULE_NAME: mixed-module
** xMODULE_NAME: module-name
** xREQUIRED_PROPERTIES: small fast reliable
** xREQUIRED_PROPERTIES: RECURSIVE_TRIGGERS
** xREQUIRED_PROPERTIES: TEMPSTORE_FILE TEMPSTORE_MEM
** xREQUIRED_PROPERTIES: AUTOVACUUM INCRVACUUM
**
*/
/* --verbosity 3 */
/* ---must-fail */
/* # must fail */
/* --verbosity 0 */
--print Hello, world.
--close all
--oom
--db 0
--new my.db
--null zilch
--testcase 1.0
SELECT 1, null;
--result 1 zilch
--glob *zil*
--notglob *ZIL*
SELECT 1, 2;
intentional error;
--run
/* ---intentional-failure */
--testcase json-1
SELECT json_array(1,2,3)
--json [1,2,3]
--testcase tableresult-1
select 1, 'a';
select 2, 'b';
--tableresult
# [a-z]
2 b
--end
--testcase json-block-1
select json_array(1,2,3);
select json_object('a',1,'b',2);
--json-block
[1,2,3]
{"a":1,"b":2}
--end
--testcase col-names-on
--column-names 1
select 1 as 'a', 2 as 'b';
--result a 1 b 2
--testcase col-names-off
--column-names 0
select 1 as 'a', 2 as 'b';
--result 1 2
--close
--print Until next time
`);
const sqt = new ns.SQLTester()
.setLogger(console.log.bind(console))
.verbosity(1)
.addTestScript(ts);
sqt.outer().outputPrefix('');
const runTests = function(){
try{
if( 0 ){
affirm( !sqt.getCurrentDb(), 'sqt.getCurrentDb()' );
sqt.openDb('/foo.db', true);
affirm( !!sqt.getCurrentDb(),'sqt.getCurrentDb()' );
affirm( 'zilch' !== sqt.nullValue() );
ts.run(sqt);
affirm( 'zilch' === sqt.nullValue() );
sqt.addTestScript(ts);
sqt.runTests();
}else{
for(const t of allTests){
sqt.addTestScript( new ns.TestScript(t) );
}
allTests.length = 0;
sqt.runTests();
}
}finally{
//log( "Metrics:", sqt.metrics );
sqt.reset();
}
};
if( globalThis.WorkerGlobalScope ){
const wPost = (type,payload)=>globalThis.postMessage({type, payload});
globalThis.onmessage = function({data}){
switch(data.type){
case 'run-tests':{
try{ runTests(); }
finally{ wPost('tests-end', sqt.metrics); }
break;
}
default:
log("unhandled onmessage: ",data);
break;
}
};
sqt.setLogger((msg)=>{
wPost('stdout', {message: msg});
});
wPost('is-ready');
//globalThis.onmessage({data:{type:'run-tests'}});
}else{
runTests();
}

127
ext/wasm/SQLTester/index.html Normal file
View File

@@ -0,0 +1,127 @@
<!doctype html>
<html lang="en-us">
<head>
<meta charset="utf-8">
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<link rel="shortcut icon" href="data:image/x-icon;," type="image/x-icon">
<!--link rel="stylesheet" href="../common/emscripten.css"/-->
<link rel="stylesheet" href="../common/testing.css"/>
<title>SQLTester</title>
</head>
<style>
fieldset {
display: flex;
flex-direction: row;
padding-right: 1em;
}
fieldset > :not(.legend) {
display: flex;
flex-direction: row;
padding-right: 1em;
}
</style>
<body>
<h1>SQLTester for JS/WASM</h1>
<p>This app reads in a build-time-defined set of SQLTester test
scripts and runs them through the test suite.
</p>
<fieldset>
<legend>Options</legend>
<span class='input-wrapper'>
<input type='checkbox' id='cb-log-reverse' checked>
<label for='cb-log-reverse'>Reverse log order?</label>
</span>
<input type='button' id='btn-run-tests' value='Run tests'/>
</fieldset>
<div id='test-output'>Test output will go here.</div>
<!--script src='SQLTester.run.mjs' type='module'></script-->
<script>
(async function(){
const W = new Worker('SQLTester.run.mjs',{
type: 'module'
});
const wPost = (type,payload)=>W.postMessage({type,payload});
const mapToString = (v)=>{
switch(typeof v){
case 'string': return v;
case 'number': case 'boolean':
case 'undefined': case 'bigint':
return ''+v;
default: break;
}
if(null===v) return 'null';
if(v instanceof Error){
v = {
message: v.message,
stack: v.stack,
errorClass: v.name
};
}
return JSON.stringify(v,undefined,2);
};
const normalizeArgs = (args)=>args.map(mapToString);
const logTarget = document.querySelector('#test-output');
const logClass = function(cssClass,...args){
const ln = document.createElement('div');
if(cssClass){
for(const c of (Array.isArray(cssClass) ? cssClass : [cssClass])){
ln.classList.add(c);
}
}
ln.append(document.createTextNode(normalizeArgs(args).join(' ')));
logTarget.append(ln);
};
{
const cbReverse = document.querySelector('#cb-log-reverse');
const cbReverseKey = 'SQLTester:cb-log-reverse';
const cbReverseIt = ()=>{
logTarget.classList[cbReverse.checked ? 'add' : 'remove']('reverse');
};
cbReverse.addEventListener('change', cbReverseIt, true);
cbReverseIt();
}
const btnRun = document.querySelector('#btn-run-tests');
const runTests = ()=>{
btnRun.setAttribute('disabled','disabled');
wPost('run-tests');
logTarget.innerText = 'Running tests...';
}
btnRun.addEventListener('click', runTests);
const log2 = function f(...args){
logClass('', ...args);
return f;
};
const log = function f(...args){
logClass('','index.html:',...args);
return f;
};
const timerId = setTimeout( ()=>{
logClass('error',"The SQLTester module is taking an unusually ",
"long time to load. More information may be available",
"in the dev console.");
}, 3000 /* assuming localhost */ );
W.onmessage = function({data}){
switch(data.type){
case 'stdout': log2(data.payload.message); break;
case 'tests-end':
btnRun.removeAttribute('disabled');
delete data.payload.nTest;
log("test results:",data.payload);
break;
case 'is-ready':
clearTimeout(timerId);
runTests(); break;
default:
log("unhandled onmessage",data);
break;
}
};
//runTests()
/* Inexplicably, */
})();
</script>
</body>
</html>

29
ext/wasm/SQLTester/touint8array.c Normal file
View File

@@ -0,0 +1,29 @@
/*
** 2023-08-29
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This file contains a tool for writing out the contents of stdin as
** a comma-separated list of numbers, one per byte.
*/
#include <stdio.h>
int main(int argc, char const **argv){
int i;
int rc = 0, colWidth = 30;
int ch;
printf("[");
for( i=0; EOF!=(ch = fgetc(stdin)); ++i ){
if( 0!=i ) printf(",");
if( i && 0==(i%colWidth) ) puts("");
printf("%d",ch);
}
printf("]");
return rc;
}

6
ext/wasm/api/sqlite3-api-glue.js
View File

@@ -888,9 +888,9 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
consistency with non-special-case wrappings.
*/
const __dbArgcMismatch = (pDb,f,n)=>{
return sqlite3.util.sqlite3_wasm_db_error(pDb, capi.SQLITE_MISUSE,
f+"() requires "+n+" argument"+
(1===n?"":'s')+".");
return util.sqlite3_wasm_db_error(pDb, capi.SQLITE_MISUSE,
f+"() requires "+n+" argument"+
(1===n?"":'s')+".");
};
/** Code duplication reducer for functions which take an encoding

75
ext/wasm/api/sqlite3-api-prologue.js
View File

@@ -772,8 +772,43 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
isSharedTypedArray,
toss: function(...args){throw new Error(args.join(' '))},
toss3,
typedArrayPart
};
typedArrayPart,
/**
Given a byte array or ArrayBuffer, this function throws if the
lead bytes of that buffer do not hold a SQLite3 database header,
else it returns without side effects.
Added in 3.44.
*/
affirmDbHeader: function(bytes){
if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes);
const header = "SQLite format 3";
if( header.length > bytes.byteLength ){
toss3("Input does not contain an SQLite3 database header.");
}
for(let i = 0; i < header.length; ++i){
if( header.charCodeAt(i) !== bytes[i] ){
toss3("Input does not contain an SQLite3 database header.");
}
}
},
/**
Given a byte array or ArrayBuffer, this function throws if the
database does not, at a cursory glance, appear to be an SQLite3
database. It only examines the size and header, but further
checks may be added in the future.
Added in 3.44.
*/
affirmIsDb: function(bytes){
if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes);
const n = bytes.byteLength;
if(n<512 || n%512!==0) {
toss3("Byte array size",n,"is invalid for an SQLite3 db.");
}
util.affirmDbHeader(bytes);
}
}/*util*/;
Object.assign(wasm, {
/**
@@ -1100,7 +1135,23 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
return 1===n
? wasm.pstack.alloc(safePtrSize ? 8 : wasm.ptrSizeof)
: wasm.pstack.allocChunks(n, safePtrSize ? 8 : wasm.ptrSizeof);
},
/**
Records the current pstack position, calls the given function,
passing it the sqlite3 object, then restores the pstack
regardless of whether the function throws. Returns the result
of the call or propagates an exception on error.
Added in 3.44.
*/
call: function(f){
const stackPos = wasm.pstack.pointer;
try{ return f(sqlite3) } finally{
wasm.pstack.restore(stackPos);
}
}
})/*wasm.pstack*/;
Object.defineProperties(wasm.pstack, {
/**
@@ -1508,6 +1559,26 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
}
};
/**
Converts SQL input from a variety of convenient formats
to plain strings.
If v is a string, it is returned as-is. If it is-a Array, its
join("") result is returned. If is is a Uint8Array, Int8Array,
or ArrayBuffer, it is assumed to hold UTF-8-encoded text and is
decoded to a string. If it looks like a WASM pointer,
wasm.cstrToJs(sql) is returned. Else undefined is returned.
Added in 3.44
*/
capi.sqlite3_js_sql_to_string = (sql)=>{
if('string' === typeof sql){
return sql;
}
const x = flexibleString(v);
return x===v ? undefined : x;
}
if( util.isUIThread() ){
/* Features specific to the main window thread... */

3
ext/wasm/api/sqlite3-api-worker1.js
View File

@@ -333,7 +333,6 @@ sqlite3.initWorker1API = function(){
if(!(globalThis.WorkerGlobalScope instanceof Function)){
toss("initWorker1API() must be run from a Worker thread.");
}
const self = this.self;
const sqlite3 = this.sqlite3 || toss("Missing this.sqlite3 object.");
const DB = sqlite3.oo1.DB;
@@ -657,5 +656,5 @@ sqlite3.initWorker1API = function(){
}, wState.xfer);
};
globalThis.postMessage({type:'sqlite3-api',result:'worker1-ready'});
}.bind({self, sqlite3});
}.bind({sqlite3});
});

101
ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js
View File

@@ -59,6 +59,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
const toss3 = sqlite3.util.toss3;
const initPromises = Object.create(null);
const capi = sqlite3.capi;
const util = sqlite3.util;
const wasm = sqlite3.wasm;
// Config opts for the VFS...
const SECTOR_SIZE = 4096;
@@ -154,8 +155,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
pool.deletePath(file.path);
}
}catch(e){
pool.storeErr(e);
return capi.SQLITE_IOERR;
return pool.storeErr(e, capi.SQLITE_IOERR);
}
}
return 0;
@@ -199,8 +199,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
}
return 0;
}catch(e){
pool.storeErr(e);
return capi.SQLITE_IOERR;
return pool.storeErr(e, capi.SQLITE_IOERR);
}
},
xSectorSize: function(pFile){
@@ -216,8 +215,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
file.sah.flush();
return 0;
}catch(e){
pool.storeErr(e);
return capi.SQLITE_IOERR;
return pool.storeErr(e, capi.SQLITE_IOERR);
}
},
xTruncate: function(pFile,sz64){
@@ -230,8 +228,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
file.sah.truncate(HEADER_OFFSET_DATA + Number(sz64));
return 0;
}catch(e){
pool.storeErr(e);
return capi.SQLITE_IOERR;
return pool.storeErr(e, capi.SQLITE_IOERR);
}
},
xUnlock: function(pFile,lockType){
@@ -251,10 +248,9 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
wasm.heap8u().subarray(pSrc, pSrc+n),
{ at: HEADER_OFFSET_DATA + Number(offset64) }
);
return nBytes === n ? 0 : capi.SQLITE_IOERR;
return n===nBytes ? 0 : toss("Unknown write() failure.");
}catch(e){
pool.storeErr(e);
return capi.SQLITE_IOERR;
return pool.storeErr(e, capi.SQLITE_IOERR);
}
}
}/*ioMethods*/;
@@ -313,8 +309,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
},
xGetLastError: function(pVfs,nOut,pOut){
const pool = getPoolForVfs(pVfs);
pool.log(`xGetLastError ${nOut}`);
const e = pool.popErr();
pool.log(`xGetLastError ${nOut} e =`,e);
if(e){
const scope = wasm.scopedAllocPush();
try{
@@ -327,7 +323,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
wasm.scopedAllocPop(scope);
}
}
return 0;
return e ? (e.sqlite3Rc || capi.SQLITE_IOERR) : 0;
},
//xSleep is optionally defined below
xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){
@@ -761,12 +757,20 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
}
/**
Sets e as this object's current error. Pass a falsy
(or no) value to clear it.
Sets e (an Error object) as this object's current error. Pass a
falsy (or no) value to clear it. If code is truthy it is
assumed to be an SQLITE_xxx result code, defaulting to
SQLITE_IOERR if code is falsy.
Returns the 2nd argument.
*/
storeErr(e){
if(e) this.error(e);
return this.$error = e;
storeErr(e,code){
if(e){
e.sqlite3Rc = code || capi.SQLITE_IOERR;
this.error(e);
}
this.$error = e;
return code;
}
/**
Pops this object's Error object and returns
@@ -869,9 +873,49 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
return b;
}
//! Impl for importDb() when its 2nd arg is a function.
async importDbChunked(name, callback){
const sah = this.#mapFilenameToSAH.get(name)
|| this.nextAvailableSAH()
|| toss("No available handles to import to.");
sah.truncate(0);
let nWrote = 0, chunk, checkedHeader = false, err = false;
try{
while( undefined !== (chunk = await callback()) ){
if(chunk instanceof ArrayBuffer) chunk = new Uint8Array(chunk);
if( 0===nWrote && chunk.byteLength>=15 ){
util.affirmDbHeader(chunk);
checkedHeader = true;
}
sah.write(chunk, {at: HEADER_OFFSET_DATA + nWrote});
nWrote += chunk.byteLength;
}
if( nWrote < 512 || 0!==nWrote % 512 ){
toss("Input size",nWrote,"is not correct for an SQLite database.");
}
if( !checkedHeader ){
const header = new Uint8Array(20);
sah.read( header, {at: 0} );
util.affirmDbHeader( header );
}
sah.write(new Uint8Array([1,1]), {
at: HEADER_OFFSET_DATA + 18
}/*force db out of WAL mode*/);
}catch(e){
this.setAssociatedPath(sah, '', 0);
throw e;
}
this.setAssociatedPath(sah, name, capi.SQLITE_OPEN_MAIN_DB);
return nWrote;
}
//! Documented elsewhere in this file.
importDb(name, bytes){
if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes);
if( bytes instanceof ArrayBuffer ) bytes = new Uint8Array(bytes);
else if( bytes instanceof Function ) return this.importDbChunked(name, bytes);
const sah = this.#mapFilenameToSAH.get(name)
|| this.nextAvailableSAH()
|| toss("No available handles to import to.");
const n = bytes.byteLength;
if(n<512 || n%512!=0){
toss("Byte array size is invalid for an SQLite db.");
@@ -882,16 +926,16 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
toss("Input does not contain an SQLite database header.");
}
}
const sah = this.#mapFilenameToSAH.get(name)
|| this.nextAvailableSAH()
|| toss("No available handles to import to.");
const nWrote = sah.write(bytes, {at: HEADER_OFFSET_DATA});
if(nWrote != n){
this.setAssociatedPath(sah, '', 0);
toss("Expected to write "+n+" bytes but wrote "+nWrote+".");
}else{
sah.write(new Uint8Array([1,1]), {at: HEADER_OFFSET_DATA+18}
/* force db out of WAL mode */);
this.setAssociatedPath(sah, name, capi.SQLITE_OPEN_MAIN_DB);
}
return nWrote;
}
}/*class OpfsSAHPool*/;
@@ -1098,6 +1142,19 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
automatically clean up any non-database files so importing them
is pointless.
If passed a function for its second argument, its behavior
changes to asynchronous and it imports its data in chunks fed to
it by the given callback function. It calls the callback (which
may be async) repeatedly, expecting either a Uint8Array or
ArrayBuffer (to denote new input) or undefined (to denote
EOF). For so long as the callback continues to return
non-undefined, it will append incoming data to the given
VFS-hosted database file. The result of the resolved Promise when
called this way is the size of the resulting database.
On succes this routine rewrites the database header bytes in the
output file (not the input array) to force disabling of WAL mode.
On a write error, the handle is removed from the pool and made
available for re-use.

98
ext/wasm/api/sqlite3-vfs-opfs.c-pp.js
View File

@@ -136,6 +136,7 @@ const installOpfsVfs = function callee(options){
const error = (...args)=>logImpl(0, ...args);
const toss = sqlite3.util.toss;
const capi = sqlite3.capi;
const util = sqlite3.util;
const wasm = sqlite3.wasm;
const sqlite3_vfs = capi.sqlite3_vfs;
const sqlite3_file = capi.sqlite3_file;
@@ -1168,40 +1169,101 @@ const installOpfsVfs = function callee(options){
doDir(opt.directory, 0);
};
/**
impl of importDb() when it's given a function as its second
argument.
*/
const importDbChunked = async function(filename, callback){
const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true);
const hFile = await hDir.getFileHandle(fnamePart, {create:true});
let sah = await hFile.createSyncAccessHandle();
let nWrote = 0, chunk, checkedHeader = false, err = false;
try{
sah.truncate(0);
while( undefined !== (chunk = await callback()) ){
if(chunk instanceof ArrayBuffer) chunk = new Uint8Array(chunk);
if( 0===nWrote && chunk.byteLength>=15 ){
util.affirmDbHeader(chunk);
checkedHeader = true;
}
sah.write(chunk, {at: nWrote});
nWrote += chunk.byteLength;
}
if( nWrote < 512 || 0!==nWrote % 512 ){
toss("Input size",nWrote,"is not correct for an SQLite database.");
}
if( !checkedHeader ){
const header = new Uint8Array(20);
sah.read( header, {at: 0} );
util.affirmDbHeader( header );
}
sah.write(new Uint8Array([1,1]), {at: 18}/*force db out of WAL mode*/);
return nWrote;
}catch(e){
await sah.close();
sah = undefined;
await hDir.removeEntry( fnamePart ).catch(()=>{});
throw e;
}finally {
if( sah ) await sah.close();
}
};
/**
Asynchronously imports the given bytes (a byte array or
ArrayBuffer) into the given database file.
If passed a function for its second argument, its behaviour
changes to async and it imports its data in chunks fed to it by
the given callback function. It calls the callback (which may
be async) repeatedly, expecting either a Uint8Array or
ArrayBuffer (to denote new input) or undefined (to denote
EOF). For so long as the callback continues to return
non-undefined, it will append incoming data to the given
VFS-hosted database file. When called this way, the resolved
value of the returned Promise is the number of bytes written to
the target file.
It very specifically requires the input to be an SQLite3
database and throws if that's not the case. It does so in
order to prevent this function from taking on a larger scope
than it is specifically intended to. i.e. we do not want it to
become a convenience for importing arbitrary files into OPFS.
Throws on error. Resolves to the number of bytes written.
This routine rewrites the database header bytes in the output
file (not the input array) to force disabling of WAL mode.
On error this throws and the state of the input file is
undefined (it depends on where the exception was triggered).
On success, resolves to the number of bytes written.
*/
opfsUtil.importDb = async function(filename, bytes){
if( bytes instanceof Function ){
return importDbChunked(filename, bytes);
}
if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes);
util.affirmIsDb(bytes);
const n = bytes.byteLength;
if(n<512 || n%512!=0){
toss("Byte array size is invalid for an SQLite db.");
}
const header = "SQLite format 3";
for(let i = 0; i < header.length; ++i){
if( header.charCodeAt(i) !== bytes[i] ){
toss("Input does not contain an SQLite database header.");
}
}
const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true);
const hFile = await hDir.getFileHandle(fnamePart, {create:true});
const sah = await hFile.createSyncAccessHandle();
sah.truncate(0);
const nWrote = sah.write(bytes, {at: 0});
sah.close();
if(nWrote != n){
toss("Expected to write "+n+" bytes but wrote "+nWrote+".");
let sah, err, nWrote = 0;
try {
const hFile = await hDir.getFileHandle(fnamePart, {create:true});
sah = await hFile.createSyncAccessHandle();
sah.truncate(0);
nWrote = sah.write(bytes, {at: 0});
if(nWrote != n){
toss("Expected to write "+n+" bytes but wrote "+nWrote+".");
}
sah.write(new Uint8Array([1,1]), {at: 18}) /* force db out of WAL mode */;
return nWrote;
}catch(e){
if( sah ){ await sah.close(); sah = undefined; }
await hDir.removeEntry( fnamePart ).catch(()=>{});
throw e;
}finally{
if( sah ) await sah.close();
}
return nWrote;
};
if(sqlite3.oo1){

122
ext/wasm/api/sqlite3-wasm.c
View File

@@ -84,6 +84,14 @@
/**********************************************************************/
/* SQLITE_ENABLE_... */
/*
** Unconditionally enable API_ARMOR in the WASM build. It ensures that
** public APIs behave predictable in the face of passing illegal NULLs
** or ranges which might otherwise invoke undefined behavior.
*/
#undef SQLITE_ENABLE_API_ARMOR
#define SQLITE_ENABLE_API_ARMOR 1
#ifndef SQLITE_ENABLE_BYTECODE_VTAB
# define SQLITE_ENABLE_BYTECODE_VTAB 1
#endif
@@ -352,7 +360,9 @@ int sqlite3_wasm_db_error(sqlite3*db, int err_code, const char *zMsg){
if( db!=0 ){
if( 0!=zMsg ){
const int nMsg = sqlite3Strlen30(zMsg);
sqlite3_mutex_enter(sqlite3_db_mutex(db));
sqlite3ErrorWithMsg(db, err_code, "%.*s", nMsg, zMsg);
sqlite3_mutex_leave(sqlite3_db_mutex(db));
}else{
sqlite3ErrorWithMsg(db, err_code, NULL);
}
@@ -1799,6 +1809,118 @@ char * sqlite3_wasm_test_str_hello(int fail){
}
return s;
}
/*
** For testing using SQLTester scripts.
**
** Return non-zero if string z matches glob pattern zGlob and zero if the
** pattern does not match.
**
** To repeat:
**
** zero == no match
** non-zero == match
**
** Globbing rules:
**
** '*' Matches any sequence of zero or more characters.
**
** '?' Matches exactly one character.
**
** [...] Matches one character from the enclosed list of
** characters.
**
** [^...] Matches one character not in the enclosed list.
**
** '#' Matches any sequence of one or more digits with an
** optional + or - sign in front, or a hexadecimal
** literal of the form 0x...
*/
static int sqlite3_wasm_SQLTester_strnotglob(const char *zGlob, const char *z){
int c, c2;
int invert;
int seen;
typedef int (*recurse_f)(const char *,const char *);
static const recurse_f recurse = sqlite3_wasm_SQLTester_strnotglob;
while( (c = (*(zGlob++)))!=0 ){
if( c=='*' ){
while( (c=(*(zGlob++))) == '*' || c=='?' ){
if( c=='?' && (*(z++))==0 ) return 0;
}
if( c==0 ){
return 1;
}else if( c=='[' ){
while( *z && recurse(zGlob-1,z)==0 ){
z++;
}
return (*z)!=0;
}
while( (c2 = (*(z++)))!=0 ){
while( c2!=c ){
c2 = *(z++);
if( c2==0 ) return 0;
}
if( recurse(zGlob,z) ) return 1;
}
return 0;
}else if( c=='?' ){
if( (*(z++))==0 ) return 0;
}else if( c=='[' ){
int prior_c = 0;
seen = 0;
invert = 0;
c = *(z++);
if( c==0 ) return 0;
c2 = *(zGlob++);
if( c2=='^' ){
invert = 1;
c2 = *(zGlob++);
}
if( c2==']' ){
if( c==']' ) seen = 1;
c2 = *(zGlob++);
}
while( c2 && c2!=']' ){
if( c2=='-' && zGlob[0]!=']' && zGlob[0]!=0 && prior_c>0 ){
c2 = *(zGlob++);
if( c>=prior_c && c<=c2 ) seen = 1;
prior_c = 0;
}else{
if( c==c2 ){
seen = 1;
}
prior_c = c2;
}
c2 = *(zGlob++);
}
if( c2==0 || (seen ^ invert)==0 ) return 0;
}else if( c=='#' ){
if( z[0]=='0'
&& (z[1]=='x' || z[1]=='X')
&& sqlite3Isxdigit(z[2])
){
z += 3;
while( sqlite3Isxdigit(z[0]) ){ z++; }
}else{
if( (z[0]=='-' || z[0]=='+') && sqlite3Isdigit(z[1]) ) z++;
if( !sqlite3Isdigit(z[0]) ) return 0;
z++;
while( sqlite3Isdigit(z[0]) ){ z++; }
}
}else{
if( c!=(*(z++)) ) return 0;
}
}
return *z==0;
}
SQLITE_WASM_EXPORT
int sqlite3_wasm_SQLTester_strglob(const char *zGlob, const char *z){
return !sqlite3_wasm_SQLTester_strnotglob(zGlob, z);
}
#endif /* SQLITE_WASM_TESTS */
#undef SQLITE_WASM_EXPORT

2
ext/wasm/api/sqlite3-worker1.c-pp.js
View File

@@ -37,7 +37,7 @@ import {default as sqlite3InitModule} from './sqlite3-bundler-friendly.mjs';
"use strict";
{
const urlParams = globalThis.location
? new URL(self.location.href).searchParams
? new URL(globalThis.location.href).searchParams
: new URLSearchParams();
let theJs = 'sqlite3.js';
if(urlParams.has('sqlite3.dir')){

46
ext/wasm/tester1.c-pp.js
View File

@@ -2939,8 +2939,27 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
let db;
try {
const exp = this.opfsDbExport;
const filename = this.opfsDbFile;
delete this.opfsDbExport;
this.opfsImportSize = await sqlite3.oo1.OpfsDb.importDb(this.opfsDbFile, exp);
this.opfsImportSize = await sqlite3.oo1.OpfsDb.importDb(filename, exp);
db = new sqlite3.oo1.OpfsDb(this.opfsDbFile);
T.assert(6 === db.selectValue('select count(*) from p')).
assert( this.opfsImportSize == exp.byteLength );
db.close();
this.opfsUnlink(filename);
T.assert(!(await sqlite3.opfs.entryExists(filename)));
// Try again with a function as an input source:
let cursor = 0;
const blockSize = 512, end = exp.byteLength;
const reader = async function(){
if(cursor >= exp.byteLength){
return undefined;
}
const rv = exp.subarray(cursor, cursor+blockSize>end ? end : cursor+blockSize);
cursor += blockSize;
return rv;
};
this.opfsImportSize = await sqlite3.oo1.OpfsDb.importDb(filename, reader);
db = new sqlite3.oo1.OpfsDb(this.opfsDbFile);
T.assert(6 === db.selectValue('select count(*) from p')).
assert( this.opfsImportSize == exp.byteLength );
@@ -3059,8 +3078,9 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
const dbytes = u1.exportFile(dbName);
T.assert(dbytes.length >= 4096);
const dbName2 = '/exported.db';
u1.importDb(dbName2, dbytes);
T.assert( 2 == u1.getFileCount() );
let nWrote = u1.importDb(dbName2, dbytes);
T.assert( 2 == u1.getFileCount() )
.assert( dbytes.byteLength == nWrote );
let db2 = new u1.OpfsSAHPoolDb(dbName2);
T.assert(db2 instanceof sqlite3.oo1.DB)
.assert(3 === db2.selectValue('select count(*) from t'));
@@ -3069,6 +3089,25 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
.assert(false === u1.unlink(dbName2))
.assert(1 === u1.getFileCount())
.assert(1 === u1.getFileNames().length);
// Try again with a function as an input source:
let cursor = 0;
const blockSize = 1024, end = dbytes.byteLength;
const reader = async function(){
if(cursor >= dbytes.byteLength){
return undefined;
}
const rv = dbytes.subarray(cursor, cursor+blockSize>end ? end : cursor+blockSize);
cursor += blockSize;
return rv;
};
nWrote = await u1.importDb(dbName2, reader);
T.assert( 2 == u1.getFileCount() );
db2 = new u1.OpfsSAHPoolDb(dbName2);
T.assert(db2 instanceof sqlite3.oo1.DB)
.assert(3 === db2.selectValue('select count(*) from t'));
db2.close();
T.assert(true === u1.unlink(dbName2))
.assert(dbytes.byteLength == nWrote);
}
T.assert(true === u1.unlink(dbName))
@@ -3214,4 +3253,3 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
TestUtil.runTests(sqlite3);
});
})(self);