1
0
mirror of https://github.com/sqlite/sqlite.git synced 2025-08-08 14:02:16 +03:00

Correct duplicate copies of sqlite3-api.js being embedded in the wasmfs-based builds.

FossilOrigin-Name: bbfcfba260f39a9c91e82d87e06b1c2cb297c03498b4530aa3e7e01ef9916012
This commit is contained in:
stephan
2022-09-28 13:01:49 +00:00
parent d98011852d
commit 5ad3631915
6 changed files with 389 additions and 216 deletions

View File

@@ -19,30 +19,110 @@
const warn = console.warn.bind(console);
let sqlite3;
/** Throws if the given sqlite3 result code is not 0. */
const checkSqliteRc = (dbh,rc)=>{
if(rc) toss("Prepare failed:",sqlite3.capi.sqlite3_errmsg(dbh));
};
const sqlToDrop = [
"SELECT type,name FROM sqlite_schema ",
"WHERE name NOT LIKE 'sqlite\\_%' escape '\\' ",
"AND name NOT LIKE '\\_%' escape '\\'"
].join('');
const clearDbWebSQL = function(db){
db.handle.transaction(function(tx){
const onErr = (e)=>console.error(e);
const callback = function(tx, result){
const rows = result.rows;
let i, n;
i = n = rows.length;
while(i--){
const row = rows.item(i);
const name = JSON.stringify(row.name);
const type = row.type;
switch(type){
case 'table': case 'view': case 'trigger':{
const sql2 = 'DROP '+type+' '+name;
warn(db.id,':',sql2);
tx.executeSql(sql2, [], atEnd, onErr);
break;
}
default:
warn("Unhandled db entry type:",type,name);
break;
}
}
};
tx.executeSql(sqlToDrop, [], callback, onErr);
db.handle.changeVersion(db.handle.version, "", ()=>{}, onErr, ()=>{});
});
};
const clearDbSqlite = function(db){
// This would be SO much easier with the oo1 API, but we specifically want to
// inject metrics we can't get via that API, and we cannot reliably (OPFS)
// open the same DB twice to clear it using that API, so...
let pStmt = 0, pSqlBegin;
const capi = sqlite3.capi, wasm = capi.wasm;
const scope = wasm.scopedAllocPush();
try {
const toDrop = [];
const ppStmt = wasm.scopedAllocPtr();
let rc = capi.sqlite3_prepare_v2(db.handle, sqlToDrop, -1, ppStmt, null);
checkSqliteRc(db.handle,rc);
pStmt = wasm.getPtrValue(ppStmt);
while(capi.SQLITE_ROW===capi.sqlite3_step(pStmt)){
toDrop.push(capi.sqlite3_column_text(pStmt,0),
capi.sqlite3_column_text(pStmt,1));
}
capi.sqlite3_finalize(pStmt);
pStmt = 0;
while(toDrop.length){
const name = toDrop.pop();
const type = toDrop.pop();
let sql2;
switch(type){
case 'table': case 'view': case 'trigger':
sql2 = 'DROP '+type+' '+name;
break;
default:
warn("Unhandled db entry type:",type,name);
continue;
}
wasm.setPtrValue(ppStmt, 0);
warn(db.id,':',sql2);
rc = capi.sqlite3_prepare_v2(db.handle, sql2, -1, ppStmt, null);
checkSqliteRc(db.handle,rc);
pStmt = wasm.getPtrValue(ppStmt);
capi.sqlite3_step(pStmt);
capi.sqlite3_finalize(pStmt);
pStmt = 0;
}
}finally{
if(pStmt) capi.sqlite3_finalize(pStmt);
wasm.scopedAllocPop(scope);
}
};
const E = (s)=>document.querySelector(s);
const App = {
e: {
output: document.querySelector('#test-output'),
selSql: document.querySelector('#sql-select'),
btnRun: document.querySelector('#sql-run'),
btnRunNext: document.querySelector('#sql-run-next'),
btnRunRemaining: document.querySelector('#sql-run-remaining'),
btnExportMetrics: document.querySelector('#export-metrics'),
btnClear: document.querySelector('#output-clear'),
btnReset: document.querySelector('#db-reset'),
cbReverseLog: document.querySelector('#cb-reverse-log-order')
output: E('#test-output'),
selSql: E('#sql-select'),
btnRun: E('#sql-run'),
btnRunNext: E('#sql-run-next'),
btnRunRemaining: E('#sql-run-remaining'),
btnExportMetrics: E('#export-metrics'),
btnClear: E('#output-clear'),
btnReset: E('#db-reset'),
cbReverseLog: E('#cb-reverse-log-order'),
selImpl: E('#select-impl')
},
db: Object.create(null),
dbs: Object.create(null),
cache:{},
metrics:{
/**
Map of sql-file to timing metrics. We currently only store
the most recent run of each file, but we really should store
all runs so that we can average out certain values which vary
significantly across runs. e.g. a mandelbrot-generating query
will have a wide range of runtimes when run 10 times in a
row.
*/
},
log: console.log.bind(console),
warn: console.warn.bind(console),
cls: function(){this.e.output.innerHTML = ''},
@@ -62,47 +142,159 @@
if(1) this.logHtml2('error', ...args);
},
openDb: function(fn, unlinkFirst=true){
if(this.db.ptr){
toss("Already have an opened db.");
}
execSql: async function(name,sql){
const db = this.getSelectedDb();
const banner = "========================================";
this.logHtml(banner,
"Running",name,'('+sql.length,'bytes) using',db.id);
const capi = this.sqlite3.capi, wasm = capi.wasm;
let pStmt = 0, pSqlBegin;
const stack = wasm.scopedAllocPush();
let pDb = 0;
try{
if(unlinkFirst && fn){
if(':'!==fn[0]) capi.wasm.sqlite3_wasm_vfs_unlink(fn);
this.clearStorage();
}
const oFlags = capi.SQLITE_OPEN_CREATE | capi.SQLITE_OPEN_READWRITE;
const ppDb = wasm.scopedAllocPtr();
const rc = capi.sqlite3_open_v2(fn, ppDb, oFlags, null);
pDb = wasm.getPtrValue(ppDb)
if(rc){
if(pDb) capi.sqlite3_close_v2(pDb);
toss("sqlite3_open_v2() failed with code",rc);
}
}finally{
wasm.scopedAllocPop(stack);
const metrics = db.metrics = Object.create(null);
metrics.prepTotal = metrics.stepTotal = 0;
metrics.stmtCount = 0;
metrics.malloc = 0;
metrics.strcpy = 0;
this.blockControls(true);
if(this.gotErr){
this.logErr("Cannot run SQL: error cleanup is pending.");
return;
}
this.db.filename = fn;
this.db.ptr = pDb;
this.logHtml("Opened db:",fn);
return this.db.ptr;
},
// Run this async so that the UI can be updated for the above header...
const dumpMetrics = ()=>{
metrics.evalSqlEnd = performance.now();
metrics.evalTimeTotal = (metrics.evalSqlEnd - metrics.evalSqlStart);
this.logHtml(db.id,"metrics:");//,JSON.stringify(metrics, undefined, ' '));
this.logHtml("prepare() count:",metrics.stmtCount);
this.logHtml("Time in prepare_v2():",metrics.prepTotal,"ms",
"("+(metrics.prepTotal / metrics.stmtCount),"ms per prepare())");
this.logHtml("Time in step():",metrics.stepTotal,"ms",
"("+(metrics.stepTotal / metrics.stmtCount),"ms per step())");
this.logHtml("Total runtime:",metrics.evalTimeTotal,"ms");
this.logHtml("Overhead (time - prep - step):",
(metrics.evalTimeTotal - metrics.prepTotal - metrics.stepTotal)+"ms");
this.logHtml(banner,"End of",name);
};
closeDb: function(unlink=false){
if(this.db.ptr){
this.sqlite3.capi.sqlite3_close_v2(this.db.ptr);
this.logHtml("Closed db",this.db.filename);
if(unlink){
capi.wasm.sqlite3_wasm_vfs_unlink(this.db.filename);
this.clearStorage();
}
this.db.ptr = this.db.filename = undefined;
let runner;
if('websql'===db.id){
runner = function(resolve, reject){
/* WebSQL cannot execute multiple statements, nor can it execute SQL without
an explicit transaction. Thus we have to do some fragile surgery on the
input SQL. Since we're only expecting carefully curated inputs, the hope is
that this will suffice. */
metrics.evalSqlStart = performance.now();
const sqls = sql.split(/;+\n/);
const rxBegin = /^BEGIN/i, rxCommit = /^COMMIT/i, rxComment = /^\s*--/;
try {
const nextSql = ()=>{
let x = sqls.shift();
while(x && rxComment.test(x)) x = sqls.shift();
return x && x.trim();
};
const transaction = function(tx){
let s;
for(;s = nextSql(); s = s.nextSql()){
if(rxBegin.test(s)) continue;
else if(rxCommit.test(s)) break;
++metrics.stmtCount;
const t = performance.now();
tx.executeSql(s);
metrics.stepTotal += performance.now() - t;
}
};
while(sqls.length){
db.handle.transaction(transaction);
}
resolve(this);
}catch(e){
this.gotErr = e;
reject(e);
return;
}
}.bind(this);
}else{/*sqlite3 db...*/
runner = function(resolve, reject){
metrics.evalSqlStart = performance.now();
try {
let t;
let sqlByteLen = sql.byteLength;
const [ppStmt, pzTail] = wasm.scopedAllocPtr(2);
t = performance.now();
pSqlBegin = wasm.scopedAlloc( sqlByteLen + 1/*SQL + NUL*/) || toss("alloc(",sqlByteLen,") failed");
metrics.malloc = performance.now() - t;
metrics.byteLength = sqlByteLen;
let pSql = pSqlBegin;
const pSqlEnd = pSqlBegin + sqlByteLen;
t = performance.now();
wasm.heap8().set(sql, pSql);
wasm.setMemValue(pSql + sqlByteLen, 0);
metrics.strcpy = performance.now() - t;
let breaker = 0;
while(pSql && wasm.getMemValue(pSql,'i8')){
wasm.setPtrValue(ppStmt, 0);
wasm.setPtrValue(pzTail, 0);
t = performance.now();
let rc = capi.sqlite3_prepare_v3(
db.handle, pSql, sqlByteLen, 0, ppStmt, pzTail
);
metrics.prepTotal += performance.now() - t;
checkSqliteRc(db.handle, rc);
pStmt = wasm.getPtrValue(ppStmt);
pSql = wasm.getPtrValue(pzTail);
sqlByteLen = pSqlEnd - pSql;
if(!pStmt) continue/*empty statement*/;
++metrics.stmtCount;
t = performance.now();
rc = capi.sqlite3_step(pStmt);
capi.sqlite3_finalize(pStmt);
pStmt = 0;
metrics.stepTotal += performance.now() - t;
switch(rc){
case capi.SQLITE_ROW:
case capi.SQLITE_DONE: break;
default: checkSqliteRc(db.handle, rc); toss("Not reached.");
}
}
resolve(this);
}catch(e){
if(pStmt) capi.sqlite3_finalize(pStmt);
this.gotErr = e;
reject(e);
return;
}finally{
wasm.scopedAllocPop(stack);
}
}.bind(this);
}
let p;
if(1){
p = new Promise(function(res,rej){
setTimeout(()=>runner(res, rej), 50)/*give UI a chance to output the "running" banner*/;
});
}else{
p = new Promise(runner);
}
return p.catch(
(e)=>this.logErr("Error via execSql("+name+",...):",e.message)
).finally(()=>{
dumpMetrics();
this.blockControls(false);
});
},
clearDb: function(){
const db = this.getSelectedDb();
if('websql'===db.id){
this.logErr("TODO: clear websql db.");
return;
}
if(!db.handle) return;
const capi = this.sqlite3, wasm = capi.wasm;
//const scope = wasm.scopedAllocPush(
this.logErr("TODO: clear db");
},
/**
Loads batch-runner.list and populates the selection list from
it. Returns a promise which resolves to nothing in particular
@@ -123,7 +315,7 @@
}
if(!r.ok) toss("Loading",infile,"failed:",r.statusText);
txt = await r.text();
const warning = document.querySelector('#warn-list');
const warning = E('#warn-list');
if(warning) warning.remove();
}catch(e){
this.logErr(e.message);
@@ -169,13 +361,6 @@
return sql;
}/*fetchFile()*/,
/** Throws if the given sqlite3 result code is not 0. */
checkRc: function(rc){
if(this.db.ptr && rc){
toss("Prepare failed:",this.sqlite3.capi.sqlite3_errmsg(this.db.ptr));
}
},
/** Disable or enable certain UI controls. */
blockControls: function(disable){
document.querySelectorAll('.disable-during-eval').forEach((e)=>e.disabled = disable);
@@ -191,11 +376,11 @@
const rc = [];
Object.keys(this.metrics).sort().forEach((k)=>{
const m = this.metrics[k];
delete m.evalFileStart;
delete m.evalFileEnd;
delete m.evalSqlStart;
delete m.evalSqlEnd;
const mk = Object.keys(m).sort();
if(!rc.length){
rc.push(['file', ...mk]);
rc.push(['db', ...mk]);
}
const row = [k.split('/').pop()/*remove dir prefix from filename*/];
rc.push(row);
@@ -205,6 +390,10 @@
},
metricsToBlob: function(colSeparator='\t'){
if(1){
this.logErr("TODO: re-do metrics dump");
return;
}
const ar = [], ma = this.metricsToArrays();
if(!ma.length){
this.logErr("Metrics are empty. Run something.");
@@ -239,114 +428,84 @@
*/
evalFile: async function(fn){
const sql = await this.fetchFile(fn);
const banner = "========================================";
this.logHtml(banner,
"Running",fn,'('+sql.length,'bytes)...');
const capi = this.sqlite3.capi, wasm = capi.wasm;
let pStmt = 0, pSqlBegin;
const stack = wasm.scopedAllocPush();
const metrics = this.metrics[fn] = Object.create(null);
metrics.prepTotal = metrics.stepTotal = 0;
metrics.stmtCount = 0;
metrics.malloc = 0;
metrics.strcpy = 0;
this.blockControls(true);
if(this.gotErr){
this.logErr("Cannot run ["+fn+"]: error cleanup is pending.");
return;
}
// Run this async so that the UI can be updated for the above header...
const ff = function(resolve, reject){
metrics.evalFileStart = performance.now();
try {
let t;
let sqlByteLen = sql.byteLength;
const [ppStmt, pzTail] = wasm.scopedAllocPtr(2);
t = performance.now();
pSqlBegin = wasm.alloc( sqlByteLen + 1/*SQL + NUL*/) || toss("alloc(",sqlByteLen,") failed");
metrics.malloc = performance.now() - t;
metrics.byteLength = sqlByteLen;
let pSql = pSqlBegin;
const pSqlEnd = pSqlBegin + sqlByteLen;
t = performance.now();
wasm.heap8().set(sql, pSql);
wasm.setMemValue(pSql + sqlByteLen, 0);
metrics.strcpy = performance.now() - t;
let breaker = 0;
while(pSql && wasm.getMemValue(pSql,'i8')){
wasm.setPtrValue(ppStmt, 0);
wasm.setPtrValue(pzTail, 0);
t = performance.now();
let rc = capi.sqlite3_prepare_v3(
this.db.ptr, pSql, sqlByteLen, 0, ppStmt, pzTail
);
metrics.prepTotal += performance.now() - t;
this.checkRc(rc);
pStmt = wasm.getPtrValue(ppStmt);
pSql = wasm.getPtrValue(pzTail);
sqlByteLen = pSqlEnd - pSql;
if(!pStmt) continue/*empty statement*/;
++metrics.stmtCount;
t = performance.now();
rc = capi.sqlite3_step(pStmt);
capi.sqlite3_finalize(pStmt);
pStmt = 0;
metrics.stepTotal += performance.now() - t;
switch(rc){
case capi.SQLITE_ROW:
case capi.SQLITE_DONE: break;
default: this.checkRc(rc); toss("Not reached.");
}
}
}catch(e){
if(pStmt) capi.sqlite3_finalize(pStmt);
this.gotErr = e;
//throw e;
reject(e);
return;
}finally{
wasm.dealloc(pSqlBegin);
wasm.scopedAllocPop(stack);
this.blockControls(false);
}
metrics.evalFileEnd = performance.now();
metrics.evalTimeTotal = (metrics.evalFileEnd - metrics.evalFileStart);
this.logHtml("Metrics:");//,JSON.stringify(metrics, undefined, ' '));
this.logHtml("prepare() count:",metrics.stmtCount);
this.logHtml("Time in prepare_v2():",metrics.prepTotal,"ms",
"("+(metrics.prepTotal / metrics.stmtCount),"ms per prepare())");
this.logHtml("Time in step():",metrics.stepTotal,"ms",
"("+(metrics.stepTotal / metrics.stmtCount),"ms per step())");
this.logHtml("Total runtime:",metrics.evalTimeTotal,"ms");
this.logHtml("Overhead (time - prep - step):",
(metrics.evalTimeTotal - metrics.prepTotal - metrics.stepTotal)+"ms");
this.logHtml(banner,"End of",fn);
resolve(this);
}.bind(this);
let p;
if(1){
p = new Promise(function(res,rej){
setTimeout(()=>ff(res, rej), 50)/*give UI a chance to output the "running" banner*/;
});
}else{
p = new Promise(ff);
}
return p.catch((e)=>this.logErr("Error via evalFile("+fn+"):",e.message));
return this.execSql(fn,sql);
}/*evalFile()*/,
clearStorage: function(){
const sz = sqlite3.capi.sqlite3_web_kvvfs_size();
const n = sqlite3.capi.sqlite3_web_kvvfs_clear(this.db.filename || '');
this.logHtml("Cleared kvvfs local/sessionStorage:",
n,"entries totaling approximately",sz,"bytes.");
/**
Clears all DB tables in all _opened_ databases. Because of
disparities between backends, we cannot simply "unlink" the
databases to clean them up.
*/
clearStorage: function(onlySelectedDb=false){
const list = onlySelectDb
? [('boolean'===typeof onlySelectDb)
? this.dbs[this.e.selImpl.value]
: onlySelectDb]
: Object.values(this.dbs);
for(let db of list){
if(db && db.handle){
this.logHtml("Clearing db",db.id);
d.clear();
}
}
},
resetDb: function(){
if(this.db.ptr){
const fn = this.db.filename;
this.closeDb(true);
this.openDb(fn,false);
/**
Fetches the handle of the db associated with
this.e.selImpl.value, opening it if needed.
*/
getSelectedDb: function(){
if(!this.dbs.memdb){
for(let opt of this.e.selImpl.options){
const d = this.dbs[opt.value] = Object.create(null);
d.id = opt.value;
switch(d.id){
case 'virtualfs':
d.filename = 'file:/virtualfs.sqlite3?vfs=unix-none';
break;
case 'memdb':
d.filename = ':memory:';
break;
case 'wasmfs-opfs':
d.filename = 'file:'+(this.sqlite3.capi.sqlite3_wasmfs_opfs_dir())+'/wasmfs-opfs.sqlite3';
break;
case 'websql':
d.filename = 'websql.db';
break;
default:
this.logErr("Unhandled db selection option (see details in the console).",opt);
toss("Unhandled db init option");
}
}
}/*first-time init*/
const dbId = this.e.selImpl.value;
const d = this.dbs[dbId];
if(d.handle) return d;
if('websql' === dbId){
d.handle = self.openDatabase('batch-runner', '0.1', 'foo', 1024 * 1024 * 50);
d.clear = ()=>clearDbWebSQL(d);
}else{
const capi = this.sqlite3.capi, wasm = capi.wasm;
const stack = wasm.scopedAllocPush();
let pDb = 0;
try{
const oFlags = capi.SQLITE_OPEN_CREATE | capi.SQLITE_OPEN_READWRITE;
const ppDb = wasm.scopedAllocPtr();
const rc = capi.sqlite3_open_v2(d.filename, ppDb, oFlags, null);
pDb = wasm.getPtrValue(ppDb)
if(rc) toss("sqlite3_open_v2() failed with code",rc);
}catch(e){
if(pDb) capi.sqlite3_close_v2(pDb);
}finally{
wasm.scopedAllocPop(stack);
}
d.handle = pDb;
d.clear = ()=>clearDbSqlite(d);
}
d.clear();
this.logHtml("Opened db:",dbId);
console.log("db =",d);
return d;
},
run: function(sqlite3){
@@ -356,38 +515,24 @@
this.logHtml("Loaded module:",capi.sqlite3_libversion(), capi.sqlite3_sourceid());
this.logHtml("WASM heap size =",wasm.heap8().length);
this.loadSqlList();
let pDir, dbFile;
if(sqlite3.capi.sqlite3_vfs_find('kvvfs')){
dbFile = 1 ? 'local' : 'session';
this.logHtml("Using KVVFS storage:",dbFile);
if(capi.sqlite3_wasmfs_opfs_dir()){
E('#warn-opfs').classList.remove('hidden');
}else{
pDir = capi.sqlite3_wasmfs_opfs_dir();
if(pDir){
dbFile = pDir+"/speedtest.db";
this.logHtml("Using persistent storage:",dbFile);
}else{
dbFile = ':memory:';
this.logHtml("Using",dbFile,"storage.");
}
E('#warn-opfs').remove();
E('option[value=wasmfs-opfs]').disabled = true;
}
if(!pDir){
document.querySelector('#warn-opfs').remove();
if('function' === typeof self.openDatabase){
E('#warn-websql').classList.remove('hidden');
}else{
E('option[value=websql]').disabled = true;
E('#warn-websql').remove();
}
this.openDb(dbFile, true);
const who = this;
const eReverseLogNotice = document.querySelector('#reverse-log-notice');
if(this.e.cbReverseLog.checked){
eReverseLogNotice.classList.remove('hidden');
this.e.output.classList.add('reverse');
}
this.e.cbReverseLog.addEventListener('change', function(){
if(this.checked){
who.e.output.classList.add('reverse');
eReverseLogNotice.classList.remove('hidden');
}else{
who.e.output.classList.remove('reverse');
eReverseLogNotice.classList.add('hidden');
}
who.e.output.classList[this.checked ? 'add' : 'remove']('reverse');
}, false);
this.e.btnClear.addEventListener('click', ()=>this.cls(), false);
this.e.btnRun.addEventListener('click', function(){
@@ -400,7 +545,7 @@
who.evalFile(who.e.selSql.value);
}, false);
this.e.btnReset.addEventListener('click', function(){
who.resetDb();
who.clearStorage(true);
}, false);
this.e.btnExportMetrics.addEventListener('click', function(){
who.logHtml2('warning',"Triggering download of metrics CSV. Check your downloads folder.");
@@ -408,6 +553,9 @@
//const m = who.metricsToArrays();
//console.log("Metrics:",who.metrics, m);
});
this.e.selImpl.addEventListener('change', function(){
who.getSelectedDb();
});
this.e.btnRunRemaining.addEventListener('click', async function(){
let v = who.e.selSql.value;
const timeStart = performance.now();
@@ -430,6 +578,7 @@
self.sqlite3TestModule.initSqlite3().then(function(theEmccModule){
self._MODULE = theEmccModule /* this is only to facilitate testing from the console */;
sqlite3 = theEmccModule.sqlite3;
console.log("App",App);
App.run(theEmccModule.sqlite3);
});
})();