database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-12-24 14:17:58 +03:00
Code Activity

Sync w/trunk, zap surplus space.

Browse Source 
FossilOrigin-Name: b8345630a2a322234bda49ee4b996f6ba20e2b080621e229a2ec5e820892a663
This commit is contained in:
larrybr
2022-11-28 14:11:48 +00:00
parent 472de149ae abbfa7aa17
commit d8d68a5b8e
21 changed files with 803 additions and 237 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ext/misc/base64.c
View File

@@ -200,7 +200,7 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){
SQLITE_LIMIT_LENGTH, -1);
char *cBuf;
ubyte *bBuf;
assert(na==1);
assert(na==1);
switch( sqlite3_value_type(av[0]) ){
case SQLITE_BLOB:
nb = nv;

8
ext/misc/base85.c
View File

@@ -148,10 +148,16 @@ static char * skipNonB85( char *s ){
return s;
}
/* Convert small integer, known to be in 0..84 inclusive, to base85 numeral.*/
/* Convert small integer, known to be in 0..84 inclusive, to base85 numeral.
* Do not use the macro form with argument expression having a side-effect.*/
#if 0
static char base85Numeral( ubyte b ){
return (b < 4)? (char)(b + '#') : (char)(b - 4 + '*');
}
#else
# define base85Numeral( dn )\
((char)(((dn) < 4)? (char)((dn) + '#') : (char)((dn) - 4 + '*')))
#endif
static char *putcs(char *pc, char *s){
char c;

2
ext/misc/basexx.c
View File

@@ -28,7 +28,7 @@
** export mop1=-DSQLITE_SHELL_EXTSRC=ext/misc/basexx.c
** export mop2=-DSQLITE_SHELL_EXTFUNCS=BASEXX
** make sqlite3 "OPTS=$mop1 $mop2"
** Win32 with Microsoft toolset on Windows:
** Win32 with Microsoft toolset on Windows:
** set mop1=-DSQLITE_SHELL_EXTSRC=ext/misc/basexx.c
** set mop2=-DSQLITE_SHELL_EXTFUNCS=BASEXX
** set mops="OPTS=%mop1% %mop2%"

5
ext/wasm/api/sqlite3-api-oo1.js
View File

@@ -201,9 +201,8 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
*/
dbCtorHelper.normalizeArgs = function(filename=':memory:',flags = 'c',vfs = null){
const arg = {};
if(1===arguments.length && 'object'===typeof arguments[0]){
const x = arguments[0];
Object.keys(x).forEach((k)=>arg[k] = x[k]);
if(1===arguments.length && arguments[0] && 'object'===typeof arguments[0]){
Object.assign(arg, arguments[0]);
if(undefined===arg.flags) arg.flags = 'c';
if(undefined===arg.vfs) arg.vfs = null;
if(undefined===arg.filename) arg.filename = ':memory:';

31
ext/wasm/api/sqlite3-api-opfs.js
View File

@@ -348,12 +348,32 @@ const installOpfsVfs = function callee(options){
'SQLITE_NOTFOUND',
'SQLITE_OPEN_CREATE',
'SQLITE_OPEN_DELETEONCLOSE',
'SQLITE_OPEN_MAIN_DB',
'SQLITE_OPEN_READONLY'
].forEach((k)=>{
if(undefined === (state.sq3Codes[k] = capi[k])){
toss("Maintenance required: not found:",k);
}
});
state.opfsFlags = Object.assign(Object.create(null),{
/**
Flag for use with xOpen(). "opfs-unlock-asap=1" enables
this. See defaultUnlockAsap, below.
*/
OPFS_UNLOCK_ASAP: 0x01,
/**
If true, any async routine which implicitly acquires a sync
access handle (i.e. an OPFS lock) will release that locks at
the end of the call which acquires it. If false, such
"autolocks" are not released until the VFS is idle for some
brief amount of time.
The benefit of enabling this is much higher concurrency. The
down-side is much-reduced performance (as much as a 4x decrease
in speedtest1).
*/
defaultUnlockAsap: false
});
/**
Runs the given operation (by name) in the async worker
@@ -844,9 +864,15 @@ const installOpfsVfs = function callee(options){
//xSleep is optionally defined below
xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){
mTimeStart('xOpen');
let opfsFlags = 0;
if(0===zName){
zName = randomFilename();
}else if('number'===typeof zName){
if(capi.sqlite3_uri_boolean(zName, "opfs-unlock-asap", 0)){
/* -----------------------^^^^^ MUST pass the untranslated
C-string here. */
opfsFlags |= state.opfsFlags.OPFS_UNLOCK_ASAP;
}
zName = wasm.cstringToJs(zName);
}
const fh = Object.create(null);
@@ -854,7 +880,7 @@ const installOpfsVfs = function callee(options){
fh.filename = zName;
fh.sab = new SharedArrayBuffer(state.fileBufferSize);
fh.flags = flags;
const rc = opRun('xOpen', pFile, zName, flags);
const rc = opRun('xOpen', pFile, zName, flags, opfsFlags);
if(!rc){
/* Recall that sqlite3_vfs::xClose() will be called, even on
error, unless pFile->pMethods is NULL. */
@@ -1145,6 +1171,9 @@ const installOpfsVfs = function callee(options){
//TODO to support fiddle and worker1 db upload:
//opfsUtil.createFile = function(absName, content=undefined){...}
//We have sqlite3.wasm.sqlite3_wasm_vfs_create_file() for this
//purpose but its interface and name are still under
//consideration.
if(sqlite3.oo1){
opfsUtil.OpfsDb = function(...args){

112
ext/wasm/api/sqlite3-opfs-async-proxy.js
View File

@@ -29,6 +29,22 @@
This file represents an implementation detail of a larger piece of
code, and not a public interface. Its details may change at any time
and are not intended to be used by any client-level code.
2022-11-27: Chrome v108 changes some async methods to synchronous, as
documented at:
https://developer.chrome.com/blog/sync-methods-for-accesshandles/
We cannot change to the sync forms at this point without breaking
clients who use Chrome v104-ish or higher. truncate(), getSize(),
flush(), and close() are now (as of v108) synchronous. Calling them
with an "await", as we have to for the async forms, is still legal
with the sync forms but is superfluous. Calling the async forms with
theFunc().then(...) is not compatible with the change to
synchronous, but we do do not use those APIs that way. i.e. we don't
_need_ to change anything for this, but at some point (after Chrome
versions (approximately) 104-107 are extinct) should change our
usage of those methods to remove the "await".
*/
"use strict";
const toss = function(...args){throw new Error(args.join(' '))};
@@ -105,7 +121,7 @@ metrics.dump = ()=>{
*/
const __openFiles = Object.create(null);
/**
__autoLocks is a Set of sqlite3_file pointers (integers) which were
__implicitLocks is a Set of sqlite3_file pointers (integers) which were
"auto-locked". i.e. those for which we obtained a sync access
handle without an explicit xLock() call. Such locks will be
released during db connection idle time, whereas a sync access
@@ -117,7 +133,7 @@ const __openFiles = Object.create(null);
penalty: speedtest1 benchmarks take up to 4x as long. By delaying
the lock release until idle time, the hit is negligible.
*/
const __autoLocks = new Set();
const __implicitLocks = new Set();
/**
Expects an OPFS file path. It gets resolved, such that ".."
@@ -166,7 +182,7 @@ const closeSyncHandle = async (fh)=>{
const h = fh.syncHandle;
delete fh.syncHandle;
delete fh.xLock;
__autoLocks.delete(fh.fid);
__implicitLocks.delete(fh.fid);
return h.close();
}
};
@@ -190,10 +206,10 @@ const closeSyncHandleNoThrow = async (fh)=>{
};
/* Release all auto-locks. */
const closeAutoLocks = async ()=>{
if(__autoLocks.size){
const releaseImplicitLocks = async ()=>{
if(__implicitLocks.size){
/* Release all auto-locks. */
for(const fid of __autoLocks){
for(const fid of __implicitLocks){
const fh = __openFiles[fid];
await closeSyncHandleNoThrow(fh);
log("Auto-unlocked",fid,fh.filenameAbs);
@@ -201,6 +217,20 @@ const closeAutoLocks = async ()=>{
}
};
/**
An experiment in improving concurrency by freeing up implicit locks
sooner. This is known to impact performance dramatically but it has
also shown to improve concurrency considerably.
If fh.releaseImplicitLocks is truthy and fh is in __implicitLocks,
this routine returns closeSyncHandleNoThrow(), else it is a no-op.
*/
const releaseImplicitLock = async (fh)=>{
if(fh.releaseImplicitLocks && __implicitLocks.has(fh.fid)){
return closeSyncHandleNoThrow(fh);
}
};
/**
An error class specifically for use with getSyncHandle(), the goal
of which is to eventually be able to distinguish unambiguously
@@ -246,7 +276,7 @@ GetSyncHandleError.convertRc = (e,rc)=>{
still fails at that point it will give up and propagate the
exception.
*/
const getSyncHandle = async (fh)=>{
const getSyncHandle = async (fh,opName)=>{
if(!fh.syncHandle){
const t = performance.now();
log("Acquiring sync handle for",fh.filenameAbs);
@@ -262,20 +292,21 @@ const getSyncHandle = async (fh)=>{
}catch(e){
if(i === maxTries){
throw new GetSyncHandleError(
e, "Error getting sync handle.",maxTries,
e, "Error getting sync handle for",opName+"().",maxTries,
"attempts failed.",fh.filenameAbs
);
}
warn("Error getting sync handle. Waiting",ms,
warn("Error getting sync handle for",opName+"(). Waiting",ms,
"ms and trying again.",fh.filenameAbs,e);
await closeAutoLocks();
//await releaseImplicitLocks();
Atomics.wait(state.sabOPView, state.opIds.retry, 0, ms);
}
}
log("Got sync handle for",fh.filenameAbs,'in',performance.now() - t,'ms');
log("Got",opName+"() sync handle for",fh.filenameAbs,
'in',performance.now() - t,'ms');
if(!fh.xLock){
__autoLocks.add(fh.fid);
log("Auto-locked",fh.fid,fh.filenameAbs);
__implicitLocks.add(fh.fid);
log("Auto-locked for",opName+"()",fh.fid,fh.filenameAbs);
}
}
return fh.syncHandle;
@@ -409,7 +440,7 @@ const vfsAsyncImpls = {
xClose: async function(fid/*sqlite3_file pointer*/){
const opName = 'xClose';
mTimeStart(opName);
__autoLocks.delete(fid);
__implicitLocks.delete(fid);
const fh = __openFiles[fid];
let rc = 0;
wTimeStart(opName);
@@ -474,13 +505,14 @@ const vfsAsyncImpls = {
wTimeStart('xFileSize');
try{
affirmLocked('xFileSize',fh);
const sz = await (await getSyncHandle(fh)).getSize();
const sz = await (await getSyncHandle(fh,'xFileSize')).getSize();
state.s11n.serialize(Number(sz));
rc = 0;
}catch(e){
state.s11n.storeException(2,e);
rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR);
}
await releaseImplicitLock(fh);
wTimeEnd();
storeAndNotify('xFileSize', rc);
mTimeEnd();
@@ -495,8 +527,8 @@ const vfsAsyncImpls = {
if( !fh.syncHandle ){
wTimeStart('xLock');
try {
await getSyncHandle(fh);
__autoLocks.delete(fid);
await getSyncHandle(fh,'xLock');
__implicitLocks.delete(fid);
}catch(e){
state.s11n.storeException(1,e);
rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_LOCK);
@@ -508,10 +540,10 @@ const vfsAsyncImpls = {
mTimeEnd();
},
xOpen: async function(fid/*sqlite3_file pointer*/, filename,
flags/*SQLITE_OPEN_...*/){
flags/*SQLITE_OPEN_...*/,
opfsFlags/*OPFS_...*/){
const opName = 'xOpen';
mTimeStart(opName);
const deleteOnClose = (state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags);
const create = (state.sq3Codes.SQLITE_OPEN_CREATE & flags);
wTimeStart('xOpen');
try{
@@ -526,14 +558,8 @@ const vfsAsyncImpls = {
return;
}
const hFile = await hDir.getFileHandle(filenamePart, {create});
/**
wa-sqlite, at this point, grabs a SyncAccessHandle and
assigns it to the syncHandle prop of the file state
object, but only for certain cases and it's unclear why it
places that limitation on it.
*/
wTimeEnd();
__openFiles[fid] = Object.assign(Object.create(null),{
const fh = Object.assign(Object.create(null),{
fid: fid,
filenameAbs: filename,
filenamePart: filenamePart,
@@ -542,8 +568,26 @@ const vfsAsyncImpls = {
sabView: state.sabFileBufView,
readOnly: create
? false : (state.sq3Codes.SQLITE_OPEN_READONLY & flags),
deleteOnClose: deleteOnClose
deleteOnClose: !!(state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags)
});
fh.releaseImplicitLocks =
(opfsFlags & state.opfsFlags.OPFS_UNLOCK_ASAP)
|| state.opfsFlags.defaultUnlockAsap;
if(0 /* this block is modelled after something wa-sqlite
does but it leads to immediate contention on journal files. */
&& (0===(flags & state.sq3Codes.SQLITE_OPEN_MAIN_DB))){
/* sqlite does not lock these files, so go ahead and grab an OPFS
lock.
https://www.sqlite.org/uri.html
*/
fh.xLock = "xOpen"/* Truthy value to keep entry from getting
flagged as auto-locked. String value so
that we can easily distinguish is later
if needed. */;
await getSyncHandle(fh,'xOpen');
}
__openFiles[fid] = fh;
storeAndNotify(opName, 0);
}catch(e){
wTimeEnd();
@@ -560,7 +604,7 @@ const vfsAsyncImpls = {
try{
affirmLocked('xRead',fh);
wTimeStart('xRead');
nRead = (await getSyncHandle(fh)).read(
nRead = (await getSyncHandle(fh,'xRead')).read(
fh.sabView.subarray(0, n),
{at: Number(offset64)}
);
@@ -575,6 +619,7 @@ const vfsAsyncImpls = {
state.s11n.storeException(1,e);
rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_READ);
}
await releaseImplicitLock(fh);
storeAndNotify('xRead',rc);
mTimeEnd();
},
@@ -603,12 +648,13 @@ const vfsAsyncImpls = {
try{
affirmLocked('xTruncate',fh);
affirmNotRO('xTruncate', fh);
await (await getSyncHandle(fh)).truncate(size);
await (await getSyncHandle(fh,'xTruncate')).truncate(size);
}catch(e){
error("xTruncate():",e,fh);
state.s11n.storeException(2,e);
rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_TRUNCATE);
}
await releaseImplicitLock(fh);
wTimeEnd();
storeAndNotify('xTruncate',rc);
mTimeEnd();
@@ -640,7 +686,7 @@ const vfsAsyncImpls = {
affirmLocked('xWrite',fh);
affirmNotRO('xWrite', fh);
rc = (
n === (await getSyncHandle(fh))
n === (await getSyncHandle(fh,'xWrite'))
.write(fh.sabView.subarray(0, n),
{at: Number(offset64)})
) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE;
@@ -649,6 +695,7 @@ const vfsAsyncImpls = {
state.s11n.storeException(1,e);
rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_WRITE);
}
await releaseImplicitLock(fh);
wTimeEnd();
storeAndNotify('xWrite',rc);
mTimeEnd();
@@ -777,13 +824,13 @@ const waitLoop = async function f(){
to do other things. If this is too high (e.g. 500ms) then
even two workers/tabs can easily run into locking errors.
*/
const waitTime = 150;
const waitTime = 100;
while(!flagAsyncShutdown){
try {
if('timed-out'===Atomics.wait(
state.sabOPView, state.opIds.whichOp, 0, waitTime
)){
await closeAutoLocks();
await releaseImplicitLocks();
continue;
}
const opId = Atomics.load(state.sabOPView, state.opIds.whichOp);
@@ -824,6 +871,7 @@ navigator.storage.getDirectory().then(function(d){
state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize);
state.opIds = opt.opIds;
state.sq3Codes = opt.sq3Codes;
state.opfsFlags = opt.opfsFlags;
Object.keys(vfsAsyncImpls).forEach((k)=>{
if(!Number.isFinite(state.opIds[k])){
toss("Maintenance required: missing state.opIds[",k,"]");

11
ext/wasm/tests/opfs/concurrency/index.html
View File

@@ -24,10 +24,13 @@
</p>
<p>
URL flags: pass a number of workers using
the <code>workers=N</code> URL flag and the worker work interval
as <code>interval=N</code> (milliseconds). Enable OPFS VFS
verbosity with <code>verbose=1-3</code> (output goes to the
dev console).
the <code>workers=N</code> URL flag. Set the time between each
workload with <code>interval=N</code> (milliseconds). Set the
number of worker iterations with <code>iterations=N</code>.
Enable OPFS VFS verbosity with <code>verbose=1-3</code> (output
goes to the dev console). Enable/disable "unlock ASAP" mode
(higher concurrency, lower speed)
with <code>unlock-asap=0-1</code>.
</p>
<p>Achtung: if it does not start to do anything within a couple of
seconds, check the dev console: Chrome often fails with "cannot allocate

34
ext/wasm/tests/opfs/concurrency/test.js
View File

@@ -56,26 +56,42 @@
options.sqlite3Dir = urlArgsJs.get('sqlite3.dir');
options.workerCount = (
urlArgsHtml.has('workers') ? +urlArgsHtml.get('workers') : 3
) || 3;
) || 4;
options.opfsVerbose = (
urlArgsHtml.has('verbose') ? +urlArgsHtml.get('verbose') : 1
) || 1;
options.interval = (
urlArgsHtml.has('interval') ? +urlArgsHtml.get('interval') : 750
) || 750;
) || 1000;
options.iterations = (
urlArgsHtml.has('iterations') ? +urlArgsHtml.get('iterations') : 10
) || 10;
options.unlockAsap = (
urlArgsHtml.has('unlock-asap') ? +urlArgsHtml.get('unlock-asap') : 0
) || 0;
const workers = [];
workers.post = (type,...args)=>{
for(const w of workers) w.postMessage({type, payload:args});
};
workers.loadedCount = 0;
workers.counts = {loaded: 0, passed: 0, failed: 0};
const checkFinished = function(){
if(workers.counts.passed + workers.counts.failed !== workers.length){
return;
}
if(workers.counts.failed>0){
logCss('tests-fail',"Finished with",workers.counts.failed,"failure(s).");
}else{
logCss('tests-pass',"All",workers.length,"workers finished.");
}
};
workers.onmessage = function(msg){
msg = msg.data;
const prefix = 'Worker #'+msg.worker+':';
switch(msg.type){
case 'loaded':
stdout(prefix,"loaded");
if(++workers.loadedCount === workers.length){
stdout("All workers loaded. Telling them to run...");
if(++workers.counts.loaded === workers.length){
stdout("All",workers.length,"workers loaded. Telling them to run...");
workers.post('run');
}
break;
@@ -83,21 +99,27 @@
case 'stderr': stderr(prefix,...msg.payload); break;
case 'error': stderr(prefix,"ERROR:",...msg.payload); break;
case 'finished':
++workers.counts.passed;
logCss('tests-pass',prefix,...msg.payload);
checkFinished();
break;
case 'failed':
++workers.counts.failed;
logCss('tests-fail',prefix,"FAILED:",...msg.payload);
checkFinished();
break;
default: logCss('error',"Unhandled message type:",msg); break;
}
};
stdout("Launching",options.workerCount,"workers...");
stdout("Launching",options.workerCount,"workers. Options:",options);
workers.uri = (
'worker.js?'
+ 'sqlite3.dir='+options.sqlite3Dir
+ '&interval='+options.interval
+ '&iterations='+options.iterations
+ '&opfs-verbose='+options.opfsVerbose
+ '&opfs-unlock-asap='+options.unlockAsap
);
for(let i = 0; i < options.workerCount; ++i){
stdout("Launching worker...");

17
ext/wasm/tests/opfs/concurrency/worker.js
View File

@@ -3,9 +3,12 @@ importScripts(
);
self.sqlite3InitModule().then(async function(sqlite3){
const urlArgs = new URL(self.location.href).searchParams;
const wName = urlArgs.get('workerId') || Math.round(Math.random()*10000);
const options = {
workerName: urlArgs.get('workerId') || Math.round(Math.random()*10000),
unlockAsap: urlArgs.get('opfs-unlock-asap') || 0 /*EXPERIMENTAL*/
};
const wPost = (type,...payload)=>{
postMessage({type, worker: wName, payload});
postMessage({type, worker: options.workerName, payload});
};
const stdout = (...args)=>wPost('stdout',...args);
const stderr = (...args)=>wPost('stderr',...args);
@@ -43,7 +46,10 @@ self.sqlite3InitModule().then(async function(sqlite3){
}
};
const run = async function(){
db = new sqlite3.opfs.OpfsDb(dbName,'c');
db = new sqlite3.opfs.OpfsDb({
filename: 'file:'+dbName+'?opfs-unlock-asap='+options.unlockAsap,
flags: 'c'
});
sqlite3.capi.sqlite3_busy_timeout(db.pointer, 5000);
db.transaction((db)=>{
db.exec([
@@ -52,7 +58,8 @@ self.sqlite3InitModule().then(async function(sqlite3){
]);
});
const maxIterations = 10;
const maxIterations =
urlArgs.has('iterations') ? (+urlArgs.get('iterations') || 10) : 10;
stdout("Starting interval-based db updates with delay of",interval.delay,"ms.");
const doWork = async ()=>{
const tm = new Date().getTime();
@@ -62,7 +69,7 @@ self.sqlite3InitModule().then(async function(sqlite3){
try{
db.exec({
sql:"INSERT OR REPLACE INTO t1(w,v) VALUES(?,?)",
bind: [wName, new Date().getTime()]
bind: [options.workerName, new Date().getTime()]
});
//stdout("Set",prefix);
}catch(e){