database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-08-07 02:42:48 +03:00
Code Activity

Speed up base85() conversions and sync w/trunk.

Browse Source 
FossilOrigin-Name: 17b823500a2ed135c1f40aa7f4d87ba5b2eab35c0abd9e0856041cf0f510cbee
This commit is contained in:
larrybr
2022-11-24 20:11:34 +00:00
parent 9b9017def4 eb84c81c53
commit 472de149ae
16 changed files with 215 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
ext/misc/base64.c
View File

@@ -64,7 +64,10 @@ SQLITE_EXTENSION_INIT1;
#define ND 0x82 /* Not above or digit-value */
#define PAD_CHAR '='
#ifndef UBYTE_TYPEDEF
typedef unsigned char ubyte;
# define UBYTE_TYPEDEF
#endif
static const ubyte b64DigitValues[128] = {
/* HT LF VT FF CR */

95
ext/misc/base85.c
View File

@@ -113,21 +113,26 @@ static void sayHelp(){
}
#endif
#ifndef UBYTE_TYPEDEF
typedef unsigned char ubyte;
# define UBYTE_TYPEDEF
#endif
/* Classify c according to interval within USASCII set w.r.t. base85
* Values of 1 and 3 are base85 numerals. Values of 0, 2, or 4 are not.
*/
#define B85_CLASS( c ) (((c)>='#')+((c)>'&')+((c)>='*')+((c)>'z'))
/* Provide digitValue to b85Numeral offset as a function of above class. */
static unsigned char b85_cOffset[] = { 0, '#', 0, '*'-4, 0 };
static ubyte b85_cOffset[] = { 0, '#', 0, '*'-4, 0 };
#define B85_DNOS( c ) b85_cOffset[B85_CLASS(c)]
/* Say whether c is a base85 numeral. */
#define IS_B85( c ) (B85_CLASS(c) & 1)
#if 0 /* Not used, */
static unsigned char base85DigitValue( char c ){
unsigned char dv = (unsigned char)(c - '#');
static ubyte base85DigitValue( char c ){
ubyte dv = (ubyte)(c - '#');
if( dv>87 ) return 0xff;
return (dv > 3)? dv-3 : dv;
}
@@ -144,46 +149,61 @@ static char * skipNonB85( char *s ){
}
/* Convert small integer, known to be in 0..84 inclusive, to base85 numeral.*/
static char base85Numeral( unsigned char b ){
static char base85Numeral( ubyte b ){
return (b < 4)? (char)(b + '#') : (char)(b - 4 + '*');
}
static char *putcs(char *pc, char *s){
char c;
while( (c = *s++)!=0 ) *pc++ = c;
return pc;
}
/* Encode a byte buffer into base85 text. If pSep!=0, it's a C string
** to be appended to encoded groups to limit their length to B85_DARK_MAX
** or to terminate the last group (to aid concatenation.)
*/
static char* toBase85( unsigned char *pIn, int nbIn, char *pOut, char *pSep ){
static char* toBase85( ubyte *pIn, int nbIn, char *pOut, char *pSep ){
int nCol = 0;
*pOut = 0;
while( nbIn > 0 ){
static signed char ncio[] = { 0, 2, 3, 4, 5 };
int nbi = (nbIn > 4)? 4 : nbIn;
unsigned long qv = 0L;
int nbe = 0;
signed char nco;
while( nbe++ < nbi ){
while( nbIn >= 4 ){
int nco = 5;
unsigned long qbv = (pIn[0]<<24)|(pIn[1]<<16)|(pIn[2]<<8)|pIn[3];
while( nco > 0 ){
unsigned nqv = (unsigned)(qbv/85UL);
unsigned char dv = qbv - 85UL*nqv;
qbv = nqv;
pOut[--nco] = base85Numeral(dv);
}
nbIn -= 4;
pIn += 4;
pOut += 5;
if( pSep && (nCol += 5)>=B85_DARK_MAX ){
pOut = putcs(pOut, pSep);
nCol = 0;
}
}
if( nbIn > 0 ){
int nco = nbIn + 1;
unsigned long qv = *pIn++;
int nbe = 1;
while( nbe++ < nbIn ){
qv = (qv<<8) | *pIn++;
}
nco = ncio[nbi];
nbIn -= nbi;
nCol += nco;
while( nco > 0 ){
unsigned char dv = (unsigned char)(qv % 85);
ubyte dv = (ubyte)(qv % 85);
qv /= 85;
pOut[--nco] = base85Numeral(dv);
}
pOut += ncio[nbi];
if( pSep && ((nCol += ncio[nbi])>=B85_DARK_MAX || nbIn<=0) ){
char *p = pSep;
while( *p ) *pOut++ = *p++;
nCol = 0;
}
*pOut = 0;
pOut += (nbIn+1);
}
if( pSep && nCol>0 ) pOut = putcs(pOut, pSep);
*pOut = 0;
return pOut;
}
/* Decode base85 text into a byte buffer. */
static unsigned char* fromBase85( char *pIn, int ncIn, unsigned char *pOut ){
static ubyte* fromBase85( char *pIn, int ncIn, ubyte *pOut ){
if( ncIn>0 && pIn[ncIn-1]=='\n' ) --ncIn;
while( ncIn>0 ){
static signed char nboi[] = { 0, 0, 1, 2, 3, 4 };
@@ -191,21 +211,30 @@ static unsigned char* fromBase85( char *pIn, int ncIn, unsigned char *pOut ){
unsigned long qv = 0L;
int nti, nbo;
ncIn -= (pUse - pIn);
if( ncIn==0 ) break;
pIn = pUse;
nti = (ncIn>5)? 5 : ncIn;
nbo = nboi[nti];
if( nbo==0 ) break;
while( nti>0 ){
char c = *pIn++;
unsigned char cdo = B85_DNOS(c);
ubyte cdo = B85_DNOS(c);
--ncIn;
if( cdo==0 ) break;
qv = 85 * qv + c - cdo;
qv = 85 * qv + (c - cdo);
--nti;
}
nbo -= nti;
while( nbo-- > 0 ){
*pOut++ = (qv >> (8*nbo))&0xff;
nbo -= nti; /* Adjust for early (non-digit) end of group. */
switch( nbo ){
case 4:
*pOut++ = (qv >> 24)&0xff;
case 3:
*pOut++ = (qv >> 16)&0xff;
case 2:
*pOut++ = (qv >> 8)&0xff;
case 1:
*pOut++ = qv&0xff;
case 0:
break;
}
}
return pOut;
@@ -252,7 +281,7 @@ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){
int nvMax = sqlite3_limit(sqlite3_context_db_handle(context),
SQLITE_LIMIT_LENGTH, -1);
char *cBuf;
unsigned char *bBuf;
ubyte *bBuf;
assert(na==1);
switch( sqlite3_value_type(av[0]) ){
case SQLITE_BLOB:
@@ -265,7 +294,7 @@ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){
}
cBuf = sqlite3_malloc(nc);
if( !cBuf ) goto memFail;
bBuf = (unsigned char*)sqlite3_value_blob(av[0]);
bBuf = (ubyte*)sqlite3_value_blob(av[0]);
nc = (int)(toBase85(bBuf, nb, cBuf, "\n") - cBuf);
sqlite3_result_text(context, cBuf, nc, sqlite3_free);
break;
@@ -335,7 +364,7 @@ static int sqlite3_base85_init
int main(int na, char *av[]){
int cin;
int rc = 0;
unsigned char bBuf[4*(B85_DARK_MAX/5)];
ubyte bBuf[4*(B85_DARK_MAX/5)];
char cBuf[5*(sizeof(bBuf)/4)+2];
size_t nio;
# ifndef OMIT_BASE85_CHECKER

3
ext/wasm/api/extern-post-js.js
View File

@@ -59,9 +59,6 @@ const toExportForES6 =
li.pop();
initModuleState.sqlite3Dir = li.join('/') + '/';
}
if(initModuleState.sqlite3Dir){
initModuleState.sqlite3Dir = initModuleState.sqlite3Dir.replace(/[/]{2,}/g,'/');
}
self.sqlite3InitModule = (...args)=>{
//console.warn("Using replaced sqlite3InitModule()",self.location);

27
ext/wasm/api/sqlite3-api-oo1.js
View File

@@ -55,12 +55,13 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
if(sqliteResultCode){
if(dbPtr instanceof DB) dbPtr = dbPtr.pointer;
toss3(
"sqlite result code",sqliteResultCode+":",
"sqlite3 result code",sqliteResultCode+":",
(dbPtr
? capi.sqlite3_errmsg(dbPtr)
: capi.sqlite3_errstr(sqliteResultCode))
);
}
return arguments[0];
};
/**
@@ -462,14 +463,16 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
Expects to be given a DB instance or an `sqlite3*` pointer (may
be null) and an sqlite3 API result code. If the result code is
not falsy, this function throws an SQLite3Error with an error
message from sqlite3_errmsg(), using dbPtr as the db handle, or
sqlite3_errstr() if dbPtr is falsy. Note that if it's passed a
non-error code like SQLITE_ROW or SQLITE_DONE, it will still
throw but the error string might be "Not an error." The various
non-0 non-error codes need to be checked for in
client code where they are expected.
message from sqlite3_errmsg(), using db (or, if db is-a DB,
db.pointer) as the db handle, or sqlite3_errstr() if db is
falsy. Note that if it's passed a non-error code like SQLITE_ROW
or SQLITE_DONE, it will still throw but the error string might be
"Not an error." The various non-0 non-error codes need to be
checked for in client code where they are expected.
If it does not throw, it returns its first argument.
*/
DB.checkRc = checkSqlite3Rc;
DB.checkRc = (db,resultCode)=>checkSqlite3Rc(db,resultCode);
DB.prototype = {
/** Returns true if this db handle is open, else false. */
@@ -1130,6 +1133,14 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
this.exec("ROLLBACK to SAVEPOINT oo1; RELEASE SAVEPOINT oo1");
throw e;
}
},
/**
A convenience form of DB.checkRc(this,resultCode). If it does
not throw, it returns this object.
*/
checkRc: function(resultCode){
return DB.checkRc(this, resultCode);
}
}/*DB.prototype*/;

128
ext/wasm/api/sqlite3-api-prologue.js
View File

@@ -11,76 +11,18 @@
***********************************************************************
This file is intended to be combined at build-time with other
related code, most notably a header and footer which wraps this whole
file into an Emscripten Module.postRun() handler which has a parameter
named "Module" (the Emscripten Module object). The exact requirements,
conventions, and build process are very much under construction and
will be (re)documented once they've stopped fluctuating so much.
related code, most notably a header and footer which wraps this
whole file into an Emscripten Module.postRun() handler which has a
parameter named "Module" (the Emscripten Module object). The sqlite3
JS API has no hard requirements on Emscripten, and does not expose
any Emscripten APIs to clients. It is structured such that its build
can be tweaked to include it in arbitrary WASM environments which
supply the necessary underlying features (e.g. a POSIX file I/O
layer).
Project home page: https://sqlite.org
Main project home page: https://sqlite.org
Documentation home page: https://sqlite.org/wasm
Specific goals of this subproject:
- Except where noted in the non-goals, provide a more-or-less
feature-complete wrapper to the sqlite3 C API, insofar as WASM
feature parity with C allows for. In fact, provide at least 4
APIs...
1) 1-to-1 bindings as exported from WASM, with no automatic
type conversions between JS and C.
2) A binding of (1) which provides certain JS/C type conversions
to greatly simplify its use.
3) A higher-level API, more akin to sql.js and node.js-style
implementations. This one speaks directly to the low-level
API. This API must be used from the same thread as the
low-level API.
4) A second higher-level API which speaks to the previous APIs via
worker messages. This one is intended for use in the main
thread, with the lower-level APIs installed in a Worker thread,
and talking to them via Worker messages. Because Workers are
asynchronouns and have only a single message channel, some
acrobatics are needed here to feed async work results back to
the client (as we cannot simply pass around callbacks between
the main and Worker threads).
- Insofar as possible, support client-side storage using JS
filesystem APIs. As of this writing, such things are still very
much under development.
Specific non-goals of this project:
- As WASM is a web-centric technology and UTF-8 is the King of
Encodings in that realm, there are no currently plans to support
the UTF16-related sqlite3 APIs. They would add a complication to
the bindings for no appreciable benefit. Though web-related
implementation details take priority, and the JavaScript
components of the API specifically focus on browser clients, the
lower-level WASM module "should" work in non-web WASM
environments.
- Supporting old or niche-market platforms. WASM is built for a
modern web and requires modern platforms.
- Though scalar User-Defined Functions (UDFs) may be created in
JavaScript, there are currently no plans to add support for
aggregate and window functions.
Attribution:
This project is endebted to the work of sql.js:
https://github.com/sql-js/sql.js
sql.js was an essential stepping stone in this code's development as
it demonstrated how to handle some of the WASM-related voodoo (like
handling pointers-to-pointers and adding JS implementations of
C-bound callback functions). These APIs have a considerably
different shape than sql.js's, however.
*/
/**
@@ -90,6 +32,10 @@
for the current environment, and then optionally be removed from
the global object using `delete self.sqlite3ApiBootstrap`.
This function is not intended for client-level use. It is intended
for use in creating bundles configured for specific WASM
environments.
This function expects a configuration object, intended to abstract
away details specific to any given WASM environment, primarily so
that it can be used without any _direct_ dependency on
@@ -126,11 +72,11 @@
environment. Defaults to `"free"`.
- `wasmfsOpfsDir`[^1]: if the environment supports persistent
storage, this directory names the "mount point" for that
directory. It must be prefixed by `/` and may contain only a
single directory-name part. Using the root directory name is not
supported by any current persistent backend. This setting is
only used in WASMFS-enabled builds.
storage using OPFS-over-WASMFS , this directory names the "mount
point" for that directory. It must be prefixed by `/` and may
contain only a single directory-name part. Using the root
directory name is not supported by any current persistent
backend. This setting is only used in WASMFS-enabled builds.
[^1] = This property may optionally be a function, in which case this
@@ -191,11 +137,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
const capi = Object.create(null);
/**
Holds state which are specific to the WASM-related
infrastructure and glue code. It is not expected that client
code will normally need these, but they're exposed here in case
it does. These APIs are _not_ to be considered an
official/stable part of the sqlite3 WASM API. They may change
as the developers' experience suggests appropriate changes.
infrastructure and glue code.
Note that a number of members of this object are injected
dynamically after the api object is fully constructed, so
@@ -228,7 +170,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
result of sqlite3.capi.sqlite3_js_rc_str() or (if that returns
falsy) a synthesized string which contains that integer.
- If passed 2 arguments and the 2nd is a object, it bevaves
- If passed 2 arguments and the 2nd is a object, it behaves
like the Error(string,object) constructor except that the first
argument is subject to the is-integer semantics from the
previous point.
@@ -686,9 +628,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
/**
The WASM IR (Intermediate Representation) value for
pointer-type values. It MUST refer to a value type of the
size described by this.ptrSizeof _or_ it may be any value
which ends in '*', which Emscripten's glue code internally
translates to i32.
size described by this.ptrSizeof.
*/
ptrIR: config.wasmPtrIR || "i32",
/**
@@ -1307,17 +1247,24 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
};
/**
Serializes the given `sqlite3*` pointer to a Uint8Array, as per
sqlite3_serialize(). On success it returns a Uint8Array. On
error it throws with a description of the problem.
A convenience wrapper around sqlite3_serialize() which serializes
the given `sqlite3*` pointer to a Uint8Array.
On success it returns a Uint8Array. If the schema is empty, an
empty array is returned.
`schema` is the schema to serialize. It may be a WASM C-string
pointer or a JS string. If it is falsy, it defaults to `"main"`.
On error it throws with a description of the problem.
*/
capi.sqlite3_js_db_export = function(pDb){
capi.sqlite3_js_db_export = function(pDb, schema=0){
if(!pDb) toss3('Invalid sqlite3* argument.');
if(!wasm.bigIntEnabled) toss3('BigInt64 support is not enabled.');
const stack = wasm.pstack.pointer;
const scope = wasm.scopedAllocPush();
let pOut;
try{
const pSize = wasm.pstack.alloc(8/*i64*/ + wasm.ptrSizeof);
const pSize = wasm.scopedAlloc(8/*i64*/ + wasm.ptrSizeof);
const ppOut = pSize + 8;
/**
Maintenance reminder, since this cost a full hour of grief
@@ -1326,8 +1273,11 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
export reads a garbage size because it's not on an 8-byte
memory boundary!
*/
const zSchema = schema
? (wasm.isPtr(schema) ? schema : wasm.scopedAllocCString(''+schema))
: 0;
let rc = wasm.exports.sqlite3_wasm_db_serialize(
pDb, ppOut, pSize, 0
pDb, zSchema, ppOut, pSize, 0
);
if(rc){
toss3("Database serialization failed with code",
@@ -1341,7 +1291,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
return rc;
}finally{
if(pOut) wasm.exports.sqlite3_free(pOut);
wasm.pstack.restore(stack);
wasm.scopedAllocPop(scope);
}
};

20
ext/wasm/api/sqlite3-wasm.c
View File

@@ -916,25 +916,27 @@ int sqlite3_wasm_db_export_chunked( sqlite3* pDb,
}
/*
** A proxy for sqlite3_serialize() which serializes the "main" schema
** A proxy for sqlite3_serialize() which serializes the schema zSchema
** of pDb, placing the serialized output in pOut and nOut. nOut may be
** NULL. If pDb or pOut are NULL then SQLITE_MISUSE is returned. If
** allocation of the serialized copy fails, SQLITE_NOMEM is returned.
** On success, 0 is returned and `*pOut` will contain a pointer to the
** memory unless mFlags includes SQLITE_SERIALIZE_NOCOPY and the
** database has no contiguous memory representation, in which case
** `*pOut` will be NULL but 0 will be returned.
** NULL. If zSchema is NULL then "main" is assumed. If pDb or pOut are
** NULL then SQLITE_MISUSE is returned. If allocation of the
** serialized copy fails, SQLITE_NOMEM is returned. On success, 0 is
** returned and `*pOut` will contain a pointer to the memory unless
** mFlags includes SQLITE_SERIALIZE_NOCOPY and the database has no
** contiguous memory representation, in which case `*pOut` will be
** NULL but 0 will be returned.
**
** If `*pOut` is not NULL, the caller is responsible for passing it to
** sqlite3_free() to free it.
*/
SQLITE_WASM_KEEP
int sqlite3_wasm_db_serialize( sqlite3 *pDb, unsigned char **pOut,
int sqlite3_wasm_db_serialize( sqlite3 *pDb, const char *zSchema,
unsigned char **pOut,
sqlite3_int64 *nOut, unsigned int mFlags ){
unsigned char * z;
if( !pDb || !pOut ) return SQLITE_MISUSE;
if(nOut) *nOut = 0;
z = sqlite3_serialize(pDb, "main", nOut, mFlags);
z = sqlite3_serialize(pDb, zSchema ? zSchema : "main", nOut, mFlags);
if( z || (SQLITE_SERIALIZE_NOCOPY & mFlags) ){
*pOut = z;
return 0;

10
ext/wasm/tester1.c-pp.js
View File

@@ -1163,6 +1163,16 @@ self.sqlite3InitModule = sqlite3InitModule;
.assert(0 === capi.sqlite3_errmsg(db.pointer).indexOf("Invalid SQL"))
.assert(dbFile === db.dbFilename())
.assert(!db.dbFilename('nope'));
//Sanity check DB.checkRc()...
let ex;
try{db.checkRc(rc)}
catch(e){ex = e}
T.assert(ex instanceof sqlite3.SQLite3Error)
.assert(0===ex.message.indexOf("sqlite3 result code"))
.assert(ex.message.indexOf("Invalid SQL")>0);
T.assert(db === db.checkRc(0))
.assert(db === sqlite3.oo1.DB.checkRc(db,0))
.assert(null === sqlite3.oo1.DB.checkRc(null,0))
})
////////////////////////////////////////////////////////////////////

5
ext/wasm/tests/opfs/concurrency/worker.js
View File

@@ -70,11 +70,12 @@ self.sqlite3InitModule().then(async function(sqlite3){
}
};
if(1){/*use setInterval()*/
interval.handle = setInterval(async ()=>{
setTimeout(async function timer(){
await doWork();
if(interval.error || maxIterations === interval.count){
clearInterval(interval.handle);
finish();
}else{
setTimeout(timer, interval.delay);
}
}, interval.delay);
}else{