database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-07-29 08:01:23 +03:00
Code Activity

Extend the importDb() method of both OPFS VFSes to (A) support reading in an async streaming fashion via a callback and (B) automatically disable WAL mode in the imported db.

Browse Source 
FossilOrigin-Name: 9b1398c96a4fd0b59e65faa8d5c98de4129f0f0357732f12cb2f5c53a08acdc2
This commit is contained in:
stephan
2023-08-18 14:16:26 +00:00
parent abfe646c12
commit ccbfe97cd5
6 changed files with 223 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
ext/wasm/api/sqlite3-api-prologue.js
View File

@ -772,8 +772,43 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
isSharedTypedArray,
toss: function(...args){throw new Error(args.join(' '))},
toss3,
typedArrayPart
};
typedArrayPart,
/**
Given a byte array or ArrayBuffer, this function throws if the
lead bytes of that buffer do not hold a SQLite3 database header,
else it returns without side effects.
Added in 3.44.
*/
affirmDbHeader: function(bytes){
if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes);
const header = "SQLite format 3";
if( header.length > bytes.byteLength ){
toss3("Input does not contain an SQLite3 database header.");
}
for(let i = 0; i < header.length; ++i){
if( header.charCodeAt(i) !== bytes[i] ){
toss3("Input does not contain an SQLite3 database header.");
}
}
},
/**
Given a byte array or ArrayBuffer, this function throws if the
database does not, at a cursory glance, appear to be an SQLite3
database. It only examines the size and header, but further
checks may be added in the future.
Added in 3.44.
*/
affirmIsDb: function(bytes){
if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes);
const n = bytes.byteLength;
if(n<512 || n%512!==0) {
toss3("Byte array size",n,"is invalid for an SQLite3 db.");
}
util.affirmDbHeader(bytes);
}
}/*util*/;
Object.assign(wasm, {
/**

61
ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js
View File

@ -59,6 +59,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
const toss3 = sqlite3.util.toss3;
const initPromises = Object.create(null);
const capi = sqlite3.capi;
const util = sqlite3.util;
const wasm = sqlite3.wasm;
// Config opts for the VFS...
const SECTOR_SIZE = 4096;
@ -869,9 +870,48 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
return b;
}
//! Impl for importDb() when its 2nd arg is a function.
async importDbChunked(name, callback){
const sah = this.#mapFilenameToSAH.get(name)
|| this.nextAvailableSAH()
|| toss("No available handles to import to.");
sah.truncate(0);
let nWrote = 0, chunk, checkedHeader = false, err = false;
try{
while( undefined !== (chunk = await callback()) ){
if(chunk instanceof ArrayBuffer) chunk = new Uint8Array(chunk);
if( 0===nWrote && chunk.byteLength>=15 ){
util.affirmDbHeader(chunk);
checkedHeader = true;
}
sah.write(chunk, {at: HEADER_OFFSET_DATA + nWrote});
nWrote += chunk.byteLength;
}
if( nWrote < 512 || 0!==nWrote % 512 ){
toss("Input size",nWrote,"is not correct for an SQLite database.");
}
if( !checkedHeader ){
const header = new Uint8Array(20);
sah.read( header, {at: 0} );
util.affirmDbHeader( header );
}
sah.write(new Uint8Array(2), {
at: HEADER_OFFSET_DATA + 18
}/*force db out of WAL mode*/);
}catch(e){
this.setAssociatedPath(sah, '', 0);
}
this.setAssociatedPath(sah, name, capi.SQLITE_OPEN_MAIN_DB);
return nWrote;
}
//! Documented elsewhere in this file.
importDb(name, bytes){
if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes);
if( bytes instanceof ArrayBuffer ) bytes = new Uint8Array(bytes);
else if( bytes instanceof Function ) return this.importDbChunked(name, bytes);
const sah = this.#mapFilenameToSAH.get(name)
|| this.nextAvailableSAH()
|| toss("No available handles to import to.");
const n = bytes.byteLength;
if(n<512 || n%512!=0){
toss("Byte array size is invalid for an SQLite db.");
@ -882,16 +922,16 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
toss("Input does not contain an SQLite database header.");
}
}
const sah = this.#mapFilenameToSAH.get(name)
|| this.nextAvailableSAH()
|| toss("No available handles to import to.");
const nWrote = sah.write(bytes, {at: HEADER_OFFSET_DATA});
if(nWrote != n){
this.setAssociatedPath(sah, '', 0);
toss("Expected to write "+n+" bytes but wrote "+nWrote+".");
}else{
sah.write(new Uint8Array([0,0]), {at: HEADER_OFFSET_DATA+18}
/* force db out of WAL mode */);
this.setAssociatedPath(sah, name, capi.SQLITE_OPEN_MAIN_DB);
}
return nWrote;
}
}/*class OpfsSAHPool*/;
@ -1098,6 +1138,19 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
automatically clean up any non-database files so importing them
is pointless.
If passed a function for its second argument, its behavior
changes to asynchronous and it imports its data in chunks fed to
it by the given callback function. It calls the callback (which
may be async) repeatedly, expecting either a Uint8Array or
ArrayBuffer (to denote new input) or undefined (to denote
EOF). For so long as the callback continues to return
non-undefined, it will append incoming data to the given
VFS-hosted database file. The result of the resolved Promise when
called this way is the size of the resulting database.
On succes this routine rewrites the database header bytes in the
output file (not the input array) to force disabling of WAL mode.
On a write error, the handle is removed from the pool and made
available for re-use.

95
ext/wasm/api/sqlite3-vfs-opfs.c-pp.js
View File

@ -136,6 +136,7 @@ const installOpfsVfs = function callee(options){
const error = (...args)=>logImpl(0, ...args);
const toss = sqlite3.util.toss;
const capi = sqlite3.capi;
const util = sqlite3.util;
const wasm = sqlite3.wasm;
const sqlite3_vfs = capi.sqlite3_vfs;
const sqlite3_file = capi.sqlite3_file;
@ -1168,40 +1169,98 @@ const installOpfsVfs = function callee(options){
doDir(opt.directory, 0);
};
/**
impl of importDb() when it's given a function as its second
argument.
*/
const importDbChunked = async function(filename, callback){
const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true);
const hFile = await hDir.getFileHandle(fnamePart, {create:true});
const sah = await hFile.createSyncAccessHandle();
sah.truncate(0);
let nWrote = 0, chunk, checkedHeader = false, err = false;
try{
while( undefined !== (chunk = await callback()) ){
if(chunk instanceof ArrayBuffer) chunk = new Uint8Array(chunk);
if( 0===nWrote && chunk.byteLength>=15 ){
util.affirmDbHeader(chunk);
checkedHeader = true;
}
sah.write(chunk, {at: nWrote});
nWrote += chunk.byteLength;
}
if( nWrote < 512 || 0!==nWrote % 512 ){
toss("Input size",nWrote,"is not correct for an SQLite database.");
}
if( !checkedHeader ){
const header = new Uint8Array(20);
sah.read( header, {at: 0} );
util.affirmDbHeader( header );
}
sah.write(new Uint8Array(2), {at: 18}/*force db out of WAL mode*/);
return nWrote;
}catch(e){
await hDir.removeEntry( fnamePart ).catch(()=>{});
throw e;
}finally {
await sah.close();
}
};
/**
Asynchronously imports the given bytes (a byte array or
ArrayBuffer) into the given database file.
If passed a function for its second argument, its behaviour
changes to async and it imports its data in chunks fed to it by
the given callback function. It calls the callback (which may
be async) repeatedly, expecting either a Uint8Array or
ArrayBuffer (to denote new input) or undefined (to denote
EOF). For so long as the callback continues to return
non-undefined, it will append incoming data to the given
VFS-hosted database file. When called this way, the resolved
value of the returned Promise is the number of bytes written to
the target file.
It very specifically requires the input to be an SQLite3
database and throws if that's not the case. It does so in
order to prevent this function from taking on a larger scope
than it is specifically intended to. i.e. we do not want it to
become a convenience for importing arbitrary files into OPFS.
Throws on error. Resolves to the number of bytes written.
This routine rewrites the database header bytes in the output
file (not the input array) to force disabling of WAL mode.
On error this throws and the state of the input file is
undefined (it depends on where the exception was triggered).
On success, resolves to the number of bytes written.
*/
opfsUtil.importDb = async function(filename, bytes){
if( bytes instanceof Function ){
return importDbChunked(filename, bytes);
}
if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes);
util.affirmIsDb(bytes);
const n = bytes.byteLength;
if(n<512 || n%512!=0){
toss("Byte array size is invalid for an SQLite db.");
}
const header = "SQLite format 3";
for(let i = 0; i < header.length; ++i){
if( header.charCodeAt(i) !== bytes[i] ){
toss("Input does not contain an SQLite database header.");
}
}
const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true);
const hFile = await hDir.getFileHandle(fnamePart, {create:true});
const sah = await hFile.createSyncAccessHandle();
sah.truncate(0);
const nWrote = sah.write(bytes, {at: 0});
sah.close();
if(nWrote != n){
toss("Expected to write "+n+" bytes but wrote "+nWrote+".");
let sah, err, nWrote = 0;
try {
const hFile = await hDir.getFileHandle(fnamePart, {create:true});
sah = await hFile.createSyncAccessHandle();
sah.truncate(0);
nWrote = sah.write(bytes, {at: 0});
if(nWrote != n){
toss("Expected to write "+n+" bytes but wrote "+nWrote+".");
}
sah.write(new Uint8Array(2), {at: 18}) /* force db out of WAL mode */;
return nWrote;
}catch(e){
await hDir.removeEntry( fnamePart ).catch(()=>{});
throw e;
}finally{
if( sah ) await sah.close();
}
return nWrote;
};
if(sqlite3.oo1){