1
0
mirror of https://github.com/sqlite/sqlite.git synced 2026-10-23 21:49:02 +03:00

Extend kvvfs export to optionally export the raw binary db pages as a list of Uint8Array instead of kvvfs-encoded strings. This is typically much larger but the pages can then be used as-is.

FossilOrigin-Name: a4f59496a53a079f8f73e4cde68f47dbd13d2d74de2ad11bc716e7e5c00f1ec0
This commit is contained in:
stephan
2025-11-30 03:02:06 +00:00
parent 6a59bb6057
commit 768c364aaa
6 changed files with 180 additions and 45 deletions

View File

@@ -116,12 +116,38 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
util.assert( 32<=kvvfsMethods.$nKeySize, "unexpected kvvfsMethods.$nKeySize: "+kvvfsMethods.$nKeySize);
const cache = Object.assign(Object.create(null),{
fixedPageSize: 8192/*used in some validation*/,
rxJournalSuffix: /-journal$/,
zKeyJrnl: wasm.allocCString("jrnl"),
zKeySz: wasm.allocCString("sz"),
keySize: kvvfsMethods.$nKeySize
keySize: kvvfsMethods.$nKeySize,
buffer: Object.assign(Object.create(null),{
n: kvvfsMethods.$nBufferSize,
pool: Object.create(null)
})
});
/**
A wasm.alloc()'d buffer large enough for all kvvfs
encoding/decoding needs (cache.buffer.n).
We leak this one-time alloc because we've no better option.
sqlite3_vfs does not have a finalizer, so we've no place to hook
in the cleanup. We "could" extend sqlite3_shutdown() to have a
cleanup list for stuff like this but that function is never
used in JS, so it's hardly worth it.
*/
cache.memBuffer = (id=0)=>cache.buffer.pool[id] ??= wasm.alloc(cache.buffer.n);
/** Freeds the buffer with the given id. */
cache.memBufferFree = (id)=>{
const b = cache.buffer.pool[id];
if( b ){
wasm.dealloc(b);
delete cache.buffer.pool[id];
}
};
const debug = sqlite3.__isUnderTest
? function(){sqlite3.config.debug("kvvfs:", ...arguments)}
: function(){};
@@ -297,7 +323,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
const other = cache.rxJournalSuffix.test(store.jzClass)
? store.jzClass.replace(cache.rxJournalSuffix,'')
: store.jzClass+'-journal';
debug("cleaning up storage handles [", store.jzClass, other,"]",store);
//debug("cleaning up storage handles [", store.jzClass, other,"]",store);
delete cache.storagePool[store.jzClass];
delete cache.storagePool[other];
if( !sqlite3.__isUnderTest ){
@@ -568,16 +594,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
debug("xRcrdRead", nBuf, zClass, wasm.cstrToJs(zClass),
wasm.cstrToJs(zKey), nV, jV, store);
}
const zV = cache.keybuf.mem ??= wasm.alloc.impl(cache.keybuf.n)
/* We leak this one-time alloc because we've no better
option. sqlite3_vfs does not have a finalizer, so
we've no place to hook in the cleanup. We "could"
extend sqlite3_shutdown() to have a cleanup list for
stuff like this but (A) that function is never used in
JS and (B) its cleanup would leave cache.keybuf
pointing to stale memory, so if the library were used
after sqlite3_shutdown() then we'd corrupt memory. */;
if( !zV ) return -3 /*OOM*/;
const zV = cache.memBuffer(0);
//if( !zV ) return -3 /*OOM*/;
const heap = wasm.heap8();
let i;
for(i = 0; i < nV; ++i){
@@ -634,7 +652,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
xOpen: function(pProtoVfs,zName,pProtoFile,flags,pOutFlags){
cache.popError();
try{
//cache.zReadBuf ??= wasm.malloc(kvvfsMethods.$nBufferSize);
if( !zName ){
zName = (cache.zEmpty ??= wasm.allocCString(""));
}
@@ -680,7 +697,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
installStorageAndJournal(s);
s.files.push(f);
s.deleteAtRefc0 = deleteAt0 || !!(capi.SQLITE_OPEN_DELETEONCLOSE & flags);
debug("xOpen installed storage handle [",nm, nm+"-journal","]", s);
//debug("xOpen installed storage handle [",nm, nm+"-journal","]", s);
}
pFileHandles.set(pProtoFile, {storage: s, file: f, jzClass});
notifyListeners('open', s, s.files.length);
@@ -874,9 +891,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
kvvfsMethods, capi.sqlite3_file.structInfo,
KVVfsFile.structInfo);
try {
cache.keybuf = Object.create(null);
cache.keybuf.n = kvvfsMethods.$nBufferSize;
util.assert( cache.keybuf.n>1024*129, "Key buffer is not large enough"
util.assert( cache.buffer.n>1024*129, "Heap buffer is not large enough"
/* Native is SQLITE_KVOS_SZ is 133073 as of this writing */ );
for(const e of Object.entries(methodOverrides.recordHandler)){
// Overwrite kvvfsMethods's callbacks
@@ -1022,9 +1037,28 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
};
/**
Copies the entire contents of the given transient storage object
into a JSON-friendly form. The returned object is structured as
follows...
Exports a kvvfs storage object to an object, optionally
JSON-friendly.
Usages:
thisfunc(storageName);
thisfunc(options);
In the latter case, the options object must be an object with
the following properties:
- "name" (string) required. The storage to export.
- "expandPages" (bool=false). If true, the .pages result property
holdes Uint8Array objects holding the raw binary-format db
pages. The default is to use kvvfs-encoded string pages
(JSON-friendly).
- "includeJournal" (bool=false). If true and the db has a current
journal, it is exported as well.
The returned object is structured as follows...
- "name": the name of the storage. This is 'local' or 'session'
for localStorage resp. sessionStorage, and an arbitrary name for
@@ -1056,11 +1090,20 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
Added in version @kvvfs-v2-added-in@.
*/
const sqlite3_js_kvvfs_export = function(storageName,includeJournal=true){
const store = storageForZClass(storageName);
const sqlite3_js_kvvfs_export = function callee(...args){
let opt;
if( 1===args.length && 'object'===typeof args[0] ){
opt = args[0];
}else{
opt = {
name: args[0],
//expandPages: true
};
}
const store = storageForZClass(opt.name);
if( !store ){
toss3(capi.SQLITE_NOTFOUND,
"There is no kvvfs storage named",storageName);
"There is no kvvfs storage named",opt.name);
}
//debug("store to export=",store);
const s = store.storage;
@@ -1070,6 +1113,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
pages: []
});
const pages = Object.create(null);
let xpages;
const keyPrefix = kvvfsKeyPrefix(rc.name);
const rxTail = keyPrefix
? /^kvvfs-[^-]+-(\w+)/ /* X... part of kvvfs-NAME-X... */
@@ -1081,7 +1125,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
let kk = (keyPrefix ? rxTail.exec(k) : undefined)?.[1] ?? k;
switch( kk ){
case 'jrnl':
if( includeJournal ) rc.journal = s.getItem(k);
if( opt.includeJournal ) rc.journal = s.getItem(k);
break;
case 'sz':
rc.size = +s.getItem(k);
@@ -1091,11 +1135,34 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
if( !util.isInt32(kk) || kk<=0 ){
toss3(capi.SQLITE_RANGE, "Malformed kvvfs key: "+k);
}
pages[kk] = s.getItem(k);
if( opt.expandPages ){
const spg = s.getItem(k),
n = spg.length,
z = cache.memBuffer(0),
zDec = cache.memBuffer(1),
heap = wasm.heap8u()/* MUST be inited last*/;
let i = 0;
for( ; i < n; ++i ){
heap[wasm.ptr.add(z, i)] = spg.codePointAt(i) & 0xff;
}
heap[wasm.ptr.add(z, i)] = 0;
//debug("Decoding",i,"page bytes");
const nDec = wasm.exports.sqlite3__wasm_kvvfs_decode(
z, zDec, cache.buffer.n
);
if( cache.fixedPageSize !== nDec ){
util.toss3(capi.SQLITE_ERROR,"Unexpected decoded page size:",nDec);
}
//debug("Decoded",nDec,"page bytes");
pages[kk] = heap.slice(zDec, wasm.ptr.add(zDec, nDec));
}else{
pages[kk] = s.getItem(k);
}
break;
}
}
}
if( opt.expandPages ) cache.memBufferFree(1);
/* Now sort the page numbers and move them into an array. In JS
property keys are always strings, so we have to coerce them to
numbers so we can get them sorted properly for the array. */
@@ -1139,7 +1206,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
|| !Array.isArray(exp.pages) ){
toss3(capi.SQLITE_MISUSE, "Malformed export object.");
}
//warn("importFromObject() is incomplete");
validateStorageName(exp.name);
let store = storageForZClass(exp.name);
const isNew = !store;
@@ -1156,7 +1222,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
toss3(capi.SQLITE_IOERR_ACCESS,
"Cannot import db storage while it is in use.");
}
sqlite3_js_kvvfs_clear(exp.name, true);
sqlite3_js_kvvfs_clear(exp.name);
}else{
store = newStorageObj(exp.name);
//warn("Installing new storage:",store);
@@ -1164,13 +1230,43 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
//debug("Importing store",store.poolEntry.files.length, store);
//debug("object to import:",exp);
const keyPrefix = kvvfsKeyPrefix(exp.name);
let zEnc;
try{
/* Force the native KVVfsFile instances to re-read the db
and page size. */;
const s = store.storage;
s.setItem(keyPrefix+'sz', exp.size);
if( exp.journal ) s.setItem(keyPrefix+'jrnl', exp.journal);
exp.pages.forEach((v,ndx)=>s.setItem(keyPrefix+(ndx+1), v));
if( exp.pages[0] instanceof Uint8Array ){
/* raw binary pages */
//debug("pages",exp.pages);
exp.pages.forEach((u,ndx)=>{
const n = u.length;
if( cache.fixedPageSize !== n ){
util.toss3(capi.SQLITE_RANGE,"Unexpected page size:", n);
}
zEnc = cache.memBuffer(1);
const zBin = cache.memBuffer(0),
heap = wasm.heap8u();
/* Copy u to the heap and encode the heaped copy. This is
_presumably_ faster than porting the encoding algo to
JS, which would involve many, many more function calls. */
let i;
for(i=0; i<n; ++i ) heap[wasm.ptr.add(zBin,i)] = u[i];
heap[wasm.ptr.add(zBin,i)] = 0;
const rc = wasm.exports.sqlite3__wasm_kvvfs_encode(zBin, i, zEnc);
util.assert( rc < cache.buffer.n,
"Impossibly long output - possibly smashed the heap" );
util.assert( 0===wasm.peek8(wasm.ptr.add(zEnc,rc)),
"Expecting NUL-terminated encoded output" );
const jenc = wasm.cstrToJs(zEnc);
//debug("(un)encoded page:",u,jenc);
s.setItem(keyPrefix+(ndx+1), jenc);
});
}else if( exp.pages[0] ){
/* kvvfs-encoded pages */
exp.pages.forEach((v,ndx)=>s.setItem(keyPrefix+(ndx+1), v));
}
if( isNew ) installStorageAndJournal(store);
}catch(e){
if( !isNew ){
@@ -1178,6 +1274,10 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
catch(ee){/*ignored*/}
}
throw e;
}finally{
if( zEnc ){
cache.memBufferFree(1);
}
}
return this;
};
@@ -1468,10 +1568,10 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
}
});
Object.assign(Object.create(ProtoCursor),{
rowid: 0,
names: Object.keys(cache.storagePool)
.filter(v=>!cache.rxJournalSuffix.test(v))
})
rowid: 0,
names: Object.keys(cache.storagePool)
.filter(v=>!cache.rxJournalSuffix.test(v))
});
const cursorState = function(cursor, reset){
const o = (cursor instanceof capi.sqlite3_vtab_cursor)
? cursor

View File

@@ -1794,6 +1794,10 @@ SQLITE_WASM_EXPORT
int sqlite3__wasm_kvvfs_decode(const char *a, char *aOut, int nOut){
return kvvfsDecode(a, aOut, nOut);
}
SQLITE_WASM_EXPORT
int sqlite3__wasm_kvvfs_encode(const char *a, int nA, char *aOut){
return kvvfsEncode(a, nA, aOut);
}
#if defined(__EMSCRIPTEN__) && defined(SQLITE_ENABLE_WASMFS)

View File

@@ -2908,6 +2908,28 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
'size', 'clear'] ){
T.assert( k[n] instanceof Function );
}
if( 0 ){
const scope = wasm.scopedAllocPush();
try{
const pg = "53514C69746520666F726D61742033b20b0101b402020d02d02l01d04l01jb02b2E91E00Dd011FD3b1FD3dxl2B010617171701377461626C656B767666736B7676667302435245415445205441424C45206B76766673286129";
const n = pg.length;
const pI = wasm.scopedAlloc( n+1 );
const nO = 8192 * 2;
const pO = wasm.scopedAlloc( nO );
const heap = wasm.heap8u();
let i;
for( i=0; i<n; ++i ){
heap[wasm.ptr.add(pI, i)] = pg.codePointAt(i) & 0xff;
}
heap[wasm.ptr.add(pI, i)] = 0;
const rc = wasm.exports.sqlite3__wasm_kvvfs_decode(pI, pO, nO);
const u = heap.slice(pO, wasm.ptr.add(pO,rc));
debug("decode rc=", rc, u);
}finally{
wasm.scopedAllocPop(scope);
}
}
}
}/*kvvfs API availability*/)
.t({
@@ -3002,7 +3024,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
db = new JDb(filename);
db.exec('insert into kvvfs(a) values(4),(5),(6)');
T.assert(6 === db.selectValue('select count(*) from kvvfs'));
const exp = exportDb(filename,true);
const exp = exportDb({name:filename,includeJournal:true});
T.assert( filename===exp.name, "Broken export filename" )
.assert( exp?.size > 0, "Missing db size" )
.assert( exp?.pages?.length > 0, "Missing db pages" );
@@ -3179,7 +3201,11 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
// It fails at this point for non-8k sizes
T.assert(expectRows === duo.selectValue(sqlCount),
"Unexpected record count.");
exp = exportDb(filename);
exp = exportDb({
name: filename,
expandPages: true
});
debug("Exported page-expanded db",exp);
if( 0 ){
debug("vacuumed export",exp);
}
@@ -3281,6 +3307,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
}
}
})/*kvvfs listeners */
.t({
name: 'kvvfs vtab',
predicate: (sqlite3)=>!!sqlite3.kvvfs.create_module,
@@ -3308,6 +3335,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
}
}
})/* kvvfs vtab */
//#if enable-see
.t({
name: 'kvvfs SEE encryption in sessionStorage',

View File

@@ -1,5 +1,5 @@
C Add\sa\scomment\sexplaining\sthe\snew\skvvfsDecode()\sreturn\s-1\sfrom\s[cd81cb70525e].
D 2025-11-29T23:41:34.784
C Extend\skvvfs\sexport\sto\soptionally\sexport\sthe\sraw\sbinary\sdb\spages\sas\sa\slist\sof\sUint8Array\sinstead\sof\skvvfs-encoded\sstrings.\sThis\sis\stypically\smuch\slarger\sbut\sthe\spages\scan\sthen\sbe\sused\sas-is.
D 2025-11-30T03:02:06.188
F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
@@ -600,11 +600,11 @@ F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667
F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89
F ext/wasm/api/sqlite3-opfs-async-proxy.js 9654b565b346dc609b75d15337f20acfa7af7d9d558da1afeb9b6d8eaa404966
F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d
F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 52c79574b05ee39ec01de1ba63a1ed36bece0858b817d2c405e36e19864610ad
F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js e01f80208b7f00a47045bc92e5c0b40c6e2232fc058f2dd81ed41d46b1cfd323
F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js a2eea6442556867b589e04107796c6e1d04a472219529eeb45b7cd221d7d048b
F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 88ce2078267a2d1af57525a32d896295f4a8db7664de0e17e82dc9ff006ed8d3
F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81
F ext/wasm/api/sqlite3-wasm.c d74ea827a644598090cf450a0bb38564332e658829d3e924f471558de669a336
F ext/wasm/api/sqlite3-wasm.c 3fd63b99dec39665ba89fcca6d001bb404ac9ff0be8c52e4e07f52749a8ba57a
F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bda1c75bd674a92a0e27cc2f3d46dbbf21e422413f8046814515a0bd7409328a
F ext/wasm/api/sqlite3-worker1.c-pp.js 802d69ead8c38dc1be52c83afbfc77e757da8a91a2e159e7ed3ecda8b8dba2e7
F ext/wasm/c-pp-lite.c f38254fba42561728c2e4764a7ba8d68700091e7c2f4418112868c0daba16783
@@ -647,7 +647,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555
F ext/wasm/test-opfs-vfs.js 1618670e466f424aa289859fe0ec8ded223e42e9e69b5c851f809baaaca1a00c
F ext/wasm/tester1-worker.c-pp.html 0e432ec2c0d99cd470484337066e8d27e7aee4641d97115338f7d962bf7b081a
F ext/wasm/tester1.c-pp.html 52d88fe2c6f21a046030a36410b4839b632f4424028197a45a3d5669ea724ddb
F ext/wasm/tester1.c-pp.js 3398107cad5c548443d1978adf8d8b3ca48dd6baa0410ae26b24b016d39cf157
F ext/wasm/tester1.c-pp.js d5bf9dd8be21377981e20974fa798ae9839ee7a279189a773299532a844a8138
F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e
F ext/wasm/tests/opfs/concurrency/test.js d08889a5bb6e61937d0b8cbb78c9efbefbf65ad09f510589c779b7cc6a803a88
F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2
@@ -718,7 +718,7 @@ F src/notify.c 57c2d1a2805d6dee32acd5d250d928ab94e02d76369ae057dee7d445fd64e878
F src/os.c 509452169d5ea739723e213b8e2481cf0e587f0e88579a912d200db5269f5f6d
F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63
F src/os_common.h 6c0eb8dd40ef3e12fe585a13e709710267a258e2c8dd1c40b1948a1d14582e06
F src/os_kv.c 110ef88f5ee976124a08a03c51ac47f46cc9904739929e77016d7ab0ca9004b0
F src/os_kv.c e7d96727db5b67e39d590a68cc61c86daf4c093c36c011a09ebfb521182ec28d
F src/os_setup.h 8efc64eda6a6c2f221387eefc2e7e45fd5a3d5c8337a7a83519ba4fbd2957ae2
F src/os_unix.c 7945ede1e85b2d1b910e1b4af9ba342e964b1e30e79f4176480a60736445cb36
F src/os_win.c a89b501fc195085c7d6c9eec7f5bd782625e94bb2a96b000f4d009703df1083f
@@ -2180,8 +2180,8 @@ F tool/version-info.c 33d0390ef484b3b1cb685d59362be891ea162123cea181cb8e6d2cf6dd
F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7
F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
P 989d097b1324cf712107bb81697fa8e9044aea0e7feacf0e6b6561e216f07989
R 465351f3f0514d80bd2330c01d6416cc
P ed9ab366d1c1880d3c06edce6c0c33ad30c7ae59725c1ec1fe3f620be1835630
R 8530361058afe18d87abefbdbe6a3870
U stephan
Z ab4d411f734d9d3878ae160021ac295e
Z 197caaef7aceb5323c153f1905283e67
# Remove this line to create a well-formed Fossil manifest.

View File

@@ -1 +1 @@
ed9ab366d1c1880d3c06edce6c0c33ad30c7ae59725c1ec1fe3f620be1835630
a4f59496a53a079f8f73e4cde68f47dbd13d2d74de2ad11bc716e7e5c00f1ec0

View File

@@ -383,7 +383,10 @@ sqlite3_kvvfs_methods sqlite3KvvfsMethods = {
** of hexadecimal and base-26 numbers, it is always clear where
** one stops and the next begins.
*/
static int kvvfsEncode(const char *aData, int nData, char *aOut){
#ifndef SQLITE_WASM
static
#endif
int kvvfsEncode(const char *aData, int nData, char *aOut){
int i, j;
const unsigned char *a = (const unsigned char*)aData;
for(i=j=0; i<nData; i++){