database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-07-29 08:01:23 +03:00
Code Activity

Fix a long-standing filename digest computation bug in the OPFS SAHPool VFS which caused all VFS-stored filenames to have a digest value of 0. See [/forumpost/042d53c928382021] and for full details.

Browse Source 
FossilOrigin-Name: 493cbe74504e8eb1ca8f2edf49fdab6bebc7fe36ffab06932a4b8c5a4eea86cd
This commit is contained in:
stephan
2025-03-16 14:05:42 +00:00
parent ab68965794 5390f95f07
commit d176ef5881
6 changed files with 326 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

79
ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js
View File

@ -79,6 +79,48 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
capi.SQLITE_OPEN_MAIN_JOURNAL |
capi.SQLITE_OPEN_SUPER_JOURNAL |
capi.SQLITE_OPEN_WAL;
const FLAG_COMPUTE_DIGEST_V2 = capi.SQLITE_OPEN_MEMORY
/* Part of the fix for
https://github.com/sqlite/sqlite-wasm/issues/97
Summary: prior to version 3.50.0 computeDigest() always computes
a value of [0,0] due to overflows, so it does not do anything
useful. Fixing it invalidates old persistent files, so we
instead only fix it for files created or updated since the bug
was discovered and fixed.
This flag determines whether we use the broken legacy
computeDigest() or the v2 variant. We only use this flag for
newly-created/overwritten files. Pre-existing files have the
broken digest stored in them so need to continue to use that.
What this means, in terms of db file compatibility between
versions:
- DBs created with versions older than this fix (<3.50.0)
can be read by post-fix versions. Such DBs which are written
to in-place (not replaced) by newer versions can still be read
by older versions, as the affected digest is only modified
when the SAH slot is assigned to a given filename.
- DBs created with post-fix versions will, when read by a pre-fix
version, be seen as having a "bad digest" and will be
unceremoniously replaced by that pre-fix version. When swapping
back to a post-fix version, that version will see that the file
entry is missing the FLAG_COMPUTE_DIGEST_V2 bit so will treat it
as a legacy file.
This flag is stored in the same memory as the various
SQLITE_OPEN_... flags and we must be careful here to not use a
flag bit which is otherwise relevant for the VFS.
SQLITE_OPEN_MEMORY is handled by sqlite3_open_v2() and friends,
not the VFS, so we'll repurpose that one. If we take a
currently-unused bit and it ends up, at some later point, being
used, we would have to invalidate existing VFS files in order to
move to another bit. Similarly, if the SQLITE_OPEN_MEMORY bit
were ever reassigned (which it won't be!), we'd invalidate all
VFS-side files.
*/;
/** Subdirectory of the VFS's space where "opaque" (randomly-named)
files are stored. Changing this effectively invalidates the data
@ -329,6 +371,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){
const pool = getPoolForVfs(pVfs);
try{
flags &= ~FLAG_COMPUTE_DIGEST_V2;
pool.log(`xOpen ${wasm.cstrToJs(zName)} ${flags}`);
// First try to open a path that already exists in the file system.
const path = (zName && wasm.peek8(zName))
@ -624,7 +667,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
const fileDigest = new Uint32Array(HEADER_DIGEST_SIZE / 4);
sah.read(fileDigest, {at: HEADER_OFFSET_DIGEST});
const compDigest = this.computeDigest(this.#apBody);
const compDigest = this.computeDigest(this.#apBody, flags);
//warn("getAssociatedPath() flags",'0x'+flags.toString(16), "compDigest", compDigest);
if(fileDigest.every((v,i) => v===compDigest[i])){
// Valid digest
const pathBytes = this.#apBody.findIndex((v)=>0===v);
@ -633,6 +677,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
// leaving stale db data laying around.
sah.truncate(HEADER_OFFSET_DATA);
}
//warn("getAssociatedPath() flags",'0x'+flags.toString(16), "compDigest", compDigest,"pathBytes",pathBytes);
return pathBytes
? textDecoder.decode(this.#apBody.subarray(0,pathBytes))
: '';
@ -655,10 +700,17 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
if(HEADER_MAX_PATH_SIZE <= enc.written + 1/*NUL byte*/){
toss("Path too long:",path);
}
if(path && flags){
/* When creating or re-writing files, update their digest, if
needed, to v2. We continue to use v1 for the (!path) case
(empty files) because there's little reason not to use a
digest of 0 for empty entries. */
flags |= FLAG_COMPUTE_DIGEST_V2;
}
this.#apBody.fill(0, enc.written, HEADER_MAX_PATH_SIZE);
this.#dvBody.setUint32(HEADER_OFFSET_FLAGS, flags);
const digest = this.computeDigest(this.#apBody);
const digest = this.computeDigest(this.#apBody, flags);
//console.warn("setAssociatedPath(",path,") digest",digest);
sah.write(this.#apBody, {at: 0});
sah.write(digest, {at: HEADER_OFFSET_DIGEST});
sah.flush();
@ -679,15 +731,22 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
metadata for each file as a validation check. Changing this
algorithm invalidates all existing databases for this VFS, so
don't do that.
See the docs for FLAG_COMPUTE_DIGEST_V2 for more details.
*/
computeDigest(byteArray){
let h1 = 0xdeadbeef;
let h2 = 0x41c6ce57;
for(const v of byteArray){
h1 = 31 * h1 + (v * 307);
h2 = 31 * h2 + (v * 307);
computeDigest(byteArray, fileFlags){
if( fileFlags & FLAG_COMPUTE_DIGEST_V2 ){
let h1 = 0xdeadbeef;
let h2 = 0x41c6ce57;
for(const v of byteArray){
h1 = Math.imul(h1 ^ v, 2654435761);
h2 = Math.imul(h2 ^ v, 104729);
}
return new Uint32Array([h1>>>0, h2>>>0]);
}else{
/* this is what the buggy legacy computation worked out to */
return new Uint32Array([0,0]);
}
return new Uint32Array([h1>>>0, h2>>>0]);
}
/**

2
ext/wasm/tester1.c-pp.js
View File

@ -3506,7 +3506,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
});
db.exec([
"create table t(a);",
"insert into t(a) values(1),(2),(3);",
"insert into t(a) values(1),(2),(1);",
"select auxtest(1,a), auxtest(1,a) from t order by a"
]);
}finally{

94
ext/wasm/tests/opfs/sahpool/digest-worker.js Normal file
View File

@ -0,0 +1,94 @@
/*
2025-01-31
The author disclaims copyright to this source code. In place of a
legal notice, here is a blessing:
* May you do good and not evil.
* May you find forgiveness for yourself and forgive others.
* May you share freely, never taking more than you give.
***********************************************************************
This file is part of testing the OPFS SAHPool VFS's computeDigest()
fix. See ./digest.html for the details.
*/
const clog = console.log.bind(console);
const wPost = (type,...args)=>postMessage({type, payload:args});
const log = (...args)=>{
clog("Worker:",...args);
wPost('log',...args);
}
const hasOpfs = ()=>{
return globalThis.FileSystemHandle
&& globalThis.FileSystemDirectoryHandle
&& globalThis.FileSystemFileHandle
&& globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle
&& navigator?.storage?.getDirectory;
};
if( !hasOpfs() ){
wPost('error',"OPFS not detected");
throw new Error("OPFS not detected");
}
clog("Importing sqlite3...");
const searchParams = new URL(self.location.href).searchParams;
importScripts(searchParams.get('sqlite3.dir') + '/sqlite3.js');
const runTests = function(sqlite3, poolUtil){
const fname = '/my.db';
let db = new poolUtil.OpfsSAHPoolDb(fname);
let n = (new Date()).valueOf();
try {
db.exec([
"create table if not exists t(a);"
]);
db.exec({
sql: "insert into t(a) values(?)",
bind: n++
});
log(fname,"record count: ",db.selectValue("select count(*) from t"));
}finally{
db.close();
}
db = new poolUtil.OpfsSAHPoolDb(fname);
try {
db.exec({
sql: "insert into t(a) values(?)",
bind: n++
});
log(fname,"record count: ",db.selectValue("select count(*) from t"));
}finally{
db.close();
}
const fname2 = '/my2.db';
db = new poolUtil.OpfsSAHPoolDb(fname2);
try {
db.exec([
"create table if not exists t(a);"
]);
db.exec({
sql: "insert into t(a) values(?)",
bind: n++
});
log(fname2,"record count: ",db.selectValue("select count(*) from t"));
}finally{
db.close();
}
};
globalThis.sqlite3InitModule().then(async function(sqlite3){
log("sqlite3 version:",sqlite3.version);
const sahPoolConfig = {
name: 'opfs-sahpool-digest',
clearOnInit: false,
initialCapacity: 6
};
return sqlite3.installOpfsSAHPoolVfs(sahPoolConfig).then(poolUtil=>{
log('vfs acquired');
runTests(sqlite3, poolUtil);
});
});

151
ext/wasm/tests/opfs/sahpool/digest.html Normal file
View File

@ -0,0 +1,151 @@
<!doctype html>
<html lang="en-us">
<head>
<meta charset="utf-8">
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<link rel="shortcut icon" href="data:image/x-icon;," type="image/x-icon">
<link rel="stylesheet" href="../../../common/emscripten.css"/>
<link rel="stylesheet" href="../../../common/testing.css"/>
<title>sqlite3 tester: OpfsSAHPool Digest</title>
<style></style>
</head>
<body><h1 id='color-target'></h1>
<p>
This is a test app for the digest calculation of the OPFS
SAHPool VFS. It requires running it with a new database created using
v3.49.0 or older, then running it again with a newer version, then
again with 3.49.0 or older.
</p>
<div class='input-wrapper'>
<input type='checkbox' id='cb-log-reverse'>
<label for='cb-log-reverse'>Reverse log order?</label>
</div>
<div id='test-output'></div>
<script>
/*
2025-02-03
The author disclaims copyright to this source code. In place of a
legal notice, here is a blessing:
* May you do good and not evil.
* May you find forgiveness for yourself and forgive others.
* May you share freely, never taking more than you give.
***********************************************************************
This is a bugfix test for the OPFS SAHPool VFS. It requires
setting up a database created using v3.49.0 or older, then
running it again with a newer version. In that case, the newer
version should be able to read the older version's db files
just fine. Revering back to the old version should also still
work - it should be able to read databases modified by the
newer version. However, a database _created_ by a version with
this fix will _not_ be legible by a version which predates
this fix, in which case the older version will see that VFS
file slot as corrupt and will clear it for re-use.
This is unfortunately rather cumbersome to test properly,
and essentially impossible to automate.
*/
(function(){
'use strict';
document.querySelector('h1').innerHTML =
document.querySelector('title').innerHTML;
const mapToString = (v)=>{
switch(typeof v){
case 'number': case 'string': case 'boolean':
case 'undefined': case 'bigint':
return ''+v;
default: break;
}
if(null===v) return 'null';
if(v instanceof Error){
v = {
message: v.message,
stack: v.stack,
errorClass: v.name
};
}
return JSON.stringify(v,undefined,2);
};
const normalizeArgs = (args)=>args.map(mapToString);
const logTarget = document.querySelector('#test-output');
const logClass = function(cssClass,...args){
const ln = document.createElement('div');
if(cssClass){
for(const c of (Array.isArray(cssClass) ? cssClass : [cssClass])){
ln.classList.add(c);
}
}
ln.append(document.createTextNode(normalizeArgs(args).join(' ')));
logTarget.append(ln);
};
const cbReverse = document.querySelector('#cb-log-reverse');
//cbReverse.setAttribute('checked','checked');
const cbReverseKey = 'tester1:cb-log-reverse';
const cbReverseIt = ()=>{
logTarget.classList[cbReverse.checked ? 'add' : 'remove']('reverse');
//localStorage.setItem(cbReverseKey, cbReverse.checked ? 1 : 0);
};
cbReverse.addEventListener('change', cbReverseIt, true);
/*if(localStorage.getItem(cbReverseKey)){
cbReverse.checked = !!(+localStorage.getItem(cbReverseKey));
}*/
cbReverseIt();
const log = (...args)=>{
//console.log(...args);
logClass('',...args);
}
const warn = (...args)=>{
console.warn(...args);
logClass('warning',...args);
}
const error = (...args)=>{
console.error(...args);
logClass('error',...args);
};
const toss = (...args)=>{
error(...args);
throw new Error(args.join(' '));
};
const endOfWork = (passed=true)=>{
const eH = document.querySelector('#color-target');
const eT = document.querySelector('title');
if(passed){
log("End of work chain. If you made it this far, you win.");
eH.innerText = 'PASS: '+eH.innerText;
eH.classList.add('tests-pass');
eT.innerText = 'PASS: '+eT.innerText;
}else{
eH.innerText = 'FAIL: '+eH.innerText;
eH.classList.add('tests-fail');
eT.innerText = 'FAIL: '+eT.innerText;
}
};
log("Running opfs-sahpool digest tests...");
const W1 = new Worker('digest-worker.js?sqlite3.dir=../../../jswasm');
W1.onmessage = function({data}){
//log("onmessage:",data);
switch(data.type){
case 'log':
log('worker says:', ...data.payload);
break;
case 'error':
error('worker says:', ...data.payload);
endOfWork(false);
break;
case 'initialized':
log(data.workerId, ': Worker initialized',...data.payload);
break;
}
};
})();
</script>
</body>
</html>

17
manifest
View File

@ -1,5 +1,5 @@
C Add\s--asan-fsanitize=...\sconfigure\sflag\sto\sthe\scanonical\sbuild\sto\soptionally\sset\s-fsantize\sflags\sfor\sthe\sfuzzcheck-asan\stool.\sTeach\sproj-check-fsanitiz\sto\sfail\sfor\sflags\swhich\sthe\scompiler\semits\sany\swarning\sfor,\sfor\sreasons\sdescribed\sin\sits\scomments.
D 2025-03-16T13:09:21.699
C Fix\sa\slong-standing\sfilename\sdigest\scomputation\sbug\sin\sthe\sOPFS\sSAHPool\sVFS\swhich\scaused\sall\sVFS-stored\sfilenames\sto\shave\sa\sdigest\svalue\sof\s0.\sSee\s[/forumpost/042d53c928382021]\sand\sfor\sfull\sdetails.
D 2025-03-16T14:05:42.209
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d
@ -645,7 +645,7 @@ F ext/wasm/api/sqlite3-api-worker1.c-pp.js f646a65257973b8c4481f8a6a216370b85644
F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89
F ext/wasm/api/sqlite3-opfs-async-proxy.js 9654b565b346dc609b75d15337f20acfa7af7d9d558da1afeb9b6d8eaa404966
F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d
F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 16d80af915bfd2529824b999b304425503094eedf34fb113d0791e002b13e5cf
F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 0f68a64e508598910e7c01214ae27d603dfc8baec6a184506fafac603a901931
F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 4ab0704ee198de7d1059eccedc7703c931510b588d10af0ee36ea5b3ebbac284
F ext/wasm/api/sqlite3-vtab-helper.c-pp.js e809739d71e8b35dfe1b55d24d91f02d04239e6aef7ca1ea92a15a29e704f616
F ext/wasm/api/sqlite3-wasm.c 6a4cd40267eaf08400895c5b9de39c56976c3b97b3c1bbe53fc2e80fa074e9c7
@ -696,10 +696,12 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555
F ext/wasm/test-opfs-vfs.js 1618670e466f424aa289859fe0ec8ded223e42e9e69b5c851f809baaaca1a00c
F ext/wasm/tester1-worker.html ebc4b820a128963afce328ecf63ab200bd923309eb939f4110510ab449e9814c
F ext/wasm/tester1.c-pp.html 1c1bc78b858af2019e663b1a31e76657b73dc24bede28ca92fbe917c3a972af2
F ext/wasm/tester1.c-pp.js f3a3cbf0207287c4caa9f84b4e2934804e4b5720c71eaf143f33c8122bd3c9f2
F ext/wasm/tester1.c-pp.js 419717b16e12703487a7ccf3ea4e63d693bdfbf7657e55a7e6c559bbccf027d3
F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e
F ext/wasm/tests/opfs/concurrency/test.js d08889a5bb6e61937d0b8cbb78c9efbefbf65ad09f510589c779b7cc6a803a88
F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2
F ext/wasm/tests/opfs/sahpool/digest-worker.js b0ab6218588f1f0a6d15a363b493ceaf29bfb87804d9e0165915a9996377cf79
F ext/wasm/tests/opfs/sahpool/digest.html 206d08a34dc8bd570b2581d3d9ab3ecad3201b516a598dd096dcf3cf8cd81df8
F ext/wasm/tests/opfs/sahpool/index.html be736567fd92d3ecb9754c145755037cbbd2bca01385e2732294b53f4c842328
F ext/wasm/tests/opfs/sahpool/sahpool-pausing.js f264925cfc82155de38cecb3d204c36e0f6991460fff0cb7c15079454679a4e2
F ext/wasm/tests/opfs/sahpool/sahpool-worker.js bd25a43fc2ab2d1bafd8f2854ad3943ef673f7c3be03e95ecf1612ff6e8e2a61
@ -2213,8 +2215,9 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350
F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7
F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
P be3a2e631100b711996b9524a54fc604966513a62d83fc916270a6226da7adab
R 699bb2f6a1869a511409b94aecd9b45c
P 013730e9b92af39cb7fd2871df9b4bc81b8990f918892bd79370704421672da0 500f2e6ec74b4c0e4ac0365ba4e0d81ed6df8dd09dc0f8af65d294c3453f8865
R 9266ed4d88e1ccecb16602d4a66164a0
T +closed 500f2e6ec74b4c0e4ac0365ba4e0d81ed6df8dd09dc0f8af65d294c3453f8865 Closed\sby\sintegrate-merge.
U stephan
Z b465670c31108cfd99bca4e96066f044
Z 8dfe9f7d9c954162ba84353f5fc2343c
# Remove this line to create a well-formed Fossil manifest.

2
manifest.uuid
View File

@ -1 +1 @@
013730e9b92af39cb7fd2871df9b4bc81b8990f918892bd79370704421672da0
493cbe74504e8eb1ca8f2edf49fdab6bebc7fe36ffab06932a4b8c5a4eea86cd