diff --git a/ext/misc/base64.c b/ext/misc/base64.c
index ec2fa70f1b..d2185b5405 100755
--- a/ext/misc/base64.c
+++ b/ext/misc/base64.c
@@ -200,7 +200,7 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){
SQLITE_LIMIT_LENGTH, -1);
char *cBuf;
ubyte *bBuf;
- assert(na==1);
+ assert(na==1);
switch( sqlite3_value_type(av[0]) ){
case SQLITE_BLOB:
nb = nv;
diff --git a/ext/misc/base85.c b/ext/misc/base85.c
index 32ae94906c..91e6992faf 100644
--- a/ext/misc/base85.c
+++ b/ext/misc/base85.c
@@ -148,10 +148,16 @@ static char * skipNonB85( char *s ){
return s;
}
-/* Convert small integer, known to be in 0..84 inclusive, to base85 numeral.*/
+/* Convert small integer, known to be in 0..84 inclusive, to base85 numeral.
+ * Do not use the macro form with argument expression having a side-effect.*/
+#if 0
static char base85Numeral( ubyte b ){
return (b < 4)? (char)(b + '#') : (char)(b - 4 + '*');
}
+#else
+# define base85Numeral( dn )\
+ ((char)(((dn) < 4)? (char)((dn) + '#') : (char)((dn) - 4 + '*')))
+#endif
static char *putcs(char *pc, char *s){
char c;
diff --git a/ext/misc/basexx.c b/ext/misc/basexx.c
index a113134c85..9cb43efeaf 100644
--- a/ext/misc/basexx.c
+++ b/ext/misc/basexx.c
@@ -28,7 +28,7 @@
** export mop1=-DSQLITE_SHELL_EXTSRC=ext/misc/basexx.c
** export mop2=-DSQLITE_SHELL_EXTFUNCS=BASEXX
** make sqlite3 "OPTS=$mop1 $mop2"
-** Win32 with Microsoft toolset on Windows:
+** Win32 with Microsoft toolset on Windows:
** set mop1=-DSQLITE_SHELL_EXTSRC=ext/misc/basexx.c
** set mop2=-DSQLITE_SHELL_EXTFUNCS=BASEXX
** set mops="OPTS=%mop1% %mop2%"
diff --git a/ext/wasm/api/sqlite3-api-oo1.js b/ext/wasm/api/sqlite3-api-oo1.js
index fc0f7372b8..9544a96c6f 100644
--- a/ext/wasm/api/sqlite3-api-oo1.js
+++ b/ext/wasm/api/sqlite3-api-oo1.js
@@ -201,9 +201,8 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
*/
dbCtorHelper.normalizeArgs = function(filename=':memory:',flags = 'c',vfs = null){
const arg = {};
- if(1===arguments.length && 'object'===typeof arguments[0]){
- const x = arguments[0];
- Object.keys(x).forEach((k)=>arg[k] = x[k]);
+ if(1===arguments.length && arguments[0] && 'object'===typeof arguments[0]){
+ Object.assign(arg, arguments[0]);
if(undefined===arg.flags) arg.flags = 'c';
if(undefined===arg.vfs) arg.vfs = null;
if(undefined===arg.filename) arg.filename = ':memory:';
diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js
index deb4c923ab..5537df89fb 100644
--- a/ext/wasm/api/sqlite3-api-opfs.js
+++ b/ext/wasm/api/sqlite3-api-opfs.js
@@ -348,12 +348,32 @@ const installOpfsVfs = function callee(options){
'SQLITE_NOTFOUND',
'SQLITE_OPEN_CREATE',
'SQLITE_OPEN_DELETEONCLOSE',
+ 'SQLITE_OPEN_MAIN_DB',
'SQLITE_OPEN_READONLY'
].forEach((k)=>{
if(undefined === (state.sq3Codes[k] = capi[k])){
toss("Maintenance required: not found:",k);
}
});
+ state.opfsFlags = Object.assign(Object.create(null),{
+ /**
+ Flag for use with xOpen(). "opfs-unlock-asap=1" enables
+ this. See defaultUnlockAsap, below.
+ */
+ OPFS_UNLOCK_ASAP: 0x01,
+ /**
+ If true, any async routine which implicitly acquires a sync
+ access handle (i.e. an OPFS lock) will release that locks at
+ the end of the call which acquires it. If false, such
+ "autolocks" are not released until the VFS is idle for some
+ brief amount of time.
+
+ The benefit of enabling this is much higher concurrency. The
+ down-side is much-reduced performance (as much as a 4x decrease
+ in speedtest1).
+ */
+ defaultUnlockAsap: false
+ });
/**
Runs the given operation (by name) in the async worker
@@ -844,9 +864,15 @@ const installOpfsVfs = function callee(options){
//xSleep is optionally defined below
xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){
mTimeStart('xOpen');
+ let opfsFlags = 0;
if(0===zName){
zName = randomFilename();
}else if('number'===typeof zName){
+ if(capi.sqlite3_uri_boolean(zName, "opfs-unlock-asap", 0)){
+ /* -----------------------^^^^^ MUST pass the untranslated
+ C-string here. */
+ opfsFlags |= state.opfsFlags.OPFS_UNLOCK_ASAP;
+ }
zName = wasm.cstringToJs(zName);
}
const fh = Object.create(null);
@@ -854,7 +880,7 @@ const installOpfsVfs = function callee(options){
fh.filename = zName;
fh.sab = new SharedArrayBuffer(state.fileBufferSize);
fh.flags = flags;
- const rc = opRun('xOpen', pFile, zName, flags);
+ const rc = opRun('xOpen', pFile, zName, flags, opfsFlags);
if(!rc){
/* Recall that sqlite3_vfs::xClose() will be called, even on
error, unless pFile->pMethods is NULL. */
@@ -1145,6 +1171,9 @@ const installOpfsVfs = function callee(options){
//TODO to support fiddle and worker1 db upload:
//opfsUtil.createFile = function(absName, content=undefined){...}
+ //We have sqlite3.wasm.sqlite3_wasm_vfs_create_file() for this
+ //purpose but its interface and name are still under
+ //consideration.
if(sqlite3.oo1){
opfsUtil.OpfsDb = function(...args){
diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js
index c208932e17..8e60969bc6 100644
--- a/ext/wasm/api/sqlite3-opfs-async-proxy.js
+++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js
@@ -29,6 +29,22 @@
This file represents an implementation detail of a larger piece of
code, and not a public interface. Its details may change at any time
and are not intended to be used by any client-level code.
+
+ 2022-11-27: Chrome v108 changes some async methods to synchronous, as
+ documented at:
+
+ https://developer.chrome.com/blog/sync-methods-for-accesshandles/
+
+ We cannot change to the sync forms at this point without breaking
+ clients who use Chrome v104-ish or higher. truncate(), getSize(),
+ flush(), and close() are now (as of v108) synchronous. Calling them
+ with an "await", as we have to for the async forms, is still legal
+ with the sync forms but is superfluous. Calling the async forms with
+ theFunc().then(...) is not compatible with the change to
+ synchronous, but we do do not use those APIs that way. i.e. we don't
+ _need_ to change anything for this, but at some point (after Chrome
+ versions (approximately) 104-107 are extinct) should change our
+ usage of those methods to remove the "await".
*/
"use strict";
const toss = function(...args){throw new Error(args.join(' '))};
@@ -105,7 +121,7 @@ metrics.dump = ()=>{
*/
const __openFiles = Object.create(null);
/**
- __autoLocks is a Set of sqlite3_file pointers (integers) which were
+ __implicitLocks is a Set of sqlite3_file pointers (integers) which were
"auto-locked". i.e. those for which we obtained a sync access
handle without an explicit xLock() call. Such locks will be
released during db connection idle time, whereas a sync access
@@ -117,7 +133,7 @@ const __openFiles = Object.create(null);
penalty: speedtest1 benchmarks take up to 4x as long. By delaying
the lock release until idle time, the hit is negligible.
*/
-const __autoLocks = new Set();
+const __implicitLocks = new Set();
/**
Expects an OPFS file path. It gets resolved, such that ".."
@@ -166,7 +182,7 @@ const closeSyncHandle = async (fh)=>{
const h = fh.syncHandle;
delete fh.syncHandle;
delete fh.xLock;
- __autoLocks.delete(fh.fid);
+ __implicitLocks.delete(fh.fid);
return h.close();
}
};
@@ -190,10 +206,10 @@ const closeSyncHandleNoThrow = async (fh)=>{
};
/* Release all auto-locks. */
-const closeAutoLocks = async ()=>{
- if(__autoLocks.size){
+const releaseImplicitLocks = async ()=>{
+ if(__implicitLocks.size){
/* Release all auto-locks. */
- for(const fid of __autoLocks){
+ for(const fid of __implicitLocks){
const fh = __openFiles[fid];
await closeSyncHandleNoThrow(fh);
log("Auto-unlocked",fid,fh.filenameAbs);
@@ -201,6 +217,20 @@ const closeAutoLocks = async ()=>{
}
};
+/**
+ An experiment in improving concurrency by freeing up implicit locks
+ sooner. This is known to impact performance dramatically but it has
+ also shown to improve concurrency considerably.
+
+ If fh.releaseImplicitLocks is truthy and fh is in __implicitLocks,
+ this routine returns closeSyncHandleNoThrow(), else it is a no-op.
+*/
+const releaseImplicitLock = async (fh)=>{
+ if(fh.releaseImplicitLocks && __implicitLocks.has(fh.fid)){
+ return closeSyncHandleNoThrow(fh);
+ }
+};
+
/**
An error class specifically for use with getSyncHandle(), the goal
of which is to eventually be able to distinguish unambiguously
@@ -246,7 +276,7 @@ GetSyncHandleError.convertRc = (e,rc)=>{
still fails at that point it will give up and propagate the
exception.
*/
-const getSyncHandle = async (fh)=>{
+const getSyncHandle = async (fh,opName)=>{
if(!fh.syncHandle){
const t = performance.now();
log("Acquiring sync handle for",fh.filenameAbs);
@@ -262,20 +292,21 @@ const getSyncHandle = async (fh)=>{
}catch(e){
if(i === maxTries){
throw new GetSyncHandleError(
- e, "Error getting sync handle.",maxTries,
+ e, "Error getting sync handle for",opName+"().",maxTries,
"attempts failed.",fh.filenameAbs
);
}
- warn("Error getting sync handle. Waiting",ms,
+ warn("Error getting sync handle for",opName+"(). Waiting",ms,
"ms and trying again.",fh.filenameAbs,e);
- await closeAutoLocks();
+ //await releaseImplicitLocks();
Atomics.wait(state.sabOPView, state.opIds.retry, 0, ms);
}
}
- log("Got sync handle for",fh.filenameAbs,'in',performance.now() - t,'ms');
+ log("Got",opName+"() sync handle for",fh.filenameAbs,
+ 'in',performance.now() - t,'ms');
if(!fh.xLock){
- __autoLocks.add(fh.fid);
- log("Auto-locked",fh.fid,fh.filenameAbs);
+ __implicitLocks.add(fh.fid);
+ log("Auto-locked for",opName+"()",fh.fid,fh.filenameAbs);
}
}
return fh.syncHandle;
@@ -409,7 +440,7 @@ const vfsAsyncImpls = {
xClose: async function(fid/*sqlite3_file pointer*/){
const opName = 'xClose';
mTimeStart(opName);
- __autoLocks.delete(fid);
+ __implicitLocks.delete(fid);
const fh = __openFiles[fid];
let rc = 0;
wTimeStart(opName);
@@ -474,13 +505,14 @@ const vfsAsyncImpls = {
wTimeStart('xFileSize');
try{
affirmLocked('xFileSize',fh);
- const sz = await (await getSyncHandle(fh)).getSize();
+ const sz = await (await getSyncHandle(fh,'xFileSize')).getSize();
state.s11n.serialize(Number(sz));
rc = 0;
}catch(e){
state.s11n.storeException(2,e);
rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR);
}
+ await releaseImplicitLock(fh);
wTimeEnd();
storeAndNotify('xFileSize', rc);
mTimeEnd();
@@ -495,8 +527,8 @@ const vfsAsyncImpls = {
if( !fh.syncHandle ){
wTimeStart('xLock');
try {
- await getSyncHandle(fh);
- __autoLocks.delete(fid);
+ await getSyncHandle(fh,'xLock');
+ __implicitLocks.delete(fid);
}catch(e){
state.s11n.storeException(1,e);
rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_LOCK);
@@ -508,10 +540,10 @@ const vfsAsyncImpls = {
mTimeEnd();
},
xOpen: async function(fid/*sqlite3_file pointer*/, filename,
- flags/*SQLITE_OPEN_...*/){
+ flags/*SQLITE_OPEN_...*/,
+ opfsFlags/*OPFS_...*/){
const opName = 'xOpen';
mTimeStart(opName);
- const deleteOnClose = (state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags);
const create = (state.sq3Codes.SQLITE_OPEN_CREATE & flags);
wTimeStart('xOpen');
try{
@@ -526,14 +558,8 @@ const vfsAsyncImpls = {
return;
}
const hFile = await hDir.getFileHandle(filenamePart, {create});
- /**
- wa-sqlite, at this point, grabs a SyncAccessHandle and
- assigns it to the syncHandle prop of the file state
- object, but only for certain cases and it's unclear why it
- places that limitation on it.
- */
wTimeEnd();
- __openFiles[fid] = Object.assign(Object.create(null),{
+ const fh = Object.assign(Object.create(null),{
fid: fid,
filenameAbs: filename,
filenamePart: filenamePart,
@@ -542,8 +568,26 @@ const vfsAsyncImpls = {
sabView: state.sabFileBufView,
readOnly: create
? false : (state.sq3Codes.SQLITE_OPEN_READONLY & flags),
- deleteOnClose: deleteOnClose
+ deleteOnClose: !!(state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags)
});
+ fh.releaseImplicitLocks =
+ (opfsFlags & state.opfsFlags.OPFS_UNLOCK_ASAP)
+ || state.opfsFlags.defaultUnlockAsap;
+ if(0 /* this block is modelled after something wa-sqlite
+ does but it leads to immediate contention on journal files. */
+ && (0===(flags & state.sq3Codes.SQLITE_OPEN_MAIN_DB))){
+ /* sqlite does not lock these files, so go ahead and grab an OPFS
+ lock.
+
+ https://www.sqlite.org/uri.html
+ */
+ fh.xLock = "xOpen"/* Truthy value to keep entry from getting
+ flagged as auto-locked. String value so
+ that we can easily distinguish is later
+ if needed. */;
+ await getSyncHandle(fh,'xOpen');
+ }
+ __openFiles[fid] = fh;
storeAndNotify(opName, 0);
}catch(e){
wTimeEnd();
@@ -560,7 +604,7 @@ const vfsAsyncImpls = {
try{
affirmLocked('xRead',fh);
wTimeStart('xRead');
- nRead = (await getSyncHandle(fh)).read(
+ nRead = (await getSyncHandle(fh,'xRead')).read(
fh.sabView.subarray(0, n),
{at: Number(offset64)}
);
@@ -575,6 +619,7 @@ const vfsAsyncImpls = {
state.s11n.storeException(1,e);
rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_READ);
}
+ await releaseImplicitLock(fh);
storeAndNotify('xRead',rc);
mTimeEnd();
},
@@ -603,12 +648,13 @@ const vfsAsyncImpls = {
try{
affirmLocked('xTruncate',fh);
affirmNotRO('xTruncate', fh);
- await (await getSyncHandle(fh)).truncate(size);
+ await (await getSyncHandle(fh,'xTruncate')).truncate(size);
}catch(e){
error("xTruncate():",e,fh);
state.s11n.storeException(2,e);
rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_TRUNCATE);
}
+ await releaseImplicitLock(fh);
wTimeEnd();
storeAndNotify('xTruncate',rc);
mTimeEnd();
@@ -640,7 +686,7 @@ const vfsAsyncImpls = {
affirmLocked('xWrite',fh);
affirmNotRO('xWrite', fh);
rc = (
- n === (await getSyncHandle(fh))
+ n === (await getSyncHandle(fh,'xWrite'))
.write(fh.sabView.subarray(0, n),
{at: Number(offset64)})
) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE;
@@ -649,6 +695,7 @@ const vfsAsyncImpls = {
state.s11n.storeException(1,e);
rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_WRITE);
}
+ await releaseImplicitLock(fh);
wTimeEnd();
storeAndNotify('xWrite',rc);
mTimeEnd();
@@ -777,13 +824,13 @@ const waitLoop = async function f(){
to do other things. If this is too high (e.g. 500ms) then
even two workers/tabs can easily run into locking errors.
*/
- const waitTime = 150;
+ const waitTime = 100;
while(!flagAsyncShutdown){
try {
if('timed-out'===Atomics.wait(
state.sabOPView, state.opIds.whichOp, 0, waitTime
)){
- await closeAutoLocks();
+ await releaseImplicitLocks();
continue;
}
const opId = Atomics.load(state.sabOPView, state.opIds.whichOp);
@@ -824,6 +871,7 @@ navigator.storage.getDirectory().then(function(d){
state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize);
state.opIds = opt.opIds;
state.sq3Codes = opt.sq3Codes;
+ state.opfsFlags = opt.opfsFlags;
Object.keys(vfsAsyncImpls).forEach((k)=>{
if(!Number.isFinite(state.opIds[k])){
toss("Maintenance required: missing state.opIds[",k,"]");
diff --git a/ext/wasm/tests/opfs/concurrency/index.html b/ext/wasm/tests/opfs/concurrency/index.html
index a082dfe997..e19f6a8da6 100644
--- a/ext/wasm/tests/opfs/concurrency/index.html
+++ b/ext/wasm/tests/opfs/concurrency/index.html
@@ -24,10 +24,13 @@
URL flags: pass a number of workers using
- the workers=N URL flag and the worker work interval
- as interval=N (milliseconds). Enable OPFS VFS
- verbosity with verbose=1-3 (output goes to the
- dev console).
+ the workers=N URL flag. Set the time between each
+ workload with interval=N (milliseconds). Set the
+ number of worker iterations with iterations=N.
+ Enable OPFS VFS verbosity with verbose=1-3 (output
+ goes to the dev console). Enable/disable "unlock ASAP" mode
+ (higher concurrency, lower speed)
+ with unlock-asap=0-1.
Achtung: if it does not start to do anything within a couple of
seconds, check the dev console: Chrome often fails with "cannot allocate
diff --git a/ext/wasm/tests/opfs/concurrency/test.js b/ext/wasm/tests/opfs/concurrency/test.js
index 27bc47b19d..044d343745 100644
--- a/ext/wasm/tests/opfs/concurrency/test.js
+++ b/ext/wasm/tests/opfs/concurrency/test.js
@@ -56,26 +56,42 @@
options.sqlite3Dir = urlArgsJs.get('sqlite3.dir');
options.workerCount = (
urlArgsHtml.has('workers') ? +urlArgsHtml.get('workers') : 3
- ) || 3;
+ ) || 4;
options.opfsVerbose = (
urlArgsHtml.has('verbose') ? +urlArgsHtml.get('verbose') : 1
) || 1;
options.interval = (
urlArgsHtml.has('interval') ? +urlArgsHtml.get('interval') : 750
- ) || 750;
+ ) || 1000;
+ options.iterations = (
+ urlArgsHtml.has('iterations') ? +urlArgsHtml.get('iterations') : 10
+ ) || 10;
+ options.unlockAsap = (
+ urlArgsHtml.has('unlock-asap') ? +urlArgsHtml.get('unlock-asap') : 0
+ ) || 0;
const workers = [];
workers.post = (type,...args)=>{
for(const w of workers) w.postMessage({type, payload:args});
};
- workers.loadedCount = 0;
+ workers.counts = {loaded: 0, passed: 0, failed: 0};
+ const checkFinished = function(){
+ if(workers.counts.passed + workers.counts.failed !== workers.length){
+ return;
+ }
+ if(workers.counts.failed>0){
+ logCss('tests-fail',"Finished with",workers.counts.failed,"failure(s).");
+ }else{
+ logCss('tests-pass',"All",workers.length,"workers finished.");
+ }
+ };
workers.onmessage = function(msg){
msg = msg.data;
const prefix = 'Worker #'+msg.worker+':';
switch(msg.type){
case 'loaded':
stdout(prefix,"loaded");
- if(++workers.loadedCount === workers.length){
- stdout("All workers loaded. Telling them to run...");
+ if(++workers.counts.loaded === workers.length){
+ stdout("All",workers.length,"workers loaded. Telling them to run...");
workers.post('run');
}
break;
@@ -83,21 +99,27 @@
case 'stderr': stderr(prefix,...msg.payload); break;
case 'error': stderr(prefix,"ERROR:",...msg.payload); break;
case 'finished':
+ ++workers.counts.passed;
logCss('tests-pass',prefix,...msg.payload);
+ checkFinished();
break;
case 'failed':
+ ++workers.counts.failed;
logCss('tests-fail',prefix,"FAILED:",...msg.payload);
+ checkFinished();
break;
default: logCss('error',"Unhandled message type:",msg); break;
}
};
- stdout("Launching",options.workerCount,"workers...");
+ stdout("Launching",options.workerCount,"workers. Options:",options);
workers.uri = (
'worker.js?'
+ 'sqlite3.dir='+options.sqlite3Dir
+ '&interval='+options.interval
+ + '&iterations='+options.iterations
+ '&opfs-verbose='+options.opfsVerbose
+ + '&opfs-unlock-asap='+options.unlockAsap
);
for(let i = 0; i < options.workerCount; ++i){
stdout("Launching worker...");
diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js
index 19b0a068e7..95fefa1954 100644
--- a/ext/wasm/tests/opfs/concurrency/worker.js
+++ b/ext/wasm/tests/opfs/concurrency/worker.js
@@ -3,9 +3,12 @@ importScripts(
);
self.sqlite3InitModule().then(async function(sqlite3){
const urlArgs = new URL(self.location.href).searchParams;
- const wName = urlArgs.get('workerId') || Math.round(Math.random()*10000);
+ const options = {
+ workerName: urlArgs.get('workerId') || Math.round(Math.random()*10000),
+ unlockAsap: urlArgs.get('opfs-unlock-asap') || 0 /*EXPERIMENTAL*/
+ };
const wPost = (type,...payload)=>{
- postMessage({type, worker: wName, payload});
+ postMessage({type, worker: options.workerName, payload});
};
const stdout = (...args)=>wPost('stdout',...args);
const stderr = (...args)=>wPost('stderr',...args);
@@ -43,7 +46,10 @@ self.sqlite3InitModule().then(async function(sqlite3){
}
};
const run = async function(){
- db = new sqlite3.opfs.OpfsDb(dbName,'c');
+ db = new sqlite3.opfs.OpfsDb({
+ filename: 'file:'+dbName+'?opfs-unlock-asap='+options.unlockAsap,
+ flags: 'c'
+ });
sqlite3.capi.sqlite3_busy_timeout(db.pointer, 5000);
db.transaction((db)=>{
db.exec([
@@ -52,7 +58,8 @@ self.sqlite3InitModule().then(async function(sqlite3){
]);
});
- const maxIterations = 10;
+ const maxIterations =
+ urlArgs.has('iterations') ? (+urlArgs.get('iterations') || 10) : 10;
stdout("Starting interval-based db updates with delay of",interval.delay,"ms.");
const doWork = async ()=>{
const tm = new Date().getTime();
@@ -62,7 +69,7 @@ self.sqlite3InitModule().then(async function(sqlite3){
try{
db.exec({
sql:"INSERT OR REPLACE INTO t1(w,v) VALUES(?,?)",
- bind: [wName, new Date().getTime()]
+ bind: [options.workerName, new Date().getTime()]
});
//stdout("Set",prefix);
}catch(e){
diff --git a/manifest b/manifest
index 10cb5bf04e..5db19a5f4b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Speed\sup\sbase85()\sconversions\sand\ssync\sw/trunk.
-D 2022-11-24T20:11:34.749
+C Sync\sw/trunk,\szap\ssurplus\sspace.
+D 2022-11-28T14:11:48.108
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -289,9 +289,9 @@ F ext/misc/README.md d6dd0fe1d8af77040216798a6a2b0c46c73054d2f0ea544fbbcdccf6f23
F ext/misc/amatch.c e3ad5532799cee9a97647f483f67f43b38796b84b5a8c60594fe782a4338f358
F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb
F ext/misc/appendvfs.c 9642c7a194a2a25dca7ad3e36af24a0a46d7702168c4ad7e59c9f9b0e16a3824
-F ext/misc/base64.c 8b200527ea933294d9a77e051e15d37e0c78f0a5f2ed1be50cb95fd41936c5ac x
-F ext/misc/base85.c b082f8dbfb823b479d19df1fa0f753e356d82ae51c3b6fc774252d4fd6877ee8
-F ext/misc/basexx.c 678dcc83894f78c26fd3662b322886777cc26bf2b40809236cd2abdad532a33c
+F ext/misc/base64.c fb140039d4e15710298e28cafe12d2bdff1b927fc57540cbcd07996f82bf9800 x
+F ext/misc/base85.c ace591246855806a70e63eb6705ce8736f4b782da2137bdbb5d39cc337346e0d
+F ext/misc/basexx.c b37ea21f00deea1029d2ddc670f5701d9370ae06f7d173c5bed79d0f26a67ec1
F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a
F ext/misc/btreeinfo.c d28ce349b40054eaa9473e835837bad7a71deec33ba13e39f963d50933bfa0f9
F ext/misc/carray.c b752f46411e4e47e34dce6f0c88bc8e51bb821ba9e49bfcd882506451c928f69
@@ -504,12 +504,12 @@ F ext/wasm/api/post-js-header.js d6ab3dfef4a06960d28a7eaa338d4e2a1a5981e9b387181
F ext/wasm/api/pre-js.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f9e91c1d5f
F ext/wasm/api/sqlite3-api-cleanup.js ecdc69dbfccfe26146f04799fcfd4a6f5790d46e7e3b9b6e9b0491f92ed8ae34
F ext/wasm/api/sqlite3-api-glue.js 056f44b82c126358a0175e08a892d56fadfce177b0d7a0012502a6acf67ea6d5
-F ext/wasm/api/sqlite3-api-oo1.js dec6c14994317ad0011714890426cdc211f4eab451c9766ea88c7ac4f535287e
-F ext/wasm/api/sqlite3-api-opfs.js 38d368e33f470f9ba196f1a2b0c9ce076c930c70df233c345a246f1ad4c26d3b
+F ext/wasm/api/sqlite3-api-oo1.js 06ad2079368e16cb9f182c18cd37bdc3932536856dff4f60582d0ca5f6c491a8
+F ext/wasm/api/sqlite3-api-opfs.js e98a8bd67dea8c20b0ec17332698b44658a6fbc4be18dd87fab2ce05284a10d7
F ext/wasm/api/sqlite3-api-prologue.js 7fce4c6a138ec3d7c285b7c125cee809e6b668d2cb0d2328a1b790b7037765bd
F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f
F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3
-F ext/wasm/api/sqlite3-opfs-async-proxy.js 1ec10873f1d59d305f6f3b435c50a1b75d693d5fb739b226f3da46fcbb11261a
+F ext/wasm/api/sqlite3-opfs-async-proxy.js 798383f6b46fd5dac122d6e35962d25b10401ddb825b5c66df1d21e6b1d8aacc
F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9
F ext/wasm/api/sqlite3-wasm.c 8b32787a3b6bb2990cbaba2304bd5b75a9652acbc8d29909b3279019b6cbaef5
F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b
@@ -555,9 +555,9 @@ F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d96
F ext/wasm/tester1-worker.html 5ef353348c37cf2e4fd0b23da562d3275523e036260b510734e9a3239ba8c987
F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399
F ext/wasm/tester1.c-pp.js 3b91f192c159088004fba6fe3441edea58421a8b88bccf3dd20978a077648d19
-F ext/wasm/tests/opfs/concurrency/index.html bb9b0f6da86df34c67fa506db9c45b7c4cf0045a211611cc6b8d2b53fa983481
-F ext/wasm/tests/opfs/concurrency/test.js 5993c08657d547d3a26b78ff3480122aed2b7361823bc127e96e558931093aff
-F ext/wasm/tests/opfs/concurrency/worker.js afccb78082b57edb17d5aba0754c823772553395df6f1aed92f82b4d9e3c32de
+F ext/wasm/tests/opfs/concurrency/index.html e8fec75ea6eddc600c8a382da7ea2579feece2263a2fb4417f2cf3e9d451744c
+F ext/wasm/tests/opfs/concurrency/test.js bfc3d7e27b207f0827f12568986b8d516a744529550b449314f5c21c9e9faf4a
+F ext/wasm/tests/opfs/concurrency/worker.js 0eff027cbd3a495acb2ac94f57ca9e4d21125ab9fda07d45f3701b0efe82d450
F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5
F ext/wasm/wasmfs.make 8fea9b4f3cde06141de1fc4c586ab405bd32c3f401554f4ebb18c797401a678d
F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
@@ -594,7 +594,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5
F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8
F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d
F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e
-F src/expr.c bc6527e3dff813c8102418e6e201870626a7fa5f69329ea7b082d602e7ed1cd9
+F src/expr.c 9e7fadc664b938c18f006be0d4f6669888f9302756ee204420c7eccaed5435a6
F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002
F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d
@@ -644,12 +644,12 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764
F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c
F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633
F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92
-F src/select.c 4c48373abb4e67129c36bc15d1f5a99a0dfd9534afeb539a2169a09ae91ccec9
-F src/shell.c.in be0463b4a48b5d7c87651867cd6a4b6f453e1a152481419f7936f0bee9a81c30
+F src/select.c c1eb8f3ee25152327f2e7e87db8cea549e57c104b63638bff4fc584d479c33f0
+F src/shell.c.in 9fda74d40b206a707aaa69fc5dc38e2c6a9137a3f4a1dcd7af581d59d92c063c
F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca
F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f
-F src/sqliteInt.h 6a24230f2928b3d1d9b0fdbedb98c862b828a4b1a9170306108e74cd6277f476
+F src/sqliteInt.h 5dd5d3d47f40b6a12be4a5fc131673bfe00c00373ed266ff4c4ec05d1991e69f
F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657
F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749
F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -709,8 +709,8 @@ F src/test_window.c cdae419fdcea5bad6dcd9368c685abdad6deb59e9fc8b84b153de513d394
F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
F src/tokenize.c 1305797eab3542a0896b552c6e7669c972c1468e11e92b370533c1f37a37082b
-F src/treeview.c 07787f67cd297a6d09d04b8d70c06769c60c9c1d9080378f93929c16f8fd3298
-F src/trigger.c 4163ada044af89d51caba1cb713a73165347b2ec05fe84a283737c134d61fcd5
+F src/treeview.c 29b1dc7e0f84ba090734febe27393d4719682af0cae1b902d5ebf0236ecebea4
+F src/trigger.c 5e68b790f022b8dafbfb0eb244786512a95c9575fc198719d2557d73e5795858
F src/update.c 5b0302c47cf31b533d5dff04c497ca1d8b9d89c39727e633fbe7b882fd5ac5aa
F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145
F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0
@@ -1407,7 +1407,7 @@ F test/printf.test 390d0d7fcffc3c4ea3c1bb4cbb267444e32b33b048ae21895f23a291844fe
F test/printf2.test 3f55c1871a5a65507416076f6eb97e738d5210aeda7595a74ee895f2224cce60
F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb
F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc
-F test/pushdown.test 5e72c51c5e33253ed639ccee1e01ce62d62b6eee5ca893cd82334e4ee7b1d7fc
+F test/pushdown.test c69f0970ea17e0afc674b89741f60c172cb6f761d81665fc71015f674f0f66ba
F test/queryonly.test 5f653159e0f552f0552d43259890c1089391dcca
F test/quick.test 1681febc928d686362d50057c642f77a02c62e57
F test/quota-glob.test 32901e9eed6705d68ca3faee2a06b73b57cb3c26
@@ -1425,7 +1425,7 @@ F test/reindex.test cd9d6021729910ece82267b4f5e1b5ac2911a7566c43b43c176a6a4732e2
F test/releasetest_data.tcl 0db8aee0c348090fd06da47020ab4ed8ec692e0723427b2f3947d4dfb806f3b0
F test/resetdb.test 8062cf10a09d8c048f8de7711e94571c38b38168db0e5877ba7561789e5eeb2b
F test/resolver01.test f4022acafda7f4d40eca94dbf16bc5fc4ac30ceb
-F test/returning1.test c43b8370a351f77aec6d71f4a2cde59b849369ed1933261a2c2c69e23e34ff5e
+F test/returning1.test 1366e04566cfe1a082d17b1e0f195ec64473c79374b3a5d4ae00c43d885dea31
F test/returningfault.test ae4c4b5e8745813287a359d9ccdb9d5c883c2e68afb18fb0767937d5de5692a4
F test/rollback.test 06680159bc6746d0f26276e339e3ae2f951c64812468308838e0a3362d911eaa
F test/rollback2.test 3f3a4e20401825017df7e7671e9f31b6de5fae5620c2b9b49917f52f8c160a8f
@@ -1501,7 +1501,7 @@ F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304
F test/shell1.test e4b4de56f454708e0747b52915135baa2cbfec4965406d6eaf02a4a5c22a9880
F test/shell2.test c536c2aab4852608f8a606262330797abc4d964a4c2c782a7760f54ea1f17a6a
F test/shell3.test 91febeac0412812bf6370abb8ed72700e32bf8f9878849414518f662dfd55e8a
-F test/shell4.test 7dc8a515705bc093d8ffe381670e8fa7a969661e8ed177c35c847e3c6dfc35e2
+F test/shell4.test 9abd0c12a7e20a4c49e84d5be208d2124fa6c09e728f56f1f4bee0f02853935f
F test/shell5.test c8b6c54f26ec537f8558273d7ed293ca3725ef42e6b12b8f151718628bd1473b
F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3
F test/shell7.test 115132f66d0463417f408562cc2cf534f6bbc6d83a6d50f0072a9eb171bae97f
@@ -1632,6 +1632,7 @@ F test/tkt-868145d012.test a5f941107ece6a64410ca4755c6329b7eb57a356
F test/tkt-8c63ff0ec.test 258b7fc8d7e4e1cb5362c7d65c143528b9c4cbed
F test/tkt-91e2e8ba6f.test 08c4f94ae07696b05c9b822da0b4e5337a2f54c5
F test/tkt-94c04eaadb.test f738c57c7f68ab8be1c054415af7774617cb6223
+F test/tkt-99378177930f87bd.test 0f932e85fa1d41f30532cb7be9718d82e491e953123b8c4c85cf025f36ffe34b
F test/tkt-9a8b09f8e6.test b2ef151d0984b2ebf237760dbeaa50724e5a0667
F test/tkt-9d68c883.test 16f7cb96781ba579bc2e19bb14b4ad609d9774b6
F test/tkt-9f2eb3abac.test cb6123ac695a08b4454c3792fbe85108f67fabf8
@@ -2063,8 +2064,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6c84ae4ba83713c751fddff8be41686bbcb525ac8135e1520436c62d0bc23d2c a2b6883ac2ef878f525ee847b170beb9f9ab9d1fa67557101be2cdae1e7f7a57
-R 637b81d4c3e7af813b40107004ae4ce7
+P 17b823500a2ed135c1f40aa7f4d87ba5b2eab35c0abd9e0856041cf0f510cbee a2449bcc2c71d0f4c3289621fbf1cb97f0f407c9f7b5bf18245b7854a07c6cfa
+R b25d12f698cc565856f0ef09be44e05e
U larrybr
-Z 5b90ebbf3db611ecc14a602bf84a6a56
+Z cf8d223b7c2dae4330a88af45b5f7bd0
# Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 30ba16d8f0..b04bf1863d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-17b823500a2ed135c1f40aa7f4d87ba5b2eab35c0abd9e0856041cf0f510cbee
\ No newline at end of file
+b8345630a2a322234bda49ee4b996f6ba20e2b080621e229a2ec5e820892a663
\ No newline at end of file
diff --git a/src/expr.c b/src/expr.c
index fb718eb23e..d30ae97668 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -53,7 +53,7 @@ char sqlite3ExprAffinity(const Expr *pExpr){
}
op = pExpr->op;
if( op==TK_REGISTER ) op = pExpr->op2;
- if( op==TK_COLUMN || op==TK_AGG_COLUMN ){
+ if( op==TK_COLUMN || (op==TK_AGG_COLUMN && pExpr->y.pTab!=0) ){
assert( ExprUseYTab(pExpr) );
assert( pExpr->y.pTab!=0 );
return sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn);
@@ -173,7 +173,9 @@ CollSeq *sqlite3ExprCollSeq(Parse *pParse, const Expr *pExpr){
while( p ){
int op = p->op;
if( op==TK_REGISTER ) op = p->op2;
- if( op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_TRIGGER ){
+ if( (op==TK_AGG_COLUMN && p->y.pTab!=0)
+ || op==TK_COLUMN || op==TK_TRIGGER
+ ){
int j;
assert( ExprUseYTab(p) );
assert( p->y.pTab!=0 );
@@ -4128,15 +4130,16 @@ expr_code_doover:
assert( pExpr->iAgg>=0 && pExpr->iAggnColumn );
pCol = &pAggInfo->aCol[pExpr->iAgg];
if( !pAggInfo->directMode ){
- assert( pCol->iMem>0 );
- return pCol->iMem;
+ return AggInfoColumnReg(pAggInfo, pExpr->iAgg);
}else if( pAggInfo->useSortingIdx ){
Table *pTab = pCol->pTab;
sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdxPTab,
pCol->iSorterColumn, target);
- if( pCol->iColumn<0 ){
+ if( pTab==0 ){
+ /* No comment added */
+ }else if( pCol->iColumn<0 ){
VdbeComment((v,"%s.rowid",pTab->zName));
- }else if( ALWAYS(pTab!=0) ){
+ }else{
VdbeComment((v,"%s.%s",
pTab->zName, pTab->aCol[pCol->iColumn].zCnName));
if( pTab->aCol[pCol->iColumn].affinity==SQLITE_AFF_REAL ){
@@ -4144,6 +4147,11 @@ expr_code_doover:
}
}
return target;
+ }else if( pExpr->y.pTab==0 ){
+ /* This case happens when the argument to an aggregate function
+ ** is rewritten by aggregateConvertIndexedExprRefToColumn() */
+ sqlite3VdbeAddOp3(v, OP_Column, pExpr->iTable, pExpr->iColumn, target);
+ return target;
}
/* Otherwise, fall thru into the TK_COLUMN case */
/* no break */ deliberate_fall_through
@@ -4441,7 +4449,7 @@ expr_code_doover:
assert( !ExprHasProperty(pExpr, EP_IntValue) );
sqlite3ErrorMsg(pParse, "misuse of aggregate: %#T()", pExpr);
}else{
- return pInfo->aFunc[pExpr->iAgg].iMem;
+ return AggInfoFuncReg(pInfo, pExpr->iAgg);
}
break;
}
@@ -4730,7 +4738,7 @@ expr_code_doover:
if( pAggInfo ){
assert( pExpr->iAgg>=0 && pExpr->iAggnColumn );
if( !pAggInfo->directMode ){
- inReg = pAggInfo->aCol[pExpr->iAgg].iMem;
+ inReg = AggInfoColumnReg(pAggInfo, pExpr->iAgg);
break;
}
if( pExpr->pAggInfo->useSortingIdx ){
@@ -6242,6 +6250,72 @@ static int addAggInfoFunc(sqlite3 *db, AggInfo *pInfo){
return i;
}
+/*
+** Search the AggInfo object for an aCol[] entry that has iTable and iColumn.
+** Return the index in aCol[] of the entry that describes that column.
+**
+** If no prior entry is found, create a new one and return -1. The
+** new column will have an idex of pAggInfo->nColumn-1.
+*/
+static void findOrCreateAggInfoColumn(
+ Parse *pParse, /* Parsing context */
+ AggInfo *pAggInfo, /* The AggInfo object to search and/or modify */
+ Expr *pExpr /* Expr describing the column to find or insert */
+){
+ struct AggInfo_col *pCol;
+ int k;
+
+ assert( pAggInfo->iFirstReg==0 );
+ pCol = pAggInfo->aCol;
+ for(k=0; knColumn; k++, pCol++){
+ if( pCol->iTable==pExpr->iTable
+ && pCol->iColumn==pExpr->iColumn
+ && pExpr->op!=TK_IF_NULL_ROW
+ ){
+ goto fix_up_expr;
+ }
+ }
+ k = addAggInfoColumn(pParse->db, pAggInfo);
+ if( k<0 ){
+ /* OOM on resize */
+ assert( pParse->db->mallocFailed );
+ return;
+ }
+ pCol = &pAggInfo->aCol[k];
+ assert( ExprUseYTab(pExpr) );
+ pCol->pTab = pExpr->y.pTab;
+ pCol->iTable = pExpr->iTable;
+ pCol->iColumn = pExpr->iColumn;
+ pCol->iSorterColumn = -1;
+ pCol->pCExpr = pExpr;
+ if( pAggInfo->pGroupBy && pExpr->op!=TK_IF_NULL_ROW ){
+ int j, n;
+ ExprList *pGB = pAggInfo->pGroupBy;
+ struct ExprList_item *pTerm = pGB->a;
+ n = pGB->nExpr;
+ for(j=0; jpExpr;
+ if( pE->op==TK_COLUMN
+ && pE->iTable==pExpr->iTable
+ && pE->iColumn==pExpr->iColumn
+ ){
+ pCol->iSorterColumn = j;
+ break;
+ }
+ }
+ }
+ if( pCol->iSorterColumn<0 ){
+ pCol->iSorterColumn = pAggInfo->nSortingColumn++;
+ }
+fix_up_expr:
+ ExprSetVVAProperty(pExpr, EP_NoReduce);
+ pExpr->pAggInfo = pAggInfo;
+ if( pExpr->op==TK_COLUMN ){
+ pExpr->op = TK_AGG_COLUMN;
+ }
+ pExpr->iAgg = (i16)k;
+}
+
/*
** This is the xExprCallback for a tree walker. It is used to
** implement sqlite3ExprAnalyzeAggregates(). See sqlite3ExprAnalyzeAggregates
@@ -6255,7 +6329,36 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){
AggInfo *pAggInfo = pNC->uNC.pAggInfo;
assert( pNC->ncFlags & NC_UAggInfo );
+ assert( pAggInfo->iFirstReg==0 );
switch( pExpr->op ){
+ default: {
+ IndexedExpr *pIEpr;
+ Expr tmp;
+ assert( pParse->iSelfTab==0 );
+ if( (pNC->ncFlags & NC_InAggFunc)==0 ) break;
+ if( pParse->pIdxEpr==0 ) break;
+ for(pIEpr=pParse->pIdxEpr; pIEpr; pIEpr=pIEpr->pIENext){
+ int iDataCur = pIEpr->iDataCur;
+ if( iDataCur<0 ) continue;
+ if( sqlite3ExprCompare(0, pExpr, pIEpr->pExpr, iDataCur)==0 ) break;
+ }
+ if( pIEpr==0 ) break;
+ if( NEVER(!ExprUseYTab(pExpr)) ) break;
+
+ /* If we reach this point, it means that expression pExpr can be
+ ** translated into a reference to an index column as described by
+ ** pIEpr.
+ */
+ memset(&tmp, 0, sizeof(tmp));
+ tmp.op = TK_AGG_COLUMN;
+ tmp.iTable = pIEpr->iIdxCur;
+ tmp.iColumn = pIEpr->iIdxCol;
+ findOrCreateAggInfoColumn(pParse, pAggInfo, &tmp);
+ pAggInfo->aCol[tmp.iAgg].pCExpr = pExpr;
+ pExpr->pAggInfo = pAggInfo;
+ pExpr->iAgg = tmp.iAgg;
+ return WRC_Prune;
+ }
case TK_IF_NULL_ROW:
case TK_AGG_COLUMN:
case TK_COLUMN: {
@@ -6267,67 +6370,9 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){
if( ALWAYS(pSrcList!=0) ){
SrcItem *pItem = pSrcList->a;
for(i=0; inSrc; i++, pItem++){
- struct AggInfo_col *pCol;
assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) );
if( pExpr->iTable==pItem->iCursor ){
- /* If we reach this point, it means that pExpr refers to a table
- ** that is in the FROM clause of the aggregate query.
- **
- ** Make an entry for the column in pAggInfo->aCol[] if there
- ** is not an entry there already.
- */
- int k;
- pCol = pAggInfo->aCol;
- for(k=0; knColumn; k++, pCol++){
- if( pCol->iTable==pExpr->iTable
- && pCol->iColumn==pExpr->iColumn
- && pExpr->op!=TK_IF_NULL_ROW
- ){
- break;
- }
- }
- if( (k>=pAggInfo->nColumn)
- && (k = addAggInfoColumn(pParse->db, pAggInfo))>=0
- ){
- pCol = &pAggInfo->aCol[k];
- assert( ExprUseYTab(pExpr) );
- pCol->pTab = pExpr->y.pTab;
- pCol->iTable = pExpr->iTable;
- pCol->iColumn = pExpr->iColumn;
- pCol->iMem = ++pParse->nMem;
- pCol->iSorterColumn = -1;
- pCol->pCExpr = pExpr;
- if( pAggInfo->pGroupBy && pExpr->op!=TK_IF_NULL_ROW ){
- int j, n;
- ExprList *pGB = pAggInfo->pGroupBy;
- struct ExprList_item *pTerm = pGB->a;
- n = pGB->nExpr;
- for(j=0; jpExpr;
- if( pE->op==TK_COLUMN
- && pE->iTable==pExpr->iTable
- && pE->iColumn==pExpr->iColumn
- ){
- pCol->iSorterColumn = j;
- break;
- }
- }
- }
- if( pCol->iSorterColumn<0 ){
- pCol->iSorterColumn = pAggInfo->nSortingColumn++;
- }
- }
- /* There is now an entry for pExpr in pAggInfo->aCol[] (either
- ** because it was there before or because we just created it).
- ** Convert the pExpr to be a TK_AGG_COLUMN referring to that
- ** pAggInfo->aCol[] entry.
- */
- ExprSetVVAProperty(pExpr, EP_NoReduce);
- pExpr->pAggInfo = pAggInfo;
- if( pExpr->op==TK_COLUMN ){
- pExpr->op = TK_AGG_COLUMN;
- }
- pExpr->iAgg = (i16)k;
+ findOrCreateAggInfoColumn(pParse, pAggInfo, pExpr);
break;
} /* endif pExpr->iTable==pItem->iCursor */
} /* end loop over pSrcList */
@@ -6357,7 +6402,6 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){
assert( !ExprHasProperty(pExpr, EP_xIsSelect) );
pItem = &pAggInfo->aFunc[i];
pItem->pFExpr = pExpr;
- pItem->iMem = ++pParse->nMem;
assert( ExprUseUToken(pExpr) );
pItem->pFunc = sqlite3FindFunction(pParse->db,
pExpr->u.zToken,
diff --git a/src/select.c b/src/select.c
index 4d5bde9a31..10498310d4 100644
--- a/src/select.c
+++ b/src/select.c
@@ -4050,6 +4050,34 @@ static ExprList *findLeftmostExprlist(Select *pSel){
return pSel->pEList;
}
+/*
+** Return true if any of the result-set columns in the compound query
+** have incompatible affinities on one or more arms of the compound.
+*/
+static int compoundHasDifferentAffinities(Select *p){
+ int ii;
+ ExprList *pList;
+ assert( p!=0 );
+ assert( p->pEList!=0 );
+ assert( p->pPrior!=0 );
+ pList = p->pEList;
+ for(ii=0; iinExpr; ii++){
+ char aff;
+ Select *pSub1;
+ assert( pList->a[ii].pExpr!=0 );
+ aff = sqlite3ExprAffinity(pList->a[ii].pExpr);
+ for(pSub1=p->pPrior; pSub1; pSub1=pSub1->pPrior){
+ assert( pSub1->pEList!=0 );
+ assert( pSub1->pEList->nExpr>ii );
+ assert( pSub1->pEList->a[ii].pExpr!=0 );
+ if( sqlite3ExprAffinity(pSub1->pEList->a[ii].pExpr)!=aff ){
+ return 1;
+ }
+ }
+ }
+ return 0;
+}
+
#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW)
/*
** This routine attempts to flatten subqueries as a performance optimization.
@@ -4153,7 +4181,8 @@ static ExprList *findLeftmostExprlist(Select *pSel){
** query or there are no RIGHT or FULL JOINs in any arm
** of the subquery. (This is a duplicate of condition (27b).)
** (17h) The corresponding result set expressions in all arms of the
-** compound must have the same affinity.
+** compound must have the same affinity. (See restriction (9)
+** on the push-down optimization.)
**
** The parent and sub-query may contain WHERE clauses. Subject to
** rules (11), (13) and (14), they may also contain ORDER BY,
@@ -4372,19 +4401,7 @@ static int flattenSubquery(
if( (p->selFlags & SF_Recursive) ) return 0;
/* Restriction (17h) */
- for(ii=0; iipEList->nExpr; ii++){
- char aff;
- assert( pSub->pEList->a[ii].pExpr!=0 );
- aff = sqlite3ExprAffinity(pSub->pEList->a[ii].pExpr);
- for(pSub1=pSub->pPrior; pSub1; pSub1=pSub1->pPrior){
- assert( pSub1->pEList!=0 );
- assert( pSub1->pEList->nExpr>ii );
- assert( pSub1->pEList->a[ii].pExpr!=0 );
- if( sqlite3ExprAffinity(pSub1->pEList->a[ii].pExpr)!=aff ){
- return 0;
- }
- }
- }
+ if( compoundHasDifferentAffinities(pSub) ) return 0;
if( pSrc->nSrc>1 ){
if( pParse->nSelect>500 ) return 0;
@@ -5029,12 +5046,14 @@ static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){
** be materialized. (This restriction is implemented in the calling
** routine.)
**
-** (8) The subquery may not be a compound that uses UNION, INTERSECT,
-** or EXCEPT. (We could, perhaps, relax this restriction to allow
-** this case if none of the comparisons operators between left and
-** right arms of the compound use a collation other than BINARY.
-** But it is a lot of work to check that case for an obscure and
-** minor optimization, so we omit it for now.)
+** (8) If the subquery is a compound that uses UNION, INTERSECT,
+** or EXCEPT, then all of the result set columns for all arms of
+** the compound must use the BINARY collating sequence.
+**
+** (9) If the subquery is a compound, then all arms of the compound must
+** have the same affinity. (This is the same as restriction (17h)
+** for query flattening.)
+**
**
** Return 0 if no changes are made and non-zero if one or more WHERE clause
** terms are duplicated into the subquery.
@@ -5051,20 +5070,44 @@ static int pushDownWhereTerms(
if( pSubq->selFlags & (SF_Recursive|SF_MultiPart) ) return 0;
if( pSrc->fg.jointype & (JT_LTORJ|JT_RIGHT) ) return 0;
-#ifndef SQLITE_OMIT_WINDOWFUNC
if( pSubq->pPrior ){
Select *pSel;
+ int notUnionAll = 0;
for(pSel=pSubq; pSel; pSel=pSel->pPrior){
u8 op = pSel->op;
assert( op==TK_ALL || op==TK_SELECT
|| op==TK_UNION || op==TK_INTERSECT || op==TK_EXCEPT );
- if( op!=TK_ALL && op!=TK_SELECT ) return 0; /* restriction (8) */
+ if( op!=TK_ALL && op!=TK_SELECT ){
+ notUnionAll = 1;
+ }
+#ifndef SQLITE_OMIT_WINDOWFUNC
if( pSel->pWin ) return 0; /* restriction (6b) */
+#endif
+ }
+ if( compoundHasDifferentAffinities(pSubq) ){
+ return 0; /* restriction (9) */
+ }
+ if( notUnionAll ){
+ /* If any of the compound arms are connected using UNION, INTERSECT,
+ ** or EXCEPT, then we must ensure that none of the columns use a
+ ** non-BINARY collating sequence. */
+ for(pSel=pSubq; pSel; pSel=pSel->pPrior){
+ int ii;
+ const ExprList *pList = pSel->pEList;
+ assert( pList!=0 );
+ for(ii=0; iinExpr; ii++){
+ CollSeq *pColl = sqlite3ExprCollSeq(pParse, pList->a[ii].pExpr);
+ if( !sqlite3IsBinary(pColl) ){
+ return 0; /* Restriction (8) */
+ }
+ }
+ }
}
}else{
+#ifndef SQLITE_OMIT_WINDOWFUNC
if( pSubq->pWin && pSubq->pWin->pPartition==0 ) return 0;
- }
#endif
+ }
#ifdef SQLITE_DEBUG
/* Only the first term of a compound can have a WITH clause. But make
@@ -6204,6 +6247,172 @@ void sqlite3SelectPrep(
sqlite3SelectAddTypeInfo(pParse, p);
}
+#if TREETRACE_ENABLED
+/*
+** Display all information about an AggInfo object
+*/
+static void printAggInfo(AggInfo *pAggInfo){
+ int ii;
+ for(ii=0; iinColumn; ii++){
+ struct AggInfo_col *pCol = &pAggInfo->aCol[ii];
+ sqlite3DebugPrintf(
+ "agg-column[%d] pTab=%s iTable=%d iColumn=%d iMem=%d"
+ " iSorterColumn=%d %s\n",
+ ii, pCol->pTab ? pCol->pTab->zName : "NULL",
+ pCol->iTable, pCol->iColumn, pAggInfo->iFirstReg+ii,
+ pCol->iSorterColumn,
+ ii>=pAggInfo->nAccumulator ? "" : " Accumulator");
+ sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0);
+ }
+ for(ii=0; iinFunc; ii++){
+ sqlite3DebugPrintf("agg-func[%d]: iMem=%d\n",
+ ii, AggInfoFuncReg(pAggInfo,ii));
+ sqlite3TreeViewExpr(0, pAggInfo->aFunc[ii].pFExpr, 0);
+ }
+}
+#endif /* TREETRACE_ENABLED */
+
+/*
+** Analyze the arguments to aggregate functions. Create new pAggInfo->aCol[]
+** entries for columns that are arguments to aggregate functions but which
+** are not otherwise used.
+**
+** The aCol[] entries in AggInfo prior to nAccumulator are columns that
+** are referenced outside of aggregate functions. These might be columns
+** that are part of the GROUP by clause, for example. Other database engines
+** would throw an error if there is a column reference that is not in the
+** GROUP BY clause and that is not part of an aggregate function argument.
+** But SQLite allows this.
+**
+** The aCol[] entries beginning with the aCol[nAccumulator] and following
+** are column references that are used exclusively as arguments to
+** aggregate functions. This routine is responsible for computing
+** (or recomputing) those aCol[] entries.
+*/
+static void analyzeAggFuncArgs(
+ Parse *pParse,
+ AggInfo *pAggInfo,
+ NameContext *pNC
+){
+ int i;
+ assert( pAggInfo!=0 );
+ assert( pAggInfo->iFirstReg==0 );
+ pNC->ncFlags |= NC_InAggFunc;
+ for(i=0; inFunc; i++){
+ Expr *pExpr = pAggInfo->aFunc[i].pFExpr;
+ assert( ExprUseXList(pExpr) );
+ sqlite3ExprAnalyzeAggList(pNC, pExpr->x.pList);
+#ifndef SQLITE_OMIT_WINDOWFUNC
+ assert( !IsWindowFunc(pExpr) );
+ if( ExprHasProperty(pExpr, EP_WinFunc) ){
+ sqlite3ExprAnalyzeAggregates(pNC, pExpr->y.pWin->pFilter);
+ }
+#endif
+ }
+ pNC->ncFlags &= ~NC_InAggFunc;
+}
+
+/*
+** An index on expressions is being used in the inner loop of an
+** aggregate query with a GROUP BY clause. This routine attempts
+** to adjust the AggInfo object to take advantage of index and to
+** perhaps use the index as a covering index.
+**
+*/
+static void optimizeAggregateUseOfIndexedExpr(
+ Parse *pParse, /* Parsing context */
+ Select *pSelect, /* The SELECT statement being processed */
+ AggInfo *pAggInfo, /* The aggregate info */
+ NameContext *pNC /* Name context used to resolve agg-func args */
+){
+ assert( pAggInfo->iFirstReg==0 );
+ pAggInfo->nColumn = pAggInfo->nAccumulator;
+ if( ALWAYS(pAggInfo->nSortingColumn>0) ){
+ if( pAggInfo->nColumn==0 ){
+ pAggInfo->nSortingColumn = 0;
+ }else{
+ pAggInfo->nSortingColumn =
+ pAggInfo->aCol[pAggInfo->nColumn-1].iSorterColumn+1;
+ }
+ }
+ analyzeAggFuncArgs(pParse, pAggInfo, pNC);
+#if TREETRACE_ENABLED
+ if( sqlite3TreeTrace & 0x20 ){
+ IndexedExpr *pIEpr;
+ TREETRACE(0x20, pParse, pSelect,
+ ("AggInfo (possibly) adjusted for Indexed Exprs\n"));
+ sqlite3TreeViewSelect(0, pSelect, 0);
+ for(pIEpr=pParse->pIdxEpr; pIEpr; pIEpr=pIEpr->pIENext){
+ printf("data-cursor=%d index={%d,%d}\n",
+ pIEpr->iDataCur, pIEpr->iIdxCur, pIEpr->iIdxCol);
+ sqlite3TreeViewExpr(0, pIEpr->pExpr, 0);
+ }
+ printAggInfo(pAggInfo);
+ }
+#else
+ (void)pSelect; /* Suppress unused-parameter warnings */
+#endif
+}
+
+/*
+** Walker callback for aggregateConvertIndexedExprRefToColumn().
+*/
+static int aggregateIdxEprRefToColCallback(Walker *pWalker, Expr *pExpr){
+ AggInfo *pAggInfo;
+ struct AggInfo_col *pCol;
+ if( pExpr->pAggInfo==0 ) return WRC_Continue;
+ if( pExpr->op==TK_AGG_COLUMN ) return WRC_Continue;
+ if( pExpr->op==TK_AGG_FUNCTION ) return WRC_Continue;
+ if( pExpr->op==TK_IF_NULL_ROW ) return WRC_Continue;
+ pAggInfo = pExpr->pAggInfo;
+ assert( pExpr->iAgg>=0 && pExpr->iAggnColumn );
+ pCol = &pAggInfo->aCol[pExpr->iAgg];
+ pExpr->op = TK_AGG_COLUMN;
+ pExpr->iTable = pCol->iTable;
+ pExpr->iColumn = pCol->iColumn;
+ return WRC_Prune;
+}
+
+/*
+** Convert every pAggInfo->aFunc[].pExpr such that any node within
+** those expressions that has pAppInfo set is changed into a TK_AGG_COLUMN
+** opcode.
+*/
+static void aggregateConvertIndexedExprRefToColumn(AggInfo *pAggInfo){
+ int i;
+ Walker w;
+ memset(&w, 0, sizeof(w));
+ w.xExprCallback = aggregateIdxEprRefToColCallback;
+ for(i=0; inFunc; i++){
+ sqlite3WalkExpr(&w, pAggInfo->aFunc[i].pFExpr);
+ }
+}
+
+
+/*
+** Allocate a block of registers so that there is one register for each
+** pAggInfo->aCol[] and pAggInfo->aFunc[] entry in pAggInfo. The first
+** register in this block is stored in pAggInfo->iFirstReg.
+**
+** This routine may only be called once for each AggInfo object. Prior
+** to calling this routine:
+**
+** * The aCol[] and aFunc[] arrays may be modified
+** * The AggInfoColumnReg() and AggInfoFuncReg() macros may not be used
+**
+** After clling this routine:
+**
+** * The aCol[] and aFunc[] arrays are fixed
+** * The AggInfoColumnReg() and AggInfoFuncReg() macros may be used
+**
+*/
+static void assignAggregateRegisters(Parse *pParse, AggInfo *pAggInfo){
+ assert( pAggInfo!=0 );
+ assert( pAggInfo->iFirstReg==0 );
+ pAggInfo->iFirstReg = pParse->nMem + 1;
+ pParse->nMem += pAggInfo->nColumn + pAggInfo->nFunc;
+}
+
/*
** Reset the aggregate accumulator.
**
@@ -6217,24 +6426,13 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){
int i;
struct AggInfo_func *pFunc;
int nReg = pAggInfo->nFunc + pAggInfo->nColumn;
+ assert( pAggInfo->iFirstReg>0 );
assert( pParse->db->pParse==pParse );
assert( pParse->db->mallocFailed==0 || pParse->nErr!=0 );
if( nReg==0 ) return;
if( pParse->nErr ) return;
-#ifdef SQLITE_DEBUG
- /* Verify that all AggInfo registers are within the range specified by
- ** AggInfo.mnReg..AggInfo.mxReg */
- assert( nReg==pAggInfo->mxReg-pAggInfo->mnReg+1 );
- for(i=0; inColumn; i++){
- assert( pAggInfo->aCol[i].iMem>=pAggInfo->mnReg
- && pAggInfo->aCol[i].iMem<=pAggInfo->mxReg );
- }
- for(i=0; inFunc; i++){
- assert( pAggInfo->aFunc[i].iMem>=pAggInfo->mnReg
- && pAggInfo->aFunc[i].iMem<=pAggInfo->mxReg );
- }
-#endif
- sqlite3VdbeAddOp3(v, OP_Null, 0, pAggInfo->mnReg, pAggInfo->mxReg);
+ sqlite3VdbeAddOp3(v, OP_Null, 0, pAggInfo->iFirstReg,
+ pAggInfo->iFirstReg+nReg-1);
for(pFunc=pAggInfo->aFunc, i=0; inFunc; i++, pFunc++){
if( pFunc->iDistinct>=0 ){
Expr *pE = pFunc->pFExpr;
@@ -6266,15 +6464,16 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){
ExprList *pList;
assert( ExprUseXList(pF->pFExpr) );
pList = pF->pFExpr->x.pList;
- sqlite3VdbeAddOp2(v, OP_AggFinal, pF->iMem, pList ? pList->nExpr : 0);
+ sqlite3VdbeAddOp2(v, OP_AggFinal, AggInfoFuncReg(pAggInfo,i),
+ pList ? pList->nExpr : 0);
sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF);
}
}
/*
-** Update the accumulator memory cells for an aggregate based on
-** the current cursor position.
+** Generate code that will update the accumulator memory cells for an
+** aggregate based on the current cursor position.
**
** If regAcc is non-zero and there are no min() or max() aggregates
** in pAggInfo, then only populate the pAggInfo->nAccumulator accumulator
@@ -6294,6 +6493,8 @@ static void updateAccumulator(
struct AggInfo_func *pF;
struct AggInfo_col *pC;
+ assert( pAggInfo->iFirstReg>0 );
+ if( pParse->nErr ) return;
pAggInfo->directMode = 1;
for(i=0, pF=pAggInfo->aFunc; inFunc; i++, pF++){
int nArg;
@@ -6354,7 +6555,7 @@ static void updateAccumulator(
if( regHit==0 && pAggInfo->nAccumulator ) regHit = ++pParse->nMem;
sqlite3VdbeAddOp4(v, OP_CollSeq, regHit, 0, 0, (char *)pColl, P4_COLLSEQ);
}
- sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, pF->iMem);
+ sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, AggInfoFuncReg(pAggInfo,i));
sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF);
sqlite3VdbeChangeP5(v, (u8)nArg);
sqlite3ReleaseTempRange(pParse, regAgg, nArg);
@@ -6369,7 +6570,7 @@ static void updateAccumulator(
addrHitTest = sqlite3VdbeAddOp1(v, OP_If, regHit); VdbeCoverage(v);
}
for(i=0, pC=pAggInfo->aCol; inAccumulator; i++, pC++){
- sqlite3ExprCode(pParse, pC->pCExpr, pC->iMem);
+ sqlite3ExprCode(pParse, pC->pCExpr, AggInfoColumnReg(pAggInfo,i));
}
pAggInfo->directMode = 0;
@@ -6629,30 +6830,6 @@ static int sameSrcAlias(SrcItem *p0, SrcList *pSrc){
return 0;
}
-#if TREETRACE_ENABLED
-/*
-** Display all information about an AggInfo object
-*/
-static void printAggInfo(AggInfo *pAggInfo){
- int ii;
- for(ii=0; iinColumn; ii++){
- struct AggInfo_col *pCol = &pAggInfo->aCol[ii];
- sqlite3DebugPrintf(
- "agg-column[%d] pTab=%s iTable=%d iColumn=%d iMem=%d"
- " iSorterColumn=%d\n",
- ii, pCol->pTab ? pCol->pTab->zName : "NULL",
- pCol->iTable, pCol->iColumn, pCol->iMem,
- pCol->iSorterColumn);
- sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0);
- }
- for(ii=0; iinFunc; ii++){
- sqlite3DebugPrintf("agg-func[%d]: iMem=%d\n",
- ii, pAggInfo->aFunc[ii].iMem);
- sqlite3TreeViewExpr(0, pAggInfo->aFunc[ii].pFExpr, 0);
- }
-}
-#endif /* TREETRACE_ENABLED */
-
/*
** Generate code for the SELECT statement given in the p argument.
**
@@ -7414,7 +7591,6 @@ int sqlite3Select(
sNC.pSrcList = pTabList;
sNC.uNC.pAggInfo = pAggInfo;
VVA_ONLY( sNC.ncFlags = NC_UAggInfo; )
- pAggInfo->mnReg = pParse->nMem+1;
pAggInfo->nSortingColumn = pGroupBy ? pGroupBy->nExpr : 0;
pAggInfo->pGroupBy = pGroupBy;
sqlite3ExprAnalyzeAggList(&sNC, pEList);
@@ -7435,20 +7611,7 @@ int sqlite3Select(
}else{
minMaxFlag = WHERE_ORDERBY_NORMAL;
}
- for(i=0; inFunc; i++){
- Expr *pExpr = pAggInfo->aFunc[i].pFExpr;
- assert( ExprUseXList(pExpr) );
- sNC.ncFlags |= NC_InAggFunc;
- sqlite3ExprAnalyzeAggList(&sNC, pExpr->x.pList);
-#ifndef SQLITE_OMIT_WINDOWFUNC
- assert( !IsWindowFunc(pExpr) );
- if( ExprHasProperty(pExpr, EP_WinFunc) ){
- sqlite3ExprAnalyzeAggregates(&sNC, pExpr->y.pWin->pFilter);
- }
-#endif
- sNC.ncFlags &= ~NC_InAggFunc;
- }
- pAggInfo->mxReg = pParse->nMem;
+ analyzeAggFuncArgs(pParse, pAggInfo, &sNC);
if( db->mallocFailed ) goto select_end;
#if TREETRACE_ENABLED
if( sqlite3TreeTrace & 0x20 ){
@@ -7536,6 +7699,10 @@ int sqlite3Select(
sqlite3ExprListDelete(db, pDistinct);
goto select_end;
}
+ if( pParse->pIdxEpr ){
+ optimizeAggregateUseOfIndexedExpr(pParse, p, pAggInfo, &sNC);
+ }
+ assignAggregateRegisters(pParse, pAggInfo);
eDist = sqlite3WhereIsDistinct(pWInfo);
TREETRACE(0x2,pParse,p,("WhereBegin returns\n"));
if( sqlite3WhereIsOrdered(pWInfo)==pGroupBy->nExpr ){
@@ -7596,6 +7763,23 @@ int sqlite3Select(
pAggInfo->useSortingIdx = 1;
}
+ /* If there entries in pAgggInfo->aFunc[] that contain subexpressions
+ ** that are indexed (and that were previously identified and tagged
+ ** in optimizeAggregateUseOfIndexedExpr()) then those subexpressions
+ ** must now be converted into a TK_AGG_COLUMN node so that the value
+ ** is correctly pulled from the index rather than being recomputed. */
+ if( pParse->pIdxEpr ){
+ aggregateConvertIndexedExprRefToColumn(pAggInfo);
+#if TREETRACE_ENABLED
+ if( sqlite3TreeTrace & 0x20 ){
+ TREETRACE(0x20, pParse, p,
+ ("AggInfo function expressions converted to reference index\n"));
+ sqlite3TreeViewSelect(0, p, 0);
+ printAggInfo(pAggInfo);
+ }
+#endif
+ }
+
/* If the index or temporary table used by the GROUP BY sort
** will naturally deliver rows in the order required by the ORDER BY
** clause, cancel the ephemeral table open coded earlier.
@@ -7774,7 +7958,8 @@ int sqlite3Select(
if( pKeyInfo ){
sqlite3VdbeChangeP4(v, -1, (char *)pKeyInfo, P4_KEYINFO);
}
- sqlite3VdbeAddOp2(v, OP_Count, iCsr, pAggInfo->aFunc[0].iMem);
+ assignAggregateRegisters(pParse, pAggInfo);
+ sqlite3VdbeAddOp2(v, OP_Count, iCsr, AggInfoFuncReg(pAggInfo,0));
sqlite3VdbeAddOp1(v, OP_Close, iCsr);
explainSimpleCount(pParse, pTab, pBest);
}else{
@@ -7810,6 +7995,7 @@ int sqlite3Select(
pDistinct = pAggInfo->aFunc[0].pFExpr->x.pList;
distFlag = pDistinct ? (WHERE_WANT_DISTINCT|WHERE_AGG_DISTINCT) : 0;
}
+ assignAggregateRegisters(pParse, pAggInfo);
/* This case runs if the aggregate has no GROUP BY clause. The
** processing is much simpler since there is only a single row
@@ -7893,7 +8079,7 @@ select_end:
if( pAggInfo && !db->mallocFailed ){
for(i=0; inColumn; i++){
Expr *pExpr = pAggInfo->aCol[i].pCExpr;
- assert( pExpr!=0 );
+ if( pExpr==0 ) continue;
assert( pExpr->pAggInfo==pAggInfo );
assert( pExpr->iAgg==i );
}
diff --git a/src/shell.c.in b/src/shell.c.in
index 4409f4240d..a829fa2387 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -10691,7 +10691,7 @@ static int do_meta_command(char *zLine, ShellState *p){
}
}else{
output_file_close(p->traceOut);
- p->traceOut = output_file_open(azArg[1], 0);
+ p->traceOut = output_file_open(z, 0);
}
}
if( p->traceOut==0 ){
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 86f33dec9e..db65082bbc 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -2739,16 +2739,15 @@ struct AggInfo {
** from source tables rather than from accumulators */
u8 useSortingIdx; /* In direct mode, reference the sorting index rather
** than the source table */
+ u16 nSortingColumn; /* Number of columns in the sorting index */
int sortingIdx; /* Cursor number of the sorting index */
int sortingIdxPTab; /* Cursor number of pseudo-table */
- int nSortingColumn; /* Number of columns in the sorting index */
- int mnReg, mxReg; /* Range of registers allocated for aCol and aFunc */
+ int iFirstReg; /* First register in range for aCol[] and aFunc[] */
ExprList *pGroupBy; /* The group by clause */
struct AggInfo_col { /* For each column used in source tables */
Table *pTab; /* Source table */
Expr *pCExpr; /* The original expression */
int iTable; /* Cursor number of the source table */
- int iMem; /* Memory location that acts as accumulator */
i16 iColumn; /* Column number within the source table */
i16 iSorterColumn; /* Column number in the sorting index */
} *aCol;
@@ -2759,7 +2758,6 @@ struct AggInfo {
struct AggInfo_func { /* For each aggregate function */
Expr *pFExpr; /* Expression encoding the function */
FuncDef *pFunc; /* The aggregate function implementation */
- int iMem; /* Memory location that acts as accumulator */
int iDistinct; /* Ephemeral table used to enforce DISTINCT */
int iDistAddr; /* Address of OP_OpenEphemeral */
} *aFunc;
@@ -2767,6 +2765,17 @@ struct AggInfo {
u32 selId; /* Select to which this AggInfo belongs */
};
+/*
+** Macros to compute aCol[] and aFunc[] register numbers.
+**
+** These macros should not be used prior to the call to
+** assignAggregateRegisters() that computes the value of pAggInfo->iFirstReg.
+** The assert()s that are part of this macro verify that constraint.
+*/
+#define AggInfoColumnReg(A,I) (assert((A)->iFirstReg),(A)->iFirstReg+(I))
+#define AggInfoFuncReg(A,I) \
+ (assert((A)->iFirstReg),(A)->iFirstReg+(A)->nColumn+(I))
+
/*
** The datatype ynVar is a signed integer, either 16-bit or 32-bit.
** Usually it is 16-bits. But if SQLITE_MAX_VARIABLE_NUMBER is greater
diff --git a/src/treeview.c b/src/treeview.c
index 89a128dff6..9df16a12bf 100644
--- a/src/treeview.c
+++ b/src/treeview.c
@@ -487,7 +487,7 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){
sqlite3TreeViewPop(&pView);
return;
}
- if( pExpr->flags || pExpr->affExpr || pExpr->vvaFlags ){
+ if( pExpr->flags || pExpr->affExpr || pExpr->vvaFlags || pExpr->pAggInfo ){
StrAccum x;
sqlite3StrAccumInit(&x, 0, zFlgs, sizeof(zFlgs), 0);
sqlite3_str_appendf(&x, " fg.af=%x.%c",
@@ -504,6 +504,9 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){
if( ExprHasVVAProperty(pExpr, EP_Immutable) ){
sqlite3_str_appendf(&x, " IMMUTABLE");
}
+ if( pExpr->pAggInfo!=0 ){
+ sqlite3_str_appendf(&x, " agg-column[%d]", pExpr->iAgg);
+ }
sqlite3StrAccumFinish(&x);
}else{
zFlgs[0] = 0;
diff --git a/src/trigger.c b/src/trigger.c
index ca64d6145b..02d8540237 100644
--- a/src/trigger.c
+++ b/src/trigger.c
@@ -61,7 +61,7 @@ Trigger *sqlite3TriggerList(Parse *pParse, Table *pTab){
if( pTrig->pTabSchema==pTab->pSchema
&& pTrig->table
&& 0==sqlite3StrICmp(pTrig->table, pTab->zName)
- && pTrig->pTabSchema!=pTmpSchema
+ && (pTrig->pTabSchema!=pTmpSchema || pTrig->bReturning)
){
pTrig->pNext = pList;
pList = pTrig;
diff --git a/test/pushdown.test b/test/pushdown.test
index af5162495b..7c9b107841 100644
--- a/test/pushdown.test
+++ b/test/pushdown.test
@@ -86,6 +86,31 @@ do_test 2.2 {
set L
} {three}
-
+# 2022-11-25 dbsqlfuzz crash-3a548de406a50e896c1bf7142692d35d339d697f
+# Disable the push-down optimization for compound subqueries if any
+# arm of the compound has an incompatible affinity.
+#
+reset_db
+do_execsql_test 3.1 {
+ CREATE TABLE t0(c0 INT);
+ INSERT INTO t0 VALUES(0);
+ CREATE TABLE t1_a(a INTEGER PRIMARY KEY, b TEXT);
+ INSERT INTO t1_a VALUES(1,'one'); --,(4,'four');
+ CREATE TABLE t1_b(c INTEGER PRIMARY KEY, d TEXT);
+ INSERT INTO t1_b VALUES(2,'two'); --,(5,'five');
+ CREATE VIEW v0 AS SELECT CAST(t0.c0 AS INTEGER) AS c0 FROM t0;
+ CREATE VIEW t1 AS SELECT a, b FROM t1_a UNION ALL SELECT c, 0 FROM t1_b;
+ SELECT t1.a, quote(t1.b), t0.c0 AS cd FROM t0 LEFT JOIN v0 ON v0.c0!=0,t1;
+} {
+ 1 'one' 0
+ 2 '0' 0
+}
+do_execsql_test 3.2 {
+ SELECT a, quote(b), cd FROM (
+ SELECT t1.a, t1.b, t0.c0 AS cd FROM t0 LEFT JOIN v0 ON v0.c0!=0,t1
+ ) WHERE a=2 AND b='0' AND cd=0;
+} {
+ 2 '0' 0
+}
finish_test
diff --git a/test/returning1.test b/test/returning1.test
index fb058a6438..92e10ee9f4 100644
--- a/test/returning1.test
+++ b/test/returning1.test
@@ -376,4 +376,28 @@ do_execsql_test 16.1 {
SELECT * FROM t2;
} {1 2 3 a b c}
+
+foreach {tn temp} {
+ 1 ""
+ 2 TEMP
+} {
+ reset_db
+ do_execsql_test 17.$tn.0 "
+ CREATE $temp TABLE foo (
+ fooid INTEGER PRIMARY KEY,
+ fooval INTEGER NOT NULL UNIQUE,
+ refcnt INTEGER NOT NULL DEFAULT 1
+ );
+ "
+ do_execsql_test 17.$tn.1 {
+ INSERT INTO foo (fooval) VALUES (17), (4711), (17)
+ ON CONFLICT DO
+ UPDATE SET refcnt = refcnt+1
+ RETURNING fooid;
+ } {
+ 1 2 1
+ }
+}
+
+
finish_test
diff --git a/test/shell4.test b/test/shell4.test
index 068072202d..193dc8b69b 100644
--- a/test/shell4.test
+++ b/test/shell4.test
@@ -125,6 +125,12 @@ SELECT * FROM t1;}}
do_test shell4-2.5 {
catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace stdout\nSELECT * FROM t1;"
} {0 {SELECT * FROM t1;}}
+do_test shell4-2.6 {
+ catchcmd ":memory:" {
+CREATE TABLE t1(x);
+.trace --stmt stdout
+SELECT * FROM t1;}
+} {0 {SELECT * FROM t1;}}
}
do_test shell4-3.1 {
diff --git a/test/tkt-99378177930f87bd.test b/test/tkt-99378177930f87bd.test
new file mode 100644
index 0000000000..b21be3f67f
--- /dev/null
+++ b/test/tkt-99378177930f87bd.test
@@ -0,0 +1,154 @@
+# 2022-11-23
+#
+# The author disclaims copyright to this source code. In place of
+# a legal notice, here is a blessing:
+#
+# May you do good and not evil.
+# May you find forgiveness for yourself and forgive others.
+# May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.
+#
+# This file implements tests to verify that the enhancement
+# request documented by ticket 99378177930f87bd is working.
+#
+# The enhancement is that if an aggregate query with a GROUP BY clause
+# uses subexpressions in the arguments to aggregate functions that are
+# also columns of an index, then the values are pulled from the index
+# rather than being recomputed. This has the potential to make some
+# indexed queries works as if the index were covering.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+do_execsql_test tkt-99378-100 {
+ CREATE TABLE t1(a INT, b TEXT, c INT, d INT);
+ INSERT INTO t1(a,b,c,d) VALUES
+ (1, '{"x":1}', 12, 3),
+ (1, '{"x":2}', 4, 5),
+ (1, '{"x":1}', 6, 11),
+ (2, '{"x":1}', 22, 3),
+ (2, '{"x":2}', 4, 5),
+ (3, '{"x":1}', 6, 7);
+ CREATE INDEX t1x ON t1(d, a, b->>'x', c);
+} {}
+do_execsql_test tkt-99378-110 {
+ SELECT a,
+ SUM(1) AS t1,
+ SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2,
+ SUM(c) AS t3,
+ SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4
+ FROM t1
+ WHERE d BETWEEN 0 and 10
+ GROUP BY a;
+} {
+ 1 2 1 16 12
+ 2 2 1 26 22
+ 3 1 1 6 6
+}
+
+# The proof that the index on the expression is being used is in the
+# fact that the byte code contains no "Function" opcodes. In other words,
+# the ->> operator (which is implemented by a function) is never invoked.
+# Instead, the b->>'x' value is pulled out of the index.
+#
+do_execsql_test tkt-99378-120 {
+ EXPLAIN
+ SELECT a,
+ SUM(1) AS t1,
+ SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2,
+ SUM(c) AS t3,
+ SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4
+ FROM t1
+ WHERE d BETWEEN 0 and 10
+ GROUP BY a;
+} {~/Function/}
+
+
+do_execsql_test tkt-99378-130 {
+ SELECT a,
+ SUM(1) AS t1,
+ SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2,
+ SUM(c) AS t3,
+ SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4
+ FROM t1
+ WHERE d BETWEEN 0 and 10
+ GROUP BY +a;
+} {
+ 1 2 1 16 12
+ 2 2 1 26 22
+ 3 1 1 6 6
+}
+do_execsql_test tkt-99378-140 {
+ EXPLAIN
+ SELECT a,
+ SUM(1) AS t1,
+ SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2,
+ SUM(c) AS t3,
+ SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4
+ FROM t1
+ WHERE d BETWEEN 0 and 10
+ GROUP BY +a;
+} {~/Function/}
+
+do_execsql_test tkt-99378-200 {
+ DROP INDEX t1x;
+ CREATE INDEX t1x ON t1(a, d, b->>'x', c);
+}
+do_execsql_test tkt-99378-210 {
+ SELECT a,
+ SUM(1) AS t1,
+ SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2,
+ SUM(c) AS t3,
+ SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4
+ FROM t1
+ WHERE d BETWEEN 0 and 10
+ GROUP BY a;
+} {
+ 1 2 1 16 12
+ 2 2 1 26 22
+ 3 1 1 6 6
+}
+do_execsql_test tkt-99378-220 {
+ EXPLAIN
+ SELECT a,
+ SUM(1) AS t1,
+ SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2,
+ SUM(c) AS t3,
+ SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4
+ FROM t1
+ WHERE d BETWEEN 0 and 10
+ GROUP BY a;
+} {~/Function/}
+do_execsql_test tkt-99378-230 {
+ SELECT a,
+ SUM(1) AS t1,
+ SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2,
+ SUM(c) AS t3,
+ SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4
+ FROM t1
+ WHERE d BETWEEN 0 and 10
+ GROUP BY a;
+} {
+ 1 2 1 16 12
+ 2 2 1 26 22
+ 3 1 1 6 6
+}
+do_execsql_test tkt-99378-240 {
+ EXPLAIN
+ SELECT a,
+ SUM(1) AS t1,
+ SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2,
+ SUM(c) AS t3,
+ SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4
+ FROM t1
+ WHERE d BETWEEN 0 and 10
+ GROUP BY a;
+} {~/Function/}
+
+
+
+
+finish_test