From 48a8352a3932c091a5337aecc66626d3e6ec0386 Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 28 Jul 2023 01:12:47 +0000 Subject: [PATCH] Add support making use of sqlite3_aggregate_context() (in a roundabout way) from Java to accumulate state within aggregate and window UDFs. FossilOrigin-Name: 640574984741c7a9472d7f8be7bce87e736d7947ce673ae4a25008d74238ad90 --- ext/jni/src/c/sqlite3-jni.c | 64 +++++++++++++-- ext/jni/src/org/sqlite/jni/SQLFunction.java | 81 +++++++++++++++++-- ext/jni/src/org/sqlite/jni/Tester1.java | 57 ++++++++----- .../src/org/sqlite/jni/sqlite3_context.java | 34 ++++++++ manifest | 18 ++--- manifest.uuid | 2 +- 6 files changed, 214 insertions(+), 42 deletions(-) diff --git a/ext/jni/src/c/sqlite3-jni.c b/ext/jni/src/c/sqlite3-jni.c index 374518ac34..907c6a9d23 100644 --- a/ext/jni/src/c/sqlite3-jni.c +++ b/ext/jni/src/c/sqlite3-jni.c @@ -271,10 +271,11 @@ enum { typedef struct NphCacheLine NphCacheLine; struct NphCacheLine { const char * zClassName /* "full/class/Name" */; - jclass klazz /* global ref to concrete NPH class */; - jmethodID midSet /* setNativePointer() */; - jmethodID midGet /* getNativePointer() */; - jmethodID midCtor /* constructor */; + jclass klazz /* global ref to concrete NPH class */; + jmethodID midSet /* setNativePointer() */; + jmethodID midGet /* getNativePointer() */; + jmethodID midCtor /* constructor */; + jmethodID midSetAgg /* sqlite3_context::setAggregateContext() */; }; typedef struct JNIEnvCacheLine JNIEnvCacheLine; @@ -713,6 +714,42 @@ static void * getNativePointer(JNIEnv * env, jobject pObj, const char *zClassNam } } +/** + Requires that jCx be a Java-side sqlite3_context wrapper for pCx. + This function calls sqlite3_aggregate_context() to allocate a tiny + sliver of memory, the address of which is set in + jCx->setAggregateContext(). The memory is only used as a key for + mapping, client-side, results of aggregate result sets across + xStep() and xFinal() methods. + + isFinal must be 1 for xFinal() calls and 0 for all others. +*/ +static void setAggregateContext(JNIEnv * env, jobject jCx, + sqlite3_context * pCx, + int isFinal){ + jmethodID setter; + void * pAgg; + struct NphCacheLine * const cacheLine = + S3Global_nph_cache(env, ClassNames.sqlite3_context); + if(cacheLine && cacheLine->klazz && cacheLine->midSetAgg){ + setter = cacheLine->midSetAgg; + assert(setter); + }else{ + jclass const klazz = + cacheLine ? cacheLine->klazz : (*env)->GetObjectClass(env, jCx); + setter = (*env)->GetMethodID(env, klazz, "setAggregateContext", "(J)V"); + if(cacheLine){ + assert(cacheLine->klazz); + assert(!cacheLine->midSetAgg); + cacheLine->midSetAgg = setter; + } + } + pAgg = sqlite3_aggregate_context(pCx, isFinal ? 0 : 8); + (*env)->CallVoidMethod(env, jCx, setter, (jlong)pAgg); + IFTHREW_REPORT; +} + + /* ** This function is NOT part of the sqlite3 public API. It is strictly ** for use by the sqlite project's own Java/JNI bindings. @@ -1054,6 +1091,11 @@ typedef struct { jobjectArray jargv; } udf_jargs; +/** + Converts the given (cx, argc, argv) into arguments for the given + UDF, placing the result in the final argument. Returns 0 on + success, SQLITE_NOMEM on allocation error. +*/ static int udf_args(sqlite3_context * const cx, int argc, sqlite3_value**argv, UDFState * const s, @@ -1102,19 +1144,23 @@ static int udf_report_exception(sqlite3_context * cx, UDFState *s, return rc; } -static int udf_xFSI(sqlite3_context* cx, int argc, +static int udf_xFSI(sqlite3_context* pCx, int argc, sqlite3_value** argv, UDFState * s, jmethodID xMethodID, const char * zFuncType){ udf_jargs args; JNIEnv * const env = s->env; - int rc = udf_args(cx, argc, argv, s, &args); + int rc = udf_args(pCx, argc, argv, s, &args); + //MARKER(("%s.%s() pCx = %p\n", s->zFuncName, zFuncType, pCx)); if(rc) return rc; //MARKER(("UDF::%s.%s()\n", s->zFuncName, zFuncType)); + if( UDF_SCALAR != s->type ){ + setAggregateContext(env, args.jcx, pCx, 0); + } (*env)->CallVoidMethod(env, s->jObj, xMethodID, args.jcx, args.jargv); IFTHREW{ - rc = udf_report_exception(cx,s, zFuncType); + rc = udf_report_exception(pCx,s, zFuncType); } UNREF_L(args.jcx); UNREF_L(args.jargv); @@ -1127,11 +1173,15 @@ static int udf_xFV(sqlite3_context* cx, UDFState * s, JNIEnv * const env = s->env; jobject jcx = new_sqlite3_context_wrapper(s->env, cx); int rc = 0; + //MARKER(("%s.%s() cx = %p\n", s->zFuncName, zFuncType, cx)); if(!jcx){ sqlite3_result_error_nomem(cx); return SQLITE_NOMEM; } //MARKER(("UDF::%s.%s()\n", s->zFuncName, zFuncType)); + if( UDF_SCALAR != s->type ){ + setAggregateContext(env, jcx, cx, 1); + } (*env)->CallVoidMethod(env, s->jObj, xMethodID, jcx); IFTHREW{ rc = udf_report_exception(cx,s, zFuncType); diff --git a/ext/jni/src/org/sqlite/jni/SQLFunction.java b/ext/jni/src/org/sqlite/jni/SQLFunction.java index 482bf45338..7e7d817504 100644 --- a/ext/jni/src/org/sqlite/jni/SQLFunction.java +++ b/ext/jni/src/org/sqlite/jni/SQLFunction.java @@ -19,9 +19,62 @@ package org.sqlite.jni; access to the callback functions needed in order to implement SQL functions in Java. This class is not used by itself: see the three inner classes. + + Note that if a given function is called multiple times in a single + SQL statement, e.g. SELECT MYFUNC(A), MYFUNC(B)..., then the + context object passed to each one will be different. This is most + significant for aggregates and window functions, since they must + assign their results to the proper context. + + TODO: add helper APIs to map sqlite3_context instances to + func-specific state and to clear that when the aggregate or window + function is done. */ public abstract class SQLFunction { + /** + ContextMap is a helper for use with aggregate and window + functions, to help them manage their accumulator state across + calls to xStep() and xFinal(). It works by mapping + sqlite3_context::getAggregateContext() to a single piece of state + which persists across a set of 0 or more SQLFunction.xStep() + calls and 1 SQLFunction.xFinal() call. + */ + public static final class ContextMap { + private java.util.Map> map + = new java.util.HashMap>(); + + /** + Should be called from a UDF's xStep() method, passing it that + method's first argument and an initial value for the persistent + state. If there is currently no mapping for + cx.getAggregateContext() within the map, one is created, else + an existing one is preferred. It returns a ValueHolder which + can be used to modify that state directly without having to put + a new result back in the underlying map. + */ + public ValueHolder xStep(sqlite3_context cx, T initialValue){ + ValueHolder rc = map.get(cx.getAggregateContext()); + if(null == rc){ + map.put(cx.getAggregateContext(), rc = new ValueHolder(initialValue)); + } + return rc; + } + + /** + Should be called from a UDF's xFinal() method and passed that + method's first argument. This function returns the value + associated with cx.getAggregateContext(), or null if + this.xStep() has not been called to set up such a mapping. That + will be the case if an aggregate is used in a statement which + has no result rows. + */ + public T xFinal(sqlite3_context cx){ + final ValueHolder h = map.remove(cx.getAggregateContext()); + return null==h ? null : h.value; + } + } + //! Subclass for creating scalar functions. public static abstract class Scalar extends SQLFunction { public abstract void xFunc(sqlite3_context cx, sqlite3_value[] args); @@ -33,18 +86,36 @@ public abstract class SQLFunction { } //! Subclass for creating aggregate functions. - public static abstract class Aggregate extends SQLFunction { + public static abstract class Aggregate extends SQLFunction { public abstract void xStep(sqlite3_context cx, sqlite3_value[] args); public abstract void xFinal(sqlite3_context cx); public void xDestroy() {} + + private final ContextMap map = new ContextMap<>(); + + /** + See ContextMap.xStep(). + */ + public final ValueHolder getAggregateState(sqlite3_context cx, T initialValue){ + return map.xStep(cx, initialValue); + } + + /** + See ContextMap.xFinal(). + */ + public final T takeAggregateState(sqlite3_context cx){ + return map.xFinal(cx); + } } //! Subclass for creating window functions. - public static abstract class Window extends SQLFunction { - public abstract void xStep(sqlite3_context cx, sqlite3_value[] args); + public static abstract class Window extends Aggregate { + public Window(){ + super(); + } + //public abstract void xStep(sqlite3_context cx, sqlite3_value[] args); public abstract void xInverse(sqlite3_context cx, sqlite3_value[] args); - public abstract void xFinal(sqlite3_context cx); + //public abstract void xFinal(sqlite3_context cx); public abstract void xValue(sqlite3_context cx); - public void xDestroy() {} } } diff --git a/ext/jni/src/org/sqlite/jni/Tester1.java b/ext/jni/src/org/sqlite/jni/Tester1.java index 4b760661db..edf0eeaafa 100644 --- a/ext/jni/src/org/sqlite/jni/Tester1.java +++ b/ext/jni/src/org/sqlite/jni/Tester1.java @@ -482,21 +482,23 @@ public class Tester1 { private static void testUdfAggregate(){ final sqlite3 db = createNewDb(); - SQLFunction func = new SQLFunction.Aggregate(){ - private int accum = 0; - @Override public void xStep(sqlite3_context cx, sqlite3_value args[]){ - this.accum += sqlite3_value_int(args[0]); + SQLFunction func = new SQLFunction.Aggregate(){ + @Override + public void xStep(sqlite3_context cx, sqlite3_value args[]){ + this.getAggregateState(cx, 0).value += sqlite3_value_int(args[0]); } - @Override public void xFinal(sqlite3_context cx){ - sqlite3_result_int(cx, this.accum); - this.accum = 0; + @Override + public void xFinal(sqlite3_context cx){ + final Integer v = this.takeAggregateState(cx); + if(null == v) sqlite3_result_null(cx); + else sqlite3_result_int(cx, v); } }; execSql(db, "CREATE TABLE t(a); INSERT INTO t(a) VALUES(1),(2),(3)"); int rc = sqlite3_create_function(db, "myfunc", 1, SQLITE_UTF8, func); affirm(0 == rc); sqlite3_stmt stmt = new sqlite3_stmt(); - sqlite3_prepare(db, "select myfunc(a) from t", stmt); + sqlite3_prepare(db, "select myfunc(a), myfunc(a+10) from t", stmt); affirm( 0 != stmt.getNativePointer() ); int n = 0; if( SQLITE_ROW == sqlite3_step(stmt) ){ @@ -514,6 +516,20 @@ public class Tester1 { } sqlite3_finalize(stmt); affirm( 1==n ); + + rc = sqlite3_prepare(db, "select myfunc(a), myfunc(a+a) from t order by a", + stmt); + affirm( 0 == rc ); + n = 0; + while( SQLITE_ROW == sqlite3_step(stmt) ){ + final int c0 = sqlite3_column_int(stmt, 0); + final int c1 = sqlite3_column_int(stmt, 1); + ++n; + affirm( 6 == c0 ); + affirm( 12 == c1 ); + } + affirm( 1 == n ); + sqlite3_finalize(stmt); sqlite3_close(db); } @@ -521,26 +537,27 @@ public class Tester1 { final sqlite3 db = createNewDb(); /* Example window function, table, and results taken from: https://sqlite.org/windowfunctions.html#udfwinfunc */ - final SQLFunction func = new SQLFunction.Window(){ - private int accum = 0; - private void xStepInverse(int v){ - this.accum += v; - } - private void xFinalValue(sqlite3_context cx){ - sqlite3_result_int(cx, this.accum); + final SQLFunction func = new SQLFunction.Window(){ + + private void xStepInverse(sqlite3_context cx, int v){ + this.getAggregateState(cx,0).value += v; } @Override public void xStep(sqlite3_context cx, sqlite3_value[] args){ - this.xStepInverse(sqlite3_value_int(args[0])); + this.xStepInverse(cx, sqlite3_value_int(args[0])); } @Override public void xInverse(sqlite3_context cx, sqlite3_value[] args){ - this.xStepInverse(-sqlite3_value_int(args[0])); + this.xStepInverse(cx, -sqlite3_value_int(args[0])); + } + + private void xFinalValue(sqlite3_context cx, Integer v){ + if(null == v) sqlite3_result_null(cx); + else sqlite3_result_int(cx, v); } @Override public void xFinal(sqlite3_context cx){ - this.xFinalValue(cx); - this.accum = 0; + xFinalValue(cx, this.takeAggregateState(cx)); } @Override public void xValue(sqlite3_context cx){ - this.xFinalValue(cx); + xFinalValue(cx, this.getAggregateState(cx,null).value); } }; int rc = sqlite3_create_function(db, "winsumint", 1, SQLITE_UTF8, func); diff --git a/ext/jni/src/org/sqlite/jni/sqlite3_context.java b/ext/jni/src/org/sqlite/jni/sqlite3_context.java index 9d053ffe9a..d6bc3012a1 100644 --- a/ext/jni/src/org/sqlite/jni/sqlite3_context.java +++ b/ext/jni/src/org/sqlite/jni/sqlite3_context.java @@ -13,8 +13,42 @@ */ package org.sqlite.jni; +/** + sqlite3_context instances are used in conjunction with user-defined + SQL functions (a.k.a. UDFs). They are opaque pointers. + + The getAggregateContext() method corresponds to C's + sqlite3_aggregate_context(), with a slightly different interface in + order to account for cross-language differences. It serves the same + purposes in a slightly different way: it provides a key which is + stable across invocations of UDF xStep() and xFinal() pairs, to + which a UDF may map state across such calls (e.g. a numeric result + which is being accumulated). +*/ public class sqlite3_context extends NativePointerHolder { public sqlite3_context() { super(); } + private long aggcx = 0; + + /** + If this object is being used in the context of an aggregate or + window UDF, the UDF binding layer will set a unique context value + here. That value will be the same across matching calls to the + xStep() and xFinal() routines, as well as xValue() and xInverse() + in window UDFs. This value can be used as a key to map state + which needs to persist across such calls, noting that such state + should be cleaned up via xFinal(). + */ + public long getAggregateContext(){ + return aggcx; + } + + /** + For use only by the JNI layer. It's permitted to call this even + though it's private. + */ + private void setAggregateContext(long n){ + aggcx = n; + } } diff --git a/manifest b/manifest index 67eaf44084..7fe2d447fd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reformulate\sjni\stests\sto\snot\srequire\sthe\s-ea\sjvm\sflag\sto\senable\sassert(). -D 2023-07-27T22:53:02.373 +C Add\ssupport\smaking\suse\sof\ssqlite3_aggregate_context()\s(in\sa\sroundabout\sway)\sfrom\sJava\sto\saccumulate\sstate\swithin\saggregate\sand\swindow\sUDFs. +D 2023-07-28T01:12:47.322 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -232,20 +232,20 @@ F ext/icu/icu.c c074519b46baa484bb5396c7e01e051034da8884bad1a1cb7f09bbe6be3f0282 F ext/icu/sqliteicu.h fa373836ed5a1ee7478bdf8a1650689294e41d0c89c1daab26e9ae78a32075a8 F ext/jni/GNUmakefile 56a014dbff9516774d895ec1ae9df0ed442765b556f79a0fc0b5bc438217200d F ext/jni/README.md 042762dbf047667783a5bd0aec303535140f302debfbd259c612edf856661623 -F ext/jni/src/c/sqlite3-jni.c 76921edc2d1abea2cb39c21bcc49acbc307cb368e96cb7803a2c134c444c3fcd +F ext/jni/src/c/sqlite3-jni.c 8d3ae5c0474548b1b95fea888227a4f617b9302a7e230bb5ff1b3735fe85fb03 F ext/jni/src/c/sqlite3-jni.h c9bb150a38dce09cc2794d5aac8fa097288d9946fbb15250fd0a23c31957f506 F ext/jni/src/org/sqlite/jni/BusyHandler.java 1b1d3e5c86cd796a0580c81b6af6550ad943baa25e47ada0dcca3aff3ebe978c F ext/jni/src/org/sqlite/jni/Collation.java 8dffbb00938007ad0967b2ab424d3c908413af1bbd3d212b9c9899910f1218d1 F ext/jni/src/org/sqlite/jni/NativePointerHolder.java 70dc7bc41f80352ff3d4331e2e24f45fcd23353b3641e2f68a81bd8262215861 F ext/jni/src/org/sqlite/jni/OutputPointer.java 08a752b58a33696c5eaf0eb9361a0966b188dec40f4a3613eb133123951f6c5f F ext/jni/src/org/sqlite/jni/ProgressHandler.java 5a1d7b2607eb2ef596fcf4492a49d1b3a5bdea3af9918e11716831ffd2f02284 -F ext/jni/src/org/sqlite/jni/SQLFunction.java 2f5d197f6c7d73b6031ba1a19598d7e3eee5ebad467eeee62c72e585bd6556a5 +F ext/jni/src/org/sqlite/jni/SQLFunction.java d77e0a4bb6bc0d65339aeacd6b20fc7e3b8a05f899c1f0ead90dda61f0a01522 F ext/jni/src/org/sqlite/jni/SQLite3Jni.java 3582b30c0fb1cb39e25b9069fe8c9e2fe4f2659f4d38437b610e46143e163610 -F ext/jni/src/org/sqlite/jni/Tester1.java 460d4a521bf3386a6aafc30c382817560b8dc1001472f6b8459cadeedb9a58ea +F ext/jni/src/org/sqlite/jni/Tester1.java 2334d1dd0efc22179654c586065c77d904830d736059b4049f9cd9e6832565bd F ext/jni/src/org/sqlite/jni/Tracer.java c2fe1eba4a76581b93b375a7b95ab1919e5ae60accfb06d6beb067b033e9bae1 F ext/jni/src/org/sqlite/jni/ValueHolder.java f022873abaabf64f3dd71ab0d6037c6e71cece3b8819fa10bf26a5461dc973ee F ext/jni/src/org/sqlite/jni/sqlite3.java c7d0500c7269882243aafb41425928d094b2fcbdbc2fd1caffc276871cd3fae3 -F ext/jni/src/org/sqlite/jni/sqlite3_context.java d781c72237e4a442adf6726b2edf15124405c28eba0387a279078858700f567c +F ext/jni/src/org/sqlite/jni/sqlite3_context.java 4a0b22226705a4f89d9c8093e0f51a8991cc0464864120970c915695afbba4e2 F ext/jni/src/org/sqlite/jni/sqlite3_stmt.java 3193693440071998a66870544d1d2314f144bea397ce4c3f83ff225d587067a0 F ext/jni/src/org/sqlite/jni/sqlite3_value.java f9d8c0766b1d1b290564cb35db8d37be54c42adc8df22ee77b8d39e3e93398cd F ext/lsm1/Makefile a553b728bba6c11201b795188c5708915cc4290f02b7df6ba7e8c4c943fd5cd9 @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7dcde2bfce54b18f391776fa1cb93c0ff6153634bedcab0007b374c06c4d4079 -R ba5fe9ad4149aefc21881fee22f6fa73 +P dc356667a8f4fa31a3fef1ae35873d834d27fd6a9f0818d6fb85e4751fde9fe5 +R 6f355aed3877133b3fb6aa6671123d94 U stephan -Z 2639e54196b7b2c8fbce104e63109714 +Z 75e450fbcee41582218a9562a53136a5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index dea2791cda..c7d6c30d99 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dc356667a8f4fa31a3fef1ae35873d834d27fd6a9f0818d6fb85e4751fde9fe5 \ No newline at end of file +640574984741c7a9472d7f8be7bce87e736d7947ce673ae4a25008d74238ad90 \ No newline at end of file