Add support making use of sqlite3_aggregate_context() (in a roundabout way) from Java to accumulate state within aggregate and window UDFs.

FossilOrigin-Name: 640574984741c7a9472d7f8be7bce87e736d7947ce673ae4a25008d74238ad90

ext/jni/src/c/sqlite3-jni.c

@@ -271,10 +271,11 @@ enum {
 typedef struct NphCacheLine NphCacheLine;
 struct NphCacheLine {
   const char * zClassName /* "full/class/Name" */;
-  jclass klazz      /* global ref to concrete NPH class */;
+  jclass klazz        /* global ref to concrete NPH class */;
-  jmethodID midSet  /* setNativePointer() */;
+  jmethodID midSet    /* setNativePointer() */;
-  jmethodID midGet  /* getNativePointer() */;
+  jmethodID midGet    /* getNativePointer() */;
-  jmethodID midCtor /* constructor */;
+  jmethodID midCtor   /* constructor */;
+  jmethodID midSetAgg /* sqlite3_context::setAggregateContext() */;
 };
 
 typedef struct JNIEnvCacheLine JNIEnvCacheLine;
@@ -713,6 +714,42 @@ static void * getNativePointer(JNIEnv * env, jobject pObj, const char *zClassNam
   }
 }
 
+/**
+   Requires that jCx be a Java-side sqlite3_context wrapper for pCx.
+   This function calls sqlite3_aggregate_context() to allocate a tiny
+   sliver of memory, the address of which is set in
+   jCx->setAggregateContext().  The memory is only used as a key for
+   mapping, client-side, results of aggregate result sets across
+   xStep() and xFinal() methods.
+
+   isFinal must be 1 for xFinal() calls and 0 for all others.
+*/
+static void setAggregateContext(JNIEnv * env, jobject jCx,
+                                sqlite3_context * pCx,
+                                int isFinal){
+  jmethodID setter;
+  void * pAgg;
+  struct NphCacheLine * const cacheLine =
+    S3Global_nph_cache(env, ClassNames.sqlite3_context);
+  if(cacheLine && cacheLine->klazz && cacheLine->midSetAgg){
+    setter = cacheLine->midSetAgg;
+    assert(setter);
+  }else{
+    jclass const klazz =
+      cacheLine ? cacheLine->klazz : (*env)->GetObjectClass(env, jCx);
+    setter = (*env)->GetMethodID(env, klazz, "setAggregateContext", "(J)V");
+    if(cacheLine){
+      assert(cacheLine->klazz);
+      assert(!cacheLine->midSetAgg);
+      cacheLine->midSetAgg = setter;
+    }
+  }
+  pAgg = sqlite3_aggregate_context(pCx, isFinal ? 0 : 8);
+  (*env)->CallVoidMethod(env, jCx, setter, (jlong)pAgg);
+  IFTHREW_REPORT;
+}
+
+
 /*
 ** This function is NOT part of the sqlite3 public API. It is strictly
 ** for use by the sqlite project's own Java/JNI bindings.
@@ -1054,6 +1091,11 @@ typedef struct {
   jobjectArray jargv;
 } udf_jargs;
 
+/**
+   Converts the given (cx, argc, argv) into arguments for the given
+   UDF, placing the result in the final argument. Returns 0 on
+   success, SQLITE_NOMEM on allocation error.
+*/
 static int udf_args(sqlite3_context * const cx,
                     int argc, sqlite3_value**argv,
                     UDFState * const s,
@@ -1102,19 +1144,23 @@ static int udf_report_exception(sqlite3_context * cx, UDFState *s,
   return rc;
 }
 
-static int udf_xFSI(sqlite3_context* cx, int argc,
+static int udf_xFSI(sqlite3_context* pCx, int argc,
                     sqlite3_value** argv,
                     UDFState * s,
                     jmethodID xMethodID,
                     const char * zFuncType){
   udf_jargs args;
   JNIEnv * const env = s->env;
-  int rc = udf_args(cx, argc, argv, s, &args);
+  int rc = udf_args(pCx, argc, argv, s, &args);
+  //MARKER(("%s.%s() pCx = %p\n", s->zFuncName, zFuncType, pCx));
   if(rc) return rc;
   //MARKER(("UDF::%s.%s()\n", s->zFuncName, zFuncType));
+  if( UDF_SCALAR != s->type ){
+    setAggregateContext(env, args.jcx, pCx, 0);
+  }
   (*env)->CallVoidMethod(env, s->jObj, xMethodID, args.jcx, args.jargv);
   IFTHREW{
-    rc = udf_report_exception(cx,s, zFuncType);
+    rc = udf_report_exception(pCx,s, zFuncType);
   }
   UNREF_L(args.jcx);
   UNREF_L(args.jargv);
@@ -1127,11 +1173,15 @@ static int udf_xFV(sqlite3_context* cx, UDFState * s,
   JNIEnv * const env = s->env;
   jobject jcx = new_sqlite3_context_wrapper(s->env, cx);
   int rc = 0;
+  //MARKER(("%s.%s() cx = %p\n", s->zFuncName, zFuncType, cx));
   if(!jcx){
     sqlite3_result_error_nomem(cx);
     return SQLITE_NOMEM;
   }
   //MARKER(("UDF::%s.%s()\n", s->zFuncName, zFuncType));
+  if( UDF_SCALAR != s->type ){
+    setAggregateContext(env, jcx, cx, 1);
+  }
   (*env)->CallVoidMethod(env, s->jObj, xMethodID, jcx);
   IFTHREW{
     rc = udf_report_exception(cx,s, zFuncType);

ext/jni/src/org/sqlite/jni/SQLFunction.java

@@ -19,9 +19,62 @@ package org.sqlite.jni;
    access to the callback functions needed in order to implement SQL
    functions in Java. This class is not used by itself: see the
    three inner classes.
+
+   Note that if a given function is called multiple times in a single
+   SQL statement, e.g. SELECT MYFUNC(A), MYFUNC(B)..., then the
+   context object passed to each one will be different. This is most
+   significant for aggregates and window functions, since they must
+   assign their results to the proper context.
+
+   TODO: add helper APIs to map sqlite3_context instances to
+   func-specific state and to clear that when the aggregate or window
+   function is done.
 */
 public abstract class SQLFunction {
 
+  /**
+     ContextMap is a helper for use with aggregate and window
+     functions, to help them manage their accumulator state across
+     calls to xStep() and xFinal(). It works by mapping
+     sqlite3_context::getAggregateContext() to a single piece of state
+     which persists across a set of 0 or more SQLFunction.xStep()
+     calls and 1 SQLFunction.xFinal() call.
+   */
+  public static final class ContextMap<T> {
+    private java.util.Map<Long,ValueHolder<T>> map
+      = new java.util.HashMap<Long,ValueHolder<T>>();
+
+    /**
+       Should be called from a UDF's xStep() method, passing it that
+       method's first argument and an initial value for the persistent
+       state. If there is currently no mapping for
+       cx.getAggregateContext() within the map, one is created, else
+       an existing one is preferred.  It returns a ValueHolder which
+       can be used to modify that state directly without having to put
+       a new result back in the underlying map.
+    */
+    public ValueHolder<T> xStep(sqlite3_context cx, T initialValue){
+      ValueHolder<T> rc = map.get(cx.getAggregateContext());
+      if(null == rc){
+        map.put(cx.getAggregateContext(), rc = new ValueHolder<T>(initialValue));
+      }
+      return rc;
+    }
+
+    /**
+       Should be called from a UDF's xFinal() method and passed that
+       method's first argument. This function returns the value
+       associated with cx.getAggregateContext(), or null if
+       this.xStep() has not been called to set up such a mapping. That
+       will be the case if an aggregate is used in a statement which
+       has no result rows.
+    */
+    public T xFinal(sqlite3_context cx){
+      final ValueHolder<T> h = map.remove(cx.getAggregateContext());
+      return null==h ? null : h.value;
+    }
+  }
+
   //! Subclass for creating scalar functions.
   public static abstract class Scalar extends SQLFunction {
     public abstract void xFunc(sqlite3_context cx, sqlite3_value[] args);
@@ -33,18 +86,36 @@ public abstract class SQLFunction {
   }
 
   //! Subclass for creating aggregate functions.
-  public static abstract class Aggregate extends SQLFunction {
+  public static abstract class Aggregate<T> extends SQLFunction {
     public abstract void xStep(sqlite3_context cx, sqlite3_value[] args);
     public abstract void xFinal(sqlite3_context cx);
     public void xDestroy() {}
+
+    private final ContextMap<T> map = new ContextMap<>();
+
+    /**
+       See ContextMap<T>.xStep().
+    */
+    public final ValueHolder<T> getAggregateState(sqlite3_context cx, T initialValue){
+      return map.xStep(cx, initialValue);
+    }
+
+    /**
+       See ContextMap<T>.xFinal().
+    */
+    public final T takeAggregateState(sqlite3_context cx){
+      return map.xFinal(cx);
+    }
   }
 
   //! Subclass for creating window functions.
-  public static abstract class Window extends SQLFunction {
+  public static abstract class Window<T> extends Aggregate<T> {
-    public abstract void xStep(sqlite3_context cx, sqlite3_value[] args);
+    public Window(){
+      super();
+    }
+    //public abstract void xStep(sqlite3_context cx, sqlite3_value[] args);
     public abstract void xInverse(sqlite3_context cx, sqlite3_value[] args);
-    public abstract void xFinal(sqlite3_context cx);
+    //public abstract void xFinal(sqlite3_context cx);
     public abstract void xValue(sqlite3_context cx);
-    public void xDestroy() {}
   }
 }

ext/jni/src/org/sqlite/jni/Tester1.java

@@ -482,21 +482,23 @@ public class Tester1 {
 
   private static void testUdfAggregate(){
     final sqlite3 db = createNewDb();
-    SQLFunction func = new SQLFunction.Aggregate(){
+    SQLFunction func = new SQLFunction.Aggregate<Integer>(){
-        private int accum = 0;
+        @Override
-        @Override public void xStep(sqlite3_context cx, sqlite3_value args[]){
+        public void xStep(sqlite3_context cx, sqlite3_value args[]){
-          this.accum += sqlite3_value_int(args[0]);
+          this.getAggregateState(cx, 0).value += sqlite3_value_int(args[0]);
         }
-        @Override public void xFinal(sqlite3_context cx){
+        @Override
-          sqlite3_result_int(cx, this.accum);
+        public void xFinal(sqlite3_context cx){
-          this.accum = 0;
+          final Integer v = this.takeAggregateState(cx);
+          if(null == v) sqlite3_result_null(cx);
+          else sqlite3_result_int(cx, v);
         }
       };
     execSql(db, "CREATE TABLE t(a); INSERT INTO t(a) VALUES(1),(2),(3)");
     int rc = sqlite3_create_function(db, "myfunc", 1, SQLITE_UTF8, func);
     affirm(0 == rc);
     sqlite3_stmt stmt = new sqlite3_stmt();
-    sqlite3_prepare(db, "select myfunc(a) from t", stmt);
+    sqlite3_prepare(db, "select myfunc(a), myfunc(a+10) from t", stmt);
     affirm( 0 != stmt.getNativePointer() );
     int n = 0;
     if( SQLITE_ROW == sqlite3_step(stmt) ){
@@ -514,6 +516,20 @@ public class Tester1 {
     }
     sqlite3_finalize(stmt);
     affirm( 1==n );
+
+    rc = sqlite3_prepare(db, "select myfunc(a), myfunc(a+a) from t order by a",
+                         stmt);
+    affirm( 0 == rc );
+    n = 0;
+    while( SQLITE_ROW == sqlite3_step(stmt) ){
+      final int c0 = sqlite3_column_int(stmt, 0);
+      final int c1 = sqlite3_column_int(stmt, 1);
+      ++n;
+      affirm( 6 == c0 );
+      affirm( 12 == c1 );
+    }
+    affirm( 1 == n );
+    sqlite3_finalize(stmt);
     sqlite3_close(db);
   }
 
@@ -521,26 +537,27 @@ public class Tester1 {
     final sqlite3 db = createNewDb();
     /* Example window function, table, and results taken from:
        https://sqlite.org/windowfunctions.html#udfwinfunc */
-    final SQLFunction func = new SQLFunction.Window(){
+    final SQLFunction func = new SQLFunction.Window<Integer>(){
-        private int accum = 0;
+
-        private void xStepInverse(int v){
+        private void xStepInverse(sqlite3_context cx, int v){
-          this.accum += v;
+          this.getAggregateState(cx,0).value += v;
-        }
-        private void xFinalValue(sqlite3_context cx){
-          sqlite3_result_int(cx, this.accum);
         }
         @Override public void xStep(sqlite3_context cx, sqlite3_value[] args){
-          this.xStepInverse(sqlite3_value_int(args[0]));
+          this.xStepInverse(cx, sqlite3_value_int(args[0]));
         }
         @Override public void xInverse(sqlite3_context cx, sqlite3_value[] args){
-          this.xStepInverse(-sqlite3_value_int(args[0]));
+          this.xStepInverse(cx, -sqlite3_value_int(args[0]));
+        }
+
+        private void xFinalValue(sqlite3_context cx, Integer v){
+          if(null == v) sqlite3_result_null(cx);
+          else sqlite3_result_int(cx, v);
         }
         @Override public void xFinal(sqlite3_context cx){
-          this.xFinalValue(cx);
+          xFinalValue(cx, this.takeAggregateState(cx));
-          this.accum = 0;
         }
         @Override public void xValue(sqlite3_context cx){
-          this.xFinalValue(cx);
+          xFinalValue(cx, this.getAggregateState(cx,null).value);
         }
       };
     int rc = sqlite3_create_function(db, "winsumint", 1, SQLITE_UTF8, func);

ext/jni/src/org/sqlite/jni/sqlite3_context.java

@@ -13,8 +13,42 @@
 */
 package org.sqlite.jni;
 
+/**
+   sqlite3_context instances are used in conjunction with user-defined
+   SQL functions (a.k.a. UDFs). They are opaque pointers.
+
+   The getAggregateContext() method corresponds to C's
+   sqlite3_aggregate_context(), with a slightly different interface in
+   order to account for cross-language differences. It serves the same
+   purposes in a slightly different way: it provides a key which is
+   stable across invocations of UDF xStep() and xFinal() pairs, to
+   which a UDF may map state across such calls (e.g. a numeric result
+   which is being accumulated).
+*/
 public class sqlite3_context extends NativePointerHolder<sqlite3_context> {
   public sqlite3_context() {
     super();
   }
+  private long aggcx = 0;
+
+  /**
+     If this object is being used in the context of an aggregate or
+     window UDF, the UDF binding layer will set a unique context value
+     here. That value will be the same across matching calls to the
+     xStep() and xFinal() routines, as well as xValue() and xInverse()
+     in window UDFs. This value can be used as a key to map state
+     which needs to persist across such calls, noting that such state
+     should be cleaned up via xFinal().
+  */
+  public long getAggregateContext(){
+    return aggcx;
+  }
+
+  /**
+     For use only by the JNI layer. It's permitted to call this even
+     though it's private.
+  */
+  private void setAggregateContext(long n){
+    aggcx = n;
+  }
 }

manifest

@@ -1,5 +1,5 @@
-C Reformulate\sjni\stests\sto\snot\srequire\sthe\s-ea\sjvm\sflag\sto\senable\sassert().
+C Add\ssupport\smaking\suse\sof\ssqlite3_aggregate_context()\s(in\sa\sroundabout\sway)\sfrom\sJava\sto\saccumulate\sstate\swithin\saggregate\sand\swindow\sUDFs.
-D 2023-07-27T22:53:02.373
+D 2023-07-28T01:12:47.322
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -232,20 +232,20 @@ F ext/icu/icu.c c074519b46baa484bb5396c7e01e051034da8884bad1a1cb7f09bbe6be3f0282
 F ext/icu/sqliteicu.h fa373836ed5a1ee7478bdf8a1650689294e41d0c89c1daab26e9ae78a32075a8
 F ext/jni/GNUmakefile 56a014dbff9516774d895ec1ae9df0ed442765b556f79a0fc0b5bc438217200d
 F ext/jni/README.md 042762dbf047667783a5bd0aec303535140f302debfbd259c612edf856661623
-F ext/jni/src/c/sqlite3-jni.c 76921edc2d1abea2cb39c21bcc49acbc307cb368e96cb7803a2c134c444c3fcd
+F ext/jni/src/c/sqlite3-jni.c 8d3ae5c0474548b1b95fea888227a4f617b9302a7e230bb5ff1b3735fe85fb03
 F ext/jni/src/c/sqlite3-jni.h c9bb150a38dce09cc2794d5aac8fa097288d9946fbb15250fd0a23c31957f506
 F ext/jni/src/org/sqlite/jni/BusyHandler.java 1b1d3e5c86cd796a0580c81b6af6550ad943baa25e47ada0dcca3aff3ebe978c
 F ext/jni/src/org/sqlite/jni/Collation.java 8dffbb00938007ad0967b2ab424d3c908413af1bbd3d212b9c9899910f1218d1
 F ext/jni/src/org/sqlite/jni/NativePointerHolder.java 70dc7bc41f80352ff3d4331e2e24f45fcd23353b3641e2f68a81bd8262215861
 F ext/jni/src/org/sqlite/jni/OutputPointer.java 08a752b58a33696c5eaf0eb9361a0966b188dec40f4a3613eb133123951f6c5f
 F ext/jni/src/org/sqlite/jni/ProgressHandler.java 5a1d7b2607eb2ef596fcf4492a49d1b3a5bdea3af9918e11716831ffd2f02284
-F ext/jni/src/org/sqlite/jni/SQLFunction.java 2f5d197f6c7d73b6031ba1a19598d7e3eee5ebad467eeee62c72e585bd6556a5
+F ext/jni/src/org/sqlite/jni/SQLFunction.java d77e0a4bb6bc0d65339aeacd6b20fc7e3b8a05f899c1f0ead90dda61f0a01522
 F ext/jni/src/org/sqlite/jni/SQLite3Jni.java 3582b30c0fb1cb39e25b9069fe8c9e2fe4f2659f4d38437b610e46143e163610
-F ext/jni/src/org/sqlite/jni/Tester1.java 460d4a521bf3386a6aafc30c382817560b8dc1001472f6b8459cadeedb9a58ea
+F ext/jni/src/org/sqlite/jni/Tester1.java 2334d1dd0efc22179654c586065c77d904830d736059b4049f9cd9e6832565bd
 F ext/jni/src/org/sqlite/jni/Tracer.java c2fe1eba4a76581b93b375a7b95ab1919e5ae60accfb06d6beb067b033e9bae1
 F ext/jni/src/org/sqlite/jni/ValueHolder.java f022873abaabf64f3dd71ab0d6037c6e71cece3b8819fa10bf26a5461dc973ee
 F ext/jni/src/org/sqlite/jni/sqlite3.java c7d0500c7269882243aafb41425928d094b2fcbdbc2fd1caffc276871cd3fae3
-F ext/jni/src/org/sqlite/jni/sqlite3_context.java d781c72237e4a442adf6726b2edf15124405c28eba0387a279078858700f567c
+F ext/jni/src/org/sqlite/jni/sqlite3_context.java 4a0b22226705a4f89d9c8093e0f51a8991cc0464864120970c915695afbba4e2
 F ext/jni/src/org/sqlite/jni/sqlite3_stmt.java 3193693440071998a66870544d1d2314f144bea397ce4c3f83ff225d587067a0
 F ext/jni/src/org/sqlite/jni/sqlite3_value.java f9d8c0766b1d1b290564cb35db8d37be54c42adc8df22ee77b8d39e3e93398cd
 F ext/lsm1/Makefile a553b728bba6c11201b795188c5708915cc4290f02b7df6ba7e8c4c943fd5cd9
@@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 7dcde2bfce54b18f391776fa1cb93c0ff6153634bedcab0007b374c06c4d4079
+P dc356667a8f4fa31a3fef1ae35873d834d27fd6a9f0818d6fb85e4751fde9fe5
-R ba5fe9ad4149aefc21881fee22f6fa73
+R 6f355aed3877133b3fb6aa6671123d94
 U stephan
-Z 2639e54196b7b2c8fbce104e63109714
+Z 75e450fbcee41582218a9562a53136a5
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-dc356667a8f4fa31a3fef1ae35873d834d27fd6a9f0818d6fb85e4751fde9fe5
+640574984741c7a9472d7f8be7bce87e736d7947ce673ae4a25008d74238ad90