Bind fts5_api::xCreateFunction() to JNI and demonstrate it with a test.

FossilOrigin-Name: c653bf16cbdccae05ab14059b140191afd5c17740fb78d756d8822986e54b17c

ext/jni/GNUmakefile

@@ -66,9 +66,12 @@ JAVA_FILES := $(patsubst %,$(dir.src.jni)/%,\
 ifeq (1,$(enable.fts5))
   JAVA_FILES += $(patsubst %,$(dir.src.jni)/%,\
     fts5_api.java \
+    fts5_tokenizer.java \
+    Fts5.java \
     Fts5Context.java \
     Fts5ExtensionApi.java \
     Fts5Function.java \
+    Fts5Tokenizer.java \
     TesterFts5.java \
   )
 endif
@@ -142,6 +145,8 @@ SQLITE_OPT := \
 # for a var which gets set in all builds but only read
 # via assert().
 
+SQLITE_OPFS += -g -DDEBUG -UNDEBUG
+
 ifeq (1,$(enable.fts5))
   SQLITE_OPT += -DSQLITE_ENABLE_FTS5
 endif
@@ -152,7 +157,8 @@ sqlite3-jni.h.in := $(dir.bld.c)/org_sqlite_jni_SQLite3Jni.h
 ifeq (1,$(enable.fts5))
   sqlite3-jni.h.in += \
     $(dir.bld.c)/org_sqlite_jni_Fts5ExtensionApi.h \
-    $(dir.bld.c)/org_sqlite_jni_fts5_api.h
+    $(dir.bld.c)/org_sqlite_jni_fts5_api.h \
+    $(dir.bld.c)/org_sqlite_jni_fts5_tokenizer.h
 endif
 sqlite3-jni.h   := $(dir.src.c)/sqlite3-jni.h
 sqlite3-jni.dll := $(dir.bld.c)/libsqlite3-jni.so

ext/jni/src/c/sqlite3-jni.c

@@ -209,6 +209,8 @@ static const struct {
   const char * const Fts5Context;
   const char * const Fts5ExtensionApi;
   const char * const fts5_api;
+  const char * const fts5_tokenizer;
+  const char * const Fts5Tokenizer;
 #endif
 } S3ClassNames = {
   "org/sqlite/jni/sqlite3",
@@ -222,7 +224,9 @@ static const struct {
 #ifdef SQLITE_ENABLE_FTS5
   "org/sqlite/jni/Fts5Context",
   "org/sqlite/jni/Fts5ExtensionApi",
-  "org/sqlite/jni/fts5_api"
+  "org/sqlite/jni/fts5_api",
+  "org/sqlite/jni/fts5_tokenizer",
+  "org/sqlite/jni/Fts5Tokenizer"
 #endif
 };
 
@@ -304,9 +308,9 @@ enum {
     Need enough space for (only) the library's NativePointerHolder
     types, a fixed count known at build-time. If we add more than this
     a fatal error will be triggered with a reminder to increase this.
-    This value needs to be, at most, the number of entries in the
+    This value needs to be exactly the number of entries in the
-    S3ClassNames object, as that value is our upper limit. The
+    S3ClassNames object. The S3ClassNames entries are the keys for
-    S3ClassNames entries are the keys for this particular cache.
+    this particular cache.
   */
   NphCache_SIZE = sizeof(S3ClassNames) / sizeof(char const *)
 };
@@ -358,6 +362,12 @@ struct JNIEnvCacheLine {
   JNIEnvCacheLine * pPrev /* Previous entry in the linked list */;
   JNIEnvCacheLine * pNext /* Next entry in the linked list */;
 #endif
+  /** TODO: NphCacheLine *pNphHit;
+
+      to help fast-track cache lookups, update this to point to the
+      most recent hit. That will speed up, e.g. the
+      sqlite3_value-to-Java-array loop.
+  */
   struct NphCacheLine nph[NphCache_SIZE];
 };
 typedef struct JNIEnvCache JNIEnvCache;
@@ -1268,49 +1278,55 @@ typedef struct {
    Converts the given (cx, argc, argv) into arguments for the given
    UDF, placing the result in the final argument. Returns 0 on
    success, SQLITE_NOMEM on allocation error.
+
+   TODO: see what we can do to optimize the
+   new_sqlite3_value_wrapper() call. e.g. find the ctor a single time
+   and call it here, rather than looking it up repeatedly.
 */
-static int udf_args(sqlite3_context * const cx,
+static int udf_args(JNIEnv *env,
+                    sqlite3_context * const cx,
                     int argc, sqlite3_value**argv,
-                    UDFState * const s,
+                    jobject * jCx, jobjectArray *jArgv){
-                    udf_jargs * const args){
   jobjectArray ja = 0;
-  JNIEnv * const env = s->env;
+  jobject jcx = new_sqlite3_context_wrapper(env, cx);
-  jobject jcx = new_sqlite3_context_wrapper(s->env, cx);
   jint i;
-  args->jcx = 0;
+  *jCx = 0;
-  args->jargv = 0;
+  *jArgv = 0;
   if(!jcx) goto error_oom;
-  ja = (*(s->env))->NewObjectArray(s->env, argc,
+  ja = (*env)->NewObjectArray(env, argc,
-                                   S3Global_env_cache(env)->globalClassObj,
+                              S3Global_env_cache(env)->globalClassObj,
-                                   NULL);
+                              NULL);
   if(!ja) goto error_oom;
   for(i = 0; i < argc; ++i){
-    jobject jsv = new_sqlite3_value_wrapper(s->env, argv[i]);
+    jobject jsv = new_sqlite3_value_wrapper(env, argv[i]);
     if(!jsv) goto error_oom;
     (*env)->SetObjectArrayElement(env, ja, i, jsv);
     UNREF_L(jsv)/*array has a ref*/;
   }
-  args->jcx = jcx;
+  *jCx = jcx;
-  args->jargv = ja;
+  *jArgv = ja;
   return 0;
 error_oom:
   sqlite3_result_error_nomem(cx);
   UNREF_L(jcx);
   UNREF_L(ja);
-  return 1;
+  return SQLITE_NOMEM;
 }
 
-static int udf_report_exception(sqlite3_context * cx, UDFState *s,
+static int udf_report_exception(sqlite3_context * cx,
+                                const char *zFuncName,
                                 const char *zFuncType){
   int rc;
   char * z =
-    sqlite3_mprintf("UDF %s.%s() threw. It should not do that.",
+    sqlite3_mprintf("Client-defined function %s.%s() threw. It should "
-                    s->zFuncName, zFuncType);
+                    "not do that.",
+                    zFuncName ? zFuncName : "<unnamed>", zFuncType);
   if(z){
     sqlite3_result_error(cx, z, -1);
     sqlite3_free(z);
     rc = SQLITE_ERROR;
   }else{
+    sqlite3_result_error_nomem(cx);
     rc = SQLITE_NOMEM;
   }
   return rc;
@@ -1325,9 +1341,9 @@ static int udf_xFSI(sqlite3_context* pCx, int argc,
                     UDFState * s,
                     jmethodID xMethodID,
                     const char * zFuncType){
-  udf_jargs args;
   JNIEnv * const env = s->env;
-  int rc = udf_args(pCx, argc, argv, s, &args);
+  udf_jargs args = {0,0};
+  int rc = udf_args(s->env, pCx, argc, argv, &args.jcx, &args.jargv);
   //MARKER(("%s.%s() pCx = %p\n", s->zFuncName, zFuncType, pCx));
   if(rc) return rc;
   //MARKER(("UDF::%s.%s()\n", s->zFuncName, zFuncType));
@@ -1337,7 +1353,7 @@ static int udf_xFSI(sqlite3_context* pCx, int argc,
   if( 0 == rc ){
     (*env)->CallVoidMethod(env, s->jObj, xMethodID, args.jcx, args.jargv);
     IFTHREW{
-      rc = udf_report_exception(pCx,s, zFuncType);
+      rc = udf_report_exception(pCx, s->zFuncName, zFuncType);
     }
   }
   UNREF_L(args.jcx);
@@ -1367,7 +1383,7 @@ static int udf_xFV(sqlite3_context* cx, UDFState * s,
   if( 0 == rc ){
     (*env)->CallVoidMethod(env, s->jObj, xMethodID, jcx);
     IFTHREW{
-      rc = udf_report_exception(cx,s, zFuncType);
+      rc = udf_report_exception(cx,s->zFuncName, zFuncType);
     }
   }
   UNREF_L(jcx);
@@ -2636,6 +2652,8 @@ JDECL(void,1do_1something_1for_1developer)(JENV_JSELF){
   Java_org_sqlite_jni_Fts5ExtensionApi_ ## Suffix
 #define JFuncNameFtsApi(Suffix)                  \
   Java_org_sqlite_jni_fts5_1api_ ## Suffix
+#define JFuncNameFtsTok(Suffix)                  \
+  Java_org_sqlite_jni_fts5_tokenizer_ ## Suffix
 
 #define JDECLFtsXA(ReturnType,Suffix)           \
   JNIEXPORT ReturnType JNICALL                  \
@@ -2643,22 +2661,82 @@ JDECL(void,1do_1something_1for_1developer)(JENV_JSELF){
 #define JDECLFtsApi(ReturnType,Suffix)          \
   JNIEXPORT ReturnType JNICALL                  \
   JFuncNameFtsApi(Suffix)
+#define JDECLFtsTok(ReturnType,Suffix)          \
+  JNIEXPORT ReturnType JNICALL                  \
+  JFuncNameFtsTok(Suffix)
 
+#define PtrGet_fts5_api(OBJ) getNativePointer(env,OBJ,S3ClassNames.fts5_api)
+#define PtrGet_fts5_tokenizer(OBJ) getNativePointer(env,OBJ,S3ClassNames.fts5_tokenizer)
 #define PtrGet_Fts5Context(OBJ) getNativePointer(env,OBJ,S3ClassNames.Fts5Context)
+#define PtrGet_Fts5Tokenizer(OBJ) getNativePointer(env,OBJ,S3ClassNames.Fts5Tokenizer)
+
+/**
+   State for binding Java-side FTS5 auxiliary functions.
+*/
+typedef struct {
+  JNIEnv * env;         /* env registered from */;
+  jobject jObj          /* functor instance */;
+  jclass klazz          /* jObj's class */;
+  char * zFuncName      /* Only for error reporting and debug logging */;
+  jmethodID jmid        /* callback member's method ID */;
+} Fts5JniAux;
+
+static void Fts5JniAux_free(Fts5JniAux * const s){
+  JNIEnv * const env = s->env;
+  if(env){
+    /*MARKER(("FTS5 aux function cleanup: %s\n", s->zFuncName));*/
+    s3jni_call_xDestroy(env, s->jObj, s->klazz);
+    UNREF_G(s->jObj);
+    UNREF_G(s->klazz);
+  }
+  sqlite3_free(s->zFuncName);
+  sqlite3_free(s);
+}
+
+static void Fts5JniAux_xDestroy(void *p){
+  if(p) Fts5JniAux_free(p);
+}
+
+static Fts5JniAux * Fts5JniAux_alloc(JNIEnv * const env, jobject jObj){
+  Fts5JniAux * s = sqlite3_malloc(sizeof(Fts5JniAux));
+  if(s){
+    const char * zSig =
+      "(Lorg/sqlite/jni/Fts5ExtensionApi;"
+      "Lorg/sqlite/jni/Fts5Context;"
+      "Lorg/sqlite/jni/sqlite3_context;"
+      "[Lorg/sqlite/jni/sqlite3_value;)V";
+    memset(s, 0, sizeof(Fts5JniAux));
+    s->env = env;
+    s->jObj = REF_G(jObj);
+    s->klazz = REF_G((*env)->GetObjectClass(env, jObj));
+    EXCEPTION_IS_FATAL("Cannot get class for FTS5 aux function object.");
+    s->jmid = (*env)->GetMethodID(env, s->klazz, "xFunction", zSig);
+    IFTHREW{
+      EXCEPTION_REPORT;
+      EXCEPTION_CLEAR;
+      Fts5JniAux_free(s);
+      s = 0;
+    }
+  }
+  return s;
+}
+
 static inline Fts5ExtensionApi const * s3jni_ftsext(void){
   return &sFts5Api/*singleton from sqlite3.c*/;
 }
 #define Fts5ExtDecl Fts5ExtensionApi const * const fext = s3jni_ftsext()
 
-#if 0
 static jobject new_Fts5Context_wrapper(JNIEnv * const env, Fts5Context *sv){
   return new_NativePointerHolder_object(env, S3ClassNames.Fts5Context, sv);
 }
-#endif
 static jobject new_fts5_api_wrapper(JNIEnv * const env, fts5_api *sv){
   return new_NativePointerHolder_object(env, S3ClassNames.fts5_api, sv);
 }
 
+/**
+   Returns a per-JNIEnv global ref to the Fts5ExtensionApi singleton
+   instance, or NULL on OOM.
+*/
 static jobject s3jni_getFts5ExensionApi(JNIEnv * const env){
   JNIEnvCacheLine * const row = S3Global_env_cache(env);
   if( !row->jFtsExt ){
@@ -2670,9 +2748,9 @@ static jobject s3jni_getFts5ExensionApi(JNIEnv * const env){
 }
 
 /*
-** Return a pointer to the fts5_api pointer for database connection db.
+** Return a pointer to the fts5_api instance for database connection
-** If an error occurs, return NULL and leave an error in the database
+** db.  If an error occurs, return NULL and leave an error in the
-** handle (accessible using sqlite3_errcode()/errmsg()).
+** database handle (accessible using sqlite3_errcode()/errmsg()).
 */
 static fts5_api *s3jni_fts5_api_from_db(sqlite3 *db){
   fts5_api *pRet = 0;
@@ -2759,6 +2837,73 @@ JDECLFtsXA(jint,xColumnTotalSize)(JENV_JSELF,jobject jCtx, jint iCol, jobject jO
   return (jint)rc;
 }
 
+/**
+   Proxy for fts5_extension_function instances plugged in via
+   fts5_api::xCreateFunction().
+*/
+static void s3jni_fts5_extension_function(Fts5ExtensionApi const *pApi,
+                                          Fts5Context *pFts,
+                                          sqlite3_context *pCx,
+                                          int argc,
+                                          sqlite3_value **argv){
+  Fts5JniAux * const pAux = pApi->xUserData(pFts);
+  JNIEnv *env;
+  jobject jpCx = 0;
+  jobjectArray jArgv = 0;
+  jobject jpFts = 0;
+  jobject jFXA;
+  int rc;
+  assert(pAux);
+  env = pAux->env;
+  jFXA = s3jni_getFts5ExensionApi(env);
+  if( !jFXA ) goto error_oom;
+  jpFts = new_Fts5Context_wrapper(env, pFts);
+  if(!jpFts) goto error_oom;
+  rc = udf_args(env, pCx, argc, argv, &jpCx, &jArgv);
+  if(rc) goto error_oom;
+  (*env)->CallVoidMethod(env, pAux->jObj, pAux->jmid,
+                         jFXA, jpFts, jpCx, jArgv);
+  IFTHREW{
+    EXCEPTION_CLEAR;
+    udf_report_exception(pCx, pAux->zFuncName, "xFunction");
+  }
+  UNREF_L(jpFts);
+  UNREF_L(jpCx);
+  UNREF_L(jArgv);
+  return;
+error_oom:
+  assert( !jArgv );
+  assert( !jpCx );
+  UNREF_L(jpFts);
+  sqlite3_result_error_nomem(pCx);
+  return;
+}
+
+JDECLFtsApi(jint,xCreateFunction)(JENV_JSELF, jstring jName, jobject jFunc){
+  fts5_api * const pApi = PtrGet_fts5_api(jSelf);
+  int rc;
+  char const * zName;
+  Fts5JniAux * pAux;
+  assert(pApi);
+  zName = JSTR_TOC(jName);
+  if(!zName) return SQLITE_NOMEM;
+  pAux = Fts5JniAux_alloc(env, jFunc);
+  if( pAux ){
+    rc = pApi->xCreateFunction(pApi, zName, pAux,
+                               s3jni_fts5_extension_function,
+                               Fts5JniAux_xDestroy);
+  }else{
+    rc = SQLITE_NOMEM;
+  }
+  if( 0==rc ){
+    pAux->zFuncName = sqlite3_mprintf("%s", zName);
+    /* OOM here is non-fatal. Ignore it. */
+  }
+  JSTR_RELEASE(jName, zName);
+  return (jint)rc;
+}
+
+
 typedef struct s3jni_fts5AuxData s3jni_fts5AuxData;
 struct s3jni_fts5AuxData {
   JNIEnv *env;
@@ -3022,10 +3167,7 @@ JDECLFtsXA(int,xSetAuxdata)(JENV_JSELF,jobject jCtx, jobject jAux){
 }
 
 /**
-   xToken() imp for xTokenize().
+   xToken() impl for xTokenize().
-
-   TODO: hold on to the byte array and avoid initializing
-   it if passed the same (z,nZ) as a previous call.
 */
 static int s3jni_xTokenize_xToken(void *p, int tFlags, const char* z,
                                   int nZ, int iStart, int iEnd){
@@ -3052,12 +3194,16 @@ static int s3jni_xTokenize_xToken(void *p, int tFlags, const char* z,
   return rc;
 }
 
-JDECLFtsXA(jint,xTokenize)(JENV_JSELF,jobject jFcx, jbyteArray jbaText,
+/**
-                           jobject jCallback){
+   Proxy for Fts5ExtensionApi.xTokenize() and fts5_tokenizer.xTokenize()
+*/
+static jint s3jni_fts5_xTokenize(JENV_JSELF, const char *zClassName,
+                                 jint tokFlags, jobject jFcx,
+                                 jbyteArray jbaText, jobject jCallback){
   Fts5ExtDecl;
   JNIEnvCacheLine * const jc = S3Global_env_cache(env);
   struct s3jni_xQueryPhraseState s;
-  int rc;
+  int rc = 0;
   jbyte * const pText = JBA_TOC(jbaText);
   jsize nText = (*env)->GetArrayLength(env, jbaText);
   jclass const klazz = jCallback ? (*env)->GetObjectClass(env, jCallback) : NULL;
@@ -3082,9 +3228,18 @@ JDECLFtsXA(jint,xTokenize)(JENV_JSELF,jobject jFcx, jbyteArray jbaText,
   s.tok.jba = REF_L(jbaText);
   s.tok.zPrev = (const char *)pText;
   s.tok.nPrev = (int)nText;
-  rc = fext->xTokenize(PtrGet_Fts5Context(jFcx),
+  if( zClassName == S3ClassNames.Fts5ExtensionApi ){
-                       (const char *)pText, (int)nText,
+    rc = fext->xTokenize(PtrGet_Fts5Context(jFcx),
-                       &s, s3jni_xTokenize_xToken);
+                         (const char *)pText, (int)nText,
+                         &s, s3jni_xTokenize_xToken);
+  }else if( zClassName == S3ClassNames.fts5_tokenizer ){
+    fts5_tokenizer * const pTok = PtrGet_fts5_tokenizer(jSelf);
+    rc = pTok->xTokenize(PtrGet_Fts5Tokenizer(jFcx), &s, tokFlags,
+                         (const char *)pText, (int)nText,
+                         s3jni_xTokenize_xToken);
+  }else{
+    (*env)->FatalError(env, "This cannot happen. Maintenance required.");
+  }
   if(s.tok.jba){
     assert( s.tok.zPrev );
     UNREF_L(s.tok.jba);
@@ -3093,6 +3248,18 @@ JDECLFtsXA(jint,xTokenize)(JENV_JSELF,jobject jFcx, jbyteArray jbaText,
   return (jint)rc;
 }
 
+JDECLFtsXA(jint,xTokenize)(JENV_JSELF,jobject jFcx, jbyteArray jbaText,
+                           jobject jCallback){
+  return s3jni_fts5_xTokenize(env, jSelf, S3ClassNames.Fts5ExtensionApi,
+                              0, jFcx, jbaText, jCallback);
+}
+
+JDECLFtsTok(jint,xTokenize)(JENV_JSELF,jobject jFcx, jint tokFlags,
+                            jbyteArray jbaText, jobject jCallback){
+  return s3jni_fts5_xTokenize(env, jSelf, S3ClassNames.Fts5Tokenizer,
+                              tokFlags, jFcx, jbaText, jCallback);
+}
+
 
 #endif /* SQLITE_ENABLE_FTS5 */
 ////////////////////////////////////////////////////////////////////////

ext/jni/src/c/sqlite3-jni.h

@@ -1771,7 +1771,7 @@ JNIEXPORT jint JNICALL Java_org_sqlite_jni_Fts5ExtensionApi_xSetAuxdata
 /*
  * Class:     org_sqlite_jni_Fts5ExtensionApi
  * Method:    xTokenize
- * Signature: (Lorg/sqlite/jni/Fts5Context;[BLorg/sqlite/jni/Fts5ExtensionApi/xTokenizeCallback;)I
+ * Signature: (Lorg/sqlite/jni/Fts5Context;[BLorg/sqlite/jni/Fts5/xTokenizeCallback;)I
  */
 JNIEXPORT jint JNICALL Java_org_sqlite_jni_Fts5ExtensionApi_xTokenize
   (JNIEnv *, jobject, jobject, jbyteArray, jobject);
@@ -1797,6 +1797,35 @@ extern "C" {
 JNIEXPORT jobject JNICALL Java_org_sqlite_jni_fts5_1api_getInstanceForDb
   (JNIEnv *, jclass, jobject);
 
+/*
+ * Class:     org_sqlite_jni_fts5_api
+ * Method:    xCreateFunction
+ * Signature: (Ljava/lang/String;Lorg/sqlite/jni/fts5_api/fts5_extension_function;)I
+ */
+JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_1api_xCreateFunction
+  (JNIEnv *, jobject, jstring, jobject);
+
+#ifdef __cplusplus
+}
+#endif
+#endif
+/* DO NOT EDIT THIS FILE - it is machine generated */
+#include <jni.h>
+/* Header for class org_sqlite_jni_fts5_tokenizer */
+
+#ifndef _Included_org_sqlite_jni_fts5_tokenizer
+#define _Included_org_sqlite_jni_fts5_tokenizer
+#ifdef __cplusplus
+extern "C" {
+#endif
+/*
+ * Class:     org_sqlite_jni_fts5_tokenizer
+ * Method:    xTokenize
+ * Signature: (Lorg/sqlite/jni/Fts5Tokenizer;I[BLorg/sqlite/jni/Fts5/xTokenizeCallback;)I
+ */
+JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_1tokenizer_xTokenize
+  (JNIEnv *, jobject, jobject, jint, jbyteArray, jobject);
+
 #ifdef __cplusplus
 }
 #endif

ext/jni/src/org/sqlite/jni/Fts5.java

@@ -0,0 +1,38 @@
+/*
+** 2023-08-05
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the JNI bindings for the sqlite3 C API.
+*/
+package org.sqlite.jni;
+
+/**
+   INCOMPLETE AND COMPLETELY UNTESTED.
+
+   A wrapper for communicating C-level (fts5_api*) instances with
+   Java. These wrappers do not own their associated pointer, they
+   simply provide a type-safe way to communicate it between Java and C
+   via JNI.
+*/
+public final class Fts5 {
+  /* Not used */
+  private Fts5(){}
+
+  //! Callback type for use with xTokenize() variants
+  public static interface xTokenizeCallback {
+    int xToken(int tFlags, byte txt[], int iStart, int iEnd);
+  }
+
+  public static final int FTS5_TOKENIZE_QUERY    = 0x0001;
+  public static final int FTS5_TOKENIZE_PREFIX   = 0x0002;
+  public static final int FTS5_TOKENIZE_DOCUMENT = 0x0004;
+  public static final int FTS5_TOKENIZE_AUX      = 0x0008;
+  public static final int FTS5_TOKEN_COLOCATED   = 0x0001;
+}

ext/jni/src/org/sqlite/jni/Fts5ExtensionApi.java

@@ -27,10 +27,6 @@ public final class Fts5ExtensionApi extends NativePointerHolder<Fts5ExtensionApi
   private Fts5ExtensionApi(){}
   private int iVersion = 2;
 
-  //! Callback type for use with xTokenize().
-  public static interface xTokenizeCallback {
-    int xToken(int tFlags, byte txt[], int iStart, int iEnd);
-  }
   public static interface xQueryPhraseCallback {
     int xCallback(Fts5ExtensionApi fapi, Fts5Context cx);
   }
@@ -68,7 +64,6 @@ public final class Fts5ExtensionApi extends NativePointerHolder<Fts5ExtensionApi
                                  @NotNull Fts5PhraseIter iter,
                                  @NotNull OutputPointer.Int32 iCol,
                                  @NotNull OutputPointer.Int32 iOff);
-
   public native int xPhraseFirstColumn(@NotNull Fts5Context cx, int iPhrase,
                                        @NotNull Fts5PhraseIter iter,
                                        @NotNull OutputPointer.Int32 iCol);
@@ -80,11 +75,8 @@ public final class Fts5ExtensionApi extends NativePointerHolder<Fts5ExtensionApi
                                        @NotNull Fts5PhraseIter iter,
                                        @NotNull OutputPointer.Int32 iCol);
   public native int xPhraseSize(@NotNull Fts5Context fcx, int iPhrase);
-
-
   public native int xQueryPhrase(@NotNull Fts5Context cx, int iPhrase,
                                  @NotNull xQueryPhraseCallback callback);
-
   public native int xRowCount(@NotNull Fts5Context fcx,
                               @NotNull OutputPointer.Int64 nRow);
   public native long xRowid(@NotNull Fts5Context cx);
@@ -95,11 +87,12 @@ public final class Fts5ExtensionApi extends NativePointerHolder<Fts5ExtensionApi
      the JNI layer will be relinquished regardless of whther pAux has
      an xDestroy() method. */
   public native int xSetAuxdata(@NotNull Fts5Context cx, @Nullable Object pAux);
-
   public native int xTokenize(@NotNull Fts5Context cx, @NotNull byte pText[],
-                              @NotNull xTokenizeCallback callback);
+                              @NotNull Fts5.xTokenizeCallback callback);
 
   /**************************************************************
   void *(*xUserData)(Fts5Context*);
+  ^^^ returns the pointer passed as the 3rd arg to
+  fts5_api::xCreateFunction.
   **************************************************************/
 }

ext/jni/src/org/sqlite/jni/Fts5Tokenizer.java

@@ -0,0 +1,30 @@
+/*
+** 2023-08-05x
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the JNI bindings for the sqlite3 C API.
+*/
+package org.sqlite.jni;
+
+/**
+   INCOMPLETE AND COMPLETELY UNTESTED.
+
+   A wrapper for communicating C-level (Fts5Tokenizer*) instances with
+   Java. These wrappers do not own their associated pointer, they
+   simply provide a type-safe way to communicate it between Java and C
+   via JNI.
+
+   At the C level, the Fts5Tokenizer type is essentially a void
+   pointer used specifically for tokenizers.
+*/
+public final class Fts5Tokenizer extends NativePointerHolder<Fts5Tokenizer> {
+  //! Only called from JNI.
+  private Fts5Tokenizer(){}
+}

ext/jni/src/org/sqlite/jni/SQLite3Jni.java

@@ -288,11 +288,6 @@ public final class SQLite3Jni {
                                                     int eTextRep,
                                                     @NotNull Collation col);
 
-  //Potential TODO, if we can sensibly map the lower-level bits to Java:
-  //public static native int sqlite3_create_fts5_function(@NotNull sqlite3 db,
-  //                                                      @NotNull String functionName,
-  //                                                      @NotNull Fts5Function func);
-
   /**
      The Java counterpart to the C-native sqlite3_create_function(),
      sqlite3_create_function_v2(), and

ext/jni/src/org/sqlite/jni/Tester1.java

@@ -83,7 +83,8 @@ public class Tester1 {
         affirm(0 == rc);
         pos = oTail.getValue();
         affirm(0 != stmt.getNativePointer());
-        rc = sqlite3_step(stmt);
+        while( SQLITE_ROW == (rc = sqlite3_step(stmt)) ){
+        }
         sqlite3_finalize(stmt);
         affirm(0 == stmt.getNativePointer());
         if(0!=rc && SQLITE_ROW!=rc && SQLITE_DONE!=rc){

ext/jni/src/org/sqlite/jni/TesterFts5.java

@@ -27,7 +27,45 @@ public class TesterFts5 {
     fts5_api fApi = fts5_api.getInstanceForDb(db);
     affirm( fApi != null );
     affirm( fApi == fts5_api.getInstanceForDb(db) /* singleton per db */ );
+
+    execSql(db, new String[] {
+        "CREATE VIRTUAL TABLE ft USING fts5(a, b);",
+        "INSERT INTO ft(rowid, a, b) VALUES(1, 'X Y', 'Y Z');",
+        "INSERT INTO ft(rowid, a, b) VALUES(2, 'A Z', 'Y Y');"
+      });
+
+    ValueHolder<Boolean> xDestroyCalled = new ValueHolder<>(false);
+    ValueHolder<Integer> xFuncCount = new ValueHolder<>(0);
+    fts5_api.fts5_extension_function func = new fts5_api.fts5_extension_function(){
+
+        public void xFunction(Fts5ExtensionApi ext, Fts5Context fCx,
+                              sqlite3_context pCx, sqlite3_value argv[]){
+          int nCols = ext.xColumnCount(fCx);
+          affirm( 2 == nCols );
+          if(false){
+            OutputPointer.String op = new OutputPointer.String();
+            for(int i = 0; i < nCols; ++i ){
+              int rc = ext.xColumnText(fCx, i, op);
+              affirm( 0 == rc );
+              outln("xFunction col "+i+": "+op.getValue());
+            }
+          }
+          ++xFuncCount.value;
+        }
+        public void xDestroy(){
+          xDestroyCalled.value = true;
+        }
+      };
+
+    int rc = fApi.xCreateFunction("myaux", func);
+    affirm( 0==rc );
+
+    affirm( 0==xFuncCount.value );
+    execSql(db, "select myaux(ft,a,b) from ft;");
+    affirm( 2==xFuncCount.value );
+    affirm( !xDestroyCalled.value );
     sqlite3_close_v2(db);
+    affirm( xDestroyCalled.value );
   }
 
   public TesterFts5(){

ext/jni/src/org/sqlite/jni/fts5_api.java

@@ -1,5 +1,5 @@
 /*
-** 2023-08-04
+** 2023-08-05
 **
 ** The author disclaims copyright to this source code.  In place of
 ** a legal notice, here is a blessing:
@@ -22,12 +22,57 @@ package org.sqlite.jni;
    via JNI.
 */
 public final class fts5_api extends NativePointerHolder<fts5_api> {
-  /* Only invoked by JNI */
+  /* Only invoked from JNI */
   private fts5_api(){}
+  public final int iVersion = 2;
 
   /**
      Returns the fts5_api instance associated with the given db, or
      null if something goes horribly wrong.
   */
   public static native fts5_api getInstanceForDb(@NotNull sqlite3 db);
+
+  public static abstract class fts5_extension_function {
+    public abstract void xFunction(Fts5ExtensionApi ext, Fts5Context fCx,
+                                   sqlite3_context pCx, sqlite3_value argv[]);
+    //! Optionally override
+    public void xDestroy(){}
+  }
+
+  // int (*xCreateTokenizer)(
+  //   fts5_api *pApi,
+  //   const char *zName,
+  //   void *pContext,
+  //   fts5_tokenizer *pTokenizer,
+  //   void (*xDestroy)(void*)
+  // );
+
+  // /* Find an existing tokenizer */
+  // int (*xFindTokenizer)(
+  //   fts5_api *pApi,
+  //   const char *zName,
+  //   void **ppContext,
+  //   fts5_tokenizer *pTokenizer
+  // );
+
+  // /* Create a new auxiliary function */
+  // int (*xCreateFunction)(
+  //   fts5_api *pApi,
+  //   const char *zName,
+  //   void *pContext,
+  //   fts5_extension_function xFunction,
+  //   void (*xDestroy)(void*)
+  // );
+
+  public native int xCreateFunction(@NotNull String name,
+                                    @NotNull fts5_extension_function xFunction);
+
+  // typedef void (*fts5_extension_function)(
+  //   const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
+  //   Fts5Context *pFts,              /* First arg to pass to pApi functions */
+  //   sqlite3_context *pCtx,          /* Context for returning result/error */
+  //   int nVal,                       /* Number of values in apVal[] array */
+  //   sqlite3_value **apVal           /* Array of trailing arguments */
+  // );
+
 }

ext/jni/src/org/sqlite/jni/fts5_tokenizer.java

@@ -0,0 +1,49 @@
+/*
+** 2023-08-05
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the JNI bindings for the sqlite3 C API.
+*/
+package org.sqlite.jni;
+
+/**
+   INCOMPLETE AND COMPLETELY UNTESTED.
+
+   A wrapper for communicating C-level (fts5_tokenizer*) instances with
+   Java. These wrappers do not own their associated pointer, they
+   simply provide a type-safe way to communicate it between Java and C
+   via JNI.
+*/
+public final class fts5_tokenizer extends NativePointerHolder<fts5_tokenizer> {
+  /* Only invoked by JNI */
+  private fts5_tokenizer(){}
+
+  // int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut);
+  // void (*xDelete)(Fts5Tokenizer*);
+
+  public native int xTokenize(@NotNull Fts5Tokenizer t, int tokFlags,
+                              @NotNull byte pText[],
+                              @NotNull Fts5.xTokenizeCallback callback);
+
+
+  // int (*xTokenize)(Fts5Tokenizer*,
+  //     void *pCtx,
+  //     int flags,            /* Mask of FTS5_TOKENIZE_* flags */
+  //     const char *pText, int nText,
+  //     int (*xToken)(
+  //       void *pCtx,         /* Copy of 2nd argument to xTokenize() */
+  //       int tflags,         /* Mask of FTS5_TOKEN_* flags */
+  //       const char *pToken, /* Pointer to buffer containing token */
+  //       int nToken,         /* Size of token in bytes */
+  //       int iStart,         /* Byte offset of token within input text */
+  //       int iEnd            /* Byte offset of end of token within input text */
+  //     )
+  // );
+}

manifest

@@ -1,5 +1,5 @@
-C Start\sadding\sfts5_api\sto\sJNI.
+C Bind\sfts5_api::xCreateFunction()\sto\sJNI\sand\sdemonstrate\sit\swith\sa\stest.
-D 2023-08-05T01:28:30.501
+D 2023-08-05T04:23:27.613
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -230,30 +230,33 @@ F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c
 F ext/icu/README.txt 7ab7ced8ae78e3a645b57e78570ff589d4c672b71370f5aa9e1cd7024f400fc9
 F ext/icu/icu.c c074519b46baa484bb5396c7e01e051034da8884bad1a1cb7f09bbe6be3f0282
 F ext/icu/sqliteicu.h fa373836ed5a1ee7478bdf8a1650689294e41d0c89c1daab26e9ae78a32075a8
-F ext/jni/GNUmakefile 9d916736e5af011664a38d27296692e237afaed41b0843f4de4e1a6136df18d3
+F ext/jni/GNUmakefile 70302fb66d4798b8341be0d702d48acc385eb68cfcf6c68d780e1d5fc218f2ff
 F ext/jni/README.md 6ff7e1f4100dee980434a6ee37a199b653bceec62e233a6e2ccde6e7ae0c58bf
-F ext/jni/src/c/sqlite3-jni.c 8c62ed298ccbe46f1b59a1ce9957a75ad8426c5dd065168316d9d2e97e54988b
+F ext/jni/src/c/sqlite3-jni.c 71a03f5348cf5b7be149d46dfe3b0210660e819383d3ba25e0733df572cf0bdc
-F ext/jni/src/c/sqlite3-jni.h 2e6450c923fe6a9c7246930d9518c1d7e0008d9cb5143868e8bd4a159fc88901
+F ext/jni/src/c/sqlite3-jni.h 526531f90d51a27e808f0758a3965b79bf92c2dd06c1fbcd3f8c37378bba7afd
 F ext/jni/src/org/sqlite/jni/BusyHandler.java 1b1d3e5c86cd796a0580c81b6af6550ad943baa25e47ada0dcca3aff3ebe978c
 F ext/jni/src/org/sqlite/jni/Collation.java 8dffbb00938007ad0967b2ab424d3c908413af1bbd3d212b9c9899910f1218d1
 F ext/jni/src/org/sqlite/jni/CollationNeeded.java ebc7cd96d46a70daa76016a308e80f70a3f21d3282787c8d139aa840fdcb1bd7
 F ext/jni/src/org/sqlite/jni/CommitHook.java 87c6a8e5138c61a8eeff018fe16d23f29219150239746032687f245938baca1a
+F ext/jni/src/org/sqlite/jni/Fts5.java 13844685231e8b4840a706db3bed84d5dfcf15be0ae7e809eac40420dba24901
 F ext/jni/src/org/sqlite/jni/Fts5Context.java 0a5a02047a6a1dd3e4a38b0e542a8dd2de365033ba30e6ae019a676305959890
-F ext/jni/src/org/sqlite/jni/Fts5ExtensionApi.java 8922069cf785492b18b15b72c29557de4f10f6462c3c560d1b4827f731f46c3f
+F ext/jni/src/org/sqlite/jni/Fts5ExtensionApi.java c2180031e76ba3079be33ef5cae7e3e383f12208845cfc4e0107a00ea82df151
 F ext/jni/src/org/sqlite/jni/Fts5Function.java 65cde7151e441fee012250a5e03277de7babcd11a0c308a832b7940574259bcc
 F ext/jni/src/org/sqlite/jni/Fts5PhraseIter.java 6642beda341c0b1b46af4e2d7f6f9ab03a7aede43277b2c92859176d6bce3be9
+F ext/jni/src/org/sqlite/jni/Fts5Tokenizer.java 91489893596b6528c0df5cd7180bd5b55809c26e2b797fb321dfcdbc1298c060
 F ext/jni/src/org/sqlite/jni/NativePointerHolder.java 9c5d901cce4f7e57c3d623f4e2476f9f79a8eed6e51b2a603f37866018e040ee
 F ext/jni/src/org/sqlite/jni/OutputPointer.java d37636dd3b82097792dae9c8c255b135153845407cdbc6689f15c475850d6c93
 F ext/jni/src/org/sqlite/jni/ProgressHandler.java 5979450e996416d28543f1d42634d308439565a99332a8bd84e424af667116cc
 F ext/jni/src/org/sqlite/jni/RollbackHook.java b04c8abcc6ade44a8a57129e33765793f69df0ba909e49ba18d73f4268d92564
 F ext/jni/src/org/sqlite/jni/SQLFunction.java 09ce81c1c637e31c3a830d4c859cce95d65f5e02ff45f8bd1985b3479381bc46
-F ext/jni/src/org/sqlite/jni/SQLite3Jni.java 78496a02c7cc65a2238f54e935af070acf4e2dbef95d7cc1ff46938c440848a4
+F ext/jni/src/org/sqlite/jni/SQLite3Jni.java 1c470a8cdb5c61218304eb76b1188e98e562b105eac816557ef512e1b48fa55a
-F ext/jni/src/org/sqlite/jni/Tester1.java ef715de2ad23ec9b982122c9e1f0dfe689d9d0d7ac6709dab2ad710811bfa50b
+F ext/jni/src/org/sqlite/jni/Tester1.java aaf6cc2c7e01e78eb208f14afa3977862eaae7dd13040acbd302544ae50c21c3
-F ext/jni/src/org/sqlite/jni/TesterFts5.java 0b1ab9f3675a593213b0f12fabe4fbe6fd16ed994d41a1f4150e760e087099f6
+F ext/jni/src/org/sqlite/jni/TesterFts5.java 0f42841c230992208a07a04b51ea39bfa719d80a763575fa511574a7409870c1
 F ext/jni/src/org/sqlite/jni/Tracer.java a5cece9f947b0af27669b8baec300b6dd7ff859c3e6a6e4a1bd8b50f9714775d
 F ext/jni/src/org/sqlite/jni/UpdateHook.java e58645a1727f8a9bbe72dc072ec5b40d9f9362cb0aa24acfe93f49ff56a9016d
 F ext/jni/src/org/sqlite/jni/ValueHolder.java f022873abaabf64f3dd71ab0d6037c6e71cece3b8819fa10bf26a5461dc973ee
-F ext/jni/src/org/sqlite/jni/fts5_api.java 6ceb87a8aea27727ff5c40abf79aa2f391e9408eeeea921b7ca02c837979ca84
+F ext/jni/src/org/sqlite/jni/fts5_api.java 794bc2bb5850333f0a4e9df557102747b6cb6064b74ecd74dcdbd0d9e0c86eb4
+F ext/jni/src/org/sqlite/jni/fts5_tokenizer.java e530b36e6437fcc500e95d5d75fbffe272bdea20d2fac6be2e1336c578fba98b
 F ext/jni/src/org/sqlite/jni/sqlite3.java 600c3ddc1ac28ee8f58669fb435fd0d21f2972c652039361fde907d4fe44eb58
 F ext/jni/src/org/sqlite/jni/sqlite3_context.java d26573fc7b309228cb49786e9078597d96232257defa955a3425d10897bca810
 F ext/jni/src/org/sqlite/jni/sqlite3_stmt.java 72a0698aeb50a183ad146cd29ee04952abb8c36021f6122656aa5ec20469f6f7
@@ -2077,8 +2080,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 23383c1dfd240ce47f504dd5c3402c9a31f166fbde5bb72d91309a5655074b33
+P 14d18fe983c83412d72fd2005a45a2b8c48d347b7bbf8ef9630ae460cff85c32
-R 1f7e3cb1d5a03c9cd76bc5a8ab1fdcba
+R df417fa0aa6fbe10d9f6667baf8e145c
 U stephan
-Z 82182ddc1d375495dd36c655052d869d
+Z 194ff6113095adcb792e4a48e3cee001
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-14d18fe983c83412d72fd2005a45a2b8c48d347b7bbf8ef9630ae460cff85c32
+c653bf16cbdccae05ab14059b140191afd5c17740fb78d756d8822986e54b17c