From ec71e555c43662481b1d704057f9660102cc0adb Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 5 Aug 2023 04:23:27 +0000 Subject: [PATCH] Bind fts5_api::xCreateFunction() to JNI and demonstrate it with a test. FossilOrigin-Name: c653bf16cbdccae05ab14059b140191afd5c17740fb78d756d8822986e54b17c --- ext/jni/GNUmakefile | 8 +- ext/jni/src/c/sqlite3-jni.c | 247 +++++++++++++++--- ext/jni/src/c/sqlite3-jni.h | 31 ++- ext/jni/src/org/sqlite/jni/Fts5.java | 38 +++ .../src/org/sqlite/jni/Fts5ExtensionApi.java | 13 +- ext/jni/src/org/sqlite/jni/Fts5Tokenizer.java | 30 +++ ext/jni/src/org/sqlite/jni/SQLite3Jni.java | 5 - ext/jni/src/org/sqlite/jni/Tester1.java | 3 +- ext/jni/src/org/sqlite/jni/TesterFts5.java | 38 +++ ext/jni/src/org/sqlite/jni/fts5_api.java | 49 +++- .../src/org/sqlite/jni/fts5_tokenizer.java | 49 ++++ manifest | 29 +- manifest.uuid | 2 +- 13 files changed, 468 insertions(+), 74 deletions(-) create mode 100644 ext/jni/src/org/sqlite/jni/Fts5.java create mode 100644 ext/jni/src/org/sqlite/jni/Fts5Tokenizer.java create mode 100644 ext/jni/src/org/sqlite/jni/fts5_tokenizer.java diff --git a/ext/jni/GNUmakefile b/ext/jni/GNUmakefile index 1375c97d5f..1440bda5f0 100644 --- a/ext/jni/GNUmakefile +++ b/ext/jni/GNUmakefile @@ -66,9 +66,12 @@ JAVA_FILES := $(patsubst %,$(dir.src.jni)/%,\ ifeq (1,$(enable.fts5)) JAVA_FILES += $(patsubst %,$(dir.src.jni)/%,\ fts5_api.java \ + fts5_tokenizer.java \ + Fts5.java \ Fts5Context.java \ Fts5ExtensionApi.java \ Fts5Function.java \ + Fts5Tokenizer.java \ TesterFts5.java \ ) endif @@ -142,6 +145,8 @@ SQLITE_OPT := \ # for a var which gets set in all builds but only read # via assert(). +SQLITE_OPFS += -g -DDEBUG -UNDEBUG + ifeq (1,$(enable.fts5)) SQLITE_OPT += -DSQLITE_ENABLE_FTS5 endif @@ -152,7 +157,8 @@ sqlite3-jni.h.in := $(dir.bld.c)/org_sqlite_jni_SQLite3Jni.h ifeq (1,$(enable.fts5)) sqlite3-jni.h.in += \ $(dir.bld.c)/org_sqlite_jni_Fts5ExtensionApi.h \ - $(dir.bld.c)/org_sqlite_jni_fts5_api.h + $(dir.bld.c)/org_sqlite_jni_fts5_api.h \ + $(dir.bld.c)/org_sqlite_jni_fts5_tokenizer.h endif sqlite3-jni.h := $(dir.src.c)/sqlite3-jni.h sqlite3-jni.dll := $(dir.bld.c)/libsqlite3-jni.so diff --git a/ext/jni/src/c/sqlite3-jni.c b/ext/jni/src/c/sqlite3-jni.c index e865c259b9..c2397921b2 100644 --- a/ext/jni/src/c/sqlite3-jni.c +++ b/ext/jni/src/c/sqlite3-jni.c @@ -209,6 +209,8 @@ static const struct { const char * const Fts5Context; const char * const Fts5ExtensionApi; const char * const fts5_api; + const char * const fts5_tokenizer; + const char * const Fts5Tokenizer; #endif } S3ClassNames = { "org/sqlite/jni/sqlite3", @@ -222,7 +224,9 @@ static const struct { #ifdef SQLITE_ENABLE_FTS5 "org/sqlite/jni/Fts5Context", "org/sqlite/jni/Fts5ExtensionApi", - "org/sqlite/jni/fts5_api" + "org/sqlite/jni/fts5_api", + "org/sqlite/jni/fts5_tokenizer", + "org/sqlite/jni/Fts5Tokenizer" #endif }; @@ -304,9 +308,9 @@ enum { Need enough space for (only) the library's NativePointerHolder types, a fixed count known at build-time. If we add more than this a fatal error will be triggered with a reminder to increase this. - This value needs to be, at most, the number of entries in the - S3ClassNames object, as that value is our upper limit. The - S3ClassNames entries are the keys for this particular cache. + This value needs to be exactly the number of entries in the + S3ClassNames object. The S3ClassNames entries are the keys for + this particular cache. */ NphCache_SIZE = sizeof(S3ClassNames) / sizeof(char const *) }; @@ -358,6 +362,12 @@ struct JNIEnvCacheLine { JNIEnvCacheLine * pPrev /* Previous entry in the linked list */; JNIEnvCacheLine * pNext /* Next entry in the linked list */; #endif + /** TODO: NphCacheLine *pNphHit; + + to help fast-track cache lookups, update this to point to the + most recent hit. That will speed up, e.g. the + sqlite3_value-to-Java-array loop. + */ struct NphCacheLine nph[NphCache_SIZE]; }; typedef struct JNIEnvCache JNIEnvCache; @@ -1268,49 +1278,55 @@ typedef struct { Converts the given (cx, argc, argv) into arguments for the given UDF, placing the result in the final argument. Returns 0 on success, SQLITE_NOMEM on allocation error. + + TODO: see what we can do to optimize the + new_sqlite3_value_wrapper() call. e.g. find the ctor a single time + and call it here, rather than looking it up repeatedly. */ -static int udf_args(sqlite3_context * const cx, +static int udf_args(JNIEnv *env, + sqlite3_context * const cx, int argc, sqlite3_value**argv, - UDFState * const s, - udf_jargs * const args){ + jobject * jCx, jobjectArray *jArgv){ jobjectArray ja = 0; - JNIEnv * const env = s->env; - jobject jcx = new_sqlite3_context_wrapper(s->env, cx); + jobject jcx = new_sqlite3_context_wrapper(env, cx); jint i; - args->jcx = 0; - args->jargv = 0; + *jCx = 0; + *jArgv = 0; if(!jcx) goto error_oom; - ja = (*(s->env))->NewObjectArray(s->env, argc, - S3Global_env_cache(env)->globalClassObj, - NULL); + ja = (*env)->NewObjectArray(env, argc, + S3Global_env_cache(env)->globalClassObj, + NULL); if(!ja) goto error_oom; for(i = 0; i < argc; ++i){ - jobject jsv = new_sqlite3_value_wrapper(s->env, argv[i]); + jobject jsv = new_sqlite3_value_wrapper(env, argv[i]); if(!jsv) goto error_oom; (*env)->SetObjectArrayElement(env, ja, i, jsv); UNREF_L(jsv)/*array has a ref*/; } - args->jcx = jcx; - args->jargv = ja; + *jCx = jcx; + *jArgv = ja; return 0; error_oom: sqlite3_result_error_nomem(cx); UNREF_L(jcx); UNREF_L(ja); - return 1; + return SQLITE_NOMEM; } -static int udf_report_exception(sqlite3_context * cx, UDFState *s, +static int udf_report_exception(sqlite3_context * cx, + const char *zFuncName, const char *zFuncType){ int rc; char * z = - sqlite3_mprintf("UDF %s.%s() threw. It should not do that.", - s->zFuncName, zFuncType); + sqlite3_mprintf("Client-defined function %s.%s() threw. It should " + "not do that.", + zFuncName ? zFuncName : "", zFuncType); if(z){ sqlite3_result_error(cx, z, -1); sqlite3_free(z); rc = SQLITE_ERROR; }else{ + sqlite3_result_error_nomem(cx); rc = SQLITE_NOMEM; } return rc; @@ -1325,9 +1341,9 @@ static int udf_xFSI(sqlite3_context* pCx, int argc, UDFState * s, jmethodID xMethodID, const char * zFuncType){ - udf_jargs args; JNIEnv * const env = s->env; - int rc = udf_args(pCx, argc, argv, s, &args); + udf_jargs args = {0,0}; + int rc = udf_args(s->env, pCx, argc, argv, &args.jcx, &args.jargv); //MARKER(("%s.%s() pCx = %p\n", s->zFuncName, zFuncType, pCx)); if(rc) return rc; //MARKER(("UDF::%s.%s()\n", s->zFuncName, zFuncType)); @@ -1337,7 +1353,7 @@ static int udf_xFSI(sqlite3_context* pCx, int argc, if( 0 == rc ){ (*env)->CallVoidMethod(env, s->jObj, xMethodID, args.jcx, args.jargv); IFTHREW{ - rc = udf_report_exception(pCx,s, zFuncType); + rc = udf_report_exception(pCx, s->zFuncName, zFuncType); } } UNREF_L(args.jcx); @@ -1367,7 +1383,7 @@ static int udf_xFV(sqlite3_context* cx, UDFState * s, if( 0 == rc ){ (*env)->CallVoidMethod(env, s->jObj, xMethodID, jcx); IFTHREW{ - rc = udf_report_exception(cx,s, zFuncType); + rc = udf_report_exception(cx,s->zFuncName, zFuncType); } } UNREF_L(jcx); @@ -2636,6 +2652,8 @@ JDECL(void,1do_1something_1for_1developer)(JENV_JSELF){ Java_org_sqlite_jni_Fts5ExtensionApi_ ## Suffix #define JFuncNameFtsApi(Suffix) \ Java_org_sqlite_jni_fts5_1api_ ## Suffix +#define JFuncNameFtsTok(Suffix) \ + Java_org_sqlite_jni_fts5_tokenizer_ ## Suffix #define JDECLFtsXA(ReturnType,Suffix) \ JNIEXPORT ReturnType JNICALL \ @@ -2643,22 +2661,82 @@ JDECL(void,1do_1something_1for_1developer)(JENV_JSELF){ #define JDECLFtsApi(ReturnType,Suffix) \ JNIEXPORT ReturnType JNICALL \ JFuncNameFtsApi(Suffix) +#define JDECLFtsTok(ReturnType,Suffix) \ + JNIEXPORT ReturnType JNICALL \ + JFuncNameFtsTok(Suffix) +#define PtrGet_fts5_api(OBJ) getNativePointer(env,OBJ,S3ClassNames.fts5_api) +#define PtrGet_fts5_tokenizer(OBJ) getNativePointer(env,OBJ,S3ClassNames.fts5_tokenizer) #define PtrGet_Fts5Context(OBJ) getNativePointer(env,OBJ,S3ClassNames.Fts5Context) +#define PtrGet_Fts5Tokenizer(OBJ) getNativePointer(env,OBJ,S3ClassNames.Fts5Tokenizer) + +/** + State for binding Java-side FTS5 auxiliary functions. +*/ +typedef struct { + JNIEnv * env; /* env registered from */; + jobject jObj /* functor instance */; + jclass klazz /* jObj's class */; + char * zFuncName /* Only for error reporting and debug logging */; + jmethodID jmid /* callback member's method ID */; +} Fts5JniAux; + +static void Fts5JniAux_free(Fts5JniAux * const s){ + JNIEnv * const env = s->env; + if(env){ + /*MARKER(("FTS5 aux function cleanup: %s\n", s->zFuncName));*/ + s3jni_call_xDestroy(env, s->jObj, s->klazz); + UNREF_G(s->jObj); + UNREF_G(s->klazz); + } + sqlite3_free(s->zFuncName); + sqlite3_free(s); +} + +static void Fts5JniAux_xDestroy(void *p){ + if(p) Fts5JniAux_free(p); +} + +static Fts5JniAux * Fts5JniAux_alloc(JNIEnv * const env, jobject jObj){ + Fts5JniAux * s = sqlite3_malloc(sizeof(Fts5JniAux)); + if(s){ + const char * zSig = + "(Lorg/sqlite/jni/Fts5ExtensionApi;" + "Lorg/sqlite/jni/Fts5Context;" + "Lorg/sqlite/jni/sqlite3_context;" + "[Lorg/sqlite/jni/sqlite3_value;)V"; + memset(s, 0, sizeof(Fts5JniAux)); + s->env = env; + s->jObj = REF_G(jObj); + s->klazz = REF_G((*env)->GetObjectClass(env, jObj)); + EXCEPTION_IS_FATAL("Cannot get class for FTS5 aux function object."); + s->jmid = (*env)->GetMethodID(env, s->klazz, "xFunction", zSig); + IFTHREW{ + EXCEPTION_REPORT; + EXCEPTION_CLEAR; + Fts5JniAux_free(s); + s = 0; + } + } + return s; +} + static inline Fts5ExtensionApi const * s3jni_ftsext(void){ return &sFts5Api/*singleton from sqlite3.c*/; } #define Fts5ExtDecl Fts5ExtensionApi const * const fext = s3jni_ftsext() -#if 0 static jobject new_Fts5Context_wrapper(JNIEnv * const env, Fts5Context *sv){ return new_NativePointerHolder_object(env, S3ClassNames.Fts5Context, sv); } -#endif static jobject new_fts5_api_wrapper(JNIEnv * const env, fts5_api *sv){ return new_NativePointerHolder_object(env, S3ClassNames.fts5_api, sv); } +/** + Returns a per-JNIEnv global ref to the Fts5ExtensionApi singleton + instance, or NULL on OOM. +*/ static jobject s3jni_getFts5ExensionApi(JNIEnv * const env){ JNIEnvCacheLine * const row = S3Global_env_cache(env); if( !row->jFtsExt ){ @@ -2670,9 +2748,9 @@ static jobject s3jni_getFts5ExensionApi(JNIEnv * const env){ } /* -** Return a pointer to the fts5_api pointer for database connection db. -** If an error occurs, return NULL and leave an error in the database -** handle (accessible using sqlite3_errcode()/errmsg()). +** Return a pointer to the fts5_api instance for database connection +** db. If an error occurs, return NULL and leave an error in the +** database handle (accessible using sqlite3_errcode()/errmsg()). */ static fts5_api *s3jni_fts5_api_from_db(sqlite3 *db){ fts5_api *pRet = 0; @@ -2759,6 +2837,73 @@ JDECLFtsXA(jint,xColumnTotalSize)(JENV_JSELF,jobject jCtx, jint iCol, jobject jO return (jint)rc; } +/** + Proxy for fts5_extension_function instances plugged in via + fts5_api::xCreateFunction(). +*/ +static void s3jni_fts5_extension_function(Fts5ExtensionApi const *pApi, + Fts5Context *pFts, + sqlite3_context *pCx, + int argc, + sqlite3_value **argv){ + Fts5JniAux * const pAux = pApi->xUserData(pFts); + JNIEnv *env; + jobject jpCx = 0; + jobjectArray jArgv = 0; + jobject jpFts = 0; + jobject jFXA; + int rc; + assert(pAux); + env = pAux->env; + jFXA = s3jni_getFts5ExensionApi(env); + if( !jFXA ) goto error_oom; + jpFts = new_Fts5Context_wrapper(env, pFts); + if(!jpFts) goto error_oom; + rc = udf_args(env, pCx, argc, argv, &jpCx, &jArgv); + if(rc) goto error_oom; + (*env)->CallVoidMethod(env, pAux->jObj, pAux->jmid, + jFXA, jpFts, jpCx, jArgv); + IFTHREW{ + EXCEPTION_CLEAR; + udf_report_exception(pCx, pAux->zFuncName, "xFunction"); + } + UNREF_L(jpFts); + UNREF_L(jpCx); + UNREF_L(jArgv); + return; +error_oom: + assert( !jArgv ); + assert( !jpCx ); + UNREF_L(jpFts); + sqlite3_result_error_nomem(pCx); + return; +} + +JDECLFtsApi(jint,xCreateFunction)(JENV_JSELF, jstring jName, jobject jFunc){ + fts5_api * const pApi = PtrGet_fts5_api(jSelf); + int rc; + char const * zName; + Fts5JniAux * pAux; + assert(pApi); + zName = JSTR_TOC(jName); + if(!zName) return SQLITE_NOMEM; + pAux = Fts5JniAux_alloc(env, jFunc); + if( pAux ){ + rc = pApi->xCreateFunction(pApi, zName, pAux, + s3jni_fts5_extension_function, + Fts5JniAux_xDestroy); + }else{ + rc = SQLITE_NOMEM; + } + if( 0==rc ){ + pAux->zFuncName = sqlite3_mprintf("%s", zName); + /* OOM here is non-fatal. Ignore it. */ + } + JSTR_RELEASE(jName, zName); + return (jint)rc; +} + + typedef struct s3jni_fts5AuxData s3jni_fts5AuxData; struct s3jni_fts5AuxData { JNIEnv *env; @@ -3022,10 +3167,7 @@ JDECLFtsXA(int,xSetAuxdata)(JENV_JSELF,jobject jCtx, jobject jAux){ } /** - xToken() imp for xTokenize(). - - TODO: hold on to the byte array and avoid initializing - it if passed the same (z,nZ) as a previous call. + xToken() impl for xTokenize(). */ static int s3jni_xTokenize_xToken(void *p, int tFlags, const char* z, int nZ, int iStart, int iEnd){ @@ -3052,12 +3194,16 @@ static int s3jni_xTokenize_xToken(void *p, int tFlags, const char* z, return rc; } -JDECLFtsXA(jint,xTokenize)(JENV_JSELF,jobject jFcx, jbyteArray jbaText, - jobject jCallback){ +/** + Proxy for Fts5ExtensionApi.xTokenize() and fts5_tokenizer.xTokenize() +*/ +static jint s3jni_fts5_xTokenize(JENV_JSELF, const char *zClassName, + jint tokFlags, jobject jFcx, + jbyteArray jbaText, jobject jCallback){ Fts5ExtDecl; JNIEnvCacheLine * const jc = S3Global_env_cache(env); struct s3jni_xQueryPhraseState s; - int rc; + int rc = 0; jbyte * const pText = JBA_TOC(jbaText); jsize nText = (*env)->GetArrayLength(env, jbaText); jclass const klazz = jCallback ? (*env)->GetObjectClass(env, jCallback) : NULL; @@ -3082,9 +3228,18 @@ JDECLFtsXA(jint,xTokenize)(JENV_JSELF,jobject jFcx, jbyteArray jbaText, s.tok.jba = REF_L(jbaText); s.tok.zPrev = (const char *)pText; s.tok.nPrev = (int)nText; - rc = fext->xTokenize(PtrGet_Fts5Context(jFcx), - (const char *)pText, (int)nText, - &s, s3jni_xTokenize_xToken); + if( zClassName == S3ClassNames.Fts5ExtensionApi ){ + rc = fext->xTokenize(PtrGet_Fts5Context(jFcx), + (const char *)pText, (int)nText, + &s, s3jni_xTokenize_xToken); + }else if( zClassName == S3ClassNames.fts5_tokenizer ){ + fts5_tokenizer * const pTok = PtrGet_fts5_tokenizer(jSelf); + rc = pTok->xTokenize(PtrGet_Fts5Tokenizer(jFcx), &s, tokFlags, + (const char *)pText, (int)nText, + s3jni_xTokenize_xToken); + }else{ + (*env)->FatalError(env, "This cannot happen. Maintenance required."); + } if(s.tok.jba){ assert( s.tok.zPrev ); UNREF_L(s.tok.jba); @@ -3093,6 +3248,18 @@ JDECLFtsXA(jint,xTokenize)(JENV_JSELF,jobject jFcx, jbyteArray jbaText, return (jint)rc; } +JDECLFtsXA(jint,xTokenize)(JENV_JSELF,jobject jFcx, jbyteArray jbaText, + jobject jCallback){ + return s3jni_fts5_xTokenize(env, jSelf, S3ClassNames.Fts5ExtensionApi, + 0, jFcx, jbaText, jCallback); +} + +JDECLFtsTok(jint,xTokenize)(JENV_JSELF,jobject jFcx, jint tokFlags, + jbyteArray jbaText, jobject jCallback){ + return s3jni_fts5_xTokenize(env, jSelf, S3ClassNames.Fts5Tokenizer, + tokFlags, jFcx, jbaText, jCallback); +} + #endif /* SQLITE_ENABLE_FTS5 */ //////////////////////////////////////////////////////////////////////// diff --git a/ext/jni/src/c/sqlite3-jni.h b/ext/jni/src/c/sqlite3-jni.h index 9f842c8aff..3baf8bef33 100644 --- a/ext/jni/src/c/sqlite3-jni.h +++ b/ext/jni/src/c/sqlite3-jni.h @@ -1771,7 +1771,7 @@ JNIEXPORT jint JNICALL Java_org_sqlite_jni_Fts5ExtensionApi_xSetAuxdata /* * Class: org_sqlite_jni_Fts5ExtensionApi * Method: xTokenize - * Signature: (Lorg/sqlite/jni/Fts5Context;[BLorg/sqlite/jni/Fts5ExtensionApi/xTokenizeCallback;)I + * Signature: (Lorg/sqlite/jni/Fts5Context;[BLorg/sqlite/jni/Fts5/xTokenizeCallback;)I */ JNIEXPORT jint JNICALL Java_org_sqlite_jni_Fts5ExtensionApi_xTokenize (JNIEnv *, jobject, jobject, jbyteArray, jobject); @@ -1797,6 +1797,35 @@ extern "C" { JNIEXPORT jobject JNICALL Java_org_sqlite_jni_fts5_1api_getInstanceForDb (JNIEnv *, jclass, jobject); +/* + * Class: org_sqlite_jni_fts5_api + * Method: xCreateFunction + * Signature: (Ljava/lang/String;Lorg/sqlite/jni/fts5_api/fts5_extension_function;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_1api_xCreateFunction + (JNIEnv *, jobject, jstring, jobject); + +#ifdef __cplusplus +} +#endif +#endif +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_sqlite_jni_fts5_tokenizer */ + +#ifndef _Included_org_sqlite_jni_fts5_tokenizer +#define _Included_org_sqlite_jni_fts5_tokenizer +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_sqlite_jni_fts5_tokenizer + * Method: xTokenize + * Signature: (Lorg/sqlite/jni/Fts5Tokenizer;I[BLorg/sqlite/jni/Fts5/xTokenizeCallback;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_1tokenizer_xTokenize + (JNIEnv *, jobject, jobject, jint, jbyteArray, jobject); + #ifdef __cplusplus } #endif diff --git a/ext/jni/src/org/sqlite/jni/Fts5.java b/ext/jni/src/org/sqlite/jni/Fts5.java new file mode 100644 index 0000000000..102cf575a8 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/Fts5.java @@ -0,0 +1,38 @@ +/* +** 2023-08-05 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni; + +/** + INCOMPLETE AND COMPLETELY UNTESTED. + + A wrapper for communicating C-level (fts5_api*) instances with + Java. These wrappers do not own their associated pointer, they + simply provide a type-safe way to communicate it between Java and C + via JNI. +*/ +public final class Fts5 { + /* Not used */ + private Fts5(){} + + //! Callback type for use with xTokenize() variants + public static interface xTokenizeCallback { + int xToken(int tFlags, byte txt[], int iStart, int iEnd); + } + + public static final int FTS5_TOKENIZE_QUERY = 0x0001; + public static final int FTS5_TOKENIZE_PREFIX = 0x0002; + public static final int FTS5_TOKENIZE_DOCUMENT = 0x0004; + public static final int FTS5_TOKENIZE_AUX = 0x0008; + public static final int FTS5_TOKEN_COLOCATED = 0x0001; +} diff --git a/ext/jni/src/org/sqlite/jni/Fts5ExtensionApi.java b/ext/jni/src/org/sqlite/jni/Fts5ExtensionApi.java index b7ef3b3883..328ed4c1d3 100644 --- a/ext/jni/src/org/sqlite/jni/Fts5ExtensionApi.java +++ b/ext/jni/src/org/sqlite/jni/Fts5ExtensionApi.java @@ -27,10 +27,6 @@ public final class Fts5ExtensionApi extends NativePointerHolder { + //! Only called from JNI. + private Fts5Tokenizer(){} +} diff --git a/ext/jni/src/org/sqlite/jni/SQLite3Jni.java b/ext/jni/src/org/sqlite/jni/SQLite3Jni.java index 03401dab75..bcaa96a100 100644 --- a/ext/jni/src/org/sqlite/jni/SQLite3Jni.java +++ b/ext/jni/src/org/sqlite/jni/SQLite3Jni.java @@ -288,11 +288,6 @@ public final class SQLite3Jni { int eTextRep, @NotNull Collation col); - //Potential TODO, if we can sensibly map the lower-level bits to Java: - //public static native int sqlite3_create_fts5_function(@NotNull sqlite3 db, - // @NotNull String functionName, - // @NotNull Fts5Function func); - /** The Java counterpart to the C-native sqlite3_create_function(), sqlite3_create_function_v2(), and diff --git a/ext/jni/src/org/sqlite/jni/Tester1.java b/ext/jni/src/org/sqlite/jni/Tester1.java index 5dd6971398..1e1dbabc50 100644 --- a/ext/jni/src/org/sqlite/jni/Tester1.java +++ b/ext/jni/src/org/sqlite/jni/Tester1.java @@ -83,7 +83,8 @@ public class Tester1 { affirm(0 == rc); pos = oTail.getValue(); affirm(0 != stmt.getNativePointer()); - rc = sqlite3_step(stmt); + while( SQLITE_ROW == (rc = sqlite3_step(stmt)) ){ + } sqlite3_finalize(stmt); affirm(0 == stmt.getNativePointer()); if(0!=rc && SQLITE_ROW!=rc && SQLITE_DONE!=rc){ diff --git a/ext/jni/src/org/sqlite/jni/TesterFts5.java b/ext/jni/src/org/sqlite/jni/TesterFts5.java index 3b17c2a49b..3b529e70cc 100644 --- a/ext/jni/src/org/sqlite/jni/TesterFts5.java +++ b/ext/jni/src/org/sqlite/jni/TesterFts5.java @@ -27,7 +27,45 @@ public class TesterFts5 { fts5_api fApi = fts5_api.getInstanceForDb(db); affirm( fApi != null ); affirm( fApi == fts5_api.getInstanceForDb(db) /* singleton per db */ ); + + execSql(db, new String[] { + "CREATE VIRTUAL TABLE ft USING fts5(a, b);", + "INSERT INTO ft(rowid, a, b) VALUES(1, 'X Y', 'Y Z');", + "INSERT INTO ft(rowid, a, b) VALUES(2, 'A Z', 'Y Y');" + }); + + ValueHolder xDestroyCalled = new ValueHolder<>(false); + ValueHolder xFuncCount = new ValueHolder<>(0); + fts5_api.fts5_extension_function func = new fts5_api.fts5_extension_function(){ + + public void xFunction(Fts5ExtensionApi ext, Fts5Context fCx, + sqlite3_context pCx, sqlite3_value argv[]){ + int nCols = ext.xColumnCount(fCx); + affirm( 2 == nCols ); + if(false){ + OutputPointer.String op = new OutputPointer.String(); + for(int i = 0; i < nCols; ++i ){ + int rc = ext.xColumnText(fCx, i, op); + affirm( 0 == rc ); + outln("xFunction col "+i+": "+op.getValue()); + } + } + ++xFuncCount.value; + } + public void xDestroy(){ + xDestroyCalled.value = true; + } + }; + + int rc = fApi.xCreateFunction("myaux", func); + affirm( 0==rc ); + + affirm( 0==xFuncCount.value ); + execSql(db, "select myaux(ft,a,b) from ft;"); + affirm( 2==xFuncCount.value ); + affirm( !xDestroyCalled.value ); sqlite3_close_v2(db); + affirm( xDestroyCalled.value ); } public TesterFts5(){ diff --git a/ext/jni/src/org/sqlite/jni/fts5_api.java b/ext/jni/src/org/sqlite/jni/fts5_api.java index a4cff7d176..6f48859769 100644 --- a/ext/jni/src/org/sqlite/jni/fts5_api.java +++ b/ext/jni/src/org/sqlite/jni/fts5_api.java @@ -1,5 +1,5 @@ /* -** 2023-08-04 +** 2023-08-05 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -22,12 +22,57 @@ package org.sqlite.jni; via JNI. */ public final class fts5_api extends NativePointerHolder { - /* Only invoked by JNI */ + /* Only invoked from JNI */ private fts5_api(){} + public final int iVersion = 2; /** Returns the fts5_api instance associated with the given db, or null if something goes horribly wrong. */ public static native fts5_api getInstanceForDb(@NotNull sqlite3 db); + + public static abstract class fts5_extension_function { + public abstract void xFunction(Fts5ExtensionApi ext, Fts5Context fCx, + sqlite3_context pCx, sqlite3_value argv[]); + //! Optionally override + public void xDestroy(){} + } + + // int (*xCreateTokenizer)( + // fts5_api *pApi, + // const char *zName, + // void *pContext, + // fts5_tokenizer *pTokenizer, + // void (*xDestroy)(void*) + // ); + + // /* Find an existing tokenizer */ + // int (*xFindTokenizer)( + // fts5_api *pApi, + // const char *zName, + // void **ppContext, + // fts5_tokenizer *pTokenizer + // ); + + // /* Create a new auxiliary function */ + // int (*xCreateFunction)( + // fts5_api *pApi, + // const char *zName, + // void *pContext, + // fts5_extension_function xFunction, + // void (*xDestroy)(void*) + // ); + + public native int xCreateFunction(@NotNull String name, + @NotNull fts5_extension_function xFunction); + + // typedef void (*fts5_extension_function)( + // const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + // Fts5Context *pFts, /* First arg to pass to pApi functions */ + // sqlite3_context *pCtx, /* Context for returning result/error */ + // int nVal, /* Number of values in apVal[] array */ + // sqlite3_value **apVal /* Array of trailing arguments */ + // ); + } diff --git a/ext/jni/src/org/sqlite/jni/fts5_tokenizer.java b/ext/jni/src/org/sqlite/jni/fts5_tokenizer.java new file mode 100644 index 0000000000..097a0cc055 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/fts5_tokenizer.java @@ -0,0 +1,49 @@ +/* +** 2023-08-05 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni; + +/** + INCOMPLETE AND COMPLETELY UNTESTED. + + A wrapper for communicating C-level (fts5_tokenizer*) instances with + Java. These wrappers do not own their associated pointer, they + simply provide a type-safe way to communicate it between Java and C + via JNI. +*/ +public final class fts5_tokenizer extends NativePointerHolder { + /* Only invoked by JNI */ + private fts5_tokenizer(){} + + // int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); + // void (*xDelete)(Fts5Tokenizer*); + + public native int xTokenize(@NotNull Fts5Tokenizer t, int tokFlags, + @NotNull byte pText[], + @NotNull Fts5.xTokenizeCallback callback); + + + // int (*xTokenize)(Fts5Tokenizer*, + // void *pCtx, + // int flags, /* Mask of FTS5_TOKENIZE_* flags */ + // const char *pText, int nText, + // int (*xToken)( + // void *pCtx, /* Copy of 2nd argument to xTokenize() */ + // int tflags, /* Mask of FTS5_TOKEN_* flags */ + // const char *pToken, /* Pointer to buffer containing token */ + // int nToken, /* Size of token in bytes */ + // int iStart, /* Byte offset of token within input text */ + // int iEnd /* Byte offset of end of token within input text */ + // ) + // ); +} diff --git a/manifest b/manifest index 61447c603c..f5e81511fe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Start\sadding\sfts5_api\sto\sJNI. -D 2023-08-05T01:28:30.501 +C Bind\sfts5_api::xCreateFunction()\sto\sJNI\sand\sdemonstrate\sit\swith\sa\stest. +D 2023-08-05T04:23:27.613 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -230,30 +230,33 @@ F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c F ext/icu/README.txt 7ab7ced8ae78e3a645b57e78570ff589d4c672b71370f5aa9e1cd7024f400fc9 F ext/icu/icu.c c074519b46baa484bb5396c7e01e051034da8884bad1a1cb7f09bbe6be3f0282 F ext/icu/sqliteicu.h fa373836ed5a1ee7478bdf8a1650689294e41d0c89c1daab26e9ae78a32075a8 -F ext/jni/GNUmakefile 9d916736e5af011664a38d27296692e237afaed41b0843f4de4e1a6136df18d3 +F ext/jni/GNUmakefile 70302fb66d4798b8341be0d702d48acc385eb68cfcf6c68d780e1d5fc218f2ff F ext/jni/README.md 6ff7e1f4100dee980434a6ee37a199b653bceec62e233a6e2ccde6e7ae0c58bf -F ext/jni/src/c/sqlite3-jni.c 8c62ed298ccbe46f1b59a1ce9957a75ad8426c5dd065168316d9d2e97e54988b -F ext/jni/src/c/sqlite3-jni.h 2e6450c923fe6a9c7246930d9518c1d7e0008d9cb5143868e8bd4a159fc88901 +F ext/jni/src/c/sqlite3-jni.c 71a03f5348cf5b7be149d46dfe3b0210660e819383d3ba25e0733df572cf0bdc +F ext/jni/src/c/sqlite3-jni.h 526531f90d51a27e808f0758a3965b79bf92c2dd06c1fbcd3f8c37378bba7afd F ext/jni/src/org/sqlite/jni/BusyHandler.java 1b1d3e5c86cd796a0580c81b6af6550ad943baa25e47ada0dcca3aff3ebe978c F ext/jni/src/org/sqlite/jni/Collation.java 8dffbb00938007ad0967b2ab424d3c908413af1bbd3d212b9c9899910f1218d1 F ext/jni/src/org/sqlite/jni/CollationNeeded.java ebc7cd96d46a70daa76016a308e80f70a3f21d3282787c8d139aa840fdcb1bd7 F ext/jni/src/org/sqlite/jni/CommitHook.java 87c6a8e5138c61a8eeff018fe16d23f29219150239746032687f245938baca1a +F ext/jni/src/org/sqlite/jni/Fts5.java 13844685231e8b4840a706db3bed84d5dfcf15be0ae7e809eac40420dba24901 F ext/jni/src/org/sqlite/jni/Fts5Context.java 0a5a02047a6a1dd3e4a38b0e542a8dd2de365033ba30e6ae019a676305959890 -F ext/jni/src/org/sqlite/jni/Fts5ExtensionApi.java 8922069cf785492b18b15b72c29557de4f10f6462c3c560d1b4827f731f46c3f +F ext/jni/src/org/sqlite/jni/Fts5ExtensionApi.java c2180031e76ba3079be33ef5cae7e3e383f12208845cfc4e0107a00ea82df151 F ext/jni/src/org/sqlite/jni/Fts5Function.java 65cde7151e441fee012250a5e03277de7babcd11a0c308a832b7940574259bcc F ext/jni/src/org/sqlite/jni/Fts5PhraseIter.java 6642beda341c0b1b46af4e2d7f6f9ab03a7aede43277b2c92859176d6bce3be9 +F ext/jni/src/org/sqlite/jni/Fts5Tokenizer.java 91489893596b6528c0df5cd7180bd5b55809c26e2b797fb321dfcdbc1298c060 F ext/jni/src/org/sqlite/jni/NativePointerHolder.java 9c5d901cce4f7e57c3d623f4e2476f9f79a8eed6e51b2a603f37866018e040ee F ext/jni/src/org/sqlite/jni/OutputPointer.java d37636dd3b82097792dae9c8c255b135153845407cdbc6689f15c475850d6c93 F ext/jni/src/org/sqlite/jni/ProgressHandler.java 5979450e996416d28543f1d42634d308439565a99332a8bd84e424af667116cc F ext/jni/src/org/sqlite/jni/RollbackHook.java b04c8abcc6ade44a8a57129e33765793f69df0ba909e49ba18d73f4268d92564 F ext/jni/src/org/sqlite/jni/SQLFunction.java 09ce81c1c637e31c3a830d4c859cce95d65f5e02ff45f8bd1985b3479381bc46 -F ext/jni/src/org/sqlite/jni/SQLite3Jni.java 78496a02c7cc65a2238f54e935af070acf4e2dbef95d7cc1ff46938c440848a4 -F ext/jni/src/org/sqlite/jni/Tester1.java ef715de2ad23ec9b982122c9e1f0dfe689d9d0d7ac6709dab2ad710811bfa50b -F ext/jni/src/org/sqlite/jni/TesterFts5.java 0b1ab9f3675a593213b0f12fabe4fbe6fd16ed994d41a1f4150e760e087099f6 +F ext/jni/src/org/sqlite/jni/SQLite3Jni.java 1c470a8cdb5c61218304eb76b1188e98e562b105eac816557ef512e1b48fa55a +F ext/jni/src/org/sqlite/jni/Tester1.java aaf6cc2c7e01e78eb208f14afa3977862eaae7dd13040acbd302544ae50c21c3 +F ext/jni/src/org/sqlite/jni/TesterFts5.java 0f42841c230992208a07a04b51ea39bfa719d80a763575fa511574a7409870c1 F ext/jni/src/org/sqlite/jni/Tracer.java a5cece9f947b0af27669b8baec300b6dd7ff859c3e6a6e4a1bd8b50f9714775d F ext/jni/src/org/sqlite/jni/UpdateHook.java e58645a1727f8a9bbe72dc072ec5b40d9f9362cb0aa24acfe93f49ff56a9016d F ext/jni/src/org/sqlite/jni/ValueHolder.java f022873abaabf64f3dd71ab0d6037c6e71cece3b8819fa10bf26a5461dc973ee -F ext/jni/src/org/sqlite/jni/fts5_api.java 6ceb87a8aea27727ff5c40abf79aa2f391e9408eeeea921b7ca02c837979ca84 +F ext/jni/src/org/sqlite/jni/fts5_api.java 794bc2bb5850333f0a4e9df557102747b6cb6064b74ecd74dcdbd0d9e0c86eb4 +F ext/jni/src/org/sqlite/jni/fts5_tokenizer.java e530b36e6437fcc500e95d5d75fbffe272bdea20d2fac6be2e1336c578fba98b F ext/jni/src/org/sqlite/jni/sqlite3.java 600c3ddc1ac28ee8f58669fb435fd0d21f2972c652039361fde907d4fe44eb58 F ext/jni/src/org/sqlite/jni/sqlite3_context.java d26573fc7b309228cb49786e9078597d96232257defa955a3425d10897bca810 F ext/jni/src/org/sqlite/jni/sqlite3_stmt.java 72a0698aeb50a183ad146cd29ee04952abb8c36021f6122656aa5ec20469f6f7 @@ -2077,8 +2080,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 23383c1dfd240ce47f504dd5c3402c9a31f166fbde5bb72d91309a5655074b33 -R 1f7e3cb1d5a03c9cd76bc5a8ab1fdcba +P 14d18fe983c83412d72fd2005a45a2b8c48d347b7bbf8ef9630ae460cff85c32 +R df417fa0aa6fbe10d9f6667baf8e145c U stephan -Z 82182ddc1d375495dd36c655052d869d +Z 194ff6113095adcb792e4a48e3cee001 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 32314a4148..3d2949e01a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -14d18fe983c83412d72fd2005a45a2b8c48d347b7bbf8ef9630ae460cff85c32 \ No newline at end of file +c653bf16cbdccae05ab14059b140191afd5c17740fb78d756d8822986e54b17c \ No newline at end of file