Add multi-thread run mode to JNI Tester1. It works but hangs on exit sometimes for Java reasons as yet not understood.

FossilOrigin-Name: bdbaf7a4534f40e550b646979e67e7b7731566bb5a2631ed376ac85a9bec40a7
2025-08-01 06:27:03 +03:00 · 2023-08-19 10:43:05 +00:00
parent 46d677e713
commit 3cf6c0f276
7 changed files with 138 additions and 83 deletions
--- a/ext/jni/GNUmakefile
+++ b/ext/jni/GNUmakefile
@ -163,14 +163,11 @@ SQLITE_OPT = \
  -DSQLITE_OMIT_LOAD_EXTENSION \
  -DSQLITE_OMIT_DEPRECATED \
  -DSQLITE_OMIT_SHARED_CACHE \
-  -DSQLITE_THREADSAFE=0 \
+  -DSQLITE_THREADSAFE=1 \
  -DSQLITE_TEMP_STORE=2 \
  -DSQLITE_USE_URI=1 \
  -DSQLITE_C=$(sqlite3.c) \
  -DSQLITE_DEBUG
-# -DSQLITE_DEBUG is just to work around a -Wall warning
-# for a var which gets set in all builds but only read
-# via assert().

 SQLITE_OPT += -g -DDEBUG -UNDEBUG

@ -223,7 +220,7 @@ $(sqlite3-jni.h): $(sqlite3-jni.h.in) $(MAKEFILE)
 $(sqlite3-jni.dll): $(sqlite3-jni.h) $(sqlite3.c) $(sqlite3.h)
 $(sqlite3-jni.dll): $(dir.bld.c) $(sqlite3-jni.c) $(SQLite3Jni.java) $(MAKEFILE)
 	$(CC) $(sqlite3-jni.dll.cflags) $(SQLITE_OPT) \
-			$(sqlite3-jni.c) -shared -o $@
+			$(sqlite3-jni.c) -shared -o $@ -lpthread
 all: $(sqlite3-jni.dll)

 .PHONY: test
@ -231,7 +228,7 @@ test.flags ?= -v
 test: $(SQLite3Jni.class) $(sqlite3-jni.dll)
 	$(bin.java) -ea -Djava.library.path=$(dir.bld.c) \
 		$(java.flags) -cp $(classpath) \
-		org.sqlite.jni.Tester1 $(if $(test.flags),-- $(test.flags),)
+		org.sqlite.jni.Tester1 $(test.flags)

 tester.scripts := $(sort $(wildcard $(dir.src)/tests/*.test))
 tester.flags ?= # --verbose
--- a/ext/jni/src/c/sqlite3-jni.c
+++ b/ext/jni/src/c/sqlite3-jni.c
@ -559,9 +559,9 @@ static struct {
 #define MUTEX_ASSERT_NOTLOCKER_ENV                                   \
  assert( (env) != S3JniGlobal.envCache.locker && "Misuse of S3JniGlobal.envCache.mutex" )
 #define MUTEX_ENV_ENTER                                       \
-  /*MARKER(("Entering ENV mutex@%p %s.\n", env, __func__));*/ \
  MUTEX_ASSERT_NOTLOCKER_ENV;                                 \
  sqlite3_mutex_enter( S3JniGlobal.envCache.mutex );          \
+  /*MARKER(("Entered ENV mutex@%p %s.\n", env, __func__));*/  \
  ++S3JniGlobal.metrics.nMutexEnv;                            \
  S3JniGlobal.envCache.locker = env
 #define MUTEX_ENV_LEAVE                                       \
@ -637,6 +637,8 @@ static S3JniEnv * S3JniGlobal_env_cache(JNIEnv * const env){
  S3JniGlobal.envCache.aHead = row;
  row->env = env;

+  //MARKER(("Initalizing cache for JNIEnv@%p\n", env));
+
  /* Grab references to various global classes and objects... */
  row->g.cObj = REF_G((*env)->FindClass(env,"java/lang/Object"));
  EXCEPTION_IS_FATAL("Error getting reference to Object class.");
@ -918,7 +920,6 @@ static void S3JniDb_set_aside(S3JniDb * const s){
  if(s){
    JNIEnv * const env = s->env;
    MUTEX_ASSERT_LOCKED_PDB;
-    assert(s->pDb && "Else this object is already in the free-list.");
    //MARKER(("state@%p for db@%p setting aside\n", s, s->pDb));
    assert(s->pPrev != s);
    assert(s->pNext != s);
@ -966,7 +967,9 @@ static void S3JniDb_free_for_env(JNIEnv *env){
  for( ; ps; ps = pNext ){
    pNext = ps->pNext;
    if(ps->env == env){
+#ifndef NDEBUG
      S3JniDb * const pPrev = ps->pPrev;
+#endif
      S3JniDb_set_aside(ps);
      assert( pPrev ? pPrev->pNext==pNext : 1 );
      assert( ps == S3JniGlobal.perDb.aFree );
@ -996,6 +999,7 @@ static int S3JniGlobal_env_uncache(JNIEnv * const env){
  if( !row ){
      return 0;
  }
+  //MARKER(("Uncaching JNIEnv@%p\n", env));
  if( row->pNext ) row->pNext->pPrev = row->pPrev;
  if( row->pPrev ) row->pPrev->pNext = row->pNext;
  if( S3JniGlobal.envCache.aHead == row ){
@ -2104,12 +2108,14 @@ static jint s3jni_close_db(JNIEnv * const env, jobject jDb, int version){
  ps = S3JniDb_for_db(env, jDb, 0);
  if(ps){
    rc = 1==version ? (jint)sqlite3_close(ps->pDb) : (jint)sqlite3_close_v2(ps->pDb);
+    if( 0==rc ){
      MUTEX_PDB_ENTER;
      S3JniDb_set_aside(ps)
        /* MUST come after close() because of ps->trace. */;
      MUTEX_PDB_LEAVE;
      NativePointerHolder_set(env, jDb, 0, &S3NphRefs.sqlite3);
    }
+  }
  return (jint)rc;
 }

@ -4391,6 +4397,10 @@ Java_org_sqlite_jni_SQLite3Jni_init(JENV_CSELF){
  jfieldID fieldId;
  const ConfigFlagEntry * pConfFlag;

+  if( 0==sqlite3_threadsafe() ){
+    (*env)->FatalError(env, "sqlite3 was not built with SQLITE_THREADSAFE.");
+    return;
+  }
  memset(&S3JniGlobal, 0, sizeof(S3JniGlobal));
  if( (*env)->GetJavaVM(env, &S3JniGlobal.jvm) ){
    (*env)->FatalError(env, "GetJavaVM() failure shouldn't be possible.");
--- a/ext/jni/src/org/sqlite/jni/OutputPointer.java
+++ b/ext/jni/src/org/sqlite/jni/OutputPointer.java
@ -36,6 +36,9 @@ package org.sqlite.jni;
   access to the object's value via the `value` property, whereas the
   JNI-level opaque types do not permit client-level code to set that
   property.
+
+   Warning: do not share instances of these classes across
+   threads. Doing so may lead to corrupting sqlite3-internal state.
 */
 public final class OutputPointer {

--- a/ext/jni/src/org/sqlite/jni/Tester1.java
+++ b/ext/jni/src/org/sqlite/jni/Tester1.java
@ -15,43 +15,48 @@ package org.sqlite.jni;
 import static org.sqlite.jni.SQLite3Jni.*;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;

-public class Tester1 {
+public class Tester1 implements Runnable {
  private static final class Metrics {
    int dbOpen;
  }

-  private String name;
+  private Integer tId;

-  Tester1(String name){
-    this.name = name;
+  Tester1(Integer id){
+    tId = id;
  }

  static final Metrics metrics = new Metrics();
-  private static final OutputPointer.sqlite3_stmt outStmt
-    = new OutputPointer.sqlite3_stmt();

-  public static void out(Object val){
+  public synchronized static void out(Object val){
    System.out.print(val);
  }

-  public static void outln(Object val){
+  public synchronized static void outln(Object val){
+    System.out.print(Thread.currentThread().getName()+": ");
    System.out.println(val);
  }

  @SuppressWarnings("unchecked")
-  public static void out(Object... vals){
+  public synchronized static void out(Object... vals){
    int n = 0;
+    System.out.print(Thread.currentThread().getName()+": ");
    for(Object v : vals) out((n++>0 ? " " : "")+v);
  }

  @SuppressWarnings("unchecked")
-  public static void outln(Object... vals){
+  public synchronized static void outln(Object... vals){
    out(vals); out("\n");
  }

-  static int affirmCount = 0;
-  public static void affirm(Boolean v, String comment){
+  static volatile int affirmCount = 0;
+  public synchronized static void affirm(Boolean v, String comment){
    ++affirmCount;
    assert( v /* prefer assert over exception if it's enabled because
                 the JNI layer sometimes has to suppress exceptions,
@ -66,11 +71,8 @@ public class Tester1 {

  private static void test1(){
    outln("libversion_number:",
-          sqlite3_libversion_number()
-          + "\n"
-          + sqlite3_libversion()
-          + "\n"
-          + SQLITE_SOURCE_ID);
+          sqlite3_libversion_number(),"\n",
+          sqlite3_libversion(),"\n",SQLITE_SOURCE_ID);
    affirm(sqlite3_libversion_number() == SQLITE_VERSION_NUMBER);
    //outln("threadsafe = "+sqlite3_threadsafe());
    affirm(SQLITE_MAX_LENGTH > 0);
@ -106,6 +108,7 @@ public class Tester1 {
    byte[] sqlChunk = sqlUtf8;
    int rc = 0;
    sqlite3_stmt stmt = null;
+    final OutputPointer.sqlite3_stmt outStmt = new OutputPointer.sqlite3_stmt();
    while(pos < sqlChunk.length){
      if(pos > 0){
        sqlChunk = Arrays.copyOfRange(sqlChunk, pos,
@ -144,7 +147,7 @@ public class Tester1 {
  }

  static sqlite3_stmt prepare(sqlite3 db, String sql){
-    outStmt.clear();
+    final OutputPointer.sqlite3_stmt outStmt = new OutputPointer.sqlite3_stmt();
    int rc = sqlite3_prepare(db, sql, outStmt);
    affirm( 0 == rc );
    final sqlite3_stmt rv = outStmt.take();
@ -202,11 +205,15 @@ public class Tester1 {
  private static void testPrepare123(){
    sqlite3 db = createNewDb();
    int rc;
+    final OutputPointer.sqlite3_stmt outStmt = new OutputPointer.sqlite3_stmt();
    rc = sqlite3_prepare(db, "CREATE TABLE t1(a);", outStmt);
    affirm(0 == rc);
-    sqlite3_stmt stmt = outStmt.get();
+    sqlite3_stmt stmt = outStmt.take();
    affirm(0 != stmt.getNativePointer());
    rc = sqlite3_step(stmt);
+    if( SQLITE_DONE != rc ){
+      outln("step failed ??? ",rc, sqlite3_errmsg(db));
+    }
    affirm(SQLITE_DONE == rc);
    sqlite3_finalize(stmt);
    affirm(0 == stmt.getNativePointer());
@ -811,6 +818,7 @@ public class Tester1 {
  private static void testBusy(){
    final String dbName = "_busy-handler.db";
    final OutputPointer.sqlite3 outDb = new OutputPointer.sqlite3();
+    final OutputPointer.sqlite3_stmt outStmt = new OutputPointer.sqlite3_stmt();

    int rc = sqlite3_open(dbName, outDb);
    ++metrics.dbOpen;
@ -1163,6 +1171,7 @@ public class Tester1 {
    testOpenDb1();
    testOpenDb2();
    testPrepare123();
+    if( true ){
      testBindFetchInt();
      testBindFetchInt64();
      testBindFetchDouble();
@ -1177,41 +1186,77 @@ public class Tester1 {
      testUdfAggregate();
      testUdfWindow();
      testTrace();
-    testBusy();
      testProgress();
      testCommitHook();
      testRollbackHook();
      testUpdateHook();
      testAuthorizer();
-    testFts5();
      if(!fromThread){
+        // skip for now: messes with affirm() counts. testFts5();
+        testBusy();
        testAutoExtension();
      }
    }
+  }

-  public void run() throws Exception{
-    runTests(true);
+  public void run(){
+    try {
+      runTests(0!=this.tId);
+    }catch(Exception e){
+      throw new RuntimeException(e);
+    }finally{
+      affirm( SQLite3Jni.uncacheJniEnv() );
+      affirm( !SQLite3Jni.uncacheJniEnv() );
+    }
  }

  public static void main(String[] args) throws Exception {
-
-    final long timeStart = System.nanoTime();
-    new Tester1("main thread").runTests(false);
-    final long timeEnd = System.nanoTime();
-
-    final java.util.List<String> liArgs =
-      java.util.Arrays.asList(args);
-    //testSleep();
-    if(liArgs.indexOf("-v")>0){
-      sqlite3_do_something_for_developer();
+    Integer nThread = null;
+    boolean doSomethingForDev = false;
+    Integer nRepeat = 1;
+    for( int i = 0; i < args.length; ){
+      String arg = args[i++];
+      if(arg.startsWith("-")){
+        arg = arg.replaceFirst("-+","");
+        if(arg.equals("v")){
+          doSomethingForDev = true;
          //listBoundMethods();
+        }else if(arg.equals("t") || arg.equals("thread")){
+          nThread = Integer.parseInt(args[i++]);
+        }else if(arg.equals("r") || arg.equals("runs")){
+          nRepeat = Integer.parseInt(args[i++]);
+        }else{
+          throw new IllegalArgumentException("Unhandled flag:"+arg);
        }
-    affirm( SQLite3Jni.uncacheJniEnv() );
-    affirm( !SQLite3Jni.uncacheJniEnv() );
+      }
+    }
+
+    final long timeStart = System.currentTimeMillis();
+    int nLoop = 0;
+    for( int n = 0; n < nRepeat; ++n ){
+      if( nThread==null || nThread<=1 ){
+        new Tester1(0).runTests(false);
+      }else{
+        final ExecutorService ex = Executors.newFixedThreadPool( nThread );
+        //final List<Future<?>> futures = new ArrayList<>();
+        ++nLoop;
+        outln("Running loop #",nLoop," over ",nThread," threads.");
+        for( int i = 0; i < nThread; ++i ){
+          ex.submit( new Tester1(i) );
+        }
+        ex.shutdown();
+        ex.awaitTermination(2, java.util.concurrent.TimeUnit.SECONDS);
+        ex.shutdownNow();
+      }
+    }
+    final long timeEnd = System.currentTimeMillis();
    outln("Tests done. Metrics:");
    outln("\tAssertions checked: "+affirmCount);
    outln("\tDatabases opened: "+metrics.dbOpen);

+    if( doSomethingForDev ){
+      sqlite3_do_something_for_developer();
+    }
    int nMethods = 0;
    int nNatives = 0;
    final java.lang.reflect.Method[] declaredMethods =
@ -1232,6 +1277,6 @@ public class Tester1 {
          nNatives+" native methods and "+
          (nMethods - nNatives)+" Java impls");
    outln("\tTotal test time = "
-          +((timeEnd - timeStart)/1000000.0)+"ms");
+          +(timeEnd - timeStart)+"ms");
  }
 }
--- a/ext/jni/src/org/sqlite/jni/TesterFts5.java
+++ b/ext/jni/src/org/sqlite/jni/TesterFts5.java
@ -72,7 +72,7 @@ public class TesterFts5 {
    affirm( xDestroyCalled.value );
  }

-  public TesterFts5(){
+  public TesterFts5(boolean outputStats){
    int oldAffirmCount = Tester1.affirmCount;
    Tester1.affirmCount = 0;
    final long timeStart = System.nanoTime();
--- a/20
+++ b/20
@ -1,5 +1,5 @@
-C Replace\sJNI::NewStringUTF()\sfor\sthe\sremaining\scases\swhere\soutput\smay\sbe\sincompatible\swith\sMUTF-8.\sIt\sis\snow\sonly\sused\swhen\swe\sknow\sthe\soutput\sto\sbe\splain\sASCII.
-D 2023-08-19T08:22:34.056
+C Add\smulti-thread\srun\smode\sto\sJNI\sTester1.\sIt\sworks\sbut\shangs\son\sexit\ssometimes\sfor\sJava\sreasons\sas\syet\snot\sunderstood.
+D 2023-08-19T10:43:05.945
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@ -231,10 +231,10 @@ F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c
 F ext/icu/README.txt 7ab7ced8ae78e3a645b57e78570ff589d4c672b71370f5aa9e1cd7024f400fc9
 F ext/icu/icu.c c074519b46baa484bb5396c7e01e051034da8884bad1a1cb7f09bbe6be3f0282
 F ext/icu/sqliteicu.h fa373836ed5a1ee7478bdf8a1650689294e41d0c89c1daab26e9ae78a32075a8
-F ext/jni/GNUmakefile a9e11b92e620058558cbc1a2d49f8ec53c78d6a989b9db0b7d0b649b9f174881
+F ext/jni/GNUmakefile 28ef565d7a2df7b8db61826a4db3806e24bfc25f0bfa2f56fdd5527c93ecdb10
 F ext/jni/README.md 7a614a2fa6c561205f7a53fd8626cf93a7b5711ff454fc1814517f796df398eb
 F ext/jni/jar-dist.make f90a553203a57934bf275bed86479485135a52f48ac5c1cfe6499ae07b0b35a4
-F ext/jni/src/c/sqlite3-jni.c a8b51e4c63572d1655dafea38b80fd63528684264c6483b5c4d1eb9098c44712
+F ext/jni/src/c/sqlite3-jni.c a4a762bff193e52a264778f64545674d5b58dbcb45478e9186d603fae2c312cd
 F ext/jni/src/c/sqlite3-jni.h f10d2f38720687c70ecdd5e44f6e8db98efee2caa05fc86b2d9e0c76e6cc0a18
 F ext/jni/src/org/sqlite/jni/Authorizer.java 1308988f7f40579ea0e4deeaec3c6be971630566bd021c31367fe3f5140db892
 F ext/jni/src/org/sqlite/jni/AutoExtension.java 18e83f6f463e306df60b2dceb65247d32af1f78af4bbbae9155411a8c6cdb093
@ -249,14 +249,14 @@ F ext/jni/src/org/sqlite/jni/Fts5Function.java 65cde7151e441fee012250a5e03277de7
 F ext/jni/src/org/sqlite/jni/Fts5PhraseIter.java 6642beda341c0b1b46af4e2d7f6f9ab03a7aede43277b2c92859176d6bce3be9
 F ext/jni/src/org/sqlite/jni/Fts5Tokenizer.java 91489893596b6528c0df5cd7180bd5b55809c26e2b797fb321dfcdbc1298c060
 F ext/jni/src/org/sqlite/jni/NativePointerHolder.java 9c5d901cce4f7e57c3d623f4e2476f9f79a8eed6e51b2a603f37866018e040ee
-F ext/jni/src/org/sqlite/jni/OutputPointer.java d81f8bd43d2296ae373692370cfad16ddde76f5c14cd2760f7b4e1113ef56d4c
+F ext/jni/src/org/sqlite/jni/OutputPointer.java 464ea85c3eba673a7b575545f69fcd8aeb398477a26d155d88cee3e2459e7802
 F ext/jni/src/org/sqlite/jni/ProgressHandler.java 6f62053a828a572de809828b1ee495380677e87daa29a1c57a0e2c06b0a131dc
 F ext/jni/src/org/sqlite/jni/ResultCode.java ba701f20213a5f259e94cfbfdd36eb7ac7ce7797f2c6c7fca2004ff12ce20f86
 F ext/jni/src/org/sqlite/jni/RollbackHook.java b04c8abcc6ade44a8a57129e33765793f69df0ba909e49ba18d73f4268d92564
 F ext/jni/src/org/sqlite/jni/SQLFunction.java 09ce81c1c637e31c3a830d4c859cce95d65f5e02ff45f8bd1985b3479381bc46
 F ext/jni/src/org/sqlite/jni/SQLite3Jni.java 26b3083501a9f42e9aa49b941f6b378213cf91ae1a8f705602773ed750043a3c
-F ext/jni/src/org/sqlite/jni/Tester1.java 68b88b3098ce60134f4298488f890871398a77477af0a1b21797c59c911060c1
-F ext/jni/src/org/sqlite/jni/TesterFts5.java 59e22dd24af033ea8827d36225a2f3297908fb6af8818ead8850c6c6847557b1
+F ext/jni/src/org/sqlite/jni/Tester1.java 655d7109a1079be898f2631930493bd86e0c0259582014bb7af41b87d21d9a27
+F ext/jni/src/org/sqlite/jni/TesterFts5.java 3914b0a7ab0ff752c1082b1ae0c09b32827d81962fff62bcd0e13b9ec3a6f03f
 F ext/jni/src/org/sqlite/jni/Tracer.java a5cece9f947b0af27669b8baec300b6dd7ff859c3e6a6e4a1bd8b50f9714775d
 F ext/jni/src/org/sqlite/jni/UpdateHook.java e58645a1727f8a9bbe72dc072ec5b40d9f9362cb0aa24acfe93f49ff56a9016d
 F ext/jni/src/org/sqlite/jni/ValueHolder.java f022873abaabf64f3dd71ab0d6037c6e71cece3b8819fa10bf26a5461dc973ee
@ -2091,8 +2091,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 00a2a3736a6dcde81d920815520040f3c47f965165e7128ca1f4062e6ec7c17c
-R 844cfc265e8ec15b67feebeb0f56ea87
+P 2d955eef25ab116c487ebc34c6f2d2836d310af239ef1993f5aeee5a3f68d590
+R 8a1b71ae57321d848d8875e47d990d30
 U stephan
-Z a99a318492abbd023ca99730c8ec8838
+Z 68dec3da5d321e77fd308ae3996624ba
 # Remove this line to create a well-formed Fossil manifest.
--- a/manifest.uuid
+++ b/manifest.uuid
@ -1 +1 @@
-2d955eef25ab116c487ebc34c6f2d2836d310af239ef1993f5aeee5a3f68d590
+bdbaf7a4534f40e550b646979e67e7b7731566bb5a2631ed376ac85a9bec40a7