Add multi-thread run mode to JNI Tester1. It works but hangs on exit sometimes for Java reasons as yet not understood.

FossilOrigin-Name: bdbaf7a4534f40e550b646979e67e7b7731566bb5a2631ed376ac85a9bec40a7

ext/jni/GNUmakefile

@@ -163,14 +163,11 @@ SQLITE_OPT = \
   -DSQLITE_OMIT_LOAD_EXTENSION \
   -DSQLITE_OMIT_DEPRECATED \
   -DSQLITE_OMIT_SHARED_CACHE \
-  -DSQLITE_THREADSAFE=0 \
+  -DSQLITE_THREADSAFE=1 \
   -DSQLITE_TEMP_STORE=2 \
   -DSQLITE_USE_URI=1 \
   -DSQLITE_C=$(sqlite3.c) \
   -DSQLITE_DEBUG
-# -DSQLITE_DEBUG is just to work around a -Wall warning
-# for a var which gets set in all builds but only read
-# via assert().
 
 SQLITE_OPT += -g -DDEBUG -UNDEBUG
 
@@ -223,7 +220,7 @@ $(sqlite3-jni.h): $(sqlite3-jni.h.in) $(MAKEFILE)
 $(sqlite3-jni.dll): $(sqlite3-jni.h) $(sqlite3.c) $(sqlite3.h)
 $(sqlite3-jni.dll): $(dir.bld.c) $(sqlite3-jni.c) $(SQLite3Jni.java) $(MAKEFILE)
 	$(CC) $(sqlite3-jni.dll.cflags) $(SQLITE_OPT) \
-			$(sqlite3-jni.c) -shared -o $@
+			$(sqlite3-jni.c) -shared -o $@ -lpthread
 all: $(sqlite3-jni.dll)
 
 .PHONY: test
@@ -231,7 +228,7 @@ test.flags ?= -v
 test: $(SQLite3Jni.class) $(sqlite3-jni.dll)
 	$(bin.java) -ea -Djava.library.path=$(dir.bld.c) \
 		$(java.flags) -cp $(classpath) \
-		org.sqlite.jni.Tester1 $(if $(test.flags),-- $(test.flags),)
+		org.sqlite.jni.Tester1 $(test.flags)
 
 tester.scripts := $(sort $(wildcard $(dir.src)/tests/*.test))
 tester.flags ?= # --verbose

ext/jni/src/c/sqlite3-jni.c

@@ -559,9 +559,9 @@ static struct {
 #define MUTEX_ASSERT_NOTLOCKER_ENV                                   \
   assert( (env) != S3JniGlobal.envCache.locker && "Misuse of S3JniGlobal.envCache.mutex" )
 #define MUTEX_ENV_ENTER                                       \
-  /*MARKER(("Entering ENV mutex@%p %s.\n", env, __func__));*/ \
   MUTEX_ASSERT_NOTLOCKER_ENV;                                 \
   sqlite3_mutex_enter( S3JniGlobal.envCache.mutex );          \
+  /*MARKER(("Entered ENV mutex@%p %s.\n", env, __func__));*/  \
   ++S3JniGlobal.metrics.nMutexEnv;                            \
   S3JniGlobal.envCache.locker = env
 #define MUTEX_ENV_LEAVE                                       \
@@ -637,6 +637,8 @@ static S3JniEnv * S3JniGlobal_env_cache(JNIEnv * const env){
   S3JniGlobal.envCache.aHead = row;
   row->env = env;
 
+  //MARKER(("Initalizing cache for JNIEnv@%p\n", env));
+
   /* Grab references to various global classes and objects... */
   row->g.cObj = REF_G((*env)->FindClass(env,"java/lang/Object"));
   EXCEPTION_IS_FATAL("Error getting reference to Object class.");
@@ -918,7 +920,6 @@ static void S3JniDb_set_aside(S3JniDb * const s){
   if(s){
     JNIEnv * const env = s->env;
     MUTEX_ASSERT_LOCKED_PDB;
-    assert(s->pDb && "Else this object is already in the free-list.");
     //MARKER(("state@%p for db@%p setting aside\n", s, s->pDb));
     assert(s->pPrev != s);
     assert(s->pNext != s);
@@ -966,7 +967,9 @@ static void S3JniDb_free_for_env(JNIEnv *env){
   for( ; ps; ps = pNext ){
     pNext = ps->pNext;
     if(ps->env == env){
+#ifndef NDEBUG
       S3JniDb * const pPrev = ps->pPrev;
+#endif
       S3JniDb_set_aside(ps);
       assert( pPrev ? pPrev->pNext==pNext : 1 );
       assert( ps == S3JniGlobal.perDb.aFree );
@@ -996,6 +999,7 @@ static int S3JniGlobal_env_uncache(JNIEnv * const env){
   if( !row ){
       return 0;
   }
+  //MARKER(("Uncaching JNIEnv@%p\n", env));
   if( row->pNext ) row->pNext->pPrev = row->pPrev;
   if( row->pPrev ) row->pPrev->pNext = row->pNext;
   if( S3JniGlobal.envCache.aHead == row ){
@@ -2104,12 +2108,14 @@ static jint s3jni_close_db(JNIEnv * const env, jobject jDb, int version){
   ps = S3JniDb_for_db(env, jDb, 0);
   if(ps){
     rc = 1==version ? (jint)sqlite3_close(ps->pDb) : (jint)sqlite3_close_v2(ps->pDb);
+    if( 0==rc ){
       MUTEX_PDB_ENTER;
       S3JniDb_set_aside(ps)
         /* MUST come after close() because of ps->trace. */;
       MUTEX_PDB_LEAVE;
       NativePointerHolder_set(env, jDb, 0, &S3NphRefs.sqlite3);
     }
+  }
   return (jint)rc;
 }
 
@@ -4391,6 +4397,10 @@ Java_org_sqlite_jni_SQLite3Jni_init(JENV_CSELF){
   jfieldID fieldId;
   const ConfigFlagEntry * pConfFlag;
 
+  if( 0==sqlite3_threadsafe() ){
+    (*env)->FatalError(env, "sqlite3 was not built with SQLITE_THREADSAFE.");
+    return;
+  }
   memset(&S3JniGlobal, 0, sizeof(S3JniGlobal));
   if( (*env)->GetJavaVM(env, &S3JniGlobal.jvm) ){
     (*env)->FatalError(env, "GetJavaVM() failure shouldn't be possible.");

ext/jni/src/org/sqlite/jni/OutputPointer.java

@@ -36,6 +36,9 @@ package org.sqlite.jni;
    access to the object's value via the `value` property, whereas the
    JNI-level opaque types do not permit client-level code to set that
    property.
+
+   Warning: do not share instances of these classes across
+   threads. Doing so may lead to corrupting sqlite3-internal state.
 */
 public final class OutputPointer {
 

ext/jni/src/org/sqlite/jni/Tester1.java

@@ -15,43 +15,48 @@ package org.sqlite.jni;
 import static org.sqlite.jni.SQLite3Jni.*;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 
-public class Tester1 {
+public class Tester1 implements Runnable {
   private static final class Metrics {
     int dbOpen;
   }
 
-  private String name;
+  private Integer tId;
 
-  Tester1(String name){
+  Tester1(Integer id){
-    this.name = name;
+    tId = id;
   }
 
   static final Metrics metrics = new Metrics();
-  private static final OutputPointer.sqlite3_stmt outStmt
-    = new OutputPointer.sqlite3_stmt();
 
-  public static void out(Object val){
+  public synchronized static void out(Object val){
     System.out.print(val);
   }
 
-  public static void outln(Object val){
+  public synchronized static void outln(Object val){
+    System.out.print(Thread.currentThread().getName()+": ");
     System.out.println(val);
   }
 
   @SuppressWarnings("unchecked")
-  public static void out(Object... vals){
+  public synchronized static void out(Object... vals){
     int n = 0;
+    System.out.print(Thread.currentThread().getName()+": ");
     for(Object v : vals) out((n++>0 ? " " : "")+v);
   }
 
   @SuppressWarnings("unchecked")
-  public static void outln(Object... vals){
+  public synchronized static void outln(Object... vals){
     out(vals); out("\n");
   }
 
-  static int affirmCount = 0;
+  static volatile int affirmCount = 0;
-  public static void affirm(Boolean v, String comment){
+  public synchronized static void affirm(Boolean v, String comment){
     ++affirmCount;
     assert( v /* prefer assert over exception if it's enabled because
                  the JNI layer sometimes has to suppress exceptions,
@@ -66,11 +71,8 @@ public class Tester1 {
 
   private static void test1(){
     outln("libversion_number:",
-          sqlite3_libversion_number()
+          sqlite3_libversion_number(),"\n",
-          + "\n"
+          sqlite3_libversion(),"\n",SQLITE_SOURCE_ID);
-          + sqlite3_libversion()
-          + "\n"
-          + SQLITE_SOURCE_ID);
     affirm(sqlite3_libversion_number() == SQLITE_VERSION_NUMBER);
     //outln("threadsafe = "+sqlite3_threadsafe());
     affirm(SQLITE_MAX_LENGTH > 0);
@@ -106,6 +108,7 @@ public class Tester1 {
     byte[] sqlChunk = sqlUtf8;
     int rc = 0;
     sqlite3_stmt stmt = null;
+    final OutputPointer.sqlite3_stmt outStmt = new OutputPointer.sqlite3_stmt();
     while(pos < sqlChunk.length){
       if(pos > 0){
         sqlChunk = Arrays.copyOfRange(sqlChunk, pos,
@@ -144,7 +147,7 @@ public class Tester1 {
   }
 
   static sqlite3_stmt prepare(sqlite3 db, String sql){
-    outStmt.clear();
+    final OutputPointer.sqlite3_stmt outStmt = new OutputPointer.sqlite3_stmt();
     int rc = sqlite3_prepare(db, sql, outStmt);
     affirm( 0 == rc );
     final sqlite3_stmt rv = outStmt.take();
@@ -202,11 +205,15 @@ public class Tester1 {
   private static void testPrepare123(){
     sqlite3 db = createNewDb();
     int rc;
+    final OutputPointer.sqlite3_stmt outStmt = new OutputPointer.sqlite3_stmt();
     rc = sqlite3_prepare(db, "CREATE TABLE t1(a);", outStmt);
     affirm(0 == rc);
-    sqlite3_stmt stmt = outStmt.get();
+    sqlite3_stmt stmt = outStmt.take();
     affirm(0 != stmt.getNativePointer());
     rc = sqlite3_step(stmt);
+    if( SQLITE_DONE != rc ){
+      outln("step failed ??? ",rc, sqlite3_errmsg(db));
+    }
     affirm(SQLITE_DONE == rc);
     sqlite3_finalize(stmt);
     affirm(0 == stmt.getNativePointer());
@@ -811,6 +818,7 @@ public class Tester1 {
   private static void testBusy(){
     final String dbName = "_busy-handler.db";
     final OutputPointer.sqlite3 outDb = new OutputPointer.sqlite3();
+    final OutputPointer.sqlite3_stmt outStmt = new OutputPointer.sqlite3_stmt();
 
     int rc = sqlite3_open(dbName, outDb);
     ++metrics.dbOpen;
@@ -1163,6 +1171,7 @@ public class Tester1 {
     testOpenDb1();
     testOpenDb2();
     testPrepare123();
+    if( true ){
       testBindFetchInt();
       testBindFetchInt64();
       testBindFetchDouble();
@@ -1177,41 +1186,77 @@ public class Tester1 {
       testUdfAggregate();
       testUdfWindow();
       testTrace();
-    testBusy();
       testProgress();
       testCommitHook();
       testRollbackHook();
       testUpdateHook();
       testAuthorizer();
-    testFts5();
       if(!fromThread){
+        // skip for now: messes with affirm() counts. testFts5();
+        testBusy();
         testAutoExtension();
       }
     }
+  }
 
-  public void run() throws Exception{
+  public void run(){
-    runTests(true);
+    try {
+      runTests(0!=this.tId);
+    }catch(Exception e){
+      throw new RuntimeException(e);
+    }finally{
+      affirm( SQLite3Jni.uncacheJniEnv() );
+      affirm( !SQLite3Jni.uncacheJniEnv() );
+    }
   }
 
   public static void main(String[] args) throws Exception {
-
+    Integer nThread = null;
-    final long timeStart = System.nanoTime();
+    boolean doSomethingForDev = false;
-    new Tester1("main thread").runTests(false);
+    Integer nRepeat = 1;
-    final long timeEnd = System.nanoTime();
+    for( int i = 0; i < args.length; ){
-
+      String arg = args[i++];
-    final java.util.List<String> liArgs =
+      if(arg.startsWith("-")){
-      java.util.Arrays.asList(args);
+        arg = arg.replaceFirst("-+","");
-    //testSleep();
+        if(arg.equals("v")){
-    if(liArgs.indexOf("-v")>0){
+          doSomethingForDev = true;
-      sqlite3_do_something_for_developer();
           //listBoundMethods();
+        }else if(arg.equals("t") || arg.equals("thread")){
+          nThread = Integer.parseInt(args[i++]);
+        }else if(arg.equals("r") || arg.equals("runs")){
+          nRepeat = Integer.parseInt(args[i++]);
+        }else{
+          throw new IllegalArgumentException("Unhandled flag:"+arg);
         }
-    affirm( SQLite3Jni.uncacheJniEnv() );
+      }
-    affirm( !SQLite3Jni.uncacheJniEnv() );
+    }
+
+    final long timeStart = System.currentTimeMillis();
+    int nLoop = 0;
+    for( int n = 0; n < nRepeat; ++n ){
+      if( nThread==null || nThread<=1 ){
+        new Tester1(0).runTests(false);
+      }else{
+        final ExecutorService ex = Executors.newFixedThreadPool( nThread );
+        //final List<Future<?>> futures = new ArrayList<>();
+        ++nLoop;
+        outln("Running loop #",nLoop," over ",nThread," threads.");
+        for( int i = 0; i < nThread; ++i ){
+          ex.submit( new Tester1(i) );
+        }
+        ex.shutdown();
+        ex.awaitTermination(2, java.util.concurrent.TimeUnit.SECONDS);
+        ex.shutdownNow();
+      }
+    }
+    final long timeEnd = System.currentTimeMillis();
     outln("Tests done. Metrics:");
     outln("\tAssertions checked: "+affirmCount);
     outln("\tDatabases opened: "+metrics.dbOpen);
 
+    if( doSomethingForDev ){
+      sqlite3_do_something_for_developer();
+    }
     int nMethods = 0;
     int nNatives = 0;
     final java.lang.reflect.Method[] declaredMethods =
@@ -1232,6 +1277,6 @@ public class Tester1 {
           nNatives+" native methods and "+
           (nMethods - nNatives)+" Java impls");
     outln("\tTotal test time = "
-          +((timeEnd - timeStart)/1000000.0)+"ms");
+          +(timeEnd - timeStart)+"ms");
   }
 }

ext/jni/src/org/sqlite/jni/TesterFts5.java

@@ -72,7 +72,7 @@ public class TesterFts5 {
     affirm( xDestroyCalled.value );
   }
 
-  public TesterFts5(){
+  public TesterFts5(boolean outputStats){
     int oldAffirmCount = Tester1.affirmCount;
     Tester1.affirmCount = 0;
     final long timeStart = System.nanoTime();

manifest

@@ -1,5 +1,5 @@
-C Replace\sJNI::NewStringUTF()\sfor\sthe\sremaining\scases\swhere\soutput\smay\sbe\sincompatible\swith\sMUTF-8.\sIt\sis\snow\sonly\sused\swhen\swe\sknow\sthe\soutput\sto\sbe\splain\sASCII.
+C Add\smulti-thread\srun\smode\sto\sJNI\sTester1.\sIt\sworks\sbut\shangs\son\sexit\ssometimes\sfor\sJava\sreasons\sas\syet\snot\sunderstood.
-D 2023-08-19T08:22:34.056
+D 2023-08-19T10:43:05.945
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -231,10 +231,10 @@ F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c
 F ext/icu/README.txt 7ab7ced8ae78e3a645b57e78570ff589d4c672b71370f5aa9e1cd7024f400fc9
 F ext/icu/icu.c c074519b46baa484bb5396c7e01e051034da8884bad1a1cb7f09bbe6be3f0282
 F ext/icu/sqliteicu.h fa373836ed5a1ee7478bdf8a1650689294e41d0c89c1daab26e9ae78a32075a8
-F ext/jni/GNUmakefile a9e11b92e620058558cbc1a2d49f8ec53c78d6a989b9db0b7d0b649b9f174881
+F ext/jni/GNUmakefile 28ef565d7a2df7b8db61826a4db3806e24bfc25f0bfa2f56fdd5527c93ecdb10
 F ext/jni/README.md 7a614a2fa6c561205f7a53fd8626cf93a7b5711ff454fc1814517f796df398eb
 F ext/jni/jar-dist.make f90a553203a57934bf275bed86479485135a52f48ac5c1cfe6499ae07b0b35a4
-F ext/jni/src/c/sqlite3-jni.c a8b51e4c63572d1655dafea38b80fd63528684264c6483b5c4d1eb9098c44712
+F ext/jni/src/c/sqlite3-jni.c a4a762bff193e52a264778f64545674d5b58dbcb45478e9186d603fae2c312cd
 F ext/jni/src/c/sqlite3-jni.h f10d2f38720687c70ecdd5e44f6e8db98efee2caa05fc86b2d9e0c76e6cc0a18
 F ext/jni/src/org/sqlite/jni/Authorizer.java 1308988f7f40579ea0e4deeaec3c6be971630566bd021c31367fe3f5140db892
 F ext/jni/src/org/sqlite/jni/AutoExtension.java 18e83f6f463e306df60b2dceb65247d32af1f78af4bbbae9155411a8c6cdb093
@@ -249,14 +249,14 @@ F ext/jni/src/org/sqlite/jni/Fts5Function.java 65cde7151e441fee012250a5e03277de7
 F ext/jni/src/org/sqlite/jni/Fts5PhraseIter.java 6642beda341c0b1b46af4e2d7f6f9ab03a7aede43277b2c92859176d6bce3be9
 F ext/jni/src/org/sqlite/jni/Fts5Tokenizer.java 91489893596b6528c0df5cd7180bd5b55809c26e2b797fb321dfcdbc1298c060
 F ext/jni/src/org/sqlite/jni/NativePointerHolder.java 9c5d901cce4f7e57c3d623f4e2476f9f79a8eed6e51b2a603f37866018e040ee
-F ext/jni/src/org/sqlite/jni/OutputPointer.java d81f8bd43d2296ae373692370cfad16ddde76f5c14cd2760f7b4e1113ef56d4c
+F ext/jni/src/org/sqlite/jni/OutputPointer.java 464ea85c3eba673a7b575545f69fcd8aeb398477a26d155d88cee3e2459e7802
 F ext/jni/src/org/sqlite/jni/ProgressHandler.java 6f62053a828a572de809828b1ee495380677e87daa29a1c57a0e2c06b0a131dc
 F ext/jni/src/org/sqlite/jni/ResultCode.java ba701f20213a5f259e94cfbfdd36eb7ac7ce7797f2c6c7fca2004ff12ce20f86
 F ext/jni/src/org/sqlite/jni/RollbackHook.java b04c8abcc6ade44a8a57129e33765793f69df0ba909e49ba18d73f4268d92564
 F ext/jni/src/org/sqlite/jni/SQLFunction.java 09ce81c1c637e31c3a830d4c859cce95d65f5e02ff45f8bd1985b3479381bc46
 F ext/jni/src/org/sqlite/jni/SQLite3Jni.java 26b3083501a9f42e9aa49b941f6b378213cf91ae1a8f705602773ed750043a3c
-F ext/jni/src/org/sqlite/jni/Tester1.java 68b88b3098ce60134f4298488f890871398a77477af0a1b21797c59c911060c1
+F ext/jni/src/org/sqlite/jni/Tester1.java 655d7109a1079be898f2631930493bd86e0c0259582014bb7af41b87d21d9a27
-F ext/jni/src/org/sqlite/jni/TesterFts5.java 59e22dd24af033ea8827d36225a2f3297908fb6af8818ead8850c6c6847557b1
+F ext/jni/src/org/sqlite/jni/TesterFts5.java 3914b0a7ab0ff752c1082b1ae0c09b32827d81962fff62bcd0e13b9ec3a6f03f
 F ext/jni/src/org/sqlite/jni/Tracer.java a5cece9f947b0af27669b8baec300b6dd7ff859c3e6a6e4a1bd8b50f9714775d
 F ext/jni/src/org/sqlite/jni/UpdateHook.java e58645a1727f8a9bbe72dc072ec5b40d9f9362cb0aa24acfe93f49ff56a9016d
 F ext/jni/src/org/sqlite/jni/ValueHolder.java f022873abaabf64f3dd71ab0d6037c6e71cece3b8819fa10bf26a5461dc973ee
@@ -2091,8 +2091,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 00a2a3736a6dcde81d920815520040f3c47f965165e7128ca1f4062e6ec7c17c
+P 2d955eef25ab116c487ebc34c6f2d2836d310af239ef1993f5aeee5a3f68d590
-R 844cfc265e8ec15b67feebeb0f56ea87
+R 8a1b71ae57321d848d8875e47d990d30
 U stephan
-Z a99a318492abbd023ca99730c8ec8838
+Z 68dec3da5d321e77fd308ae3996624ba
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-2d955eef25ab116c487ebc34c6f2d2836d310af239ef1993f5aeee5a3f68d590
+bdbaf7a4534f40e550b646979e67e7b7731566bb5a2631ed376ac85a9bec40a7