More work on the JNI multi-threaded test runner.

FossilOrigin-Name: 9a74ad716bded1e14333bf7c72392916f800d58a96240eabe4988ca5fc9e8752

ext/jni/src/c/sqlite3-jni.c

@@ -599,8 +599,6 @@ static S3JniGlobalType S3JniGlobal = {};
   /*MARKER(("Leaving PerDb mutex@%p %s.\n", env));*/  \
   SJG.perDb.locker = 0;                               \
   sqlite3_mutex_leave( SJG.perDb.mutex )
-#define MUTEX_PDB_ASSERT_LOCKED  \
-  assert( 0 != SJG.perDb.locker && "Misuse of S3JniGlobal.perDb.mutex" )
 
 #define OOM_CHECK(VAR) if(!(VAR)) s3jni_oom(env)
 static inline void s3jni_oom(JNIEnv * const env){
@@ -903,12 +901,12 @@ static void S3JniHook_unref(JNIEnv * const env, S3JniHook * const s, int doXDest
 }
 
 /**
-   Clears s's state and moves it to the free-list.
+   Clears s's state and moves it to the free-list. Requires that
+   S3JniGlobal.perDb.mutex be unlocked.
 */
-static void S3JniDb_set_aside(S3JniDb * const s){
+static void S3JniDb_set_aside(JNIEnv * env, S3JniDb * const s){
   if(s){
-    LocalJniGetEnv;
-    MUTEX_PDB_ASSERT_LOCKED;
+    MUTEX_PDB_ENTER;
     //MARKER(("state@%p for db@%p setting aside\n", s, s->pDb));
     assert(s->pPrev != s);
     assert(s->pNext != s);
@@ -938,6 +936,7 @@ static void S3JniDb_set_aside(S3JniDb * const s){
     SJG.perDb.aFree = s;
     //MARKER(("%p->pPrev@%p, pNext@%p\n", s, s->pPrev, s->pNext));
     //if(s->pNext) MARKER(("next: %p->pPrev@%p\n", s->pNext, s->pNext->pPrev));
+    MUTEX_PDB_LEAVE;
   }
 }
 
@@ -1962,10 +1961,8 @@ static jint s3jni_close_db(JNIEnv * const env, jobject jDb, int version){
   if(ps){
     rc = 1==version ? (jint)sqlite3_close(ps->pDb) : (jint)sqlite3_close_v2(ps->pDb);
     if( 0==rc ){
-      MUTEX_PDB_ENTER;
-      S3JniDb_set_aside(ps)
+      S3JniDb_set_aside(env, ps)
         /* MUST come after close() because of ps->trace. */;
-      MUTEX_PDB_LEAVE;
       NativePointerHolder_set(env, jDb, 0, &S3NphRefs.sqlite3);
     }
   }
@@ -2537,9 +2534,7 @@ static int s3jni_open_post(JNIEnv * const env, S3JniEnv * const jc,
       assert( ps->pDb == *ppDb /* set up via s3jni_run_java_auto_extensions() */);
     }
   }else{
-    MUTEX_PDB_ENTER;
-    S3JniDb_set_aside(ps);
-    MUTEX_PDB_LEAVE;
+    S3JniDb_set_aside(env, ps);
     ps = 0;
   }
   OutputPointer_set_sqlite3(env, jOut, ps ? ps->jDb : 0);
@@ -3306,7 +3301,8 @@ JDECL(jbyteArray,1value_1text16be)(JENV_CSELF, jobject jpSVal){
 }
 
 JDECL(void,1do_1something_1for_1developer)(JENV_CSELF){
-  MARKER(("\nVarious bits of internal info:\n"));
+  MARKER(("\nVarious bits of internal info:\n"
+          "Any metrics here are invalid in multi-thread use.\n"));
   puts("FTS5 is "
 #ifdef SQLITE_ENABLE_FTS5
        "available"
@@ -3338,7 +3334,7 @@ JDECL(void,1do_1something_1for_1developer)(JENV_CSELF){
          "\n\tenv %u"
          "\n\tnph inits %u"
          "\n\tperDb %u"
-         "\n\tautoExt %u container access\n",
+         "\n\tautoExt %u list accesses\n",
          SJG.metrics.nMutexEnv, SJG.metrics.nMutexEnv2,
          SJG.metrics.nMutexPerDb, SJG.metrics.nMutexAutoExt);
   printf("S3JniDb: %u alloced (*%u = %u bytes), %u recycled\n",

ext/jni/src/org/sqlite/jni/NativePointerHolder.java

@@ -28,6 +28,6 @@ package org.sqlite.jni;
 */
 public class NativePointerHolder<ContextType> {
   //! Only set from JNI, where access permissions don't matter.
-  private long nativePointer = 0;
+  private volatile long nativePointer = 0;
   public final long getNativePointer(){ return nativePointer; }
 }

ext/jni/src/org/sqlite/jni/Tester1.java

@@ -25,7 +25,9 @@ public class Tester1 implements Runnable {
   //! True when running in multi-threaded mode.
   private static boolean mtMode = false;
   private static boolean takeNaps = false;
-
+  private static boolean shuffle = false;
+  private static boolean listRunTests = false;
+  private static List<java.lang.reflect.Method> testMethods = null;
   private static final class Metrics {
     int dbOpen;
   }
@@ -159,7 +161,7 @@ public class Tester1 implements Runnable {
     return rv;
   }
 
-  private void testCompileOption(){
+  private void showCompileOption(){
     int i = 0;
     String optName;
     outln("compile options:");
@@ -1175,31 +1177,41 @@ public class Tester1 implements Runnable {
   }
 
   private void runTests(boolean fromThread) throws Exception {
-    if(false) testCompileOption();
+    if(false) showCompileOption();
+    List<java.lang.reflect.Method> mlist = testMethods;
+    affirm( null!=mlist );
+    if( shuffle ){
+      mlist = new ArrayList<>( testMethods.subList(0, testMethods.size()) );
+      java.util.Collections.shuffle(
+        mlist
+        //java.util.concurrent.ThreadLocalRandom.current()
+      );
+    }
+    if( listRunTests ){
+      synchronized(this.getClass()){
+        out("Initial test"," list: ");
+        for(java.lang.reflect.Method m : testMethods){
+          out(m.getName()+" ");
+        }
+        outln();
+
+        out("Running"," tests: ");
+        for(java.lang.reflect.Method m : mlist){
+          out(m.getName()+" ");
+        }
+        outln();
+        out("(That list excludes some which are hard-coded to run.)\n");
+      }
+    }
     testToUtf8();
     test1();
-    nap(); testOpenDb1();
-    nap(); testOpenDb2();
-    nap(); testCollation();
-    nap(); testPrepare123();
-    nap(); testBindFetchInt();
-    nap(); testBindFetchInt64();
-    nap(); testBindFetchDouble();
-    nap(); testBindFetchText();
-    nap(); testBindFetchBlob();
-    nap(); testSql();
-    nap(); testStatus();
-    nap(); testUdf1();
-    nap(); testUdfJavaObject();
-    nap(); testUdfAggregate();
-    nap(); testUdfWindow();
-    nap(); testTrace();
-    nap(); testProgress();
-    nap(); testCommitHook();
-    nap(); testRollbackHook();
-    nap(); testUpdateHook();
-    nap(); testAuthorizer();
-    nap(); testAutoExtension();
+    int n = 0;
+    for(java.lang.reflect.Method m : mlist){
+      ++n;
+      nap();
+      m.invoke(this);
+    }
+    affirm( n == mlist.size() );
     if(!fromThread){
       testBusy();
       if( !mtMode ){
@@ -1219,6 +1231,27 @@ public class Tester1 implements Runnable {
     }
   }
 
+  /**
+     Runs the basic sqlite3 JNI binding sanity-check suite.
+
+     CLI flags:
+
+     -t|-thread N: runs the tests in N threads
+      concurrently. Default=1.
+
+     -r|-repeat N: repeats the tests in a loop N times, each one
+      consisting of the -thread value's threads.
+
+     -shuffle: randomizes the order of most of the test functions.
+
+     -naps: sleep small random intervals between tests in order to add
+     some chaos for cross-thread contention.
+
+     -list-tests: outputs the list of tests being run, minus some
+      which are hard-coded,
+
+     -v: emit some developer-mode info at the end.
+  */
   public static void main(String[] args) throws Exception {
     Integer nThread = null;
     boolean doSomethingForDev = false;
@@ -1232,8 +1265,12 @@ public class Tester1 implements Runnable {
           //listBoundMethods();
         }else if(arg.equals("t") || arg.equals("thread")){
           nThread = Integer.parseInt(args[i++]);
-        }else if(arg.equals("r") || arg.equals("runs")){
+        }else if(arg.equals("r") || arg.equals("repeat")){
           nRepeat = Integer.parseInt(args[i++]);
+        }else if(arg.equals("shuffle")){
+          shuffle = true;
+        }else if(arg.equals("list-tests")){
+          listRunTests = true;
         }else if(arg.equals("naps")){
           takeNaps = true;
         }else{
@@ -1242,6 +1279,24 @@ public class Tester1 implements Runnable {
       }
     }
 
+    {
+      // Build list of tests to run from the methods named test*().
+      testMethods = new ArrayList<>();
+      final List<String> excludes = new ArrayList<>();
+      // Tests we want to control the order of:
+      excludes.add("testSleep");
+      excludes.add("testToUtf8");
+      excludes.add("test1");
+      excludes.add("testBusy");
+      excludes.add("testFts5");
+      for(java.lang.reflect.Method m : Tester1.class.getDeclaredMethods()){
+        final String name = m.getName();
+        if( name.startsWith("test") && excludes.indexOf(name)<0 ){
+          testMethods.add(m);
+        }
+      }
+    }
+
     final long timeStart = System.currentTimeMillis();
     int nLoop = 0;
     outln("libversion_number: ",

manifest

@@ -1,5 +1,5 @@
-C Disassociate\sJNI\sdb\shandles\sfrom\sthe\sthread\sthat\screated\sthem,\sas\sit's\sno\slonger\srelevant.
-D 2023-08-22T18:36:30.981
+C More\swork\son\sthe\sJNI\smulti-threaded\stest\srunner.
+D 2023-08-22T20:10:28.237
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -235,7 +235,7 @@ F ext/icu/sqliteicu.h fa373836ed5a1ee7478bdf8a1650689294e41d0c89c1daab26e9ae78a3
 F ext/jni/GNUmakefile 1ccd09095447709ffd7a4f32514fd586512491c6bed06d009bab4294b451ed62
 F ext/jni/README.md 975b35173debbbf3a4ab7166e14d2ffa2bacff9b6850414f09cc919805e81ba4
 F ext/jni/jar-dist.make 9a03d10dbb5a74c724bfec4b76fd9e4c9865cbbc858d731cb48f38ac897d73a3
-F ext/jni/src/c/sqlite3-jni.c 50edc462e8fdf54f9b8ede692a7c865c5e4315930899276664dd6744764d4723
+F ext/jni/src/c/sqlite3-jni.c c38c18875b946a3bdc4eda0b2f19ad53b895118979ec85a630706c1c5575079b
 F ext/jni/src/c/sqlite3-jni.h 8b0ab1a3f0f92b75d4ff50db4a88b66a137cfb561268eb15bb3993ed174dbb74
 F ext/jni/src/org/sqlite/jni/Authorizer.java 1308988f7f40579ea0e4deeaec3c6be971630566bd021c31367fe3f5140db892
 F ext/jni/src/org/sqlite/jni/AutoExtension.java 3b62c915e45ce73f63343ca9195ec63592244d616a1908b7587bdd45de1b97dd
@@ -249,14 +249,14 @@ F ext/jni/src/org/sqlite/jni/Fts5ExtensionApi.java 10cb2e0eb4dc5cf4241a7ccc0442a
 F ext/jni/src/org/sqlite/jni/Fts5Function.java 65cde7151e441fee012250a5e03277de7babcd11a0c308a832b7940574259bcc
 F ext/jni/src/org/sqlite/jni/Fts5PhraseIter.java 6642beda341c0b1b46af4e2d7f6f9ab03a7aede43277b2c92859176d6bce3be9
 F ext/jni/src/org/sqlite/jni/Fts5Tokenizer.java 91489893596b6528c0df5cd7180bd5b55809c26e2b797fb321dfcdbc1298c060
-F ext/jni/src/org/sqlite/jni/NativePointerHolder.java 9c5d901cce4f7e57c3d623f4e2476f9f79a8eed6e51b2a603f37866018e040ee
+F ext/jni/src/org/sqlite/jni/NativePointerHolder.java 8110d4cfb20884e8ed241de7420c615b040a9f9c441d9cff06f34833399244a8
 F ext/jni/src/org/sqlite/jni/OutputPointer.java 464ea85c3eba673a7b575545f69fcd8aeb398477a26d155d88cee3e2459e7802
 F ext/jni/src/org/sqlite/jni/ProgressHandler.java 6f62053a828a572de809828b1ee495380677e87daa29a1c57a0e2c06b0a131dc
 F ext/jni/src/org/sqlite/jni/ResultCode.java ba701f20213a5f259e94cfbfdd36eb7ac7ce7797f2c6c7fca2004ff12ce20f86
 F ext/jni/src/org/sqlite/jni/RollbackHook.java b04c8abcc6ade44a8a57129e33765793f69df0ba909e49ba18d73f4268d92564
 F ext/jni/src/org/sqlite/jni/SQLFunction.java 8c1ad92c35bcc1b2f7256cf6e229b31340ed6d1a404d487f0a9adb28ba7fc332
 F ext/jni/src/org/sqlite/jni/SQLite3Jni.java 2f36370cfdec01d309720392b2c3e4af6afce0b6ece8188b5c3ed688a5a1e63a
-F ext/jni/src/org/sqlite/jni/Tester1.java 58a058f718215ff32fbdf8026a2d4eb88f9d7e939a5640d5a944efafdfda4b7c
+F ext/jni/src/org/sqlite/jni/Tester1.java da8bc65f52d310ae17b372eeaef25726be47d3a2052e8a33ce44606a7dc451d7
 F ext/jni/src/org/sqlite/jni/TesterFts5.java c729d5b3cb91888b7e2a3a3ef450852f184697df78721574f6c0bf9043e4b84c
 F ext/jni/src/org/sqlite/jni/Tracer.java a5cece9f947b0af27669b8baec300b6dd7ff859c3e6a6e4a1bd8b50f9714775d
 F ext/jni/src/org/sqlite/jni/UpdateHook.java e58645a1727f8a9bbe72dc072ec5b40d9f9362cb0aa24acfe93f49ff56a9016d
@@ -2092,8 +2092,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 02e868690f97ca728b0f2dd018aa79a9d13c85dd85b164caa895d319ae8f3ff5
-R 4fc9fbe1829e4bd8b0e17e9933291453
+P 8b78b737e66a399b04e555a8197f63a73198a4105cb2f37ffd5b0e6014302caf
+R 31a03bd5ee6de6309ec09a8781691d85
 U stephan
-Z 9fda786e7e193bbf92304385fbc27c96
+Z 973ac484c97d56f1bb0793972fc0262a
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-8b78b737e66a399b04e555a8197f63a73198a4105cb2f37ffd5b0e6014302caf
+9a74ad716bded1e14333bf7c72392916f800d58a96240eabe4988ca5fc9e8752