Initial sketches of a line-by-line parser for SQLTester to overcome its compatibility shortcomings. Far from complete.

FossilOrigin-Name: 43534cd042499c1bef44ca5c4a8305a710d99e70e8b0adce6df50c6a1f0402b9

ext/jni/GNUmakefile

@@ -87,6 +87,7 @@ JAVA_FILES.tester := $(patsubst %,$(dir.src.jni.tester)/%,\
   Outer.java \
   SQLTester.java \
   TestScript.java \
+  TestScript2.java \
 )
 
 CLASS_FILES.main := $(JAVA_FILES.main:.java=.class)

ext/jni/src/org/sqlite/jni/tester/SQLTester.java

@@ -80,6 +80,7 @@ public class SQLTester {
   private int iCurrentDb = 0;
   private final String initialDbName = "test.db";
   private TestScript currentScript;
+  private TestScript2 currentScript2;
 
   public SQLTester(){
     reset();
@@ -155,8 +156,32 @@ public class SQLTester {
     Util.unlink(initialDbName);
   }
 
+
+  //! Not yet funcional
+  private void runTests2() throws Exception {
+    try {
+      for(String f : listInFiles){
+        reset();
+        setupInitialDb();
+        ++nTestFile;
+        final TestScript2 ts = new TestScript2(f);
+        currentScript2 = ts;
+        try{
+          ts.run(this);
+        }catch(SkipTestRemainder e){
+          /* not an error */
+          ++nAbortedScript;
+        }
+        outln("<<<<<----- ",nTest," test(s) in ",ts.getFilename());
+      }
+    }finally{
+      currentScript2 = null;
+    }
+    Util.unlink(initialDbName);
+  }
+
   private StringBuilder clearBuffer(StringBuilder b){
-    b.delete(0, b.length());
+    b.setLength(0);;
     return b;
   }
 
@@ -403,11 +428,14 @@ public class SQLTester {
 
   public static void main(String[] argv) throws Exception{
     final SQLTester t = new SQLTester();
+    boolean v2 = false;
     for(String a : argv){
       if(a.startsWith("-")){
         final String flag = a.replaceFirst("-+","");
         if( flag.equals("verbose") ){
           t.setVerbosity(t.getVerbosity() + 1);
+        }else if( flag.equals("2") ){
+          v2 = true;
         }else{
           throw new IllegalArgumentException("Unhandled flag: "+flag);
         }
@@ -415,7 +443,8 @@ public class SQLTester {
       }
       t.addTestScript(a);
     }
-    t.runTests();
+    if( v2 ) t.runTests2();
+    else t.runTests();
     t.outln("Processed ",t.nTotalTest," test(s) in ",t.nTestFile," file(s).");
     if( t.nAbortedScript > 0 ){
       t.outln("Aborted ",t.nAbortedScript," script(s).");

ext/jni/src/org/sqlite/jni/tester/TestScript2.java

@@ -0,0 +1,166 @@
+/*
+** 2023-08-08
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file contains the TestScript2 part of the SQLTester framework.
+*/
+package org.sqlite.jni.tester;
+//import java.util.regex.*;
+import java.util.Arrays;
+import java.nio.charset.StandardCharsets;
+
+/**
+   This class represents a single test script. It handles (or
+   delegates) its the reading-in and parsing, but the details of
+   evaluation are delegated elsewhere.
+*/
+class TestScript2 {
+  private String filename = null;
+  private final Cursor curs = new Cursor();
+  private final Outer outer = new Outer();
+
+  private static final class Cursor {
+    private final StringBuilder sb = new StringBuilder();
+    byte[] src = null;
+    int pos = 0;
+    int lineNo = 1;
+    boolean inComment = false;
+
+    void reset(){
+      sb.setLength(0); pos = 0; lineNo = 1; inComment = false;
+    }
+  }
+
+  private byte[] readFile(String filename) throws Exception {
+    return java.nio.file.Files.readAllBytes(java.nio.file.Paths.get(filename));
+  }
+
+  /**
+     Initializes the script with the content of the given file.
+     Throws if it cannot read the file.
+  */
+  public TestScript2(String filename) throws Exception{
+    this.filename = filename;
+    setVerbosity(2);
+    curs.src = readFile(filename);
+  }
+
+  public String getFilename(){
+    return filename;
+  }
+
+  public void setVerbosity(int level){
+    outer.setVerbosity(level);
+  }
+
+  @SuppressWarnings("unchecked")
+  private <T> TestScript2 verbose(T... vals){
+    outer.verbose(vals);
+    return this;
+  }
+
+  @SuppressWarnings("unchecked")
+  private void tossSyntax(Object... msg){
+    StringBuilder sb = new StringBuilder();
+    sb.append(this.filename).append(":").append(curs.lineNo).
+      append(": ");
+    for(Object o : msg) sb.append(o);
+    throw new RuntimeException(sb.toString());
+  }
+
+  private void reset(){
+    curs.reset();
+  }
+
+  /**
+     Returns the next line from the buffer, minus the trailing EOL.
+     If skipLeadingWs is true then all leading whitespace (including
+     blank links) is skipped over and will not appear in the resulting
+     string.
+
+     Returns null when all input is consumed. Throws if it reads
+     illegally-encoded input, e.g. (non-)characters in the range
+     128-256.
+  */
+  String getLine(boolean skipLeadingWs){
+    curs.sb.setLength(0);
+    byte b = 0, prevB = 0;
+    int i = curs.pos;
+    if(skipLeadingWs) {
+      /* Skip any leading spaces, including newlines. This will eliminate
+         blank lines. */
+      for(; i < curs.src.length; ++i, prevB=b){
+        b = curs.src[i];
+        switch((int)b){
+          case 32/*space*/: case 9/*tab*/: case 13/*CR*/: continue;
+          case 10/*NL*/: ++curs.lineNo; continue;
+          default: break;
+        }
+        break;
+      }
+    }
+    if( i==curs.src.length ){
+      return null /* EOF */;
+    }
+    boolean doBreak = false;
+    final byte[] aChar = {0,0,0,0} /* multi-byte char buffer */;
+    int nChar = 0 /* number of bytes in the char */;
+    for(; i < curs.src.length && !doBreak; ++i){
+      b = curs.src[i];
+      switch( (int)b ){
+        case 13/*CR*/: continue;
+        case 10/*NL*/:
+          ++curs.lineNo;
+          if(curs.sb.length()>0) doBreak = true;
+          break;
+        default:
+          /* Multi-byte chars need to be gathered up and appended at
+             one time. Appending individual bytes to the StringBuffer
+             appends their integer value. */
+          nChar = 1;
+          switch( b & 0xF0 ){
+            case 0xC0: nChar = 2; break;
+            case 0xE0: nChar = 3; break;
+            case 0xF0: nChar = 4; break;
+            default:
+              if( b > 127 ) tossSyntax("Invalid character (#"+(int)b+").");
+              break;
+          }
+          if( 1==nChar ){
+            curs.sb.append((char)b);
+          }else{
+            for(int x = 0; x < nChar; ++x) aChar[x] = curs.src[i+x];
+            curs.sb.append(new String(Arrays.copyOf(aChar, nChar),
+                                      StandardCharsets.UTF_8));
+            i += nChar-1;
+          }
+          break;
+      }
+    }
+    curs.pos = i;
+    if( 0==curs.sb.length() && i==curs.src.length ){
+      return null /* EOF */;
+    }
+    return curs.sb.toString();
+  }/*getLine()*/
+
+  /**
+     Runs this test script in the context of the given tester object.
+  */
+  @SuppressWarnings("unchecked")
+  public void run(SQLTester tester) throws Exception {
+    reset();
+    setVerbosity(tester.getVerbosity());
+    String line;
+    while( null != (line = getLine(false)) ){
+      verbose("LINE #",curs.lineNo-1,": ",line);
+    }
+  }
+}

ext/jni/src/tests/000-000-sanity.test2

@@ -0,0 +1,10 @@
+  /*
+** This is a comment. There are many like it but this one is mine.
+**
+  */
+--command 1
+
+
+--command 🤩😃
+  SELECT 1;
+  SELECT 2;

manifest

@@ -1,5 +1,5 @@
-C Adapt\sTestScript\sto\sskip\sREQUIRED_PROPERTIES,\sper\s[7a07863e082664da],\sand\simprove\sa\scouple\sof\sadject\sreasons-for-skipping\smessages.
-D 2023-08-09T18:25:50.693
+C Initial\ssketches\sof\sa\sline-by-line\sparser\sfor\sSQLTester\sto\sovercome\sits\scompatibility\sshortcomings.\sFar\sfrom\scomplete.
+D 2023-08-09T19:51:39.077
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -230,7 +230,7 @@ F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c
 F ext/icu/README.txt 7ab7ced8ae78e3a645b57e78570ff589d4c672b71370f5aa9e1cd7024f400fc9
 F ext/icu/icu.c c074519b46baa484bb5396c7e01e051034da8884bad1a1cb7f09bbe6be3f0282
 F ext/icu/sqliteicu.h fa373836ed5a1ee7478bdf8a1650689294e41d0c89c1daab26e9ae78a32075a8
-F ext/jni/GNUmakefile 52f402abb8c4695a58f734d20455cf1a5afaaa10ceacc47bcbf1b06a8d5d27e8
+F ext/jni/GNUmakefile d7300b7e124214afde7f11bddd5c0d336a9be0220fe2b74e787078e1aa2db778
 F ext/jni/README.md e965674505e105626127ad45e628e4d19fcd379cdafc4d23c814c1ac2c55681d
 F ext/jni/src/c/sqlite3-jni.c bae09ff8bf45f19a506a4eaaf693d26b81f0dd0a410b82475e04dde4b1c5a520
 F ext/jni/src/c/sqlite3-jni.h 84a3fc3d308e347a2f6b24e4cb8bbafdfa8e75361302047d788e51a307cb2328
@@ -266,9 +266,11 @@ F ext/jni/src/org/sqlite/jni/sqlite3_context.java d26573fc7b309228cb49786e907859
 F ext/jni/src/org/sqlite/jni/sqlite3_stmt.java 78e6d1b95ac600a9475e9db4623f69449322b0c93d1bd4e1616e76ed547ed9fc
 F ext/jni/src/org/sqlite/jni/sqlite3_value.java 3d1d4903e267bc0bc81d57d21f5e85978eff389a1a6ed46726dbe75f85e6914a
 F ext/jni/src/org/sqlite/jni/tester/Outer.java b06acf9c79e8dbc8fea4a98b00724a6a76e3ee4503eb114671d2885f8fb3df8b
-F ext/jni/src/org/sqlite/jni/tester/SQLTester.java 0869fa25e32295e1742e0ab4c2f20e25e47ab5f6d520ab31b7ea075249781548
+F ext/jni/src/org/sqlite/jni/tester/SQLTester.java 35ea65f416b41b1e21119f58109e35518a39cbc1edbdd8883cb581772665c18d
 F ext/jni/src/org/sqlite/jni/tester/TestScript.java 18f55e1e3001c4ccfc359d57448729227c3eaf4a7c774964fe6418e07aefd541
+F ext/jni/src/org/sqlite/jni/tester/TestScript2.java 1c8426039f2050cf623dab17e5d05a976ee061429c7be1eb5a3e73f7b00daf65
 F ext/jni/src/org/sqlite/jni/tester/test-script-interpreter.md ab7169b08566a082ef55c9ef8a553827f99958ed3e076f31eef757563fae51ba
+F ext/jni/src/tests/000-000-sanity.test2 0b4bb90e12cfa96e24999f5890ce46305b6a83efcf9ef0ff4df16521f636a9d5
 F ext/jni/src/tests/000_first.test cd5fb732520cf36d7a3e5ad94a274c7327a9504b01a1a7f98e1f946df6c539fd
 F ext/jni/src/tests/010_ignored.test e17e874c6ab3c437f1293d88093cf06286083b65bf162317f91bbfd92f961b70
 F ext/lsm1/Makefile a553b728bba6c11201b795188c5708915cc4290f02b7df6ba7e8c4c943fd5cd9
@@ -2090,8 +2092,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 7a07863e082664da2efcf4ecd36785d2583abbda12526cdb643cf1aa0568292e
-R 2aec460c6025f35f30c5bc23045915cc
+P f937097e9b22a6c78c242cbf00c71bdc57f04b1b9a15ae24058bc2813c99688c
+R e3dd73f0afa45739c2d0daea22589f05
 U stephan
-Z c1a2fe06f34d73f23c401dc70e624dbe
+Z 24fe8a8f37b2f666f1b6b1c10180bf33
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-f937097e9b22a6c78c242cbf00c71bdc57f04b1b9a15ae24058bc2813c99688c
+43534cd042499c1bef44ca5c4a8305a710d99e70e8b0adce6df50c6a1f0402b9