Updates to the fuzzer query invariant checker - tracking changes made

over in dbsqlfuzz.

FossilOrigin-Name: 4ca16a304ad10fbb48f78b4384b347fe883e1a4f222f113ac981e89845c3e113

manifest

@@ -1,5 +1,5 @@
-C Changes\sto\sMakefile.msc\sto\sfix\sthe\sbuild\son\sWindows\sfollowing\sthe\sprevious\nmerge.
+C Updates\sto\sthe\sfuzzer\squery\sinvariant\schecker\s-\stracking\schanges\smade\nover\sin\sdbsqlfuzz.
-D 2022-10-12T18:30:08.886
+D 2022-10-12T18:40:25.766
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -1149,7 +1149,7 @@ F test/fuzzdata8.db 653423800b7671e67caa740e977d80e1360f0d69e9992851f3ea5c4a69a2
 F test/fuzzer1.test 3d4c4b7e547aba5e5511a2991e3e3d07166cfbb8
 F test/fuzzer2.test a85ef814ce071293bce1ad8dffa217cbbaad4c14
 F test/fuzzerfault.test f64c4aef4c9e9edf1d6dc0d3f1e65dcc81e67c996403c88d14f09b74807a42bc
-F test/fuzzinvariants.c d7bb4a0fcc0ac344bcb72f1b86e4ae0acba5ea26dddde8160ee3db6520f10c64
+F test/fuzzinvariants.c a91f439e512c72045bddaf0f0ee3b876b2fe645e076aefb085658f1337066f7e
 F test/gcfault.test dd28c228a38976d6336a3fc42d7e5f1ad060cb8c
 F test/gencol1.test cc0dbb0ee116e5602e18ea7d47f2a0f76b26e09a823b7c36ef254370c2b0f3c1
 F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98
@@ -2034,8 +2034,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 98f954942a8d4824b5c4bd1d27cfeff7a71a029dc5a54220b76d59e367996581
+P 368fa6b25bc803ded7c1a0184615980902657879370caec22ceea42496ec0566
-R 54aab607449d14558025d70793f3c47c
+R b6417444abfe9dc882e90feaa0e7d85d
 U drh
-Z e026cd3c8f91b1cd55f89d31b35f702a
+Z 6e9d8b3a258c07649344995d3cb09455
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-368fa6b25bc803ded7c1a0184615980902657879370caec22ceea42496ec0566
+4ca16a304ad10fbb48f78b4384b347fe883e1a4f222f113ac981e89845c3e113

test/fuzzinvariants.c

@@ -46,11 +46,14 @@ static void reportInvariantFailed(sqlite3_stmt*,sqlite3_stmt*,int);
 **
 **     SQLITE_OK          This check was successful.
 **
-**     SQLITE_DONE        iCnt is out of range.
+**     SQLITE_DONE        iCnt is out of range.  The caller typically sets
+**                        up a loop on iCnt starting with zero, and increments
+**                        iCnt until this code is returned.
 **
 **     SQLITE_CORRUPT     The invariant failed, but the underlying database
 **                        file is indicating that it is corrupt, which might
-**                        be the cause of the malfunction.
+**                        be the cause of the malfunction.  The *pCorrupt
+**                        value will also be set.
 **
 **     SQLITE_INTERNAL    The invariant failed, and the database file is not
 **                        corrupt.  (This never happens because this function
@@ -142,6 +145,20 @@ int fuzz_invariant(
       */
       goto not_a_fault;
     }
+    if( sqlite3_strlike("%limit%)%order%by%", sqlite3_sql(pTestStmt),0)==0 ){
+      /* crash-89bd6a6f8c6166e9a4c5f47b3e70b225f69b76c6
+      ** Original statement is:
+      **
+      **    SELECT a,b,c* FROM t1 LIMIT 1%5<4
+      **
+      ** When running:
+      **
+      **    SELECT * FROM (...) ORDER BY 1
+      **
+      ** A different subset of the rows come out
+      */
+      goto not_a_fault;
+    }
     rc = sqlite3_prepare_v2(db, 
             "SELECT 1 FROM bytecode(?1) WHERE opcode='VOpen'", -1, &pCk, 0);
     if( rc==SQLITE_OK ){
@@ -166,6 +183,24 @@ not_a_fault:
 ** Generate SQL used to test a statement invariant.
 **
 ** Return 0 if the iCnt is out of range.
+**
+** iCnt meanings:
+**
+**   0     SELECT * FROM (<query>)
+**   1     SELECT DISTINCT * FROM (<query>)
+**   2     SELECT * FROM (<query>) WHERE ORDER BY 1
+**   3     SELECT DISTINCT * FROM (<query>) ORDER BY 1
+**   4     SELECT * FROM (<query>) WHERE <all-columns>=<all-values>
+**   5     SELECT DISTINCT * FROM (<query>) WHERE <all-columns=<all-values
+**   6     SELECT * FROM (<query>) WHERE <all-column>=<all-value> ORDER BY 1
+**   7     SELECT DISTINCT * FROM (<query>) WHERE <all-column>=<all-value>
+**                           ORDER BY 1
+**   N+0   SELECT * FROM (<query>) WHERE <nth-column>=<value>
+**   N+1   SELECT DISTINCT * FROM (<query>) WHERE <Nth-column>=<value>
+**   N+2   SELECT * FROM (<query>) WHERE <Nth-column>=<value> ORDER BY 1
+**   N+3   SELECT DISTINCT * FROM (<query>) WHERE <Nth-column>=<value>
+**                           ORDER BY N
+**
 */
 static char *fuzz_invariant_sql(sqlite3_stmt *pStmt, int iCnt){
   const char *zIn;
@@ -182,7 +217,6 @@ static char *fuzz_invariant_sql(sqlite3_stmt *pStmt, int iCnt){
   int bOrderBy = 0;
   int nParam = sqlite3_bind_parameter_count(pStmt);
 
-  iCnt++;
   switch( iCnt % 4 ){
     case 1:  bDistinct = 1;              break;
     case 2:  bOrderBy = 1;               break;
@@ -197,9 +231,10 @@ static char *fuzz_invariant_sql(sqlite3_stmt *pStmt, int iCnt){
   while( nIn>0 && (isspace(zIn[nIn-1]) || zIn[nIn-1]==';') ) nIn--;
   if( strchr(zIn, '?') ) return 0;
   pTest = sqlite3_str_new(0);
-  sqlite3_str_appendf(pTest, "SELECT %s* FROM (%s",
+  sqlite3_str_appendf(pTest, "SELECT %s* FROM (",  
-                      bDistinct ? "DISTINCT " : "", zIn);
+                      bDistinct ? "DISTINCT " : "");
-  sqlite3_str_appendf(pTest, ")");
+  sqlite3_str_append(pTest, zIn, nIn);
+  sqlite3_str_append(pTest, ")", 1);
   rc = sqlite3_prepare_v2(db, sqlite3_str_value(pTest), -1, &pBase, 0);
   if( rc ){
     sqlite3_finalize(pBase);
@@ -216,7 +251,8 @@ static char *fuzz_invariant_sql(sqlite3_stmt *pStmt, int iCnt){
       ** WHERE clause. */
       continue;
     }
-    if( i+1!=iCnt ) continue;
+    if( iCnt==0 ) continue;
+    if( iCnt>1 && i+2!=iCnt ) continue;
     if( zColName==0 ) continue;
     if( sqlite3_column_type(pStmt, i)==SQLITE_NULL ){
       sqlite3_str_appendf(pTest, " %s \"%w\" ISNULL", zAnd, zColName);
@@ -228,7 +264,7 @@ static char *fuzz_invariant_sql(sqlite3_stmt *pStmt, int iCnt){
   }
   if( pBase!=pStmt ) sqlite3_finalize(pBase);
   if( bOrderBy ){
-    sqlite3_str_appendf(pTest, " ORDER BY 1");
+    sqlite3_str_appendf(pTest, " ORDER BY %d", iCnt>2 ? iCnt-1 : 1);
   }
   return sqlite3_str_finish(pTest);
 }
@@ -259,10 +295,30 @@ static int sameValue(sqlite3_stmt *pS1, int i1, sqlite3_stmt *pS2, int i2){
       break;
     }
     case SQLITE_TEXT: {
-      const char *z1 = (const char*)sqlite3_column_text(pS1,i1);
+      int e1 = sqlite3_value_encoding(sqlite3_column_value(pS1,i1));
-      const char *z2 = (const char*)sqlite3_column_text(pS2,i2);
+      int e2 = sqlite3_value_encoding(sqlite3_column_value(pS2,i2));
-      x = ((z1==0 && z2==0) || (z1!=0 && z2!=0 && strcmp(z1,z1)==0));
+      if( e1!=e2 ){
-      break;
+        const char *z1 = (const char*)sqlite3_column_text(pS1,i1);
+        const char *z2 = (const char*)sqlite3_column_text(pS2,i2);
+        x = ((z1==0 && z2==0) || (z1!=0 && z2!=0 && strcmp(z1,z1)==0));
+        printf("Encodings differ.  %d on left and %d on right\n", e1, e2);
+        break;
+      }
+      if( e1!=SQLITE_UTF8 ){
+        int len1 = sqlite3_column_bytes16(pS1,i1);
+        const unsigned char *b1 = sqlite3_column_blob(pS1,i1);
+        int len2 = sqlite3_column_bytes16(pS2,i2);
+        const unsigned char *b2 = sqlite3_column_blob(pS2,i2);
+        if( len1!=len2 ){
+          x = 0;
+        }else if( len1==0 ){
+          x = 1;
+        }else{
+          x = (b1!=0 && b2!=0 && memcmp(b1,b2,len1)==0);
+        }
+        break;
+      }
+      /* Fall through into the SQLITE_BLOB case */
     }
     case SQLITE_BLOB: {
       int len1 = sqlite3_column_bytes(pS1,i1);
@@ -282,11 +338,23 @@ static int sameValue(sqlite3_stmt *pS1, int i1, sqlite3_stmt *pS2, int i2){
   return x;
 }
 
+/*
+** Print binary data as hex
+*/
+static void printHex(const unsigned char *a, int n, int mx){
+  int j;
+  for(j=0; j<mx && j<n; j++){
+    printf("%02x", a[j]);
+  }
+  if( j<n ) printf("...");
+}
+
 /*
 ** Print a single row from the prepared statement
 */
 static void printRow(sqlite3_stmt *pStmt, int iRow){
-  int i, nCol;
+  int i, n, nCol;
+  unsigned const char *data;
   nCol = sqlite3_column_count(pStmt);
   for(i=0; i<nCol; i++){
     printf("row%d.col%d = ", iRow, i);
@@ -304,18 +372,44 @@ static void printRow(sqlite3_stmt *pStmt, int iRow){
         break;
       }
       case SQLITE_TEXT: {
-        printf("(text) \"%s\"\n", sqlite3_column_text(pStmt, i));
+        switch( sqlite3_value_encoding(sqlite3_column_value(pStmt,i)) ){
+          case SQLITE_UTF8: {
+            printf("(utf8) x'");
+            n = sqlite3_column_bytes(pStmt, i);
+            data = sqlite3_column_blob(pStmt, i);
+            printHex(data, n, 35);
+            printf("'\n");
+            break;
+          }
+          case SQLITE_UTF16BE: {
+            printf("(utf16be) x'");
+            n = sqlite3_column_bytes16(pStmt, i);
+            data = sqlite3_column_blob(pStmt, i);
+            printHex(data, n, 35);
+            printf("'\n");
+            break;
+          }
+          case SQLITE_UTF16LE: {
+            printf("(utf16le) x'");
+            n = sqlite3_column_bytes16(pStmt, i);
+            data = sqlite3_column_blob(pStmt, i);
+            printHex(data, n, 35);
+            printf("'\n");
+            break;
+          }
+          default: {
+            printf("Illegal return from sqlite3_value_encoding(): %d\n",
+                sqlite3_value_encoding(sqlite3_column_value(pStmt,i)));
+            abort();
+          }
+        }
         break;
       }
       case SQLITE_BLOB: {
-        int n = sqlite3_column_bytes(pStmt, i);
+        n = sqlite3_column_bytes(pStmt, i);
-        int j;
+        data = sqlite3_column_blob(pStmt, i);
-        unsigned const char *data = sqlite3_column_blob(pStmt, i);
         printf("(blob %d bytes) x'", n);
-        for(j=0; j<20 && j<n; j++){
+        printHex(data, n, 35);
-          printf("%02x", data[j]);
-        }
-        if( j<n ) printf("...");
         printf("'\n");
         break;
       }