Add a hack to debug out a description of the WHERE clause of a SELECT (or other) statement. Use this in script tool/schemalint.tcl to automatically recommend indexes that might speed up specific queries.

FossilOrigin-Name: c6fa01c28ef7ceea2963a92dfffe62eed451b05c

manifest

@@ -1,5 +1,5 @@
-C Improvements\sto\sthe\sSQLITE_CONFIG_PAGECACHE\sdocumentation.\s\sEnhance\sthe\ncommand-line\sshell\sto\sbe\sable\sto\stake\sadvantage\sof\sthe\sfull\srange\sof\nSQLITE_CONFIG_PAGECACHE\scapabilities,\ssuch\sas\ssetting\spMem==NULL\sand\sN<0.
+C Add\sa\shack\sto\sdebug\sout\sa\sdescription\sof\sthe\sWHERE\sclause\sof\sa\sSELECT\s(or\sother)\sstatement.\sUse\sthis\sin\sscript\stool/schemalint.tcl\sto\sautomatically\srecommend\sindexes\sthat\smight\sspeed\sup\sspecific\squeries.
-D 2015-11-11T15:28:52.898
+D 2015-11-11T18:08:58.267
 F Makefile.in d828db6afa6c1fa060d01e33e4674408df1942a1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e928e68168df69b353300ac87c10105206653a03
@@ -416,7 +416,7 @@ F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb
 F src/wal.c 18b0ed49830cf04fe2d68224b41838a73ac6cd24
 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4
 F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba
-F src/where.c 6aceb72cc58dc06922a9e1604d559c8ca4c3e728
+F src/where.c 6176426332d5a67df4222adfeae8c22a933f8c84
 F src/whereInt.h 7892bb54cf9ca0ae5c7e6094491b94c9286dc647
 F src/wherecode.c 4c96182e7b25e4be54008dee2da5b9c2f8480b9b
 F src/whereexpr.c e63244ca06c503e5f3c5b7f3c9aea0db826089ed
@@ -972,6 +972,7 @@ F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5
 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38
 F test/schema4.test e6a66e20cc69f0e306667c08be7fda3d11707dc5
 F test/schema5.test 29699b4421f183c8f0e88bd28ce7d75d13ea653e
+F test/schemalint.test 22f26e6e9d8fe437b6344a97c91fd14c95e813a7
 F test/securedel.test 21749c32ccc30f1ea9e4b9f33295a6521ec20fa0
 F test/securedel2.test 2d54c28e46eb1fd6902089958b20b1b056c6f1c5
 F test/select1.test be62204d2bd9a5a8a149e9974cfddce893d8f686
@@ -1378,6 +1379,7 @@ F tool/replace.tcl 7727c60a04299b65a92f5e1590896fea0f25b9e0
 F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a
 F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5
 F tool/run-speed-test.sh 0ae485af4fe9f826e2b494be8c81f8ca9e222a4a
+F tool/schemalint.tcl 2eb60e950e91061c27d771582fe87e9518d0988d
 F tool/showdb.c d4476e000a64eca9f5e2c2f68741e747b9778e8d
 F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818
 F tool/showlocks.c 9920bcc64f58378ff1118caead34147201f48c68
@@ -1402,7 +1404,10 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P e43e1171fd7837a08069dc25df4eac14db1c2afe
+P 2518d5c971c4b32d9227b3bb7259162e3e27b00b
-R 9a1d72f8baac116c36385e92e48c0378
+R 9a794d90ebfe5dc636fc86a7e99d61d5
-U drh
+T *branch * schemalint
-Z 9d52ec874b9a01b7b0eb1dab69dbb993
+T *sym-schemalint *
+T -sym-trunk *
+U dan
+Z 95ad0fe90f5fe7059fb61a18885ffca1

manifest.uuid

@@ -1 +1 @@
-2518d5c971c4b32d9227b3bb7259162e3e27b00b
+c6fa01c28ef7ceea2963a92dfffe62eed451b05c

src/where.c

@@ -3901,6 +3901,155 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){
   return 0;
 }
 
+#ifdef SQLITE_SCHEMA_LINT
+static char *whereAppendPrintf(sqlite3 *db, const char *zFmt, ...){
+  va_list ap;
+  char *zRes = 0;
+  va_start(ap, zFmt);
+  zRes = sqlite3_vmprintf(zFmt, ap);
+  if( zRes==0 ){
+    db->mallocFailed = 1;
+  }else if( db->mallocFailed ){
+    sqlite3_free(zRes);
+    zRes = 0;
+  }
+  va_end(ap);
+  return zRes;
+}
+
+/*
+** Append a representation of term pTerm to the string in zIn and return
+** the result. Or, if an OOM occurs, free zIn and return a NULL pointer.
+*/
+static char *whereAppendSingleTerm(
+  Parse *pParse,
+  Table *pTab,
+  int bOr,
+  char *zIn,
+  WhereTerm *pTerm
+){
+  char *zBuf;
+  sqlite3 *db = pParse->db;
+  Expr *pX = pTerm->pExpr;
+  CollSeq *pColl;
+  const char *zOp = 0;
+
+  if( pTerm->eOperator & (WO_IS|WO_EQ|WO_IN) ){
+    zOp = "eq";
+  }else if( pTerm->eOperator & (WO_LT|WO_LE|WO_GE|WO_GT) ){
+    zOp = "range";
+  }
+  pColl = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight);
+
+  if( zOp ){
+    const char *zFmt = bOr ? "%z{{%s %s %s %lld}}" : "%z{%s %s %s %lld}";
+    zBuf = whereAppendPrintf(db, zFmt, zIn, 
+        zOp, pTab->aCol[pTerm->u.leftColumn].zName, 
+        (pColl ? pColl->zName : "BINARY"),
+        pTerm->prereqRight
+    );
+  }else{
+    zBuf = zIn;
+  }
+
+  return zBuf;
+}
+
+static char *whereTraceWC(
+  Parse *pParse, 
+  struct SrcList_item *pItem,
+  char *zIn,
+  WhereClause *pWC
+){
+  sqlite3 *db = pParse->db;
+  Table *pTab = pItem->pTab;
+  char *zBuf = zIn;
+  int iCol;
+  int ii;
+  int bFirst = 1;
+
+  /* List of WO_SINGLE constraints */
+  for(iCol=0; iCol<pTab->nCol; iCol++){
+    int opMask = WO_SINGLE; 
+    WhereScan scan;
+    WhereTerm *pTerm;
+    for(pTerm=whereScanInit(&scan, pWC, pItem->iCursor, iCol, opMask, 0);
+        pTerm;
+        pTerm=whereScanNext(&scan)
+    ){
+      assert( iCol==pTerm->u.leftColumn );
+      if( bFirst==0 ) zBuf = whereAppendPrintf(db, "%z ", zBuf);
+      zBuf = whereAppendSingleTerm(pParse, pTab, pWC->op==TK_OR, zBuf, pTerm);
+      bFirst = 0;
+    }
+  }
+
+  /* Add composite - (WO_OR|WO_AND) - constraints */
+  for(ii=0; ii<pWC->nTerm; ii++){
+    WhereTerm *pTerm = &pWC->a[ii];
+    if( pTerm->eOperator & (WO_OR|WO_AND) ){
+      const char *zFmt = ((pTerm->eOperator&WO_OR) ? "%z%s{or " : "%z%s{");
+      zBuf = whereAppendPrintf(db, zFmt, zBuf, bFirst ? "" : " ");
+      zBuf = whereTraceWC(pParse, pItem, zBuf, &pTerm->u.pOrInfo->wc);
+      zBuf = whereAppendPrintf(db, "%z}", zBuf);
+      bFirst = 0;
+    }
+  }
+
+  return zBuf;
+}
+
+static void whereTraceBuilder(
+  Parse *pParse,
+  WhereLoopBuilder *p
+){
+  sqlite3 *db = pParse->db;
+  if( db->xTrace ){
+    WhereInfo *pWInfo = p->pWInfo;
+    int nTablist = pWInfo->pTabList->nSrc;
+    int ii;
+
+    /* Loop through each element of the FROM clause. Ignore any sub-selects
+    ** or views. Invoke the xTrace() callback once for each real table. */
+    for(ii=0; ii<nTablist; ii++){
+      char *zBuf = 0;
+      int iCol;
+      int nCol;
+      Table *pTab;
+
+      struct SrcList_item *pItem = &pWInfo->pTabList->a[ii];
+      if( pItem->pSelect ) continue;
+      pTab = pItem->pTab;
+      nCol = pTab->nCol;
+
+      /* Append the table name to the buffer. */
+      zBuf = whereAppendPrintf(db, "%s", pTab->zName);
+
+      /* Append the list of columns required to create a covering index */
+      zBuf = whereAppendPrintf(db, "%z {cols", zBuf);
+      if( 0==(pItem->colUsed & ((u64)1 << (sizeof(Bitmask)*8-1))) ){
+        for(iCol=0; iCol<nCol; iCol++){
+          if( iCol==(sizeof(Bitmask)*8-1) ) break;
+          if( pItem->colUsed & ((u64)1 << iCol) ){
+            zBuf = whereAppendPrintf(db, "%z %s", zBuf, pTab->aCol[iCol].zName);
+          }
+        }
+      }
+      zBuf = whereAppendPrintf(db, "%z} ", zBuf);
+
+      /* Append the contents of WHERE clause */
+      zBuf = whereTraceWC(pParse, pItem, zBuf, p->pWC);
+
+      /* Pass the buffer to the xTrace() callback, then free it */
+      db->xTrace(db->pTraceArg, zBuf);
+      sqlite3DbFree(db, zBuf);
+    }
+  }
+}
+#else
+# define whereTraceBuilder(x,y)
+#endif
+
 /*
 ** Generate the beginning of the loop used for WHERE clause processing.
 ** The return value is a pointer to an opaque structure that contains
@@ -4161,6 +4310,9 @@ WhereInfo *sqlite3WhereBegin(
   }
 #endif
 
+  /* Schema-lint xTrace callback */
+  whereTraceBuilder(pParse, &sWLB);
+
   if( nTabList!=1 || whereShortCut(&sWLB)==0 ){
     rc = whereLoopAddAll(&sWLB);
     if( rc ) goto whereBeginError;

test/schemalint.test

@@ -0,0 +1,81 @@
+
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix schemalint
+
+proc xTrace {zMsg} {
+  lappend ::trace_out $zMsg
+}
+db trace xTrace
+
+proc do_trace_test {tn sql res} {
+  uplevel [list do_test $tn [subst -nocommands {
+    set ::trace_out [list]
+    set stmt [sqlite3_prepare db "$sql" -1 x]
+    sqlite3_finalize [set stmt]
+    set ::trace_out
+  }] [list {*}$res]]
+}
+
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, b, c);
+  CREATE TABLE t2(x, y, z);
+}
+
+do_trace_test 1.1 {
+  SELECT b, c, y, z FROM t1, t2 WHERE c=? AND z=?
+} {
+  {t1 {cols b c} {eq c BINARY 0}}
+  {t2 {cols y z} {eq z BINARY 0}}
+}
+
+do_trace_test 1.2 {
+  SELECT a FROM t1 WHERE b>10
+} {
+  {t1 {cols a b} {range b BINARY 0}}
+}
+
+do_trace_test 1.3 {
+  SELECT b FROM t1 WHERE b IN (10, 20, 30)
+} {
+  {t1 {cols b} {eq b BINARY 0}}
+}
+
+do_trace_test 1.4 {
+  SELECT * FROM t1, t2 WHERE x=a
+} {
+  {t1 {cols a b c} {eq a BINARY 2}} 
+  {t2 {cols x y z} {eq x BINARY 1}}
+}
+
+do_trace_test 1.5 {
+  SELECT * FROM t1 WHERE a IN (1, 2, 3)
+} {
+  {t1 {cols a b c} {eq a BINARY 0}}
+}
+
+#-----------------------------------------------------------------------
+# Cases involving OR clauses in the WHERE clause.
+#
+do_trace_test 2.1 {
+  SELECT * FROM t1 WHERE a=? OR b=?
+} {
+  {t1 {cols a b c} {or {{eq a BINARY 0}} {{eq b BINARY 0}}}}
+}
+
+do_trace_test 2.2 {
+  SELECT * FROM t1 WHERE a=? OR (b=? AND c=?)
+} {
+  {t1 {cols a b c} {or {{eq a BINARY 0}} {{eq b BINARY 0} {eq c BINARY 0}}}}
+}
+
+do_trace_test 2.3 {
+  SELECT * FROM t1 WHERE (a=? AND b=?) OR c=?
+} {
+  {t1 {cols a b c} {or {{eq c BINARY 0}} {{eq a BINARY 0} {eq b BINARY 0}}}}
+}
+
+finish_test
+

tool/schemalint.tcl

@@ -0,0 +1,259 @@
+
+
+
+set ::G(lSelect)  [list]           ;# List of SELECT statements to analyze
+set ::G(database) ""               ;# Name of database or SQL schema file
+set ::G(trace)    [list]           ;# List of data from xTrace()
+set ::G(verbose)  0                ;# True if -verbose option was passed 
+
+proc usage {} {
+  puts stderr "Usage: $::argv0 ?SWITCHES? DATABASE/SCHEMA"
+  puts stderr "  Switches are:"
+  puts stderr "  -select SQL     (recommend indexes for SQL statement)"
+  puts stderr "  -verbose        (increase verbosity of output)"
+  puts stderr ""
+  exit
+}
+
+proc process_cmdline_args {argv} {
+  global G
+  set nArg [llength $argv]
+  set G(database) [lindex $argv end]
+
+  for {set i 0} {$i < [llength $argv]-1} {incr i} {
+    set k [lindex $argv $i]
+    switch -- $k {
+      -select {
+        incr i
+        if {$i>=[llength $argv]-1} usage
+        lappend G(lSelect) [lindex $argv $i]
+      }
+      -verbose {
+        set G(verbose) 1
+      }
+      default {
+        usage
+      }
+    }
+  }
+}
+
+proc open_database {} {
+  global G
+  sqlite3 db ""
+
+  # Check if the "database" file is really an SQLite database. If so, copy
+  # it into the temp db just opened. Otherwise, assume that it is an SQL
+  # schema and execute it directly.
+  set fd [open $G(database)]
+  set hdr [read $fd 16]
+  if {$hdr == "SQLite format 3\000"} {
+    close $fd
+    sqlite3 db2 $G(database)
+    sqlite3_backup B db main db2 main
+    B step 2000000000
+    set rc [B finish]
+    db2 close
+    if {$rc != "SQLITE_OK"} { error "Failed to load database $G(database)" }
+  } else {
+    append hdr [read $fd]
+    db eval $hdr
+    close $fd
+  }
+}
+
+proc analyze_selects {} {
+  global G
+  set G(trace) ""
+
+  # Collect a line of xTrace output for each loop in the set of SELECT
+  # statements.
+  proc xTrace {zMsg} { lappend ::G(trace) $zMsg }
+  db trace "lappend ::G(trace)"
+  foreach s $G(lSelect) {
+    set stmt [sqlite3_prepare_v2 db $s -1 dummy]
+    set rc [sqlite3_finalize $stmt]
+    if {$rc!="SQLITE_OK"} {
+      error "Failed to compile SQL: [sqlite3_errmsg db]"
+    }
+  }
+
+  db trace ""
+  if {$G(verbose)} {
+    foreach t $G(trace) { puts "trace: $t" }
+  }
+
+  # puts $G(trace)
+}
+
+# The argument is a list of the form:
+#
+#    key1 {value1.1 value1.2} key2 {value2.1 value 2.2...}
+#
+# Values lists may be of any length greater than zero. This function returns
+# a list of lists created by pivoting on each values list. i.e. a list
+# consisting of the elements:
+#
+#   {{key1 value1.1} {key2 value2.1}}
+#   {{key1 value1.2} {key2 value2.1}}
+#   {{key1 value1.1} {key2 value2.2}}
+#   {{key1 value1.2} {key2 value2.2}}
+#
+proc expand_eq_list {L} {
+  set ll [list {}]
+  for {set i 0} {$i < [llength $L]} {incr i 2} {
+    set key [lindex $L $i]
+    set new [list]
+    foreach piv [lindex $L $i+1] {
+      foreach l $ll {
+        lappend new [concat $l [list [list $key $piv]]]
+      }
+    }
+    set ll $new
+  }
+
+  return $ll
+}
+
+proc eqset_to_index {tname eqset {range {}}} {
+  global G
+  set lCols [list]
+  set idxname $tname
+  foreach e [concat [lsort $eqset] [list $range]] {
+    if {[llength $e]==0} continue
+    foreach {c collate} $e {}
+    lappend lCols "$c collate $collate"
+    append idxname "_$c"
+    if {[string compare -nocase binary $collate]!=0} {
+      append idxname [string tolower $collate]
+    }
+  }
+
+  set create_index "CREATE INDEX $idxname ON ${tname}("
+  append create_index [join $lCols ", "]
+  append create_index ");"
+
+  set G(trial.$idxname) $create_index
+}
+
+proc expand_or_cons {L} {
+  set lRet [list [list]]
+  foreach elem $L {
+    set type [lindex $elem 0]
+    if {$type=="eq" || $type=="range"} {
+      set lNew [list]
+      for {set i 0} {$i < [llength $lRet]} {incr i} {
+        lappend lNew [concat [lindex $lRet $i] [list $elem]]
+      }
+      set lRet $lNew
+    } elseif {$type=="or"} {
+      set lNew [list]
+      foreach branch [lrange $elem 1 end] {
+        foreach b [expand_or_cons $branch] {
+          for {set i 0} {$i < [llength $lRet]} {incr i} {
+            lappend lNew [concat [lindex $lRet $i] $b]
+          }
+        }
+      }
+      set lRet $lNew
+    } 
+  }
+  return $lRet
+}
+
+proc find_trial_indexes {} {
+  global G
+  foreach t $G(trace) {
+    set tname [lindex $t 0]
+    catch { array unset mask }
+
+    foreach lCons [expand_or_cons [lrange $t 2 end]] {
+      set constraints [list]
+
+      foreach a $lCons {
+        set type [lindex $a 0]
+        if {$type=="eq" || $type=="range"} {
+          set m [lindex $a 3]
+          foreach k [array names mask] { set mask([expr ($k & $m)]) 1 }
+          set mask($m) 1
+          lappend constraints $a
+        }
+      }
+
+      foreach k [array names mask] {
+        catch {array unset eq}
+        foreach a $constraints {
+          foreach {type col collate m} $a {
+            if {($m & $k)==$m} {
+              if {$type=="eq"} {
+                lappend eq($col) $collate
+              } else {
+                set range($col.$collate) 1
+              }
+            }
+          }
+        }
+
+        #puts "mask=$k eq=[array get eq] range=[array get range]"
+        
+        set ranges [array names range]
+        foreach eqset [expand_eq_list [array get eq]] {
+          if {[llength $ranges]==0} {
+            eqset_to_index $tname $eqset
+          } else {
+            foreach r $ranges {
+              set bSeen 0
+              foreach {c collate} [split $r .] {}
+              foreach e $eqset {
+                if {[lindex $e 0] == $c} {
+                  set bSeen 1
+                  break
+                }
+              }
+              if {$bSeen} {
+                eqset_to_index $tname $eqset
+              } else {
+                eqset_to_index $tname $eqset [list $c $collate]
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+  if {$G(verbose)} {
+    foreach k [array names G trial.*] { puts "index: $G($k)" }
+  }
+}
+
+proc run_trials {} {
+  global G
+
+  foreach k [array names G trial.*] {
+    set idxname [lindex [split $k .] 1]
+    db eval $G($k)
+    set pgno [db one {SELECT rootpage FROM sqlite_master WHERE name = $idxname}]
+    set IDX($pgno) $idxname
+  }
+  db eval ANALYZE
+
+  catch { array unset used }
+  foreach s $G(lSelect) {
+    db eval "EXPLAIN $s" x {
+      if {($x(opcode)=="OpenRead" || $x(opcode)=="ReopenIdx")} {
+        if {[info exists IDX($x(p2))]} { set used($IDX($x(p2))) 1 }
+      }
+    }
+    foreach idx [array names used] {
+      puts $G(trial.$idx)
+    }
+  }
+}
+
+process_cmdline_args $argv
+open_database
+analyze_selects
+find_trial_indexes
+run_trials
+