From e7dd68c2d44350a48ca2f4010b81f2470a265933 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 11 Nov 2015 18:08:58 +0000
Subject: [PATCH 01/68] Add a hack to debug out a description of the WHERE
 clause of a SELECT (or other) statement. Use this in script
 tool/schemalint.tcl to automatically recommend indexes that might speed up
 specific queries.

FossilOrigin-Name: c6fa01c28ef7ceea2963a92dfffe62eed451b05c
---
 manifest             |  19 ++--
 manifest.uuid        |   2 +-
 src/where.c          | 152 +++++++++++++++++++++++++
 test/schemalint.test |  81 ++++++++++++++
 tool/schemalint.tcl  | 259 +++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 505 insertions(+), 8 deletions(-)
 create mode 100644 test/schemalint.test
 create mode 100644 tool/schemalint.tcl

diff --git a/manifest b/manifest
index 5e942ee5cf..ce318ef3fd 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improvements\sto\sthe\sSQLITE_CONFIG_PAGECACHE\sdocumentation.\s\sEnhance\sthe\ncommand-line\sshell\sto\sbe\sable\sto\stake\sadvantage\sof\sthe\sfull\srange\sof\nSQLITE_CONFIG_PAGECACHE\scapabilities,\ssuch\sas\ssetting\spMem==NULL\sand\sN<0.
-D 2015-11-11T15:28:52.898
+C Add\sa\shack\sto\sdebug\sout\sa\sdescription\sof\sthe\sWHERE\sclause\sof\sa\sSELECT\s(or\sother)\sstatement.\sUse\sthis\sin\sscript\stool/schemalint.tcl\sto\sautomatically\srecommend\sindexes\sthat\smight\sspeed\sup\sspecific\squeries.
+D 2015-11-11T18:08:58.267
 F Makefile.in d828db6afa6c1fa060d01e33e4674408df1942a1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e928e68168df69b353300ac87c10105206653a03
@@ -416,7 +416,7 @@ F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb
 F src/wal.c 18b0ed49830cf04fe2d68224b41838a73ac6cd24
 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4
 F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba
-F src/where.c 6aceb72cc58dc06922a9e1604d559c8ca4c3e728
+F src/where.c 6176426332d5a67df4222adfeae8c22a933f8c84
 F src/whereInt.h 7892bb54cf9ca0ae5c7e6094491b94c9286dc647
 F src/wherecode.c 4c96182e7b25e4be54008dee2da5b9c2f8480b9b
 F src/whereexpr.c e63244ca06c503e5f3c5b7f3c9aea0db826089ed
@@ -972,6 +972,7 @@ F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5
 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38
 F test/schema4.test e6a66e20cc69f0e306667c08be7fda3d11707dc5
 F test/schema5.test 29699b4421f183c8f0e88bd28ce7d75d13ea653e
+F test/schemalint.test 22f26e6e9d8fe437b6344a97c91fd14c95e813a7
 F test/securedel.test 21749c32ccc30f1ea9e4b9f33295a6521ec20fa0
 F test/securedel2.test 2d54c28e46eb1fd6902089958b20b1b056c6f1c5
 F test/select1.test be62204d2bd9a5a8a149e9974cfddce893d8f686
@@ -1378,6 +1379,7 @@ F tool/replace.tcl 7727c60a04299b65a92f5e1590896fea0f25b9e0
 F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a
 F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5
 F tool/run-speed-test.sh 0ae485af4fe9f826e2b494be8c81f8ca9e222a4a
+F tool/schemalint.tcl 2eb60e950e91061c27d771582fe87e9518d0988d
 F tool/showdb.c d4476e000a64eca9f5e2c2f68741e747b9778e8d
 F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818
 F tool/showlocks.c 9920bcc64f58378ff1118caead34147201f48c68
@@ -1402,7 +1404,10 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P e43e1171fd7837a08069dc25df4eac14db1c2afe
-R 9a1d72f8baac116c36385e92e48c0378
-U drh
-Z 9d52ec874b9a01b7b0eb1dab69dbb993
+P 2518d5c971c4b32d9227b3bb7259162e3e27b00b
+R 9a794d90ebfe5dc636fc86a7e99d61d5
+T *branch * schemalint
+T *sym-schemalint *
+T -sym-trunk *
+U dan
+Z 95ad0fe90f5fe7059fb61a18885ffca1
diff --git a/manifest.uuid b/manifest.uuid
index bf01d0ad61..079b2e76d7 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2518d5c971c4b32d9227b3bb7259162e3e27b00b
\ No newline at end of file
+c6fa01c28ef7ceea2963a92dfffe62eed451b05c
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index 1c87706ea2..becfb5e8ce 100644
--- a/src/where.c
+++ b/src/where.c
@@ -3901,6 +3901,155 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){
   return 0;
 }
 
+#ifdef SQLITE_SCHEMA_LINT
+static char *whereAppendPrintf(sqlite3 *db, const char *zFmt, ...){
+  va_list ap;
+  char *zRes = 0;
+  va_start(ap, zFmt);
+  zRes = sqlite3_vmprintf(zFmt, ap);
+  if( zRes==0 ){
+    db->mallocFailed = 1;
+  }else if( db->mallocFailed ){
+    sqlite3_free(zRes);
+    zRes = 0;
+  }
+  va_end(ap);
+  return zRes;
+}
+
+/*
+** Append a representation of term pTerm to the string in zIn and return
+** the result. Or, if an OOM occurs, free zIn and return a NULL pointer.
+*/
+static char *whereAppendSingleTerm(
+  Parse *pParse,
+  Table *pTab,
+  int bOr,
+  char *zIn,
+  WhereTerm *pTerm
+){
+  char *zBuf;
+  sqlite3 *db = pParse->db;
+  Expr *pX = pTerm->pExpr;
+  CollSeq *pColl;
+  const char *zOp = 0;
+
+  if( pTerm->eOperator & (WO_IS|WO_EQ|WO_IN) ){
+    zOp = "eq";
+  }else if( pTerm->eOperator & (WO_LT|WO_LE|WO_GE|WO_GT) ){
+    zOp = "range";
+  }
+  pColl = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight);
+
+  if( zOp ){
+    const char *zFmt = bOr ? "%z{{%s %s %s %lld}}" : "%z{%s %s %s %lld}";
+    zBuf = whereAppendPrintf(db, zFmt, zIn, 
+        zOp, pTab->aCol[pTerm->u.leftColumn].zName, 
+        (pColl ? pColl->zName : "BINARY"),
+        pTerm->prereqRight
+    );
+  }else{
+    zBuf = zIn;
+  }
+
+  return zBuf;
+}
+
+static char *whereTraceWC(
+  Parse *pParse, 
+  struct SrcList_item *pItem,
+  char *zIn,
+  WhereClause *pWC
+){
+  sqlite3 *db = pParse->db;
+  Table *pTab = pItem->pTab;
+  char *zBuf = zIn;
+  int iCol;
+  int ii;
+  int bFirst = 1;
+
+  /* List of WO_SINGLE constraints */
+  for(iCol=0; iCol<pTab->nCol; iCol++){
+    int opMask = WO_SINGLE; 
+    WhereScan scan;
+    WhereTerm *pTerm;
+    for(pTerm=whereScanInit(&scan, pWC, pItem->iCursor, iCol, opMask, 0);
+        pTerm;
+        pTerm=whereScanNext(&scan)
+    ){
+      assert( iCol==pTerm->u.leftColumn );
+      if( bFirst==0 ) zBuf = whereAppendPrintf(db, "%z ", zBuf);
+      zBuf = whereAppendSingleTerm(pParse, pTab, pWC->op==TK_OR, zBuf, pTerm);
+      bFirst = 0;
+    }
+  }
+
+  /* Add composite - (WO_OR|WO_AND) - constraints */
+  for(ii=0; ii<pWC->nTerm; ii++){
+    WhereTerm *pTerm = &pWC->a[ii];
+    if( pTerm->eOperator & (WO_OR|WO_AND) ){
+      const char *zFmt = ((pTerm->eOperator&WO_OR) ? "%z%s{or " : "%z%s{");
+      zBuf = whereAppendPrintf(db, zFmt, zBuf, bFirst ? "" : " ");
+      zBuf = whereTraceWC(pParse, pItem, zBuf, &pTerm->u.pOrInfo->wc);
+      zBuf = whereAppendPrintf(db, "%z}", zBuf);
+      bFirst = 0;
+    }
+  }
+
+  return zBuf;
+}
+
+static void whereTraceBuilder(
+  Parse *pParse,
+  WhereLoopBuilder *p
+){
+  sqlite3 *db = pParse->db;
+  if( db->xTrace ){
+    WhereInfo *pWInfo = p->pWInfo;
+    int nTablist = pWInfo->pTabList->nSrc;
+    int ii;
+
+    /* Loop through each element of the FROM clause. Ignore any sub-selects
+    ** or views. Invoke the xTrace() callback once for each real table. */
+    for(ii=0; ii<nTablist; ii++){
+      char *zBuf = 0;
+      int iCol;
+      int nCol;
+      Table *pTab;
+
+      struct SrcList_item *pItem = &pWInfo->pTabList->a[ii];
+      if( pItem->pSelect ) continue;
+      pTab = pItem->pTab;
+      nCol = pTab->nCol;
+
+      /* Append the table name to the buffer. */
+      zBuf = whereAppendPrintf(db, "%s", pTab->zName);
+
+      /* Append the list of columns required to create a covering index */
+      zBuf = whereAppendPrintf(db, "%z {cols", zBuf);
+      if( 0==(pItem->colUsed & ((u64)1 << (sizeof(Bitmask)*8-1))) ){
+        for(iCol=0; iCol<nCol; iCol++){
+          if( iCol==(sizeof(Bitmask)*8-1) ) break;
+          if( pItem->colUsed & ((u64)1 << iCol) ){
+            zBuf = whereAppendPrintf(db, "%z %s", zBuf, pTab->aCol[iCol].zName);
+          }
+        }
+      }
+      zBuf = whereAppendPrintf(db, "%z} ", zBuf);
+
+      /* Append the contents of WHERE clause */
+      zBuf = whereTraceWC(pParse, pItem, zBuf, p->pWC);
+
+      /* Pass the buffer to the xTrace() callback, then free it */
+      db->xTrace(db->pTraceArg, zBuf);
+      sqlite3DbFree(db, zBuf);
+    }
+  }
+}
+#else
+# define whereTraceBuilder(x,y)
+#endif
+
 /*
 ** Generate the beginning of the loop used for WHERE clause processing.
 ** The return value is a pointer to an opaque structure that contains
@@ -4161,6 +4310,9 @@ WhereInfo *sqlite3WhereBegin(
   }
 #endif
 
+  /* Schema-lint xTrace callback */
+  whereTraceBuilder(pParse, &sWLB);
+
   if( nTabList!=1 || whereShortCut(&sWLB)==0 ){
     rc = whereLoopAddAll(&sWLB);
     if( rc ) goto whereBeginError;
diff --git a/test/schemalint.test b/test/schemalint.test
new file mode 100644
index 0000000000..f367aba29c
--- /dev/null
+++ b/test/schemalint.test
@@ -0,0 +1,81 @@
+
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix schemalint
+
+proc xTrace {zMsg} {
+  lappend ::trace_out $zMsg
+}
+db trace xTrace
+
+proc do_trace_test {tn sql res} {
+  uplevel [list do_test $tn [subst -nocommands {
+    set ::trace_out [list]
+    set stmt [sqlite3_prepare db "$sql" -1 x]
+    sqlite3_finalize [set stmt]
+    set ::trace_out
+  }] [list {*}$res]]
+}
+
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, b, c);
+  CREATE TABLE t2(x, y, z);
+}
+
+do_trace_test 1.1 {
+  SELECT b, c, y, z FROM t1, t2 WHERE c=? AND z=?
+} {
+  {t1 {cols b c} {eq c BINARY 0}}
+  {t2 {cols y z} {eq z BINARY 0}}
+}
+
+do_trace_test 1.2 {
+  SELECT a FROM t1 WHERE b>10
+} {
+  {t1 {cols a b} {range b BINARY 0}}
+}
+
+do_trace_test 1.3 {
+  SELECT b FROM t1 WHERE b IN (10, 20, 30)
+} {
+  {t1 {cols b} {eq b BINARY 0}}
+}
+
+do_trace_test 1.4 {
+  SELECT * FROM t1, t2 WHERE x=a
+} {
+  {t1 {cols a b c} {eq a BINARY 2}} 
+  {t2 {cols x y z} {eq x BINARY 1}}
+}
+
+do_trace_test 1.5 {
+  SELECT * FROM t1 WHERE a IN (1, 2, 3)
+} {
+  {t1 {cols a b c} {eq a BINARY 0}}
+}
+
+#-----------------------------------------------------------------------
+# Cases involving OR clauses in the WHERE clause.
+#
+do_trace_test 2.1 {
+  SELECT * FROM t1 WHERE a=? OR b=?
+} {
+  {t1 {cols a b c} {or {{eq a BINARY 0}} {{eq b BINARY 0}}}}
+}
+
+do_trace_test 2.2 {
+  SELECT * FROM t1 WHERE a=? OR (b=? AND c=?)
+} {
+  {t1 {cols a b c} {or {{eq a BINARY 0}} {{eq b BINARY 0} {eq c BINARY 0}}}}
+}
+
+do_trace_test 2.3 {
+  SELECT * FROM t1 WHERE (a=? AND b=?) OR c=?
+} {
+  {t1 {cols a b c} {or {{eq c BINARY 0}} {{eq a BINARY 0} {eq b BINARY 0}}}}
+}
+
+finish_test
+
diff --git a/tool/schemalint.tcl b/tool/schemalint.tcl
new file mode 100644
index 0000000000..41f9ff70eb
--- /dev/null
+++ b/tool/schemalint.tcl
@@ -0,0 +1,259 @@
+
+
+
+set ::G(lSelect)  [list]           ;# List of SELECT statements to analyze
+set ::G(database) ""               ;# Name of database or SQL schema file
+set ::G(trace)    [list]           ;# List of data from xTrace()
+set ::G(verbose)  0                ;# True if -verbose option was passed 
+
+proc usage {} {
+  puts stderr "Usage: $::argv0 ?SWITCHES? DATABASE/SCHEMA"
+  puts stderr "  Switches are:"
+  puts stderr "  -select SQL     (recommend indexes for SQL statement)"
+  puts stderr "  -verbose        (increase verbosity of output)"
+  puts stderr ""
+  exit
+}
+
+proc process_cmdline_args {argv} {
+  global G
+  set nArg [llength $argv]
+  set G(database) [lindex $argv end]
+
+  for {set i 0} {$i < [llength $argv]-1} {incr i} {
+    set k [lindex $argv $i]
+    switch -- $k {
+      -select {
+        incr i
+        if {$i>=[llength $argv]-1} usage
+        lappend G(lSelect) [lindex $argv $i]
+      }
+      -verbose {
+        set G(verbose) 1
+      }
+      default {
+        usage
+      }
+    }
+  }
+}
+
+proc open_database {} {
+  global G
+  sqlite3 db ""
+
+  # Check if the "database" file is really an SQLite database. If so, copy
+  # it into the temp db just opened. Otherwise, assume that it is an SQL
+  # schema and execute it directly.
+  set fd [open $G(database)]
+  set hdr [read $fd 16]
+  if {$hdr == "SQLite format 3\000"} {
+    close $fd
+    sqlite3 db2 $G(database)
+    sqlite3_backup B db main db2 main
+    B step 2000000000
+    set rc [B finish]
+    db2 close
+    if {$rc != "SQLITE_OK"} { error "Failed to load database $G(database)" }
+  } else {
+    append hdr [read $fd]
+    db eval $hdr
+    close $fd
+  }
+}
+
+proc analyze_selects {} {
+  global G
+  set G(trace) ""
+
+  # Collect a line of xTrace output for each loop in the set of SELECT
+  # statements.
+  proc xTrace {zMsg} { lappend ::G(trace) $zMsg }
+  db trace "lappend ::G(trace)"
+  foreach s $G(lSelect) {
+    set stmt [sqlite3_prepare_v2 db $s -1 dummy]
+    set rc [sqlite3_finalize $stmt]
+    if {$rc!="SQLITE_OK"} {
+      error "Failed to compile SQL: [sqlite3_errmsg db]"
+    }
+  }
+
+  db trace ""
+  if {$G(verbose)} {
+    foreach t $G(trace) { puts "trace: $t" }
+  }
+
+  # puts $G(trace)
+}
+
+# The argument is a list of the form:
+#
+#    key1 {value1.1 value1.2} key2 {value2.1 value 2.2...}
+#
+# Values lists may be of any length greater than zero. This function returns
+# a list of lists created by pivoting on each values list. i.e. a list
+# consisting of the elements:
+#
+#   {{key1 value1.1} {key2 value2.1}}
+#   {{key1 value1.2} {key2 value2.1}}
+#   {{key1 value1.1} {key2 value2.2}}
+#   {{key1 value1.2} {key2 value2.2}}
+#
+proc expand_eq_list {L} {
+  set ll [list {}]
+  for {set i 0} {$i < [llength $L]} {incr i 2} {
+    set key [lindex $L $i]
+    set new [list]
+    foreach piv [lindex $L $i+1] {
+      foreach l $ll {
+        lappend new [concat $l [list [list $key $piv]]]
+      }
+    }
+    set ll $new
+  }
+
+  return $ll
+}
+
+proc eqset_to_index {tname eqset {range {}}} {
+  global G
+  set lCols [list]
+  set idxname $tname
+  foreach e [concat [lsort $eqset] [list $range]] {
+    if {[llength $e]==0} continue
+    foreach {c collate} $e {}
+    lappend lCols "$c collate $collate"
+    append idxname "_$c"
+    if {[string compare -nocase binary $collate]!=0} {
+      append idxname [string tolower $collate]
+    }
+  }
+
+  set create_index "CREATE INDEX $idxname ON ${tname}("
+  append create_index [join $lCols ", "]
+  append create_index ");"
+
+  set G(trial.$idxname) $create_index
+}
+
+proc expand_or_cons {L} {
+  set lRet [list [list]]
+  foreach elem $L {
+    set type [lindex $elem 0]
+    if {$type=="eq" || $type=="range"} {
+      set lNew [list]
+      for {set i 0} {$i < [llength $lRet]} {incr i} {
+        lappend lNew [concat [lindex $lRet $i] [list $elem]]
+      }
+      set lRet $lNew
+    } elseif {$type=="or"} {
+      set lNew [list]
+      foreach branch [lrange $elem 1 end] {
+        foreach b [expand_or_cons $branch] {
+          for {set i 0} {$i < [llength $lRet]} {incr i} {
+            lappend lNew [concat [lindex $lRet $i] $b]
+          }
+        }
+      }
+      set lRet $lNew
+    } 
+  }
+  return $lRet
+}
+
+proc find_trial_indexes {} {
+  global G
+  foreach t $G(trace) {
+    set tname [lindex $t 0]
+    catch { array unset mask }
+
+    foreach lCons [expand_or_cons [lrange $t 2 end]] {
+      set constraints [list]
+
+      foreach a $lCons {
+        set type [lindex $a 0]
+        if {$type=="eq" || $type=="range"} {
+          set m [lindex $a 3]
+          foreach k [array names mask] { set mask([expr ($k & $m)]) 1 }
+          set mask($m) 1
+          lappend constraints $a
+        }
+      }
+
+      foreach k [array names mask] {
+        catch {array unset eq}
+        foreach a $constraints {
+          foreach {type col collate m} $a {
+            if {($m & $k)==$m} {
+              if {$type=="eq"} {
+                lappend eq($col) $collate
+              } else {
+                set range($col.$collate) 1
+              }
+            }
+          }
+        }
+
+        #puts "mask=$k eq=[array get eq] range=[array get range]"
+        
+        set ranges [array names range]
+        foreach eqset [expand_eq_list [array get eq]] {
+          if {[llength $ranges]==0} {
+            eqset_to_index $tname $eqset
+          } else {
+            foreach r $ranges {
+              set bSeen 0
+              foreach {c collate} [split $r .] {}
+              foreach e $eqset {
+                if {[lindex $e 0] == $c} {
+                  set bSeen 1
+                  break
+                }
+              }
+              if {$bSeen} {
+                eqset_to_index $tname $eqset
+              } else {
+                eqset_to_index $tname $eqset [list $c $collate]
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+  if {$G(verbose)} {
+    foreach k [array names G trial.*] { puts "index: $G($k)" }
+  }
+}
+
+proc run_trials {} {
+  global G
+
+  foreach k [array names G trial.*] {
+    set idxname [lindex [split $k .] 1]
+    db eval $G($k)
+    set pgno [db one {SELECT rootpage FROM sqlite_master WHERE name = $idxname}]
+    set IDX($pgno) $idxname
+  }
+  db eval ANALYZE
+
+  catch { array unset used }
+  foreach s $G(lSelect) {
+    db eval "EXPLAIN $s" x {
+      if {($x(opcode)=="OpenRead" || $x(opcode)=="ReopenIdx")} {
+        if {[info exists IDX($x(p2))]} { set used($IDX($x(p2))) 1 }
+      }
+    }
+    foreach idx [array names used] {
+      puts $G(trial.$idx)
+    }
+  }
+}
+
+process_cmdline_args $argv
+open_database
+analyze_selects
+find_trial_indexes
+run_trials
+

From ef5de04b4edaa859fe7d17886448d5dc3e2cefd1 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 20 Nov 2015 20:55:27 +0000
Subject: [PATCH 02/68] Add support for ORDER BY clauses to schemalint.tcl.

FossilOrigin-Name: 93bdf70e859915ff3696ba0fc68f91ceb2e1a971
---
 manifest             | 19 ++++++++-----------
 manifest.uuid        |  2 +-
 src/where.c          | 33 +++++++++++++++++++++++++++++----
 test/schemalint.test | 19 +++++++++++++++++++
 tool/schemalint.tcl  |  7 +++++++
 5 files changed, 64 insertions(+), 16 deletions(-)

diff --git a/manifest b/manifest
index ce318ef3fd..e9c4624905 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sa\shack\sto\sdebug\sout\sa\sdescription\sof\sthe\sWHERE\sclause\sof\sa\sSELECT\s(or\sother)\sstatement.\sUse\sthis\sin\sscript\stool/schemalint.tcl\sto\sautomatically\srecommend\sindexes\sthat\smight\sspeed\sup\sspecific\squeries.
-D 2015-11-11T18:08:58.267
+C Add\ssupport\sfor\sORDER\sBY\sclauses\sto\sschemalint.tcl.
+D 2015-11-20T20:55:27.428
 F Makefile.in d828db6afa6c1fa060d01e33e4674408df1942a1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e928e68168df69b353300ac87c10105206653a03
@@ -416,7 +416,7 @@ F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb
 F src/wal.c 18b0ed49830cf04fe2d68224b41838a73ac6cd24
 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4
 F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba
-F src/where.c 6176426332d5a67df4222adfeae8c22a933f8c84
+F src/where.c 5804902239731a7025240f4338d98cdc57315636
 F src/whereInt.h 7892bb54cf9ca0ae5c7e6094491b94c9286dc647
 F src/wherecode.c 4c96182e7b25e4be54008dee2da5b9c2f8480b9b
 F src/whereexpr.c e63244ca06c503e5f3c5b7f3c9aea0db826089ed
@@ -972,7 +972,7 @@ F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5
 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38
 F test/schema4.test e6a66e20cc69f0e306667c08be7fda3d11707dc5
 F test/schema5.test 29699b4421f183c8f0e88bd28ce7d75d13ea653e
-F test/schemalint.test 22f26e6e9d8fe437b6344a97c91fd14c95e813a7
+F test/schemalint.test 7fba0e262353c8c3dd5ee406cb11cf90f8b566fe
 F test/securedel.test 21749c32ccc30f1ea9e4b9f33295a6521ec20fa0
 F test/securedel2.test 2d54c28e46eb1fd6902089958b20b1b056c6f1c5
 F test/select1.test be62204d2bd9a5a8a149e9974cfddce893d8f686
@@ -1379,7 +1379,7 @@ F tool/replace.tcl 7727c60a04299b65a92f5e1590896fea0f25b9e0
 F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a
 F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5
 F tool/run-speed-test.sh 0ae485af4fe9f826e2b494be8c81f8ca9e222a4a
-F tool/schemalint.tcl 2eb60e950e91061c27d771582fe87e9518d0988d
+F tool/schemalint.tcl 160fe2ed74df02e859ab32a522fa1f2e8e0c4230
 F tool/showdb.c d4476e000a64eca9f5e2c2f68741e747b9778e8d
 F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818
 F tool/showlocks.c 9920bcc64f58378ff1118caead34147201f48c68
@@ -1404,10 +1404,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 2518d5c971c4b32d9227b3bb7259162e3e27b00b
-R 9a794d90ebfe5dc636fc86a7e99d61d5
-T *branch * schemalint
-T *sym-schemalint *
-T -sym-trunk *
+P c6fa01c28ef7ceea2963a92dfffe62eed451b05c
+R 6fc6a71e3dbe84052a5d99204cd615ee
 U dan
-Z 95ad0fe90f5fe7059fb61a18885ffca1
+Z 64248603e4a564821f72e660b22a8811
diff --git a/manifest.uuid b/manifest.uuid
index 079b2e76d7..a959ffa2b1 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c6fa01c28ef7ceea2963a92dfffe62eed451b05c
\ No newline at end of file
+93bdf70e859915ff3696ba0fc68f91ceb2e1a971
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index becfb5e8ce..4ba35d6ac9 100644
--- a/src/where.c
+++ b/src/where.c
@@ -3957,6 +3957,7 @@ static char *whereAppendSingleTerm(
 
 static char *whereTraceWC(
   Parse *pParse, 
+  int bInitialSpace,
   struct SrcList_item *pItem,
   char *zIn,
   WhereClause *pWC
@@ -3966,7 +3967,7 @@ static char *whereTraceWC(
   char *zBuf = zIn;
   int iCol;
   int ii;
-  int bFirst = 1;
+  int bFirst = !bInitialSpace;
 
   /* List of WO_SINGLE constraints */
   for(iCol=0; iCol<pTab->nCol; iCol++){
@@ -3990,7 +3991,7 @@ static char *whereTraceWC(
     if( pTerm->eOperator & (WO_OR|WO_AND) ){
       const char *zFmt = ((pTerm->eOperator&WO_OR) ? "%z%s{or " : "%z%s{");
       zBuf = whereAppendPrintf(db, zFmt, zBuf, bFirst ? "" : " ");
-      zBuf = whereTraceWC(pParse, pItem, zBuf, &pTerm->u.pOrInfo->wc);
+      zBuf = whereTraceWC(pParse, 0, pItem, zBuf, &pTerm->u.pOrInfo->wc);
       zBuf = whereAppendPrintf(db, "%z}", zBuf);
       bFirst = 0;
     }
@@ -4005,6 +4006,7 @@ static void whereTraceBuilder(
 ){
   sqlite3 *db = pParse->db;
   if( db->xTrace ){
+    ExprList *pOrderBy = p->pOrderBy;
     WhereInfo *pWInfo = p->pWInfo;
     int nTablist = pWInfo->pTabList->nSrc;
     int ii;
@@ -4035,10 +4037,33 @@ static void whereTraceBuilder(
           }
         }
       }
-      zBuf = whereAppendPrintf(db, "%z} ", zBuf);
+      zBuf = whereAppendPrintf(db, "%z}",zBuf);
 
       /* Append the contents of WHERE clause */
-      zBuf = whereTraceWC(pParse, pItem, zBuf, p->pWC);
+      zBuf = whereTraceWC(pParse, 1, pItem, zBuf, p->pWC);
+
+      /* Append the ORDER BY clause, if any */
+      if( pOrderBy ){
+        int i;
+        int bFirst = 1;
+        for(i=0; i<pOrderBy->nExpr; i++){
+          Expr *pExpr = pOrderBy->a[i].pExpr; 
+          CollSeq *pColl = sqlite3ExprCollSeq(pParse, pExpr);
+
+          pExpr = sqlite3ExprSkipCollate(pExpr);
+          if( pExpr->op==TK_COLUMN && pExpr->iTable==pItem->iCursor ){
+            if( pExpr->iColumn>=0 ){
+              const char *zName = pTab->aCol[pExpr->iColumn].zName;
+              zBuf = whereAppendPrintf(db, "%z%s%s %s %s", zBuf,
+                  bFirst ? " {orderby " : " ", zName, pColl->zName,
+                  (pOrderBy->a[i].sortOrder ? "DESC" : "ASC")
+              );
+              bFirst = 0;
+            }
+          }
+        }
+        if( bFirst==0 ) zBuf = whereAppendPrintf(db, "%z}", zBuf);
+      }
 
       /* Pass the buffer to the xTrace() callback, then free it */
       db->xTrace(db->pTraceArg, zBuf);
diff --git a/test/schemalint.test b/test/schemalint.test
index f367aba29c..d354c1fdd7 100644
--- a/test/schemalint.test
+++ b/test/schemalint.test
@@ -77,5 +77,24 @@ do_trace_test 2.3 {
   {t1 {cols a b c} {or {{eq c BINARY 0}} {{eq a BINARY 0} {eq b BINARY 0}}}}
 }
 
+#-----------------------------------------------------------------------
+# Cases involving ORDER BY.
+#
+do_trace_test 3.1 {
+  SELECT * FROM t1 ORDER BY a;
+} {{t1 {cols a b c} {orderby a BINARY ASC}}}
+
+do_trace_test 3.2 {
+  SELECT * FROM t1 WHERE a=? ORDER BY b;
+} {{t1 {cols a b c} {eq a BINARY 0} {orderby b BINARY ASC}}}
+
+do_trace_test 3.3 {
+  SELECT min(a) FROM t1;
+} {{t1 {cols a} {orderby a BINARY ASC}}}
+
+do_trace_test 3.4 {
+  SELECT max(a) FROM t1;
+} {{t1 {cols a} {orderby a BINARY DESC}}}
+
 finish_test
 
diff --git a/tool/schemalint.tcl b/tool/schemalint.tcl
index 41f9ff70eb..94e1ec02ec 100644
--- a/tool/schemalint.tcl
+++ b/tool/schemalint.tcl
@@ -115,6 +115,9 @@ proc expand_eq_list {L} {
   return $ll
 }
 
+#--------------------------------------------------------------------------
+# Formulate a CREATE INDEX statement that creates an index on table $tname.
+#
 proc eqset_to_index {tname eqset {range {}}} {
   global G
   set lCols [list]
@@ -167,6 +170,10 @@ proc find_trial_indexes {} {
     set tname [lindex $t 0]
     catch { array unset mask }
 
+    if {[lindex $t end 0]=="orderby"} {
+      set orderby [lrange [lindex $t end] 1 end]
+    }
+
     foreach lCons [expand_or_cons [lrange $t 2 end]] {
       set constraints [list]
 

From 1b8b839a8db3692a559c1ca3f90fb18bb3e82624 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 23 Nov 2015 17:10:51 +0000
Subject: [PATCH 03/68] Fix ORDER BY handling in the schemalint.tcl script. Add
 internal self-tests to the same script.

FossilOrigin-Name: b8f277c9b45c4b30e6690e838a30311aa8d84876
---
 manifest            |  12 +-
 manifest.uuid       |   2 +-
 tool/schemalint.tcl | 265 +++++++++++++++++++++++++++++++++++---------
 3 files changed, 220 insertions(+), 59 deletions(-)

diff --git a/manifest b/manifest
index e9c4624905..fb61058c44 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\ssupport\sfor\sORDER\sBY\sclauses\sto\sschemalint.tcl.
-D 2015-11-20T20:55:27.428
+C Fix\sORDER\sBY\shandling\sin\sthe\sschemalint.tcl\sscript.\sAdd\sinternal\sself-tests\sto\sthe\ssame\sscript.
+D 2015-11-23T17:10:51.938
 F Makefile.in d828db6afa6c1fa060d01e33e4674408df1942a1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e928e68168df69b353300ac87c10105206653a03
@@ -1379,7 +1379,7 @@ F tool/replace.tcl 7727c60a04299b65a92f5e1590896fea0f25b9e0
 F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a
 F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5
 F tool/run-speed-test.sh 0ae485af4fe9f826e2b494be8c81f8ca9e222a4a
-F tool/schemalint.tcl 160fe2ed74df02e859ab32a522fa1f2e8e0c4230
+F tool/schemalint.tcl e49c9b25c8c43f0531eca96edaf85ce8bd7f308b
 F tool/showdb.c d4476e000a64eca9f5e2c2f68741e747b9778e8d
 F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818
 F tool/showlocks.c 9920bcc64f58378ff1118caead34147201f48c68
@@ -1404,7 +1404,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P c6fa01c28ef7ceea2963a92dfffe62eed451b05c
-R 6fc6a71e3dbe84052a5d99204cd615ee
+P 93bdf70e859915ff3696ba0fc68f91ceb2e1a971
+R 4554e59743437c1fee4689a6514241d8
 U dan
-Z 64248603e4a564821f72e660b22a8811
+Z f43cf07c3dedebc0e4a9e76dc40982d5
diff --git a/manifest.uuid b/manifest.uuid
index a959ffa2b1..7790e3b6f1 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-93bdf70e859915ff3696ba0fc68f91ceb2e1a971
\ No newline at end of file
+b8f277c9b45c4b30e6690e838a30311aa8d84876
\ No newline at end of file
diff --git a/tool/schemalint.tcl b/tool/schemalint.tcl
index 94e1ec02ec..933b277ea2 100644
--- a/tool/schemalint.tcl
+++ b/tool/schemalint.tcl
@@ -1,22 +1,18 @@
 
-
-
-set ::G(lSelect)  [list]           ;# List of SELECT statements to analyze
-set ::G(database) ""               ;# Name of database or SQL schema file
-set ::G(trace)    [list]           ;# List of data from xTrace()
-set ::G(verbose)  0                ;# True if -verbose option was passed 
+set ::VERBOSE 0
 
 proc usage {} {
   puts stderr "Usage: $::argv0 ?SWITCHES? DATABASE/SCHEMA"
   puts stderr "  Switches are:"
   puts stderr "  -select SQL     (recommend indexes for SQL statement)"
   puts stderr "  -verbose        (increase verbosity of output)"
+  puts stderr "  -test           (run internal tests and then exit)"
   puts stderr ""
   exit
 }
 
-proc process_cmdline_args {argv} {
-  global G
+proc process_cmdline_args {ctxvar argv} {
+  upvar $ctxvar G
   set nArg [llength $argv]
   set G(database) [lindex $argv end]
 
@@ -29,17 +25,24 @@ proc process_cmdline_args {argv} {
         lappend G(lSelect) [lindex $argv $i]
       }
       -verbose {
-        set G(verbose) 1
+        set ::VERBOSE 1
+      }
+      -test {
+        sqlidx_internal_tests
       }
       default {
         usage
       }
     }
   }
+
+  if {$G(database)=="-test"} {
+    sqlidx_internal_tests
+  }
 }
 
-proc open_database {} {
-  global G
+proc open_database {ctxvar} {
+  upvar $ctxvar G
   sqlite3 db ""
 
   # Check if the "database" file is really an SQLite database. If so, copy
@@ -62,14 +65,17 @@ proc open_database {} {
   }
 }
 
-proc analyze_selects {} {
-  global G
+proc analyze_selects {ctxvar} {
+  upvar $ctxvar G
   set G(trace) ""
 
   # Collect a line of xTrace output for each loop in the set of SELECT
   # statements.
-  proc xTrace {zMsg} { lappend ::G(trace) $zMsg }
-  db trace "lappend ::G(trace)"
+  proc xTrace {zMsg} { 
+    upvar G G
+    lappend G(trace) $zMsg 
+  }
+  db trace xTrace
   foreach s $G(lSelect) {
     set stmt [sqlite3_prepare_v2 db $s -1 dummy]
     set rc [sqlite3_finalize $stmt]
@@ -79,7 +85,7 @@ proc analyze_selects {} {
   }
 
   db trace ""
-  if {$G(verbose)} {
+  if {$::VERBOSE} {
     foreach t $G(trace) { puts "trace: $t" }
   }
 
@@ -118,11 +124,12 @@ proc expand_eq_list {L} {
 #--------------------------------------------------------------------------
 # Formulate a CREATE INDEX statement that creates an index on table $tname.
 #
-proc eqset_to_index {tname eqset {range {}}} {
-  global G
+proc eqset_to_index {ctxvar tname eqset {range {}}} {
+  upvar $ctxvar G
+
   set lCols [list]
   set idxname $tname
-  foreach e [concat [lsort $eqset] [list $range]] {
+  foreach e [lsort $eqset] { 
     if {[llength $e]==0} continue
     foreach {c collate} $e {}
     lappend lCols "$c collate $collate"
@@ -132,6 +139,19 @@ proc eqset_to_index {tname eqset {range {}}} {
     }
   }
 
+  foreach {c collate dir} $range {
+    append idxname "_$c"
+    if {[string compare -nocase binary $collate]!=0} {
+      append idxname [string tolower $collate]
+    }
+    if {$dir=="DESC"} {
+      lappend lCols "$c collate $collate DESC"
+      append idxname "desc"
+    } else {
+      lappend lCols "$c collate $collate"
+    }
+  }
+
   set create_index "CREATE INDEX $idxname ON ${tname}("
   append create_index [join $lCols ", "]
   append create_index ");"
@@ -164,64 +184,97 @@ proc expand_or_cons {L} {
   return $lRet
 }
 
-proc find_trial_indexes {} {
-  global G
+proc find_trial_indexes {ctxvar} {
+  upvar $ctxvar G
   foreach t $G(trace) {
     set tname [lindex $t 0]
     catch { array unset mask }
 
+    set orderby [list]
     if {[lindex $t end 0]=="orderby"} {
       set orderby [lrange [lindex $t end] 1 end]
     }
 
     foreach lCons [expand_or_cons [lrange $t 2 end]] {
-      set constraints [list]
 
+      # Populate the array mask() so that it contains an entry for each
+      # combination of prerequisite scans that may lead to distinct sets
+      # of constraints being usable.
+      #
+      catch { array unset mask }
+      set mask(0) 1
       foreach a $lCons {
         set type [lindex $a 0]
         if {$type=="eq" || $type=="range"} {
           set m [lindex $a 3]
           foreach k [array names mask] { set mask([expr ($k & $m)]) 1 }
           set mask($m) 1
-          lappend constraints $a
         }
       }
 
+      # Loop once for each distinct prerequisite scan mask identified in
+      # the previous block.
+      #
       foreach k [array names mask] {
+
+        # Identify the constraints available for prerequisite mask $k. For
+        # each == constraint, set an entry in the eq() array as follows:
+        # 
+        #   set eq(<col>) <collation>
+        #
+        # If there is more than one == constraint for a column, and they use
+        # different collation sequences, <collation> is replaced with a list
+        # of the possible collation sequences. For example, for:
+        #
+        #   SELECT * FROM t1 WHERE a=? COLLATE BINARY AND a=? COLLATE NOCASE
+        #
+        # Set the following entry in the eq() array:
+        #
+        #   set eq(a) {binary nocase}
+        #
+        # For each range constraint found an entry is appended to the $ranges
+        # list. The entry is itself a list of the form {<col> <collation>}.
+        #
         catch {array unset eq}
-        foreach a $constraints {
-          foreach {type col collate m} $a {
-            if {($m & $k)==$m} {
-              if {$type=="eq"} {
-                lappend eq($col) $collate
-              } else {
-                set range($col.$collate) 1
+        set ranges [list]
+        foreach a $lCons {
+          set type [lindex $a 0]
+          if {$type=="eq" || $type=="range"} {
+            foreach {type col collate m} $a {
+              if {($m & $k)==$m} {
+                if {$type=="eq"} {
+                  lappend eq($col) $collate
+                } else {
+                  lappend ranges [list $col $collate ASC]
+                }
               }
             }
           }
         }
+        set ranges [lsort -unique $ranges]
+        if {$orderby != ""} {
+          lappend ranges $orderby
+        }
 
-        #puts "mask=$k eq=[array get eq] range=[array get range]"
-        
-        set ranges [array names range]
         foreach eqset [expand_eq_list [array get eq]] {
-          if {[llength $ranges]==0} {
-            eqset_to_index $tname $eqset
-          } else {
-            foreach r $ranges {
+          if {$eqset != ""} {
+            eqset_to_index G $tname $eqset
+          }
+
+          foreach r $ranges {
+            set tail [list]
+            foreach {c collate dir} $r {
               set bSeen 0
-              foreach {c collate} [split $r .] {}
               foreach e $eqset {
                 if {[lindex $e 0] == $c} {
                   set bSeen 1
                   break
                 }
               }
-              if {$bSeen} {
-                eqset_to_index $tname $eqset
-              } else {
-                eqset_to_index $tname $eqset [list $c $collate]
-              }
+              if {$bSeen==0} { lappend tail {*}$r }
+            }
+            if {[llength $tail]} {
+              eqset_to_index G $tname $eqset $r
             }
           }
         }
@@ -229,13 +282,14 @@ proc find_trial_indexes {} {
     }
   }
 
-  if {$G(verbose)} {
+  if {$::VERBOSE} {
     foreach k [array names G trial.*] { puts "index: $G($k)" }
   }
 }
 
-proc run_trials {} {
-  global G
+proc run_trials {ctxvar} {
+  upvar $ctxvar G
+  set ret [list]
 
   foreach k [array names G trial.*] {
     set idxname [lindex [split $k .] 1]
@@ -253,14 +307,121 @@ proc run_trials {} {
       }
     }
     foreach idx [array names used] {
-      puts $G(trial.$idx)
+      lappend ret $G(trial.$idx)
     }
   }
+
+  set ret
 }
 
-process_cmdline_args $argv
-open_database
-analyze_selects
-find_trial_indexes
-run_trials
+proc sqlidx_init_context {varname} {
+  upvar $varname G
+  set G(lSelect)  [list]           ;# List of SELECT statements to analyze
+  set G(database) ""               ;# Name of database or SQL schema file
+  set G(trace)    [list]           ;# List of data from xTrace()
+}
+
+#-------------------------------------------------------------------------
+# The following is test code only.
+#
+proc sqlidx_one_test {tn schema select expected} {
+#  if {$tn!=2} return
+  sqlidx_init_context C
+
+  sqlite3 db ""
+  db eval $schema
+  lappend C(lSelect) $select
+  analyze_selects C
+  find_trial_indexes C
+
+  set idxlist [run_trials C]
+  if {$idxlist != [list {*}$expected]} {
+    puts stderr "Test $tn failed"
+    puts stderr "Expected: $expected"
+    puts stderr "Got: $idxlist"
+    exit -1
+  }
+
+  db close
+}
+
+proc sqlidx_internal_tests {} {
+
+  # No indexes for a query with no constraints.
+  sqlidx_one_test 0 {
+    CREATE TABLE t1(a, b, c);
+  } {
+    SELECT * FROM t1;
+  } {
+  }
+
+  sqlidx_one_test 1 {
+    CREATE TABLE t1(a, b, c);
+    CREATE TABLE t2(x, y, z);
+  } {
+    SELECT a FROM t1, t2 WHERE a=? AND x=c
+  } {
+    {CREATE INDEX t2_x ON t2(x collate BINARY);}
+    {CREATE INDEX t1_a_c ON t1(a collate BINARY, c collate BINARY);}
+  }
+
+  sqlidx_one_test 2 {
+    CREATE TABLE t1(a, b, c);
+  } {
+    SELECT * FROM t1 WHERE b>?;
+  } {
+    {CREATE INDEX t1_b ON t1(b collate BINARY);}
+  }
+
+  sqlidx_one_test 3 {
+    CREATE TABLE t1(a, b, c);
+  } {
+    SELECT * FROM t1 WHERE b COLLATE nocase BETWEEN ? AND ?
+  } {
+    {CREATE INDEX t1_bnocase ON t1(b collate NOCASE);}
+  }
+
+  sqlidx_one_test 4 {
+    CREATE TABLE t1(a, b, c);
+  } {
+    SELECT a FROM t1 ORDER BY b;
+  } {
+    {CREATE INDEX t1_b ON t1(b collate BINARY);}
+  }
+
+  sqlidx_one_test 5 {
+    CREATE TABLE t1(a, b, c);
+  } {
+    SELECT a FROM t1 WHERE a=? ORDER BY b;
+  } {
+    {CREATE INDEX t1_a_b ON t1(a collate BINARY, b collate BINARY);}
+  }
+
+  sqlidx_one_test 5 {
+    CREATE TABLE t1(a, b, c);
+  } {
+    SELECT min(a) FROM t1
+  } {
+    {CREATE INDEX t1_a ON t1(a collate BINARY);}
+  }
+
+  sqlidx_one_test 6 {
+    CREATE TABLE t1(a, b, c);
+  } {
+    SELECT * FROM t1 ORDER BY a ASC, b COLLATE nocase DESC, c ASC;
+  } {
+    {CREATE INDEX t1_a_bnocasedesc_c ON t1(a collate BINARY, b collate NOCASE DESC, c collate BINARY);}
+  }
+
+  exit
+}
+# End of internal test code.
+#-------------------------------------------------------------------------
+
+sqlidx_init_context D
+process_cmdline_args D $argv
+open_database D
+analyze_selects D
+find_trial_indexes D
+foreach idx [run_trials D] { puts $idx }
 

From 323f7d3fc6ca95c55eb0e5ae06e6bacb328a10a7 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 23 Nov 2015 18:28:07 +0000
Subject: [PATCH 04/68] In the CREATE INDEX statements output by
 schemalint.tcl, avoid declaring an explicit collation sequence that is the
 same as the column's default.

FossilOrigin-Name: d3aa067c830e98f2074630c4613c557b0ce90a57
---
 manifest            | 12 +++----
 manifest.uuid       |  2 +-
 tool/schemalint.tcl | 77 ++++++++++++++++++++++++++++++---------------
 3 files changed, 59 insertions(+), 32 deletions(-)

diff --git a/manifest b/manifest
index cd40a978a6..bff0419492 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\slatest\strunk\schanges\swith\sthis\sbranch.
-D 2015-11-23T17:14:03.947
+C In\sthe\sCREATE\sINDEX\sstatements\soutput\sby\sschemalint.tcl,\savoid\sdeclaring\san\sexplicit\scollation\ssequence\sthat\sis\sthe\ssame\sas\sthe\scolumn's\sdefault.
+D 2015-11-23T18:28:07.584
 F Makefile.in d828db6afa6c1fa060d01e33e4674408df1942a1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e928e68168df69b353300ac87c10105206653a03
@@ -1381,7 +1381,7 @@ F tool/replace.tcl 7727c60a04299b65a92f5e1590896fea0f25b9e0
 F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a
 F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5
 F tool/run-speed-test.sh 0ae485af4fe9f826e2b494be8c81f8ca9e222a4a
-F tool/schemalint.tcl e49c9b25c8c43f0531eca96edaf85ce8bd7f308b
+F tool/schemalint.tcl ad61083d39cb40e848b9acc1273e4c487bb55cd7
 F tool/showdb.c d4476e000a64eca9f5e2c2f68741e747b9778e8d
 F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818
 F tool/showlocks.c 9920bcc64f58378ff1118caead34147201f48c68
@@ -1406,7 +1406,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P b8f277c9b45c4b30e6690e838a30311aa8d84876 60de5f23424552c98aa760ac89149a3d51f895be
-R 00875769a91e68d074e499acf18a7d59
+P 8f1ef0904d055b5510ec9043810ebf22a8c5e253
+R 4b9b445c1607ad5c9f9bdcf33b4eb416
 U dan
-Z 5eb820ab8a7550b698dcf8a7d000e044
+Z 6d9149db2c512107280c4e32c490b8cb
diff --git a/manifest.uuid b/manifest.uuid
index b038ee0b5d..ed4290e83d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-8f1ef0904d055b5510ec9043810ebf22a8c5e253
\ No newline at end of file
+d3aa067c830e98f2074630c4613c557b0ce90a57
\ No newline at end of file
diff --git a/tool/schemalint.tcl b/tool/schemalint.tcl
index 933b277ea2..7ed74f9054 100644
--- a/tool/schemalint.tcl
+++ b/tool/schemalint.tcl
@@ -124,32 +124,33 @@ proc expand_eq_list {L} {
 #--------------------------------------------------------------------------
 # Formulate a CREATE INDEX statement that creates an index on table $tname.
 #
-proc eqset_to_index {ctxvar tname eqset {range {}}} {
+proc eqset_to_index {ctxvar aCollVar tname eqset {range {}}} {
   upvar $ctxvar G
+  upvar $aCollVar aColl
+
+  set rangeset [list]
+  foreach e [lsort $eqset] {
+    lappend rangeset [lindex $e 0] [lindex $e 1] ASC
+  }
+  set rangeset [concat $rangeset $range]
 
   set lCols [list]
   set idxname $tname
-  foreach e [lsort $eqset] { 
-    if {[llength $e]==0} continue
-    foreach {c collate} $e {}
-    lappend lCols "$c collate $collate"
-    append idxname "_$c"
-    if {[string compare -nocase binary $collate]!=0} {
-      append idxname [string tolower $collate]
-    }
-  }
 
-  foreach {c collate dir} $range {
+  foreach {c collate dir} $rangeset {
     append idxname "_$c"
-    if {[string compare -nocase binary $collate]!=0} {
+    set coldef $c
+
+    if {[string compare -nocase $collate $aColl($c)]!=0} {
       append idxname [string tolower $collate]
+      append coldef " COLLATE $collate"
     }
+
     if {$dir=="DESC"} {
-      lappend lCols "$c collate $collate DESC"
+      append coldef " DESC"
       append idxname "desc"
-    } else {
-      lappend lCols "$c collate $collate"
     }
+    lappend lCols $coldef
   }
 
   set create_index "CREATE INDEX $idxname ON ${tname}("
@@ -184,12 +185,30 @@ proc expand_or_cons {L} {
   return $lRet
 }
 
+proc sqlidx_get_coll_map {tname arrayvar} {
+  upvar $arrayvar aColl
+  set colnames [list]
+  db eval "PRAGMA table_info = $tname" x { lappend colnames $x(name) }
+  db eval "CREATE INDEX schemalint_test ON ${tname}([join $colnames ,])"
+
+  db eval "PRAGMA index_xinfo = schemalint_test" x { 
+    set aColl($x(name)) $x(coll)
+  }
+  db eval "DROP INDEX schemalint_test"
+}
+
 proc find_trial_indexes {ctxvar} {
   upvar $ctxvar G
   foreach t $G(trace) {
     set tname [lindex $t 0]
     catch { array unset mask }
 
+    # Invoke "PRAGMA table_info" on the table. Use the results to create
+    # an array mapping from column name to collation sequence. Store the
+    # array in local variable aColl.
+    #
+    sqlidx_get_coll_map $tname aColl
+
     set orderby [list]
     if {[lindex $t end 0]=="orderby"} {
       set orderby [lrange [lindex $t end] 1 end]
@@ -258,7 +277,7 @@ proc find_trial_indexes {ctxvar} {
 
         foreach eqset [expand_eq_list [array get eq]] {
           if {$eqset != ""} {
-            eqset_to_index G $tname $eqset
+            eqset_to_index G aColl $tname $eqset
           }
 
           foreach r $ranges {
@@ -274,7 +293,7 @@ proc find_trial_indexes {ctxvar} {
               if {$bSeen==0} { lappend tail {*}$r }
             }
             if {[llength $tail]} {
-              eqset_to_index G $tname $eqset $r
+              eqset_to_index G aColl $tname $eqset $r
             }
           }
         }
@@ -361,8 +380,8 @@ proc sqlidx_internal_tests {} {
   } {
     SELECT a FROM t1, t2 WHERE a=? AND x=c
   } {
-    {CREATE INDEX t2_x ON t2(x collate BINARY);}
-    {CREATE INDEX t1_a_c ON t1(a collate BINARY, c collate BINARY);}
+    {CREATE INDEX t2_x ON t2(x);}
+    {CREATE INDEX t1_a_c ON t1(a, c);}
   }
 
   sqlidx_one_test 2 {
@@ -370,7 +389,7 @@ proc sqlidx_internal_tests {} {
   } {
     SELECT * FROM t1 WHERE b>?;
   } {
-    {CREATE INDEX t1_b ON t1(b collate BINARY);}
+    {CREATE INDEX t1_b ON t1(b);}
   }
 
   sqlidx_one_test 3 {
@@ -378,7 +397,7 @@ proc sqlidx_internal_tests {} {
   } {
     SELECT * FROM t1 WHERE b COLLATE nocase BETWEEN ? AND ?
   } {
-    {CREATE INDEX t1_bnocase ON t1(b collate NOCASE);}
+    {CREATE INDEX t1_bnocase ON t1(b COLLATE NOCASE);}
   }
 
   sqlidx_one_test 4 {
@@ -386,7 +405,7 @@ proc sqlidx_internal_tests {} {
   } {
     SELECT a FROM t1 ORDER BY b;
   } {
-    {CREATE INDEX t1_b ON t1(b collate BINARY);}
+    {CREATE INDEX t1_b ON t1(b);}
   }
 
   sqlidx_one_test 5 {
@@ -394,7 +413,7 @@ proc sqlidx_internal_tests {} {
   } {
     SELECT a FROM t1 WHERE a=? ORDER BY b;
   } {
-    {CREATE INDEX t1_a_b ON t1(a collate BINARY, b collate BINARY);}
+    {CREATE INDEX t1_a_b ON t1(a, b);}
   }
 
   sqlidx_one_test 5 {
@@ -402,7 +421,7 @@ proc sqlidx_internal_tests {} {
   } {
     SELECT min(a) FROM t1
   } {
-    {CREATE INDEX t1_a ON t1(a collate BINARY);}
+    {CREATE INDEX t1_a ON t1(a);}
   }
 
   sqlidx_one_test 6 {
@@ -410,7 +429,15 @@ proc sqlidx_internal_tests {} {
   } {
     SELECT * FROM t1 ORDER BY a ASC, b COLLATE nocase DESC, c ASC;
   } {
-    {CREATE INDEX t1_a_bnocasedesc_c ON t1(a collate BINARY, b collate NOCASE DESC, c collate BINARY);}
+    {CREATE INDEX t1_a_bnocasedesc_c ON t1(a, b COLLATE NOCASE DESC, c);}
+  }
+
+  sqlidx_one_test 7 {
+    CREATE TABLE t1(a COLLATE NOCase, b, c);
+  } {
+    SELECT * FROM t1 WHERE a=?
+  } {
+    {CREATE INDEX t1_a ON t1(a);}
   }
 
   exit

From c45bf341afda3c3fddb297519292c11a9658a9d3 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 30 Nov 2015 18:17:55 +0000
Subject: [PATCH 05/68] Fix the schemalint.tcl script to handle identifiers
 that require quoting.

FossilOrigin-Name: 451e0fafbe5b7e9c67d9b584d5e16796c3196881
---
 manifest             |  16 +++---
 manifest.uuid        |   2 +-
 src/where.c          |  10 ++--
 test/schemalint.test |  28 +++++-----
 tool/schemalint.tcl  | 120 ++++++++++++++++++++++++++++++++++++++++---
 5 files changed, 143 insertions(+), 33 deletions(-)

diff --git a/manifest b/manifest
index bff0419492..16b150e61d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\sthe\sCREATE\sINDEX\sstatements\soutput\sby\sschemalint.tcl,\savoid\sdeclaring\san\sexplicit\scollation\ssequence\sthat\sis\sthe\ssame\sas\sthe\scolumn's\sdefault.
-D 2015-11-23T18:28:07.584
+C Fix\sthe\sschemalint.tcl\sscript\sto\shandle\sidentifiers\sthat\srequire\squoting.
+D 2015-11-30T18:17:55.036
 F Makefile.in d828db6afa6c1fa060d01e33e4674408df1942a1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e928e68168df69b353300ac87c10105206653a03
@@ -416,7 +416,7 @@ F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb
 F src/wal.c 18b0ed49830cf04fe2d68224b41838a73ac6cd24
 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4
 F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba
-F src/where.c 5804902239731a7025240f4338d98cdc57315636
+F src/where.c be09f4a0513f885845ae481324c2e07603f88b38
 F src/whereInt.h 7892bb54cf9ca0ae5c7e6094491b94c9286dc647
 F src/wherecode.c 4c96182e7b25e4be54008dee2da5b9c2f8480b9b
 F src/whereexpr.c bd4877cd4dd11f6ab551ef0054535ca3c6224950
@@ -973,7 +973,7 @@ F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5
 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38
 F test/schema4.test e6a66e20cc69f0e306667c08be7fda3d11707dc5
 F test/schema5.test 29699b4421f183c8f0e88bd28ce7d75d13ea653e
-F test/schemalint.test 7fba0e262353c8c3dd5ee406cb11cf90f8b566fe
+F test/schemalint.test cee9f375637fcc8c6e77d971abe044445c23e024
 F test/securedel.test 21749c32ccc30f1ea9e4b9f33295a6521ec20fa0
 F test/securedel2.test 2d54c28e46eb1fd6902089958b20b1b056c6f1c5
 F test/select1.test be62204d2bd9a5a8a149e9974cfddce893d8f686
@@ -1381,7 +1381,7 @@ F tool/replace.tcl 7727c60a04299b65a92f5e1590896fea0f25b9e0
 F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a
 F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5
 F tool/run-speed-test.sh 0ae485af4fe9f826e2b494be8c81f8ca9e222a4a
-F tool/schemalint.tcl ad61083d39cb40e848b9acc1273e4c487bb55cd7
+F tool/schemalint.tcl 2f44d0874061a948f6ef53505062db4ac9806cf7
 F tool/showdb.c d4476e000a64eca9f5e2c2f68741e747b9778e8d
 F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818
 F tool/showlocks.c 9920bcc64f58378ff1118caead34147201f48c68
@@ -1406,7 +1406,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 8f1ef0904d055b5510ec9043810ebf22a8c5e253
-R 4b9b445c1607ad5c9f9bdcf33b4eb416
+P d3aa067c830e98f2074630c4613c557b0ce90a57
+R aed657578ea11902053a31be3048325d
 U dan
-Z 6d9149db2c512107280c4e32c490b8cb
+Z 94438cf65b515737c9a10851485e87ac
diff --git a/manifest.uuid b/manifest.uuid
index ed4290e83d..f83d5d89f6 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d3aa067c830e98f2074630c4613c557b0ce90a57
\ No newline at end of file
+451e0fafbe5b7e9c67d9b584d5e16796c3196881
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index 4ba35d6ac9..49e9fc8382 100644
--- a/src/where.c
+++ b/src/where.c
@@ -3942,7 +3942,8 @@ static char *whereAppendSingleTerm(
   pColl = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight);
 
   if( zOp ){
-    const char *zFmt = bOr ? "%z{{%s %s %s %lld}}" : "%z{%s %s %s %lld}";
+    const char *zFmt = bOr ? "%z{{%s \"%w\" \"%w\" %lld}}" :
+                             "%z{%s \"%w\" \"%w\" %lld}";
     zBuf = whereAppendPrintf(db, zFmt, zIn, 
         zOp, pTab->aCol[pTerm->u.leftColumn].zName, 
         (pColl ? pColl->zName : "BINARY"),
@@ -4025,7 +4026,7 @@ static void whereTraceBuilder(
       nCol = pTab->nCol;
 
       /* Append the table name to the buffer. */
-      zBuf = whereAppendPrintf(db, "%s", pTab->zName);
+      zBuf = whereAppendPrintf(db, "\"%w\"", pTab->zName);
 
       /* Append the list of columns required to create a covering index */
       zBuf = whereAppendPrintf(db, "%z {cols", zBuf);
@@ -4033,7 +4034,8 @@ static void whereTraceBuilder(
         for(iCol=0; iCol<nCol; iCol++){
           if( iCol==(sizeof(Bitmask)*8-1) ) break;
           if( pItem->colUsed & ((u64)1 << iCol) ){
-            zBuf = whereAppendPrintf(db, "%z %s", zBuf, pTab->aCol[iCol].zName);
+            const char *zName = pTab->aCol[iCol].zName;
+            zBuf = whereAppendPrintf(db, "%z \"%w\"", zBuf, zName);
           }
         }
       }
@@ -4054,7 +4056,7 @@ static void whereTraceBuilder(
           if( pExpr->op==TK_COLUMN && pExpr->iTable==pItem->iCursor ){
             if( pExpr->iColumn>=0 ){
               const char *zName = pTab->aCol[pExpr->iColumn].zName;
-              zBuf = whereAppendPrintf(db, "%z%s%s %s %s", zBuf,
+              zBuf = whereAppendPrintf(db, "%z%s\"%w\" \"%w\" %s", zBuf,
                   bFirst ? " {orderby " : " ", zName, pColl->zName,
                   (pOrderBy->a[i].sortOrder ? "DESC" : "ASC")
               );
diff --git a/test/schemalint.test b/test/schemalint.test
index d354c1fdd7..7e0eba494d 100644
--- a/test/schemalint.test
+++ b/test/schemalint.test
@@ -27,33 +27,33 @@ do_execsql_test 1.0 {
 do_trace_test 1.1 {
   SELECT b, c, y, z FROM t1, t2 WHERE c=? AND z=?
 } {
-  {t1 {cols b c} {eq c BINARY 0}}
-  {t2 {cols y z} {eq z BINARY 0}}
+  {"t1" {cols "b" "c"} {eq "c" "BINARY" 0}}
+  {"t2" {cols "y" "z"} {eq "z" "BINARY" 0}}
 }
 
 do_trace_test 1.2 {
   SELECT a FROM t1 WHERE b>10
 } {
-  {t1 {cols a b} {range b BINARY 0}}
+  {"t1" {cols "a" "b"} {range "b" "BINARY" 0}}
 }
 
 do_trace_test 1.3 {
   SELECT b FROM t1 WHERE b IN (10, 20, 30)
 } {
-  {t1 {cols b} {eq b BINARY 0}}
+  {"t1" {cols "b"} {eq "b" "BINARY" 0}}
 }
 
 do_trace_test 1.4 {
   SELECT * FROM t1, t2 WHERE x=a
 } {
-  {t1 {cols a b c} {eq a BINARY 2}} 
-  {t2 {cols x y z} {eq x BINARY 1}}
+  {"t1" {cols "a" "b" "c"} {eq "a" "BINARY" 2}} 
+  {"t2" {cols "x" "y" "z"} {eq "x" "BINARY" 1}}
 }
 
 do_trace_test 1.5 {
   SELECT * FROM t1 WHERE a IN (1, 2, 3)
 } {
-  {t1 {cols a b c} {eq a BINARY 0}}
+  {"t1" {cols "a" "b" "c"} {eq "a" "BINARY" 0}}
 }
 
 #-----------------------------------------------------------------------
@@ -62,19 +62,19 @@ do_trace_test 1.5 {
 do_trace_test 2.1 {
   SELECT * FROM t1 WHERE a=? OR b=?
 } {
-  {t1 {cols a b c} {or {{eq a BINARY 0}} {{eq b BINARY 0}}}}
+  {"t1" {cols "a" "b" "c"} {or {{eq "a" "BINARY" 0}} {{eq "b" "BINARY" 0}}}}
 }
 
 do_trace_test 2.2 {
   SELECT * FROM t1 WHERE a=? OR (b=? AND c=?)
 } {
-  {t1 {cols a b c} {or {{eq a BINARY 0}} {{eq b BINARY 0} {eq c BINARY 0}}}}
+  {"t1" {cols "a" "b" "c"} {or {{eq "a" "BINARY" 0}} {{eq "b" "BINARY" 0} {eq "c" "BINARY" 0}}}}
 }
 
 do_trace_test 2.3 {
   SELECT * FROM t1 WHERE (a=? AND b=?) OR c=?
 } {
-  {t1 {cols a b c} {or {{eq c BINARY 0}} {{eq a BINARY 0} {eq b BINARY 0}}}}
+  {"t1" {cols "a" "b" "c"} {or {{eq "c" "BINARY" 0}} {{eq "a" "BINARY" 0} {eq "b" "BINARY" 0}}}}
 }
 
 #-----------------------------------------------------------------------
@@ -82,19 +82,19 @@ do_trace_test 2.3 {
 #
 do_trace_test 3.1 {
   SELECT * FROM t1 ORDER BY a;
-} {{t1 {cols a b c} {orderby a BINARY ASC}}}
+} {{"t1" {cols "a" "b" "c"} {orderby "a" "BINARY" ASC}}}
 
 do_trace_test 3.2 {
   SELECT * FROM t1 WHERE a=? ORDER BY b;
-} {{t1 {cols a b c} {eq a BINARY 0} {orderby b BINARY ASC}}}
+} {{"t1" {cols "a" "b" "c"} {eq "a" "BINARY" 0} {orderby "b" "BINARY" ASC}}}
 
 do_trace_test 3.3 {
   SELECT min(a) FROM t1;
-} {{t1 {cols a} {orderby a BINARY ASC}}}
+} {{"t1" {cols "a"} {orderby "a" "BINARY" ASC}}}
 
 do_trace_test 3.4 {
   SELECT max(a) FROM t1;
-} {{t1 {cols a} {orderby a BINARY DESC}}}
+} {{"t1" {cols "a"} {orderby "a" "BINARY" DESC}}}
 
 finish_test
 
diff --git a/tool/schemalint.tcl b/tool/schemalint.tcl
index 7ed74f9054..b6ebd4390d 100644
--- a/tool/schemalint.tcl
+++ b/tool/schemalint.tcl
@@ -11,6 +11,41 @@ proc usage {} {
   exit
 }
 
+# Return the quoted version of identfier $id. Quotes are only added if 
+# they are required by SQLite.
+#
+# This command currently assumes that quotes are required if the 
+# identifier contains any ASCII-range characters that are not 
+# alpha-numeric or underscores.
+#
+proc quote {id} {
+  if {[requires_quote $id]} {
+    set x [string map {\" \"\"} $id]
+    return "\"$x\""
+  }
+  return $id
+}
+proc requires_quote {id} {
+  foreach c [split $id {}] {
+    if {[string is alnum $c]==0 && $c!="_"} {
+      return 1
+    }
+  }
+  return 0
+}
+
+# The argument passed to this command is a Tcl list of identifiers. The
+# value returned is the same list, except with each item quoted and the
+# elements comma-separated.
+#
+proc list_to_sql {L} {
+  set ret [list]
+  foreach l $L {
+    lappend ret [quote $l]
+  }
+  join $ret ", "
+}
+
 proc process_cmdline_args {ctxvar argv} {
   upvar $ctxvar G
   set nArg [llength $argv]
@@ -139,11 +174,11 @@ proc eqset_to_index {ctxvar aCollVar tname eqset {range {}}} {
 
   foreach {c collate dir} $rangeset {
     append idxname "_$c"
-    set coldef $c
+    set coldef [quote $c]
 
     if {[string compare -nocase $collate $aColl($c)]!=0} {
       append idxname [string tolower $collate]
-      append coldef " COLLATE $collate"
+      append coldef " COLLATE [quote $collate]"
     }
 
     if {$dir=="DESC"} {
@@ -153,7 +188,7 @@ proc eqset_to_index {ctxvar aCollVar tname eqset {range {}}} {
     lappend lCols $coldef
   }
 
-  set create_index "CREATE INDEX $idxname ON ${tname}("
+  set create_index "CREATE INDEX [quote $idxname] ON [quote $tname]("
   append create_index [join $lCols ", "]
   append create_index ");"
 
@@ -185,12 +220,27 @@ proc expand_or_cons {L} {
   return $lRet
 }
 
+#--------------------------------------------------------------------------
+# Argument $tname is the name of a table in the main database opened by
+# database handle [db]. $arrayvar is the name of an array variable in the
+# caller's context. This command populates the array with an entry mapping 
+# from column name to default collation sequence for each column of table
+# $tname. For example, if a table is declared:
+#
+#   CREATE TABLE t1(a COLLATE nocase, b, c COLLATE binary)
+#
+# the mapping is populated with:
+#
+#   map(a) -> "nocase"
+#   map(b) -> "binary"
+#   map(c) -> "binary"
+#
 proc sqlidx_get_coll_map {tname arrayvar} {
   upvar $arrayvar aColl
   set colnames [list]
-  db eval "PRAGMA table_info = $tname" x { lappend colnames $x(name) }
-  db eval "CREATE INDEX schemalint_test ON ${tname}([join $colnames ,])"
-
+  set qname [quote $tname]
+  db eval "PRAGMA table_info = $qname" x { lappend colnames $x(name) }
+  db eval "CREATE INDEX schemalint_test ON ${qname}([list_to_sql $colnames])"
   db eval "PRAGMA index_xinfo = schemalint_test" x { 
     set aColl($x(name)) $x(coll)
   }
@@ -348,6 +398,7 @@ proc sqlidx_one_test {tn schema select expected} {
   sqlidx_init_context C
 
   sqlite3 db ""
+  db collate "a b c" [list string compare]
   db eval $schema
   lappend C(lSelect) $select
   analyze_selects C
@@ -362,9 +413,14 @@ proc sqlidx_one_test {tn schema select expected} {
   }
 
   db close
+
+  upvar nTest nTest
+  incr nTest
 }
 
 proc sqlidx_internal_tests {} {
+  set nTest 0
+
 
   # No indexes for a query with no constraints.
   sqlidx_one_test 0 {
@@ -440,6 +496,58 @@ proc sqlidx_internal_tests {} {
     {CREATE INDEX t1_a ON t1(a);}
   }
 
+  # Tables with names that require quotes.
+  #
+  sqlidx_one_test 8.1 {
+    CREATE TABLE "t t"(a, b, c);
+  } {
+    SELECT * FROM "t t" WHERE a=?
+  } {
+    {CREATE INDEX "t t_a" ON "t t"(a);}
+  }
+  sqlidx_one_test 8.2 {
+    CREATE TABLE "t t"(a, b, c);
+  } {
+    SELECT * FROM "t t" WHERE b BETWEEN ? AND ?
+  } {
+    {CREATE INDEX "t t_b" ON "t t"(b);}
+  }
+  
+  # Columns with names that require quotes.
+  #
+  sqlidx_one_test 9.1 {
+    CREATE TABLE t3(a, "b b", c);
+  } {
+    SELECT * FROM t3 WHERE "b b" = ?
+  } {
+    {CREATE INDEX "t3_b b" ON t3("b b");}
+  }
+  sqlidx_one_test 9.2 {
+    CREATE TABLE t3(a, "b b", c);
+  } {
+    SELECT * FROM t3 ORDER BY "b b"
+  } {
+    {CREATE INDEX "t3_b b" ON t3("b b");}
+  }
+
+  # Collations with names that require quotes.
+  #
+  sqlidx_one_test 10.1 {
+    CREATE TABLE t4(a, b, c);
+  } {
+    SELECT * FROM t4 ORDER BY c COLLATE "a b c"
+  } {
+    {CREATE INDEX "t4_ca b c" ON t4(c COLLATE "a b c");}
+  }
+  sqlidx_one_test 10.2 {
+    CREATE TABLE t4(a, b, c);
+  } {
+    SELECT * FROM t4 WHERE c = ? COLLATE "a b c"
+  } {
+    {CREATE INDEX "t4_ca b c" ON t4(c COLLATE "a b c");}
+  }
+
+  puts "All $nTest tests passed"
   exit
 }
 # End of internal test code.

From ff3b534b9f4349f1b84f3e7f04f7293dfb5b6ac2 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 30 Nov 2015 19:16:00 +0000
Subject: [PATCH 06/68] Add a rule to main.mk to build the schemalint.tcl
 script into an executable. Similar to the way the sqlite3_analyzer executable
 is built.

FossilOrigin-Name: b8251065db9dbe5463490316baa09dc636551377
---
 main.mk             | 14 ++++++++++++++
 manifest            | 14 +++++++-------
 manifest.uuid       |  2 +-
 tool/schemalint.tcl | 10 +++++++++-
 4 files changed, 31 insertions(+), 9 deletions(-)

diff --git a/main.mk b/main.mk
index 838e0753d9..9ee3af7289 100644
--- a/main.mk
+++ b/main.mk
@@ -715,6 +715,20 @@ sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl
 sqlite3_analyzer$(EXE): sqlite3_analyzer.c
 	$(TCCX) $(TCL_FLAGS) sqlite3_analyzer.c -o $@ $(LIBTCL) $(THREADLIB) 
 
+sqlite3_schemalint.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/schemalint.tcl
+	echo "#define TCLSH 2" > $@
+	echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@
+	cat sqlite3.c $(TOP)/src/tclsqlite.c >> $@
+	echo "static const char *tclsh_main_loop(void){" >> $@
+	echo "static const char *zMainloop = " >> $@
+	tclsh $(TOP)/tool/tostr.tcl $(TOP)/tool/schemalint.tcl >> $@
+	echo "; return zMainloop; }" >> $@
+
+sqlite3_schemalint$(EXE): $(TESTSRC) sqlite3_schemalint.c
+	$(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS)                  \
+		sqlite3_schemalint.c $(TESTSRC)               \
+		-o sqlite3_schemalint$(EXE) $(LIBTCL) $(THREADLIB)
+
 # Rules to build the 'testfixture' application.
 #
 TESTFIXTURE_FLAGS  = -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
diff --git a/manifest b/manifest
index 16b150e61d..7fe67fd96f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sthe\sschemalint.tcl\sscript\sto\shandle\sidentifiers\sthat\srequire\squoting.
-D 2015-11-30T18:17:55.036
+C Add\sa\srule\sto\smain.mk\sto\sbuild\sthe\sschemalint.tcl\sscript\sinto\san\sexecutable.\sSimilar\sto\sthe\sway\sthe\ssqlite3_analyzer\sexecutable\sis\sbuilt.
+D 2015-11-30T19:16:00.031
 F Makefile.in d828db6afa6c1fa060d01e33e4674408df1942a1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e928e68168df69b353300ac87c10105206653a03
@@ -263,7 +263,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk ae99be5eb22933b1ecf80f94d41d25a3ea80aaf3
+F main.mk 0aec7410e847a886ed8260b829e4e72f060b0431
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -1381,7 +1381,7 @@ F tool/replace.tcl 7727c60a04299b65a92f5e1590896fea0f25b9e0
 F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a
 F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5
 F tool/run-speed-test.sh 0ae485af4fe9f826e2b494be8c81f8ca9e222a4a
-F tool/schemalint.tcl 2f44d0874061a948f6ef53505062db4ac9806cf7
+F tool/schemalint.tcl 045481a427ecdb72d11a0b0ef8725e7b892a09b9
 F tool/showdb.c d4476e000a64eca9f5e2c2f68741e747b9778e8d
 F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818
 F tool/showlocks.c 9920bcc64f58378ff1118caead34147201f48c68
@@ -1406,7 +1406,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P d3aa067c830e98f2074630c4613c557b0ce90a57
-R aed657578ea11902053a31be3048325d
+P 451e0fafbe5b7e9c67d9b584d5e16796c3196881
+R 8aa86746f90305c3990fbec7bea3f626
 U dan
-Z 94438cf65b515737c9a10851485e87ac
+Z e9b16724c3379c54748ede3fe8135436
diff --git a/manifest.uuid b/manifest.uuid
index f83d5d89f6..d723846430 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-451e0fafbe5b7e9c67d9b584d5e16796c3196881
\ No newline at end of file
+b8251065db9dbe5463490316baa09dc636551377
\ No newline at end of file
diff --git a/tool/schemalint.tcl b/tool/schemalint.tcl
index b6ebd4390d..99e3476661 100644
--- a/tool/schemalint.tcl
+++ b/tool/schemalint.tcl
@@ -1,3 +1,4 @@
+if {[catch {
 
 set ::VERBOSE 0
 
@@ -553,10 +554,17 @@ proc sqlidx_internal_tests {} {
 # End of internal test code.
 #-------------------------------------------------------------------------
 
+if {[info exists ::argv0]==0} { set ::argv0 [info nameofexec] }
+if {[info exists ::argv]==0} usage
 sqlidx_init_context D
-process_cmdline_args D $argv
+process_cmdline_args D $::argv
 open_database D
 analyze_selects D
 find_trial_indexes D
 foreach idx [run_trials D] { puts $idx }
 
+} err]} {
+  puts "ERROR: $err"
+  puts $errorInfo
+  exit 1
+}

From a494ca8d12761b3b1ee89019811a3803ed5a252f Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 22 Jan 2016 14:32:20 +0000
Subject: [PATCH 07/68] Update the schemalint.tcl script so that the argument
 to a -select option may be either an SQL statement or the name of a file
 containing an SQL statement

FossilOrigin-Name: d4e37767671dc946b8a21d0158fc3044e36f0c7e
---
 manifest            | 12 ++++++------
 manifest.uuid       |  2 +-
 tool/schemalint.tcl | 14 +++++++++++++-
 3 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 7fe67fd96f..cc75817d97 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sa\srule\sto\smain.mk\sto\sbuild\sthe\sschemalint.tcl\sscript\sinto\san\sexecutable.\sSimilar\sto\sthe\sway\sthe\ssqlite3_analyzer\sexecutable\sis\sbuilt.
-D 2015-11-30T19:16:00.031
+C Update\sthe\sschemalint.tcl\sscript\sso\sthat\sthe\sargument\sto\sa\s-select\soption\smay\sbe\seither\san\sSQL\sstatement\sor\sthe\sname\sof\sa\sfile\scontaining\san\sSQL\sstatement
+D 2016-01-22T14:32:20.666
 F Makefile.in d828db6afa6c1fa060d01e33e4674408df1942a1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e928e68168df69b353300ac87c10105206653a03
@@ -1381,7 +1381,7 @@ F tool/replace.tcl 7727c60a04299b65a92f5e1590896fea0f25b9e0
 F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a
 F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5
 F tool/run-speed-test.sh 0ae485af4fe9f826e2b494be8c81f8ca9e222a4a
-F tool/schemalint.tcl 045481a427ecdb72d11a0b0ef8725e7b892a09b9
+F tool/schemalint.tcl 7b94e303b02d1b1724c56201ed9fa2845dc8d3a5
 F tool/showdb.c d4476e000a64eca9f5e2c2f68741e747b9778e8d
 F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818
 F tool/showlocks.c 9920bcc64f58378ff1118caead34147201f48c68
@@ -1406,7 +1406,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 451e0fafbe5b7e9c67d9b584d5e16796c3196881
-R 8aa86746f90305c3990fbec7bea3f626
+P b8251065db9dbe5463490316baa09dc636551377
+R 6851215977a26ab67201b1a70341660d
 U dan
-Z e9b16724c3379c54748ede3fe8135436
+Z 63a5088a2207bef48ca5ab65d0652925
diff --git a/manifest.uuid b/manifest.uuid
index d723846430..3a22438944 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b8251065db9dbe5463490316baa09dc636551377
\ No newline at end of file
+d4e37767671dc946b8a21d0158fc3044e36f0c7e
\ No newline at end of file
diff --git a/tool/schemalint.tcl b/tool/schemalint.tcl
index 99e3476661..2755ba8828 100644
--- a/tool/schemalint.tcl
+++ b/tool/schemalint.tcl
@@ -47,6 +47,13 @@ proc list_to_sql {L} {
   join $ret ", "
 }
 
+proc readfile {zFile} {
+  set fd [open $zFile]
+  set data [read $fd]
+  close $fd
+  return $data
+}
+
 proc process_cmdline_args {ctxvar argv} {
   upvar $ctxvar G
   set nArg [llength $argv]
@@ -58,7 +65,12 @@ proc process_cmdline_args {ctxvar argv} {
       -select {
         incr i
         if {$i>=[llength $argv]-1} usage
-        lappend G(lSelect) [lindex $argv $i]
+        set zSelect [lindex $argv $i]
+        if {[file readable $zSelect]} {
+          lappend G(lSelect) [readfile $zSelect]
+        } else {
+          lappend G(lSelect) $zSelect
+        }
       }
       -verbose {
         set ::VERBOSE 1

From 336bfe06b96d76f2e71cb1d9bc83ceb97d172cf5 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 22 Jan 2016 14:44:02 +0000
Subject: [PATCH 08/68] Fix handling of transitive constraints in
 schemalint.tcl.

FossilOrigin-Name: 44edc1aa3b412ddbe2a242075e2bf36a99437688
---
 manifest            | 14 +++++++-------
 manifest.uuid       |  2 +-
 src/where.c         |  8 +++++---
 tool/schemalint.tcl | 12 ++++++++++++
 4 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index cc75817d97..3f50c20486 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\sthe\sschemalint.tcl\sscript\sso\sthat\sthe\sargument\sto\sa\s-select\soption\smay\sbe\seither\san\sSQL\sstatement\sor\sthe\sname\sof\sa\sfile\scontaining\san\sSQL\sstatement
-D 2016-01-22T14:32:20.666
+C Fix\shandling\sof\stransitive\sconstraints\sin\sschemalint.tcl.
+D 2016-01-22T14:44:02.574
 F Makefile.in d828db6afa6c1fa060d01e33e4674408df1942a1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e928e68168df69b353300ac87c10105206653a03
@@ -416,7 +416,7 @@ F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb
 F src/wal.c 18b0ed49830cf04fe2d68224b41838a73ac6cd24
 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4
 F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba
-F src/where.c be09f4a0513f885845ae481324c2e07603f88b38
+F src/where.c 1e987b91aaafe2bb363488e43fcdac137f5b8b59
 F src/whereInt.h 7892bb54cf9ca0ae5c7e6094491b94c9286dc647
 F src/wherecode.c 4c96182e7b25e4be54008dee2da5b9c2f8480b9b
 F src/whereexpr.c bd4877cd4dd11f6ab551ef0054535ca3c6224950
@@ -1381,7 +1381,7 @@ F tool/replace.tcl 7727c60a04299b65a92f5e1590896fea0f25b9e0
 F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a
 F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5
 F tool/run-speed-test.sh 0ae485af4fe9f826e2b494be8c81f8ca9e222a4a
-F tool/schemalint.tcl 7b94e303b02d1b1724c56201ed9fa2845dc8d3a5
+F tool/schemalint.tcl 49690356702d6cac07e2bb1790eac73862e92926
 F tool/showdb.c d4476e000a64eca9f5e2c2f68741e747b9778e8d
 F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818
 F tool/showlocks.c 9920bcc64f58378ff1118caead34147201f48c68
@@ -1406,7 +1406,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P b8251065db9dbe5463490316baa09dc636551377
-R 6851215977a26ab67201b1a70341660d
+P d4e37767671dc946b8a21d0158fc3044e36f0c7e
+R cbe06eb4db20d1a79810102395b5ccc1
 U dan
-Z 63a5088a2207bef48ca5ab65d0652925
+Z e0e53a7502c2ab11481d0d1a7c5964ef
diff --git a/manifest.uuid b/manifest.uuid
index 3a22438944..8879269b0a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d4e37767671dc946b8a21d0158fc3044e36f0c7e
\ No newline at end of file
+44edc1aa3b412ddbe2a242075e2bf36a99437688
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index 49e9fc8382..770b2f6b39 100644
--- a/src/where.c
+++ b/src/where.c
@@ -3924,6 +3924,7 @@ static char *whereAppendPrintf(sqlite3 *db, const char *zFmt, ...){
 static char *whereAppendSingleTerm(
   Parse *pParse,
   Table *pTab,
+  int iCol,
   int bOr,
   char *zIn,
   WhereTerm *pTerm
@@ -3945,7 +3946,7 @@ static char *whereAppendSingleTerm(
     const char *zFmt = bOr ? "%z{{%s \"%w\" \"%w\" %lld}}" :
                              "%z{%s \"%w\" \"%w\" %lld}";
     zBuf = whereAppendPrintf(db, zFmt, zIn, 
-        zOp, pTab->aCol[pTerm->u.leftColumn].zName, 
+        zOp, pTab->aCol[iCol].zName, 
         (pColl ? pColl->zName : "BINARY"),
         pTerm->prereqRight
     );
@@ -3969,6 +3970,7 @@ static char *whereTraceWC(
   int iCol;
   int ii;
   int bFirst = !bInitialSpace;
+  int bOr = (pWC->op==TK_OR);
 
   /* List of WO_SINGLE constraints */
   for(iCol=0; iCol<pTab->nCol; iCol++){
@@ -3979,9 +3981,9 @@ static char *whereTraceWC(
         pTerm;
         pTerm=whereScanNext(&scan)
     ){
-      assert( iCol==pTerm->u.leftColumn );
+      /* assert( iCol==pTerm->u.leftColumn ); */
       if( bFirst==0 ) zBuf = whereAppendPrintf(db, "%z ", zBuf);
-      zBuf = whereAppendSingleTerm(pParse, pTab, pWC->op==TK_OR, zBuf, pTerm);
+      zBuf = whereAppendSingleTerm(pParse, pTab, iCol, bOr, zBuf, pTerm);
       bFirst = 0;
     }
   }
diff --git a/tool/schemalint.tcl b/tool/schemalint.tcl
index 2755ba8828..1c3f2e27c3 100644
--- a/tool/schemalint.tcl
+++ b/tool/schemalint.tcl
@@ -560,6 +560,18 @@ proc sqlidx_internal_tests {} {
     {CREATE INDEX "t4_ca b c" ON t4(c COLLATE "a b c");}
   }
 
+  # Transitive constraints
+  #
+  sqlidx_one_test 11.1 {
+    CREATE TABLE t5(a, b);
+    CREATE TABLE t6(c, d);
+  } {
+    SELECT * FROM t5, t6 WHERE a=? AND b=c AND c=?
+  } {
+    {CREATE INDEX t6_c ON t6(c);} 
+    {CREATE INDEX t5_a_b ON t5(a, b);}
+  }
+
   puts "All $nTest tests passed"
   exit
 }

From 3e6ac1643c3beebc9d2c23f819e25a229a0e8a1d Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 11 Feb 2016 21:01:16 +0000
Subject: [PATCH 09/68] Experimental integration of schemalint functionality
 with the shell tool. Does not work yet.

FossilOrigin-Name: ed49f297bcee86674ed673e195610b8cc1d35647
---
 main.mk             |   2 +-
 manifest            |  31 +--
 manifest.uuid       |   2 +-
 src/main.c          |   7 +
 src/pragma.c        |   1 +
 src/shell.c         |  36 +++-
 src/shell_indexes.c | 468 ++++++++++++++++++++++++++++++++++++++++++++
 src/sqlite.h.in     |   9 +
 src/sqliteInt.h     |   4 +
 src/where.c         | 191 ++++++------------
 10 files changed, 601 insertions(+), 150 deletions(-)
 create mode 100644 src/shell_indexes.c

diff --git a/main.mk b/main.mk
index 22d40fa258..a10f043695 100644
--- a/main.mk
+++ b/main.mk
@@ -472,7 +472,7 @@ libsqlite3.a:	$(LIBOBJ)
 	$(AR) libsqlite3.a $(LIBOBJ)
 	$(RANLIB) libsqlite3.a
 
-sqlite3$(EXE):	$(TOP)/src/shell.c libsqlite3.a sqlite3.h
+sqlite3$(EXE):	$(TOP)/src/shell.c libsqlite3.a sqlite3.h $(TOP)/src/shell_indexes.c
 	$(TCCX) $(READLINE_FLAGS) -o sqlite3$(EXE) $(SHELL_OPT) \
 		$(TOP)/src/shell.c libsqlite3.a $(LIBREADLINE) $(TLIBS) $(THREADLIB)
 
diff --git a/manifest b/manifest
index 78c07a5223..7da4cdeb1d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\slatest\strunk\schanges\swith\sthis\sbranch.
-D 2016-02-09T15:10:56.225
+C Experimental\sintegration\sof\sschemalint\sfunctionality\swith\sthe\sshell\stool.\sDoes\snot\swork\syet.
+D 2016-02-11T21:01:16.253
 F Makefile.in dac2776c84e0d533b158a9af6e57e05c4a6b19f3
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc b0493f10caddb8adf992a4e6f1943141fc7c6816
@@ -13,7 +13,7 @@ F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903
 F autoconf/Makefile.am 1c1657650775960804945dc392e14d9e43c5ed84
 F autoconf/Makefile.msc a35b2aab24d1603f3f0ae65cf01686c2578d319c
 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7
-F autoconf/README.txt e9757a381e5ce2553dbaa6247bb8ad00eb8d87aa w autoconf/README
+F autoconf/README.txt e9757a381e5ce2553dbaa6247bb8ad00eb8d87aa
 F autoconf/configure.ac 72a5e42beb090b32bca580285dc0ab3c4670adb8
 F autoconf/tea/Makefile.in b438a7020446c8a8156e8d97c8914a04833da6fd
 F autoconf/tea/README 3e9a3c060f29a44344ab50aec506f4db903fb873
@@ -272,7 +272,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk cd48a5d8a6dc59229f4f3fe40771104f2918f789
+F main.mk eeff3d12ebe5945dac88147c641407c22a731131
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -313,7 +313,7 @@ F src/insert.c 046199e085e69e05af7bef197d53c5b4b402b6fa
 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d
 F src/legacy.c b1b0880fc474abfab89e737b0ecfde0bd7a60902
 F src/loadext.c 84996d7d70a605597d79c1f1d7b2012a5fd34f2b
-F src/main.c b67a45397b93b7ba8fbd6bfcb03423d245baed05
+F src/main.c 816b9a98a6aca0fd643e77f3610d6a4a1a4c7e24
 F src/malloc.c 337e9808b5231855fe28857950f4f60ae42c417f
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c 6919bcf12f221868ea066eec27e579fed95ce98b
@@ -341,7 +341,7 @@ F src/parse.y d7bff41d460f2df96fb890f36700e85cb0fc5634
 F src/pcache.c 73895411fa6b7bd6f0091212feabbe833b358d23
 F src/pcache.h 4d0ccaad264d360981ec5e6a2b596d6e85242545
 F src/pcache1.c 72f644dc9e1468c72922eff5904048427b817051
-F src/pragma.c 80ee77226d0008d9188356a6cbbe6010866e1bee
+F src/pragma.c cfd521558fccd3864ec664af09a061e9e692583f
 F src/pragma.h 64c78a648751b9f4f297276c4eb7507b14b4628c
 F src/prepare.c c12b786713df3e8270c0f85f988c5359d8b4d87c
 F src/printf.c 63e6fb12bbe702dd664dc3703776c090383a5a26
@@ -349,11 +349,12 @@ F src/random.c ba2679f80ec82c4190062d756f22d0c358180696
 F src/resolve.c 9f7ce3a3c087afb7597b7c916c99126ff3f12f0c
 F src/rowset.c 9fe4b3ad7cc00944386bb600233d8f523de07a6e
 F src/select.c ff80004a9a6ece891a8d9327a88e7b6e2588ee6d
-F src/shell.c dcd7a83645ef2a58ee9c6d0ea4714d877d7835c4
-F src/sqlite.h.in cf22ad1d52dca2c9862d63833e581028119aab7e
+F src/shell.c 2cde87e03712204231167c4a6c61b0eb5129e105
+F src/shell_indexes.c 3cff393ee86d15fbfbe31f30cd752b46d7779b52
+F src/sqlite.h.in c7db059d3b810b70b83d9ed1436fa813eba22462
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h dfbe62ffd95b99afe2140d8c35b180d11924072d
-F src/sqliteInt.h 3aeaff9611acd790c8e76719b33db09ab885d537
+F src/sqliteInt.h a1d0d9613ed7da3657396795e44991fef188c8ee
 F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46
 F src/status.c 70912d7be68e9e2dbc4010c93d344af61d4c59ba
 F src/table.c 51b46b2a62d1b3a959633d593b89bab5e2c9155e
@@ -427,7 +428,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c d21b99fd1458159d0b1ecdccc8ee6ada4fdc4c54
 F src/wal.h 2f7c831cf3b071fa548bf2d5cac640846a7ff19c
 F src/walker.c 0f142b5bd3ed2041fc52d773880748b212e63354
-F src/where.c 438b89caa0cbe17cd32703a8f93baca9789d0474
+F src/where.c 89d5845353fe6d2e77bce52a2c8bea0781c69dad
 F src/whereInt.h 78b6b4de94db84aecbdc07fe3e38f648eb391e9a
 F src/wherecode.c 791a784bbf8749d560fdb0b990b607bc4f44a38d
 F src/whereexpr.c de117970b29471177a6901d60ad83a194671dc03
@@ -980,7 +981,7 @@ F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0
 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd
 F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7
 F test/savepoint7.test db3db281486c925095f305aad09fe806e5188ff3
-F test/savepointfault.test f044eac64b59f09746c7020ee261734de82bf9b2 w test/savepoint3.test
+F test/savepointfault.test f044eac64b59f09746c7020ee261734de82bf9b2
 F test/scanstatus.test 5253c219e331318a437f436268e0e82345700285
 F test/schema.test 8f7999be894260f151adf15c2c7540f1c6d6a481
 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5
@@ -1346,7 +1347,7 @@ F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2
 F test/whereI.test eab5b226bbc344ac70d7dc09b963a064860ae6d7
 F test/whereJ.test 55a3221706a7ab706293f17cc8f96da563bf0767
 F test/whereK.test f8e3cf26a8513ecc7f514f54df9f0572c046c42b
-F test/wherefault.test 1374c3aa198388925246475f84ad4cd5f9528864 w test/where8m.test
+F test/wherefault.test 1374c3aa198388925246475f84ad4cd5f9528864
 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31
 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c
 F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c
@@ -1429,7 +1430,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 9341491c3a11d5a66e4f88d2af9b0d3799b4f27a ca72be8618e5d466d6f35819ca8bbd2b84269959
-R c3affb84a750d99cc1d1198e66d07e76
+P 1a4182eedd0143c3f71b3d97f1d1bb25adeba617
+R 6253a80a5a7097c3b24b24ce68900613
 U dan
-Z dbcdec085ed65802fe686b0a9369fd51
+Z bddf3d3a7cd183a6a2362ed54e10358b
diff --git a/manifest.uuid b/manifest.uuid
index b2c40d21e2..38f9cc7372 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-1a4182eedd0143c3f71b3d97f1d1bb25adeba617
\ No newline at end of file
+ed49f297bcee86674ed673e195610b8cc1d35647
\ No newline at end of file
diff --git a/src/main.c b/src/main.c
index 922af1315a..72f12d2ac2 100644
--- a/src/main.c
+++ b/src/main.c
@@ -792,6 +792,13 @@ int sqlite3_db_config(sqlite3 *db, int op, ...){
       rc = setupLookaside(db, pBuf, sz, cnt);
       break;
     }
+#ifdef SQLITE_SCHEMA_LINT
+    case SQLITE_DBCONFIG_WHEREINFO: {
+      db->xWhereInfo = va_arg(ap, void(*)(void*, int, const char*, int, i64));
+      db->pWhereInfoCtx = va_arg(ap, void*);
+      break;
+    }
+#endif
     default: {
       static const struct {
         int op;      /* The opcode */
diff --git a/src/pragma.c b/src/pragma.c
index c34d5421c2..2b4058ca16 100644
--- a/src/pragma.c
+++ b/src/pragma.c
@@ -1048,6 +1048,7 @@ void sqlite3Pragma(
   ** type:       Column declaration type.
   ** notnull:    True if 'NOT NULL' is part of column declaration
   ** dflt_value: The default value for the column, if any.
+  ** pk:         Non-zero for PK fields.
   */
   case PragTyp_TABLE_INFO: if( zRight ){
     Table *pTab;
diff --git a/src/shell.c b/src/shell.c
index 3f8b22f4fa..e2fe41291d 100644
--- a/src/shell.c
+++ b/src/shell.c
@@ -156,6 +156,7 @@ static void setTextMode(FILE *out){
 # define setTextMode(X)
 #endif
 
+#include "shell_indexes.c"
 
 /* True if the timer is enabled */
 static int enableTimer = 0;
@@ -592,7 +593,8 @@ typedef struct ShellState ShellState;
 struct ShellState {
   sqlite3 *db;           /* The database */
   int echoOn;            /* True to echo input commands */
-  int autoEQP;           /* Run EXPLAIN QUERY PLAN prior to seach SQL stmt */
+  int autoEQP;           /* Run EXPLAIN QUERY PLAN prior to each SQL stmt */
+  int bRecommend;        /* Instead of sqlite3_exec(), recommend indexes */
   int statsOn;           /* True to display memory stats before each finalize */
   int scanstatsOn;       /* True to display scan stats before each finalize */
   int countChanges;      /* True to display change counts */
@@ -1544,6 +1546,19 @@ static void explain_data_delete(ShellState *p){
   p->iIndent = 0;
 }
 
+typedef struct RecCommandCtx RecCommandCtx;
+struct RecCommandCtx {
+  int (*xCallback)(void*,int,char**,char**,int*);
+  ShellState *pArg;
+};
+
+static void recCommandOut(void *pCtx, const char *zLine){
+  const char *zCol = "output";
+  RecCommandCtx *p = (RecCommandCtx*)pCtx;
+  int t = SQLITE_TEXT;
+  p->xCallback(p->pArg, 1, (char**)&zLine, (char**)&zCol, &t);
+}
+
 /*
 ** Execute a statement or set of statements.  Print 
 ** any result rows/columns depending on the current mode 
@@ -1570,6 +1585,13 @@ static int shell_exec(
     *pzErrMsg = NULL;
   }
 
+  if( pArg->bRecommend ){
+    RecCommandCtx ctx;
+    ctx.xCallback = xCallback;
+    ctx.pArg = pArg;
+    rc = shellIndexesCommand(db, zSql, recCommandOut, &ctx, pzErrMsg);
+  }else
+
   while( zSql[0] && (SQLITE_OK == rc) ){
     rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zLeftover);
     if( SQLITE_OK != rc ){
@@ -3609,6 +3631,15 @@ static int do_meta_command(char *zLine, ShellState *p){
     sqlite3_close(pSrc);
   }else
 
+  if( c=='r' && n>=2 && strncmp(azArg[0], "recommend", n)==0 ){
+    if( nArg==2 ){
+      p->bRecommend = booleanValue(azArg[1]);
+    }else{
+      raw_printf(stderr, "Usage: .recommend on|off\n");
+      rc = 1;
+    }
+  }else
+
 
   if( c=='s' && strncmp(azArg[0], "scanstats", n)==0 ){
     if( nArg==2 ){
@@ -4903,6 +4934,9 @@ int SQLITE_CDECL main(int argc, char **argv){
           if( bail_on_error ) return rc;
         }
       }
+
+    }else if( strcmp(z, "-recommend") ){
+      data.bRecommend = 1;
     }else{
       utf8_printf(stderr,"%s: Error: unknown option: %s\n", Argv0, z);
       raw_printf(stderr,"Use -help for a list of options.\n");
diff --git a/src/shell_indexes.c b/src/shell_indexes.c
new file mode 100644
index 0000000000..8e105fec1b
--- /dev/null
+++ b/src/shell_indexes.c
@@ -0,0 +1,468 @@
+/*
+** 2016 February 10
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+*/
+
+typedef sqlite3_int64 i64;
+
+typedef struct IdxConstraint IdxConstraint;
+typedef struct IdxContext IdxContext;
+typedef struct IdxScan IdxScan;
+typedef struct IdxWhere IdxWhere;
+
+/*
+** A single constraint. Equivalent to either "col = ?" or "col < ?".
+**
+** pLink:
+**   ... todo ...
+*/
+struct IdxConstraint {
+  char *zColl;                    /* Collation sequence */
+  int bRange;                     /* True for range, false for eq */
+  int iCol;                       /* Constrained table column */
+  i64 depmask;                    /* Dependency mask */
+  IdxConstraint *pNext;           /* Next constraint in pEq or pRange list */
+  IdxConstraint *pLink;           /* See above */
+};
+
+/*
+** A WHERE clause. Made up of IdxConstraint objects.
+**
+**   a=? AND b=? AND (c=? OR d=?) AND (e=? OR f=?)
+**
+*/
+struct IdxWhere {
+  IdxConstraint *pEq;             /* List of == constraints */
+  IdxConstraint *pRange;          /* List of < constraints */
+  IdxWhere **apOr;                /* Array of OR branches (joined by pNextOr) */
+  IdxWhere *pNextOr;              /* Next in OR'd terms */
+  IdxWhere *pParent;              /* Parent object (or NULL) */
+};
+
+/*
+** A single scan of a single table.
+*/
+struct IdxScan {
+  char *zTable;                   /* Name of table to scan */
+  int iDb;                        /* Database containing table zTable */
+  i64 covering;                   /* Mask of columns required for cov. index */
+  IdxConstraint *pOrder;          /* ORDER BY columns */
+  IdxWhere where;                 /* WHERE Constraints */
+  IdxScan *pNextScan;             /* Next IdxScan object for same query */
+};
+
+/*
+** Context object passed to idxWhereInfo()
+*/
+struct IdxContext {
+  IdxWhere *pCurrent;             /* Current where clause */
+  IdxScan *pScan;                 /* List of scan objects */
+  sqlite3 *dbm;                   /* In-memory db for this analysis */
+  int rc;                         /* Error code (if error has occurred) */
+};
+
+typedef struct PragmaTable PragmaTable;
+typedef struct PragmaCursor PragmaCursor;
+
+struct PragmaTable {
+  sqlite3_vtab base;
+  sqlite3 *db;
+};
+
+struct PragmaCursor {
+  sqlite3_vtab_cursor base;
+  sqlite3_stmt *pStmt;
+  i64 iRowid;
+};
+
+/*
+** Connect to or create a pragma virtual table.
+*/
+static int pragmaConnect(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  const char *zSchema = 
+    "CREATE TABLE a(tbl HIDDEN, cid, name, type, notnull, dflt_value, pk)";
+  PragmaTable *pTab = 0;
+  int rc = SQLITE_OK;
+
+  rc = sqlite3_declare_vtab(db, zSchema);
+  if( rc==SQLITE_OK ){
+    pTab = (PragmaTable *)sqlite3_malloc64(sizeof(PragmaTable));
+    if( pTab==0 ) rc = SQLITE_NOMEM;
+  }
+
+  assert( rc==SQLITE_OK || pTab==0 );
+  if( rc==SQLITE_OK ){
+    memset(pTab, 0, sizeof(PragmaTable));
+    pTab->db = db;
+  }
+
+  *ppVtab = (sqlite3_vtab*)pTab;
+  return rc;
+}
+
+/*
+** Disconnect from or destroy a pragma virtual table.
+*/
+static int pragmaDisconnect(sqlite3_vtab *pVtab){
+  sqlite3_free(pVtab);
+  return SQLITE_OK;
+}
+
+/*
+** xBestIndex method for pragma virtual tables.
+*/
+static int pragmaBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
+  int i;
+
+  pIdxInfo->estimatedCost = 1.0e6;  /* Initial cost estimate */
+
+  /* Look for a valid tbl=? constraint. */
+  for(i=0; i<pIdxInfo->nConstraint; i++){
+    if( pIdxInfo->aConstraint[i].usable==0 ) continue;
+    if( pIdxInfo->aConstraint[i].op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
+    if( pIdxInfo->aConstraint[i].iColumn!=0 ) continue;
+    pIdxInfo->idxNum = 1;
+    pIdxInfo->estimatedCost = 1.0;
+    pIdxInfo->aConstraintUsage[i].argvIndex = 1;
+    pIdxInfo->aConstraintUsage[i].omit = 1;
+    break;
+  }
+  if( i==pIdxInfo->nConstraint ){
+    tab->zErrMsg = sqlite3_mprintf("missing required tbl=? constraint");
+    return SQLITE_ERROR;
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Open a new pragma cursor.
+*/
+static int pragmaOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
+  PragmaTable *pTab = (PragmaTable *)pVTab;
+  PragmaCursor *pCsr;
+
+  pCsr = (PragmaCursor*)sqlite3_malloc64(sizeof(PragmaCursor));
+  if( pCsr==0 ){
+    return SQLITE_NOMEM;
+  }else{
+    memset(pCsr, 0, sizeof(PragmaCursor));
+    pCsr->base.pVtab = pVTab;
+  }
+
+  *ppCursor = (sqlite3_vtab_cursor*)pCsr;
+  return SQLITE_OK;
+}
+
+/*
+** Move a statvfs cursor to the next entry in the file.
+*/
+static int pragmaNext(sqlite3_vtab_cursor *pCursor){
+  PragmaCursor *pCsr = (PragmaCursor*)pCursor;
+  int rc = SQLITE_OK;
+
+  if( sqlite3_step(pCsr->pStmt)!=SQLITE_ROW ){
+    rc = sqlite3_finalize(pCsr->pStmt);
+    pCsr->pStmt = 0;
+  }
+  pCsr->iRowid++;
+  return rc;
+}
+
+static int pragmaEof(sqlite3_vtab_cursor *pCursor){
+  PragmaCursor *pCsr = (PragmaCursor*)pCursor;
+  return pCsr->pStmt==0;
+}
+
+static int pragmaFilter(
+  sqlite3_vtab_cursor *pCursor, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  PragmaCursor *pCsr = (PragmaCursor*)pCursor;
+  PragmaTable *pTab = (PragmaTable*)(pCursor->pVtab);
+  char *zSql;
+  const char *zTbl;
+  int rc = SQLITE_OK;
+
+  if( pCsr->pStmt ){
+    sqlite3_finalize(pCsr->pStmt);
+    pCsr->pStmt = 0;
+  }
+  pCsr->iRowid = 0;
+
+  assert( argc==1 );
+  zTbl = (const char*)sqlite3_value_text(argv[0]);
+  zSql = sqlite3_mprintf("PRAGMA table_info(%Q)", zTbl);
+  if( zSql==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0);
+  }
+  return pragmaNext(pCursor);;
+}
+
+/*
+** xColumn method.
+*/
+static int pragmaColumn(
+  sqlite3_vtab_cursor *pCursor, 
+  sqlite3_context *ctx, 
+  int iCol
+){
+  PragmaCursor *pCsr = (PragmaCursor *)pCursor;
+  if( iCol>0 ){
+    sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pStmt, iCol-1));
+  }
+  return SQLITE_OK;
+}
+
+static int pragmaRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
+  PragmaCursor *pCsr = (PragmaCursor *)pCursor;
+  *pRowid = pCsr->iRowid;
+  return SQLITE_OK;
+}
+
+static int registerPragmaVtabs(sqlite3 *db){
+  static sqlite3_module pragma_module = {
+    0,                            /* iVersion */
+    pragmaConnect,                /* xCreate */
+    pragmaConnect,                /* xConnect */
+    pragmaBestIndex,              /* xBestIndex */
+    pragmaDisconnect,             /* xDisconnect */
+    pragmaDisconnect,             /* xDestroy */
+    pragmaOpen,                   /* xOpen - open a cursor */
+    pragmaClose,                  /* xClose - close a cursor */
+    pragmaFilter,                 /* xFilter - configure scan constraints */
+    pragmaNext,                   /* xNext - advance a cursor */
+    pragmaEof,                    /* xEof - check for end of scan */
+    pragmaColumn,                 /* xColumn - read data */
+    pragmaRowid,                  /* xRowid - read data */
+    0,                            /* xUpdate */
+    0,                            /* xBegin */
+    0,                            /* xSync */
+    0,                            /* xCommit */
+    0,                            /* xRollback */
+    0,                            /* xFindMethod */
+    0,                            /* xRename */
+  };
+  return sqlite3_create_module(db, "pragma_table_info", &pragma_module, 0);
+}
+
+/*
+** Allocate and return nByte bytes of zeroed memory using sqlite3_malloc(). 
+** If the allocation fails, set *pRc to SQLITE_NOMEM and return NULL.
+*/
+static void *idxMalloc(int *pRc, int nByte){
+  void *pRet;
+  assert( *pRc==SQLITE_OK );
+  assert( nByte>0 );
+  pRet = sqlite3_malloc(nByte);
+  if( pRet ){
+    memset(pRet, 0, nByte);
+  }else{
+    *pRc = SQLITE_NOMEM;
+  }
+  return pRet;
+}
+
+/*
+** Allocate and return a new IdxConstraint object. Set the IdxConstraint.zColl
+** variable to point to a copy of nul-terminated string zColl.
+*/
+static IdxConstraint *idxNewConstraint(int *pRc, const char *zColl){
+  IdxConstraint *pNew;
+  int nColl = strlen(zColl);
+
+  assert( *pRc==SQLITE_OK );
+  pNew = (IdxConstraint*)idxMalloc(pRc, sizeof(IdxConstraint) * nColl + 1);
+  if( pNew ){
+    pNew->zColl = (char*)&pNew[1];
+    memcpy(pNew->zColl, zColl, nColl+1);
+  }
+  return pNew;
+}
+
+/*
+** SQLITE_DBCONFIG_WHEREINFO callback.
+*/
+static void idxWhereInfo(
+  void *pCtx,                     /* Pointer to IdxContext structure */
+  int eOp, 
+  const char *zVal, 
+  int iVal, 
+  i64 mask
+){
+  IdxContext *p = (IdxContext*)pCtx;
+
+#if 1
+  const char *zOp = 
+    eOp==SQLITE_WHEREINFO_TABLE ? "TABLE" :
+    eOp==SQLITE_WHEREINFO_EQUALS ? "EQUALS" :
+    eOp==SQLITE_WHEREINFO_RANGE ? "RANGE" :
+    eOp==SQLITE_WHEREINFO_ORDERBY ? "ORDERBY" :
+    eOp==SQLITE_WHEREINFO_NEXTOR ? "NEXTOR" :
+    eOp==SQLITE_WHEREINFO_ENDOR ? "ENDOR" :
+    eOp==SQLITE_WHEREINFO_BEGINOR ? "BEGINOR" :
+    "!error!";
+  printf("op=%s zVal=%s iVal=%d mask=%llx\n", zOp, zVal, iVal, mask);
+#endif
+
+  if( p->rc==SQLITE_OK ){
+    assert( eOp==SQLITE_WHEREINFO_TABLE || p->pScan!=0 );
+    switch( eOp ){
+      case SQLITE_WHEREINFO_TABLE: {
+        int nVal = strlen(zVal);
+        IdxScan *pNew = (IdxScan*)idxMalloc(&p->rc, sizeof(IdxScan) + nVal + 1);
+        if( !pNew ) return;
+        pNew->zTable = (char*)&pNew[1];
+        memcpy(pNew->zTable, zVal, nVal+1);
+        pNew->pNextScan = p->pScan;
+        pNew->covering = mask;
+        p->pScan = pNew;
+        p->pCurrent = &pNew->where;
+        break;
+      }
+
+      case SQLITE_WHEREINFO_ORDERBY: {
+        IdxConstraint *pNew = idxNewConstraint(&p->rc, zVal);
+        IdxConstraint **pp;
+        if( pNew==0 ) return;
+        pNew->iCol = iVal;
+        for(pp=&p->pScan->pOrder; *pp; pp=&(*pp)->pNext);
+        *pp = pNew;
+        break;
+      }
+
+      case SQLITE_WHEREINFO_EQUALS:
+      case SQLITE_WHEREINFO_RANGE: {
+        IdxConstraint *pNew = idxNewConstraint(&p->rc, zVal);
+        if( pNew==0 ) return;
+        pNew->iCol = iVal;
+        pNew->depmask = mask;
+
+        if( eOp==SQLITE_WHEREINFO_RANGE ){
+          pNew->pNext = p->pCurrent->pRange;
+          p->pCurrent->pRange = pNew;
+        }else{
+          pNew->pNext = p->pCurrent->pEq;
+          p->pCurrent->pEq = pNew;
+        }
+        break;
+      }
+
+      case SQLITE_WHEREINFO_BEGINOR: {
+        assert( 0 );
+        break;
+      }
+      case SQLITE_WHEREINFO_ENDOR: {
+        assert( 0 );
+        break;
+      }
+      case SQLITE_WHEREINFO_NEXTOR: {
+        assert( 0 );
+        break;
+      }
+    }
+  }
+}
+
+/*
+** An error associated with database handle db has just occurred. Pass
+** the error message to callback function xOut.
+*/
+static void idxDatabaseError(
+  sqlite3 *db,                    /* Database handle */
+  char **pzErrmsg                 /* Write error here */
+){
+  *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
+}
+
+static int idxCreateTables(sqlite3 *db, sqlite3 *dbm, IdxScan *pScan){
+  int rc = SQLITE_OK;
+  IdxScan *pIter;
+  for(pIter=pScan; pIter; pIter=pIter->pNextScan){
+  }
+}
+
+static void idxScanFree(IdxScan *pScan){
+}
+
+/*
+** The xOut callback is invoked to return command output to the user. The
+** second argument is always a nul-terminated string. The first argument is
+** passed zero if the string contains normal output or non-zero if it is an
+** error message.
+*/
+int shellIndexesCommand(
+  sqlite3 *db,                         /* Database handle */
+  const char *zSql,                    /* SQL to find indexes for */
+  void (*xOut)(void*, const char*),    /* Output callback */
+  void *pOutCtx,                       /* Context for xOut() */
+  char **pzErrmsg                      /* OUT: Error message (sqlite3_malloc) */
+){
+  int rc = SQLITE_OK;
+  sqlite3 *dbm = 0;
+  IdxContext ctx;
+  sqlite3_stmt *pStmt = 0;        /* Statement compiled from zSql */
+
+  memset(&ctx, 0, sizeof(IdxContext));
+
+  /* Open an in-memory database to work with. The main in-memory 
+  ** database schema contains tables similar to those in the users 
+  ** database (handle db). The attached in-memory db (aux) contains
+  ** application tables used by the code in this file.  */
+  rc = sqlite3_open(":memory:", &dbm);
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_exec(dbm, 
+        "ATTACH ':memory:' AS aux;"
+        "CREATE TABLE aux.depmask(mask PRIMARY KEY) WITHOUT ROWID;"
+        , 0, 0, 0
+    );
+  }
+  if( rc!=SQLITE_OK ){
+    idxDatabaseError(dbm, pzErrmsg);
+    goto indexes_out;
+  }
+
+  /* Analyze the SELECT statement in zSql. */
+  ctx.dbm = dbm;
+  sqlite3_db_config(db, SQLITE_DBCONFIG_WHEREINFO, idxWhereInfo, (void*)&ctx);
+  rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0);
+  sqlite3_db_config(db, SQLITE_DBCONFIG_WHEREINFO, (void*)0, (void*)0);
+  if( rc!=SQLITE_OK ){
+    idxDatabaseError(db, pzErrmsg);
+    goto indexes_out;
+  }
+
+  /* Create tables within the main in-memory database. These tables
+  ** have the same names, columns and declared types as the tables in
+  ** the user database. All constraints except for PRIMARY KEY are
+  ** removed. */
+  rc = idxCreateTables(db, dbm, ctx.pScan);
+  if( rc!=SQLITE_OK ){
+    goto indexes_out;
+  }
+
+  /* Create candidate indexes within the in-memory database file */
+
+ indexes_out:
+  idxScanFree(ctx.pScan);
+  sqlite3_close(dbm);
+  return rc;
+}
+
+
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index fce396c0f6..03d3d72f6f 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -1909,6 +1909,15 @@ struct sqlite3_mem_methods {
 #define SQLITE_DBCONFIG_LOOKASIDE       1001  /* void* int int */
 #define SQLITE_DBCONFIG_ENABLE_FKEY     1002  /* int int* */
 #define SQLITE_DBCONFIG_ENABLE_TRIGGER  1003  /* int int* */
+#define SQLITE_DBCONFIG_WHEREINFO       1004  /* xWhereInfo void* */
+
+#define SQLITE_WHEREINFO_TABLE   1
+#define SQLITE_WHEREINFO_EQUALS  2
+#define SQLITE_WHEREINFO_RANGE   3
+#define SQLITE_WHEREINFO_ORDERBY 4
+#define SQLITE_WHEREINFO_BEGINOR 5
+#define SQLITE_WHEREINFO_ENDOR   6
+#define SQLITE_WHEREINFO_NEXTOR  7
 
 
 /*
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 760c1f4d21..57c58195e0 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1274,6 +1274,10 @@ struct sqlite3 {
 #ifdef SQLITE_USER_AUTHENTICATION
   sqlite3_userauth auth;        /* User authentication information */
 #endif
+#ifdef SQLITE_SCHEMA_LINT
+  void (*xWhereInfo)(void*, int, const char*, int, i64);
+  void *pWhereInfoCtx;
+#endif
 };
 
 /*
diff --git a/src/where.c b/src/where.c
index af6bb20384..3724fa33e0 100644
--- a/src/where.c
+++ b/src/where.c
@@ -3906,176 +3906,103 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){
 }
 
 #ifdef SQLITE_SCHEMA_LINT
-static char *whereAppendPrintf(sqlite3 *db, const char *zFmt, ...){
-  va_list ap;
-  char *zRes = 0;
-  va_start(ap, zFmt);
-  zRes = sqlite3_vmprintf(zFmt, ap);
-  if( zRes==0 ){
-    db->mallocFailed = 1;
-  }else if( db->mallocFailed ){
-    sqlite3_free(zRes);
-    zRes = 0;
-  }
-  va_end(ap);
-  return zRes;
-}
-
-/*
-** Append a representation of term pTerm to the string in zIn and return
-** the result. Or, if an OOM occurs, free zIn and return a NULL pointer.
-*/
-static char *whereAppendSingleTerm(
-  Parse *pParse,
-  Table *pTab,
-  int iCol,
-  int bOr,
-  char *zIn,
-  WhereTerm *pTerm
-){
-  char *zBuf;
-  sqlite3 *db = pParse->db;
-  Expr *pX = pTerm->pExpr;
-  CollSeq *pColl;
-  const char *zOp = 0;
-
-  if( pTerm->eOperator & (WO_IS|WO_EQ|WO_IN) ){
-    zOp = "eq";
-  }else if( pTerm->eOperator & (WO_LT|WO_LE|WO_GE|WO_GT) ){
-    zOp = "range";
-  }
-  pColl = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight);
-
-  if( zOp ){
-    const char *zFmt = bOr ? "%z{{%s \"%w\" \"%w\" %lld}}" :
-                             "%z{%s \"%w\" \"%w\" %lld}";
-    zBuf = whereAppendPrintf(db, zFmt, zIn, 
-        zOp, pTab->aCol[iCol].zName, 
-        (pColl ? pColl->zName : "BINARY"),
-        pTerm->prereqRight
-    );
-  }else{
-    zBuf = zIn;
-  }
-
-  return zBuf;
-}
-
-static char *whereTraceWC(
+static void whereTraceWC(
   Parse *pParse, 
-  int bInitialSpace,
   struct SrcList_item *pItem,
-  char *zIn,
-  WhereClause *pWC
+  WhereClause *pWC,
+  int bOr
 ){
   sqlite3 *db = pParse->db;
   Table *pTab = pItem->pTab;
-  char *zBuf = zIn;
-  int iCol;
+  void (*x)(void*, int, const char*, int, i64) = db->xWhereInfo;
+  void *pCtx = db->pWhereInfoCtx;
+  int bFirst = 1;                 /* True until first callback is made */
   int ii;
-  int bFirst = !bInitialSpace;
-  int bOr = (pWC->op==TK_OR);
 
-  /* List of WO_SINGLE constraints */
-  for(iCol=0; iCol<pTab->nCol; iCol++){
+  /* Issue callbacks for WO_SINGLE constraints */
+  for(ii=0; ii<pTab->nCol; ii++){
     int opMask = WO_SINGLE; 
     WhereScan scan;
     WhereTerm *pTerm;
-    for(pTerm=whereScanInit(&scan, pWC, pItem->iCursor, iCol, opMask, 0);
+    for(pTerm=whereScanInit(&scan, pWC, pItem->iCursor, ii, opMask, 0);
         pTerm;
         pTerm=whereScanNext(&scan)
     ){
-      /* assert( iCol==pTerm->u.leftColumn ); */
-      if( bFirst==0 ) zBuf = whereAppendPrintf(db, "%z ", zBuf);
-      zBuf = whereAppendSingleTerm(pParse, pTab, iCol, bOr, zBuf, pTerm);
+      int eOp;
+      Expr *pX = pTerm->pExpr;
+      CollSeq *pC = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight);
+      if( pTerm->eOperator & (WO_IS|WO_EQ|WO_IN) ){
+        eOp = SQLITE_WHEREINFO_EQUALS;
+      }else{
+        eOp = SQLITE_WHEREINFO_RANGE;
+      }
+      if( bOr && !bFirst ) x(pCtx, SQLITE_WHEREINFO_NEXTOR, 0, 0, 0);
+      x(pCtx, eOp, (pC ? pC->zName : "BINARY"), ii, pTerm->prereqRight);
       bFirst = 0;
     }
   }
 
-  /* Add composite - (WO_OR|WO_AND) - constraints */
+  /* Callbacks for composite - (WO_OR|WO_AND) - constraints */
   for(ii=0; ii<pWC->nTerm; ii++){
     WhereTerm *pTerm = &pWC->a[ii];
-    if( pTerm->eOperator & (WO_OR|WO_AND) ){
-      const char *zFmt = ((pTerm->eOperator&WO_OR) ? "%z%s{or " : "%z%s{");
-      zBuf = whereAppendPrintf(db, zFmt, zBuf, bFirst ? "" : " ");
-      zBuf = whereTraceWC(pParse, 0, pItem, zBuf, &pTerm->u.pOrInfo->wc);
-      zBuf = whereAppendPrintf(db, "%z}", zBuf);
+    if( pTerm->eOperator & WO_OR ){
+      assert( bOr==0 );
+      x(pCtx, SQLITE_WHEREINFO_BEGINOR, 0, 0, 0);
+      whereTraceWC(pParse, pItem, &pTerm->u.pOrInfo->wc, 1);
+      x(pCtx, SQLITE_WHEREINFO_ENDOR, 0, 0, 0);
+    }
+    if( pTerm->eOperator & WO_AND ){
+      if( bOr && !bFirst ) x(pCtx, SQLITE_WHEREINFO_NEXTOR, 0, 0, 0);
+      whereTraceWC(pParse, pItem, &pTerm->u.pAndInfo->wc, 0);
       bFirst = 0;
     }
   }
-
-  return zBuf;
 }
 
+
 static void whereTraceBuilder(
   Parse *pParse,
   WhereLoopBuilder *p
 ){
   sqlite3 *db = pParse->db;
-  if( db->xTrace ){
-    ExprList *pOrderBy = p->pOrderBy;
-    WhereInfo *pWInfo = p->pWInfo;
-    int nTablist = pWInfo->pTabList->nSrc;
+  if( db->xWhereInfo && db->init.busy==0 ){
+    void (*x)(void*, int, const char*, int, i64) = db->xWhereInfo;
+    void *pCtx = db->pWhereInfoCtx;
     int ii;
+    int nTab = p->pWInfo->pTabList->nSrc;
 
     /* Loop through each element of the FROM clause. Ignore any sub-selects
-    ** or views. Invoke the xTrace() callback once for each real table. */
-    for(ii=0; ii<nTablist; ii++){
-      char *zBuf = 0;
-      int iCol;
-      int nCol;
-      Table *pTab;
+    ** or views. Invoke the xWhereInfo() callback multiple times for each
+    ** real table.  */
+    for(ii=0; ii<p->pWInfo->pTabList->nSrc; ii++){
+      struct SrcList_item *pItem = &p->pWInfo->pTabList->a[ii];
+      if( pItem->pSelect==0 ){
+        Table *pTab = pItem->pTab;
+        int iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
 
-      struct SrcList_item *pItem = &pWInfo->pTabList->a[ii];
-      if( pItem->pSelect ) continue;
-      pTab = pItem->pTab;
-      nCol = pTab->nCol;
+        /* Table name callback */
+        x(pCtx, SQLITE_WHEREINFO_TABLE, pTab->zName, iDb, pItem->colUsed);
 
-      /* Append the table name to the buffer. */
-      zBuf = whereAppendPrintf(db, "\"%w\"", pTab->zName);
-
-      /* Append the list of columns required to create a covering index */
-      zBuf = whereAppendPrintf(db, "%z {cols", zBuf);
-      if( 0==(pItem->colUsed & ((u64)1 << (sizeof(Bitmask)*8-1))) ){
-        for(iCol=0; iCol<nCol; iCol++){
-          if( iCol==(sizeof(Bitmask)*8-1) ) break;
-          if( pItem->colUsed & ((u64)1 << iCol) ){
-            const char *zName = pTab->aCol[iCol].zName;
-            zBuf = whereAppendPrintf(db, "%z \"%w\"", zBuf, zName);
-          }
-        }
-      }
-      zBuf = whereAppendPrintf(db, "%z}",zBuf);
-
-      /* Append the contents of WHERE clause */
-      zBuf = whereTraceWC(pParse, 1, pItem, zBuf, p->pWC);
-
-      /* Append the ORDER BY clause, if any */
-      if( pOrderBy ){
-        int i;
-        int bFirst = 1;
-        for(i=0; i<pOrderBy->nExpr; i++){
-          Expr *pExpr = pOrderBy->a[i].pExpr; 
-          CollSeq *pColl = sqlite3ExprCollSeq(pParse, pExpr);
-
-          pExpr = sqlite3ExprSkipCollate(pExpr);
-          if( pExpr->op==TK_COLUMN && pExpr->iTable==pItem->iCursor ){
-            if( pExpr->iColumn>=0 ){
-              const char *zName = pTab->aCol[pExpr->iColumn].zName;
-              zBuf = whereAppendPrintf(db, "%z%s\"%w\" \"%w\" %s", zBuf,
-                  bFirst ? " {orderby " : " ", zName, pColl->zName,
-                  (pOrderBy->a[i].sortOrder ? "DESC" : "ASC")
-              );
-              bFirst = 0;
+        /* ORDER BY callbacks */
+        if( p->pOrderBy ){
+          int i;
+          int bFirst = 1;
+          for(i=0; i<p->pOrderBy->nExpr; i++){
+            Expr *pExpr = p->pOrderBy->a[i].pExpr; 
+            CollSeq *pColl = sqlite3ExprCollSeq(pParse, pExpr);
+            pExpr = sqlite3ExprSkipCollate(pExpr);
+            if( pExpr->op==TK_COLUMN && pExpr->iTable==pItem->iCursor ){
+              int iCol = pExpr->iColumn;
+              if( iCol>=0 ){
+                x(pCtx, SQLITE_WHEREINFO_ORDERBY, pColl->zName, iCol, 0); 
+              }
             }
           }
         }
-        if( bFirst==0 ) zBuf = whereAppendPrintf(db, "%z}", zBuf);
-      }
 
-      /* Pass the buffer to the xTrace() callback, then free it */
-      db->xTrace(db->pTraceArg, zBuf);
-      sqlite3DbFree(db, zBuf);
+        /* WHERE callbacks */
+        whereTraceWC(pParse, pItem, p->pWC, 0);
+      }
     }
   }
 }

From 47dd4867bc22c57668cc9fd11e49dbd1d70ccf55 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 15 Feb 2016 20:12:16 +0000
Subject: [PATCH 10/68] Progress towards integrating schemalint into the shell
 tool. Some cases work now.

FossilOrigin-Name: 58d4cf26e15f90463148bec63d6ab514ffbbae60
---
 manifest            |  12 +-
 manifest.uuid       |   2 +-
 src/shell_indexes.c | 505 ++++++++++++++++++++++++++++++++++++++++++--
 3 files changed, 496 insertions(+), 23 deletions(-)

diff --git a/manifest b/manifest
index 7da4cdeb1d..c7c4b5f60c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Experimental\sintegration\sof\sschemalint\sfunctionality\swith\sthe\sshell\stool.\sDoes\snot\swork\syet.
-D 2016-02-11T21:01:16.253
+C Progress\stowards\sintegrating\sschemalint\sinto\sthe\sshell\stool.\sSome\scases\swork\snow.
+D 2016-02-15T20:12:16.368
 F Makefile.in dac2776c84e0d533b158a9af6e57e05c4a6b19f3
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc b0493f10caddb8adf992a4e6f1943141fc7c6816
@@ -350,7 +350,7 @@ F src/resolve.c 9f7ce3a3c087afb7597b7c916c99126ff3f12f0c
 F src/rowset.c 9fe4b3ad7cc00944386bb600233d8f523de07a6e
 F src/select.c ff80004a9a6ece891a8d9327a88e7b6e2588ee6d
 F src/shell.c 2cde87e03712204231167c4a6c61b0eb5129e105
-F src/shell_indexes.c 3cff393ee86d15fbfbe31f30cd752b46d7779b52
+F src/shell_indexes.c eefa3e9b7d992d10d1f71e27a2e12024a838b365
 F src/sqlite.h.in c7db059d3b810b70b83d9ed1436fa813eba22462
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h dfbe62ffd95b99afe2140d8c35b180d11924072d
@@ -1430,7 +1430,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 1a4182eedd0143c3f71b3d97f1d1bb25adeba617
-R 6253a80a5a7097c3b24b24ce68900613
+P ed49f297bcee86674ed673e195610b8cc1d35647
+R 6a53f87d207675e944487abae6392564
 U dan
-Z bddf3d3a7cd183a6a2362ed54e10358b
+Z dd362fd8eca18359c7fdd88fbcc31963
diff --git a/manifest.uuid b/manifest.uuid
index 38f9cc7372..9d51efbd2d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ed49f297bcee86674ed673e195610b8cc1d35647
\ No newline at end of file
+58d4cf26e15f90463148bec63d6ab514ffbbae60
\ No newline at end of file
diff --git a/src/shell_indexes.c b/src/shell_indexes.c
index 8e105fec1b..b7b44e47e0 100644
--- a/src/shell_indexes.c
+++ b/src/shell_indexes.c
@@ -18,6 +18,9 @@ typedef struct IdxContext IdxContext;
 typedef struct IdxScan IdxScan;
 typedef struct IdxWhere IdxWhere;
 
+typedef struct IdxColumn IdxColumn;
+typedef struct IdxTable IdxTable;
+
 /*
 ** A single constraint. Equivalent to either "col = ?" or "col < ?".
 **
@@ -34,16 +37,22 @@ struct IdxConstraint {
 };
 
 /*
-** A WHERE clause. Made up of IdxConstraint objects.
+** A WHERE clause. Made up of IdxConstraint objects. Example WHERE clause:
 **
 **   a=? AND b=? AND (c=? OR d=?) AND (e=? OR f=?)
 **
+** The above
+**
+**
+**
 */
 struct IdxWhere {
   IdxConstraint *pEq;             /* List of == constraints */
   IdxConstraint *pRange;          /* List of < constraints */
-  IdxWhere **apOr;                /* Array of OR branches (joined by pNextOr) */
-  IdxWhere *pNextOr;              /* Next in OR'd terms */
+  IdxWhere *pOr;                  /* List of OR constraints */
+
+  IdxWhere *pNextOr;              /* Next in OR constraints of same IdxWhere */
+  IdxWhere *pSibling;             /* Next branch in single OR constraint */
   IdxWhere *pParent;              /* Parent object (or NULL) */
 };
 
@@ -51,6 +60,7 @@ struct IdxWhere {
 ** A single scan of a single table.
 */
 struct IdxScan {
+  IdxTable *pTable;               /* Table-info */
   char *zTable;                   /* Name of table to scan */
   int iDb;                        /* Database containing table zTable */
   i64 covering;                   /* Mask of columns required for cov. index */
@@ -64,11 +74,27 @@ struct IdxScan {
 */
 struct IdxContext {
   IdxWhere *pCurrent;             /* Current where clause */
+  int rc;                         /* Error code (if error has occurred) */
   IdxScan *pScan;                 /* List of scan objects */
   sqlite3 *dbm;                   /* In-memory db for this analysis */
-  int rc;                         /* Error code (if error has occurred) */
+  sqlite3 *db;                    /* User database under analysis */
+  sqlite3_stmt *pInsertMask;      /* To write to aux.depmask */
 };
 
+/*
+** Data regarding a database table. Extracted from "PRAGMA table_info"
+*/
+struct IdxColumn {
+  char *zName;
+  char *zColl;
+  int iPk;
+};
+struct IdxTable {
+  int nCol;
+  IdxColumn *aCol;
+};
+
+
 typedef struct PragmaTable PragmaTable;
 typedef struct PragmaCursor PragmaCursor;
 
@@ -94,7 +120,7 @@ static int pragmaConnect(
   char **pzErr
 ){
   const char *zSchema = 
-    "CREATE TABLE a(tbl HIDDEN, cid, name, type, notnull, dflt_value, pk)";
+    "CREATE TABLE a(tbl HIDDEN, cid, name, type, isnotnull, dflt_value, pk)";
   PragmaTable *pTab = 0;
   int rc = SQLITE_OK;
 
@@ -102,6 +128,8 @@ static int pragmaConnect(
   if( rc==SQLITE_OK ){
     pTab = (PragmaTable *)sqlite3_malloc64(sizeof(PragmaTable));
     if( pTab==0 ) rc = SQLITE_NOMEM;
+  }else{
+    *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db));
   }
 
   assert( rc==SQLITE_OK || pTab==0 );
@@ -152,7 +180,6 @@ static int pragmaBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
 ** Open a new pragma cursor.
 */
 static int pragmaOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
-  PragmaTable *pTab = (PragmaTable *)pVTab;
   PragmaCursor *pCsr;
 
   pCsr = (PragmaCursor*)sqlite3_malloc64(sizeof(PragmaCursor));
@@ -167,6 +194,13 @@ static int pragmaOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
   return SQLITE_OK;
 }
 
+static int pragmaClose(sqlite3_vtab_cursor *pCursor){
+  PragmaCursor *pCsr = (PragmaCursor*)pCursor;
+  sqlite3_finalize(pCsr->pStmt);
+  sqlite3_free(pCsr);
+  return SQLITE_OK;
+}
+
 /*
 ** Move a statvfs cursor to the next entry in the file.
 */
@@ -212,6 +246,7 @@ static int pragmaFilter(
   }else{
     rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0);
   }
+  if( rc ) return rc;
   return pragmaNext(pCursor);;
 }
 
@@ -308,7 +343,7 @@ static void idxWhereInfo(
 ){
   IdxContext *p = (IdxContext*)pCtx;
 
-#if 1
+#if 0
   const char *zOp = 
     eOp==SQLITE_WHEREINFO_TABLE ? "TABLE" :
     eOp==SQLITE_WHEREINFO_EQUALS ? "EQUALS" :
@@ -361,19 +396,36 @@ static void idxWhereInfo(
           pNew->pNext = p->pCurrent->pEq;
           p->pCurrent->pEq = pNew;
         }
+
+        sqlite3_bind_int64(p->pInsertMask, 1, mask);
+        sqlite3_step(p->pInsertMask);
+        p->rc = sqlite3_reset(p->pInsertMask);
         break;
       }
 
       case SQLITE_WHEREINFO_BEGINOR: {
-        assert( 0 );
-        break;
-      }
-      case SQLITE_WHEREINFO_ENDOR: {
-        assert( 0 );
+        IdxWhere *pNew = (IdxWhere*)idxMalloc(&p->rc, sizeof(IdxWhere));
+        if( pNew==0 ) return;
+        pNew->pParent = p->pCurrent;
+        pNew->pNextOr = p->pCurrent->pOr;
+        p->pCurrent->pOr = pNew;
+        p->pCurrent = pNew;
         break;
       }
+
       case SQLITE_WHEREINFO_NEXTOR: {
-        assert( 0 );
+        IdxWhere *pNew = (IdxWhere*)idxMalloc(&p->rc, sizeof(IdxWhere));
+        if( pNew==0 ) return;
+        pNew->pParent = p->pCurrent->pParent;
+        assert( p->pCurrent->pSibling==0 );
+        p->pCurrent->pSibling = pNew;
+        p->pCurrent = pNew;
+        break;
+      }
+
+      case SQLITE_WHEREINFO_ENDOR: {
+        assert( p->pCurrent->pParent );
+        p->pCurrent = p->pCurrent->pParent;
         break;
       }
     }
@@ -391,16 +443,418 @@ static void idxDatabaseError(
   *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
 }
 
-static int idxCreateTables(sqlite3 *db, sqlite3 *dbm, IdxScan *pScan){
+static char *idxQueryToList(
+  sqlite3 *db, 
+  const char *zBind,
+  int *pRc,
+  char **pzErrmsg,
+  const char *zSql
+){
+  char *zRet = 0;
+  if( *pRc==SQLITE_OK ){
+    sqlite3_stmt *pStmt = 0;
+    int rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+    if( rc==SQLITE_OK ){
+      sqlite3_bind_text(pStmt, 1, zBind, -1, SQLITE_TRANSIENT);
+      while( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){
+        const char *z = (const char*)sqlite3_column_text(pStmt, 0);
+        zRet = sqlite3_mprintf("%z%s%Q", zRet, zRet?", ":"", z);
+        if( zRet==0 ){
+          rc = SQLITE_NOMEM;
+        }
+      }
+      rc = sqlite3_finalize(pStmt);
+    }
+
+    if( rc ){
+      idxDatabaseError(db, pzErrmsg);
+      sqlite3_free(zRet);
+      zRet = 0;
+    }
+    *pRc = rc;
+  }
+
+  return zRet;
+}
+
+static int idxGetTableInfo(
+  sqlite3 *db,
+  IdxScan *pScan,
+  char **pzErrmsg
+){
+  const char *zSql = "SELECT name, pk FROM pragma_table_info(?)";
+  sqlite3_stmt *p1 = 0;
+  int nCol = 0;
+  int nByte = sizeof(IdxTable);
+  IdxTable *pNew = 0;
+  int rc, rc2;
+  char *pCsr;
+
+  rc = sqlite3_prepare_v2(db, zSql, -1, &p1, 0);
+  if( rc!=SQLITE_OK ){
+    idxDatabaseError(db, pzErrmsg);
+    return rc;
+  }
+  sqlite3_bind_text(p1, 1, pScan->zTable, -1, SQLITE_TRANSIENT);
+  while( SQLITE_ROW==sqlite3_step(p1) ){
+    const char *zCol = sqlite3_column_text(p1, 0);
+    nByte += 1 + strlen(zCol);
+    rc = sqlite3_table_column_metadata(
+        db, "main", pScan->zTable, zCol, 0, &zCol, 0, 0, 0
+    );
+    nByte += 1 + strlen(zCol);
+    nCol++;
+  }
+  rc2 = sqlite3_reset(p1);
+  if( rc==SQLITE_OK ) rc = rc2;
+
+  nByte += sizeof(IdxColumn) * nCol;
+  if( rc==SQLITE_OK ){
+    pNew = idxMalloc(&rc, nByte);
+  }
+  if( rc==SQLITE_OK ){
+    pNew->aCol = (IdxColumn*)&pNew[1];
+    pNew->nCol = nCol;
+    pCsr = (char*)&pNew->aCol[nCol];
+  }
+
+  nCol = 0;
+  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){
+    const char *zCol = sqlite3_column_text(p1, 0);
+    int nCopy = strlen(zCol) + 1;
+    pNew->aCol[nCol].zName = pCsr;
+    pNew->aCol[nCol].iPk = sqlite3_column_int(p1, 1);
+    memcpy(pCsr, zCol, nCopy);
+    pCsr += nCopy;
+
+    rc = sqlite3_table_column_metadata(
+        db, "main", pScan->zTable, zCol, 0, &zCol, 0, 0, 0
+    );
+    if( rc==SQLITE_OK ){
+      nCopy = strlen(zCol) + 1;
+      pNew->aCol[nCol].zColl = pCsr;
+      memcpy(pCsr, zCol, nCopy);
+      pCsr += nCopy;
+    }
+
+    nCol++;
+  }
+  rc2 = sqlite3_finalize(p1);
+  if( rc==SQLITE_OK ) rc = rc2;
+
+  if( rc==SQLITE_OK ){
+    pScan->pTable = pNew;
+  }else{
+    sqlite3_free(pNew);
+  }
+
+  return rc;
+}
+
+
+static int idxCreateTables(
+  sqlite3 *db,                    /* User database */
+  sqlite3 *dbm,                   /* In-memory database to create tables in */
+  IdxScan *pScan,                 /* List of scans */
+  char **pzErrmsg                 /* OUT: Error message */
+){
   int rc = SQLITE_OK;
   IdxScan *pIter;
-  for(pIter=pScan; pIter; pIter=pIter->pNextScan){
+  for(pIter=pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
+    int nPk = 0;
+    char *zCols = 0;
+    char *zPk = 0;
+    char *zCreate = 0;
+    int iCol;
+
+    rc = idxGetTableInfo(db, pIter, pzErrmsg);
+
+    for(iCol=0; rc==SQLITE_OK && iCol<pIter->pTable->nCol; iCol++){
+      IdxColumn *pCol = &pIter->pTable->aCol[iCol];
+      if( pCol->iPk>nPk ) nPk = pCol->iPk;
+      zCols = sqlite3_mprintf("%z%s%Q", zCols, (zCols?", ":""), pCol->zName);
+      if( zCols==0 ) rc = SQLITE_NOMEM;
+    }
+
+    for(iCol=1; rc==SQLITE_OK && iCol<=nPk; iCol++){
+      int j;
+      for(j=0; j<pIter->pTable->nCol; j++){
+        IdxColumn *pCol = &pIter->pTable->aCol[j];
+        if( pCol->iPk==iCol ){
+          zPk = sqlite3_mprintf("%z%s%Q", zPk, (zPk?", ":""), pCol->zName);
+          if( zPk==0 ) rc = SQLITE_NOMEM;
+          break;
+        }
+      }
+    }
+
+    if( rc==SQLITE_OK ){
+      if( zPk ){
+        zCreate = sqlite3_mprintf("CREATE TABLE %Q(%s, PRIMARY KEY(%s))",
+            pIter->zTable, zCols, zPk
+        );
+      }else{
+        zCreate = sqlite3_mprintf("CREATE TABLE %Q(%s)", pIter->zTable, zCols);
+      }
+      if( zCreate==0 ) rc = SQLITE_NOMEM;
+    }
+
+    if( rc==SQLITE_OK ){
+#if 1
+      printf("/* %s */\n", zCreate);
+#endif
+      rc = sqlite3_exec(dbm, zCreate, 0, 0, pzErrmsg);
+    }
+    sqlite3_free(zCols);
+    sqlite3_free(zPk);
+    sqlite3_free(zCreate);
   }
+  return rc;
+}
+
+/*
+** This function is a no-op if *pRc is set to anything other than 
+** SQLITE_OK when it is called.
+**
+** If *pRc is initially set to SQLITE_OK, then the text specified by
+** the printf() style arguments is appended to zIn and the result returned
+** in a buffer allocated by sqlite3_malloc(). sqlite3_free() is called on
+** zIn before returning.
+*/
+static char *idxAppendText(int *pRc, char *zIn, const char *zFmt, ...){
+  va_list ap;
+  char *zAppend = 0;
+  char *zRet = 0;
+  int nIn = zIn ? strlen(zIn) : 0;
+  int nAppend = 0;
+  va_start(ap, zFmt);
+  if( *pRc==SQLITE_OK ){
+    zAppend = sqlite3_vmprintf(zFmt, ap);
+    if( zAppend ){
+      nAppend = strlen(zAppend);
+      zRet = (char*)sqlite3_malloc(nIn + nAppend);
+    }
+    if( zAppend && zRet ){
+      memcpy(zRet, zIn, nIn);
+      memcpy(&zRet[nIn], zAppend, nAppend+1);
+    }else{
+      sqlite3_free(zRet);
+      zRet = 0;
+      *pRc = SQLITE_NOMEM;
+    }
+    sqlite3_free(zAppend);
+    sqlite3_free(zIn);
+  }
+  va_end(ap);
+  return zRet;
+}
+
+static char *idxAppendColDefn(
+  int *pRc, 
+  char *zIn, 
+  IdxTable *pTab, 
+  IdxConstraint *pCons
+){
+  char *zRet = zIn;
+  IdxColumn *p = &pTab->aCol[pCons->iCol];
+  if( zRet ) zRet = idxAppendText(pRc, zRet, ", ");
+  zRet = idxAppendText(pRc, zRet, "%Q", p->zName);
+  if( sqlite3_stricmp(p->zColl, pCons->zColl) ){
+    zRet = idxAppendText(pRc, zRet, " COLLATE %Q", pCons->zColl);
+  }
+  return zRet;
+}
+
+static int idxCreateFromCons(
+  sqlite3 *dbm,
+  IdxScan *pScan,
+  IdxConstraint *pEq, 
+  IdxConstraint *pTail
+){
+  int rc = SQLITE_OK;
+  if( pEq || pTail ){
+    IdxTable *pTab = pScan->pTable;
+    char *zCols = 0;
+    char *zIdx = 0;
+    IdxConstraint *pCons;
+    int h = 0;
+
+    for(pCons=pEq; pCons; pCons=pCons->pLink){
+      zCols = idxAppendColDefn(&rc, zCols, pTab, pCons);
+    }
+    for(pCons=pTail; pCons; pCons=pCons->pLink){
+      zCols = idxAppendColDefn(&rc, zCols, pTab, pCons);
+    }
+
+    /* Hash the list of columns to come up with a name for the index */
+    if( rc==SQLITE_OK ){
+      int i;
+      for(i=0; zCols[i]; i++){
+        h += ((h<<3) + zCols[i]);
+      }
+
+      zIdx = sqlite3_mprintf("CREATE INDEX IF NOT EXISTS "
+          "'%q_idx_%08x' ON %Q(%s)", pScan->zTable, h, pScan->zTable, zCols
+      );
+      if( !zIdx ){
+        rc = SQLITE_NOMEM;
+      }else{
+        rc = sqlite3_exec(dbm, zIdx, 0, 0, 0);
+        printf("/* %s */\n", zIdx);
+      }
+    }
+
+    sqlite3_free(zIdx);
+    sqlite3_free(zCols);
+  }
+  return rc;
+}
+
+static int idxCreateFromWhere(
+    sqlite3*, i64, IdxScan*, IdxWhere*, IdxConstraint*, IdxConstraint*
+);
+
+static int idxCreateForeachOr(
+  sqlite3 *dbm, 
+  i64 mask,                       /* Consider only these constraints */
+  IdxScan *pScan,                 /* Create indexes for this scan */
+  IdxWhere *pWhere,               /* Read constraints from here */
+  IdxConstraint *pEq,             /* == constraints for inclusion */
+  IdxConstraint *pTail            /* range/ORDER BY constraints for inclusion */
+){
+  int rc = SQLITE_OK;
+  IdxWhere *p1;
+  IdxWhere *p2;
+  for(p1=pWhere->pOr; p1 && rc==SQLITE_OK; p1=p1->pNextOr){
+    rc = idxCreateFromWhere(dbm, mask, pScan, p1, pEq, pTail);
+    for(p2=p1->pSibling; p2 && rc==SQLITE_OK; p2=p2->pSibling){
+      rc = idxCreateFromWhere(dbm, mask, pScan, p2, pEq, pTail);
+    }
+  }
+  return rc;
+}
+
+static int idxCreateFromWhere(
+  sqlite3 *dbm, 
+  i64 mask,                       /* Consider only these constraints */
+  IdxScan *pScan,                 /* Create indexes for this scan */
+  IdxWhere *pWhere,               /* Read constraints from here */
+  IdxConstraint *pEq,             /* == constraints for inclusion */
+  IdxConstraint *pTail            /* range/ORDER BY constraints for inclusion */
+){
+  IdxConstraint *p1 = pEq;
+  IdxConstraint *pCon;
+  int rc;
+
+  /* Gather up all the == constraints that match the mask. */
+  for(pCon=pWhere->pEq; pCon; pCon=pCon->pNext){
+    if( (mask & pCon->depmask)==pCon->depmask ){
+      pCon->pLink = p1;
+      p1 = pCon;
+    }
+  }
+
+  /* Create an index using the == constraints collected above. And the
+  ** range constraint/ORDER BY terms passed in by the caller, if any. */
+  rc = idxCreateFromCons(dbm, pScan, p1, pTail);
+  if( rc==SQLITE_OK ){
+    rc = idxCreateForeachOr(dbm, mask, pScan, pWhere, p1, pTail);
+  }
+
+  /* If no range/ORDER BY passed by the caller, create a version of the
+  ** index for each range constraint that matches the mask. */
+  if( pTail==0 ){
+    for(pCon=pWhere->pRange; rc==SQLITE_OK && pCon; pCon=pCon->pNext){
+      assert( pCon->pLink==0 );
+      if( (mask & pCon->depmask)==pCon->depmask ){
+        rc = idxCreateFromCons(dbm, pScan, p1, pCon);
+        if( rc==SQLITE_OK ){
+          rc = idxCreateForeachOr(dbm, mask, pScan, pWhere, p1, pCon);
+        }
+      }
+    }
+  }
+
+  return rc;
+}
+
+static int idxPrepareStmt(
+  sqlite3 *db,                    /* Database handle to compile against */
+  const char *zSql,               /* SQL statement to compile */
+  sqlite3_stmt **ppStmt,          /* OUT: Compiled SQL statement */
+  char **pzErrmsg                 /* OUT: sqlite3_malloc()ed error message */
+){
+  int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0);
+  if( rc!=SQLITE_OK ){
+    *ppStmt = 0;
+    idxDatabaseError(db, pzErrmsg);
+  }
+  return rc;
+}
+
+/*
+** Create candidate indexes in database [dbm] based on the data in 
+** linked-list pScan.
+*/
+static int idxCreateCandidates(
+  sqlite3 *dbm,
+  IdxScan *pScan,
+  char **pzErrmsg
+){
+  int rc2;
+  int rc = SQLITE_OK;
+  sqlite3_stmt *pDepmask;         /* Foreach depmask */
+  IdxScan *pIter;
+
+  rc = idxPrepareStmt(dbm, "SELECT mask FROM depmask", &pDepmask, pzErrmsg);
+
+  for(pIter=pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
+    IdxWhere *pWhere = &pIter->where;
+    while( SQLITE_ROW==sqlite3_step(pDepmask) && rc==SQLITE_OK ){
+      i64 mask = sqlite3_column_int64(pDepmask, 0);
+      rc = idxCreateFromWhere(dbm, mask, pIter, pWhere, 0, 0);
+      if( rc==SQLITE_OK && pIter->pOrder ){
+        rc = idxCreateFromWhere(dbm, mask, pIter, pWhere, 0, pIter->pOrder);
+      }
+    }
+  }
+
+  rc2 = sqlite3_finalize(pDepmask);
+  if( rc==SQLITE_OK ) rc = rc2;
+  return rc;
 }
 
 static void idxScanFree(IdxScan *pScan){
 }
 
+int idxFindIndexes(
+  sqlite3 *dbm,                        /* Database handle */
+  const char *zSql,                    /* SQL to find indexes for */
+  void (*xOut)(void*, const char*),    /* Output callback */
+  void *pOutCtx,                       /* Context for xOut() */
+  char **pzErrmsg                      /* OUT: Error message (sqlite3_malloc) */
+){
+  char *zExplain;
+  sqlite3_stmt *pExplain;
+  int rc;
+
+  zExplain = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", zSql);
+  if( zExplain==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    rc = idxPrepareStmt(dbm, zExplain, &pExplain, pzErrmsg);
+    sqlite3_free(zExplain);
+  }
+  if( rc!=SQLITE_OK ) return rc;
+
+  while( sqlite3_step(pExplain)==SQLITE_ROW ){
+    int iCol;
+    // for(iCol=0; iCol<sqlite3_column_count(pExplain); iCol++){ }
+    xOut(pOutCtx, sqlite3_column_text(pExplain, 3));
+  }
+  rc = sqlite3_finalize(pExplain);
+}
+
 /*
 ** The xOut callback is invoked to return command output to the user. The
 ** second argument is always a nul-terminated string. The first argument is
@@ -419,6 +873,8 @@ int shellIndexesCommand(
   IdxContext ctx;
   sqlite3_stmt *pStmt = 0;        /* Statement compiled from zSql */
 
+  rc = registerPragmaVtabs(db);
+  if( rc ) return rc;
   memset(&ctx, 0, sizeof(IdxContext));
 
   /* Open an in-memory database to work with. The main in-memory 
@@ -430,9 +886,19 @@ int shellIndexesCommand(
     rc = sqlite3_exec(dbm, 
         "ATTACH ':memory:' AS aux;"
         "CREATE TABLE aux.depmask(mask PRIMARY KEY) WITHOUT ROWID;"
+        "INSERT INTO aux.depmask VALUES(0);"
         , 0, 0, 0
     );
   }
+
+  /* Prepare an INSERT statement for writing to aux.depmask */
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_prepare_v2(dbm, 
+        "INSERT OR IGNORE INTO depmask SELECT mask | ?1 FROM depmask;", -1,
+        &ctx.pInsertMask, 0
+    );
+  }
+
   if( rc!=SQLITE_OK ){
     idxDatabaseError(dbm, pzErrmsg);
     goto indexes_out;
@@ -443,6 +909,7 @@ int shellIndexesCommand(
   sqlite3_db_config(db, SQLITE_DBCONFIG_WHEREINFO, idxWhereInfo, (void*)&ctx);
   rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0);
   sqlite3_db_config(db, SQLITE_DBCONFIG_WHEREINFO, (void*)0, (void*)0);
+
   if( rc!=SQLITE_OK ){
     idxDatabaseError(db, pzErrmsg);
     goto indexes_out;
@@ -452,12 +919,18 @@ int shellIndexesCommand(
   ** have the same names, columns and declared types as the tables in
   ** the user database. All constraints except for PRIMARY KEY are
   ** removed. */
-  rc = idxCreateTables(db, dbm, ctx.pScan);
+  rc = idxCreateTables(db, dbm, ctx.pScan, pzErrmsg);
   if( rc!=SQLITE_OK ){
     goto indexes_out;
   }
 
   /* Create candidate indexes within the in-memory database file */
+  rc = idxCreateCandidates(dbm, ctx.pScan, pzErrmsg);
+  if( rc!=SQLITE_OK ){
+    goto indexes_out;
+  }
+
+  rc = idxFindIndexes(dbm, zSql, xOut, pOutCtx, pzErrmsg);
 
  indexes_out:
   idxScanFree(ctx.pScan);

From 94975e27a3b56d6efaa1576cae33e89be045ea59 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 16 Feb 2016 18:37:37 +0000
Subject: [PATCH 11/68] Fix further issues in schemalint.

FossilOrigin-Name: 73a7f010937828c5195a198604f976e8458cef73
---
 manifest            |  12 +-
 manifest.uuid       |   2 +-
 src/shell_indexes.c | 426 +++++++++++++++-----------------------------
 3 files changed, 154 insertions(+), 286 deletions(-)

diff --git a/manifest b/manifest
index c7c4b5f60c..f1eaa8735a 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Progress\stowards\sintegrating\sschemalint\sinto\sthe\sshell\stool.\sSome\scases\swork\snow.
-D 2016-02-15T20:12:16.368
+C Fix\sfurther\sissues\sin\sschemalint.
+D 2016-02-16T18:37:37.844
 F Makefile.in dac2776c84e0d533b158a9af6e57e05c4a6b19f3
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc b0493f10caddb8adf992a4e6f1943141fc7c6816
@@ -350,7 +350,7 @@ F src/resolve.c 9f7ce3a3c087afb7597b7c916c99126ff3f12f0c
 F src/rowset.c 9fe4b3ad7cc00944386bb600233d8f523de07a6e
 F src/select.c ff80004a9a6ece891a8d9327a88e7b6e2588ee6d
 F src/shell.c 2cde87e03712204231167c4a6c61b0eb5129e105
-F src/shell_indexes.c eefa3e9b7d992d10d1f71e27a2e12024a838b365
+F src/shell_indexes.c 27d3b064078066c9a284a93149c4a151821f55bf
 F src/sqlite.h.in c7db059d3b810b70b83d9ed1436fa813eba22462
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h dfbe62ffd95b99afe2140d8c35b180d11924072d
@@ -1430,7 +1430,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P ed49f297bcee86674ed673e195610b8cc1d35647
-R 6a53f87d207675e944487abae6392564
+P 58d4cf26e15f90463148bec63d6ab514ffbbae60
+R f221e3cac7f52c2c660e427ee329aa03
 U dan
-Z dd362fd8eca18359c7fdd88fbcc31963
+Z 8b18cd934aae857f866e3f00c12cba6f
diff --git a/manifest.uuid b/manifest.uuid
index 9d51efbd2d..4b3aa263c2 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-58d4cf26e15f90463148bec63d6ab514ffbbae60
\ No newline at end of file
+73a7f010937828c5195a198604f976e8458cef73
\ No newline at end of file
diff --git a/src/shell_indexes.c b/src/shell_indexes.c
index b7b44e47e0..33f872efd7 100644
--- a/src/shell_indexes.c
+++ b/src/shell_indexes.c
@@ -25,7 +25,8 @@ typedef struct IdxTable IdxTable;
 ** A single constraint. Equivalent to either "col = ?" or "col < ?".
 **
 ** pLink:
-**   ... todo ...
+**   Used to temporarily link IdxConstraint objects into lists while
+**   creating candidate indexes.
 */
 struct IdxConstraint {
   char *zColl;                    /* Collation sequence */
@@ -39,18 +40,41 @@ struct IdxConstraint {
 /*
 ** A WHERE clause. Made up of IdxConstraint objects. Example WHERE clause:
 **
-**   a=? AND b=? AND (c=? OR d=?) AND (e=? OR f=?)
+**   a=? AND b=? AND ((c=? AND d=?) OR e=?) AND (f=? OR g=?) AND h>?
 **
-** The above
+** The above is decomposed into 5 AND connected clauses. The first two are
+** added to the IdxWhere.pEq linked list, the following two into 
+** IdxWhere.pOr and the last into IdxWhere.pRange.
 **
+** IdxWhere.pEq and IdxWhere.pRange are simple linked lists of IdxConstraint
+** objects linked by the IdxConstraint.pNext field.
 **
+** The list headed at IdxWhere.pOr and linked by IdxWhere.pNextOr contains
+** all "OR" terms that belong to the current WHERE clause. In the example
+** above, there are two OR terms:
 **
+**   ((c=? AND d=?) OR e=?)
+**   (f=? OR g=?)
+**
+** Within an OR term, the OR connected sub-expressions are termed siblings.
+** These are connected into a linked list by the pSibling pointers. Each OR
+** term above consists of two siblings.
+**
+**   pOr -> (c=? AND d=?) -> pNextOr -> (f=?)
+**               |                        |
+**            pSibling                 pSibling
+**               |                        |
+**               V                        V
+**             (e=?)                    (g=?)
+**
+** IdxWhere.pParent is only used while constructing a tree of IdxWhere 
+** structures. It is NULL for the root IdxWhere. For all others, the parent
+** WHERE clause.
 */
 struct IdxWhere {
   IdxConstraint *pEq;             /* List of == constraints */
   IdxConstraint *pRange;          /* List of < constraints */
   IdxWhere *pOr;                  /* List of OR constraints */
-
   IdxWhere *pNextOr;              /* Next in OR constraints of same IdxWhere */
   IdxWhere *pSibling;             /* Next branch in single OR constraint */
   IdxWhere *pParent;              /* Parent object (or NULL) */
@@ -94,209 +118,6 @@ struct IdxTable {
   IdxColumn *aCol;
 };
 
-
-typedef struct PragmaTable PragmaTable;
-typedef struct PragmaCursor PragmaCursor;
-
-struct PragmaTable {
-  sqlite3_vtab base;
-  sqlite3 *db;
-};
-
-struct PragmaCursor {
-  sqlite3_vtab_cursor base;
-  sqlite3_stmt *pStmt;
-  i64 iRowid;
-};
-
-/*
-** Connect to or create a pragma virtual table.
-*/
-static int pragmaConnect(
-  sqlite3 *db,
-  void *pAux,
-  int argc, const char *const*argv,
-  sqlite3_vtab **ppVtab,
-  char **pzErr
-){
-  const char *zSchema = 
-    "CREATE TABLE a(tbl HIDDEN, cid, name, type, isnotnull, dflt_value, pk)";
-  PragmaTable *pTab = 0;
-  int rc = SQLITE_OK;
-
-  rc = sqlite3_declare_vtab(db, zSchema);
-  if( rc==SQLITE_OK ){
-    pTab = (PragmaTable *)sqlite3_malloc64(sizeof(PragmaTable));
-    if( pTab==0 ) rc = SQLITE_NOMEM;
-  }else{
-    *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db));
-  }
-
-  assert( rc==SQLITE_OK || pTab==0 );
-  if( rc==SQLITE_OK ){
-    memset(pTab, 0, sizeof(PragmaTable));
-    pTab->db = db;
-  }
-
-  *ppVtab = (sqlite3_vtab*)pTab;
-  return rc;
-}
-
-/*
-** Disconnect from or destroy a pragma virtual table.
-*/
-static int pragmaDisconnect(sqlite3_vtab *pVtab){
-  sqlite3_free(pVtab);
-  return SQLITE_OK;
-}
-
-/*
-** xBestIndex method for pragma virtual tables.
-*/
-static int pragmaBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
-  int i;
-
-  pIdxInfo->estimatedCost = 1.0e6;  /* Initial cost estimate */
-
-  /* Look for a valid tbl=? constraint. */
-  for(i=0; i<pIdxInfo->nConstraint; i++){
-    if( pIdxInfo->aConstraint[i].usable==0 ) continue;
-    if( pIdxInfo->aConstraint[i].op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
-    if( pIdxInfo->aConstraint[i].iColumn!=0 ) continue;
-    pIdxInfo->idxNum = 1;
-    pIdxInfo->estimatedCost = 1.0;
-    pIdxInfo->aConstraintUsage[i].argvIndex = 1;
-    pIdxInfo->aConstraintUsage[i].omit = 1;
-    break;
-  }
-  if( i==pIdxInfo->nConstraint ){
-    tab->zErrMsg = sqlite3_mprintf("missing required tbl=? constraint");
-    return SQLITE_ERROR;
-  }
-  return SQLITE_OK;
-}
-
-/*
-** Open a new pragma cursor.
-*/
-static int pragmaOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
-  PragmaCursor *pCsr;
-
-  pCsr = (PragmaCursor*)sqlite3_malloc64(sizeof(PragmaCursor));
-  if( pCsr==0 ){
-    return SQLITE_NOMEM;
-  }else{
-    memset(pCsr, 0, sizeof(PragmaCursor));
-    pCsr->base.pVtab = pVTab;
-  }
-
-  *ppCursor = (sqlite3_vtab_cursor*)pCsr;
-  return SQLITE_OK;
-}
-
-static int pragmaClose(sqlite3_vtab_cursor *pCursor){
-  PragmaCursor *pCsr = (PragmaCursor*)pCursor;
-  sqlite3_finalize(pCsr->pStmt);
-  sqlite3_free(pCsr);
-  return SQLITE_OK;
-}
-
-/*
-** Move a statvfs cursor to the next entry in the file.
-*/
-static int pragmaNext(sqlite3_vtab_cursor *pCursor){
-  PragmaCursor *pCsr = (PragmaCursor*)pCursor;
-  int rc = SQLITE_OK;
-
-  if( sqlite3_step(pCsr->pStmt)!=SQLITE_ROW ){
-    rc = sqlite3_finalize(pCsr->pStmt);
-    pCsr->pStmt = 0;
-  }
-  pCsr->iRowid++;
-  return rc;
-}
-
-static int pragmaEof(sqlite3_vtab_cursor *pCursor){
-  PragmaCursor *pCsr = (PragmaCursor*)pCursor;
-  return pCsr->pStmt==0;
-}
-
-static int pragmaFilter(
-  sqlite3_vtab_cursor *pCursor, 
-  int idxNum, const char *idxStr,
-  int argc, sqlite3_value **argv
-){
-  PragmaCursor *pCsr = (PragmaCursor*)pCursor;
-  PragmaTable *pTab = (PragmaTable*)(pCursor->pVtab);
-  char *zSql;
-  const char *zTbl;
-  int rc = SQLITE_OK;
-
-  if( pCsr->pStmt ){
-    sqlite3_finalize(pCsr->pStmt);
-    pCsr->pStmt = 0;
-  }
-  pCsr->iRowid = 0;
-
-  assert( argc==1 );
-  zTbl = (const char*)sqlite3_value_text(argv[0]);
-  zSql = sqlite3_mprintf("PRAGMA table_info(%Q)", zTbl);
-  if( zSql==0 ){
-    rc = SQLITE_NOMEM;
-  }else{
-    rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0);
-  }
-  if( rc ) return rc;
-  return pragmaNext(pCursor);;
-}
-
-/*
-** xColumn method.
-*/
-static int pragmaColumn(
-  sqlite3_vtab_cursor *pCursor, 
-  sqlite3_context *ctx, 
-  int iCol
-){
-  PragmaCursor *pCsr = (PragmaCursor *)pCursor;
-  if( iCol>0 ){
-    sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pStmt, iCol-1));
-  }
-  return SQLITE_OK;
-}
-
-static int pragmaRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
-  PragmaCursor *pCsr = (PragmaCursor *)pCursor;
-  *pRowid = pCsr->iRowid;
-  return SQLITE_OK;
-}
-
-static int registerPragmaVtabs(sqlite3 *db){
-  static sqlite3_module pragma_module = {
-    0,                            /* iVersion */
-    pragmaConnect,                /* xCreate */
-    pragmaConnect,                /* xConnect */
-    pragmaBestIndex,              /* xBestIndex */
-    pragmaDisconnect,             /* xDisconnect */
-    pragmaDisconnect,             /* xDestroy */
-    pragmaOpen,                   /* xOpen - open a cursor */
-    pragmaClose,                  /* xClose - close a cursor */
-    pragmaFilter,                 /* xFilter - configure scan constraints */
-    pragmaNext,                   /* xNext - advance a cursor */
-    pragmaEof,                    /* xEof - check for end of scan */
-    pragmaColumn,                 /* xColumn - read data */
-    pragmaRowid,                  /* xRowid - read data */
-    0,                            /* xUpdate */
-    0,                            /* xBegin */
-    0,                            /* xSync */
-    0,                            /* xCommit */
-    0,                            /* xRollback */
-    0,                            /* xFindMethod */
-    0,                            /* xRename */
-  };
-  return sqlite3_create_module(db, "pragma_table_info", &pragma_module, 0);
-}
-
 /*
 ** Allocate and return nByte bytes of zeroed memory using sqlite3_malloc(). 
 ** If the allocation fails, set *pRc to SQLITE_NOMEM and return NULL.
@@ -477,12 +298,48 @@ static char *idxQueryToList(
   return zRet;
 }
 
+static int idxPrepareStmt(
+  sqlite3 *db,                    /* Database handle to compile against */
+  sqlite3_stmt **ppStmt,          /* OUT: Compiled SQL statement */
+  char **pzErrmsg,                /* OUT: sqlite3_malloc()ed error message */
+  const char *zSql                /* SQL statement to compile */
+){
+  int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0);
+  if( rc!=SQLITE_OK ){
+    *ppStmt = 0;
+    idxDatabaseError(db, pzErrmsg);
+  }
+  return rc;
+}
+
+static int idxPrintfPrepareStmt(
+  sqlite3 *db,                    /* Database handle to compile against */
+  sqlite3_stmt **ppStmt,          /* OUT: Compiled SQL statement */
+  char **pzErrmsg,                /* OUT: sqlite3_malloc()ed error message */
+  const char *zFmt,               /* printf() format of SQL statement */
+  ...                             /* Trailing printf() arguments */
+){
+  va_list ap;
+  int rc;
+  char *zSql;
+  va_start(ap, zFmt);
+  zSql = sqlite3_vmprintf(zFmt, ap);
+  if( zSql==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    rc = idxPrepareStmt(db, ppStmt, pzErrmsg, zSql);
+    sqlite3_free(zSql);
+  }
+  va_end(ap);
+  return rc;
+}
+
 static int idxGetTableInfo(
   sqlite3 *db,
   IdxScan *pScan,
   char **pzErrmsg
 ){
-  const char *zSql = "SELECT name, pk FROM pragma_table_info(?)";
+  const char *zTbl = pScan->zTable;
   sqlite3_stmt *p1 = 0;
   int nCol = 0;
   int nByte = sizeof(IdxTable);
@@ -490,17 +347,12 @@ static int idxGetTableInfo(
   int rc, rc2;
   char *pCsr;
 
-  rc = sqlite3_prepare_v2(db, zSql, -1, &p1, 0);
-  if( rc!=SQLITE_OK ){
-    idxDatabaseError(db, pzErrmsg);
-    return rc;
-  }
-  sqlite3_bind_text(p1, 1, pScan->zTable, -1, SQLITE_TRANSIENT);
-  while( SQLITE_ROW==sqlite3_step(p1) ){
-    const char *zCol = sqlite3_column_text(p1, 0);
+  rc = idxPrintfPrepareStmt(db, &p1, pzErrmsg, "PRAGMA table_info=%Q", zTbl);
+  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){
+    const char *zCol = sqlite3_column_text(p1, 1);
     nByte += 1 + strlen(zCol);
     rc = sqlite3_table_column_metadata(
-        db, "main", pScan->zTable, zCol, 0, &zCol, 0, 0, 0
+        db, "main", zTbl, zCol, 0, &zCol, 0, 0, 0
     );
     nByte += 1 + strlen(zCol);
     nCol++;
@@ -520,15 +372,15 @@ static int idxGetTableInfo(
 
   nCol = 0;
   while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){
-    const char *zCol = sqlite3_column_text(p1, 0);
+    const char *zCol = sqlite3_column_text(p1, 1);
     int nCopy = strlen(zCol) + 1;
     pNew->aCol[nCol].zName = pCsr;
-    pNew->aCol[nCol].iPk = sqlite3_column_int(p1, 1);
+    pNew->aCol[nCol].iPk = sqlite3_column_int(p1, 5);
     memcpy(pCsr, zCol, nCopy);
     pCsr += nCopy;
 
     rc = sqlite3_table_column_metadata(
-        db, "main", pScan->zTable, zCol, 0, &zCol, 0, 0, 0
+        db, "main", zTbl, zCol, 0, &zCol, 0, 0, 0
     );
     if( rc==SQLITE_OK ){
       nCopy = strlen(zCol) + 1;
@@ -600,7 +452,7 @@ static int idxCreateTables(
     }
 
     if( rc==SQLITE_OK ){
-#if 1
+#if 0
       printf("/* %s */\n", zCreate);
 #endif
       rc = sqlite3_exec(dbm, zCreate, 0, 0, pzErrmsg);
@@ -700,7 +552,9 @@ static int idxCreateFromCons(
         rc = SQLITE_NOMEM;
       }else{
         rc = sqlite3_exec(dbm, zIdx, 0, 0, 0);
+#if 0
         printf("/* %s */\n", zIdx);
+#endif
       }
     }
 
@@ -778,20 +632,6 @@ static int idxCreateFromWhere(
   return rc;
 }
 
-static int idxPrepareStmt(
-  sqlite3 *db,                    /* Database handle to compile against */
-  const char *zSql,               /* SQL statement to compile */
-  sqlite3_stmt **ppStmt,          /* OUT: Compiled SQL statement */
-  char **pzErrmsg                 /* OUT: sqlite3_malloc()ed error message */
-){
-  int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0);
-  if( rc!=SQLITE_OK ){
-    *ppStmt = 0;
-    idxDatabaseError(db, pzErrmsg);
-  }
-  return rc;
-}
-
 /*
 ** Create candidate indexes in database [dbm] based on the data in 
 ** linked-list pScan.
@@ -806,7 +646,7 @@ static int idxCreateCandidates(
   sqlite3_stmt *pDepmask;         /* Foreach depmask */
   IdxScan *pIter;
 
-  rc = idxPrepareStmt(dbm, "SELECT mask FROM depmask", &pDepmask, pzErrmsg);
+  rc = idxPrepareStmt(dbm, &pDepmask, pzErrmsg, "SELECT mask FROM depmask");
 
   for(pIter=pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
     IdxWhere *pWhere = &pIter->where;
@@ -832,27 +672,65 @@ int idxFindIndexes(
   const char *zSql,                    /* SQL to find indexes for */
   void (*xOut)(void*, const char*),    /* Output callback */
   void *pOutCtx,                       /* Context for xOut() */
-  char **pzErrmsg                      /* OUT: Error message (sqlite3_malloc) */
+  char **pzErr                         /* OUT: Error message (sqlite3_malloc) */
 ){
-  char *zExplain;
-  sqlite3_stmt *pExplain;
-  int rc;
+  sqlite3_stmt *pExplain = 0;
+  sqlite3_stmt *pSelect = 0;
+  int rc, rc2;
 
-  zExplain = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", zSql);
-  if( zExplain==0 ){
-    rc = SQLITE_NOMEM;
-  }else{
-    rc = idxPrepareStmt(dbm, zExplain, &pExplain, pzErrmsg);
-    sqlite3_free(zExplain);
+  rc = idxPrintfPrepareStmt(dbm, &pExplain, pzErr,"EXPLAIN QUERY PLAN %s",zSql);
+  if( rc==SQLITE_OK ){
+    rc = idxPrepareStmt(dbm, &pSelect, pzErr, 
+        "SELECT sql FROM sqlite_master WHERE name = ?"
+    );
   }
-  if( rc!=SQLITE_OK ) return rc;
 
-  while( sqlite3_step(pExplain)==SQLITE_ROW ){
-    int iCol;
-    // for(iCol=0; iCol<sqlite3_column_count(pExplain); iCol++){ }
-    xOut(pOutCtx, sqlite3_column_text(pExplain, 3));
+  while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){
+    int i;
+    const char *zDetail = (const char*)sqlite3_column_text(pExplain, 3);
+    int nDetail = strlen(zDetail);
+
+    for(i=0; i<nDetail; i++){
+      if( memcmp(&zDetail[i], " USING INDEX ", 13)==0 ){
+        int nIdx = 0;
+        const char *zIdx = &zDetail[i+13];
+        while( zIdx[nIdx]!='\0' && zIdx[nIdx]!=' ' ) nIdx++;
+        sqlite3_bind_text(pSelect, 1, zIdx, nIdx, SQLITE_STATIC);
+        if( SQLITE_ROW==sqlite3_step(pSelect) ){
+          xOut(pOutCtx, sqlite3_column_text(pSelect, 0));
+        }
+        rc = sqlite3_reset(pSelect);
+        break;
+      }
+    }
   }
-  rc = sqlite3_finalize(pExplain);
+  rc2 = sqlite3_reset(pExplain);
+  if( rc==SQLITE_OK ) rc = rc2;
+  if( rc==SQLITE_OK ) xOut(pOutCtx, "");
+
+  while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){
+    int iSelectid = sqlite3_column_int(pExplain, 0);
+    int iOrder = sqlite3_column_int(pExplain, 1);
+    int iFrom = sqlite3_column_int(pExplain, 2);
+    const char *zDetail = (const char*)sqlite3_column_text(pExplain, 3);
+    char *zOut;
+
+    zOut = sqlite3_mprintf("%d|%d|%d|%s", iSelectid, iOrder, iFrom, zDetail);
+    if( zOut==0 ){
+      rc = SQLITE_NOMEM;
+    }else{
+      xOut(pOutCtx, zOut);
+      sqlite3_free(zOut);
+    }
+  }
+
+ find_indexes_out:
+  rc2 = sqlite3_finalize(pExplain);
+  if( rc==SQLITE_OK ) rc = rc2;
+  rc2 = sqlite3_finalize(pSelect);
+  if( rc==SQLITE_OK ) rc = rc2;
+
+  return rc;
 }
 
 /*
@@ -873,8 +751,6 @@ int shellIndexesCommand(
   IdxContext ctx;
   sqlite3_stmt *pStmt = 0;        /* Statement compiled from zSql */
 
-  rc = registerPragmaVtabs(db);
-  if( rc ) return rc;
   memset(&ctx, 0, sizeof(IdxContext));
 
   /* Open an in-memory database to work with. The main in-memory 
@@ -887,52 +763,44 @@ int shellIndexesCommand(
         "ATTACH ':memory:' AS aux;"
         "CREATE TABLE aux.depmask(mask PRIMARY KEY) WITHOUT ROWID;"
         "INSERT INTO aux.depmask VALUES(0);"
-        , 0, 0, 0
+        , 0, 0, pzErrmsg
     );
   }
 
   /* Prepare an INSERT statement for writing to aux.depmask */
   if( rc==SQLITE_OK ){
-    rc = sqlite3_prepare_v2(dbm, 
-        "INSERT OR IGNORE INTO depmask SELECT mask | ?1 FROM depmask;", -1,
-        &ctx.pInsertMask, 0
+    rc = idxPrepareStmt(dbm, &ctx.pInsertMask, pzErrmsg,
+        "INSERT OR IGNORE INTO depmask SELECT mask | ?1 FROM depmask;"
     );
   }
 
-  if( rc!=SQLITE_OK ){
-    idxDatabaseError(dbm, pzErrmsg);
-    goto indexes_out;
-  }
-
   /* Analyze the SELECT statement in zSql. */
-  ctx.dbm = dbm;
-  sqlite3_db_config(db, SQLITE_DBCONFIG_WHEREINFO, idxWhereInfo, (void*)&ctx);
-  rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0);
-  sqlite3_db_config(db, SQLITE_DBCONFIG_WHEREINFO, (void*)0, (void*)0);
-
-  if( rc!=SQLITE_OK ){
-    idxDatabaseError(db, pzErrmsg);
-    goto indexes_out;
+  if( rc==SQLITE_OK ){
+    ctx.dbm = dbm;
+    sqlite3_db_config(db, SQLITE_DBCONFIG_WHEREINFO, idxWhereInfo, (void*)&ctx);
+    rc = idxPrepareStmt(db, &pStmt, pzErrmsg, zSql);
+    sqlite3_db_config(db, SQLITE_DBCONFIG_WHEREINFO, (void*)0, (void*)0);
+    sqlite3_finalize(pStmt);
   }
 
   /* Create tables within the main in-memory database. These tables
   ** have the same names, columns and declared types as the tables in
   ** the user database. All constraints except for PRIMARY KEY are
   ** removed. */
-  rc = idxCreateTables(db, dbm, ctx.pScan, pzErrmsg);
-  if( rc!=SQLITE_OK ){
-    goto indexes_out;
+  if( rc==SQLITE_OK ){
+    rc = idxCreateTables(db, dbm, ctx.pScan, pzErrmsg);
   }
 
   /* Create candidate indexes within the in-memory database file */
-  rc = idxCreateCandidates(dbm, ctx.pScan, pzErrmsg);
-  if( rc!=SQLITE_OK ){
-    goto indexes_out;
+  if( rc==SQLITE_OK ){
+    rc = idxCreateCandidates(dbm, ctx.pScan, pzErrmsg);
   }
 
-  rc = idxFindIndexes(dbm, zSql, xOut, pOutCtx, pzErrmsg);
+  /* Create candidate indexes within the in-memory database file */
+  if( rc==SQLITE_OK ){
+    rc = idxFindIndexes(dbm, zSql, xOut, pOutCtx, pzErrmsg);
+  }
 
- indexes_out:
   idxScanFree(ctx.pScan);
   sqlite3_close(dbm);
   return rc;

From 1c93844dd30973b13907a837876e7aa161e4bbed Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 17 Feb 2016 20:06:12 +0000
Subject: [PATCH 12/68] Schemalint changes: Avoid creating candidate indexes if
 a compatible index exists. Do not quote identifiers that do not require it.

FossilOrigin-Name: cf0f7eeb4f6490b1e3f05b45e83b87cd64640846
---
 manifest            |  12 +--
 manifest.uuid       |   2 +-
 src/shell_indexes.c | 181 ++++++++++++++++++++++++++++++++------------
 3 files changed, 138 insertions(+), 57 deletions(-)

diff --git a/manifest b/manifest
index f1eaa8735a..ff57fedd26 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sfurther\sissues\sin\sschemalint.
-D 2016-02-16T18:37:37.844
+C Schemalint\schanges:\sAvoid\screating\scandidate\sindexes\sif\sa\scompatible\sindex\sexists.\sDo\snot\squote\sidentifiers\sthat\sdo\snot\srequire\sit.
+D 2016-02-17T20:06:12.566
 F Makefile.in dac2776c84e0d533b158a9af6e57e05c4a6b19f3
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc b0493f10caddb8adf992a4e6f1943141fc7c6816
@@ -350,7 +350,7 @@ F src/resolve.c 9f7ce3a3c087afb7597b7c916c99126ff3f12f0c
 F src/rowset.c 9fe4b3ad7cc00944386bb600233d8f523de07a6e
 F src/select.c ff80004a9a6ece891a8d9327a88e7b6e2588ee6d
 F src/shell.c 2cde87e03712204231167c4a6c61b0eb5129e105
-F src/shell_indexes.c 27d3b064078066c9a284a93149c4a151821f55bf
+F src/shell_indexes.c 6cc207072469f1ded8c3bb9de1d4b6590a28abb8
 F src/sqlite.h.in c7db059d3b810b70b83d9ed1436fa813eba22462
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h dfbe62ffd95b99afe2140d8c35b180d11924072d
@@ -1430,7 +1430,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 58d4cf26e15f90463148bec63d6ab514ffbbae60
-R f221e3cac7f52c2c660e427ee329aa03
+P 73a7f010937828c5195a198604f976e8458cef73
+R 3a9a4af7276c9c00e5b2666c0bfe324e
 U dan
-Z 8b18cd934aae857f866e3f00c12cba6f
+Z 82976e1002963ffa4a1c502f357a9ae9
diff --git a/manifest.uuid b/manifest.uuid
index 4b3aa263c2..2e9ca33ba0 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-73a7f010937828c5195a198604f976e8458cef73
\ No newline at end of file
+cf0f7eeb4f6490b1e3f05b45e83b87cd64640846
\ No newline at end of file
diff --git a/src/shell_indexes.c b/src/shell_indexes.c
index 33f872efd7..54b59cd421 100644
--- a/src/shell_indexes.c
+++ b/src/shell_indexes.c
@@ -33,6 +33,7 @@ struct IdxConstraint {
   int bRange;                     /* True for range, false for eq */
   int iCol;                       /* Constrained table column */
   i64 depmask;                    /* Dependency mask */
+  int bFlag;                      /* Used by idxFindCompatible() */
   IdxConstraint *pNext;           /* Next constraint in pEq or pRange list */
   IdxConstraint *pLink;           /* See above */
 };
@@ -413,53 +414,20 @@ static int idxCreateTables(
   int rc = SQLITE_OK;
   IdxScan *pIter;
   for(pIter=pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
-    int nPk = 0;
-    char *zCols = 0;
-    char *zPk = 0;
-    char *zCreate = 0;
-    int iCol;
-
     rc = idxGetTableInfo(db, pIter, pzErrmsg);
-
-    for(iCol=0; rc==SQLITE_OK && iCol<pIter->pTable->nCol; iCol++){
-      IdxColumn *pCol = &pIter->pTable->aCol[iCol];
-      if( pCol->iPk>nPk ) nPk = pCol->iPk;
-      zCols = sqlite3_mprintf("%z%s%Q", zCols, (zCols?", ":""), pCol->zName);
-      if( zCols==0 ) rc = SQLITE_NOMEM;
-    }
-
-    for(iCol=1; rc==SQLITE_OK && iCol<=nPk; iCol++){
-      int j;
-      for(j=0; j<pIter->pTable->nCol; j++){
-        IdxColumn *pCol = &pIter->pTable->aCol[j];
-        if( pCol->iPk==iCol ){
-          zPk = sqlite3_mprintf("%z%s%Q", zPk, (zPk?", ":""), pCol->zName);
-          if( zPk==0 ) rc = SQLITE_NOMEM;
-          break;
-        }
-      }
-    }
-
     if( rc==SQLITE_OK ){
-      if( zPk ){
-        zCreate = sqlite3_mprintf("CREATE TABLE %Q(%s, PRIMARY KEY(%s))",
-            pIter->zTable, zCols, zPk
-        );
-      }else{
-        zCreate = sqlite3_mprintf("CREATE TABLE %Q(%s)", pIter->zTable, zCols);
+      int rc2;
+      sqlite3_stmt *pSql = 0;
+      rc = idxPrintfPrepareStmt(db, &pSql, pzErrmsg, 
+          "SELECT sql FROM sqlite_master WHERE tbl_name = %Q", pIter->zTable
+      );
+      while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
+        const char *zSql = (const char*)sqlite3_column_text(pSql, 0);
+        rc = sqlite3_exec(dbm, zSql, 0, 0, pzErrmsg);
       }
-      if( zCreate==0 ) rc = SQLITE_NOMEM;
+      rc2 = sqlite3_finalize(pSql);
+      if( rc==SQLITE_OK ) rc = rc2;
     }
-
-    if( rc==SQLITE_OK ){
-#if 0
-      printf("/* %s */\n", zCreate);
-#endif
-      rc = sqlite3_exec(dbm, zCreate, 0, 0, pzErrmsg);
-    }
-    sqlite3_free(zCols);
-    sqlite3_free(zPk);
-    sqlite3_free(zCreate);
   }
   return rc;
 }
@@ -501,6 +469,20 @@ static char *idxAppendText(int *pRc, char *zIn, const char *zFmt, ...){
   return zRet;
 }
 
+static int idxIdentifierRequiresQuotes(const char *zId){
+  int i;
+  for(i=0; zId[i]; i++){
+    if( !(zId[i]=='_')
+     && !(zId[i]>='0' && zId[i]<='9')
+     && !(zId[i]>='a' && zId[i]<='z')
+     && !(zId[i]>='A' && zId[i]<='Z')
+    ){
+      return 1;
+    }
+  }
+  return 0;
+}
+
 static char *idxAppendColDefn(
   int *pRc, 
   char *zIn, 
@@ -510,13 +492,100 @@ static char *idxAppendColDefn(
   char *zRet = zIn;
   IdxColumn *p = &pTab->aCol[pCons->iCol];
   if( zRet ) zRet = idxAppendText(pRc, zRet, ", ");
-  zRet = idxAppendText(pRc, zRet, "%Q", p->zName);
+
+  if( idxIdentifierRequiresQuotes(p->zName) ){
+    zRet = idxAppendText(pRc, zRet, "%Q", p->zName);
+  }else{
+    zRet = idxAppendText(pRc, zRet, "%s", p->zName);
+  }
+
   if( sqlite3_stricmp(p->zColl, pCons->zColl) ){
-    zRet = idxAppendText(pRc, zRet, " COLLATE %Q", pCons->zColl);
+    if( idxIdentifierRequiresQuotes(pCons->zColl) ){
+      zRet = idxAppendText(pRc, zRet, " COLLATE %Q", pCons->zColl);
+    }else{
+      zRet = idxAppendText(pRc, zRet, " COLLATE %s", pCons->zColl);
+    }
   }
   return zRet;
 }
 
+/*
+** Search database dbm for an index compatible with the one idxCreateFromCons()
+** would create from arguments pScan, pEq and pTail. If no error occurs and 
+** such an index is found, return non-zero. Or, if no such index is found,
+** return zero.
+**
+** If an error occurs, set *pRc to an SQLite error code and return zero.
+*/
+static int idxFindCompatible(
+  int *pRc,                       /* OUT: Error code */
+  sqlite3* dbm,                   /* Database to search */
+  IdxScan *pScan,                 /* Scan for table to search for index on */
+  IdxConstraint *pEq,             /* List of == constraints */
+  IdxConstraint *pTail            /* List of range constraints */
+){
+  const char *zTbl = pScan->zTable;
+  sqlite3_stmt *pIdxList = 0;
+  IdxConstraint *pIter;
+  int nEq = 0;                    /* Number of elements in pEq */
+  int rc, rc2;
+
+  /* Count the elements in list pEq */
+  for(pIter=pEq; pIter; pIter=pIter->pNext) nEq++;
+
+  rc = idxPrintfPrepareStmt(dbm, &pIdxList, 0, "PRAGMA index_list=%Q", zTbl);
+  while( rc==SQLITE_OK && sqlite3_step(pIdxList)==SQLITE_ROW ){
+    int bMatch = 1;
+    IdxConstraint *pT = pTail;
+    sqlite3_stmt *pInfo = 0;
+    const char *zIdx = (const char*)sqlite3_column_text(pIdxList, 1);
+
+    /* Zero the IdxConstraint.bFlag values in the pEq list */
+    for(pIter=pEq; pIter; pIter=pIter->pNext) pIter->bFlag = 0;
+
+    rc = idxPrintfPrepareStmt(dbm, &pInfo, 0, "PRAGMA index_xInfo=%Q", zIdx);
+    while( rc==SQLITE_OK && sqlite3_step(pInfo)==SQLITE_ROW ){
+      int iIdx = sqlite3_column_int(pInfo, 0);
+      int iCol = sqlite3_column_int(pInfo, 1);
+      const char *zColl = (const char*)sqlite3_column_text(pInfo, 4);
+
+      if( iIdx<nEq ){
+        for(pIter=pEq; pIter; pIter=pIter->pNext){
+          if( pIter->bFlag ) continue;
+          if( pIter->iCol!=iCol ) continue;
+          if( sqlite3_stricmp(pIter->zColl, zColl) ) continue;
+          pIter->bFlag = 1;
+          break;
+        }
+        if( pIter==0 ){
+          bMatch = 0;
+          break;
+        }
+      }else{
+        if( pT ){
+          if( pT->iCol!=iCol || sqlite3_stricmp(pT->zColl, zColl) ){
+            bMatch = 0;
+            break;
+          }
+          pT = pT->pLink;
+        }
+      }
+    }
+    rc2 = sqlite3_finalize(pInfo);
+    if( rc==SQLITE_OK ) rc = rc2;
+
+    if( rc==SQLITE_OK && bMatch ){
+      sqlite3_finalize(pIdxList);
+      return 1;
+    }
+  }
+  rc2 = sqlite3_finalize(pIdxList);
+  if( rc==SQLITE_OK ) rc = rc2;
+
+  *pRc = rc;
+  return 0;
+}
+
 static int idxCreateFromCons(
   sqlite3 *dbm,
   IdxScan *pScan,
@@ -524,12 +593,13 @@ static int idxCreateFromCons(
   IdxConstraint *pTail
 ){
   int rc = SQLITE_OK;
-  if( pEq || pTail ){
+  if( (pEq || pTail) && 0==idxFindCompatible(&rc, dbm, pScan, pEq, pTail) ){
     IdxTable *pTab = pScan->pTable;
     char *zCols = 0;
     char *zIdx = 0;
     IdxConstraint *pCons;
     int h = 0;
+    const char *zFmt;
 
     for(pCons=pEq; pCons; pCons=pCons->pLink){
       zCols = idxAppendColDefn(&rc, zCols, pTab, pCons);
@@ -545,9 +615,12 @@ static int idxCreateFromCons(
         h += ((h<<3) + zCols[i]);
       }
 
-      zIdx = sqlite3_mprintf("CREATE INDEX IF NOT EXISTS "
-          "'%q_idx_%08x' ON %Q(%s)", pScan->zTable, h, pScan->zTable, zCols
-      );
+      if( idxIdentifierRequiresQuotes(pScan->zTable) ){
+        zFmt = "CREATE INDEX '%q_idx_%08x' ON %Q(%s)";
+      }else{
+        zFmt = "CREATE INDEX %s_idx_%08x ON %s(%s)";
+      }
+      zIdx = sqlite3_mprintf(zFmt, pScan->zTable, h, pScan->zTable, zCols);
       if( !zIdx ){
         rc = SQLITE_NOMEM;
       }else{
@@ -665,6 +738,12 @@ static int idxCreateCandidates(
 }
 
 static void idxScanFree(IdxScan *pScan){
+  IdxScan *pIter;
+  IdxScan *pNext;
+  for(pIter=pScan; pIter; pIter=pNext){
+    pNext = pIter->pNextScan;
+
+  }
 }
 
 int idxFindIndexes(
@@ -796,12 +875,14 @@ int shellIndexesCommand(
     rc = idxCreateCandidates(dbm, ctx.pScan, pzErrmsg);
   }
 
-  /* Create candidate indexes within the in-memory database file */
+  /* Figure out which of the candidate indexes are preferred by the query
+  ** planner and report the results to the user.  */
   if( rc==SQLITE_OK ){
     rc = idxFindIndexes(dbm, zSql, xOut, pOutCtx, pzErrmsg);
   }
 
   idxScanFree(ctx.pScan);
+  sqlite3_finalize(ctx.pInsertMask);
   sqlite3_close(dbm);
   return rc;
 }

From b12dfd01259fc16b7f0ac522aeec185daf7c128f Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 18 Feb 2016 19:10:02 +0000
Subject: [PATCH 13/68] Have the schemalint output distinguish between existing
 and recommended indexes.

FossilOrigin-Name: 4ab3df25f1fee7c8fea19d0c64b3e0e4d3b9c3cf
---
 manifest            | 12 +++----
 manifest.uuid       |  2 +-
 src/shell_indexes.c | 78 ++++++++++++++++++++++++++++++---------------
 3 files changed, 59 insertions(+), 33 deletions(-)

diff --git a/manifest b/manifest
index ff57fedd26..c8c283d441 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Schemalint\schanges:\sAvoid\screating\scandidate\sindexes\sif\sa\scompatible\sindex\sexists.\sDo\snot\squote\sidentifiers\sthat\sdo\snot\srequire\sit.
-D 2016-02-17T20:06:12.566
+C Have\sthe\sschemalint\soutput\sdistinguish\sbetween\sexisting\sand\srecommended\sindexes.
+D 2016-02-18T19:10:02.440
 F Makefile.in dac2776c84e0d533b158a9af6e57e05c4a6b19f3
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc b0493f10caddb8adf992a4e6f1943141fc7c6816
@@ -350,7 +350,7 @@ F src/resolve.c 9f7ce3a3c087afb7597b7c916c99126ff3f12f0c
 F src/rowset.c 9fe4b3ad7cc00944386bb600233d8f523de07a6e
 F src/select.c ff80004a9a6ece891a8d9327a88e7b6e2588ee6d
 F src/shell.c 2cde87e03712204231167c4a6c61b0eb5129e105
-F src/shell_indexes.c 6cc207072469f1ded8c3bb9de1d4b6590a28abb8
+F src/shell_indexes.c 277eb75b8cfb3c2bcf76e062baaa419779f824e7
 F src/sqlite.h.in c7db059d3b810b70b83d9ed1436fa813eba22462
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h dfbe62ffd95b99afe2140d8c35b180d11924072d
@@ -1430,7 +1430,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 73a7f010937828c5195a198604f976e8458cef73
-R 3a9a4af7276c9c00e5b2666c0bfe324e
+P cf0f7eeb4f6490b1e3f05b45e83b87cd64640846
+R f54b8a1f9b5e206798931d383b1eb3a6
 U dan
-Z 82976e1002963ffa4a1c502f357a9ae9
+Z 5c18fceb59d8462f5aec79ee219a18a2
diff --git a/manifest.uuid b/manifest.uuid
index 2e9ca33ba0..9ede1c42e8 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-cf0f7eeb4f6490b1e3f05b45e83b87cd64640846
\ No newline at end of file
+4ab3df25f1fee7c8fea19d0c64b3e0e4d3b9c3cf
\ No newline at end of file
diff --git a/src/shell_indexes.c b/src/shell_indexes.c
index 54b59cd421..755fff6403 100644
--- a/src/shell_indexes.c
+++ b/src/shell_indexes.c
@@ -98,12 +98,14 @@ struct IdxScan {
 ** Context object passed to idxWhereInfo()
 */
 struct IdxContext {
+  char **pzErrmsg;
   IdxWhere *pCurrent;             /* Current where clause */
   int rc;                         /* Error code (if error has occurred) */
   IdxScan *pScan;                 /* List of scan objects */
   sqlite3 *dbm;                   /* In-memory db for this analysis */
   sqlite3 *db;                    /* User database under analysis */
   sqlite3_stmt *pInsertMask;      /* To write to aux.depmask */
+  i64 iIdxRowid;                  /* Rowid of first index created */
 };
 
 /*
@@ -587,11 +589,12 @@ static int idxFindCompatible(
 }
 
 static int idxCreateFromCons(
-  sqlite3 *dbm,
+  IdxContext *pCtx,
   IdxScan *pScan,
   IdxConstraint *pEq, 
   IdxConstraint *pTail
 ){
+  sqlite3 *dbm = pCtx->dbm;
   int rc = SQLITE_OK;
   if( (pEq || pTail) && 0==idxFindCompatible(&rc, dbm, pScan, pEq, pTail) ){
     IdxTable *pTab = pScan->pTable;
@@ -608,8 +611,8 @@ static int idxCreateFromCons(
       zCols = idxAppendColDefn(&rc, zCols, pTab, pCons);
     }
 
-    /* Hash the list of columns to come up with a name for the index */
     if( rc==SQLITE_OK ){
+      /* Hash the list of columns to come up with a name for the index */
       int i;
       for(i=0; zCols[i]; i++){
         h += ((h<<3) + zCols[i]);
@@ -630,6 +633,18 @@ static int idxCreateFromCons(
 #endif
       }
     }
+    if( rc==SQLITE_OK && pCtx->iIdxRowid==0 ){
+      int rc2;
+      sqlite3_stmt *pLast = 0;
+      rc = idxPrepareStmt(dbm, &pLast, pCtx->pzErrmsg, 
+          "SELECT max(rowid) FROM sqlite_master"
+      );
+      if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLast) ){
+        pCtx->iIdxRowid = sqlite3_column_int64(pLast, 0);
+      }
+      rc2 = sqlite3_finalize(pLast);
+      if( rc==SQLITE_OK ) rc = rc2;
+    }
 
     sqlite3_free(zIdx);
     sqlite3_free(zCols);
@@ -638,11 +653,11 @@ static int idxCreateFromCons(
 }
 
 static int idxCreateFromWhere(
-    sqlite3*, i64, IdxScan*, IdxWhere*, IdxConstraint*, IdxConstraint*
+    IdxContext*, i64, IdxScan*, IdxWhere*, IdxConstraint*, IdxConstraint*
 );
 
 static int idxCreateForeachOr(
-  sqlite3 *dbm, 
+  IdxContext *pCtx, 
   i64 mask,                       /* Consider only these constraints */
   IdxScan *pScan,                 /* Create indexes for this scan */
   IdxWhere *pWhere,               /* Read constraints from here */
@@ -653,22 +668,23 @@ static int idxCreateForeachOr(
   IdxWhere *p1;
   IdxWhere *p2;
   for(p1=pWhere->pOr; p1 && rc==SQLITE_OK; p1=p1->pNextOr){
-    rc = idxCreateFromWhere(dbm, mask, pScan, p1, pEq, pTail);
+    rc = idxCreateFromWhere(pCtx, mask, pScan, p1, pEq, pTail);
     for(p2=p1->pSibling; p2 && rc==SQLITE_OK; p2=p2->pSibling){
-      rc = idxCreateFromWhere(dbm, mask, pScan, p2, pEq, pTail);
+      rc = idxCreateFromWhere(pCtx, mask, pScan, p2, pEq, pTail);
     }
   }
   return rc;
 }
 
 static int idxCreateFromWhere(
-  sqlite3 *dbm, 
+  IdxContext *pCtx, 
   i64 mask,                       /* Consider only these constraints */
   IdxScan *pScan,                 /* Create indexes for this scan */
   IdxWhere *pWhere,               /* Read constraints from here */
   IdxConstraint *pEq,             /* == constraints for inclusion */
   IdxConstraint *pTail            /* range/ORDER BY constraints for inclusion */
 ){
+  sqlite3 *dbm = pCtx->dbm;
   IdxConstraint *p1 = pEq;
   IdxConstraint *pCon;
   int rc;
@@ -683,9 +699,9 @@ static int idxCreateFromWhere(
 
   /* Create an index using the == constraints collected above. And the
   ** range constraint/ORDER BY terms passed in by the caller, if any. */
-  rc = idxCreateFromCons(dbm, pScan, p1, pTail);
+  rc = idxCreateFromCons(pCtx, pScan, p1, pTail);
   if( rc==SQLITE_OK ){
-    rc = idxCreateForeachOr(dbm, mask, pScan, pWhere, p1, pTail);
+    rc = idxCreateForeachOr(pCtx, mask, pScan, pWhere, p1, pTail);
   }
 
   /* If no range/ORDER BY passed by the caller, create a version of the
@@ -694,9 +710,9 @@ static int idxCreateFromWhere(
     for(pCon=pWhere->pRange; rc==SQLITE_OK && pCon; pCon=pCon->pNext){
       assert( pCon->pLink==0 );
       if( (mask & pCon->depmask)==pCon->depmask ){
-        rc = idxCreateFromCons(dbm, pScan, p1, pCon);
+        rc = idxCreateFromCons(pCtx, pScan, p1, pCon);
         if( rc==SQLITE_OK ){
-          rc = idxCreateForeachOr(dbm, mask, pScan, pWhere, p1, pCon);
+          rc = idxCreateForeachOr(pCtx, mask, pScan, pWhere, p1, pCon);
         }
       }
     }
@@ -709,25 +725,24 @@ static int idxCreateFromWhere(
 ** Create candidate indexes in database [dbm] based on the data in 
 ** linked-list pScan.
 */
-static int idxCreateCandidates(
-  sqlite3 *dbm,
-  IdxScan *pScan,
-  char **pzErrmsg
-){
+static int idxCreateCandidates(IdxContext *pCtx){
+  sqlite3 *dbm = pCtx->dbm;
   int rc2;
   int rc = SQLITE_OK;
   sqlite3_stmt *pDepmask;         /* Foreach depmask */
   IdxScan *pIter;
 
-  rc = idxPrepareStmt(dbm, &pDepmask, pzErrmsg, "SELECT mask FROM depmask");
+  rc = idxPrepareStmt(pCtx->dbm, &pDepmask, pCtx->pzErrmsg, 
+      "SELECT mask FROM depmask"
+  );
 
-  for(pIter=pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
+  for(pIter=pCtx->pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
     IdxWhere *pWhere = &pIter->where;
     while( SQLITE_ROW==sqlite3_step(pDepmask) && rc==SQLITE_OK ){
       i64 mask = sqlite3_column_int64(pDepmask, 0);
-      rc = idxCreateFromWhere(dbm, mask, pIter, pWhere, 0, 0);
+      rc = idxCreateFromWhere(pCtx, mask, pIter, pWhere, 0, 0);
       if( rc==SQLITE_OK && pIter->pOrder ){
-        rc = idxCreateFromWhere(dbm, mask, pIter, pWhere, 0, pIter->pOrder);
+        rc = idxCreateFromWhere(pCtx, mask, pIter, pWhere, 0, pIter->pOrder);
       }
     }
   }
@@ -747,20 +762,22 @@ static void idxScanFree(IdxScan *pScan){
 }
 
 int idxFindIndexes(
-  sqlite3 *dbm,                        /* Database handle */
+  IdxContext *pCtx,
   const char *zSql,                    /* SQL to find indexes for */
   void (*xOut)(void*, const char*),    /* Output callback */
   void *pOutCtx,                       /* Context for xOut() */
   char **pzErr                         /* OUT: Error message (sqlite3_malloc) */
 ){
+  sqlite3 *dbm = pCtx->dbm;
   sqlite3_stmt *pExplain = 0;
   sqlite3_stmt *pSelect = 0;
   int rc, rc2;
+  int bFound = 0;
 
   rc = idxPrintfPrepareStmt(dbm, &pExplain, pzErr,"EXPLAIN QUERY PLAN %s",zSql);
   if( rc==SQLITE_OK ){
     rc = idxPrepareStmt(dbm, &pSelect, pzErr, 
-        "SELECT sql FROM sqlite_master WHERE name = ?"
+        "SELECT rowid, sql FROM sqlite_master WHERE name = ?"
     );
   }
 
@@ -776,7 +793,12 @@ int idxFindIndexes(
         while( zIdx[nIdx]!='\0' && zIdx[nIdx]!=' ' ) nIdx++;
         sqlite3_bind_text(pSelect, 1, zIdx, nIdx, SQLITE_STATIC);
         if( SQLITE_ROW==sqlite3_step(pSelect) ){
-          xOut(pOutCtx, sqlite3_column_text(pSelect, 0));
+          i64 iRowid = sqlite3_column_int64(pSelect, 0);
+          const char *zSql = (const char*)sqlite3_column_text(pSelect, 1);
+          if( iRowid>=pCtx->iIdxRowid ){
+            xOut(pOutCtx, zSql);
+            bFound = 1;
+          }
         }
         rc = sqlite3_reset(pSelect);
         break;
@@ -785,7 +807,10 @@ int idxFindIndexes(
   }
   rc2 = sqlite3_reset(pExplain);
   if( rc==SQLITE_OK ) rc = rc2;
-  if( rc==SQLITE_OK ) xOut(pOutCtx, "");
+  if( rc==SQLITE_OK ){
+    if( bFound==0 ) xOut(pOutCtx, "(no new indexes)");
+    xOut(pOutCtx, "");
+  }
 
   while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){
     int iSelectid = sqlite3_column_int(pExplain, 0);
@@ -841,6 +866,7 @@ int shellIndexesCommand(
     rc = sqlite3_exec(dbm, 
         "ATTACH ':memory:' AS aux;"
         "CREATE TABLE aux.depmask(mask PRIMARY KEY) WITHOUT ROWID;"
+        "CREATE TABLE aux.indexes(name PRIMARY KEY) WITHOUT ROWID;"
         "INSERT INTO aux.depmask VALUES(0);"
         , 0, 0, pzErrmsg
     );
@@ -872,13 +898,13 @@ int shellIndexesCommand(
 
   /* Create candidate indexes within the in-memory database file */
   if( rc==SQLITE_OK ){
-    rc = idxCreateCandidates(dbm, ctx.pScan, pzErrmsg);
+    rc = idxCreateCandidates(&ctx);
   }
 
   /* Figure out which of the candidate indexes are preferred by the query
   ** planner and report the results to the user.  */
   if( rc==SQLITE_OK ){
-    rc = idxFindIndexes(dbm, zSql, xOut, pOutCtx, pzErrmsg);
+    rc = idxFindIndexes(&ctx, zSql, xOut, pOutCtx, pzErrmsg);
   }
 
   idxScanFree(ctx.pScan);

From 850493ac1fd888939dc92bde3418d53d38e9fcbe Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 19 Feb 2016 07:53:43 +0000
Subject: [PATCH 14/68] Fix a couple of bugs in the schemalint code.

FossilOrigin-Name: 02fbf699c07286f842d9617755f071b0fffc5d40
---
 manifest            | 12 ++++++------
 manifest.uuid       |  2 +-
 src/shell_indexes.c | 47 +++++++++++++++++++++++++++++++++------------
 3 files changed, 42 insertions(+), 19 deletions(-)

diff --git a/manifest b/manifest
index c8c283d441..549a6f5bf1 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Have\sthe\sschemalint\soutput\sdistinguish\sbetween\sexisting\sand\srecommended\sindexes.
-D 2016-02-18T19:10:02.440
+C Fix\sa\scouple\sof\sbugs\sin\sthe\sschemalint\scode.
+D 2016-02-19T07:53:43.883
 F Makefile.in dac2776c84e0d533b158a9af6e57e05c4a6b19f3
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc b0493f10caddb8adf992a4e6f1943141fc7c6816
@@ -350,7 +350,7 @@ F src/resolve.c 9f7ce3a3c087afb7597b7c916c99126ff3f12f0c
 F src/rowset.c 9fe4b3ad7cc00944386bb600233d8f523de07a6e
 F src/select.c ff80004a9a6ece891a8d9327a88e7b6e2588ee6d
 F src/shell.c 2cde87e03712204231167c4a6c61b0eb5129e105
-F src/shell_indexes.c 277eb75b8cfb3c2bcf76e062baaa419779f824e7
+F src/shell_indexes.c c0099c01c7af01038ab4315621814df535b700a3
 F src/sqlite.h.in c7db059d3b810b70b83d9ed1436fa813eba22462
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h dfbe62ffd95b99afe2140d8c35b180d11924072d
@@ -1430,7 +1430,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P cf0f7eeb4f6490b1e3f05b45e83b87cd64640846
-R f54b8a1f9b5e206798931d383b1eb3a6
+P 4ab3df25f1fee7c8fea19d0c64b3e0e4d3b9c3cf
+R 0c9cae22954d1f32b48770c565f201e2
 U dan
-Z 5c18fceb59d8462f5aec79ee219a18a2
+Z cecc60af647c055f880ee4bdf2018855
diff --git a/manifest.uuid b/manifest.uuid
index 9ede1c42e8..675cfe73ee 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4ab3df25f1fee7c8fea19d0c64b3e0e4d3b9c3cf
\ No newline at end of file
+02fbf699c07286f842d9617755f071b0fffc5d40
\ No newline at end of file
diff --git a/src/shell_indexes.c b/src/shell_indexes.c
index 755fff6403..cd8a5e7288 100644
--- a/src/shell_indexes.c
+++ b/src/shell_indexes.c
@@ -95,7 +95,7 @@ struct IdxScan {
 };
 
 /*
-** Context object passed to idxWhereInfo()
+** Context object passed to idxWhereInfo() and other functions.
 */
 struct IdxContext {
   char **pzErrmsg;
@@ -454,7 +454,7 @@ static char *idxAppendText(int *pRc, char *zIn, const char *zFmt, ...){
     zAppend = sqlite3_vmprintf(zFmt, ap);
     if( zAppend ){
       nAppend = strlen(zAppend);
-      zRet = (char*)sqlite3_malloc(nIn + nAppend);
+      zRet = (char*)sqlite3_malloc(nIn + nAppend + 1);
     }
     if( zAppend && zRet ){
       memcpy(zRet, zIn, nIn);
@@ -532,8 +532,9 @@ static int idxFindCompatible(
   int nEq = 0;                    /* Number of elements in pEq */
   int rc, rc2;
 
+
   /* Count the elements in list pEq */
-  for(pIter=pEq; pIter; pIter=pIter->pNext) nEq++;
+  for(pIter=pEq; pIter; pIter=pIter->pLink) nEq++;
 
   rc = idxPrintfPrepareStmt(dbm, &pIdxList, 0, "PRAGMA index_list=%Q", zTbl);
   while( rc==SQLITE_OK && sqlite3_step(pIdxList)==SQLITE_ROW ){
@@ -543,7 +544,7 @@ static int idxFindCompatible(
     const char *zIdx = (const char*)sqlite3_column_text(pIdxList, 1);
 
     /* Zero the IdxConstraint.bFlag values in the pEq list */
-    for(pIter=pEq; pIter; pIter=pIter->pNext) pIter->bFlag = 0;
+    for(pIter=pEq; pIter; pIter=pIter->pLink) pIter->bFlag = 0;
 
     rc = idxPrintfPrepareStmt(dbm, &pInfo, 0, "PRAGMA index_xInfo=%Q", zIdx);
     while( rc==SQLITE_OK && sqlite3_step(pInfo)==SQLITE_ROW ){
@@ -552,7 +553,7 @@ static int idxFindCompatible(
       const char *zColl = (const char*)sqlite3_column_text(pInfo, 4);
 
       if( iIdx<nEq ){
-        for(pIter=pEq; pIter; pIter=pIter->pNext){
+        for(pIter=pEq; pIter; pIter=pIter->pLink){
           if( pIter->bFlag ) continue;
           if( pIter->iCol!=iCol ) continue;
           if( sqlite3_stricmp(pIter->zColl, zColl) ) continue;
@@ -627,10 +628,8 @@ static int idxCreateFromCons(
       if( !zIdx ){
         rc = SQLITE_NOMEM;
       }else{
-        rc = sqlite3_exec(dbm, zIdx, 0, 0, 0);
-#if 0
-        printf("/* %s */\n", zIdx);
-#endif
+        rc = sqlite3_exec(dbm, zIdx, 0, 0, pCtx->pzErrmsg);
+        /* printf("%s\n", zIdx); */
       }
     }
     if( rc==SQLITE_OK && pCtx->iIdxRowid==0 ){
@@ -676,6 +675,18 @@ static int idxCreateForeachOr(
   return rc;
 }
 
+/*
+** Return true if list pList (linked by IdxConstraint.pLink) contains
+** a constraint compatible with *p. Otherwise return false.
+*/
+static int idxFindConstraint(IdxConstraint *pList, IdxConstraint *p){
+  IdxConstraint *pCmp;
+  for(pCmp=pList; pCmp; pCmp=pCmp->pLink){
+    if( p->iCol==pCmp->iCol ) return 1;
+  }
+  return 0;
+}
+
 static int idxCreateFromWhere(
   IdxContext *pCtx, 
   i64 mask,                       /* Consider only these constraints */
@@ -691,7 +702,10 @@ static int idxCreateFromWhere(
 
   /* Gather up all the == constraints that match the mask. */
   for(pCon=pWhere->pEq; pCon; pCon=pCon->pNext){
-    if( (mask & pCon->depmask)==pCon->depmask ){
+    if( (mask & pCon->depmask)==pCon->depmask 
+     && idxFindConstraint(p1, pCon)==0
+     && idxFindConstraint(pTail, pCon)==0
+    ){
       pCon->pLink = p1;
       p1 = pCon;
     }
@@ -709,7 +723,10 @@ static int idxCreateFromWhere(
   if( pTail==0 ){
     for(pCon=pWhere->pRange; rc==SQLITE_OK && pCon; pCon=pCon->pNext){
       assert( pCon->pLink==0 );
-      if( (mask & pCon->depmask)==pCon->depmask ){
+      if( (mask & pCon->depmask)==pCon->depmask
+        && idxFindConstraint(pEq, pCon)==0
+        && idxFindConstraint(pTail, pCon)==0
+      ){
         rc = idxCreateFromCons(pCtx, pScan, p1, pCon);
         if( rc==SQLITE_OK ){
           rc = idxCreateForeachOr(pCtx, mask, pScan, pWhere, p1, pCon);
@@ -787,9 +804,14 @@ int idxFindIndexes(
     int nDetail = strlen(zDetail);
 
     for(i=0; i<nDetail; i++){
+      const char *zIdx = 0;
       if( memcmp(&zDetail[i], " USING INDEX ", 13)==0 ){
+        zIdx = &zDetail[i+13];
+      }else if( memcmp(&zDetail[i], " USING COVERING INDEX ", 22)==0 ){
+        zIdx = &zDetail[i+22];
+      }
+      if( zIdx ){
         int nIdx = 0;
-        const char *zIdx = &zDetail[i+13];
         while( zIdx[nIdx]!='\0' && zIdx[nIdx]!=' ' ) nIdx++;
         sqlite3_bind_text(pSelect, 1, zIdx, nIdx, SQLITE_STATIC);
         if( SQLITE_ROW==sqlite3_step(pSelect) ){
@@ -856,6 +878,7 @@ int shellIndexesCommand(
   sqlite3_stmt *pStmt = 0;        /* Statement compiled from zSql */
 
   memset(&ctx, 0, sizeof(IdxContext));
+  ctx.pzErrmsg = pzErrmsg;
 
   /* Open an in-memory database to work with. The main in-memory 
   ** database schema contains tables similar to those in the users 

From 14479504381b65251f332d819e7f03a36ad08ea1 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 22 Feb 2016 19:51:08 +0000
Subject: [PATCH 15/68] Add test script shell6.test, containing tests for
 schemalint.

FossilOrigin-Name: 0b73406595c9a077399b0f4c17af3a826cf3612f
---
 manifest            |  15 ++--
 manifest.uuid       |   2 +-
 src/shell_indexes.c |  21 ++++-
 src/where.c         |   6 +-
 test/shell6.test    | 191 ++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 221 insertions(+), 14 deletions(-)
 create mode 100644 test/shell6.test

diff --git a/manifest b/manifest
index 549a6f5bf1..1ea78374dd 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\scouple\sof\sbugs\sin\sthe\sschemalint\scode.
-D 2016-02-19T07:53:43.883
+C Add\stest\sscript\sshell6.test,\scontaining\stests\sfor\sschemalint.
+D 2016-02-22T19:51:08.971
 F Makefile.in dac2776c84e0d533b158a9af6e57e05c4a6b19f3
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc b0493f10caddb8adf992a4e6f1943141fc7c6816
@@ -350,7 +350,7 @@ F src/resolve.c 9f7ce3a3c087afb7597b7c916c99126ff3f12f0c
 F src/rowset.c 9fe4b3ad7cc00944386bb600233d8f523de07a6e
 F src/select.c ff80004a9a6ece891a8d9327a88e7b6e2588ee6d
 F src/shell.c 2cde87e03712204231167c4a6c61b0eb5129e105
-F src/shell_indexes.c c0099c01c7af01038ab4315621814df535b700a3
+F src/shell_indexes.c e10b3c2c4bc9a87f7bff1cf622473717b1a00698
 F src/sqlite.h.in c7db059d3b810b70b83d9ed1436fa813eba22462
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h dfbe62ffd95b99afe2140d8c35b180d11924072d
@@ -428,7 +428,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c d21b99fd1458159d0b1ecdccc8ee6ada4fdc4c54
 F src/wal.h 2f7c831cf3b071fa548bf2d5cac640846a7ff19c
 F src/walker.c 0f142b5bd3ed2041fc52d773880748b212e63354
-F src/where.c 89d5845353fe6d2e77bce52a2c8bea0781c69dad
+F src/where.c 0ecce6da62ad521cac83dfa1b60a37936b36b6a1
 F src/whereInt.h 78b6b4de94db84aecbdc07fe3e38f648eb391e9a
 F src/wherecode.c 791a784bbf8749d560fdb0b990b607bc4f44a38d
 F src/whereexpr.c de117970b29471177a6901d60ad83a194671dc03
@@ -1025,6 +1025,7 @@ F test/shell2.test 12b8bf901b0e3a8ac58cf5c0c63a0a388d4d1862
 F test/shell3.test 5e8545ec72c4413a0e8d4c6be56496e3c257ca29
 F test/shell4.test ddf0a99044e2245a87fc17423e3aaa1445b3243b
 F test/shell5.test c04e9f9f948305706b88377c464c7f08ce7479f9
+F test/shell6.test dc93ef3f42c5c385e66b97729374fa1c017ea5ed
 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3
 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5
 F test/shrink.test 1b4330b1fd9e818c04726d45cb28db73087535ce
@@ -1430,7 +1431,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 4ab3df25f1fee7c8fea19d0c64b3e0e4d3b9c3cf
-R 0c9cae22954d1f32b48770c565f201e2
+P 02fbf699c07286f842d9617755f071b0fffc5d40
+R a01d8436d914e04ca0b643f78b3bf6ab
 U dan
-Z cecc60af647c055f880ee4bdf2018855
+Z 48d2e83e55b4a0cbf023823a82d88ee7
diff --git a/manifest.uuid b/manifest.uuid
index 675cfe73ee..a71d648d17 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-02fbf699c07286f842d9617755f071b0fffc5d40
\ No newline at end of file
+0b73406595c9a077399b0f4c17af3a826cf3612f
\ No newline at end of file
diff --git a/src/shell_indexes.c b/src/shell_indexes.c
index cd8a5e7288..738a035f02 100644
--- a/src/shell_indexes.c
+++ b/src/shell_indexes.c
@@ -34,6 +34,7 @@ struct IdxConstraint {
   int iCol;                       /* Constrained table column */
   i64 depmask;                    /* Dependency mask */
   int bFlag;                      /* Used by idxFindCompatible() */
+  int bDesc;                      /* True if ORDER BY <expr> DESC */
   IdxConstraint *pNext;           /* Next constraint in pEq or pRange list */
   IdxConstraint *pLink;           /* See above */
 };
@@ -198,11 +199,17 @@ static void idxWhereInfo(
 
       case SQLITE_WHEREINFO_ORDERBY: {
         IdxConstraint *pNew = idxNewConstraint(&p->rc, zVal);
-        IdxConstraint **pp;
         if( pNew==0 ) return;
         pNew->iCol = iVal;
-        for(pp=&p->pScan->pOrder; *pp; pp=&(*pp)->pNext);
-        *pp = pNew;
+        pNew->bDesc = (int)mask;
+        if( p->pScan->pOrder==0 ){
+          p->pScan->pOrder = pNew;
+        }else{
+          IdxConstraint *pIter;
+          for(pIter=p->pScan->pOrder; pIter->pNext; pIter=pIter->pNext);
+          pIter->pNext = pNew;
+          pIter->pLink = pNew;
+        }
         break;
       }
 
@@ -508,6 +515,10 @@ static char *idxAppendColDefn(
       zRet = idxAppendText(pRc, zRet, " COLLATE %s", pCons->zColl);
     }
   }
+
+  if( pCons->bDesc ){
+    zRet = idxAppendText(pRc, zRet, " DESC");
+  }
   return zRet;
 }
 
@@ -812,7 +823,9 @@ int idxFindIndexes(
       }
       if( zIdx ){
         int nIdx = 0;
-        while( zIdx[nIdx]!='\0' && zIdx[nIdx]!=' ' ) nIdx++;
+        while( zIdx[nIdx]!='\0' && (zIdx[nIdx]!=' ' || zIdx[nIdx+1]!='(') ){
+          nIdx++;
+        }
         sqlite3_bind_text(pSelect, 1, zIdx, nIdx, SQLITE_STATIC);
         if( SQLITE_ROW==sqlite3_step(pSelect) ){
           i64 iRowid = sqlite3_column_int64(pSelect, 0);
diff --git a/src/where.c b/src/where.c
index 3724fa33e0..6275fc8d1e 100644
--- a/src/where.c
+++ b/src/where.c
@@ -3990,11 +3990,13 @@ static void whereTraceBuilder(
           for(i=0; i<p->pOrderBy->nExpr; i++){
             Expr *pExpr = p->pOrderBy->a[i].pExpr; 
             CollSeq *pColl = sqlite3ExprCollSeq(pParse, pExpr);
+            assert( pColl || pParse->rc );
             pExpr = sqlite3ExprSkipCollate(pExpr);
             if( pExpr->op==TK_COLUMN && pExpr->iTable==pItem->iCursor ){
               int iCol = pExpr->iColumn;
-              if( iCol>=0 ){
-                x(pCtx, SQLITE_WHEREINFO_ORDERBY, pColl->zName, iCol, 0); 
+              if( pColl && iCol>=0 ){
+                int bDesc = p->pOrderBy->a[i].sortOrder;
+                x(pCtx, SQLITE_WHEREINFO_ORDERBY, pColl->zName, iCol, bDesc); 
               }
             }
           }
diff --git a/test/shell6.test b/test/shell6.test
new file mode 100644
index 0000000000..2b200a2757
--- /dev/null
+++ b/test/shell6.test
@@ -0,0 +1,191 @@
+# 2009 Nov 11
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# The focus of this file is testing the CLI shell tool. Specifically,
+# the ".recommend" command.
+#
+#
+
+# Test plan:
+#
+#   shell1-1.*: Basic command line option handling.
+#   shell1-2.*: Basic "dot" command token parsing.
+#   shell1-3.*: Basic test that "dot" command can be called.
+#
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix shell6
+
+if {$tcl_platform(platform)=="windows"} {
+  set CLI "sqlite3.exe"
+} else {
+  set CLI "./sqlite3"
+}
+if {![file executable $CLI]} {
+  finish_test
+  return
+}
+
+
+proc squish {txt} {
+  regsub -all {[[:space:]]+} $txt { }
+}
+
+proc do_rec_test {tn sql res} {
+  set res [squish [string trim $res]]
+  set tst [subst -nocommands { 
+    squish [lindex [catchcmd [list -rec test.db {$sql;}]] 1]
+  }]
+  uplevel [list do_test $tn $tst $res]
+}
+
+proc do_setup_rec_test {tn setup sql res} {
+  reset_db
+  db eval $setup
+  uplevel [list do_rec_test $tn $sql $res]
+}
+
+
+do_setup_rec_test 1.1 { CREATE TABLE t1(a, b, c) } {
+  SELECT * FROM t1
+} {
+  (no new indexes)
+  0|0|0|SCAN TABLE t1
+}
+
+do_setup_rec_test 1.2 {
+  CREATE TABLE t1(a, b, c);
+} {
+  SELECT * FROM t1 WHERE b>?;
+} {
+  CREATE INDEX t1_idx_00000062 ON t1(b) 
+  0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_00000062 (b>?)
+}
+
+do_setup_rec_test 1.3 {
+  CREATE TABLE t1(a, b, c);
+} {
+  SELECT * FROM t1 WHERE b COLLATE nocase BETWEEN ? AND ?
+} {
+  CREATE INDEX t1_idx_3e094c27 ON t1(b COLLATE NOCASE) 
+  0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_3e094c27 (b>? AND b<?)
+}
+
+do_setup_rec_test 1.4 {
+  CREATE TABLE t1(a, b, c);
+} {
+  SELECT a FROM t1 ORDER BY b;
+} {
+  CREATE INDEX t1_idx_00000062 ON t1(b) 
+  0|0|0|SCAN TABLE t1 USING INDEX t1_idx_00000062
+}
+
+do_setup_rec_test 1.5 {
+  CREATE TABLE t1(a, b, c);
+} {
+  SELECT a FROM t1 WHERE a=? ORDER BY b;
+} {
+  CREATE INDEX t1_idx_000123a7 ON t1(a, b) 
+  0|0|0|SEARCH TABLE t1 USING COVERING INDEX t1_idx_000123a7 (a=?)
+}
+
+do_setup_rec_test 1.6 {
+  CREATE TABLE t1(a, b, c);
+} {
+  SELECT min(a) FROM t1
+} {
+  CREATE INDEX t1_idx_00000061 ON t1(a) 
+  0|0|0|SEARCH TABLE t1 USING COVERING INDEX t1_idx_00000061
+}
+
+do_setup_rec_test 1.7 {
+  CREATE TABLE t1(a, b, c);
+} {
+  SELECT * FROM t1 ORDER BY a, b, c;
+} {
+  CREATE INDEX t1_idx_033e95fe ON t1(a, b, c) 
+  0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_033e95fe
+}
+
+do_setup_rec_test 1.8 {
+  CREATE TABLE t1(a, b, c);
+} {
+  SELECT * FROM t1 ORDER BY a ASC, b COLLATE nocase DESC, c ASC;
+} {
+  CREATE INDEX t1_idx_5be6e222 ON t1(a, b COLLATE NOCASE DESC, c)
+  0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_5be6e222
+}
+
+do_setup_rec_test 1.9 {
+  CREATE TABLE t1(a COLLATE NOCase, b, c);
+} {
+  SELECT * FROM t1 WHERE a=?
+} {
+  CREATE INDEX t1_idx_00000061 ON t1(a) 
+  0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_00000061 (a=?)
+}
+
+
+# Tables with names that require quotes.
+#
+do_setup_rec_test 8.1 {
+  CREATE TABLE "t t"(a, b, c);
+} {
+  SELECT * FROM "t t" WHERE a=?
+} {
+  CREATE INDEX 't t_idx_00000061' ON 't t'(a)
+  0|0|0|SEARCH TABLE t t USING INDEX t t_idx_00000061 (a=?) 
+}
+
+do_setup_rec_test 8.2 {
+  CREATE TABLE "t t"(a, b, c);
+} {
+  SELECT * FROM "t t" WHERE b BETWEEN ? AND ?
+} {
+  CREATE INDEX 't t_idx_00000062' ON 't t'(b) 
+  0|0|0|SEARCH TABLE t t USING INDEX t t_idx_00000062 (b>? AND b<?)
+}
+
+# Columns with names that require quotes.
+#
+do_setup_rec_test 9.1 {
+  CREATE TABLE t3(a, "b b", c);
+} {
+  SELECT * FROM t3 WHERE "b b" = ?
+} {
+  CREATE INDEX t3_idx_00050c52 ON t3('b b') 
+  0|0|0|SEARCH TABLE t3 USING INDEX t3_idx_00050c52 (b b=?)
+}
+
+do_setup_rec_test 9.2 {
+  CREATE TABLE t3(a, "b b", c);
+} {
+  SELECT * FROM t3 ORDER BY "b b"
+} {
+  CREATE INDEX t3_idx_00050c52 ON t3('b b') 
+  0|0|0|SCAN TABLE t3 USING INDEX t3_idx_00050c52
+}
+
+# Transitive constraints
+#
+do_setup_rec_test 10.1 {
+  CREATE TABLE t5(a, b);
+  CREATE TABLE t6(c, d);
+} {
+  SELECT * FROM t5, t6 WHERE a=? AND b=c AND c=?
+} {
+  CREATE INDEX t6_idx_00000063 ON t6(c) 
+  CREATE INDEX t5_idx_000123a7 ON t5(a, b) 
+  0|0|1|SEARCH TABLE t6 USING INDEX t6_idx_00000063 (c=?) 
+  0|1|0|SEARCH TABLE t5 USING COVERING INDEX t5_idx_000123a7 (a=? AND b=?)
+}
+
+finish_test

From 94c67eca890b703f6f14218553adc89d4781ac95 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 9 Mar 2016 08:07:31 +0000
Subject: [PATCH 16/68] Fix a problem in the schemalint code that comes up when
 a sub-query uses one or more of the same tables as its parent.

FossilOrigin-Name: fc18cc9293fb0080b7152c16baac49f44e2db7b3
---
 manifest             |  17 ++++----
 manifest.uuid        |   2 +-
 src/shell_indexes.c  |  48 ++++++++++++++++++---
 src/where.c          |   1 -
 test/schemalint.test | 100 -------------------------------------------
 test/shell6.test     |   2 +-
 6 files changed, 52 insertions(+), 118 deletions(-)
 delete mode 100644 test/schemalint.test

diff --git a/manifest b/manifest
index 1ea78374dd..7b7e824272 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\stest\sscript\sshell6.test,\scontaining\stests\sfor\sschemalint.
-D 2016-02-22T19:51:08.971
+C Fix\sa\sproblem\sin\sthe\sschemalint\scode\sthat\scomes\sup\swhen\sa\ssub-query\suses\sone\sor\smore\sof\sthe\ssame\stables\sas\sits\sparent.
+D 2016-03-09T08:07:31.577
 F Makefile.in dac2776c84e0d533b158a9af6e57e05c4a6b19f3
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc b0493f10caddb8adf992a4e6f1943141fc7c6816
@@ -350,7 +350,7 @@ F src/resolve.c 9f7ce3a3c087afb7597b7c916c99126ff3f12f0c
 F src/rowset.c 9fe4b3ad7cc00944386bb600233d8f523de07a6e
 F src/select.c ff80004a9a6ece891a8d9327a88e7b6e2588ee6d
 F src/shell.c 2cde87e03712204231167c4a6c61b0eb5129e105
-F src/shell_indexes.c e10b3c2c4bc9a87f7bff1cf622473717b1a00698
+F src/shell_indexes.c 23c4b7f7bb8e9b0a8b912c389a85922939bb0769
 F src/sqlite.h.in c7db059d3b810b70b83d9ed1436fa813eba22462
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h dfbe62ffd95b99afe2140d8c35b180d11924072d
@@ -428,7 +428,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c d21b99fd1458159d0b1ecdccc8ee6ada4fdc4c54
 F src/wal.h 2f7c831cf3b071fa548bf2d5cac640846a7ff19c
 F src/walker.c 0f142b5bd3ed2041fc52d773880748b212e63354
-F src/where.c 0ecce6da62ad521cac83dfa1b60a37936b36b6a1
+F src/where.c 59f267a53d29db13880597011323745b69e14f24
 F src/whereInt.h 78b6b4de94db84aecbdc07fe3e38f648eb391e9a
 F src/wherecode.c 791a784bbf8749d560fdb0b990b607bc4f44a38d
 F src/whereexpr.c de117970b29471177a6901d60ad83a194671dc03
@@ -988,7 +988,6 @@ F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5
 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38
 F test/schema4.test e6a66e20cc69f0e306667c08be7fda3d11707dc5
 F test/schema5.test 29699b4421f183c8f0e88bd28ce7d75d13ea653e
-F test/schemalint.test cee9f375637fcc8c6e77d971abe044445c23e024
 F test/securedel.test 21749c32ccc30f1ea9e4b9f33295a6521ec20fa0
 F test/securedel2.test 2d54c28e46eb1fd6902089958b20b1b056c6f1c5
 F test/select1.test be62204d2bd9a5a8a149e9974cfddce893d8f686
@@ -1025,7 +1024,7 @@ F test/shell2.test 12b8bf901b0e3a8ac58cf5c0c63a0a388d4d1862
 F test/shell3.test 5e8545ec72c4413a0e8d4c6be56496e3c257ca29
 F test/shell4.test ddf0a99044e2245a87fc17423e3aaa1445b3243b
 F test/shell5.test c04e9f9f948305706b88377c464c7f08ce7479f9
-F test/shell6.test dc93ef3f42c5c385e66b97729374fa1c017ea5ed
+F test/shell6.test 4ce53ce53e1206742e003b5ae609b6c2be438df1
 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3
 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5
 F test/shrink.test 1b4330b1fd9e818c04726d45cb28db73087535ce
@@ -1431,7 +1430,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 02fbf699c07286f842d9617755f071b0fffc5d40
-R a01d8436d914e04ca0b643f78b3bf6ab
+P 0b73406595c9a077399b0f4c17af3a826cf3612f
+R d77c52a71510d468f62d262501df9c9f
 U dan
-Z 48d2e83e55b4a0cbf023823a82d88ee7
+Z c51c58b805618dc47dc45d8f0cd33733
diff --git a/manifest.uuid b/manifest.uuid
index a71d648d17..e9344a7af6 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0b73406595c9a077399b0f4c17af3a826cf3612f
\ No newline at end of file
+fc18cc9293fb0080b7152c16baac49f44e2db7b3
\ No newline at end of file
diff --git a/src/shell_indexes.c b/src/shell_indexes.c
index 738a035f02..c5e172cb97 100644
--- a/src/shell_indexes.c
+++ b/src/shell_indexes.c
@@ -413,7 +413,6 @@ static int idxGetTableInfo(
   return rc;
 }
 
-
 static int idxCreateTables(
   sqlite3 *db,                    /* User database */
   sqlite3 *dbm,                   /* In-memory database to create tables in */
@@ -424,6 +423,22 @@ static int idxCreateTables(
   IdxScan *pIter;
   for(pIter=pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
     rc = idxGetTableInfo(db, pIter, pzErrmsg);
+
+    /* Test if table has already been created. If so, jump to the next
+    ** iteration of the loop.  */
+    if( rc==SQLITE_OK ){
+      sqlite3_stmt *pSql = 0;
+      rc = idxPrintfPrepareStmt(dbm, &pSql, pzErrmsg, 
+          "SELECT 1 FROM sqlite_master WHERE tbl_name = %Q", pIter->zTable
+      );
+      if( rc==SQLITE_OK ){
+        int bSkip = 0;
+        if( sqlite3_step(pSql)==SQLITE_ROW ) bSkip = 1;
+        rc = sqlite3_finalize(pSql);
+        if( bSkip ) continue;
+      }
+    }
+
     if( rc==SQLITE_OK ){
       int rc2;
       sqlite3_stmt *pSql = 0;
@@ -799,6 +814,7 @@ int idxFindIndexes(
   sqlite3 *dbm = pCtx->dbm;
   sqlite3_stmt *pExplain = 0;
   sqlite3_stmt *pSelect = 0;
+  sqlite3_stmt *pInsert = 0;
   int rc, rc2;
   int bFound = 0;
 
@@ -808,6 +824,11 @@ int idxFindIndexes(
         "SELECT rowid, sql FROM sqlite_master WHERE name = ?"
     );
   }
+  if( rc==SQLITE_OK ){
+    rc = idxPrepareStmt(dbm, &pInsert, pzErr,
+        "INSERT OR IGNORE INTO aux.indexes VALUES(?)"
+    );
+  }
 
   while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){
     int i;
@@ -831,8 +852,10 @@ int idxFindIndexes(
           i64 iRowid = sqlite3_column_int64(pSelect, 0);
           const char *zSql = (const char*)sqlite3_column_text(pSelect, 1);
           if( iRowid>=pCtx->iIdxRowid ){
-            xOut(pOutCtx, zSql);
-            bFound = 1;
+            sqlite3_bind_text(pInsert, 1, zSql, -1, SQLITE_STATIC);
+            sqlite3_step(pInsert);
+            rc = sqlite3_reset(pInsert);
+            if( rc ) goto find_indexes_out;
           }
         }
         rc = sqlite3_reset(pSelect);
@@ -843,8 +866,19 @@ int idxFindIndexes(
   rc2 = sqlite3_reset(pExplain);
   if( rc==SQLITE_OK ) rc = rc2;
   if( rc==SQLITE_OK ){
-    if( bFound==0 ) xOut(pOutCtx, "(no new indexes)");
-    xOut(pOutCtx, "");
+    sqlite3_stmt *pLoop = 0;
+    rc = idxPrepareStmt(dbm, &pLoop, pzErr, "SELECT name FROM aux.indexes");
+    if( rc==SQLITE_OK ){
+      while( SQLITE_ROW==sqlite3_step(pLoop) ){
+        bFound = 1;
+        xOut(pOutCtx, sqlite3_column_text(pLoop, 0));
+      }
+      rc = sqlite3_finalize(pLoop);
+    }
+    if( rc==SQLITE_OK ){
+      if( bFound==0 ) xOut(pOutCtx, "(no new indexes)");
+      xOut(pOutCtx, "");
+    }
   }
 
   while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){
@@ -868,6 +902,8 @@ int idxFindIndexes(
   if( rc==SQLITE_OK ) rc = rc2;
   rc2 = sqlite3_finalize(pSelect);
   if( rc==SQLITE_OK ) rc = rc2;
+  rc2 = sqlite3_finalize(pInsert);
+  if( rc==SQLITE_OK ) rc = rc2;
 
   return rc;
 }
@@ -911,7 +947,7 @@ int shellIndexesCommand(
   /* Prepare an INSERT statement for writing to aux.depmask */
   if( rc==SQLITE_OK ){
     rc = idxPrepareStmt(dbm, &ctx.pInsertMask, pzErrmsg,
-        "INSERT OR IGNORE INTO depmask SELECT mask | ?1 FROM depmask;"
+        "INSERT OR IGNORE INTO aux.depmask SELECT mask | ?1 FROM aux.depmask;"
     );
   }
 
diff --git a/src/where.c b/src/where.c
index 6275fc8d1e..abe1015b95 100644
--- a/src/where.c
+++ b/src/where.c
@@ -3990,7 +3990,6 @@ static void whereTraceBuilder(
           for(i=0; i<p->pOrderBy->nExpr; i++){
             Expr *pExpr = p->pOrderBy->a[i].pExpr; 
             CollSeq *pColl = sqlite3ExprCollSeq(pParse, pExpr);
-            assert( pColl || pParse->rc );
             pExpr = sqlite3ExprSkipCollate(pExpr);
             if( pExpr->op==TK_COLUMN && pExpr->iTable==pItem->iCursor ){
               int iCol = pExpr->iColumn;
diff --git a/test/schemalint.test b/test/schemalint.test
deleted file mode 100644
index 7e0eba494d..0000000000
--- a/test/schemalint.test
+++ /dev/null
@@ -1,100 +0,0 @@
-
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-set testprefix schemalint
-
-proc xTrace {zMsg} {
-  lappend ::trace_out $zMsg
-}
-db trace xTrace
-
-proc do_trace_test {tn sql res} {
-  uplevel [list do_test $tn [subst -nocommands {
-    set ::trace_out [list]
-    set stmt [sqlite3_prepare db "$sql" -1 x]
-    sqlite3_finalize [set stmt]
-    set ::trace_out
-  }] [list {*}$res]]
-}
-
-
-do_execsql_test 1.0 {
-  CREATE TABLE t1(a, b, c);
-  CREATE TABLE t2(x, y, z);
-}
-
-do_trace_test 1.1 {
-  SELECT b, c, y, z FROM t1, t2 WHERE c=? AND z=?
-} {
-  {"t1" {cols "b" "c"} {eq "c" "BINARY" 0}}
-  {"t2" {cols "y" "z"} {eq "z" "BINARY" 0}}
-}
-
-do_trace_test 1.2 {
-  SELECT a FROM t1 WHERE b>10
-} {
-  {"t1" {cols "a" "b"} {range "b" "BINARY" 0}}
-}
-
-do_trace_test 1.3 {
-  SELECT b FROM t1 WHERE b IN (10, 20, 30)
-} {
-  {"t1" {cols "b"} {eq "b" "BINARY" 0}}
-}
-
-do_trace_test 1.4 {
-  SELECT * FROM t1, t2 WHERE x=a
-} {
-  {"t1" {cols "a" "b" "c"} {eq "a" "BINARY" 2}} 
-  {"t2" {cols "x" "y" "z"} {eq "x" "BINARY" 1}}
-}
-
-do_trace_test 1.5 {
-  SELECT * FROM t1 WHERE a IN (1, 2, 3)
-} {
-  {"t1" {cols "a" "b" "c"} {eq "a" "BINARY" 0}}
-}
-
-#-----------------------------------------------------------------------
-# Cases involving OR clauses in the WHERE clause.
-#
-do_trace_test 2.1 {
-  SELECT * FROM t1 WHERE a=? OR b=?
-} {
-  {"t1" {cols "a" "b" "c"} {or {{eq "a" "BINARY" 0}} {{eq "b" "BINARY" 0}}}}
-}
-
-do_trace_test 2.2 {
-  SELECT * FROM t1 WHERE a=? OR (b=? AND c=?)
-} {
-  {"t1" {cols "a" "b" "c"} {or {{eq "a" "BINARY" 0}} {{eq "b" "BINARY" 0} {eq "c" "BINARY" 0}}}}
-}
-
-do_trace_test 2.3 {
-  SELECT * FROM t1 WHERE (a=? AND b=?) OR c=?
-} {
-  {"t1" {cols "a" "b" "c"} {or {{eq "c" "BINARY" 0}} {{eq "a" "BINARY" 0} {eq "b" "BINARY" 0}}}}
-}
-
-#-----------------------------------------------------------------------
-# Cases involving ORDER BY.
-#
-do_trace_test 3.1 {
-  SELECT * FROM t1 ORDER BY a;
-} {{"t1" {cols "a" "b" "c"} {orderby "a" "BINARY" ASC}}}
-
-do_trace_test 3.2 {
-  SELECT * FROM t1 WHERE a=? ORDER BY b;
-} {{"t1" {cols "a" "b" "c"} {eq "a" "BINARY" 0} {orderby "b" "BINARY" ASC}}}
-
-do_trace_test 3.3 {
-  SELECT min(a) FROM t1;
-} {{"t1" {cols "a"} {orderby "a" "BINARY" ASC}}}
-
-do_trace_test 3.4 {
-  SELECT max(a) FROM t1;
-} {{"t1" {cols "a"} {orderby "a" "BINARY" DESC}}}
-
-finish_test
-
diff --git a/test/shell6.test b/test/shell6.test
index 2b200a2757..ce8c24ceba 100644
--- a/test/shell6.test
+++ b/test/shell6.test
@@ -182,8 +182,8 @@ do_setup_rec_test 10.1 {
 } {
   SELECT * FROM t5, t6 WHERE a=? AND b=c AND c=?
 } {
-  CREATE INDEX t6_idx_00000063 ON t6(c) 
   CREATE INDEX t5_idx_000123a7 ON t5(a, b) 
+  CREATE INDEX t6_idx_00000063 ON t6(c) 
   0|0|1|SEARCH TABLE t6 USING INDEX t6_idx_00000063 (c=?) 
   0|1|0|SEARCH TABLE t5 USING COVERING INDEX t5_idx_000123a7 (a=? AND b=?)
 }

From b5c39ac0c916f03c85f3b1e328c554632969bf45 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 31 Mar 2017 08:00:03 +0000
Subject: [PATCH 17/68] Update shell6.test to account for the fact that tests
 are now run in a separate directory.

FossilOrigin-Name: 1e3622de8b51fdb219eb1934533a95fe6dc5158f44d358bc2efdcb53a4a9fdab
---
 manifest         | 12 ++++++------
 manifest.uuid    |  2 +-
 test/shell6.test |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 4540ae7f75..b8f39b0c2a 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\sthis\sbranch\swith\slatest\strunk\schanges.
-D 2016-03-15T09:42:39.387
+C Update\sshell6.test\sto\saccount\sfor\sthe\sfact\sthat\stests\sare\snow\srun\sin\sa\nseparate\sdirectory.
+D 2017-03-31T08:00:03.133
 F Makefile.in f53429fb2f313c099283659d0df6f20f932c861f
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc df0bf9ff7f8b3f4dd9fb4cc43f92fe58f6ec5c66
@@ -1031,7 +1031,7 @@ F test/shell2.test 12b8bf901b0e3a8ac58cf5c0c63a0a388d4d1862
 F test/shell3.test 5e8545ec72c4413a0e8d4c6be56496e3c257ca29
 F test/shell4.test f43e250139dc5dc5f0f2ec1752c50284a1ede102
 F test/shell5.test c04e9f9f948305706b88377c464c7f08ce7479f9
-F test/shell6.test 4ce53ce53e1206742e003b5ae609b6c2be438df1
+F test/shell6.test 05c2488ff2bb11ac300b9170e8f5b6e03d58982c77035cdca5dbf036f0d3c5b7
 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3
 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5
 F test/shrink.test 1b4330b1fd9e818c04726d45cb28db73087535ce
@@ -1459,7 +1459,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 59caca4387e97231834a3bffdbed8636be4e8e19 9f194f90c07d5415b1d6bb10ec049f74999d6885
-R 5b90dca0c628375a48db229132bb19b2
+P 249cd361b840913794b7cd2f2d42777dcd547a60
+R b5c9ea0357d4d69dec4a40a72a5670a9
 U dan
-Z dd17403945ec5cf2087723aff15fb7cd
+Z 84e2a466cf9b475b6898bdfe1581ffe0
diff --git a/manifest.uuid b/manifest.uuid
index 234da9ff63..8cfc3a9b80 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-249cd361b840913794b7cd2f2d42777dcd547a60
\ No newline at end of file
+1e3622de8b51fdb219eb1934533a95fe6dc5158f44d358bc2efdcb53a4a9fdab
\ No newline at end of file
diff --git a/test/shell6.test b/test/shell6.test
index ce8c24ceba..19b094a80d 100644
--- a/test/shell6.test
+++ b/test/shell6.test
@@ -27,7 +27,7 @@ set testprefix shell6
 if {$tcl_platform(platform)=="windows"} {
   set CLI "sqlite3.exe"
 } else {
-  set CLI "./sqlite3"
+  set CLI ".././sqlite3"
 }
 if {![file executable $CLI]} {
   finish_test

From e86573fa512a2e06e7ae0b737c0b839974355598 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 4 Apr 2017 04:23:06 +0000
Subject: [PATCH 18/68] Add the sqlite3_whereinfo_hook() API - an experimental
 API replacing the DBCONFIG_WHEREINFO hack on this branch.

FossilOrigin-Name: a54aef35da11f7508a8888a159372036a362fc52afa1df752dc835db334c4330
---
 manifest            |  22 +++----
 manifest.uuid       |   2 +-
 src/main.c          |  27 ++++++--
 src/shell_indexes.c |  33 ++--------
 src/sqlite.h.in     | 152 ++++++++++++++++++++++++++++++++++++++++----
 src/sqliteInt.h     |   4 +-
 src/where.c         |  38 +++--------
 7 files changed, 189 insertions(+), 89 deletions(-)

diff --git a/manifest b/manifest
index 90f504d2aa..fc73fb3e18 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\slatest\strunk\sinto\sthis\sbranch.
-D 2017-03-31T11:20:20.468
+C Add\sthe\ssqlite3_whereinfo_hook()\sAPI\s-\san\sexperimental\sAPI\sreplacing\sthe\nDBCONFIG_WHEREINFO\shack\son\sthis\sbranch.
+D 2017-04-04T04:23:06.108
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a
@@ -36,7 +36,7 @@ F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad
 F doc/lemon.html b5a3c07d33ecb8e019ce8f7660fe2dbbad9d7977
 F doc/pager-invariants.txt 27fed9a70ddad2088750c4a2b493b63853da2710
 F doc/vfs-shm.txt e101f27ea02a8387ce46a05be2b1a902a021d37a
-F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd w ext/README.txt
+F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd
 F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
@@ -366,7 +366,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c d4bb3a135948553d18cf992f76f7ed7b18aa0327f250607b5a6671e55d9947d5
 F src/legacy.c e88ed13c2d531decde75d42c2e35623fb9ce3cb0
 F src/loadext.c a72909474dadce771d3669bf84bf689424f6f87d471fee898589c3ef9b2acfd9
-F src/main.c 36974edf7461f708b337389c59b30ec91257420c129c1f5ee022bc2c2784d320
+F src/main.c 4b93bda0f1f916f4cb618a6fce358f7e01ab060971c586c4cea90ad0c83a83f5
 F src/malloc.c 89c98e3619d362dcffa5c1c639b364b65b474751
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -403,11 +403,11 @@ F src/resolve.c 3e518b962d932a997fae373366880fc028c75706
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 2496d0cc6368dabe7ad2e4c7f5ed3ad9aa3b4d11cd90f33fa1d1ef72493f43aa
 F src/shell.c e524688c2544167f835ba43e24309f8707ca60c8ab6eb5c263a12c8618a233b8
-F src/shell_indexes.c 23c4b7f7bb8e9b0a8b912c389a85922939bb0769
-F src/sqlite.h.in 3db120c750634c0acc1ca291140a4cae75cb41cb0507f53c31b838c040954366
+F src/shell_indexes.c 12cc58b62492bddc88f879086b4cd1045dbfd4e52040ecb6c39b971a6aac80ba
+F src/sqlite.h.in ae5c9cbf2e77492c319fca08769575d9695e64718a16d32324944d24e291bcf7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 58fd0676d3111d02e62e5a35992a7d3da5d3f88753acc174f2d37b774fbbdd28
-F src/sqliteInt.h 22a0a101c51a0fae4156088f7c613b99a73d20ba6153a1400d1ae3011cf1082c
+F src/sqliteInt.h 9af1569ac748f72169b4ad0004d03ea14c75b59eb955a2c0ecca75a16ea2015f
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -483,7 +483,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 40c543f0a2195d1b0dc88ef12142bea690009344
 F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71
 F src/walker.c b71a992b413b3a022572eccf29ef4b4890223791
-F src/where.c 30c790ab98b759ac3d7fa564e1321489151fac7d1587525bc5b9698394a41ecf
+F src/where.c cbd1e8562fa37de359963749e1dc3d0dd1a42a894d10a62928bffc709490fde6
 F src/whereInt.h 2d50c2b74a33be44cb68fdecee30b4d93552f1f4
 F src/wherecode.c 677e95413c472c0b413023b6b69a47f40fce1b04
 F src/whereexpr.c 130cdd1a43af71b19755270fb1224874cf55158c
@@ -1571,7 +1571,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 1e3622de8b51fdb219eb1934533a95fe6dc5158f44d358bc2efdcb53a4a9fdab c8000e94cca59dabf83d6cb75b40441aaf793d29880582dc4baa17246449b5fe
-R a9131615092ed98b863a64f66624daea
+P cb721d0b36268a7b0ef493fa4d7f6bcbaa9ead8b1990e3c3fae015fa1d545226
+R 061e66f6528edc67cfbe5cb82be7b02d
 U dan
-Z 924aec6608b5903c59e65e7507c45eb8
+Z d08fd2524fbe07ca7104a150ba1edcbe
diff --git a/manifest.uuid b/manifest.uuid
index 1b590b1f1f..64b0731eb5 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-cb721d0b36268a7b0ef493fa4d7f6bcbaa9ead8b1990e3c3fae015fa1d545226
\ No newline at end of file
+a54aef35da11f7508a8888a159372036a362fc52afa1df752dc835db334c4330
\ No newline at end of file
diff --git a/src/main.c b/src/main.c
index eed82b0e66..df715852ae 100644
--- a/src/main.c
+++ b/src/main.c
@@ -801,13 +801,6 @@ int sqlite3_db_config(sqlite3 *db, int op, ...){
       rc = setupLookaside(db, pBuf, sz, cnt);
       break;
     }
-#ifdef SQLITE_SCHEMA_LINT
-    case SQLITE_DBCONFIG_WHEREINFO: {
-      db->xWhereInfo = va_arg(ap, void(*)(void*, int, const char*, int, i64));
-      db->pWhereInfoCtx = va_arg(ap, void*);
-      break;
-    }
-#endif
     default: {
       static const struct {
         int op;      /* The opcode */
@@ -2004,6 +1997,26 @@ void *sqlite3_preupdate_hook(
 }
 #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */
 
+#ifdef SQLITE_ENABLE_WHEREINFO_HOOK
+/*
+** Register a where-info hook.
+*/
+void *sqlite3_whereinfo_hook(
+  sqlite3 *db,                    /* Register callback with this db handle */
+  void (*xWhereInfo)(void*, int, const char*, int, sqlite3_uint64),
+  void *pCtx                      /* User pointer passed to callback */
+){
+  void *pRet;
+  sqlite3_mutex_enter(db->mutex);
+  pRet = db->pWhereInfoCtx;
+  db->xWhereInfo = xWhereInfo;
+  db->pWhereInfoCtx = pCtx;
+  sqlite3_mutex_leave(db->mutex);
+  return pRet;
+}
+#endif /* SQLITE_ENABLE_WHEREINFO_HOOK */
+
+
 #ifndef SQLITE_OMIT_WAL
 /*
 ** The sqlite3_wal_hook() callback registered by sqlite3_wal_autocheckpoint().
diff --git a/src/shell_indexes.c b/src/shell_indexes.c
index c5e172cb97..efdc24fac3 100644
--- a/src/shell_indexes.c
+++ b/src/shell_indexes.c
@@ -12,6 +12,7 @@
 */
 
 typedef sqlite3_int64 i64;
+typedef sqlite3_uint64 u64;
 
 typedef struct IdxConstraint IdxConstraint;
 typedef struct IdxContext IdxContext;
@@ -164,7 +165,7 @@ static void idxWhereInfo(
   int eOp, 
   const char *zVal, 
   int iVal, 
-  i64 mask
+  u64 mask
 ){
   IdxContext *p = (IdxContext*)pCtx;
 
@@ -233,32 +234,6 @@ static void idxWhereInfo(
         p->rc = sqlite3_reset(p->pInsertMask);
         break;
       }
-
-      case SQLITE_WHEREINFO_BEGINOR: {
-        IdxWhere *pNew = (IdxWhere*)idxMalloc(&p->rc, sizeof(IdxWhere));
-        if( pNew==0 ) return;
-        pNew->pParent = p->pCurrent;
-        pNew->pNextOr = p->pCurrent->pOr;
-        p->pCurrent->pOr = pNew;
-        p->pCurrent = pNew;
-        break;
-      }
-
-      case SQLITE_WHEREINFO_NEXTOR: {
-        IdxWhere *pNew = (IdxWhere*)idxMalloc(&p->rc, sizeof(IdxWhere));
-        if( pNew==0 ) return;
-        pNew->pParent = p->pCurrent->pParent;
-        assert( p->pCurrent->pSibling==0 );
-        p->pCurrent->pSibling = pNew;
-        p->pCurrent = pNew;
-        break;
-      }
-
-      case SQLITE_WHEREINFO_ENDOR: {
-        assert( p->pCurrent->pParent );
-        p->pCurrent = p->pCurrent->pParent;
-        break;
-      }
     }
   }
 }
@@ -954,9 +929,9 @@ int shellIndexesCommand(
   /* Analyze the SELECT statement in zSql. */
   if( rc==SQLITE_OK ){
     ctx.dbm = dbm;
-    sqlite3_db_config(db, SQLITE_DBCONFIG_WHEREINFO, idxWhereInfo, (void*)&ctx);
+    sqlite3_whereinfo_hook(db, idxWhereInfo, (void*)&ctx);
     rc = idxPrepareStmt(db, &pStmt, pzErrmsg, zSql);
-    sqlite3_db_config(db, SQLITE_DBCONFIG_WHEREINFO, (void*)0, (void*)0);
+    sqlite3_whereinfo_hook(db, 0, 0);
     sqlite3_finalize(pStmt);
   }
 
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index f3067b64eb..3f9fb2fccb 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -2017,17 +2017,6 @@ struct sqlite3_mem_methods {
 #define SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION 1005 /* int int* */
 #define SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE      1006 /* int int* */
 
-#define SQLITE_DBCONFIG_WHEREINFO             1007  /* xWhereInfo void* */
-
-#define SQLITE_WHEREINFO_TABLE   1
-#define SQLITE_WHEREINFO_EQUALS  2
-#define SQLITE_WHEREINFO_RANGE   3
-#define SQLITE_WHEREINFO_ORDERBY 4
-#define SQLITE_WHEREINFO_BEGINOR 5
-#define SQLITE_WHEREINFO_ENDOR   6
-#define SQLITE_WHEREINFO_NEXTOR  7
-
-
 /*
 ** CAPI3REF: Enable Or Disable Extended Result Codes
 ** METHOD: sqlite3
@@ -8496,6 +8485,147 @@ SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp(
 */
 SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb);
 
+/*
+** This function is used to register a whereinfo hook with the database
+** handle passed as the first argument. Once registered, the whereinfo hook
+** is invoked zero or more times while preparing a query to provide
+** information to the application. It is intended to be used by expert 
+** systems to recommend indexes that could be added to the database in order
+** to improve query response time.
+**
+** An SQLite query plan describes the way data is read from zero or more
+** database tables. For each table read, the data required may be
+** constrained by equality or range constraints on one or more columns, 
+** and it may or may not be required in order sorted by one or more 
+** columns. For example, the following query:
+**
+** <pre>
+**     SELECT * FROM t1 WHERE t1.a = ? ORDER BY t1.b;
+** </pre>
+**
+** reads data from table t1. It requires only those rows for which t1.a
+** is set to a specific value, and requires them sorted in order of
+** column t1.b. Internally, SQLite determines this and attempts to locate
+** an index that can be used to efficiently find the required subset of
+** rows and/or allows the rows to be read from the database in the 
+** required order. In this case, ideally an index of the form:
+**
+** <pre>
+**     CREATE INDEX i1 ON t1(a, b);
+** </pre>
+**
+** The data passed to the whereinfo hook during query preparation do
+** not describe the actual query plan to the application. Instead, it
+** describes the parts of the query that SQLite could use an index to
+** optimize if a suitable index existed. In this case, that only
+** rows with t1.a=? are required, and that they are required sorted
+** in order by t1.b.
+**
+** Each time the whereinfo hook is invoked, the first argument is a
+** copy of the (void*) pointer passed as the second argument to this
+** API function. The second is always one of the SQLITE_WHEREINFO_XXX
+** constants defined below.
+**
+** For each table read by a query, the whereinfo hook is invoked as follows:
+**
+** <ul>
+**   <li> Once with SQLITE_WHEREINFO_TABLE as the second argument. This
+**        indicates the table that subsequent callbacks (until the next
+**        SQLITE_WHEREINFO_TABLE) apply to.
+**
+**   <li> If SQLite requires rows in a specific order, once with
+**        SQLITE_WHEREINFO_ORDERBY for each column of the table that is
+**        one of the sort keys.
+**
+**   <li> If there are any "<col> = ?" constraints that restrict the rows
+**        required by SQLite, once with SQLITE_WHEREINFO_EQUALS for each
+**        such constraint.
+**
+**   <li> If there are any "<col> > ?" constraints (or any other range
+**        constraint) that restrict the rows required by SQLite, once with 
+**        SQLITE_WHEREINFO_RANGE for each such constraint.
+** </ul>
+**
+** The third, fourth and fifth arguments passed to the whereinfo callback
+** are interpreted differently, depending on the SQLITE_WHEREINFO_XXX value
+** as follows:
+**
+** <dl>
+**   <dt> SQLITE_WHEREINFO_TABLE
+**   <dd> The third argument passed in this case is the name of the table.
+**        The fourth is the index of the database in which the table is
+**        located (0 for "main", 1 for "temp" or higher for an attached
+**        database). The fifth argument is a bitmask that indicates which
+**        of the tables columns may be required by the query. If the leftmost
+**        column of the table is used in some way, bit 0 of the bitmask is
+**        set. If the next-to-leftmost is used, bit 1 etc. Bit 63 is used to
+**        represent all columns with an index of 63 or higher. If bit 63
+**        is set, the application should assume that the query requires all
+**        columns from the table with an index of 63 or greater.
+**
+**   <dt> SQLITE_WHEREINFO_ORDERBY
+**   <dd> The third argument passed in this case is the name of the collation
+**        sequence to sort by. The fourth is the index of the table column to
+**        sort by (0 for the leftmost column, 1 for the next-to-leftmost
+**        etc.). The fifth argument is a boolean flag - true for a DESC sort
+**        or false for ASC.
+**
+**   <dt> SQLITE_WHEREINFO_EQUALS
+**   <dd> The third argument passed in this case is the name of the collation
+**        sequence used by the constraint. The fourth is the index of the 
+**        table column in the constraint. If the current table is not part
+**        of a join, then the value passed as the fifth argument is always
+**        zero. Or, if it is part of a join, then the fifth parameter passed
+**        to this callback is a mask of other tables that the current
+**        constraint depends on. For example, in the query:
+**
+**   <pre>
+**      SELECT * FROM t1, t2 WHERE t1.a = (t2.b+1);
+**   </pre>
+**
+**        the fifth parameter passed to the the SQLITE_WHEREINFO_EQUALS
+**        whereinfo callback would have the bit assigned to table "t2"
+**        set to true. There is no way for the application to determine 
+**        the specific bit in the mask assigned to any table, but the bit
+**        assignments are consistent while parsing a single query.
+**
+**   <dt> SQLITE_WHEREINFO_RANGE
+**   <dd> As for SQLITE_WHEREINFO_EQUALS.
+** </dl>
+**
+** Note that if a WHERE clause includes an OR expression, then there may be
+** more than one set of callbacks for a single table. For example, the
+** following SQL:
+**
+** <pre>
+**    SELECT * FROM t1 WHERE t1.a=? OR t1.b=?
+** </pre>
+**
+** Provokes the same callbacks as the following two queries executed in
+** series.
+**
+** <pre>
+**    SELECT * FROM t1 WHERE t1.a=?;
+**    SELECT * FROM t1 WHERE t1.b=?;
+** </pre>
+*/
+SQLITE_EXPERIMENTAL void *sqlite3_whereinfo_hook(
+  sqlite3 *db,                    /* Register callback with this db handle */
+  void (*xWhereInfo)(
+    void*,                        /* Copy of pCtx */
+    int,                          /* SQLITE_WHEREINFO_XXX constant */
+    const char*,
+    int,
+    sqlite3_uint64
+  ),
+  void *pCtx                      /* User pointer passed to callback */
+);
+
+#define SQLITE_WHEREINFO_TABLE   1
+#define SQLITE_WHEREINFO_EQUALS  2
+#define SQLITE_WHEREINFO_RANGE   3
+#define SQLITE_WHEREINFO_ORDERBY 4
+
 /*
 ** Undo the hack that converts floating point types to integer for
 ** builds on processors without floating point support.
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 0f6ffd5175..c5f5ccf0da 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1431,8 +1431,8 @@ struct sqlite3 {
 #ifdef SQLITE_USER_AUTHENTICATION
   sqlite3_userauth auth;        /* User authentication information */
 #endif
-#ifdef SQLITE_SCHEMA_LINT
-  void (*xWhereInfo)(void*, int, const char*, int, i64);
+#ifdef SQLITE_ENABLE_WHEREINFO_HOOK
+  void (*xWhereInfo)(void*, int, const char*, int, u64);
   void *pWhereInfoCtx;
 #endif
 };
diff --git a/src/where.c b/src/where.c
index c7e4ba200a..ba2ccfe65e 100644
--- a/src/where.c
+++ b/src/where.c
@@ -3277,7 +3277,7 @@ static int whereLoopAddOr(
   WhereLoopBuilder sSubBuild;
   WhereOrSet sSum, sCur;
   struct SrcList_item *pItem;
-  
+
   pWC = pBuilder->pWC;
   pWCEnd = pWC->a + pWC->nTerm;
   pNew = pBuilder->pNew;
@@ -3294,7 +3294,7 @@ static int whereLoopAddOr(
       WhereTerm *pOrTerm;
       int once = 1;
       int i, j;
-    
+
       sSubBuild = *pBuilder;
       sSubBuild.pOrderBy = 0;
       sSubBuild.pOrSet = &sCur;
@@ -4277,18 +4277,16 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){
   return 0;
 }
 
-#ifdef SQLITE_SCHEMA_LINT
+#ifdef SQLITE_ENABLE_WHEREINFO_HOOK
 static void whereTraceWC(
   Parse *pParse, 
   struct SrcList_item *pItem,
-  WhereClause *pWC,
-  int bOr
+  WhereClause *pWC
 ){
   sqlite3 *db = pParse->db;
   Table *pTab = pItem->pTab;
   void (*x)(void*, int, const char*, int, i64) = db->xWhereInfo;
   void *pCtx = db->pWhereInfoCtx;
-  int bFirst = 1;                 /* True until first callback is made */
   int ii;
 
   /* Issue callbacks for WO_SINGLE constraints */
@@ -4308,30 +4306,15 @@ static void whereTraceWC(
       }else{
         eOp = SQLITE_WHEREINFO_RANGE;
       }
-      if( bOr && !bFirst ) x(pCtx, SQLITE_WHEREINFO_NEXTOR, 0, 0, 0);
       x(pCtx, eOp, (pC ? pC->zName : "BINARY"), ii, pTerm->prereqRight);
-      bFirst = 0;
-    }
-  }
-
-  /* Callbacks for composite - (WO_OR|WO_AND) - constraints */
-  for(ii=0; ii<pWC->nTerm; ii++){
-    WhereTerm *pTerm = &pWC->a[ii];
-    if( pTerm->eOperator & WO_OR ){
-      assert( bOr==0 );
-      x(pCtx, SQLITE_WHEREINFO_BEGINOR, 0, 0, 0);
-      whereTraceWC(pParse, pItem, &pTerm->u.pOrInfo->wc, 1);
-      x(pCtx, SQLITE_WHEREINFO_ENDOR, 0, 0, 0);
-    }
-    if( pTerm->eOperator & WO_AND ){
-      if( bOr && !bFirst ) x(pCtx, SQLITE_WHEREINFO_NEXTOR, 0, 0, 0);
-      whereTraceWC(pParse, pItem, &pTerm->u.pAndInfo->wc, 0);
-      bFirst = 0;
     }
   }
 }
 
-
+/*
+** If there is a where-info hook attached to the database handle, issue all
+** required callbacks for the current sqlite3WhereBegin() call.
+*/
 static void whereTraceBuilder(
   Parse *pParse,
   WhereLoopBuilder *p
@@ -4358,7 +4341,6 @@ static void whereTraceBuilder(
         /* ORDER BY callbacks */
         if( p->pOrderBy ){
           int i;
-          int bFirst = 1;
           for(i=0; i<p->pOrderBy->nExpr; i++){
             Expr *pExpr = p->pOrderBy->a[i].pExpr; 
             CollSeq *pColl = sqlite3ExprCollSeq(pParse, pExpr);
@@ -4374,7 +4356,7 @@ static void whereTraceBuilder(
         }
 
         /* WHERE callbacks */
-        whereTraceWC(pParse, pItem, p->pWC, 0);
+        whereTraceWC(pParse, pItem, p->pWC);
       }
     }
   }
@@ -4654,7 +4636,7 @@ WhereInfo *sqlite3WhereBegin(
   }
 #endif
 
-  /* Schema-lint xTrace callback */
+  /* Invoke the where-info hook, if one has been registered. */
   whereTraceBuilder(pParse, &sWLB);
 
   if( nTabList!=1 || whereShortCut(&sWLB)==0 ){

From 53761f4d521be3b719a96106eaf89627abf20694 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 4 Apr 2017 17:50:31 +0000
Subject: [PATCH 19/68] Add some support for OR terms to
 sqlite3_whereinfo_hook().

FossilOrigin-Name: 5cd070000da1d9e399090677b4db75dc5639c33211385d6eb84f14a4d0b617cd
---
 manifest            |  16 +++---
 manifest.uuid       |   2 +-
 src/shell_indexes.c |   7 ++-
 src/where.c         | 117 ++++++++++++++++++++++++++++++++++++++++++--
 test/shell6.test    |  14 ++++++
 5 files changed, 138 insertions(+), 18 deletions(-)

diff --git a/manifest b/manifest
index fc73fb3e18..0e690d166d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\ssqlite3_whereinfo_hook()\sAPI\s-\san\sexperimental\sAPI\sreplacing\sthe\nDBCONFIG_WHEREINFO\shack\son\sthis\sbranch.
-D 2017-04-04T04:23:06.108
+C Add\ssome\ssupport\sfor\sOR\sterms\sto\ssqlite3_whereinfo_hook().
+D 2017-04-04T17:50:31.913
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a
@@ -403,7 +403,7 @@ F src/resolve.c 3e518b962d932a997fae373366880fc028c75706
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 2496d0cc6368dabe7ad2e4c7f5ed3ad9aa3b4d11cd90f33fa1d1ef72493f43aa
 F src/shell.c e524688c2544167f835ba43e24309f8707ca60c8ab6eb5c263a12c8618a233b8
-F src/shell_indexes.c 12cc58b62492bddc88f879086b4cd1045dbfd4e52040ecb6c39b971a6aac80ba
+F src/shell_indexes.c d40ea0a81112df7bdccd7232238bee0bbb39699085ea78cc08fd863bf052a63b
 F src/sqlite.h.in ae5c9cbf2e77492c319fca08769575d9695e64718a16d32324944d24e291bcf7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 58fd0676d3111d02e62e5a35992a7d3da5d3f88753acc174f2d37b774fbbdd28
@@ -483,7 +483,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 40c543f0a2195d1b0dc88ef12142bea690009344
 F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71
 F src/walker.c b71a992b413b3a022572eccf29ef4b4890223791
-F src/where.c cbd1e8562fa37de359963749e1dc3d0dd1a42a894d10a62928bffc709490fde6
+F src/where.c be352441558b55e21964ff53f0483776594789d0ede12dd9c58b022873c858f8
 F src/whereInt.h 2d50c2b74a33be44cb68fdecee30b4d93552f1f4
 F src/wherecode.c 677e95413c472c0b413023b6b69a47f40fce1b04
 F src/whereexpr.c 130cdd1a43af71b19755270fb1224874cf55158c
@@ -1120,7 +1120,7 @@ F test/shell2.test e242a9912f44f4c23c3d1d802a83e934e84c853b
 F test/shell3.test 9b95ba643eaa228376f06a898fb410ee9b6e57c1
 F test/shell4.test 89ad573879a745974ff2df20ff97c5d6ffffbd5d
 F test/shell5.test 50a732c1c2158b1cd62cf53975ce1ea7ce6b9dc9
-F test/shell6.test 05c2488ff2bb11ac300b9170e8f5b6e03d58982c77035cdca5dbf036f0d3c5b7
+F test/shell6.test 081067c1afcb38da50134ffd5ccc0a59ede14f41959486f733ffbba689c0ccfa
 F test/shell7.test 07751911b294698e0c5df67bcbd29e7d2f0f2907
 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3
 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5
@@ -1571,7 +1571,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P cb721d0b36268a7b0ef493fa4d7f6bcbaa9ead8b1990e3c3fae015fa1d545226
-R 061e66f6528edc67cfbe5cb82be7b02d
+P a54aef35da11f7508a8888a159372036a362fc52afa1df752dc835db334c4330
+R 51e547bc1c25ea79dd375d8f020224f3
 U dan
-Z d08fd2524fbe07ca7104a150ba1edcbe
+Z fe378068686ea107c39185021a17b4ae
diff --git a/manifest.uuid b/manifest.uuid
index 64b0731eb5..7669c54b05 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a54aef35da11f7508a8888a159372036a362fc52afa1df752dc835db334c4330
\ No newline at end of file
+5cd070000da1d9e399090677b4db75dc5639c33211385d6eb84f14a4d0b617cd
\ No newline at end of file
diff --git a/src/shell_indexes.c b/src/shell_indexes.c
index efdc24fac3..e061f04d3f 100644
--- a/src/shell_indexes.c
+++ b/src/shell_indexes.c
@@ -175,9 +175,6 @@ static void idxWhereInfo(
     eOp==SQLITE_WHEREINFO_EQUALS ? "EQUALS" :
     eOp==SQLITE_WHEREINFO_RANGE ? "RANGE" :
     eOp==SQLITE_WHEREINFO_ORDERBY ? "ORDERBY" :
-    eOp==SQLITE_WHEREINFO_NEXTOR ? "NEXTOR" :
-    eOp==SQLITE_WHEREINFO_ENDOR ? "ENDOR" :
-    eOp==SQLITE_WHEREINFO_BEGINOR ? "BEGINOR" :
     "!error!";
   printf("op=%s zVal=%s iVal=%d mask=%llx\n", zOp, zVal, iVal, mask);
 #endif
@@ -630,7 +627,9 @@ static int idxCreateFromCons(
         rc = SQLITE_NOMEM;
       }else{
         rc = sqlite3_exec(dbm, zIdx, 0, 0, pCtx->pzErrmsg);
-        /* printf("%s\n", zIdx); */
+#if 0
+        printf("CANDIDATE: %s\n", zIdx);
+#endif
       }
     }
     if( rc==SQLITE_OK && pCtx->iIdxRowid==0 ){
diff --git a/src/where.c b/src/where.c
index ba2ccfe65e..a423fa1839 100644
--- a/src/where.c
+++ b/src/where.c
@@ -4278,6 +4278,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){
 }
 
 #ifdef SQLITE_ENABLE_WHEREINFO_HOOK
+
 static void whereTraceWC(
   Parse *pParse, 
   struct SrcList_item *pItem,
@@ -4285,7 +4286,7 @@ static void whereTraceWC(
 ){
   sqlite3 *db = pParse->db;
   Table *pTab = pItem->pTab;
-  void (*x)(void*, int, const char*, int, i64) = db->xWhereInfo;
+  void (*x)(void*, int, const char*, int, u64) = db->xWhereInfo;
   void *pCtx = db->pWhereInfoCtx;
   int ii;
 
@@ -4311,6 +4312,109 @@ static void whereTraceWC(
   }
 }
 
+/*
+** If there are any OR terms in WHERE clause pWC, make the associated
+** where-info hook callbacks.
+*/
+static void whereTraceOR(
+  Parse *pParse, 
+  struct SrcList_item *pItem,
+  WhereClause *pWC
+){
+  sqlite3 *db = pParse->db;
+  WhereClause tempWC;
+  struct TermAndIdx {
+    WhereTerm *pTerm;
+    int iIdx;
+  } aOr[4];
+  int nOr = 0;
+  Table *pTab = pItem->pTab;
+  int iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
+  int ii;
+
+  memset(aOr, 0, sizeof(aOr));
+
+  /* Iterate through OR nodes */
+  for(ii=0; ii<pWC->nTerm; ii++){
+    WhereTerm *pTerm = &pWC->a[ii];
+    if( pTerm->eOperator & WO_OR ){
+      /* Check that each branch of this OR term contains at least
+      ** one reference to the table currently being processed. If that
+      ** is not the case, this term can be ignored.  */
+      WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc;
+      WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm];
+      WhereTerm *pOrTerm;
+      WhereClause *pTermWC;
+      WhereScan scan;
+
+      for(pOrTerm=pOrWC->a; pOrTerm<pOrWCEnd; pOrTerm++){
+        int iCol;
+        if( (pOrTerm->eOperator & WO_AND)!=0 ){
+          pTermWC = &pOrTerm->u.pAndInfo->wc;
+        }else{
+          tempWC.pWInfo = pWC->pWInfo;
+          tempWC.pOuter = pWC;
+          tempWC.op = TK_AND;
+          tempWC.nTerm = 1;
+          tempWC.a = pOrTerm;
+          pTermWC = &tempWC;
+        }
+
+        for(iCol=0; iCol<pTab->nCol; iCol++){
+          int iCsr = pItem->iCursor;
+          if( !whereScanInit(&scan, pTermWC, iCsr, iCol, WO_SINGLE, 0) ){
+            break;
+          }
+        }
+        if( iCol==pTab->nCol ) break;
+      }
+
+      if( pOrTerm==pOrWCEnd ){
+        aOr[nOr].pTerm = pTerm;
+        aOr[nOr].iIdx = pOrWC->nTerm;
+        nOr++;
+        if( nOr==ArraySize(aOr) ) break;
+      }
+    }
+  }
+
+  while( 1 ){
+    for(ii=0; ii<nOr; ii++){
+      if( aOr[ii].iIdx==0 ){
+        aOr[ii].iIdx = aOr[ii].pTerm->u.pOrInfo->wc.nTerm;
+      }else{
+        aOr[ii].iIdx--;
+        break;
+      }
+    }
+    if( ii==nOr ) break;
+
+    /* Table name callback */
+    db->xWhereInfo(db->pWhereInfoCtx, 
+        SQLITE_WHEREINFO_TABLE, pTab->zName, iDb, pItem->colUsed
+    );
+    /* whereTraceWC(pParse, pItem, pWC); */
+    for(ii=0; ii<nOr; ii++){
+      WhereClause * const pOrWC = &aOr[ii].pTerm->u.pOrInfo->wc;
+      if( aOr[ii].iIdx<pOrWC->nTerm ){
+        WhereClause *pTermWC;
+        WhereTerm *pOrTerm = &pOrWC->a[aOr[ii].iIdx];
+        if( (pOrTerm->eOperator & WO_AND)!=0 ){
+          pTermWC = &pOrTerm->u.pAndInfo->wc;
+        }else{
+          tempWC.pWInfo = pWC->pWInfo;
+          tempWC.pOuter = pWC;
+          tempWC.op = TK_AND;
+          tempWC.nTerm = 1;
+          tempWC.a = pOrTerm;
+          pTermWC = &tempWC;
+        }
+        whereTraceWC(pParse, pItem, pTermWC);
+      }
+    }
+  }
+}
+
 /*
 ** If there is a where-info hook attached to the database handle, issue all
 ** required callbacks for the current sqlite3WhereBegin() call.
@@ -4321,16 +4425,16 @@ static void whereTraceBuilder(
 ){
   sqlite3 *db = pParse->db;
   if( db->xWhereInfo && db->init.busy==0 ){
-    void (*x)(void*, int, const char*, int, i64) = db->xWhereInfo;
+    void (*x)(void*, int, const char*, int, u64) = db->xWhereInfo;
     void *pCtx = db->pWhereInfoCtx;
     int ii;
-    int nTab = p->pWInfo->pTabList->nSrc;
+    SrcList *pTabList = p->pWInfo->pTabList;
 
     /* Loop through each element of the FROM clause. Ignore any sub-selects
     ** or views. Invoke the xWhereInfo() callback multiple times for each
     ** real table.  */
-    for(ii=0; ii<p->pWInfo->pTabList->nSrc; ii++){
-      struct SrcList_item *pItem = &p->pWInfo->pTabList->a[ii];
+    for(ii=0; ii<pTabList->nSrc; ii++){
+      struct SrcList_item *pItem = &pTabList->a[ii];
       if( pItem->pSelect==0 ){
         Table *pTab = pItem->pTab;
         int iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
@@ -4357,6 +4461,9 @@ static void whereTraceBuilder(
 
         /* WHERE callbacks */
         whereTraceWC(pParse, pItem, p->pWC);
+
+        /* OR-clause processing */
+        whereTraceOR(pParse, pItem, p->pWC);
       }
     }
   }
diff --git a/test/shell6.test b/test/shell6.test
index 19b094a80d..c44ccf0bd1 100644
--- a/test/shell6.test
+++ b/test/shell6.test
@@ -188,4 +188,18 @@ do_setup_rec_test 10.1 {
   0|1|0|SEARCH TABLE t5 USING COVERING INDEX t5_idx_000123a7 (a=? AND b=?)
 }
 
+# OR terms.
+#
+do_setup_rec_test 11.1 {
+  CREATE TABLE t7(a, b);
+} {
+  SELECT * FROM t7 WHERE a=? OR b=?
+} {
+  CREATE INDEX t7_idx_00000061 ON t7(a) 
+  CREATE INDEX t7_idx_00000062 ON t7(b)
+  0|0|0|SEARCH TABLE t7 USING INDEX t7_idx_00000061 (a=?) 
+  0|0|0|SEARCH TABLE t7 USING INDEX t7_idx_00000062 (b=?)
+}
+
 finish_test
+

From d8ac297233b9af6cc74f92a14834eaacda781676 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 6 Apr 2017 18:44:18 +0000
Subject: [PATCH 20/68] Changes to allow indexes to be recommended for queries
 on SQL views.

FossilOrigin-Name: 0884ff1da2e27b146c764b73cf299a1f2cfe213c4a79bde34dec02d1fc946e70
---
 manifest            |  14 ++---
 manifest.uuid       |   2 +-
 src/shell_indexes.c | 148 +++++++-------------------------------------
 test/shell6.test    |  32 +++++-----
 4 files changed, 47 insertions(+), 149 deletions(-)

diff --git a/manifest b/manifest
index 0e690d166d..ce8fc15893 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\ssome\ssupport\sfor\sOR\sterms\sto\ssqlite3_whereinfo_hook().
-D 2017-04-04T17:50:31.913
+C Changes\sto\sallow\sindexes\sto\sbe\srecommended\sfor\squeries\son\sSQL\sviews.
+D 2017-04-06T18:44:18.391
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a
@@ -403,7 +403,7 @@ F src/resolve.c 3e518b962d932a997fae373366880fc028c75706
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 2496d0cc6368dabe7ad2e4c7f5ed3ad9aa3b4d11cd90f33fa1d1ef72493f43aa
 F src/shell.c e524688c2544167f835ba43e24309f8707ca60c8ab6eb5c263a12c8618a233b8
-F src/shell_indexes.c d40ea0a81112df7bdccd7232238bee0bbb39699085ea78cc08fd863bf052a63b
+F src/shell_indexes.c 1f5ab036ec189411aeea27e6e74ab0009d831764d5d8517455dcb6b6a734beb7
 F src/sqlite.h.in ae5c9cbf2e77492c319fca08769575d9695e64718a16d32324944d24e291bcf7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 58fd0676d3111d02e62e5a35992a7d3da5d3f88753acc174f2d37b774fbbdd28
@@ -1120,7 +1120,7 @@ F test/shell2.test e242a9912f44f4c23c3d1d802a83e934e84c853b
 F test/shell3.test 9b95ba643eaa228376f06a898fb410ee9b6e57c1
 F test/shell4.test 89ad573879a745974ff2df20ff97c5d6ffffbd5d
 F test/shell5.test 50a732c1c2158b1cd62cf53975ce1ea7ce6b9dc9
-F test/shell6.test 081067c1afcb38da50134ffd5ccc0a59ede14f41959486f733ffbba689c0ccfa
+F test/shell6.test f37998b26dfde19beaaf06a4cb60c476f66a7b54affff3870a2a011402c13efc
 F test/shell7.test 07751911b294698e0c5df67bcbd29e7d2f0f2907
 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3
 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5
@@ -1571,7 +1571,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a54aef35da11f7508a8888a159372036a362fc52afa1df752dc835db334c4330
-R 51e547bc1c25ea79dd375d8f020224f3
+P 5cd070000da1d9e399090677b4db75dc5639c33211385d6eb84f14a4d0b617cd
+R aa13547c61fd31f43629472eaf9b4008
 U dan
-Z fe378068686ea107c39185021a17b4ae
+Z 2f3b5e2a515dd18e4123d763066c9a33
diff --git a/manifest.uuid b/manifest.uuid
index 7669c54b05..3b9f56b1a1 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5cd070000da1d9e399090677b4db75dc5639c33211385d6eb84f14a4d0b617cd
\ No newline at end of file
+0884ff1da2e27b146c764b73cf299a1f2cfe213c4a79bde34dec02d1fc946e70
\ No newline at end of file
diff --git a/src/shell_indexes.c b/src/shell_indexes.c
index e061f04d3f..c05672e9e0 100644
--- a/src/shell_indexes.c
+++ b/src/shell_indexes.c
@@ -43,44 +43,18 @@ struct IdxConstraint {
 /*
 ** A WHERE clause. Made up of IdxConstraint objects. Example WHERE clause:
 **
-**   a=? AND b=? AND ((c=? AND d=?) OR e=?) AND (f=? OR g=?) AND h>?
+**   a=? AND b=? AND c=? AND d=? AND e>? AND f<?
 **
-** The above is decomposed into 5 AND connected clauses. The first two are
+** The above is decomposed into 6 AND connected clauses. The first four are
 ** added to the IdxWhere.pEq linked list, the following two into 
-** IdxWhere.pOr and the last into IdxWhere.pRange.
+** IdxWhere.pRange.
 **
 ** IdxWhere.pEq and IdxWhere.pRange are simple linked lists of IdxConstraint
 ** objects linked by the IdxConstraint.pNext field.
-**
-** The list headed at IdxWhere.pOr and linked by IdxWhere.pNextOr contains
-** all "OR" terms that belong to the current WHERE clause. In the example
-** above, there are two OR terms:
-**
-**   ((c=? AND d=?) OR e=?)
-**   (f=? OR g=?)
-**
-** Within an OR term, the OR connected sub-expressions are termed siblings.
-** These are connected into a linked list by the pSibling pointers. Each OR
-** term above consists of two siblings.
-**
-**   pOr -> (c=? AND d=?) -> pNextOr -> (f=?)
-**               |                        |
-**            pSibling                 pSibling
-**               |                        |
-**               V                        V
-**             (e=?)                    (g=?)
-**
-** IdxWhere.pParent is only used while constructing a tree of IdxWhere 
-** structures. It is NULL for the root IdxWhere. For all others, the parent
-** WHERE clause.
 */
 struct IdxWhere {
   IdxConstraint *pEq;             /* List of == constraints */
   IdxConstraint *pRange;          /* List of < constraints */
-  IdxWhere *pOr;                  /* List of OR constraints */
-  IdxWhere *pNextOr;              /* Next in OR constraints of same IdxWhere */
-  IdxWhere *pSibling;             /* Next branch in single OR constraint */
-  IdxWhere *pParent;              /* Parent object (or NULL) */
 };
 
 /*
@@ -246,40 +220,6 @@ static void idxDatabaseError(
   *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
 }
 
-static char *idxQueryToList(
-  sqlite3 *db, 
-  const char *zBind,
-  int *pRc,
-  char **pzErrmsg,
-  const char *zSql
-){
-  char *zRet = 0;
-  if( *pRc==SQLITE_OK ){
-    sqlite3_stmt *pStmt = 0;
-    int rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
-    if( rc==SQLITE_OK ){
-      sqlite3_bind_text(pStmt, 1, zBind, -1, SQLITE_TRANSIENT);
-      while( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){
-        const char *z = (const char*)sqlite3_column_text(pStmt, 0);
-        zRet = sqlite3_mprintf("%z%s%Q", zRet, zRet?", ":"", z);
-        if( zRet==0 ){
-          rc = SQLITE_NOMEM;
-        }
-      }
-      rc = sqlite3_finalize(pStmt);
-    }
-
-    if( rc ){
-      idxDatabaseError(db, pzErrmsg);
-      sqlite3_free(zRet);
-      zRet = 0;
-    }
-    *pRc = rc;
-  }
-
-  return zRet;
-}
-
 static int idxPrepareStmt(
   sqlite3 *db,                    /* Database handle to compile against */
   sqlite3_stmt **ppStmt,          /* OUT: Compiled SQL statement */
@@ -331,7 +271,7 @@ static int idxGetTableInfo(
 
   rc = idxPrintfPrepareStmt(db, &p1, pzErrmsg, "PRAGMA table_info=%Q", zTbl);
   while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){
-    const char *zCol = sqlite3_column_text(p1, 1);
+    const char *zCol = (const char*)sqlite3_column_text(p1, 1);
     nByte += 1 + strlen(zCol);
     rc = sqlite3_table_column_metadata(
         db, "main", zTbl, zCol, 0, &zCol, 0, 0, 0
@@ -354,7 +294,7 @@ static int idxGetTableInfo(
 
   nCol = 0;
   while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){
-    const char *zCol = sqlite3_column_text(p1, 1);
+    const char *zCol = (const char*)sqlite3_column_text(p1, 1);
     int nCopy = strlen(zCol) + 1;
     pNew->aCol[nCol].zName = pCsr;
     pNew->aCol[nCol].iPk = sqlite3_column_int(p1, 5);
@@ -392,38 +332,24 @@ static int idxCreateTables(
   char **pzErrmsg                 /* OUT: Error message */
 ){
   int rc = SQLITE_OK;
+  int rc2;
   IdxScan *pIter;
+  sqlite3_stmt *pSql = 0;
+
+  /* Copy the entire schema of database [db] into [dbm]. */
+  rc = idxPrintfPrepareStmt(db, &pSql, pzErrmsg, 
+      "SELECT sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%%'"
+  );
+  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
+    const char *zSql = (const char*)sqlite3_column_text(pSql, 0);
+    rc = sqlite3_exec(dbm, zSql, 0, 0, pzErrmsg);
+  }
+  rc2 = sqlite3_finalize(pSql);
+  if( rc==SQLITE_OK ) rc = rc2;
+
+  /* Load IdxTable objects */
   for(pIter=pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
     rc = idxGetTableInfo(db, pIter, pzErrmsg);
-
-    /* Test if table has already been created. If so, jump to the next
-    ** iteration of the loop.  */
-    if( rc==SQLITE_OK ){
-      sqlite3_stmt *pSql = 0;
-      rc = idxPrintfPrepareStmt(dbm, &pSql, pzErrmsg, 
-          "SELECT 1 FROM sqlite_master WHERE tbl_name = %Q", pIter->zTable
-      );
-      if( rc==SQLITE_OK ){
-        int bSkip = 0;
-        if( sqlite3_step(pSql)==SQLITE_ROW ) bSkip = 1;
-        rc = sqlite3_finalize(pSql);
-        if( bSkip ) continue;
-      }
-    }
-
-    if( rc==SQLITE_OK ){
-      int rc2;
-      sqlite3_stmt *pSql = 0;
-      rc = idxPrintfPrepareStmt(db, &pSql, pzErrmsg, 
-          "SELECT sql FROM sqlite_master WHERE tbl_name = %Q", pIter->zTable
-      );
-      while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
-        const char *zSql = (const char*)sqlite3_column_text(pSql, 0);
-        rc = sqlite3_exec(dbm, zSql, 0, 0, pzErrmsg);
-      }
-      rc2 = sqlite3_finalize(pSql);
-      if( rc==SQLITE_OK ) rc = rc2;
-    }
   }
   return rc;
 }
@@ -655,26 +581,6 @@ static int idxCreateFromWhere(
     IdxContext*, i64, IdxScan*, IdxWhere*, IdxConstraint*, IdxConstraint*
 );
 
-static int idxCreateForeachOr(
-  IdxContext *pCtx, 
-  i64 mask,                       /* Consider only these constraints */
-  IdxScan *pScan,                 /* Create indexes for this scan */
-  IdxWhere *pWhere,               /* Read constraints from here */
-  IdxConstraint *pEq,             /* == constraints for inclusion */
-  IdxConstraint *pTail            /* range/ORDER BY constraints for inclusion */
-){
-  int rc = SQLITE_OK;
-  IdxWhere *p1;
-  IdxWhere *p2;
-  for(p1=pWhere->pOr; p1 && rc==SQLITE_OK; p1=p1->pNextOr){
-    rc = idxCreateFromWhere(pCtx, mask, pScan, p1, pEq, pTail);
-    for(p2=p1->pSibling; p2 && rc==SQLITE_OK; p2=p2->pSibling){
-      rc = idxCreateFromWhere(pCtx, mask, pScan, p2, pEq, pTail);
-    }
-  }
-  return rc;
-}
-
 /*
 ** Return true if list pList (linked by IdxConstraint.pLink) contains
 ** a constraint compatible with *p. Otherwise return false.
@@ -695,7 +601,6 @@ static int idxCreateFromWhere(
   IdxConstraint *pEq,             /* == constraints for inclusion */
   IdxConstraint *pTail            /* range/ORDER BY constraints for inclusion */
 ){
-  sqlite3 *dbm = pCtx->dbm;
   IdxConstraint *p1 = pEq;
   IdxConstraint *pCon;
   int rc;
@@ -714,9 +619,6 @@ static int idxCreateFromWhere(
   /* Create an index using the == constraints collected above. And the
   ** range constraint/ORDER BY terms passed in by the caller, if any. */
   rc = idxCreateFromCons(pCtx, pScan, p1, pTail);
-  if( rc==SQLITE_OK ){
-    rc = idxCreateForeachOr(pCtx, mask, pScan, pWhere, p1, pTail);
-  }
 
   /* If no range/ORDER BY passed by the caller, create a version of the
   ** index for each range constraint that matches the mask. */
@@ -728,9 +630,6 @@ static int idxCreateFromWhere(
         && idxFindConstraint(pTail, pCon)==0
       ){
         rc = idxCreateFromCons(pCtx, pScan, p1, pCon);
-        if( rc==SQLITE_OK ){
-          rc = idxCreateForeachOr(pCtx, mask, pScan, pWhere, p1, pCon);
-        }
       }
     }
   }
@@ -749,7 +648,7 @@ static int idxCreateCandidates(IdxContext *pCtx){
   sqlite3_stmt *pDepmask;         /* Foreach depmask */
   IdxScan *pIter;
 
-  rc = idxPrepareStmt(pCtx->dbm, &pDepmask, pCtx->pzErrmsg, 
+  rc = idxPrepareStmt(dbm, &pDepmask, pCtx->pzErrmsg, 
       "SELECT mask FROM depmask"
   );
 
@@ -774,7 +673,6 @@ static void idxScanFree(IdxScan *pScan){
   IdxScan *pNext;
   for(pIter=pScan; pIter; pIter=pNext){
     pNext = pIter->pNextScan;
-
   }
 }
 
@@ -841,11 +739,11 @@ int idxFindIndexes(
   if( rc==SQLITE_OK ) rc = rc2;
   if( rc==SQLITE_OK ){
     sqlite3_stmt *pLoop = 0;
-    rc = idxPrepareStmt(dbm, &pLoop, pzErr, "SELECT name FROM aux.indexes");
+    rc = idxPrepareStmt(dbm, &pLoop, pzErr,"SELECT name||';' FROM aux.indexes");
     if( rc==SQLITE_OK ){
       while( SQLITE_ROW==sqlite3_step(pLoop) ){
         bFound = 1;
-        xOut(pOutCtx, sqlite3_column_text(pLoop, 0));
+        xOut(pOutCtx, (const char*)sqlite3_column_text(pLoop, 0));
       }
       rc = sqlite3_finalize(pLoop);
     }
diff --git a/test/shell6.test b/test/shell6.test
index c44ccf0bd1..2f7c251827 100644
--- a/test/shell6.test
+++ b/test/shell6.test
@@ -66,7 +66,7 @@ do_setup_rec_test 1.2 {
 } {
   SELECT * FROM t1 WHERE b>?;
 } {
-  CREATE INDEX t1_idx_00000062 ON t1(b) 
+  CREATE INDEX t1_idx_00000062 ON t1(b);
   0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_00000062 (b>?)
 }
 
@@ -75,7 +75,7 @@ do_setup_rec_test 1.3 {
 } {
   SELECT * FROM t1 WHERE b COLLATE nocase BETWEEN ? AND ?
 } {
-  CREATE INDEX t1_idx_3e094c27 ON t1(b COLLATE NOCASE) 
+  CREATE INDEX t1_idx_3e094c27 ON t1(b COLLATE NOCASE);
   0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_3e094c27 (b>? AND b<?)
 }
 
@@ -84,7 +84,7 @@ do_setup_rec_test 1.4 {
 } {
   SELECT a FROM t1 ORDER BY b;
 } {
-  CREATE INDEX t1_idx_00000062 ON t1(b) 
+  CREATE INDEX t1_idx_00000062 ON t1(b);
   0|0|0|SCAN TABLE t1 USING INDEX t1_idx_00000062
 }
 
@@ -93,7 +93,7 @@ do_setup_rec_test 1.5 {
 } {
   SELECT a FROM t1 WHERE a=? ORDER BY b;
 } {
-  CREATE INDEX t1_idx_000123a7 ON t1(a, b) 
+  CREATE INDEX t1_idx_000123a7 ON t1(a, b);
   0|0|0|SEARCH TABLE t1 USING COVERING INDEX t1_idx_000123a7 (a=?)
 }
 
@@ -102,7 +102,7 @@ do_setup_rec_test 1.6 {
 } {
   SELECT min(a) FROM t1
 } {
-  CREATE INDEX t1_idx_00000061 ON t1(a) 
+  CREATE INDEX t1_idx_00000061 ON t1(a);
   0|0|0|SEARCH TABLE t1 USING COVERING INDEX t1_idx_00000061
 }
 
@@ -111,7 +111,7 @@ do_setup_rec_test 1.7 {
 } {
   SELECT * FROM t1 ORDER BY a, b, c;
 } {
-  CREATE INDEX t1_idx_033e95fe ON t1(a, b, c) 
+  CREATE INDEX t1_idx_033e95fe ON t1(a, b, c);
   0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_033e95fe
 }
 
@@ -120,7 +120,7 @@ do_setup_rec_test 1.8 {
 } {
   SELECT * FROM t1 ORDER BY a ASC, b COLLATE nocase DESC, c ASC;
 } {
-  CREATE INDEX t1_idx_5be6e222 ON t1(a, b COLLATE NOCASE DESC, c)
+  CREATE INDEX t1_idx_5be6e222 ON t1(a, b COLLATE NOCASE DESC, c);
   0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_5be6e222
 }
 
@@ -129,7 +129,7 @@ do_setup_rec_test 1.9 {
 } {
   SELECT * FROM t1 WHERE a=?
 } {
-  CREATE INDEX t1_idx_00000061 ON t1(a) 
+  CREATE INDEX t1_idx_00000061 ON t1(a);
   0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_00000061 (a=?)
 }
 
@@ -141,7 +141,7 @@ do_setup_rec_test 8.1 {
 } {
   SELECT * FROM "t t" WHERE a=?
 } {
-  CREATE INDEX 't t_idx_00000061' ON 't t'(a)
+  CREATE INDEX 't t_idx_00000061' ON 't t'(a);
   0|0|0|SEARCH TABLE t t USING INDEX t t_idx_00000061 (a=?) 
 }
 
@@ -150,7 +150,7 @@ do_setup_rec_test 8.2 {
 } {
   SELECT * FROM "t t" WHERE b BETWEEN ? AND ?
 } {
-  CREATE INDEX 't t_idx_00000062' ON 't t'(b) 
+  CREATE INDEX 't t_idx_00000062' ON 't t'(b);
   0|0|0|SEARCH TABLE t t USING INDEX t t_idx_00000062 (b>? AND b<?)
 }
 
@@ -161,7 +161,7 @@ do_setup_rec_test 9.1 {
 } {
   SELECT * FROM t3 WHERE "b b" = ?
 } {
-  CREATE INDEX t3_idx_00050c52 ON t3('b b') 
+  CREATE INDEX t3_idx_00050c52 ON t3('b b');
   0|0|0|SEARCH TABLE t3 USING INDEX t3_idx_00050c52 (b b=?)
 }
 
@@ -170,7 +170,7 @@ do_setup_rec_test 9.2 {
 } {
   SELECT * FROM t3 ORDER BY "b b"
 } {
-  CREATE INDEX t3_idx_00050c52 ON t3('b b') 
+  CREATE INDEX t3_idx_00050c52 ON t3('b b');
   0|0|0|SCAN TABLE t3 USING INDEX t3_idx_00050c52
 }
 
@@ -182,8 +182,8 @@ do_setup_rec_test 10.1 {
 } {
   SELECT * FROM t5, t6 WHERE a=? AND b=c AND c=?
 } {
-  CREATE INDEX t5_idx_000123a7 ON t5(a, b) 
-  CREATE INDEX t6_idx_00000063 ON t6(c) 
+  CREATE INDEX t5_idx_000123a7 ON t5(a, b);
+  CREATE INDEX t6_idx_00000063 ON t6(c);
   0|0|1|SEARCH TABLE t6 USING INDEX t6_idx_00000063 (c=?) 
   0|1|0|SEARCH TABLE t5 USING COVERING INDEX t5_idx_000123a7 (a=? AND b=?)
 }
@@ -195,8 +195,8 @@ do_setup_rec_test 11.1 {
 } {
   SELECT * FROM t7 WHERE a=? OR b=?
 } {
-  CREATE INDEX t7_idx_00000061 ON t7(a) 
-  CREATE INDEX t7_idx_00000062 ON t7(b)
+  CREATE INDEX t7_idx_00000061 ON t7(a);
+  CREATE INDEX t7_idx_00000062 ON t7(b);
   0|0|0|SEARCH TABLE t7 USING INDEX t7_idx_00000061 (a=?) 
   0|0|0|SEARCH TABLE t7 USING INDEX t7_idx_00000062 (b=?)
 }

From 8e0b8e06884c56eb0e275d3f3cfb18510de84db8 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 7 Apr 2017 20:14:22 +0000
Subject: [PATCH 21/68] Refactor code to suggest indexes from the shell tool
 into an extension in ext/expert. Unfinished.

FossilOrigin-Name: 305e19f976ca064638a294e0817bf547ea745e1eb74746c5855514e6ced9c5fa
---
 ext/expert/expert.c                           | 104 ++++
 .../expert/sqlite3expert.c                    | 476 +++++++++++++-----
 ext/expert/sqlite3expert.h                    |  57 +++
 main.mk                                       |   6 +-
 manifest                                      |  18 +-
 manifest.uuid                                 |   2 +-
 src/shell.c                                   |  34 --
 7 files changed, 518 insertions(+), 179 deletions(-)
 create mode 100644 ext/expert/expert.c
 rename src/shell_indexes.c => ext/expert/sqlite3expert.c (67%)
 create mode 100644 ext/expert/sqlite3expert.h

diff --git a/ext/expert/expert.c b/ext/expert/expert.c
new file mode 100644
index 0000000000..5a5cda185f
--- /dev/null
+++ b/ext/expert/expert.c
@@ -0,0 +1,104 @@
+/*
+** 2017 April 07
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+*/
+
+
+#include <sqlite3.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "sqlite3expert.h"
+
+
+static void option_requires_argument(const char *zOpt){
+  fprintf(stderr, "Option requires an argument: %s\n", zOpt);
+  exit(-3);
+}
+
+static void usage(char **argv){
+  fprintf(stderr, "\n");
+  fprintf(stderr, "Usage %s ?OPTIONS? DATABASE\n", argv[0]);
+  fprintf(stderr, "\n");
+  fprintf(stderr, "Options are:\n");
+  fprintf(stderr, "  -sql SQL   (analyze SQL statements passed as argument)\n");
+  fprintf(stderr, "  -file FILE (read SQL statements from file FILE)\n");
+  exit(-1);
+}
+
+static int readSqlFromFile(sqlite3expert *p, const char *zFile, char **pzErr){
+  return SQLITE_OK;
+}
+
+int main(int argc, char **argv){
+  const char *zDb;
+  int rc = 0;
+  char *zErr = 0;
+  int i;
+
+  sqlite3 *db = 0;
+  sqlite3expert *p = 0;
+
+  if( argc<2 ) usage(argv);
+  zDb = argv[argc-1];
+  rc = sqlite3_open(zDb, &db);
+  if( rc!=SQLITE_OK ){
+    fprintf(stderr, "Cannot open db file: %s - %s\n", zDb, sqlite3_errmsg(db));
+    exit(-2);
+  }
+
+  p = sqlite3_expert_new(db, &zErr);
+  if( p==0 ){
+    fprintf(stderr, "Cannot run analysis: %s\n", zErr);
+    rc = 1;
+  }else{
+    for(i=1; i<(argc-1); i++){
+      char *zArg = argv[i];
+      int nArg = strlen(zArg);
+      if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-file", nArg) ){
+        if( ++i==(argc-1) ) option_requires_argument("-file");
+        rc = readSqlFromFile(p, argv[i], &zErr);
+      }
+
+      else if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-sql", nArg) ){
+        if( ++i==(argc-1) ) option_requires_argument("-sql");
+        rc = sqlite3_expert_sql(p, argv[i], &zErr);
+      }
+
+      else{
+        usage(argv);
+      }
+    }
+  }
+
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_expert_analyze(p, &zErr);
+  }
+
+  if( rc==SQLITE_OK ){
+    int nQuery = sqlite3_expert_count(p);
+    for(i=0; i<nQuery; i++){
+      const char *zSql = sqlite3_expert_report(p, i, EXPERT_REPORT_SQL);
+      const char *zIdx = sqlite3_expert_report(p, i, EXPERT_REPORT_INDEXES);
+      const char *zEQP = sqlite3_expert_report(p, i, EXPERT_REPORT_PLAN);
+      fprintf(stdout, "-- query %d ----------------------------------\n", i+1);
+      fprintf(stdout, "%s\n\n%s\n%s\n", zSql, zIdx, zEQP);
+    }
+  }else if( zErr ){
+    fprintf(stderr, "Error: %s\n", zErr);
+  }
+
+  sqlite3_expert_destroy(p);
+  sqlite3_free(zErr);
+  return rc;
+}
+
+
diff --git a/src/shell_indexes.c b/ext/expert/sqlite3expert.c
similarity index 67%
rename from src/shell_indexes.c
rename to ext/expert/sqlite3expert.c
index c05672e9e0..3bcfbbc6e8 100644
--- a/src/shell_indexes.c
+++ b/ext/expert/sqlite3expert.c
@@ -11,12 +11,18 @@
 *************************************************************************
 */
 
+#include "sqlite3expert.h"
+#include <assert.h>
+#include <string.h>
+#include <stdio.h>
+
 typedef sqlite3_int64 i64;
 typedef sqlite3_uint64 u64;
 
 typedef struct IdxConstraint IdxConstraint;
 typedef struct IdxContext IdxContext;
 typedef struct IdxScan IdxScan;
+typedef struct IdxStatement IdxStatement;
 typedef struct IdxWhere IdxWhere;
 
 typedef struct IdxColumn IdxColumn;
@@ -70,6 +76,19 @@ struct IdxScan {
   IdxScan *pNextScan;             /* Next IdxScan object for same query */
 };
 
+/*
+** Data regarding a database table. Extracted from "PRAGMA table_info"
+*/
+struct IdxColumn {
+  char *zName;
+  char *zColl;
+  int iPk;
+};
+struct IdxTable {
+  int nCol;
+  IdxColumn *aCol;
+};
+
 /*
 ** Context object passed to idxWhereInfo() and other functions.
 */
@@ -84,19 +103,31 @@ struct IdxContext {
   i64 iIdxRowid;                  /* Rowid of first index created */
 };
 
+struct IdxStatement {
+  int iId;                        /* Statement number */
+  char *zSql;                     /* SQL statement */
+  char *zIdx;                     /* Indexes */
+  char *zEQP;                     /* Plan */
+  IdxStatement *pNext;
+};
+
 /*
-** Data regarding a database table. Extracted from "PRAGMA table_info"
+** sqlite3expert object.
 */
-struct IdxColumn {
-  char *zName;
-  char *zColl;
-  int iPk;
-};
-struct IdxTable {
-  int nCol;
-  IdxColumn *aCol;
+struct sqlite3expert {
+  sqlite3 *db;                    /* Users database */
+  sqlite3 *dbm;                   /* In-memory db for this analysis */
+
+  int bRun;                       /* True once analysis has run */
+  char **pzErrmsg;
+
+  IdxScan *pScan;                 /* List of scan objects */
+  IdxStatement *pStatement;       /* List of IdxStatement objects */
+  int rc;                         /* Error code from whereinfo hook */
+  i64 iIdxRowid;                  /* Rowid of first index created */
 };
 
+
 /*
 ** Allocate and return nByte bytes of zeroed memory using sqlite3_malloc(). 
 ** If the allocation fails, set *pRc to SQLITE_NOMEM and return NULL.
@@ -132,7 +163,7 @@ static IdxConstraint *idxNewConstraint(int *pRc, const char *zColl){
 }
 
 /*
-** SQLITE_DBCONFIG_WHEREINFO callback.
+** sqlite3_whereinfo_hook() callback.
 */
 static void idxWhereInfo(
   void *pCtx,                     /* Pointer to IdxContext structure */
@@ -141,7 +172,7 @@ static void idxWhereInfo(
   int iVal, 
   u64 mask
 ){
-  IdxContext *p = (IdxContext*)pCtx;
+  sqlite3expert *p = (sqlite3expert*)pCtx;
 
 #if 0
   const char *zOp = 
@@ -165,7 +196,6 @@ static void idxWhereInfo(
         pNew->pNextScan = p->pScan;
         pNew->covering = mask;
         p->pScan = pNew;
-        p->pCurrent = &pNew->where;
         break;
       }
 
@@ -193,16 +223,17 @@ static void idxWhereInfo(
         pNew->depmask = mask;
 
         if( eOp==SQLITE_WHEREINFO_RANGE ){
-          pNew->pNext = p->pCurrent->pRange;
-          p->pCurrent->pRange = pNew;
+          pNew->pNext = p->pScan->where.pRange;
+          p->pScan->where.pRange = pNew;
         }else{
-          pNew->pNext = p->pCurrent->pEq;
-          p->pCurrent->pEq = pNew;
+          pNew->pNext = p->pScan->where.pEq;
+          p->pScan->where.pEq = pNew;
         }
-
+#if 0
         sqlite3_bind_int64(p->pInsertMask, 1, mask);
         sqlite3_step(p->pInsertMask);
         p->rc = sqlite3_reset(p->pInsertMask);
+#endif
         break;
       }
     }
@@ -325,35 +356,6 @@ static int idxGetTableInfo(
   return rc;
 }
 
-static int idxCreateTables(
-  sqlite3 *db,                    /* User database */
-  sqlite3 *dbm,                   /* In-memory database to create tables in */
-  IdxScan *pScan,                 /* List of scans */
-  char **pzErrmsg                 /* OUT: Error message */
-){
-  int rc = SQLITE_OK;
-  int rc2;
-  IdxScan *pIter;
-  sqlite3_stmt *pSql = 0;
-
-  /* Copy the entire schema of database [db] into [dbm]. */
-  rc = idxPrintfPrepareStmt(db, &pSql, pzErrmsg, 
-      "SELECT sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%%'"
-  );
-  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
-    const char *zSql = (const char*)sqlite3_column_text(pSql, 0);
-    rc = sqlite3_exec(dbm, zSql, 0, 0, pzErrmsg);
-  }
-  rc2 = sqlite3_finalize(pSql);
-  if( rc==SQLITE_OK ) rc = rc2;
-
-  /* Load IdxTable objects */
-  for(pIter=pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
-    rc = idxGetTableInfo(db, pIter, pzErrmsg);
-  }
-  return rc;
-}
-
 /*
 ** This function is a no-op if *pRc is set to anything other than 
 ** SQLITE_OK when it is called.
@@ -456,7 +458,6 @@ static int idxFindCompatible(
   int nEq = 0;                    /* Number of elements in pEq */
   int rc, rc2;
 
-
   /* Count the elements in list pEq */
   for(pIter=pEq; pIter; pIter=pIter->pLink) nEq++;
 
@@ -514,12 +515,12 @@ static int idxFindCompatible(
 }
 
 static int idxCreateFromCons(
-  IdxContext *pCtx,
+  sqlite3expert *p,
   IdxScan *pScan,
   IdxConstraint *pEq, 
   IdxConstraint *pTail
 ){
-  sqlite3 *dbm = pCtx->dbm;
+  sqlite3 *dbm = p->dbm;
   int rc = SQLITE_OK;
   if( (pEq || pTail) && 0==idxFindCompatible(&rc, dbm, pScan, pEq, pTail) ){
     IdxTable *pTab = pScan->pTable;
@@ -552,20 +553,20 @@ static int idxCreateFromCons(
       if( !zIdx ){
         rc = SQLITE_NOMEM;
       }else{
-        rc = sqlite3_exec(dbm, zIdx, 0, 0, pCtx->pzErrmsg);
-#if 0
+        rc = sqlite3_exec(dbm, zIdx, 0, 0, p->pzErrmsg);
+#if 1
         printf("CANDIDATE: %s\n", zIdx);
 #endif
       }
     }
-    if( rc==SQLITE_OK && pCtx->iIdxRowid==0 ){
+    if( rc==SQLITE_OK && p->iIdxRowid==0 ){
       int rc2;
       sqlite3_stmt *pLast = 0;
-      rc = idxPrepareStmt(dbm, &pLast, pCtx->pzErrmsg, 
+      rc = idxPrepareStmt(dbm, &pLast, p->pzErrmsg, 
           "SELECT max(rowid) FROM sqlite_master"
       );
       if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLast) ){
-        pCtx->iIdxRowid = sqlite3_column_int64(pLast, 0);
+        p->iIdxRowid = sqlite3_column_int64(pLast, 0);
       }
       rc2 = sqlite3_finalize(pLast);
       if( rc==SQLITE_OK ) rc = rc2;
@@ -578,7 +579,7 @@ static int idxCreateFromCons(
 }
 
 static int idxCreateFromWhere(
-    IdxContext*, i64, IdxScan*, IdxWhere*, IdxConstraint*, IdxConstraint*
+    sqlite3expert*, i64, IdxScan*, IdxWhere*, IdxConstraint*, IdxConstraint*
 );
 
 /*
@@ -594,7 +595,7 @@ static int idxFindConstraint(IdxConstraint *pList, IdxConstraint *p){
 }
 
 static int idxCreateFromWhere(
-  IdxContext *pCtx, 
+  sqlite3expert *p, 
   i64 mask,                       /* Consider only these constraints */
   IdxScan *pScan,                 /* Create indexes for this scan */
   IdxWhere *pWhere,               /* Read constraints from here */
@@ -618,7 +619,7 @@ static int idxCreateFromWhere(
 
   /* Create an index using the == constraints collected above. And the
   ** range constraint/ORDER BY terms passed in by the caller, if any. */
-  rc = idxCreateFromCons(pCtx, pScan, p1, pTail);
+  rc = idxCreateFromCons(p, pScan, p1, pTail);
 
   /* If no range/ORDER BY passed by the caller, create a version of the
   ** index for each range constraint that matches the mask. */
@@ -629,7 +630,7 @@ static int idxCreateFromWhere(
         && idxFindConstraint(pEq, pCon)==0
         && idxFindConstraint(pTail, pCon)==0
       ){
-        rc = idxCreateFromCons(pCtx, pScan, p1, pCon);
+        rc = idxCreateFromCons(p, pScan, p1, pCon);
       }
     }
   }
@@ -641,56 +642,90 @@ static int idxCreateFromWhere(
 ** Create candidate indexes in database [dbm] based on the data in 
 ** linked-list pScan.
 */
-static int idxCreateCandidates(IdxContext *pCtx){
-  sqlite3 *dbm = pCtx->dbm;
+static int idxCreateCandidates(sqlite3expert *p, char **pzErr){
+  sqlite3 *dbm = p->dbm;
   int rc2;
   int rc = SQLITE_OK;
-  sqlite3_stmt *pDepmask;         /* Foreach depmask */
+  sqlite3_stmt *pDepmask = 0;     /* Foreach depmask */
+  sqlite3_stmt *pInsert = 0;      /* insert */
   IdxScan *pIter;
 
-  rc = idxPrepareStmt(dbm, &pDepmask, pCtx->pzErrmsg, 
-      "SELECT mask FROM depmask"
+  rc = idxPrepareStmt(dbm, &pInsert, pzErr, 
+      "INSERT OR IGNORE INTO aux.depmask SELECT mask | ?1 FROM aux.depmask;"
   );
+  if( rc==SQLITE_OK ){
+    rc = idxPrepareStmt(dbm, &pDepmask, pzErr, "SELECT mask FROM depmask");
+  }
 
-  for(pIter=pCtx->pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
+  for(pIter=p->pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
     IdxWhere *pWhere = &pIter->where;
+    IdxConstraint *pCons;
+    rc = sqlite3_exec(dbm, 
+        "DELETE FROM aux.depmask;"
+        "INSERT INTO aux.depmask VALUES(0);"
+        , 0, 0, pzErr
+    );
+    for(pCons=pIter->where.pEq; pCons; pCons=pCons->pNext){
+      sqlite3_bind_int64(pInsert, 1, pCons->depmask);
+      sqlite3_step(pInsert);
+      rc = sqlite3_reset(pInsert);
+    }
+
     while( SQLITE_ROW==sqlite3_step(pDepmask) && rc==SQLITE_OK ){
       i64 mask = sqlite3_column_int64(pDepmask, 0);
-      rc = idxCreateFromWhere(pCtx, mask, pIter, pWhere, 0, 0);
+      rc = idxCreateFromWhere(p, mask, pIter, pWhere, 0, 0);
       if( rc==SQLITE_OK && pIter->pOrder ){
-        rc = idxCreateFromWhere(pCtx, mask, pIter, pWhere, 0, pIter->pOrder);
+        rc = idxCreateFromWhere(p, mask, pIter, pWhere, 0, pIter->pOrder);
       }
     }
+    rc2 = sqlite3_reset(pDepmask);
+    if( rc==SQLITE_OK ) rc = rc2;
   }
 
   rc2 = sqlite3_finalize(pDepmask);
   if( rc==SQLITE_OK ) rc = rc2;
+  rc2 = sqlite3_finalize(pInsert);
+  if( rc==SQLITE_OK ) rc = rc2;
   return rc;
 }
 
-static void idxScanFree(IdxScan *pScan){
-  IdxScan *pIter;
-  IdxScan *pNext;
-  for(pIter=pScan; pIter; pIter=pNext){
-    pNext = pIter->pNextScan;
-  }
+/*
+** Free all elements of the linked list starting from pScan up until pLast
+** (pLast is not freed).
+*/
+static void idxScanFree(IdxScan *pScan, IdxScan *pLast){
+  /* TODO! */
 }
 
+/*
+** Free all elements of the linked list starting from pStatement up 
+** until pLast (pLast is not freed).
+*/
+static void idxStatementFree(IdxStatement *pStatement, IdxStatement *pLast){
+  /* TODO! */
+}
+
+static void idxFinalize(int *pRc, sqlite3_stmt *pStmt){
+  int rc = sqlite3_finalize(pStmt);
+  if( *pRc==SQLITE_OK ) *pRc = rc;
+}
+static void idxReset(int *pRc, sqlite3_stmt *pStmt){
+  int rc = sqlite3_reset(pStmt);
+  if( *pRc==SQLITE_OK ) *pRc = rc;
+}
+
+
 int idxFindIndexes(
-  IdxContext *pCtx,
-  const char *zSql,                    /* SQL to find indexes for */
-  void (*xOut)(void*, const char*),    /* Output callback */
-  void *pOutCtx,                       /* Context for xOut() */
+  sqlite3expert *p,
   char **pzErr                         /* OUT: Error message (sqlite3_malloc) */
 ){
-  sqlite3 *dbm = pCtx->dbm;
-  sqlite3_stmt *pExplain = 0;
+  IdxStatement *pStmt;
+  sqlite3 *dbm = p->dbm;
   sqlite3_stmt *pSelect = 0;
   sqlite3_stmt *pInsert = 0;
   int rc, rc2;
   int bFound = 0;
 
-  rc = idxPrintfPrepareStmt(dbm, &pExplain, pzErr,"EXPLAIN QUERY PLAN %s",zSql);
   if( rc==SQLITE_OK ){
     rc = idxPrepareStmt(dbm, &pSelect, pzErr, 
         "SELECT rowid, sql FROM sqlite_master WHERE name = ?"
@@ -702,76 +737,81 @@ int idxFindIndexes(
     );
   }
 
-  while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){
-    int i;
-    const char *zDetail = (const char*)sqlite3_column_text(pExplain, 3);
-    int nDetail = strlen(zDetail);
+  for(pStmt=p->pStatement; rc==SQLITE_OK && pStmt; pStmt=pStmt->pNext){
+    sqlite3_stmt *pExplain = 0;
+    rc = sqlite3_exec(dbm, "DELETE FROM aux.indexes", 0, 0, 0);
+    if( rc==SQLITE_OK ){
+      rc = idxPrintfPrepareStmt(dbm, &pExplain, pzErr,
+          "EXPLAIN QUERY PLAN %s", pStmt->zSql
+      );
+    }
+    while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){
+      int i;
+      const char *zDetail = (const char*)sqlite3_column_text(pExplain, 3);
+      int nDetail = strlen(zDetail);
 
-    for(i=0; i<nDetail; i++){
-      const char *zIdx = 0;
-      if( memcmp(&zDetail[i], " USING INDEX ", 13)==0 ){
-        zIdx = &zDetail[i+13];
-      }else if( memcmp(&zDetail[i], " USING COVERING INDEX ", 22)==0 ){
-        zIdx = &zDetail[i+22];
-      }
-      if( zIdx ){
-        int nIdx = 0;
-        while( zIdx[nIdx]!='\0' && (zIdx[nIdx]!=' ' || zIdx[nIdx+1]!='(') ){
-          nIdx++;
+      for(i=0; i<nDetail; i++){
+        const char *zIdx = 0;
+        if( memcmp(&zDetail[i], " USING INDEX ", 13)==0 ){
+          zIdx = &zDetail[i+13];
+        }else if( memcmp(&zDetail[i], " USING COVERING INDEX ", 22)==0 ){
+          zIdx = &zDetail[i+22];
         }
-        sqlite3_bind_text(pSelect, 1, zIdx, nIdx, SQLITE_STATIC);
-        if( SQLITE_ROW==sqlite3_step(pSelect) ){
-          i64 iRowid = sqlite3_column_int64(pSelect, 0);
-          const char *zSql = (const char*)sqlite3_column_text(pSelect, 1);
-          if( iRowid>=pCtx->iIdxRowid ){
-            sqlite3_bind_text(pInsert, 1, zSql, -1, SQLITE_STATIC);
-            sqlite3_step(pInsert);
-            rc = sqlite3_reset(pInsert);
-            if( rc ) goto find_indexes_out;
+        if( zIdx ){
+          int nIdx = 0;
+          while( zIdx[nIdx]!='\0' && (zIdx[nIdx]!=' ' || zIdx[nIdx+1]!='(') ){
+            nIdx++;
           }
+          sqlite3_bind_text(pSelect, 1, zIdx, nIdx, SQLITE_STATIC);
+          if( SQLITE_ROW==sqlite3_step(pSelect) ){
+            i64 iRowid = sqlite3_column_int64(pSelect, 0);
+            const char *zSql = (const char*)sqlite3_column_text(pSelect, 1);
+            if( iRowid>=p->iIdxRowid ){
+              sqlite3_bind_text(pInsert, 1, zSql, -1, SQLITE_STATIC);
+              sqlite3_step(pInsert);
+              rc = sqlite3_reset(pInsert);
+              if( rc ) goto find_indexes_out;
+            }
+          }
+          rc = sqlite3_reset(pSelect);
+          break;
         }
-        rc = sqlite3_reset(pSelect);
-        break;
       }
     }
-  }
-  rc2 = sqlite3_reset(pExplain);
-  if( rc==SQLITE_OK ) rc = rc2;
-  if( rc==SQLITE_OK ){
-    sqlite3_stmt *pLoop = 0;
-    rc = idxPrepareStmt(dbm, &pLoop, pzErr,"SELECT name||';' FROM aux.indexes");
+    idxReset(&rc, pExplain);
     if( rc==SQLITE_OK ){
-      while( SQLITE_ROW==sqlite3_step(pLoop) ){
-        bFound = 1;
-        xOut(pOutCtx, (const char*)sqlite3_column_text(pLoop, 0));
+      sqlite3_stmt *pLoop = 0;
+      rc = idxPrepareStmt(dbm,&pLoop,pzErr,"SELECT name||';' FROM aux.indexes");
+      if( rc==SQLITE_OK ){
+        while( SQLITE_ROW==sqlite3_step(pLoop) ){
+          bFound = 1;
+          pStmt->zIdx = idxAppendText(&rc, pStmt->zIdx, "%s\n",
+              (const char*)sqlite3_column_text(pLoop, 0)
+          );
+        }
+        idxFinalize(&rc, pLoop);
+      }
+      if( bFound==0 ){
+        pStmt->zIdx = idxAppendText(&rc, pStmt->zIdx, "(no new indexes)\n");
       }
-      rc = sqlite3_finalize(pLoop);
     }
-    if( rc==SQLITE_OK ){
-      if( bFound==0 ) xOut(pOutCtx, "(no new indexes)");
-      xOut(pOutCtx, "");
-    }
-  }
 
-  while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){
-    int iSelectid = sqlite3_column_int(pExplain, 0);
-    int iOrder = sqlite3_column_int(pExplain, 1);
-    int iFrom = sqlite3_column_int(pExplain, 2);
-    const char *zDetail = (const char*)sqlite3_column_text(pExplain, 3);
-    char *zOut;
+    while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){
+      int iSelectid = sqlite3_column_int(pExplain, 0);
+      int iOrder = sqlite3_column_int(pExplain, 1);
+      int iFrom = sqlite3_column_int(pExplain, 2);
+      const char *zDetail = (const char*)sqlite3_column_text(pExplain, 3);
 
-    zOut = sqlite3_mprintf("%d|%d|%d|%s", iSelectid, iOrder, iFrom, zDetail);
-    if( zOut==0 ){
-      rc = SQLITE_NOMEM;
-    }else{
-      xOut(pOutCtx, zOut);
-      sqlite3_free(zOut);
+      pStmt->zEQP = idxAppendText(&rc, pStmt->zEQP, "%d|%d|%d|%s\n", 
+          iSelectid, iOrder, iFrom, zDetail
+      );
     }
+
+    rc2 = sqlite3_finalize(pExplain);
+    if( rc==SQLITE_OK ) rc = rc2;
   }
 
  find_indexes_out:
-  rc2 = sqlite3_finalize(pExplain);
-  if( rc==SQLITE_OK ) rc = rc2;
   rc2 = sqlite3_finalize(pSelect);
   if( rc==SQLITE_OK ) rc = rc2;
   rc2 = sqlite3_finalize(pInsert);
@@ -794,6 +834,7 @@ int shellIndexesCommand(
   char **pzErrmsg                      /* OUT: Error message (sqlite3_malloc) */
 ){
   int rc = SQLITE_OK;
+#if 0
   sqlite3 *dbm = 0;
   IdxContext ctx;
   sqlite3_stmt *pStmt = 0;        /* Statement compiled from zSql */
@@ -851,10 +892,175 @@ int shellIndexesCommand(
     rc = idxFindIndexes(&ctx, zSql, xOut, pOutCtx, pzErrmsg);
   }
 
-  idxScanFree(ctx.pScan);
+  idxScanFree(ctx.pScan, 0);
   sqlite3_finalize(ctx.pInsertMask);
   sqlite3_close(dbm);
+#endif
   return rc;
 }
 
+/*************************************************************************/
+
+/*
+** Allocate a new sqlite3expert object.
+*/
+sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){
+  int rc = SQLITE_OK;
+  sqlite3expert *pNew;
+
+  pNew = (sqlite3expert*)idxMalloc(&rc, sizeof(sqlite3expert));
+  pNew->db = db;
+
+  /* Open an in-memory database to work with. The main in-memory 
+  ** database schema contains tables similar to those in the users 
+  ** database (handle db). The attached in-memory db (aux) contains
+  ** application tables used by the code in this file.  */
+  rc = sqlite3_open(":memory:", &pNew->dbm);
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_exec(pNew->dbm,
+        "ATTACH ':memory:' AS aux;"
+        "CREATE TABLE aux.depmask(mask PRIMARY KEY) WITHOUT ROWID;"
+        "CREATE TABLE aux.indexes(name PRIMARY KEY) WITHOUT ROWID;"
+        , 0, 0, pzErrmsg
+    );
+  }
+
+  /* Copy the entire schema of database [db] into [dbm]. */
+  if( rc==SQLITE_OK ){
+    sqlite3_stmt *pSql;
+    int rc2;
+    rc = idxPrintfPrepareStmt(pNew->db, &pSql, pzErrmsg, 
+        "SELECT sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%%'"
+    );
+    while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
+      const char *zSql = (const char*)sqlite3_column_text(pSql, 0);
+      rc = sqlite3_exec(pNew->dbm, zSql, 0, 0, pzErrmsg);
+    }
+    rc2 = sqlite3_finalize(pSql);
+    if( rc==SQLITE_OK ) rc = rc2;
+  }
+
+  /* If an error has occurred, free the new object and reutrn NULL. Otherwise,
+  ** return the new sqlite3expert handle.  */
+  if( rc!=SQLITE_OK ){
+    sqlite3_expert_destroy(pNew);
+    pNew = 0;
+  }
+  return pNew;
+}
+
+/*
+** Add an SQL statement to the analysis.
+*/
+int sqlite3_expert_sql(
+  sqlite3expert *p,               /* From sqlite3_expert_new() */
+  const char *zSql,               /* SQL statement to add */
+  char **pzErr                    /* OUT: Error message (if any) */
+){
+  IdxScan *pScanOrig = p->pScan;
+  IdxStatement *pStmtOrig = p->pStatement;
+  int rc = SQLITE_OK;
+  const char *zStmt = zSql;
+
+  if( p->bRun ) return SQLITE_MISUSE;
+
+  sqlite3_whereinfo_hook(p->db, idxWhereInfo, p);
+  while( rc==SQLITE_OK && zStmt && zStmt[0] ){
+    sqlite3_stmt *pStmt = 0;
+    rc = sqlite3_prepare_v2(p->db, zStmt, -1, &pStmt, &zStmt);
+    if( rc==SQLITE_OK ){
+      if( pStmt ){
+        IdxStatement *pNew;
+        const char *z = sqlite3_sql(pStmt);
+        int n = strlen(z);
+        pNew = (IdxStatement*)idxMalloc(&rc, sizeof(IdxStatement) + n+1);
+        if( rc==SQLITE_OK ){
+          pNew->zSql = (char*)&pNew[1];
+          memcpy(pNew->zSql, z, n+1);
+          pNew->pNext = p->pStatement;
+          if( p->pStatement ) pNew->iId = p->pStatement->iId+1;
+          p->pStatement = pNew;
+        }
+        sqlite3_finalize(pStmt);
+      }
+    }else{
+      idxDatabaseError(p->db, pzErr);
+    }
+  }
+  sqlite3_whereinfo_hook(p->db, 0, 0);
+
+  if( rc!=SQLITE_OK ){
+    idxScanFree(p->pScan, pScanOrig);
+    idxStatementFree(p->pStatement, pStmtOrig);
+    p->pScan = pScanOrig;
+    p->pStatement = pStmtOrig;
+  }
+
+  return rc;
+}
+
+int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr){
+  int rc = SQLITE_OK;
+  IdxScan *pIter;
+
+  /* Load IdxTable objects */
+  for(pIter=p->pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
+    rc = idxGetTableInfo(p->dbm, pIter, pzErr);
+  }
+
+
+  /* Create candidate indexes within the in-memory database file */
+  if( rc==SQLITE_OK ){
+    rc = idxCreateCandidates(p, pzErr);
+  }
+
+  /* Figure out which of the candidate indexes are preferred by the query
+  ** planner and report the results to the user.  */
+  if( rc==SQLITE_OK ){
+    rc = idxFindIndexes(p, pzErr);
+  }
+
+  if( rc==SQLITE_OK ){
+    p->bRun = 1;
+  }
+  return rc;
+}
+
+int sqlite3_expert_count(sqlite3expert *p){
+  int nRet = 0;
+  if( p->pStatement ) nRet = p->pStatement->iId+1;
+  return nRet;
+}
+
+const char *sqlite3_expert_report(sqlite3expert *p, int iStmt, int eReport){
+  const char *zRet = 0;
+  IdxStatement *pStmt;
+
+  if( p->bRun==0 ) return 0;
+  for(pStmt=p->pStatement; pStmt && pStmt->iId!=iStmt; pStmt=pStmt->pNext);
+  if( pStmt ){
+    switch( eReport ){
+      case EXPERT_REPORT_SQL:
+        zRet = pStmt->zSql;
+        break;
+      case EXPERT_REPORT_INDEXES:
+        zRet = pStmt->zIdx;
+        break;
+      case EXPERT_REPORT_PLAN:
+        zRet = pStmt->zEQP;
+        break;
+    }
+  }
+  return zRet;
+}
+
+/*
+** Free an sqlite3expert object.
+*/
+void sqlite3_expert_destroy(sqlite3expert *p){
+  sqlite3_close(p->dbm);
+  idxScanFree(p->pScan, 0);
+  idxStatementFree(p->pStatement, 0);
+  sqlite3_free(p);
+}
 
diff --git a/ext/expert/sqlite3expert.h b/ext/expert/sqlite3expert.h
new file mode 100644
index 0000000000..6384f8e556
--- /dev/null
+++ b/ext/expert/sqlite3expert.h
@@ -0,0 +1,57 @@
+/*
+** 2017 April 07
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+*/
+
+
+#include "sqlite3.h"
+
+typedef struct sqlite3expert sqlite3expert;
+
+/*
+** Create a new sqlite3expert object.
+*/
+sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErr);
+
+/*
+** Add an SQL statement to the analysis.
+*/
+int sqlite3_expert_sql(
+  sqlite3expert *p,               /* From sqlite3_expert_new() */
+  const char *zSql,               /* SQL statement to add */
+  char **pzErr                    /* OUT: Error message (if any) */
+);
+
+int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr);
+
+/*
+** Return the total number of SQL queries loaded via sqlite3_expert_sql().
+*/
+int sqlite3_expert_count(sqlite3expert*);
+
+/*
+** Return a component of the report.
+*/
+const char *sqlite3_expert_report(sqlite3expert*, int iStmt, int eReport);
+
+/*
+** Values for the third argument passed to sqlite3_expert_report().
+*/
+#define EXPERT_REPORT_SQL        1
+#define EXPERT_REPORT_INDEXES    2
+#define EXPERT_REPORT_PLAN       3
+
+/*
+** Free an (sqlite3expert*) handle allocated by sqlite3-expert_new().
+*/
+void sqlite3_expert_destroy(sqlite3expert*);
+
+
diff --git a/main.mk b/main.mk
index b22d07d5a2..cb41cb7e00 100644
--- a/main.mk
+++ b/main.mk
@@ -491,7 +491,7 @@ libsqlite3.a:	$(LIBOBJ)
 	$(AR) libsqlite3.a $(LIBOBJ)
 	$(RANLIB) libsqlite3.a
 
-sqlite3$(EXE):	$(TOP)/src/shell.c libsqlite3.a sqlite3.h $(TOP)/src/shell_indexes.c
+sqlite3$(EXE):	$(TOP)/src/shell.c libsqlite3.a sqlite3.h
 	$(TCCX) $(READLINE_FLAGS) -o sqlite3$(EXE) $(SHELL_OPT) \
 		$(TOP)/src/shell.c libsqlite3.a $(LIBREADLINE) $(TLIBS) $(THREADLIB)
 
@@ -761,6 +761,9 @@ sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl
 sqlite3_analyzer$(EXE): sqlite3_analyzer.c
 	$(TCCX) $(TCL_FLAGS) sqlite3_analyzer.c -o $@ $(LIBTCL) $(THREADLIB) 
 
+sqlite3_expert$(EXE): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c
+	$(TCCX) -DSQLITE_ENABLE_WHEREINFO_HOOK $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert $(THREADLIB)
+
 sqlite3_schemalint.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/schemalint.tcl
 	echo "#define TCLSH 2" > $@
 	echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@
@@ -979,6 +982,7 @@ clean:
 	rm -f sqlite3rc.h
 	rm -f shell.c sqlite3ext.h
 	rm -f sqlite3_analyzer sqlite3_analyzer.exe sqlite3_analyzer.c
+	rm -f sqlite3_expert sqlite3_expert.exe 
 	rm -f sqlite-*-output.vsix
 	rm -f mptester mptester.exe
 	rm -f fuzzershell fuzzershell.exe
diff --git a/manifest b/manifest
index ce8fc15893..8a7b97520c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Changes\sto\sallow\sindexes\sto\sbe\srecommended\sfor\squeries\son\sSQL\sviews.
-D 2017-04-06T18:44:18.391
+C Refactor\scode\sto\ssuggest\sindexes\sfrom\sthe\sshell\stool\sinto\san\sextension\sin\next/expert.\sUnfinished.
+D 2017-04-07T20:14:22.417
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a
@@ -40,6 +40,9 @@ F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd
 F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
+F ext/expert/expert.c c73a0da702a2e9f5fd48046ab182683a73ee0318cefa3f45826536d015f39021
+F ext/expert/sqlite3expert.c 75ee320cf38b50df331232cbb60ec3b7703dd2770cc8df4ebbb445f664f6827d
+F ext/expert/sqlite3expert.h feeaee4ab73ba52426329781bbb28032ce18cf5abd2bf6221bac2df4c32b3013
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
 F ext/fts1/ft_hash.c 3927bd880e65329bdc6f506555b228b28924921b
 F ext/fts1/ft_hash.h 06df7bba40dadd19597aa400a875dbc2fed705ea
@@ -325,7 +328,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk e6f8f6895b75e4615a284b546933ca88fd56f3b85b560f4fb0da731abaa14f80
+F main.mk 6d4f1f1f78a6ac453d35f18c4e696fdefbe65dfec9530a41c5579ef8ec076072
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -402,8 +405,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 3e518b962d932a997fae373366880fc028c75706
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 2496d0cc6368dabe7ad2e4c7f5ed3ad9aa3b4d11cd90f33fa1d1ef72493f43aa
-F src/shell.c e524688c2544167f835ba43e24309f8707ca60c8ab6eb5c263a12c8618a233b8
-F src/shell_indexes.c 1f5ab036ec189411aeea27e6e74ab0009d831764d5d8517455dcb6b6a734beb7
+F src/shell.c ceb2b2f1f958ea2c47a7f37972d0f715fbf9dcf6a34a5e98c886b85e3ce6a238
 F src/sqlite.h.in ae5c9cbf2e77492c319fca08769575d9695e64718a16d32324944d24e291bcf7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 58fd0676d3111d02e62e5a35992a7d3da5d3f88753acc174f2d37b774fbbdd28
@@ -1571,7 +1573,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5cd070000da1d9e399090677b4db75dc5639c33211385d6eb84f14a4d0b617cd
-R aa13547c61fd31f43629472eaf9b4008
+P 0884ff1da2e27b146c764b73cf299a1f2cfe213c4a79bde34dec02d1fc946e70
+R 2f1dd4cb0b48a4b54a46f30181a94d12
 U dan
-Z 2f3b5e2a515dd18e4123d763066c9a33
+Z bc46ca2174a3c856c4daa1344a9d88d3
diff --git a/manifest.uuid b/manifest.uuid
index 3b9f56b1a1..9b195887ca 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0884ff1da2e27b146c764b73cf299a1f2cfe213c4a79bde34dec02d1fc946e70
\ No newline at end of file
+305e19f976ca064638a294e0817bf547ea745e1eb74746c5855514e6ced9c5fa
\ No newline at end of file
diff --git a/src/shell.c b/src/shell.c
index e51694bc76..1236b12a0e 100644
--- a/src/shell.c
+++ b/src/shell.c
@@ -166,7 +166,6 @@ static void setTextMode(FILE *file, int isOutput){
 # define setTextMode(X,Y)
 #endif
 
-#include "shell_indexes.c"
 
 /* True if the timer is enabled */
 static int enableTimer = 0;
@@ -1364,7 +1363,6 @@ struct ShellState {
   sqlite3 *db;           /* The database */
   int autoExplain;       /* Automatically turn on .explain mode */
   int autoEQP;           /* Run EXPLAIN QUERY PLAN prior to seach SQL stmt */
-  int bRecommend;        /* Instead of sqlite3_exec(), recommend indexes */
   int statsOn;           /* True to display memory stats before each finalize */
   int scanstatsOn;       /* True to display scan stats before each finalize */
   int outCount;          /* Revert to stdout when reaching zero */
@@ -2586,19 +2584,6 @@ static void explain_data_delete(ShellState *p){
   p->iIndent = 0;
 }
 
-typedef struct RecCommandCtx RecCommandCtx;
-struct RecCommandCtx {
-  int (*xCallback)(void*,int,char**,char**,int*);
-  ShellState *pArg;
-};
-
-static void recCommandOut(void *pCtx, const char *zLine){
-  const char *zCol = "output";
-  RecCommandCtx *p = (RecCommandCtx*)pCtx;
-  int t = SQLITE_TEXT;
-  p->xCallback(p->pArg, 1, (char**)&zLine, (char**)&zCol, &t);
-}
-
 /*
 ** Disable and restore .wheretrace and .selecttrace settings.
 */
@@ -2723,13 +2708,6 @@ static int shell_exec(
     *pzErrMsg = NULL;
   }
 
-  if( pArg->bRecommend ){
-    RecCommandCtx ctx;
-    ctx.xCallback = xCallback;
-    ctx.pArg = pArg;
-    rc = shellIndexesCommand(db, zSql, recCommandOut, &ctx, pzErrMsg);
-  }else
-
   while( zSql[0] && (SQLITE_OK == rc) ){
     static const char *zStmtSql;
     rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zLeftover);
@@ -5522,15 +5500,6 @@ static int do_meta_command(char *zLine, ShellState *p){
     sqlite3_close(pSrc);
   }else
 
-  if( c=='r' && n>=2 && strncmp(azArg[0], "recommend", n)==0 ){
-    if( nArg==2 ){
-      p->bRecommend = booleanValue(azArg[1]);
-    }else{
-      raw_printf(stderr, "Usage: .recommend on|off\n");
-      rc = 1;
-    }
-  }else
-
 
   if( c=='s' && strncmp(azArg[0], "scanstats", n)==0 ){
     if( nArg==2 ){
@@ -7337,9 +7306,6 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
           if( bail_on_error ) return rc;
         }
       }
-
-    }else if( strcmp(z, "-recommend") ){
-      data.bRecommend = 1;
     }else{
       utf8_printf(stderr,"%s: Error: unknown option: %s\n", Argv0, z);
       raw_printf(stderr,"Use -help for a list of options.\n");

From 65e67ed1b234403355c6f06404c4162493d5f6c7 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 8 Apr 2017 17:41:24 +0000
Subject: [PATCH 22/68] Use hash tables instead of in-memory database tables
 for a few purposes in sqlite3expert.c.

FossilOrigin-Name: bf10e68d9e4d5eae7ae6148a7ad64c9596f2ed8ccd36065adb09a1f9e7dae50b
---
 ext/expert/sqlite3expert.c | 438 +++++++++++++++++--------------------
 manifest                   |  12 +-
 manifest.uuid              |   2 +-
 3 files changed, 209 insertions(+), 243 deletions(-)

diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index 3bcfbbc6e8..fb442a0b3c 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -100,7 +100,6 @@ struct IdxContext {
   sqlite3 *dbm;                   /* In-memory db for this analysis */
   sqlite3 *db;                    /* User database under analysis */
   sqlite3_stmt *pInsertMask;      /* To write to aux.depmask */
-  i64 iIdxRowid;                  /* Rowid of first index created */
 };
 
 struct IdxStatement {
@@ -111,6 +110,21 @@ struct IdxStatement {
   IdxStatement *pNext;
 };
 
+
+#define IDX_HASH_SIZE 1023
+typedef struct IdxHashEntry IdxHashEntry;
+typedef struct IdxHash IdxHash;
+struct IdxHashEntry {
+  char *zKey;                     /* nul-terminated key */
+  char *zVal;                     /* nul-terminated value string */
+  IdxHashEntry *pHashNext;        /* Next entry in same hash bucket */
+  IdxHashEntry *pNext;            /* Next entry in hash */
+};
+struct IdxHash {
+  IdxHashEntry *pFirst;
+  IdxHashEntry *aHash[IDX_HASH_SIZE];
+};
+
 /*
 ** sqlite3expert object.
 */
@@ -124,7 +138,8 @@ struct sqlite3expert {
   IdxScan *pScan;                 /* List of scan objects */
   IdxStatement *pStatement;       /* List of IdxStatement objects */
   int rc;                         /* Error code from whereinfo hook */
-  i64 iIdxRowid;                  /* Rowid of first index created */
+
+  IdxHash hIdx;                   /* Hash containing all candidate indexes */
 };
 
 
@@ -145,6 +160,126 @@ static void *idxMalloc(int *pRc, int nByte){
   return pRet;
 }
 
+/*************************************************************************
+** Start of hash table implementations.
+*/
+typedef struct IdxHash64Entry IdxHash64Entry;
+typedef struct IdxHash64 IdxHash64;
+struct IdxHash64Entry {
+  u64 iVal;
+  IdxHash64Entry *pNext;          /* Next entry in hash table */
+  IdxHash64Entry *pHashNext;      /* Next entry in same hash bucket */
+};
+struct IdxHash64 {
+  IdxHash64Entry *pFirst;         /* Most recently added entry in hash table */
+  IdxHash64Entry *aHash[IDX_HASH_SIZE];
+};
+
+static void idxHash64Init(IdxHash64 *pHash){
+  memset(pHash, 0, sizeof(IdxHash64));
+}
+static void idxHash64Clear(IdxHash64 *pHash){
+  IdxHash64Entry *pEntry;
+  IdxHash64Entry *pNext;
+  for(pEntry=pHash->pFirst; pEntry; pEntry=pNext){
+    pNext = pEntry->pNext;
+    sqlite3_free(pEntry);
+  }
+  memset(pHash, 0, sizeof(IdxHash64));
+}
+static void idxHash64Add(int *pRc, IdxHash64 *pHash, u64 iVal){
+  int iHash = (int)((iVal*7) % IDX_HASH_SIZE);
+  IdxHash64Entry *pEntry;
+  assert( iHash>=0 );
+
+  for(pEntry=pHash->aHash[iHash]; pEntry; pEntry=pEntry->pHashNext){
+    if( pEntry->iVal==iVal ) return;
+  }
+  pEntry = idxMalloc(pRc, sizeof(IdxHash64Entry));
+  if( pEntry ){
+    pEntry->iVal = iVal;
+    pEntry->pHashNext = pHash->aHash[iHash];
+    pHash->aHash[iHash] = pEntry;
+    pEntry->pNext = pHash->pFirst;
+    pHash->pFirst = pEntry;
+  }
+}
+
+static void idxHashInit(IdxHash *pHash){
+  memset(pHash, 0, sizeof(IdxHash));
+}
+static void idxHashClear(IdxHash *pHash){
+  int i;
+  for(i=0; i<IDX_HASH_SIZE; i++){
+    IdxHashEntry *pEntry;
+    IdxHashEntry *pNext;
+    for(pEntry=pHash->aHash[i]; pEntry; pEntry=pNext){
+      pNext = pEntry->pHashNext;
+      sqlite3_free(pEntry);
+    }
+  }
+  memset(pHash, 0, sizeof(IdxHash));
+}
+static int idxHashString(const char *z, int n){
+  unsigned int ret = 0;
+  int i;
+  for(i=0; i<n; i++){
+    ret += (ret<<3) + (unsigned char)(z[i]);
+  }
+  return (int)(ret % IDX_HASH_SIZE);
+}
+
+static int idxHashAdd(
+  int *pRc, 
+  IdxHash *pHash, 
+  const char *zKey,
+  const char *zVal
+){
+  int nKey = strlen(zKey);
+  int iHash = idxHashString(zKey, nKey);
+  int nVal = (zVal ? strlen(zVal) : 0);
+  IdxHashEntry *pEntry;
+  assert( iHash>=0 );
+  for(pEntry=pHash->aHash[iHash]; pEntry; pEntry=pEntry->pHashNext){
+    if( strlen(pEntry->zKey)==nKey && 0==memcmp(pEntry->zKey, zKey, nKey) ){
+      return 1;
+    }
+  }
+  pEntry = idxMalloc(pRc, sizeof(IdxHashEntry) + nKey+1 + nVal+1);
+  if( pEntry ){
+    pEntry->zKey = (char*)&pEntry[1];
+    memcpy(pEntry->zKey, zKey, nKey);
+    if( zVal ){
+      pEntry->zVal = &pEntry->zKey[nKey+1];
+      memcpy(pEntry->zVal, zVal, nVal);
+    }
+    pEntry->pHashNext = pHash->aHash[iHash];
+    pHash->aHash[iHash] = pEntry;
+
+    pEntry->pNext = pHash->pFirst;
+    pHash->pFirst = pEntry;
+  }
+  return 0;
+}
+
+static const char *idxHashSearch(IdxHash *pHash, const char *zKey, int nKey){
+  int iHash;
+  IdxHashEntry *pEntry;
+  if( nKey<0 ) nKey = strlen(zKey);
+  iHash = idxHashString(zKey, nKey);
+  assert( iHash>=0 );
+  for(pEntry=pHash->aHash[iHash]; pEntry; pEntry=pEntry->pHashNext){
+    if( strlen(pEntry->zKey)==nKey && 0==memcmp(pEntry->zKey, zKey, nKey) ){
+      return pEntry->zVal;
+    }
+  }
+  return 0;
+}
+
+/*
+** End of hash table implementations.
+**************************************************************************/
+
 /*
 ** Allocate and return a new IdxConstraint object. Set the IdxConstraint.zColl
 ** variable to point to a copy of nul-terminated string zColl.
@@ -229,11 +364,6 @@ static void idxWhereInfo(
           pNew->pNext = p->pScan->where.pEq;
           p->pScan->where.pEq = pNew;
         }
-#if 0
-        sqlite3_bind_int64(p->pInsertMask, 1, mask);
-        sqlite3_step(p->pInsertMask);
-        p->rc = sqlite3_reset(p->pInsertMask);
-#endif
         break;
       }
     }
@@ -287,6 +417,11 @@ static int idxPrintfPrepareStmt(
   return rc;
 }
 
+static void idxFinalize(int *pRc, sqlite3_stmt *pStmt){
+  int rc = sqlite3_finalize(pStmt);
+  if( *pRc==SQLITE_OK ) *pRc = rc;
+}
+
 static int idxGetTableInfo(
   sqlite3 *db,
   IdxScan *pScan,
@@ -344,8 +479,7 @@ static int idxGetTableInfo(
 
     nCol++;
   }
-  rc2 = sqlite3_finalize(p1);
-  if( rc==SQLITE_OK ) rc = rc2;
+  idxFinalize(&rc, p1);
 
   if( rc==SQLITE_OK ){
     pScan->pTable = pNew;
@@ -456,7 +590,7 @@ static int idxFindCompatible(
   sqlite3_stmt *pIdxList = 0;
   IdxConstraint *pIter;
   int nEq = 0;                    /* Number of elements in pEq */
-  int rc, rc2;
+  int rc;
 
   /* Count the elements in list pEq */
   for(pIter=pEq; pIter; pIter=pIter->pLink) nEq++;
@@ -499,16 +633,14 @@ static int idxFindCompatible(
         }
       }
     }
-    rc2 = sqlite3_finalize(pInfo);
-    if( rc==SQLITE_OK ) rc = rc2;
+    idxFinalize(&rc, pInfo);
 
     if( rc==SQLITE_OK && bMatch ){
       sqlite3_finalize(pIdxList);
       return 1;
     }
   }
-  rc2 = sqlite3_finalize(pIdxList);
-  if( rc==SQLITE_OK ) rc = rc2;
+  idxFinalize(&rc, pIdxList);
 
   *pRc = rc;
   return 0;
@@ -539,49 +671,37 @@ static int idxCreateFromCons(
 
     if( rc==SQLITE_OK ){
       /* Hash the list of columns to come up with a name for the index */
+      char *zName;                /* Index name */
       int i;
       for(i=0; zCols[i]; i++){
         h += ((h<<3) + zCols[i]);
       }
-
-      if( idxIdentifierRequiresQuotes(pScan->zTable) ){
-        zFmt = "CREATE INDEX '%q_idx_%08x' ON %Q(%s)";
-      }else{
-        zFmt = "CREATE INDEX %s_idx_%08x ON %s(%s)";
-      }
-      zIdx = sqlite3_mprintf(zFmt, pScan->zTable, h, pScan->zTable, zCols);
-      if( !zIdx ){
+      zName = sqlite3_mprintf("%s_idx_%08x", pScan->zTable, h);
+      if( zName==0 ){ 
         rc = SQLITE_NOMEM;
       }else{
-        rc = sqlite3_exec(dbm, zIdx, 0, 0, p->pzErrmsg);
-#if 1
-        printf("CANDIDATE: %s\n", zIdx);
-#endif
+        if( idxIdentifierRequiresQuotes(pScan->zTable) ){
+          zFmt = "CREATE INDEX '%q' ON %Q(%s)";
+        }else{
+          zFmt = "CREATE INDEX %s ON %s(%s)";
+        }
+        zIdx = sqlite3_mprintf(zFmt, zName, pScan->zTable, zCols);
+        if( !zIdx ){
+          rc = SQLITE_NOMEM;
+        }else{
+          rc = sqlite3_exec(dbm, zIdx, 0, 0, p->pzErrmsg);
+          idxHashAdd(&rc, &p->hIdx, zName, zIdx);
+        }
+        sqlite3_free(zName);
+        sqlite3_free(zIdx);
       }
     }
-    if( rc==SQLITE_OK && p->iIdxRowid==0 ){
-      int rc2;
-      sqlite3_stmt *pLast = 0;
-      rc = idxPrepareStmt(dbm, &pLast, p->pzErrmsg, 
-          "SELECT max(rowid) FROM sqlite_master"
-      );
-      if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLast) ){
-        p->iIdxRowid = sqlite3_column_int64(pLast, 0);
-      }
-      rc2 = sqlite3_finalize(pLast);
-      if( rc==SQLITE_OK ) rc = rc2;
-    }
 
-    sqlite3_free(zIdx);
     sqlite3_free(zCols);
   }
   return rc;
 }
 
-static int idxCreateFromWhere(
-    sqlite3expert*, i64, IdxScan*, IdxWhere*, IdxConstraint*, IdxConstraint*
-);
-
 /*
 ** Return true if list pList (linked by IdxConstraint.pLink) contains
 ** a constraint compatible with *p. Otherwise return false.
@@ -643,49 +763,34 @@ static int idxCreateFromWhere(
 ** linked-list pScan.
 */
 static int idxCreateCandidates(sqlite3expert *p, char **pzErr){
-  sqlite3 *dbm = p->dbm;
-  int rc2;
   int rc = SQLITE_OK;
-  sqlite3_stmt *pDepmask = 0;     /* Foreach depmask */
-  sqlite3_stmt *pInsert = 0;      /* insert */
   IdxScan *pIter;
-
-  rc = idxPrepareStmt(dbm, &pInsert, pzErr, 
-      "INSERT OR IGNORE INTO aux.depmask SELECT mask | ?1 FROM aux.depmask;"
-  );
-  if( rc==SQLITE_OK ){
-    rc = idxPrepareStmt(dbm, &pDepmask, pzErr, "SELECT mask FROM depmask");
-  }
+  IdxHash64 hMask;
+  idxHash64Init(&hMask);
 
   for(pIter=p->pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
+    IdxHash64Entry *pEntry;
     IdxWhere *pWhere = &pIter->where;
     IdxConstraint *pCons;
-    rc = sqlite3_exec(dbm, 
-        "DELETE FROM aux.depmask;"
-        "INSERT INTO aux.depmask VALUES(0);"
-        , 0, 0, pzErr
-    );
+
+    idxHash64Add(&rc, &hMask, 0);
     for(pCons=pIter->where.pEq; pCons; pCons=pCons->pNext){
-      sqlite3_bind_int64(pInsert, 1, pCons->depmask);
-      sqlite3_step(pInsert);
-      rc = sqlite3_reset(pInsert);
+      for(pEntry=hMask.pFirst; pEntry; pEntry=pEntry->pNext){
+        idxHash64Add(&rc, &hMask, pEntry->iVal | (u64)pCons->depmask);
+      }
     }
 
-    while( SQLITE_ROW==sqlite3_step(pDepmask) && rc==SQLITE_OK ){
-      i64 mask = sqlite3_column_int64(pDepmask, 0);
+    for(pEntry=hMask.pFirst; pEntry; pEntry=pEntry->pNext){
+      i64 mask = (i64)pEntry->iVal;
       rc = idxCreateFromWhere(p, mask, pIter, pWhere, 0, 0);
       if( rc==SQLITE_OK && pIter->pOrder ){
         rc = idxCreateFromWhere(p, mask, pIter, pWhere, 0, pIter->pOrder);
       }
     }
-    rc2 = sqlite3_reset(pDepmask);
-    if( rc==SQLITE_OK ) rc = rc2;
+
+    idxHash64Clear(&hMask);
   }
 
-  rc2 = sqlite3_finalize(pDepmask);
-  if( rc==SQLITE_OK ) rc = rc2;
-  rc2 = sqlite3_finalize(pInsert);
-  if( rc==SQLITE_OK ) rc = rc2;
   return rc;
 }
 
@@ -705,15 +810,6 @@ static void idxStatementFree(IdxStatement *pStatement, IdxStatement *pLast){
   /* TODO! */
 }
 
-static void idxFinalize(int *pRc, sqlite3_stmt *pStmt){
-  int rc = sqlite3_finalize(pStmt);
-  if( *pRc==SQLITE_OK ) *pRc = rc;
-}
-static void idxReset(int *pRc, sqlite3_stmt *pStmt){
-  int rc = sqlite3_reset(pStmt);
-  if( *pRc==SQLITE_OK ) *pRc = rc;
-}
-
 
 int idxFindIndexes(
   sqlite3expert *p,
@@ -721,34 +817,25 @@ int idxFindIndexes(
 ){
   IdxStatement *pStmt;
   sqlite3 *dbm = p->dbm;
-  sqlite3_stmt *pSelect = 0;
-  sqlite3_stmt *pInsert = 0;
-  int rc, rc2;
-  int bFound = 0;
+  int rc = SQLITE_OK;
 
-  if( rc==SQLITE_OK ){
-    rc = idxPrepareStmt(dbm, &pSelect, pzErr, 
-        "SELECT rowid, sql FROM sqlite_master WHERE name = ?"
-    );
-  }
-  if( rc==SQLITE_OK ){
-    rc = idxPrepareStmt(dbm, &pInsert, pzErr,
-        "INSERT OR IGNORE INTO aux.indexes VALUES(?)"
-    );
-  }
+  IdxHash hIdx;
+  idxHashInit(&hIdx);
 
   for(pStmt=p->pStatement; rc==SQLITE_OK && pStmt; pStmt=pStmt->pNext){
+    IdxHashEntry *pEntry;
     sqlite3_stmt *pExplain = 0;
-    rc = sqlite3_exec(dbm, "DELETE FROM aux.indexes", 0, 0, 0);
-    if( rc==SQLITE_OK ){
-      rc = idxPrintfPrepareStmt(dbm, &pExplain, pzErr,
-          "EXPLAIN QUERY PLAN %s", pStmt->zSql
-      );
-    }
+    idxHashClear(&hIdx);
+    rc = idxPrintfPrepareStmt(dbm, &pExplain, pzErr,
+        "EXPLAIN QUERY PLAN %s", pStmt->zSql
+    );
     while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){
-      int i;
+      int iSelectid = sqlite3_column_int(pExplain, 0);
+      int iOrder = sqlite3_column_int(pExplain, 1);
+      int iFrom = sqlite3_column_int(pExplain, 2);
       const char *zDetail = (const char*)sqlite3_column_text(pExplain, 3);
       int nDetail = strlen(zDetail);
+      int i;
 
       for(i=0; i<nDetail; i++){
         const char *zIdx = 0;
@@ -758,149 +845,39 @@ int idxFindIndexes(
           zIdx = &zDetail[i+22];
         }
         if( zIdx ){
+          const char *zSql;
           int nIdx = 0;
           while( zIdx[nIdx]!='\0' && (zIdx[nIdx]!=' ' || zIdx[nIdx+1]!='(') ){
             nIdx++;
           }
-          sqlite3_bind_text(pSelect, 1, zIdx, nIdx, SQLITE_STATIC);
-          if( SQLITE_ROW==sqlite3_step(pSelect) ){
-            i64 iRowid = sqlite3_column_int64(pSelect, 0);
-            const char *zSql = (const char*)sqlite3_column_text(pSelect, 1);
-            if( iRowid>=p->iIdxRowid ){
-              sqlite3_bind_text(pInsert, 1, zSql, -1, SQLITE_STATIC);
-              sqlite3_step(pInsert);
-              rc = sqlite3_reset(pInsert);
-              if( rc ) goto find_indexes_out;
-            }
+          zSql = idxHashSearch(&p->hIdx, zIdx, nIdx);
+          if( zSql ){
+            idxHashAdd(&rc, &hIdx, zSql, 0);
+            if( rc ) goto find_indexes_out;
           }
-          rc = sqlite3_reset(pSelect);
           break;
         }
       }
-    }
-    idxReset(&rc, pExplain);
-    if( rc==SQLITE_OK ){
-      sqlite3_stmt *pLoop = 0;
-      rc = idxPrepareStmt(dbm,&pLoop,pzErr,"SELECT name||';' FROM aux.indexes");
-      if( rc==SQLITE_OK ){
-        while( SQLITE_ROW==sqlite3_step(pLoop) ){
-          bFound = 1;
-          pStmt->zIdx = idxAppendText(&rc, pStmt->zIdx, "%s\n",
-              (const char*)sqlite3_column_text(pLoop, 0)
-          );
-        }
-        idxFinalize(&rc, pLoop);
-      }
-      if( bFound==0 ){
-        pStmt->zIdx = idxAppendText(&rc, pStmt->zIdx, "(no new indexes)\n");
-      }
-    }
-
-    while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){
-      int iSelectid = sqlite3_column_int(pExplain, 0);
-      int iOrder = sqlite3_column_int(pExplain, 1);
-      int iFrom = sqlite3_column_int(pExplain, 2);
-      const char *zDetail = (const char*)sqlite3_column_text(pExplain, 3);
 
       pStmt->zEQP = idxAppendText(&rc, pStmt->zEQP, "%d|%d|%d|%s\n", 
           iSelectid, iOrder, iFrom, zDetail
       );
     }
 
-    rc2 = sqlite3_finalize(pExplain);
-    if( rc==SQLITE_OK ) rc = rc2;
+    for(pEntry=hIdx.pFirst; pEntry; pEntry=pEntry->pNext){
+      pStmt->zIdx = idxAppendText(&rc, pStmt->zIdx, "%s\n", pEntry->zKey);
+    }
+    if( pStmt->zIdx==0 ){
+      pStmt->zIdx = idxAppendText(&rc, 0, "(no new indexes)\n");
+    }
+
+    idxFinalize(&rc, pExplain);
   }
 
  find_indexes_out:
-  rc2 = sqlite3_finalize(pSelect);
-  if( rc==SQLITE_OK ) rc = rc2;
-  rc2 = sqlite3_finalize(pInsert);
-  if( rc==SQLITE_OK ) rc = rc2;
-
   return rc;
 }
 
-/*
-** The xOut callback is invoked to return command output to the user. The
-** second argument is always a nul-terminated string. The first argument is
-** passed zero if the string contains normal output or non-zero if it is an
-** error message.
-*/
-int shellIndexesCommand(
-  sqlite3 *db,                         /* Database handle */
-  const char *zSql,                    /* SQL to find indexes for */
-  void (*xOut)(void*, const char*),    /* Output callback */
-  void *pOutCtx,                       /* Context for xOut() */
-  char **pzErrmsg                      /* OUT: Error message (sqlite3_malloc) */
-){
-  int rc = SQLITE_OK;
-#if 0
-  sqlite3 *dbm = 0;
-  IdxContext ctx;
-  sqlite3_stmt *pStmt = 0;        /* Statement compiled from zSql */
-
-  memset(&ctx, 0, sizeof(IdxContext));
-  ctx.pzErrmsg = pzErrmsg;
-
-  /* Open an in-memory database to work with. The main in-memory 
-  ** database schema contains tables similar to those in the users 
-  ** database (handle db). The attached in-memory db (aux) contains
-  ** application tables used by the code in this file.  */
-  rc = sqlite3_open(":memory:", &dbm);
-  if( rc==SQLITE_OK ){
-    rc = sqlite3_exec(dbm, 
-        "ATTACH ':memory:' AS aux;"
-        "CREATE TABLE aux.depmask(mask PRIMARY KEY) WITHOUT ROWID;"
-        "CREATE TABLE aux.indexes(name PRIMARY KEY) WITHOUT ROWID;"
-        "INSERT INTO aux.depmask VALUES(0);"
-        , 0, 0, pzErrmsg
-    );
-  }
-
-  /* Prepare an INSERT statement for writing to aux.depmask */
-  if( rc==SQLITE_OK ){
-    rc = idxPrepareStmt(dbm, &ctx.pInsertMask, pzErrmsg,
-        "INSERT OR IGNORE INTO aux.depmask SELECT mask | ?1 FROM aux.depmask;"
-    );
-  }
-
-  /* Analyze the SELECT statement in zSql. */
-  if( rc==SQLITE_OK ){
-    ctx.dbm = dbm;
-    sqlite3_whereinfo_hook(db, idxWhereInfo, (void*)&ctx);
-    rc = idxPrepareStmt(db, &pStmt, pzErrmsg, zSql);
-    sqlite3_whereinfo_hook(db, 0, 0);
-    sqlite3_finalize(pStmt);
-  }
-
-  /* Create tables within the main in-memory database. These tables
-  ** have the same names, columns and declared types as the tables in
-  ** the user database. All constraints except for PRIMARY KEY are
-  ** removed. */
-  if( rc==SQLITE_OK ){
-    rc = idxCreateTables(db, dbm, ctx.pScan, pzErrmsg);
-  }
-
-  /* Create candidate indexes within the in-memory database file */
-  if( rc==SQLITE_OK ){
-    rc = idxCreateCandidates(&ctx);
-  }
-
-  /* Figure out which of the candidate indexes are preferred by the query
-  ** planner and report the results to the user.  */
-  if( rc==SQLITE_OK ){
-    rc = idxFindIndexes(&ctx, zSql, xOut, pOutCtx, pzErrmsg);
-  }
-
-  idxScanFree(ctx.pScan, 0);
-  sqlite3_finalize(ctx.pInsertMask);
-  sqlite3_close(dbm);
-#endif
-  return rc;
-}
-
-/*************************************************************************/
-
 /*
 ** Allocate a new sqlite3expert object.
 */
@@ -913,22 +890,12 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){
 
   /* Open an in-memory database to work with. The main in-memory 
   ** database schema contains tables similar to those in the users 
-  ** database (handle db). The attached in-memory db (aux) contains
-  ** application tables used by the code in this file.  */
+  ** database (handle db).  */
   rc = sqlite3_open(":memory:", &pNew->dbm);
-  if( rc==SQLITE_OK ){
-    rc = sqlite3_exec(pNew->dbm,
-        "ATTACH ':memory:' AS aux;"
-        "CREATE TABLE aux.depmask(mask PRIMARY KEY) WITHOUT ROWID;"
-        "CREATE TABLE aux.indexes(name PRIMARY KEY) WITHOUT ROWID;"
-        , 0, 0, pzErrmsg
-    );
-  }
 
   /* Copy the entire schema of database [db] into [dbm]. */
   if( rc==SQLITE_OK ){
     sqlite3_stmt *pSql;
-    int rc2;
     rc = idxPrintfPrepareStmt(pNew->db, &pSql, pzErrmsg, 
         "SELECT sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%%'"
     );
@@ -936,8 +903,7 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){
       const char *zSql = (const char*)sqlite3_column_text(pSql, 0);
       rc = sqlite3_exec(pNew->dbm, zSql, 0, 0, pzErrmsg);
     }
-    rc2 = sqlite3_finalize(pSql);
-    if( rc==SQLITE_OK ) rc = rc2;
+    idxFinalize(&rc, pSql);
   }
 
   /* If an error has occurred, free the new object and reutrn NULL. Otherwise,
diff --git a/manifest b/manifest
index 8a7b97520c..83ba986696 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Refactor\scode\sto\ssuggest\sindexes\sfrom\sthe\sshell\stool\sinto\san\sextension\sin\next/expert.\sUnfinished.
-D 2017-04-07T20:14:22.417
+C Use\shash\stables\sinstead\sof\sin-memory\sdatabase\stables\sfor\sa\sfew\spurposes\sin\nsqlite3expert.c.
+D 2017-04-08T17:41:24.364
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a
@@ -41,7 +41,7 @@ F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/expert.c c73a0da702a2e9f5fd48046ab182683a73ee0318cefa3f45826536d015f39021
-F ext/expert/sqlite3expert.c 75ee320cf38b50df331232cbb60ec3b7703dd2770cc8df4ebbb445f664f6827d
+F ext/expert/sqlite3expert.c c0e3ee75550fe73c3f6a536801071b97536e0907e898993086ab478853139327
 F ext/expert/sqlite3expert.h feeaee4ab73ba52426329781bbb28032ce18cf5abd2bf6221bac2df4c32b3013
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
 F ext/fts1/ft_hash.c 3927bd880e65329bdc6f506555b228b28924921b
@@ -1573,7 +1573,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0884ff1da2e27b146c764b73cf299a1f2cfe213c4a79bde34dec02d1fc946e70
-R 2f1dd4cb0b48a4b54a46f30181a94d12
+P 305e19f976ca064638a294e0817bf547ea745e1eb74746c5855514e6ced9c5fa
+R 628a1adb0870c895045a02cfe74a8f4c
 U dan
-Z bc46ca2174a3c856c4daa1344a9d88d3
+Z c1b7449477bbba31dce4f7f111235b67
diff --git a/manifest.uuid b/manifest.uuid
index 9b195887ca..04abc2e7d4 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-305e19f976ca064638a294e0817bf547ea745e1eb74746c5855514e6ced9c5fa
\ No newline at end of file
+bf10e68d9e4d5eae7ae6148a7ad64c9596f2ed8ccd36065adb09a1f9e7dae50b
\ No newline at end of file

From e7c3aca60161d6537e4375c2a94442439e5c7a40 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 8 Apr 2017 18:56:32 +0000
Subject: [PATCH 23/68] Rename shell6.test to expert1.test. Have it invoke the
 sqlite3_expert binary if it is present.

FossilOrigin-Name: be0deff940bea4f653d644113f42529a32ff7039d1d168119233aaf000a22f40
---
 ext/expert/expert.c                | 18 ++++++++++++++++--
 ext/expert/sqlite3expert.c         |  2 +-
 manifest                           | 16 ++++++++--------
 manifest.uuid                      |  2 +-
 test/{shell6.test => expert1.test} | 15 ++++++---------
 5 files changed, 32 insertions(+), 21 deletions(-)
 rename test/{shell6.test => expert1.test} (93%)

diff --git a/ext/expert/expert.c b/ext/expert/expert.c
index 5a5cda185f..e211faa08e 100644
--- a/ext/expert/expert.c
+++ b/ext/expert/expert.c
@@ -24,6 +24,10 @@ static void option_requires_argument(const char *zOpt){
   exit(-3);
 }
 
+static int option_integer_arg(const char *zVal){
+  return atoi(zVal);
+}
+
 static void usage(char **argv){
   fprintf(stderr, "\n");
   fprintf(stderr, "Usage %s ?OPTIONS? DATABASE\n", argv[0]);
@@ -31,6 +35,7 @@ static void usage(char **argv){
   fprintf(stderr, "Options are:\n");
   fprintf(stderr, "  -sql SQL   (analyze SQL statements passed as argument)\n");
   fprintf(stderr, "  -file FILE (read SQL statements from file FILE)\n");
+  fprintf(stderr, "  -verbose LEVEL (integer verbosity level. default 1)\n");
   exit(-1);
 }
 
@@ -43,6 +48,7 @@ int main(int argc, char **argv){
   int rc = 0;
   char *zErr = 0;
   int i;
+  int iVerbose = 1;               /* -verbose option */
 
   sqlite3 *db = 0;
   sqlite3expert *p = 0;
@@ -73,6 +79,11 @@ int main(int argc, char **argv){
         rc = sqlite3_expert_sql(p, argv[i], &zErr);
       }
 
+      else if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-verbose", nArg) ){
+        if( ++i==(argc-1) ) option_requires_argument("-verbose");
+        iVerbose = option_integer_arg(argv[i]);
+      }
+
       else{
         usage(argv);
       }
@@ -89,8 +100,11 @@ int main(int argc, char **argv){
       const char *zSql = sqlite3_expert_report(p, i, EXPERT_REPORT_SQL);
       const char *zIdx = sqlite3_expert_report(p, i, EXPERT_REPORT_INDEXES);
       const char *zEQP = sqlite3_expert_report(p, i, EXPERT_REPORT_PLAN);
-      fprintf(stdout, "-- query %d ----------------------------------\n", i+1);
-      fprintf(stdout, "%s\n\n%s\n%s\n", zSql, zIdx, zEQP);
+      if( iVerbose>0 ){
+        fprintf(stdout, "-- query %d ----------------------------------\n",i+1);
+        fprintf(stdout, "%s\n\n", zSql);
+      }
+      fprintf(stdout, "%s\n%s\n", zIdx, zEQP);
     }
   }else if( zErr ){
     fprintf(stderr, "Error: %s\n", zErr);
diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index fb442a0b3c..6d674a748b 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -865,7 +865,7 @@ int idxFindIndexes(
     }
 
     for(pEntry=hIdx.pFirst; pEntry; pEntry=pEntry->pNext){
-      pStmt->zIdx = idxAppendText(&rc, pStmt->zIdx, "%s\n", pEntry->zKey);
+      pStmt->zIdx = idxAppendText(&rc, pStmt->zIdx, "%s;\n", pEntry->zKey);
     }
     if( pStmt->zIdx==0 ){
       pStmt->zIdx = idxAppendText(&rc, 0, "(no new indexes)\n");
diff --git a/manifest b/manifest
index 83ba986696..5deeb9c637 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Use\shash\stables\sinstead\sof\sin-memory\sdatabase\stables\sfor\sa\sfew\spurposes\sin\nsqlite3expert.c.
-D 2017-04-08T17:41:24.364
+C Rename\sshell6.test\sto\sexpert1.test.\sHave\sit\sinvoke\sthe\ssqlite3_expert\sbinary\nif\sit\sis\spresent.
+D 2017-04-08T18:56:32.081
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a
@@ -40,8 +40,8 @@ F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd
 F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
-F ext/expert/expert.c c73a0da702a2e9f5fd48046ab182683a73ee0318cefa3f45826536d015f39021
-F ext/expert/sqlite3expert.c c0e3ee75550fe73c3f6a536801071b97536e0907e898993086ab478853139327
+F ext/expert/expert.c 861735e296c68cd442c20d5f6e75d07c957227009b79b31581a0971481e35d97
+F ext/expert/sqlite3expert.c 864ebebab5fd12cad8fb5debd26ff083dffe87ade15c1b6dcc7a7c3ae0afdd1b
 F ext/expert/sqlite3expert.h feeaee4ab73ba52426329781bbb28032ce18cf5abd2bf6221bac2df4c32b3013
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
 F ext/fts1/ft_hash.c 3927bd880e65329bdc6f506555b228b28924921b
@@ -699,6 +699,7 @@ F test/exclusive.test 9a57bd66e39144b888ca75c309914fcdefb4e3f9
 F test/exclusive2.test 984090e8e9d1b331d2e8111daf6e5d61dda0bef7
 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7
 F test/exists.test 79a75323c78f02bbe9c251ea502a092f9ef63dac
+F test/expert1.test 4108ab79ee454679e3d46e3703943ac49676b6174703a13ed81f9ee8cd794e68
 F test/expr.test 66a2c9ac34f74f036faa4092f5402c7d3162fc93
 F test/extension01.test 00d13cec817f331a687a243e0e5a2d87b0e358c9
 F test/extraquick.test cb254400bd42bfb777ff675356aabf3287978f79
@@ -1122,7 +1123,6 @@ F test/shell2.test e242a9912f44f4c23c3d1d802a83e934e84c853b
 F test/shell3.test 9b95ba643eaa228376f06a898fb410ee9b6e57c1
 F test/shell4.test 89ad573879a745974ff2df20ff97c5d6ffffbd5d
 F test/shell5.test 50a732c1c2158b1cd62cf53975ce1ea7ce6b9dc9
-F test/shell6.test f37998b26dfde19beaaf06a4cb60c476f66a7b54affff3870a2a011402c13efc
 F test/shell7.test 07751911b294698e0c5df67bcbd29e7d2f0f2907
 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3
 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5
@@ -1573,7 +1573,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 305e19f976ca064638a294e0817bf547ea745e1eb74746c5855514e6ced9c5fa
-R 628a1adb0870c895045a02cfe74a8f4c
+P bf10e68d9e4d5eae7ae6148a7ad64c9596f2ed8ccd36065adb09a1f9e7dae50b
+R e14200231c0812aa1e853be6a898ba2f
 U dan
-Z c1b7449477bbba31dce4f7f111235b67
+Z b04d01a3056a5795f1f1bb4407f0bf9d
diff --git a/manifest.uuid b/manifest.uuid
index 04abc2e7d4..6f644ebe90 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-bf10e68d9e4d5eae7ae6148a7ad64c9596f2ed8ccd36065adb09a1f9e7dae50b
\ No newline at end of file
+be0deff940bea4f653d644113f42529a32ff7039d1d168119233aaf000a22f40
\ No newline at end of file
diff --git a/test/shell6.test b/test/expert1.test
similarity index 93%
rename from test/shell6.test
rename to test/expert1.test
index 2f7c251827..b5651040d8 100644
--- a/test/shell6.test
+++ b/test/expert1.test
@@ -16,20 +16,17 @@
 
 # Test plan:
 #
-#   shell1-1.*: Basic command line option handling.
-#   shell1-2.*: Basic "dot" command token parsing.
-#   shell1-3.*: Basic test that "dot" command can be called.
 #
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
-set testprefix shell6
+set testprefix expert1
 
 if {$tcl_platform(platform)=="windows"} {
-  set CLI "sqlite3.exe"
+  set CMD "sqlite3_expert.exe"
 } else {
-  set CLI ".././sqlite3"
+  set CMD ".././sqlite3_expert"
 }
-if {![file executable $CLI]} {
+if {![file executable $CMD]} {
   finish_test
   return
 }
@@ -42,7 +39,7 @@ proc squish {txt} {
 proc do_rec_test {tn sql res} {
   set res [squish [string trim $res]]
   set tst [subst -nocommands { 
-    squish [lindex [catchcmd [list -rec test.db {$sql;}]] 1]
+    squish [string trim [exec $::CMD -verbose 0 -sql {$sql;} test.db]]
   }]
   uplevel [list do_test $tn $tst $res]
 }
@@ -195,8 +192,8 @@ do_setup_rec_test 11.1 {
 } {
   SELECT * FROM t7 WHERE a=? OR b=?
 } {
-  CREATE INDEX t7_idx_00000061 ON t7(a);
   CREATE INDEX t7_idx_00000062 ON t7(b);
+  CREATE INDEX t7_idx_00000061 ON t7(a);
   0|0|0|SEARCH TABLE t7 USING INDEX t7_idx_00000061 (a=?) 
   0|0|0|SEARCH TABLE t7 USING INDEX t7_idx_00000062 (b=?)
 }

From 958151b00571ee62c626875a2c30cc099155f041 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sun, 9 Apr 2017 08:38:37 +0000
Subject: [PATCH 24/68] Fix the -file option on the sqlite3_expert program.

FossilOrigin-Name: 0857c48e02a76490fc623364f77363165dea94ec254f93d8f0fd0bac2968c572
---
 ext/expert/expert.c | 25 ++++++++++++++++++++++++-
 manifest            | 12 ++++++------
 manifest.uuid       |  2 +-
 3 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/ext/expert/expert.c b/ext/expert/expert.c
index e211faa08e..fca2d05194 100644
--- a/ext/expert/expert.c
+++ b/ext/expert/expert.c
@@ -40,7 +40,30 @@ static void usage(char **argv){
 }
 
 static int readSqlFromFile(sqlite3expert *p, const char *zFile, char **pzErr){
-  return SQLITE_OK;
+  FILE *in = fopen(zFile, "rb");
+  long nIn;
+  size_t nRead;
+  char *pBuf;
+  int rc;
+  if( in==0 ){
+    *pzErr = sqlite3_mprintf("failed to open file %s\n", zFile);
+    return SQLITE_ERROR;
+  }
+  fseek(in, 0, SEEK_END);
+  nIn = ftell(in);
+  rewind(in);
+  pBuf = sqlite3_malloc64( nIn+1 );
+  nRead = fread(pBuf, nIn, 1, in);
+  fclose(in);
+  if( nRead!=1 ){
+    sqlite3_free(pBuf);
+    *pzErr = sqlite3_mprintf("failed to read file %s\n", zFile);
+    return SQLITE_ERROR;
+  }
+  pBuf[nIn] = 0;
+  rc = sqlite3_expert_sql(p, pBuf, pzErr);
+  sqlite3_free(pBuf);
+  return rc;
 }
 
 int main(int argc, char **argv){
diff --git a/manifest b/manifest
index bcb6036c74..cfcf08aa4c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\slatest\strunk\swith\sthis\sbranch.
-D 2017-04-08T19:00:37.586
+C Fix\sthe\s-file\soption\son\sthe\ssqlite3_expert\sprogram.
+D 2017-04-09T08:38:37.326
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a
@@ -40,7 +40,7 @@ F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd
 F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
-F ext/expert/expert.c 861735e296c68cd442c20d5f6e75d07c957227009b79b31581a0971481e35d97
+F ext/expert/expert.c bf0fd71921cb7b807cda9a76fb380e3d6e6b980d6167093b2952b41ec9ad8f46
 F ext/expert/sqlite3expert.c 864ebebab5fd12cad8fb5debd26ff083dffe87ade15c1b6dcc7a7c3ae0afdd1b
 F ext/expert/sqlite3expert.h feeaee4ab73ba52426329781bbb28032ce18cf5abd2bf6221bac2df4c32b3013
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
@@ -1575,7 +1575,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P be0deff940bea4f653d644113f42529a32ff7039d1d168119233aaf000a22f40 204e72f0080e8f08f99978870bd3cb9d59b068ecffee82192d707c650548b43b
-R 6e0d9b80ca81b8d3b8a7b8f8dc2a80a0
+P e051e8f21b909c5e89a7203eef6501fecd64f82a634c4e10398b5ef7d716eb37
+R c2522cb6c6416b4473f74ba522229374
 U dan
-Z 4063bdd74acb02b7df335075f419d3f7
+Z 7f4db390eaebfa3a6aedc09b4bf9a775
diff --git a/manifest.uuid b/manifest.uuid
index 581e5de589..53bb31e126 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e051e8f21b909c5e89a7203eef6501fecd64f82a634c4e10398b5ef7d716eb37
\ No newline at end of file
+0857c48e02a76490fc623364f77363165dea94ec254f93d8f0fd0bac2968c572
\ No newline at end of file

From cd84474ece326bf56480b6cc9e118f6bb4e7b7d2 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 10 Apr 2017 16:13:20 +0000
Subject: [PATCH 25/68] Changes to allow the code in sqlite3expert.c to be
 tested directly (via the API in sqlite3expert.h) instead of by invoking the
 sqlite3_expert application. Fix memory leaks and other problems.

FossilOrigin-Name: 5dd9831721b70a89a26728adcd49c7f6103ef8266891a79c2db34d913702709e
---
 {test => ext/expert}/expert1.test |  81 +++++++----
 ext/expert/sqlite3expert.c        |  60 +++++---
 ext/expert/test_expert.c          | 218 ++++++++++++++++++++++++++++++
 main.mk                           |   2 +
 manifest                          |  21 +--
 manifest.uuid                     |   2 +-
 src/tclsqlite.c                   |   2 +
 test/permutations.test            |   1 +
 8 files changed, 330 insertions(+), 57 deletions(-)
 rename {test => ext/expert}/expert1.test (74%)
 create mode 100644 ext/expert/test_expert.c

diff --git a/test/expert1.test b/ext/expert/expert1.test
similarity index 74%
rename from test/expert1.test
rename to ext/expert/expert1.test
index b5651040d8..684c618780 100644
--- a/test/expert1.test
+++ b/ext/expert/expert1.test
@@ -26,39 +26,62 @@ if {$tcl_platform(platform)=="windows"} {
 } else {
   set CMD ".././sqlite3_expert"
 }
-if {![file executable $CMD]} {
-  finish_test
-  return
-}
-
 
 proc squish {txt} {
   regsub -all {[[:space:]]+} $txt { }
 }
 
-proc do_rec_test {tn sql res} {
-  set res [squish [string trim $res]]
-  set tst [subst -nocommands { 
-    squish [string trim [exec $::CMD -verbose 0 -sql {$sql;} test.db]]
-  }]
-  uplevel [list do_test $tn $tst $res]
-}
-
 proc do_setup_rec_test {tn setup sql res} {
   reset_db
   db eval $setup
   uplevel [list do_rec_test $tn $sql $res]
 }
 
+foreach {tn setup} {
+  1 {
+    if {![file executable $CMD]} { continue }
 
-do_setup_rec_test 1.1 { CREATE TABLE t1(a, b, c) } {
+    proc do_rec_test {tn sql res} {
+      set res [squish [string trim $res]]
+      set tst [subst -nocommands { 
+        squish [string trim [exec $::CMD -verbose 0 -sql {$sql;} test.db]]
+      }]
+      uplevel [list do_test $tn $tst $res]
+    }
+  }
+  2 {
+    if {[info commands sqlite3_expert_new]==""} { continue }
+
+    proc do_rec_test {tn sql res} {
+      set expert [sqlite3_expert_new db]
+      $expert sql $sql
+      $expert analyze
+
+      set result [list]
+      for {set i 0} {$i < [$expert count]} {incr i} {
+        lappend result [string trim [$expert report $i indexes]]
+        lappend result [string trim [$expert report $i plan]]
+      }
+
+      $expert destroy
+
+      set tst [subst -nocommands {set {} [squish [join {$result}]]}]
+      uplevel [list do_test $tn $tst [string trim [squish $res]]]
+    }
+  }
+} {
+
+  eval $setup
+
+
+do_setup_rec_test $tn.1.1 { CREATE TABLE t1(a, b, c) } {
   SELECT * FROM t1
 } {
   (no new indexes)
   0|0|0|SCAN TABLE t1
 }
 
-do_setup_rec_test 1.2 {
+do_setup_rec_test $tn.1.2 {
   CREATE TABLE t1(a, b, c);
 } {
   SELECT * FROM t1 WHERE b>?;
@@ -67,7 +90,7 @@ do_setup_rec_test 1.2 {
   0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_00000062 (b>?)
 }
 
-do_setup_rec_test 1.3 {
+do_setup_rec_test $tn.1.3 {
   CREATE TABLE t1(a, b, c);
 } {
   SELECT * FROM t1 WHERE b COLLATE nocase BETWEEN ? AND ?
@@ -76,7 +99,7 @@ do_setup_rec_test 1.3 {
   0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_3e094c27 (b>? AND b<?)
 }
 
-do_setup_rec_test 1.4 {
+do_setup_rec_test $tn.1.4 {
   CREATE TABLE t1(a, b, c);
 } {
   SELECT a FROM t1 ORDER BY b;
@@ -85,7 +108,7 @@ do_setup_rec_test 1.4 {
   0|0|0|SCAN TABLE t1 USING INDEX t1_idx_00000062
 }
 
-do_setup_rec_test 1.5 {
+do_setup_rec_test $tn.1.5 {
   CREATE TABLE t1(a, b, c);
 } {
   SELECT a FROM t1 WHERE a=? ORDER BY b;
@@ -94,7 +117,7 @@ do_setup_rec_test 1.5 {
   0|0|0|SEARCH TABLE t1 USING COVERING INDEX t1_idx_000123a7 (a=?)
 }
 
-do_setup_rec_test 1.6 {
+do_setup_rec_test $tn.1.6 {
   CREATE TABLE t1(a, b, c);
 } {
   SELECT min(a) FROM t1
@@ -103,7 +126,7 @@ do_setup_rec_test 1.6 {
   0|0|0|SEARCH TABLE t1 USING COVERING INDEX t1_idx_00000061
 }
 
-do_setup_rec_test 1.7 {
+do_setup_rec_test $tn.1.7 {
   CREATE TABLE t1(a, b, c);
 } {
   SELECT * FROM t1 ORDER BY a, b, c;
@@ -112,7 +135,7 @@ do_setup_rec_test 1.7 {
   0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_033e95fe
 }
 
-do_setup_rec_test 1.8 {
+do_setup_rec_test $tn.1.8 {
   CREATE TABLE t1(a, b, c);
 } {
   SELECT * FROM t1 ORDER BY a ASC, b COLLATE nocase DESC, c ASC;
@@ -121,7 +144,7 @@ do_setup_rec_test 1.8 {
   0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_5be6e222
 }
 
-do_setup_rec_test 1.9 {
+do_setup_rec_test $tn.1.9 {
   CREATE TABLE t1(a COLLATE NOCase, b, c);
 } {
   SELECT * FROM t1 WHERE a=?
@@ -133,7 +156,7 @@ do_setup_rec_test 1.9 {
 
 # Tables with names that require quotes.
 #
-do_setup_rec_test 8.1 {
+do_setup_rec_test $tn.8.1 {
   CREATE TABLE "t t"(a, b, c);
 } {
   SELECT * FROM "t t" WHERE a=?
@@ -142,7 +165,7 @@ do_setup_rec_test 8.1 {
   0|0|0|SEARCH TABLE t t USING INDEX t t_idx_00000061 (a=?) 
 }
 
-do_setup_rec_test 8.2 {
+do_setup_rec_test $tn.8.2 {
   CREATE TABLE "t t"(a, b, c);
 } {
   SELECT * FROM "t t" WHERE b BETWEEN ? AND ?
@@ -153,7 +176,7 @@ do_setup_rec_test 8.2 {
 
 # Columns with names that require quotes.
 #
-do_setup_rec_test 9.1 {
+do_setup_rec_test $tn.9.1 {
   CREATE TABLE t3(a, "b b", c);
 } {
   SELECT * FROM t3 WHERE "b b" = ?
@@ -162,7 +185,7 @@ do_setup_rec_test 9.1 {
   0|0|0|SEARCH TABLE t3 USING INDEX t3_idx_00050c52 (b b=?)
 }
 
-do_setup_rec_test 9.2 {
+do_setup_rec_test $tn.9.2 {
   CREATE TABLE t3(a, "b b", c);
 } {
   SELECT * FROM t3 ORDER BY "b b"
@@ -173,7 +196,7 @@ do_setup_rec_test 9.2 {
 
 # Transitive constraints
 #
-do_setup_rec_test 10.1 {
+do_setup_rec_test $tn.10.1 {
   CREATE TABLE t5(a, b);
   CREATE TABLE t6(c, d);
 } {
@@ -187,7 +210,7 @@ do_setup_rec_test 10.1 {
 
 # OR terms.
 #
-do_setup_rec_test 11.1 {
+do_setup_rec_test $tn.11.1 {
   CREATE TABLE t7(a, b);
 } {
   SELECT * FROM t7 WHERE a=? OR b=?
@@ -198,5 +221,7 @@ do_setup_rec_test 11.1 {
   0|0|0|SEARCH TABLE t7 USING INDEX t7_idx_00000062 (b=?)
 }
 
+}
+
 finish_test
 
diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index 6d674a748b..6bcc697fcf 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -11,6 +11,8 @@
 *************************************************************************
 */
 
+#if !defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHEREINFO_HOOK)
+
 #include "sqlite3expert.h"
 #include <assert.h>
 #include <string.h>
@@ -20,7 +22,6 @@ typedef sqlite3_int64 i64;
 typedef sqlite3_uint64 u64;
 
 typedef struct IdxConstraint IdxConstraint;
-typedef struct IdxContext IdxContext;
 typedef struct IdxScan IdxScan;
 typedef struct IdxStatement IdxStatement;
 typedef struct IdxWhere IdxWhere;
@@ -89,19 +90,6 @@ struct IdxTable {
   IdxColumn *aCol;
 };
 
-/*
-** Context object passed to idxWhereInfo() and other functions.
-*/
-struct IdxContext {
-  char **pzErrmsg;
-  IdxWhere *pCurrent;             /* Current where clause */
-  int rc;                         /* Error code (if error has occurred) */
-  IdxScan *pScan;                 /* List of scan objects */
-  sqlite3 *dbm;                   /* In-memory db for this analysis */
-  sqlite3 *db;                    /* User database under analysis */
-  sqlite3_stmt *pInsertMask;      /* To write to aux.depmask */
-};
-
 struct IdxStatement {
   int iId;                        /* Statement number */
   char *zSql;                     /* SQL statement */
@@ -301,7 +289,7 @@ static IdxConstraint *idxNewConstraint(int *pRc, const char *zColl){
 ** sqlite3_whereinfo_hook() callback.
 */
 static void idxWhereInfo(
-  void *pCtx,                     /* Pointer to IdxContext structure */
+  void *pCtx,                     /* Pointer to sqlite3expert structure */
   int eOp, 
   const char *zVal, 
   int iVal, 
@@ -794,12 +782,31 @@ static int idxCreateCandidates(sqlite3expert *p, char **pzErr){
   return rc;
 }
 
+static void idxConstraintFree(IdxConstraint *pConstraint){
+  IdxConstraint *pNext;
+  IdxConstraint *p;
+
+  for(p=pConstraint; p; p=pNext){
+    pNext = p->pNext;
+    sqlite3_free(p);
+  }
+}
+
 /*
 ** Free all elements of the linked list starting from pScan up until pLast
 ** (pLast is not freed).
 */
 static void idxScanFree(IdxScan *pScan, IdxScan *pLast){
-  /* TODO! */
+  IdxScan *p;
+  IdxScan *pNext;
+  for(p=pScan; p!=pLast; p=pNext){
+    pNext = p->pNextScan;
+    idxConstraintFree(p->pOrder);
+    idxConstraintFree(p->where.pEq);
+    idxConstraintFree(p->where.pRange);
+    sqlite3_free(p->pTable);
+    sqlite3_free(p);
+  }
 }
 
 /*
@@ -807,7 +814,14 @@ static void idxScanFree(IdxScan *pScan, IdxScan *pLast){
 ** until pLast (pLast is not freed).
 */
 static void idxStatementFree(IdxStatement *pStatement, IdxStatement *pLast){
-  /* TODO! */
+  IdxStatement *p;
+  IdxStatement *pNext;
+  for(p=pStatement; p!=pLast; p=pNext){
+    pNext = p->pNext;
+    sqlite3_free(p->zEQP);
+    sqlite3_free(p->zIdx);
+    sqlite3_free(p);
+  }
 }
 
 
@@ -875,6 +889,7 @@ int idxFindIndexes(
   }
 
  find_indexes_out:
+  idxHashClear(&hIdx);
   return rc;
 }
 
@@ -974,7 +989,6 @@ int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr){
     rc = idxGetTableInfo(p->dbm, pIter, pzErr);
   }
 
-
   /* Create candidate indexes within the in-memory database file */
   if( rc==SQLITE_OK ){
     rc = idxCreateCandidates(p, pzErr);
@@ -992,12 +1006,19 @@ int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr){
   return rc;
 }
 
+/*
+** Return the total number of statements that have been added to this
+** sqlite3expert using sqlite3_expert_sql().
+*/
 int sqlite3_expert_count(sqlite3expert *p){
   int nRet = 0;
   if( p->pStatement ) nRet = p->pStatement->iId+1;
   return nRet;
 }
 
+/*
+** Return a component of the report.
+*/
 const char *sqlite3_expert_report(sqlite3expert *p, int iStmt, int eReport){
   const char *zRet = 0;
   IdxStatement *pStmt;
@@ -1027,6 +1048,9 @@ void sqlite3_expert_destroy(sqlite3expert *p){
   sqlite3_close(p->dbm);
   idxScanFree(p->pScan, 0);
   idxStatementFree(p->pStatement, 0);
+  idxHashClear(&p->hIdx);
   sqlite3_free(p);
 }
 
+#endif /* !defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHEREINFO_HOOK) */
+
diff --git a/ext/expert/test_expert.c b/ext/expert/test_expert.c
new file mode 100644
index 0000000000..a37887e587
--- /dev/null
+++ b/ext/expert/test_expert.c
@@ -0,0 +1,218 @@
+/*
+** 2017 April 07
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+*/
+
+#if defined(SQLITE_TEST) && defined(SQLITE_ENABLE_WHEREINFO_HOOK)
+
+#include "sqlite3expert.h"
+#include <assert.h>
+#include <string.h>
+
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
+
+/*
+** Extract an sqlite3* db handle from the object passed as the second
+** argument. If successful, set *pDb to point to the db handle and return
+** TCL_OK. Otherwise, return TCL_ERROR.
+*/
+static int dbHandleFromObj(Tcl_Interp *interp, Tcl_Obj *pObj, sqlite3 **pDb){
+  Tcl_CmdInfo info;
+  if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(pObj), &info) ){
+    Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(pObj), 0);
+    return TCL_ERROR;
+  }
+
+  *pDb = *(sqlite3 **)info.objClientData;
+  return TCL_OK;
+}
+
+
+/*
+** Tclcmd:  $expert sql SQL
+**          $expert analyze
+**          $expert count
+**          $expert report STMT EREPORT
+**          $expert destroy
+*/
+static int SQLITE_TCLAPI testExpertCmd(
+  void *clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  sqlite3expert *pExpert = (sqlite3expert*)clientData;
+  struct Subcmd {
+    const char *zSub;
+    int nArg;
+    const char *zMsg;
+  } aSub[] = {
+    { "sql",       1, "TABLE",        }, /* 0 */
+    { "analyze",   0, "",             }, /* 1 */
+    { "count",     0, "",             }, /* 2 */
+    { "report",    2, "STMT EREPORT", }, /* 3 */
+    { "destroy",   0, "",             }, /* 4 */
+    { 0 }
+  };
+  int iSub;
+  int rc = TCL_OK;
+  char *zErr = 0;
+
+  if( objc<2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "SUBCOMMAND ...");
+    return TCL_ERROR;
+  }
+  rc = Tcl_GetIndexFromObjStruct(interp, 
+      objv[1], aSub, sizeof(aSub[0]), "sub-command", 0, &iSub
+  );
+  if( rc!=TCL_OK ) return rc;
+  if( objc!=2+aSub[iSub].nArg ){
+    Tcl_WrongNumArgs(interp, 2, objv, aSub[iSub].zMsg);
+    return TCL_ERROR;
+  }
+
+  switch( iSub ){
+    case 0: {      /* sql */
+      char *zArg = Tcl_GetString(objv[2]);
+      rc = sqlite3_expert_sql(pExpert, zArg, &zErr);
+      break;
+    }
+
+    case 1: {      /* analyze */
+      rc = sqlite3_expert_analyze(pExpert, &zErr);
+      break;
+    }
+
+    case 2: {      /* count */
+      int n = sqlite3_expert_count(pExpert);
+      Tcl_SetObjResult(interp, Tcl_NewIntObj(n));
+      break;
+    }
+
+    case 3: {      /* report */
+      const char *aEnum[] = {
+        "sql", "indexes", "plan", 0
+      };
+      int iEnum;
+      int iStmt;
+      const char *zReport;
+
+      if( Tcl_GetIntFromObj(interp, objv[2], &iStmt) 
+       || Tcl_GetIndexFromObj(interp, objv[3], aEnum, "report", 0, &iEnum)
+      ){
+        return TCL_ERROR;
+      }
+
+      assert( EXPERT_REPORT_SQL==1 );
+      assert( EXPERT_REPORT_INDEXES==2 );
+      assert( EXPERT_REPORT_PLAN==3 );
+      zReport = sqlite3_expert_report(pExpert, iStmt, 1+iEnum);
+      Tcl_SetObjResult(interp, Tcl_NewStringObj(zReport, -1));
+      break;
+    }
+
+    default:       /* destroy */
+      assert( iSub==4 );     
+      Tcl_DeleteCommand(interp, Tcl_GetString(objv[0]));
+      break;
+  }
+
+  if( rc!=TCL_OK ){
+    if( zErr ){
+      Tcl_SetObjResult(interp, Tcl_NewStringObj(zErr, -1));
+    }else{
+      extern const char *sqlite3ErrName(int);
+      Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
+    }
+  }
+  sqlite3_free(zErr);
+  return rc;
+}
+
+static void SQLITE_TCLAPI testExpertDel(void *clientData){
+  sqlite3expert *pExpert = (sqlite3expert*)clientData;
+  sqlite3_expert_destroy(pExpert);
+}
+
+/*
+** sqlite3_expert_new DB
+*/
+static int SQLITE_TCLAPI test_sqlite3_expert_new(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  static int iCmd = 0;
+  sqlite3 *db;
+  char *zCmd = 0;
+  char *zErr = 0;
+  sqlite3expert *pExpert;
+  int rc = TCL_OK;
+
+  if( objc!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "DB");
+    return TCL_ERROR;
+  }
+  if( dbHandleFromObj(interp, objv[1], &db) ){
+    return TCL_ERROR;
+  }
+
+  zCmd = sqlite3_mprintf("sqlite3expert%d", ++iCmd);
+  if( zCmd==0 ){
+    Tcl_AppendResult(interp, "out of memory", (char*)0);
+    return TCL_ERROR;
+  }
+
+  pExpert = sqlite3_expert_new(db, &zErr);
+  if( pExpert==0 ){
+    Tcl_AppendResult(interp, zErr, (char*)0);
+    rc = TCL_ERROR;
+  }else{
+    void *p = (void*)pExpert;
+    Tcl_CreateObjCommand(interp, zCmd, testExpertCmd, p, testExpertDel);
+    Tcl_SetObjResult(interp, Tcl_NewStringObj(zCmd, -1));
+  }
+
+  sqlite3_free(zCmd);
+  sqlite3_free(zErr);
+  return rc;
+}
+
+int TestExpert_Init(Tcl_Interp *interp){
+  struct Cmd {
+    const char *zCmd;
+    Tcl_ObjCmdProc *xProc;
+  } aCmd[] = {
+    { "sqlite3_expert_new", test_sqlite3_expert_new },
+  };
+  int i;
+
+  for(i=0; i<sizeof(aCmd)/sizeof(struct Cmd); i++){
+    struct Cmd *p = &aCmd[i];
+    Tcl_CreateObjCommand(interp, p->zCmd, p->xProc, 0, 0);
+  }
+
+  return TCL_OK;
+}
+
+#else /* defined(SQLITE_TEST) && defined(SQLITE_ENABLE_WHEREINFO_HOOK) */
+int TestExpert_Init(Tcl_Interp *interp){
+  return TCL_OK;
+}
+#endif
diff --git a/main.mk b/main.mk
index cb41cb7e00..ea4f9891f7 100644
--- a/main.mk
+++ b/main.mk
@@ -274,6 +274,8 @@ SRC += \
 # Source code to the test files.
 #
 TESTSRC = \
+  $(TOP)/ext/expert/sqlite3expert.c \
+  $(TOP)/ext/expert/test_expert.c \
   $(TOP)/ext/fts3/fts3_term.c \
   $(TOP)/ext/fts3/fts3_test.c \
   $(TOP)/ext/rbu/test_rbu.c \
diff --git a/manifest b/manifest
index cfcf08aa4c..7660557a22 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sthe\s-file\soption\son\sthe\ssqlite3_expert\sprogram.
-D 2017-04-09T08:38:37.326
+C Changes\sto\sallow\sthe\scode\sin\ssqlite3expert.c\sto\sbe\stested\sdirectly\s(via\sthe\nAPI\sin\ssqlite3expert.h)\sinstead\sof\sby\sinvoking\sthe\ssqlite3_expert\sapplication.\nFix\smemory\sleaks\sand\sother\sproblems.
+D 2017-04-10T16:13:20.707
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a
@@ -41,8 +41,10 @@ F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/expert.c bf0fd71921cb7b807cda9a76fb380e3d6e6b980d6167093b2952b41ec9ad8f46
-F ext/expert/sqlite3expert.c 864ebebab5fd12cad8fb5debd26ff083dffe87ade15c1b6dcc7a7c3ae0afdd1b
+F ext/expert/expert1.test c1b1405f3ac20e9f71dacdf7bd68ff22e273b249a419260b123ebe385daf2db5 w test/expert1.test
+F ext/expert/sqlite3expert.c b87f13e90b999b5b10c4ec004b6a935150c00d3af1a16944e262172b9b831b8c
 F ext/expert/sqlite3expert.h feeaee4ab73ba52426329781bbb28032ce18cf5abd2bf6221bac2df4c32b3013
+F ext/expert/test_expert.c bad0611732d07180d586bd589cbb7713dc3ab0338c52bff29680eb2007678c05
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
 F ext/fts1/ft_hash.c 3927bd880e65329bdc6f506555b228b28924921b
 F ext/fts1/ft_hash.h 06df7bba40dadd19597aa400a875dbc2fed705ea
@@ -328,7 +330,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 6d4f1f1f78a6ac453d35f18c4e696fdefbe65dfec9530a41c5579ef8ec076072
+F main.mk ef818c7b1bb21f657e3bfb363cc7167264d688ca404a666e6ddda6029e94c43b
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -413,7 +415,7 @@ F src/sqliteInt.h 36eec0868d25b50e64a05bc7355557a0aa9bd14330c16b7d7568520b2bf2bb
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
-F src/tclsqlite.c 6c2151b6d8d98e183a04466d40df8889c0574d79
+F src/tclsqlite.c 2e0f7f63de8329526fbcb14fa5261d3574b2a06dd330f4df680120a3e6156133
 F src/test1.c 8a98191a1da8e100f77cdb5cc716df67d405028d
 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5
 F src/test3.c d03f5b5da9a2410b7a91c64b0d3306ed28ab6fee
@@ -699,7 +701,6 @@ F test/exclusive.test 9a57bd66e39144b888ca75c309914fcdefb4e3f9
 F test/exclusive2.test 984090e8e9d1b331d2e8111daf6e5d61dda0bef7
 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7
 F test/exists.test 79a75323c78f02bbe9c251ea502a092f9ef63dac
-F test/expert1.test 4108ab79ee454679e3d46e3703943ac49676b6174703a13ed81f9ee8cd794e68
 F test/expr.test 66a2c9ac34f74f036faa4092f5402c7d3162fc93
 F test/extension01.test 00d13cec817f331a687a243e0e5a2d87b0e358c9
 F test/extraquick.test cb254400bd42bfb777ff675356aabf3287978f79
@@ -1031,7 +1032,7 @@ F test/parser1.test 391b9bf9a229547a129c61ac345ed1a6f5eb1854
 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b
 F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442
 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff
-F test/permutations.test af720e7d139e7e5417341d0f0eef2b911c0b067852138dc2f5b6a451b5725118
+F test/permutations.test 9c0da2079fa37e7509957c9efbbdc282dea4ed0e732d19e6f216d53ae431a67d
 F test/pragma.test 1e94755164a3a3264cd39836de4bebcb7809e5f8
 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f
 F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed
@@ -1575,7 +1576,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e051e8f21b909c5e89a7203eef6501fecd64f82a634c4e10398b5ef7d716eb37
-R c2522cb6c6416b4473f74ba522229374
+P 0857c48e02a76490fc623364f77363165dea94ec254f93d8f0fd0bac2968c572
+R edcb3d0fdfbb918bffbc8d72b5429335
 U dan
-Z 7f4db390eaebfa3a6aedc09b4bf9a775
+Z 0d7d47356215948792ad81449f48b82b
diff --git a/manifest.uuid b/manifest.uuid
index 53bb31e126..a059662f4d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0857c48e02a76490fc623364f77363165dea94ec254f93d8f0fd0bac2968c572
\ No newline at end of file
+5dd9831721b70a89a26728adcd49c7f6103ef8266891a79c2db34d913702709e
\ No newline at end of file
diff --git a/src/tclsqlite.c b/src/tclsqlite.c
index 9df023b45c..a4a0ed3f30 100644
--- a/src/tclsqlite.c
+++ b/src/tclsqlite.c
@@ -4129,6 +4129,7 @@ static void init_all(Tcl_Interp *interp){
 #if defined(SQLITE_ENABLE_SESSION) && defined(SQLITE_ENABLE_PREUPDATE_HOOK)
     extern int TestSession_Init(Tcl_Interp*);
 #endif
+    extern int TestExpert_Init(Tcl_Interp*);
     extern int Fts5tcl_Init(Tcl_Interp *);
     extern int SqliteRbu_Init(Tcl_Interp*);
     extern int Sqlitetesttcl_Init(Tcl_Interp*);
@@ -4177,6 +4178,7 @@ static void init_all(Tcl_Interp *interp){
 #if defined(SQLITE_ENABLE_SESSION) && defined(SQLITE_ENABLE_PREUPDATE_HOOK)
     TestSession_Init(interp);
 #endif
+    TestExpert_Init(interp);
     Fts5tcl_Init(interp);
     SqliteRbu_Init(interp);
     Sqlitetesttcl_Init(interp);
diff --git a/test/permutations.test b/test/permutations.test
index 628e7ba44e..7e47155154 100644
--- a/test/permutations.test
+++ b/test/permutations.test
@@ -89,6 +89,7 @@ foreach f [glob $testdir/*.test] { lappend alltests [file tail $f] }
 foreach f [glob -nocomplain       \
     $testdir/../ext/rtree/*.test  \
     $testdir/../ext/fts5/test/*.test   \
+    $testdir/../ext/expert/*.test   \
 ] { 
   lappend alltests $f 
 }

From 96d43a05ecbc611a27751446c808d1710b2d8e6e Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 10 Apr 2017 20:00:26 +0000
Subject: [PATCH 26/68] Add ext/expert/README.md.

FossilOrigin-Name: 9318f1b9ed2d8da3a82ea69179e2d56a99d326c7721642665f87f6a4534e7bf0
---
 ext/expert/README.md       |  68 +++++++++++
 ext/expert/expert1.test    |   4 +-
 ext/expert/sqlite3expert.c | 237 +++++++++++++++++++++++--------------
 manifest                   |  15 +--
 manifest.uuid              |   2 +-
 5 files changed, 228 insertions(+), 98 deletions(-)
 create mode 100644 ext/expert/README.md

diff --git a/ext/expert/README.md b/ext/expert/README.md
new file mode 100644
index 0000000000..3cede6c09d
--- /dev/null
+++ b/ext/expert/README.md
@@ -0,0 +1,68 @@
+## SQLite Expert Extension
+
+This folder contains code for a simple system to propose useful indexes
+given a database and a set of SQL queries. It works as follows:
+
+  1. The user database schema is copied to a temporary database.
+
+  1. All SQL queries are prepared against the temporary database. The
+     **sqlite3\_whereinfo\_hook()** API is used to record information regarding
+     the WHERE and ORDER BY clauses attached to each query.
+
+  1. The information gathered in step 2 is used to create (possibly a large
+     number of) candidate indexes.
+
+  1. The SQL queries are prepared a second time. If the planner uses any
+     of the indexes created in step 3, they are recommended to the user.
+
+No ANALYZE data is available to the planner in step 4 above. This can lead to sub-optimal results.
+
+This extension requires that SQLite be built with the 
+SQLITE\_ENABLE\_WHEREINFO\_HOOK pre-processor symbol defined.
+
+# C API
+
+The SQLite expert C API is defined in sqlite3expert.h. Most uses will proceed
+as follows:
+
+  1. An sqlite3expert object is created by calling **sqlite3\_expert\_new()**.
+     A database handle opened by the user is passed as an argument.
+
+  1. The sqlite3expert object is configured with one or more SQL statements
+     by making one or more calls to **sqlite3\_expert\_sql()**. Each call may
+     specify a single SQL statement, or multiple statements separated by
+     semi-colons.
+
+  1. **sqlite3\_expert\_analyze()** is called to run the analysis.
+
+  1. One or more calls are made to **sqlite3\_expert\_report()** to extract
+     components of the results of the analysis.
+
+  1. **sqlite3\_expert\_destroy()** is called to free all resources.
+
+Refer to comments in sqlite3expert.h for further details.
+
+# sqlite3_expert application
+
+The file "expert.c" contains the code for a command line application that
+uses the API described above. It can be compiled with (for example):
+
+<pre>
+  gcc -O2 -DSQLITE_ENABLE_WHEREINFO_HOOK sqlite3.c expert.c sqlite3expert.c -o sqlite3_expert
+</pre>
+
+Assuming the database is "test.db", it can then be run to analyze a single query:
+
+<pre>
+  ./sqlite3_expert -sql &lt;sql-query&gt; test.db
+</pre>
+
+Or an entire text file worth of queries with:
+
+<pre>
+  ./sqlite3_expert -file &lt;text-file&gt; test.db
+</pre>
+
+
+
+
diff --git a/ext/expert/expert1.test b/ext/expert/expert1.test
index 684c618780..2e8c1c7e76 100644
--- a/ext/expert/expert1.test
+++ b/ext/expert/expert1.test
@@ -17,7 +17,9 @@
 # Test plan:
 #
 #
-set testdir [file dirname $argv0]
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+}
 source $testdir/tester.tcl
 set testprefix expert1
 
diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index 6bcc697fcf..faa6966932 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -1,5 +1,5 @@
 /*
-** 2016 February 10
+** 2017 April 09
 **
 ** The author disclaims copyright to this source code.  In place of
 ** a legal notice, here is a blessing:
@@ -21,16 +21,15 @@
 typedef sqlite3_int64 i64;
 typedef sqlite3_uint64 u64;
 
+typedef struct IdxColumn IdxColumn;
 typedef struct IdxConstraint IdxConstraint;
 typedef struct IdxScan IdxScan;
 typedef struct IdxStatement IdxStatement;
-typedef struct IdxWhere IdxWhere;
-
-typedef struct IdxColumn IdxColumn;
 typedef struct IdxTable IdxTable;
 
 /*
-** A single constraint. Equivalent to either "col = ?" or "col < ?".
+** A single constraint. Equivalent to either "col = ?" or "col < ?" (or
+** any other type of single-ended range constraint on a column).
 **
 ** pLink:
 **   Used to temporarily link IdxConstraint objects into lists while
@@ -47,38 +46,22 @@ struct IdxConstraint {
   IdxConstraint *pLink;           /* See above */
 };
 
-/*
-** A WHERE clause. Made up of IdxConstraint objects. Example WHERE clause:
-**
-**   a=? AND b=? AND c=? AND d=? AND e>? AND f<?
-**
-** The above is decomposed into 6 AND connected clauses. The first four are
-** added to the IdxWhere.pEq linked list, the following two into 
-** IdxWhere.pRange.
-**
-** IdxWhere.pEq and IdxWhere.pRange are simple linked lists of IdxConstraint
-** objects linked by the IdxConstraint.pNext field.
-*/
-struct IdxWhere {
-  IdxConstraint *pEq;             /* List of == constraints */
-  IdxConstraint *pRange;          /* List of < constraints */
-};
-
 /*
 ** A single scan of a single table.
 */
 struct IdxScan {
-  IdxTable *pTable;               /* Table-info */
   char *zTable;                   /* Name of table to scan */
   int iDb;                        /* Database containing table zTable */
   i64 covering;                   /* Mask of columns required for cov. index */
   IdxConstraint *pOrder;          /* ORDER BY columns */
-  IdxWhere where;                 /* WHERE Constraints */
-  IdxScan *pNextScan;             /* Next IdxScan object for same query */
+  IdxConstraint *pEq;             /* List of == constraints */
+  IdxConstraint *pRange;          /* List of < constraints */
+  IdxScan *pNextScan;             /* Next IdxScan object for same analysis */
 };
 
 /*
-** Data regarding a database table. Extracted from "PRAGMA table_info"
+** Information regarding a single database table. Extracted from 
+** "PRAGMA table_info" by function idxGetTableInfo().
 */
 struct IdxColumn {
   char *zName;
@@ -90,6 +73,10 @@ struct IdxTable {
   IdxColumn *aCol;
 };
 
+/*
+** Each statement being analyzed is represented by an instance of this
+** structure.
+*/
 struct IdxStatement {
   int iId;                        /* Statement number */
   char *zSql;                     /* SQL statement */
@@ -99,6 +86,15 @@ struct IdxStatement {
 };
 
 
+/*
+** A hash table for storing strings. With space for a payload string
+** with each entry. Methods are:
+**
+**   idxHashInit()
+**   idxHashClear()
+**   idxHashAdd()
+**   idxHashSearch()
+*/
 #define IDX_HASH_SIZE 1023
 typedef struct IdxHashEntry IdxHashEntry;
 typedef struct IdxHash IdxHash;
@@ -113,20 +109,36 @@ struct IdxHash {
   IdxHashEntry *aHash[IDX_HASH_SIZE];
 };
 
+/*
+** A hash table for storing a set of 64-bit values. Methods are:
+**
+**   idxHash64Init()
+**   idxHash64Clear()
+**   idxHash64Add()
+*/
+typedef struct IdxHash64Entry IdxHash64Entry;
+typedef struct IdxHash64 IdxHash64;
+struct IdxHash64Entry {
+  u64 iVal;
+  IdxHash64Entry *pNext;          /* Next entry in hash table */
+  IdxHash64Entry *pHashNext;      /* Next entry in same hash bucket */
+};
+struct IdxHash64 {
+  IdxHash64Entry *pFirst;         /* Most recently added entry in hash table */
+  IdxHash64Entry *aHash[IDX_HASH_SIZE];
+};
+
 /*
 ** sqlite3expert object.
 */
 struct sqlite3expert {
-  sqlite3 *db;                    /* Users database */
+  sqlite3 *db;                    /* User database */
   sqlite3 *dbm;                   /* In-memory db for this analysis */
-
   int bRun;                       /* True once analysis has run */
   char **pzErrmsg;
-
   IdxScan *pScan;                 /* List of scan objects */
   IdxStatement *pStatement;       /* List of IdxStatement objects */
   int rc;                         /* Error code from whereinfo hook */
-
   IdxHash hIdx;                   /* Hash containing all candidate indexes */
 };
 
@@ -148,24 +160,16 @@ static void *idxMalloc(int *pRc, int nByte){
   return pRet;
 }
 
-/*************************************************************************
-** Start of hash table implementations.
+/*
+** Initialize an IdxHash64 hash table.
 */
-typedef struct IdxHash64Entry IdxHash64Entry;
-typedef struct IdxHash64 IdxHash64;
-struct IdxHash64Entry {
-  u64 iVal;
-  IdxHash64Entry *pNext;          /* Next entry in hash table */
-  IdxHash64Entry *pHashNext;      /* Next entry in same hash bucket */
-};
-struct IdxHash64 {
-  IdxHash64Entry *pFirst;         /* Most recently added entry in hash table */
-  IdxHash64Entry *aHash[IDX_HASH_SIZE];
-};
-
 static void idxHash64Init(IdxHash64 *pHash){
   memset(pHash, 0, sizeof(IdxHash64));
 }
+
+/*
+** Reset an IdxHash64 hash table.
+*/
 static void idxHash64Clear(IdxHash64 *pHash){
   IdxHash64Entry *pEntry;
   IdxHash64Entry *pNext;
@@ -175,6 +179,11 @@ static void idxHash64Clear(IdxHash64 *pHash){
   }
   memset(pHash, 0, sizeof(IdxHash64));
 }
+
+/*
+** Add iVal to the IdxHash64 hash table passed as the second argument. This
+** function is a no-op if iVal is already present in the hash table.
+*/
 static void idxHash64Add(int *pRc, IdxHash64 *pHash, u64 iVal){
   int iHash = (int)((iVal*7) % IDX_HASH_SIZE);
   IdxHash64Entry *pEntry;
@@ -193,9 +202,16 @@ static void idxHash64Add(int *pRc, IdxHash64 *pHash, u64 iVal){
   }
 }
 
+/*
+** Initialize an IdxHash hash table.
+*/
 static void idxHashInit(IdxHash *pHash){
   memset(pHash, 0, sizeof(IdxHash));
 }
+
+/*
+** Reset an IdxHash hash table.
+*/
 static void idxHashClear(IdxHash *pHash){
   int i;
   for(i=0; i<IDX_HASH_SIZE; i++){
@@ -208,6 +224,11 @@ static void idxHashClear(IdxHash *pHash){
   }
   memset(pHash, 0, sizeof(IdxHash));
 }
+
+/*
+** Return the index of the hash bucket that the string specified by the
+** arguments to this function belongs.
+*/
 static int idxHashString(const char *z, int n){
   unsigned int ret = 0;
   int i;
@@ -217,6 +238,11 @@ static int idxHashString(const char *z, int n){
   return (int)(ret % IDX_HASH_SIZE);
 }
 
+/*
+** If zKey is already present in the hash table, return non-zero and do
+** nothing. Otherwise, add an entry with key zKey and payload string zVal to
+** the hash table passed as the second argument. 
+*/
 static int idxHashAdd(
   int *pRc, 
   IdxHash *pHash, 
@@ -250,6 +276,12 @@ static int idxHashAdd(
   return 0;
 }
 
+/*
+** If the hash table contains an entry with a key equal to the string
+** passed as the final two arguments to this function, return a pointer
+** to the payload string. Otherwise, if zKey/nKey is not present in the
+** hash table, return NULL.
+*/
 static const char *idxHashSearch(IdxHash *pHash, const char *zKey, int nKey){
   int iHash;
   IdxHashEntry *pEntry;
@@ -264,10 +296,6 @@ static const char *idxHashSearch(IdxHash *pHash, const char *zKey, int nKey){
   return 0;
 }
 
-/*
-** End of hash table implementations.
-**************************************************************************/
-
 /*
 ** Allocate and return a new IdxConstraint object. Set the IdxConstraint.zColl
 ** variable to point to a copy of nul-terminated string zColl.
@@ -346,11 +374,11 @@ static void idxWhereInfo(
         pNew->depmask = mask;
 
         if( eOp==SQLITE_WHEREINFO_RANGE ){
-          pNew->pNext = p->pScan->where.pRange;
-          p->pScan->where.pRange = pNew;
+          pNew->pNext = p->pScan->pRange;
+          p->pScan->pRange = pNew;
         }else{
-          pNew->pNext = p->pScan->where.pEq;
-          p->pScan->where.pEq = pNew;
+          pNew->pNext = p->pScan->pEq;
+          p->pScan->pEq = pNew;
         }
         break;
       }
@@ -369,6 +397,9 @@ static void idxDatabaseError(
   *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
 }
 
+/*
+** Prepare an SQL statement.
+*/
 static int idxPrepareStmt(
   sqlite3 *db,                    /* Database handle to compile against */
   sqlite3_stmt **ppStmt,          /* OUT: Compiled SQL statement */
@@ -383,6 +414,9 @@ static int idxPrepareStmt(
   return rc;
 }
 
+/*
+** Prepare an SQL statement using the results of a printf() formatting.
+*/
 static int idxPrintfPrepareStmt(
   sqlite3 *db,                    /* Database handle to compile against */
   sqlite3_stmt **ppStmt,          /* OUT: Compiled SQL statement */
@@ -405,17 +439,32 @@ static int idxPrintfPrepareStmt(
   return rc;
 }
 
+/*
+** Finalize SQL statement pStmt. If (*pRc) is SQLITE_OK when this function
+** is called, set it to the return value of sqlite3_finalize() before
+** returning. Otherwise, discard the sqlite3_finalize() return value.
+*/
 static void idxFinalize(int *pRc, sqlite3_stmt *pStmt){
   int rc = sqlite3_finalize(pStmt);
   if( *pRc==SQLITE_OK ) *pRc = rc;
 }
 
+/*
+** Attempt to allocate an IdxTable structure corresponding to table zTab
+** in the main database of connection db. If successful, set (*ppOut) to
+** point to the new object and return SQLITE_OK. Otherwise, return an
+** SQLite error code and set (*ppOut) to NULL. In this case *pzErrmsg may be
+** set to point to an error string.
+**
+** It is the responsibility of the caller to eventually free either the
+** IdxTable object or error message using sqlite3_free().
+*/
 static int idxGetTableInfo(
-  sqlite3 *db,
-  IdxScan *pScan,
-  char **pzErrmsg
+  sqlite3 *db,                    /* Database connection to read details from */
+  const char *zTab,               /* Table name */
+  IdxTable **ppOut,               /* OUT: New object (if successful) */
+  char **pzErrmsg                 /* OUT: Error message (if not) */
 ){
-  const char *zTbl = pScan->zTable;
   sqlite3_stmt *p1 = 0;
   int nCol = 0;
   int nByte = sizeof(IdxTable);
@@ -423,12 +472,12 @@ static int idxGetTableInfo(
   int rc, rc2;
   char *pCsr;
 
-  rc = idxPrintfPrepareStmt(db, &p1, pzErrmsg, "PRAGMA table_info=%Q", zTbl);
+  rc = idxPrintfPrepareStmt(db, &p1, pzErrmsg, "PRAGMA table_info=%Q", zTab);
   while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){
     const char *zCol = (const char*)sqlite3_column_text(p1, 1);
     nByte += 1 + strlen(zCol);
     rc = sqlite3_table_column_metadata(
-        db, "main", zTbl, zCol, 0, &zCol, 0, 0, 0
+        db, "main", zTab, zCol, 0, &zCol, 0, 0, 0
     );
     nByte += 1 + strlen(zCol);
     nCol++;
@@ -456,7 +505,7 @@ static int idxGetTableInfo(
     pCsr += nCopy;
 
     rc = sqlite3_table_column_metadata(
-        db, "main", zTbl, zCol, 0, &zCol, 0, 0, 0
+        db, "main", zTab, zCol, 0, &zCol, 0, 0, 0
     );
     if( rc==SQLITE_OK ){
       nCopy = strlen(zCol) + 1;
@@ -469,12 +518,12 @@ static int idxGetTableInfo(
   }
   idxFinalize(&rc, p1);
 
-  if( rc==SQLITE_OK ){
-    pScan->pTable = pNew;
-  }else{
+  if( rc!=SQLITE_OK ){
     sqlite3_free(pNew);
+    pNew = 0;
   }
 
+  *ppOut = pNew;
   return rc;
 }
 
@@ -515,6 +564,10 @@ static char *idxAppendText(int *pRc, char *zIn, const char *zFmt, ...){
   return zRet;
 }
 
+/*
+** Return true if zId must be quoted in order to use it as an SQL
+** identifier, or false otherwise.
+*/
 static int idxIdentifierRequiresQuotes(const char *zId){
   int i;
   for(i=0; zId[i]; i++){
@@ -529,10 +582,14 @@ static int idxIdentifierRequiresQuotes(const char *zId){
   return 0;
 }
 
+/*
+** This function appends an index column definition suitable for constraint
+** pCons to the string passed as zIn and returns the result.
+*/
 static char *idxAppendColDefn(
-  int *pRc, 
-  char *zIn, 
-  IdxTable *pTab, 
+  int *pRc,                       /* IN/OUT: Error code */
+  char *zIn,                      /* Column defn accumulated so far */
+  IdxTable *pTab,                 /* Table index will be created on */
   IdxConstraint *pCons
 ){
   char *zRet = zIn;
@@ -636,6 +693,7 @@ static int idxFindCompatible(
 
 static int idxCreateFromCons(
   sqlite3expert *p,
+  IdxTable *pTab,
   IdxScan *pScan,
   IdxConstraint *pEq, 
   IdxConstraint *pTail
@@ -643,7 +701,6 @@ static int idxCreateFromCons(
   sqlite3 *dbm = p->dbm;
   int rc = SQLITE_OK;
   if( (pEq || pTail) && 0==idxFindCompatible(&rc, dbm, pScan, pEq, pTail) ){
-    IdxTable *pTab = pScan->pTable;
     char *zCols = 0;
     char *zIdx = 0;
     IdxConstraint *pCons;
@@ -704,9 +761,9 @@ static int idxFindConstraint(IdxConstraint *pList, IdxConstraint *p){
 
 static int idxCreateFromWhere(
   sqlite3expert *p, 
+  IdxTable *pTab,
   i64 mask,                       /* Consider only these constraints */
   IdxScan *pScan,                 /* Create indexes for this scan */
-  IdxWhere *pWhere,               /* Read constraints from here */
   IdxConstraint *pEq,             /* == constraints for inclusion */
   IdxConstraint *pTail            /* range/ORDER BY constraints for inclusion */
 ){
@@ -715,7 +772,7 @@ static int idxCreateFromWhere(
   int rc;
 
   /* Gather up all the == constraints that match the mask. */
-  for(pCon=pWhere->pEq; pCon; pCon=pCon->pNext){
+  for(pCon=pScan->pEq; pCon; pCon=pCon->pNext){
     if( (mask & pCon->depmask)==pCon->depmask 
      && idxFindConstraint(p1, pCon)==0
      && idxFindConstraint(pTail, pCon)==0
@@ -727,18 +784,18 @@ static int idxCreateFromWhere(
 
   /* Create an index using the == constraints collected above. And the
   ** range constraint/ORDER BY terms passed in by the caller, if any. */
-  rc = idxCreateFromCons(p, pScan, p1, pTail);
+  rc = idxCreateFromCons(p, pTab, pScan, p1, pTail);
 
   /* If no range/ORDER BY passed by the caller, create a version of the
   ** index for each range constraint that matches the mask. */
   if( pTail==0 ){
-    for(pCon=pWhere->pRange; rc==SQLITE_OK && pCon; pCon=pCon->pNext){
+    for(pCon=pScan->pRange; rc==SQLITE_OK && pCon; pCon=pCon->pNext){
       assert( pCon->pLink==0 );
       if( (mask & pCon->depmask)==pCon->depmask
         && idxFindConstraint(pEq, pCon)==0
         && idxFindConstraint(pTail, pCon)==0
       ){
-        rc = idxCreateFromCons(p, pScan, p1, pCon);
+        rc = idxCreateFromCons(p, pTab, pScan, p1, pCon);
       }
     }
   }
@@ -758,30 +815,36 @@ static int idxCreateCandidates(sqlite3expert *p, char **pzErr){
 
   for(pIter=p->pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
     IdxHash64Entry *pEntry;
-    IdxWhere *pWhere = &pIter->where;
     IdxConstraint *pCons;
+    IdxTable *pTab = 0;
+
+    rc = idxGetTableInfo(p->dbm, pIter->zTable, &pTab, pzErr);
 
     idxHash64Add(&rc, &hMask, 0);
-    for(pCons=pIter->where.pEq; pCons; pCons=pCons->pNext){
+    for(pCons=pIter->pEq; pCons; pCons=pCons->pNext){
       for(pEntry=hMask.pFirst; pEntry; pEntry=pEntry->pNext){
         idxHash64Add(&rc, &hMask, pEntry->iVal | (u64)pCons->depmask);
       }
     }
 
-    for(pEntry=hMask.pFirst; pEntry; pEntry=pEntry->pNext){
+    for(pEntry=hMask.pFirst; rc==SQLITE_OK && pEntry; pEntry=pEntry->pNext){
       i64 mask = (i64)pEntry->iVal;
-      rc = idxCreateFromWhere(p, mask, pIter, pWhere, 0, 0);
+      rc = idxCreateFromWhere(p, pTab, mask, pIter, 0, 0);
       if( rc==SQLITE_OK && pIter->pOrder ){
-        rc = idxCreateFromWhere(p, mask, pIter, pWhere, 0, pIter->pOrder);
+        rc = idxCreateFromWhere(p, pTab, mask, pIter, 0, pIter->pOrder);
       }
     }
 
+    sqlite3_free(pTab);
     idxHash64Clear(&hMask);
   }
 
   return rc;
 }
 
+/*
+** Free all elements of the linked list starting at pConstraint.
+*/
 static void idxConstraintFree(IdxConstraint *pConstraint){
   IdxConstraint *pNext;
   IdxConstraint *p;
@@ -802,9 +865,8 @@ static void idxScanFree(IdxScan *pScan, IdxScan *pLast){
   for(p=pScan; p!=pLast; p=pNext){
     pNext = p->pNextScan;
     idxConstraintFree(p->pOrder);
-    idxConstraintFree(p->where.pEq);
-    idxConstraintFree(p->where.pRange);
-    sqlite3_free(p->pTable);
+    idxConstraintFree(p->pEq);
+    idxConstraintFree(p->pRange);
     sqlite3_free(p);
   }
 }
@@ -825,6 +887,11 @@ static void idxStatementFree(IdxStatement *pStatement, IdxStatement *pLast){
 }
 
 
+/*
+** This function is called after candidate indexes have been created. It
+** runs all the queries to see which indexes they prefer, and populates
+** IdxStatement.zIdx and IdxStatement.zEQP with the results.
+*/
 int idxFindIndexes(
   sqlite3expert *p,
   char **pzErr                         /* OUT: Error message (sqlite3_malloc) */
@@ -981,18 +1048,10 @@ int sqlite3_expert_sql(
 }
 
 int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr){
-  int rc = SQLITE_OK;
-  IdxScan *pIter;
-
-  /* Load IdxTable objects */
-  for(pIter=p->pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
-    rc = idxGetTableInfo(p->dbm, pIter, pzErr);
-  }
+  int rc;
 
   /* Create candidate indexes within the in-memory database file */
-  if( rc==SQLITE_OK ){
-    rc = idxCreateCandidates(p, pzErr);
-  }
+  rc = idxCreateCandidates(p, pzErr);
 
   /* Figure out which of the candidate indexes are preferred by the query
   ** planner and report the results to the user.  */
diff --git a/manifest b/manifest
index 7660557a22..ab763c7f7c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Changes\sto\sallow\sthe\scode\sin\ssqlite3expert.c\sto\sbe\stested\sdirectly\s(via\sthe\nAPI\sin\ssqlite3expert.h)\sinstead\sof\sby\sinvoking\sthe\ssqlite3_expert\sapplication.\nFix\smemory\sleaks\sand\sother\sproblems.
-D 2017-04-10T16:13:20.707
+C Add\sext/expert/README.md.
+D 2017-04-10T20:00:26.414
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a
@@ -40,9 +40,10 @@ F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd
 F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
+F ext/expert/README.md 9f15075ec5ad772808eff55ef044c31140fd1146aa0a3c47eafd155e71851b01
 F ext/expert/expert.c bf0fd71921cb7b807cda9a76fb380e3d6e6b980d6167093b2952b41ec9ad8f46
-F ext/expert/expert1.test c1b1405f3ac20e9f71dacdf7bd68ff22e273b249a419260b123ebe385daf2db5 w test/expert1.test
-F ext/expert/sqlite3expert.c b87f13e90b999b5b10c4ec004b6a935150c00d3af1a16944e262172b9b831b8c
+F ext/expert/expert1.test d4451ccdca910ec3652a7a1994210fe2b7bc8f56431ed0cf2ff3dee798c89084
+F ext/expert/sqlite3expert.c 8bcb83b3723239dc6a2a199e7a030741b7ecf47814828a1da7ea559aa42f094a
 F ext/expert/sqlite3expert.h feeaee4ab73ba52426329781bbb28032ce18cf5abd2bf6221bac2df4c32b3013
 F ext/expert/test_expert.c bad0611732d07180d586bd589cbb7713dc3ab0338c52bff29680eb2007678c05
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
@@ -1576,7 +1577,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0857c48e02a76490fc623364f77363165dea94ec254f93d8f0fd0bac2968c572
-R edcb3d0fdfbb918bffbc8d72b5429335
+P 5dd9831721b70a89a26728adcd49c7f6103ef8266891a79c2db34d913702709e
+R 119e273740189bcdef018d8609f3a756
 U dan
-Z 0d7d47356215948792ad81449f48b82b
+Z 664c10ad907cc8fb129a4f4feaf498a1
diff --git a/manifest.uuid b/manifest.uuid
index a059662f4d..935458cfd1 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5dd9831721b70a89a26728adcd49c7f6103ef8266891a79c2db34d913702709e
\ No newline at end of file
+9318f1b9ed2d8da3a82ea69179e2d56a99d326c7721642665f87f6a4534e7bf0
\ No newline at end of file

From a4e61024d75cd1ac531553cee0a0dae13b460474 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 11 Apr 2017 17:43:12 +0000
Subject: [PATCH 27/68] Add header comments to the API functions in
 sqlite3expert.h. Include a list of all candidate indexes in the report output
 by the sqlite3_expert program.

FossilOrigin-Name: 0c45c5eb9f0f171b8d7c5f0d2973f9f59915467506cdff1450f3e4b2134a01ca
---
 ext/expert/expert.c        |  8 +++-
 ext/expert/expert1.test    |  4 +-
 ext/expert/sqlite3expert.c | 38 ++++++++--------
 ext/expert/sqlite3expert.h | 88 +++++++++++++++++++++++++++++++++++---
 manifest                   | 18 ++++----
 manifest.uuid              |  2 +-
 6 files changed, 124 insertions(+), 34 deletions(-)

diff --git a/ext/expert/expert.c b/ext/expert/expert.c
index fca2d05194..04be60b9d1 100644
--- a/ext/expert/expert.c
+++ b/ext/expert/expert.c
@@ -119,12 +119,18 @@ int main(int argc, char **argv){
 
   if( rc==SQLITE_OK ){
     int nQuery = sqlite3_expert_count(p);
+    if( iVerbose>0 ){
+      const char *zCand = sqlite3_expert_report(p,0,EXPERT_REPORT_CANDIDATES);
+      fprintf(stdout, "-- Candidates -------------------------------\n");
+      fprintf(stdout, "%s\n", zCand);
+    }
     for(i=0; i<nQuery; i++){
       const char *zSql = sqlite3_expert_report(p, i, EXPERT_REPORT_SQL);
       const char *zIdx = sqlite3_expert_report(p, i, EXPERT_REPORT_INDEXES);
       const char *zEQP = sqlite3_expert_report(p, i, EXPERT_REPORT_PLAN);
+      if( zIdx==0 ) zIdx = "(no new indexes)";
       if( iVerbose>0 ){
-        fprintf(stdout, "-- query %d ----------------------------------\n",i+1);
+        fprintf(stdout, "-- Query %d ----------------------------------\n",i+1);
         fprintf(stdout, "%s\n\n", zSql);
       }
       fprintf(stdout, "%s\n%s\n", zIdx, zEQP);
diff --git a/ext/expert/expert1.test b/ext/expert/expert1.test
index 2e8c1c7e76..11d5bd66d4 100644
--- a/ext/expert/expert1.test
+++ b/ext/expert/expert1.test
@@ -61,7 +61,9 @@ foreach {tn setup} {
 
       set result [list]
       for {set i 0} {$i < [$expert count]} {incr i} {
-        lappend result [string trim [$expert report $i indexes]]
+        set idx [string trim [$expert report $i indexes]]
+        if {$idx==""} {set idx "(no new indexes)"}
+        lappend result $idx
         lappend result [string trim [$expert report $i plan]]
       }
 
diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index faa6966932..1f77a15bc3 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -134,12 +134,13 @@ struct IdxHash64 {
 struct sqlite3expert {
   sqlite3 *db;                    /* User database */
   sqlite3 *dbm;                   /* In-memory db for this analysis */
-  int bRun;                       /* True once analysis has run */
-  char **pzErrmsg;
   IdxScan *pScan;                 /* List of scan objects */
   IdxStatement *pStatement;       /* List of IdxStatement objects */
+  int bRun;                       /* True once analysis has run */
+  char **pzErrmsg;
   int rc;                         /* Error code from whereinfo hook */
   IdxHash hIdx;                   /* Hash containing all candidate indexes */
+  char *zCandidates;              /* For EXPERT_REPORT_CANDIDATES */
 };
 
 
@@ -948,9 +949,6 @@ int idxFindIndexes(
     for(pEntry=hIdx.pFirst; pEntry; pEntry=pEntry->pNext){
       pStmt->zIdx = idxAppendText(&rc, pStmt->zIdx, "%s;\n", pEntry->zKey);
     }
-    if( pStmt->zIdx==0 ){
-      pStmt->zIdx = idxAppendText(&rc, 0, "(no new indexes)\n");
-    }
 
     idxFinalize(&rc, pExplain);
   }
@@ -1049,10 +1047,15 @@ int sqlite3_expert_sql(
 
 int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr){
   int rc;
+  IdxHashEntry *pEntry;
 
   /* Create candidate indexes within the in-memory database file */
   rc = idxCreateCandidates(p, pzErr);
 
+  for(pEntry=p->hIdx.pFirst; pEntry; pEntry=pEntry->pNext){
+    p->zCandidates = idxAppendText(&rc, p->zCandidates, "%s;\n", pEntry->zVal);
+  }
+
   /* Figure out which of the candidate indexes are preferred by the query
   ** planner and report the results to the user.  */
   if( rc==SQLITE_OK ){
@@ -1084,18 +1087,19 @@ const char *sqlite3_expert_report(sqlite3expert *p, int iStmt, int eReport){
 
   if( p->bRun==0 ) return 0;
   for(pStmt=p->pStatement; pStmt && pStmt->iId!=iStmt; pStmt=pStmt->pNext);
-  if( pStmt ){
-    switch( eReport ){
-      case EXPERT_REPORT_SQL:
-        zRet = pStmt->zSql;
-        break;
-      case EXPERT_REPORT_INDEXES:
-        zRet = pStmt->zIdx;
-        break;
-      case EXPERT_REPORT_PLAN:
-        zRet = pStmt->zEQP;
-        break;
-    }
+  switch( eReport ){
+    case EXPERT_REPORT_SQL:
+      if( pStmt ) zRet = pStmt->zSql;
+      break;
+    case EXPERT_REPORT_INDEXES:
+      if( pStmt ) zRet = pStmt->zIdx;
+      break;
+    case EXPERT_REPORT_PLAN:
+      if( pStmt ) zRet = pStmt->zEQP;
+      break;
+    case EXPERT_REPORT_CANDIDATES:
+      zRet = p->zCandidates;
+      break;
   }
   return zRet;
 }
diff --git a/ext/expert/sqlite3expert.h b/ext/expert/sqlite3expert.h
index 6384f8e556..455f41d991 100644
--- a/ext/expert/sqlite3expert.h
+++ b/ext/expert/sqlite3expert.h
@@ -18,27 +18,102 @@ typedef struct sqlite3expert sqlite3expert;
 
 /*
 ** Create a new sqlite3expert object.
+**
+** If successful, a pointer to the new object is returned and (*pzErr) set
+** to NULL. Or, if an error occurs, NULL is returned and (*pzErr) set to
+** an English-language error message. In this case it is the responsibility
+** of the caller to eventually free the error message buffer using
+** sqlite3_free().
 */
 sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErr);
 
 /*
-** Add an SQL statement to the analysis.
+** Specify zero or more SQL statements to be included in the analysis.
+**
+** Buffer zSql must contain zero or more complete SQL statements. This
+** function parses all statements contained in the buffer and adds them
+** to the internal list of statements to analyze. If successful, SQLITE_OK
+** is returned and (*pzErr) set to NULL. Or, if an error occurs - for example
+** due to a error in the SQL - an SQLite error code is returned and (*pzErr)
+** may be set to point to an English language error message. In this case
+** the caller is responsible for eventually freeing the error message buffer
+** using sqlite3_free().
+**
+** If an error does occur while processing one of the statements in the
+** buffer passed as the second argument, none of the statements in the
+** buffer are added to the analysis.
+**
+** This function must be called before sqlite3_expert_analyze(). If a call
+** to this function is made on an sqlite3expert object that has already
+** been passed to sqlite3_expert_analyze() SQLITE_MISUSE is returned
+** immediately and no statements are added to the analysis.
 */
 int sqlite3_expert_sql(
-  sqlite3expert *p,               /* From sqlite3_expert_new() */
-  const char *zSql,               /* SQL statement to add */
+  sqlite3expert *p,               /* From a successful sqlite3_expert_new() */
+  const char *zSql,               /* SQL statement(s) to add */
   char **pzErr                    /* OUT: Error message (if any) */
 );
 
+/*
+** This function is called after the sqlite3expert object has been configured
+** with all SQL statements using sqlite3_expert_sql() to actually perform
+** the analysis. Once this function has been called, it is not possible to
+** add further SQL statements to the analysis.
+**
+** If successful, SQLITE_OK is returned and (*pzErr) is set to NULL. Or, if
+** an error occurs, an SQLite error code is returned and (*pzErr) set to 
+** point to a buffer containing an English language error message. In this
+** case it is the responsibility of the caller to eventually free the buffer
+** using sqlite3_free().
+**
+** If an error does occur within this function, the sqlite3expert object
+** is no longer useful for any purpose. At that point it is no longer
+** possible to add further SQL statements to the object or to re-attempt
+** the analysis. The sqlite3expert object must still be freed using a call
+** sqlite3_expert_destroy().
+*/
 int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr);
 
 /*
-** Return the total number of SQL queries loaded via sqlite3_expert_sql().
+** Return the total number of statements loaded using sqlite3_expert_sql().
+** The total number of SQL statements may be different from the total number
+** to calls to sqlite3_expert_sql().
 */
 int sqlite3_expert_count(sqlite3expert*);
 
 /*
 ** Return a component of the report.
+**
+** This function is called after sqlite3_expert_analyze() to extract the
+** results of the analysis. Each call to this function returns either a
+** NULL pointer or a pointer to a buffer containing a nul-terminated string.
+** The value passed as the third argument must be one of the EXPERT_REPORT_*
+** #define constants defined below.
+**
+** For some EXPERT_REPORT_* parameters, the buffer returned contains 
+** information relating to a specific SQL statement. In these cases that
+** SQL statement is identified by the value passed as the second argument.
+** SQL statements are numbered from 0 in the order in which they are parsed.
+** If an out-of-range value (less than zero or equal to or greater than the
+** value returned by sqlite3_expert_count()) is passed as the second argument
+** along with such an EXPERT_REPORT_* parameter, NULL is always returned.
+**
+** EXPERT_REPORT_SQL:
+**   Return the text of SQL statement iStmt.
+**
+** EXPERT_REPORT_INDEXES:
+**   Return a buffer containing the CREATE INDEX statements for all recommended
+**   indexes for statement iStmt. If there are no new recommeded indexes, NULL 
+**   is returned.
+**
+** EXPERT_REPORT_PLAN:
+**   Return a buffer containing the EXPLAIN QUERY PLAN output for SQL query
+**   iStmt after the proposed indexes have been added to the database schema.
+**
+** EXPERT_REPORT_CANDIDATES:
+**   Return a pointer to a buffer containing the CREATE INDEX statements 
+**   for all indexes that were tested (for all SQL statements). The iStmt
+**   parameter is ignored for EXPERT_REPORT_CANDIDATES calls.
 */
 const char *sqlite3_expert_report(sqlite3expert*, int iStmt, int eReport);
 
@@ -48,9 +123,12 @@ const char *sqlite3_expert_report(sqlite3expert*, int iStmt, int eReport);
 #define EXPERT_REPORT_SQL        1
 #define EXPERT_REPORT_INDEXES    2
 #define EXPERT_REPORT_PLAN       3
+#define EXPERT_REPORT_CANDIDATES 4
 
 /*
-** Free an (sqlite3expert*) handle allocated by sqlite3-expert_new().
+** Free an (sqlite3expert*) handle and all associated resources. There 
+** should be one call to this function for each successful call to 
+** sqlite3-expert_new().
 */
 void sqlite3_expert_destroy(sqlite3expert*);
 
diff --git a/manifest b/manifest
index ab763c7f7c..8f08e742b8 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sext/expert/README.md.
-D 2017-04-10T20:00:26.414
+C Add\sheader\scomments\sto\sthe\sAPI\sfunctions\sin\ssqlite3expert.h.\sInclude\sa\slist\sof\nall\scandidate\sindexes\sin\sthe\sreport\soutput\sby\sthe\ssqlite3_expert\sprogram.
+D 2017-04-11T17:43:12.842
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a
@@ -41,10 +41,10 @@ F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md 9f15075ec5ad772808eff55ef044c31140fd1146aa0a3c47eafd155e71851b01
-F ext/expert/expert.c bf0fd71921cb7b807cda9a76fb380e3d6e6b980d6167093b2952b41ec9ad8f46
-F ext/expert/expert1.test d4451ccdca910ec3652a7a1994210fe2b7bc8f56431ed0cf2ff3dee798c89084
-F ext/expert/sqlite3expert.c 8bcb83b3723239dc6a2a199e7a030741b7ecf47814828a1da7ea559aa42f094a
-F ext/expert/sqlite3expert.h feeaee4ab73ba52426329781bbb28032ce18cf5abd2bf6221bac2df4c32b3013
+F ext/expert/expert.c 5bcc2fba8509fd1ddeb755df9273e9667ce82cda97b0dd6240792d9b9e86a873
+F ext/expert/expert1.test cc33f9390f205bfeb6a30c7618b24a5675f4b9cb844c9154c11398a7f1477e81
+F ext/expert/sqlite3expert.c 2b22a5fbd093a7020ea9625928292265f31b8e78f6feabb987e71a79b2a29089
+F ext/expert/sqlite3expert.h b1c9eedeb647fd734c4206ae6851635284cfbfa5fb688eff74c3265c9f949b4d
 F ext/expert/test_expert.c bad0611732d07180d586bd589cbb7713dc3ab0338c52bff29680eb2007678c05
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
 F ext/fts1/ft_hash.c 3927bd880e65329bdc6f506555b228b28924921b
@@ -1577,7 +1577,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5dd9831721b70a89a26728adcd49c7f6103ef8266891a79c2db34d913702709e
-R 119e273740189bcdef018d8609f3a756
+P 9318f1b9ed2d8da3a82ea69179e2d56a99d326c7721642665f87f6a4534e7bf0
+R 15f16c821c74aae47eac0f21fc6788f2
 U dan
-Z 664c10ad907cc8fb129a4f4feaf498a1
+Z 914bc7396f378320666a7fa77328351c
diff --git a/manifest.uuid b/manifest.uuid
index 935458cfd1..8933394847 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-9318f1b9ed2d8da3a82ea69179e2d56a99d326c7721642665f87f6a4534e7bf0
\ No newline at end of file
+0c45c5eb9f0f171b8d7c5f0d2973f9f59915467506cdff1450f3e4b2134a01ca
\ No newline at end of file

From c2309693b98d2b45a1629b28537507edf35d5e88 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 11 Apr 2017 18:29:14 +0000
Subject: [PATCH 28/68] Fix a formatting issue in the output of the
 sqlite3_expert program.

FossilOrigin-Name: cc8c3581060ffef02290b680183e6f6bc9837ba3550e74c8aaabdc7c45edc223
---
 ext/expert/expert.c |  2 +-
 manifest            | 12 ++++++------
 manifest.uuid       |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/ext/expert/expert.c b/ext/expert/expert.c
index 04be60b9d1..19b52c867d 100644
--- a/ext/expert/expert.c
+++ b/ext/expert/expert.c
@@ -128,7 +128,7 @@ int main(int argc, char **argv){
       const char *zSql = sqlite3_expert_report(p, i, EXPERT_REPORT_SQL);
       const char *zIdx = sqlite3_expert_report(p, i, EXPERT_REPORT_INDEXES);
       const char *zEQP = sqlite3_expert_report(p, i, EXPERT_REPORT_PLAN);
-      if( zIdx==0 ) zIdx = "(no new indexes)";
+      if( zIdx==0 ) zIdx = "(no new indexes)\n";
       if( iVerbose>0 ){
         fprintf(stdout, "-- Query %d ----------------------------------\n",i+1);
         fprintf(stdout, "%s\n\n", zSql);
diff --git a/manifest b/manifest
index 8f08e742b8..dd5849b713 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sheader\scomments\sto\sthe\sAPI\sfunctions\sin\ssqlite3expert.h.\sInclude\sa\slist\sof\nall\scandidate\sindexes\sin\sthe\sreport\soutput\sby\sthe\ssqlite3_expert\sprogram.
-D 2017-04-11T17:43:12.842
+C Fix\sa\sformatting\sissue\sin\sthe\soutput\sof\sthe\ssqlite3_expert\sprogram.
+D 2017-04-11T18:29:14.206
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a
@@ -41,7 +41,7 @@ F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md 9f15075ec5ad772808eff55ef044c31140fd1146aa0a3c47eafd155e71851b01
-F ext/expert/expert.c 5bcc2fba8509fd1ddeb755df9273e9667ce82cda97b0dd6240792d9b9e86a873
+F ext/expert/expert.c 6349cf8d26c847f5f0fa7e25772b614c67f60f3c850dca0d75d55eb27cf3f69b
 F ext/expert/expert1.test cc33f9390f205bfeb6a30c7618b24a5675f4b9cb844c9154c11398a7f1477e81
 F ext/expert/sqlite3expert.c 2b22a5fbd093a7020ea9625928292265f31b8e78f6feabb987e71a79b2a29089
 F ext/expert/sqlite3expert.h b1c9eedeb647fd734c4206ae6851635284cfbfa5fb688eff74c3265c9f949b4d
@@ -1577,7 +1577,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 9318f1b9ed2d8da3a82ea69179e2d56a99d326c7721642665f87f6a4534e7bf0
-R 15f16c821c74aae47eac0f21fc6788f2
+P 0c45c5eb9f0f171b8d7c5f0d2973f9f59915467506cdff1450f3e4b2134a01ca
+R 33f15deae23049b95df8ad5185895aff
 U dan
-Z 914bc7396f378320666a7fa77328351c
+Z 2806083003fa51636f951574e2a01b7d
diff --git a/manifest.uuid b/manifest.uuid
index 8933394847..70191b5b97 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0c45c5eb9f0f171b8d7c5f0d2973f9f59915467506cdff1450f3e4b2134a01ca
\ No newline at end of file
+cc8c3581060ffef02290b680183e6f6bc9837ba3550e74c8aaabdc7c45edc223
\ No newline at end of file

From 0824ccf29bb9f5c72261d69a7b43740ae6a27735 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 14 Apr 2017 19:41:37 +0000
Subject: [PATCH 29/68] Modify the code in ext/expert/ to use the vtab
 interface instead of sqlite3_whereinfo_hook(). Remove
 sqlite3_whereinfo_hook().

FossilOrigin-Name: 3bb6585004090dbf92dd5e9abdf0fd2c921e64b5b3121c4fb7446db764ab59e5
---
 ext/expert/expert1.test    |  25 ++-
 ext/expert/sqlite3expert.c | 352 ++++++++++++++++++++++++++++---------
 manifest                   |  22 +--
 manifest.uuid              |   2 +-
 src/main.c                 |  20 ---
 src/sqlite.h.in            | 144 +--------------
 src/sqliteInt.h            |   5 +-
 src/where.c                | 243 +++++--------------------
 8 files changed, 344 insertions(+), 469 deletions(-)

diff --git a/ext/expert/expert1.test b/ext/expert/expert1.test
index 11d5bd66d4..437ba45600 100644
--- a/ext/expert/expert1.test
+++ b/ext/expert/expert1.test
@@ -139,14 +139,14 @@ do_setup_rec_test $tn.1.7 {
   0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_033e95fe
 }
 
-do_setup_rec_test $tn.1.8 {
-  CREATE TABLE t1(a, b, c);
-} {
-  SELECT * FROM t1 ORDER BY a ASC, b COLLATE nocase DESC, c ASC;
-} {
-  CREATE INDEX t1_idx_5be6e222 ON t1(a, b COLLATE NOCASE DESC, c);
-  0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_5be6e222
-}
+#do_setup_rec_test $tn.1.8 {
+#  CREATE TABLE t1(a, b, c);
+#} {
+#  SELECT * FROM t1 ORDER BY a ASC, b COLLATE nocase DESC, c ASC;
+#} {
+#  CREATE INDEX t1_idx_5be6e222 ON t1(a, b COLLATE NOCASE DESC, c);
+#  0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_5be6e222
+#}
 
 do_setup_rec_test $tn.1.9 {
   CREATE TABLE t1(a COLLATE NOCase, b, c);
@@ -157,6 +157,15 @@ do_setup_rec_test $tn.1.9 {
   0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_00000061 (a=?)
 }
 
+do_setup_rec_test $tn.1.10 {
+  CREATE TABLE t1(a, b COLLATE nocase, c);
+} {
+  SELECT * FROM t1 ORDER BY a ASC, b DESC, c ASC;
+} {
+  CREATE INDEX t1_idx_5cb97285 ON t1(a, b DESC, c);
+  0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_5cb97285
+}
+
 
 # Tables with names that require quotes.
 #
diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index 1f77a15bc3..3cd110c367 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -50,7 +50,7 @@ struct IdxConstraint {
 ** A single scan of a single table.
 */
 struct IdxScan {
-  char *zTable;                   /* Name of table to scan */
+  IdxTable *pTab;                 /* Associated table object */
   int iDb;                        /* Database containing table zTable */
   i64 covering;                   /* Mask of columns required for cov. index */
   IdxConstraint *pOrder;          /* ORDER BY columns */
@@ -70,7 +70,9 @@ struct IdxColumn {
 };
 struct IdxTable {
   int nCol;
+  char *zName;                    /* Table name */
   IdxColumn *aCol;
+  IdxTable *pNext;                /* Next table in linked list of all tables */
 };
 
 /*
@@ -134,6 +136,9 @@ struct IdxHash64 {
 struct sqlite3expert {
   sqlite3 *db;                    /* User database */
   sqlite3 *dbm;                   /* In-memory db for this analysis */
+  sqlite3 *dbv;                   /* Vtab schema for this analysis */
+  IdxTable *pTable;               /* List of all IdxTable objects */
+
   IdxScan *pScan;                 /* List of scan objects */
   IdxStatement *pStatement;       /* List of IdxStatement objects */
   int bRun;                       /* True once analysis has run */
@@ -314,79 +319,189 @@ static IdxConstraint *idxNewConstraint(int *pRc, const char *zColl){
   return pNew;
 }
 
-/*
-** sqlite3_whereinfo_hook() callback.
+
+/*************************************************************************
+** Beginning of virtual table implementation.
 */
-static void idxWhereInfo(
-  void *pCtx,                     /* Pointer to sqlite3expert structure */
-  int eOp, 
-  const char *zVal, 
-  int iVal, 
-  u64 mask
+typedef struct ExpertVtab ExpertVtab;
+struct ExpertVtab {
+  sqlite3_vtab base;
+  IdxTable *pTab;
+  sqlite3expert *pExpert;
+};
+
+static char *expertDequote(const char *zIn){
+  int n = strlen(zIn);
+  char *zRet = sqlite3_malloc(n);
+
+  assert( zIn[0]=='\'' );
+  assert( zIn[n-1]=='\'' );
+
+  if( zRet ){
+    int iOut = 0;
+    int iIn = 0;
+    for(iIn=1; iIn<(n-1); iIn++){
+      if( zIn[iIn]=='\'' ){
+        assert( zIn[iIn+1]=='\'' );
+        iIn++;
+      }
+      zRet[iOut++] = zIn[iIn];
+    }
+    zRet[iOut] = '\0';
+  }
+
+  return zRet;
+}
+
+/* 
+** This function is the implementation of both the xConnect and xCreate
+** methods of the r-tree virtual table.
+**
+**   argv[0]   -> module name
+**   argv[1]   -> database name
+**   argv[2]   -> table name
+**   argv[...] -> column names...
+*/
+static int expertConnect(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
 ){
-  sqlite3expert *p = (sqlite3expert*)pCtx;
+  sqlite3expert *pExpert = (sqlite3expert*)pAux;
+  ExpertVtab *p = 0;
+  int rc;
 
-#if 0
-  const char *zOp = 
-    eOp==SQLITE_WHEREINFO_TABLE ? "TABLE" :
-    eOp==SQLITE_WHEREINFO_EQUALS ? "EQUALS" :
-    eOp==SQLITE_WHEREINFO_RANGE ? "RANGE" :
-    eOp==SQLITE_WHEREINFO_ORDERBY ? "ORDERBY" :
-    "!error!";
-  printf("op=%s zVal=%s iVal=%d mask=%llx\n", zOp, zVal, iVal, mask);
-#endif
-
-  if( p->rc==SQLITE_OK ){
-    assert( eOp==SQLITE_WHEREINFO_TABLE || p->pScan!=0 );
-    switch( eOp ){
-      case SQLITE_WHEREINFO_TABLE: {
-        int nVal = strlen(zVal);
-        IdxScan *pNew = (IdxScan*)idxMalloc(&p->rc, sizeof(IdxScan) + nVal + 1);
-        if( !pNew ) return;
-        pNew->zTable = (char*)&pNew[1];
-        memcpy(pNew->zTable, zVal, nVal+1);
-        pNew->pNextScan = p->pScan;
-        pNew->covering = mask;
-        p->pScan = pNew;
-        break;
+  if( argc!=4 ){
+    *pzErr = sqlite3_mprintf("internal error!");
+    rc = SQLITE_ERROR;
+  }else{
+    char *zCreateTable = expertDequote(argv[3]);
+    if( zCreateTable ){
+      rc = sqlite3_declare_vtab(db, zCreateTable);
+      if( rc==SQLITE_OK ){
+        p = idxMalloc(&rc, sizeof(ExpertVtab));
       }
-
-      case SQLITE_WHEREINFO_ORDERBY: {
-        IdxConstraint *pNew = idxNewConstraint(&p->rc, zVal);
-        if( pNew==0 ) return;
-        pNew->iCol = iVal;
-        pNew->bDesc = (int)mask;
-        if( p->pScan->pOrder==0 ){
-          p->pScan->pOrder = pNew;
-        }else{
-          IdxConstraint *pIter;
-          for(pIter=p->pScan->pOrder; pIter->pNext; pIter=pIter->pNext);
-          pIter->pNext = pNew;
-          pIter->pLink = pNew;
-        }
-        break;
+      if( rc==SQLITE_OK ){
+        p->pExpert = pExpert;
+        p->pTab = pExpert->pTable;
+        assert( sqlite3_stricmp(p->pTab->zName, argv[2])==0 );
       }
+      sqlite3_free(zCreateTable);
+    }else{
+      rc = SQLITE_NOMEM;
+    }
+  }
 
-      case SQLITE_WHEREINFO_EQUALS:
-      case SQLITE_WHEREINFO_RANGE: {
-        IdxConstraint *pNew = idxNewConstraint(&p->rc, zVal);
-        if( pNew==0 ) return;
-        pNew->iCol = iVal;
-        pNew->depmask = mask;
+  *ppVtab = (sqlite3_vtab*)p;
+  return rc;
+}
 
-        if( eOp==SQLITE_WHEREINFO_RANGE ){
-          pNew->pNext = p->pScan->pRange;
-          p->pScan->pRange = pNew;
-        }else{
-          pNew->pNext = p->pScan->pEq;
-          p->pScan->pEq = pNew;
+static int expertDisconnect(sqlite3_vtab *pVtab){
+  ExpertVtab *p = (ExpertVtab*)pVtab;
+  sqlite3_free(p);
+  return SQLITE_OK;
+}
+
+static int expertBestIndex(sqlite3_vtab *pVtab, sqlite3_index_info *pIdxInfo){
+  ExpertVtab *p = (ExpertVtab*)pVtab;
+  sqlite3 *dbv = p->pExpert->dbv;
+  int rc = SQLITE_OK;
+  int n = 0;
+  IdxScan *pScan;
+  const int opmask = 
+    SQLITE_INDEX_CONSTRAINT_EQ | SQLITE_INDEX_CONSTRAINT_GT |
+    SQLITE_INDEX_CONSTRAINT_LT | SQLITE_INDEX_CONSTRAINT_GE |
+    SQLITE_INDEX_CONSTRAINT_LE;
+
+  pScan = idxMalloc(&rc, sizeof(IdxScan));
+  if( pScan ){
+    int i;
+
+    /* Link the new scan object into the list */
+    pScan->pTab = p->pTab;
+    pScan->pNextScan = p->pExpert->pScan;
+    p->pExpert->pScan = pScan;
+
+    /* Add the constraints to the IdxScan object */
+    for(i=0; i<pIdxInfo->nConstraint; i++){
+      int op = pIdxInfo->aConstraint[i].op;
+      if( op&opmask ){
+        IdxConstraint *pNew;
+        const char *zColl = sqlite3_vtab_collation(dbv, i);
+        pNew = idxNewConstraint(&rc, zColl);
+        if( pNew ){
+          pNew->iCol = pIdxInfo->aConstraint[i].iColumn;
+          if( op==SQLITE_INDEX_CONSTRAINT_EQ ){
+            pNew->pNext = pScan->pEq;
+            pScan->pEq = pNew;
+          }else{
+            pNew->bRange = 1;
+            pNew->pNext = pScan->pRange;
+            pScan->pRange = pNew;
+          }
         }
-        break;
+        if( pIdxInfo->aConstraint[i].usable ){
+          n++;
+          pIdxInfo->aConstraintUsage[i].argvIndex = n;
+        }
+      }
+    }
+
+    /* Add the ORDER BY to the IdxScan object */
+    for(i=pIdxInfo->nOrderBy-1; i>=0; i--){
+      IdxConstraint *pNew;
+      const char *zColl = sqlite3_vtab_collation(dbv, i+pIdxInfo->nConstraint);
+      pNew = idxNewConstraint(&rc, zColl);
+      if( pNew ){
+        pNew->iCol = pIdxInfo->aOrderBy[i].iColumn;
+        pNew->bDesc = pIdxInfo->aOrderBy[i].desc;
+        pNew->pNext = pScan->pOrder;
+        pNew->pLink = pScan->pOrder;
+        pScan->pOrder = pNew;
+        n++;
       }
     }
   }
+
+  pIdxInfo->estimatedCost = 1000000.0 / n;
+  return rc;
 }
 
+static int idxRegisterVtab(sqlite3expert *p){
+  static sqlite3_module expertModule = {
+    2,                            /* iVersion */
+    expertConnect,                /* xCreate - create a table */
+    expertConnect,                /* xConnect - connect to an existing table */
+    expertBestIndex,              /* xBestIndex - Determine search strategy */
+    expertDisconnect,             /* xDisconnect - Disconnect from a table */
+    expertDisconnect,             /* xDestroy - Drop a table */
+    0,                            /* xOpen - open a cursor */
+    0,                            /* xClose - close a cursor */
+    0,                            /* xFilter - configure scan constraints */
+    0,                            /* xNext - advance a cursor */
+    0,                            /* xEof */
+    0,                            /* xColumn - read data */
+    0,                            /* xRowid - read data */
+    0,                            /* xUpdate - write data */
+    0,                            /* xBegin - begin transaction */
+    0,                            /* xSync - sync transaction */
+    0,                            /* xCommit - commit transaction */
+    0,                            /* xRollback - rollback transaction */
+    0,                            /* xFindFunction - function overloading */
+    0,                            /* xRename - rename the table */
+    0,                            /* xSavepoint */
+    0,                            /* xRelease */
+    0,                            /* xRollbackTo */
+  };
+
+  return sqlite3_create_module(p->dbv, "expert", &expertModule, (void*)p);
+}
+/*
+** End of virtual table implementation.
+*************************************************************************/
+
 /*
 ** An error associated with database handle db has just occurred. Pass
 ** the error message to callback function xOut.
@@ -468,7 +583,8 @@ static int idxGetTableInfo(
 ){
   sqlite3_stmt *p1 = 0;
   int nCol = 0;
-  int nByte = sizeof(IdxTable);
+  int nTab = strlen(zTab);
+  int nByte = sizeof(IdxTable) + nTab + 1;
   IdxTable *pNew = 0;
   int rc, rc2;
   char *pCsr;
@@ -522,6 +638,9 @@ static int idxGetTableInfo(
   if( rc!=SQLITE_OK ){
     sqlite3_free(pNew);
     pNew = 0;
+  }else{
+    pNew->zName = pCsr;
+    memcpy(pNew->zName, zTab, nTab+1);
   }
 
   *ppOut = pNew;
@@ -632,7 +751,7 @@ static int idxFindCompatible(
   IdxConstraint *pEq,             /* List of == constraints */
   IdxConstraint *pTail            /* List of range constraints */
 ){
-  const char *zTbl = pScan->zTable;
+  const char *zTbl = pScan->pTab->zName;
   sqlite3_stmt *pIdxList = 0;
   IdxConstraint *pIter;
   int nEq = 0;                    /* Number of elements in pEq */
@@ -717,21 +836,22 @@ static int idxCreateFromCons(
 
     if( rc==SQLITE_OK ){
       /* Hash the list of columns to come up with a name for the index */
+      const char *zTable = pScan->pTab->zName;
       char *zName;                /* Index name */
       int i;
       for(i=0; zCols[i]; i++){
         h += ((h<<3) + zCols[i]);
       }
-      zName = sqlite3_mprintf("%s_idx_%08x", pScan->zTable, h);
+      zName = sqlite3_mprintf("%s_idx_%08x", zTable, h);
       if( zName==0 ){ 
         rc = SQLITE_NOMEM;
       }else{
-        if( idxIdentifierRequiresQuotes(pScan->zTable) ){
+        if( idxIdentifierRequiresQuotes(zTable) ){
           zFmt = "CREATE INDEX '%q' ON %Q(%s)";
         }else{
           zFmt = "CREATE INDEX %s ON %s(%s)";
         }
-        zIdx = sqlite3_mprintf(zFmt, zName, pScan->zTable, zCols);
+        zIdx = sqlite3_mprintf(zFmt, zName, zTable, zCols);
         if( !zIdx ){
           rc = SQLITE_NOMEM;
         }else{
@@ -817,9 +937,7 @@ static int idxCreateCandidates(sqlite3expert *p, char **pzErr){
   for(pIter=p->pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
     IdxHash64Entry *pEntry;
     IdxConstraint *pCons;
-    IdxTable *pTab = 0;
-
-    rc = idxGetTableInfo(p->dbm, pIter->zTable, &pTab, pzErr);
+    IdxTable *pTab = pIter->pTab;
 
     idxHash64Add(&rc, &hMask, 0);
     for(pCons=pIter->pEq; pCons; pCons=pCons->pNext){
@@ -836,7 +954,6 @@ static int idxCreateCandidates(sqlite3expert *p, char **pzErr){
       }
     }
 
-    sqlite3_free(pTab);
     idxHash64Clear(&hMask);
   }
 
@@ -958,6 +1075,60 @@ int idxFindIndexes(
   return rc;
 }
 
+static int idxCreateVtabSchema(sqlite3expert *p, char **pzErrmsg){
+  int rc = idxRegisterVtab(p);
+  sqlite3_stmt *pSchema = 0;
+
+  /* For each table in the main db schema:
+  **
+  **   1) Add an entry to the p->pTable list, and
+  **   2) Create the equivalent virtual table in dbv.
+  */
+  rc = idxPrepareStmt(p->db, &pSchema, pzErrmsg,
+      "SELECT type, name, sql FROM sqlite_master "
+      "WHERE type IN ('table','view') ORDER BY 1"
+  );
+  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSchema) ){
+    const char *zType = (const char*)sqlite3_column_text(pSchema, 0);
+    const char *zName = (const char*)sqlite3_column_text(pSchema, 1);
+    const char *zSql = (const char*)sqlite3_column_text(pSchema, 2);
+
+    if( zType[0]=='v' ){
+      rc = sqlite3_exec(p->dbv, zSql, 0, 0, pzErrmsg);
+    }else{
+      IdxTable *pTab;
+      rc = idxGetTableInfo(p->db, zName, &pTab, pzErrmsg);
+      if( rc==SQLITE_OK ){
+        int i;
+        char *zInner = 0;
+        char *zOuter = 0;
+        pTab->pNext = p->pTable;
+        p->pTable = pTab;
+
+        /* The statement the vtab will pass to sqlite3_declare_vtab() */
+        zInner = idxAppendText(&rc, 0, "CREATE TABLE x(");
+        for(i=0; i<pTab->nCol; i++){
+          zInner = idxAppendText(&rc, zInner, "%s%Q COLLATE %s", 
+              (i==0 ? "" : ", "), pTab->aCol[i].zName, pTab->aCol[i].zColl
+          );
+        }
+        zInner = idxAppendText(&rc, zInner, ")");
+
+        /* The CVT statement to create the vtab */
+        zOuter = idxAppendText(&rc, 0, 
+            "CREATE VIRTUAL TABLE %Q USING expert(%Q)", zName, zInner
+        );
+        if( rc==SQLITE_OK ){
+          rc = sqlite3_exec(p->dbv, zOuter, 0, 0, pzErrmsg);
+        }
+        sqlite3_free(zInner);
+        sqlite3_free(zOuter);
+      }
+    }
+  }
+  return rc;
+}
+
 /*
 ** Allocate a new sqlite3expert object.
 */
@@ -966,12 +1137,20 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){
   sqlite3expert *pNew;
 
   pNew = (sqlite3expert*)idxMalloc(&rc, sizeof(sqlite3expert));
-  pNew->db = db;
 
-  /* Open an in-memory database to work with. The main in-memory 
-  ** database schema contains tables similar to those in the users 
-  ** database (handle db).  */
-  rc = sqlite3_open(":memory:", &pNew->dbm);
+  /* Open two in-memory databases to work with. The "vtab database" (dbv)
+  ** will contain a virtual table corresponding to each real table in
+  ** the user database schema, and a copy of each view. It is used to
+  ** collect information regarding the WHERE, ORDER BY and other clauses
+  ** of the user's query.
+  */
+  if( rc==SQLITE_OK ){
+    pNew->db = db;
+    rc = sqlite3_open(":memory:", &pNew->dbv);
+  }
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_open(":memory:", &pNew->dbm);
+  }
 
   /* Copy the entire schema of database [db] into [dbm]. */
   if( rc==SQLITE_OK ){
@@ -986,6 +1165,11 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){
     idxFinalize(&rc, pSql);
   }
 
+  /* Create the vtab schema */
+  if( rc==SQLITE_OK ){
+    rc = idxCreateVtabSchema(pNew, pzErrmsg);
+  }
+
   /* If an error has occurred, free the new object and reutrn NULL. Otherwise,
   ** return the new sqlite3expert handle.  */
   if( rc!=SQLITE_OK ){
@@ -1010,10 +1194,9 @@ int sqlite3_expert_sql(
 
   if( p->bRun ) return SQLITE_MISUSE;
 
-  sqlite3_whereinfo_hook(p->db, idxWhereInfo, p);
   while( rc==SQLITE_OK && zStmt && zStmt[0] ){
     sqlite3_stmt *pStmt = 0;
-    rc = sqlite3_prepare_v2(p->db, zStmt, -1, &pStmt, &zStmt);
+    rc = sqlite3_prepare_v2(p->dbv, zStmt, -1, &pStmt, &zStmt);
     if( rc==SQLITE_OK ){
       if( pStmt ){
         IdxStatement *pNew;
@@ -1033,7 +1216,6 @@ int sqlite3_expert_sql(
       idxDatabaseError(p->db, pzErr);
     }
   }
-  sqlite3_whereinfo_hook(p->db, 0, 0);
 
   if( rc!=SQLITE_OK ){
     idxScanFree(p->pScan, pScanOrig);
@@ -1052,6 +1234,7 @@ int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr){
   /* Create candidate indexes within the in-memory database file */
   rc = idxCreateCandidates(p, pzErr);
 
+  /* Formulate the EXPERT_REPORT_CANDIDATES text */
   for(pEntry=p->hIdx.pFirst; pEntry; pEntry=pEntry->pNext){
     p->zCandidates = idxAppendText(&rc, p->zCandidates, "%s;\n", pEntry->zVal);
   }
@@ -1108,11 +1291,14 @@ const char *sqlite3_expert_report(sqlite3expert *p, int iStmt, int eReport){
 ** Free an sqlite3expert object.
 */
 void sqlite3_expert_destroy(sqlite3expert *p){
-  sqlite3_close(p->dbm);
-  idxScanFree(p->pScan, 0);
-  idxStatementFree(p->pStatement, 0);
-  idxHashClear(&p->hIdx);
-  sqlite3_free(p);
+  if( p ){
+    sqlite3_close(p->dbm);
+    sqlite3_close(p->dbv);
+    idxScanFree(p->pScan, 0);
+    idxStatementFree(p->pStatement, 0);
+    idxHashClear(&p->hIdx);
+    sqlite3_free(p);
+  }
 }
 
 #endif /* !defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHEREINFO_HOOK) */
diff --git a/manifest b/manifest
index 44bc9ecdb4..89f2ca08db 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\sthis\sbranch\swith\slatest\schanges\sfrom\strunk.
-D 2017-04-13T16:19:40.114
+C Modify\sthe\scode\sin\sext/expert/\sto\suse\sthe\svtab\sinterface\sinstead\sof\nsqlite3_whereinfo_hook().\sRemove\ssqlite3_whereinfo_hook().
+D 2017-04-14T19:41:37.126
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a
@@ -42,8 +42,8 @@ F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md 9f15075ec5ad772808eff55ef044c31140fd1146aa0a3c47eafd155e71851b01
 F ext/expert/expert.c 6349cf8d26c847f5f0fa7e25772b614c67f60f3c850dca0d75d55eb27cf3f69b
-F ext/expert/expert1.test cc33f9390f205bfeb6a30c7618b24a5675f4b9cb844c9154c11398a7f1477e81
-F ext/expert/sqlite3expert.c 2b22a5fbd093a7020ea9625928292265f31b8e78f6feabb987e71a79b2a29089
+F ext/expert/expert1.test cd630eda18a2508eade4c39a1eafe32e7437a33973391e5dddfc7fd0f3163684
+F ext/expert/sqlite3expert.c 9473b011d7e0be5b52157f3b1fc153d7e5f7d2b43af110180843e7a03972439f
 F ext/expert/sqlite3expert.h b1c9eedeb647fd734c4206ae6851635284cfbfa5fb688eff74c3265c9f949b4d
 F ext/expert/test_expert.c bad0611732d07180d586bd589cbb7713dc3ab0338c52bff29680eb2007678c05
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
@@ -372,7 +372,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c d4bb3a135948553d18cf992f76f7ed7b18aa0327f250607b5a6671e55d9947d5
 F src/legacy.c e88ed13c2d531decde75d42c2e35623fb9ce3cb0
 F src/loadext.c a72909474dadce771d3669bf84bf689424f6f87d471fee898589c3ef9b2acfd9
-F src/main.c 4b93bda0f1f916f4cb618a6fce358f7e01ab060971c586c4cea90ad0c83a83f5
+F src/main.c 158326243c5ddc8b98a1e983fa488650cf76d760
 F src/malloc.c e20bb2b48abec52d3faf01cce12e8b4f95973755fafec98d45162dfdab111978
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -409,10 +409,10 @@ F src/resolve.c 3e518b962d932a997fae373366880fc028c75706
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c e6f9afd8a5ef35bd186e51a6bea6d3d46bc93a530f26a21fe8a0a43dbeca9415
 F src/shell.c 70f4957b988572315e97c56941fdc81fd35907fee36b7b2e7be5ec4c7e9d065d
-F src/sqlite.h.in cf20591fa0eb09e435db647ab28b61159262cbebac69ddad3c8c01accfb6c856
+F src/sqlite.h.in fa0a24af458b49de8d5654fd1f3b61422ef513cee9b17f39ce3f5e5731ce4f6a
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 58fd0676d3111d02e62e5a35992a7d3da5d3f88753acc174f2d37b774fbbdd28
-F src/sqliteInt.h 5bcafb7c36f7f8765ed1e7031b7eb5f5e84cfdfe5ea4b3af01178528bde259c8
+F src/sqliteInt.h 8ac5d9b92084de271b913387968720f4dfe17302eb48a51c6ae161b6149057ad
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -488,7 +488,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 40c543f0a2195d1b0dc88ef12142bea690009344
 F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71
 F src/walker.c b71a992b413b3a022572eccf29ef4b4890223791
-F src/where.c f5acfb6fbac65e7da7b0e718fa6c6e784dee37eb29dad6efd42880ca117e7277
+F src/where.c 64f2c18c72e06bc935c64a53fd5cea6446ffedeba87c45905408bea124c201b6
 F src/whereInt.h 7a21ef633e26acbf46df04add2eba6e0a2100c78dc5879049e93f981fc3344df
 F src/wherecode.c 943e32e9dccd0af802e0683ae11071c8bd808364e5908a5fb66758bd404c8681
 F src/whereexpr.c e913aaa7b73ffcce66abcea5f197e2c538d48b5df78d0b7bba8ff4d73cc2e745
@@ -1578,7 +1578,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0f66a093935100efd731e14aa63b57360ddd517c1ac97edd1ea9a9de95e1f3cc 59c70108fd179932ccdd860f93e1cd68b77476d3b1af1af00cf6e378c9984862
-R 67cf9bfa549dda32e4c23afbb89f9c17
+P 5fcd840cf9b6a5c3ee4ef1e8f92f6c30f96a7899a3d774ee9be8a816916f2c3b
+R 754b00e271f01cdc7a8ee810736e1b12
 U dan
-Z 6319da725d84003b27573110707fc385
+Z bc14af8147812d1c0312a546deaaf0cc
diff --git a/manifest.uuid b/manifest.uuid
index 123a103b07..7b25e887aa 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5fcd840cf9b6a5c3ee4ef1e8f92f6c30f96a7899a3d774ee9be8a816916f2c3b
\ No newline at end of file
+3bb6585004090dbf92dd5e9abdf0fd2c921e64b5b3121c4fb7446db764ab59e5
\ No newline at end of file
diff --git a/src/main.c b/src/main.c
index df715852ae..4ac5327e4f 100644
--- a/src/main.c
+++ b/src/main.c
@@ -1997,26 +1997,6 @@ void *sqlite3_preupdate_hook(
 }
 #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */
 
-#ifdef SQLITE_ENABLE_WHEREINFO_HOOK
-/*
-** Register a where-info hook.
-*/
-void *sqlite3_whereinfo_hook(
-  sqlite3 *db,                    /* Register callback with this db handle */
-  void (*xWhereInfo)(void*, int, const char*, int, sqlite3_uint64),
-  void *pCtx                      /* User pointer passed to callback */
-){
-  void *pRet;
-  sqlite3_mutex_enter(db->mutex);
-  pRet = db->pWhereInfoCtx;
-  db->xWhereInfo = xWhereInfo;
-  db->pWhereInfoCtx = pCtx;
-  sqlite3_mutex_leave(db->mutex);
-  return pRet;
-}
-#endif /* SQLITE_ENABLE_WHEREINFO_HOOK */
-
-
 #ifndef SQLITE_OMIT_WAL
 /*
 ** The sqlite3_wal_hook() callback registered by sqlite3_wal_autocheckpoint().
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 923e7974e5..3c54efd36a 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -2017,6 +2017,7 @@ struct sqlite3_mem_methods {
 #define SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION 1005 /* int int* */
 #define SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE      1006 /* int int* */
 
+
 /*
 ** CAPI3REF: Enable Or Disable Extended Result Codes
 ** METHOD: sqlite3
@@ -8036,6 +8037,8 @@ int sqlite3_vtab_config(sqlite3*, int op, ...);
 */
 int sqlite3_vtab_on_conflict(sqlite3 *);
 
+SQLITE_EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3*, int);
+
 /*
 ** CAPI3REF: Conflict resolution modes
 ** KEYWORDS: {conflict resolution mode}
@@ -8482,147 +8485,6 @@ SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp(
 */
 SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb);
 
-/*
-** This function is used to register a whereinfo hook with the database
-** handle passed as the first argument. Once registered, the whereinfo hook
-** is invoked zero or more times while preparing a query to provide
-** information to the application. It is intended to be used by expert 
-** systems to recommend indexes that could be added to the database in order
-** to improve query response time.
-**
-** An SQLite query plan describes the way data is read from zero or more
-** database tables. For each table read, the data required may be
-** constrained by equality or range constraints on one or more columns, 
-** and it may or may not be required in order sorted by one or more 
-** columns. For example, the following query:
-**
-** <pre>
-**     SELECT * FROM t1 WHERE t1.a = ? ORDER BY t1.b;
-** </pre>
-**
-** reads data from table t1. It requires only those rows for which t1.a
-** is set to a specific value, and requires them sorted in order of
-** column t1.b. Internally, SQLite determines this and attempts to locate
-** an index that can be used to efficiently find the required subset of
-** rows and/or allows the rows to be read from the database in the 
-** required order. In this case, ideally an index of the form:
-**
-** <pre>
-**     CREATE INDEX i1 ON t1(a, b);
-** </pre>
-**
-** The data passed to the whereinfo hook during query preparation do
-** not describe the actual query plan to the application. Instead, it
-** describes the parts of the query that SQLite could use an index to
-** optimize if a suitable index existed. In this case, that only
-** rows with t1.a=? are required, and that they are required sorted
-** in order by t1.b.
-**
-** Each time the whereinfo hook is invoked, the first argument is a
-** copy of the (void*) pointer passed as the second argument to this
-** API function. The second is always one of the SQLITE_WHEREINFO_XXX
-** constants defined below.
-**
-** For each table read by a query, the whereinfo hook is invoked as follows:
-**
-** <ul>
-**   <li> Once with SQLITE_WHEREINFO_TABLE as the second argument. This
-**        indicates the table that subsequent callbacks (until the next
-**        SQLITE_WHEREINFO_TABLE) apply to.
-**
-**   <li> If SQLite requires rows in a specific order, once with
-**        SQLITE_WHEREINFO_ORDERBY for each column of the table that is
-**        one of the sort keys.
-**
-**   <li> If there are any "<col> = ?" constraints that restrict the rows
-**        required by SQLite, once with SQLITE_WHEREINFO_EQUALS for each
-**        such constraint.
-**
-**   <li> If there are any "<col> > ?" constraints (or any other range
-**        constraint) that restrict the rows required by SQLite, once with 
-**        SQLITE_WHEREINFO_RANGE for each such constraint.
-** </ul>
-**
-** The third, fourth and fifth arguments passed to the whereinfo callback
-** are interpreted differently, depending on the SQLITE_WHEREINFO_XXX value
-** as follows:
-**
-** <dl>
-**   <dt> SQLITE_WHEREINFO_TABLE
-**   <dd> The third argument passed in this case is the name of the table.
-**        The fourth is the index of the database in which the table is
-**        located (0 for "main", 1 for "temp" or higher for an attached
-**        database). The fifth argument is a bitmask that indicates which
-**        of the tables columns may be required by the query. If the leftmost
-**        column of the table is used in some way, bit 0 of the bitmask is
-**        set. If the next-to-leftmost is used, bit 1 etc. Bit 63 is used to
-**        represent all columns with an index of 63 or higher. If bit 63
-**        is set, the application should assume that the query requires all
-**        columns from the table with an index of 63 or greater.
-**
-**   <dt> SQLITE_WHEREINFO_ORDERBY
-**   <dd> The third argument passed in this case is the name of the collation
-**        sequence to sort by. The fourth is the index of the table column to
-**        sort by (0 for the leftmost column, 1 for the next-to-leftmost
-**        etc.). The fifth argument is a boolean flag - true for a DESC sort
-**        or false for ASC.
-**
-**   <dt> SQLITE_WHEREINFO_EQUALS
-**   <dd> The third argument passed in this case is the name of the collation
-**        sequence used by the constraint. The fourth is the index of the 
-**        table column in the constraint. If the current table is not part
-**        of a join, then the value passed as the fifth argument is always
-**        zero. Or, if it is part of a join, then the fifth parameter passed
-**        to this callback is a mask of other tables that the current
-**        constraint depends on. For example, in the query:
-**
-**   <pre>
-**      SELECT * FROM t1, t2 WHERE t1.a = (t2.b+1);
-**   </pre>
-**
-**        the fifth parameter passed to the the SQLITE_WHEREINFO_EQUALS
-**        whereinfo callback would have the bit assigned to table "t2"
-**        set to true. There is no way for the application to determine 
-**        the specific bit in the mask assigned to any table, but the bit
-**        assignments are consistent while parsing a single query.
-**
-**   <dt> SQLITE_WHEREINFO_RANGE
-**   <dd> As for SQLITE_WHEREINFO_EQUALS.
-** </dl>
-**
-** Note that if a WHERE clause includes an OR expression, then there may be
-** more than one set of callbacks for a single table. For example, the
-** following SQL:
-**
-** <pre>
-**    SELECT * FROM t1 WHERE t1.a=? OR t1.b=?
-** </pre>
-**
-** Provokes the same callbacks as the following two queries executed in
-** series.
-**
-** <pre>
-**    SELECT * FROM t1 WHERE t1.a=?;
-**    SELECT * FROM t1 WHERE t1.b=?;
-** </pre>
-*/
-SQLITE_EXPERIMENTAL void *sqlite3_whereinfo_hook(
-  sqlite3 *db,                    /* Register callback with this db handle */
-  void (*xWhereInfo)(
-    void*,                        /* Copy of pCtx */
-    int,                          /* SQLITE_WHEREINFO_XXX constant */
-    const char*,
-    int,
-    sqlite3_uint64
-  ),
-  void *pCtx                      /* User pointer passed to callback */
-);
-
-#define SQLITE_WHEREINFO_TABLE   1
-#define SQLITE_WHEREINFO_EQUALS  2
-#define SQLITE_WHEREINFO_RANGE   3
-#define SQLITE_WHEREINFO_ORDERBY 4
-
 /*
 ** Undo the hack that converts floating point types to integer for
 ** builds on processors without floating point support.
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 0c5d4b74d7..22892c967f 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1399,6 +1399,7 @@ struct sqlite3 {
   VtabCtx *pVtabCtx;            /* Context for active vtab connect/create */
   VTable **aVTrans;             /* Virtual tables with open transactions */
   VTable *pDisconnect;    /* Disconnect these in next sqlite3_prepare() */
+  void *pVtabWC;                /* For sqlite3_vtab_collation() */
 #endif
   Hash aFunc;                   /* Hash table of connection functions */
   Hash aCollSeq;                /* All collating sequences */
@@ -1431,10 +1432,6 @@ struct sqlite3 {
 #ifdef SQLITE_USER_AUTHENTICATION
   sqlite3_userauth auth;        /* User authentication information */
 #endif
-#ifdef SQLITE_ENABLE_WHEREINFO_HOOK
-  void (*xWhereInfo)(void*, int, const char*, int, u64);
-  void *pWhereInfoCtx;
-#endif
 };
 
 /*
diff --git a/src/where.c b/src/where.c
index 3cbd959771..df7582832a 100644
--- a/src/where.c
+++ b/src/where.c
@@ -885,7 +885,8 @@ static sqlite3_index_info *allocateIndexInfo(
   */
   pIdxInfo = sqlite3DbMallocZero(pParse->db, sizeof(*pIdxInfo)
                            + (sizeof(*pIdxCons) + sizeof(*pUsage))*nTerm
-                           + sizeof(*pIdxOrderBy)*nOrderBy );
+                           + sizeof(*pIdxOrderBy)*nOrderBy 
+                           );
   if( pIdxInfo==0 ){
     sqlite3ErrorMsg(pParse, "out of memory");
     return 0;
@@ -3116,6 +3117,34 @@ static int whereLoopAddVirtualOne(
 }
 
 
+struct BestIndexCtx {
+  WhereClause *pWC;
+  sqlite3_index_info *pIdxInfo;
+  ExprList *pOrderBy;
+  Parse *pParse;
+};
+
+const char *sqlite3_vtab_collation(sqlite3 *db, int iCons){
+  struct BestIndexCtx *p = (struct BestIndexCtx*)db->pVtabWC;
+  const char *zRet = 0;
+  if( p && iCons>=0 ){
+    if( iCons<p->pIdxInfo->nConstraint ){
+      int iTerm = p->pIdxInfo->aConstraint[iCons].iTermOffset;
+      Expr *pX = p->pWC->a[iTerm].pExpr;
+      CollSeq *pC = sqlite3BinaryCompareCollSeq(p->pParse,pX->pLeft,pX->pRight);
+      zRet = (pC ? pC->zName : "BINARY");
+    }else{
+      iCons -= p->pIdxInfo->nConstraint;
+      if( iCons<p->pIdxInfo->nOrderBy ){
+        Expr *pX = p->pOrderBy->a[iCons].pExpr;
+        CollSeq *pC = sqlite3ExprCollSeq(p->pParse, pX);
+        zRet = (pC ? pC->zName : "BINARY");
+      }
+    }
+  }
+  return zRet;
+}
+
 /*
 ** Add all WhereLoop objects for a table of the join identified by
 ** pBuilder->pNew->iTab.  That table is guaranteed to be a virtual table.
@@ -3157,6 +3186,8 @@ static int whereLoopAddVirtual(
   WhereLoop *pNew;
   Bitmask mBest;               /* Tables used by best possible plan */
   u16 mNoOmit;
+  struct BestIndexCtx bic;
+  void *pSaved;
 
   assert( (mPrereq & mUnusable)==0 );
   pWInfo = pBuilder->pWInfo;
@@ -3178,6 +3209,13 @@ static int whereLoopAddVirtual(
     return SQLITE_NOMEM_BKPT;
   }
 
+  bic.pWC = pWC;
+  bic.pIdxInfo = p;
+  bic.pParse = pParse;
+  bic.pOrderBy = pBuilder->pOrderBy;
+  pSaved = pParse->db->pVtabWC;
+  pParse->db->pVtabWC = (void*)&bic;
+
   /* First call xBestIndex() with all constraints usable. */
   WHERETRACE(0x40, ("  VirtualOne: all usable\n"));
   rc = whereLoopAddVirtualOne(pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn);
@@ -3254,6 +3292,7 @@ static int whereLoopAddVirtual(
 
   if( p->needToFreeIdxStr ) sqlite3_free(p->idxStr);
   sqlite3DbFreeNN(pParse->db, p);
+  pParse->db->pVtabWC = pSaved;
   return rc;
 }
 #endif /* SQLITE_OMIT_VIRTUALTABLE */
@@ -3277,7 +3316,7 @@ static int whereLoopAddOr(
   WhereLoopBuilder sSubBuild;
   WhereOrSet sSum, sCur;
   struct SrcList_item *pItem;
-
+  
   pWC = pBuilder->pWC;
   pWCEnd = pWC->a + pWC->nTerm;
   pNew = pBuilder->pNew;
@@ -3294,7 +3333,7 @@ static int whereLoopAddOr(
       WhereTerm *pOrTerm;
       int once = 1;
       int i, j;
-
+    
       sSubBuild = *pBuilder;
       sSubBuild.pOrderBy = 0;
       sSubBuild.pOrSet = &sCur;
@@ -4278,201 +4317,6 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){
   return 0;
 }
 
-#ifdef SQLITE_ENABLE_WHEREINFO_HOOK
-
-static void whereTraceWC(
-  Parse *pParse, 
-  struct SrcList_item *pItem,
-  WhereClause *pWC
-){
-  sqlite3 *db = pParse->db;
-  Table *pTab = pItem->pTab;
-  void (*x)(void*, int, const char*, int, u64) = db->xWhereInfo;
-  void *pCtx = db->pWhereInfoCtx;
-  int ii;
-
-  /* Issue callbacks for WO_SINGLE constraints */
-  for(ii=0; ii<pTab->nCol; ii++){
-    int opMask = WO_SINGLE; 
-    WhereScan scan;
-    WhereTerm *pTerm;
-    for(pTerm=whereScanInit(&scan, pWC, pItem->iCursor, ii, opMask, 0);
-        pTerm;
-        pTerm=whereScanNext(&scan)
-    ){
-      int eOp;
-      Expr *pX = pTerm->pExpr;
-      CollSeq *pC = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight);
-      if( pTerm->eOperator & (WO_IS|WO_EQ|WO_IN) ){
-        eOp = SQLITE_WHEREINFO_EQUALS;
-      }else{
-        eOp = SQLITE_WHEREINFO_RANGE;
-      }
-      x(pCtx, eOp, (pC ? pC->zName : "BINARY"), ii, pTerm->prereqRight);
-    }
-  }
-}
-
-/*
-** If there are any OR terms in WHERE clause pWC, make the associated
-** where-info hook callbacks.
-*/
-static void whereTraceOR(
-  Parse *pParse, 
-  struct SrcList_item *pItem,
-  WhereClause *pWC
-){
-  sqlite3 *db = pParse->db;
-  WhereClause tempWC;
-  struct TermAndIdx {
-    WhereTerm *pTerm;
-    int iIdx;
-  } aOr[4];
-  int nOr = 0;
-  Table *pTab = pItem->pTab;
-  int iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
-  int ii;
-
-  memset(aOr, 0, sizeof(aOr));
-
-  /* Iterate through OR nodes */
-  for(ii=0; ii<pWC->nTerm; ii++){
-    WhereTerm *pTerm = &pWC->a[ii];
-    if( pTerm->eOperator & WO_OR ){
-      /* Check that each branch of this OR term contains at least
-      ** one reference to the table currently being processed. If that
-      ** is not the case, this term can be ignored.  */
-      WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc;
-      WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm];
-      WhereTerm *pOrTerm;
-      WhereClause *pTermWC;
-      WhereScan scan;
-
-      for(pOrTerm=pOrWC->a; pOrTerm<pOrWCEnd; pOrTerm++){
-        int iCol;
-        if( (pOrTerm->eOperator & WO_AND)!=0 ){
-          pTermWC = &pOrTerm->u.pAndInfo->wc;
-        }else{
-          tempWC.pWInfo = pWC->pWInfo;
-          tempWC.pOuter = pWC;
-          tempWC.op = TK_AND;
-          tempWC.nTerm = 1;
-          tempWC.a = pOrTerm;
-          pTermWC = &tempWC;
-        }
-
-        for(iCol=0; iCol<pTab->nCol; iCol++){
-          int iCsr = pItem->iCursor;
-          if( !whereScanInit(&scan, pTermWC, iCsr, iCol, WO_SINGLE, 0) ){
-            break;
-          }
-        }
-        if( iCol==pTab->nCol ) break;
-      }
-
-      if( pOrTerm==pOrWCEnd ){
-        aOr[nOr].pTerm = pTerm;
-        aOr[nOr].iIdx = pOrWC->nTerm;
-        nOr++;
-        if( nOr==ArraySize(aOr) ) break;
-      }
-    }
-  }
-
-  while( 1 ){
-    for(ii=0; ii<nOr; ii++){
-      if( aOr[ii].iIdx==0 ){
-        aOr[ii].iIdx = aOr[ii].pTerm->u.pOrInfo->wc.nTerm;
-      }else{
-        aOr[ii].iIdx--;
-        break;
-      }
-    }
-    if( ii==nOr ) break;
-
-    /* Table name callback */
-    db->xWhereInfo(db->pWhereInfoCtx, 
-        SQLITE_WHEREINFO_TABLE, pTab->zName, iDb, pItem->colUsed
-    );
-    /* whereTraceWC(pParse, pItem, pWC); */
-    for(ii=0; ii<nOr; ii++){
-      WhereClause * const pOrWC = &aOr[ii].pTerm->u.pOrInfo->wc;
-      if( aOr[ii].iIdx<pOrWC->nTerm ){
-        WhereClause *pTermWC;
-        WhereTerm *pOrTerm = &pOrWC->a[aOr[ii].iIdx];
-        if( (pOrTerm->eOperator & WO_AND)!=0 ){
-          pTermWC = &pOrTerm->u.pAndInfo->wc;
-        }else{
-          tempWC.pWInfo = pWC->pWInfo;
-          tempWC.pOuter = pWC;
-          tempWC.op = TK_AND;
-          tempWC.nTerm = 1;
-          tempWC.a = pOrTerm;
-          pTermWC = &tempWC;
-        }
-        whereTraceWC(pParse, pItem, pTermWC);
-      }
-    }
-  }
-}
-
-/*
-** If there is a where-info hook attached to the database handle, issue all
-** required callbacks for the current sqlite3WhereBegin() call.
-*/
-static void whereTraceBuilder(
-  Parse *pParse,
-  WhereLoopBuilder *p
-){
-  sqlite3 *db = pParse->db;
-  if( db->xWhereInfo && db->init.busy==0 ){
-    void (*x)(void*, int, const char*, int, u64) = db->xWhereInfo;
-    void *pCtx = db->pWhereInfoCtx;
-    int ii;
-    SrcList *pTabList = p->pWInfo->pTabList;
-
-    /* Loop through each element of the FROM clause. Ignore any sub-selects
-    ** or views. Invoke the xWhereInfo() callback multiple times for each
-    ** real table.  */
-    for(ii=0; ii<pTabList->nSrc; ii++){
-      struct SrcList_item *pItem = &pTabList->a[ii];
-      if( pItem->pSelect==0 ){
-        Table *pTab = pItem->pTab;
-        int iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
-
-        /* Table name callback */
-        x(pCtx, SQLITE_WHEREINFO_TABLE, pTab->zName, iDb, pItem->colUsed);
-
-        /* ORDER BY callbacks */
-        if( p->pOrderBy ){
-          int i;
-          for(i=0; i<p->pOrderBy->nExpr; i++){
-            Expr *pExpr = p->pOrderBy->a[i].pExpr; 
-            CollSeq *pColl = sqlite3ExprCollSeq(pParse, pExpr);
-            pExpr = sqlite3ExprSkipCollate(pExpr);
-            if( pExpr->op==TK_COLUMN && pExpr->iTable==pItem->iCursor ){
-              int iCol = pExpr->iColumn;
-              if( pColl && iCol>=0 ){
-                int bDesc = p->pOrderBy->a[i].sortOrder;
-                x(pCtx, SQLITE_WHEREINFO_ORDERBY, pColl->zName, iCol, bDesc); 
-              }
-            }
-          }
-        }
-
-        /* WHERE callbacks */
-        whereTraceWC(pParse, pItem, p->pWC);
-
-        /* OR-clause processing */
-        whereTraceOR(pParse, pItem, p->pWC);
-      }
-    }
-  }
-}
-#else
-# define whereTraceBuilder(x,y)
-#endif
-
 /*
 ** Generate the beginning of the loop used for WHERE clause processing.
 ** The return value is a pointer to an opaque structure that contains
@@ -4745,9 +4589,6 @@ WhereInfo *sqlite3WhereBegin(
   }
 #endif
 
-  /* Invoke the where-info hook, if one has been registered. */
-  whereTraceBuilder(pParse, &sWLB);
-
   if( nTabList!=1 || whereShortCut(&sWLB)==0 ){
     rc = whereLoopAddAll(&sWLB);
     if( rc ) goto whereBeginError;

From 79610f5d090160c0df4d80f3badd9b50f8e959e2 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 15 Apr 2017 14:16:04 +0000
Subject: [PATCH 30/68] Fix memory leaks in the code on this branch. Make use
 of the sqlite3_index_constraint.usage field. Do not try to handle ORDER BY
 terms with explicit COLLATE clauses - they don't get passed to the vtab layer
 anyway.

FossilOrigin-Name: 0cd75a872c89958a7f418720a0e8c6f638f8284c488f666015c19136feae6be8
---
 ext/expert/sqlite3expert.c | 147 +++++++++----------------------------
 manifest                   |  14 ++--
 manifest.uuid              |   2 +-
 src/where.c                |  21 ++----
 4 files changed, 48 insertions(+), 136 deletions(-)

diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index 3cd110c367..153159fbbf 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -39,7 +39,6 @@ struct IdxConstraint {
   char *zColl;                    /* Collation sequence */
   int bRange;                     /* True for range, false for eq */
   int iCol;                       /* Constrained table column */
-  i64 depmask;                    /* Dependency mask */
   int bFlag;                      /* Used by idxFindCompatible() */
   int bDesc;                      /* True if ORDER BY <expr> DESC */
   IdxConstraint *pNext;           /* Next constraint in pEq or pRange list */
@@ -111,25 +110,6 @@ struct IdxHash {
   IdxHashEntry *aHash[IDX_HASH_SIZE];
 };
 
-/*
-** A hash table for storing a set of 64-bit values. Methods are:
-**
-**   idxHash64Init()
-**   idxHash64Clear()
-**   idxHash64Add()
-*/
-typedef struct IdxHash64Entry IdxHash64Entry;
-typedef struct IdxHash64 IdxHash64;
-struct IdxHash64Entry {
-  u64 iVal;
-  IdxHash64Entry *pNext;          /* Next entry in hash table */
-  IdxHash64Entry *pHashNext;      /* Next entry in same hash bucket */
-};
-struct IdxHash64 {
-  IdxHash64Entry *pFirst;         /* Most recently added entry in hash table */
-  IdxHash64Entry *aHash[IDX_HASH_SIZE];
-};
-
 /*
 ** sqlite3expert object.
 */
@@ -166,48 +146,6 @@ static void *idxMalloc(int *pRc, int nByte){
   return pRet;
 }
 
-/*
-** Initialize an IdxHash64 hash table.
-*/
-static void idxHash64Init(IdxHash64 *pHash){
-  memset(pHash, 0, sizeof(IdxHash64));
-}
-
-/*
-** Reset an IdxHash64 hash table.
-*/
-static void idxHash64Clear(IdxHash64 *pHash){
-  IdxHash64Entry *pEntry;
-  IdxHash64Entry *pNext;
-  for(pEntry=pHash->pFirst; pEntry; pEntry=pNext){
-    pNext = pEntry->pNext;
-    sqlite3_free(pEntry);
-  }
-  memset(pHash, 0, sizeof(IdxHash64));
-}
-
-/*
-** Add iVal to the IdxHash64 hash table passed as the second argument. This
-** function is a no-op if iVal is already present in the hash table.
-*/
-static void idxHash64Add(int *pRc, IdxHash64 *pHash, u64 iVal){
-  int iHash = (int)((iVal*7) % IDX_HASH_SIZE);
-  IdxHash64Entry *pEntry;
-  assert( iHash>=0 );
-
-  for(pEntry=pHash->aHash[iHash]; pEntry; pEntry=pEntry->pHashNext){
-    if( pEntry->iVal==iVal ) return;
-  }
-  pEntry = idxMalloc(pRc, sizeof(IdxHash64Entry));
-  if( pEntry ){
-    pEntry->iVal = iVal;
-    pEntry->pHashNext = pHash->aHash[iHash];
-    pHash->aHash[iHash] = pEntry;
-    pEntry->pNext = pHash->pFirst;
-    pHash->pFirst = pEntry;
-  }
-}
-
 /*
 ** Initialize an IdxHash hash table.
 */
@@ -426,14 +364,14 @@ static int expertBestIndex(sqlite3_vtab *pVtab, sqlite3_index_info *pIdxInfo){
 
     /* Add the constraints to the IdxScan object */
     for(i=0; i<pIdxInfo->nConstraint; i++){
-      int op = pIdxInfo->aConstraint[i].op;
-      if( op&opmask ){
+      struct sqlite3_index_constraint *pCons = &pIdxInfo->aConstraint[i];
+      if( pCons->usable && (pCons->op & opmask) ){
         IdxConstraint *pNew;
         const char *zColl = sqlite3_vtab_collation(dbv, i);
         pNew = idxNewConstraint(&rc, zColl);
         if( pNew ){
-          pNew->iCol = pIdxInfo->aConstraint[i].iColumn;
-          if( op==SQLITE_INDEX_CONSTRAINT_EQ ){
+          pNew->iCol = pCons->iColumn;
+          if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ ){
             pNew->pNext = pScan->pEq;
             pScan->pEq = pNew;
           }else{
@@ -442,20 +380,17 @@ static int expertBestIndex(sqlite3_vtab *pVtab, sqlite3_index_info *pIdxInfo){
             pScan->pRange = pNew;
           }
         }
-        if( pIdxInfo->aConstraint[i].usable ){
-          n++;
-          pIdxInfo->aConstraintUsage[i].argvIndex = n;
-        }
+        n++;
+        pIdxInfo->aConstraintUsage[i].argvIndex = n;
       }
     }
 
     /* Add the ORDER BY to the IdxScan object */
     for(i=pIdxInfo->nOrderBy-1; i>=0; i--){
-      IdxConstraint *pNew;
-      const char *zColl = sqlite3_vtab_collation(dbv, i+pIdxInfo->nConstraint);
-      pNew = idxNewConstraint(&rc, zColl);
+      int iCol = pIdxInfo->aOrderBy[i].iColumn;
+      IdxConstraint *pNew = idxNewConstraint(&rc, p->pTab->aCol[iCol].zColl);
       if( pNew ){
-        pNew->iCol = pIdxInfo->aOrderBy[i].iColumn;
+        pNew->iCol = iCol;
         pNew->bDesc = pIdxInfo->aOrderBy[i].desc;
         pNew->pNext = pScan->pOrder;
         pNew->pLink = pScan->pOrder;
@@ -813,7 +748,6 @@ static int idxFindCompatible(
 
 static int idxCreateFromCons(
   sqlite3expert *p,
-  IdxTable *pTab,
   IdxScan *pScan,
   IdxConstraint *pEq, 
   IdxConstraint *pTail
@@ -821,6 +755,7 @@ static int idxCreateFromCons(
   sqlite3 *dbm = p->dbm;
   int rc = SQLITE_OK;
   if( (pEq || pTail) && 0==idxFindCompatible(&rc, dbm, pScan, pEq, pTail) ){
+    IdxTable *pTab = pScan->pTab;
     char *zCols = 0;
     char *zIdx = 0;
     IdxConstraint *pCons;
@@ -882,22 +817,16 @@ static int idxFindConstraint(IdxConstraint *pList, IdxConstraint *p){
 
 static int idxCreateFromWhere(
   sqlite3expert *p, 
-  IdxTable *pTab,
-  i64 mask,                       /* Consider only these constraints */
   IdxScan *pScan,                 /* Create indexes for this scan */
-  IdxConstraint *pEq,             /* == constraints for inclusion */
   IdxConstraint *pTail            /* range/ORDER BY constraints for inclusion */
 ){
-  IdxConstraint *p1 = pEq;
+  IdxConstraint *p1 = 0;
   IdxConstraint *pCon;
   int rc;
 
-  /* Gather up all the == constraints that match the mask. */
+  /* Gather up all the == constraints. */
   for(pCon=pScan->pEq; pCon; pCon=pCon->pNext){
-    if( (mask & pCon->depmask)==pCon->depmask 
-     && idxFindConstraint(p1, pCon)==0
-     && idxFindConstraint(pTail, pCon)==0
-    ){
+    if( !idxFindConstraint(p1, pCon) && !idxFindConstraint(pTail, pCon) ){
       pCon->pLink = p1;
       p1 = pCon;
     }
@@ -905,18 +834,15 @@ static int idxCreateFromWhere(
 
   /* Create an index using the == constraints collected above. And the
   ** range constraint/ORDER BY terms passed in by the caller, if any. */
-  rc = idxCreateFromCons(p, pTab, pScan, p1, pTail);
+  rc = idxCreateFromCons(p, pScan, p1, pTail);
 
   /* If no range/ORDER BY passed by the caller, create a version of the
-  ** index for each range constraint that matches the mask. */
+  ** index for each range constraint.  */
   if( pTail==0 ){
     for(pCon=pScan->pRange; rc==SQLITE_OK && pCon; pCon=pCon->pNext){
       assert( pCon->pLink==0 );
-      if( (mask & pCon->depmask)==pCon->depmask
-        && idxFindConstraint(pEq, pCon)==0
-        && idxFindConstraint(pTail, pCon)==0
-      ){
-        rc = idxCreateFromCons(p, pTab, pScan, p1, pCon);
+      if( !idxFindConstraint(p1, pCon) && !idxFindConstraint(pTail, pCon) ){
+        rc = idxCreateFromCons(p, pScan, p1, pCon);
       }
     }
   }
@@ -931,30 +857,12 @@ static int idxCreateFromWhere(
 static int idxCreateCandidates(sqlite3expert *p, char **pzErr){
   int rc = SQLITE_OK;
   IdxScan *pIter;
-  IdxHash64 hMask;
-  idxHash64Init(&hMask);
 
   for(pIter=p->pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
-    IdxHash64Entry *pEntry;
-    IdxConstraint *pCons;
-    IdxTable *pTab = pIter->pTab;
-
-    idxHash64Add(&rc, &hMask, 0);
-    for(pCons=pIter->pEq; pCons; pCons=pCons->pNext){
-      for(pEntry=hMask.pFirst; pEntry; pEntry=pEntry->pNext){
-        idxHash64Add(&rc, &hMask, pEntry->iVal | (u64)pCons->depmask);
-      }
+    rc = idxCreateFromWhere(p, pIter, 0);
+    if( rc==SQLITE_OK && pIter->pOrder ){
+      rc = idxCreateFromWhere(p, pIter, pIter->pOrder);
     }
-
-    for(pEntry=hMask.pFirst; rc==SQLITE_OK && pEntry; pEntry=pEntry->pNext){
-      i64 mask = (i64)pEntry->iVal;
-      rc = idxCreateFromWhere(p, pTab, mask, pIter, 0, 0);
-      if( rc==SQLITE_OK && pIter->pOrder ){
-        rc = idxCreateFromWhere(p, pTab, mask, pIter, 0, pIter->pOrder);
-      }
-    }
-
-    idxHash64Clear(&hMask);
   }
 
   return rc;
@@ -1004,6 +912,18 @@ static void idxStatementFree(IdxStatement *pStatement, IdxStatement *pLast){
   }
 }
 
+/*
+** Free the linked list of IdxTable objects starting at pTab.
+*/
+static void idxTableFree(IdxTable *pTab){
+  IdxTable *pIter;
+  IdxTable *pNext;
+  for(pIter=pTab; pIter; pIter=pNext){
+    pNext = pIter->pNext;
+    sqlite3_free(pIter);
+  }
+}
+
 
 /*
 ** This function is called after candidate indexes have been created. It
@@ -1126,6 +1046,7 @@ static int idxCreateVtabSchema(sqlite3expert *p, char **pzErrmsg){
       }
     }
   }
+  idxFinalize(&rc, pSchema);
   return rc;
 }
 
@@ -1296,7 +1217,9 @@ void sqlite3_expert_destroy(sqlite3expert *p){
     sqlite3_close(p->dbv);
     idxScanFree(p->pScan, 0);
     idxStatementFree(p->pStatement, 0);
+    idxTableFree(p->pTable);
     idxHashClear(&p->hIdx);
+    sqlite3_free(p->zCandidates);
     sqlite3_free(p);
   }
 }
diff --git a/manifest b/manifest
index 89f2ca08db..6db1d15746 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Modify\sthe\scode\sin\sext/expert/\sto\suse\sthe\svtab\sinterface\sinstead\sof\nsqlite3_whereinfo_hook().\sRemove\ssqlite3_whereinfo_hook().
-D 2017-04-14T19:41:37.126
+C Fix\smemory\sleaks\sin\sthe\scode\son\sthis\sbranch.\sMake\suse\sof\sthe\nsqlite3_index_constraint.usage\sfield.\sDo\snot\stry\sto\shandle\sORDER\sBY\sterms\swith\nexplicit\sCOLLATE\sclauses\s-\sthey\sdon't\sget\spassed\sto\sthe\svtab\slayer\sanyway.
+D 2017-04-15T14:16:04.093
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a
@@ -43,7 +43,7 @@ F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md 9f15075ec5ad772808eff55ef044c31140fd1146aa0a3c47eafd155e71851b01
 F ext/expert/expert.c 6349cf8d26c847f5f0fa7e25772b614c67f60f3c850dca0d75d55eb27cf3f69b
 F ext/expert/expert1.test cd630eda18a2508eade4c39a1eafe32e7437a33973391e5dddfc7fd0f3163684
-F ext/expert/sqlite3expert.c 9473b011d7e0be5b52157f3b1fc153d7e5f7d2b43af110180843e7a03972439f
+F ext/expert/sqlite3expert.c d4a0a45be58874b4cf54316e5d776726489967140399b1e77a17dbb66558cb38
 F ext/expert/sqlite3expert.h b1c9eedeb647fd734c4206ae6851635284cfbfa5fb688eff74c3265c9f949b4d
 F ext/expert/test_expert.c bad0611732d07180d586bd589cbb7713dc3ab0338c52bff29680eb2007678c05
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
@@ -488,7 +488,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 40c543f0a2195d1b0dc88ef12142bea690009344
 F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71
 F src/walker.c b71a992b413b3a022572eccf29ef4b4890223791
-F src/where.c 64f2c18c72e06bc935c64a53fd5cea6446ffedeba87c45905408bea124c201b6
+F src/where.c 7b4d39ffdb82a6c5abd8678c66a4cd16d8bdba3bbe158a9e2f181a78e05b07ef
 F src/whereInt.h 7a21ef633e26acbf46df04add2eba6e0a2100c78dc5879049e93f981fc3344df
 F src/wherecode.c 943e32e9dccd0af802e0683ae11071c8bd808364e5908a5fb66758bd404c8681
 F src/whereexpr.c e913aaa7b73ffcce66abcea5f197e2c538d48b5df78d0b7bba8ff4d73cc2e745
@@ -1578,7 +1578,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5fcd840cf9b6a5c3ee4ef1e8f92f6c30f96a7899a3d774ee9be8a816916f2c3b
-R 754b00e271f01cdc7a8ee810736e1b12
+P 3bb6585004090dbf92dd5e9abdf0fd2c921e64b5b3121c4fb7446db764ab59e5
+R 039cfbfa3ee0c0fcda94663d31b151d9
 U dan
-Z bc14af8147812d1c0312a546deaaf0cc
+Z 80d209b7df04543ba8ed8a1a58dd2c4a
diff --git a/manifest.uuid b/manifest.uuid
index 7b25e887aa..de896ff6d4 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-3bb6585004090dbf92dd5e9abdf0fd2c921e64b5b3121c4fb7446db764ab59e5
\ No newline at end of file
+0cd75a872c89958a7f418720a0e8c6f638f8284c488f666015c19136feae6be8
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index df7582832a..d49c3d42f2 100644
--- a/src/where.c
+++ b/src/where.c
@@ -3120,27 +3120,17 @@ static int whereLoopAddVirtualOne(
 struct BestIndexCtx {
   WhereClause *pWC;
   sqlite3_index_info *pIdxInfo;
-  ExprList *pOrderBy;
   Parse *pParse;
 };
 
 const char *sqlite3_vtab_collation(sqlite3 *db, int iCons){
   struct BestIndexCtx *p = (struct BestIndexCtx*)db->pVtabWC;
   const char *zRet = 0;
-  if( p && iCons>=0 ){
-    if( iCons<p->pIdxInfo->nConstraint ){
-      int iTerm = p->pIdxInfo->aConstraint[iCons].iTermOffset;
-      Expr *pX = p->pWC->a[iTerm].pExpr;
-      CollSeq *pC = sqlite3BinaryCompareCollSeq(p->pParse,pX->pLeft,pX->pRight);
-      zRet = (pC ? pC->zName : "BINARY");
-    }else{
-      iCons -= p->pIdxInfo->nConstraint;
-      if( iCons<p->pIdxInfo->nOrderBy ){
-        Expr *pX = p->pOrderBy->a[iCons].pExpr;
-        CollSeq *pC = sqlite3ExprCollSeq(p->pParse, pX);
-        zRet = (pC ? pC->zName : "BINARY");
-      }
-    }
+  if( p && iCons>=0 && iCons<p->pIdxInfo->nConstraint ){
+    int iTerm = p->pIdxInfo->aConstraint[iCons].iTermOffset;
+    Expr *pX = p->pWC->a[iTerm].pExpr;
+    CollSeq *pC = sqlite3BinaryCompareCollSeq(p->pParse,pX->pLeft,pX->pRight);
+    zRet = (pC ? pC->zName : "BINARY");
   }
   return zRet;
 }
@@ -3212,7 +3202,6 @@ static int whereLoopAddVirtual(
   bic.pWC = pWC;
   bic.pIdxInfo = p;
   bic.pParse = pParse;
-  bic.pOrderBy = pBuilder->pOrderBy;
   pSaved = pParse->db->pVtabWC;
   pParse->db->pVtabWC = (void*)&bic;
 

From e01b9281fc5b47a6052cc3fcaa7a36c0bb1f550d Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 15 Apr 2017 14:30:01 +0000
Subject: [PATCH 31/68] Add header comment for sqlite3_vtab_collation().

FossilOrigin-Name: d238694ca445ccb4eeb3e3269a5f872f998f795945d0f9dd95c11d0e42d4d538
---
 manifest        | 16 ++++++++--------
 manifest.uuid   |  2 +-
 src/sqlite.h.in | 14 ++++++++++++++
 src/sqliteInt.h |  2 +-
 src/where.c     | 19 +++++++++++++++----
 5 files changed, 39 insertions(+), 14 deletions(-)

diff --git a/manifest b/manifest
index 6db1d15746..38a861353e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\smemory\sleaks\sin\sthe\scode\son\sthis\sbranch.\sMake\suse\sof\sthe\nsqlite3_index_constraint.usage\sfield.\sDo\snot\stry\sto\shandle\sORDER\sBY\sterms\swith\nexplicit\sCOLLATE\sclauses\s-\sthey\sdon't\sget\spassed\sto\sthe\svtab\slayer\sanyway.
-D 2017-04-15T14:16:04.093
+C Add\sheader\scomment\sfor\ssqlite3_vtab_collation().
+D 2017-04-15T14:30:01.495
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a
@@ -409,10 +409,10 @@ F src/resolve.c 3e518b962d932a997fae373366880fc028c75706
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c e6f9afd8a5ef35bd186e51a6bea6d3d46bc93a530f26a21fe8a0a43dbeca9415
 F src/shell.c 70f4957b988572315e97c56941fdc81fd35907fee36b7b2e7be5ec4c7e9d065d
-F src/sqlite.h.in fa0a24af458b49de8d5654fd1f3b61422ef513cee9b17f39ce3f5e5731ce4f6a
+F src/sqlite.h.in 18b4f1367bb80c07a6883c75c18267de2b977960adf1f30c783a731691eae3b0
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 58fd0676d3111d02e62e5a35992a7d3da5d3f88753acc174f2d37b774fbbdd28
-F src/sqliteInt.h 8ac5d9b92084de271b913387968720f4dfe17302eb48a51c6ae161b6149057ad
+F src/sqliteInt.h bd00872a45aa8eaa262301436bb63dc0afa685ceab7361790b065d5802269650
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -488,7 +488,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 40c543f0a2195d1b0dc88ef12142bea690009344
 F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71
 F src/walker.c b71a992b413b3a022572eccf29ef4b4890223791
-F src/where.c 7b4d39ffdb82a6c5abd8678c66a4cd16d8bdba3bbe158a9e2f181a78e05b07ef
+F src/where.c 4b0d7b6696f2de64b3bdd2bc600f51ef6cd832fe269f2c0ad330974885c30652
 F src/whereInt.h 7a21ef633e26acbf46df04add2eba6e0a2100c78dc5879049e93f981fc3344df
 F src/wherecode.c 943e32e9dccd0af802e0683ae11071c8bd808364e5908a5fb66758bd404c8681
 F src/whereexpr.c e913aaa7b73ffcce66abcea5f197e2c538d48b5df78d0b7bba8ff4d73cc2e745
@@ -1578,7 +1578,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3bb6585004090dbf92dd5e9abdf0fd2c921e64b5b3121c4fb7446db764ab59e5
-R 039cfbfa3ee0c0fcda94663d31b151d9
+P 0cd75a872c89958a7f418720a0e8c6f638f8284c488f666015c19136feae6be8
+R c9114bbd4adb78d55a86a8431d68fb8b
 U dan
-Z 80d209b7df04543ba8ed8a1a58dd2c4a
+Z d3545883deff75eff19284d6064a11b5
diff --git a/manifest.uuid b/manifest.uuid
index de896ff6d4..409e30cf73 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0cd75a872c89958a7f418720a0e8c6f638f8284c488f666015c19136feae6be8
\ No newline at end of file
+d238694ca445ccb4eeb3e3269a5f872f998f795945d0f9dd95c11d0e42d4d538
\ No newline at end of file
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 3c54efd36a..219e17b9fe 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -8037,6 +8037,20 @@ int sqlite3_vtab_config(sqlite3*, int op, ...);
 */
 int sqlite3_vtab_on_conflict(sqlite3 *);
 
+/*
+** CAPI3REF: Determine The Collation For a Virtual Table Constraint
+**
+** This function may only be called from within a call to the [xBestIndex]
+** method of a [virtual table implementation]. 
+**
+** The first argument must be the database handle with which the virtual 
+** table is associated (the one passed to the [xConnect] or [xCreate] method 
+** to create the sqlite3_vtab object. The second argument must be an index
+** into the aConstraint[] array belonging to the sqlite3_index_info structure
+** passed to xBestIndex. This function returns a pointer to a buffer 
+** containing the name of the collation sequence for the corresponding
+** constraint.
+*/
 SQLITE_EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3*, int);
 
 /*
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 22892c967f..259c020fd6 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1399,7 +1399,7 @@ struct sqlite3 {
   VtabCtx *pVtabCtx;            /* Context for active vtab connect/create */
   VTable **aVTrans;             /* Virtual tables with open transactions */
   VTable *pDisconnect;    /* Disconnect these in next sqlite3_prepare() */
-  void *pVtabWC;                /* For sqlite3_vtab_collation() */
+  void *pBestIndexCtx;          /* For sqlite3_vtab_collation() */
 #endif
   Hash aFunc;                   /* Hash table of connection functions */
   Hash aCollSeq;                /* All collating sequences */
diff --git a/src/where.c b/src/where.c
index d49c3d42f2..b1926a2d40 100644
--- a/src/where.c
+++ b/src/where.c
@@ -3117,14 +3117,25 @@ static int whereLoopAddVirtualOne(
 }
 
 
+/*
+** Context object used to pass information from whereLoopAddVirtual()
+** to sqlite3_vtab_collation().
+*/
 struct BestIndexCtx {
   WhereClause *pWC;
   sqlite3_index_info *pIdxInfo;
   Parse *pParse;
 };
 
+/*
+** If this function is invoked from within an xBestIndex() callback, it
+** returns a pointer to a buffer containing the name of the collation
+** sequence associated with element iCons of the sqlite3_index_info.aConstraint
+** array. Or, if iCons is out of range or there is no active xBestIndex
+** call, return NULL.
+*/
 const char *sqlite3_vtab_collation(sqlite3 *db, int iCons){
-  struct BestIndexCtx *p = (struct BestIndexCtx*)db->pVtabWC;
+  struct BestIndexCtx *p = (struct BestIndexCtx*)db->pBestIndexCtx;
   const char *zRet = 0;
   if( p && iCons>=0 && iCons<p->pIdxInfo->nConstraint ){
     int iTerm = p->pIdxInfo->aConstraint[iCons].iTermOffset;
@@ -3202,8 +3213,8 @@ static int whereLoopAddVirtual(
   bic.pWC = pWC;
   bic.pIdxInfo = p;
   bic.pParse = pParse;
-  pSaved = pParse->db->pVtabWC;
-  pParse->db->pVtabWC = (void*)&bic;
+  pSaved = pParse->db->pBestIndexCtx;
+  pParse->db->pBestIndexCtx = (void*)&bic;
 
   /* First call xBestIndex() with all constraints usable. */
   WHERETRACE(0x40, ("  VirtualOne: all usable\n"));
@@ -3281,7 +3292,7 @@ static int whereLoopAddVirtual(
 
   if( p->needToFreeIdxStr ) sqlite3_free(p->idxStr);
   sqlite3DbFreeNN(pParse->db, p);
-  pParse->db->pVtabWC = pSaved;
+  pParse->db->pBestIndexCtx = pSaved;
   return rc;
 }
 #endif /* SQLITE_OMIT_VIRTUALTABLE */

From 2abf90096ff64257a12f602e5754d3c33e97a41a Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 15 Apr 2017 16:52:12 +0000
Subject: [PATCH 32/68] Fix problems with handling constraints on the rowid
 column in sqlite3expert.c.

FossilOrigin-Name: 2e6308798ae2db30564deb35ba3896597448edabbcac6efc4ff084552e42de30
---
 ext/expert/expert1.test    | 58 +++++++++++++++++++++++++++-----------
 ext/expert/sqlite3expert.c | 20 +++++++------
 manifest                   | 14 ++++-----
 manifest.uuid              |  2 +-
 4 files changed, 61 insertions(+), 33 deletions(-)

diff --git a/ext/expert/expert1.test b/ext/expert/expert1.test
index 437ba45600..bcde9b2dfd 100644
--- a/ext/expert/expert1.test
+++ b/ext/expert/expert1.test
@@ -78,14 +78,14 @@ foreach {tn setup} {
   eval $setup
 
 
-do_setup_rec_test $tn.1.1 { CREATE TABLE t1(a, b, c) } {
+do_setup_rec_test $tn.1 { CREATE TABLE t1(a, b, c) } {
   SELECT * FROM t1
 } {
   (no new indexes)
   0|0|0|SCAN TABLE t1
 }
 
-do_setup_rec_test $tn.1.2 {
+do_setup_rec_test $tn.2 {
   CREATE TABLE t1(a, b, c);
 } {
   SELECT * FROM t1 WHERE b>?;
@@ -94,7 +94,7 @@ do_setup_rec_test $tn.1.2 {
   0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_00000062 (b>?)
 }
 
-do_setup_rec_test $tn.1.3 {
+do_setup_rec_test $tn.3 {
   CREATE TABLE t1(a, b, c);
 } {
   SELECT * FROM t1 WHERE b COLLATE nocase BETWEEN ? AND ?
@@ -103,7 +103,7 @@ do_setup_rec_test $tn.1.3 {
   0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_3e094c27 (b>? AND b<?)
 }
 
-do_setup_rec_test $tn.1.4 {
+do_setup_rec_test $tn.4 {
   CREATE TABLE t1(a, b, c);
 } {
   SELECT a FROM t1 ORDER BY b;
@@ -112,7 +112,7 @@ do_setup_rec_test $tn.1.4 {
   0|0|0|SCAN TABLE t1 USING INDEX t1_idx_00000062
 }
 
-do_setup_rec_test $tn.1.5 {
+do_setup_rec_test $tn.5 {
   CREATE TABLE t1(a, b, c);
 } {
   SELECT a FROM t1 WHERE a=? ORDER BY b;
@@ -121,7 +121,7 @@ do_setup_rec_test $tn.1.5 {
   0|0|0|SEARCH TABLE t1 USING COVERING INDEX t1_idx_000123a7 (a=?)
 }
 
-do_setup_rec_test $tn.1.6 {
+do_setup_rec_test $tn.6 {
   CREATE TABLE t1(a, b, c);
 } {
   SELECT min(a) FROM t1
@@ -130,7 +130,7 @@ do_setup_rec_test $tn.1.6 {
   0|0|0|SEARCH TABLE t1 USING COVERING INDEX t1_idx_00000061
 }
 
-do_setup_rec_test $tn.1.7 {
+do_setup_rec_test $tn.7 {
   CREATE TABLE t1(a, b, c);
 } {
   SELECT * FROM t1 ORDER BY a, b, c;
@@ -148,7 +148,7 @@ do_setup_rec_test $tn.1.7 {
 #  0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_5be6e222
 #}
 
-do_setup_rec_test $tn.1.9 {
+do_setup_rec_test $tn.8.1 {
   CREATE TABLE t1(a COLLATE NOCase, b, c);
 } {
   SELECT * FROM t1 WHERE a=?
@@ -156,8 +156,7 @@ do_setup_rec_test $tn.1.9 {
   CREATE INDEX t1_idx_00000061 ON t1(a);
   0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_00000061 (a=?)
 }
-
-do_setup_rec_test $tn.1.10 {
+do_setup_rec_test $tn.8.2 {
   CREATE TABLE t1(a, b COLLATE nocase, c);
 } {
   SELECT * FROM t1 ORDER BY a ASC, b DESC, c ASC;
@@ -169,7 +168,7 @@ do_setup_rec_test $tn.1.10 {
 
 # Tables with names that require quotes.
 #
-do_setup_rec_test $tn.8.1 {
+do_setup_rec_test $tn.9.1 {
   CREATE TABLE "t t"(a, b, c);
 } {
   SELECT * FROM "t t" WHERE a=?
@@ -178,7 +177,7 @@ do_setup_rec_test $tn.8.1 {
   0|0|0|SEARCH TABLE t t USING INDEX t t_idx_00000061 (a=?) 
 }
 
-do_setup_rec_test $tn.8.2 {
+do_setup_rec_test $tn.9.2 {
   CREATE TABLE "t t"(a, b, c);
 } {
   SELECT * FROM "t t" WHERE b BETWEEN ? AND ?
@@ -189,7 +188,7 @@ do_setup_rec_test $tn.8.2 {
 
 # Columns with names that require quotes.
 #
-do_setup_rec_test $tn.9.1 {
+do_setup_rec_test $tn.10.1 {
   CREATE TABLE t3(a, "b b", c);
 } {
   SELECT * FROM t3 WHERE "b b" = ?
@@ -198,7 +197,7 @@ do_setup_rec_test $tn.9.1 {
   0|0|0|SEARCH TABLE t3 USING INDEX t3_idx_00050c52 (b b=?)
 }
 
-do_setup_rec_test $tn.9.2 {
+do_setup_rec_test $tn.10.2 {
   CREATE TABLE t3(a, "b b", c);
 } {
   SELECT * FROM t3 ORDER BY "b b"
@@ -209,7 +208,7 @@ do_setup_rec_test $tn.9.2 {
 
 # Transitive constraints
 #
-do_setup_rec_test $tn.10.1 {
+do_setup_rec_test $tn.11.1 {
   CREATE TABLE t5(a, b);
   CREATE TABLE t6(c, d);
 } {
@@ -223,7 +222,7 @@ do_setup_rec_test $tn.10.1 {
 
 # OR terms.
 #
-do_setup_rec_test $tn.11.1 {
+do_setup_rec_test $tn.12.1 {
   CREATE TABLE t7(a, b);
 } {
   SELECT * FROM t7 WHERE a=? OR b=?
@@ -234,6 +233,33 @@ do_setup_rec_test $tn.11.1 {
   0|0|0|SEARCH TABLE t7 USING INDEX t7_idx_00000062 (b=?)
 }
 
+# rowid terms.
+#
+do_setup_rec_test $tn.13.1 {
+  CREATE TABLE t8(a, b);
+} {
+  SELECT * FROM t8 WHERE rowid=?
+} {
+  (no new indexes)
+  0|0|0|SEARCH TABLE t8 USING INTEGER PRIMARY KEY (rowid=?)
+}
+do_setup_rec_test $tn.13.2 {
+  CREATE TABLE t8(a, b);
+} {
+  SELECT * FROM t8 ORDER BY rowid
+} {
+  (no new indexes)
+  0|0|0|SCAN TABLE t8
+}
+do_setup_rec_test $tn.13.3 {
+  CREATE TABLE t8(a, b);
+} {
+  SELECT * FROM t8 WHERE a=? ORDER BY rowid
+} {
+  CREATE INDEX t8_idx_00000061 ON t8(a); 
+  0|0|0|SEARCH TABLE t8 USING INDEX t8_idx_00000061 (a=?)
+}
+
 }
 
 finish_test
diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index 153159fbbf..ab419d3f08 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -365,7 +365,7 @@ static int expertBestIndex(sqlite3_vtab *pVtab, sqlite3_index_info *pIdxInfo){
     /* Add the constraints to the IdxScan object */
     for(i=0; i<pIdxInfo->nConstraint; i++){
       struct sqlite3_index_constraint *pCons = &pIdxInfo->aConstraint[i];
-      if( pCons->usable && (pCons->op & opmask) ){
+      if( pCons->usable && pCons->iColumn>=0 && (pCons->op & opmask) ){
         IdxConstraint *pNew;
         const char *zColl = sqlite3_vtab_collation(dbv, i);
         pNew = idxNewConstraint(&rc, zColl);
@@ -388,14 +388,16 @@ static int expertBestIndex(sqlite3_vtab *pVtab, sqlite3_index_info *pIdxInfo){
     /* Add the ORDER BY to the IdxScan object */
     for(i=pIdxInfo->nOrderBy-1; i>=0; i--){
       int iCol = pIdxInfo->aOrderBy[i].iColumn;
-      IdxConstraint *pNew = idxNewConstraint(&rc, p->pTab->aCol[iCol].zColl);
-      if( pNew ){
-        pNew->iCol = iCol;
-        pNew->bDesc = pIdxInfo->aOrderBy[i].desc;
-        pNew->pNext = pScan->pOrder;
-        pNew->pLink = pScan->pOrder;
-        pScan->pOrder = pNew;
-        n++;
+      if( iCol>=0 ){
+        IdxConstraint *pNew = idxNewConstraint(&rc, p->pTab->aCol[iCol].zColl);
+        if( pNew ){
+          pNew->iCol = iCol;
+          pNew->bDesc = pIdxInfo->aOrderBy[i].desc;
+          pNew->pNext = pScan->pOrder;
+          pNew->pLink = pScan->pOrder;
+          pScan->pOrder = pNew;
+          n++;
+        }
       }
     }
   }
diff --git a/manifest b/manifest
index a544938625..8f2777fd7d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\slatest\strunk\schanges\sinto\sthis\sbranch.
-D 2017-04-15T15:47:19.800
+C Fix\sproblems\swith\shandling\sconstraints\son\sthe\srowid\scolumn\sin\ssqlite3expert.c.
+D 2017-04-15T16:52:12.782
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6
@@ -42,8 +42,8 @@ F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md 9f15075ec5ad772808eff55ef044c31140fd1146aa0a3c47eafd155e71851b01
 F ext/expert/expert.c 6349cf8d26c847f5f0fa7e25772b614c67f60f3c850dca0d75d55eb27cf3f69b
-F ext/expert/expert1.test cd630eda18a2508eade4c39a1eafe32e7437a33973391e5dddfc7fd0f3163684
-F ext/expert/sqlite3expert.c d4a0a45be58874b4cf54316e5d776726489967140399b1e77a17dbb66558cb38
+F ext/expert/expert1.test c08c95fd81e80073def8bdbf30b67934a9c20193b3632e5f27565ef88f964809
+F ext/expert/sqlite3expert.c d73a4813af4c8097bb574113c91abf9c0ec06a7af47c80227a160a7c413265b5
 F ext/expert/sqlite3expert.h b1c9eedeb647fd734c4206ae6851635284cfbfa5fb688eff74c3265c9f949b4d
 F ext/expert/test_expert.c bad0611732d07180d586bd589cbb7713dc3ab0338c52bff29680eb2007678c05
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
@@ -1579,7 +1579,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d238694ca445ccb4eeb3e3269a5f872f998f795945d0f9dd95c11d0e42d4d538 89f9e4363aa19f306e55f749c442eae2f8994f6a47c65e645a79b308b450d5e5
-R ee849516fe355a9533f6086faf663944
+P 2d0c458e013cb2d02fbeabed8dabd66f55141aac194611f0e599b3c95af1964f
+R d323d2b5ffa5632b2c495cb6bacbb016
 U dan
-Z 67dd241955788961a5e3c1b5613b9072
+Z 2344908fdbfea0ffd0d694cf678615cc
diff --git a/manifest.uuid b/manifest.uuid
index 69d4418f08..afc4462a6b 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2d0c458e013cb2d02fbeabed8dabd66f55141aac194611f0e599b3c95af1964f
\ No newline at end of file
+2e6308798ae2db30564deb35ba3896597448edabbcac6efc4ff084552e42de30
\ No newline at end of file

From 280db65e2cc87e8e842ed7fe3800de72e8bb8521 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 17 Apr 2017 17:03:08 +0000
Subject: [PATCH 33/68] Add support for analyzing trigger programs to the
 sqlite3_expert code.

FossilOrigin-Name: 159e8022a9d6701532b8b60e0c41154bc434c1bbdb107c8c97a78fb1140fa745
---
 ext/expert/expert1.test    |  15 +++
 ext/expert/sqlite3expert.c | 189 ++++++++++++++++++++++++++++++++++++-
 manifest                   |  30 +++---
 manifest.uuid              |   2 +-
 src/build.c                |  12 +--
 src/main.c                 |   5 +-
 src/prepare.c              |   2 +-
 src/sqlite.h.in            |  11 +++
 src/sqliteInt.h            |   3 +-
 src/trigger.c              |   2 +-
 src/vdbe.c                 |   6 +-
 src/vdbeaux.c              | 185 +++++++++++++++++++-----------------
 12 files changed, 340 insertions(+), 122 deletions(-)

diff --git a/ext/expert/expert1.test b/ext/expert/expert1.test
index bcde9b2dfd..a817865229 100644
--- a/ext/expert/expert1.test
+++ b/ext/expert/expert1.test
@@ -260,6 +260,21 @@ do_setup_rec_test $tn.13.3 {
   0|0|0|SEARCH TABLE t8 USING INDEX t8_idx_00000061 (a=?)
 }
 
+# Triggers
+#
+do_setup_rec_test $tn.14 {
+  CREATE TABLE t9(a, b, c);
+  CREATE TABLE t10(a, b, c);
+  CREATE TRIGGER t9t AFTER INSERT ON t9 BEGIN
+    UPDATE t10 SET a=new.a WHERE b = new.b;
+  END;
+} {
+  INSERT INTO t9 VALUES(?, ?, ?);
+} {
+  CREATE INDEX t10_idx_00000062 ON t10(b); 
+  0|0|0|SEARCH TABLE t10 USING INDEX t10_idx_00000062 (b=?)
+}
+
 }
 
 finish_test
diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index ab419d3f08..8316a18ae3 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -26,6 +26,7 @@ typedef struct IdxConstraint IdxConstraint;
 typedef struct IdxScan IdxScan;
 typedef struct IdxStatement IdxStatement;
 typedef struct IdxTable IdxTable;
+typedef struct IdxWrite IdxWrite;
 
 /*
 ** A single constraint. Equivalent to either "col = ?" or "col < ?" (or
@@ -74,6 +75,17 @@ struct IdxTable {
   IdxTable *pNext;                /* Next table in linked list of all tables */
 };
 
+/*
+** An object of the following type is created for each unique table/write-op
+** seen. The objects are stored in a singly-linked list beginning at
+** sqlite3expert.pWrite.
+*/
+struct IdxWrite {
+  IdxTable *pTab;
+  int eOp;                        /* SQLITE_UPDATE, DELETE or INSERT */
+  IdxWrite *pNext;
+};
+
 /*
 ** Each statement being analyzed is represented by an instance of this
 ** structure.
@@ -118,8 +130,8 @@ struct sqlite3expert {
   sqlite3 *dbm;                   /* In-memory db for this analysis */
   sqlite3 *dbv;                   /* Vtab schema for this analysis */
   IdxTable *pTable;               /* List of all IdxTable objects */
-
   IdxScan *pScan;                 /* List of scan objects */
+  IdxWrite *pWrite;               /* List of write objects */
   IdxStatement *pStatement;       /* List of IdxStatement objects */
   int bRun;                       /* True once analysis has run */
   char **pzErrmsg;
@@ -406,6 +418,15 @@ static int expertBestIndex(sqlite3_vtab *pVtab, sqlite3_index_info *pIdxInfo){
   return rc;
 }
 
+static int expertUpdate(
+  sqlite3_vtab *pVtab, 
+  int nData, 
+  sqlite3_value **azData, 
+  sqlite_int64 *pRowid
+){
+  return SQLITE_OK;
+}
+
 static int idxRegisterVtab(sqlite3expert *p){
   static sqlite3_module expertModule = {
     2,                            /* iVersion */
@@ -421,7 +442,7 @@ static int idxRegisterVtab(sqlite3expert *p){
     0,                            /* xEof */
     0,                            /* xColumn - read data */
     0,                            /* xRowid - read data */
-    0,                            /* xUpdate - write data */
+    expertUpdate,                 /* xUpdate - write data */
     0,                            /* xBegin - begin transaction */
     0,                            /* xSync - sync transaction */
     0,                            /* xCommit - commit transaction */
@@ -926,6 +947,19 @@ static void idxTableFree(IdxTable *pTab){
   }
 }
 
+/*
+** Free the linked list of IdxWrite objects starting at pTab.
+*/
+static void idxWriteFree(IdxWrite *pTab){
+  IdxWrite *pIter;
+  IdxWrite *pNext;
+  for(pIter=pTab; pIter; pIter=pNext){
+    pNext = pIter->pNext;
+    sqlite3_free(pIter);
+  }
+}
+
+
 
 /*
 ** This function is called after candidate indexes have been created. It
@@ -997,6 +1031,139 @@ int idxFindIndexes(
   return rc;
 }
 
+static int idxAuthCallback(
+  void *pCtx,
+  int eOp,
+  const char *z3,
+  const char *z4,
+  const char *zDb,
+  const char *zTrigger
+){
+  int rc = SQLITE_OK;
+  if( eOp==SQLITE_INSERT || eOp==SQLITE_UPDATE || eOp==SQLITE_DELETE ){
+    if( sqlite3_stricmp(zDb, "main")==0 ){
+      sqlite3expert *p = (sqlite3expert*)pCtx;
+      IdxTable *pTab;
+      for(pTab=p->pTable; pTab; pTab=pTab->pNext){
+        if( 0==sqlite3_stricmp(z3, pTab->zName) ) break;
+      }
+      if( pTab ){
+        IdxWrite *pWrite;
+        for(pWrite=p->pWrite; pWrite; pWrite=pWrite->pNext){
+          if( pWrite->pTab==pTab && pWrite->eOp==eOp ) break;
+        }
+        if( pWrite==0 ){
+          pWrite = idxMalloc(&rc, sizeof(IdxWrite));
+          if( rc==SQLITE_OK ){
+            pWrite->pTab = pTab;
+            pWrite->eOp = eOp;
+            pWrite->pNext = p->pWrite;
+            p->pWrite = pWrite;
+          }
+        }
+      }
+    }
+  }
+  return rc;
+}
+
+static int idxProcessOneTrigger(
+  sqlite3expert *p, 
+  IdxWrite *pWrite, 
+  char **pzErr
+){
+  static const char *zInt = "t592690916721053953805701627921227776";
+  static const char *zDrop = "DROP TABLE t592690916721053953805701627921227776";
+  IdxTable *pTab = pWrite->pTab;
+  const char *zTab = pTab->zName;
+  const char *zSql = 
+    "SELECT 'CREATE TEMP' || substr(sql, 7) FROM sqlite_master "
+    "WHERE tbl_name = %Q AND type IN ('table', 'trigger') "
+    "ORDER BY type;";
+  sqlite3_stmt *pSelect = 0;
+  int rc = SQLITE_OK;
+  char *zWrite = 0;
+
+  /* Create the table and its triggers in the temp schema */
+  rc = idxPrintfPrepareStmt(p->db, &pSelect, pzErr, zSql, zTab, zTab);
+  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSelect) ){
+    const char *zCreate = (const char*)sqlite3_column_text(pSelect, 0);
+    rc = sqlite3_exec(p->dbv, zCreate, 0, 0, pzErr);
+  }
+  idxFinalize(&rc, pSelect);
+
+  /* Rename the table in the temp schema to zInt */
+  if( rc==SQLITE_OK ){
+    char *z = sqlite3_mprintf("ALTER TABLE temp.%Q RENAME TO %Q", zTab, zInt);
+    if( z==0 ){
+      rc = SQLITE_NOMEM;
+    }else{
+      rc = sqlite3_exec(p->dbv, z, 0, 0, pzErr);
+      sqlite3_free(z);
+    }
+  }
+
+  switch( pWrite->eOp ){
+    case SQLITE_INSERT: {
+      int i;
+      zWrite = idxAppendText(&rc, zWrite, "INSERT INTO %Q VALUES(", zInt);
+      for(i=0; i<pTab->nCol; i++){
+        zWrite = idxAppendText(&rc, zWrite, "%s?", i==0 ? "" : ", ");
+      }
+      zWrite = idxAppendText(&rc, zWrite, ")");
+      break;
+    }
+    case SQLITE_UPDATE: {
+      int i;
+      zWrite = idxAppendText(&rc, zWrite, "UPDATE %Q SET ", zInt);
+      for(i=0; i<pTab->nCol; i++){
+        zWrite = idxAppendText(&rc, zWrite, "%s%Q=?", i==0 ? "" : ", ", 
+            pTab->aCol[i].zName
+        );
+      }
+      break;
+    }
+    default: {
+      assert( pWrite->eOp==SQLITE_DELETE );
+      if( rc==SQLITE_OK ){
+        zWrite = sqlite3_mprintf("DELETE FROM %Q", zInt);
+        if( zWrite==0 ) rc = SQLITE_NOMEM;
+      }
+    }
+  }
+
+  if( rc==SQLITE_OK ){
+    sqlite3_stmt *pX = 0;
+    rc = sqlite3_prepare_v2(p->dbv, zWrite, -1, &pX, 0);
+    idxFinalize(&rc, pX);
+  }
+  sqlite3_free(zWrite);
+
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_exec(p->dbv, zDrop, 0, 0, pzErr);
+  }
+
+  return rc;
+}
+
+static int idxProcessTriggers(sqlite3expert *p, char **pzErr){
+  int rc = SQLITE_OK;
+  IdxWrite *pEnd = 0;
+  IdxWrite *pFirst = p->pWrite;
+
+  while( rc==SQLITE_OK && pFirst!=pEnd ){
+    IdxWrite *pIter;
+    for(pIter=pFirst; rc==SQLITE_OK && pIter!=pEnd; pIter=pIter->pNext){
+      rc = idxProcessOneTrigger(p, pIter, pzErr);
+    }
+    pEnd = pFirst;
+    pFirst = p->pWrite;
+  }
+
+  return rc;
+}
+
+
 static int idxCreateVtabSchema(sqlite3expert *p, char **pzErrmsg){
   int rc = idxRegisterVtab(p);
   sqlite3_stmt *pSchema = 0;
@@ -1073,7 +1240,11 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){
   }
   if( rc==SQLITE_OK ){
     rc = sqlite3_open(":memory:", &pNew->dbm);
+    if( rc==SQLITE_OK ){
+      sqlite3_db_config(pNew->dbm, SQLITE_DBCONFIG_FULL_EQP, 1, (int*)0);
+    }
   }
+  
 
   /* Copy the entire schema of database [db] into [dbm]. */
   if( rc==SQLITE_OK ){
@@ -1093,6 +1264,11 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){
     rc = idxCreateVtabSchema(pNew, pzErrmsg);
   }
 
+  /* Register the auth callback with dbv */
+  if( rc==SQLITE_OK ){
+    sqlite3_set_authorizer(pNew->dbv, idxAuthCallback, (void*)pNew);
+  }
+
   /* If an error has occurred, free the new object and reutrn NULL. Otherwise,
   ** return the new sqlite3expert handle.  */
   if( rc!=SQLITE_OK ){
@@ -1136,7 +1312,7 @@ int sqlite3_expert_sql(
         sqlite3_finalize(pStmt);
       }
     }else{
-      idxDatabaseError(p->db, pzErr);
+      idxDatabaseError(p->dbv, pzErr);
     }
   }
 
@@ -1154,8 +1330,12 @@ int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr){
   int rc;
   IdxHashEntry *pEntry;
 
+  rc = idxProcessTriggers(p, pzErr);
+
   /* Create candidate indexes within the in-memory database file */
-  rc = idxCreateCandidates(p, pzErr);
+  if( rc==SQLITE_OK ){
+    rc = idxCreateCandidates(p, pzErr);
+  }
 
   /* Formulate the EXPERT_REPORT_CANDIDATES text */
   for(pEntry=p->hIdx.pFirst; pEntry; pEntry=pEntry->pNext){
@@ -1220,6 +1400,7 @@ void sqlite3_expert_destroy(sqlite3expert *p){
     idxScanFree(p->pScan, 0);
     idxStatementFree(p->pStatement, 0);
     idxTableFree(p->pTable);
+    idxWriteFree(p->pWrite);
     idxHashClear(&p->hIdx);
     sqlite3_free(p->zCandidates);
     sqlite3_free(p);
diff --git a/manifest b/manifest
index 8f2777fd7d..7bc6efbb2d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sproblems\swith\shandling\sconstraints\son\sthe\srowid\scolumn\sin\ssqlite3expert.c.
-D 2017-04-15T16:52:12.782
+C Add\ssupport\sfor\sanalyzing\strigger\sprograms\sto\sthe\ssqlite3_expert\scode.
+D 2017-04-17T17:03:08.723
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6
@@ -42,8 +42,8 @@ F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md 9f15075ec5ad772808eff55ef044c31140fd1146aa0a3c47eafd155e71851b01
 F ext/expert/expert.c 6349cf8d26c847f5f0fa7e25772b614c67f60f3c850dca0d75d55eb27cf3f69b
-F ext/expert/expert1.test c08c95fd81e80073def8bdbf30b67934a9c20193b3632e5f27565ef88f964809
-F ext/expert/sqlite3expert.c d73a4813af4c8097bb574113c91abf9c0ec06a7af47c80227a160a7c413265b5
+F ext/expert/expert1.test 6a50a1538dc9e4ff360fb117298aa3b085beed030cbe15dd36803da1a9f70702
+F ext/expert/sqlite3expert.c f9f41caf6d941b52ef76043f3191ab05294f4d286425e34923a7d74876014f0b
 F ext/expert/sqlite3expert.h b1c9eedeb647fd734c4206ae6851635284cfbfa5fb688eff74c3265c9f949b4d
 F ext/expert/test_expert.c bad0611732d07180d586bd589cbb7713dc3ab0338c52bff29680eb2007678c05
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
@@ -353,7 +353,7 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
 F src/btree.c 24ae5472bd0b53b4130ecdda389deb621af721d1fcb50890b878102b00bd10fa
 F src/btree.h 80f518c0788be6cec8d9f8e13bd8e380df299d2b5e4ac340dc887b0642647cfc
 F src/btreeInt.h a392d353104b4add58b4a59cb185f5d5693dde832c565b77d8d4c343ed98f610
-F src/build.c 4026a9c554b233e50c5e9ad46963e676cf54dd2306d952aa1eaa07a1bc9ce14f
+F src/build.c 3fd46781483b527ee18508e7854e87e60a259211bb9bbf16b6fafaf08a043a64
 F src/callback.c 2e76147783386374bf01b227f752c81ec872d730
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c 47d91a25ad8f199a71a5b1b7b169d6dd0d6e98c5719eca801568798743d1161c
@@ -372,7 +372,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c d4bb3a135948553d18cf992f76f7ed7b18aa0327f250607b5a6671e55d9947d5
 F src/legacy.c e88ed13c2d531decde75d42c2e35623fb9ce3cb0
 F src/loadext.c a72909474dadce771d3669bf84bf689424f6f87d471fee898589c3ef9b2acfd9
-F src/main.c 158326243c5ddc8b98a1e983fa488650cf76d760
+F src/main.c ffb658429483c0d1c604014c631871f64b8db19666d8b70f75c7512d003ac1ad
 F src/malloc.c e20bb2b48abec52d3faf01cce12e8b4f95973755fafec98d45162dfdab111978
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -402,17 +402,17 @@ F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490
 F src/pcache1.c 1195a21fe28e223e024f900b2011e80df53793f0356a24caace4188b098540dc
 F src/pragma.c 150821702fc90694b46c3432c1402fc970a4c5b8595cb13c21aeb568f9a78fc3
 F src/pragma.h 37a1311d0388db480388d7ec09054f7103045eff20d4971f8a433b77f40b9921
-F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a
+F src/prepare.c 7c46b5c7be9e19a1bf87777f0b7f9fb257b5ff9856c46de49f2354acfbeb4c86
 F src/printf.c 8757834f1b54dae512fb25eb1acc8e94a0d15dd2290b58f2563f65973265adb2
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 3e518b962d932a997fae373366880fc028c75706
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 4588dcfb0fa430012247a209ba08e17904dd32ec7690e9cb6c85e0ef012b0518
 F src/shell.c 70f4957b988572315e97c56941fdc81fd35907fee36b7b2e7be5ec4c7e9d065d
-F src/sqlite.h.in 18b4f1367bb80c07a6883c75c18267de2b977960adf1f30c783a731691eae3b0
+F src/sqlite.h.in 900a07463a87be50b9954817f4c24a0660b4c4ddc1bfe83dedea484c6ac98425
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 58fd0676d3111d02e62e5a35992a7d3da5d3f88753acc174f2d37b774fbbdd28
-F src/sqliteInt.h bd00872a45aa8eaa262301436bb63dc0afa685ceab7361790b065d5802269650
+F src/sqliteInt.h 0e520ab49f019221dd5a17b6e4006523ce4f33d88b20bcf9115d11952a487c39
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -469,16 +469,16 @@ F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
 F src/tokenize.c 1003d6d90c6783206c711f0a9397656fa5b055209f4d092caa43bb3bf5215db5
 F src/treeview.c b92d57c1ac59f4a3f6b189506921a2b48098f6f4d6afd0b715bc2815ef6af092
-F src/trigger.c c9f0810043b265724fdb1bdd466894f984dfc182
+F src/trigger.c 134b8e7b61317ab7b2a2dd12eb1b9aa2e23ac5bc4a05e63e35b3609b6b30a7c0
 F src/update.c c443935c652af9365e033f756550b5032d02e1b06eb2cb890ed7511ae0c051dc
 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c
 F src/util.c ca8440ede81e155d15cff7c101654f60b55a9ae6
 F src/vacuum.c 1fe4555cd8c9b263afb85b5b4ee3a4a4181ad569
-F src/vdbe.c 808fda3d50f544120d27c731449b524b4ec8f8b0f734b228831078f0ba53ecb9
+F src/vdbe.c 314f0a70ddc29e63f131dfbe6f53c277875d3028cdf5654a709e21cab39fe0c9
 F src/vdbe.h f7d1456e28875c2dcb964056589b5b7149ab7edf39edeca801596a39bb3d3848
 F src/vdbeInt.h c070bc5c8b913bda0ceaa995cd4d939ded5e4fc96cf7c3c1c602d41b871f8ade
 F src/vdbeapi.c 5b08d82592bcff4470601fe78aaabebd50837860
-F src/vdbeaux.c 6b3f6ce909e206d4c918988b13b7fa687e92b4471d137e0f2a37edac80ec60be
+F src/vdbeaux.c 526b617ac6b5e167a6bd581e067f1ee1dbcb06e7802cff46b76fb1c02ed7d34e
 F src/vdbeblob.c 359891617358deefc85bef7bcf787fa6b77facb9
 F src/vdbemem.c 3122f5a21064198c10ee1b4686937aab27d5395712d9af905b7fa1affc47a453
 F src/vdbesort.c e72fe02a2121386ba767ede8942e9450878b8fc873abf3d1b6824485f092570c
@@ -1579,7 +1579,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2d0c458e013cb2d02fbeabed8dabd66f55141aac194611f0e599b3c95af1964f
-R d323d2b5ffa5632b2c495cb6bacbb016
+P 2e6308798ae2db30564deb35ba3896597448edabbcac6efc4ff084552e42de30
+R 3935ba65365bf244da478b55268b3a1f
 U dan
-Z 2344908fdbfea0ffd0d694cf678615cc
+Z 871de5d26699f20ec603c0a41beb3b11
diff --git a/manifest.uuid b/manifest.uuid
index afc4462a6b..8158ffa2b0 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2e6308798ae2db30564deb35ba3896597448edabbcac6efc4ff084552e42de30
\ No newline at end of file
+159e8022a9d6701532b8b60e0c41154bc434c1bbdb107c8c97a78fb1140fa745
\ No newline at end of file
diff --git a/src/build.c b/src/build.c
index e04406d857..47339e82b6 100644
--- a/src/build.c
+++ b/src/build.c
@@ -479,7 +479,7 @@ void sqlite3UnlinkAndDeleteIndex(sqlite3 *db, int iDb, const char *zIdxName){
     }
     freeIndex(db, pIndex);
   }
-  db->flags |= SQLITE_InternChanges;
+  db->bInternChanges = 1;
 }
 
 /*
@@ -551,7 +551,7 @@ void sqlite3ResetAllSchemasOfConnection(sqlite3 *db){
       sqlite3SchemaClear(pDb->pSchema);
     }
   }
-  db->flags &= ~SQLITE_InternChanges;
+  db->bInternChanges = 0;
   sqlite3VtabUnlockList(db);
   sqlite3BtreeLeaveAll(db);
   sqlite3CollapseDatabaseArray(db);
@@ -561,7 +561,7 @@ void sqlite3ResetAllSchemasOfConnection(sqlite3 *db){
 ** This routine is called when a commit occurs.
 */
 void sqlite3CommitInternalChanges(sqlite3 *db){
-  db->flags &= ~SQLITE_InternChanges;
+  db->bInternChanges = 0;
 }
 
 /*
@@ -665,7 +665,7 @@ void sqlite3UnlinkAndDeleteTable(sqlite3 *db, int iDb, const char *zTabName){
   pDb = &db->aDb[iDb];
   p = sqlite3HashInsert(&pDb->pSchema->tblHash, zTabName, 0);
   sqlite3DeleteTable(db, p);
-  db->flags |= SQLITE_InternChanges;
+  db->bInternChanges = 1;
 }
 
 /*
@@ -2051,7 +2051,7 @@ void sqlite3EndTable(
       return;
     }
     pParse->pNewTable = 0;
-    db->flags |= SQLITE_InternChanges;
+    db->bInternChanges = 1;
 
 #ifndef SQLITE_OMIT_ALTERTABLE
     if( !p->pSelect ){
@@ -3320,7 +3320,7 @@ void sqlite3CreateIndex(
       sqlite3OomFault(db);
       goto exit_create_index;
     }
-    db->flags |= SQLITE_InternChanges;
+    db->bInternChanges = 1;
     if( pTblName!=0 ){
       pIndex->tnum = db->init.newTnum;
     }
diff --git a/src/main.c b/src/main.c
index 4ac5327e4f..db508abc57 100644
--- a/src/main.c
+++ b/src/main.c
@@ -811,6 +811,7 @@ int sqlite3_db_config(sqlite3 *db, int op, ...){
         { SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, SQLITE_Fts3Tokenizer  },
         { SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, SQLITE_LoadExtension  },
         { SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE,      SQLITE_NoCkptOnClose  },
+        { SQLITE_DBCONFIG_FULL_EQP,              SQLITE_FullEQP  },
       };
       unsigned int i;
       rc = SQLITE_ERROR; /* IMP: R-42790-23372 */
@@ -1252,7 +1253,7 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){
   ** the database rollback and schema reset, which can cause false
   ** corruption reports in some cases.  */
   sqlite3BtreeEnterAll(db);
-  schemaChange = (db->flags & SQLITE_InternChanges)!=0 && db->init.busy==0;
+  schemaChange = db->bInternChanges && db->init.busy==0;
 
   for(i=0; i<db->nDb; i++){
     Btree *p = db->aDb[i].pBt;
@@ -1266,7 +1267,7 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){
   sqlite3VtabRollback(db);
   sqlite3EndBenignMalloc();
 
-  if( (db->flags&SQLITE_InternChanges)!=0 && db->init.busy==0 ){
+  if( db->bInternChanges && db->init.busy==0 ){
     sqlite3ExpirePreparedStatements(db);
     sqlite3ResetAllSchemasOfConnection(db);
   }
diff --git a/src/prepare.c b/src/prepare.c
index 74127bc76b..aa36cca166 100644
--- a/src/prepare.c
+++ b/src/prepare.c
@@ -354,7 +354,7 @@ error_out:
 */
 int sqlite3Init(sqlite3 *db, char **pzErrMsg){
   int i, rc;
-  int commit_internal = !(db->flags&SQLITE_InternChanges);
+  int commit_internal = db->bInternChanges==0;
   
   assert( sqlite3_mutex_held(db->mutex) );
   assert( sqlite3BtreeHoldsMutex(db->aDb[0].pBt) );
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 219e17b9fe..6a5de5f5b9 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -2007,6 +2007,16 @@ struct sqlite3_mem_methods {
 ** have been disabled - 0 if they are not disabled, 1 if they are.
 ** </dd>
 **
+** <dt>SQLITE_DBCONFIG_FULL_EQP</dt>
+** <dd> By default, the output of EXPLAIN QUERY PLAN commands does not 
+** include output for any operations performed by trigger programs. This
+** option is used to set or clear (the default) a flag that governs this
+** behavior. The first parameter passed to this operation is an integer -
+** non-zero to enable output for trigger programs, or zero to disable it.
+** The second parameter is a pointer to an integer into which is written 
+** 0 or 1 to indicate whether output-for-triggers has been disabled - 0 if 
+** it is not disabled, 1 if it is.  
+** </dd>
 ** </dl>
 */
 #define SQLITE_DBCONFIG_MAINDBNAME            1000 /* const char* */
@@ -2016,6 +2026,7 @@ struct sqlite3_mem_methods {
 #define SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 1004 /* int int* */
 #define SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION 1005 /* int int* */
 #define SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE      1006 /* int int* */
+#define SQLITE_DBCONFIG_FULL_EQP              1007 /* int int* */
 
 
 /*
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 259c020fd6..3ce77bf7d5 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1334,6 +1334,7 @@ struct sqlite3 {
   u8 mTrace;                    /* zero or more SQLITE_TRACE flags */
   u8 skipBtreeMutex;            /* True if no shared-cache backends */
   u8 nSqlExec;                  /* Number of pending OP_SqlExec opcodes */
+  u8 bInternChanges;            /* There are uncommited schema changes */
   int nextPagesize;             /* Pagesize after VACUUM if >0 */
   u32 magic;                    /* Magic number for detect library misuse */
   int nChange;                  /* Value returned by sqlite3_changes() */
@@ -1449,7 +1450,7 @@ struct sqlite3 {
 **      SQLITE_CacheSpill    == PAGER_CACHE_SPILL
 */
 #define SQLITE_VdbeTrace      0x00000001  /* True to trace VDBE execution */
-#define SQLITE_InternChanges  0x00000002  /* Uncommitted Hash table changes */
+#define SQLITE_FullEQP        0x00000002  /* Include triggers in EQP output */
 #define SQLITE_FullColNames   0x00000004  /* Show full column names on SELECT */
 #define SQLITE_FullFSync      0x00000008  /* Use full fsync on the backend */
 #define SQLITE_CkptFullFSync  0x00000010  /* Use full fsync for checkpoint */
diff --git a/src/trigger.c b/src/trigger.c
index bdf964084b..197dcda6c3 100644
--- a/src/trigger.c
+++ b/src/trigger.c
@@ -584,7 +584,7 @@ void sqlite3UnlinkAndDeleteTrigger(sqlite3 *db, int iDb, const char *zName){
       *pp = (*pp)->pNext;
     }
     sqlite3DeleteTrigger(db, pTrigger);
-    db->flags |= SQLITE_InternChanges;
+    db->bInternChanges = 1;
   }
 }
 
diff --git a/src/vdbe.c b/src/vdbe.c
index a990afb11d..4f8657829f 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -3037,7 +3037,7 @@ case OP_Savepoint: {
         int isSchemaChange;
         iSavepoint = db->nSavepoint - iSavepoint - 1;
         if( p1==SAVEPOINT_ROLLBACK ){
-          isSchemaChange = (db->flags & SQLITE_InternChanges)!=0;
+          isSchemaChange = db->bInternChanges;
           for(ii=0; ii<db->nDb; ii++){
             rc = sqlite3BtreeTripAllCursors(db->aDb[ii].pBt,
                                        SQLITE_ABORT_ROLLBACK,
@@ -3056,7 +3056,7 @@ case OP_Savepoint: {
         if( isSchemaChange ){
           sqlite3ExpirePreparedStatements(db);
           sqlite3ResetAllSchemasOfConnection(db);
-          db->flags = (db->flags | SQLITE_InternChanges);
+          db->bInternChanges = 1;
         }
       }
   
@@ -3336,7 +3336,7 @@ case OP_SetCookie: {
   if( pOp->p2==BTREE_SCHEMA_VERSION ){
     /* When the schema cookie changes, record the new cookie internally */
     pDb->pSchema->schema_cookie = pOp->p3;
-    db->flags |= SQLITE_InternChanges;
+    db->bInternChanges = 1;
   }else if( pOp->p2==BTREE_FILE_FORMAT ){
     /* Record changes in the file format */
     pDb->pSchema->file_format = pOp->p3;
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index 00a5ec91a9..5de43f42ce 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -1611,6 +1611,8 @@ int sqlite3VdbeList(
   int i;                               /* Loop counter */
   int rc = SQLITE_OK;                  /* Return code */
   Mem *pMem = &p->aMem[1];             /* First Mem of result set */
+  int bFull = (p->explain==1 || (db->flags & SQLITE_FullEQP));
+  Op *pOp;
 
   assert( p->explain );
   assert( p->magic==VDBE_MAGIC_RUN );
@@ -1638,7 +1640,7 @@ int sqlite3VdbeList(
   ** encountered, but p->pc will eventually catch up to nRow.
   */
   nRow = p->nOp;
-  if( p->explain==1 ){
+  if( bFull ){
     /* The first 8 memory cells are used for the result set.  So we will
     ** commandeer the 9th cell to use as storage for an array of pointers
     ** to trigger subprograms.  The VDBE is guaranteed to have at least 9
@@ -1658,17 +1660,11 @@ int sqlite3VdbeList(
 
   do{
     i = p->pc++;
-  }while( i<nRow && p->explain==2 && p->aOp[i].opcode!=OP_Explain );
-  if( i>=nRow ){
-    p->rc = SQLITE_OK;
-    rc = SQLITE_DONE;
-  }else if( db->u1.isInterrupted ){
-    p->rc = SQLITE_INTERRUPT;
-    rc = SQLITE_ERROR;
-    sqlite3VdbeError(p, sqlite3ErrStr(p->rc));
-  }else{
-    char *zP4;
-    Op *pOp;
+    if( i>=nRow ){
+      p->rc = SQLITE_OK;
+      rc = SQLITE_DONE;
+      break;
+    }
     if( i<p->nOp ){
       /* The output line number is small enough that we are still in the
       ** main program. */
@@ -1683,94 +1679,107 @@ int sqlite3VdbeList(
       }
       pOp = &apSub[j]->aOp[i];
     }
-    if( p->explain==1 ){
-      pMem->flags = MEM_Int;
-      pMem->u.i = i;                                /* Program counter */
-      pMem++;
-  
-      pMem->flags = MEM_Static|MEM_Str|MEM_Term;
-      pMem->z = (char*)sqlite3OpcodeName(pOp->opcode); /* Opcode */
-      assert( pMem->z!=0 );
-      pMem->n = sqlite3Strlen30(pMem->z);
-      pMem->enc = SQLITE_UTF8;
-      pMem++;
 
-      /* When an OP_Program opcode is encounter (the only opcode that has
-      ** a P4_SUBPROGRAM argument), expand the size of the array of subprograms
-      ** kept in p->aMem[9].z to hold the new program - assuming this subprogram
-      ** has not already been seen.
-      */
-      if( pOp->p4type==P4_SUBPROGRAM ){
-        int nByte = (nSub+1)*sizeof(SubProgram*);
-        int j;
-        for(j=0; j<nSub; j++){
-          if( apSub[j]==pOp->p4.pProgram ) break;
-        }
-        if( j==nSub && SQLITE_OK==sqlite3VdbeMemGrow(pSub, nByte, nSub!=0) ){
-          apSub = (SubProgram **)pSub->z;
-          apSub[nSub++] = pOp->p4.pProgram;
-          pSub->flags |= MEM_Blob;
-          pSub->n = nSub*sizeof(SubProgram*);
-        }
+    /* When an OP_Program opcode is encounter (the only opcode that has
+    ** a P4_SUBPROGRAM argument), expand the size of the array of subprograms
+    ** kept in p->aMem[9].z to hold the new program - assuming this subprogram
+    ** has not already been seen.
+    */
+    if( bFull && pOp->p4type==P4_SUBPROGRAM ){
+      int nByte = (nSub+1)*sizeof(SubProgram*);
+      int j;
+      for(j=0; j<nSub; j++){
+        if( apSub[j]==pOp->p4.pProgram ) break;
+      }
+      if( j==nSub ){
+        rc = sqlite3VdbeMemGrow(pSub, nByte, nSub!=0);
+        if( rc!=SQLITE_OK ) break;
+        apSub = (SubProgram **)pSub->z;
+        apSub[nSub++] = pOp->p4.pProgram;
+        pSub->flags |= MEM_Blob;
+        pSub->n = nSub*sizeof(SubProgram*);
+        nRow += pOp->p4.pProgram->nOp;
       }
     }
+  }while( p->explain==2 && pOp->opcode!=OP_Explain );
 
-    pMem->flags = MEM_Int;
-    pMem->u.i = pOp->p1;                          /* P1 */
-    pMem++;
-
-    pMem->flags = MEM_Int;
-    pMem->u.i = pOp->p2;                          /* P2 */
-    pMem++;
-
-    pMem->flags = MEM_Int;
-    pMem->u.i = pOp->p3;                          /* P3 */
-    pMem++;
-
-    if( sqlite3VdbeMemClearAndResize(pMem, 100) ){ /* P4 */
-      assert( p->db->mallocFailed );
-      return SQLITE_ERROR;
-    }
-    pMem->flags = MEM_Str|MEM_Term;
-    zP4 = displayP4(pOp, pMem->z, pMem->szMalloc);
-    if( zP4!=pMem->z ){
-      pMem->n = 0;
-      sqlite3VdbeMemSetStr(pMem, zP4, -1, SQLITE_UTF8, 0);
+  if( rc==SQLITE_OK ){
+    if( db->u1.isInterrupted ){
+      p->rc = SQLITE_INTERRUPT;
+      rc = SQLITE_ERROR;
+      sqlite3VdbeError(p, sqlite3ErrStr(p->rc));
     }else{
-      assert( pMem->z!=0 );
-      pMem->n = sqlite3Strlen30(pMem->z);
-      pMem->enc = SQLITE_UTF8;
-    }
-    pMem++;
-
-    if( p->explain==1 ){
-      if( sqlite3VdbeMemClearAndResize(pMem, 4) ){
-        assert( p->db->mallocFailed );
-        return SQLITE_ERROR;
+      char *zP4;
+      if( p->explain==1 ){
+        pMem->flags = MEM_Int;
+        pMem->u.i = i;                                /* Program counter */
+        pMem++;
+    
+        pMem->flags = MEM_Static|MEM_Str|MEM_Term;
+        pMem->z = (char*)sqlite3OpcodeName(pOp->opcode); /* Opcode */
+        assert( pMem->z!=0 );
+        pMem->n = sqlite3Strlen30(pMem->z);
+        pMem->enc = SQLITE_UTF8;
+        pMem++;
       }
-      pMem->flags = MEM_Str|MEM_Term;
-      pMem->n = 2;
-      sqlite3_snprintf(3, pMem->z, "%.2x", pOp->p5);   /* P5 */
-      pMem->enc = SQLITE_UTF8;
+
+      pMem->flags = MEM_Int;
+      pMem->u.i = pOp->p1;                          /* P1 */
       pMem++;
-  
-#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS
-      if( sqlite3VdbeMemClearAndResize(pMem, 500) ){
+
+      pMem->flags = MEM_Int;
+      pMem->u.i = pOp->p2;                          /* P2 */
+      pMem++;
+
+      pMem->flags = MEM_Int;
+      pMem->u.i = pOp->p3;                          /* P3 */
+      pMem++;
+
+      if( sqlite3VdbeMemClearAndResize(pMem, 100) ){ /* P4 */
         assert( p->db->mallocFailed );
         return SQLITE_ERROR;
       }
       pMem->flags = MEM_Str|MEM_Term;
-      pMem->n = displayComment(pOp, zP4, pMem->z, 500);
-      pMem->enc = SQLITE_UTF8;
-#else
-      pMem->flags = MEM_Null;                       /* Comment */
-#endif
-    }
+      zP4 = displayP4(pOp, pMem->z, pMem->szMalloc);
+      if( zP4!=pMem->z ){
+        pMem->n = 0;
+        sqlite3VdbeMemSetStr(pMem, zP4, -1, SQLITE_UTF8, 0);
+      }else{
+        assert( pMem->z!=0 );
+        pMem->n = sqlite3Strlen30(pMem->z);
+        pMem->enc = SQLITE_UTF8;
+      }
+      pMem++;
 
-    p->nResColumn = 8 - 4*(p->explain-1);
-    p->pResultSet = &p->aMem[1];
-    p->rc = SQLITE_OK;
-    rc = SQLITE_ROW;
+      if( p->explain==1 ){
+        if( sqlite3VdbeMemClearAndResize(pMem, 4) ){
+          assert( p->db->mallocFailed );
+          return SQLITE_ERROR;
+        }
+        pMem->flags = MEM_Str|MEM_Term;
+        pMem->n = 2;
+        sqlite3_snprintf(3, pMem->z, "%.2x", pOp->p5);   /* P5 */
+        pMem->enc = SQLITE_UTF8;
+        pMem++;
+    
+#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS
+        if( sqlite3VdbeMemClearAndResize(pMem, 500) ){
+          assert( p->db->mallocFailed );
+          return SQLITE_ERROR;
+        }
+        pMem->flags = MEM_Str|MEM_Term;
+        pMem->n = displayComment(pOp, zP4, pMem->z, 500);
+        pMem->enc = SQLITE_UTF8;
+#else
+        pMem->flags = MEM_Null;                       /* Comment */
+#endif
+      }
+
+      p->nResColumn = 8 - 4*(p->explain-1);
+      p->pResultSet = &p->aMem[1];
+      p->rc = SQLITE_OK;
+      rc = SQLITE_ROW;
+    }
   }
   return rc;
 }

From 7853002c712e439537276d6e820b8b9f56b505c8 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 18 Apr 2017 09:04:48 +0000
Subject: [PATCH 34/68] Fix sqlite3_expert handling of triggers on views.

FossilOrigin-Name: ff4976da667872614331d88e68fb67d347874f164a1c7950dd738c7c2320b954
---
 ext/expert/expert.c        |  4 ++--
 ext/expert/sqlite3expert.c | 14 +++++++++++---
 manifest                   | 14 +++++++-------
 manifest.uuid              |  2 +-
 4 files changed, 21 insertions(+), 13 deletions(-)

diff --git a/ext/expert/expert.c b/ext/expert/expert.c
index 19b52c867d..3aafde1d3f 100644
--- a/ext/expert/expert.c
+++ b/ext/expert/expert.c
@@ -135,8 +135,8 @@ int main(int argc, char **argv){
       }
       fprintf(stdout, "%s\n%s\n", zIdx, zEQP);
     }
-  }else if( zErr ){
-    fprintf(stderr, "Error: %s\n", zErr);
+  }else{
+    fprintf(stderr, "Error: %s\n", zErr ? zErr : "?");
   }
 
   sqlite3_expert_destroy(p);
diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index 8316a18ae3..093e4438f5 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -1136,6 +1136,9 @@ static int idxProcessOneTrigger(
     sqlite3_stmt *pX = 0;
     rc = sqlite3_prepare_v2(p->dbv, zWrite, -1, &pX, 0);
     idxFinalize(&rc, pX);
+    if( rc!=SQLITE_OK ){
+      idxDatabaseError(p->dbv, pzErr);
+    }
   }
   sqlite3_free(zWrite);
 
@@ -1174,15 +1177,20 @@ static int idxCreateVtabSchema(sqlite3expert *p, char **pzErrmsg){
   **   2) Create the equivalent virtual table in dbv.
   */
   rc = idxPrepareStmt(p->db, &pSchema, pzErrmsg,
-      "SELECT type, name, sql FROM sqlite_master "
-      "WHERE type IN ('table','view') ORDER BY 1"
+      "SELECT type, name, sql, 1 FROM sqlite_master "
+      "WHERE type IN ('table','view') AND name NOT LIKE 'sqlite_%%' "
+      " UNION ALL "
+      "SELECT type, name, sql, 2 FROM sqlite_master "
+      "WHERE type = 'trigger'"
+      "  AND tbl_name IN(SELECT name FROM sqlite_master WHERE type = 'view') "
+      "ORDER BY 4, 1"
   );
   while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSchema) ){
     const char *zType = (const char*)sqlite3_column_text(pSchema, 0);
     const char *zName = (const char*)sqlite3_column_text(pSchema, 1);
     const char *zSql = (const char*)sqlite3_column_text(pSchema, 2);
 
-    if( zType[0]=='v' ){
+    if( zType[0]=='v' || zType[1]=='r' ){
       rc = sqlite3_exec(p->dbv, zSql, 0, 0, pzErrmsg);
     }else{
       IdxTable *pTab;
diff --git a/manifest b/manifest
index 7bc6efbb2d..cde195ed9d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\ssupport\sfor\sanalyzing\strigger\sprograms\sto\sthe\ssqlite3_expert\scode.
-D 2017-04-17T17:03:08.723
+C Fix\ssqlite3_expert\shandling\sof\striggers\son\sviews.
+D 2017-04-18T09:04:48.670
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6
@@ -41,9 +41,9 @@ F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md 9f15075ec5ad772808eff55ef044c31140fd1146aa0a3c47eafd155e71851b01
-F ext/expert/expert.c 6349cf8d26c847f5f0fa7e25772b614c67f60f3c850dca0d75d55eb27cf3f69b
+F ext/expert/expert.c 22d2dd096d479049bc332506fc8c0294bf53b7ebfe60af99635d8c87839bb40b
 F ext/expert/expert1.test 6a50a1538dc9e4ff360fb117298aa3b085beed030cbe15dd36803da1a9f70702
-F ext/expert/sqlite3expert.c f9f41caf6d941b52ef76043f3191ab05294f4d286425e34923a7d74876014f0b
+F ext/expert/sqlite3expert.c 8befe20906cf8f4928c754763cbf0a81f42fc1ee242fe8e5869c3bb4adba7300
 F ext/expert/sqlite3expert.h b1c9eedeb647fd734c4206ae6851635284cfbfa5fb688eff74c3265c9f949b4d
 F ext/expert/test_expert.c bad0611732d07180d586bd589cbb7713dc3ab0338c52bff29680eb2007678c05
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
@@ -1579,7 +1579,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2e6308798ae2db30564deb35ba3896597448edabbcac6efc4ff084552e42de30
-R 3935ba65365bf244da478b55268b3a1f
+P 159e8022a9d6701532b8b60e0c41154bc434c1bbdb107c8c97a78fb1140fa745
+R 52d2320865ff7653f66201ecd375f3d6
 U dan
-Z 871de5d26699f20ec603c0a41beb3b11
+Z e02fbf739534a0fe797908906bbc0c24
diff --git a/manifest.uuid b/manifest.uuid
index 8158ffa2b0..6969132da7 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-159e8022a9d6701532b8b60e0c41154bc434c1bbdb107c8c97a78fb1140fa745
\ No newline at end of file
+ff4976da667872614331d88e68fb67d347874f164a1c7950dd738c7c2320b954
\ No newline at end of file

From a6ed5a4f397a97ca92d392fb5481da0f2058c41f Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 18 Apr 2017 20:10:16 +0000
Subject: [PATCH 35/68] Have sqlite3_expert_analyze() populate the sqlite_stat1
 table before running queries through the planner for the second time.

FossilOrigin-Name: a157fcfde5afc27ae38e7cf4669fcc8e60e23d9d301ffe2e541dd69f895b493b
---
 ext/expert/expert1.test    |  79 ++++++++++
 ext/expert/sqlite3expert.c | 306 +++++++++++++++++++++++++++++++++++--
 ext/expert/test_expert.c   |   3 +-
 manifest                   |  16 +-
 manifest.uuid              |   2 +-
 5 files changed, 387 insertions(+), 19 deletions(-)

diff --git a/ext/expert/expert1.test b/ext/expert/expert1.test
index a817865229..7995327557 100644
--- a/ext/expert/expert1.test
+++ b/ext/expert/expert1.test
@@ -275,7 +275,86 @@ do_setup_rec_test $tn.14 {
   0|0|0|SEARCH TABLE t10 USING INDEX t10_idx_00000062 (b=?)
 }
 
+do_setup_rec_test $tn.15 {
+  CREATE TABLE t1(a, b);
+  CREATE TABLE t2(c, d);
+
+  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
+  INSERT INTO t1 SELECT (i-1)/50, (i-1)/20 FROM s;
+
+  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
+  INSERT INTO t2 SELECT (i-1)/20, (i-1)/5 FROM s;
+} {
+  SELECT * FROM t2, t1 WHERE b=? AND d=? AND t2.rowid=t1.rowid
+} {
+  CREATE INDEX t2_idx_00000064 ON t2(d);
+  0|0|0|SEARCH TABLE t2 USING INDEX t2_idx_00000064 (d=?) 
+  0|1|1|SEARCH TABLE t1 USING INTEGER PRIMARY KEY (rowid=?)
 }
 
+}
+
+proc do_candidates_test {tn sql res} {
+  set res [squish [string trim $res]]
+
+  set expert [sqlite3_expert_new db]
+  $expert sql $sql
+  $expert analyze
+
+  set candidates [squish [string trim [$expert report 0 candidates]]]
+  $expert destroy
+
+  uplevel [list do_test $tn [list set {} $candidates] $res]
+}
+
+
+reset_db
+do_execsql_test 3.0 {
+  CREATE TABLE t1(a, b);
+  CREATE TABLE t2(c, d);
+
+  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
+  INSERT INTO t1 SELECT (i-1)/50, (i-1)/20 FROM s;
+
+  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
+  INSERT INTO t2 SELECT (i-1)/20, (i-1)/5 FROM s;
+}
+do_candidates_test 3.1 {
+  SELECT * FROM t1,t2 WHERE (b=? OR a=?) AND (c=? OR d=?)
+} {
+  CREATE INDEX t1_idx_00000062 ON t1(b); -- stat1: 100 20 
+  CREATE INDEX t1_idx_00000061 ON t1(a); -- stat1: 100 50 
+  CREATE INDEX t2_idx_00000063 ON t2(c); -- stat1: 100 20 
+  CREATE INDEX t2_idx_00000064 ON t2(d); -- stat1: 100 5
+}
+
+do_candidates_test 3.2 {
+  SELECT * FROM t1,t2 WHERE a=? AND b=? AND c=? AND d=?
+} {
+  CREATE INDEX t1_idx_000123a7 ON t1(a, b); -- stat1: 100 50 17
+  CREATE INDEX t2_idx_0001295b ON t2(c, d); -- stat1: 100 20 5
+}
+
+do_execsql_test 3.2 {
+  CREATE INDEX t1_idx_00000061 ON t1(a); -- stat1: 100 50 
+  CREATE INDEX t1_idx_00000062 ON t1(b); -- stat1: 100 20 
+  CREATE INDEX t1_idx_000123a7 ON t1(a, b); -- stat1: 100 50 16
+
+  CREATE INDEX t2_idx_00000063 ON t2(c); -- stat1: 100 20 
+  CREATE INDEX t2_idx_00000064 ON t2(d); -- stat1: 100 5
+  CREATE INDEX t2_idx_0001295b ON t2(c, d); -- stat1: 100 20 5
+
+  ANALYZE;
+  SELECT * FROM sqlite_stat1 ORDER BY 1, 2;
+} {
+  t1 t1_idx_00000061 {100 50} 
+  t1 t1_idx_00000062 {100 20}
+  t1 t1_idx_000123a7 {100 50 17}
+  t2 t2_idx_00000063 {100 20} 
+  t2 t2_idx_00000064 {100 5} 
+  t2 t2_idx_0001295b {100 20 5}
+}
+
+
 finish_test
 
diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index 093e4438f5..bba5190d5b 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -114,6 +114,7 @@ typedef struct IdxHash IdxHash;
 struct IdxHashEntry {
   char *zKey;                     /* nul-terminated key */
   char *zVal;                     /* nul-terminated value string */
+  char *zVal2;                    /* nul-terminated value string 2 */
   IdxHashEntry *pHashNext;        /* Next entry in same hash bucket */
   IdxHashEntry *pNext;            /* Next entry in hash */
 };
@@ -175,6 +176,7 @@ static void idxHashClear(IdxHash *pHash){
     IdxHashEntry *pNext;
     for(pEntry=pHash->aHash[i]; pEntry; pEntry=pNext){
       pNext = pEntry->pHashNext;
+      sqlite3_free(pEntry->zVal2);
       sqlite3_free(pEntry);
     }
   }
@@ -232,13 +234,7 @@ static int idxHashAdd(
   return 0;
 }
 
-/*
-** If the hash table contains an entry with a key equal to the string
-** passed as the final two arguments to this function, return a pointer
-** to the payload string. Otherwise, if zKey/nKey is not present in the
-** hash table, return NULL.
-*/
-static const char *idxHashSearch(IdxHash *pHash, const char *zKey, int nKey){
+static IdxHashEntry *idxHashFind(IdxHash *pHash, const char *zKey, int nKey){
   int iHash;
   IdxHashEntry *pEntry;
   if( nKey<0 ) nKey = strlen(zKey);
@@ -246,12 +242,24 @@ static const char *idxHashSearch(IdxHash *pHash, const char *zKey, int nKey){
   assert( iHash>=0 );
   for(pEntry=pHash->aHash[iHash]; pEntry; pEntry=pEntry->pHashNext){
     if( strlen(pEntry->zKey)==nKey && 0==memcmp(pEntry->zKey, zKey, nKey) ){
-      return pEntry->zVal;
+      return pEntry;
     }
   }
   return 0;
 }
 
+/*
+** If the hash table contains an entry with a key equal to the string
+** passed as the final two arguments to this function, return a pointer
+** to the payload string. Otherwise, if zKey/nKey is not present in the
+** hash table, return NULL.
+*/
+static const char *idxHashSearch(IdxHash *pHash, const char *zKey, int nKey){
+  IdxHashEntry *pEntry = idxHashFind(pHash, zKey, nKey);
+  if( pEntry ) return pEntry->zVal;
+  return 0;
+}
+
 /*
 ** Allocate and return a new IdxConstraint object. Set the IdxConstraint.zColl
 ** variable to point to a copy of nul-terminated string zColl.
@@ -1227,6 +1235,277 @@ static int idxCreateVtabSchema(sqlite3expert *p, char **pzErrmsg){
   return rc;
 }
 
+struct IdxRemCtx {
+  int nSlot;
+  struct IdxRemSlot {
+    int eType;                    /* SQLITE_NULL, INTEGER, REAL, TEXT, BLOB */
+    i64 iVal;                     /* SQLITE_INTEGER value */
+    double rVal;                  /* SQLITE_FLOAT value */
+    int nByte;                    /* Bytes of space allocated at z */
+    int n;                        /* Size of buffer z */
+    char *z;                      /* SQLITE_TEXT/BLOB value */
+  } aSlot[1];
+};
+
+/*
+** Implementation of scalar function rem().
+*/
+static void idxRemFunc(
+  sqlite3_context *pCtx,
+  int argc,
+  sqlite3_value **argv
+){
+  struct IdxRemCtx *p = (struct IdxRemCtx*)sqlite3_user_data(pCtx);
+  struct IdxRemSlot *pSlot;
+  int iSlot;
+  assert( argc==2 );
+
+  iSlot = sqlite3_value_int(argv[0]);
+  assert( iSlot<=p->nSlot );
+  pSlot = &p->aSlot[iSlot];
+
+  switch( pSlot->eType ){
+    case SQLITE_NULL:
+      /* no-op */
+      break;
+
+    case SQLITE_INTEGER:
+      sqlite3_result_int64(pCtx, pSlot->iVal);
+      break;
+
+    case SQLITE_FLOAT:
+      sqlite3_result_double(pCtx, pSlot->rVal);
+      break;
+
+    case SQLITE_BLOB:
+      sqlite3_result_blob(pCtx, pSlot->z, pSlot->n, SQLITE_TRANSIENT);
+      break;
+
+    case SQLITE_TEXT:
+      sqlite3_result_text(pCtx, pSlot->z, pSlot->n, SQLITE_TRANSIENT);
+      break;
+  }
+
+  pSlot->eType = sqlite3_value_type(argv[1]);
+  switch( pSlot->eType ){
+    case SQLITE_NULL:
+      /* no-op */
+      break;
+
+    case SQLITE_INTEGER:
+      pSlot->iVal = sqlite3_value_int64(argv[1]);
+      break;
+
+    case SQLITE_FLOAT:
+      pSlot->rVal = sqlite3_value_double(argv[1]);
+      break;
+
+    case SQLITE_BLOB:
+    case SQLITE_TEXT: {
+      int nByte = sqlite3_value_bytes(argv[1]);
+      if( nByte>pSlot->nByte ){
+        char *zNew = (char*)sqlite3_realloc(pSlot->z, nByte*2);
+        if( zNew==0 ){
+          sqlite3_result_error_nomem(pCtx);
+          return;
+        }
+        pSlot->nByte = nByte*2;
+        pSlot->z = zNew;
+      }
+      pSlot->n = nByte;
+      if( pSlot->eType==SQLITE_BLOB ){
+        memcpy(pSlot->z, sqlite3_value_blob(argv[1]), nByte);
+      }else{
+        memcpy(pSlot->z, sqlite3_value_text(argv[1]), nByte);
+      }
+      break;
+    }
+  }
+}
+
+static int idxLargestIndex(sqlite3 *db, int *pnMax, char **pzErr){
+  int rc = SQLITE_OK;
+  const char *zMax = 
+    "SELECT max(i.seqno) FROM "
+    "  sqlite_master AS s, "
+    "  pragma_index_list(s.name) AS l, "
+    "  pragma_index_info(l.name) AS i "
+    "WHERE s.type = 'table'";
+  sqlite3_stmt *pMax = 0;
+
+  *pnMax = 0;
+  rc = idxPrepareStmt(db, &pMax, pzErr, zMax);
+  if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pMax) ){
+    *pnMax = sqlite3_column_int(pMax, 0) + 1;
+  }
+  idxFinalize(&rc, pMax);
+
+  return rc;
+}
+
+static int idxPopulateOneStat1(
+  sqlite3expert *p,
+  sqlite3_stmt *pIndexXInfo,
+  sqlite3_stmt *pWriteStat,
+  const char *zTab,
+  const char *zIdx,
+  char **pzErr
+){
+  char *zCols = 0;
+  char *zOrder = 0;
+  char *zQuery = 0;
+  int nCol = 0;
+  int i;
+  sqlite3_stmt *pQuery = 0;
+  int *aStat = 0;
+  int rc = SQLITE_OK;
+
+  /* Formulate the query text */
+  sqlite3_bind_text(pIndexXInfo, 1, zIdx, -1, SQLITE_STATIC);
+  while( SQLITE_OK==rc && SQLITE_ROW==sqlite3_step(pIndexXInfo) ){
+    const char *zComma = zCols==0 ? "" : ", ";
+    const char *zName = (const char*)sqlite3_column_text(pIndexXInfo, 0);
+    const char *zColl = (const char*)sqlite3_column_text(pIndexXInfo, 1);
+    zCols = idxAppendText(&rc, zCols, 
+        "%sx.%Q IS rem(%d, x.%Q) COLLATE %s", zComma, zName, nCol, zName, zColl
+    );
+    zOrder = idxAppendText(&rc, zOrder, "%s%d", zComma, ++nCol);
+  }
+  if( rc==SQLITE_OK ){
+    zQuery = sqlite3_mprintf(
+        "SELECT %s FROM %Q x ORDER BY %s", zCols, zTab, zOrder
+    );
+  }
+  sqlite3_free(zCols);
+  sqlite3_free(zOrder);
+
+  /* Formulate the query text */
+  if( rc==SQLITE_OK ){
+    rc = idxPrepareStmt(p->db, &pQuery, pzErr, zQuery);
+  }
+  sqlite3_free(zQuery);
+
+  if( rc==SQLITE_OK ){
+    aStat = (int*)idxMalloc(&rc, sizeof(int)*(nCol+1));
+  }
+  if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pQuery) ){
+    IdxHashEntry *pEntry;
+    char *zStat = 0;
+    for(i=0; i<=nCol; i++) aStat[i] = 1;
+    while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pQuery) ){
+      aStat[0]++;
+      for(i=0; i<nCol; i++){
+        if( sqlite3_column_int(pQuery, i)==0 ) break;
+      }
+      for(/*no-op*/; i<nCol; i++){
+        aStat[i+1]++;
+      }
+    }
+
+    if( rc==SQLITE_OK ){
+      int s0 = aStat[0];
+      zStat = sqlite3_mprintf("%d", s0);
+      if( zStat==0 ) rc = SQLITE_NOMEM;
+      for(i=1; rc==SQLITE_OK && i<=nCol; i++){
+        zStat = idxAppendText(&rc, zStat, " %d", (s0+aStat[i]/2) / aStat[i]);
+      }
+    }
+
+    if( rc==SQLITE_OK ){
+      sqlite3_bind_text(pWriteStat, 1, zTab, -1, SQLITE_STATIC);
+      sqlite3_bind_text(pWriteStat, 2, zIdx, -1, SQLITE_STATIC);
+      sqlite3_bind_text(pWriteStat, 3, zStat, -1, SQLITE_STATIC);
+      sqlite3_step(pWriteStat);
+      rc = sqlite3_reset(pWriteStat);
+    }
+
+    pEntry = idxHashFind(&p->hIdx, zIdx, strlen(zIdx));
+    if( pEntry ){
+      assert( pEntry->zVal2==0 );
+      pEntry->zVal2 = zStat;
+    }else{
+      sqlite3_free(zStat);
+    }
+  }
+  sqlite3_free(aStat);
+  idxFinalize(&rc, pQuery);
+
+  return rc;
+}
+
+/*
+** This function is called as part of sqlite3_expert_analyze(). Candidate
+** indexes have already been created in database sqlite3expert.dbm, this
+** function populates sqlite_stat1 table in the same database.
+**
+** The stat1 data is generated by querying the 
+*/
+static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
+  int rc = SQLITE_OK;
+  int nMax =0;
+  struct IdxRemCtx *pCtx = 0;
+  int i;
+  sqlite3_stmt *pAllIndex = 0;
+  sqlite3_stmt *pIndexXInfo = 0;
+  sqlite3_stmt *pWrite = 0;
+
+  const char *zAllIndex = 
+    "SELECT s.name, l.name FROM "
+    "  sqlite_master AS s, "
+    "  pragma_index_list(s.name) AS l "
+    "WHERE s.type = 'table'";
+  const char *zIndexXInfo = 
+    "SELECT name, coll FROM pragma_index_xinfo(?) WHERE key";
+  const char *zWrite = "INSERT INTO sqlite_stat1 VALUES(?, ?, ?)";
+
+  rc = idxLargestIndex(p->dbm, &nMax, pzErr);
+  if( nMax<=0 || rc!=SQLITE_OK ) return rc;
+
+  rc = sqlite3_exec(p->dbm, "ANALYZE; PRAGMA writable_schema=1", 0, 0, 0);
+
+  if( rc==SQLITE_OK ){
+    int nByte = sizeof(struct IdxRemCtx) + (sizeof(struct IdxRemSlot) * nMax);
+    pCtx = (struct IdxRemCtx*)idxMalloc(&rc, nByte);
+  }
+
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_create_function(
+        p->db, "rem", 2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0
+    );
+  }
+
+  if( rc==SQLITE_OK ){
+    pCtx->nSlot = nMax+1;
+    rc = idxPrepareStmt(p->dbm, &pAllIndex, pzErr, zAllIndex);
+  }
+  if( rc==SQLITE_OK ){
+    rc = idxPrepareStmt(p->dbm, &pIndexXInfo, pzErr, zIndexXInfo);
+  }
+  if( rc==SQLITE_OK ){
+    rc = idxPrepareStmt(p->dbm, &pWrite, pzErr, zWrite);
+  }
+
+  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pAllIndex) ){
+    const char *zTab = (const char*)sqlite3_column_text(pAllIndex, 0);
+    const char *zIdx = (const char*)sqlite3_column_text(pAllIndex, 1);
+    rc = idxPopulateOneStat1(p, pIndexXInfo, pWrite, zTab, zIdx, pzErr);
+  }
+
+  idxFinalize(&rc, pAllIndex);
+  idxFinalize(&rc, pIndexXInfo);
+  idxFinalize(&rc, pWrite);
+
+  for(i=0; i<pCtx->nSlot; i++){
+    sqlite3_free(pCtx->aSlot[i].z);
+  }
+  sqlite3_free(pCtx);
+
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_exec(p->dbm, "ANALYZE sqlite_master", 0, 0, 0);
+  }
+  return rc;
+}
+
 /*
 ** Allocate a new sqlite3expert object.
 */
@@ -1338,6 +1617,7 @@ int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr){
   int rc;
   IdxHashEntry *pEntry;
 
+  /* Do trigger processing to collect any extra IdxScan structures */
   rc = idxProcessTriggers(p, pzErr);
 
   /* Create candidate indexes within the in-memory database file */
@@ -1345,9 +1625,17 @@ int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr){
     rc = idxCreateCandidates(p, pzErr);
   }
 
+  /* Generate the stat1 data */
+  if( rc==SQLITE_OK ){
+    rc = idxPopulateStat1(p, pzErr);
+  }
+
   /* Formulate the EXPERT_REPORT_CANDIDATES text */
   for(pEntry=p->hIdx.pFirst; pEntry; pEntry=pEntry->pNext){
-    p->zCandidates = idxAppendText(&rc, p->zCandidates, "%s;\n", pEntry->zVal);
+    p->zCandidates = idxAppendText(&rc, p->zCandidates, 
+        "%s;%s%s\n", pEntry->zVal, 
+        pEntry->zVal2 ? " -- stat1: " : "", pEntry->zVal2
+    );
   }
 
   /* Figure out which of the candidate indexes are preferred by the query
diff --git a/ext/expert/test_expert.c b/ext/expert/test_expert.c
index a37887e587..587f95c91b 100644
--- a/ext/expert/test_expert.c
+++ b/ext/expert/test_expert.c
@@ -106,7 +106,7 @@ static int SQLITE_TCLAPI testExpertCmd(
 
     case 3: {      /* report */
       const char *aEnum[] = {
-        "sql", "indexes", "plan", 0
+        "sql", "indexes", "plan", "candidates", 0
       };
       int iEnum;
       int iStmt;
@@ -121,6 +121,7 @@ static int SQLITE_TCLAPI testExpertCmd(
       assert( EXPERT_REPORT_SQL==1 );
       assert( EXPERT_REPORT_INDEXES==2 );
       assert( EXPERT_REPORT_PLAN==3 );
+      assert( EXPERT_REPORT_CANDIDATES==4 );
       zReport = sqlite3_expert_report(pExpert, iStmt, 1+iEnum);
       Tcl_SetObjResult(interp, Tcl_NewStringObj(zReport, -1));
       break;
diff --git a/manifest b/manifest
index cde195ed9d..5fa07a1f45 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\ssqlite3_expert\shandling\sof\striggers\son\sviews.
-D 2017-04-18T09:04:48.670
+C Have\ssqlite3_expert_analyze()\spopulate\sthe\ssqlite_stat1\stable\sbefore\srunning\nqueries\sthrough\sthe\splanner\sfor\sthe\ssecond\stime.
+D 2017-04-18T20:10:16.786
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6
@@ -42,10 +42,10 @@ F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md 9f15075ec5ad772808eff55ef044c31140fd1146aa0a3c47eafd155e71851b01
 F ext/expert/expert.c 22d2dd096d479049bc332506fc8c0294bf53b7ebfe60af99635d8c87839bb40b
-F ext/expert/expert1.test 6a50a1538dc9e4ff360fb117298aa3b085beed030cbe15dd36803da1a9f70702
-F ext/expert/sqlite3expert.c 8befe20906cf8f4928c754763cbf0a81f42fc1ee242fe8e5869c3bb4adba7300
+F ext/expert/expert1.test 1033e43071b69dc2f4e88fbf03fc7f18846c9865cac14f28c80f581437f09acb
+F ext/expert/sqlite3expert.c 713388c6c440c6759a1e0898c7936a014dc9791237e62780412229e4a79b0035
 F ext/expert/sqlite3expert.h b1c9eedeb647fd734c4206ae6851635284cfbfa5fb688eff74c3265c9f949b4d
-F ext/expert/test_expert.c bad0611732d07180d586bd589cbb7713dc3ab0338c52bff29680eb2007678c05
+F ext/expert/test_expert.c b01a5115f9444a9b416582c985138f5dfdb279848ce8b7452be383530be27f01
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
 F ext/fts1/ft_hash.c 3927bd880e65329bdc6f506555b228b28924921b
 F ext/fts1/ft_hash.h 06df7bba40dadd19597aa400a875dbc2fed705ea
@@ -1579,7 +1579,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 159e8022a9d6701532b8b60e0c41154bc434c1bbdb107c8c97a78fb1140fa745
-R 52d2320865ff7653f66201ecd375f3d6
+P ff4976da667872614331d88e68fb67d347874f164a1c7950dd738c7c2320b954
+R d621eb0093377628a3811751fb5daa84
 U dan
-Z e02fbf739534a0fe797908906bbc0c24
+Z 02b917b4563ed84ab0581f0a345c83b3
diff --git a/manifest.uuid b/manifest.uuid
index 6969132da7..68ccc22188 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ff4976da667872614331d88e68fb67d347874f164a1c7950dd738c7c2320b954
\ No newline at end of file
+a157fcfde5afc27ae38e7cf4669fcc8e60e23d9d301ffe2e541dd69f895b493b
\ No newline at end of file

From e53b4f977484247b9b893d191252e6ce5bfd8e87 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 20 Apr 2017 09:54:04 +0000
Subject: [PATCH 36/68] Add an option to generate stat1 data based on a subset
 of the user database table contents to sqlite3_expert.

FossilOrigin-Name: c69c3e21db6e141f7e24226c6432f2ed31fe5f177bd23781915871f8600ee56a
---
 ext/expert/expert.c        |  10 ++-
 ext/expert/sqlite3expert.c | 133 ++++++++++++++++++++++++++++++++++---
 ext/expert/sqlite3expert.h |  33 +++++++++
 manifest                   |  16 ++---
 manifest.uuid              |   2 +-
 5 files changed, 175 insertions(+), 19 deletions(-)

diff --git a/ext/expert/expert.c b/ext/expert/expert.c
index 3aafde1d3f..baa7be940f 100644
--- a/ext/expert/expert.c
+++ b/ext/expert/expert.c
@@ -36,6 +36,7 @@ static void usage(char **argv){
   fprintf(stderr, "  -sql SQL   (analyze SQL statements passed as argument)\n");
   fprintf(stderr, "  -file FILE (read SQL statements from file FILE)\n");
   fprintf(stderr, "  -verbose LEVEL (integer verbosity level. default 1)\n");
+  fprintf(stderr, "  -sample PERCENT (percent of db to sample. default 100)\n");
   exit(-1);
 }
 
@@ -97,11 +98,18 @@ int main(int argc, char **argv){
         rc = readSqlFromFile(p, argv[i], &zErr);
       }
 
-      else if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-sql", nArg) ){
+      else if( nArg>=3 && 0==sqlite3_strnicmp(zArg, "-sql", nArg) ){
         if( ++i==(argc-1) ) option_requires_argument("-sql");
         rc = sqlite3_expert_sql(p, argv[i], &zErr);
       }
 
+      else if( nArg>=3 && 0==sqlite3_strnicmp(zArg, "-sample", nArg) ){
+        int iSample;
+        if( ++i==(argc-1) ) option_requires_argument("-sample");
+        iSample = option_integer_arg(argv[i]);
+        sqlite3_expert_config(p, EXPERT_CONFIG_SAMPLE, iSample);
+      }
+
       else if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-verbose", nArg) ){
         if( ++i==(argc-1) ) option_requires_argument("-verbose");
         iVerbose = option_integer_arg(argv[i]);
diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index bba5190d5b..03963e1f7e 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -28,6 +28,9 @@ typedef struct IdxStatement IdxStatement;
 typedef struct IdxTable IdxTable;
 typedef struct IdxWrite IdxWrite;
 
+#define UNIQUE_TABLE_NAME "t592690916721053953805701627921227776"
+
+
 /*
 ** A single constraint. Equivalent to either "col = ?" or "col < ?" (or
 ** any other type of single-ended range constraint on a column).
@@ -127,6 +130,7 @@ struct IdxHash {
 ** sqlite3expert object.
 */
 struct sqlite3expert {
+  int iSample;                    /* Percentage of tables to sample for stat1 */
   sqlite3 *db;                    /* User database */
   sqlite3 *dbm;                   /* In-memory db for this analysis */
   sqlite3 *dbv;                   /* Vtab schema for this analysis */
@@ -1080,8 +1084,8 @@ static int idxProcessOneTrigger(
   IdxWrite *pWrite, 
   char **pzErr
 ){
-  static const char *zInt = "t592690916721053953805701627921227776";
-  static const char *zDrop = "DROP TABLE t592690916721053953805701627921227776";
+  static const char *zInt = UNIQUE_TABLE_NAME;
+  static const char *zDrop = "DROP TABLE " UNIQUE_TABLE_NAME;
   IdxTable *pTab = pWrite->pTab;
   const char *zTab = pTab->zName;
   const char *zSql = 
@@ -1235,6 +1239,38 @@ static int idxCreateVtabSchema(sqlite3expert *p, char **pzErrmsg){
   return rc;
 }
 
+struct IdxSampleCtx {
+  int iTarget;
+  double target;                  /* Target nRet/nRow value */
+  double nRow;                    /* Number of rows seen */
+  double nRet;                    /* Number of rows returned */
+};
+
+static void idxSampleFunc(
+  sqlite3_context *pCtx,
+  int argc,
+  sqlite3_value **argv
+){
+  struct IdxSampleCtx *p = (struct IdxSampleCtx*)sqlite3_user_data(pCtx);
+  int bRet;
+
+  assert( argc==0 );
+  if( p->nRow==0.0 ){
+    bRet = 1;
+  }else{
+    bRet = (p->nRet / p->nRow) <= p->target;
+    if( bRet==0 ){
+      unsigned short rnd;
+      sqlite3_randomness(2, (void*)&rnd);
+      bRet = ((int)rnd % 100) <= p->iTarget;
+    }
+  }
+
+  sqlite3_result_int(pCtx, bRet);
+  p->nRow += 1.0;
+  p->nRet += (double)bRet;
+}
+
 struct IdxRemCtx {
   int nSlot;
   struct IdxRemSlot {
@@ -1360,6 +1396,8 @@ static int idxPopulateOneStat1(
   int *aStat = 0;
   int rc = SQLITE_OK;
 
+  assert( p->iSample>0 );
+
   /* Formulate the query text */
   sqlite3_bind_text(pIndexXInfo, 1, zIdx, -1, SQLITE_STATIC);
   while( SQLITE_OK==rc && SQLITE_ROW==sqlite3_step(pIndexXInfo) ){
@@ -1372,9 +1410,15 @@ static int idxPopulateOneStat1(
     zOrder = idxAppendText(&rc, zOrder, "%s%d", zComma, ++nCol);
   }
   if( rc==SQLITE_OK ){
-    zQuery = sqlite3_mprintf(
-        "SELECT %s FROM %Q x ORDER BY %s", zCols, zTab, zOrder
-    );
+    if( p->iSample==100 ){
+      zQuery = sqlite3_mprintf(
+          "SELECT %s FROM %Q x ORDER BY %s", zCols, zTab, zOrder
+      );
+    }else{
+      zQuery = sqlite3_mprintf(
+          "SELECT %s FROM temp."UNIQUE_TABLE_NAME" x ORDER BY %s", zCols, zOrder
+      );
+    }
   }
   sqlite3_free(zCols);
   sqlite3_free(zOrder);
@@ -1433,6 +1477,25 @@ static int idxPopulateOneStat1(
   return rc;
 }
 
+static int idxBuildSampleTable(sqlite3expert *p, const char *zTab){
+  int rc;
+  char *zSql;
+
+  rc = sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0);
+  if( rc!=SQLITE_OK ) return rc;
+
+  zSql = sqlite3_mprintf(
+      "CREATE TABLE temp." UNIQUE_TABLE_NAME 
+      "  AS SELECT * FROM %Q WHERE sample()"
+      , zTab
+  );
+  if( zSql==0 ) return SQLITE_NOMEM;
+  rc = sqlite3_exec(p->db, zSql, 0, 0, 0);
+  sqlite3_free(zSql);
+
+  return rc;
+}
+
 /*
 ** This function is called as part of sqlite3_expert_analyze(). Candidate
 ** indexes have already been created in database sqlite3expert.dbm, this
@@ -1444,13 +1507,15 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
   int rc = SQLITE_OK;
   int nMax =0;
   struct IdxRemCtx *pCtx = 0;
+  struct IdxSampleCtx samplectx; 
   int i;
+  i64 iPrev = -100000;
   sqlite3_stmt *pAllIndex = 0;
   sqlite3_stmt *pIndexXInfo = 0;
   sqlite3_stmt *pWrite = 0;
 
-  const char *zAllIndex = 
-    "SELECT s.name, l.name FROM "
+  const char *zAllIndex =
+    "SELECT s.rowid, s.name, l.name FROM "
     "  sqlite_master AS s, "
     "  pragma_index_list(s.name) AS l "
     "WHERE s.type = 'table'";
@@ -1458,6 +1523,9 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
     "SELECT name, coll FROM pragma_index_xinfo(?) WHERE key";
   const char *zWrite = "INSERT INTO sqlite_stat1 VALUES(?, ?, ?)";
 
+  /* If iSample==0, no sqlite_stat1 data is required. */
+  if( p->iSample==0 ) return SQLITE_OK;
+
   rc = idxLargestIndex(p->dbm, &nMax, pzErr);
   if( nMax<=0 || rc!=SQLITE_OK ) return rc;
 
@@ -1473,6 +1541,11 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
         p->db, "rem", 2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0
     );
   }
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_create_function(
+        p->db, "sample", 0, SQLITE_UTF8, (void*)&samplectx, idxSampleFunc, 0, 0
+    );
+  }
 
   if( rc==SQLITE_OK ){
     pCtx->nSlot = nMax+1;
@@ -1486,9 +1559,24 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
   }
 
   while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pAllIndex) ){
-    const char *zTab = (const char*)sqlite3_column_text(pAllIndex, 0);
-    const char *zIdx = (const char*)sqlite3_column_text(pAllIndex, 1);
+    i64 iRowid = sqlite3_column_int64(pAllIndex, 0);
+    const char *zTab = (const char*)sqlite3_column_text(pAllIndex, 1);
+    const char *zIdx = (const char*)sqlite3_column_text(pAllIndex, 2);
+    if( p->iSample<100 && iPrev!=iRowid ){
+      samplectx.target = (double)p->iSample / 100.0;
+      samplectx.iTarget = p->iSample;
+      samplectx.nRow = 0.0;
+      samplectx.nRet = 0.0;
+      rc = idxBuildSampleTable(p, zTab);
+      if( rc!=SQLITE_OK ) break;
+    }
     rc = idxPopulateOneStat1(p, pIndexXInfo, pWrite, zTab, zIdx, pzErr);
+    iPrev = iRowid;
+  }
+  if( p->iSample<100 ){
+    rc = sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp." UNIQUE_TABLE_NAME,
+        0,0,0
+    );
   }
 
   idxFinalize(&rc, pAllIndex);
@@ -1503,6 +1591,8 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
   if( rc==SQLITE_OK ){
     rc = sqlite3_exec(p->dbm, "ANALYZE sqlite_master", 0, 0, 0);
   }
+
+  sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0);
   return rc;
 }
 
@@ -1523,6 +1613,7 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){
   */
   if( rc==SQLITE_OK ){
     pNew->db = db;
+    pNew->iSample = 100;
     rc = sqlite3_open(":memory:", &pNew->dbv);
   }
   if( rc==SQLITE_OK ){
@@ -1565,6 +1656,30 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){
   return pNew;
 }
 
+/*
+** Configure an sqlite3expert object.
+*/
+int sqlite3_expert_config(sqlite3expert *p, int op, ...){
+  int rc = SQLITE_OK;
+  va_list ap;
+  va_start(ap, op);
+  switch( op ){
+    case EXPERT_CONFIG_SAMPLE: {
+      int iVal = va_arg(ap, int);
+      if( iVal<0 ) iVal = 0;
+      if( iVal>100 ) iVal = 100;
+      p->iSample = iVal;
+      break;
+    }
+    default:
+      rc = SQLITE_NOTFOUND;
+      break;
+  }
+
+  va_end(ap);
+  return rc;
+}
+
 /*
 ** Add an SQL statement to the analysis.
 */
diff --git a/ext/expert/sqlite3expert.h b/ext/expert/sqlite3expert.h
index 455f41d991..39135dc274 100644
--- a/ext/expert/sqlite3expert.h
+++ b/ext/expert/sqlite3expert.h
@@ -27,6 +27,38 @@ typedef struct sqlite3expert sqlite3expert;
 */
 sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErr);
 
+/*
+** Configure an sqlite3expert object.
+**
+** EXPERT_CONFIG_SAMPLE:
+**   By default, sqlite3_expert_analyze() generates sqlite_stat1 data for
+**   each candidate index. This involves scanning and sorting the entire
+**   contents of each user database table once for each candidate index
+**   associated with the table. For large databases, this can be 
+**   prohibitively slow. This option allows the sqlite3expert object to
+**   be configured so that sqlite_stat1 data is instead generated based on a
+**   subset of each table, or so that no sqlite_stat1 data is used at all.
+**
+**   A single integer argument is passed to this option. If the value is less
+**   than or equal to zero, then no sqlite_stat1 data is generated or used by
+**   the analysis - indexes are recommended based on the database schema only.
+**   Or, if the value is 100 or greater, complete sqlite_stat1 data is
+**   generated for each candidate index (this is the default). Finally, if the
+**   value falls between 0 and 100, then it represents the percentage of user
+**   table rows that should be considered when generating sqlite_stat1 data.
+**
+**   Examples:
+**
+**     // Do not generate any sqlite_stat1 data
+**     sqlite3_expert_config(pExpert, EXPERT_CONFIG_SAMPLE, 0);
+**
+**     // Generate sqlite_stat1 data based on 10% of the rows in each table.
+**     sqlite3_expert_config(pExpert, EXPERT_CONFIG_SAMPLE, 10);
+*/
+int sqlite3_expert_config(sqlite3expert *p, int op, ...);
+
+#define EXPERT_CONFIG_SAMPLE 1    /* int */
+
 /*
 ** Specify zero or more SQL statements to be included in the analysis.
 **
@@ -54,6 +86,7 @@ int sqlite3_expert_sql(
   char **pzErr                    /* OUT: Error message (if any) */
 );
 
+
 /*
 ** This function is called after the sqlite3expert object has been configured
 ** with all SQL statements using sqlite3_expert_sql() to actually perform
diff --git a/manifest b/manifest
index 5fa07a1f45..5ae9923e9e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Have\ssqlite3_expert_analyze()\spopulate\sthe\ssqlite_stat1\stable\sbefore\srunning\nqueries\sthrough\sthe\splanner\sfor\sthe\ssecond\stime.
-D 2017-04-18T20:10:16.786
+C Add\san\soption\sto\sgenerate\sstat1\sdata\sbased\son\sa\ssubset\sof\sthe\suser\sdatabase\ntable\scontents\sto\ssqlite3_expert.
+D 2017-04-20T09:54:04.700
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6
@@ -41,10 +41,10 @@ F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md 9f15075ec5ad772808eff55ef044c31140fd1146aa0a3c47eafd155e71851b01
-F ext/expert/expert.c 22d2dd096d479049bc332506fc8c0294bf53b7ebfe60af99635d8c87839bb40b
+F ext/expert/expert.c 33842ef151d84c5f8000f9c7b938998c6b999eaef7ce1f4eeb0df8ffe6739496
 F ext/expert/expert1.test 1033e43071b69dc2f4e88fbf03fc7f18846c9865cac14f28c80f581437f09acb
-F ext/expert/sqlite3expert.c 713388c6c440c6759a1e0898c7936a014dc9791237e62780412229e4a79b0035
-F ext/expert/sqlite3expert.h b1c9eedeb647fd734c4206ae6851635284cfbfa5fb688eff74c3265c9f949b4d
+F ext/expert/sqlite3expert.c af3b336f83bcd2a586f6119d4040ac36ccf45162c48e3780ed63ab119fd04fe1
+F ext/expert/sqlite3expert.h af6354f8ee5c9e025024e63fec3bd640a802afcc3099a44d804752cf0791d811
 F ext/expert/test_expert.c b01a5115f9444a9b416582c985138f5dfdb279848ce8b7452be383530be27f01
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
 F ext/fts1/ft_hash.c 3927bd880e65329bdc6f506555b228b28924921b
@@ -1579,7 +1579,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ff4976da667872614331d88e68fb67d347874f164a1c7950dd738c7c2320b954
-R d621eb0093377628a3811751fb5daa84
+P a157fcfde5afc27ae38e7cf4669fcc8e60e23d9d301ffe2e541dd69f895b493b
+R 2a75643c384b85cff6c5d9d5615ac2a2
 U dan
-Z 02b917b4563ed84ab0581f0a345c83b3
+Z 7876a034c88841c26664b8a83eb3817f
diff --git a/manifest.uuid b/manifest.uuid
index 68ccc22188..d816bcc63e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a157fcfde5afc27ae38e7cf4669fcc8e60e23d9d301ffe2e541dd69f895b493b
\ No newline at end of file
+c69c3e21db6e141f7e24226c6432f2ed31fe5f177bd23781915871f8600ee56a
\ No newline at end of file

From 2ed99def5b67ab87c79255368bf939afa5fdb22b Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 20 Apr 2017 16:08:33 +0000
Subject: [PATCH 37/68] Avoid creating a temp table in the user database in the
 sqlite3_expert code. Trouble is, this makes sampling for stat1 data much
 slower.

FossilOrigin-Name: c62e358243d96cb38a7ce2aa679fc640b62bf46080eab4bd5fc2acf5997d6cd5
---
 ext/expert/sqlite3expert.c | 270 +++++++++++++++++++++++++++----------
 manifest                   |  15 ++-
 manifest.uuid              |   2 +-
 3 files changed, 210 insertions(+), 77 deletions(-)

diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index 03963e1f7e..d578973101 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -281,6 +281,59 @@ static IdxConstraint *idxNewConstraint(int *pRc, const char *zColl){
   return pNew;
 }
 
+/*
+** An error associated with database handle db has just occurred. Pass
+** the error message to callback function xOut.
+*/
+static void idxDatabaseError(
+  sqlite3 *db,                    /* Database handle */
+  char **pzErrmsg                 /* Write error here */
+){
+  *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
+}
+
+/*
+** Prepare an SQL statement.
+*/
+static int idxPrepareStmt(
+  sqlite3 *db,                    /* Database handle to compile against */
+  sqlite3_stmt **ppStmt,          /* OUT: Compiled SQL statement */
+  char **pzErrmsg,                /* OUT: sqlite3_malloc()ed error message */
+  const char *zSql                /* SQL statement to compile */
+){
+  int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0);
+  if( rc!=SQLITE_OK ){
+    *ppStmt = 0;
+    idxDatabaseError(db, pzErrmsg);
+  }
+  return rc;
+}
+
+/*
+** Prepare an SQL statement using the results of a printf() formatting.
+*/
+static int idxPrintfPrepareStmt(
+  sqlite3 *db,                    /* Database handle to compile against */
+  sqlite3_stmt **ppStmt,          /* OUT: Compiled SQL statement */
+  char **pzErrmsg,                /* OUT: sqlite3_malloc()ed error message */
+  const char *zFmt,               /* printf() format of SQL statement */
+  ...                             /* Trailing printf() arguments */
+){
+  va_list ap;
+  int rc;
+  char *zSql;
+  va_start(ap, zFmt);
+  zSql = sqlite3_vmprintf(zFmt, ap);
+  if( zSql==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    rc = idxPrepareStmt(db, ppStmt, pzErrmsg, zSql);
+    sqlite3_free(zSql);
+  }
+  va_end(ap);
+  return rc;
+}
+
 
 /*************************************************************************
 ** Beginning of virtual table implementation.
@@ -292,6 +345,17 @@ struct ExpertVtab {
   sqlite3expert *pExpert;
 };
 
+typedef struct ExpertCsr ExpertCsr;
+struct ExpertCsr {
+  sqlite3_vtab_cursor base;
+  sqlite3_stmt *pData;
+
+  int iTarget;                    /* Target as a percentage */
+  double target;                  /* Target nRet/nRow value */
+  double nRow;                    /* Rows seen */
+  double nRet;                    /* Rows returned */
+};
+
 static char *expertDequote(const char *zIn){
   int n = strlen(zIn);
   char *zRet = sqlite3_malloc(n);
@@ -389,7 +453,11 @@ static int expertBestIndex(sqlite3_vtab *pVtab, sqlite3_index_info *pIdxInfo){
     /* Add the constraints to the IdxScan object */
     for(i=0; i<pIdxInfo->nConstraint; i++){
       struct sqlite3_index_constraint *pCons = &pIdxInfo->aConstraint[i];
-      if( pCons->usable && pCons->iColumn>=0 && (pCons->op & opmask) ){
+      if( pCons->usable 
+       && pCons->iColumn>=0 
+       && p->pTab->aCol[pCons->iColumn].iPk==0
+       && (pCons->op & opmask) 
+      ){
         IdxConstraint *pNew;
         const char *zColl = sqlite3_vtab_collation(dbv, i);
         pNew = idxNewConstraint(&rc, zColl);
@@ -439,6 +507,122 @@ static int expertUpdate(
   return SQLITE_OK;
 }
 
+/* 
+** Virtual table module xOpen method.
+*/
+static int expertOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
+  int rc = SQLITE_OK;
+  ExpertCsr *pCsr;
+  pCsr = idxMalloc(&rc, sizeof(ExpertCsr));
+  *ppCursor = (sqlite3_vtab_cursor*)pCsr;
+  return rc;
+}
+
+/* 
+** Virtual table module xClose method.
+*/
+static int expertClose(sqlite3_vtab_cursor *cur){
+  ExpertCsr *pCsr = (ExpertCsr*)cur;
+  sqlite3_finalize(pCsr->pData);
+  sqlite3_free(pCsr);
+  return SQLITE_OK;
+}
+
+/*
+** Virtual table module xEof method.
+**
+** Return non-zero if the cursor does not currently point to a valid 
+** record (i.e if the scan has finished), or zero otherwise.
+*/
+static int expertEof(sqlite3_vtab_cursor *cur){
+  ExpertCsr *pCsr = (ExpertCsr*)cur;
+  return pCsr->pData==0;
+}
+
+/* 
+** Virtual table module xNext method.
+*/
+static int expertNext(sqlite3_vtab_cursor *cur){
+  ExpertCsr *pCsr = (ExpertCsr*)cur;
+  int rc = SQLITE_OK;
+  int bRet;
+  assert( pCsr->pData );
+
+  do {
+    rc = sqlite3_step(pCsr->pData);
+    if( rc!=SQLITE_ROW ){
+      rc = sqlite3_finalize(pCsr->pData);
+      pCsr->pData = 0;
+      bRet = 1;
+    }else{
+      rc = SQLITE_OK;
+      bRet = (pCsr->nRow==0.0 || pCsr->nRow/pCsr->nRet < pCsr->target);
+      if( bRet==0 ){
+        unsigned short rnd;
+        sqlite3_randomness(2, (void*)&rnd);
+        bRet = ((int)rnd % 100) <= pCsr->iTarget;
+      }
+    }
+    pCsr->nRow += 1.0;
+  }while( bRet==0 );
+
+  pCsr->nRet += 1.0;
+  return rc;
+}
+
+/* 
+** Virtual table module xRowid method.
+*/
+static int expertRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
+  *pRowid = 0;
+  return SQLITE_OK;
+}
+
+/* 
+** Virtual table module xColumn method.
+*/
+static int expertColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){
+  ExpertCsr *pCsr = (ExpertCsr*)cur;
+  sqlite3_value *pVal;
+  pVal = sqlite3_column_value(pCsr->pData, i);
+  if( pVal ){
+    sqlite3_result_value(ctx, pVal);
+  }
+  return SQLITE_OK;
+}
+
+/* 
+** Virtual table module xFilter method.
+*/
+static int expertFilter(
+  sqlite3_vtab_cursor *cur, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  ExpertCsr *pCsr = (ExpertCsr*)cur;
+  ExpertVtab *pVtab = (ExpertVtab*)(cur->pVtab);
+  sqlite3expert *pExpert = pVtab->pExpert;
+  int rc;
+
+  rc = sqlite3_finalize(pCsr->pData);
+  pCsr->pData = 0;
+  pCsr->nRow = 0.0;
+  pCsr->nRet = 0.0;
+  pCsr->iTarget = pExpert->iSample;
+  pCsr->target = (double)pExpert->iSample / 100.0;
+
+  if( rc==SQLITE_OK ){
+    rc = idxPrintfPrepareStmt(pExpert->db, &pCsr->pData, &pVtab->base.zErrMsg,
+        "SELECT * FROM main.%Q", pVtab->pTab->zName
+    );
+  }
+
+  if( rc==SQLITE_OK ){
+    rc = expertNext(cur);
+  }
+  return rc;
+}
+
 static int idxRegisterVtab(sqlite3expert *p){
   static sqlite3_module expertModule = {
     2,                            /* iVersion */
@@ -447,13 +631,13 @@ static int idxRegisterVtab(sqlite3expert *p){
     expertBestIndex,              /* xBestIndex - Determine search strategy */
     expertDisconnect,             /* xDisconnect - Disconnect from a table */
     expertDisconnect,             /* xDestroy - Drop a table */
-    0,                            /* xOpen - open a cursor */
-    0,                            /* xClose - close a cursor */
-    0,                            /* xFilter - configure scan constraints */
-    0,                            /* xNext - advance a cursor */
-    0,                            /* xEof */
-    0,                            /* xColumn - read data */
-    0,                            /* xRowid - read data */
+    expertOpen,                   /* xOpen - open a cursor */
+    expertClose,                  /* xClose - close a cursor */
+    expertFilter,                 /* xFilter - configure scan constraints */
+    expertNext,                   /* xNext - advance a cursor */
+    expertEof,                    /* xEof */
+    expertColumn,                 /* xColumn - read data */
+    expertRowid,                  /* xRowid - read data */
     expertUpdate,                 /* xUpdate - write data */
     0,                            /* xBegin - begin transaction */
     0,                            /* xSync - sync transaction */
@@ -471,60 +655,6 @@ static int idxRegisterVtab(sqlite3expert *p){
 /*
 ** End of virtual table implementation.
 *************************************************************************/
-
-/*
-** An error associated with database handle db has just occurred. Pass
-** the error message to callback function xOut.
-*/
-static void idxDatabaseError(
-  sqlite3 *db,                    /* Database handle */
-  char **pzErrmsg                 /* Write error here */
-){
-  *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
-}
-
-/*
-** Prepare an SQL statement.
-*/
-static int idxPrepareStmt(
-  sqlite3 *db,                    /* Database handle to compile against */
-  sqlite3_stmt **ppStmt,          /* OUT: Compiled SQL statement */
-  char **pzErrmsg,                /* OUT: sqlite3_malloc()ed error message */
-  const char *zSql                /* SQL statement to compile */
-){
-  int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0);
-  if( rc!=SQLITE_OK ){
-    *ppStmt = 0;
-    idxDatabaseError(db, pzErrmsg);
-  }
-  return rc;
-}
-
-/*
-** Prepare an SQL statement using the results of a printf() formatting.
-*/
-static int idxPrintfPrepareStmt(
-  sqlite3 *db,                    /* Database handle to compile against */
-  sqlite3_stmt **ppStmt,          /* OUT: Compiled SQL statement */
-  char **pzErrmsg,                /* OUT: sqlite3_malloc()ed error message */
-  const char *zFmt,               /* printf() format of SQL statement */
-  ...                             /* Trailing printf() arguments */
-){
-  va_list ap;
-  int rc;
-  char *zSql;
-  va_start(ap, zFmt);
-  zSql = sqlite3_vmprintf(zFmt, ap);
-  if( zSql==0 ){
-    rc = SQLITE_NOMEM;
-  }else{
-    rc = idxPrepareStmt(db, ppStmt, pzErrmsg, zSql);
-    sqlite3_free(zSql);
-  }
-  va_end(ap);
-  return rc;
-}
-
 /*
 ** Finalize SQL statement pStmt. If (*pRc) is SQLITE_OK when this function
 ** is called, set it to the return value of sqlite3_finalize() before
@@ -557,7 +687,7 @@ static int idxGetTableInfo(
   int nByte = sizeof(IdxTable) + nTab + 1;
   IdxTable *pNew = 0;
   int rc, rc2;
-  char *pCsr;
+  char *pCsr = 0;
 
   rc = idxPrintfPrepareStmt(db, &p1, pzErrmsg, "PRAGMA table_info=%Q", zTab);
   while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){
@@ -1425,7 +1555,8 @@ static int idxPopulateOneStat1(
 
   /* Formulate the query text */
   if( rc==SQLITE_OK ){
-    rc = idxPrepareStmt(p->db, &pQuery, pzErr, zQuery);
+    sqlite3 *dbrem = (p->iSample==100 ? p->db : p->dbv);
+    rc = idxPrepareStmt(dbrem, &pQuery, pzErr, zQuery);
   }
   sqlite3_free(zQuery);
 
@@ -1481,16 +1612,14 @@ static int idxBuildSampleTable(sqlite3expert *p, const char *zTab){
   int rc;
   char *zSql;
 
-  rc = sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0);
+  rc = sqlite3_exec(p->dbv,"DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0);
   if( rc!=SQLITE_OK ) return rc;
 
   zSql = sqlite3_mprintf(
-      "CREATE TABLE temp." UNIQUE_TABLE_NAME 
-      "  AS SELECT * FROM %Q WHERE sample()"
-      , zTab
+      "CREATE TABLE temp." UNIQUE_TABLE_NAME " AS SELECT * FROM %Q", zTab
   );
   if( zSql==0 ) return SQLITE_NOMEM;
-  rc = sqlite3_exec(p->db, zSql, 0, 0, 0);
+  rc = sqlite3_exec(p->dbv, zSql, 0, 0, 0);
   sqlite3_free(zSql);
 
   return rc;
@@ -1537,8 +1666,9 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
   }
 
   if( rc==SQLITE_OK ){
+    sqlite3 *dbrem = (p->iSample==100 ? p->db : p->dbv);
     rc = sqlite3_create_function(
-        p->db, "rem", 2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0
+        dbrem, "rem", 2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0
     );
   }
   if( rc==SQLITE_OK ){
diff --git a/manifest b/manifest
index 5ae9923e9e..86d64841b6 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\san\soption\sto\sgenerate\sstat1\sdata\sbased\son\sa\ssubset\sof\sthe\suser\sdatabase\ntable\scontents\sto\ssqlite3_expert.
-D 2017-04-20T09:54:04.700
+C Avoid\screating\sa\stemp\stable\sin\sthe\suser\sdatabase\sin\sthe\ssqlite3_expert\scode.\nTrouble\sis,\sthis\smakes\ssampling\sfor\sstat1\sdata\smuch\sslower.
+D 2017-04-20T16:08:33.333
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6
@@ -43,7 +43,7 @@ F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md 9f15075ec5ad772808eff55ef044c31140fd1146aa0a3c47eafd155e71851b01
 F ext/expert/expert.c 33842ef151d84c5f8000f9c7b938998c6b999eaef7ce1f4eeb0df8ffe6739496
 F ext/expert/expert1.test 1033e43071b69dc2f4e88fbf03fc7f18846c9865cac14f28c80f581437f09acb
-F ext/expert/sqlite3expert.c af3b336f83bcd2a586f6119d4040ac36ccf45162c48e3780ed63ab119fd04fe1
+F ext/expert/sqlite3expert.c 68acd26b0520408f0c5e76bd9ddaf70d776d397f8aa1bd6eda91f42f34c0a8cf
 F ext/expert/sqlite3expert.h af6354f8ee5c9e025024e63fec3bd640a802afcc3099a44d804752cf0791d811
 F ext/expert/test_expert.c b01a5115f9444a9b416582c985138f5dfdb279848ce8b7452be383530be27f01
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
@@ -1579,7 +1579,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a157fcfde5afc27ae38e7cf4669fcc8e60e23d9d301ffe2e541dd69f895b493b
-R 2a75643c384b85cff6c5d9d5615ac2a2
+P c69c3e21db6e141f7e24226c6432f2ed31fe5f177bd23781915871f8600ee56a
+R 02fc06fd99e34f4a0d95661da624b768
+T *branch * schemalint-failure
+T *sym-schemalint-failure *
+T -sym-schemalint *
 U dan
-Z 7876a034c88841c26664b8a83eb3817f
+Z 8a7b9cb83c65b8cfdb4d171254abdcee
diff --git a/manifest.uuid b/manifest.uuid
index d816bcc63e..1472955024 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c69c3e21db6e141f7e24226c6432f2ed31fe5f177bd23781915871f8600ee56a
\ No newline at end of file
+c62e358243d96cb38a7ce2aa679fc640b62bf46080eab4bd5fc2acf5997d6cd5
\ No newline at end of file

From 6ebd7ff4827d2902f1e5bec3eba7d9c7d34564e3 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 20 Apr 2017 16:18:43 +0000
Subject: [PATCH 38/68] Avoid adding INTEGER PRIMARY KEY columns to candidate
 indexes.

FossilOrigin-Name: 4577fea5cd9d91ea241e9be82797ca1a4447f536e1e4b78a4a569aeb52e78fcb
---
 ext/expert/sqlite3expert.c |  8 ++++++--
 manifest                   | 12 ++++++------
 manifest.uuid              |  2 +-
 3 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index 03963e1f7e..1966ab700f 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -389,7 +389,11 @@ static int expertBestIndex(sqlite3_vtab *pVtab, sqlite3_index_info *pIdxInfo){
     /* Add the constraints to the IdxScan object */
     for(i=0; i<pIdxInfo->nConstraint; i++){
       struct sqlite3_index_constraint *pCons = &pIdxInfo->aConstraint[i];
-      if( pCons->usable && pCons->iColumn>=0 && (pCons->op & opmask) ){
+      if( pCons->usable 
+       && pCons->iColumn>=0 
+       && p->pTab->aCol[pCons->iColumn].iPk==0
+       && (pCons->op & opmask) 
+      ){
         IdxConstraint *pNew;
         const char *zColl = sqlite3_vtab_collation(dbv, i);
         pNew = idxNewConstraint(&rc, zColl);
@@ -557,7 +561,7 @@ static int idxGetTableInfo(
   int nByte = sizeof(IdxTable) + nTab + 1;
   IdxTable *pNew = 0;
   int rc, rc2;
-  char *pCsr;
+  char *pCsr = 0;
 
   rc = idxPrintfPrepareStmt(db, &p1, pzErrmsg, "PRAGMA table_info=%Q", zTab);
   while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){
diff --git a/manifest b/manifest
index 5ae9923e9e..e63525e54c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\san\soption\sto\sgenerate\sstat1\sdata\sbased\son\sa\ssubset\sof\sthe\suser\sdatabase\ntable\scontents\sto\ssqlite3_expert.
-D 2017-04-20T09:54:04.700
+C Avoid\sadding\sINTEGER\sPRIMARY\sKEY\scolumns\sto\scandidate\sindexes.
+D 2017-04-20T16:18:43.967
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6
@@ -43,7 +43,7 @@ F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md 9f15075ec5ad772808eff55ef044c31140fd1146aa0a3c47eafd155e71851b01
 F ext/expert/expert.c 33842ef151d84c5f8000f9c7b938998c6b999eaef7ce1f4eeb0df8ffe6739496
 F ext/expert/expert1.test 1033e43071b69dc2f4e88fbf03fc7f18846c9865cac14f28c80f581437f09acb
-F ext/expert/sqlite3expert.c af3b336f83bcd2a586f6119d4040ac36ccf45162c48e3780ed63ab119fd04fe1
+F ext/expert/sqlite3expert.c 403b261dbea217f1b221dd4ab19774d1acfc4eba44c1ed35ab38060168f81b6d
 F ext/expert/sqlite3expert.h af6354f8ee5c9e025024e63fec3bd640a802afcc3099a44d804752cf0791d811
 F ext/expert/test_expert.c b01a5115f9444a9b416582c985138f5dfdb279848ce8b7452be383530be27f01
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
@@ -1579,7 +1579,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a157fcfde5afc27ae38e7cf4669fcc8e60e23d9d301ffe2e541dd69f895b493b
-R 2a75643c384b85cff6c5d9d5615ac2a2
+P c69c3e21db6e141f7e24226c6432f2ed31fe5f177bd23781915871f8600ee56a
+R 755161800d19d245e3f0050f91b54ff6
 U dan
-Z 7876a034c88841c26664b8a83eb3817f
+Z c9b10ac09e4b6010a00208af302face2
diff --git a/manifest.uuid b/manifest.uuid
index d816bcc63e..a34fd3239d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c69c3e21db6e141f7e24226c6432f2ed31fe5f177bd23781915871f8600ee56a
\ No newline at end of file
+4577fea5cd9d91ea241e9be82797ca1a4447f536e1e4b78a4a569aeb52e78fcb
\ No newline at end of file

From bc3f784cd2695c5d7c6cc4958088b93662a9597a Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 20 Apr 2017 16:43:32 +0000
Subject: [PATCH 39/68] Speed this branch up a bit by filtering before the
 virtual table layer when sampling user data.

FossilOrigin-Name: 8e57c31340dd9ffc457da63c5996fb1b573f8154f864ec2b52c15f399906ac8b
---
 ext/expert/sqlite3expert.c | 46 ++++++++++----------------------------
 manifest                   | 15 +++++--------
 manifest.uuid              |  2 +-
 3 files changed, 19 insertions(+), 44 deletions(-)

diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index d578973101..3ee74b5710 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -349,11 +349,6 @@ typedef struct ExpertCsr ExpertCsr;
 struct ExpertCsr {
   sqlite3_vtab_cursor base;
   sqlite3_stmt *pData;
-
-  int iTarget;                    /* Target as a percentage */
-  double target;                  /* Target nRet/nRow value */
-  double nRow;                    /* Rows seen */
-  double nRet;                    /* Rows returned */
 };
 
 static char *expertDequote(const char *zIn){
@@ -545,28 +540,16 @@ static int expertEof(sqlite3_vtab_cursor *cur){
 static int expertNext(sqlite3_vtab_cursor *cur){
   ExpertCsr *pCsr = (ExpertCsr*)cur;
   int rc = SQLITE_OK;
-  int bRet;
+
   assert( pCsr->pData );
+  rc = sqlite3_step(pCsr->pData);
+  if( rc!=SQLITE_ROW ){
+    rc = sqlite3_finalize(pCsr->pData);
+    pCsr->pData = 0;
+  }else{
+    rc = SQLITE_OK;
+  }
 
-  do {
-    rc = sqlite3_step(pCsr->pData);
-    if( rc!=SQLITE_ROW ){
-      rc = sqlite3_finalize(pCsr->pData);
-      pCsr->pData = 0;
-      bRet = 1;
-    }else{
-      rc = SQLITE_OK;
-      bRet = (pCsr->nRow==0.0 || pCsr->nRow/pCsr->nRet < pCsr->target);
-      if( bRet==0 ){
-        unsigned short rnd;
-        sqlite3_randomness(2, (void*)&rnd);
-        bRet = ((int)rnd % 100) <= pCsr->iTarget;
-      }
-    }
-    pCsr->nRow += 1.0;
-  }while( bRet==0 );
-
-  pCsr->nRet += 1.0;
   return rc;
 }
 
@@ -606,14 +589,9 @@ static int expertFilter(
 
   rc = sqlite3_finalize(pCsr->pData);
   pCsr->pData = 0;
-  pCsr->nRow = 0.0;
-  pCsr->nRet = 0.0;
-  pCsr->iTarget = pExpert->iSample;
-  pCsr->target = (double)pExpert->iSample / 100.0;
-
   if( rc==SQLITE_OK ){
     rc = idxPrintfPrepareStmt(pExpert->db, &pCsr->pData, &pVtab->base.zErrMsg,
-        "SELECT * FROM main.%Q", pVtab->pTab->zName
+        "SELECT * FROM main.%Q WHERE sample()", pVtab->pTab->zName
     );
   }
 
@@ -1703,9 +1681,9 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
     rc = idxPopulateOneStat1(p, pIndexXInfo, pWrite, zTab, zIdx, pzErr);
     iPrev = iRowid;
   }
-  if( p->iSample<100 ){
-    rc = sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp." UNIQUE_TABLE_NAME,
-        0,0,0
+  if( rc==SQLITE_OK && p->iSample<100 ){
+    rc = sqlite3_exec(p->dbv, 
+        "DROP TABLE IF EXISTS temp." UNIQUE_TABLE_NAME, 0,0,0
     );
   }
 
diff --git a/manifest b/manifest
index 86d64841b6..78e4264cc9 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Avoid\screating\sa\stemp\stable\sin\sthe\suser\sdatabase\sin\sthe\ssqlite3_expert\scode.\nTrouble\sis,\sthis\smakes\ssampling\sfor\sstat1\sdata\smuch\sslower.
-D 2017-04-20T16:08:33.333
+C Speed\sthis\sbranch\sup\sa\sbit\sby\sfiltering\sbefore\sthe\svirtual\stable\slayer\swhen\nsampling\suser\sdata.
+D 2017-04-20T16:43:32.927
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6
@@ -43,7 +43,7 @@ F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md 9f15075ec5ad772808eff55ef044c31140fd1146aa0a3c47eafd155e71851b01
 F ext/expert/expert.c 33842ef151d84c5f8000f9c7b938998c6b999eaef7ce1f4eeb0df8ffe6739496
 F ext/expert/expert1.test 1033e43071b69dc2f4e88fbf03fc7f18846c9865cac14f28c80f581437f09acb
-F ext/expert/sqlite3expert.c 68acd26b0520408f0c5e76bd9ddaf70d776d397f8aa1bd6eda91f42f34c0a8cf
+F ext/expert/sqlite3expert.c fde366d8c1b1970b2c18196ca2e64d01c2106bd9431c371a26e8d5b79f37f90b
 F ext/expert/sqlite3expert.h af6354f8ee5c9e025024e63fec3bd640a802afcc3099a44d804752cf0791d811
 F ext/expert/test_expert.c b01a5115f9444a9b416582c985138f5dfdb279848ce8b7452be383530be27f01
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
@@ -1579,10 +1579,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c69c3e21db6e141f7e24226c6432f2ed31fe5f177bd23781915871f8600ee56a
-R 02fc06fd99e34f4a0d95661da624b768
-T *branch * schemalint-failure
-T *sym-schemalint-failure *
-T -sym-schemalint *
+P c62e358243d96cb38a7ce2aa679fc640b62bf46080eab4bd5fc2acf5997d6cd5
+R 73307a63b82b2606f65f28fc989fd3ae
 U dan
-Z 8a7b9cb83c65b8cfdb4d171254abdcee
+Z b05e3391f67ce73794363163fdd05070
diff --git a/manifest.uuid b/manifest.uuid
index 1472955024..a17ccd88b9 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c62e358243d96cb38a7ce2aa679fc640b62bf46080eab4bd5fc2acf5997d6cd5
\ No newline at end of file
+8e57c31340dd9ffc457da63c5996fb1b573f8154f864ec2b52c15f399906ac8b
\ No newline at end of file

From 5b9dbd1ed0a8470aa6536b0b9d28f85ea53fccbf Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 21 Apr 2017 19:53:39 +0000
Subject: [PATCH 40/68] Update the README.md file in the ext/expert/ directory.

FossilOrigin-Name: 3b2ff4e0692dfca395d4523b7d5cd0dfd5c319c1072a2a873631fa477cee0b79
---
 ext/expert/README.md | 36 +++++++++++++++++++++++++-----------
 manifest             | 12 ++++++------
 manifest.uuid        |  2 +-
 3 files changed, 32 insertions(+), 18 deletions(-)

diff --git a/ext/expert/README.md b/ext/expert/README.md
index 3cede6c09d..b3023f2c1a 100644
--- a/ext/expert/README.md
+++ b/ext/expert/README.md
@@ -5,21 +5,20 @@ given a database and a set of SQL queries. It works as follows:
 
   1. The user database schema is copied to a temporary database.
 
-  1. All SQL queries are prepared against the temporary database. The
-     **sqlite3\_whereinfo\_hook()** API is used to record information regarding
-     the WHERE and ORDER BY clauses attached to each query.
+  1. All SQL queries are prepared against the temporary database. I
+     Information regarding the WHERE and ORDER BY clauses, and other query
+     features that affect index selection, are recorded.
 
   1. The information gathered in step 2 is used to create (possibly a large
      number of) candidate indexes.
 
+  1. A subset of the data in the user database is used to generate statistics
+     for all existing indexes and the candidate indexes generated in step 3
+     above.
+
   1. The SQL queries are prepared a second time. If the planner uses any
      of the indexes created in step 3, they are recommended to the user.
 
-No ANALYZE data is available to the planner in step 4 above. This can lead to sub-optimal results.
-
-This extension requires that SQLite be built with the 
-SQLITE\_ENABLE\_WHEREINFO\_HOOK pre-processor symbol defined.
-
 # C API
 
 The SQLite expert C API is defined in sqlite3expert.h. Most uses will proceed
@@ -32,6 +31,10 @@ as follows:
      by making one or more calls to **sqlite3\_expert\_sql()**. Each call may
      specify a single SQL statement, or multiple statements separated by
      semi-colons.
+  
+  1. Optionally, the **sqlite3\_expert\_config()** API may be used to 
+     configure the size of the data subset used to generate index statistics.
+     Using a smaller subset of the data can speed up the analysis.
 
   1. **sqlite3\_expert\_analyze()** is called to run the analysis.
 
@@ -48,10 +51,11 @@ The file "expert.c" contains the code for a command line application that
 uses the API described above. It can be compiled with (for example):
 
 <pre>
-  gcc -O2 -DSQLITE_ENABLE_WHEREINFO_HOOK sqlite3.c expert.c sqlite3expert.c -o sqlite3_expert
+  gcc -O2 sqlite3.c expert.c sqlite3expert.c -o sqlite3_expert
 </pre>
 
-Assuming the database is "test.db", it can then be run to analyze a single query:
+Assuming the database is named "test.db", it can then be run to analyze a
+single query:
 
 <pre>
   ./sqlite3_expert -sql &lt;sql-query&gt; test.db
@@ -63,6 +67,16 @@ Or an entire text file worth of queries with:
   ./sqlite3_expert -file &lt;text-file&gt; test.db
 </pre>
 
+By default, sqlite3_expert generates index statistics using all the data in
+the user database. For a large database, this may be prohibitively time
+consuming. The "-sample" option may be used to configure sqlite3_expert to
+generate statistics based on an integer percentage of the user database as
+follows:
 
+<pre>
+  # Generate statistics based on 25% of the user database rows:
+  ./sqlite3_expert -sample 25 -sql &lt;sql-query&gt; test.db
 
-
+  # Do not generate any statistics at all:
+  ./sqlite3_expert -sample 0 -sql &lt;sql-query&gt; test.db
+</pre>
diff --git a/manifest b/manifest
index 66b5295d63..a85be5f925 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\slatest\strunk\schanges\sinto\sthis\sbranch.
-D 2017-04-20T17:35:46.403
+C Update\sthe\sREADME.md\sfile\sin\sthe\sext/expert/\sdirectory.
+D 2017-04-21T19:53:39.781
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6
@@ -40,7 +40,7 @@ F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd
 F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
-F ext/expert/README.md 9f15075ec5ad772808eff55ef044c31140fd1146aa0a3c47eafd155e71851b01
+F ext/expert/README.md 58a5af07981d6946464eb4b069f36a18dff1d594bcf331a02b4485dac5cd5207
 F ext/expert/expert.c 33842ef151d84c5f8000f9c7b938998c6b999eaef7ce1f4eeb0df8ffe6739496
 F ext/expert/expert1.test 1033e43071b69dc2f4e88fbf03fc7f18846c9865cac14f28c80f581437f09acb
 F ext/expert/sqlite3expert.c 4bc1820a70d68b478884a26a2215df8c1f2d44fb40d9cd8c47d2046c8ce0c8bc
@@ -1582,7 +1582,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 4e366996434e63f06cc451d2011f1f1464360f03430b8260e48f78a612b4e9d6 6b21d0fdebdccfaf63590d9ca9a279c22b8baec07c1a669b9f617f25bd857384
-R 3ff876810132a5a27eb9c4a943446c06
+P b1533bc455f52f570c0f4b8aaa0da802757dc89b0e45b9a9b31aa591a44bf7bd
+R 1bfa4bacff37532498e779dfe65e0523
 U dan
-Z 9bcc0d98d05bcef127f4de36c5b9d364
+Z 425c38161b659ffbe58ff987543a301c
diff --git a/manifest.uuid b/manifest.uuid
index 10e306f449..e42f6c368b 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b1533bc455f52f570c0f4b8aaa0da802757dc89b0e45b9a9b31aa591a44bf7bd
\ No newline at end of file
+3b2ff4e0692dfca395d4523b7d5cd0dfd5c319c1072a2a873631fa477cee0b79
\ No newline at end of file

From 6b0fa916f570dd80b462c924d05048d52c5651f5 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 21 Apr 2017 19:56:53 +0000
Subject: [PATCH 41/68] Fix formatting errors in the previous commit.

FossilOrigin-Name: da9a2e5aa977f7e8e9e4365f7b34bb4f482029a3d44646100773cedc8ea9b959
---
 ext/expert/README.md | 13 +++++++------
 manifest             | 12 ++++++------
 manifest.uuid        |  2 +-
 3 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/ext/expert/README.md b/ext/expert/README.md
index b3023f2c1a..3c2e0a65e0 100644
--- a/ext/expert/README.md
+++ b/ext/expert/README.md
@@ -5,12 +5,13 @@ given a database and a set of SQL queries. It works as follows:
 
   1. The user database schema is copied to a temporary database.
 
-  1. All SQL queries are prepared against the temporary database. I
+  1. All SQL queries are prepared against the temporary database.
      Information regarding the WHERE and ORDER BY clauses, and other query
-     features that affect index selection, are recorded.
+     features that affect index selection are recorded.
 
-  1. The information gathered in step 2 is used to create (possibly a large
-     number of) candidate indexes.
+  1. The information gathered in step 2 is used to create candidate indexes
+     - indexes that the planner might have made use of in the previous step,
+     had they been available.
 
   1. A subset of the data in the user database is used to generate statistics
      for all existing indexes and the candidate indexes generated in step 3
@@ -67,9 +68,9 @@ Or an entire text file worth of queries with:
   ./sqlite3_expert -file &lt;text-file&gt; test.db
 </pre>
 
-By default, sqlite3_expert generates index statistics using all the data in
+By default, sqlite3\_expert generates index statistics using all the data in
 the user database. For a large database, this may be prohibitively time
-consuming. The "-sample" option may be used to configure sqlite3_expert to
+consuming. The "-sample" option may be used to configure sqlite3\_expert to
 generate statistics based on an integer percentage of the user database as
 follows:
 
diff --git a/manifest b/manifest
index a85be5f925..f684b3a0fc 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\sthe\sREADME.md\sfile\sin\sthe\sext/expert/\sdirectory.
-D 2017-04-21T19:53:39.781
+C Fix\sformatting\serrors\sin\sthe\sprevious\scommit.
+D 2017-04-21T19:56:53.579
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6
@@ -40,7 +40,7 @@ F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd
 F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
-F ext/expert/README.md 58a5af07981d6946464eb4b069f36a18dff1d594bcf331a02b4485dac5cd5207
+F ext/expert/README.md ffa4899855d90ae1a2a29b29754c7abad9ed8c42f187c72e913cb671264dbd5d
 F ext/expert/expert.c 33842ef151d84c5f8000f9c7b938998c6b999eaef7ce1f4eeb0df8ffe6739496
 F ext/expert/expert1.test 1033e43071b69dc2f4e88fbf03fc7f18846c9865cac14f28c80f581437f09acb
 F ext/expert/sqlite3expert.c 4bc1820a70d68b478884a26a2215df8c1f2d44fb40d9cd8c47d2046c8ce0c8bc
@@ -1582,7 +1582,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b1533bc455f52f570c0f4b8aaa0da802757dc89b0e45b9a9b31aa591a44bf7bd
-R 1bfa4bacff37532498e779dfe65e0523
+P 3b2ff4e0692dfca395d4523b7d5cd0dfd5c319c1072a2a873631fa477cee0b79
+R 95ece7093798132a04553def12c0836e
 U dan
-Z 425c38161b659ffbe58ff987543a301c
+Z 360220848d9d809f77758310da99ccdb
diff --git a/manifest.uuid b/manifest.uuid
index e42f6c368b..77c9b27734 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-3b2ff4e0692dfca395d4523b7d5cd0dfd5c319c1072a2a873631fa477cee0b79
\ No newline at end of file
+da9a2e5aa977f7e8e9e4365f7b34bb4f482029a3d44646100773cedc8ea9b959
\ No newline at end of file

From f3ac8ba569f51cf2946511471f121aefcf449406 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 21 Apr 2017 19:58:35 +0000
Subject: [PATCH 42/68] Another minor formatting fix.

FossilOrigin-Name: 9fa2ce3c2b75408594d387684984cd946765776d30bc622a1f4e64d6fe1856d5
---
 ext/expert/README.md |  6 +++---
 manifest             | 12 ++++++------
 manifest.uuid        |  2 +-
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/ext/expert/README.md b/ext/expert/README.md
index 3c2e0a65e0..28886fd1f2 100644
--- a/ext/expert/README.md
+++ b/ext/expert/README.md
@@ -9,9 +9,9 @@ given a database and a set of SQL queries. It works as follows:
      Information regarding the WHERE and ORDER BY clauses, and other query
      features that affect index selection are recorded.
 
-  1. The information gathered in step 2 is used to create candidate indexes
-     - indexes that the planner might have made use of in the previous step,
-     had they been available.
+  1. The information gathered in step 2 is used to create candidate 
+     indexes - indexes that the planner might have made use of in the previous
+     step, had they been available.
 
   1. A subset of the data in the user database is used to generate statistics
      for all existing indexes and the candidate indexes generated in step 3
diff --git a/manifest b/manifest
index f684b3a0fc..02667e9490 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sformatting\serrors\sin\sthe\sprevious\scommit.
-D 2017-04-21T19:56:53.579
+C Another\sminor\sformatting\sfix.
+D 2017-04-21T19:58:35.957
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6
@@ -40,7 +40,7 @@ F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd
 F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
-F ext/expert/README.md ffa4899855d90ae1a2a29b29754c7abad9ed8c42f187c72e913cb671264dbd5d
+F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3
 F ext/expert/expert.c 33842ef151d84c5f8000f9c7b938998c6b999eaef7ce1f4eeb0df8ffe6739496
 F ext/expert/expert1.test 1033e43071b69dc2f4e88fbf03fc7f18846c9865cac14f28c80f581437f09acb
 F ext/expert/sqlite3expert.c 4bc1820a70d68b478884a26a2215df8c1f2d44fb40d9cd8c47d2046c8ce0c8bc
@@ -1582,7 +1582,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3b2ff4e0692dfca395d4523b7d5cd0dfd5c319c1072a2a873631fa477cee0b79
-R 95ece7093798132a04553def12c0836e
+P da9a2e5aa977f7e8e9e4365f7b34bb4f482029a3d44646100773cedc8ea9b959
+R 5d55911452a98d7b9a566802f4defda8
 U dan
-Z 360220848d9d809f77758310da99ccdb
+Z bc8b305bc880d9cdf8a4b644dabd4fac
diff --git a/manifest.uuid b/manifest.uuid
index 77c9b27734..d9908b5ae2 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-da9a2e5aa977f7e8e9e4365f7b34bb4f482029a3d44646100773cedc8ea9b959
\ No newline at end of file
+9fa2ce3c2b75408594d387684984cd946765776d30bc622a1f4e64d6fe1856d5
\ No newline at end of file

From 382bce037cad6c70635b23f0ee63669e326fad99 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 2 May 2017 20:42:30 +0000
Subject: [PATCH 43/68] In the sqlite3_expert command-line tool, allow two-dash
 options.  Do not accept the database name if it begins with "-".

FossilOrigin-Name: af7d1596044980e0a18baa3ab6674779724dffbf18a152c72e53f11a08999e68
---
 ext/expert/expert.c |  4 ++--
 manifest            | 12 ++++++------
 manifest.uuid       |  2 +-
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/ext/expert/expert.c b/ext/expert/expert.c
index baa7be940f..1c4b758d8f 100644
--- a/ext/expert/expert.c
+++ b/ext/expert/expert.c
@@ -79,6 +79,7 @@ int main(int argc, char **argv){
 
   if( argc<2 ) usage(argv);
   zDb = argv[argc-1];
+  if( zDb[0]=='-' ) usage(argv);
   rc = sqlite3_open(zDb, &db);
   if( rc!=SQLITE_OK ){
     fprintf(stderr, "Cannot open db file: %s - %s\n", zDb, sqlite3_errmsg(db));
@@ -92,6 +93,7 @@ int main(int argc, char **argv){
   }else{
     for(i=1; i<(argc-1); i++){
       char *zArg = argv[i];
+      if( zArg[0]=='-' && zArg[1]=='-' && zArg[2]!=0 ) zArg++;
       int nArg = strlen(zArg);
       if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-file", nArg) ){
         if( ++i==(argc-1) ) option_requires_argument("-file");
@@ -151,5 +153,3 @@ int main(int argc, char **argv){
   sqlite3_free(zErr);
   return rc;
 }
-
-
diff --git a/manifest b/manifest
index 9ca3ce06e1..7ad30c6537 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\sthe\slatest\senhancements\sfrom\strunk.
-D 2017-05-02T19:45:14.234
+C In\sthe\ssqlite3_expert\scommand-line\stool,\sallow\stwo-dash\soptions.\s\sDo\snot\naccept\sthe\sdatabase\sname\sif\sit\sbegins\swith\s"-".
+D 2017-05-02T20:42:30.619
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6
@@ -41,7 +41,7 @@ F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3
-F ext/expert/expert.c 33842ef151d84c5f8000f9c7b938998c6b999eaef7ce1f4eeb0df8ffe6739496
+F ext/expert/expert.c dc88a3e73fdeb0d21e7d60b791120961853af931f3c509b508f4ac4a0233438e
 F ext/expert/expert1.test 1033e43071b69dc2f4e88fbf03fc7f18846c9865cac14f28c80f581437f09acb
 F ext/expert/sqlite3expert.c 4bc1820a70d68b478884a26a2215df8c1f2d44fb40d9cd8c47d2046c8ce0c8bc
 F ext/expert/sqlite3expert.h af6354f8ee5c9e025024e63fec3bd640a802afcc3099a44d804752cf0791d811
@@ -1585,7 +1585,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 11f4761c3a84e2cc9df62f117a003af8c57f3d226eec5a40d6241b121e78d002 430f539cbb3f806fb89191e0b759a5f8b49d9e5b6c95fe9a4b55a1aa0875762a
-R c37204de6c890801155672b7d23b8abd
+P a7dcf6a79f7e1c5884baee2909a4bf3174ae06d561dae87b390856e573f81b49
+R 84109358bc8c579f4172a289a09c559d
 U drh
-Z 640e08616c5ac7ee4ced91411da47a09
+Z 3bebc4378f2040a907bd12dcd9347c44
diff --git a/manifest.uuid b/manifest.uuid
index 17b09356de..c25e85551e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a7dcf6a79f7e1c5884baee2909a4bf3174ae06d561dae87b390856e573f81b49
\ No newline at end of file
+af7d1596044980e0a18baa3ab6674779724dffbf18a152c72e53f11a08999e68
\ No newline at end of file

From 138bd6df4199ecf7e3d75c9ea9bc8ce86c3055c4 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 3 May 2017 12:15:20 +0000
Subject: [PATCH 44/68] In sqlite3expert.c, do not copy the schema for virtual
 tables.  Updates to makefiles to make building easier.

FossilOrigin-Name: da15752dccf6090e40ec825db89048eca2b30185882225bf81f1891e914c2e7f
---
 Makefile.in                | 19 +++++++++++++++++++
 ext/expert/sqlite3expert.c |  7 +------
 main.mk                    |  6 +++---
 manifest                   | 16 ++++++++--------
 manifest.uuid              |  2 +-
 5 files changed, 32 insertions(+), 18 deletions(-)

diff --git a/Makefile.in b/Makefile.in
index 9076ffc085..0cde7a64f6 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -416,6 +416,8 @@ TESTSRC = \
 # Statically linked extensions
 #
 TESTSRC += \
+  $(TOP)/ext/expert/sqlite3expert.c \
+  $(TOP)/ext/expert/test_expert.c \
   $(TOP)/ext/misc/amatch.c \
   $(TOP)/ext/misc/carray.c \
   $(TOP)/ext/misc/closure.c \
@@ -1155,6 +1157,23 @@ sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl
 sqlite3_analyzer$(TEXE): sqlite3_analyzer.c
 	$(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS)
 
+sqlite3_expert$(TEXE): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c
+	$(LTLINK)	$(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert $(TLIBS)
+
+sqlite3_schemalint.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/schemalint.tcl
+	echo "#define TCLSH 2" > $@
+	echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@
+	cat sqlite3.c $(TOP)/src/tclsqlite.c >> $@
+	echo "static const char *tclsh_main_loop(void){" >> $@
+	echo "static const char *zMainloop = " >> $@
+	$(TCLSH_CMD) $(TOP)/tool/tostr.tcl $(TOP)/tool/schemalint.tcl >> $@
+	echo "; return zMainloop; }" >> $@
+
+sqlite3_schemalint$(TEXE): $(TESTSRC) sqlite3_schemalint.c
+	$(LTLINK) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS)                  \
+		sqlite3_schemalint.c $(TESTSRC)               \
+		-o sqlite3_schemalint$(EXE) $(LIBTCL) $(TLIBS)
+
 dbdump$(TEXE): $(TOP)/ext/misc/dbdump.c sqlite3.lo
 	$(LTLINK) -DDBDUMP_STANDALONE -o $@ \
            $(TOP)/ext/misc/dbdump.c sqlite3.lo $(TLIBS)
diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index 8cacc9d706..1e02328211 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -10,9 +10,6 @@
 **
 *************************************************************************
 */
-
-#if !defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHEREINFO_HOOK)
-
 #include "sqlite3expert.h"
 #include <assert.h>
 #include <string.h>
@@ -1745,6 +1742,7 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){
     sqlite3_stmt *pSql;
     rc = idxPrintfPrepareStmt(pNew->db, &pSql, pzErrmsg, 
         "SELECT sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%%'"
+        " AND sql NOT LIKE 'CREATE VIRTUAL %%'"
     );
     while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
       const char *zSql = (const char*)sqlite3_column_text(pSql, 0);
@@ -1933,6 +1931,3 @@ void sqlite3_expert_destroy(sqlite3expert *p){
     sqlite3_free(p);
   }
 }
-
-#endif /* !defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHEREINFO_HOOK) */
-
diff --git a/main.mk b/main.mk
index ea4f9891f7..218598b549 100644
--- a/main.mk
+++ b/main.mk
@@ -764,7 +764,7 @@ sqlite3_analyzer$(EXE): sqlite3_analyzer.c
 	$(TCCX) $(TCL_FLAGS) sqlite3_analyzer.c -o $@ $(LIBTCL) $(THREADLIB) 
 
 sqlite3_expert$(EXE): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c
-	$(TCCX) -DSQLITE_ENABLE_WHEREINFO_HOOK $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert $(THREADLIB)
+	$(TCCX) $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert $(THREADLIB)
 
 sqlite3_schemalint.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/schemalint.tcl
 	echo "#define TCLSH 2" > $@
@@ -775,9 +775,9 @@ sqlite3_schemalint.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/schemalint.tc
 	tclsh $(TOP)/tool/tostr.tcl $(TOP)/tool/schemalint.tcl >> $@
 	echo "; return zMainloop; }" >> $@
 
-sqlite3_schemalint$(EXE): $(TESTSRC) sqlite3_schemalint.c
+sqlite3_schemalint$(EXE): $(TESTSRC) sqlite3_schemalint.c $(TOP)/ext/session/test_session.c
 	$(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS)                  \
-		sqlite3_schemalint.c $(TESTSRC)               \
+		sqlite3_schemalint.c $(TESTSRC) $(TOP)/ext/session/test_session.c  \
 		-o sqlite3_schemalint$(EXE) $(LIBTCL) $(THREADLIB)
 
 dbdump$(EXE):	$(TOP)/ext/misc/dbdump.c sqlite3.o
diff --git a/manifest b/manifest
index 7ad30c6537..ff6e6888b8 100644
--- a/manifest
+++ b/manifest
@@ -1,6 +1,6 @@
-C In\sthe\ssqlite3_expert\scommand-line\stool,\sallow\stwo-dash\soptions.\s\sDo\snot\naccept\sthe\sdatabase\sname\sif\sit\sbegins\swith\s"-".
-D 2017-05-02T20:42:30.619
-F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
+C In\ssqlite3expert.c,\sdo\snot\scopy\sthe\sschema\sfor\svirtual\stables.\s\sUpdates\sto\nmakefiles\sto\smake\sbuilding\seasier.
+D 2017-05-03T12:15:20.951
+F Makefile.in 2c991e7b1a2bb23c147406c3630b54d99c4931ae1fa0e8c6b6bf40a7a2fd02a3
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6
 F README.md 2b15fae33852f2f53996774c21fb41e1d94181c4401a0e43ac93e11f2cc901b9
@@ -43,7 +43,7 @@ F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3
 F ext/expert/expert.c dc88a3e73fdeb0d21e7d60b791120961853af931f3c509b508f4ac4a0233438e
 F ext/expert/expert1.test 1033e43071b69dc2f4e88fbf03fc7f18846c9865cac14f28c80f581437f09acb
-F ext/expert/sqlite3expert.c 4bc1820a70d68b478884a26a2215df8c1f2d44fb40d9cd8c47d2046c8ce0c8bc
+F ext/expert/sqlite3expert.c 87bac32f90492adfd99d3dc142d5c9af38a3bd961637202285077968051bc6e8
 F ext/expert/sqlite3expert.h af6354f8ee5c9e025024e63fec3bd640a802afcc3099a44d804752cf0791d811
 F ext/expert/test_expert.c 85f5c743a899063fa48296d21de2f32c26d09a21c8582b2a0bc482e8de183e7a
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
@@ -332,7 +332,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk ef818c7b1bb21f657e3bfb363cc7167264d688ca404a666e6ddda6029e94c43b
+F main.mk 7fcb8db53d07ea8124c2c43dac5fcee8095e4abeab1acdf114833b8805aa72c9
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -1585,7 +1585,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a7dcf6a79f7e1c5884baee2909a4bf3174ae06d561dae87b390856e573f81b49
-R 84109358bc8c579f4172a289a09c559d
+P af7d1596044980e0a18baa3ab6674779724dffbf18a152c72e53f11a08999e68
+R 446fb26db86c2ead652773ba0b926ac7
 U drh
-Z 3bebc4378f2040a907bd12dcd9347c44
+Z 183bc096a41586435af9a58068b8a076
diff --git a/manifest.uuid b/manifest.uuid
index c25e85551e..388a18ee4f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-af7d1596044980e0a18baa3ab6674779724dffbf18a152c72e53f11a08999e68
\ No newline at end of file
+da15752dccf6090e40ec825db89048eca2b30185882225bf81f1891e914c2e7f
\ No newline at end of file

From bd0f1dbd06baae25459d270633300f3dd7ec5484 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 3 May 2017 12:50:46 +0000
Subject: [PATCH 45/68] Get sqlite3_expert building on Windows.

FossilOrigin-Name: d8254047b30f7c1be486bf39d4420678604573b951b5cc83c19ebf74aba0864c
---
 Makefile.msc               | 18 ++++++++++++++++++
 ext/expert/expert.c        |  2 +-
 ext/expert/sqlite3expert.c | 36 +++++++++++++++++++-----------------
 manifest                   | 16 ++++++++--------
 manifest.uuid              |  2 +-
 5 files changed, 47 insertions(+), 27 deletions(-)

diff --git a/Makefile.msc b/Makefile.msc
index 5ef8decc38..143a90246f 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -1399,6 +1399,8 @@ TESTSRC = \
 # Statically linked extensions.
 #
 TESTEXT = \
+  $(TOP)\ext\expert\sqlite3expert.c \
+  $(TOP)\ext\expert\test_expert.c \
   $(TOP)\ext\misc\amatch.c \
   $(TOP)\ext\misc\carray.c \
   $(TOP)\ext\misc\closure.c \
@@ -2181,6 +2183,22 @@ sqlite3_analyzer.exe:	sqlite3_analyzer.c $(LIBRESOBJS)
 	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \
 		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
 
+sqlite3_expert.exe: $(SQLITE3C) $(TOP)\ext\expert\sqlite3expert.h $(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c
+	$(LTLINK) $(NO_WARN)	$(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c $(SQLITE3C) $(TLIBS)
+
+sqlite3_schemalint.c: $(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\schemalint.tcl $(SQLITE_TCL_DEP)
+	echo "#define TCLSH 2" > $@
+	echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@
+	copy $@ + $(SQLITE3C) + $(TOP)\src\tclsqlite.c >> $@
+	echo "static const char *tclsh_main_loop(void){" >> $@
+	echo "static const char *zMainloop = " >> $@
+	$(TCLSH_CMD) $(TOP)\tool\tostr.tcl $(TOP)\tool\schemalint.tcl >> $@
+	echo "; return zMainloop; }" >> $@
+
+sqlite3_schemalint.exe: $(TESTSRC) sqlite3_schemalint.c
+	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_schemalint.c \
+		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
+
 dbdump.exe:	$(TOP)\ext\misc\dbdump.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DDBDUMP_STANDALONE $(TOP)\ext\misc\dbdump.c $(SQLITE3C) \
 		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS)
diff --git a/ext/expert/expert.c b/ext/expert/expert.c
index 1c4b758d8f..13fc87ea89 100644
--- a/ext/expert/expert.c
+++ b/ext/expert/expert.c
@@ -94,7 +94,7 @@ int main(int argc, char **argv){
     for(i=1; i<(argc-1); i++){
       char *zArg = argv[i];
       if( zArg[0]=='-' && zArg[1]=='-' && zArg[2]!=0 ) zArg++;
-      int nArg = strlen(zArg);
+      int nArg = (int)strlen(zArg);
       if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-file", nArg) ){
         if( ++i==(argc-1) ) option_requires_argument("-file");
         rc = readSqlFromFile(p, argv[i], &zErr);
diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index 1e02328211..8627d196f0 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -25,6 +25,8 @@ typedef struct IdxStatement IdxStatement;
 typedef struct IdxTable IdxTable;
 typedef struct IdxWrite IdxWrite;
 
+#define STRLEN  (int)strlen
+
 /*
 ** A temp table name that we assume no user database will actually use.
 ** If this assumption proves incorrect triggers on the table with the
@@ -212,13 +214,13 @@ static int idxHashAdd(
   const char *zKey,
   const char *zVal
 ){
-  int nKey = strlen(zKey);
+  int nKey = STRLEN(zKey);
   int iHash = idxHashString(zKey, nKey);
-  int nVal = (zVal ? strlen(zVal) : 0);
+  int nVal = (zVal ? STRLEN(zVal) : 0);
   IdxHashEntry *pEntry;
   assert( iHash>=0 );
   for(pEntry=pHash->aHash[iHash]; pEntry; pEntry=pEntry->pHashNext){
-    if( strlen(pEntry->zKey)==nKey && 0==memcmp(pEntry->zKey, zKey, nKey) ){
+    if( STRLEN(pEntry->zKey)==nKey && 0==memcmp(pEntry->zKey, zKey, nKey) ){
       return 1;
     }
   }
@@ -246,11 +248,11 @@ static int idxHashAdd(
 static IdxHashEntry *idxHashFind(IdxHash *pHash, const char *zKey, int nKey){
   int iHash;
   IdxHashEntry *pEntry;
-  if( nKey<0 ) nKey = strlen(zKey);
+  if( nKey<0 ) nKey = STRLEN(zKey);
   iHash = idxHashString(zKey, nKey);
   assert( iHash>=0 );
   for(pEntry=pHash->aHash[iHash]; pEntry; pEntry=pEntry->pHashNext){
-    if( strlen(pEntry->zKey)==nKey && 0==memcmp(pEntry->zKey, zKey, nKey) ){
+    if( STRLEN(pEntry->zKey)==nKey && 0==memcmp(pEntry->zKey, zKey, nKey) ){
       return pEntry;
     }
   }
@@ -275,7 +277,7 @@ static const char *idxHashSearch(IdxHash *pHash, const char *zKey, int nKey){
 */
 static IdxConstraint *idxNewConstraint(int *pRc, const char *zColl){
   IdxConstraint *pNew;
-  int nColl = strlen(zColl);
+  int nColl = STRLEN(zColl);
 
   assert( *pRc==SQLITE_OK );
   pNew = (IdxConstraint*)idxMalloc(pRc, sizeof(IdxConstraint) * nColl + 1);
@@ -357,7 +359,7 @@ struct ExpertCsr {
 };
 
 static char *expertDequote(const char *zIn){
-  int n = strlen(zIn);
+  int n = STRLEN(zIn);
   char *zRet = sqlite3_malloc(n);
 
   assert( zIn[0]=='\'' );
@@ -666,7 +668,7 @@ static int idxGetTableInfo(
 ){
   sqlite3_stmt *p1 = 0;
   int nCol = 0;
-  int nTab = strlen(zTab);
+  int nTab = STRLEN(zTab);
   int nByte = sizeof(IdxTable) + nTab + 1;
   IdxTable *pNew = 0;
   int rc, rc2;
@@ -675,11 +677,11 @@ static int idxGetTableInfo(
   rc = idxPrintfPrepareStmt(db, &p1, pzErrmsg, "PRAGMA table_info=%Q", zTab);
   while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){
     const char *zCol = (const char*)sqlite3_column_text(p1, 1);
-    nByte += 1 + strlen(zCol);
+    nByte += 1 + STRLEN(zCol);
     rc = sqlite3_table_column_metadata(
         db, "main", zTab, zCol, 0, &zCol, 0, 0, 0
     );
-    nByte += 1 + strlen(zCol);
+    nByte += 1 + STRLEN(zCol);
     nCol++;
   }
   rc2 = sqlite3_reset(p1);
@@ -698,7 +700,7 @@ static int idxGetTableInfo(
   nCol = 0;
   while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){
     const char *zCol = (const char*)sqlite3_column_text(p1, 1);
-    int nCopy = strlen(zCol) + 1;
+    int nCopy = STRLEN(zCol) + 1;
     pNew->aCol[nCol].zName = pCsr;
     pNew->aCol[nCol].iPk = sqlite3_column_int(p1, 5);
     memcpy(pCsr, zCol, nCopy);
@@ -708,7 +710,7 @@ static int idxGetTableInfo(
         db, "main", zTab, zCol, 0, &zCol, 0, 0, 0
     );
     if( rc==SQLITE_OK ){
-      nCopy = strlen(zCol) + 1;
+      nCopy = STRLEN(zCol) + 1;
       pNew->aCol[nCol].zColl = pCsr;
       memcpy(pCsr, zCol, nCopy);
       pCsr += nCopy;
@@ -743,13 +745,13 @@ static char *idxAppendText(int *pRc, char *zIn, const char *zFmt, ...){
   va_list ap;
   char *zAppend = 0;
   char *zRet = 0;
-  int nIn = zIn ? strlen(zIn) : 0;
+  int nIn = zIn ? STRLEN(zIn) : 0;
   int nAppend = 0;
   va_start(ap, zFmt);
   if( *pRc==SQLITE_OK ){
     zAppend = sqlite3_vmprintf(zFmt, ap);
     if( zAppend ){
-      nAppend = strlen(zAppend);
+      nAppend = STRLEN(zAppend);
       zRet = (char*)sqlite3_malloc(nIn + nAppend + 1);
     }
     if( zAppend && zRet ){
@@ -1114,7 +1116,7 @@ int idxFindIndexes(
       int iOrder = sqlite3_column_int(pExplain, 1);
       int iFrom = sqlite3_column_int(pExplain, 2);
       const char *zDetail = (const char*)sqlite3_column_text(pExplain, 3);
-      int nDetail = strlen(zDetail);
+      int nDetail = STRLEN(zDetail);
       int i;
 
       for(i=0; i<nDetail; i++){
@@ -1577,7 +1579,7 @@ static int idxPopulateOneStat1(
       rc = sqlite3_reset(pWriteStat);
     }
 
-    pEntry = idxHashFind(&p->hIdx, zIdx, strlen(zIdx));
+    pEntry = idxHashFind(&p->hIdx, zIdx, STRLEN(zIdx));
     if( pEntry ){
       assert( pEntry->zVal2==0 );
       pEntry->zVal2 = zStat;
@@ -1816,7 +1818,7 @@ int sqlite3_expert_sql(
       if( pStmt ){
         IdxStatement *pNew;
         const char *z = sqlite3_sql(pStmt);
-        int n = strlen(z);
+        int n = STRLEN(z);
         pNew = (IdxStatement*)idxMalloc(&rc, sizeof(IdxStatement) + n+1);
         if( rc==SQLITE_OK ){
           pNew->zSql = (char*)&pNew[1];
diff --git a/manifest b/manifest
index ff6e6888b8..770ed52d6a 100644
--- a/manifest
+++ b/manifest
@@ -1,8 +1,8 @@
-C In\ssqlite3expert.c,\sdo\snot\scopy\sthe\sschema\sfor\svirtual\stables.\s\sUpdates\sto\nmakefiles\sto\smake\sbuilding\seasier.
-D 2017-05-03T12:15:20.951
+C Get\ssqlite3_expert\sbuilding\son\sWindows.
+D 2017-05-03T12:50:46.615
 F Makefile.in 2c991e7b1a2bb23c147406c3630b54d99c4931ae1fa0e8c6b6bf40a7a2fd02a3
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6
+F Makefile.msc 2b4876693e76420704bd2defe9f957b902ccb35ebe1fd071b80f70e862b7d444
 F README.md 2b15fae33852f2f53996774c21fb41e1d94181c4401a0e43ac93e11f2cc901b9
 F VERSION 0a0e02e16b44ea735b40118fc844311b2ab0d35b25fbeda5120aee62f973f663
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -41,9 +41,9 @@ F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3
-F ext/expert/expert.c dc88a3e73fdeb0d21e7d60b791120961853af931f3c509b508f4ac4a0233438e
+F ext/expert/expert.c 4791c5e064aea81b2b829fa95228b22283380ee370ea88a1e580103b75516ebf
 F ext/expert/expert1.test 1033e43071b69dc2f4e88fbf03fc7f18846c9865cac14f28c80f581437f09acb
-F ext/expert/sqlite3expert.c 87bac32f90492adfd99d3dc142d5c9af38a3bd961637202285077968051bc6e8
+F ext/expert/sqlite3expert.c 6ed4e84a06d1a29b2cf3009c0266573b88602d098055caa46c467154a64e0959
 F ext/expert/sqlite3expert.h af6354f8ee5c9e025024e63fec3bd640a802afcc3099a44d804752cf0791d811
 F ext/expert/test_expert.c 85f5c743a899063fa48296d21de2f32c26d09a21c8582b2a0bc482e8de183e7a
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
@@ -1585,7 +1585,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P af7d1596044980e0a18baa3ab6674779724dffbf18a152c72e53f11a08999e68
-R 446fb26db86c2ead652773ba0b926ac7
+P da15752dccf6090e40ec825db89048eca2b30185882225bf81f1891e914c2e7f
+R d4fb7977c87e361df2272c02e0401fc7
 U drh
-Z 183bc096a41586435af9a58068b8a076
+Z 1174ddb23c34b04fd900b7d4a8068d04
diff --git a/manifest.uuid b/manifest.uuid
index 388a18ee4f..8d1a6daaf2 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-da15752dccf6090e40ec825db89048eca2b30185882225bf81f1891e914c2e7f
\ No newline at end of file
+d8254047b30f7c1be486bf39d4420678604573b951b5cc83c19ebf74aba0864c
\ No newline at end of file

From bd72749f7d9c387e28a93b3877f5d416a85b7014 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 3 May 2017 13:05:08 +0000
Subject: [PATCH 46/68] Fix a harmless compiler warning on Windows.

FossilOrigin-Name: 593e5dd00cdf8fbc680951d68b53b38262c61c467703f6a8eb477226cf6beedd
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/vdbeaux.c |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 770ed52d6a..77a8c112c8 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Get\ssqlite3_expert\sbuilding\son\sWindows.
-D 2017-05-03T12:50:46.615
+C Fix\sa\sharmless\scompiler\swarning\son\sWindows.
+D 2017-05-03T13:05:08.456
 F Makefile.in 2c991e7b1a2bb23c147406c3630b54d99c4931ae1fa0e8c6b6bf40a7a2fd02a3
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 2b4876693e76420704bd2defe9f957b902ccb35ebe1fd071b80f70e862b7d444
@@ -479,7 +479,7 @@ F src/vdbe.c 356042d11e05064c43242020e8de97acef9fc8931cfc39ae7cf4cf91d6e42c19
 F src/vdbe.h f7d1456e28875c2dcb964056589b5b7149ab7edf39edeca801596a39bb3d3848
 F src/vdbeInt.h c070bc5c8b913bda0ceaa995cd4d939ded5e4fc96cf7c3c1c602d41b871f8ade
 F src/vdbeapi.c 5b08d82592bcff4470601fe78aaabebd50837860
-F src/vdbeaux.c 98ced78bb4d8f1c66a4519591804cbf34530f19c295a8589833aaa6004ea8731
+F src/vdbeaux.c dd4106907d8a52f80876c7cec030511220146056d0bb5d274f8528e251ffaed9
 F src/vdbeblob.c 359891617358deefc85bef7bcf787fa6b77facb9
 F src/vdbemem.c 2c70f8f5de6c71fb99a22c5b83be9fab5c47cdd8e279fa44a8c00cfed06d7e89
 F src/vdbesort.c e72fe02a2121386ba767ede8942e9450878b8fc873abf3d1b6824485f092570c
@@ -1585,7 +1585,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P da15752dccf6090e40ec825db89048eca2b30185882225bf81f1891e914c2e7f
-R d4fb7977c87e361df2272c02e0401fc7
+P d8254047b30f7c1be486bf39d4420678604573b951b5cc83c19ebf74aba0864c
+R efc1806e3149882e34f308cbcfce19a8
 U drh
-Z 1174ddb23c34b04fd900b7d4a8068d04
+Z 5eec4984a758352eb665dd5f89e1a5b7
diff --git a/manifest.uuid b/manifest.uuid
index 8d1a6daaf2..03edeee37c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d8254047b30f7c1be486bf39d4420678604573b951b5cc83c19ebf74aba0864c
\ No newline at end of file
+593e5dd00cdf8fbc680951d68b53b38262c61c467703f6a8eb477226cf6beedd
\ No newline at end of file
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index bde66dc1ba..de9ea10cbb 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -1612,7 +1612,7 @@ int sqlite3VdbeList(
   int rc = SQLITE_OK;                  /* Return code */
   Mem *pMem = &p->aMem[1];             /* First Mem of result set */
   int bFull = (p->explain==1 || (db->flags & SQLITE_FullEQP));
-  Op *pOp;
+  Op *pOp = 0;
 
   assert( p->explain );
   assert( p->magic==VDBE_MAGIC_RUN );

From e064d572e79203edb90e56f226ea1da792cb5ff3 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 4 May 2017 14:02:05 +0000
Subject: [PATCH 47/68] Remove the tool/schemalint.tcl script. And related
 Makefile entries. It is superseded by sqlite3_expert.

FossilOrigin-Name: 269bf52e27611cd00fa7f73ee98b395a30dec215232cf1c34f6b741112ba530b
---
 Makefile.in         |  14 --
 Makefile.msc        |  13 -
 main.mk             |  14 --
 manifest            |  19 +-
 manifest.uuid       |   2 +-
 tool/schemalint.tcl | 594 --------------------------------------------
 6 files changed, 10 insertions(+), 646 deletions(-)
 delete mode 100644 tool/schemalint.tcl

diff --git a/Makefile.in b/Makefile.in
index 0cde7a64f6..ac7062d5d4 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -1160,20 +1160,6 @@ sqlite3_analyzer$(TEXE): sqlite3_analyzer.c
 sqlite3_expert$(TEXE): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c
 	$(LTLINK)	$(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert $(TLIBS)
 
-sqlite3_schemalint.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/schemalint.tcl
-	echo "#define TCLSH 2" > $@
-	echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@
-	cat sqlite3.c $(TOP)/src/tclsqlite.c >> $@
-	echo "static const char *tclsh_main_loop(void){" >> $@
-	echo "static const char *zMainloop = " >> $@
-	$(TCLSH_CMD) $(TOP)/tool/tostr.tcl $(TOP)/tool/schemalint.tcl >> $@
-	echo "; return zMainloop; }" >> $@
-
-sqlite3_schemalint$(TEXE): $(TESTSRC) sqlite3_schemalint.c
-	$(LTLINK) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS)                  \
-		sqlite3_schemalint.c $(TESTSRC)               \
-		-o sqlite3_schemalint$(EXE) $(LIBTCL) $(TLIBS)
-
 dbdump$(TEXE): $(TOP)/ext/misc/dbdump.c sqlite3.lo
 	$(LTLINK) -DDBDUMP_STANDALONE -o $@ \
            $(TOP)/ext/misc/dbdump.c sqlite3.lo $(TLIBS)
diff --git a/Makefile.msc b/Makefile.msc
index 143a90246f..c8658ed15e 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -2186,19 +2186,6 @@ sqlite3_analyzer.exe:	sqlite3_analyzer.c $(LIBRESOBJS)
 sqlite3_expert.exe: $(SQLITE3C) $(TOP)\ext\expert\sqlite3expert.h $(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c
 	$(LTLINK) $(NO_WARN)	$(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c $(SQLITE3C) $(TLIBS)
 
-sqlite3_schemalint.c: $(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\schemalint.tcl $(SQLITE_TCL_DEP)
-	echo "#define TCLSH 2" > $@
-	echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@
-	copy $@ + $(SQLITE3C) + $(TOP)\src\tclsqlite.c >> $@
-	echo "static const char *tclsh_main_loop(void){" >> $@
-	echo "static const char *zMainloop = " >> $@
-	$(TCLSH_CMD) $(TOP)\tool\tostr.tcl $(TOP)\tool\schemalint.tcl >> $@
-	echo "; return zMainloop; }" >> $@
-
-sqlite3_schemalint.exe: $(TESTSRC) sqlite3_schemalint.c
-	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_schemalint.c \
-		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
-
 dbdump.exe:	$(TOP)\ext\misc\dbdump.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DDBDUMP_STANDALONE $(TOP)\ext\misc\dbdump.c $(SQLITE3C) \
 		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS)
diff --git a/main.mk b/main.mk
index 218598b549..8a0ab0674a 100644
--- a/main.mk
+++ b/main.mk
@@ -766,20 +766,6 @@ sqlite3_analyzer$(EXE): sqlite3_analyzer.c
 sqlite3_expert$(EXE): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c
 	$(TCCX) $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert $(THREADLIB)
 
-sqlite3_schemalint.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/schemalint.tcl
-	echo "#define TCLSH 2" > $@
-	echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@
-	cat sqlite3.c $(TOP)/src/tclsqlite.c >> $@
-	echo "static const char *tclsh_main_loop(void){" >> $@
-	echo "static const char *zMainloop = " >> $@
-	tclsh $(TOP)/tool/tostr.tcl $(TOP)/tool/schemalint.tcl >> $@
-	echo "; return zMainloop; }" >> $@
-
-sqlite3_schemalint$(EXE): $(TESTSRC) sqlite3_schemalint.c $(TOP)/ext/session/test_session.c
-	$(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS)                  \
-		sqlite3_schemalint.c $(TESTSRC) $(TOP)/ext/session/test_session.c  \
-		-o sqlite3_schemalint$(EXE) $(LIBTCL) $(THREADLIB)
-
 dbdump$(EXE):	$(TOP)/ext/misc/dbdump.c sqlite3.o
 	$(TCCX) -DDBDUMP_STANDALONE -o dbdump$(EXE) \
             $(TOP)/ext/misc/dbdump.c sqlite3.o $(THREADLIB)
diff --git a/manifest b/manifest
index 77a8c112c8..c5b72ddd17 100644
--- a/manifest
+++ b/manifest
@@ -1,8 +1,8 @@
-C Fix\sa\sharmless\scompiler\swarning\son\sWindows.
-D 2017-05-03T13:05:08.456
-F Makefile.in 2c991e7b1a2bb23c147406c3630b54d99c4931ae1fa0e8c6b6bf40a7a2fd02a3
+C Remove\sthe\stool/schemalint.tcl\sscript.\sAnd\srelated\sMakefile\sentries.\sIt\sis\nsuperseded\sby\ssqlite3_expert.
+D 2017-05-04T14:02:05.547
+F Makefile.in 2594c46dc86cb8cf0bd397c89c16b946ba45cd8c3459471a8634a9a9412a4724
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 2b4876693e76420704bd2defe9f957b902ccb35ebe1fd071b80f70e862b7d444
+F Makefile.msc 35879b76bc9136af684cc54b4178ebc6ab231ac0f24990bef703ad80a9330641
 F README.md 2b15fae33852f2f53996774c21fb41e1d94181c4401a0e43ac93e11f2cc901b9
 F VERSION 0a0e02e16b44ea735b40118fc844311b2ab0d35b25fbeda5120aee62f973f663
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -332,7 +332,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 7fcb8db53d07ea8124c2c43dac5fcee8095e4abeab1acdf114833b8805aa72c9
+F main.mk e916822e73ef2cf567d9fb079735d0bd1fb7afc89f75252a62c72d50730a49f3
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -1537,7 +1537,6 @@ F tool/replace.tcl 60f91e8dd06ab81f74d213ecbd9c9945f32ac048
 F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a
 F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5
 F tool/run-speed-test.sh f95d19fd669b68c4c38b6b475242841d47c66076
-F tool/schemalint.tcl 49690356702d6cac07e2bb1790eac73862e92926
 F tool/showdb.c e6bc9dba233bf1b57ca0a525a2bba762db4e223de84990739db3f09c46151b1e
 F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818
 F tool/showlocks.c 9920bcc64f58378ff1118caead34147201f48c68
@@ -1585,7 +1584,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d8254047b30f7c1be486bf39d4420678604573b951b5cc83c19ebf74aba0864c
-R efc1806e3149882e34f308cbcfce19a8
-U drh
-Z 5eec4984a758352eb665dd5f89e1a5b7
+P 593e5dd00cdf8fbc680951d68b53b38262c61c467703f6a8eb477226cf6beedd
+R bd679d9017e39a4713f5824e1924f07c
+U dan
+Z f03ecb4affa26d84072c6437b5574775
diff --git a/manifest.uuid b/manifest.uuid
index 03edeee37c..fc8dafbb37 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-593e5dd00cdf8fbc680951d68b53b38262c61c467703f6a8eb477226cf6beedd
\ No newline at end of file
+269bf52e27611cd00fa7f73ee98b395a30dec215232cf1c34f6b741112ba530b
\ No newline at end of file
diff --git a/tool/schemalint.tcl b/tool/schemalint.tcl
deleted file mode 100644
index 1c3f2e27c3..0000000000
--- a/tool/schemalint.tcl
+++ /dev/null
@@ -1,594 +0,0 @@
-if {[catch {
-
-set ::VERBOSE 0
-
-proc usage {} {
-  puts stderr "Usage: $::argv0 ?SWITCHES? DATABASE/SCHEMA"
-  puts stderr "  Switches are:"
-  puts stderr "  -select SQL     (recommend indexes for SQL statement)"
-  puts stderr "  -verbose        (increase verbosity of output)"
-  puts stderr "  -test           (run internal tests and then exit)"
-  puts stderr ""
-  exit
-}
-
-# Return the quoted version of identfier $id. Quotes are only added if 
-# they are required by SQLite.
-#
-# This command currently assumes that quotes are required if the 
-# identifier contains any ASCII-range characters that are not 
-# alpha-numeric or underscores.
-#
-proc quote {id} {
-  if {[requires_quote $id]} {
-    set x [string map {\" \"\"} $id]
-    return "\"$x\""
-  }
-  return $id
-}
-proc requires_quote {id} {
-  foreach c [split $id {}] {
-    if {[string is alnum $c]==0 && $c!="_"} {
-      return 1
-    }
-  }
-  return 0
-}
-
-# The argument passed to this command is a Tcl list of identifiers. The
-# value returned is the same list, except with each item quoted and the
-# elements comma-separated.
-#
-proc list_to_sql {L} {
-  set ret [list]
-  foreach l $L {
-    lappend ret [quote $l]
-  }
-  join $ret ", "
-}
-
-proc readfile {zFile} {
-  set fd [open $zFile]
-  set data [read $fd]
-  close $fd
-  return $data
-}
-
-proc process_cmdline_args {ctxvar argv} {
-  upvar $ctxvar G
-  set nArg [llength $argv]
-  set G(database) [lindex $argv end]
-
-  for {set i 0} {$i < [llength $argv]-1} {incr i} {
-    set k [lindex $argv $i]
-    switch -- $k {
-      -select {
-        incr i
-        if {$i>=[llength $argv]-1} usage
-        set zSelect [lindex $argv $i]
-        if {[file readable $zSelect]} {
-          lappend G(lSelect) [readfile $zSelect]
-        } else {
-          lappend G(lSelect) $zSelect
-        }
-      }
-      -verbose {
-        set ::VERBOSE 1
-      }
-      -test {
-        sqlidx_internal_tests
-      }
-      default {
-        usage
-      }
-    }
-  }
-
-  if {$G(database)=="-test"} {
-    sqlidx_internal_tests
-  }
-}
-
-proc open_database {ctxvar} {
-  upvar $ctxvar G
-  sqlite3 db ""
-
-  # Check if the "database" file is really an SQLite database. If so, copy
-  # it into the temp db just opened. Otherwise, assume that it is an SQL
-  # schema and execute it directly.
-  set fd [open $G(database)]
-  set hdr [read $fd 16]
-  if {$hdr == "SQLite format 3\000"} {
-    close $fd
-    sqlite3 db2 $G(database)
-    sqlite3_backup B db main db2 main
-    B step 2000000000
-    set rc [B finish]
-    db2 close
-    if {$rc != "SQLITE_OK"} { error "Failed to load database $G(database)" }
-  } else {
-    append hdr [read $fd]
-    db eval $hdr
-    close $fd
-  }
-}
-
-proc analyze_selects {ctxvar} {
-  upvar $ctxvar G
-  set G(trace) ""
-
-  # Collect a line of xTrace output for each loop in the set of SELECT
-  # statements.
-  proc xTrace {zMsg} { 
-    upvar G G
-    lappend G(trace) $zMsg 
-  }
-  db trace xTrace
-  foreach s $G(lSelect) {
-    set stmt [sqlite3_prepare_v2 db $s -1 dummy]
-    set rc [sqlite3_finalize $stmt]
-    if {$rc!="SQLITE_OK"} {
-      error "Failed to compile SQL: [sqlite3_errmsg db]"
-    }
-  }
-
-  db trace ""
-  if {$::VERBOSE} {
-    foreach t $G(trace) { puts "trace: $t" }
-  }
-
-  # puts $G(trace)
-}
-
-# The argument is a list of the form:
-#
-#    key1 {value1.1 value1.2} key2 {value2.1 value 2.2...}
-#
-# Values lists may be of any length greater than zero. This function returns
-# a list of lists created by pivoting on each values list. i.e. a list
-# consisting of the elements:
-#
-#   {{key1 value1.1} {key2 value2.1}}
-#   {{key1 value1.2} {key2 value2.1}}
-#   {{key1 value1.1} {key2 value2.2}}
-#   {{key1 value1.2} {key2 value2.2}}
-#
-proc expand_eq_list {L} {
-  set ll [list {}]
-  for {set i 0} {$i < [llength $L]} {incr i 2} {
-    set key [lindex $L $i]
-    set new [list]
-    foreach piv [lindex $L $i+1] {
-      foreach l $ll {
-        lappend new [concat $l [list [list $key $piv]]]
-      }
-    }
-    set ll $new
-  }
-
-  return $ll
-}
-
-#--------------------------------------------------------------------------
-# Formulate a CREATE INDEX statement that creates an index on table $tname.
-#
-proc eqset_to_index {ctxvar aCollVar tname eqset {range {}}} {
-  upvar $ctxvar G
-  upvar $aCollVar aColl
-
-  set rangeset [list]
-  foreach e [lsort $eqset] {
-    lappend rangeset [lindex $e 0] [lindex $e 1] ASC
-  }
-  set rangeset [concat $rangeset $range]
-
-  set lCols [list]
-  set idxname $tname
-
-  foreach {c collate dir} $rangeset {
-    append idxname "_$c"
-    set coldef [quote $c]
-
-    if {[string compare -nocase $collate $aColl($c)]!=0} {
-      append idxname [string tolower $collate]
-      append coldef " COLLATE [quote $collate]"
-    }
-
-    if {$dir=="DESC"} {
-      append coldef " DESC"
-      append idxname "desc"
-    }
-    lappend lCols $coldef
-  }
-
-  set create_index "CREATE INDEX [quote $idxname] ON [quote $tname]("
-  append create_index [join $lCols ", "]
-  append create_index ");"
-
-  set G(trial.$idxname) $create_index
-}
-
-proc expand_or_cons {L} {
-  set lRet [list [list]]
-  foreach elem $L {
-    set type [lindex $elem 0]
-    if {$type=="eq" || $type=="range"} {
-      set lNew [list]
-      for {set i 0} {$i < [llength $lRet]} {incr i} {
-        lappend lNew [concat [lindex $lRet $i] [list $elem]]
-      }
-      set lRet $lNew
-    } elseif {$type=="or"} {
-      set lNew [list]
-      foreach branch [lrange $elem 1 end] {
-        foreach b [expand_or_cons $branch] {
-          for {set i 0} {$i < [llength $lRet]} {incr i} {
-            lappend lNew [concat [lindex $lRet $i] $b]
-          }
-        }
-      }
-      set lRet $lNew
-    } 
-  }
-  return $lRet
-}
-
-#--------------------------------------------------------------------------
-# Argument $tname is the name of a table in the main database opened by
-# database handle [db]. $arrayvar is the name of an array variable in the
-# caller's context. This command populates the array with an entry mapping 
-# from column name to default collation sequence for each column of table
-# $tname. For example, if a table is declared:
-#
-#   CREATE TABLE t1(a COLLATE nocase, b, c COLLATE binary)
-#
-# the mapping is populated with:
-#
-#   map(a) -> "nocase"
-#   map(b) -> "binary"
-#   map(c) -> "binary"
-#
-proc sqlidx_get_coll_map {tname arrayvar} {
-  upvar $arrayvar aColl
-  set colnames [list]
-  set qname [quote $tname]
-  db eval "PRAGMA table_info = $qname" x { lappend colnames $x(name) }
-  db eval "CREATE INDEX schemalint_test ON ${qname}([list_to_sql $colnames])"
-  db eval "PRAGMA index_xinfo = schemalint_test" x { 
-    set aColl($x(name)) $x(coll)
-  }
-  db eval "DROP INDEX schemalint_test"
-}
-
-proc find_trial_indexes {ctxvar} {
-  upvar $ctxvar G
-  foreach t $G(trace) {
-    set tname [lindex $t 0]
-    catch { array unset mask }
-
-    # Invoke "PRAGMA table_info" on the table. Use the results to create
-    # an array mapping from column name to collation sequence. Store the
-    # array in local variable aColl.
-    #
-    sqlidx_get_coll_map $tname aColl
-
-    set orderby [list]
-    if {[lindex $t end 0]=="orderby"} {
-      set orderby [lrange [lindex $t end] 1 end]
-    }
-
-    foreach lCons [expand_or_cons [lrange $t 2 end]] {
-
-      # Populate the array mask() so that it contains an entry for each
-      # combination of prerequisite scans that may lead to distinct sets
-      # of constraints being usable.
-      #
-      catch { array unset mask }
-      set mask(0) 1
-      foreach a $lCons {
-        set type [lindex $a 0]
-        if {$type=="eq" || $type=="range"} {
-          set m [lindex $a 3]
-          foreach k [array names mask] { set mask([expr ($k & $m)]) 1 }
-          set mask($m) 1
-        }
-      }
-
-      # Loop once for each distinct prerequisite scan mask identified in
-      # the previous block.
-      #
-      foreach k [array names mask] {
-
-        # Identify the constraints available for prerequisite mask $k. For
-        # each == constraint, set an entry in the eq() array as follows:
-        # 
-        #   set eq(<col>) <collation>
-        #
-        # If there is more than one == constraint for a column, and they use
-        # different collation sequences, <collation> is replaced with a list
-        # of the possible collation sequences. For example, for:
-        #
-        #   SELECT * FROM t1 WHERE a=? COLLATE BINARY AND a=? COLLATE NOCASE
-        #
-        # Set the following entry in the eq() array:
-        #
-        #   set eq(a) {binary nocase}
-        #
-        # For each range constraint found an entry is appended to the $ranges
-        # list. The entry is itself a list of the form {<col> <collation>}.
-        #
-        catch {array unset eq}
-        set ranges [list]
-        foreach a $lCons {
-          set type [lindex $a 0]
-          if {$type=="eq" || $type=="range"} {
-            foreach {type col collate m} $a {
-              if {($m & $k)==$m} {
-                if {$type=="eq"} {
-                  lappend eq($col) $collate
-                } else {
-                  lappend ranges [list $col $collate ASC]
-                }
-              }
-            }
-          }
-        }
-        set ranges [lsort -unique $ranges]
-        if {$orderby != ""} {
-          lappend ranges $orderby
-        }
-
-        foreach eqset [expand_eq_list [array get eq]] {
-          if {$eqset != ""} {
-            eqset_to_index G aColl $tname $eqset
-          }
-
-          foreach r $ranges {
-            set tail [list]
-            foreach {c collate dir} $r {
-              set bSeen 0
-              foreach e $eqset {
-                if {[lindex $e 0] == $c} {
-                  set bSeen 1
-                  break
-                }
-              }
-              if {$bSeen==0} { lappend tail {*}$r }
-            }
-            if {[llength $tail]} {
-              eqset_to_index G aColl $tname $eqset $r
-            }
-          }
-        }
-      }
-    }
-  }
-
-  if {$::VERBOSE} {
-    foreach k [array names G trial.*] { puts "index: $G($k)" }
-  }
-}
-
-proc run_trials {ctxvar} {
-  upvar $ctxvar G
-  set ret [list]
-
-  foreach k [array names G trial.*] {
-    set idxname [lindex [split $k .] 1]
-    db eval $G($k)
-    set pgno [db one {SELECT rootpage FROM sqlite_master WHERE name = $idxname}]
-    set IDX($pgno) $idxname
-  }
-  db eval ANALYZE
-
-  catch { array unset used }
-  foreach s $G(lSelect) {
-    db eval "EXPLAIN $s" x {
-      if {($x(opcode)=="OpenRead" || $x(opcode)=="ReopenIdx")} {
-        if {[info exists IDX($x(p2))]} { set used($IDX($x(p2))) 1 }
-      }
-    }
-    foreach idx [array names used] {
-      lappend ret $G(trial.$idx)
-    }
-  }
-
-  set ret
-}
-
-proc sqlidx_init_context {varname} {
-  upvar $varname G
-  set G(lSelect)  [list]           ;# List of SELECT statements to analyze
-  set G(database) ""               ;# Name of database or SQL schema file
-  set G(trace)    [list]           ;# List of data from xTrace()
-}
-
-#-------------------------------------------------------------------------
-# The following is test code only.
-#
-proc sqlidx_one_test {tn schema select expected} {
-#  if {$tn!=2} return
-  sqlidx_init_context C
-
-  sqlite3 db ""
-  db collate "a b c" [list string compare]
-  db eval $schema
-  lappend C(lSelect) $select
-  analyze_selects C
-  find_trial_indexes C
-
-  set idxlist [run_trials C]
-  if {$idxlist != [list {*}$expected]} {
-    puts stderr "Test $tn failed"
-    puts stderr "Expected: $expected"
-    puts stderr "Got: $idxlist"
-    exit -1
-  }
-
-  db close
-
-  upvar nTest nTest
-  incr nTest
-}
-
-proc sqlidx_internal_tests {} {
-  set nTest 0
-
-
-  # No indexes for a query with no constraints.
-  sqlidx_one_test 0 {
-    CREATE TABLE t1(a, b, c);
-  } {
-    SELECT * FROM t1;
-  } {
-  }
-
-  sqlidx_one_test 1 {
-    CREATE TABLE t1(a, b, c);
-    CREATE TABLE t2(x, y, z);
-  } {
-    SELECT a FROM t1, t2 WHERE a=? AND x=c
-  } {
-    {CREATE INDEX t2_x ON t2(x);}
-    {CREATE INDEX t1_a_c ON t1(a, c);}
-  }
-
-  sqlidx_one_test 2 {
-    CREATE TABLE t1(a, b, c);
-  } {
-    SELECT * FROM t1 WHERE b>?;
-  } {
-    {CREATE INDEX t1_b ON t1(b);}
-  }
-
-  sqlidx_one_test 3 {
-    CREATE TABLE t1(a, b, c);
-  } {
-    SELECT * FROM t1 WHERE b COLLATE nocase BETWEEN ? AND ?
-  } {
-    {CREATE INDEX t1_bnocase ON t1(b COLLATE NOCASE);}
-  }
-
-  sqlidx_one_test 4 {
-    CREATE TABLE t1(a, b, c);
-  } {
-    SELECT a FROM t1 ORDER BY b;
-  } {
-    {CREATE INDEX t1_b ON t1(b);}
-  }
-
-  sqlidx_one_test 5 {
-    CREATE TABLE t1(a, b, c);
-  } {
-    SELECT a FROM t1 WHERE a=? ORDER BY b;
-  } {
-    {CREATE INDEX t1_a_b ON t1(a, b);}
-  }
-
-  sqlidx_one_test 5 {
-    CREATE TABLE t1(a, b, c);
-  } {
-    SELECT min(a) FROM t1
-  } {
-    {CREATE INDEX t1_a ON t1(a);}
-  }
-
-  sqlidx_one_test 6 {
-    CREATE TABLE t1(a, b, c);
-  } {
-    SELECT * FROM t1 ORDER BY a ASC, b COLLATE nocase DESC, c ASC;
-  } {
-    {CREATE INDEX t1_a_bnocasedesc_c ON t1(a, b COLLATE NOCASE DESC, c);}
-  }
-
-  sqlidx_one_test 7 {
-    CREATE TABLE t1(a COLLATE NOCase, b, c);
-  } {
-    SELECT * FROM t1 WHERE a=?
-  } {
-    {CREATE INDEX t1_a ON t1(a);}
-  }
-
-  # Tables with names that require quotes.
-  #
-  sqlidx_one_test 8.1 {
-    CREATE TABLE "t t"(a, b, c);
-  } {
-    SELECT * FROM "t t" WHERE a=?
-  } {
-    {CREATE INDEX "t t_a" ON "t t"(a);}
-  }
-  sqlidx_one_test 8.2 {
-    CREATE TABLE "t t"(a, b, c);
-  } {
-    SELECT * FROM "t t" WHERE b BETWEEN ? AND ?
-  } {
-    {CREATE INDEX "t t_b" ON "t t"(b);}
-  }
-  
-  # Columns with names that require quotes.
-  #
-  sqlidx_one_test 9.1 {
-    CREATE TABLE t3(a, "b b", c);
-  } {
-    SELECT * FROM t3 WHERE "b b" = ?
-  } {
-    {CREATE INDEX "t3_b b" ON t3("b b");}
-  }
-  sqlidx_one_test 9.2 {
-    CREATE TABLE t3(a, "b b", c);
-  } {
-    SELECT * FROM t3 ORDER BY "b b"
-  } {
-    {CREATE INDEX "t3_b b" ON t3("b b");}
-  }
-
-  # Collations with names that require quotes.
-  #
-  sqlidx_one_test 10.1 {
-    CREATE TABLE t4(a, b, c);
-  } {
-    SELECT * FROM t4 ORDER BY c COLLATE "a b c"
-  } {
-    {CREATE INDEX "t4_ca b c" ON t4(c COLLATE "a b c");}
-  }
-  sqlidx_one_test 10.2 {
-    CREATE TABLE t4(a, b, c);
-  } {
-    SELECT * FROM t4 WHERE c = ? COLLATE "a b c"
-  } {
-    {CREATE INDEX "t4_ca b c" ON t4(c COLLATE "a b c");}
-  }
-
-  # Transitive constraints
-  #
-  sqlidx_one_test 11.1 {
-    CREATE TABLE t5(a, b);
-    CREATE TABLE t6(c, d);
-  } {
-    SELECT * FROM t5, t6 WHERE a=? AND b=c AND c=?
-  } {
-    {CREATE INDEX t6_c ON t6(c);} 
-    {CREATE INDEX t5_a_b ON t5(a, b);}
-  }
-
-  puts "All $nTest tests passed"
-  exit
-}
-# End of internal test code.
-#-------------------------------------------------------------------------
-
-if {[info exists ::argv0]==0} { set ::argv0 [info nameofexec] }
-if {[info exists ::argv]==0} usage
-sqlidx_init_context D
-process_cmdline_args D $::argv
-open_database D
-analyze_selects D
-find_trial_indexes D
-foreach idx [run_trials D] { puts $idx }
-
-} err]} {
-  puts "ERROR: $err"
-  puts $errorInfo
-  exit 1
-}

From 7fdb9a7edcd0f17f17c62127ee3912c655aee173 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 11 Oct 2017 20:10:22 +0000
Subject: [PATCH 48/68] Change some internal details to bring this branch
 closer to the code on trunk.

FossilOrigin-Name: 58e42cfd7d4af2279dab8dd895dca7afbdf9035445bc81bd4a7461ea24e846fc
---
 manifest        | 26 +++++++++++++-------------
 manifest.uuid   |  2 +-
 src/build.c     | 12 ++++++------
 src/main.c      | 20 +++++++++++++++++---
 src/prepare.c   |  2 +-
 src/sqliteInt.h |  4 ++--
 src/trigger.c   |  2 +-
 src/vdbe.c      |  6 +++---
 src/vdbeaux.c   |  2 +-
 9 files changed, 45 insertions(+), 31 deletions(-)

diff --git a/manifest b/manifest
index fd41b85331..c1b89c6095 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\schanges\sfrom\strunk.
-D 2017-05-15T17:56:48.105
+C Change\ssome\sinternal\sdetails\sto\sbring\sthis\sbranch\scloser\sto\sthe\scode\son\strunk.
+D 2017-10-11T20:10:22.514
 F Makefile.in 2594c46dc86cb8cf0bd397c89c16b946ba45cd8c3459471a8634a9a9412a4724
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 19636d6f00b600680edca5e437cd895599cffc11ceaa6ddac866373b98a9e644
@@ -355,7 +355,7 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
 F src/btree.c 8c1fd4cfa2b0bf021386e0a1f4e30b64eea7a2c1bc2e0c3e5901a626b1ab6aa9
 F src/btree.h 80f518c0788be6cec8d9f8e13bd8e380df299d2b5e4ac340dc887b0642647cfc
 F src/btreeInt.h a392d353104b4add58b4a59cb185f5d5693dde832c565b77d8d4c343ed98f610
-F src/build.c 3fd46781483b527ee18508e7854e87e60a259211bb9bbf16b6fafaf08a043a64
+F src/build.c 4026a9c554b233e50c5e9ad46963e676cf54dd2306d952aa1eaa07a1bc9ce14f
 F src/callback.c 2e76147783386374bf01b227f752c81ec872d730
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c 47d91a25ad8f199a71a5b1b7b169d6dd0d6e98c5719eca801568798743d1161c
@@ -374,7 +374,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c d4bb3a135948553d18cf992f76f7ed7b18aa0327f250607b5a6671e55d9947d5
 F src/legacy.c e88ed13c2d531decde75d42c2e35623fb9ce3cb0
 F src/loadext.c a72909474dadce771d3669bf84bf689424f6f87d471fee898589c3ef9b2acfd9
-F src/main.c 253de9f00311bd2b9898c93f22556a741e1a63310785c70356141326765fb64a
+F src/main.c d88cf6d3a281c983f4e00a9fbe8be011504e5d98f06237639b3f546da57d197b
 F src/malloc.c e20bb2b48abec52d3faf01cce12e8b4f95973755fafec98d45162dfdab111978
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -404,7 +404,7 @@ F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490
 F src/pcache1.c 1195a21fe28e223e024f900b2011e80df53793f0356a24caace4188b098540dc
 F src/pragma.c 7fef375edafdb7ae9ba938b992aa726e18bf07b0599cfed040a088a262744b7a
 F src/pragma.h 37a1311d0388db480388d7ec09054f7103045eff20d4971f8a433b77f40b9921
-F src/prepare.c 7c46b5c7be9e19a1bf87777f0b7f9fb257b5ff9856c46de49f2354acfbeb4c86
+F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a
 F src/printf.c 8757834f1b54dae512fb25eb1acc8e94a0d15dd2290b58f2563f65973265adb2
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 3e518b962d932a997fae373366880fc028c75706
@@ -414,7 +414,7 @@ F src/shell.c a37d96b20b3644d0eb905df5aa7a0fcf9f6e73c15898337230c760a24a8df794
 F src/sqlite.h.in 1672a3a22b877d5fbd5d3d243591e11e241c5abc321dbf969429c4bf5656acd9
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 58fd0676d3111d02e62e5a35992a7d3da5d3f88753acc174f2d37b774fbbdd28
-F src/sqliteInt.h a8be6c63ce04fc759e3d8ca2dee2fa2d4128b0a4bf2031c3f6e482fd5c5abdfe
+F src/sqliteInt.h c4d3d3902d7beea52c4fdeb8fda1b9e80bcff0f51230751ec98e2def7cbac801
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -471,16 +471,16 @@ F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
 F src/tokenize.c 1003d6d90c6783206c711f0a9397656fa5b055209f4d092caa43bb3bf5215db5
 F src/treeview.c 6cf8d7fe9e63fae57dad1bb57f6615e14eac0c527e43d868e805042cae8ed1f7
-F src/trigger.c 134b8e7b61317ab7b2a2dd12eb1b9aa2e23ac5bc4a05e63e35b3609b6b30a7c0
+F src/trigger.c c9f0810043b265724fdb1bdd466894f984dfc182
 F src/update.c c443935c652af9365e033f756550b5032d02e1b06eb2cb890ed7511ae0c051dc
 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c
 F src/util.c fc081ec6f63448dcd80d3dfad35baecfa104823254a815b081a4d9fe76e1db23
 F src/vacuum.c 1fe4555cd8c9b263afb85b5b4ee3a4a4181ad569
-F src/vdbe.c 356042d11e05064c43242020e8de97acef9fc8931cfc39ae7cf4cf91d6e42c19
+F src/vdbe.c 9bac2bc2313ed682e6f48ccff6644d3263341885bfcbb3cdea7b720c722be2d5
 F src/vdbe.h f7d1456e28875c2dcb964056589b5b7149ab7edf39edeca801596a39bb3d3848
 F src/vdbeInt.h 1ecdacc1322fdd3241ec30c32a480e328a6f864e532dc53fae8e0ab68121aebf
 F src/vdbeapi.c dc904b3c5e459727993c2421e653e29d63223846d129fae98adc782b0a996481
-F src/vdbeaux.c 73f10e298ed1e0565a30d8f8c909534ded5a32075a27542cc624d711ddcdc6b4
+F src/vdbeaux.c 4f54ba67ffc192e06b3b470c09c5db69044673da4e16f371670b60094794ad99
 F src/vdbeblob.c 359891617358deefc85bef7bcf787fa6b77facb9
 F src/vdbemem.c 2c70f8f5de6c71fb99a22c5b83be9fab5c47cdd8e279fa44a8c00cfed06d7e89
 F src/vdbesort.c e72fe02a2121386ba767ede8942e9450878b8fc873abf3d1b6824485f092570c
@@ -1586,7 +1586,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 269bf52e27611cd00fa7f73ee98b395a30dec215232cf1c34f6b741112ba530b bb0d9281588b8cc24bf2f1f10d0c56277004226adaa2ce5037782503b283b45d
-R 7d3a8849d7369ade14c69fc7df3fcb28
-U drh
-Z 969ec953bff5414f5fad1a53a1834b03
+P 6e0f64ab5eafda5d9e61e00c89af3c1ea2c5aa29821da1bdbcab040957b12403
+R ae231083620f188faca2d9c880ca5095
+U dan
+Z b2966b026a12abf27911ccd1f8c99303
diff --git a/manifest.uuid b/manifest.uuid
index 92a71c9011..8fd1e9c5ad 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6e0f64ab5eafda5d9e61e00c89af3c1ea2c5aa29821da1bdbcab040957b12403
\ No newline at end of file
+58e42cfd7d4af2279dab8dd895dca7afbdf9035445bc81bd4a7461ea24e846fc
\ No newline at end of file
diff --git a/src/build.c b/src/build.c
index 47339e82b6..e04406d857 100644
--- a/src/build.c
+++ b/src/build.c
@@ -479,7 +479,7 @@ void sqlite3UnlinkAndDeleteIndex(sqlite3 *db, int iDb, const char *zIdxName){
     }
     freeIndex(db, pIndex);
   }
-  db->bInternChanges = 1;
+  db->flags |= SQLITE_InternChanges;
 }
 
 /*
@@ -551,7 +551,7 @@ void sqlite3ResetAllSchemasOfConnection(sqlite3 *db){
       sqlite3SchemaClear(pDb->pSchema);
     }
   }
-  db->bInternChanges = 0;
+  db->flags &= ~SQLITE_InternChanges;
   sqlite3VtabUnlockList(db);
   sqlite3BtreeLeaveAll(db);
   sqlite3CollapseDatabaseArray(db);
@@ -561,7 +561,7 @@ void sqlite3ResetAllSchemasOfConnection(sqlite3 *db){
 ** This routine is called when a commit occurs.
 */
 void sqlite3CommitInternalChanges(sqlite3 *db){
-  db->bInternChanges = 0;
+  db->flags &= ~SQLITE_InternChanges;
 }
 
 /*
@@ -665,7 +665,7 @@ void sqlite3UnlinkAndDeleteTable(sqlite3 *db, int iDb, const char *zTabName){
   pDb = &db->aDb[iDb];
   p = sqlite3HashInsert(&pDb->pSchema->tblHash, zTabName, 0);
   sqlite3DeleteTable(db, p);
-  db->bInternChanges = 1;
+  db->flags |= SQLITE_InternChanges;
 }
 
 /*
@@ -2051,7 +2051,7 @@ void sqlite3EndTable(
       return;
     }
     pParse->pNewTable = 0;
-    db->bInternChanges = 1;
+    db->flags |= SQLITE_InternChanges;
 
 #ifndef SQLITE_OMIT_ALTERTABLE
     if( !p->pSelect ){
@@ -3320,7 +3320,7 @@ void sqlite3CreateIndex(
       sqlite3OomFault(db);
       goto exit_create_index;
     }
-    db->bInternChanges = 1;
+    db->flags |= SQLITE_InternChanges;
     if( pTblName!=0 ){
       pIndex->tnum = db->init.newTnum;
     }
diff --git a/src/main.c b/src/main.c
index d39c49a73d..2212224741 100644
--- a/src/main.c
+++ b/src/main.c
@@ -801,6 +801,21 @@ int sqlite3_db_config(sqlite3 *db, int op, ...){
       rc = setupLookaside(db, pBuf, sz, cnt);
       break;
     }
+    case SQLITE_DBCONFIG_FULL_EQP: {
+      int onoff = va_arg(ap, int);
+      int *pRes = va_arg(ap, int*);
+      if( onoff>0 ){
+        db->bFullEQP = 1;
+      }else if( onoff==0 ){
+        db->bFullEQP = 0;
+      }
+      sqlite3ExpirePreparedStatements(db);
+      if( pRes ){
+        *pRes = db->bFullEQP;
+      }
+      rc = SQLITE_OK;
+      break;
+    }
     default: {
       static const struct {
         int op;      /* The opcode */
@@ -811,7 +826,6 @@ int sqlite3_db_config(sqlite3 *db, int op, ...){
         { SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, SQLITE_Fts3Tokenizer  },
         { SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, SQLITE_LoadExtension  },
         { SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE,      SQLITE_NoCkptOnClose  },
-        { SQLITE_DBCONFIG_FULL_EQP,              SQLITE_FullEQP  },
       };
       unsigned int i;
       rc = SQLITE_ERROR; /* IMP: R-42790-23372 */
@@ -1253,7 +1267,7 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){
   ** the database rollback and schema reset, which can cause false
   ** corruption reports in some cases.  */
   sqlite3BtreeEnterAll(db);
-  schemaChange = db->bInternChanges && db->init.busy==0;
+  schemaChange = (db->flags & SQLITE_InternChanges)!=0 && db->init.busy==0;
 
   for(i=0; i<db->nDb; i++){
     Btree *p = db->aDb[i].pBt;
@@ -1267,7 +1281,7 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){
   sqlite3VtabRollback(db);
   sqlite3EndBenignMalloc();
 
-  if( db->bInternChanges && db->init.busy==0 ){
+  if( (db->flags&SQLITE_InternChanges)!=0 && db->init.busy==0 ){
     sqlite3ExpirePreparedStatements(db);
     sqlite3ResetAllSchemasOfConnection(db);
   }
diff --git a/src/prepare.c b/src/prepare.c
index aa36cca166..74127bc76b 100644
--- a/src/prepare.c
+++ b/src/prepare.c
@@ -354,7 +354,7 @@ error_out:
 */
 int sqlite3Init(sqlite3 *db, char **pzErrMsg){
   int i, rc;
-  int commit_internal = db->bInternChanges==0;
+  int commit_internal = !(db->flags&SQLITE_InternChanges);
   
   assert( sqlite3_mutex_held(db->mutex) );
   assert( sqlite3BtreeHoldsMutex(db->aDb[0].pBt) );
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index d1c80d5618..af99b6c085 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1334,7 +1334,7 @@ struct sqlite3 {
   u8 mTrace;                    /* zero or more SQLITE_TRACE flags */
   u8 skipBtreeMutex;            /* True if no shared-cache backends */
   u8 nSqlExec;                  /* Number of pending OP_SqlExec opcodes */
-  u8 bInternChanges;            /* There are uncommited schema changes */
+  u8 bFullEQP;                  /* Include triggers in EQP output */
   int nextPagesize;             /* Pagesize after VACUUM if >0 */
   u32 magic;                    /* Magic number for detect library misuse */
   int nChange;                  /* Value returned by sqlite3_changes() */
@@ -1450,7 +1450,7 @@ struct sqlite3 {
 **      SQLITE_CacheSpill    == PAGER_CACHE_SPILL
 */
 #define SQLITE_VdbeTrace      0x00000001  /* True to trace VDBE execution */
-#define SQLITE_FullEQP        0x00000002  /* Include triggers in EQP output */
+#define SQLITE_InternChanges  0x00000002  /* Uncommitted Hash table changes */
 #define SQLITE_FullColNames   0x00000004  /* Show full column names on SELECT */
 #define SQLITE_FullFSync      0x00000008  /* Use full fsync on the backend */
 #define SQLITE_CkptFullFSync  0x00000010  /* Use full fsync for checkpoint */
diff --git a/src/trigger.c b/src/trigger.c
index 197dcda6c3..bdf964084b 100644
--- a/src/trigger.c
+++ b/src/trigger.c
@@ -584,7 +584,7 @@ void sqlite3UnlinkAndDeleteTrigger(sqlite3 *db, int iDb, const char *zName){
       *pp = (*pp)->pNext;
     }
     sqlite3DeleteTrigger(db, pTrigger);
-    db->bInternChanges = 1;
+    db->flags |= SQLITE_InternChanges;
   }
 }
 
diff --git a/src/vdbe.c b/src/vdbe.c
index 6d1b334cdf..e6c964245f 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -3056,7 +3056,7 @@ case OP_Savepoint: {
         int isSchemaChange;
         iSavepoint = db->nSavepoint - iSavepoint - 1;
         if( p1==SAVEPOINT_ROLLBACK ){
-          isSchemaChange = db->bInternChanges;
+          isSchemaChange = (db->flags & SQLITE_InternChanges)!=0;
           for(ii=0; ii<db->nDb; ii++){
             rc = sqlite3BtreeTripAllCursors(db->aDb[ii].pBt,
                                        SQLITE_ABORT_ROLLBACK,
@@ -3075,7 +3075,7 @@ case OP_Savepoint: {
         if( isSchemaChange ){
           sqlite3ExpirePreparedStatements(db);
           sqlite3ResetAllSchemasOfConnection(db);
-          db->bInternChanges = 1;
+          db->flags = (db->flags | SQLITE_InternChanges);
         }
       }
   
@@ -3355,7 +3355,7 @@ case OP_SetCookie: {
   if( pOp->p2==BTREE_SCHEMA_VERSION ){
     /* When the schema cookie changes, record the new cookie internally */
     pDb->pSchema->schema_cookie = pOp->p3;
-    db->bInternChanges = 1;
+    db->flags |= SQLITE_InternChanges;
   }else if( pOp->p2==BTREE_FILE_FORMAT ){
     /* Record changes in the file format */
     pDb->pSchema->file_format = pOp->p3;
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index 87b1d16316..97da4628bd 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -1611,7 +1611,7 @@ int sqlite3VdbeList(
   int i;                               /* Loop counter */
   int rc = SQLITE_OK;                  /* Return code */
   Mem *pMem = &p->aMem[1];             /* First Mem of result set */
-  int bFull = (p->explain==1 || (db->flags & SQLITE_FullEQP));
+  int bFull = (p->explain==1 || db->bFullEQP);
   Op *pOp = 0;
 
   assert( p->explain );

From 85b76a284e63192324782f953bf3df053189def2 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 12 Oct 2017 20:24:09 +0000
Subject: [PATCH 49/68] Fix the EXPLAIN processing so that it returns
 SQLITE_ERROR on an OOM, as it should.

FossilOrigin-Name: 4ec63ef233c9239d3d383a99c234ef6604d55a9838c16f57e56c2a120aeb63c4
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/vdbeaux.c |  9 ++++++---
 3 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index 57a59acd5c..6df14defa2 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\sthis\sbranch\sto\smatch\slatest\strunk.
-D 2017-10-12T14:13:31.376
+C Fix\sthe\sEXPLAIN\sprocessing\sso\sthat\sit\sreturns\sSQLITE_ERROR\son\san\sOOM,\sas\sit\nshould.
+D 2017-10-12T20:24:09.195
 F Makefile.in 51259a193f348a96b0ebc60ad84cfe83fdeecdd1d0ecb24f0e278a35ac1449da
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e3fd2bfdf18211892e3f09db7e2841753fe61cc0747a0b0e87d77406840c3f6e
@@ -535,7 +535,7 @@ F src/vdbe.c 176c0897af0aedecd3abc9afaf7fa80eaa7cf5eaf62583de256a9961df474373
 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97
 F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9
 F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1
-F src/vdbeaux.c 32530e6e635649244399e2230b41f7dcfc95c40b5a4dcf81611b8aeec6b9d064
+F src/vdbeaux.c 2b10f70d6479f990e66d0e7bbd3c58b85cd8388c1dba5b89698b7a42c185d0f2
 F src/vdbeblob.c 635a79b60340a6a14a622ea8dcb081f0a66b1ac3836870c587f232eec08c0286
 F src/vdbemem.c 5c1533bf756918b4e46b2ed2bb82c29c7c651e1e37bbd0a0d8731a68787598ff
 F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f
@@ -1662,7 +1662,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2719cf5c5bbe8e31d18368d54d968af3878ad2e15f0666e18d7b567d7439c451 36acc0a97fdcc6f54f29c68c4e131702f69c3e59e58237ff4e5c647928699956
-R 7d4d36d7e390bddaee9373964ffc0b0e
-U dan
-Z 45c234430da48e38efb9390ff8797b6d
+P d325da6c50f2f9c10f50b3839f87c155876c339879bb8bdb82567b5ff34ef7ba
+R 6202e14e92fa1dfd47a763d4d423b320
+U drh
+Z 6985a8a5847e041f83568bb74bfda40b
diff --git a/manifest.uuid b/manifest.uuid
index 6530f4e47e..ca99924a85 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d325da6c50f2f9c10f50b3839f87c155876c339879bb8bdb82567b5ff34ef7ba
\ No newline at end of file
+4ec63ef233c9239d3d383a99c234ef6604d55a9838c16f57e56c2a120aeb63c4
\ No newline at end of file
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index 727a43ef97..399981f480 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -1653,7 +1653,7 @@ int sqlite3VdbeList(
   releaseMemArray(pMem, 8);
   p->pResultSet = 0;
 
-  if( p->rc==SQLITE_NOMEM_BKPT ){
+  if( p->rc==SQLITE_NOMEM ){
     /* This happens if a malloc() inside a call to sqlite3_column_text() or
     ** sqlite3_column_text16() failed.  */
     sqlite3OomFault(db);
@@ -1720,8 +1720,11 @@ int sqlite3VdbeList(
         if( apSub[j]==pOp->p4.pProgram ) break;
       }
       if( j==nSub ){
-        rc = sqlite3VdbeMemGrow(pSub, nByte, nSub!=0);
-        if( rc!=SQLITE_OK ) break;
+        p->rc = sqlite3VdbeMemGrow(pSub, nByte, nSub!=0);
+        if( p->rc!=SQLITE_OK ){
+          rc = SQLITE_ERROR;
+          break;
+        }
         apSub = (SubProgram **)pSub->z;
         apSub[nSub++] = pOp->p4.pProgram;
         pSub->flags |= MEM_Blob;

From ae542401d801b95b5805f3f291a2f64ee074df90 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 13 Oct 2017 14:20:15 +0000
Subject: [PATCH 50/68] Update main.mk to build the sqlite3_expert program with
 -DSQLITE_THREADSAFE=0 and -DSQLITE_OMIT_LOAD_EXTENSION. To minimize
 dependencies.

FossilOrigin-Name: 4c68ad7da48c941eb1ca0210a85d95467f3f3640bdc55a02686166471806ca4e
---
 main.mk       |  2 +-
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/main.mk b/main.mk
index bfbfee1a4e..a92a8137f5 100644
--- a/main.mk
+++ b/main.mk
@@ -788,7 +788,7 @@ sqlite3_analyzer$(EXE): sqlite3_analyzer.c
 	$(TCCX) $(TCL_FLAGS) sqlite3_analyzer.c -o $@ $(LIBTCL) $(THREADLIB) 
 
 sqlite3_expert$(EXE): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c
-	$(TCCX) $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert $(THREADLIB)
+	$(TCCX) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert $(THREADLIB)
 
 dbdump$(EXE):	$(TOP)/ext/misc/dbdump.c sqlite3.o
 	$(TCCX) -DDBDUMP_STANDALONE -o dbdump$(EXE) \
diff --git a/manifest b/manifest
index 6df14defa2..bf28bea373 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sthe\sEXPLAIN\sprocessing\sso\sthat\sit\sreturns\sSQLITE_ERROR\son\san\sOOM,\sas\sit\nshould.
-D 2017-10-12T20:24:09.195
+C Update\smain.mk\sto\sbuild\sthe\ssqlite3_expert\sprogram\swith\s-DSQLITE_THREADSAFE=0\nand\s-DSQLITE_OMIT_LOAD_EXTENSION.\sTo\sminimize\sdependencies.
+D 2017-10-13T14:20:15.203
 F Makefile.in 51259a193f348a96b0ebc60ad84cfe83fdeecdd1d0ecb24f0e278a35ac1449da
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e3fd2bfdf18211892e3f09db7e2841753fe61cc0747a0b0e87d77406840c3f6e
@@ -388,7 +388,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 4ba2640f924e022ec66cadcb629e7e1d01aeb1932a004f92abc6238eac1c6a56
+F main.mk 6ec9f289c46fdc66928782d7ff0d1b07c2e4c6ca553a3fe4f94e793ac53e8db6
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -1662,7 +1662,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d325da6c50f2f9c10f50b3839f87c155876c339879bb8bdb82567b5ff34ef7ba
-R 6202e14e92fa1dfd47a763d4d423b320
-U drh
-Z 6985a8a5847e041f83568bb74bfda40b
+P 4ec63ef233c9239d3d383a99c234ef6604d55a9838c16f57e56c2a120aeb63c4
+R d11eafd1958fe2d9bdec327b55b3ebe5
+U dan
+Z 3e15c410e04c5e53e4b05cff4437d5be
diff --git a/manifest.uuid b/manifest.uuid
index ca99924a85..424d577e4a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4ec63ef233c9239d3d383a99c234ef6604d55a9838c16f57e56c2a120aeb63c4
\ No newline at end of file
+4c68ad7da48c941eb1ca0210a85d95467f3f3640bdc55a02686166471806ca4e
\ No newline at end of file

From 2da1504432a7e0efb4ccd119db29eb3dea2d6ad8 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 13 Oct 2017 16:24:32 +0000
Subject: [PATCH 51/68] Fix main.mk to name the win32 executable
 "sqlite3_expert.exe", not "sqlite3_expert".

FossilOrigin-Name: e38571d518ff399d1033b33843dae60bc61181cdf692dcd8e2e14746bf0d1e61
---
 main.mk       |  2 +-
 manifest      | 13 +++++++------
 manifest.uuid |  2 +-
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/main.mk b/main.mk
index a92a8137f5..b1bdf7a21c 100644
--- a/main.mk
+++ b/main.mk
@@ -788,7 +788,7 @@ sqlite3_analyzer$(EXE): sqlite3_analyzer.c
 	$(TCCX) $(TCL_FLAGS) sqlite3_analyzer.c -o $@ $(LIBTCL) $(THREADLIB) 
 
 sqlite3_expert$(EXE): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c
-	$(TCCX) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert $(THREADLIB)
+	$(TCCX) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert$(EXE) $(THREADLIB)
 
 dbdump$(EXE):	$(TOP)/ext/misc/dbdump.c sqlite3.o
 	$(TCCX) -DDBDUMP_STANDALONE -o dbdump$(EXE) \
diff --git a/manifest b/manifest
index bf28bea373..b371ad7c22 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\smain.mk\sto\sbuild\sthe\ssqlite3_expert\sprogram\swith\s-DSQLITE_THREADSAFE=0\nand\s-DSQLITE_OMIT_LOAD_EXTENSION.\sTo\sminimize\sdependencies.
-D 2017-10-13T14:20:15.203
+C Fix\smain.mk\sto\sname\sthe\swin32\sexecutable\s"sqlite3_expert.exe",\snot\n"sqlite3_expert".
+D 2017-10-13T16:24:32.830
 F Makefile.in 51259a193f348a96b0ebc60ad84cfe83fdeecdd1d0ecb24f0e278a35ac1449da
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e3fd2bfdf18211892e3f09db7e2841753fe61cc0747a0b0e87d77406840c3f6e
@@ -388,7 +388,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 6ec9f289c46fdc66928782d7ff0d1b07c2e4c6ca553a3fe4f94e793ac53e8db6
+F main.mk 88c1647059bb48a7314285794b2090e7c931d665d526fa4d88952f5040bee9b3
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -1662,7 +1662,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 4ec63ef233c9239d3d383a99c234ef6604d55a9838c16f57e56c2a120aeb63c4
-R d11eafd1958fe2d9bdec327b55b3ebe5
+P 4c68ad7da48c941eb1ca0210a85d95467f3f3640bdc55a02686166471806ca4e
+R 549f733ecdac0d819a10aad7a58032a8
+T +closed 5981969cad5afebb8f8098489d0419cc9ead560a0f22a484230f1886011cd57c
 U dan
-Z 3e15c410e04c5e53e4b05cff4437d5be
+Z b4137401f555133414b7d9121a7d2894
diff --git a/manifest.uuid b/manifest.uuid
index 424d577e4a..2f5abff820 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4c68ad7da48c941eb1ca0210a85d95467f3f3640bdc55a02686166471806ca4e
\ No newline at end of file
+e38571d518ff399d1033b33843dae60bc61181cdf692dcd8e2e14746bf0d1e61
\ No newline at end of file

From 087316ceecb47f18f0f0eb750e7fc3273f50dcca Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 15 Dec 2017 12:22:21 +0000
Subject: [PATCH 52/68] In the LEMON parser generator, provide reduce actions
 with access to the lookahead token.

FossilOrigin-Name: 42af190f4f86ad60de02800054010fafd484ac86ca41e2a13799b2e583eea98c
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 tool/lempar.c | 12 ++++++++++--
 3 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index 1c8585e662..40cc2db652 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\svalueFromExpr()\sonly\sgenerate\sa\sOOM\sfault\sif\sthere\shave\sbeen\snow\sprior\nfaults.
-D 2017-12-13T23:47:55.478
+C In\sthe\sLEMON\sparser\sgenerator,\sprovide\sreduce\sactions\swith\saccess\sto\sthe\nlookahead\stoken.
+D 2017-12-15T12:22:21.137
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a2492b29176edc3c754aa7a2f7daa20cd3fa20a56e3ee64e376092836177c42a
@@ -1602,7 +1602,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
 F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f
 F tool/lemon.c e6056373044d55296d21f81467dba7632bbb81dc49af072b3f0e76338771497e
-F tool/lempar.c 105d0d9cbe5a25d24d4769241ffbfc63ac7c09e6ccee0dc43dcc8a4c4ae4e426
+F tool/lempar.c 967ebf585cd09b11b89d255d213865109a9c4ff075680d22580a2826de288c89
 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9
 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca
@@ -1680,7 +1680,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6de21deac469ab25378656f6f58115a92f5892428c6f2f3545c9bafac37e4a41
-R 44abcc3575bcbddb8f261c0eaf812ba5
+P 3765aaf712998af5ffb6bc680a0c1419f2b5deb47ecbc1835ba5879127c4dbe3
+R 545f6d4adf455d1fa5987c428909cf33
 U drh
-Z 3fcdf29f818607817f06d87e8d680060
+Z 7981b7f16ad8b88f8d8fe8aa12f607a9
diff --git a/manifest.uuid b/manifest.uuid
index f81e89e40c..999df6a24e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-3765aaf712998af5ffb6bc680a0c1419f2b5deb47ecbc1835ba5879127c4dbe3
\ No newline at end of file
+42af190f4f86ad60de02800054010fafd484ac86ca41e2a13799b2e583eea98c
\ No newline at end of file
diff --git a/tool/lempar.c b/tool/lempar.c
index 37a5892195..da81ddd4bc 100644
--- a/tool/lempar.c
+++ b/tool/lempar.c
@@ -651,10 +651,18 @@ static void yy_accept(yyParser*);  /* Forward Declaration */
 /*
 ** Perform a reduce action and the shift that must immediately
 ** follow the reduce.
+**
+** The yyLookahead and yyLookaheadToken parameters provide reduce actions
+** access to the lookahead token (if any).  The yyLookahead will be YYNOCODE
+** if the lookahead token has already been consumed.  As this procedure is
+** only called from one place, optimizing compilers will in-line it, which
+** means that the extra parameters have no performance impact.
 */
 static void yy_reduce(
   yyParser *yypParser,         /* The parser */
-  unsigned int yyruleno        /* Number of the rule by which to reduce */
+  unsigned int yyruleno,       /* Number of the rule by which to reduce */
+  int yyLookahead,             /* Lookahead token, or YYNOCODE if none */
+  ParseTOKENTYPE yyLookaheadToken  /* Value of the lookahead token */
 ){
   int yygoto;                     /* The next state */
   int yyact;                      /* The next action */
@@ -853,7 +861,7 @@ void Parse(
 #endif
       yymajor = YYNOCODE;
     }else if( yyact <= YY_MAX_REDUCE ){
-      yy_reduce(yypParser,yyact-YY_MIN_REDUCE);
+      yy_reduce(yypParser,yyact-YY_MIN_REDUCE,yymajor,yyminor);
     }else{
       assert( yyact == YY_ERROR_ACTION );
       yyminorunion.yy0 = yyminor;

From 2c44e370f198037202dc1c44d8d3efe7aabceb70 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 15 Dec 2017 20:21:17 +0000
Subject: [PATCH 53/68] Enhance the "swarmvtab" extension. See header comments
 in ext/misc/unionvtab.c for details.

FossilOrigin-Name: 01c173651ab22b7b0c139eded6f2ad8504efd09088df8ae6a3471230ebf2306f
---
 ext/misc/unionvtab.c | 305 +++++++++++++++++++++++++++++++++++++++----
 manifest             |  19 +--
 manifest.uuid        |   2 +-
 test/swarmvtab.test  |   2 +-
 test/swarmvtab2.test |   2 +-
 test/swarmvtab3.test | 233 +++++++++++++++++++++++++++++++++
 6 files changed, 522 insertions(+), 41 deletions(-)
 create mode 100644 test/swarmvtab3.test

diff --git a/ext/misc/unionvtab.c b/ext/misc/unionvtab.c
index fc87915b38..14cce84510 100644
--- a/ext/misc/unionvtab.c
+++ b/ext/misc/unionvtab.c
@@ -56,6 +56,8 @@
 **
 ** SWARMVTAB
 **
+**  LEGACY SYNTAX:
+**
 **   A "swarmvtab" virtual table is created similarly to a unionvtab table:
 **
 **     CREATE VIRTUAL TABLE <name>
@@ -66,13 +68,78 @@
 **   the database file containing the source table.  The <callback> option
 **   is optional.  If included, it is the name of an application-defined
 **   SQL function that is invoked with the URI of the file, if the file
-**   does not already exist on disk.
+**   does not already exist on disk when required by swarmvtab.
+**
+**  NEW SYNTAX:
+**
+**   Using the new syntax, a swarmvtab table is created with:
+**
+**      CREATE VIRTUAL TABLE <name> USING swarmvtab(
+**        <sql-statement> [, <options>]
+**      );
+**
+**   where valid <options> are:
+**
+**      missing=<udf-function-name>
+**      openclose=<udf-function-name>
+**      maxopen=<integer>
+**      <sql-parameter>=<text-value>
+**
+**   The <sql-statement> must return the same 4 columns as for a swarmvtab
+**   table in legacy mode. However, it may also return a 5th column - the
+**   "context" column. The text value returned in this column is not used
+**   at all by the swarmvtab implementation, except that it is passed as
+**   an additional argument to the two UDF functions that may be invoked
+**   (see below).
+**
+**   The "missing" option, if present, specifies the name of an SQL UDF
+**   function to be invoked if a database file is not already present on
+**   disk when required by swarmvtab. If the <sql-statement> did not provide
+**   a context column, it is invoked as:
+**
+**     SELECT <missing-udf>(<database filename/uri>);
+**
+**   Or, if there was a context column:
+**
+**     SELECT <missing-udf>(<database filename/uri>, <context>);
+**
+**   The "openclose" option may also specify a UDF function. This function
+**   is invoked right before swarmvtab opens a database, and right after
+**   it closes one. The first argument - or first two arguments, if
+**   <sql-statement> supplied the context column - is the same as for
+**   the "missing" UDF. Following this, the UDF is passed integer value
+**   0 before a db is opened, and 1 right after it is closed. If both
+**   a missing and openclose UDF is supplied, the application should expect
+**   the following sequence of calls (for a single database):
+**
+**      SELECT <openclose-udf>(<db filename>, <context>, 0);
+**      if( db not already on disk ){
+**          SELECT <missing-udf>(<db filename>, <context>);
+**      }
+**      ... swarmvtab uses database ...
+**      SELECT <openclose-udf>(<db filename>, <context>, 1);
+**
+**   The "maxopen" option is used to configure the maximum number of
+**   database files swarmvtab will hold open simultaneously (default 9).
+**
+**   If an option name begins with a ":" character, then it is assumed
+**   to be an SQL parameter. In this case, the specified text value is
+**   bound to the same variable of the <sql-statement> before it is 
+**   executed. It is an error of the named SQL parameter does not exist.
+**   For example:
+**
+**     CREATE VIRTUAL TABLE swarm USING swarmvtab(
+**       'SELECT :path || localfile, tbl, min, max FROM swarmdir',
+**       :path='/home/user/databases/'
+**       missing='missing_func'
+**     );
 */
 
 #include "sqlite3ext.h"
 SQLITE_EXTENSION_INIT1
 #include <assert.h>
 #include <string.h>
+#include <stdlib.h>
 
 #ifndef SQLITE_OMIT_VIRTUALTABLE
 
@@ -128,6 +195,7 @@ struct UnionSrc {
 
   /* Fields used by swarmvtab only */
   char *zFile;                    /* Database file containing table zTab */
+  char *zContext;                 /* Context string, if any */
   int nUser;                      /* Current number of users */
   sqlite3 *db;                    /* Database handle */
   UnionSrc *pNextClosable;        /* Next in list of closable sources */
@@ -145,8 +213,11 @@ struct UnionTab {
   UnionSrc *aSrc;                 /* Array of source tables, sorted by rowid */
 
   /* Used by swarmvtab only */
+  int bHasContext;                /* Has context strings */
   char *zSourceStr;               /* Expected unionSourceToStr() value */
-  char *zNotFoundCallback;        /* UDF to invoke if file not found on open */
+  sqlite3_stmt *pNotFound;        /* UDF to invoke if file not found on open */
+  sqlite3_stmt *pOpenClose;       /* UDF to invoke on open and close */
+
   UnionSrc *pClosable;            /* First in list of closable sources */
   int nOpen;                      /* Current number of open sources */
   int nMaxOpen;                   /* Maximum number of open sources */
@@ -351,6 +422,39 @@ static void unionFinalize(int *pRc, sqlite3_stmt *pStmt, char **pzErr){
   }
 }
 
+/*
+** If an "openclose" UDF was supplied when this virtual table was created,
+** invoke it now. The first argument passed is the name of the database
+** file for source pSrc. The second is integer value bClose.
+**
+** If successful, return SQLITE_OK. Otherwise an SQLite error code. In this
+** case if argument pzErr is not NULL, also set (*pzErr) to an English
+** language error message. The caller is responsible for eventually freeing 
+** any error message using sqlite3_free().
+*/
+static int unionInvokeOpenClose(
+  UnionTab *pTab, 
+  UnionSrc *pSrc, 
+  int bClose,
+  char **pzErr
+){
+  int rc = SQLITE_OK;
+  if( pTab->pOpenClose ){
+    sqlite3_bind_text(pTab->pOpenClose, 1, pSrc->zFile, -1, SQLITE_STATIC);
+    if( pTab->bHasContext ){
+      sqlite3_bind_text(pTab->pOpenClose, 2, pSrc->zContext, -1, SQLITE_STATIC);
+    }
+    sqlite3_bind_int(pTab->pOpenClose, 2+pTab->bHasContext, bClose);
+    sqlite3_step(pTab->pOpenClose);
+    if( SQLITE_OK!=(rc = sqlite3_reset(pTab->pOpenClose)) ){
+      if( pzErr ){
+        *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db));
+      }
+    }
+  }
+  return rc;
+}
+
 /*
 ** This function is a no-op for unionvtab. For swarmvtab, it attempts to
 ** close open database files until at most nMax are open. An SQLite error
@@ -358,13 +462,16 @@ static void unionFinalize(int *pRc, sqlite3_stmt *pStmt, char **pzErr){
 */
 static void unionCloseSources(UnionTab *pTab, int nMax){
   while( pTab->pClosable && pTab->nOpen>nMax ){
+    UnionSrc *p;
     UnionSrc **pp;
     for(pp=&pTab->pClosable; (*pp)->pNextClosable; pp=&(*pp)->pNextClosable);
-    assert( (*pp)->db );
-    sqlite3_close((*pp)->db);
-    (*pp)->db = 0;
+    p = *pp;
+    assert( p->db );
+    sqlite3_close(p->db);
+    p->db = 0;
     *pp = 0;
     pTab->nOpen--;
+    unionInvokeOpenClose(pTab, p, 1, 0);
   }
 }
 
@@ -377,13 +484,18 @@ static int unionDisconnect(sqlite3_vtab *pVtab){
     int i;
     for(i=0; i<pTab->nSrc; i++){
       UnionSrc *pSrc = &pTab->aSrc[i];
+      if( pSrc->db ){
+        unionInvokeOpenClose(pTab, pSrc, 1, 0);
+      }
       sqlite3_free(pSrc->zDb);
       sqlite3_free(pSrc->zTab);
       sqlite3_free(pSrc->zFile);
+      sqlite3_free(pSrc->zContext);
       sqlite3_close(pSrc->db);
     }
+    sqlite3_finalize(pTab->pNotFound);
+    sqlite3_finalize(pTab->pOpenClose);
     sqlite3_free(pTab->zSourceStr);
-    sqlite3_free(pTab->zNotFoundCallback);
     sqlite3_free(pTab->aSrc);
     sqlite3_free(pTab);
   }
@@ -496,29 +608,31 @@ static int unionSourceCheck(UnionTab *pTab, char **pzErr){
   return rc;
 }
 
-
 /*
 ** Try to open the swarmvtab database.  If initially unable, invoke the
 ** not-found callback UDF and then try again.
 */
 static int unionOpenDatabaseInner(UnionTab *pTab, UnionSrc *pSrc, char **pzErr){
-  int rc = SQLITE_OK;
-  static const int openFlags = 
-       SQLITE_OPEN_READONLY | SQLITE_OPEN_URI;
+  static const int openFlags = SQLITE_OPEN_READONLY | SQLITE_OPEN_URI;
+  int rc;
+
+  rc = unionInvokeOpenClose(pTab, pSrc, 0, pzErr);
+  if( rc!=SQLITE_OK ) return rc;
+
   rc = sqlite3_open_v2(pSrc->zFile, &pSrc->db, openFlags, 0);
   if( rc==SQLITE_OK ) return rc;
-  if( pTab->zNotFoundCallback ){
-    char *zSql = sqlite3_mprintf("SELECT \"%w\"(%Q);",
-                    pTab->zNotFoundCallback, pSrc->zFile);
+  if( pTab->pNotFound ){
     sqlite3_close(pSrc->db);
     pSrc->db = 0;
-    if( zSql==0 ){
-      *pzErr = sqlite3_mprintf("out of memory");
-      return SQLITE_NOMEM;
+    sqlite3_bind_text(pTab->pNotFound, 1, pSrc->zFile, -1, SQLITE_STATIC);
+    if( pTab->bHasContext ){
+      sqlite3_bind_text(pTab->pNotFound, 2, pSrc->zContext, -1, SQLITE_STATIC);
+    }
+    sqlite3_step(pTab->pNotFound);
+    if( SQLITE_OK!=(rc = sqlite3_reset(pTab->pNotFound)) ){
+      *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db));
+      return rc;
     }
-    rc = sqlite3_exec(pTab->db, zSql, 0, 0, pzErr);
-    sqlite3_free(zSql);
-    if( rc ) return rc;
     rc = sqlite3_open_v2(pSrc->zFile, &pSrc->db, openFlags, 0);
   }
   if( rc!=SQLITE_OK ){
@@ -572,6 +686,7 @@ static int unionOpenDatabase(UnionTab *pTab, int iSrc, char **pzErr){
     }else{
       sqlite3_close(pSrc->db);
       pSrc->db = 0;
+      unionInvokeOpenClose(pTab, pSrc, 1, 0);
     }
   }
 
@@ -627,6 +742,132 @@ static int unionFinalizeCsrStmt(UnionCsr *pCsr){
   return rc;
 }
 
+/* 
+** Return true if the argument is a space, tab, CR or LF character.
+*/
+static int union_isspace(char c){
+  return (c==' ' || c=='\n' || c=='\r' || c=='\t');
+}
+
+/* 
+** Return true if the argument is an alphanumeric character in the 
+** ASCII range.
+*/
+static int union_isidchar(char c){
+  return ((c>='a' && c<='z') || (c>='A' && c<'Z') || (c>='0' && c<='9'));
+}
+
+/*
+** This function is called to handle all arguments following the first 
+** (the SQL statement) passed to a swarmvtab (not unionvtab) CREATE 
+** VIRTUAL TABLE statement. It may bind parameters to the SQL statement 
+** or configure members of the UnionTab object passed as the second
+** argument.
+**
+** Refer to header comments at the top of this file for a description
+** of the arguments parsed.
+**
+** This function is a no-op if *pRc is other than SQLITE_OK when it is
+** called. Otherwise, if an error occurs, *pRc is set to an SQLite error
+** code. In this case *pzErr may be set to point to a buffer containing
+** an English language error message. It is the responsibility of the 
+** caller to eventually free the buffer using sqlite3_free().
+*/
+static void unionConfigureVtab(
+  int *pRc,                       /* IN/OUT: Error code */
+  UnionTab *pTab,                 /* Table to configure */
+  sqlite3_stmt *pStmt,            /* SQL statement to find sources */
+  int nArg,                       /* Number of entries in azArg[] array */
+  const char * const *azArg,      /* Array of arguments to consider */
+  char **pzErr                    /* OUT: Error message */
+){
+  int rc = *pRc;
+  int i;
+  if( rc==SQLITE_OK ){
+    pTab->bHasContext = (sqlite3_column_count(pStmt)>4);
+  }
+  for(i=0; rc==SQLITE_OK && i<nArg; i++){
+    char *zArg = unionStrdup(&rc, azArg[i]);
+    if( zArg ){
+      int nOpt = 0;               /* Size of option name in bytes */
+      char *zOpt;                 /* Pointer to option name */
+      char *zVal;                 /* Pointer to value */
+
+      unionDequote(zArg);
+      zOpt = zArg;
+      while( union_isspace(*zOpt) ) zOpt++;
+      zVal = zOpt;
+      if( *zVal==':' ) zVal++;
+      while( union_isidchar(*zVal) ) zVal++;
+      nOpt = zVal-zOpt;
+
+      while( union_isspace(*zVal) ) zVal++;
+      if( *zVal=='=' ){
+        zOpt[nOpt] = '\0';
+        zVal++;
+        while( union_isspace(*zVal) ) zVal++;
+        zVal = unionStrdup(&rc, zVal);
+        if( zVal ){
+          unionDequote(zVal);
+          if( zOpt[0]==':' ){
+            /* A value to bind to the SQL statement */
+            int iParam = sqlite3_bind_parameter_index(pStmt, zOpt);
+            if( iParam==0 ){
+              *pzErr = sqlite3_mprintf(
+                  "swarmvtab: no such SQL parameter: %s", zOpt
+              );
+              rc = SQLITE_ERROR;
+            }else{
+              rc = sqlite3_bind_text(pStmt, iParam, zVal, -1, SQLITE_TRANSIENT);
+            }
+          }else if( nOpt==7 && 0==sqlite3_strnicmp(zOpt, "maxopen", 7) ){
+            pTab->nMaxOpen = atoi(zVal);
+            if( pTab->nMaxOpen<=0 ){
+              *pzErr = sqlite3_mprintf("swarmvtab: illegal maxopen value");
+              rc = SQLITE_ERROR;
+            }
+          }else if( nOpt==7 && 0==sqlite3_strnicmp(zOpt, "missing", 7) ){
+            if( pTab->pNotFound ){
+              *pzErr = sqlite3_mprintf(
+                  "swarmvtab: duplicate \"missing\" option");
+              rc = SQLITE_ERROR;
+            }else{
+              pTab->pNotFound = unionPreparePrintf(&rc, pzErr, pTab->db,
+                  "SELECT \"%w\"(?%s)", zVal, pTab->bHasContext ? ",?" : ""
+              );
+            }
+          }else if( nOpt==9 && 0==sqlite3_strnicmp(zOpt, "openclose", 9) ){
+            if( pTab->pOpenClose ){
+              *pzErr = sqlite3_mprintf(
+                  "swarmvtab: duplicate \"openclose\" option");
+              rc = SQLITE_ERROR;
+            }else{
+              pTab->pOpenClose = unionPreparePrintf(&rc, pzErr, pTab->db,
+                  "SELECT \"%w\"(?,?%s)", zVal, pTab->bHasContext ? ",?" : ""
+              );
+            }
+          }else{
+            *pzErr = sqlite3_mprintf("swarmvtab: unrecognized option: %s",zOpt);
+            rc = SQLITE_ERROR;
+          }
+          sqlite3_free(zVal);
+        }
+      }else{
+        if( i==0 && nArg==1 ){
+          pTab->pNotFound = unionPreparePrintf(&rc, pzErr, pTab->db,
+              "SELECT \"%w\"(?)", zArg
+          );
+        }else{
+          *pzErr = sqlite3_mprintf( "swarmvtab: parse error: %s", azArg[i]);
+          rc = SQLITE_ERROR;
+        }
+      }
+      sqlite3_free(zArg);
+    }
+  }
+  *pRc = rc;
+}
+
 /* 
 ** xConnect/xCreate method.
 **
@@ -654,7 +895,7 @@ static int unionConnect(
     /* unionvtab tables may only be created in the temp schema */
     *pzErr = sqlite3_mprintf("%s tables must be created in TEMP schema", zVtab);
     rc = SQLITE_ERROR;
-  }else if( argc!=4 && argc!=5 ){
+  }else if( argc<4 || (argc>4 && bSwarm==0) ){
     *pzErr = sqlite3_mprintf("wrong number of arguments for %s", zVtab);
     rc = SQLITE_ERROR;
   }else{
@@ -673,6 +914,17 @@ static int unionConnect(
 
     /* Allocate the UnionTab structure */
     pTab = unionMalloc(&rc, sizeof(UnionTab));
+    if( pTab ){
+      assert( rc==SQLITE_OK );
+      pTab->db = db;
+      pTab->bSwarm = bSwarm;
+      pTab->nMaxOpen = SWARMVTAB_MAX_OPEN;
+    }
+
+    /* Parse other CVT arguments, if any */
+    if( bSwarm ){
+      unionConfigureVtab(&rc, pTab, pStmt, argc-4, &argv[4], pzErr);
+    }
 
     /* Iterate through the rows returned by the SQL statement specified
     ** as an argument to the CREATE VIRTUAL TABLE statement. */
@@ -715,17 +967,15 @@ static int unionConnect(
         }else{
           pSrc->zDb = unionStrdup(&rc, zDb);
         }
+        if( pTab->bHasContext ){
+          const char *zContext = (const char*)sqlite3_column_text(pStmt, 4);
+          pSrc->zContext = unionStrdup(&rc, zContext);
+        }
       }
     }
     unionFinalize(&rc, pStmt, pzErr);
     pStmt = 0;
 
-    /* Capture the not-found callback UDF name */
-    if( rc==SQLITE_OK && argc>=5 ){
-      pTab->zNotFoundCallback = unionStrdup(&rc, argv[4]);
-      unionDequote(pTab->zNotFoundCallback);
-    }
-
     /* It is an error if the SELECT statement returned zero rows. If only
     ** because there is no way to determine the schema of the virtual 
     ** table in this case.  */
@@ -738,9 +988,6 @@ static int unionConnect(
     ** compatible schemas. For swarmvtab, attach the first database and
     ** check that the first table is a rowid table only.  */
     if( rc==SQLITE_OK ){
-      pTab->db = db;
-      pTab->bSwarm = bSwarm;
-      pTab->nMaxOpen = SWARMVTAB_MAX_OPEN;
       if( bSwarm ){
         rc = unionOpenDatabase(pTab, 0, pzErr);
       }else{
diff --git a/manifest b/manifest
index 40cc2db652..657245305a 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\sthe\sLEMON\sparser\sgenerator,\sprovide\sreduce\sactions\swith\saccess\sto\sthe\nlookahead\stoken.
-D 2017-12-15T12:22:21.137
+C Enhance\sthe\s"swarmvtab"\sextension.\sSee\sheader\scomments\sin\sext/misc/unionvtab.c\nfor\sdetails.
+D 2017-12-15T20:21:17.831
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a2492b29176edc3c754aa7a2f7daa20cd3fa20a56e3ee64e376092836177c42a
@@ -287,7 +287,7 @@ F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52
 F ext/misc/spellfix.c 41cf26c6b89fcaa8798ae10ae64d39c1f1d9d6995152e545bd491c13058b8fac
 F ext/misc/stmt.c 6f16443abb3551e3f5813bb13ba19a30e7032830015b0f92fe0c0453045c0a11
 F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512
-F ext/misc/unionvtab.c 1e0ebc5078e1a916db191bcd88f87e94ea7ba4aa563ee30ff706261cb4b39461
+F ext/misc/unionvtab.c de36c2c45583d68f99e45b392311967066b02e2651d05697da783698b245b387
 F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95
 F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
@@ -1260,8 +1260,9 @@ F test/subselect.test 0966aa8e720224dbd6a5e769a3ec2a723e332303
 F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a
 F test/subtype1.test 7fe09496352f97053af1437150751be2d0a0cae8
 F test/superlock.test ec94f0556b6488d97f71c79f9061ae08d9ab8f12
-F test/swarmvtab.test c2279311b44de032f86a8295a9b06818d864856f9428b4c99eee91a0d419cf25
-F test/swarmvtab2.test 9a3a68a1e58d00f4ed6c68d12d52f2df971b9e22a80a41f6f8c1409abba8e5b4
+F test/swarmvtab.test 9a3fd5ab3e9b3c976ad1b3d7646aab725114f2ac26b59395d0778b33bab6cdaf
+F test/swarmvtab2.test c948cb2fdfc5b01d85e8f6d6504854202dc1a0782ab2a0ed61538f27cbd0aa5c
+F test/swarmvtab3.test c4c8d09e56ae99b90187ac225458f13f373873ea296fc442c7ad7511f25e7314
 F test/swarmvtabfault.test 00aec54665909490f5c383f3cae3b5d18bd97c12490b429ff8752a3027acfa42
 F test/symlink.test c9ebe7330d228249e447038276bfc8a7b22f4849
 F test/sync.test 2f84bdbc2b2df1fcb0220575b4b9f8cea94b7529
@@ -1680,7 +1681,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3765aaf712998af5ffb6bc680a0c1419f2b5deb47ecbc1835ba5879127c4dbe3
-R 545f6d4adf455d1fa5987c428909cf33
-U drh
-Z 7981b7f16ad8b88f8d8fe8aa12f607a9
+P 42af190f4f86ad60de02800054010fafd484ac86ca41e2a13799b2e583eea98c
+R 23f432208fb9ea9e9693297333855e09
+U dan
+Z ee61a0a249e7c85137482dc433509d6a
diff --git a/manifest.uuid b/manifest.uuid
index 999df6a24e..72e6a84022 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-42af190f4f86ad60de02800054010fafd484ac86ca41e2a13799b2e583eea98c
\ No newline at end of file
+01c173651ab22b7b0c139eded6f2ad8504efd09088df8ae6a3471230ebf2306f
\ No newline at end of file
diff --git a/test/swarmvtab.test b/test/swarmvtab.test
index 4cdcf29ca8..9d2919bee1 100644
--- a/test/swarmvtab.test
+++ b/test/swarmvtab.test
@@ -213,7 +213,7 @@ do_catchsql_test 3.1 {
         ("test.db2", "t1", 11, 20)
     ', 'fetch_db_no_such_function'
   );
-} {1 {no such function: fetch_db_no_such_function}}
+} {1 {sql error: no such function: fetch_db_no_such_function}}
 
 do_catchsql_test 3.2 {
   CREATE VIRTUAL TABLE temp.xyz USING swarmvtab(
diff --git a/test/swarmvtab2.test b/test/swarmvtab2.test
index cf1bdd0af9..1cc7fbb378 100644
--- a/test/swarmvtab2.test
+++ b/test/swarmvtab2.test
@@ -14,7 +14,7 @@
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
-set testprefix swarmvtab
+set testprefix swarmvtab2
 do_not_use_codec
 
 ifcapable !vtab {
diff --git a/test/swarmvtab3.test b/test/swarmvtab3.test
new file mode 100644
index 0000000000..70d6c7dca0
--- /dev/null
+++ b/test/swarmvtab3.test
@@ -0,0 +1,233 @@
+# 2017-07-15
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is the "swarmvtab" extension
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix swarmvtab3
+do_not_use_codec
+
+ifcapable !vtab {
+  finish_test
+  return
+}
+
+load_static_extension db unionvtab
+
+set nFile $sqlite_open_file_count
+
+do_execsql_test 1.0 {
+  CREATE TEMP TABLE swarm(id, tbl, minval, maxval);
+}
+
+# Set up 100 databases with filenames "remote_test.dbN", where N is between
+# 0 and 99.
+do_test 1.1 {
+  for {set i 0} {$i < 100} {incr i} {
+    set file remote_test.db$i
+    forcedelete $file
+    forcedelete test.db$i
+    sqlite3 rrr $file
+    rrr eval {
+      CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
+      INSERT INTO t1 VALUES($i, $i);
+    }
+    rrr close
+    db eval {
+      INSERT INTO swarm VALUES($i, 't1', $i, $i);
+    }
+    set ::dbcache(test.db$i) 0
+  }
+} {}
+
+proc missing_db {filename} {
+  set remote "remote_$filename"
+  forcedelete $filename
+  file copy $remote $filename
+}
+db func missing_db missing_db
+
+proc openclose_db {filename bClose} {
+  if {$bClose} {
+    incr ::dbcache($filename) -1
+  } else {
+    incr ::dbcache($filename) 1
+  }
+  if {$::dbcache($filename)==0} {
+    forcedelete $filename
+  }
+}
+db func openclose_db openclose_db
+
+proc check_dbcache {} {
+  set n 0
+  for {set i 0} {$i<100} {incr i} {
+    set exists [file exists test.db$i]
+    if {$exists!=($::dbcache(test.db$i)!=0)} {
+      error "inconsistent ::dbcache and disk"
+    }
+    incr n $exists
+  }
+  return $n
+}
+
+foreach {tn nMaxOpen cvt} {
+  1 5 {
+    CREATE VIRTUAL TABLE temp.s USING swarmvtab(
+        'SELECT :prefix || id, tbl, minval, minval FROM swarm',
+        :prefix='test.db',
+        missing=missing_db,
+        openclose=openclose_db,
+        maxopen=5
+    )
+  }
+
+  2 3 {
+    CREATE VIRTUAL TABLE temp.s USING swarmvtab(
+        'SELECT :prefix || id, tbl, minval, minval FROM swarm',
+        :prefix='test.db',
+        missing =       'missing_db',
+        openclose=[openclose_db],
+        maxopen = 3
+    )
+  }
+
+  3 1 {
+    CREATE VIRTUAL TABLE temp.s USING swarmvtab(
+        'SELECT :prefix||''.''||:suffix||id, tbl, minval, minval FROM swarm',
+        :prefix=test, :suffix=db,
+        missing =       'missing_db',
+        openclose=[openclose_db],
+        maxopen = 1
+    )
+  }
+
+} {
+  execsql { DROP TABLE IF EXISTS s }
+  
+  do_execsql_test 1.$tn.1 $cvt
+
+  do_execsql_test 1.$tn.2 {
+    SELECT b FROM s WHERE a<10;
+  } {0 1 2 3 4 5 6 7 8 9}
+
+  do_test 1.$tn.3 { check_dbcache } $nMaxOpen
+
+  do_execsql_test 1.$tn.4 {
+    SELECT b FROM s WHERE (b%10)=0;
+  } {0 10 20 30 40 50 60 70 80 90}
+
+  do_test 1.$tn.5 { check_dbcache } $nMaxOpen
+}
+
+execsql { DROP TABLE IF EXISTS s }
+for {set i 0} {$i < 100} {incr i} {
+  forcedelete remote_test.db$i
+}
+
+#----------------------------------------------------------------------------
+#
+do_execsql_test 2.0 {
+  DROP TABLE IF EXISTS swarm;
+  CREATE TEMP TABLE swarm(file, tbl, minval, maxval, ctx);
+}
+
+catch { array unset ::dbcache }
+
+# Set up 100 databases with filenames "remote_test.dbN", where N is a
+# random integer between 0 and 1,000,000
+# 0 and 99.
+do_test 2.1 {
+  for {set i 0} {$i < 100} {incr i} {
+    while 1 {
+      set ctx [expr abs(int(rand() *1000000))]
+      if {[info exists ::dbcache($ctx)]==0} break
+    }
+
+    set file test_remote.db$ctx
+    forcedelete $file
+    forcedelete test.db$i
+    sqlite3 rrr $file
+    rrr eval {
+      CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
+      INSERT INTO t1 VALUES($i, $i);
+    }
+    rrr close
+    db eval {
+      INSERT INTO swarm VALUES('test.db' || $i, 't1', $i, $i, $file)
+    }
+    set ::dbcache(test.db$i) 0
+  }
+} {}
+
+proc missing_db {filename ctx} {
+  file copy $ctx $filename
+}
+db func missing_db missing_db
+
+proc openclose_db {filename ctx bClose} {
+  if {$bClose} {
+    incr ::dbcache($filename) -1
+  } else {
+    incr ::dbcache($filename) 1
+  }
+  if {$::dbcache($filename)==0} {
+    forcedelete $filename
+  }
+}
+db func openclose_db openclose_db
+
+proc check_dbcache {} {
+  set n 0
+  foreach k [array names ::dbcache] {
+    set exists [file exists $k]
+    if {$exists!=($::dbcache($k)!=0)} {
+      error "inconsistent ::dbcache and disk ($k)"
+    }
+    incr n $exists
+  }
+  return $n
+}
+
+foreach {tn nMaxOpen cvt} {
+  2 5 {
+    CREATE VIRTUAL TABLE temp.s USING swarmvtab(
+        'SELECT file, tbl, minval, minval, ctx FROM swarm',
+        missing=missing_db,
+        openclose=openclose_db,
+        maxopen=5
+    )
+  }
+} {
+  execsql { DROP TABLE IF EXISTS s }
+  
+  do_execsql_test 1.$tn.1 $cvt
+
+  do_execsql_test 1.$tn.2 {
+    SELECT b FROM s WHERE a<10;
+  } {0 1 2 3 4 5 6 7 8 9}
+
+  do_test 1.$tn.3 { check_dbcache } $nMaxOpen
+
+  do_execsql_test 1.$tn.4 {
+    SELECT b FROM s WHERE (b%10)=0;
+  } {0 10 20 30 40 50 60 70 80 90}
+
+  do_test 1.$tn.5 { check_dbcache } $nMaxOpen
+}
+
+db close
+forcedelete {*}[glob test_remote.db*]
+
+finish_test
+

From 468c649331dc631a5821adb24083bf5d96dfb9dc Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 16 Dec 2017 04:37:15 +0000
Subject: [PATCH 54/68] Add unnecessary initializations to some local variables
 in the rtree module to suppress false-positive compiler warnings coming out
 of MSVC.

FossilOrigin-Name: 64487d658cb3b6c8c67f1e198c70813c963de52599f3ea974bdc2aa432e74de9
---
 ext/rtree/rtree.c |  4 ++--
 manifest          | 14 +++++++-------
 manifest.uuid     |  2 +-
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c
index ff15a192a1..ddabacf408 100644
--- a/ext/rtree/rtree.c
+++ b/ext/rtree/rtree.c
@@ -2021,7 +2021,7 @@ static int ChooseLeaf(
 ){
   int rc;
   int ii;
-  RtreeNode *pNode;
+  RtreeNode *pNode = 0;
   rc = nodeAcquire(pRtree, 1, 0, &pNode);
 
   for(ii=0; rc==SQLITE_OK && ii<(pRtree->iDepth-iHeight); ii++){
@@ -2896,7 +2896,7 @@ static int rtreeDeleteRowid(Rtree *pRtree, sqlite3_int64 iDelete){
   */
   if( rc==SQLITE_OK && pRtree->iDepth>0 && NCELL(pRoot)==1 ){
     int rc2;
-    RtreeNode *pChild;
+    RtreeNode *pChild = 0;
     i64 iChild = nodeGetRowid(pRtree, pRoot, 0);
     rc = nodeAcquire(pRtree, iChild, pRoot, &pChild);
     if( rc==SQLITE_OK ){
diff --git a/manifest b/manifest
index 657245305a..c09e354d38 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Enhance\sthe\s"swarmvtab"\sextension.\sSee\sheader\scomments\sin\sext/misc/unionvtab.c\nfor\sdetails.
-D 2017-12-15T20:21:17.831
+C Add\sunnecessary\sinitializations\sto\ssome\slocal\svariables\sin\sthe\srtree\smodule\nto\ssuppress\sfalse-positive\scompiler\swarnings\scoming\sout\sof\sMSVC.
+D 2017-12-16T04:37:15.013
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a2492b29176edc3c754aa7a2f7daa20cd3fa20a56e3ee64e376092836177c42a
@@ -338,7 +338,7 @@ F ext/repair/test/checkfreelist01.test 3e8aa6aeb4007680c94a8d07b41c339aa635cc782
 F ext/repair/test/checkindex01.test 6945d0ffc0c1dc993b2ce88036b26e0f5d6fcc65da70fc9df27c2647bb358b0f
 F ext/repair/test/test.tcl 686d76d888dffd021f64260abf29a55c57b2cedfa7fc69150b42b1d6119aac3c
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
-F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f
+F ext/rtree/rtree.c 2111f685ae07988622c241f819b56fea60782f56e32f97e334473c59f6083481
 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e
 F ext/rtree/rtree1.test 82a353747fcab1083d114b2ac84723dfefdbf86c1a6e1df57bf588c7d4285436
 F ext/rtree/rtree2.test 5f25b01acd03470067a2d52783b2eb0a50bf836803d4342d20ca39e541220fe2
@@ -1681,7 +1681,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 42af190f4f86ad60de02800054010fafd484ac86ca41e2a13799b2e583eea98c
-R 23f432208fb9ea9e9693297333855e09
-U dan
-Z ee61a0a249e7c85137482dc433509d6a
+P 01c173651ab22b7b0c139eded6f2ad8504efd09088df8ae6a3471230ebf2306f
+R e261edbb5cf2d21265b23f8e0074b2f2
+U drh
+Z c0a264eb11df5fc17a94051fa60030ca
diff --git a/manifest.uuid b/manifest.uuid
index 72e6a84022..9c14ed954f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-01c173651ab22b7b0c139eded6f2ad8504efd09088df8ae6a3471230ebf2306f
\ No newline at end of file
+64487d658cb3b6c8c67f1e198c70813c963de52599f3ea974bdc2aa432e74de9
\ No newline at end of file

From 43efc18669bc081f8937f8dc3f91a3068fd257a7 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 19 Dec 2017 17:42:13 +0000
Subject: [PATCH 55/68] Experimentally add the SQLite expert functionality to
 the shell tool.

FossilOrigin-Name: 51068dbaeaef13bb80af8126b8c4f3a454dee63de5127d706db50bf789533e60
---
 ext/expert/expert1.test |  18 +++--
 main.mk                 |   4 +-
 manifest                |  22 +++---
 manifest.uuid           |   2 +-
 src/shell.c.in          | 154 ++++++++++++++++++++++++++++++++++++++++
 test/tester.tcl         |   8 ++-
 6 files changed, 189 insertions(+), 19 deletions(-)

diff --git a/ext/expert/expert1.test b/ext/expert/expert1.test
index 7995327557..04396baf48 100644
--- a/ext/expert/expert1.test
+++ b/ext/expert/expert1.test
@@ -23,11 +23,8 @@ if {![info exists testdir]} {
 source $testdir/tester.tcl
 set testprefix expert1
 
-if {$tcl_platform(platform)=="windows"} {
-  set CMD "sqlite3_expert.exe"
-} else {
-  set CMD ".././sqlite3_expert"
-}
+set CLI [test_binary_name sqlite3]
+set CMD [test_binary_name sqlite3_expert]
 
 proc squish {txt} {
   regsub -all {[[:space:]]+} $txt { }
@@ -73,6 +70,17 @@ foreach {tn setup} {
       uplevel [list do_test $tn $tst [string trim [squish $res]]]
     }
   }
+  3 {
+    if {![file executable $CLI]} { continue }
+
+    proc do_rec_test {tn sql res} {
+      set res [squish [string trim $res]]
+      set tst [subst -nocommands { 
+        squish [string trim [exec $::CLI test.db ".expert" {$sql;}]]
+      }]
+      uplevel [list do_test $tn $tst $res]
+    }
+  }
 } {
 
   eval $setup
diff --git a/main.mk b/main.mk
index 13a8673e13..8692450441 100644
--- a/main.mk
+++ b/main.mk
@@ -692,7 +692,9 @@ SHELL_SRC = \
 	$(TOP)/src/shell.c.in \
 	$(TOP)/ext/misc/shathree.c \
 	$(TOP)/ext/misc/fileio.c \
-	$(TOP)/ext/misc/completion.c
+	$(TOP)/ext/misc/completion.c \
+	$(TOP)/ext/expert/sqlite3expert.c \
+	$(TOP)/ext/expert/sqlite3expert.h
 
 shell.c:	$(SHELL_SRC) $(TOP)/tool/mkshellc.tcl
 	tclsh $(TOP)/tool/mkshellc.tcl >shell.c
diff --git a/manifest b/manifest
index c3b80112dd..f9adc5dc9a 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\ssqlite3_vtab_collation()\sfunction,\swhich\sallows\san\sxBestIndex\scallback\nto\sdetermine\sthe\scollation\ssequence\sthat\sSQLite\swill\suse\sfor\sa\scomparison.\sAnd\nthe\sSQLITE_DBCONFIG_FULL_EQP\sconfiguration\soption,\swhich\senhances\sthe\soutput\nof\s"EXPLAIN\sQUERY\sPLAN"\sso\sthat\sit\sincludes\sstatements\srun\sby\striggers.\sAnd\nthe\scode\sfor\sthe\ssqlite3_expert\sextension\sand\scommand\sline\sapplication.
-D 2017-12-16T19:36:52.802
+C Experimentally\sadd\sthe\sSQLite\sexpert\sfunctionality\sto\sthe\sshell\stool.
+D 2017-12-19T17:42:13.030
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -42,7 +42,7 @@ F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3
 F ext/expert/expert.c 4791c5e064aea81b2b829fa95228b22283380ee370ea88a1e580103b75516ebf
-F ext/expert/expert1.test 1033e43071b69dc2f4e88fbf03fc7f18846c9865cac14f28c80f581437f09acb
+F ext/expert/expert1.test 4ffbecb378dc542d6fe030f936e2aff79a42c8cdf9fbcbff07db629fb7857b22
 F ext/expert/sqlite3expert.c 6ed4e84a06d1a29b2cf3009c0266573b88602d098055caa46c467154a64e0959
 F ext/expert/sqlite3expert.h af6354f8ee5c9e025024e63fec3bd640a802afcc3099a44d804752cf0791d811
 F ext/expert/test_expert.c 85f5c743a899063fa48296d21de2f32c26d09a21c8582b2a0bc482e8de183e7a
@@ -401,7 +401,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 6ef9e2b1c3f1e46c9e3c2b362531cbc277f7bea1a66fe610a3a7c4173b091ba4
+F main.mk 50bac9920024b5485f06398b3980f09e97ab28cd4b5b6dcd829d2a5e3ce22e7a
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -479,7 +479,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
-F src/shell.c.in 6ffed0c589f5aff180789a8c8abf5b2d3e2eea7470c86b30e797887cb0c9d0e5
+F src/shell.c.in 87a048fabcf0080a78bcdd01e57933369951f7fa7d628f04bad48e900c1899a1
 F src/sqlite.h.in 95afbec6e623843d702e727765efc833586fecc688867f41f87be7db3ff1fa62
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1287,7 +1287,7 @@ F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30
 F test/temptable2.test cd396beb41117a5302fff61767c35fa4270a0d5e
 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637
 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc
-F test/tester.tcl 9948bd856ce8a1c127f2f7900365387a42a917ce0dc87185bdd128fa5b11aff2
+F test/tester.tcl 3ed81b9e1d9718a8d9603596c8a877793d054294053c4277a3d3897eabab3866
 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5
 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58
 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7
@@ -1687,8 +1687,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 64487d658cb3b6c8c67f1e198c70813c963de52599f3ea974bdc2aa432e74de9 d5b597b52a1213cdf382d96f4df3535727be0852b25bafd12bbef54da946c5f2
-R 864c298149f75794a8b0c88eee2c8573
-T +closed d5b597b52a1213cdf382d96f4df3535727be0852b25bafd12bbef54da946c5f2
+P 4c782c950204c09c1d8f857c39c4cf476539ec4e7eee6fd86419d47cf0f8b9e0
+R 02a097d24d17c339f9cf394885a43a5e
+T *branch * expert-in-shell
+T *sym-expert-in-shell *
+T -sym-trunk *
 U dan
-Z 532f71905753147684a848f63aad67ba
+Z d356e0f9c90762c0d67a4b31b4b9510d
diff --git a/manifest.uuid b/manifest.uuid
index 7b68239262..c7ada37e5a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4c782c950204c09c1d8f857c39c4cf476539ec4e7eee6fd86419d47cf0f8b9e0
\ No newline at end of file
+51068dbaeaef13bb80af8126b8c4f3a454dee63de5127d706db50bf789533e60
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 062f76e475..cdd824835a 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -796,6 +796,8 @@ static void shellAddSchemaName(
 INCLUDE ../ext/misc/shathree.c
 INCLUDE ../ext/misc/fileio.c
 INCLUDE ../ext/misc/completion.c
+INCLUDE ../ext/expert/sqlite3expert.h
+INCLUDE ../ext/expert/sqlite3expert.c
 
 #if defined(SQLITE_ENABLE_SESSION)
 /*
@@ -822,6 +824,12 @@ struct SavedModeInfo {
   int colWidth[100];  /* Column widths prior to ".explain on" */
 };
 
+typedef struct ExpertInfo ExpertInfo;
+struct ExpertInfo {
+  sqlite3expert *pExpert;
+  int bVerbose;
+};
+
 /*
 ** State information about the database connection is contained in an
 ** instance of the following structure.
@@ -866,6 +874,7 @@ struct ShellState {
   int nSession;             /* Number of active sessions */
   OpenSession aSession[4];  /* Array of sessions.  [0] is in focus. */
 #endif
+  ExpertInfo expert;        /* Valid if previous command was ".expert OPT..." */
 };
 
 /*
@@ -2249,6 +2258,79 @@ static void exec_prepared_stmt(
   }
 }
 
+/*
+** This function is called to process SQL if the previous shell command
+** was ".expert". It passes the SQL in the second argument directly to
+** the sqlite3expert object.
+**
+** If successful, SQLITE_OK is returned. Otherwise, an SQLite error
+** code. In this case, (*pzErr) may be set to point to a buffer containing
+** an English language error message. It is the responsibility of the
+** caller to eventually free this buffer using sqlite3_free().
+*/
+static int expertHandleSQL(
+  ShellState *pState, 
+  const char *zSql, 
+  char **pzErr
+){
+  assert( pState->expert.pExpert );
+  assert( pzErr==0 || *pzErr==0 );
+  return sqlite3_expert_sql(pState->expert.pExpert, zSql, pzErr);
+}
+
+/*
+** This function is called either to silently clean up the object
+** created by the ".expert" command (if bCancel==1), or to generate a 
+** report from it and then clean it up (if bCancel==0).
+**
+** If successful, SQLITE_OK is returned. Otherwise, an SQLite error
+** code. In this case, (*pzErr) may be set to point to a buffer containing
+** an English language error message. It is the responsibility of the
+** caller to eventually free this buffer using sqlite3_free().
+*/
+static int expertFinish(
+  ShellState *pState,
+  int bCancel,
+  char **pzErr
+){
+  int rc = SQLITE_OK;
+  sqlite3expert *p = pState->expert.pExpert;
+  assert( p );
+  assert( bCancel || pzErr==0 || *pzErr==0 );
+  if( bCancel==0 ){
+    FILE *out = pState->out;
+    int bVerbose = pState->expert.bVerbose;
+
+    rc = sqlite3_expert_analyze(p, pzErr);
+    if( rc==SQLITE_OK ){
+      int nQuery = sqlite3_expert_count(p);
+      int i;
+
+      if( bVerbose ){
+        const char *zCand = sqlite3_expert_report(p,0,EXPERT_REPORT_CANDIDATES);
+        raw_printf(out, "-- Candidates -----------------------------\n");
+        raw_printf(out, "%s\n", zCand);
+      }
+      for(i=0; i<nQuery; i++){
+        const char *zSql = sqlite3_expert_report(p, i, EXPERT_REPORT_SQL);
+        const char *zIdx = sqlite3_expert_report(p, i, EXPERT_REPORT_INDEXES);
+        const char *zEQP = sqlite3_expert_report(p, i, EXPERT_REPORT_PLAN);
+        if( zIdx==0 ) zIdx = "(no new indexes)\n";
+        if( bVerbose ){
+          raw_printf(out, "-- Query %d --------------------------------\n",i+1);
+          raw_printf(out, "%s\n\n", zSql);
+        }
+        raw_printf(out, "%s\n", zIdx);
+        raw_printf(out, "%s\n", zEQP);
+      }
+    }
+  }
+  sqlite3_expert_destroy(p);
+  pState->expert.pExpert = 0;
+  return rc;
+}
+
+
 /*
 ** Execute a statement or set of statements.  Print
 ** any result rows/columns depending on the current mode
@@ -2275,6 +2357,11 @@ static int shell_exec(
     *pzErrMsg = NULL;
   }
 
+  if( pArg->expert.pExpert ){
+    rc = expertHandleSQL(pArg, zSql, pzErrMsg);
+    return expertFinish(pArg, (rc!=SQLITE_OK), pzErrMsg);
+  }
+
   while( zSql[0] && (SQLITE_OK == rc) ){
     static const char *zStmtSql;
     rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zLeftover);
@@ -4068,6 +4155,64 @@ static int lintDotCommand(
   return SQLITE_ERROR;
 }
 
+/*
+** Implementation of ".expert" dot command.
+*/
+static int expertDotCommand(
+  ShellState *pState,             /* Current shell tool state */
+  char **azArg,                   /* Array of arguments passed to dot command */
+  int nArg                        /* Number of entries in azArg[] */
+){
+  int rc = SQLITE_OK;
+  char *zErr = 0;
+  int i;
+  int iSample = 0;
+
+  assert( pState->expert.pExpert==0 );
+  memset(&pState->expert, 0, sizeof(ExpertInfo));
+
+  for(i=1; rc==SQLITE_OK && i<nArg; i++){
+    char *z = azArg[i];
+    int n;
+    if( z[0]=='-' && z[1]=='-' ) z++;
+    n = strlen(z);
+    if( n>=2 && 0==strncmp(z, "-verbose", n) ){
+      pState->expert.bVerbose = 1;
+    }
+    else if( n>=2 && 0==strncmp(z, "-sample", n) ){
+      if( i==(nArg-1) ){
+        raw_printf(stderr, "option requires an argument: %s\n", z);
+        rc = SQLITE_ERROR;
+      }else{
+        iSample = (int)integerValue(azArg[++i]);
+        if( iSample<0 || iSample>100 ){
+          raw_printf(stderr, "value out of range: %s\n", azArg[i]);
+          rc = SQLITE_ERROR;
+        }
+      }
+    }
+    else{
+      raw_printf(stderr, "unknown option: %s\n", z);
+      rc = SQLITE_ERROR;
+    }
+  }
+
+  if( rc==SQLITE_OK ){
+    pState->expert.pExpert = sqlite3_expert_new(pState->db, &zErr);
+    if( pState->expert.pExpert==0 ){
+      raw_printf(stderr, "sqlite3_expert_new: %s\n", zErr);
+      rc = SQLITE_ERROR;
+    }else{
+      sqlite3_expert_config(
+          pState->expert.pExpert, EXPERT_CONFIG_SAMPLE, iSample
+      );
+    }
+  }
+
+  return rc;
+}
+
+
 
 /*
 ** If an input line begins with "." then invoke this routine to
@@ -4082,6 +4227,10 @@ static int do_meta_command(char *zLine, ShellState *p){
   int rc = 0;
   char *azArg[50];
 
+  if( p->expert.pExpert ){
+    expertFinish(p, 1, 0);
+  }
+
   /* Parse the input line into tokens.
   */
   while( zLine[h] && nArg<ArraySize(azArg) ){
@@ -4436,6 +4585,11 @@ static int do_meta_command(char *zLine, ShellState *p){
     }
   }else
 
+  if( c=='e' && strncmp(azArg[0], "expert", n)==0 ){
+    open_db(p, 0);
+    expertDotCommand(p, azArg, nArg);
+  }else
+
   if( c=='f' && strncmp(azArg[0], "fullschema", n)==0 ){
     ShellState data;
     char *zErrMsg = 0;
diff --git a/test/tester.tcl b/test/tester.tcl
index 10a20a47d6..d0d6c92a60 100644
--- a/test/tester.tcl
+++ b/test/tester.tcl
@@ -2271,13 +2271,17 @@ proc test_restore_config_pagecache {} {
   sqlite3 db test.db
 }
 
-proc test_find_binary {nm} {
+proc test_binary_name {nm} {
   if {$::tcl_platform(platform)=="windows"} {
     set ret "$nm.exe"
   } else {
     set ret $nm
   }
-  set ret [file normalize [file join $::cmdlinearg(TESTFIXTURE_HOME) $ret]]
+  file normalize [file join $::cmdlinearg(TESTFIXTURE_HOME) $ret]
+}
+
+proc test_find_binary {nm} {
+  set ret [test_binary_name $nm]
   if {![file executable $ret]} {
     finish_test
     return ""

From e42e1bc5943f6e9e8358af755e6011a907d7527c Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 19 Dec 2017 18:56:28 +0000
Subject: [PATCH 56/68] Fix crash in sqlite3_vtab_collation() when called for
 an IS NOT NULL constraint.

FossilOrigin-Name: ad38d2c4f073705c02c7b38675e8ae86fe4a794d54eb796e7ed51a905824d5f5
---
 ext/expert/expert1.test |  9 +++++++++
 manifest                | 15 +++++++--------
 manifest.uuid           |  2 +-
 src/where.c             |  5 ++++-
 4 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/ext/expert/expert1.test b/ext/expert/expert1.test
index 7995327557..ff71207dcd 100644
--- a/ext/expert/expert1.test
+++ b/ext/expert/expert1.test
@@ -292,6 +292,15 @@ do_setup_rec_test $tn.15 {
   0|1|1|SEARCH TABLE t1 USING INTEGER PRIMARY KEY (rowid=?)
 }
 
+do_setup_rec_test $tn.16 {
+  CREATE TABLE t1(a, b);
+} {
+  SELECT * FROM t1 WHERE b IS NOT NULL;
+} {
+  (no new indexes)
+  0|0|0|SCAN TABLE t1
+}
+
 }
 
 proc do_candidates_test {tn sql res} {
diff --git a/manifest b/manifest
index c3b80112dd..19e4965990 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\ssqlite3_vtab_collation()\sfunction,\swhich\sallows\san\sxBestIndex\scallback\nto\sdetermine\sthe\scollation\ssequence\sthat\sSQLite\swill\suse\sfor\sa\scomparison.\sAnd\nthe\sSQLITE_DBCONFIG_FULL_EQP\sconfiguration\soption,\swhich\senhances\sthe\soutput\nof\s"EXPLAIN\sQUERY\sPLAN"\sso\sthat\sit\sincludes\sstatements\srun\sby\striggers.\sAnd\nthe\scode\sfor\sthe\ssqlite3_expert\sextension\sand\scommand\sline\sapplication.
-D 2017-12-16T19:36:52.802
+C Fix\scrash\sin\ssqlite3_vtab_collation()\swhen\scalled\sfor\san\sIS\sNOT\sNULL\nconstraint.
+D 2017-12-19T18:56:28.856
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -42,7 +42,7 @@ F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3
 F ext/expert/expert.c 4791c5e064aea81b2b829fa95228b22283380ee370ea88a1e580103b75516ebf
-F ext/expert/expert1.test 1033e43071b69dc2f4e88fbf03fc7f18846c9865cac14f28c80f581437f09acb
+F ext/expert/expert1.test 939265e01b8a40551b1338cc3c38b378d7c99479ececbfc5095b2e757f7b8944
 F ext/expert/sqlite3expert.c 6ed4e84a06d1a29b2cf3009c0266573b88602d098055caa46c467154a64e0959
 F ext/expert/sqlite3expert.h af6354f8ee5c9e025024e63fec3bd640a802afcc3099a44d804752cf0791d811
 F ext/expert/test_expert.c 85f5c743a899063fa48296d21de2f32c26d09a21c8582b2a0bc482e8de183e7a
@@ -561,7 +561,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 5a3f464edd64596f601683ed321d12e6fd93c5fb9afdfb3653d6ffd0fee9c48f
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f
-F src/where.c 1617e7e42daf0ace7799204e3e9b7c9e75069aa2a0c8ddd24d1b9569b2a0b7e4
+F src/where.c 0cdfda28708fe75699df2fcdb44d067dc94e7b1f83e3a622dca96a27aaaa2e42
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
 F src/wherecode.c ff2f079097a3bdce6ebabfde1419fba448c9ce5feb7cb964e8bfa2a4e27274ef
 F src/whereexpr.c 427ea8e96ec24f2a7814c67b8024ad664a9c7656264c4566c34743cb23186e46
@@ -1687,8 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 64487d658cb3b6c8c67f1e198c70813c963de52599f3ea974bdc2aa432e74de9 d5b597b52a1213cdf382d96f4df3535727be0852b25bafd12bbef54da946c5f2
-R 864c298149f75794a8b0c88eee2c8573
-T +closed d5b597b52a1213cdf382d96f4df3535727be0852b25bafd12bbef54da946c5f2
+P 4c782c950204c09c1d8f857c39c4cf476539ec4e7eee6fd86419d47cf0f8b9e0
+R 467752e7571913f5d5399376bb34bcd1
 U dan
-Z 532f71905753147684a848f63aad67ba
+Z b7b7f4c9939a0a5799e54a0fcc200864
diff --git a/manifest.uuid b/manifest.uuid
index 7b68239262..c2cbc0a906 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4c782c950204c09c1d8f857c39c4cf476539ec4e7eee6fd86419d47cf0f8b9e0
\ No newline at end of file
+ad38d2c4f073705c02c7b38675e8ae86fe4a794d54eb796e7ed51a905824d5f5
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index 0694031a51..f0ee78a1da 100644
--- a/src/where.c
+++ b/src/where.c
@@ -3160,9 +3160,12 @@ const char *sqlite3_vtab_collation(sqlite3 *db, int iCons){
   struct BestIndexCtx *p = (struct BestIndexCtx*)db->pBestIndexCtx;
   const char *zRet = 0;
   if( p && iCons>=0 && iCons<p->pIdxInfo->nConstraint ){
+    CollSeq *pC = 0;
     int iTerm = p->pIdxInfo->aConstraint[iCons].iTermOffset;
     Expr *pX = p->pWC->a[iTerm].pExpr;
-    CollSeq *pC = sqlite3BinaryCompareCollSeq(p->pParse,pX->pLeft,pX->pRight);
+    if( pX->pLeft ){
+      pC = sqlite3BinaryCompareCollSeq(p->pParse, pX->pLeft, pX->pRight);
+    }
     zRet = (pC ? pC->zName : "BINARY");
   }
   return zRet;

From 0872e268abf94766a92dcacc91681066a4e07d40 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 20 Dec 2017 23:46:29 +0000
Subject: [PATCH 57/68] Lowercase local variable names in the SHA3 extension in
 order to avoid collisions with macros in termios.h.

FossilOrigin-Name: 3ec7371161bd617e40328aa015b09acc2b37b0b5d269a87050a0c57163f92801
---
 ext/misc/shathree.c | 544 ++++++++++++++++++++++----------------------
 manifest            |  14 +-
 manifest.uuid       |   2 +-
 3 files changed, 280 insertions(+), 280 deletions(-)

diff --git a/ext/misc/shathree.c b/ext/misc/shathree.c
index 612e395ccb..e5c95407d6 100644
--- a/ext/misc/shathree.c
+++ b/ext/misc/shathree.c
@@ -78,9 +78,9 @@ struct SHA3Context {
 */
 static void KeccakF1600Step(SHA3Context *p){
   int i;
-  u64 B0, B1, B2, B3, B4;
-  u64 C0, C1, C2, C3, C4;
-  u64 D0, D1, D2, D3, D4;
+  u64 b0, b1, b2, b3, b4;
+  u64 c0, c1, c2, c3, c4;
+  u64 d0, d1, d2, d3, d4;
   static const u64 RC[] = {
     0x0000000000000001ULL,  0x0000000000008082ULL,
     0x800000000000808aULL,  0x8000000080008000ULL,
@@ -95,301 +95,301 @@ static void KeccakF1600Step(SHA3Context *p){
     0x8000000080008081ULL,  0x8000000000008080ULL,
     0x0000000080000001ULL,  0x8000000080008008ULL
   };
-# define A00 (p->u.s[0])
-# define A01 (p->u.s[1])
-# define A02 (p->u.s[2])
-# define A03 (p->u.s[3])
-# define A04 (p->u.s[4])
-# define A10 (p->u.s[5])
-# define A11 (p->u.s[6])
-# define A12 (p->u.s[7])
-# define A13 (p->u.s[8])
-# define A14 (p->u.s[9])
-# define A20 (p->u.s[10])
-# define A21 (p->u.s[11])
-# define A22 (p->u.s[12])
-# define A23 (p->u.s[13])
-# define A24 (p->u.s[14])
-# define A30 (p->u.s[15])
-# define A31 (p->u.s[16])
-# define A32 (p->u.s[17])
-# define A33 (p->u.s[18])
-# define A34 (p->u.s[19])
-# define A40 (p->u.s[20])
-# define A41 (p->u.s[21])
-# define A42 (p->u.s[22])
-# define A43 (p->u.s[23])
-# define A44 (p->u.s[24])
+# define a00 (p->u.s[0])
+# define a01 (p->u.s[1])
+# define a02 (p->u.s[2])
+# define a03 (p->u.s[3])
+# define a04 (p->u.s[4])
+# define a10 (p->u.s[5])
+# define a11 (p->u.s[6])
+# define a12 (p->u.s[7])
+# define a13 (p->u.s[8])
+# define a14 (p->u.s[9])
+# define a20 (p->u.s[10])
+# define a21 (p->u.s[11])
+# define a22 (p->u.s[12])
+# define a23 (p->u.s[13])
+# define a24 (p->u.s[14])
+# define a30 (p->u.s[15])
+# define a31 (p->u.s[16])
+# define a32 (p->u.s[17])
+# define a33 (p->u.s[18])
+# define a34 (p->u.s[19])
+# define a40 (p->u.s[20])
+# define a41 (p->u.s[21])
+# define a42 (p->u.s[22])
+# define a43 (p->u.s[23])
+# define a44 (p->u.s[24])
 # define ROL64(a,x) ((a<<x)|(a>>(64-x)))
 
   for(i=0; i<24; i+=4){
-    C0 = A00^A10^A20^A30^A40;
-    C1 = A01^A11^A21^A31^A41;
-    C2 = A02^A12^A22^A32^A42;
-    C3 = A03^A13^A23^A33^A43;
-    C4 = A04^A14^A24^A34^A44;
-    D0 = C4^ROL64(C1, 1);
-    D1 = C0^ROL64(C2, 1);
-    D2 = C1^ROL64(C3, 1);
-    D3 = C2^ROL64(C4, 1);
-    D4 = C3^ROL64(C0, 1);
+    c0 = a00^a10^a20^a30^a40;
+    c1 = a01^a11^a21^a31^a41;
+    c2 = a02^a12^a22^a32^a42;
+    c3 = a03^a13^a23^a33^a43;
+    c4 = a04^a14^a24^a34^a44;
+    d0 = c4^ROL64(c1, 1);
+    d1 = c0^ROL64(c2, 1);
+    d2 = c1^ROL64(c3, 1);
+    d3 = c2^ROL64(c4, 1);
+    d4 = c3^ROL64(c0, 1);
 
-    B0 = (A00^D0);
-    B1 = ROL64((A11^D1), 44);
-    B2 = ROL64((A22^D2), 43);
-    B3 = ROL64((A33^D3), 21);
-    B4 = ROL64((A44^D4), 14);
-    A00 =   B0 ^((~B1)&  B2 );
-    A00 ^= RC[i];
-    A11 =   B1 ^((~B2)&  B3 );
-    A22 =   B2 ^((~B3)&  B4 );
-    A33 =   B3 ^((~B4)&  B0 );
-    A44 =   B4 ^((~B0)&  B1 );
+    b0 = (a00^d0);
+    b1 = ROL64((a11^d1), 44);
+    b2 = ROL64((a22^d2), 43);
+    b3 = ROL64((a33^d3), 21);
+    b4 = ROL64((a44^d4), 14);
+    a00 =   b0 ^((~b1)&  b2 );
+    a00 ^= RC[i];
+    a11 =   b1 ^((~b2)&  b3 );
+    a22 =   b2 ^((~b3)&  b4 );
+    a33 =   b3 ^((~b4)&  b0 );
+    a44 =   b4 ^((~b0)&  b1 );
 
-    B2 = ROL64((A20^D0), 3);
-    B3 = ROL64((A31^D1), 45);
-    B4 = ROL64((A42^D2), 61);
-    B0 = ROL64((A03^D3), 28);
-    B1 = ROL64((A14^D4), 20);
-    A20 =   B0 ^((~B1)&  B2 );
-    A31 =   B1 ^((~B2)&  B3 );
-    A42 =   B2 ^((~B3)&  B4 );
-    A03 =   B3 ^((~B4)&  B0 );
-    A14 =   B4 ^((~B0)&  B1 );
+    b2 = ROL64((a20^d0), 3);
+    b3 = ROL64((a31^d1), 45);
+    b4 = ROL64((a42^d2), 61);
+    b0 = ROL64((a03^d3), 28);
+    b1 = ROL64((a14^d4), 20);
+    a20 =   b0 ^((~b1)&  b2 );
+    a31 =   b1 ^((~b2)&  b3 );
+    a42 =   b2 ^((~b3)&  b4 );
+    a03 =   b3 ^((~b4)&  b0 );
+    a14 =   b4 ^((~b0)&  b1 );
 
-    B4 = ROL64((A40^D0), 18);
-    B0 = ROL64((A01^D1), 1);
-    B1 = ROL64((A12^D2), 6);
-    B2 = ROL64((A23^D3), 25);
-    B3 = ROL64((A34^D4), 8);
-    A40 =   B0 ^((~B1)&  B2 );
-    A01 =   B1 ^((~B2)&  B3 );
-    A12 =   B2 ^((~B3)&  B4 );
-    A23 =   B3 ^((~B4)&  B0 );
-    A34 =   B4 ^((~B0)&  B1 );
+    b4 = ROL64((a40^d0), 18);
+    b0 = ROL64((a01^d1), 1);
+    b1 = ROL64((a12^d2), 6);
+    b2 = ROL64((a23^d3), 25);
+    b3 = ROL64((a34^d4), 8);
+    a40 =   b0 ^((~b1)&  b2 );
+    a01 =   b1 ^((~b2)&  b3 );
+    a12 =   b2 ^((~b3)&  b4 );
+    a23 =   b3 ^((~b4)&  b0 );
+    a34 =   b4 ^((~b0)&  b1 );
 
-    B1 = ROL64((A10^D0), 36);
-    B2 = ROL64((A21^D1), 10);
-    B3 = ROL64((A32^D2), 15);
-    B4 = ROL64((A43^D3), 56);
-    B0 = ROL64((A04^D4), 27);
-    A10 =   B0 ^((~B1)&  B2 );
-    A21 =   B1 ^((~B2)&  B3 );
-    A32 =   B2 ^((~B3)&  B4 );
-    A43 =   B3 ^((~B4)&  B0 );
-    A04 =   B4 ^((~B0)&  B1 );
+    b1 = ROL64((a10^d0), 36);
+    b2 = ROL64((a21^d1), 10);
+    b3 = ROL64((a32^d2), 15);
+    b4 = ROL64((a43^d3), 56);
+    b0 = ROL64((a04^d4), 27);
+    a10 =   b0 ^((~b1)&  b2 );
+    a21 =   b1 ^((~b2)&  b3 );
+    a32 =   b2 ^((~b3)&  b4 );
+    a43 =   b3 ^((~b4)&  b0 );
+    a04 =   b4 ^((~b0)&  b1 );
 
-    B3 = ROL64((A30^D0), 41);
-    B4 = ROL64((A41^D1), 2);
-    B0 = ROL64((A02^D2), 62);
-    B1 = ROL64((A13^D3), 55);
-    B2 = ROL64((A24^D4), 39);
-    A30 =   B0 ^((~B1)&  B2 );
-    A41 =   B1 ^((~B2)&  B3 );
-    A02 =   B2 ^((~B3)&  B4 );
-    A13 =   B3 ^((~B4)&  B0 );
-    A24 =   B4 ^((~B0)&  B1 );
+    b3 = ROL64((a30^d0), 41);
+    b4 = ROL64((a41^d1), 2);
+    b0 = ROL64((a02^d2), 62);
+    b1 = ROL64((a13^d3), 55);
+    b2 = ROL64((a24^d4), 39);
+    a30 =   b0 ^((~b1)&  b2 );
+    a41 =   b1 ^((~b2)&  b3 );
+    a02 =   b2 ^((~b3)&  b4 );
+    a13 =   b3 ^((~b4)&  b0 );
+    a24 =   b4 ^((~b0)&  b1 );
 
-    C0 = A00^A20^A40^A10^A30;
-    C1 = A11^A31^A01^A21^A41;
-    C2 = A22^A42^A12^A32^A02;
-    C3 = A33^A03^A23^A43^A13;
-    C4 = A44^A14^A34^A04^A24;
-    D0 = C4^ROL64(C1, 1);
-    D1 = C0^ROL64(C2, 1);
-    D2 = C1^ROL64(C3, 1);
-    D3 = C2^ROL64(C4, 1);
-    D4 = C3^ROL64(C0, 1);
+    c0 = a00^a20^a40^a10^a30;
+    c1 = a11^a31^a01^a21^a41;
+    c2 = a22^a42^a12^a32^a02;
+    c3 = a33^a03^a23^a43^a13;
+    c4 = a44^a14^a34^a04^a24;
+    d0 = c4^ROL64(c1, 1);
+    d1 = c0^ROL64(c2, 1);
+    d2 = c1^ROL64(c3, 1);
+    d3 = c2^ROL64(c4, 1);
+    d4 = c3^ROL64(c0, 1);
 
-    B0 = (A00^D0);
-    B1 = ROL64((A31^D1), 44);
-    B2 = ROL64((A12^D2), 43);
-    B3 = ROL64((A43^D3), 21);
-    B4 = ROL64((A24^D4), 14);
-    A00 =   B0 ^((~B1)&  B2 );
-    A00 ^= RC[i+1];
-    A31 =   B1 ^((~B2)&  B3 );
-    A12 =   B2 ^((~B3)&  B4 );
-    A43 =   B3 ^((~B4)&  B0 );
-    A24 =   B4 ^((~B0)&  B1 );
+    b0 = (a00^d0);
+    b1 = ROL64((a31^d1), 44);
+    b2 = ROL64((a12^d2), 43);
+    b3 = ROL64((a43^d3), 21);
+    b4 = ROL64((a24^d4), 14);
+    a00 =   b0 ^((~b1)&  b2 );
+    a00 ^= RC[i+1];
+    a31 =   b1 ^((~b2)&  b3 );
+    a12 =   b2 ^((~b3)&  b4 );
+    a43 =   b3 ^((~b4)&  b0 );
+    a24 =   b4 ^((~b0)&  b1 );
 
-    B2 = ROL64((A40^D0), 3);
-    B3 = ROL64((A21^D1), 45);
-    B4 = ROL64((A02^D2), 61);
-    B0 = ROL64((A33^D3), 28);
-    B1 = ROL64((A14^D4), 20);
-    A40 =   B0 ^((~B1)&  B2 );
-    A21 =   B1 ^((~B2)&  B3 );
-    A02 =   B2 ^((~B3)&  B4 );
-    A33 =   B3 ^((~B4)&  B0 );
-    A14 =   B4 ^((~B0)&  B1 );
+    b2 = ROL64((a40^d0), 3);
+    b3 = ROL64((a21^d1), 45);
+    b4 = ROL64((a02^d2), 61);
+    b0 = ROL64((a33^d3), 28);
+    b1 = ROL64((a14^d4), 20);
+    a40 =   b0 ^((~b1)&  b2 );
+    a21 =   b1 ^((~b2)&  b3 );
+    a02 =   b2 ^((~b3)&  b4 );
+    a33 =   b3 ^((~b4)&  b0 );
+    a14 =   b4 ^((~b0)&  b1 );
 
-    B4 = ROL64((A30^D0), 18);
-    B0 = ROL64((A11^D1), 1);
-    B1 = ROL64((A42^D2), 6);
-    B2 = ROL64((A23^D3), 25);
-    B3 = ROL64((A04^D4), 8);
-    A30 =   B0 ^((~B1)&  B2 );
-    A11 =   B1 ^((~B2)&  B3 );
-    A42 =   B2 ^((~B3)&  B4 );
-    A23 =   B3 ^((~B4)&  B0 );
-    A04 =   B4 ^((~B0)&  B1 );
+    b4 = ROL64((a30^d0), 18);
+    b0 = ROL64((a11^d1), 1);
+    b1 = ROL64((a42^d2), 6);
+    b2 = ROL64((a23^d3), 25);
+    b3 = ROL64((a04^d4), 8);
+    a30 =   b0 ^((~b1)&  b2 );
+    a11 =   b1 ^((~b2)&  b3 );
+    a42 =   b2 ^((~b3)&  b4 );
+    a23 =   b3 ^((~b4)&  b0 );
+    a04 =   b4 ^((~b0)&  b1 );
 
-    B1 = ROL64((A20^D0), 36);
-    B2 = ROL64((A01^D1), 10);
-    B3 = ROL64((A32^D2), 15);
-    B4 = ROL64((A13^D3), 56);
-    B0 = ROL64((A44^D4), 27);
-    A20 =   B0 ^((~B1)&  B2 );
-    A01 =   B1 ^((~B2)&  B3 );
-    A32 =   B2 ^((~B3)&  B4 );
-    A13 =   B3 ^((~B4)&  B0 );
-    A44 =   B4 ^((~B0)&  B1 );
+    b1 = ROL64((a20^d0), 36);
+    b2 = ROL64((a01^d1), 10);
+    b3 = ROL64((a32^d2), 15);
+    b4 = ROL64((a13^d3), 56);
+    b0 = ROL64((a44^d4), 27);
+    a20 =   b0 ^((~b1)&  b2 );
+    a01 =   b1 ^((~b2)&  b3 );
+    a32 =   b2 ^((~b3)&  b4 );
+    a13 =   b3 ^((~b4)&  b0 );
+    a44 =   b4 ^((~b0)&  b1 );
 
-    B3 = ROL64((A10^D0), 41);
-    B4 = ROL64((A41^D1), 2);
-    B0 = ROL64((A22^D2), 62);
-    B1 = ROL64((A03^D3), 55);
-    B2 = ROL64((A34^D4), 39);
-    A10 =   B0 ^((~B1)&  B2 );
-    A41 =   B1 ^((~B2)&  B3 );
-    A22 =   B2 ^((~B3)&  B4 );
-    A03 =   B3 ^((~B4)&  B0 );
-    A34 =   B4 ^((~B0)&  B1 );
+    b3 = ROL64((a10^d0), 41);
+    b4 = ROL64((a41^d1), 2);
+    b0 = ROL64((a22^d2), 62);
+    b1 = ROL64((a03^d3), 55);
+    b2 = ROL64((a34^d4), 39);
+    a10 =   b0 ^((~b1)&  b2 );
+    a41 =   b1 ^((~b2)&  b3 );
+    a22 =   b2 ^((~b3)&  b4 );
+    a03 =   b3 ^((~b4)&  b0 );
+    a34 =   b4 ^((~b0)&  b1 );
 
-    C0 = A00^A40^A30^A20^A10;
-    C1 = A31^A21^A11^A01^A41;
-    C2 = A12^A02^A42^A32^A22;
-    C3 = A43^A33^A23^A13^A03;
-    C4 = A24^A14^A04^A44^A34;
-    D0 = C4^ROL64(C1, 1);
-    D1 = C0^ROL64(C2, 1);
-    D2 = C1^ROL64(C3, 1);
-    D3 = C2^ROL64(C4, 1);
-    D4 = C3^ROL64(C0, 1);
+    c0 = a00^a40^a30^a20^a10;
+    c1 = a31^a21^a11^a01^a41;
+    c2 = a12^a02^a42^a32^a22;
+    c3 = a43^a33^a23^a13^a03;
+    c4 = a24^a14^a04^a44^a34;
+    d0 = c4^ROL64(c1, 1);
+    d1 = c0^ROL64(c2, 1);
+    d2 = c1^ROL64(c3, 1);
+    d3 = c2^ROL64(c4, 1);
+    d4 = c3^ROL64(c0, 1);
 
-    B0 = (A00^D0);
-    B1 = ROL64((A21^D1), 44);
-    B2 = ROL64((A42^D2), 43);
-    B3 = ROL64((A13^D3), 21);
-    B4 = ROL64((A34^D4), 14);
-    A00 =   B0 ^((~B1)&  B2 );
-    A00 ^= RC[i+2];
-    A21 =   B1 ^((~B2)&  B3 );
-    A42 =   B2 ^((~B3)&  B4 );
-    A13 =   B3 ^((~B4)&  B0 );
-    A34 =   B4 ^((~B0)&  B1 );
+    b0 = (a00^d0);
+    b1 = ROL64((a21^d1), 44);
+    b2 = ROL64((a42^d2), 43);
+    b3 = ROL64((a13^d3), 21);
+    b4 = ROL64((a34^d4), 14);
+    a00 =   b0 ^((~b1)&  b2 );
+    a00 ^= RC[i+2];
+    a21 =   b1 ^((~b2)&  b3 );
+    a42 =   b2 ^((~b3)&  b4 );
+    a13 =   b3 ^((~b4)&  b0 );
+    a34 =   b4 ^((~b0)&  b1 );
 
-    B2 = ROL64((A30^D0), 3);
-    B3 = ROL64((A01^D1), 45);
-    B4 = ROL64((A22^D2), 61);
-    B0 = ROL64((A43^D3), 28);
-    B1 = ROL64((A14^D4), 20);
-    A30 =   B0 ^((~B1)&  B2 );
-    A01 =   B1 ^((~B2)&  B3 );
-    A22 =   B2 ^((~B3)&  B4 );
-    A43 =   B3 ^((~B4)&  B0 );
-    A14 =   B4 ^((~B0)&  B1 );
+    b2 = ROL64((a30^d0), 3);
+    b3 = ROL64((a01^d1), 45);
+    b4 = ROL64((a22^d2), 61);
+    b0 = ROL64((a43^d3), 28);
+    b1 = ROL64((a14^d4), 20);
+    a30 =   b0 ^((~b1)&  b2 );
+    a01 =   b1 ^((~b2)&  b3 );
+    a22 =   b2 ^((~b3)&  b4 );
+    a43 =   b3 ^((~b4)&  b0 );
+    a14 =   b4 ^((~b0)&  b1 );
 
-    B4 = ROL64((A10^D0), 18);
-    B0 = ROL64((A31^D1), 1);
-    B1 = ROL64((A02^D2), 6);
-    B2 = ROL64((A23^D3), 25);
-    B3 = ROL64((A44^D4), 8);
-    A10 =   B0 ^((~B1)&  B2 );
-    A31 =   B1 ^((~B2)&  B3 );
-    A02 =   B2 ^((~B3)&  B4 );
-    A23 =   B3 ^((~B4)&  B0 );
-    A44 =   B4 ^((~B0)&  B1 );
+    b4 = ROL64((a10^d0), 18);
+    b0 = ROL64((a31^d1), 1);
+    b1 = ROL64((a02^d2), 6);
+    b2 = ROL64((a23^d3), 25);
+    b3 = ROL64((a44^d4), 8);
+    a10 =   b0 ^((~b1)&  b2 );
+    a31 =   b1 ^((~b2)&  b3 );
+    a02 =   b2 ^((~b3)&  b4 );
+    a23 =   b3 ^((~b4)&  b0 );
+    a44 =   b4 ^((~b0)&  b1 );
 
-    B1 = ROL64((A40^D0), 36);
-    B2 = ROL64((A11^D1), 10);
-    B3 = ROL64((A32^D2), 15);
-    B4 = ROL64((A03^D3), 56);
-    B0 = ROL64((A24^D4), 27);
-    A40 =   B0 ^((~B1)&  B2 );
-    A11 =   B1 ^((~B2)&  B3 );
-    A32 =   B2 ^((~B3)&  B4 );
-    A03 =   B3 ^((~B4)&  B0 );
-    A24 =   B4 ^((~B0)&  B1 );
+    b1 = ROL64((a40^d0), 36);
+    b2 = ROL64((a11^d1), 10);
+    b3 = ROL64((a32^d2), 15);
+    b4 = ROL64((a03^d3), 56);
+    b0 = ROL64((a24^d4), 27);
+    a40 =   b0 ^((~b1)&  b2 );
+    a11 =   b1 ^((~b2)&  b3 );
+    a32 =   b2 ^((~b3)&  b4 );
+    a03 =   b3 ^((~b4)&  b0 );
+    a24 =   b4 ^((~b0)&  b1 );
 
-    B3 = ROL64((A20^D0), 41);
-    B4 = ROL64((A41^D1), 2);
-    B0 = ROL64((A12^D2), 62);
-    B1 = ROL64((A33^D3), 55);
-    B2 = ROL64((A04^D4), 39);
-    A20 =   B0 ^((~B1)&  B2 );
-    A41 =   B1 ^((~B2)&  B3 );
-    A12 =   B2 ^((~B3)&  B4 );
-    A33 =   B3 ^((~B4)&  B0 );
-    A04 =   B4 ^((~B0)&  B1 );
+    b3 = ROL64((a20^d0), 41);
+    b4 = ROL64((a41^d1), 2);
+    b0 = ROL64((a12^d2), 62);
+    b1 = ROL64((a33^d3), 55);
+    b2 = ROL64((a04^d4), 39);
+    a20 =   b0 ^((~b1)&  b2 );
+    a41 =   b1 ^((~b2)&  b3 );
+    a12 =   b2 ^((~b3)&  b4 );
+    a33 =   b3 ^((~b4)&  b0 );
+    a04 =   b4 ^((~b0)&  b1 );
 
-    C0 = A00^A30^A10^A40^A20;
-    C1 = A21^A01^A31^A11^A41;
-    C2 = A42^A22^A02^A32^A12;
-    C3 = A13^A43^A23^A03^A33;
-    C4 = A34^A14^A44^A24^A04;
-    D0 = C4^ROL64(C1, 1);
-    D1 = C0^ROL64(C2, 1);
-    D2 = C1^ROL64(C3, 1);
-    D3 = C2^ROL64(C4, 1);
-    D4 = C3^ROL64(C0, 1);
+    c0 = a00^a30^a10^a40^a20;
+    c1 = a21^a01^a31^a11^a41;
+    c2 = a42^a22^a02^a32^a12;
+    c3 = a13^a43^a23^a03^a33;
+    c4 = a34^a14^a44^a24^a04;
+    d0 = c4^ROL64(c1, 1);
+    d1 = c0^ROL64(c2, 1);
+    d2 = c1^ROL64(c3, 1);
+    d3 = c2^ROL64(c4, 1);
+    d4 = c3^ROL64(c0, 1);
 
-    B0 = (A00^D0);
-    B1 = ROL64((A01^D1), 44);
-    B2 = ROL64((A02^D2), 43);
-    B3 = ROL64((A03^D3), 21);
-    B4 = ROL64((A04^D4), 14);
-    A00 =   B0 ^((~B1)&  B2 );
-    A00 ^= RC[i+3];
-    A01 =   B1 ^((~B2)&  B3 );
-    A02 =   B2 ^((~B3)&  B4 );
-    A03 =   B3 ^((~B4)&  B0 );
-    A04 =   B4 ^((~B0)&  B1 );
+    b0 = (a00^d0);
+    b1 = ROL64((a01^d1), 44);
+    b2 = ROL64((a02^d2), 43);
+    b3 = ROL64((a03^d3), 21);
+    b4 = ROL64((a04^d4), 14);
+    a00 =   b0 ^((~b1)&  b2 );
+    a00 ^= RC[i+3];
+    a01 =   b1 ^((~b2)&  b3 );
+    a02 =   b2 ^((~b3)&  b4 );
+    a03 =   b3 ^((~b4)&  b0 );
+    a04 =   b4 ^((~b0)&  b1 );
 
-    B2 = ROL64((A10^D0), 3);
-    B3 = ROL64((A11^D1), 45);
-    B4 = ROL64((A12^D2), 61);
-    B0 = ROL64((A13^D3), 28);
-    B1 = ROL64((A14^D4), 20);
-    A10 =   B0 ^((~B1)&  B2 );
-    A11 =   B1 ^((~B2)&  B3 );
-    A12 =   B2 ^((~B3)&  B4 );
-    A13 =   B3 ^((~B4)&  B0 );
-    A14 =   B4 ^((~B0)&  B1 );
+    b2 = ROL64((a10^d0), 3);
+    b3 = ROL64((a11^d1), 45);
+    b4 = ROL64((a12^d2), 61);
+    b0 = ROL64((a13^d3), 28);
+    b1 = ROL64((a14^d4), 20);
+    a10 =   b0 ^((~b1)&  b2 );
+    a11 =   b1 ^((~b2)&  b3 );
+    a12 =   b2 ^((~b3)&  b4 );
+    a13 =   b3 ^((~b4)&  b0 );
+    a14 =   b4 ^((~b0)&  b1 );
 
-    B4 = ROL64((A20^D0), 18);
-    B0 = ROL64((A21^D1), 1);
-    B1 = ROL64((A22^D2), 6);
-    B2 = ROL64((A23^D3), 25);
-    B3 = ROL64((A24^D4), 8);
-    A20 =   B0 ^((~B1)&  B2 );
-    A21 =   B1 ^((~B2)&  B3 );
-    A22 =   B2 ^((~B3)&  B4 );
-    A23 =   B3 ^((~B4)&  B0 );
-    A24 =   B4 ^((~B0)&  B1 );
+    b4 = ROL64((a20^d0), 18);
+    b0 = ROL64((a21^d1), 1);
+    b1 = ROL64((a22^d2), 6);
+    b2 = ROL64((a23^d3), 25);
+    b3 = ROL64((a24^d4), 8);
+    a20 =   b0 ^((~b1)&  b2 );
+    a21 =   b1 ^((~b2)&  b3 );
+    a22 =   b2 ^((~b3)&  b4 );
+    a23 =   b3 ^((~b4)&  b0 );
+    a24 =   b4 ^((~b0)&  b1 );
 
-    B1 = ROL64((A30^D0), 36);
-    B2 = ROL64((A31^D1), 10);
-    B3 = ROL64((A32^D2), 15);
-    B4 = ROL64((A33^D3), 56);
-    B0 = ROL64((A34^D4), 27);
-    A30 =   B0 ^((~B1)&  B2 );
-    A31 =   B1 ^((~B2)&  B3 );
-    A32 =   B2 ^((~B3)&  B4 );
-    A33 =   B3 ^((~B4)&  B0 );
-    A34 =   B4 ^((~B0)&  B1 );
+    b1 = ROL64((a30^d0), 36);
+    b2 = ROL64((a31^d1), 10);
+    b3 = ROL64((a32^d2), 15);
+    b4 = ROL64((a33^d3), 56);
+    b0 = ROL64((a34^d4), 27);
+    a30 =   b0 ^((~b1)&  b2 );
+    a31 =   b1 ^((~b2)&  b3 );
+    a32 =   b2 ^((~b3)&  b4 );
+    a33 =   b3 ^((~b4)&  b0 );
+    a34 =   b4 ^((~b0)&  b1 );
 
-    B3 = ROL64((A40^D0), 41);
-    B4 = ROL64((A41^D1), 2);
-    B0 = ROL64((A42^D2), 62);
-    B1 = ROL64((A43^D3), 55);
-    B2 = ROL64((A44^D4), 39);
-    A40 =   B0 ^((~B1)&  B2 );
-    A41 =   B1 ^((~B2)&  B3 );
-    A42 =   B2 ^((~B3)&  B4 );
-    A43 =   B3 ^((~B4)&  B0 );
-    A44 =   B4 ^((~B0)&  B1 );
+    b3 = ROL64((a40^d0), 41);
+    b4 = ROL64((a41^d1), 2);
+    b0 = ROL64((a42^d2), 62);
+    b1 = ROL64((a43^d3), 55);
+    b2 = ROL64((a44^d4), 39);
+    a40 =   b0 ^((~b1)&  b2 );
+    a41 =   b1 ^((~b2)&  b3 );
+    a42 =   b2 ^((~b3)&  b4 );
+    a43 =   b3 ^((~b4)&  b0 );
+    a44 =   b4 ^((~b0)&  b1 );
   }
 }
 
diff --git a/manifest b/manifest
index 19e4965990..c1dd4fcf09 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\scrash\sin\ssqlite3_vtab_collation()\swhen\scalled\sfor\san\sIS\sNOT\sNULL\nconstraint.
-D 2017-12-19T18:56:28.856
+C Lowercase\slocal\svariable\snames\sin\sthe\sSHA3\sextension\sin\sorder\sto\savoid\ncollisions\swith\smacros\sin\stermios.h.
+D 2017-12-20T23:46:29.934
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -288,7 +288,7 @@ F ext/misc/rot13.c 540a169cb0d74f15522a8930b0cccdcb37a4fd071d219a5a083a319fc6e8d
 F ext/misc/scrub.c 1c5bfb8b0cd18b602fcb55755e84abf0023ac2fb
 F ext/misc/series.c f3c0dba5c5c749ce1782b53076108f87cf0b71041eb6023f727a9c50681da564
 F ext/misc/sha1.c 0b9e9b855354910d3ca467bf39099d570e73db56
-F ext/misc/shathree.c fa185d7aee0ad0aca5e091b4a2db7baff11796170e5793b5de99e511a13af448
+F ext/misc/shathree.c 9e960ba50483214c6a7a4b1517f8d8cef799e9db381195178c3fd3ad207e10c0
 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52
 F ext/misc/spellfix.c 41cf26c6b89fcaa8798ae10ae64d39c1f1d9d6995152e545bd491c13058b8fac
 F ext/misc/stmt.c 6f16443abb3551e3f5813bb13ba19a30e7032830015b0f92fe0c0453045c0a11
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 4c782c950204c09c1d8f857c39c4cf476539ec4e7eee6fd86419d47cf0f8b9e0
-R 467752e7571913f5d5399376bb34bcd1
-U dan
-Z b7b7f4c9939a0a5799e54a0fcc200864
+P ad38d2c4f073705c02c7b38675e8ae86fe4a794d54eb796e7ed51a905824d5f5
+R 5daaca0715e80715d469ec030422b144
+U drh
+Z 1310a0a150441991e0cda5ffa4262cdc
diff --git a/manifest.uuid b/manifest.uuid
index c2cbc0a906..984372aeb6 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ad38d2c4f073705c02c7b38675e8ae86fe4a794d54eb796e7ed51a905824d5f5
\ No newline at end of file
+3ec7371161bd617e40328aa015b09acc2b37b0b5d269a87050a0c57163f92801
\ No newline at end of file

From 3c5d285762ee7293bd262a2e7f19c350ff275660 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 21 Dec 2017 14:46:11 +0000
Subject: [PATCH 58/68] Add the SQLITE_DBCONFIG_MAX preprocessor macro which
 will always be equal to the largest SQLITE_DECONFIG value.

FossilOrigin-Name: 95d4103348d14f9d4c3fee30bd1d882f717b047b2b92f8fef52bec4878281907
---
 manifest        | 13 ++++++-------
 manifest.uuid   |  2 +-
 src/sqlite.h.in |  3 ++-
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index df2d29e242..9e1163e1b8 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\sexperimental\s".expert"\scommand\sto\sthe\ssqlite3.exe\sshell.
-D 2017-12-21T02:17:02.540
+C Add\sthe\sSQLITE_DBCONFIG_MAX\spreprocessor\smacro\swhich\swill\salways\sbe\sequal\sto\nthe\slargest\sSQLITE_DECONFIG\svalue.
+D 2017-12-21T14:46:11.149
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -480,7 +480,7 @@ F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
 F src/shell.c.in 87a048fabcf0080a78bcdd01e57933369951f7fa7d628f04bad48e900c1899a1
-F src/sqlite.h.in 95afbec6e623843d702e727765efc833586fecc688867f41f87be7db3ff1fa62
+F src/sqlite.h.in e00a8c4910e64bd72c1808e9dff80a7a6323f1f036cdd3fdf927c8eae6bb33d2
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
 F src/sqliteInt.h 4d6580ab911e9bc36447ef05bf4d2aab929dde161ac117c7de6220eef06d6175
@@ -1687,8 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3ec7371161bd617e40328aa015b09acc2b37b0b5d269a87050a0c57163f92801 51068dbaeaef13bb80af8126b8c4f3a454dee63de5127d706db50bf789533e60
-R 4f8ab13364d6bba7088ea515e80d086d
-T +closed 51068dbaeaef13bb80af8126b8c4f3a454dee63de5127d706db50bf789533e60
+P 0821bae7afefed98102c81104b4a477e81816bb1f43353c80865411771e3c5a7
+R 0d03176c5debc93f715c8f8149153033
 U drh
-Z b5148bb6ba613d796ac4de4909e11d71
+Z b349f9c48e8b618dcebb916f768288b4
diff --git a/manifest.uuid b/manifest.uuid
index c17fb2dc41..f0979207cd 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0821bae7afefed98102c81104b4a477e81816bb1f43353c80865411771e3c5a7
\ No newline at end of file
+95d4103348d14f9d4c3fee30bd1d882f717b047b2b92f8fef52bec4878281907
\ No newline at end of file
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index f4d44b52ff..671d3dcebb 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -2091,6 +2091,7 @@ struct sqlite3_mem_methods {
 #define SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE      1006 /* int int* */
 #define SQLITE_DBCONFIG_ENABLE_QPSG           1007 /* int int* */
 #define SQLITE_DBCONFIG_FULL_EQP              1008 /* int int* */
+#define SQLITE_DBCONFIG_MAX                   1008 /* Largest DBCONFIG */
 
 /*
 ** CAPI3REF: Enable Or Disable Extended Result Codes
@@ -7041,7 +7042,7 @@ int sqlite3_test_control(int op, ...);
 #define SQLITE_TESTCTRL_ISINIT                  23
 #define SQLITE_TESTCTRL_SORTER_MMAP             24
 #define SQLITE_TESTCTRL_IMPOSTER                25
-#define SQLITE_TESTCTRL_LAST                    25
+#define SQLITE_TESTCTRL_LAST                    25  /* Largest TESTCTRL */
 
 /*
 ** CAPI3REF: SQLite Runtime Status

From 36e31c6915763de55f7e2968a20084fc7252fcd9 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 21 Dec 2017 18:23:26 +0000
Subject: [PATCH 59/68] Change the name of SQLITE_DBCONFIG_FULL_EQP to be
 SQLITE_DBCONFIG_TRIGGER_EQP (which we can do without breaking compatibility
 because the former name has not yet appeared in an official release) and
 streamline its implementation.

FossilOrigin-Name: fffc7685d19f78ec322a4e834ad727af20a17e2e1c35680e4b1c4162c4786f60
---
 ext/expert/sqlite3expert.c |  2 +-
 manifest                   | 20 ++++++++++----------
 manifest.uuid              |  2 +-
 src/main.c                 | 16 +---------------
 src/sqlite.h.in            |  4 ++--
 src/sqliteInt.h            |  5 +++--
 src/vdbeaux.c              |  6 +++---
 7 files changed, 21 insertions(+), 34 deletions(-)

diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index 8627d196f0..b8b19c66a6 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -1734,7 +1734,7 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){
   if( rc==SQLITE_OK ){
     rc = sqlite3_open(":memory:", &pNew->dbm);
     if( rc==SQLITE_OK ){
-      sqlite3_db_config(pNew->dbm, SQLITE_DBCONFIG_FULL_EQP, 1, (int*)0);
+      sqlite3_db_config(pNew->dbm, SQLITE_DBCONFIG_TRIGGER_EQP, 1, (int*)0);
     }
   }
   
diff --git a/manifest b/manifest
index 9e1163e1b8..361a345ff6 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\sSQLITE_DBCONFIG_MAX\spreprocessor\smacro\swhich\swill\salways\sbe\sequal\sto\nthe\slargest\sSQLITE_DECONFIG\svalue.
-D 2017-12-21T14:46:11.149
+C Change\sthe\sname\sof\sSQLITE_DBCONFIG_FULL_EQP\sto\sbe\sSQLITE_DBCONFIG_TRIGGER_EQP\n(which\swe\scan\sdo\swithout\sbreaking\scompatibility\sbecause\sthe\sformer\sname\shas\nnot\syet\sappeared\sin\san\sofficial\srelease)\sand\sstreamline\sits\simplementation.
+D 2017-12-21T18:23:26.999
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -43,7 +43,7 @@ F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3
 F ext/expert/expert.c 4791c5e064aea81b2b829fa95228b22283380ee370ea88a1e580103b75516ebf
 F ext/expert/expert1.test 0c71a3453ce3a0b4dbe952713aec0ae8d416dd846820dd027b08f305f5278b30
-F ext/expert/sqlite3expert.c 6ed4e84a06d1a29b2cf3009c0266573b88602d098055caa46c467154a64e0959
+F ext/expert/sqlite3expert.c be6452d15a85a59dfa503c3a890d72bd924ac0c39c0af6075b6437a38d7a64da
 F ext/expert/sqlite3expert.h af6354f8ee5c9e025024e63fec3bd640a802afcc3099a44d804752cf0791d811
 F ext/expert/test_expert.c 85f5c743a899063fa48296d21de2f32c26d09a21c8582b2a0bc482e8de183e7a
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
@@ -443,7 +443,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c cb67cc56ef2ddd13e6944b2c0dd08a920bcd9503230adef8b9928d338097c722
 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e
 F src/loadext.c 20865b183bb8a3723d59cf1efffc3c50217eb452c1021d077b908c94da26b0b2
-F src/main.c 903c4e3eb189d4ce94748dc7f442f9136177cb2eedb27185f83a525edb0c4530
+F src/main.c 7ce55fa3c0bf669944de309ebab1655ed06ec67869adb0372c7a1062e461c448
 F src/malloc.c a02c9e69bc76bee0f639416b947a946412890b606301454727feadcb313536d6
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -480,10 +480,10 @@ F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
 F src/shell.c.in 87a048fabcf0080a78bcdd01e57933369951f7fa7d628f04bad48e900c1899a1
-F src/sqlite.h.in e00a8c4910e64bd72c1808e9dff80a7a6323f1f036cdd3fdf927c8eae6bb33d2
+F src/sqlite.h.in 53611410ade98671c0a95b209273e52dfd0ec1268ebc9213622d221d83b50bfe
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
-F src/sqliteInt.h 4d6580ab911e9bc36447ef05bf4d2aab929dde161ac117c7de6220eef06d6175
+F src/sqliteInt.h 392de8f6d2e3d6c88cd1a998dc2fa90bbdb146cb440cc7a1abc62d478c69df12
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -551,7 +551,7 @@ F src/vdbe.c 3393b508d9ad084ffce232a7c53e375ef5ac99b50b685c5131fcdfce97a9d534
 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97
 F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9
 F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1
-F src/vdbeaux.c cf474d29372a30d0db84b0a9a74ab07117c9b831e1585a0fa335e2b445f4a95f
+F src/vdbeaux.c 7ae48b180e5dd5d282e6752d155f1ab7929196d8e6577b82742044188152ca85
 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191
 F src/vdbemem.c 8478f7fb1948bf8fdeec7c2cb59ea58155c31258b9cd43c56d485e03ed40bd07
 F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0821bae7afefed98102c81104b4a477e81816bb1f43353c80865411771e3c5a7
-R 0d03176c5debc93f715c8f8149153033
+P 95d4103348d14f9d4c3fee30bd1d882f717b047b2b92f8fef52bec4878281907
+R 8b3e23acfaff32d9421a31731659429a
 U drh
-Z b349f9c48e8b618dcebb916f768288b4
+Z aa5f57e4aaa4de538c8fad51c00ee6a2
diff --git a/manifest.uuid b/manifest.uuid
index f0979207cd..bd309e6fc0 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-95d4103348d14f9d4c3fee30bd1d882f717b047b2b92f8fef52bec4878281907
\ No newline at end of file
+fffc7685d19f78ec322a4e834ad727af20a17e2e1c35680e4b1c4162c4786f60
\ No newline at end of file
diff --git a/src/main.c b/src/main.c
index 52d3be9531..32ce1889f6 100644
--- a/src/main.c
+++ b/src/main.c
@@ -806,21 +806,6 @@ int sqlite3_db_config(sqlite3 *db, int op, ...){
       rc = setupLookaside(db, pBuf, sz, cnt);
       break;
     }
-    case SQLITE_DBCONFIG_FULL_EQP: {
-      int onoff = va_arg(ap, int);
-      int *pRes = va_arg(ap, int*);
-      if( onoff>0 ){
-        db->bFullEQP = 1;
-      }else if( onoff==0 ){
-        db->bFullEQP = 0;
-      }
-      sqlite3ExpirePreparedStatements(db);
-      if( pRes ){
-        *pRes = db->bFullEQP;
-      }
-      rc = SQLITE_OK;
-      break;
-    }
     default: {
       static const struct {
         int op;      /* The opcode */
@@ -832,6 +817,7 @@ int sqlite3_db_config(sqlite3 *db, int op, ...){
         { SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, SQLITE_LoadExtension  },
         { SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE,      SQLITE_NoCkptOnClose  },
         { SQLITE_DBCONFIG_ENABLE_QPSG,           SQLITE_EnableQPSG     },
+        { SQLITE_DBCONFIG_TRIGGER_EQP,           SQLITE_TriggerEQP     },
       };
       unsigned int i;
       rc = SQLITE_ERROR; /* IMP: R-42790-23372 */
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 671d3dcebb..67c90589cc 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -2070,7 +2070,7 @@ struct sqlite3_mem_methods {
 ** the QPSG active, SQLite will always use the same query plan in the field as
 ** was used during testing in the lab.
 ** </dd>
-** <dt>SQLITE_DBCONFIG_FULL_EQP</dt>
+** <dt>SQLITE_DBCONFIG_TRIGGER_EQP</dt>
 ** <dd> By default, the output of EXPLAIN QUERY PLAN commands does not 
 ** include output for any operations performed by trigger programs. This
 ** option is used to set or clear (the default) a flag that governs this
@@ -2090,7 +2090,7 @@ struct sqlite3_mem_methods {
 #define SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION 1005 /* int int* */
 #define SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE      1006 /* int int* */
 #define SQLITE_DBCONFIG_ENABLE_QPSG           1007 /* int int* */
-#define SQLITE_DBCONFIG_FULL_EQP              1008 /* int int* */
+#define SQLITE_DBCONFIG_TRIGGER_EQP           1008 /* int int* */
 #define SQLITE_DBCONFIG_MAX                   1008 /* Largest DBCONFIG */
 
 /*
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index efe823a654..8d4aea723b 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1356,7 +1356,6 @@ struct sqlite3 {
   u8 mTrace;                    /* zero or more SQLITE_TRACE flags */
   u8 skipBtreeMutex;            /* True if no shared-cache backends */
   u8 nSqlExec;                  /* Number of pending OP_SqlExec opcodes */
-  u8 bFullEQP;                  /* Include triggers in EQP output */
   int nextPagesize;             /* Pagesize after VACUUM if >0 */
   u32 magic;                    /* Magic number for detect library misuse */
   int nChange;                  /* Value returned by sqlite3_changes() */
@@ -1497,7 +1496,9 @@ struct sqlite3 {
 #define SQLITE_QueryOnly      0x00100000  /* Disable database changes */
 #define SQLITE_CellSizeCk     0x00200000  /* Check btree cell sizes on load */
 #define SQLITE_Fts3Tokenizer  0x00400000  /* Enable fts3_tokenizer(2) */
-#define SQLITE_EnableQPSG     0x00800000  /* Query Planner Stability Guarantee */
+#define SQLITE_EnableQPSG     0x00800000  /* Query Planner Stability Guarantee*/
+#define SQLITE_TriggerEQP     0x01000000  /* Show trigger EXPLAIN QUERY PLAN */
+
 /* Flags used only if debugging */
 #ifdef SQLITE_DEBUG
 #define SQLITE_SqlTrace       0x08000000  /* Debug print SQL as it executes */
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index 1c354e0be1..bc4bbda8e4 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -1639,7 +1639,7 @@ int sqlite3VdbeList(
   int i;                               /* Loop counter */
   int rc = SQLITE_OK;                  /* Return code */
   Mem *pMem = &p->aMem[1];             /* First Mem of result set */
-  int bFull = (p->explain==1 || db->bFullEQP);
+  int bListSubprogs = (p->explain==1 || (db->flags & SQLITE_TriggerEQP)!=0);
   Op *pOp = 0;
 
   assert( p->explain );
@@ -1668,7 +1668,7 @@ int sqlite3VdbeList(
   ** encountered, but p->pc will eventually catch up to nRow.
   */
   nRow = p->nOp;
-  if( bFull ){
+  if( bListSubprogs ){
     /* The first 8 memory cells are used for the result set.  So we will
     ** commandeer the 9th cell to use as storage for an array of pointers
     ** to trigger subprograms.  The VDBE is guaranteed to have at least 9
@@ -1713,7 +1713,7 @@ int sqlite3VdbeList(
     ** kept in p->aMem[9].z to hold the new program - assuming this subprogram
     ** has not already been seen.
     */
-    if( bFull && pOp->p4type==P4_SUBPROGRAM ){
+    if( bListSubprogs && pOp->p4type==P4_SUBPROGRAM ){
       int nByte = (nSub+1)*sizeof(SubProgram*);
       int j;
       for(j=0; j<nSub; j++){

From 2e1ea57f8bd08f8abdbb976b0772dbd995511718 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 21 Dec 2017 18:55:24 +0000
Subject: [PATCH 60/68] Add the ".expert" command to the shell tool's ".help"
 output.

FossilOrigin-Name: fc6193af5d216b4066fbc47e75a7d0538fd5fda40b94ee15e2ff4037ea89221b
---
 manifest       | 14 +++++++-------
 manifest.uuid  |  2 +-
 src/shell.c.in |  1 +
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 361a345ff6..b9641d26a6 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Change\sthe\sname\sof\sSQLITE_DBCONFIG_FULL_EQP\sto\sbe\sSQLITE_DBCONFIG_TRIGGER_EQP\n(which\swe\scan\sdo\swithout\sbreaking\scompatibility\sbecause\sthe\sformer\sname\shas\nnot\syet\sappeared\sin\san\sofficial\srelease)\sand\sstreamline\sits\simplementation.
-D 2017-12-21T18:23:26.999
+C Add\sthe\s".expert"\scommand\sto\sthe\sshell\stool's\s".help"\soutput.
+D 2017-12-21T18:55:24.380
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -479,7 +479,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
-F src/shell.c.in 87a048fabcf0080a78bcdd01e57933369951f7fa7d628f04bad48e900c1899a1
+F src/shell.c.in ef343d708f32e43bc5abff8a4a769e026252b77ff37583cbd7ad30a9cc169f26
 F src/sqlite.h.in 53611410ade98671c0a95b209273e52dfd0ec1268ebc9213622d221d83b50bfe
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 95d4103348d14f9d4c3fee30bd1d882f717b047b2b92f8fef52bec4878281907
-R 8b3e23acfaff32d9421a31731659429a
-U drh
-Z aa5f57e4aaa4de538c8fad51c00ee6a2
+P fffc7685d19f78ec322a4e834ad727af20a17e2e1c35680e4b1c4162c4786f60
+R d5bcde1015d19d1cdfe7f85d9291f783
+U dan
+Z f2c8e8756f16d691313b2a4f36b60cc0
diff --git a/manifest.uuid b/manifest.uuid
index bd309e6fc0..247556897e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-fffc7685d19f78ec322a4e834ad727af20a17e2e1c35680e4b1c4162c4786f60
\ No newline at end of file
+fc6193af5d216b4066fbc47e75a7d0538fd5fda40b94ee15e2ff4037ea89221b
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index cdd824835a..261a1971e7 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -2779,6 +2779,7 @@ static char zHelp[] =
   ".echo on|off           Turn command echo on or off\n"
   ".eqp on|off|full       Enable or disable automatic EXPLAIN QUERY PLAN\n"
   ".exit                  Exit this program\n"
+  ".expert                EXPERIMENTAL. Suggest indexes for specified queries\n"
 /* Because explain mode comes on automatically now, the ".explain" mode
 ** is removed from the help screen.  It is still supported for legacy, however */
 /*".explain ?on|off|auto? Turn EXPLAIN output mode on or off or to automatic\n"*/

From ada7045eabf23116f9c6b422fff7ca604198ce58 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 21 Dec 2017 21:02:27 +0000
Subject: [PATCH 61/68] Add the ".eqp trigger" option to the ".eqp" command in
 the command-line shell.  Implemented using the new
 SQLITE_DBCONFIG_TRIGGER_EQP control.

FossilOrigin-Name: 2c51644a12a638d89e4f7cc3fd561236ce424f2d4e1db31f1e8388f77add02b8
---
 manifest       | 14 +++++++-------
 manifest.uuid  |  2 +-
 src/shell.c.in | 27 +++++++++++++++++++++------
 3 files changed, 29 insertions(+), 14 deletions(-)

diff --git a/manifest b/manifest
index b9641d26a6..62b8ff3da8 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\s".expert"\scommand\sto\sthe\sshell\stool's\s".help"\soutput.
-D 2017-12-21T18:55:24.380
+C Add\sthe\s".eqp\strigger"\soption\sto\sthe\s".eqp"\scommand\sin\sthe\scommand-line\nshell.\s\sImplemented\susing\sthe\snew\sSQLITE_DBCONFIG_TRIGGER_EQP\scontrol.
+D 2017-12-21T21:02:27.410
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -479,7 +479,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
-F src/shell.c.in ef343d708f32e43bc5abff8a4a769e026252b77ff37583cbd7ad30a9cc169f26
+F src/shell.c.in 339169a3d1307b5566ebe9ce15832d03439206106724c78cc3d9125a7b851795
 F src/sqlite.h.in 53611410ade98671c0a95b209273e52dfd0ec1268ebc9213622d221d83b50bfe
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P fffc7685d19f78ec322a4e834ad727af20a17e2e1c35680e4b1c4162c4786f60
-R d5bcde1015d19d1cdfe7f85d9291f783
-U dan
-Z f2c8e8756f16d691313b2a4f36b60cc0
+P fc6193af5d216b4066fbc47e75a7d0538fd5fda40b94ee15e2ff4037ea89221b
+R 9e177759a0c3bcdc85d791f3df26085a
+U drh
+Z 1ea40405a7cdaae369b8eb1d1f8aee13
diff --git a/manifest.uuid b/manifest.uuid
index 247556897e..48249eb7dd 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-fc6193af5d216b4066fbc47e75a7d0538fd5fda40b94ee15e2ff4037ea89221b
\ No newline at end of file
+2c51644a12a638d89e4f7cc3fd561236ce424f2d4e1db31f1e8388f77add02b8
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 261a1971e7..13b1fcde39 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -877,6 +877,13 @@ struct ShellState {
   ExpertInfo expert;        /* Valid if previous command was ".expert OPT..." */
 };
 
+/* Allowed values for ShellState.autoEQP
+*/
+#define AUTOEQP_off      0
+#define AUTOEQP_on       1
+#define AUTOEQP_trigger  2
+#define AUTOEQP_full     3
+
 /*
 ** These are the allowed shellFlgs values
 */
@@ -2395,7 +2402,12 @@ static int shell_exec(
       if( pArg && pArg->autoEQP && sqlite3_strlike("EXPLAIN%",zStmtSql,0)!=0 ){
         sqlite3_stmt *pExplain;
         char *zEQP;
+        int triggerEQP = 0;
         disable_debug_trace_modes();
+        sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, -1, &triggerEQP);
+        if( pArg->autoEQP>=AUTOEQP_trigger ){
+          sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, 1, 0);
+        }
         zEQP = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", zStmtSql);
         rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0);
         if( rc==SQLITE_OK ){
@@ -2408,7 +2420,7 @@ static int shell_exec(
         }
         sqlite3_finalize(pExplain);
         sqlite3_free(zEQP);
-        if( pArg->autoEQP>=2 ){
+        if( pArg->autoEQP>=AUTOEQP_full ){
           /* Also do an EXPLAIN for ".eqp full" mode */
           zEQP = sqlite3_mprintf("EXPLAIN %s", zStmtSql);
           rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0);
@@ -2421,6 +2433,7 @@ static int shell_exec(
           sqlite3_finalize(pExplain);
           sqlite3_free(zEQP);
         }
+        sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, triggerEQP, 0);
         restore_debug_trace_modes();
       }
 
@@ -4547,12 +4560,14 @@ static int do_meta_command(char *zLine, ShellState *p){
   if( c=='e' && strncmp(azArg[0], "eqp", n)==0 ){
     if( nArg==2 ){
       if( strcmp(azArg[1],"full")==0 ){
-        p->autoEQP = 2;
+        p->autoEQP = AUTOEQP_full;
+      }else if( strcmp(azArg[1],"trigger")==0 ){
+        p->autoEQP = AUTOEQP_trigger;
       }else{
         p->autoEQP = booleanValue(azArg[1]);
       }
     }else{
-      raw_printf(stderr, "Usage: .eqp on|off|full\n");
+      raw_printf(stderr, "Usage: .eqp off|on|trigger|full\n");
       rc = 1;
     }
   }else
@@ -5905,7 +5920,7 @@ static int do_meta_command(char *zLine, ShellState *p){
   }else
 
   if( c=='s' && strncmp(azArg[0], "show", n)==0 ){
-    static const char *azBool[] = { "off", "on", "full", "unk" };
+    static const char *azBool[] = { "off", "on", "trigger", "full"};
     int i;
     if( nArg!=1 ){
       raw_printf(stderr, "Usage: .show\n");
@@ -7057,9 +7072,9 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
     }else if( strcmp(z,"-echo")==0 ){
       ShellSetFlag(&data, SHFLG_Echo);
     }else if( strcmp(z,"-eqp")==0 ){
-      data.autoEQP = 1;
+      data.autoEQP = AUTOEQP_on;
     }else if( strcmp(z,"-eqpfull")==0 ){
-      data.autoEQP = 2;
+      data.autoEQP = AUTOEQP_full;
     }else if( strcmp(z,"-stats")==0 ){
       data.statsOn = 1;
     }else if( strcmp(z,"-scanstats")==0 ){

From ef7231b8653d7e8e5d29d2f19805361099f491c6 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 21 Dec 2017 21:41:13 +0000
Subject: [PATCH 62/68] Fix SQLITE_DBCONFIG_TRIGGER_EQP so that it works even
 if SQLITE_DEBUG is not defined.

FossilOrigin-Name: afbbfff72002089fa73b5473cc98360df14288d489e93e667332d6e884ef60da
---
 manifest        | 12 ++++++------
 manifest.uuid   |  2 +-
 src/wherecode.c |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 62b8ff3da8..990fbed2ba 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\s".eqp\strigger"\soption\sto\sthe\s".eqp"\scommand\sin\sthe\scommand-line\nshell.\s\sImplemented\susing\sthe\snew\sSQLITE_DBCONFIG_TRIGGER_EQP\scontrol.
-D 2017-12-21T21:02:27.410
+C Fix\sSQLITE_DBCONFIG_TRIGGER_EQP\sso\sthat\sit\sworks\seven\sif\sSQLITE_DEBUG\sis\nnot\sdefined.
+D 2017-12-21T21:41:13.558
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -563,7 +563,7 @@ F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f
 F src/where.c 0cdfda28708fe75699df2fcdb44d067dc94e7b1f83e3a622dca96a27aaaa2e42
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
-F src/wherecode.c ff2f079097a3bdce6ebabfde1419fba448c9ce5feb7cb964e8bfa2a4e27274ef
+F src/wherecode.c af1e79154aaa88cd802d6f2e5b945f67eaca7c958d1525fbf8ee19d5bd7b9020
 F src/whereexpr.c 427ea8e96ec24f2a7814c67b8024ad664a9c7656264c4566c34743cb23186e46
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P fc6193af5d216b4066fbc47e75a7d0538fd5fda40b94ee15e2ff4037ea89221b
-R 9e177759a0c3bcdc85d791f3df26085a
+P 2c51644a12a638d89e4f7cc3fd561236ce424f2d4e1db31f1e8388f77add02b8
+R f003797eb1e0446ed94e2b0c9d22122e
 U drh
-Z 1ea40405a7cdaae369b8eb1d1f8aee13
+Z c4f59d8ddc7f0db54ec7405d6ea2d997
diff --git a/manifest.uuid b/manifest.uuid
index 48249eb7dd..194b1d49ef 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2c51644a12a638d89e4f7cc3fd561236ce424f2d4e1db31f1e8388f77add02b8
\ No newline at end of file
+afbbfff72002089fa73b5473cc98360df14288d489e93e667332d6e884ef60da
\ No newline at end of file
diff --git a/src/wherecode.c b/src/wherecode.c
index 3c166a1210..32dd2048bf 100644
--- a/src/wherecode.c
+++ b/src/wherecode.c
@@ -128,7 +128,7 @@ int sqlite3WhereExplainOneScan(
 ){
   int ret = 0;
 #if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS)
-  if( pParse->explain==2 )
+  if( sqlite3ParseToplevel(pParse)->explain==2 )
 #endif
   {
     struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom];

From efc88d020cc6872923616d4c22cc98488f117da2 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 22 Dec 2017 00:52:50 +0000
Subject: [PATCH 63/68] Modify the new sqlite3_vtab_collation() interface so
 that it takes a pointer to the sqlite3_index_info object passed into
 xBestIndex rather than an sqlite3 connection pointer, which the xBestIndex
 method might not have access to.

FossilOrigin-Name: 5c1fe6666019147a26480b5db1bf2f474a5d072c234c736f16ed5d2a9a040b3f
---
 ext/expert/sqlite3expert.c |  3 +-
 manifest                   | 18 +++++------
 manifest.uuid              |  2 +-
 src/sqlite.h.in            | 11 +++----
 src/sqliteInt.h            |  3 +-
 src/where.c                | 63 +++++++++++++++++++-------------------
 6 files changed, 48 insertions(+), 52 deletions(-)

diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index b8b19c66a6..e4060de681 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -434,7 +434,6 @@ static int expertDisconnect(sqlite3_vtab *pVtab){
 
 static int expertBestIndex(sqlite3_vtab *pVtab, sqlite3_index_info *pIdxInfo){
   ExpertVtab *p = (ExpertVtab*)pVtab;
-  sqlite3 *dbv = p->pExpert->dbv;
   int rc = SQLITE_OK;
   int n = 0;
   IdxScan *pScan;
@@ -461,7 +460,7 @@ static int expertBestIndex(sqlite3_vtab *pVtab, sqlite3_index_info *pIdxInfo){
        && (pCons->op & opmask) 
       ){
         IdxConstraint *pNew;
-        const char *zColl = sqlite3_vtab_collation(dbv, i);
+        const char *zColl = sqlite3_vtab_collation(pIdxInfo, i);
         pNew = idxNewConstraint(&rc, zColl);
         if( pNew ){
           pNew->iCol = pCons->iColumn;
diff --git a/manifest b/manifest
index 990fbed2ba..ba93b34166 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sSQLITE_DBCONFIG_TRIGGER_EQP\sso\sthat\sit\sworks\seven\sif\sSQLITE_DEBUG\sis\nnot\sdefined.
-D 2017-12-21T21:41:13.558
+C Modify\sthe\snew\ssqlite3_vtab_collation()\sinterface\sso\sthat\sit\stakes\sa\npointer\sto\sthe\ssqlite3_index_info\sobject\spassed\sinto\sxBestIndex\srather\sthan\nan\ssqlite3\sconnection\spointer,\swhich\sthe\sxBestIndex\smethod\smight\snot\shave\naccess\sto.
+D 2017-12-22T00:52:50.426
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -43,7 +43,7 @@ F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3
 F ext/expert/expert.c 4791c5e064aea81b2b829fa95228b22283380ee370ea88a1e580103b75516ebf
 F ext/expert/expert1.test 0c71a3453ce3a0b4dbe952713aec0ae8d416dd846820dd027b08f305f5278b30
-F ext/expert/sqlite3expert.c be6452d15a85a59dfa503c3a890d72bd924ac0c39c0af6075b6437a38d7a64da
+F ext/expert/sqlite3expert.c 252f3129f12a0e9df094a14711db98265c9c6d7afa033ec906d94e920f5c7ba7
 F ext/expert/sqlite3expert.h af6354f8ee5c9e025024e63fec3bd640a802afcc3099a44d804752cf0791d811
 F ext/expert/test_expert.c 85f5c743a899063fa48296d21de2f32c26d09a21c8582b2a0bc482e8de183e7a
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
@@ -480,10 +480,10 @@ F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
 F src/shell.c.in 339169a3d1307b5566ebe9ce15832d03439206106724c78cc3d9125a7b851795
-F src/sqlite.h.in 53611410ade98671c0a95b209273e52dfd0ec1268ebc9213622d221d83b50bfe
+F src/sqlite.h.in 2126192945019d4cdce335cb236b440a05ec75c93e4cd94c9c6d6e7fcc654cc4
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
-F src/sqliteInt.h 392de8f6d2e3d6c88cd1a998dc2fa90bbdb146cb440cc7a1abc62d478c69df12
+F src/sqliteInt.h 003b78433baae4e5c997f99f2f9cf98d90754f256baeacb32f8189569a48251f
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -561,7 +561,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 5a3f464edd64596f601683ed321d12e6fd93c5fb9afdfb3653d6ffd0fee9c48f
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f
-F src/where.c 0cdfda28708fe75699df2fcdb44d067dc94e7b1f83e3a622dca96a27aaaa2e42
+F src/where.c 5876c9100b622f7b9e5ee7f579b8b6a71ae5ba627724cea4546d9114c32b3cb5
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
 F src/wherecode.c af1e79154aaa88cd802d6f2e5b945f67eaca7c958d1525fbf8ee19d5bd7b9020
 F src/whereexpr.c 427ea8e96ec24f2a7814c67b8024ad664a9c7656264c4566c34743cb23186e46
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2c51644a12a638d89e4f7cc3fd561236ce424f2d4e1db31f1e8388f77add02b8
-R f003797eb1e0446ed94e2b0c9d22122e
+P afbbfff72002089fa73b5473cc98360df14288d489e93e667332d6e884ef60da
+R b5eb5cd81beccd0fb58b9d1b00a03e31
 U drh
-Z c4f59d8ddc7f0db54ec7405d6ea2d997
+Z 10e32a088c82fd051b685f8b12f5082b
diff --git a/manifest.uuid b/manifest.uuid
index 194b1d49ef..4ed62ec7c2 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-afbbfff72002089fa73b5473cc98360df14288d489e93e667332d6e884ef60da
\ No newline at end of file
+5c1fe6666019147a26480b5db1bf2f474a5d072c234c736f16ed5d2a9a040b3f
\ No newline at end of file
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 67c90589cc..f161eea6f2 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -8302,15 +8302,14 @@ int sqlite3_vtab_on_conflict(sqlite3 *);
 ** This function may only be called from within a call to the [xBestIndex]
 ** method of a [virtual table implementation]. 
 **
-** The first argument must be the database handle with which the virtual 
-** table is associated (the one passed to the [xConnect] or [xCreate] method 
-** to create the sqlite3_vtab object. The second argument must be an index
-** into the aConstraint[] array belonging to the sqlite3_index_info structure
-** passed to xBestIndex. This function returns a pointer to a buffer 
+** The first argument must be the sqlite3_index_info object that is the
+** first parameter to the xBestIndex() method. The second argument must be
+** an index into the aConstraint[] array belonging to the sqlite3_index_info
+** structure passed to xBestIndex. This function returns a pointer to a buffer 
 ** containing the name of the collation sequence for the corresponding
 ** constraint.
 */
-SQLITE_EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3*, int);
+SQLITE_EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3_index_info*,int);
 
 /*
 ** CAPI3REF: Conflict resolution modes
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 8d4aea723b..b0c4711b03 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1420,8 +1420,7 @@ struct sqlite3 {
   Hash aModule;                 /* populated by sqlite3_create_module() */
   VtabCtx *pVtabCtx;            /* Context for active vtab connect/create */
   VTable **aVTrans;             /* Virtual tables with open transactions */
-  VTable *pDisconnect;    /* Disconnect these in next sqlite3_prepare() */
-  void *pBestIndexCtx;          /* For sqlite3_vtab_collation() */
+  VTable *pDisconnect;          /* Disconnect these in next sqlite3_prepare() */
 #endif
   Hash aFunc;                   /* Hash table of connection functions */
   Hash aCollSeq;                /* All collating sequences */
diff --git a/src/where.c b/src/where.c
index f0ee78a1da..d3aec354ba 100644
--- a/src/where.c
+++ b/src/where.c
@@ -19,6 +19,21 @@
 #include "sqliteInt.h"
 #include "whereInt.h"
 
+/*
+** Extra information appended to the end of sqlite3_index_info but not
+** visible to the xBestIndex function, at least not directly.  The
+** sqlite3_vtab_collation() interface knows how to reach it, however.
+**
+** This object is not an API and can be changed from one release to the
+** next.  As long as allocateIndexInfo() and sqlite3_vtab_collation()
+** agree on the structure, all will be well.
+*/
+typedef struct HiddenIndexInfo HiddenIndexInfo;
+struct HiddenIndexInfo {
+  WhereClause *pWC;   /* The Where clause being analyzed */
+  Parse *pParse;      /* The parsing context */
+};
+
 /* Forward declaration of methods */
 static int whereLoopResize(sqlite3*, WhereLoop*, int);
 
@@ -841,11 +856,11 @@ end_auto_index_create:
 ** by passing the pointer returned by this function to sqlite3_free().
 */
 static sqlite3_index_info *allocateIndexInfo(
-  Parse *pParse,
-  WhereClause *pWC,
+  Parse *pParse,                  /* The parsing context */
+  WhereClause *pWC,               /* The WHERE clause being analyzed */
   Bitmask mUnusable,              /* Ignore terms with these prereqs */
-  struct SrcList_item *pSrc,
-  ExprList *pOrderBy,
+  struct SrcList_item *pSrc,      /* The FROM clause term that is the vtab */
+  ExprList *pOrderBy,             /* The ORDER BY clause */
   u16 *pmNoOmit                   /* Mask of terms not to omit */
 ){
   int i, j;
@@ -853,6 +868,7 @@ static sqlite3_index_info *allocateIndexInfo(
   struct sqlite3_index_constraint *pIdxCons;
   struct sqlite3_index_orderby *pIdxOrderBy;
   struct sqlite3_index_constraint_usage *pUsage;
+  struct HiddenIndexInfo *pHidden;
   WhereTerm *pTerm;
   int nOrderBy;
   sqlite3_index_info *pIdxInfo;
@@ -894,7 +910,7 @@ static sqlite3_index_info *allocateIndexInfo(
   */
   pIdxInfo = sqlite3DbMallocZero(pParse->db, sizeof(*pIdxInfo)
                            + (sizeof(*pIdxCons) + sizeof(*pUsage))*nTerm
-                           + sizeof(*pIdxOrderBy)*nOrderBy );
+                           + sizeof(*pIdxOrderBy)*nOrderBy + sizeof(*pHidden) );
   if( pIdxInfo==0 ){
     sqlite3ErrorMsg(pParse, "out of memory");
     return 0;
@@ -905,7 +921,8 @@ static sqlite3_index_info *allocateIndexInfo(
   ** changing them.  We have to do some funky casting in order to
   ** initialize those fields.
   */
-  pIdxCons = (struct sqlite3_index_constraint*)&pIdxInfo[1];
+  pHidden = (struct HiddenIndexInfo*)&pIdxInfo[1];
+  pIdxCons = (struct sqlite3_index_constraint*)&pHidden[1];
   pIdxOrderBy = (struct sqlite3_index_orderby*)&pIdxCons[nTerm];
   pUsage = (struct sqlite3_index_constraint_usage*)&pIdxOrderBy[nOrderBy];
   *(int*)&pIdxInfo->nConstraint = nTerm;
@@ -915,6 +932,8 @@ static sqlite3_index_info *allocateIndexInfo(
   *(struct sqlite3_index_constraint_usage**)&pIdxInfo->aConstraintUsage =
                                                                    pUsage;
 
+  pHidden->pWC = pWC;
+  pHidden->pParse = pParse;
   for(i=j=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){
     u16 op;
     if( pTerm->leftCursor != pSrc->iCursor ) continue;
@@ -3138,17 +3157,6 @@ static int whereLoopAddVirtualOne(
   return rc;
 }
 
-
-/*
-** Context object used to pass information from whereLoopAddVirtual()
-** to sqlite3_vtab_collation().
-*/
-struct BestIndexCtx {
-  WhereClause *pWC;
-  sqlite3_index_info *pIdxInfo;
-  Parse *pParse;
-};
-
 /*
 ** If this function is invoked from within an xBestIndex() callback, it
 ** returns a pointer to a buffer containing the name of the collation
@@ -3156,15 +3164,15 @@ struct BestIndexCtx {
 ** array. Or, if iCons is out of range or there is no active xBestIndex
 ** call, return NULL.
 */
-const char *sqlite3_vtab_collation(sqlite3 *db, int iCons){
-  struct BestIndexCtx *p = (struct BestIndexCtx*)db->pBestIndexCtx;
+const char *sqlite3_vtab_collation(sqlite3_index_info *pIdxInfo, int iCons){
+  HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
   const char *zRet = 0;
-  if( p && iCons>=0 && iCons<p->pIdxInfo->nConstraint ){
+  if( iCons>=0 && iCons<pIdxInfo->nConstraint ){
     CollSeq *pC = 0;
-    int iTerm = p->pIdxInfo->aConstraint[iCons].iTermOffset;
-    Expr *pX = p->pWC->a[iTerm].pExpr;
+    int iTerm = pIdxInfo->aConstraint[iCons].iTermOffset;
+    Expr *pX = pHidden->pWC->a[iTerm].pExpr;
     if( pX->pLeft ){
-      pC = sqlite3BinaryCompareCollSeq(p->pParse, pX->pLeft, pX->pRight);
+      pC = sqlite3BinaryCompareCollSeq(pHidden->pParse, pX->pLeft, pX->pRight);
     }
     zRet = (pC ? pC->zName : "BINARY");
   }
@@ -3212,8 +3220,6 @@ static int whereLoopAddVirtual(
   WhereLoop *pNew;
   Bitmask mBest;               /* Tables used by best possible plan */
   u16 mNoOmit;
-  struct BestIndexCtx bic;
-  void *pSaved;
 
   assert( (mPrereq & mUnusable)==0 );
   pWInfo = pBuilder->pWInfo;
@@ -3235,12 +3241,6 @@ static int whereLoopAddVirtual(
     return SQLITE_NOMEM_BKPT;
   }
 
-  bic.pWC = pWC;
-  bic.pIdxInfo = p;
-  bic.pParse = pParse;
-  pSaved = pParse->db->pBestIndexCtx;
-  pParse->db->pBestIndexCtx = (void*)&bic;
-
   /* First call xBestIndex() with all constraints usable. */
   WHERETRACE(0x40, ("  VirtualOne: all usable\n"));
   rc = whereLoopAddVirtualOne(pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn);
@@ -3317,7 +3317,6 @@ static int whereLoopAddVirtual(
 
   if( p->needToFreeIdxStr ) sqlite3_free(p->idxStr);
   sqlite3DbFreeNN(pParse->db, p);
-  pParse->db->pBestIndexCtx = pSaved;
   return rc;
 }
 #endif /* SQLITE_OMIT_VIRTUALTABLE */

From 5a699a01ca773e2f4cc90677f94efb3ff01b9a2a Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 22 Dec 2017 19:53:02 +0000
Subject: [PATCH 64/68] In the sqlite3TreeViewExprList() routine, show the "AS"
 alias name for each expression in the list, if it exists.

FossilOrigin-Name: 5efd854fe21470336ba4140294b6c90ef39af32e103b92c664438d7485c50f9a
---
 manifest       | 12 ++++++------
 manifest.uuid  |  2 +-
 src/treeview.c | 12 ++++++++++--
 3 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index ba93b34166..23b0f9e47e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Modify\sthe\snew\ssqlite3_vtab_collation()\sinterface\sso\sthat\sit\stakes\sa\npointer\sto\sthe\ssqlite3_index_info\sobject\spassed\sinto\sxBestIndex\srather\sthan\nan\ssqlite3\sconnection\spointer,\swhich\sthe\sxBestIndex\smethod\smight\snot\shave\naccess\sto.
-D 2017-12-22T00:52:50.426
+C In\sthe\ssqlite3TreeViewExprList()\sroutine,\sshow\sthe\s"AS"\salias\sname\sfor\s\neach\sexpression\sin\sthe\slist,\sif\sit\sexists.
+D 2017-12-22T19:53:02.654
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -541,7 +541,7 @@ F src/test_windirent.h 5d67483a55442e31e1bde0f4a230e6e932ad5906
 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
 F src/tokenize.c 1003d6d90c6783206c711f0a9397656fa5b055209f4d092caa43bb3bf5215db5
-F src/treeview.c 08a83195de8fad3f00542e3c8b3c1eb1222c999817c9e301ffb7f332882b96dd
+F src/treeview.c eae35972ff44f67064de2eaf35f04afe94e7aea3271a8b3bcebb3f954880fec3
 F src/trigger.c 775053eecf6b73062e243404b56f5064446254d5cce17d8704d5cdffd72a546a
 F src/update.c 961bd1265d4d1e5cd65c9a54fa5122fb7aefcb003fcf2de0c092fceb7e58972c
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P afbbfff72002089fa73b5473cc98360df14288d489e93e667332d6e884ef60da
-R b5eb5cd81beccd0fb58b9d1b00a03e31
+P 5c1fe6666019147a26480b5db1bf2f474a5d072c234c736f16ed5d2a9a040b3f
+R d3af759da34c77156099ceec10b188ae
 U drh
-Z 10e32a088c82fd051b685f8b12f5082b
+Z f7d659c7910069dbd59e1c3feefe9c0a
diff --git a/manifest.uuid b/manifest.uuid
index 4ed62ec7c2..d2b59b3c8a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5c1fe6666019147a26480b5db1bf2f474a5d072c234c736f16ed5d2a9a040b3f
\ No newline at end of file
+5efd854fe21470336ba4140294b6c90ef39af32e103b92c664438d7485c50f9a
\ No newline at end of file
diff --git a/src/treeview.c b/src/treeview.c
index 8d171a4733..6dd386ffe6 100644
--- a/src/treeview.c
+++ b/src/treeview.c
@@ -507,12 +507,20 @@ void sqlite3TreeViewBareExprList(
     sqlite3TreeViewLine(pView, "%s", zLabel);
     for(i=0; i<pList->nExpr; i++){
       int j = pList->a[i].u.x.iOrderByCol;
-      if( j ){
+      char *zName = pList->a[i].zName;
+      if( j || zName ){
         sqlite3TreeViewPush(pView, 0);
+      }
+      if( zName ){
+        sqlite3TreeViewLine(pView, "AS %s", zName);
+      }
+      if( j ){
         sqlite3TreeViewLine(pView, "iOrderByCol=%d", j);
       }
       sqlite3TreeViewExpr(pView, pList->a[i].pExpr, i<pList->nExpr-1);
-      if( j ) sqlite3TreeViewPop(pView);
+      if( j || zName ){
+        sqlite3TreeViewPop(pView);
+      }
     }
   }
 }

From 07859486c6194c4d9f67725101acd3b25f9f177a Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 23 Dec 2017 11:51:40 +0000
Subject: [PATCH 65/68] Add a SELECTTRACE() macro to indicate when column names
 are assigned to a SELECT statement.  This helps with debugging for tickets
 like [de3403bf5ae5f72e] and [3b4450072511e621].

FossilOrigin-Name: 8f194008c3aaa4ef287200e37bc5278ba9c377a7091ee3f95bad66513226b083
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/select.c  |  1 +
 3 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index 23b0f9e47e..d9e65a145e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\sthe\ssqlite3TreeViewExprList()\sroutine,\sshow\sthe\s"AS"\salias\sname\sfor\s\neach\sexpression\sin\sthe\slist,\sif\sit\sexists.
-D 2017-12-22T19:53:02.654
+C Add\sa\sSELECTTRACE()\smacro\sto\sindicate\swhen\scolumn\snames\sare\sassigned\sto\na\sSELECT\sstatement.\s\sThis\shelps\swith\sdebugging\sfor\stickets\slike\n[de3403bf5ae5f72e]\sand\s[3b4450072511e621].
+D 2017-12-23T11:51:40.933
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -478,7 +478,7 @@ F src/printf.c 9506b4b96e59c0467047155f09015750cb2878aeda3d39e5610c1192ddc3c41c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
+F src/select.c 6c84733262885f70675b7b4d5c3f5d21562cd378cf06d5b0cb3bdbdb4785b85f
 F src/shell.c.in 339169a3d1307b5566ebe9ce15832d03439206106724c78cc3d9125a7b851795
 F src/sqlite.h.in 2126192945019d4cdce335cb236b440a05ec75c93e4cd94c9c6d6e7fcc654cc4
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5c1fe6666019147a26480b5db1bf2f474a5d072c234c736f16ed5d2a9a040b3f
-R d3af759da34c77156099ceec10b188ae
+P 5efd854fe21470336ba4140294b6c90ef39af32e103b92c664438d7485c50f9a
+R 3deaa3da40d43e03f04500b413c136e7
 U drh
-Z f7d659c7910069dbd59e1c3feefe9c0a
+Z 1c7db6ece307b188f8e97c3dc484c292
diff --git a/manifest.uuid b/manifest.uuid
index d2b59b3c8a..8c8a394848 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5efd854fe21470336ba4140294b6c90ef39af32e103b92c664438d7485c50f9a
\ No newline at end of file
+8f194008c3aaa4ef287200e37bc5278ba9c377a7091ee3f95bad66513226b083
\ No newline at end of file
diff --git a/src/select.c b/src/select.c
index 97eaf21b4d..e37fffd147 100644
--- a/src/select.c
+++ b/src/select.c
@@ -1596,6 +1596,7 @@ static void generateColumnNames(
   if( pParse->colNamesSet || db->mallocFailed ) return;
   /* Column names are determined by the left-most term of a compound select */
   while( pSelect->pPrior ) pSelect = pSelect->pPrior;
+  SELECTTRACE(1,pParse,pSelect,("generating column names\n"));
   pTabList = pSelect->pSrc;
   pEList = pSelect->pEList;
   assert( v!=0 );

From 755b0fd358b355db11ce81e4a2b8b267567ca072 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 23 Dec 2017 12:33:40 +0000
Subject: [PATCH 66/68] Move the generation of output column names earlier in
 the case of a CREATE TABLE AS.  This is a fix for ticket [3b4450072511e62]
 and a continuation of check-in [ade7ddf1998190b2b63] that fixes cases of
 ticket [de3403bf5ae5f72ed6] that were missed previously.

FossilOrigin-Name: 6b2ff26c25bb9da344add79c93fb3e49fa034a89b38ef56e08e18d21de61f707
---
 manifest          | 16 ++++++++--------
 manifest.uuid     |  2 +-
 src/build.c       |  9 ++++-----
 src/select.c      | 11 +++++------
 test/colname.test | 21 +++++++++++++++++++++
 5 files changed, 39 insertions(+), 20 deletions(-)

diff --git a/manifest b/manifest
index d9e65a145e..3d7fcbcc18 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sa\sSELECTTRACE()\smacro\sto\sindicate\swhen\scolumn\snames\sare\sassigned\sto\na\sSELECT\sstatement.\s\sThis\shelps\swith\sdebugging\sfor\stickets\slike\n[de3403bf5ae5f72e]\sand\s[3b4450072511e621].
-D 2017-12-23T11:51:40.933
+C Move\sthe\sgeneration\sof\soutput\scolumn\snames\searlier\sin\sthe\scase\sof\sa\nCREATE\sTABLE\sAS.\s\sThis\sis\sa\sfix\sfor\sticket\s[3b4450072511e62]\sand\sa\ncontinuation\sof\scheck-in\s[ade7ddf1998190b2b63]\sthat\sfixes\scases\sof\nticket\s[de3403bf5ae5f72ed6]\sthat\swere\smissed\spreviously.
+D 2017-12-23T12:33:40.443
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -423,7 +423,7 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
 F src/btree.c b83a6b03f160528020bb965f0c3a40af5286cd4923c3870fd218177f03a120a7
 F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09
 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc
-F src/build.c 87b68e3b45559ec404b12f095f0ba5f06f91a6dd2d21bd8443e41d8ac2e67196
+F src/build.c ed567f088edbc305dad33a6b14e08f8216a3860f6bad1d180450d5a5414bf346
 F src/callback.c fe677cb5f5abb02f7a772a62a98c2f516426081df68856e8f2d5f950929b966a
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
@@ -478,7 +478,7 @@ F src/printf.c 9506b4b96e59c0467047155f09015750cb2878aeda3d39e5610c1192ddc3c41c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c 6c84733262885f70675b7b4d5c3f5d21562cd378cf06d5b0cb3bdbdb4785b85f
+F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
 F src/shell.c.in 339169a3d1307b5566ebe9ce15832d03439206106724c78cc3d9125a7b851795
 F src/sqlite.h.in 2126192945019d4cdce335cb236b440a05ec75c93e4cd94c9c6d6e7fcc654cc4
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
@@ -682,7 +682,7 @@ F test/collate9.test 3adcc799229545940df2f25308dd1ad65869145a
 F test/collateA.test b8218ab90d1fa5c59dcf156efabb1b2599c580d6
 F test/collateB.test 1e68906951b846570f29f20102ed91d29e634854ee47454d725f2151ecac0b95
 F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1
-F test/colname.test c47639d26cbeba6977457e5ef2c2c55c5b6c889478dd7eb0ed858ba894e7fa93
+F test/colname.test a7ecb8f1d6d8b30a6cf8fa84a2cd6f6e91cad8296376fabe485cf93cd5eb6229
 F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db
 F test/conflict2.test bb0b94cf7196c64a3cbd815c66d3ee98c2fecd9c
 F test/conflict3.test a83db76a6c3503b2fa057c7bfb08c318d8a422202d8bc5b86226e078e5b49ff9
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5efd854fe21470336ba4140294b6c90ef39af32e103b92c664438d7485c50f9a
-R 3deaa3da40d43e03f04500b413c136e7
+P 8f194008c3aaa4ef287200e37bc5278ba9c377a7091ee3f95bad66513226b083
+R 64e4839a73a8deb3e3e75445ed44ca78
 U drh
-Z 1c7db6ece307b188f8e97c3dc484c292
+Z d26f24526e4b1ec13c6ca34a79230a0c
diff --git a/manifest.uuid b/manifest.uuid
index 8c8a394848..521110fac7 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-8f194008c3aaa4ef287200e37bc5278ba9c377a7091ee3f95bad66513226b083
\ No newline at end of file
+6b2ff26c25bb9da344add79c93fb3e49fa034a89b38ef56e08e18d21de61f707
\ No newline at end of file
diff --git a/src/build.c b/src/build.c
index 9582f136c8..01d8972415 100644
--- a/src/build.c
+++ b/src/build.c
@@ -1965,11 +1965,6 @@ void sqlite3EndTable(
       pParse->nTab = 2;
       addrTop = sqlite3VdbeCurrentAddr(v) + 1;
       sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop);
-      sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield);
-      sqlite3Select(pParse, pSelect, &dest);
-      sqlite3VdbeEndCoroutine(v, regYield);
-      sqlite3VdbeJumpHere(v, addrTop - 1);
-      if( pParse->nErr ) return;
       pSelTab = sqlite3ResultSetOfSelect(pParse, pSelect);
       if( pSelTab==0 ) return;
       assert( p->aCol==0 );
@@ -1978,6 +1973,10 @@ void sqlite3EndTable(
       pSelTab->nCol = 0;
       pSelTab->aCol = 0;
       sqlite3DeleteTable(db, pSelTab);
+      sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield);
+      sqlite3Select(pParse, pSelect, &dest);
+      sqlite3VdbeEndCoroutine(v, regYield);
+      sqlite3VdbeJumpHere(v, addrTop - 1);
       addrInsLoop = sqlite3VdbeAddOp1(v, OP_Yield, dest.iSDParm);
       VdbeCoverage(v);
       sqlite3VdbeAddOp3(v, OP_MakeRecord, dest.iSdst, dest.nSdst, regRec);
diff --git a/src/select.c b/src/select.c
index e37fffd147..1a4b0a93af 100644
--- a/src/select.c
+++ b/src/select.c
@@ -1381,8 +1381,9 @@ static const char *columnTypeImpl(
 
   assert( pExpr!=0 );
   assert( pNC->pSrcList!=0 );
+  assert( pExpr->op!=TK_AGG_COLUMN );  /* This routine runes before aggregates
+                                       ** are processed */
   switch( pExpr->op ){
-    case TK_AGG_COLUMN:
     case TK_COLUMN: {
       /* The expression is a column. Locate the table the column is being
       ** extracted from in NameContext.pSrcList. This table may be real
@@ -1391,8 +1392,6 @@ static const char *columnTypeImpl(
       Table *pTab = 0;            /* Table structure column is extracted from */
       Select *pS = 0;             /* Select the column is extracted from */
       int iCol = pExpr->iColumn;  /* Index of column in pTab */
-      testcase( pExpr->op==TK_AGG_COLUMN );
-      testcase( pExpr->op==TK_COLUMN );
       while( pNC && !pTab ){
         SrcList *pTabList = pNC->pSrcList;
         for(j=0;j<pTabList->nSrc && pTabList->a[j].iCursor!=pExpr->iTable;j++);
@@ -1705,12 +1704,12 @@ int sqlite3ColumnsFromExprList(
         pColExpr = pColExpr->pRight;
         assert( pColExpr!=0 );
       }
-      if( (pColExpr->op==TK_COLUMN || pColExpr->op==TK_AGG_COLUMN)
-       && pColExpr->pTab!=0 
-      ){
+      assert( pColExpr->op!=TK_AGG_COLUMN );
+      if( pColExpr->op==TK_COLUMN ){
         /* For columns use the column name name */
         int iCol = pColExpr->iColumn;
         Table *pTab = pColExpr->pTab;
+        assert( pTab!=0 );
         if( iCol<0 ) iCol = pTab->iPKey;
         zName = iCol>=0 ? pTab->aCol[iCol].zName : "rowid";
       }else if( pColExpr->op==TK_ID ){
diff --git a/test/colname.test b/test/colname.test
index 2e4ae89008..5a40286773 100644
--- a/test/colname.test
+++ b/test/colname.test
@@ -378,6 +378,27 @@ do_test colname-9.210 {
   execsql2 {SELECT t1.a, v3.a AS n FROM t1 JOIN v3}
 } {a 1 n 3}
 
+# 2017-12-23:  Ticket https://www.sqlite.org/src/info/3b4450072511e621
+# Inconsistent column names in CREATE TABLE AS
+#
+# Verify that the names of columns in the created table of a CREATE TABLE AS
+# are the same as the names of result columns in the SELECT statement.
+#
+do_execsql_test colname-9.300 {
+  DROP TABLE IF EXISTS t1;
+  DROP TABLE IF EXISTS t2;
+  CREATE TABLE t1(aaa INT);
+  INSERT INTO t1(aaa) VALUES(123);
+}
+do_test colname-9.310 {
+  execsql2 {SELECT BBb FROM (SELECT aaa AS Bbb FROM t1)}
+} {Bbb 123}
+do_execsql_test colname-9.320 {
+  CREATE TABLE t2 AS SELECT BBb FROM (SELECT aaa AS Bbb FROM t1);
+  SELECT name FROM pragma_table_info('t2');
+} {Bbb}
+
+
 # Make sure the quotation marks get removed from the column names
 # when constructing a new table from an aggregate SELECT.
 # Email from Juergen Palm on 2017-07-11.

From ab42a2fe8ea01e7e1f21eddd4945e9ee3aff9cc5 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 23 Dec 2017 13:55:43 +0000
Subject: [PATCH 67/68] Improve the error message that comes back when
 sqlite3_load_extension() fails because the named file exists but is not a
 valid shared library.

FossilOrigin-Name: 05fee1a21ea398f1e4d6f1cf361657eff25ed6cd8f85ab398262dcfd30da57e9
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/loadext.c |  7 ++++++-
 3 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 3d7fcbcc18..1122cef153 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Move\sthe\sgeneration\sof\soutput\scolumn\snames\searlier\sin\sthe\scase\sof\sa\nCREATE\sTABLE\sAS.\s\sThis\sis\sa\sfix\sfor\sticket\s[3b4450072511e62]\sand\sa\ncontinuation\sof\scheck-in\s[ade7ddf1998190b2b63]\sthat\sfixes\scases\sof\nticket\s[de3403bf5ae5f72ed6]\sthat\swere\smissed\spreviously.
-D 2017-12-23T12:33:40.443
+C Improve\sthe\serror\smessage\sthat\scomes\sback\swhen\ssqlite3_load_extension()\nfails\sbecause\sthe\snamed\sfile\sexists\sbut\sis\snot\sa\svalid\sshared\slibrary.
+D 2017-12-23T13:55:43.819
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -442,7 +442,7 @@ F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c cb67cc56ef2ddd13e6944b2c0dd08a920bcd9503230adef8b9928d338097c722
 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e
-F src/loadext.c 20865b183bb8a3723d59cf1efffc3c50217eb452c1021d077b908c94da26b0b2
+F src/loadext.c c2eb35ab31035effc0429e367147a06c2a7aeb86a9114b0cbe9b882a9fb8f131
 F src/main.c 7ce55fa3c0bf669944de309ebab1655ed06ec67869adb0372c7a1062e461c448
 F src/malloc.c a02c9e69bc76bee0f639416b947a946412890b606301454727feadcb313536d6
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 8f194008c3aaa4ef287200e37bc5278ba9c377a7091ee3f95bad66513226b083
-R 64e4839a73a8deb3e3e75445ed44ca78
+P 6b2ff26c25bb9da344add79c93fb3e49fa034a89b38ef56e08e18d21de61f707
+R 7577ffeaa1cca163e13b3e78ba9bdfee
 U drh
-Z d26f24526e4b1ec13c6ca34a79230a0c
+Z 436076bd78ea4f3cfe09ab636dacef01
diff --git a/manifest.uuid b/manifest.uuid
index 521110fac7..304225ca85 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6b2ff26c25bb9da344add79c93fb3e49fa034a89b38ef56e08e18d21de61f707
\ No newline at end of file
+05fee1a21ea398f1e4d6f1cf361657eff25ed6cd8f85ab398262dcfd30da57e9
\ No newline at end of file
diff --git a/src/loadext.c b/src/loadext.c
index 6751425936..c28600aef8 100644
--- a/src/loadext.c
+++ b/src/loadext.c
@@ -496,8 +496,13 @@ static int sqlite3LoadExtension(
 #if SQLITE_OS_UNIX || SQLITE_OS_WIN
   for(ii=0; ii<ArraySize(azEndings) && handle==0; ii++){
     char *zAltFile = sqlite3_mprintf("%s.%s", zFile, azEndings[ii]);
+    int bOk = 0;
     if( zAltFile==0 ) return SQLITE_NOMEM_BKPT;
-    handle = sqlite3OsDlOpen(pVfs, zAltFile);
+    if( sqlite3OsAccess(pVfs,zAltFile,SQLITE_ACCESS_EXISTS,&bOk)==SQLITE_OK
+     && bOk
+    ){
+      handle = sqlite3OsDlOpen(pVfs, zAltFile);
+    }
     sqlite3_free(zAltFile);
   }
 #endif

From 88a921ce602c7d940a0ba4ed3eeeef31a4ce1455 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 23 Dec 2017 14:39:36 +0000
Subject: [PATCH 68/68] Simplification to the error handling logic in the
 extension loader.

FossilOrigin-Name: 07c773148d8db185fa54991df09298b64f4fef28879e6c9395759265e8183977
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/loadext.c |  9 +++------
 3 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/manifest b/manifest
index 1122cef153..f187cc0290 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improve\sthe\serror\smessage\sthat\scomes\sback\swhen\ssqlite3_load_extension()\nfails\sbecause\sthe\snamed\sfile\sexists\sbut\sis\snot\sa\svalid\sshared\slibrary.
-D 2017-12-23T13:55:43.819
+C Simplification\sto\sthe\serror\shandling\slogic\sin\sthe\sextension\sloader.
+D 2017-12-23T14:39:36.160
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -442,7 +442,7 @@ F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c cb67cc56ef2ddd13e6944b2c0dd08a920bcd9503230adef8b9928d338097c722
 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e
-F src/loadext.c c2eb35ab31035effc0429e367147a06c2a7aeb86a9114b0cbe9b882a9fb8f131
+F src/loadext.c 55bcc3c741059a1056859e8adaf133aa179e22be12215c0936b2f354ef71209b
 F src/main.c 7ce55fa3c0bf669944de309ebab1655ed06ec67869adb0372c7a1062e461c448
 F src/malloc.c a02c9e69bc76bee0f639416b947a946412890b606301454727feadcb313536d6
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6b2ff26c25bb9da344add79c93fb3e49fa034a89b38ef56e08e18d21de61f707
-R 7577ffeaa1cca163e13b3e78ba9bdfee
+P 05fee1a21ea398f1e4d6f1cf361657eff25ed6cd8f85ab398262dcfd30da57e9
+R 7fbfe3e61eca395fa5baf5f121a4d2ad
 U drh
-Z 436076bd78ea4f3cfe09ab636dacef01
+Z d82d9c01768cefc5beb206b92e1398bf
diff --git a/manifest.uuid b/manifest.uuid
index 304225ca85..8454418cd5 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-05fee1a21ea398f1e4d6f1cf361657eff25ed6cd8f85ab398262dcfd30da57e9
\ No newline at end of file
+07c773148d8db185fa54991df09298b64f4fef28879e6c9395759265e8183977
\ No newline at end of file
diff --git a/src/loadext.c b/src/loadext.c
index c28600aef8..abc6b3ff1d 100644
--- a/src/loadext.c
+++ b/src/loadext.c
@@ -496,13 +496,10 @@ static int sqlite3LoadExtension(
 #if SQLITE_OS_UNIX || SQLITE_OS_WIN
   for(ii=0; ii<ArraySize(azEndings) && handle==0; ii++){
     char *zAltFile = sqlite3_mprintf("%s.%s", zFile, azEndings[ii]);
-    int bOk = 0;
+    int bExists = 0;
     if( zAltFile==0 ) return SQLITE_NOMEM_BKPT;
-    if( sqlite3OsAccess(pVfs,zAltFile,SQLITE_ACCESS_EXISTS,&bOk)==SQLITE_OK
-     && bOk
-    ){
-      handle = sqlite3OsDlOpen(pVfs, zAltFile);
-    }
+    sqlite3OsAccess(pVfs, zAltFile, SQLITE_ACCESS_EXISTS, &bExists);
+    if( bExists )  handle = sqlite3OsDlOpen(pVfs, zAltFile);
     sqlite3_free(zAltFile);
   }
 #endif