From e7dd68c2d44350a48ca2f4010b81f2470a265933 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 11 Nov 2015 18:08:58 +0000
Subject: [PATCH 001/488] Add a hack to debug out a description of the WHERE
 clause of a SELECT (or other) statement. Use this in script
 tool/schemalint.tcl to automatically recommend indexes that might speed up
 specific queries.

FossilOrigin-Name: c6fa01c28ef7ceea2963a92dfffe62eed451b05c
---
 manifest             |  19 ++--
 manifest.uuid        |   2 +-
 src/where.c          | 152 +++++++++++++++++++++++++
 test/schemalint.test |  81 ++++++++++++++
 tool/schemalint.tcl  | 259 +++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 505 insertions(+), 8 deletions(-)
 create mode 100644 test/schemalint.test
 create mode 100644 tool/schemalint.tcl

diff --git a/manifest b/manifest
index 5e942ee5cf..ce318ef3fd 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improvements\sto\sthe\sSQLITE_CONFIG_PAGECACHE\sdocumentation.\s\sEnhance\sthe\ncommand-line\sshell\sto\sbe\sable\sto\stake\sadvantage\sof\sthe\sfull\srange\sof\nSQLITE_CONFIG_PAGECACHE\scapabilities,\ssuch\sas\ssetting\spMem==NULL\sand\sN<0.
-D 2015-11-11T15:28:52.898
+C Add\sa\shack\sto\sdebug\sout\sa\sdescription\sof\sthe\sWHERE\sclause\sof\sa\sSELECT\s(or\sother)\sstatement.\sUse\sthis\sin\sscript\stool/schemalint.tcl\sto\sautomatically\srecommend\sindexes\sthat\smight\sspeed\sup\sspecific\squeries.
+D 2015-11-11T18:08:58.267
 F Makefile.in d828db6afa6c1fa060d01e33e4674408df1942a1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e928e68168df69b353300ac87c10105206653a03
@@ -416,7 +416,7 @@ F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb
 F src/wal.c 18b0ed49830cf04fe2d68224b41838a73ac6cd24
 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4
 F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba
-F src/where.c 6aceb72cc58dc06922a9e1604d559c8ca4c3e728
+F src/where.c 6176426332d5a67df4222adfeae8c22a933f8c84
 F src/whereInt.h 7892bb54cf9ca0ae5c7e6094491b94c9286dc647
 F src/wherecode.c 4c96182e7b25e4be54008dee2da5b9c2f8480b9b
 F src/whereexpr.c e63244ca06c503e5f3c5b7f3c9aea0db826089ed
@@ -972,6 +972,7 @@ F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5
 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38
 F test/schema4.test e6a66e20cc69f0e306667c08be7fda3d11707dc5
 F test/schema5.test 29699b4421f183c8f0e88bd28ce7d75d13ea653e
+F test/schemalint.test 22f26e6e9d8fe437b6344a97c91fd14c95e813a7
 F test/securedel.test 21749c32ccc30f1ea9e4b9f33295a6521ec20fa0
 F test/securedel2.test 2d54c28e46eb1fd6902089958b20b1b056c6f1c5
 F test/select1.test be62204d2bd9a5a8a149e9974cfddce893d8f686
@@ -1378,6 +1379,7 @@ F tool/replace.tcl 7727c60a04299b65a92f5e1590896fea0f25b9e0
 F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a
 F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5
 F tool/run-speed-test.sh 0ae485af4fe9f826e2b494be8c81f8ca9e222a4a
+F tool/schemalint.tcl 2eb60e950e91061c27d771582fe87e9518d0988d
 F tool/showdb.c d4476e000a64eca9f5e2c2f68741e747b9778e8d
 F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818
 F tool/showlocks.c 9920bcc64f58378ff1118caead34147201f48c68
@@ -1402,7 +1404,10 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P e43e1171fd7837a08069dc25df4eac14db1c2afe
-R 9a1d72f8baac116c36385e92e48c0378
-U drh
-Z 9d52ec874b9a01b7b0eb1dab69dbb993
+P 2518d5c971c4b32d9227b3bb7259162e3e27b00b
+R 9a794d90ebfe5dc636fc86a7e99d61d5
+T *branch * schemalint
+T *sym-schemalint *
+T -sym-trunk *
+U dan
+Z 95ad0fe90f5fe7059fb61a18885ffca1
diff --git a/manifest.uuid b/manifest.uuid
index bf01d0ad61..079b2e76d7 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2518d5c971c4b32d9227b3bb7259162e3e27b00b
\ No newline at end of file
+c6fa01c28ef7ceea2963a92dfffe62eed451b05c
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index 1c87706ea2..becfb5e8ce 100644
--- a/src/where.c
+++ b/src/where.c
@@ -3901,6 +3901,155 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){
   return 0;
 }
 
+#ifdef SQLITE_SCHEMA_LINT
+static char *whereAppendPrintf(sqlite3 *db, const char *zFmt, ...){
+  va_list ap;
+  char *zRes = 0;
+  va_start(ap, zFmt);
+  zRes = sqlite3_vmprintf(zFmt, ap);
+  if( zRes==0 ){
+    db->mallocFailed = 1;
+  }else if( db->mallocFailed ){
+    sqlite3_free(zRes);
+    zRes = 0;
+  }
+  va_end(ap);
+  return zRes;
+}
+
+/*
+** Append a representation of term pTerm to the string in zIn and return
+** the result. Or, if an OOM occurs, free zIn and return a NULL pointer.
+*/
+static char *whereAppendSingleTerm(
+  Parse *pParse,
+  Table *pTab,
+  int bOr,
+  char *zIn,
+  WhereTerm *pTerm
+){
+  char *zBuf;
+  sqlite3 *db = pParse->db;
+  Expr *pX = pTerm->pExpr;
+  CollSeq *pColl;
+  const char *zOp = 0;
+
+  if( pTerm->eOperator & (WO_IS|WO_EQ|WO_IN) ){
+    zOp = "eq";
+  }else if( pTerm->eOperator & (WO_LT|WO_LE|WO_GE|WO_GT) ){
+    zOp = "range";
+  }
+  pColl = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight);
+
+  if( zOp ){
+    const char *zFmt = bOr ? "%z{{%s %s %s %lld}}" : "%z{%s %s %s %lld}";
+    zBuf = whereAppendPrintf(db, zFmt, zIn, 
+        zOp, pTab->aCol[pTerm->u.leftColumn].zName, 
+        (pColl ? pColl->zName : "BINARY"),
+        pTerm->prereqRight
+    );
+  }else{
+    zBuf = zIn;
+  }
+
+  return zBuf;
+}
+
+static char *whereTraceWC(
+  Parse *pParse, 
+  struct SrcList_item *pItem,
+  char *zIn,
+  WhereClause *pWC
+){
+  sqlite3 *db = pParse->db;
+  Table *pTab = pItem->pTab;
+  char *zBuf = zIn;
+  int iCol;
+  int ii;
+  int bFirst = 1;
+
+  /* List of WO_SINGLE constraints */
+  for(iCol=0; iCol<pTab->nCol; iCol++){
+    int opMask = WO_SINGLE; 
+    WhereScan scan;
+    WhereTerm *pTerm;
+    for(pTerm=whereScanInit(&scan, pWC, pItem->iCursor, iCol, opMask, 0);
+        pTerm;
+        pTerm=whereScanNext(&scan)
+    ){
+      assert( iCol==pTerm->u.leftColumn );
+      if( bFirst==0 ) zBuf = whereAppendPrintf(db, "%z ", zBuf);
+      zBuf = whereAppendSingleTerm(pParse, pTab, pWC->op==TK_OR, zBuf, pTerm);
+      bFirst = 0;
+    }
+  }
+
+  /* Add composite - (WO_OR|WO_AND) - constraints */
+  for(ii=0; ii<pWC->nTerm; ii++){
+    WhereTerm *pTerm = &pWC->a[ii];
+    if( pTerm->eOperator & (WO_OR|WO_AND) ){
+      const char *zFmt = ((pTerm->eOperator&WO_OR) ? "%z%s{or " : "%z%s{");
+      zBuf = whereAppendPrintf(db, zFmt, zBuf, bFirst ? "" : " ");
+      zBuf = whereTraceWC(pParse, pItem, zBuf, &pTerm->u.pOrInfo->wc);
+      zBuf = whereAppendPrintf(db, "%z}", zBuf);
+      bFirst = 0;
+    }
+  }
+
+  return zBuf;
+}
+
+static void whereTraceBuilder(
+  Parse *pParse,
+  WhereLoopBuilder *p
+){
+  sqlite3 *db = pParse->db;
+  if( db->xTrace ){
+    WhereInfo *pWInfo = p->pWInfo;
+    int nTablist = pWInfo->pTabList->nSrc;
+    int ii;
+
+    /* Loop through each element of the FROM clause. Ignore any sub-selects
+    ** or views. Invoke the xTrace() callback once for each real table. */
+    for(ii=0; ii<nTablist; ii++){
+      char *zBuf = 0;
+      int iCol;
+      int nCol;
+      Table *pTab;
+
+      struct SrcList_item *pItem = &pWInfo->pTabList->a[ii];
+      if( pItem->pSelect ) continue;
+      pTab = pItem->pTab;
+      nCol = pTab->nCol;
+
+      /* Append the table name to the buffer. */
+      zBuf = whereAppendPrintf(db, "%s", pTab->zName);
+
+      /* Append the list of columns required to create a covering index */
+      zBuf = whereAppendPrintf(db, "%z {cols", zBuf);
+      if( 0==(pItem->colUsed & ((u64)1 << (sizeof(Bitmask)*8-1))) ){
+        for(iCol=0; iCol<nCol; iCol++){
+          if( iCol==(sizeof(Bitmask)*8-1) ) break;
+          if( pItem->colUsed & ((u64)1 << iCol) ){
+            zBuf = whereAppendPrintf(db, "%z %s", zBuf, pTab->aCol[iCol].zName);
+          }
+        }
+      }
+      zBuf = whereAppendPrintf(db, "%z} ", zBuf);
+
+      /* Append the contents of WHERE clause */
+      zBuf = whereTraceWC(pParse, pItem, zBuf, p->pWC);
+
+      /* Pass the buffer to the xTrace() callback, then free it */
+      db->xTrace(db->pTraceArg, zBuf);
+      sqlite3DbFree(db, zBuf);
+    }
+  }
+}
+#else
+# define whereTraceBuilder(x,y)
+#endif
+
 /*
 ** Generate the beginning of the loop used for WHERE clause processing.
 ** The return value is a pointer to an opaque structure that contains
@@ -4161,6 +4310,9 @@ WhereInfo *sqlite3WhereBegin(
   }
 #endif
 
+  /* Schema-lint xTrace callback */
+  whereTraceBuilder(pParse, &sWLB);
+
   if( nTabList!=1 || whereShortCut(&sWLB)==0 ){
     rc = whereLoopAddAll(&sWLB);
     if( rc ) goto whereBeginError;
diff --git a/test/schemalint.test b/test/schemalint.test
new file mode 100644
index 0000000000..f367aba29c
--- /dev/null
+++ b/test/schemalint.test
@@ -0,0 +1,81 @@
+
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix schemalint
+
+proc xTrace {zMsg} {
+  lappend ::trace_out $zMsg
+}
+db trace xTrace
+
+proc do_trace_test {tn sql res} {
+  uplevel [list do_test $tn [subst -nocommands {
+    set ::trace_out [list]
+    set stmt [sqlite3_prepare db "$sql" -1 x]
+    sqlite3_finalize [set stmt]
+    set ::trace_out
+  }] [list {*}$res]]
+}
+
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, b, c);
+  CREATE TABLE t2(x, y, z);
+}
+
+do_trace_test 1.1 {
+  SELECT b, c, y, z FROM t1, t2 WHERE c=? AND z=?
+} {
+  {t1 {cols b c} {eq c BINARY 0}}
+  {t2 {cols y z} {eq z BINARY 0}}
+}
+
+do_trace_test 1.2 {
+  SELECT a FROM t1 WHERE b>10
+} {
+  {t1 {cols a b} {range b BINARY 0}}
+}
+
+do_trace_test 1.3 {
+  SELECT b FROM t1 WHERE b IN (10, 20, 30)
+} {
+  {t1 {cols b} {eq b BINARY 0}}
+}
+
+do_trace_test 1.4 {
+  SELECT * FROM t1, t2 WHERE x=a
+} {
+  {t1 {cols a b c} {eq a BINARY 2}} 
+  {t2 {cols x y z} {eq x BINARY 1}}
+}
+
+do_trace_test 1.5 {
+  SELECT * FROM t1 WHERE a IN (1, 2, 3)
+} {
+  {t1 {cols a b c} {eq a BINARY 0}}
+}
+
+#-----------------------------------------------------------------------
+# Cases involving OR clauses in the WHERE clause.
+#
+do_trace_test 2.1 {
+  SELECT * FROM t1 WHERE a=? OR b=?
+} {
+  {t1 {cols a b c} {or {{eq a BINARY 0}} {{eq b BINARY 0}}}}
+}
+
+do_trace_test 2.2 {
+  SELECT * FROM t1 WHERE a=? OR (b=? AND c=?)
+} {
+  {t1 {cols a b c} {or {{eq a BINARY 0}} {{eq b BINARY 0} {eq c BINARY 0}}}}
+}
+
+do_trace_test 2.3 {
+  SELECT * FROM t1 WHERE (a=? AND b=?) OR c=?
+} {
+  {t1 {cols a b c} {or {{eq c BINARY 0}} {{eq a BINARY 0} {eq b BINARY 0}}}}
+}
+
+finish_test
+
diff --git a/tool/schemalint.tcl b/tool/schemalint.tcl
new file mode 100644
index 0000000000..41f9ff70eb
--- /dev/null
+++ b/tool/schemalint.tcl
@@ -0,0 +1,259 @@
+
+
+
+set ::G(lSelect)  [list]           ;# List of SELECT statements to analyze
+set ::G(database) ""               ;# Name of database or SQL schema file
+set ::G(trace)    [list]           ;# List of data from xTrace()
+set ::G(verbose)  0                ;# True if -verbose option was passed 
+
+proc usage {} {
+  puts stderr "Usage: $::argv0 ?SWITCHES? DATABASE/SCHEMA"
+  puts stderr "  Switches are:"
+  puts stderr "  -select SQL     (recommend indexes for SQL statement)"
+  puts stderr "  -verbose        (increase verbosity of output)"
+  puts stderr ""
+  exit
+}
+
+proc process_cmdline_args {argv} {
+  global G
+  set nArg [llength $argv]
+  set G(database) [lindex $argv end]
+
+  for {set i 0} {$i < [llength $argv]-1} {incr i} {
+    set k [lindex $argv $i]
+    switch -- $k {
+      -select {
+        incr i
+        if {$i>=[llength $argv]-1} usage
+        lappend G(lSelect) [lindex $argv $i]
+      }
+      -verbose {
+        set G(verbose) 1
+      }
+      default {
+        usage
+      }
+    }
+  }
+}
+
+proc open_database {} {
+  global G
+  sqlite3 db ""
+
+  # Check if the "database" file is really an SQLite database. If so, copy
+  # it into the temp db just opened. Otherwise, assume that it is an SQL
+  # schema and execute it directly.
+  set fd [open $G(database)]
+  set hdr [read $fd 16]
+  if {$hdr == "SQLite format 3\000"} {
+    close $fd
+    sqlite3 db2 $G(database)
+    sqlite3_backup B db main db2 main
+    B step 2000000000
+    set rc [B finish]
+    db2 close
+    if {$rc != "SQLITE_OK"} { error "Failed to load database $G(database)" }
+  } else {
+    append hdr [read $fd]
+    db eval $hdr
+    close $fd
+  }
+}
+
+proc analyze_selects {} {
+  global G
+  set G(trace) ""
+
+  # Collect a line of xTrace output for each loop in the set of SELECT
+  # statements.
+  proc xTrace {zMsg} { lappend ::G(trace) $zMsg }
+  db trace "lappend ::G(trace)"
+  foreach s $G(lSelect) {
+    set stmt [sqlite3_prepare_v2 db $s -1 dummy]
+    set rc [sqlite3_finalize $stmt]
+    if {$rc!="SQLITE_OK"} {
+      error "Failed to compile SQL: [sqlite3_errmsg db]"
+    }
+  }
+
+  db trace ""
+  if {$G(verbose)} {
+    foreach t $G(trace) { puts "trace: $t" }
+  }
+
+  # puts $G(trace)
+}
+
+# The argument is a list of the form:
+#
+#    key1 {value1.1 value1.2} key2 {value2.1 value 2.2...}
+#
+# Values lists may be of any length greater than zero. This function returns
+# a list of lists created by pivoting on each values list. i.e. a list
+# consisting of the elements:
+#
+#   {{key1 value1.1} {key2 value2.1}}
+#   {{key1 value1.2} {key2 value2.1}}
+#   {{key1 value1.1} {key2 value2.2}}
+#   {{key1 value1.2} {key2 value2.2}}
+#
+proc expand_eq_list {L} {
+  set ll [list {}]
+  for {set i 0} {$i < [llength $L]} {incr i 2} {
+    set key [lindex $L $i]
+    set new [list]
+    foreach piv [lindex $L $i+1] {
+      foreach l $ll {
+        lappend new [concat $l [list [list $key $piv]]]
+      }
+    }
+    set ll $new
+  }
+
+  return $ll
+}
+
+proc eqset_to_index {tname eqset {range {}}} {
+  global G
+  set lCols [list]
+  set idxname $tname
+  foreach e [concat [lsort $eqset] [list $range]] {
+    if {[llength $e]==0} continue
+    foreach {c collate} $e {}
+    lappend lCols "$c collate $collate"
+    append idxname "_$c"
+    if {[string compare -nocase binary $collate]!=0} {
+      append idxname [string tolower $collate]
+    }
+  }
+
+  set create_index "CREATE INDEX $idxname ON ${tname}("
+  append create_index [join $lCols ", "]
+  append create_index ");"
+
+  set G(trial.$idxname) $create_index
+}
+
+proc expand_or_cons {L} {
+  set lRet [list [list]]
+  foreach elem $L {
+    set type [lindex $elem 0]
+    if {$type=="eq" || $type=="range"} {
+      set lNew [list]
+      for {set i 0} {$i < [llength $lRet]} {incr i} {
+        lappend lNew [concat [lindex $lRet $i] [list $elem]]
+      }
+      set lRet $lNew
+    } elseif {$type=="or"} {
+      set lNew [list]
+      foreach branch [lrange $elem 1 end] {
+        foreach b [expand_or_cons $branch] {
+          for {set i 0} {$i < [llength $lRet]} {incr i} {
+            lappend lNew [concat [lindex $lRet $i] $b]
+          }
+        }
+      }
+      set lRet $lNew
+    } 
+  }
+  return $lRet
+}
+
+proc find_trial_indexes {} {
+  global G
+  foreach t $G(trace) {
+    set tname [lindex $t 0]
+    catch { array unset mask }
+
+    foreach lCons [expand_or_cons [lrange $t 2 end]] {
+      set constraints [list]
+
+      foreach a $lCons {
+        set type [lindex $a 0]
+        if {$type=="eq" || $type=="range"} {
+          set m [lindex $a 3]
+          foreach k [array names mask] { set mask([expr ($k & $m)]) 1 }
+          set mask($m) 1
+          lappend constraints $a
+        }
+      }
+
+      foreach k [array names mask] {
+        catch {array unset eq}
+        foreach a $constraints {
+          foreach {type col collate m} $a {
+            if {($m & $k)==$m} {
+              if {$type=="eq"} {
+                lappend eq($col) $collate
+              } else {
+                set range($col.$collate) 1
+              }
+            }
+          }
+        }
+
+        #puts "mask=$k eq=[array get eq] range=[array get range]"
+        
+        set ranges [array names range]
+        foreach eqset [expand_eq_list [array get eq]] {
+          if {[llength $ranges]==0} {
+            eqset_to_index $tname $eqset
+          } else {
+            foreach r $ranges {
+              set bSeen 0
+              foreach {c collate} [split $r .] {}
+              foreach e $eqset {
+                if {[lindex $e 0] == $c} {
+                  set bSeen 1
+                  break
+                }
+              }
+              if {$bSeen} {
+                eqset_to_index $tname $eqset
+              } else {
+                eqset_to_index $tname $eqset [list $c $collate]
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+  if {$G(verbose)} {
+    foreach k [array names G trial.*] { puts "index: $G($k)" }
+  }
+}
+
+proc run_trials {} {
+  global G
+
+  foreach k [array names G trial.*] {
+    set idxname [lindex [split $k .] 1]
+    db eval $G($k)
+    set pgno [db one {SELECT rootpage FROM sqlite_master WHERE name = $idxname}]
+    set IDX($pgno) $idxname
+  }
+  db eval ANALYZE
+
+  catch { array unset used }
+  foreach s $G(lSelect) {
+    db eval "EXPLAIN $s" x {
+      if {($x(opcode)=="OpenRead" || $x(opcode)=="ReopenIdx")} {
+        if {[info exists IDX($x(p2))]} { set used($IDX($x(p2))) 1 }
+      }
+    }
+    foreach idx [array names used] {
+      puts $G(trial.$idx)
+    }
+  }
+}
+
+process_cmdline_args $argv
+open_database
+analyze_selects
+find_trial_indexes
+run_trials
+

From ef5de04b4edaa859fe7d17886448d5dc3e2cefd1 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 20 Nov 2015 20:55:27 +0000
Subject: [PATCH 002/488] Add support for ORDER BY clauses to schemalint.tcl.

FossilOrigin-Name: 93bdf70e859915ff3696ba0fc68f91ceb2e1a971
---
 manifest             | 19 ++++++++-----------
 manifest.uuid        |  2 +-
 src/where.c          | 33 +++++++++++++++++++++++++++++----
 test/schemalint.test | 19 +++++++++++++++++++
 tool/schemalint.tcl  |  7 +++++++
 5 files changed, 64 insertions(+), 16 deletions(-)

diff --git a/manifest b/manifest
index ce318ef3fd..e9c4624905 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sa\shack\sto\sdebug\sout\sa\sdescription\sof\sthe\sWHERE\sclause\sof\sa\sSELECT\s(or\sother)\sstatement.\sUse\sthis\sin\sscript\stool/schemalint.tcl\sto\sautomatically\srecommend\sindexes\sthat\smight\sspeed\sup\sspecific\squeries.
-D 2015-11-11T18:08:58.267
+C Add\ssupport\sfor\sORDER\sBY\sclauses\sto\sschemalint.tcl.
+D 2015-11-20T20:55:27.428
 F Makefile.in d828db6afa6c1fa060d01e33e4674408df1942a1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e928e68168df69b353300ac87c10105206653a03
@@ -416,7 +416,7 @@ F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb
 F src/wal.c 18b0ed49830cf04fe2d68224b41838a73ac6cd24
 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4
 F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba
-F src/where.c 6176426332d5a67df4222adfeae8c22a933f8c84
+F src/where.c 5804902239731a7025240f4338d98cdc57315636
 F src/whereInt.h 7892bb54cf9ca0ae5c7e6094491b94c9286dc647
 F src/wherecode.c 4c96182e7b25e4be54008dee2da5b9c2f8480b9b
 F src/whereexpr.c e63244ca06c503e5f3c5b7f3c9aea0db826089ed
@@ -972,7 +972,7 @@ F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5
 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38
 F test/schema4.test e6a66e20cc69f0e306667c08be7fda3d11707dc5
 F test/schema5.test 29699b4421f183c8f0e88bd28ce7d75d13ea653e
-F test/schemalint.test 22f26e6e9d8fe437b6344a97c91fd14c95e813a7
+F test/schemalint.test 7fba0e262353c8c3dd5ee406cb11cf90f8b566fe
 F test/securedel.test 21749c32ccc30f1ea9e4b9f33295a6521ec20fa0
 F test/securedel2.test 2d54c28e46eb1fd6902089958b20b1b056c6f1c5
 F test/select1.test be62204d2bd9a5a8a149e9974cfddce893d8f686
@@ -1379,7 +1379,7 @@ F tool/replace.tcl 7727c60a04299b65a92f5e1590896fea0f25b9e0
 F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a
 F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5
 F tool/run-speed-test.sh 0ae485af4fe9f826e2b494be8c81f8ca9e222a4a
-F tool/schemalint.tcl 2eb60e950e91061c27d771582fe87e9518d0988d
+F tool/schemalint.tcl 160fe2ed74df02e859ab32a522fa1f2e8e0c4230
 F tool/showdb.c d4476e000a64eca9f5e2c2f68741e747b9778e8d
 F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818
 F tool/showlocks.c 9920bcc64f58378ff1118caead34147201f48c68
@@ -1404,10 +1404,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 2518d5c971c4b32d9227b3bb7259162e3e27b00b
-R 9a794d90ebfe5dc636fc86a7e99d61d5
-T *branch * schemalint
-T *sym-schemalint *
-T -sym-trunk *
+P c6fa01c28ef7ceea2963a92dfffe62eed451b05c
+R 6fc6a71e3dbe84052a5d99204cd615ee
 U dan
-Z 95ad0fe90f5fe7059fb61a18885ffca1
+Z 64248603e4a564821f72e660b22a8811
diff --git a/manifest.uuid b/manifest.uuid
index 079b2e76d7..a959ffa2b1 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c6fa01c28ef7ceea2963a92dfffe62eed451b05c
\ No newline at end of file
+93bdf70e859915ff3696ba0fc68f91ceb2e1a971
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index becfb5e8ce..4ba35d6ac9 100644
--- a/src/where.c
+++ b/src/where.c
@@ -3957,6 +3957,7 @@ static char *whereAppendSingleTerm(
 
 static char *whereTraceWC(
   Parse *pParse, 
+  int bInitialSpace,
   struct SrcList_item *pItem,
   char *zIn,
   WhereClause *pWC
@@ -3966,7 +3967,7 @@ static char *whereTraceWC(
   char *zBuf = zIn;
   int iCol;
   int ii;
-  int bFirst = 1;
+  int bFirst = !bInitialSpace;
 
   /* List of WO_SINGLE constraints */
   for(iCol=0; iCol<pTab->nCol; iCol++){
@@ -3990,7 +3991,7 @@ static char *whereTraceWC(
     if( pTerm->eOperator & (WO_OR|WO_AND) ){
       const char *zFmt = ((pTerm->eOperator&WO_OR) ? "%z%s{or " : "%z%s{");
       zBuf = whereAppendPrintf(db, zFmt, zBuf, bFirst ? "" : " ");
-      zBuf = whereTraceWC(pParse, pItem, zBuf, &pTerm->u.pOrInfo->wc);
+      zBuf = whereTraceWC(pParse, 0, pItem, zBuf, &pTerm->u.pOrInfo->wc);
       zBuf = whereAppendPrintf(db, "%z}", zBuf);
       bFirst = 0;
     }
@@ -4005,6 +4006,7 @@ static void whereTraceBuilder(
 ){
   sqlite3 *db = pParse->db;
   if( db->xTrace ){
+    ExprList *pOrderBy = p->pOrderBy;
     WhereInfo *pWInfo = p->pWInfo;
     int nTablist = pWInfo->pTabList->nSrc;
     int ii;
@@ -4035,10 +4037,33 @@ static void whereTraceBuilder(
           }
         }
       }
-      zBuf = whereAppendPrintf(db, "%z} ", zBuf);
+      zBuf = whereAppendPrintf(db, "%z}",zBuf);
 
       /* Append the contents of WHERE clause */
-      zBuf = whereTraceWC(pParse, pItem, zBuf, p->pWC);
+      zBuf = whereTraceWC(pParse, 1, pItem, zBuf, p->pWC);
+
+      /* Append the ORDER BY clause, if any */
+      if( pOrderBy ){
+        int i;
+        int bFirst = 1;
+        for(i=0; i<pOrderBy->nExpr; i++){
+          Expr *pExpr = pOrderBy->a[i].pExpr; 
+          CollSeq *pColl = sqlite3ExprCollSeq(pParse, pExpr);
+
+          pExpr = sqlite3ExprSkipCollate(pExpr);
+          if( pExpr->op==TK_COLUMN && pExpr->iTable==pItem->iCursor ){
+            if( pExpr->iColumn>=0 ){
+              const char *zName = pTab->aCol[pExpr->iColumn].zName;
+              zBuf = whereAppendPrintf(db, "%z%s%s %s %s", zBuf,
+                  bFirst ? " {orderby " : " ", zName, pColl->zName,
+                  (pOrderBy->a[i].sortOrder ? "DESC" : "ASC")
+              );
+              bFirst = 0;
+            }
+          }
+        }
+        if( bFirst==0 ) zBuf = whereAppendPrintf(db, "%z}", zBuf);
+      }
 
       /* Pass the buffer to the xTrace() callback, then free it */
       db->xTrace(db->pTraceArg, zBuf);
diff --git a/test/schemalint.test b/test/schemalint.test
index f367aba29c..d354c1fdd7 100644
--- a/test/schemalint.test
+++ b/test/schemalint.test
@@ -77,5 +77,24 @@ do_trace_test 2.3 {
   {t1 {cols a b c} {or {{eq c BINARY 0}} {{eq a BINARY 0} {eq b BINARY 0}}}}
 }
 
+#-----------------------------------------------------------------------
+# Cases involving ORDER BY.
+#
+do_trace_test 3.1 {
+  SELECT * FROM t1 ORDER BY a;
+} {{t1 {cols a b c} {orderby a BINARY ASC}}}
+
+do_trace_test 3.2 {
+  SELECT * FROM t1 WHERE a=? ORDER BY b;
+} {{t1 {cols a b c} {eq a BINARY 0} {orderby b BINARY ASC}}}
+
+do_trace_test 3.3 {
+  SELECT min(a) FROM t1;
+} {{t1 {cols a} {orderby a BINARY ASC}}}
+
+do_trace_test 3.4 {
+  SELECT max(a) FROM t1;
+} {{t1 {cols a} {orderby a BINARY DESC}}}
+
 finish_test
 
diff --git a/tool/schemalint.tcl b/tool/schemalint.tcl
index 41f9ff70eb..94e1ec02ec 100644
--- a/tool/schemalint.tcl
+++ b/tool/schemalint.tcl
@@ -115,6 +115,9 @@ proc expand_eq_list {L} {
   return $ll
 }
 
+#--------------------------------------------------------------------------
+# Formulate a CREATE INDEX statement that creates an index on table $tname.
+#
 proc eqset_to_index {tname eqset {range {}}} {
   global G
   set lCols [list]
@@ -167,6 +170,10 @@ proc find_trial_indexes {} {
     set tname [lindex $t 0]
     catch { array unset mask }
 
+    if {[lindex $t end 0]=="orderby"} {
+      set orderby [lrange [lindex $t end] 1 end]
+    }
+
     foreach lCons [expand_or_cons [lrange $t 2 end]] {
       set constraints [list]
 

From 1b8b839a8db3692a559c1ca3f90fb18bb3e82624 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 23 Nov 2015 17:10:51 +0000
Subject: [PATCH 003/488] Fix ORDER BY handling in the schemalint.tcl script.
 Add internal self-tests to the same script.

FossilOrigin-Name: b8f277c9b45c4b30e6690e838a30311aa8d84876
---
 manifest            |  12 +-
 manifest.uuid       |   2 +-
 tool/schemalint.tcl | 265 +++++++++++++++++++++++++++++++++++---------
 3 files changed, 220 insertions(+), 59 deletions(-)

diff --git a/manifest b/manifest
index e9c4624905..fb61058c44 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\ssupport\sfor\sORDER\sBY\sclauses\sto\sschemalint.tcl.
-D 2015-11-20T20:55:27.428
+C Fix\sORDER\sBY\shandling\sin\sthe\sschemalint.tcl\sscript.\sAdd\sinternal\sself-tests\sto\sthe\ssame\sscript.
+D 2015-11-23T17:10:51.938
 F Makefile.in d828db6afa6c1fa060d01e33e4674408df1942a1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e928e68168df69b353300ac87c10105206653a03
@@ -1379,7 +1379,7 @@ F tool/replace.tcl 7727c60a04299b65a92f5e1590896fea0f25b9e0
 F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a
 F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5
 F tool/run-speed-test.sh 0ae485af4fe9f826e2b494be8c81f8ca9e222a4a
-F tool/schemalint.tcl 160fe2ed74df02e859ab32a522fa1f2e8e0c4230
+F tool/schemalint.tcl e49c9b25c8c43f0531eca96edaf85ce8bd7f308b
 F tool/showdb.c d4476e000a64eca9f5e2c2f68741e747b9778e8d
 F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818
 F tool/showlocks.c 9920bcc64f58378ff1118caead34147201f48c68
@@ -1404,7 +1404,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P c6fa01c28ef7ceea2963a92dfffe62eed451b05c
-R 6fc6a71e3dbe84052a5d99204cd615ee
+P 93bdf70e859915ff3696ba0fc68f91ceb2e1a971
+R 4554e59743437c1fee4689a6514241d8
 U dan
-Z 64248603e4a564821f72e660b22a8811
+Z f43cf07c3dedebc0e4a9e76dc40982d5
diff --git a/manifest.uuid b/manifest.uuid
index a959ffa2b1..7790e3b6f1 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-93bdf70e859915ff3696ba0fc68f91ceb2e1a971
\ No newline at end of file
+b8f277c9b45c4b30e6690e838a30311aa8d84876
\ No newline at end of file
diff --git a/tool/schemalint.tcl b/tool/schemalint.tcl
index 94e1ec02ec..933b277ea2 100644
--- a/tool/schemalint.tcl
+++ b/tool/schemalint.tcl
@@ -1,22 +1,18 @@
 
-
-
-set ::G(lSelect)  [list]           ;# List of SELECT statements to analyze
-set ::G(database) ""               ;# Name of database or SQL schema file
-set ::G(trace)    [list]           ;# List of data from xTrace()
-set ::G(verbose)  0                ;# True if -verbose option was passed 
+set ::VERBOSE 0
 
 proc usage {} {
   puts stderr "Usage: $::argv0 ?SWITCHES? DATABASE/SCHEMA"
   puts stderr "  Switches are:"
   puts stderr "  -select SQL     (recommend indexes for SQL statement)"
   puts stderr "  -verbose        (increase verbosity of output)"
+  puts stderr "  -test           (run internal tests and then exit)"
   puts stderr ""
   exit
 }
 
-proc process_cmdline_args {argv} {
-  global G
+proc process_cmdline_args {ctxvar argv} {
+  upvar $ctxvar G
   set nArg [llength $argv]
   set G(database) [lindex $argv end]
 
@@ -29,17 +25,24 @@ proc process_cmdline_args {argv} {
         lappend G(lSelect) [lindex $argv $i]
       }
       -verbose {
-        set G(verbose) 1
+        set ::VERBOSE 1
+      }
+      -test {
+        sqlidx_internal_tests
       }
       default {
         usage
       }
     }
   }
+
+  if {$G(database)=="-test"} {
+    sqlidx_internal_tests
+  }
 }
 
-proc open_database {} {
-  global G
+proc open_database {ctxvar} {
+  upvar $ctxvar G
   sqlite3 db ""
 
   # Check if the "database" file is really an SQLite database. If so, copy
@@ -62,14 +65,17 @@ proc open_database {} {
   }
 }
 
-proc analyze_selects {} {
-  global G
+proc analyze_selects {ctxvar} {
+  upvar $ctxvar G
   set G(trace) ""
 
   # Collect a line of xTrace output for each loop in the set of SELECT
   # statements.
-  proc xTrace {zMsg} { lappend ::G(trace) $zMsg }
-  db trace "lappend ::G(trace)"
+  proc xTrace {zMsg} { 
+    upvar G G
+    lappend G(trace) $zMsg 
+  }
+  db trace xTrace
   foreach s $G(lSelect) {
     set stmt [sqlite3_prepare_v2 db $s -1 dummy]
     set rc [sqlite3_finalize $stmt]
@@ -79,7 +85,7 @@ proc analyze_selects {} {
   }
 
   db trace ""
-  if {$G(verbose)} {
+  if {$::VERBOSE} {
     foreach t $G(trace) { puts "trace: $t" }
   }
 
@@ -118,11 +124,12 @@ proc expand_eq_list {L} {
 #--------------------------------------------------------------------------
 # Formulate a CREATE INDEX statement that creates an index on table $tname.
 #
-proc eqset_to_index {tname eqset {range {}}} {
-  global G
+proc eqset_to_index {ctxvar tname eqset {range {}}} {
+  upvar $ctxvar G
+
   set lCols [list]
   set idxname $tname
-  foreach e [concat [lsort $eqset] [list $range]] {
+  foreach e [lsort $eqset] { 
     if {[llength $e]==0} continue
     foreach {c collate} $e {}
     lappend lCols "$c collate $collate"
@@ -132,6 +139,19 @@ proc eqset_to_index {tname eqset {range {}}} {
     }
   }
 
+  foreach {c collate dir} $range {
+    append idxname "_$c"
+    if {[string compare -nocase binary $collate]!=0} {
+      append idxname [string tolower $collate]
+    }
+    if {$dir=="DESC"} {
+      lappend lCols "$c collate $collate DESC"
+      append idxname "desc"
+    } else {
+      lappend lCols "$c collate $collate"
+    }
+  }
+
   set create_index "CREATE INDEX $idxname ON ${tname}("
   append create_index [join $lCols ", "]
   append create_index ");"
@@ -164,64 +184,97 @@ proc expand_or_cons {L} {
   return $lRet
 }
 
-proc find_trial_indexes {} {
-  global G
+proc find_trial_indexes {ctxvar} {
+  upvar $ctxvar G
   foreach t $G(trace) {
     set tname [lindex $t 0]
     catch { array unset mask }
 
+    set orderby [list]
     if {[lindex $t end 0]=="orderby"} {
       set orderby [lrange [lindex $t end] 1 end]
     }
 
     foreach lCons [expand_or_cons [lrange $t 2 end]] {
-      set constraints [list]
 
+      # Populate the array mask() so that it contains an entry for each
+      # combination of prerequisite scans that may lead to distinct sets
+      # of constraints being usable.
+      #
+      catch { array unset mask }
+      set mask(0) 1
       foreach a $lCons {
         set type [lindex $a 0]
         if {$type=="eq" || $type=="range"} {
           set m [lindex $a 3]
           foreach k [array names mask] { set mask([expr ($k & $m)]) 1 }
           set mask($m) 1
-          lappend constraints $a
         }
       }
 
+      # Loop once for each distinct prerequisite scan mask identified in
+      # the previous block.
+      #
       foreach k [array names mask] {
+
+        # Identify the constraints available for prerequisite mask $k. For
+        # each == constraint, set an entry in the eq() array as follows:
+        # 
+        #   set eq(<col>) <collation>
+        #
+        # If there is more than one == constraint for a column, and they use
+        # different collation sequences, <collation> is replaced with a list
+        # of the possible collation sequences. For example, for:
+        #
+        #   SELECT * FROM t1 WHERE a=? COLLATE BINARY AND a=? COLLATE NOCASE
+        #
+        # Set the following entry in the eq() array:
+        #
+        #   set eq(a) {binary nocase}
+        #
+        # For each range constraint found an entry is appended to the $ranges
+        # list. The entry is itself a list of the form {<col> <collation>}.
+        #
         catch {array unset eq}
-        foreach a $constraints {
-          foreach {type col collate m} $a {
-            if {($m & $k)==$m} {
-              if {$type=="eq"} {
-                lappend eq($col) $collate
-              } else {
-                set range($col.$collate) 1
+        set ranges [list]
+        foreach a $lCons {
+          set type [lindex $a 0]
+          if {$type=="eq" || $type=="range"} {
+            foreach {type col collate m} $a {
+              if {($m & $k)==$m} {
+                if {$type=="eq"} {
+                  lappend eq($col) $collate
+                } else {
+                  lappend ranges [list $col $collate ASC]
+                }
               }
             }
           }
         }
+        set ranges [lsort -unique $ranges]
+        if {$orderby != ""} {
+          lappend ranges $orderby
+        }
 
-        #puts "mask=$k eq=[array get eq] range=[array get range]"
-        
-        set ranges [array names range]
         foreach eqset [expand_eq_list [array get eq]] {
-          if {[llength $ranges]==0} {
-            eqset_to_index $tname $eqset
-          } else {
-            foreach r $ranges {
+          if {$eqset != ""} {
+            eqset_to_index G $tname $eqset
+          }
+
+          foreach r $ranges {
+            set tail [list]
+            foreach {c collate dir} $r {
               set bSeen 0
-              foreach {c collate} [split $r .] {}
               foreach e $eqset {
                 if {[lindex $e 0] == $c} {
                   set bSeen 1
                   break
                 }
               }
-              if {$bSeen} {
-                eqset_to_index $tname $eqset
-              } else {
-                eqset_to_index $tname $eqset [list $c $collate]
-              }
+              if {$bSeen==0} { lappend tail {*}$r }
+            }
+            if {[llength $tail]} {
+              eqset_to_index G $tname $eqset $r
             }
           }
         }
@@ -229,13 +282,14 @@ proc find_trial_indexes {} {
     }
   }
 
-  if {$G(verbose)} {
+  if {$::VERBOSE} {
     foreach k [array names G trial.*] { puts "index: $G($k)" }
   }
 }
 
-proc run_trials {} {
-  global G
+proc run_trials {ctxvar} {
+  upvar $ctxvar G
+  set ret [list]
 
   foreach k [array names G trial.*] {
     set idxname [lindex [split $k .] 1]
@@ -253,14 +307,121 @@ proc run_trials {} {
       }
     }
     foreach idx [array names used] {
-      puts $G(trial.$idx)
+      lappend ret $G(trial.$idx)
     }
   }
+
+  set ret
 }
 
-process_cmdline_args $argv
-open_database
-analyze_selects
-find_trial_indexes
-run_trials
+proc sqlidx_init_context {varname} {
+  upvar $varname G
+  set G(lSelect)  [list]           ;# List of SELECT statements to analyze
+  set G(database) ""               ;# Name of database or SQL schema file
+  set G(trace)    [list]           ;# List of data from xTrace()
+}
+
+#-------------------------------------------------------------------------
+# The following is test code only.
+#
+proc sqlidx_one_test {tn schema select expected} {
+#  if {$tn!=2} return
+  sqlidx_init_context C
+
+  sqlite3 db ""
+  db eval $schema
+  lappend C(lSelect) $select
+  analyze_selects C
+  find_trial_indexes C
+
+  set idxlist [run_trials C]
+  if {$idxlist != [list {*}$expected]} {
+    puts stderr "Test $tn failed"
+    puts stderr "Expected: $expected"
+    puts stderr "Got: $idxlist"
+    exit -1
+  }
+
+  db close
+}
+
+proc sqlidx_internal_tests {} {
+
+  # No indexes for a query with no constraints.
+  sqlidx_one_test 0 {
+    CREATE TABLE t1(a, b, c);
+  } {
+    SELECT * FROM t1;
+  } {
+  }
+
+  sqlidx_one_test 1 {
+    CREATE TABLE t1(a, b, c);
+    CREATE TABLE t2(x, y, z);
+  } {
+    SELECT a FROM t1, t2 WHERE a=? AND x=c
+  } {
+    {CREATE INDEX t2_x ON t2(x collate BINARY);}
+    {CREATE INDEX t1_a_c ON t1(a collate BINARY, c collate BINARY);}
+  }
+
+  sqlidx_one_test 2 {
+    CREATE TABLE t1(a, b, c);
+  } {
+    SELECT * FROM t1 WHERE b>?;
+  } {
+    {CREATE INDEX t1_b ON t1(b collate BINARY);}
+  }
+
+  sqlidx_one_test 3 {
+    CREATE TABLE t1(a, b, c);
+  } {
+    SELECT * FROM t1 WHERE b COLLATE nocase BETWEEN ? AND ?
+  } {
+    {CREATE INDEX t1_bnocase ON t1(b collate NOCASE);}
+  }
+
+  sqlidx_one_test 4 {
+    CREATE TABLE t1(a, b, c);
+  } {
+    SELECT a FROM t1 ORDER BY b;
+  } {
+    {CREATE INDEX t1_b ON t1(b collate BINARY);}
+  }
+
+  sqlidx_one_test 5 {
+    CREATE TABLE t1(a, b, c);
+  } {
+    SELECT a FROM t1 WHERE a=? ORDER BY b;
+  } {
+    {CREATE INDEX t1_a_b ON t1(a collate BINARY, b collate BINARY);}
+  }
+
+  sqlidx_one_test 5 {
+    CREATE TABLE t1(a, b, c);
+  } {
+    SELECT min(a) FROM t1
+  } {
+    {CREATE INDEX t1_a ON t1(a collate BINARY);}
+  }
+
+  sqlidx_one_test 6 {
+    CREATE TABLE t1(a, b, c);
+  } {
+    SELECT * FROM t1 ORDER BY a ASC, b COLLATE nocase DESC, c ASC;
+  } {
+    {CREATE INDEX t1_a_bnocasedesc_c ON t1(a collate BINARY, b collate NOCASE DESC, c collate BINARY);}
+  }
+
+  exit
+}
+# End of internal test code.
+#-------------------------------------------------------------------------
+
+sqlidx_init_context D
+process_cmdline_args D $argv
+open_database D
+analyze_selects D
+find_trial_indexes D
+foreach idx [run_trials D] { puts $idx }
 

From 323f7d3fc6ca95c55eb0e5ae06e6bacb328a10a7 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 23 Nov 2015 18:28:07 +0000
Subject: [PATCH 004/488] In the CREATE INDEX statements output by
 schemalint.tcl, avoid declaring an explicit collation sequence that is the
 same as the column's default.

FossilOrigin-Name: d3aa067c830e98f2074630c4613c557b0ce90a57
---
 manifest            | 12 +++----
 manifest.uuid       |  2 +-
 tool/schemalint.tcl | 77 ++++++++++++++++++++++++++++++---------------
 3 files changed, 59 insertions(+), 32 deletions(-)

diff --git a/manifest b/manifest
index cd40a978a6..bff0419492 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\slatest\strunk\schanges\swith\sthis\sbranch.
-D 2015-11-23T17:14:03.947
+C In\sthe\sCREATE\sINDEX\sstatements\soutput\sby\sschemalint.tcl,\savoid\sdeclaring\san\sexplicit\scollation\ssequence\sthat\sis\sthe\ssame\sas\sthe\scolumn's\sdefault.
+D 2015-11-23T18:28:07.584
 F Makefile.in d828db6afa6c1fa060d01e33e4674408df1942a1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e928e68168df69b353300ac87c10105206653a03
@@ -1381,7 +1381,7 @@ F tool/replace.tcl 7727c60a04299b65a92f5e1590896fea0f25b9e0
 F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a
 F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5
 F tool/run-speed-test.sh 0ae485af4fe9f826e2b494be8c81f8ca9e222a4a
-F tool/schemalint.tcl e49c9b25c8c43f0531eca96edaf85ce8bd7f308b
+F tool/schemalint.tcl ad61083d39cb40e848b9acc1273e4c487bb55cd7
 F tool/showdb.c d4476e000a64eca9f5e2c2f68741e747b9778e8d
 F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818
 F tool/showlocks.c 9920bcc64f58378ff1118caead34147201f48c68
@@ -1406,7 +1406,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P b8f277c9b45c4b30e6690e838a30311aa8d84876 60de5f23424552c98aa760ac89149a3d51f895be
-R 00875769a91e68d074e499acf18a7d59
+P 8f1ef0904d055b5510ec9043810ebf22a8c5e253
+R 4b9b445c1607ad5c9f9bdcf33b4eb416
 U dan
-Z 5eb820ab8a7550b698dcf8a7d000e044
+Z 6d9149db2c512107280c4e32c490b8cb
diff --git a/manifest.uuid b/manifest.uuid
index b038ee0b5d..ed4290e83d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-8f1ef0904d055b5510ec9043810ebf22a8c5e253
\ No newline at end of file
+d3aa067c830e98f2074630c4613c557b0ce90a57
\ No newline at end of file
diff --git a/tool/schemalint.tcl b/tool/schemalint.tcl
index 933b277ea2..7ed74f9054 100644
--- a/tool/schemalint.tcl
+++ b/tool/schemalint.tcl
@@ -124,32 +124,33 @@ proc expand_eq_list {L} {
 #--------------------------------------------------------------------------
 # Formulate a CREATE INDEX statement that creates an index on table $tname.
 #
-proc eqset_to_index {ctxvar tname eqset {range {}}} {
+proc eqset_to_index {ctxvar aCollVar tname eqset {range {}}} {
   upvar $ctxvar G
+  upvar $aCollVar aColl
+
+  set rangeset [list]
+  foreach e [lsort $eqset] {
+    lappend rangeset [lindex $e 0] [lindex $e 1] ASC
+  }
+  set rangeset [concat $rangeset $range]
 
   set lCols [list]
   set idxname $tname
-  foreach e [lsort $eqset] { 
-    if {[llength $e]==0} continue
-    foreach {c collate} $e {}
-    lappend lCols "$c collate $collate"
-    append idxname "_$c"
-    if {[string compare -nocase binary $collate]!=0} {
-      append idxname [string tolower $collate]
-    }
-  }
 
-  foreach {c collate dir} $range {
+  foreach {c collate dir} $rangeset {
     append idxname "_$c"
-    if {[string compare -nocase binary $collate]!=0} {
+    set coldef $c
+
+    if {[string compare -nocase $collate $aColl($c)]!=0} {
       append idxname [string tolower $collate]
+      append coldef " COLLATE $collate"
     }
+
     if {$dir=="DESC"} {
-      lappend lCols "$c collate $collate DESC"
+      append coldef " DESC"
       append idxname "desc"
-    } else {
-      lappend lCols "$c collate $collate"
     }
+    lappend lCols $coldef
   }
 
   set create_index "CREATE INDEX $idxname ON ${tname}("
@@ -184,12 +185,30 @@ proc expand_or_cons {L} {
   return $lRet
 }
 
+proc sqlidx_get_coll_map {tname arrayvar} {
+  upvar $arrayvar aColl
+  set colnames [list]
+  db eval "PRAGMA table_info = $tname" x { lappend colnames $x(name) }
+  db eval "CREATE INDEX schemalint_test ON ${tname}([join $colnames ,])"
+
+  db eval "PRAGMA index_xinfo = schemalint_test" x { 
+    set aColl($x(name)) $x(coll)
+  }
+  db eval "DROP INDEX schemalint_test"
+}
+
 proc find_trial_indexes {ctxvar} {
   upvar $ctxvar G
   foreach t $G(trace) {
     set tname [lindex $t 0]
     catch { array unset mask }
 
+    # Invoke "PRAGMA table_info" on the table. Use the results to create
+    # an array mapping from column name to collation sequence. Store the
+    # array in local variable aColl.
+    #
+    sqlidx_get_coll_map $tname aColl
+
     set orderby [list]
     if {[lindex $t end 0]=="orderby"} {
       set orderby [lrange [lindex $t end] 1 end]
@@ -258,7 +277,7 @@ proc find_trial_indexes {ctxvar} {
 
         foreach eqset [expand_eq_list [array get eq]] {
           if {$eqset != ""} {
-            eqset_to_index G $tname $eqset
+            eqset_to_index G aColl $tname $eqset
           }
 
           foreach r $ranges {
@@ -274,7 +293,7 @@ proc find_trial_indexes {ctxvar} {
               if {$bSeen==0} { lappend tail {*}$r }
             }
             if {[llength $tail]} {
-              eqset_to_index G $tname $eqset $r
+              eqset_to_index G aColl $tname $eqset $r
             }
           }
         }
@@ -361,8 +380,8 @@ proc sqlidx_internal_tests {} {
   } {
     SELECT a FROM t1, t2 WHERE a=? AND x=c
   } {
-    {CREATE INDEX t2_x ON t2(x collate BINARY);}
-    {CREATE INDEX t1_a_c ON t1(a collate BINARY, c collate BINARY);}
+    {CREATE INDEX t2_x ON t2(x);}
+    {CREATE INDEX t1_a_c ON t1(a, c);}
   }
 
   sqlidx_one_test 2 {
@@ -370,7 +389,7 @@ proc sqlidx_internal_tests {} {
   } {
     SELECT * FROM t1 WHERE b>?;
   } {
-    {CREATE INDEX t1_b ON t1(b collate BINARY);}
+    {CREATE INDEX t1_b ON t1(b);}
   }
 
   sqlidx_one_test 3 {
@@ -378,7 +397,7 @@ proc sqlidx_internal_tests {} {
   } {
     SELECT * FROM t1 WHERE b COLLATE nocase BETWEEN ? AND ?
   } {
-    {CREATE INDEX t1_bnocase ON t1(b collate NOCASE);}
+    {CREATE INDEX t1_bnocase ON t1(b COLLATE NOCASE);}
   }
 
   sqlidx_one_test 4 {
@@ -386,7 +405,7 @@ proc sqlidx_internal_tests {} {
   } {
     SELECT a FROM t1 ORDER BY b;
   } {
-    {CREATE INDEX t1_b ON t1(b collate BINARY);}
+    {CREATE INDEX t1_b ON t1(b);}
   }
 
   sqlidx_one_test 5 {
@@ -394,7 +413,7 @@ proc sqlidx_internal_tests {} {
   } {
     SELECT a FROM t1 WHERE a=? ORDER BY b;
   } {
-    {CREATE INDEX t1_a_b ON t1(a collate BINARY, b collate BINARY);}
+    {CREATE INDEX t1_a_b ON t1(a, b);}
   }
 
   sqlidx_one_test 5 {
@@ -402,7 +421,7 @@ proc sqlidx_internal_tests {} {
   } {
     SELECT min(a) FROM t1
   } {
-    {CREATE INDEX t1_a ON t1(a collate BINARY);}
+    {CREATE INDEX t1_a ON t1(a);}
   }
 
   sqlidx_one_test 6 {
@@ -410,7 +429,15 @@ proc sqlidx_internal_tests {} {
   } {
     SELECT * FROM t1 ORDER BY a ASC, b COLLATE nocase DESC, c ASC;
   } {
-    {CREATE INDEX t1_a_bnocasedesc_c ON t1(a collate BINARY, b collate NOCASE DESC, c collate BINARY);}
+    {CREATE INDEX t1_a_bnocasedesc_c ON t1(a, b COLLATE NOCASE DESC, c);}
+  }
+
+  sqlidx_one_test 7 {
+    CREATE TABLE t1(a COLLATE NOCase, b, c);
+  } {
+    SELECT * FROM t1 WHERE a=?
+  } {
+    {CREATE INDEX t1_a ON t1(a);}
   }
 
   exit

From c45bf341afda3c3fddb297519292c11a9658a9d3 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 30 Nov 2015 18:17:55 +0000
Subject: [PATCH 005/488] Fix the schemalint.tcl script to handle identifiers
 that require quoting.

FossilOrigin-Name: 451e0fafbe5b7e9c67d9b584d5e16796c3196881
---
 manifest             |  16 +++---
 manifest.uuid        |   2 +-
 src/where.c          |  10 ++--
 test/schemalint.test |  28 +++++-----
 tool/schemalint.tcl  | 120 ++++++++++++++++++++++++++++++++++++++++---
 5 files changed, 143 insertions(+), 33 deletions(-)

diff --git a/manifest b/manifest
index bff0419492..16b150e61d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\sthe\sCREATE\sINDEX\sstatements\soutput\sby\sschemalint.tcl,\savoid\sdeclaring\san\sexplicit\scollation\ssequence\sthat\sis\sthe\ssame\sas\sthe\scolumn's\sdefault.
-D 2015-11-23T18:28:07.584
+C Fix\sthe\sschemalint.tcl\sscript\sto\shandle\sidentifiers\sthat\srequire\squoting.
+D 2015-11-30T18:17:55.036
 F Makefile.in d828db6afa6c1fa060d01e33e4674408df1942a1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e928e68168df69b353300ac87c10105206653a03
@@ -416,7 +416,7 @@ F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb
 F src/wal.c 18b0ed49830cf04fe2d68224b41838a73ac6cd24
 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4
 F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba
-F src/where.c 5804902239731a7025240f4338d98cdc57315636
+F src/where.c be09f4a0513f885845ae481324c2e07603f88b38
 F src/whereInt.h 7892bb54cf9ca0ae5c7e6094491b94c9286dc647
 F src/wherecode.c 4c96182e7b25e4be54008dee2da5b9c2f8480b9b
 F src/whereexpr.c bd4877cd4dd11f6ab551ef0054535ca3c6224950
@@ -973,7 +973,7 @@ F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5
 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38
 F test/schema4.test e6a66e20cc69f0e306667c08be7fda3d11707dc5
 F test/schema5.test 29699b4421f183c8f0e88bd28ce7d75d13ea653e
-F test/schemalint.test 7fba0e262353c8c3dd5ee406cb11cf90f8b566fe
+F test/schemalint.test cee9f375637fcc8c6e77d971abe044445c23e024
 F test/securedel.test 21749c32ccc30f1ea9e4b9f33295a6521ec20fa0
 F test/securedel2.test 2d54c28e46eb1fd6902089958b20b1b056c6f1c5
 F test/select1.test be62204d2bd9a5a8a149e9974cfddce893d8f686
@@ -1381,7 +1381,7 @@ F tool/replace.tcl 7727c60a04299b65a92f5e1590896fea0f25b9e0
 F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a
 F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5
 F tool/run-speed-test.sh 0ae485af4fe9f826e2b494be8c81f8ca9e222a4a
-F tool/schemalint.tcl ad61083d39cb40e848b9acc1273e4c487bb55cd7
+F tool/schemalint.tcl 2f44d0874061a948f6ef53505062db4ac9806cf7
 F tool/showdb.c d4476e000a64eca9f5e2c2f68741e747b9778e8d
 F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818
 F tool/showlocks.c 9920bcc64f58378ff1118caead34147201f48c68
@@ -1406,7 +1406,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 8f1ef0904d055b5510ec9043810ebf22a8c5e253
-R 4b9b445c1607ad5c9f9bdcf33b4eb416
+P d3aa067c830e98f2074630c4613c557b0ce90a57
+R aed657578ea11902053a31be3048325d
 U dan
-Z 6d9149db2c512107280c4e32c490b8cb
+Z 94438cf65b515737c9a10851485e87ac
diff --git a/manifest.uuid b/manifest.uuid
index ed4290e83d..f83d5d89f6 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d3aa067c830e98f2074630c4613c557b0ce90a57
\ No newline at end of file
+451e0fafbe5b7e9c67d9b584d5e16796c3196881
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index 4ba35d6ac9..49e9fc8382 100644
--- a/src/where.c
+++ b/src/where.c
@@ -3942,7 +3942,8 @@ static char *whereAppendSingleTerm(
   pColl = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight);
 
   if( zOp ){
-    const char *zFmt = bOr ? "%z{{%s %s %s %lld}}" : "%z{%s %s %s %lld}";
+    const char *zFmt = bOr ? "%z{{%s \"%w\" \"%w\" %lld}}" :
+                             "%z{%s \"%w\" \"%w\" %lld}";
     zBuf = whereAppendPrintf(db, zFmt, zIn, 
         zOp, pTab->aCol[pTerm->u.leftColumn].zName, 
         (pColl ? pColl->zName : "BINARY"),
@@ -4025,7 +4026,7 @@ static void whereTraceBuilder(
       nCol = pTab->nCol;
 
       /* Append the table name to the buffer. */
-      zBuf = whereAppendPrintf(db, "%s", pTab->zName);
+      zBuf = whereAppendPrintf(db, "\"%w\"", pTab->zName);
 
       /* Append the list of columns required to create a covering index */
       zBuf = whereAppendPrintf(db, "%z {cols", zBuf);
@@ -4033,7 +4034,8 @@ static void whereTraceBuilder(
         for(iCol=0; iCol<nCol; iCol++){
           if( iCol==(sizeof(Bitmask)*8-1) ) break;
           if( pItem->colUsed & ((u64)1 << iCol) ){
-            zBuf = whereAppendPrintf(db, "%z %s", zBuf, pTab->aCol[iCol].zName);
+            const char *zName = pTab->aCol[iCol].zName;
+            zBuf = whereAppendPrintf(db, "%z \"%w\"", zBuf, zName);
           }
         }
       }
@@ -4054,7 +4056,7 @@ static void whereTraceBuilder(
           if( pExpr->op==TK_COLUMN && pExpr->iTable==pItem->iCursor ){
             if( pExpr->iColumn>=0 ){
               const char *zName = pTab->aCol[pExpr->iColumn].zName;
-              zBuf = whereAppendPrintf(db, "%z%s%s %s %s", zBuf,
+              zBuf = whereAppendPrintf(db, "%z%s\"%w\" \"%w\" %s", zBuf,
                   bFirst ? " {orderby " : " ", zName, pColl->zName,
                   (pOrderBy->a[i].sortOrder ? "DESC" : "ASC")
               );
diff --git a/test/schemalint.test b/test/schemalint.test
index d354c1fdd7..7e0eba494d 100644
--- a/test/schemalint.test
+++ b/test/schemalint.test
@@ -27,33 +27,33 @@ do_execsql_test 1.0 {
 do_trace_test 1.1 {
   SELECT b, c, y, z FROM t1, t2 WHERE c=? AND z=?
 } {
-  {t1 {cols b c} {eq c BINARY 0}}
-  {t2 {cols y z} {eq z BINARY 0}}
+  {"t1" {cols "b" "c"} {eq "c" "BINARY" 0}}
+  {"t2" {cols "y" "z"} {eq "z" "BINARY" 0}}
 }
 
 do_trace_test 1.2 {
   SELECT a FROM t1 WHERE b>10
 } {
-  {t1 {cols a b} {range b BINARY 0}}
+  {"t1" {cols "a" "b"} {range "b" "BINARY" 0}}
 }
 
 do_trace_test 1.3 {
   SELECT b FROM t1 WHERE b IN (10, 20, 30)
 } {
-  {t1 {cols b} {eq b BINARY 0}}
+  {"t1" {cols "b"} {eq "b" "BINARY" 0}}
 }
 
 do_trace_test 1.4 {
   SELECT * FROM t1, t2 WHERE x=a
 } {
-  {t1 {cols a b c} {eq a BINARY 2}} 
-  {t2 {cols x y z} {eq x BINARY 1}}
+  {"t1" {cols "a" "b" "c"} {eq "a" "BINARY" 2}} 
+  {"t2" {cols "x" "y" "z"} {eq "x" "BINARY" 1}}
 }
 
 do_trace_test 1.5 {
   SELECT * FROM t1 WHERE a IN (1, 2, 3)
 } {
-  {t1 {cols a b c} {eq a BINARY 0}}
+  {"t1" {cols "a" "b" "c"} {eq "a" "BINARY" 0}}
 }
 
 #-----------------------------------------------------------------------
@@ -62,19 +62,19 @@ do_trace_test 1.5 {
 do_trace_test 2.1 {
   SELECT * FROM t1 WHERE a=? OR b=?
 } {
-  {t1 {cols a b c} {or {{eq a BINARY 0}} {{eq b BINARY 0}}}}
+  {"t1" {cols "a" "b" "c"} {or {{eq "a" "BINARY" 0}} {{eq "b" "BINARY" 0}}}}
 }
 
 do_trace_test 2.2 {
   SELECT * FROM t1 WHERE a=? OR (b=? AND c=?)
 } {
-  {t1 {cols a b c} {or {{eq a BINARY 0}} {{eq b BINARY 0} {eq c BINARY 0}}}}
+  {"t1" {cols "a" "b" "c"} {or {{eq "a" "BINARY" 0}} {{eq "b" "BINARY" 0} {eq "c" "BINARY" 0}}}}
 }
 
 do_trace_test 2.3 {
   SELECT * FROM t1 WHERE (a=? AND b=?) OR c=?
 } {
-  {t1 {cols a b c} {or {{eq c BINARY 0}} {{eq a BINARY 0} {eq b BINARY 0}}}}
+  {"t1" {cols "a" "b" "c"} {or {{eq "c" "BINARY" 0}} {{eq "a" "BINARY" 0} {eq "b" "BINARY" 0}}}}
 }
 
 #-----------------------------------------------------------------------
@@ -82,19 +82,19 @@ do_trace_test 2.3 {
 #
 do_trace_test 3.1 {
   SELECT * FROM t1 ORDER BY a;
-} {{t1 {cols a b c} {orderby a BINARY ASC}}}
+} {{"t1" {cols "a" "b" "c"} {orderby "a" "BINARY" ASC}}}
 
 do_trace_test 3.2 {
   SELECT * FROM t1 WHERE a=? ORDER BY b;
-} {{t1 {cols a b c} {eq a BINARY 0} {orderby b BINARY ASC}}}
+} {{"t1" {cols "a" "b" "c"} {eq "a" "BINARY" 0} {orderby "b" "BINARY" ASC}}}
 
 do_trace_test 3.3 {
   SELECT min(a) FROM t1;
-} {{t1 {cols a} {orderby a BINARY ASC}}}
+} {{"t1" {cols "a"} {orderby "a" "BINARY" ASC}}}
 
 do_trace_test 3.4 {
   SELECT max(a) FROM t1;
-} {{t1 {cols a} {orderby a BINARY DESC}}}
+} {{"t1" {cols "a"} {orderby "a" "BINARY" DESC}}}
 
 finish_test
 
diff --git a/tool/schemalint.tcl b/tool/schemalint.tcl
index 7ed74f9054..b6ebd4390d 100644
--- a/tool/schemalint.tcl
+++ b/tool/schemalint.tcl
@@ -11,6 +11,41 @@ proc usage {} {
   exit
 }
 
+# Return the quoted version of identfier $id. Quotes are only added if 
+# they are required by SQLite.
+#
+# This command currently assumes that quotes are required if the 
+# identifier contains any ASCII-range characters that are not 
+# alpha-numeric or underscores.
+#
+proc quote {id} {
+  if {[requires_quote $id]} {
+    set x [string map {\" \"\"} $id]
+    return "\"$x\""
+  }
+  return $id
+}
+proc requires_quote {id} {
+  foreach c [split $id {}] {
+    if {[string is alnum $c]==0 && $c!="_"} {
+      return 1
+    }
+  }
+  return 0
+}
+
+# The argument passed to this command is a Tcl list of identifiers. The
+# value returned is the same list, except with each item quoted and the
+# elements comma-separated.
+#
+proc list_to_sql {L} {
+  set ret [list]
+  foreach l $L {
+    lappend ret [quote $l]
+  }
+  join $ret ", "
+}
+
 proc process_cmdline_args {ctxvar argv} {
   upvar $ctxvar G
   set nArg [llength $argv]
@@ -139,11 +174,11 @@ proc eqset_to_index {ctxvar aCollVar tname eqset {range {}}} {
 
   foreach {c collate dir} $rangeset {
     append idxname "_$c"
-    set coldef $c
+    set coldef [quote $c]
 
     if {[string compare -nocase $collate $aColl($c)]!=0} {
       append idxname [string tolower $collate]
-      append coldef " COLLATE $collate"
+      append coldef " COLLATE [quote $collate]"
     }
 
     if {$dir=="DESC"} {
@@ -153,7 +188,7 @@ proc eqset_to_index {ctxvar aCollVar tname eqset {range {}}} {
     lappend lCols $coldef
   }
 
-  set create_index "CREATE INDEX $idxname ON ${tname}("
+  set create_index "CREATE INDEX [quote $idxname] ON [quote $tname]("
   append create_index [join $lCols ", "]
   append create_index ");"
 
@@ -185,12 +220,27 @@ proc expand_or_cons {L} {
   return $lRet
 }
 
+#--------------------------------------------------------------------------
+# Argument $tname is the name of a table in the main database opened by
+# database handle [db]. $arrayvar is the name of an array variable in the
+# caller's context. This command populates the array with an entry mapping 
+# from column name to default collation sequence for each column of table
+# $tname. For example, if a table is declared:
+#
+#   CREATE TABLE t1(a COLLATE nocase, b, c COLLATE binary)
+#
+# the mapping is populated with:
+#
+#   map(a) -> "nocase"
+#   map(b) -> "binary"
+#   map(c) -> "binary"
+#
 proc sqlidx_get_coll_map {tname arrayvar} {
   upvar $arrayvar aColl
   set colnames [list]
-  db eval "PRAGMA table_info = $tname" x { lappend colnames $x(name) }
-  db eval "CREATE INDEX schemalint_test ON ${tname}([join $colnames ,])"
-
+  set qname [quote $tname]
+  db eval "PRAGMA table_info = $qname" x { lappend colnames $x(name) }
+  db eval "CREATE INDEX schemalint_test ON ${qname}([list_to_sql $colnames])"
   db eval "PRAGMA index_xinfo = schemalint_test" x { 
     set aColl($x(name)) $x(coll)
   }
@@ -348,6 +398,7 @@ proc sqlidx_one_test {tn schema select expected} {
   sqlidx_init_context C
 
   sqlite3 db ""
+  db collate "a b c" [list string compare]
   db eval $schema
   lappend C(lSelect) $select
   analyze_selects C
@@ -362,9 +413,14 @@ proc sqlidx_one_test {tn schema select expected} {
   }
 
   db close
+
+  upvar nTest nTest
+  incr nTest
 }
 
 proc sqlidx_internal_tests {} {
+  set nTest 0
+
 
   # No indexes for a query with no constraints.
   sqlidx_one_test 0 {
@@ -440,6 +496,58 @@ proc sqlidx_internal_tests {} {
     {CREATE INDEX t1_a ON t1(a);}
   }
 
+  # Tables with names that require quotes.
+  #
+  sqlidx_one_test 8.1 {
+    CREATE TABLE "t t"(a, b, c);
+  } {
+    SELECT * FROM "t t" WHERE a=?
+  } {
+    {CREATE INDEX "t t_a" ON "t t"(a);}
+  }
+  sqlidx_one_test 8.2 {
+    CREATE TABLE "t t"(a, b, c);
+  } {
+    SELECT * FROM "t t" WHERE b BETWEEN ? AND ?
+  } {
+    {CREATE INDEX "t t_b" ON "t t"(b);}
+  }
+  
+  # Columns with names that require quotes.
+  #
+  sqlidx_one_test 9.1 {
+    CREATE TABLE t3(a, "b b", c);
+  } {
+    SELECT * FROM t3 WHERE "b b" = ?
+  } {
+    {CREATE INDEX "t3_b b" ON t3("b b");}
+  }
+  sqlidx_one_test 9.2 {
+    CREATE TABLE t3(a, "b b", c);
+  } {
+    SELECT * FROM t3 ORDER BY "b b"
+  } {
+    {CREATE INDEX "t3_b b" ON t3("b b");}
+  }
+
+  # Collations with names that require quotes.
+  #
+  sqlidx_one_test 10.1 {
+    CREATE TABLE t4(a, b, c);
+  } {
+    SELECT * FROM t4 ORDER BY c COLLATE "a b c"
+  } {
+    {CREATE INDEX "t4_ca b c" ON t4(c COLLATE "a b c");}
+  }
+  sqlidx_one_test 10.2 {
+    CREATE TABLE t4(a, b, c);
+  } {
+    SELECT * FROM t4 WHERE c = ? COLLATE "a b c"
+  } {
+    {CREATE INDEX "t4_ca b c" ON t4(c COLLATE "a b c");}
+  }
+
+  puts "All $nTest tests passed"
   exit
 }
 # End of internal test code.

From ff3b534b9f4349f1b84f3e7f04f7293dfb5b6ac2 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 30 Nov 2015 19:16:00 +0000
Subject: [PATCH 006/488] Add a rule to main.mk to build the schemalint.tcl
 script into an executable. Similar to the way the sqlite3_analyzer executable
 is built.

FossilOrigin-Name: b8251065db9dbe5463490316baa09dc636551377
---
 main.mk             | 14 ++++++++++++++
 manifest            | 14 +++++++-------
 manifest.uuid       |  2 +-
 tool/schemalint.tcl | 10 +++++++++-
 4 files changed, 31 insertions(+), 9 deletions(-)

diff --git a/main.mk b/main.mk
index 838e0753d9..9ee3af7289 100644
--- a/main.mk
+++ b/main.mk
@@ -715,6 +715,20 @@ sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl
 sqlite3_analyzer$(EXE): sqlite3_analyzer.c
 	$(TCCX) $(TCL_FLAGS) sqlite3_analyzer.c -o $@ $(LIBTCL) $(THREADLIB) 
 
+sqlite3_schemalint.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/schemalint.tcl
+	echo "#define TCLSH 2" > $@
+	echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@
+	cat sqlite3.c $(TOP)/src/tclsqlite.c >> $@
+	echo "static const char *tclsh_main_loop(void){" >> $@
+	echo "static const char *zMainloop = " >> $@
+	tclsh $(TOP)/tool/tostr.tcl $(TOP)/tool/schemalint.tcl >> $@
+	echo "; return zMainloop; }" >> $@
+
+sqlite3_schemalint$(EXE): $(TESTSRC) sqlite3_schemalint.c
+	$(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS)                  \
+		sqlite3_schemalint.c $(TESTSRC)               \
+		-o sqlite3_schemalint$(EXE) $(LIBTCL) $(THREADLIB)
+
 # Rules to build the 'testfixture' application.
 #
 TESTFIXTURE_FLAGS  = -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
diff --git a/manifest b/manifest
index 16b150e61d..7fe67fd96f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sthe\sschemalint.tcl\sscript\sto\shandle\sidentifiers\sthat\srequire\squoting.
-D 2015-11-30T18:17:55.036
+C Add\sa\srule\sto\smain.mk\sto\sbuild\sthe\sschemalint.tcl\sscript\sinto\san\sexecutable.\sSimilar\sto\sthe\sway\sthe\ssqlite3_analyzer\sexecutable\sis\sbuilt.
+D 2015-11-30T19:16:00.031
 F Makefile.in d828db6afa6c1fa060d01e33e4674408df1942a1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e928e68168df69b353300ac87c10105206653a03
@@ -263,7 +263,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk ae99be5eb22933b1ecf80f94d41d25a3ea80aaf3
+F main.mk 0aec7410e847a886ed8260b829e4e72f060b0431
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -1381,7 +1381,7 @@ F tool/replace.tcl 7727c60a04299b65a92f5e1590896fea0f25b9e0
 F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a
 F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5
 F tool/run-speed-test.sh 0ae485af4fe9f826e2b494be8c81f8ca9e222a4a
-F tool/schemalint.tcl 2f44d0874061a948f6ef53505062db4ac9806cf7
+F tool/schemalint.tcl 045481a427ecdb72d11a0b0ef8725e7b892a09b9
 F tool/showdb.c d4476e000a64eca9f5e2c2f68741e747b9778e8d
 F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818
 F tool/showlocks.c 9920bcc64f58378ff1118caead34147201f48c68
@@ -1406,7 +1406,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P d3aa067c830e98f2074630c4613c557b0ce90a57
-R aed657578ea11902053a31be3048325d
+P 451e0fafbe5b7e9c67d9b584d5e16796c3196881
+R 8aa86746f90305c3990fbec7bea3f626
 U dan
-Z 94438cf65b515737c9a10851485e87ac
+Z e9b16724c3379c54748ede3fe8135436
diff --git a/manifest.uuid b/manifest.uuid
index f83d5d89f6..d723846430 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-451e0fafbe5b7e9c67d9b584d5e16796c3196881
\ No newline at end of file
+b8251065db9dbe5463490316baa09dc636551377
\ No newline at end of file
diff --git a/tool/schemalint.tcl b/tool/schemalint.tcl
index b6ebd4390d..99e3476661 100644
--- a/tool/schemalint.tcl
+++ b/tool/schemalint.tcl
@@ -1,3 +1,4 @@
+if {[catch {
 
 set ::VERBOSE 0
 
@@ -553,10 +554,17 @@ proc sqlidx_internal_tests {} {
 # End of internal test code.
 #-------------------------------------------------------------------------
 
+if {[info exists ::argv0]==0} { set ::argv0 [info nameofexec] }
+if {[info exists ::argv]==0} usage
 sqlidx_init_context D
-process_cmdline_args D $argv
+process_cmdline_args D $::argv
 open_database D
 analyze_selects D
 find_trial_indexes D
 foreach idx [run_trials D] { puts $idx }
 
+} err]} {
+  puts "ERROR: $err"
+  puts $errorInfo
+  exit 1
+}

From a494ca8d12761b3b1ee89019811a3803ed5a252f Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 22 Jan 2016 14:32:20 +0000
Subject: [PATCH 007/488] Update the schemalint.tcl script so that the argument
 to a -select option may be either an SQL statement or the name of a file
 containing an SQL statement

FossilOrigin-Name: d4e37767671dc946b8a21d0158fc3044e36f0c7e
---
 manifest            | 12 ++++++------
 manifest.uuid       |  2 +-
 tool/schemalint.tcl | 14 +++++++++++++-
 3 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 7fe67fd96f..cc75817d97 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sa\srule\sto\smain.mk\sto\sbuild\sthe\sschemalint.tcl\sscript\sinto\san\sexecutable.\sSimilar\sto\sthe\sway\sthe\ssqlite3_analyzer\sexecutable\sis\sbuilt.
-D 2015-11-30T19:16:00.031
+C Update\sthe\sschemalint.tcl\sscript\sso\sthat\sthe\sargument\sto\sa\s-select\soption\smay\sbe\seither\san\sSQL\sstatement\sor\sthe\sname\sof\sa\sfile\scontaining\san\sSQL\sstatement
+D 2016-01-22T14:32:20.666
 F Makefile.in d828db6afa6c1fa060d01e33e4674408df1942a1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e928e68168df69b353300ac87c10105206653a03
@@ -1381,7 +1381,7 @@ F tool/replace.tcl 7727c60a04299b65a92f5e1590896fea0f25b9e0
 F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a
 F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5
 F tool/run-speed-test.sh 0ae485af4fe9f826e2b494be8c81f8ca9e222a4a
-F tool/schemalint.tcl 045481a427ecdb72d11a0b0ef8725e7b892a09b9
+F tool/schemalint.tcl 7b94e303b02d1b1724c56201ed9fa2845dc8d3a5
 F tool/showdb.c d4476e000a64eca9f5e2c2f68741e747b9778e8d
 F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818
 F tool/showlocks.c 9920bcc64f58378ff1118caead34147201f48c68
@@ -1406,7 +1406,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 451e0fafbe5b7e9c67d9b584d5e16796c3196881
-R 8aa86746f90305c3990fbec7bea3f626
+P b8251065db9dbe5463490316baa09dc636551377
+R 6851215977a26ab67201b1a70341660d
 U dan
-Z e9b16724c3379c54748ede3fe8135436
+Z 63a5088a2207bef48ca5ab65d0652925
diff --git a/manifest.uuid b/manifest.uuid
index d723846430..3a22438944 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b8251065db9dbe5463490316baa09dc636551377
\ No newline at end of file
+d4e37767671dc946b8a21d0158fc3044e36f0c7e
\ No newline at end of file
diff --git a/tool/schemalint.tcl b/tool/schemalint.tcl
index 99e3476661..2755ba8828 100644
--- a/tool/schemalint.tcl
+++ b/tool/schemalint.tcl
@@ -47,6 +47,13 @@ proc list_to_sql {L} {
   join $ret ", "
 }
 
+proc readfile {zFile} {
+  set fd [open $zFile]
+  set data [read $fd]
+  close $fd
+  return $data
+}
+
 proc process_cmdline_args {ctxvar argv} {
   upvar $ctxvar G
   set nArg [llength $argv]
@@ -58,7 +65,12 @@ proc process_cmdline_args {ctxvar argv} {
       -select {
         incr i
         if {$i>=[llength $argv]-1} usage
-        lappend G(lSelect) [lindex $argv $i]
+        set zSelect [lindex $argv $i]
+        if {[file readable $zSelect]} {
+          lappend G(lSelect) [readfile $zSelect]
+        } else {
+          lappend G(lSelect) $zSelect
+        }
       }
       -verbose {
         set ::VERBOSE 1

From 336bfe06b96d76f2e71cb1d9bc83ceb97d172cf5 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 22 Jan 2016 14:44:02 +0000
Subject: [PATCH 008/488] Fix handling of transitive constraints in
 schemalint.tcl.

FossilOrigin-Name: 44edc1aa3b412ddbe2a242075e2bf36a99437688
---
 manifest            | 14 +++++++-------
 manifest.uuid       |  2 +-
 src/where.c         |  8 +++++---
 tool/schemalint.tcl | 12 ++++++++++++
 4 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index cc75817d97..3f50c20486 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\sthe\sschemalint.tcl\sscript\sso\sthat\sthe\sargument\sto\sa\s-select\soption\smay\sbe\seither\san\sSQL\sstatement\sor\sthe\sname\sof\sa\sfile\scontaining\san\sSQL\sstatement
-D 2016-01-22T14:32:20.666
+C Fix\shandling\sof\stransitive\sconstraints\sin\sschemalint.tcl.
+D 2016-01-22T14:44:02.574
 F Makefile.in d828db6afa6c1fa060d01e33e4674408df1942a1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e928e68168df69b353300ac87c10105206653a03
@@ -416,7 +416,7 @@ F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb
 F src/wal.c 18b0ed49830cf04fe2d68224b41838a73ac6cd24
 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4
 F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba
-F src/where.c be09f4a0513f885845ae481324c2e07603f88b38
+F src/where.c 1e987b91aaafe2bb363488e43fcdac137f5b8b59
 F src/whereInt.h 7892bb54cf9ca0ae5c7e6094491b94c9286dc647
 F src/wherecode.c 4c96182e7b25e4be54008dee2da5b9c2f8480b9b
 F src/whereexpr.c bd4877cd4dd11f6ab551ef0054535ca3c6224950
@@ -1381,7 +1381,7 @@ F tool/replace.tcl 7727c60a04299b65a92f5e1590896fea0f25b9e0
 F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a
 F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5
 F tool/run-speed-test.sh 0ae485af4fe9f826e2b494be8c81f8ca9e222a4a
-F tool/schemalint.tcl 7b94e303b02d1b1724c56201ed9fa2845dc8d3a5
+F tool/schemalint.tcl 49690356702d6cac07e2bb1790eac73862e92926
 F tool/showdb.c d4476e000a64eca9f5e2c2f68741e747b9778e8d
 F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818
 F tool/showlocks.c 9920bcc64f58378ff1118caead34147201f48c68
@@ -1406,7 +1406,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P b8251065db9dbe5463490316baa09dc636551377
-R 6851215977a26ab67201b1a70341660d
+P d4e37767671dc946b8a21d0158fc3044e36f0c7e
+R cbe06eb4db20d1a79810102395b5ccc1
 U dan
-Z 63a5088a2207bef48ca5ab65d0652925
+Z e0e53a7502c2ab11481d0d1a7c5964ef
diff --git a/manifest.uuid b/manifest.uuid
index 3a22438944..8879269b0a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d4e37767671dc946b8a21d0158fc3044e36f0c7e
\ No newline at end of file
+44edc1aa3b412ddbe2a242075e2bf36a99437688
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index 49e9fc8382..770b2f6b39 100644
--- a/src/where.c
+++ b/src/where.c
@@ -3924,6 +3924,7 @@ static char *whereAppendPrintf(sqlite3 *db, const char *zFmt, ...){
 static char *whereAppendSingleTerm(
   Parse *pParse,
   Table *pTab,
+  int iCol,
   int bOr,
   char *zIn,
   WhereTerm *pTerm
@@ -3945,7 +3946,7 @@ static char *whereAppendSingleTerm(
     const char *zFmt = bOr ? "%z{{%s \"%w\" \"%w\" %lld}}" :
                              "%z{%s \"%w\" \"%w\" %lld}";
     zBuf = whereAppendPrintf(db, zFmt, zIn, 
-        zOp, pTab->aCol[pTerm->u.leftColumn].zName, 
+        zOp, pTab->aCol[iCol].zName, 
         (pColl ? pColl->zName : "BINARY"),
         pTerm->prereqRight
     );
@@ -3969,6 +3970,7 @@ static char *whereTraceWC(
   int iCol;
   int ii;
   int bFirst = !bInitialSpace;
+  int bOr = (pWC->op==TK_OR);
 
   /* List of WO_SINGLE constraints */
   for(iCol=0; iCol<pTab->nCol; iCol++){
@@ -3979,9 +3981,9 @@ static char *whereTraceWC(
         pTerm;
         pTerm=whereScanNext(&scan)
     ){
-      assert( iCol==pTerm->u.leftColumn );
+      /* assert( iCol==pTerm->u.leftColumn ); */
       if( bFirst==0 ) zBuf = whereAppendPrintf(db, "%z ", zBuf);
-      zBuf = whereAppendSingleTerm(pParse, pTab, pWC->op==TK_OR, zBuf, pTerm);
+      zBuf = whereAppendSingleTerm(pParse, pTab, iCol, bOr, zBuf, pTerm);
       bFirst = 0;
     }
   }
diff --git a/tool/schemalint.tcl b/tool/schemalint.tcl
index 2755ba8828..1c3f2e27c3 100644
--- a/tool/schemalint.tcl
+++ b/tool/schemalint.tcl
@@ -560,6 +560,18 @@ proc sqlidx_internal_tests {} {
     {CREATE INDEX "t4_ca b c" ON t4(c COLLATE "a b c");}
   }
 
+  # Transitive constraints
+  #
+  sqlidx_one_test 11.1 {
+    CREATE TABLE t5(a, b);
+    CREATE TABLE t6(c, d);
+  } {
+    SELECT * FROM t5, t6 WHERE a=? AND b=c AND c=?
+  } {
+    {CREATE INDEX t6_c ON t6(c);} 
+    {CREATE INDEX t5_a_b ON t5(a, b);}
+  }
+
   puts "All $nTest tests passed"
   exit
 }

From 3e6ac1643c3beebc9d2c23f819e25a229a0e8a1d Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 11 Feb 2016 21:01:16 +0000
Subject: [PATCH 009/488] Experimental integration of schemalint functionality
 with the shell tool. Does not work yet.

FossilOrigin-Name: ed49f297bcee86674ed673e195610b8cc1d35647
---
 main.mk             |   2 +-
 manifest            |  31 +--
 manifest.uuid       |   2 +-
 src/main.c          |   7 +
 src/pragma.c        |   1 +
 src/shell.c         |  36 +++-
 src/shell_indexes.c | 468 ++++++++++++++++++++++++++++++++++++++++++++
 src/sqlite.h.in     |   9 +
 src/sqliteInt.h     |   4 +
 src/where.c         | 191 ++++++------------
 10 files changed, 601 insertions(+), 150 deletions(-)
 create mode 100644 src/shell_indexes.c

diff --git a/main.mk b/main.mk
index 22d40fa258..a10f043695 100644
--- a/main.mk
+++ b/main.mk
@@ -472,7 +472,7 @@ libsqlite3.a:	$(LIBOBJ)
 	$(AR) libsqlite3.a $(LIBOBJ)
 	$(RANLIB) libsqlite3.a
 
-sqlite3$(EXE):	$(TOP)/src/shell.c libsqlite3.a sqlite3.h
+sqlite3$(EXE):	$(TOP)/src/shell.c libsqlite3.a sqlite3.h $(TOP)/src/shell_indexes.c
 	$(TCCX) $(READLINE_FLAGS) -o sqlite3$(EXE) $(SHELL_OPT) \
 		$(TOP)/src/shell.c libsqlite3.a $(LIBREADLINE) $(TLIBS) $(THREADLIB)
 
diff --git a/manifest b/manifest
index 78c07a5223..7da4cdeb1d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\slatest\strunk\schanges\swith\sthis\sbranch.
-D 2016-02-09T15:10:56.225
+C Experimental\sintegration\sof\sschemalint\sfunctionality\swith\sthe\sshell\stool.\sDoes\snot\swork\syet.
+D 2016-02-11T21:01:16.253
 F Makefile.in dac2776c84e0d533b158a9af6e57e05c4a6b19f3
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc b0493f10caddb8adf992a4e6f1943141fc7c6816
@@ -13,7 +13,7 @@ F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903
 F autoconf/Makefile.am 1c1657650775960804945dc392e14d9e43c5ed84
 F autoconf/Makefile.msc a35b2aab24d1603f3f0ae65cf01686c2578d319c
 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7
-F autoconf/README.txt e9757a381e5ce2553dbaa6247bb8ad00eb8d87aa w autoconf/README
+F autoconf/README.txt e9757a381e5ce2553dbaa6247bb8ad00eb8d87aa
 F autoconf/configure.ac 72a5e42beb090b32bca580285dc0ab3c4670adb8
 F autoconf/tea/Makefile.in b438a7020446c8a8156e8d97c8914a04833da6fd
 F autoconf/tea/README 3e9a3c060f29a44344ab50aec506f4db903fb873
@@ -272,7 +272,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk cd48a5d8a6dc59229f4f3fe40771104f2918f789
+F main.mk eeff3d12ebe5945dac88147c641407c22a731131
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -313,7 +313,7 @@ F src/insert.c 046199e085e69e05af7bef197d53c5b4b402b6fa
 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d
 F src/legacy.c b1b0880fc474abfab89e737b0ecfde0bd7a60902
 F src/loadext.c 84996d7d70a605597d79c1f1d7b2012a5fd34f2b
-F src/main.c b67a45397b93b7ba8fbd6bfcb03423d245baed05
+F src/main.c 816b9a98a6aca0fd643e77f3610d6a4a1a4c7e24
 F src/malloc.c 337e9808b5231855fe28857950f4f60ae42c417f
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c 6919bcf12f221868ea066eec27e579fed95ce98b
@@ -341,7 +341,7 @@ F src/parse.y d7bff41d460f2df96fb890f36700e85cb0fc5634
 F src/pcache.c 73895411fa6b7bd6f0091212feabbe833b358d23
 F src/pcache.h 4d0ccaad264d360981ec5e6a2b596d6e85242545
 F src/pcache1.c 72f644dc9e1468c72922eff5904048427b817051
-F src/pragma.c 80ee77226d0008d9188356a6cbbe6010866e1bee
+F src/pragma.c cfd521558fccd3864ec664af09a061e9e692583f
 F src/pragma.h 64c78a648751b9f4f297276c4eb7507b14b4628c
 F src/prepare.c c12b786713df3e8270c0f85f988c5359d8b4d87c
 F src/printf.c 63e6fb12bbe702dd664dc3703776c090383a5a26
@@ -349,11 +349,12 @@ F src/random.c ba2679f80ec82c4190062d756f22d0c358180696
 F src/resolve.c 9f7ce3a3c087afb7597b7c916c99126ff3f12f0c
 F src/rowset.c 9fe4b3ad7cc00944386bb600233d8f523de07a6e
 F src/select.c ff80004a9a6ece891a8d9327a88e7b6e2588ee6d
-F src/shell.c dcd7a83645ef2a58ee9c6d0ea4714d877d7835c4
-F src/sqlite.h.in cf22ad1d52dca2c9862d63833e581028119aab7e
+F src/shell.c 2cde87e03712204231167c4a6c61b0eb5129e105
+F src/shell_indexes.c 3cff393ee86d15fbfbe31f30cd752b46d7779b52
+F src/sqlite.h.in c7db059d3b810b70b83d9ed1436fa813eba22462
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h dfbe62ffd95b99afe2140d8c35b180d11924072d
-F src/sqliteInt.h 3aeaff9611acd790c8e76719b33db09ab885d537
+F src/sqliteInt.h a1d0d9613ed7da3657396795e44991fef188c8ee
 F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46
 F src/status.c 70912d7be68e9e2dbc4010c93d344af61d4c59ba
 F src/table.c 51b46b2a62d1b3a959633d593b89bab5e2c9155e
@@ -427,7 +428,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c d21b99fd1458159d0b1ecdccc8ee6ada4fdc4c54
 F src/wal.h 2f7c831cf3b071fa548bf2d5cac640846a7ff19c
 F src/walker.c 0f142b5bd3ed2041fc52d773880748b212e63354
-F src/where.c 438b89caa0cbe17cd32703a8f93baca9789d0474
+F src/where.c 89d5845353fe6d2e77bce52a2c8bea0781c69dad
 F src/whereInt.h 78b6b4de94db84aecbdc07fe3e38f648eb391e9a
 F src/wherecode.c 791a784bbf8749d560fdb0b990b607bc4f44a38d
 F src/whereexpr.c de117970b29471177a6901d60ad83a194671dc03
@@ -980,7 +981,7 @@ F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0
 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd
 F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7
 F test/savepoint7.test db3db281486c925095f305aad09fe806e5188ff3
-F test/savepointfault.test f044eac64b59f09746c7020ee261734de82bf9b2 w test/savepoint3.test
+F test/savepointfault.test f044eac64b59f09746c7020ee261734de82bf9b2
 F test/scanstatus.test 5253c219e331318a437f436268e0e82345700285
 F test/schema.test 8f7999be894260f151adf15c2c7540f1c6d6a481
 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5
@@ -1346,7 +1347,7 @@ F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2
 F test/whereI.test eab5b226bbc344ac70d7dc09b963a064860ae6d7
 F test/whereJ.test 55a3221706a7ab706293f17cc8f96da563bf0767
 F test/whereK.test f8e3cf26a8513ecc7f514f54df9f0572c046c42b
-F test/wherefault.test 1374c3aa198388925246475f84ad4cd5f9528864 w test/where8m.test
+F test/wherefault.test 1374c3aa198388925246475f84ad4cd5f9528864
 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31
 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c
 F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c
@@ -1429,7 +1430,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 9341491c3a11d5a66e4f88d2af9b0d3799b4f27a ca72be8618e5d466d6f35819ca8bbd2b84269959
-R c3affb84a750d99cc1d1198e66d07e76
+P 1a4182eedd0143c3f71b3d97f1d1bb25adeba617
+R 6253a80a5a7097c3b24b24ce68900613
 U dan
-Z dbcdec085ed65802fe686b0a9369fd51
+Z bddf3d3a7cd183a6a2362ed54e10358b
diff --git a/manifest.uuid b/manifest.uuid
index b2c40d21e2..38f9cc7372 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-1a4182eedd0143c3f71b3d97f1d1bb25adeba617
\ No newline at end of file
+ed49f297bcee86674ed673e195610b8cc1d35647
\ No newline at end of file
diff --git a/src/main.c b/src/main.c
index 922af1315a..72f12d2ac2 100644
--- a/src/main.c
+++ b/src/main.c
@@ -792,6 +792,13 @@ int sqlite3_db_config(sqlite3 *db, int op, ...){
       rc = setupLookaside(db, pBuf, sz, cnt);
       break;
     }
+#ifdef SQLITE_SCHEMA_LINT
+    case SQLITE_DBCONFIG_WHEREINFO: {
+      db->xWhereInfo = va_arg(ap, void(*)(void*, int, const char*, int, i64));
+      db->pWhereInfoCtx = va_arg(ap, void*);
+      break;
+    }
+#endif
     default: {
       static const struct {
         int op;      /* The opcode */
diff --git a/src/pragma.c b/src/pragma.c
index c34d5421c2..2b4058ca16 100644
--- a/src/pragma.c
+++ b/src/pragma.c
@@ -1048,6 +1048,7 @@ void sqlite3Pragma(
   ** type:       Column declaration type.
   ** notnull:    True if 'NOT NULL' is part of column declaration
   ** dflt_value: The default value for the column, if any.
+  ** pk:         Non-zero for PK fields.
   */
   case PragTyp_TABLE_INFO: if( zRight ){
     Table *pTab;
diff --git a/src/shell.c b/src/shell.c
index 3f8b22f4fa..e2fe41291d 100644
--- a/src/shell.c
+++ b/src/shell.c
@@ -156,6 +156,7 @@ static void setTextMode(FILE *out){
 # define setTextMode(X)
 #endif
 
+#include "shell_indexes.c"
 
 /* True if the timer is enabled */
 static int enableTimer = 0;
@@ -592,7 +593,8 @@ typedef struct ShellState ShellState;
 struct ShellState {
   sqlite3 *db;           /* The database */
   int echoOn;            /* True to echo input commands */
-  int autoEQP;           /* Run EXPLAIN QUERY PLAN prior to seach SQL stmt */
+  int autoEQP;           /* Run EXPLAIN QUERY PLAN prior to each SQL stmt */
+  int bRecommend;        /* Instead of sqlite3_exec(), recommend indexes */
   int statsOn;           /* True to display memory stats before each finalize */
   int scanstatsOn;       /* True to display scan stats before each finalize */
   int countChanges;      /* True to display change counts */
@@ -1544,6 +1546,19 @@ static void explain_data_delete(ShellState *p){
   p->iIndent = 0;
 }
 
+typedef struct RecCommandCtx RecCommandCtx;
+struct RecCommandCtx {
+  int (*xCallback)(void*,int,char**,char**,int*);
+  ShellState *pArg;
+};
+
+static void recCommandOut(void *pCtx, const char *zLine){
+  const char *zCol = "output";
+  RecCommandCtx *p = (RecCommandCtx*)pCtx;
+  int t = SQLITE_TEXT;
+  p->xCallback(p->pArg, 1, (char**)&zLine, (char**)&zCol, &t);
+}
+
 /*
 ** Execute a statement or set of statements.  Print 
 ** any result rows/columns depending on the current mode 
@@ -1570,6 +1585,13 @@ static int shell_exec(
     *pzErrMsg = NULL;
   }
 
+  if( pArg->bRecommend ){
+    RecCommandCtx ctx;
+    ctx.xCallback = xCallback;
+    ctx.pArg = pArg;
+    rc = shellIndexesCommand(db, zSql, recCommandOut, &ctx, pzErrMsg);
+  }else
+
   while( zSql[0] && (SQLITE_OK == rc) ){
     rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zLeftover);
     if( SQLITE_OK != rc ){
@@ -3609,6 +3631,15 @@ static int do_meta_command(char *zLine, ShellState *p){
     sqlite3_close(pSrc);
   }else
 
+  if( c=='r' && n>=2 && strncmp(azArg[0], "recommend", n)==0 ){
+    if( nArg==2 ){
+      p->bRecommend = booleanValue(azArg[1]);
+    }else{
+      raw_printf(stderr, "Usage: .recommend on|off\n");
+      rc = 1;
+    }
+  }else
+
 
   if( c=='s' && strncmp(azArg[0], "scanstats", n)==0 ){
     if( nArg==2 ){
@@ -4903,6 +4934,9 @@ int SQLITE_CDECL main(int argc, char **argv){
           if( bail_on_error ) return rc;
         }
       }
+
+    }else if( strcmp(z, "-recommend") ){
+      data.bRecommend = 1;
     }else{
       utf8_printf(stderr,"%s: Error: unknown option: %s\n", Argv0, z);
       raw_printf(stderr,"Use -help for a list of options.\n");
diff --git a/src/shell_indexes.c b/src/shell_indexes.c
new file mode 100644
index 0000000000..8e105fec1b
--- /dev/null
+++ b/src/shell_indexes.c
@@ -0,0 +1,468 @@
+/*
+** 2016 February 10
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+*/
+
+typedef sqlite3_int64 i64;
+
+typedef struct IdxConstraint IdxConstraint;
+typedef struct IdxContext IdxContext;
+typedef struct IdxScan IdxScan;
+typedef struct IdxWhere IdxWhere;
+
+/*
+** A single constraint. Equivalent to either "col = ?" or "col < ?".
+**
+** pLink:
+**   ... todo ...
+*/
+struct IdxConstraint {
+  char *zColl;                    /* Collation sequence */
+  int bRange;                     /* True for range, false for eq */
+  int iCol;                       /* Constrained table column */
+  i64 depmask;                    /* Dependency mask */
+  IdxConstraint *pNext;           /* Next constraint in pEq or pRange list */
+  IdxConstraint *pLink;           /* See above */
+};
+
+/*
+** A WHERE clause. Made up of IdxConstraint objects.
+**
+**   a=? AND b=? AND (c=? OR d=?) AND (e=? OR f=?)
+**
+*/
+struct IdxWhere {
+  IdxConstraint *pEq;             /* List of == constraints */
+  IdxConstraint *pRange;          /* List of < constraints */
+  IdxWhere **apOr;                /* Array of OR branches (joined by pNextOr) */
+  IdxWhere *pNextOr;              /* Next in OR'd terms */
+  IdxWhere *pParent;              /* Parent object (or NULL) */
+};
+
+/*
+** A single scan of a single table.
+*/
+struct IdxScan {
+  char *zTable;                   /* Name of table to scan */
+  int iDb;                        /* Database containing table zTable */
+  i64 covering;                   /* Mask of columns required for cov. index */
+  IdxConstraint *pOrder;          /* ORDER BY columns */
+  IdxWhere where;                 /* WHERE Constraints */
+  IdxScan *pNextScan;             /* Next IdxScan object for same query */
+};
+
+/*
+** Context object passed to idxWhereInfo()
+*/
+struct IdxContext {
+  IdxWhere *pCurrent;             /* Current where clause */
+  IdxScan *pScan;                 /* List of scan objects */
+  sqlite3 *dbm;                   /* In-memory db for this analysis */
+  int rc;                         /* Error code (if error has occurred) */
+};
+
+typedef struct PragmaTable PragmaTable;
+typedef struct PragmaCursor PragmaCursor;
+
+struct PragmaTable {
+  sqlite3_vtab base;
+  sqlite3 *db;
+};
+
+struct PragmaCursor {
+  sqlite3_vtab_cursor base;
+  sqlite3_stmt *pStmt;
+  i64 iRowid;
+};
+
+/*
+** Connect to or create a pragma virtual table.
+*/
+static int pragmaConnect(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  const char *zSchema = 
+    "CREATE TABLE a(tbl HIDDEN, cid, name, type, notnull, dflt_value, pk)";
+  PragmaTable *pTab = 0;
+  int rc = SQLITE_OK;
+
+  rc = sqlite3_declare_vtab(db, zSchema);
+  if( rc==SQLITE_OK ){
+    pTab = (PragmaTable *)sqlite3_malloc64(sizeof(PragmaTable));
+    if( pTab==0 ) rc = SQLITE_NOMEM;
+  }
+
+  assert( rc==SQLITE_OK || pTab==0 );
+  if( rc==SQLITE_OK ){
+    memset(pTab, 0, sizeof(PragmaTable));
+    pTab->db = db;
+  }
+
+  *ppVtab = (sqlite3_vtab*)pTab;
+  return rc;
+}
+
+/*
+** Disconnect from or destroy a pragma virtual table.
+*/
+static int pragmaDisconnect(sqlite3_vtab *pVtab){
+  sqlite3_free(pVtab);
+  return SQLITE_OK;
+}
+
+/*
+** xBestIndex method for pragma virtual tables.
+*/
+static int pragmaBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
+  int i;
+
+  pIdxInfo->estimatedCost = 1.0e6;  /* Initial cost estimate */
+
+  /* Look for a valid tbl=? constraint. */
+  for(i=0; i<pIdxInfo->nConstraint; i++){
+    if( pIdxInfo->aConstraint[i].usable==0 ) continue;
+    if( pIdxInfo->aConstraint[i].op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
+    if( pIdxInfo->aConstraint[i].iColumn!=0 ) continue;
+    pIdxInfo->idxNum = 1;
+    pIdxInfo->estimatedCost = 1.0;
+    pIdxInfo->aConstraintUsage[i].argvIndex = 1;
+    pIdxInfo->aConstraintUsage[i].omit = 1;
+    break;
+  }
+  if( i==pIdxInfo->nConstraint ){
+    tab->zErrMsg = sqlite3_mprintf("missing required tbl=? constraint");
+    return SQLITE_ERROR;
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Open a new pragma cursor.
+*/
+static int pragmaOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
+  PragmaTable *pTab = (PragmaTable *)pVTab;
+  PragmaCursor *pCsr;
+
+  pCsr = (PragmaCursor*)sqlite3_malloc64(sizeof(PragmaCursor));
+  if( pCsr==0 ){
+    return SQLITE_NOMEM;
+  }else{
+    memset(pCsr, 0, sizeof(PragmaCursor));
+    pCsr->base.pVtab = pVTab;
+  }
+
+  *ppCursor = (sqlite3_vtab_cursor*)pCsr;
+  return SQLITE_OK;
+}
+
+/*
+** Move a statvfs cursor to the next entry in the file.
+*/
+static int pragmaNext(sqlite3_vtab_cursor *pCursor){
+  PragmaCursor *pCsr = (PragmaCursor*)pCursor;
+  int rc = SQLITE_OK;
+
+  if( sqlite3_step(pCsr->pStmt)!=SQLITE_ROW ){
+    rc = sqlite3_finalize(pCsr->pStmt);
+    pCsr->pStmt = 0;
+  }
+  pCsr->iRowid++;
+  return rc;
+}
+
+static int pragmaEof(sqlite3_vtab_cursor *pCursor){
+  PragmaCursor *pCsr = (PragmaCursor*)pCursor;
+  return pCsr->pStmt==0;
+}
+
+static int pragmaFilter(
+  sqlite3_vtab_cursor *pCursor, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  PragmaCursor *pCsr = (PragmaCursor*)pCursor;
+  PragmaTable *pTab = (PragmaTable*)(pCursor->pVtab);
+  char *zSql;
+  const char *zTbl;
+  int rc = SQLITE_OK;
+
+  if( pCsr->pStmt ){
+    sqlite3_finalize(pCsr->pStmt);
+    pCsr->pStmt = 0;
+  }
+  pCsr->iRowid = 0;
+
+  assert( argc==1 );
+  zTbl = (const char*)sqlite3_value_text(argv[0]);
+  zSql = sqlite3_mprintf("PRAGMA table_info(%Q)", zTbl);
+  if( zSql==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0);
+  }
+  return pragmaNext(pCursor);;
+}
+
+/*
+** xColumn method.
+*/
+static int pragmaColumn(
+  sqlite3_vtab_cursor *pCursor, 
+  sqlite3_context *ctx, 
+  int iCol
+){
+  PragmaCursor *pCsr = (PragmaCursor *)pCursor;
+  if( iCol>0 ){
+    sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pStmt, iCol-1));
+  }
+  return SQLITE_OK;
+}
+
+static int pragmaRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
+  PragmaCursor *pCsr = (PragmaCursor *)pCursor;
+  *pRowid = pCsr->iRowid;
+  return SQLITE_OK;
+}
+
+static int registerPragmaVtabs(sqlite3 *db){
+  static sqlite3_module pragma_module = {
+    0,                            /* iVersion */
+    pragmaConnect,                /* xCreate */
+    pragmaConnect,                /* xConnect */
+    pragmaBestIndex,              /* xBestIndex */
+    pragmaDisconnect,             /* xDisconnect */
+    pragmaDisconnect,             /* xDestroy */
+    pragmaOpen,                   /* xOpen - open a cursor */
+    pragmaClose,                  /* xClose - close a cursor */
+    pragmaFilter,                 /* xFilter - configure scan constraints */
+    pragmaNext,                   /* xNext - advance a cursor */
+    pragmaEof,                    /* xEof - check for end of scan */
+    pragmaColumn,                 /* xColumn - read data */
+    pragmaRowid,                  /* xRowid - read data */
+    0,                            /* xUpdate */
+    0,                            /* xBegin */
+    0,                            /* xSync */
+    0,                            /* xCommit */
+    0,                            /* xRollback */
+    0,                            /* xFindMethod */
+    0,                            /* xRename */
+  };
+  return sqlite3_create_module(db, "pragma_table_info", &pragma_module, 0);
+}
+
+/*
+** Allocate and return nByte bytes of zeroed memory using sqlite3_malloc(). 
+** If the allocation fails, set *pRc to SQLITE_NOMEM and return NULL.
+*/
+static void *idxMalloc(int *pRc, int nByte){
+  void *pRet;
+  assert( *pRc==SQLITE_OK );
+  assert( nByte>0 );
+  pRet = sqlite3_malloc(nByte);
+  if( pRet ){
+    memset(pRet, 0, nByte);
+  }else{
+    *pRc = SQLITE_NOMEM;
+  }
+  return pRet;
+}
+
+/*
+** Allocate and return a new IdxConstraint object. Set the IdxConstraint.zColl
+** variable to point to a copy of nul-terminated string zColl.
+*/
+static IdxConstraint *idxNewConstraint(int *pRc, const char *zColl){
+  IdxConstraint *pNew;
+  int nColl = strlen(zColl);
+
+  assert( *pRc==SQLITE_OK );
+  pNew = (IdxConstraint*)idxMalloc(pRc, sizeof(IdxConstraint) * nColl + 1);
+  if( pNew ){
+    pNew->zColl = (char*)&pNew[1];
+    memcpy(pNew->zColl, zColl, nColl+1);
+  }
+  return pNew;
+}
+
+/*
+** SQLITE_DBCONFIG_WHEREINFO callback.
+*/
+static void idxWhereInfo(
+  void *pCtx,                     /* Pointer to IdxContext structure */
+  int eOp, 
+  const char *zVal, 
+  int iVal, 
+  i64 mask
+){
+  IdxContext *p = (IdxContext*)pCtx;
+
+#if 1
+  const char *zOp = 
+    eOp==SQLITE_WHEREINFO_TABLE ? "TABLE" :
+    eOp==SQLITE_WHEREINFO_EQUALS ? "EQUALS" :
+    eOp==SQLITE_WHEREINFO_RANGE ? "RANGE" :
+    eOp==SQLITE_WHEREINFO_ORDERBY ? "ORDERBY" :
+    eOp==SQLITE_WHEREINFO_NEXTOR ? "NEXTOR" :
+    eOp==SQLITE_WHEREINFO_ENDOR ? "ENDOR" :
+    eOp==SQLITE_WHEREINFO_BEGINOR ? "BEGINOR" :
+    "!error!";
+  printf("op=%s zVal=%s iVal=%d mask=%llx\n", zOp, zVal, iVal, mask);
+#endif
+
+  if( p->rc==SQLITE_OK ){
+    assert( eOp==SQLITE_WHEREINFO_TABLE || p->pScan!=0 );
+    switch( eOp ){
+      case SQLITE_WHEREINFO_TABLE: {
+        int nVal = strlen(zVal);
+        IdxScan *pNew = (IdxScan*)idxMalloc(&p->rc, sizeof(IdxScan) + nVal + 1);
+        if( !pNew ) return;
+        pNew->zTable = (char*)&pNew[1];
+        memcpy(pNew->zTable, zVal, nVal+1);
+        pNew->pNextScan = p->pScan;
+        pNew->covering = mask;
+        p->pScan = pNew;
+        p->pCurrent = &pNew->where;
+        break;
+      }
+
+      case SQLITE_WHEREINFO_ORDERBY: {
+        IdxConstraint *pNew = idxNewConstraint(&p->rc, zVal);
+        IdxConstraint **pp;
+        if( pNew==0 ) return;
+        pNew->iCol = iVal;
+        for(pp=&p->pScan->pOrder; *pp; pp=&(*pp)->pNext);
+        *pp = pNew;
+        break;
+      }
+
+      case SQLITE_WHEREINFO_EQUALS:
+      case SQLITE_WHEREINFO_RANGE: {
+        IdxConstraint *pNew = idxNewConstraint(&p->rc, zVal);
+        if( pNew==0 ) return;
+        pNew->iCol = iVal;
+        pNew->depmask = mask;
+
+        if( eOp==SQLITE_WHEREINFO_RANGE ){
+          pNew->pNext = p->pCurrent->pRange;
+          p->pCurrent->pRange = pNew;
+        }else{
+          pNew->pNext = p->pCurrent->pEq;
+          p->pCurrent->pEq = pNew;
+        }
+        break;
+      }
+
+      case SQLITE_WHEREINFO_BEGINOR: {
+        assert( 0 );
+        break;
+      }
+      case SQLITE_WHEREINFO_ENDOR: {
+        assert( 0 );
+        break;
+      }
+      case SQLITE_WHEREINFO_NEXTOR: {
+        assert( 0 );
+        break;
+      }
+    }
+  }
+}
+
+/*
+** An error associated with database handle db has just occurred. Pass
+** the error message to callback function xOut.
+*/
+static void idxDatabaseError(
+  sqlite3 *db,                    /* Database handle */
+  char **pzErrmsg                 /* Write error here */
+){
+  *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
+}
+
+static int idxCreateTables(sqlite3 *db, sqlite3 *dbm, IdxScan *pScan){
+  int rc = SQLITE_OK;
+  IdxScan *pIter;
+  for(pIter=pScan; pIter; pIter=pIter->pNextScan){
+  }
+}
+
+static void idxScanFree(IdxScan *pScan){
+}
+
+/*
+** The xOut callback is invoked to return command output to the user. The
+** second argument is always a nul-terminated string. The first argument is
+** passed zero if the string contains normal output or non-zero if it is an
+** error message.
+*/
+int shellIndexesCommand(
+  sqlite3 *db,                         /* Database handle */
+  const char *zSql,                    /* SQL to find indexes for */
+  void (*xOut)(void*, const char*),    /* Output callback */
+  void *pOutCtx,                       /* Context for xOut() */
+  char **pzErrmsg                      /* OUT: Error message (sqlite3_malloc) */
+){
+  int rc = SQLITE_OK;
+  sqlite3 *dbm = 0;
+  IdxContext ctx;
+  sqlite3_stmt *pStmt = 0;        /* Statement compiled from zSql */
+
+  memset(&ctx, 0, sizeof(IdxContext));
+
+  /* Open an in-memory database to work with. The main in-memory 
+  ** database schema contains tables similar to those in the users 
+  ** database (handle db). The attached in-memory db (aux) contains
+  ** application tables used by the code in this file.  */
+  rc = sqlite3_open(":memory:", &dbm);
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_exec(dbm, 
+        "ATTACH ':memory:' AS aux;"
+        "CREATE TABLE aux.depmask(mask PRIMARY KEY) WITHOUT ROWID;"
+        , 0, 0, 0
+    );
+  }
+  if( rc!=SQLITE_OK ){
+    idxDatabaseError(dbm, pzErrmsg);
+    goto indexes_out;
+  }
+
+  /* Analyze the SELECT statement in zSql. */
+  ctx.dbm = dbm;
+  sqlite3_db_config(db, SQLITE_DBCONFIG_WHEREINFO, idxWhereInfo, (void*)&ctx);
+  rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0);
+  sqlite3_db_config(db, SQLITE_DBCONFIG_WHEREINFO, (void*)0, (void*)0);
+  if( rc!=SQLITE_OK ){
+    idxDatabaseError(db, pzErrmsg);
+    goto indexes_out;
+  }
+
+  /* Create tables within the main in-memory database. These tables
+  ** have the same names, columns and declared types as the tables in
+  ** the user database. All constraints except for PRIMARY KEY are
+  ** removed. */
+  rc = idxCreateTables(db, dbm, ctx.pScan);
+  if( rc!=SQLITE_OK ){
+    goto indexes_out;
+  }
+
+  /* Create candidate indexes within the in-memory database file */
+
+ indexes_out:
+  idxScanFree(ctx.pScan);
+  sqlite3_close(dbm);
+  return rc;
+}
+
+
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index fce396c0f6..03d3d72f6f 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -1909,6 +1909,15 @@ struct sqlite3_mem_methods {
 #define SQLITE_DBCONFIG_LOOKASIDE       1001  /* void* int int */
 #define SQLITE_DBCONFIG_ENABLE_FKEY     1002  /* int int* */
 #define SQLITE_DBCONFIG_ENABLE_TRIGGER  1003  /* int int* */
+#define SQLITE_DBCONFIG_WHEREINFO       1004  /* xWhereInfo void* */
+
+#define SQLITE_WHEREINFO_TABLE   1
+#define SQLITE_WHEREINFO_EQUALS  2
+#define SQLITE_WHEREINFO_RANGE   3
+#define SQLITE_WHEREINFO_ORDERBY 4
+#define SQLITE_WHEREINFO_BEGINOR 5
+#define SQLITE_WHEREINFO_ENDOR   6
+#define SQLITE_WHEREINFO_NEXTOR  7
 
 
 /*
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 760c1f4d21..57c58195e0 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1274,6 +1274,10 @@ struct sqlite3 {
 #ifdef SQLITE_USER_AUTHENTICATION
   sqlite3_userauth auth;        /* User authentication information */
 #endif
+#ifdef SQLITE_SCHEMA_LINT
+  void (*xWhereInfo)(void*, int, const char*, int, i64);
+  void *pWhereInfoCtx;
+#endif
 };
 
 /*
diff --git a/src/where.c b/src/where.c
index af6bb20384..3724fa33e0 100644
--- a/src/where.c
+++ b/src/where.c
@@ -3906,176 +3906,103 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){
 }
 
 #ifdef SQLITE_SCHEMA_LINT
-static char *whereAppendPrintf(sqlite3 *db, const char *zFmt, ...){
-  va_list ap;
-  char *zRes = 0;
-  va_start(ap, zFmt);
-  zRes = sqlite3_vmprintf(zFmt, ap);
-  if( zRes==0 ){
-    db->mallocFailed = 1;
-  }else if( db->mallocFailed ){
-    sqlite3_free(zRes);
-    zRes = 0;
-  }
-  va_end(ap);
-  return zRes;
-}
-
-/*
-** Append a representation of term pTerm to the string in zIn and return
-** the result. Or, if an OOM occurs, free zIn and return a NULL pointer.
-*/
-static char *whereAppendSingleTerm(
-  Parse *pParse,
-  Table *pTab,
-  int iCol,
-  int bOr,
-  char *zIn,
-  WhereTerm *pTerm
-){
-  char *zBuf;
-  sqlite3 *db = pParse->db;
-  Expr *pX = pTerm->pExpr;
-  CollSeq *pColl;
-  const char *zOp = 0;
-
-  if( pTerm->eOperator & (WO_IS|WO_EQ|WO_IN) ){
-    zOp = "eq";
-  }else if( pTerm->eOperator & (WO_LT|WO_LE|WO_GE|WO_GT) ){
-    zOp = "range";
-  }
-  pColl = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight);
-
-  if( zOp ){
-    const char *zFmt = bOr ? "%z{{%s \"%w\" \"%w\" %lld}}" :
-                             "%z{%s \"%w\" \"%w\" %lld}";
-    zBuf = whereAppendPrintf(db, zFmt, zIn, 
-        zOp, pTab->aCol[iCol].zName, 
-        (pColl ? pColl->zName : "BINARY"),
-        pTerm->prereqRight
-    );
-  }else{
-    zBuf = zIn;
-  }
-
-  return zBuf;
-}
-
-static char *whereTraceWC(
+static void whereTraceWC(
   Parse *pParse, 
-  int bInitialSpace,
   struct SrcList_item *pItem,
-  char *zIn,
-  WhereClause *pWC
+  WhereClause *pWC,
+  int bOr
 ){
   sqlite3 *db = pParse->db;
   Table *pTab = pItem->pTab;
-  char *zBuf = zIn;
-  int iCol;
+  void (*x)(void*, int, const char*, int, i64) = db->xWhereInfo;
+  void *pCtx = db->pWhereInfoCtx;
+  int bFirst = 1;                 /* True until first callback is made */
   int ii;
-  int bFirst = !bInitialSpace;
-  int bOr = (pWC->op==TK_OR);
 
-  /* List of WO_SINGLE constraints */
-  for(iCol=0; iCol<pTab->nCol; iCol++){
+  /* Issue callbacks for WO_SINGLE constraints */
+  for(ii=0; ii<pTab->nCol; ii++){
     int opMask = WO_SINGLE; 
     WhereScan scan;
     WhereTerm *pTerm;
-    for(pTerm=whereScanInit(&scan, pWC, pItem->iCursor, iCol, opMask, 0);
+    for(pTerm=whereScanInit(&scan, pWC, pItem->iCursor, ii, opMask, 0);
         pTerm;
         pTerm=whereScanNext(&scan)
     ){
-      /* assert( iCol==pTerm->u.leftColumn ); */
-      if( bFirst==0 ) zBuf = whereAppendPrintf(db, "%z ", zBuf);
-      zBuf = whereAppendSingleTerm(pParse, pTab, iCol, bOr, zBuf, pTerm);
+      int eOp;
+      Expr *pX = pTerm->pExpr;
+      CollSeq *pC = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight);
+      if( pTerm->eOperator & (WO_IS|WO_EQ|WO_IN) ){
+        eOp = SQLITE_WHEREINFO_EQUALS;
+      }else{
+        eOp = SQLITE_WHEREINFO_RANGE;
+      }
+      if( bOr && !bFirst ) x(pCtx, SQLITE_WHEREINFO_NEXTOR, 0, 0, 0);
+      x(pCtx, eOp, (pC ? pC->zName : "BINARY"), ii, pTerm->prereqRight);
       bFirst = 0;
     }
   }
 
-  /* Add composite - (WO_OR|WO_AND) - constraints */
+  /* Callbacks for composite - (WO_OR|WO_AND) - constraints */
   for(ii=0; ii<pWC->nTerm; ii++){
     WhereTerm *pTerm = &pWC->a[ii];
-    if( pTerm->eOperator & (WO_OR|WO_AND) ){
-      const char *zFmt = ((pTerm->eOperator&WO_OR) ? "%z%s{or " : "%z%s{");
-      zBuf = whereAppendPrintf(db, zFmt, zBuf, bFirst ? "" : " ");
-      zBuf = whereTraceWC(pParse, 0, pItem, zBuf, &pTerm->u.pOrInfo->wc);
-      zBuf = whereAppendPrintf(db, "%z}", zBuf);
+    if( pTerm->eOperator & WO_OR ){
+      assert( bOr==0 );
+      x(pCtx, SQLITE_WHEREINFO_BEGINOR, 0, 0, 0);
+      whereTraceWC(pParse, pItem, &pTerm->u.pOrInfo->wc, 1);
+      x(pCtx, SQLITE_WHEREINFO_ENDOR, 0, 0, 0);
+    }
+    if( pTerm->eOperator & WO_AND ){
+      if( bOr && !bFirst ) x(pCtx, SQLITE_WHEREINFO_NEXTOR, 0, 0, 0);
+      whereTraceWC(pParse, pItem, &pTerm->u.pAndInfo->wc, 0);
       bFirst = 0;
     }
   }
-
-  return zBuf;
 }
 
+
 static void whereTraceBuilder(
   Parse *pParse,
   WhereLoopBuilder *p
 ){
   sqlite3 *db = pParse->db;
-  if( db->xTrace ){
-    ExprList *pOrderBy = p->pOrderBy;
-    WhereInfo *pWInfo = p->pWInfo;
-    int nTablist = pWInfo->pTabList->nSrc;
+  if( db->xWhereInfo && db->init.busy==0 ){
+    void (*x)(void*, int, const char*, int, i64) = db->xWhereInfo;
+    void *pCtx = db->pWhereInfoCtx;
     int ii;
+    int nTab = p->pWInfo->pTabList->nSrc;
 
     /* Loop through each element of the FROM clause. Ignore any sub-selects
-    ** or views. Invoke the xTrace() callback once for each real table. */
-    for(ii=0; ii<nTablist; ii++){
-      char *zBuf = 0;
-      int iCol;
-      int nCol;
-      Table *pTab;
+    ** or views. Invoke the xWhereInfo() callback multiple times for each
+    ** real table.  */
+    for(ii=0; ii<p->pWInfo->pTabList->nSrc; ii++){
+      struct SrcList_item *pItem = &p->pWInfo->pTabList->a[ii];
+      if( pItem->pSelect==0 ){
+        Table *pTab = pItem->pTab;
+        int iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
 
-      struct SrcList_item *pItem = &pWInfo->pTabList->a[ii];
-      if( pItem->pSelect ) continue;
-      pTab = pItem->pTab;
-      nCol = pTab->nCol;
+        /* Table name callback */
+        x(pCtx, SQLITE_WHEREINFO_TABLE, pTab->zName, iDb, pItem->colUsed);
 
-      /* Append the table name to the buffer. */
-      zBuf = whereAppendPrintf(db, "\"%w\"", pTab->zName);
-
-      /* Append the list of columns required to create a covering index */
-      zBuf = whereAppendPrintf(db, "%z {cols", zBuf);
-      if( 0==(pItem->colUsed & ((u64)1 << (sizeof(Bitmask)*8-1))) ){
-        for(iCol=0; iCol<nCol; iCol++){
-          if( iCol==(sizeof(Bitmask)*8-1) ) break;
-          if( pItem->colUsed & ((u64)1 << iCol) ){
-            const char *zName = pTab->aCol[iCol].zName;
-            zBuf = whereAppendPrintf(db, "%z \"%w\"", zBuf, zName);
-          }
-        }
-      }
-      zBuf = whereAppendPrintf(db, "%z}",zBuf);
-
-      /* Append the contents of WHERE clause */
-      zBuf = whereTraceWC(pParse, 1, pItem, zBuf, p->pWC);
-
-      /* Append the ORDER BY clause, if any */
-      if( pOrderBy ){
-        int i;
-        int bFirst = 1;
-        for(i=0; i<pOrderBy->nExpr; i++){
-          Expr *pExpr = pOrderBy->a[i].pExpr; 
-          CollSeq *pColl = sqlite3ExprCollSeq(pParse, pExpr);
-
-          pExpr = sqlite3ExprSkipCollate(pExpr);
-          if( pExpr->op==TK_COLUMN && pExpr->iTable==pItem->iCursor ){
-            if( pExpr->iColumn>=0 ){
-              const char *zName = pTab->aCol[pExpr->iColumn].zName;
-              zBuf = whereAppendPrintf(db, "%z%s\"%w\" \"%w\" %s", zBuf,
-                  bFirst ? " {orderby " : " ", zName, pColl->zName,
-                  (pOrderBy->a[i].sortOrder ? "DESC" : "ASC")
-              );
-              bFirst = 0;
+        /* ORDER BY callbacks */
+        if( p->pOrderBy ){
+          int i;
+          int bFirst = 1;
+          for(i=0; i<p->pOrderBy->nExpr; i++){
+            Expr *pExpr = p->pOrderBy->a[i].pExpr; 
+            CollSeq *pColl = sqlite3ExprCollSeq(pParse, pExpr);
+            pExpr = sqlite3ExprSkipCollate(pExpr);
+            if( pExpr->op==TK_COLUMN && pExpr->iTable==pItem->iCursor ){
+              int iCol = pExpr->iColumn;
+              if( iCol>=0 ){
+                x(pCtx, SQLITE_WHEREINFO_ORDERBY, pColl->zName, iCol, 0); 
+              }
             }
           }
         }
-        if( bFirst==0 ) zBuf = whereAppendPrintf(db, "%z}", zBuf);
-      }
 
-      /* Pass the buffer to the xTrace() callback, then free it */
-      db->xTrace(db->pTraceArg, zBuf);
-      sqlite3DbFree(db, zBuf);
+        /* WHERE callbacks */
+        whereTraceWC(pParse, pItem, p->pWC, 0);
+      }
     }
   }
 }

From 47dd4867bc22c57668cc9fd11e49dbd1d70ccf55 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 15 Feb 2016 20:12:16 +0000
Subject: [PATCH 010/488] Progress towards integrating schemalint into the
 shell tool. Some cases work now.

FossilOrigin-Name: 58d4cf26e15f90463148bec63d6ab514ffbbae60
---
 manifest            |  12 +-
 manifest.uuid       |   2 +-
 src/shell_indexes.c | 505 ++++++++++++++++++++++++++++++++++++++++++--
 3 files changed, 496 insertions(+), 23 deletions(-)

diff --git a/manifest b/manifest
index 7da4cdeb1d..c7c4b5f60c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Experimental\sintegration\sof\sschemalint\sfunctionality\swith\sthe\sshell\stool.\sDoes\snot\swork\syet.
-D 2016-02-11T21:01:16.253
+C Progress\stowards\sintegrating\sschemalint\sinto\sthe\sshell\stool.\sSome\scases\swork\snow.
+D 2016-02-15T20:12:16.368
 F Makefile.in dac2776c84e0d533b158a9af6e57e05c4a6b19f3
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc b0493f10caddb8adf992a4e6f1943141fc7c6816
@@ -350,7 +350,7 @@ F src/resolve.c 9f7ce3a3c087afb7597b7c916c99126ff3f12f0c
 F src/rowset.c 9fe4b3ad7cc00944386bb600233d8f523de07a6e
 F src/select.c ff80004a9a6ece891a8d9327a88e7b6e2588ee6d
 F src/shell.c 2cde87e03712204231167c4a6c61b0eb5129e105
-F src/shell_indexes.c 3cff393ee86d15fbfbe31f30cd752b46d7779b52
+F src/shell_indexes.c eefa3e9b7d992d10d1f71e27a2e12024a838b365
 F src/sqlite.h.in c7db059d3b810b70b83d9ed1436fa813eba22462
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h dfbe62ffd95b99afe2140d8c35b180d11924072d
@@ -1430,7 +1430,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 1a4182eedd0143c3f71b3d97f1d1bb25adeba617
-R 6253a80a5a7097c3b24b24ce68900613
+P ed49f297bcee86674ed673e195610b8cc1d35647
+R 6a53f87d207675e944487abae6392564
 U dan
-Z bddf3d3a7cd183a6a2362ed54e10358b
+Z dd362fd8eca18359c7fdd88fbcc31963
diff --git a/manifest.uuid b/manifest.uuid
index 38f9cc7372..9d51efbd2d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ed49f297bcee86674ed673e195610b8cc1d35647
\ No newline at end of file
+58d4cf26e15f90463148bec63d6ab514ffbbae60
\ No newline at end of file
diff --git a/src/shell_indexes.c b/src/shell_indexes.c
index 8e105fec1b..b7b44e47e0 100644
--- a/src/shell_indexes.c
+++ b/src/shell_indexes.c
@@ -18,6 +18,9 @@ typedef struct IdxContext IdxContext;
 typedef struct IdxScan IdxScan;
 typedef struct IdxWhere IdxWhere;
 
+typedef struct IdxColumn IdxColumn;
+typedef struct IdxTable IdxTable;
+
 /*
 ** A single constraint. Equivalent to either "col = ?" or "col < ?".
 **
@@ -34,16 +37,22 @@ struct IdxConstraint {
 };
 
 /*
-** A WHERE clause. Made up of IdxConstraint objects.
+** A WHERE clause. Made up of IdxConstraint objects. Example WHERE clause:
 **
 **   a=? AND b=? AND (c=? OR d=?) AND (e=? OR f=?)
 **
+** The above
+**
+**
+**
 */
 struct IdxWhere {
   IdxConstraint *pEq;             /* List of == constraints */
   IdxConstraint *pRange;          /* List of < constraints */
-  IdxWhere **apOr;                /* Array of OR branches (joined by pNextOr) */
-  IdxWhere *pNextOr;              /* Next in OR'd terms */
+  IdxWhere *pOr;                  /* List of OR constraints */
+
+  IdxWhere *pNextOr;              /* Next in OR constraints of same IdxWhere */
+  IdxWhere *pSibling;             /* Next branch in single OR constraint */
   IdxWhere *pParent;              /* Parent object (or NULL) */
 };
 
@@ -51,6 +60,7 @@ struct IdxWhere {
 ** A single scan of a single table.
 */
 struct IdxScan {
+  IdxTable *pTable;               /* Table-info */
   char *zTable;                   /* Name of table to scan */
   int iDb;                        /* Database containing table zTable */
   i64 covering;                   /* Mask of columns required for cov. index */
@@ -64,11 +74,27 @@ struct IdxScan {
 */
 struct IdxContext {
   IdxWhere *pCurrent;             /* Current where clause */
+  int rc;                         /* Error code (if error has occurred) */
   IdxScan *pScan;                 /* List of scan objects */
   sqlite3 *dbm;                   /* In-memory db for this analysis */
-  int rc;                         /* Error code (if error has occurred) */
+  sqlite3 *db;                    /* User database under analysis */
+  sqlite3_stmt *pInsertMask;      /* To write to aux.depmask */
 };
 
+/*
+** Data regarding a database table. Extracted from "PRAGMA table_info"
+*/
+struct IdxColumn {
+  char *zName;
+  char *zColl;
+  int iPk;
+};
+struct IdxTable {
+  int nCol;
+  IdxColumn *aCol;
+};
+
+
 typedef struct PragmaTable PragmaTable;
 typedef struct PragmaCursor PragmaCursor;
 
@@ -94,7 +120,7 @@ static int pragmaConnect(
   char **pzErr
 ){
   const char *zSchema = 
-    "CREATE TABLE a(tbl HIDDEN, cid, name, type, notnull, dflt_value, pk)";
+    "CREATE TABLE a(tbl HIDDEN, cid, name, type, isnotnull, dflt_value, pk)";
   PragmaTable *pTab = 0;
   int rc = SQLITE_OK;
 
@@ -102,6 +128,8 @@ static int pragmaConnect(
   if( rc==SQLITE_OK ){
     pTab = (PragmaTable *)sqlite3_malloc64(sizeof(PragmaTable));
     if( pTab==0 ) rc = SQLITE_NOMEM;
+  }else{
+    *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db));
   }
 
   assert( rc==SQLITE_OK || pTab==0 );
@@ -152,7 +180,6 @@ static int pragmaBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
 ** Open a new pragma cursor.
 */
 static int pragmaOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
-  PragmaTable *pTab = (PragmaTable *)pVTab;
   PragmaCursor *pCsr;
 
   pCsr = (PragmaCursor*)sqlite3_malloc64(sizeof(PragmaCursor));
@@ -167,6 +194,13 @@ static int pragmaOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
   return SQLITE_OK;
 }
 
+static int pragmaClose(sqlite3_vtab_cursor *pCursor){
+  PragmaCursor *pCsr = (PragmaCursor*)pCursor;
+  sqlite3_finalize(pCsr->pStmt);
+  sqlite3_free(pCsr);
+  return SQLITE_OK;
+}
+
 /*
 ** Move a statvfs cursor to the next entry in the file.
 */
@@ -212,6 +246,7 @@ static int pragmaFilter(
   }else{
     rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0);
   }
+  if( rc ) return rc;
   return pragmaNext(pCursor);;
 }
 
@@ -308,7 +343,7 @@ static void idxWhereInfo(
 ){
   IdxContext *p = (IdxContext*)pCtx;
 
-#if 1
+#if 0
   const char *zOp = 
     eOp==SQLITE_WHEREINFO_TABLE ? "TABLE" :
     eOp==SQLITE_WHEREINFO_EQUALS ? "EQUALS" :
@@ -361,19 +396,36 @@ static void idxWhereInfo(
           pNew->pNext = p->pCurrent->pEq;
           p->pCurrent->pEq = pNew;
         }
+
+        sqlite3_bind_int64(p->pInsertMask, 1, mask);
+        sqlite3_step(p->pInsertMask);
+        p->rc = sqlite3_reset(p->pInsertMask);
         break;
       }
 
       case SQLITE_WHEREINFO_BEGINOR: {
-        assert( 0 );
-        break;
-      }
-      case SQLITE_WHEREINFO_ENDOR: {
-        assert( 0 );
+        IdxWhere *pNew = (IdxWhere*)idxMalloc(&p->rc, sizeof(IdxWhere));
+        if( pNew==0 ) return;
+        pNew->pParent = p->pCurrent;
+        pNew->pNextOr = p->pCurrent->pOr;
+        p->pCurrent->pOr = pNew;
+        p->pCurrent = pNew;
         break;
       }
+
       case SQLITE_WHEREINFO_NEXTOR: {
-        assert( 0 );
+        IdxWhere *pNew = (IdxWhere*)idxMalloc(&p->rc, sizeof(IdxWhere));
+        if( pNew==0 ) return;
+        pNew->pParent = p->pCurrent->pParent;
+        assert( p->pCurrent->pSibling==0 );
+        p->pCurrent->pSibling = pNew;
+        p->pCurrent = pNew;
+        break;
+      }
+
+      case SQLITE_WHEREINFO_ENDOR: {
+        assert( p->pCurrent->pParent );
+        p->pCurrent = p->pCurrent->pParent;
         break;
       }
     }
@@ -391,16 +443,418 @@ static void idxDatabaseError(
   *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
 }
 
-static int idxCreateTables(sqlite3 *db, sqlite3 *dbm, IdxScan *pScan){
+static char *idxQueryToList(
+  sqlite3 *db, 
+  const char *zBind,
+  int *pRc,
+  char **pzErrmsg,
+  const char *zSql
+){
+  char *zRet = 0;
+  if( *pRc==SQLITE_OK ){
+    sqlite3_stmt *pStmt = 0;
+    int rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+    if( rc==SQLITE_OK ){
+      sqlite3_bind_text(pStmt, 1, zBind, -1, SQLITE_TRANSIENT);
+      while( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){
+        const char *z = (const char*)sqlite3_column_text(pStmt, 0);
+        zRet = sqlite3_mprintf("%z%s%Q", zRet, zRet?", ":"", z);
+        if( zRet==0 ){
+          rc = SQLITE_NOMEM;
+        }
+      }
+      rc = sqlite3_finalize(pStmt);
+    }
+
+    if( rc ){
+      idxDatabaseError(db, pzErrmsg);
+      sqlite3_free(zRet);
+      zRet = 0;
+    }
+    *pRc = rc;
+  }
+
+  return zRet;
+}
+
+static int idxGetTableInfo(
+  sqlite3 *db,
+  IdxScan *pScan,
+  char **pzErrmsg
+){
+  const char *zSql = "SELECT name, pk FROM pragma_table_info(?)";
+  sqlite3_stmt *p1 = 0;
+  int nCol = 0;
+  int nByte = sizeof(IdxTable);
+  IdxTable *pNew = 0;
+  int rc, rc2;
+  char *pCsr;
+
+  rc = sqlite3_prepare_v2(db, zSql, -1, &p1, 0);
+  if( rc!=SQLITE_OK ){
+    idxDatabaseError(db, pzErrmsg);
+    return rc;
+  }
+  sqlite3_bind_text(p1, 1, pScan->zTable, -1, SQLITE_TRANSIENT);
+  while( SQLITE_ROW==sqlite3_step(p1) ){
+    const char *zCol = sqlite3_column_text(p1, 0);
+    nByte += 1 + strlen(zCol);
+    rc = sqlite3_table_column_metadata(
+        db, "main", pScan->zTable, zCol, 0, &zCol, 0, 0, 0
+    );
+    nByte += 1 + strlen(zCol);
+    nCol++;
+  }
+  rc2 = sqlite3_reset(p1);
+  if( rc==SQLITE_OK ) rc = rc2;
+
+  nByte += sizeof(IdxColumn) * nCol;
+  if( rc==SQLITE_OK ){
+    pNew = idxMalloc(&rc, nByte);
+  }
+  if( rc==SQLITE_OK ){
+    pNew->aCol = (IdxColumn*)&pNew[1];
+    pNew->nCol = nCol;
+    pCsr = (char*)&pNew->aCol[nCol];
+  }
+
+  nCol = 0;
+  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){
+    const char *zCol = sqlite3_column_text(p1, 0);
+    int nCopy = strlen(zCol) + 1;
+    pNew->aCol[nCol].zName = pCsr;
+    pNew->aCol[nCol].iPk = sqlite3_column_int(p1, 1);
+    memcpy(pCsr, zCol, nCopy);
+    pCsr += nCopy;
+
+    rc = sqlite3_table_column_metadata(
+        db, "main", pScan->zTable, zCol, 0, &zCol, 0, 0, 0
+    );
+    if( rc==SQLITE_OK ){
+      nCopy = strlen(zCol) + 1;
+      pNew->aCol[nCol].zColl = pCsr;
+      memcpy(pCsr, zCol, nCopy);
+      pCsr += nCopy;
+    }
+
+    nCol++;
+  }
+  rc2 = sqlite3_finalize(p1);
+  if( rc==SQLITE_OK ) rc = rc2;
+
+  if( rc==SQLITE_OK ){
+    pScan->pTable = pNew;
+  }else{
+    sqlite3_free(pNew);
+  }
+
+  return rc;
+}
+
+
+static int idxCreateTables(
+  sqlite3 *db,                    /* User database */
+  sqlite3 *dbm,                   /* In-memory database to create tables in */
+  IdxScan *pScan,                 /* List of scans */
+  char **pzErrmsg                 /* OUT: Error message */
+){
   int rc = SQLITE_OK;
   IdxScan *pIter;
-  for(pIter=pScan; pIter; pIter=pIter->pNextScan){
+  for(pIter=pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
+    int nPk = 0;
+    char *zCols = 0;
+    char *zPk = 0;
+    char *zCreate = 0;
+    int iCol;
+
+    rc = idxGetTableInfo(db, pIter, pzErrmsg);
+
+    for(iCol=0; rc==SQLITE_OK && iCol<pIter->pTable->nCol; iCol++){
+      IdxColumn *pCol = &pIter->pTable->aCol[iCol];
+      if( pCol->iPk>nPk ) nPk = pCol->iPk;
+      zCols = sqlite3_mprintf("%z%s%Q", zCols, (zCols?", ":""), pCol->zName);
+      if( zCols==0 ) rc = SQLITE_NOMEM;
+    }
+
+    for(iCol=1; rc==SQLITE_OK && iCol<=nPk; iCol++){
+      int j;
+      for(j=0; j<pIter->pTable->nCol; j++){
+        IdxColumn *pCol = &pIter->pTable->aCol[j];
+        if( pCol->iPk==iCol ){
+          zPk = sqlite3_mprintf("%z%s%Q", zPk, (zPk?", ":""), pCol->zName);
+          if( zPk==0 ) rc = SQLITE_NOMEM;
+          break;
+        }
+      }
+    }
+
+    if( rc==SQLITE_OK ){
+      if( zPk ){
+        zCreate = sqlite3_mprintf("CREATE TABLE %Q(%s, PRIMARY KEY(%s))",
+            pIter->zTable, zCols, zPk
+        );
+      }else{
+        zCreate = sqlite3_mprintf("CREATE TABLE %Q(%s)", pIter->zTable, zCols);
+      }
+      if( zCreate==0 ) rc = SQLITE_NOMEM;
+    }
+
+    if( rc==SQLITE_OK ){
+#if 1
+      printf("/* %s */\n", zCreate);
+#endif
+      rc = sqlite3_exec(dbm, zCreate, 0, 0, pzErrmsg);
+    }
+    sqlite3_free(zCols);
+    sqlite3_free(zPk);
+    sqlite3_free(zCreate);
   }
+  return rc;
+}
+
+/*
+** This function is a no-op if *pRc is set to anything other than 
+** SQLITE_OK when it is called.
+**
+** If *pRc is initially set to SQLITE_OK, then the text specified by
+** the printf() style arguments is appended to zIn and the result returned
+** in a buffer allocated by sqlite3_malloc(). sqlite3_free() is called on
+** zIn before returning.
+*/
+static char *idxAppendText(int *pRc, char *zIn, const char *zFmt, ...){
+  va_list ap;
+  char *zAppend = 0;
+  char *zRet = 0;
+  int nIn = zIn ? strlen(zIn) : 0;
+  int nAppend = 0;
+  va_start(ap, zFmt);
+  if( *pRc==SQLITE_OK ){
+    zAppend = sqlite3_vmprintf(zFmt, ap);
+    if( zAppend ){
+      nAppend = strlen(zAppend);
+      zRet = (char*)sqlite3_malloc(nIn + nAppend);
+    }
+    if( zAppend && zRet ){
+      memcpy(zRet, zIn, nIn);
+      memcpy(&zRet[nIn], zAppend, nAppend+1);
+    }else{
+      sqlite3_free(zRet);
+      zRet = 0;
+      *pRc = SQLITE_NOMEM;
+    }
+    sqlite3_free(zAppend);
+    sqlite3_free(zIn);
+  }
+  va_end(ap);
+  return zRet;
+}
+
+static char *idxAppendColDefn(
+  int *pRc, 
+  char *zIn, 
+  IdxTable *pTab, 
+  IdxConstraint *pCons
+){
+  char *zRet = zIn;
+  IdxColumn *p = &pTab->aCol[pCons->iCol];
+  if( zRet ) zRet = idxAppendText(pRc, zRet, ", ");
+  zRet = idxAppendText(pRc, zRet, "%Q", p->zName);
+  if( sqlite3_stricmp(p->zColl, pCons->zColl) ){
+    zRet = idxAppendText(pRc, zRet, " COLLATE %Q", pCons->zColl);
+  }
+  return zRet;
+}
+
+static int idxCreateFromCons(
+  sqlite3 *dbm,
+  IdxScan *pScan,
+  IdxConstraint *pEq, 
+  IdxConstraint *pTail
+){
+  int rc = SQLITE_OK;
+  if( pEq || pTail ){
+    IdxTable *pTab = pScan->pTable;
+    char *zCols = 0;
+    char *zIdx = 0;
+    IdxConstraint *pCons;
+    int h = 0;
+
+    for(pCons=pEq; pCons; pCons=pCons->pLink){
+      zCols = idxAppendColDefn(&rc, zCols, pTab, pCons);
+    }
+    for(pCons=pTail; pCons; pCons=pCons->pLink){
+      zCols = idxAppendColDefn(&rc, zCols, pTab, pCons);
+    }
+
+    /* Hash the list of columns to come up with a name for the index */
+    if( rc==SQLITE_OK ){
+      int i;
+      for(i=0; zCols[i]; i++){
+        h += ((h<<3) + zCols[i]);
+      }
+
+      zIdx = sqlite3_mprintf("CREATE INDEX IF NOT EXISTS "
+          "'%q_idx_%08x' ON %Q(%s)", pScan->zTable, h, pScan->zTable, zCols
+      );
+      if( !zIdx ){
+        rc = SQLITE_NOMEM;
+      }else{
+        rc = sqlite3_exec(dbm, zIdx, 0, 0, 0);
+        printf("/* %s */\n", zIdx);
+      }
+    }
+
+    sqlite3_free(zIdx);
+    sqlite3_free(zCols);
+  }
+  return rc;
+}
+
+static int idxCreateFromWhere(
+    sqlite3*, i64, IdxScan*, IdxWhere*, IdxConstraint*, IdxConstraint*
+);
+
+static int idxCreateForeachOr(
+  sqlite3 *dbm, 
+  i64 mask,                       /* Consider only these constraints */
+  IdxScan *pScan,                 /* Create indexes for this scan */
+  IdxWhere *pWhere,               /* Read constraints from here */
+  IdxConstraint *pEq,             /* == constraints for inclusion */
+  IdxConstraint *pTail            /* range/ORDER BY constraints for inclusion */
+){
+  int rc = SQLITE_OK;
+  IdxWhere *p1;
+  IdxWhere *p2;
+  for(p1=pWhere->pOr; p1 && rc==SQLITE_OK; p1=p1->pNextOr){
+    rc = idxCreateFromWhere(dbm, mask, pScan, p1, pEq, pTail);
+    for(p2=p1->pSibling; p2 && rc==SQLITE_OK; p2=p2->pSibling){
+      rc = idxCreateFromWhere(dbm, mask, pScan, p2, pEq, pTail);
+    }
+  }
+  return rc;
+}
+
+static int idxCreateFromWhere(
+  sqlite3 *dbm, 
+  i64 mask,                       /* Consider only these constraints */
+  IdxScan *pScan,                 /* Create indexes for this scan */
+  IdxWhere *pWhere,               /* Read constraints from here */
+  IdxConstraint *pEq,             /* == constraints for inclusion */
+  IdxConstraint *pTail            /* range/ORDER BY constraints for inclusion */
+){
+  IdxConstraint *p1 = pEq;
+  IdxConstraint *pCon;
+  int rc;
+
+  /* Gather up all the == constraints that match the mask. */
+  for(pCon=pWhere->pEq; pCon; pCon=pCon->pNext){
+    if( (mask & pCon->depmask)==pCon->depmask ){
+      pCon->pLink = p1;
+      p1 = pCon;
+    }
+  }
+
+  /* Create an index using the == constraints collected above. And the
+  ** range constraint/ORDER BY terms passed in by the caller, if any. */
+  rc = idxCreateFromCons(dbm, pScan, p1, pTail);
+  if( rc==SQLITE_OK ){
+    rc = idxCreateForeachOr(dbm, mask, pScan, pWhere, p1, pTail);
+  }
+
+  /* If no range/ORDER BY passed by the caller, create a version of the
+  ** index for each range constraint that matches the mask. */
+  if( pTail==0 ){
+    for(pCon=pWhere->pRange; rc==SQLITE_OK && pCon; pCon=pCon->pNext){
+      assert( pCon->pLink==0 );
+      if( (mask & pCon->depmask)==pCon->depmask ){
+        rc = idxCreateFromCons(dbm, pScan, p1, pCon);
+        if( rc==SQLITE_OK ){
+          rc = idxCreateForeachOr(dbm, mask, pScan, pWhere, p1, pCon);
+        }
+      }
+    }
+  }
+
+  return rc;
+}
+
+static int idxPrepareStmt(
+  sqlite3 *db,                    /* Database handle to compile against */
+  const char *zSql,               /* SQL statement to compile */
+  sqlite3_stmt **ppStmt,          /* OUT: Compiled SQL statement */
+  char **pzErrmsg                 /* OUT: sqlite3_malloc()ed error message */
+){
+  int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0);
+  if( rc!=SQLITE_OK ){
+    *ppStmt = 0;
+    idxDatabaseError(db, pzErrmsg);
+  }
+  return rc;
+}
+
+/*
+** Create candidate indexes in database [dbm] based on the data in 
+** linked-list pScan.
+*/
+static int idxCreateCandidates(
+  sqlite3 *dbm,
+  IdxScan *pScan,
+  char **pzErrmsg
+){
+  int rc2;
+  int rc = SQLITE_OK;
+  sqlite3_stmt *pDepmask;         /* Foreach depmask */
+  IdxScan *pIter;
+
+  rc = idxPrepareStmt(dbm, "SELECT mask FROM depmask", &pDepmask, pzErrmsg);
+
+  for(pIter=pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
+    IdxWhere *pWhere = &pIter->where;
+    while( SQLITE_ROW==sqlite3_step(pDepmask) && rc==SQLITE_OK ){
+      i64 mask = sqlite3_column_int64(pDepmask, 0);
+      rc = idxCreateFromWhere(dbm, mask, pIter, pWhere, 0, 0);
+      if( rc==SQLITE_OK && pIter->pOrder ){
+        rc = idxCreateFromWhere(dbm, mask, pIter, pWhere, 0, pIter->pOrder);
+      }
+    }
+  }
+
+  rc2 = sqlite3_finalize(pDepmask);
+  if( rc==SQLITE_OK ) rc = rc2;
+  return rc;
 }
 
 static void idxScanFree(IdxScan *pScan){
 }
 
+int idxFindIndexes(
+  sqlite3 *dbm,                        /* Database handle */
+  const char *zSql,                    /* SQL to find indexes for */
+  void (*xOut)(void*, const char*),    /* Output callback */
+  void *pOutCtx,                       /* Context for xOut() */
+  char **pzErrmsg                      /* OUT: Error message (sqlite3_malloc) */
+){
+  char *zExplain;
+  sqlite3_stmt *pExplain;
+  int rc;
+
+  zExplain = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", zSql);
+  if( zExplain==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    rc = idxPrepareStmt(dbm, zExplain, &pExplain, pzErrmsg);
+    sqlite3_free(zExplain);
+  }
+  if( rc!=SQLITE_OK ) return rc;
+
+  while( sqlite3_step(pExplain)==SQLITE_ROW ){
+    int iCol;
+    // for(iCol=0; iCol<sqlite3_column_count(pExplain); iCol++){ }
+    xOut(pOutCtx, sqlite3_column_text(pExplain, 3));
+  }
+  rc = sqlite3_finalize(pExplain);
+}
+
 /*
 ** The xOut callback is invoked to return command output to the user. The
 ** second argument is always a nul-terminated string. The first argument is
@@ -419,6 +873,8 @@ int shellIndexesCommand(
   IdxContext ctx;
   sqlite3_stmt *pStmt = 0;        /* Statement compiled from zSql */
 
+  rc = registerPragmaVtabs(db);
+  if( rc ) return rc;
   memset(&ctx, 0, sizeof(IdxContext));
 
   /* Open an in-memory database to work with. The main in-memory 
@@ -430,9 +886,19 @@ int shellIndexesCommand(
     rc = sqlite3_exec(dbm, 
         "ATTACH ':memory:' AS aux;"
         "CREATE TABLE aux.depmask(mask PRIMARY KEY) WITHOUT ROWID;"
+        "INSERT INTO aux.depmask VALUES(0);"
         , 0, 0, 0
     );
   }
+
+  /* Prepare an INSERT statement for writing to aux.depmask */
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_prepare_v2(dbm, 
+        "INSERT OR IGNORE INTO depmask SELECT mask | ?1 FROM depmask;", -1,
+        &ctx.pInsertMask, 0
+    );
+  }
+
   if( rc!=SQLITE_OK ){
     idxDatabaseError(dbm, pzErrmsg);
     goto indexes_out;
@@ -443,6 +909,7 @@ int shellIndexesCommand(
   sqlite3_db_config(db, SQLITE_DBCONFIG_WHEREINFO, idxWhereInfo, (void*)&ctx);
   rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0);
   sqlite3_db_config(db, SQLITE_DBCONFIG_WHEREINFO, (void*)0, (void*)0);
+
   if( rc!=SQLITE_OK ){
     idxDatabaseError(db, pzErrmsg);
     goto indexes_out;
@@ -452,12 +919,18 @@ int shellIndexesCommand(
   ** have the same names, columns and declared types as the tables in
   ** the user database. All constraints except for PRIMARY KEY are
   ** removed. */
-  rc = idxCreateTables(db, dbm, ctx.pScan);
+  rc = idxCreateTables(db, dbm, ctx.pScan, pzErrmsg);
   if( rc!=SQLITE_OK ){
     goto indexes_out;
   }
 
   /* Create candidate indexes within the in-memory database file */
+  rc = idxCreateCandidates(dbm, ctx.pScan, pzErrmsg);
+  if( rc!=SQLITE_OK ){
+    goto indexes_out;
+  }
+
+  rc = idxFindIndexes(dbm, zSql, xOut, pOutCtx, pzErrmsg);
 
  indexes_out:
   idxScanFree(ctx.pScan);

From 94975e27a3b56d6efaa1576cae33e89be045ea59 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 16 Feb 2016 18:37:37 +0000
Subject: [PATCH 011/488] Fix further issues in schemalint.

FossilOrigin-Name: 73a7f010937828c5195a198604f976e8458cef73
---
 manifest            |  12 +-
 manifest.uuid       |   2 +-
 src/shell_indexes.c | 426 +++++++++++++++-----------------------------
 3 files changed, 154 insertions(+), 286 deletions(-)

diff --git a/manifest b/manifest
index c7c4b5f60c..f1eaa8735a 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Progress\stowards\sintegrating\sschemalint\sinto\sthe\sshell\stool.\sSome\scases\swork\snow.
-D 2016-02-15T20:12:16.368
+C Fix\sfurther\sissues\sin\sschemalint.
+D 2016-02-16T18:37:37.844
 F Makefile.in dac2776c84e0d533b158a9af6e57e05c4a6b19f3
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc b0493f10caddb8adf992a4e6f1943141fc7c6816
@@ -350,7 +350,7 @@ F src/resolve.c 9f7ce3a3c087afb7597b7c916c99126ff3f12f0c
 F src/rowset.c 9fe4b3ad7cc00944386bb600233d8f523de07a6e
 F src/select.c ff80004a9a6ece891a8d9327a88e7b6e2588ee6d
 F src/shell.c 2cde87e03712204231167c4a6c61b0eb5129e105
-F src/shell_indexes.c eefa3e9b7d992d10d1f71e27a2e12024a838b365
+F src/shell_indexes.c 27d3b064078066c9a284a93149c4a151821f55bf
 F src/sqlite.h.in c7db059d3b810b70b83d9ed1436fa813eba22462
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h dfbe62ffd95b99afe2140d8c35b180d11924072d
@@ -1430,7 +1430,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P ed49f297bcee86674ed673e195610b8cc1d35647
-R 6a53f87d207675e944487abae6392564
+P 58d4cf26e15f90463148bec63d6ab514ffbbae60
+R f221e3cac7f52c2c660e427ee329aa03
 U dan
-Z dd362fd8eca18359c7fdd88fbcc31963
+Z 8b18cd934aae857f866e3f00c12cba6f
diff --git a/manifest.uuid b/manifest.uuid
index 9d51efbd2d..4b3aa263c2 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-58d4cf26e15f90463148bec63d6ab514ffbbae60
\ No newline at end of file
+73a7f010937828c5195a198604f976e8458cef73
\ No newline at end of file
diff --git a/src/shell_indexes.c b/src/shell_indexes.c
index b7b44e47e0..33f872efd7 100644
--- a/src/shell_indexes.c
+++ b/src/shell_indexes.c
@@ -25,7 +25,8 @@ typedef struct IdxTable IdxTable;
 ** A single constraint. Equivalent to either "col = ?" or "col < ?".
 **
 ** pLink:
-**   ... todo ...
+**   Used to temporarily link IdxConstraint objects into lists while
+**   creating candidate indexes.
 */
 struct IdxConstraint {
   char *zColl;                    /* Collation sequence */
@@ -39,18 +40,41 @@ struct IdxConstraint {
 /*
 ** A WHERE clause. Made up of IdxConstraint objects. Example WHERE clause:
 **
-**   a=? AND b=? AND (c=? OR d=?) AND (e=? OR f=?)
+**   a=? AND b=? AND ((c=? AND d=?) OR e=?) AND (f=? OR g=?) AND h>?
 **
-** The above
+** The above is decomposed into 5 AND connected clauses. The first two are
+** added to the IdxWhere.pEq linked list, the following two into 
+** IdxWhere.pOr and the last into IdxWhere.pRange.
 **
+** IdxWhere.pEq and IdxWhere.pRange are simple linked lists of IdxConstraint
+** objects linked by the IdxConstraint.pNext field.
 **
+** The list headed at IdxWhere.pOr and linked by IdxWhere.pNextOr contains
+** all "OR" terms that belong to the current WHERE clause. In the example
+** above, there are two OR terms:
 **
+**   ((c=? AND d=?) OR e=?)
+**   (f=? OR g=?)
+**
+** Within an OR term, the OR connected sub-expressions are termed siblings.
+** These are connected into a linked list by the pSibling pointers. Each OR
+** term above consists of two siblings.
+**
+**   pOr -> (c=? AND d=?) -> pNextOr -> (f=?)
+**               |                        |
+**            pSibling                 pSibling
+**               |                        |
+**               V                        V
+**             (e=?)                    (g=?)
+**
+** IdxWhere.pParent is only used while constructing a tree of IdxWhere 
+** structures. It is NULL for the root IdxWhere. For all others, the parent
+** WHERE clause.
 */
 struct IdxWhere {
   IdxConstraint *pEq;             /* List of == constraints */
   IdxConstraint *pRange;          /* List of < constraints */
   IdxWhere *pOr;                  /* List of OR constraints */
-
   IdxWhere *pNextOr;              /* Next in OR constraints of same IdxWhere */
   IdxWhere *pSibling;             /* Next branch in single OR constraint */
   IdxWhere *pParent;              /* Parent object (or NULL) */
@@ -94,209 +118,6 @@ struct IdxTable {
   IdxColumn *aCol;
 };
 
-
-typedef struct PragmaTable PragmaTable;
-typedef struct PragmaCursor PragmaCursor;
-
-struct PragmaTable {
-  sqlite3_vtab base;
-  sqlite3 *db;
-};
-
-struct PragmaCursor {
-  sqlite3_vtab_cursor base;
-  sqlite3_stmt *pStmt;
-  i64 iRowid;
-};
-
-/*
-** Connect to or create a pragma virtual table.
-*/
-static int pragmaConnect(
-  sqlite3 *db,
-  void *pAux,
-  int argc, const char *const*argv,
-  sqlite3_vtab **ppVtab,
-  char **pzErr
-){
-  const char *zSchema = 
-    "CREATE TABLE a(tbl HIDDEN, cid, name, type, isnotnull, dflt_value, pk)";
-  PragmaTable *pTab = 0;
-  int rc = SQLITE_OK;
-
-  rc = sqlite3_declare_vtab(db, zSchema);
-  if( rc==SQLITE_OK ){
-    pTab = (PragmaTable *)sqlite3_malloc64(sizeof(PragmaTable));
-    if( pTab==0 ) rc = SQLITE_NOMEM;
-  }else{
-    *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db));
-  }
-
-  assert( rc==SQLITE_OK || pTab==0 );
-  if( rc==SQLITE_OK ){
-    memset(pTab, 0, sizeof(PragmaTable));
-    pTab->db = db;
-  }
-
-  *ppVtab = (sqlite3_vtab*)pTab;
-  return rc;
-}
-
-/*
-** Disconnect from or destroy a pragma virtual table.
-*/
-static int pragmaDisconnect(sqlite3_vtab *pVtab){
-  sqlite3_free(pVtab);
-  return SQLITE_OK;
-}
-
-/*
-** xBestIndex method for pragma virtual tables.
-*/
-static int pragmaBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
-  int i;
-
-  pIdxInfo->estimatedCost = 1.0e6;  /* Initial cost estimate */
-
-  /* Look for a valid tbl=? constraint. */
-  for(i=0; i<pIdxInfo->nConstraint; i++){
-    if( pIdxInfo->aConstraint[i].usable==0 ) continue;
-    if( pIdxInfo->aConstraint[i].op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
-    if( pIdxInfo->aConstraint[i].iColumn!=0 ) continue;
-    pIdxInfo->idxNum = 1;
-    pIdxInfo->estimatedCost = 1.0;
-    pIdxInfo->aConstraintUsage[i].argvIndex = 1;
-    pIdxInfo->aConstraintUsage[i].omit = 1;
-    break;
-  }
-  if( i==pIdxInfo->nConstraint ){
-    tab->zErrMsg = sqlite3_mprintf("missing required tbl=? constraint");
-    return SQLITE_ERROR;
-  }
-  return SQLITE_OK;
-}
-
-/*
-** Open a new pragma cursor.
-*/
-static int pragmaOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
-  PragmaCursor *pCsr;
-
-  pCsr = (PragmaCursor*)sqlite3_malloc64(sizeof(PragmaCursor));
-  if( pCsr==0 ){
-    return SQLITE_NOMEM;
-  }else{
-    memset(pCsr, 0, sizeof(PragmaCursor));
-    pCsr->base.pVtab = pVTab;
-  }
-
-  *ppCursor = (sqlite3_vtab_cursor*)pCsr;
-  return SQLITE_OK;
-}
-
-static int pragmaClose(sqlite3_vtab_cursor *pCursor){
-  PragmaCursor *pCsr = (PragmaCursor*)pCursor;
-  sqlite3_finalize(pCsr->pStmt);
-  sqlite3_free(pCsr);
-  return SQLITE_OK;
-}
-
-/*
-** Move a statvfs cursor to the next entry in the file.
-*/
-static int pragmaNext(sqlite3_vtab_cursor *pCursor){
-  PragmaCursor *pCsr = (PragmaCursor*)pCursor;
-  int rc = SQLITE_OK;
-
-  if( sqlite3_step(pCsr->pStmt)!=SQLITE_ROW ){
-    rc = sqlite3_finalize(pCsr->pStmt);
-    pCsr->pStmt = 0;
-  }
-  pCsr->iRowid++;
-  return rc;
-}
-
-static int pragmaEof(sqlite3_vtab_cursor *pCursor){
-  PragmaCursor *pCsr = (PragmaCursor*)pCursor;
-  return pCsr->pStmt==0;
-}
-
-static int pragmaFilter(
-  sqlite3_vtab_cursor *pCursor, 
-  int idxNum, const char *idxStr,
-  int argc, sqlite3_value **argv
-){
-  PragmaCursor *pCsr = (PragmaCursor*)pCursor;
-  PragmaTable *pTab = (PragmaTable*)(pCursor->pVtab);
-  char *zSql;
-  const char *zTbl;
-  int rc = SQLITE_OK;
-
-  if( pCsr->pStmt ){
-    sqlite3_finalize(pCsr->pStmt);
-    pCsr->pStmt = 0;
-  }
-  pCsr->iRowid = 0;
-
-  assert( argc==1 );
-  zTbl = (const char*)sqlite3_value_text(argv[0]);
-  zSql = sqlite3_mprintf("PRAGMA table_info(%Q)", zTbl);
-  if( zSql==0 ){
-    rc = SQLITE_NOMEM;
-  }else{
-    rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0);
-  }
-  if( rc ) return rc;
-  return pragmaNext(pCursor);;
-}
-
-/*
-** xColumn method.
-*/
-static int pragmaColumn(
-  sqlite3_vtab_cursor *pCursor, 
-  sqlite3_context *ctx, 
-  int iCol
-){
-  PragmaCursor *pCsr = (PragmaCursor *)pCursor;
-  if( iCol>0 ){
-    sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pStmt, iCol-1));
-  }
-  return SQLITE_OK;
-}
-
-static int pragmaRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
-  PragmaCursor *pCsr = (PragmaCursor *)pCursor;
-  *pRowid = pCsr->iRowid;
-  return SQLITE_OK;
-}
-
-static int registerPragmaVtabs(sqlite3 *db){
-  static sqlite3_module pragma_module = {
-    0,                            /* iVersion */
-    pragmaConnect,                /* xCreate */
-    pragmaConnect,                /* xConnect */
-    pragmaBestIndex,              /* xBestIndex */
-    pragmaDisconnect,             /* xDisconnect */
-    pragmaDisconnect,             /* xDestroy */
-    pragmaOpen,                   /* xOpen - open a cursor */
-    pragmaClose,                  /* xClose - close a cursor */
-    pragmaFilter,                 /* xFilter - configure scan constraints */
-    pragmaNext,                   /* xNext - advance a cursor */
-    pragmaEof,                    /* xEof - check for end of scan */
-    pragmaColumn,                 /* xColumn - read data */
-    pragmaRowid,                  /* xRowid - read data */
-    0,                            /* xUpdate */
-    0,                            /* xBegin */
-    0,                            /* xSync */
-    0,                            /* xCommit */
-    0,                            /* xRollback */
-    0,                            /* xFindMethod */
-    0,                            /* xRename */
-  };
-  return sqlite3_create_module(db, "pragma_table_info", &pragma_module, 0);
-}
-
 /*
 ** Allocate and return nByte bytes of zeroed memory using sqlite3_malloc(). 
 ** If the allocation fails, set *pRc to SQLITE_NOMEM and return NULL.
@@ -477,12 +298,48 @@ static char *idxQueryToList(
   return zRet;
 }
 
+static int idxPrepareStmt(
+  sqlite3 *db,                    /* Database handle to compile against */
+  sqlite3_stmt **ppStmt,          /* OUT: Compiled SQL statement */
+  char **pzErrmsg,                /* OUT: sqlite3_malloc()ed error message */
+  const char *zSql                /* SQL statement to compile */
+){
+  int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0);
+  if( rc!=SQLITE_OK ){
+    *ppStmt = 0;
+    idxDatabaseError(db, pzErrmsg);
+  }
+  return rc;
+}
+
+static int idxPrintfPrepareStmt(
+  sqlite3 *db,                    /* Database handle to compile against */
+  sqlite3_stmt **ppStmt,          /* OUT: Compiled SQL statement */
+  char **pzErrmsg,                /* OUT: sqlite3_malloc()ed error message */
+  const char *zFmt,               /* printf() format of SQL statement */
+  ...                             /* Trailing printf() arguments */
+){
+  va_list ap;
+  int rc;
+  char *zSql;
+  va_start(ap, zFmt);
+  zSql = sqlite3_vmprintf(zFmt, ap);
+  if( zSql==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    rc = idxPrepareStmt(db, ppStmt, pzErrmsg, zSql);
+    sqlite3_free(zSql);
+  }
+  va_end(ap);
+  return rc;
+}
+
 static int idxGetTableInfo(
   sqlite3 *db,
   IdxScan *pScan,
   char **pzErrmsg
 ){
-  const char *zSql = "SELECT name, pk FROM pragma_table_info(?)";
+  const char *zTbl = pScan->zTable;
   sqlite3_stmt *p1 = 0;
   int nCol = 0;
   int nByte = sizeof(IdxTable);
@@ -490,17 +347,12 @@ static int idxGetTableInfo(
   int rc, rc2;
   char *pCsr;
 
-  rc = sqlite3_prepare_v2(db, zSql, -1, &p1, 0);
-  if( rc!=SQLITE_OK ){
-    idxDatabaseError(db, pzErrmsg);
-    return rc;
-  }
-  sqlite3_bind_text(p1, 1, pScan->zTable, -1, SQLITE_TRANSIENT);
-  while( SQLITE_ROW==sqlite3_step(p1) ){
-    const char *zCol = sqlite3_column_text(p1, 0);
+  rc = idxPrintfPrepareStmt(db, &p1, pzErrmsg, "PRAGMA table_info=%Q", zTbl);
+  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){
+    const char *zCol = sqlite3_column_text(p1, 1);
     nByte += 1 + strlen(zCol);
     rc = sqlite3_table_column_metadata(
-        db, "main", pScan->zTable, zCol, 0, &zCol, 0, 0, 0
+        db, "main", zTbl, zCol, 0, &zCol, 0, 0, 0
     );
     nByte += 1 + strlen(zCol);
     nCol++;
@@ -520,15 +372,15 @@ static int idxGetTableInfo(
 
   nCol = 0;
   while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){
-    const char *zCol = sqlite3_column_text(p1, 0);
+    const char *zCol = sqlite3_column_text(p1, 1);
     int nCopy = strlen(zCol) + 1;
     pNew->aCol[nCol].zName = pCsr;
-    pNew->aCol[nCol].iPk = sqlite3_column_int(p1, 1);
+    pNew->aCol[nCol].iPk = sqlite3_column_int(p1, 5);
     memcpy(pCsr, zCol, nCopy);
     pCsr += nCopy;
 
     rc = sqlite3_table_column_metadata(
-        db, "main", pScan->zTable, zCol, 0, &zCol, 0, 0, 0
+        db, "main", zTbl, zCol, 0, &zCol, 0, 0, 0
     );
     if( rc==SQLITE_OK ){
       nCopy = strlen(zCol) + 1;
@@ -600,7 +452,7 @@ static int idxCreateTables(
     }
 
     if( rc==SQLITE_OK ){
-#if 1
+#if 0
       printf("/* %s */\n", zCreate);
 #endif
       rc = sqlite3_exec(dbm, zCreate, 0, 0, pzErrmsg);
@@ -700,7 +552,9 @@ static int idxCreateFromCons(
         rc = SQLITE_NOMEM;
       }else{
         rc = sqlite3_exec(dbm, zIdx, 0, 0, 0);
+#if 0
         printf("/* %s */\n", zIdx);
+#endif
       }
     }
 
@@ -778,20 +632,6 @@ static int idxCreateFromWhere(
   return rc;
 }
 
-static int idxPrepareStmt(
-  sqlite3 *db,                    /* Database handle to compile against */
-  const char *zSql,               /* SQL statement to compile */
-  sqlite3_stmt **ppStmt,          /* OUT: Compiled SQL statement */
-  char **pzErrmsg                 /* OUT: sqlite3_malloc()ed error message */
-){
-  int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0);
-  if( rc!=SQLITE_OK ){
-    *ppStmt = 0;
-    idxDatabaseError(db, pzErrmsg);
-  }
-  return rc;
-}
-
 /*
 ** Create candidate indexes in database [dbm] based on the data in 
 ** linked-list pScan.
@@ -806,7 +646,7 @@ static int idxCreateCandidates(
   sqlite3_stmt *pDepmask;         /* Foreach depmask */
   IdxScan *pIter;
 
-  rc = idxPrepareStmt(dbm, "SELECT mask FROM depmask", &pDepmask, pzErrmsg);
+  rc = idxPrepareStmt(dbm, &pDepmask, pzErrmsg, "SELECT mask FROM depmask");
 
   for(pIter=pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
     IdxWhere *pWhere = &pIter->where;
@@ -832,27 +672,65 @@ int idxFindIndexes(
   const char *zSql,                    /* SQL to find indexes for */
   void (*xOut)(void*, const char*),    /* Output callback */
   void *pOutCtx,                       /* Context for xOut() */
-  char **pzErrmsg                      /* OUT: Error message (sqlite3_malloc) */
+  char **pzErr                         /* OUT: Error message (sqlite3_malloc) */
 ){
-  char *zExplain;
-  sqlite3_stmt *pExplain;
-  int rc;
+  sqlite3_stmt *pExplain = 0;
+  sqlite3_stmt *pSelect = 0;
+  int rc, rc2;
 
-  zExplain = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", zSql);
-  if( zExplain==0 ){
-    rc = SQLITE_NOMEM;
-  }else{
-    rc = idxPrepareStmt(dbm, zExplain, &pExplain, pzErrmsg);
-    sqlite3_free(zExplain);
+  rc = idxPrintfPrepareStmt(dbm, &pExplain, pzErr,"EXPLAIN QUERY PLAN %s",zSql);
+  if( rc==SQLITE_OK ){
+    rc = idxPrepareStmt(dbm, &pSelect, pzErr, 
+        "SELECT sql FROM sqlite_master WHERE name = ?"
+    );
   }
-  if( rc!=SQLITE_OK ) return rc;
 
-  while( sqlite3_step(pExplain)==SQLITE_ROW ){
-    int iCol;
-    // for(iCol=0; iCol<sqlite3_column_count(pExplain); iCol++){ }
-    xOut(pOutCtx, sqlite3_column_text(pExplain, 3));
+  while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){
+    int i;
+    const char *zDetail = (const char*)sqlite3_column_text(pExplain, 3);
+    int nDetail = strlen(zDetail);
+
+    for(i=0; i<nDetail; i++){
+      if( memcmp(&zDetail[i], " USING INDEX ", 13)==0 ){
+        int nIdx = 0;
+        const char *zIdx = &zDetail[i+13];
+        while( zIdx[nIdx]!='\0' && zIdx[nIdx]!=' ' ) nIdx++;
+        sqlite3_bind_text(pSelect, 1, zIdx, nIdx, SQLITE_STATIC);
+        if( SQLITE_ROW==sqlite3_step(pSelect) ){
+          xOut(pOutCtx, sqlite3_column_text(pSelect, 0));
+        }
+        rc = sqlite3_reset(pSelect);
+        break;
+      }
+    }
   }
-  rc = sqlite3_finalize(pExplain);
+  rc2 = sqlite3_reset(pExplain);
+  if( rc==SQLITE_OK ) rc = rc2;
+  if( rc==SQLITE_OK ) xOut(pOutCtx, "");
+
+  while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){
+    int iSelectid = sqlite3_column_int(pExplain, 0);
+    int iOrder = sqlite3_column_int(pExplain, 1);
+    int iFrom = sqlite3_column_int(pExplain, 2);
+    const char *zDetail = (const char*)sqlite3_column_text(pExplain, 3);
+    char *zOut;
+
+    zOut = sqlite3_mprintf("%d|%d|%d|%s", iSelectid, iOrder, iFrom, zDetail);
+    if( zOut==0 ){
+      rc = SQLITE_NOMEM;
+    }else{
+      xOut(pOutCtx, zOut);
+      sqlite3_free(zOut);
+    }
+  }
+
+ find_indexes_out:
+  rc2 = sqlite3_finalize(pExplain);
+  if( rc==SQLITE_OK ) rc = rc2;
+  rc2 = sqlite3_finalize(pSelect);
+  if( rc==SQLITE_OK ) rc = rc2;
+
+  return rc;
 }
 
 /*
@@ -873,8 +751,6 @@ int shellIndexesCommand(
   IdxContext ctx;
   sqlite3_stmt *pStmt = 0;        /* Statement compiled from zSql */
 
-  rc = registerPragmaVtabs(db);
-  if( rc ) return rc;
   memset(&ctx, 0, sizeof(IdxContext));
 
   /* Open an in-memory database to work with. The main in-memory 
@@ -887,52 +763,44 @@ int shellIndexesCommand(
         "ATTACH ':memory:' AS aux;"
         "CREATE TABLE aux.depmask(mask PRIMARY KEY) WITHOUT ROWID;"
         "INSERT INTO aux.depmask VALUES(0);"
-        , 0, 0, 0
+        , 0, 0, pzErrmsg
     );
   }
 
   /* Prepare an INSERT statement for writing to aux.depmask */
   if( rc==SQLITE_OK ){
-    rc = sqlite3_prepare_v2(dbm, 
-        "INSERT OR IGNORE INTO depmask SELECT mask | ?1 FROM depmask;", -1,
-        &ctx.pInsertMask, 0
+    rc = idxPrepareStmt(dbm, &ctx.pInsertMask, pzErrmsg,
+        "INSERT OR IGNORE INTO depmask SELECT mask | ?1 FROM depmask;"
     );
   }
 
-  if( rc!=SQLITE_OK ){
-    idxDatabaseError(dbm, pzErrmsg);
-    goto indexes_out;
-  }
-
   /* Analyze the SELECT statement in zSql. */
-  ctx.dbm = dbm;
-  sqlite3_db_config(db, SQLITE_DBCONFIG_WHEREINFO, idxWhereInfo, (void*)&ctx);
-  rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0);
-  sqlite3_db_config(db, SQLITE_DBCONFIG_WHEREINFO, (void*)0, (void*)0);
-
-  if( rc!=SQLITE_OK ){
-    idxDatabaseError(db, pzErrmsg);
-    goto indexes_out;
+  if( rc==SQLITE_OK ){
+    ctx.dbm = dbm;
+    sqlite3_db_config(db, SQLITE_DBCONFIG_WHEREINFO, idxWhereInfo, (void*)&ctx);
+    rc = idxPrepareStmt(db, &pStmt, pzErrmsg, zSql);
+    sqlite3_db_config(db, SQLITE_DBCONFIG_WHEREINFO, (void*)0, (void*)0);
+    sqlite3_finalize(pStmt);
   }
 
   /* Create tables within the main in-memory database. These tables
   ** have the same names, columns and declared types as the tables in
   ** the user database. All constraints except for PRIMARY KEY are
   ** removed. */
-  rc = idxCreateTables(db, dbm, ctx.pScan, pzErrmsg);
-  if( rc!=SQLITE_OK ){
-    goto indexes_out;
+  if( rc==SQLITE_OK ){
+    rc = idxCreateTables(db, dbm, ctx.pScan, pzErrmsg);
   }
 
   /* Create candidate indexes within the in-memory database file */
-  rc = idxCreateCandidates(dbm, ctx.pScan, pzErrmsg);
-  if( rc!=SQLITE_OK ){
-    goto indexes_out;
+  if( rc==SQLITE_OK ){
+    rc = idxCreateCandidates(dbm, ctx.pScan, pzErrmsg);
   }
 
-  rc = idxFindIndexes(dbm, zSql, xOut, pOutCtx, pzErrmsg);
+  /* Create candidate indexes within the in-memory database file */
+  if( rc==SQLITE_OK ){
+    rc = idxFindIndexes(dbm, zSql, xOut, pOutCtx, pzErrmsg);
+  }
 
- indexes_out:
   idxScanFree(ctx.pScan);
   sqlite3_close(dbm);
   return rc;

From 1c93844dd30973b13907a837876e7aa161e4bbed Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 17 Feb 2016 20:06:12 +0000
Subject: [PATCH 012/488] Schemalint changes: Avoid creating candidate indexes
 if a compatible index exists. Do not quote identifiers that do not require
 it.

FossilOrigin-Name: cf0f7eeb4f6490b1e3f05b45e83b87cd64640846
---
 manifest            |  12 +--
 manifest.uuid       |   2 +-
 src/shell_indexes.c | 181 ++++++++++++++++++++++++++++++++------------
 3 files changed, 138 insertions(+), 57 deletions(-)

diff --git a/manifest b/manifest
index f1eaa8735a..ff57fedd26 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sfurther\sissues\sin\sschemalint.
-D 2016-02-16T18:37:37.844
+C Schemalint\schanges:\sAvoid\screating\scandidate\sindexes\sif\sa\scompatible\sindex\sexists.\sDo\snot\squote\sidentifiers\sthat\sdo\snot\srequire\sit.
+D 2016-02-17T20:06:12.566
 F Makefile.in dac2776c84e0d533b158a9af6e57e05c4a6b19f3
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc b0493f10caddb8adf992a4e6f1943141fc7c6816
@@ -350,7 +350,7 @@ F src/resolve.c 9f7ce3a3c087afb7597b7c916c99126ff3f12f0c
 F src/rowset.c 9fe4b3ad7cc00944386bb600233d8f523de07a6e
 F src/select.c ff80004a9a6ece891a8d9327a88e7b6e2588ee6d
 F src/shell.c 2cde87e03712204231167c4a6c61b0eb5129e105
-F src/shell_indexes.c 27d3b064078066c9a284a93149c4a151821f55bf
+F src/shell_indexes.c 6cc207072469f1ded8c3bb9de1d4b6590a28abb8
 F src/sqlite.h.in c7db059d3b810b70b83d9ed1436fa813eba22462
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h dfbe62ffd95b99afe2140d8c35b180d11924072d
@@ -1430,7 +1430,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 58d4cf26e15f90463148bec63d6ab514ffbbae60
-R f221e3cac7f52c2c660e427ee329aa03
+P 73a7f010937828c5195a198604f976e8458cef73
+R 3a9a4af7276c9c00e5b2666c0bfe324e
 U dan
-Z 8b18cd934aae857f866e3f00c12cba6f
+Z 82976e1002963ffa4a1c502f357a9ae9
diff --git a/manifest.uuid b/manifest.uuid
index 4b3aa263c2..2e9ca33ba0 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-73a7f010937828c5195a198604f976e8458cef73
\ No newline at end of file
+cf0f7eeb4f6490b1e3f05b45e83b87cd64640846
\ No newline at end of file
diff --git a/src/shell_indexes.c b/src/shell_indexes.c
index 33f872efd7..54b59cd421 100644
--- a/src/shell_indexes.c
+++ b/src/shell_indexes.c
@@ -33,6 +33,7 @@ struct IdxConstraint {
   int bRange;                     /* True for range, false for eq */
   int iCol;                       /* Constrained table column */
   i64 depmask;                    /* Dependency mask */
+  int bFlag;                      /* Used by idxFindCompatible() */
   IdxConstraint *pNext;           /* Next constraint in pEq or pRange list */
   IdxConstraint *pLink;           /* See above */
 };
@@ -413,53 +414,20 @@ static int idxCreateTables(
   int rc = SQLITE_OK;
   IdxScan *pIter;
   for(pIter=pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
-    int nPk = 0;
-    char *zCols = 0;
-    char *zPk = 0;
-    char *zCreate = 0;
-    int iCol;
-
     rc = idxGetTableInfo(db, pIter, pzErrmsg);
-
-    for(iCol=0; rc==SQLITE_OK && iCol<pIter->pTable->nCol; iCol++){
-      IdxColumn *pCol = &pIter->pTable->aCol[iCol];
-      if( pCol->iPk>nPk ) nPk = pCol->iPk;
-      zCols = sqlite3_mprintf("%z%s%Q", zCols, (zCols?", ":""), pCol->zName);
-      if( zCols==0 ) rc = SQLITE_NOMEM;
-    }
-
-    for(iCol=1; rc==SQLITE_OK && iCol<=nPk; iCol++){
-      int j;
-      for(j=0; j<pIter->pTable->nCol; j++){
-        IdxColumn *pCol = &pIter->pTable->aCol[j];
-        if( pCol->iPk==iCol ){
-          zPk = sqlite3_mprintf("%z%s%Q", zPk, (zPk?", ":""), pCol->zName);
-          if( zPk==0 ) rc = SQLITE_NOMEM;
-          break;
-        }
-      }
-    }
-
     if( rc==SQLITE_OK ){
-      if( zPk ){
-        zCreate = sqlite3_mprintf("CREATE TABLE %Q(%s, PRIMARY KEY(%s))",
-            pIter->zTable, zCols, zPk
-        );
-      }else{
-        zCreate = sqlite3_mprintf("CREATE TABLE %Q(%s)", pIter->zTable, zCols);
+      int rc2;
+      sqlite3_stmt *pSql = 0;
+      rc = idxPrintfPrepareStmt(db, &pSql, pzErrmsg, 
+          "SELECT sql FROM sqlite_master WHERE tbl_name = %Q", pIter->zTable
+      );
+      while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
+        const char *zSql = (const char*)sqlite3_column_text(pSql, 0);
+        rc = sqlite3_exec(dbm, zSql, 0, 0, pzErrmsg);
       }
-      if( zCreate==0 ) rc = SQLITE_NOMEM;
+      rc2 = sqlite3_finalize(pSql);
+      if( rc==SQLITE_OK ) rc = rc2;
     }
-
-    if( rc==SQLITE_OK ){
-#if 0
-      printf("/* %s */\n", zCreate);
-#endif
-      rc = sqlite3_exec(dbm, zCreate, 0, 0, pzErrmsg);
-    }
-    sqlite3_free(zCols);
-    sqlite3_free(zPk);
-    sqlite3_free(zCreate);
   }
   return rc;
 }
@@ -501,6 +469,20 @@ static char *idxAppendText(int *pRc, char *zIn, const char *zFmt, ...){
   return zRet;
 }
 
+static int idxIdentifierRequiresQuotes(const char *zId){
+  int i;
+  for(i=0; zId[i]; i++){
+    if( !(zId[i]=='_')
+     && !(zId[i]>='0' && zId[i]<='9')
+     && !(zId[i]>='a' && zId[i]<='z')
+     && !(zId[i]>='A' && zId[i]<='Z')
+    ){
+      return 1;
+    }
+  }
+  return 0;
+}
+
 static char *idxAppendColDefn(
   int *pRc, 
   char *zIn, 
@@ -510,13 +492,100 @@ static char *idxAppendColDefn(
   char *zRet = zIn;
   IdxColumn *p = &pTab->aCol[pCons->iCol];
   if( zRet ) zRet = idxAppendText(pRc, zRet, ", ");
-  zRet = idxAppendText(pRc, zRet, "%Q", p->zName);
+
+  if( idxIdentifierRequiresQuotes(p->zName) ){
+    zRet = idxAppendText(pRc, zRet, "%Q", p->zName);
+  }else{
+    zRet = idxAppendText(pRc, zRet, "%s", p->zName);
+  }
+
   if( sqlite3_stricmp(p->zColl, pCons->zColl) ){
-    zRet = idxAppendText(pRc, zRet, " COLLATE %Q", pCons->zColl);
+    if( idxIdentifierRequiresQuotes(pCons->zColl) ){
+      zRet = idxAppendText(pRc, zRet, " COLLATE %Q", pCons->zColl);
+    }else{
+      zRet = idxAppendText(pRc, zRet, " COLLATE %s", pCons->zColl);
+    }
   }
   return zRet;
 }
 
+/*
+** Search database dbm for an index compatible with the one idxCreateFromCons()
+** would create from arguments pScan, pEq and pTail. If no error occurs and 
+** such an index is found, return non-zero. Or, if no such index is found,
+** return zero.
+**
+** If an error occurs, set *pRc to an SQLite error code and return zero.
+*/
+static int idxFindCompatible(
+  int *pRc,                       /* OUT: Error code */
+  sqlite3* dbm,                   /* Database to search */
+  IdxScan *pScan,                 /* Scan for table to search for index on */
+  IdxConstraint *pEq,             /* List of == constraints */
+  IdxConstraint *pTail            /* List of range constraints */
+){
+  const char *zTbl = pScan->zTable;
+  sqlite3_stmt *pIdxList = 0;
+  IdxConstraint *pIter;
+  int nEq = 0;                    /* Number of elements in pEq */
+  int rc, rc2;
+
+  /* Count the elements in list pEq */
+  for(pIter=pEq; pIter; pIter=pIter->pNext) nEq++;
+
+  rc = idxPrintfPrepareStmt(dbm, &pIdxList, 0, "PRAGMA index_list=%Q", zTbl);
+  while( rc==SQLITE_OK && sqlite3_step(pIdxList)==SQLITE_ROW ){
+    int bMatch = 1;
+    IdxConstraint *pT = pTail;
+    sqlite3_stmt *pInfo = 0;
+    const char *zIdx = (const char*)sqlite3_column_text(pIdxList, 1);
+
+    /* Zero the IdxConstraint.bFlag values in the pEq list */
+    for(pIter=pEq; pIter; pIter=pIter->pNext) pIter->bFlag = 0;
+
+    rc = idxPrintfPrepareStmt(dbm, &pInfo, 0, "PRAGMA index_xInfo=%Q", zIdx);
+    while( rc==SQLITE_OK && sqlite3_step(pInfo)==SQLITE_ROW ){
+      int iIdx = sqlite3_column_int(pInfo, 0);
+      int iCol = sqlite3_column_int(pInfo, 1);
+      const char *zColl = (const char*)sqlite3_column_text(pInfo, 4);
+
+      if( iIdx<nEq ){
+        for(pIter=pEq; pIter; pIter=pIter->pNext){
+          if( pIter->bFlag ) continue;
+          if( pIter->iCol!=iCol ) continue;
+          if( sqlite3_stricmp(pIter->zColl, zColl) ) continue;
+          pIter->bFlag = 1;
+          break;
+        }
+        if( pIter==0 ){
+          bMatch = 0;
+          break;
+        }
+      }else{
+        if( pT ){
+          if( pT->iCol!=iCol || sqlite3_stricmp(pT->zColl, zColl) ){
+            bMatch = 0;
+            break;
+          }
+          pT = pT->pLink;
+        }
+      }
+    }
+    rc2 = sqlite3_finalize(pInfo);
+    if( rc==SQLITE_OK ) rc = rc2;
+
+    if( rc==SQLITE_OK && bMatch ){
+      sqlite3_finalize(pIdxList);
+      return 1;
+    }
+  }
+  rc2 = sqlite3_finalize(pIdxList);
+  if( rc==SQLITE_OK ) rc = rc2;
+
+  *pRc = rc;
+  return 0;
+}
+
 static int idxCreateFromCons(
   sqlite3 *dbm,
   IdxScan *pScan,
@@ -524,12 +593,13 @@ static int idxCreateFromCons(
   IdxConstraint *pTail
 ){
   int rc = SQLITE_OK;
-  if( pEq || pTail ){
+  if( (pEq || pTail) && 0==idxFindCompatible(&rc, dbm, pScan, pEq, pTail) ){
     IdxTable *pTab = pScan->pTable;
     char *zCols = 0;
     char *zIdx = 0;
     IdxConstraint *pCons;
     int h = 0;
+    const char *zFmt;
 
     for(pCons=pEq; pCons; pCons=pCons->pLink){
       zCols = idxAppendColDefn(&rc, zCols, pTab, pCons);
@@ -545,9 +615,12 @@ static int idxCreateFromCons(
         h += ((h<<3) + zCols[i]);
       }
 
-      zIdx = sqlite3_mprintf("CREATE INDEX IF NOT EXISTS "
-          "'%q_idx_%08x' ON %Q(%s)", pScan->zTable, h, pScan->zTable, zCols
-      );
+      if( idxIdentifierRequiresQuotes(pScan->zTable) ){
+        zFmt = "CREATE INDEX '%q_idx_%08x' ON %Q(%s)";
+      }else{
+        zFmt = "CREATE INDEX %s_idx_%08x ON %s(%s)";
+      }
+      zIdx = sqlite3_mprintf(zFmt, pScan->zTable, h, pScan->zTable, zCols);
       if( !zIdx ){
         rc = SQLITE_NOMEM;
       }else{
@@ -665,6 +738,12 @@ static int idxCreateCandidates(
 }
 
 static void idxScanFree(IdxScan *pScan){
+  IdxScan *pIter;
+  IdxScan *pNext;
+  for(pIter=pScan; pIter; pIter=pNext){
+    pNext = pIter->pNextScan;
+
+  }
 }
 
 int idxFindIndexes(
@@ -796,12 +875,14 @@ int shellIndexesCommand(
     rc = idxCreateCandidates(dbm, ctx.pScan, pzErrmsg);
   }
 
-  /* Create candidate indexes within the in-memory database file */
+  /* Figure out which of the candidate indexes are preferred by the query
+  ** planner and report the results to the user.  */
   if( rc==SQLITE_OK ){
     rc = idxFindIndexes(dbm, zSql, xOut, pOutCtx, pzErrmsg);
   }
 
   idxScanFree(ctx.pScan);
+  sqlite3_finalize(ctx.pInsertMask);
   sqlite3_close(dbm);
   return rc;
 }

From b12dfd01259fc16b7f0ac522aeec185daf7c128f Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 18 Feb 2016 19:10:02 +0000
Subject: [PATCH 013/488] Have the schemalint output distinguish between
 existing and recommended indexes.

FossilOrigin-Name: 4ab3df25f1fee7c8fea19d0c64b3e0e4d3b9c3cf
---
 manifest            | 12 +++----
 manifest.uuid       |  2 +-
 src/shell_indexes.c | 78 ++++++++++++++++++++++++++++++---------------
 3 files changed, 59 insertions(+), 33 deletions(-)

diff --git a/manifest b/manifest
index ff57fedd26..c8c283d441 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Schemalint\schanges:\sAvoid\screating\scandidate\sindexes\sif\sa\scompatible\sindex\sexists.\sDo\snot\squote\sidentifiers\sthat\sdo\snot\srequire\sit.
-D 2016-02-17T20:06:12.566
+C Have\sthe\sschemalint\soutput\sdistinguish\sbetween\sexisting\sand\srecommended\sindexes.
+D 2016-02-18T19:10:02.440
 F Makefile.in dac2776c84e0d533b158a9af6e57e05c4a6b19f3
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc b0493f10caddb8adf992a4e6f1943141fc7c6816
@@ -350,7 +350,7 @@ F src/resolve.c 9f7ce3a3c087afb7597b7c916c99126ff3f12f0c
 F src/rowset.c 9fe4b3ad7cc00944386bb600233d8f523de07a6e
 F src/select.c ff80004a9a6ece891a8d9327a88e7b6e2588ee6d
 F src/shell.c 2cde87e03712204231167c4a6c61b0eb5129e105
-F src/shell_indexes.c 6cc207072469f1ded8c3bb9de1d4b6590a28abb8
+F src/shell_indexes.c 277eb75b8cfb3c2bcf76e062baaa419779f824e7
 F src/sqlite.h.in c7db059d3b810b70b83d9ed1436fa813eba22462
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h dfbe62ffd95b99afe2140d8c35b180d11924072d
@@ -1430,7 +1430,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 73a7f010937828c5195a198604f976e8458cef73
-R 3a9a4af7276c9c00e5b2666c0bfe324e
+P cf0f7eeb4f6490b1e3f05b45e83b87cd64640846
+R f54b8a1f9b5e206798931d383b1eb3a6
 U dan
-Z 82976e1002963ffa4a1c502f357a9ae9
+Z 5c18fceb59d8462f5aec79ee219a18a2
diff --git a/manifest.uuid b/manifest.uuid
index 2e9ca33ba0..9ede1c42e8 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-cf0f7eeb4f6490b1e3f05b45e83b87cd64640846
\ No newline at end of file
+4ab3df25f1fee7c8fea19d0c64b3e0e4d3b9c3cf
\ No newline at end of file
diff --git a/src/shell_indexes.c b/src/shell_indexes.c
index 54b59cd421..755fff6403 100644
--- a/src/shell_indexes.c
+++ b/src/shell_indexes.c
@@ -98,12 +98,14 @@ struct IdxScan {
 ** Context object passed to idxWhereInfo()
 */
 struct IdxContext {
+  char **pzErrmsg;
   IdxWhere *pCurrent;             /* Current where clause */
   int rc;                         /* Error code (if error has occurred) */
   IdxScan *pScan;                 /* List of scan objects */
   sqlite3 *dbm;                   /* In-memory db for this analysis */
   sqlite3 *db;                    /* User database under analysis */
   sqlite3_stmt *pInsertMask;      /* To write to aux.depmask */
+  i64 iIdxRowid;                  /* Rowid of first index created */
 };
 
 /*
@@ -587,11 +589,12 @@ static int idxFindCompatible(
 }
 
 static int idxCreateFromCons(
-  sqlite3 *dbm,
+  IdxContext *pCtx,
   IdxScan *pScan,
   IdxConstraint *pEq, 
   IdxConstraint *pTail
 ){
+  sqlite3 *dbm = pCtx->dbm;
   int rc = SQLITE_OK;
   if( (pEq || pTail) && 0==idxFindCompatible(&rc, dbm, pScan, pEq, pTail) ){
     IdxTable *pTab = pScan->pTable;
@@ -608,8 +611,8 @@ static int idxCreateFromCons(
       zCols = idxAppendColDefn(&rc, zCols, pTab, pCons);
     }
 
-    /* Hash the list of columns to come up with a name for the index */
     if( rc==SQLITE_OK ){
+      /* Hash the list of columns to come up with a name for the index */
       int i;
       for(i=0; zCols[i]; i++){
         h += ((h<<3) + zCols[i]);
@@ -630,6 +633,18 @@ static int idxCreateFromCons(
 #endif
       }
     }
+    if( rc==SQLITE_OK && pCtx->iIdxRowid==0 ){
+      int rc2;
+      sqlite3_stmt *pLast = 0;
+      rc = idxPrepareStmt(dbm, &pLast, pCtx->pzErrmsg, 
+          "SELECT max(rowid) FROM sqlite_master"
+      );
+      if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLast) ){
+        pCtx->iIdxRowid = sqlite3_column_int64(pLast, 0);
+      }
+      rc2 = sqlite3_finalize(pLast);
+      if( rc==SQLITE_OK ) rc = rc2;
+    }
 
     sqlite3_free(zIdx);
     sqlite3_free(zCols);
@@ -638,11 +653,11 @@ static int idxCreateFromCons(
 }
 
 static int idxCreateFromWhere(
-    sqlite3*, i64, IdxScan*, IdxWhere*, IdxConstraint*, IdxConstraint*
+    IdxContext*, i64, IdxScan*, IdxWhere*, IdxConstraint*, IdxConstraint*
 );
 
 static int idxCreateForeachOr(
-  sqlite3 *dbm, 
+  IdxContext *pCtx, 
   i64 mask,                       /* Consider only these constraints */
   IdxScan *pScan,                 /* Create indexes for this scan */
   IdxWhere *pWhere,               /* Read constraints from here */
@@ -653,22 +668,23 @@ static int idxCreateForeachOr(
   IdxWhere *p1;
   IdxWhere *p2;
   for(p1=pWhere->pOr; p1 && rc==SQLITE_OK; p1=p1->pNextOr){
-    rc = idxCreateFromWhere(dbm, mask, pScan, p1, pEq, pTail);
+    rc = idxCreateFromWhere(pCtx, mask, pScan, p1, pEq, pTail);
     for(p2=p1->pSibling; p2 && rc==SQLITE_OK; p2=p2->pSibling){
-      rc = idxCreateFromWhere(dbm, mask, pScan, p2, pEq, pTail);
+      rc = idxCreateFromWhere(pCtx, mask, pScan, p2, pEq, pTail);
     }
   }
   return rc;
 }
 
 static int idxCreateFromWhere(
-  sqlite3 *dbm, 
+  IdxContext *pCtx, 
   i64 mask,                       /* Consider only these constraints */
   IdxScan *pScan,                 /* Create indexes for this scan */
   IdxWhere *pWhere,               /* Read constraints from here */
   IdxConstraint *pEq,             /* == constraints for inclusion */
   IdxConstraint *pTail            /* range/ORDER BY constraints for inclusion */
 ){
+  sqlite3 *dbm = pCtx->dbm;
   IdxConstraint *p1 = pEq;
   IdxConstraint *pCon;
   int rc;
@@ -683,9 +699,9 @@ static int idxCreateFromWhere(
 
   /* Create an index using the == constraints collected above. And the
   ** range constraint/ORDER BY terms passed in by the caller, if any. */
-  rc = idxCreateFromCons(dbm, pScan, p1, pTail);
+  rc = idxCreateFromCons(pCtx, pScan, p1, pTail);
   if( rc==SQLITE_OK ){
-    rc = idxCreateForeachOr(dbm, mask, pScan, pWhere, p1, pTail);
+    rc = idxCreateForeachOr(pCtx, mask, pScan, pWhere, p1, pTail);
   }
 
   /* If no range/ORDER BY passed by the caller, create a version of the
@@ -694,9 +710,9 @@ static int idxCreateFromWhere(
     for(pCon=pWhere->pRange; rc==SQLITE_OK && pCon; pCon=pCon->pNext){
       assert( pCon->pLink==0 );
       if( (mask & pCon->depmask)==pCon->depmask ){
-        rc = idxCreateFromCons(dbm, pScan, p1, pCon);
+        rc = idxCreateFromCons(pCtx, pScan, p1, pCon);
         if( rc==SQLITE_OK ){
-          rc = idxCreateForeachOr(dbm, mask, pScan, pWhere, p1, pCon);
+          rc = idxCreateForeachOr(pCtx, mask, pScan, pWhere, p1, pCon);
         }
       }
     }
@@ -709,25 +725,24 @@ static int idxCreateFromWhere(
 ** Create candidate indexes in database [dbm] based on the data in 
 ** linked-list pScan.
 */
-static int idxCreateCandidates(
-  sqlite3 *dbm,
-  IdxScan *pScan,
-  char **pzErrmsg
-){
+static int idxCreateCandidates(IdxContext *pCtx){
+  sqlite3 *dbm = pCtx->dbm;
   int rc2;
   int rc = SQLITE_OK;
   sqlite3_stmt *pDepmask;         /* Foreach depmask */
   IdxScan *pIter;
 
-  rc = idxPrepareStmt(dbm, &pDepmask, pzErrmsg, "SELECT mask FROM depmask");
+  rc = idxPrepareStmt(pCtx->dbm, &pDepmask, pCtx->pzErrmsg, 
+      "SELECT mask FROM depmask"
+  );
 
-  for(pIter=pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
+  for(pIter=pCtx->pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
     IdxWhere *pWhere = &pIter->where;
     while( SQLITE_ROW==sqlite3_step(pDepmask) && rc==SQLITE_OK ){
       i64 mask = sqlite3_column_int64(pDepmask, 0);
-      rc = idxCreateFromWhere(dbm, mask, pIter, pWhere, 0, 0);
+      rc = idxCreateFromWhere(pCtx, mask, pIter, pWhere, 0, 0);
       if( rc==SQLITE_OK && pIter->pOrder ){
-        rc = idxCreateFromWhere(dbm, mask, pIter, pWhere, 0, pIter->pOrder);
+        rc = idxCreateFromWhere(pCtx, mask, pIter, pWhere, 0, pIter->pOrder);
       }
     }
   }
@@ -747,20 +762,22 @@ static void idxScanFree(IdxScan *pScan){
 }
 
 int idxFindIndexes(
-  sqlite3 *dbm,                        /* Database handle */
+  IdxContext *pCtx,
   const char *zSql,                    /* SQL to find indexes for */
   void (*xOut)(void*, const char*),    /* Output callback */
   void *pOutCtx,                       /* Context for xOut() */
   char **pzErr                         /* OUT: Error message (sqlite3_malloc) */
 ){
+  sqlite3 *dbm = pCtx->dbm;
   sqlite3_stmt *pExplain = 0;
   sqlite3_stmt *pSelect = 0;
   int rc, rc2;
+  int bFound = 0;
 
   rc = idxPrintfPrepareStmt(dbm, &pExplain, pzErr,"EXPLAIN QUERY PLAN %s",zSql);
   if( rc==SQLITE_OK ){
     rc = idxPrepareStmt(dbm, &pSelect, pzErr, 
-        "SELECT sql FROM sqlite_master WHERE name = ?"
+        "SELECT rowid, sql FROM sqlite_master WHERE name = ?"
     );
   }
 
@@ -776,7 +793,12 @@ int idxFindIndexes(
         while( zIdx[nIdx]!='\0' && zIdx[nIdx]!=' ' ) nIdx++;
         sqlite3_bind_text(pSelect, 1, zIdx, nIdx, SQLITE_STATIC);
         if( SQLITE_ROW==sqlite3_step(pSelect) ){
-          xOut(pOutCtx, sqlite3_column_text(pSelect, 0));
+          i64 iRowid = sqlite3_column_int64(pSelect, 0);
+          const char *zSql = (const char*)sqlite3_column_text(pSelect, 1);
+          if( iRowid>=pCtx->iIdxRowid ){
+            xOut(pOutCtx, zSql);
+            bFound = 1;
+          }
         }
         rc = sqlite3_reset(pSelect);
         break;
@@ -785,7 +807,10 @@ int idxFindIndexes(
   }
   rc2 = sqlite3_reset(pExplain);
   if( rc==SQLITE_OK ) rc = rc2;
-  if( rc==SQLITE_OK ) xOut(pOutCtx, "");
+  if( rc==SQLITE_OK ){
+    if( bFound==0 ) xOut(pOutCtx, "(no new indexes)");
+    xOut(pOutCtx, "");
+  }
 
   while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){
     int iSelectid = sqlite3_column_int(pExplain, 0);
@@ -841,6 +866,7 @@ int shellIndexesCommand(
     rc = sqlite3_exec(dbm, 
         "ATTACH ':memory:' AS aux;"
         "CREATE TABLE aux.depmask(mask PRIMARY KEY) WITHOUT ROWID;"
+        "CREATE TABLE aux.indexes(name PRIMARY KEY) WITHOUT ROWID;"
         "INSERT INTO aux.depmask VALUES(0);"
         , 0, 0, pzErrmsg
     );
@@ -872,13 +898,13 @@ int shellIndexesCommand(
 
   /* Create candidate indexes within the in-memory database file */
   if( rc==SQLITE_OK ){
-    rc = idxCreateCandidates(dbm, ctx.pScan, pzErrmsg);
+    rc = idxCreateCandidates(&ctx);
   }
 
   /* Figure out which of the candidate indexes are preferred by the query
   ** planner and report the results to the user.  */
   if( rc==SQLITE_OK ){
-    rc = idxFindIndexes(dbm, zSql, xOut, pOutCtx, pzErrmsg);
+    rc = idxFindIndexes(&ctx, zSql, xOut, pOutCtx, pzErrmsg);
   }
 
   idxScanFree(ctx.pScan);

From 850493ac1fd888939dc92bde3418d53d38e9fcbe Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 19 Feb 2016 07:53:43 +0000
Subject: [PATCH 014/488] Fix a couple of bugs in the schemalint code.

FossilOrigin-Name: 02fbf699c07286f842d9617755f071b0fffc5d40
---
 manifest            | 12 ++++++------
 manifest.uuid       |  2 +-
 src/shell_indexes.c | 47 +++++++++++++++++++++++++++++++++------------
 3 files changed, 42 insertions(+), 19 deletions(-)

diff --git a/manifest b/manifest
index c8c283d441..549a6f5bf1 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Have\sthe\sschemalint\soutput\sdistinguish\sbetween\sexisting\sand\srecommended\sindexes.
-D 2016-02-18T19:10:02.440
+C Fix\sa\scouple\sof\sbugs\sin\sthe\sschemalint\scode.
+D 2016-02-19T07:53:43.883
 F Makefile.in dac2776c84e0d533b158a9af6e57e05c4a6b19f3
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc b0493f10caddb8adf992a4e6f1943141fc7c6816
@@ -350,7 +350,7 @@ F src/resolve.c 9f7ce3a3c087afb7597b7c916c99126ff3f12f0c
 F src/rowset.c 9fe4b3ad7cc00944386bb600233d8f523de07a6e
 F src/select.c ff80004a9a6ece891a8d9327a88e7b6e2588ee6d
 F src/shell.c 2cde87e03712204231167c4a6c61b0eb5129e105
-F src/shell_indexes.c 277eb75b8cfb3c2bcf76e062baaa419779f824e7
+F src/shell_indexes.c c0099c01c7af01038ab4315621814df535b700a3
 F src/sqlite.h.in c7db059d3b810b70b83d9ed1436fa813eba22462
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h dfbe62ffd95b99afe2140d8c35b180d11924072d
@@ -1430,7 +1430,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P cf0f7eeb4f6490b1e3f05b45e83b87cd64640846
-R f54b8a1f9b5e206798931d383b1eb3a6
+P 4ab3df25f1fee7c8fea19d0c64b3e0e4d3b9c3cf
+R 0c9cae22954d1f32b48770c565f201e2
 U dan
-Z 5c18fceb59d8462f5aec79ee219a18a2
+Z cecc60af647c055f880ee4bdf2018855
diff --git a/manifest.uuid b/manifest.uuid
index 9ede1c42e8..675cfe73ee 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4ab3df25f1fee7c8fea19d0c64b3e0e4d3b9c3cf
\ No newline at end of file
+02fbf699c07286f842d9617755f071b0fffc5d40
\ No newline at end of file
diff --git a/src/shell_indexes.c b/src/shell_indexes.c
index 755fff6403..cd8a5e7288 100644
--- a/src/shell_indexes.c
+++ b/src/shell_indexes.c
@@ -95,7 +95,7 @@ struct IdxScan {
 };
 
 /*
-** Context object passed to idxWhereInfo()
+** Context object passed to idxWhereInfo() and other functions.
 */
 struct IdxContext {
   char **pzErrmsg;
@@ -454,7 +454,7 @@ static char *idxAppendText(int *pRc, char *zIn, const char *zFmt, ...){
     zAppend = sqlite3_vmprintf(zFmt, ap);
     if( zAppend ){
       nAppend = strlen(zAppend);
-      zRet = (char*)sqlite3_malloc(nIn + nAppend);
+      zRet = (char*)sqlite3_malloc(nIn + nAppend + 1);
     }
     if( zAppend && zRet ){
       memcpy(zRet, zIn, nIn);
@@ -532,8 +532,9 @@ static int idxFindCompatible(
   int nEq = 0;                    /* Number of elements in pEq */
   int rc, rc2;
 
+
   /* Count the elements in list pEq */
-  for(pIter=pEq; pIter; pIter=pIter->pNext) nEq++;
+  for(pIter=pEq; pIter; pIter=pIter->pLink) nEq++;
 
   rc = idxPrintfPrepareStmt(dbm, &pIdxList, 0, "PRAGMA index_list=%Q", zTbl);
   while( rc==SQLITE_OK && sqlite3_step(pIdxList)==SQLITE_ROW ){
@@ -543,7 +544,7 @@ static int idxFindCompatible(
     const char *zIdx = (const char*)sqlite3_column_text(pIdxList, 1);
 
     /* Zero the IdxConstraint.bFlag values in the pEq list */
-    for(pIter=pEq; pIter; pIter=pIter->pNext) pIter->bFlag = 0;
+    for(pIter=pEq; pIter; pIter=pIter->pLink) pIter->bFlag = 0;
 
     rc = idxPrintfPrepareStmt(dbm, &pInfo, 0, "PRAGMA index_xInfo=%Q", zIdx);
     while( rc==SQLITE_OK && sqlite3_step(pInfo)==SQLITE_ROW ){
@@ -552,7 +553,7 @@ static int idxFindCompatible(
       const char *zColl = (const char*)sqlite3_column_text(pInfo, 4);
 
       if( iIdx<nEq ){
-        for(pIter=pEq; pIter; pIter=pIter->pNext){
+        for(pIter=pEq; pIter; pIter=pIter->pLink){
           if( pIter->bFlag ) continue;
           if( pIter->iCol!=iCol ) continue;
           if( sqlite3_stricmp(pIter->zColl, zColl) ) continue;
@@ -627,10 +628,8 @@ static int idxCreateFromCons(
       if( !zIdx ){
         rc = SQLITE_NOMEM;
       }else{
-        rc = sqlite3_exec(dbm, zIdx, 0, 0, 0);
-#if 0
-        printf("/* %s */\n", zIdx);
-#endif
+        rc = sqlite3_exec(dbm, zIdx, 0, 0, pCtx->pzErrmsg);
+        /* printf("%s\n", zIdx); */
       }
     }
     if( rc==SQLITE_OK && pCtx->iIdxRowid==0 ){
@@ -676,6 +675,18 @@ static int idxCreateForeachOr(
   return rc;
 }
 
+/*
+** Return true if list pList (linked by IdxConstraint.pLink) contains
+** a constraint compatible with *p. Otherwise return false.
+*/
+static int idxFindConstraint(IdxConstraint *pList, IdxConstraint *p){
+  IdxConstraint *pCmp;
+  for(pCmp=pList; pCmp; pCmp=pCmp->pLink){
+    if( p->iCol==pCmp->iCol ) return 1;
+  }
+  return 0;
+}
+
 static int idxCreateFromWhere(
   IdxContext *pCtx, 
   i64 mask,                       /* Consider only these constraints */
@@ -691,7 +702,10 @@ static int idxCreateFromWhere(
 
   /* Gather up all the == constraints that match the mask. */
   for(pCon=pWhere->pEq; pCon; pCon=pCon->pNext){
-    if( (mask & pCon->depmask)==pCon->depmask ){
+    if( (mask & pCon->depmask)==pCon->depmask 
+     && idxFindConstraint(p1, pCon)==0
+     && idxFindConstraint(pTail, pCon)==0
+    ){
       pCon->pLink = p1;
       p1 = pCon;
     }
@@ -709,7 +723,10 @@ static int idxCreateFromWhere(
   if( pTail==0 ){
     for(pCon=pWhere->pRange; rc==SQLITE_OK && pCon; pCon=pCon->pNext){
       assert( pCon->pLink==0 );
-      if( (mask & pCon->depmask)==pCon->depmask ){
+      if( (mask & pCon->depmask)==pCon->depmask
+        && idxFindConstraint(pEq, pCon)==0
+        && idxFindConstraint(pTail, pCon)==0
+      ){
         rc = idxCreateFromCons(pCtx, pScan, p1, pCon);
         if( rc==SQLITE_OK ){
           rc = idxCreateForeachOr(pCtx, mask, pScan, pWhere, p1, pCon);
@@ -787,9 +804,14 @@ int idxFindIndexes(
     int nDetail = strlen(zDetail);
 
     for(i=0; i<nDetail; i++){
+      const char *zIdx = 0;
       if( memcmp(&zDetail[i], " USING INDEX ", 13)==0 ){
+        zIdx = &zDetail[i+13];
+      }else if( memcmp(&zDetail[i], " USING COVERING INDEX ", 22)==0 ){
+        zIdx = &zDetail[i+22];
+      }
+      if( zIdx ){
         int nIdx = 0;
-        const char *zIdx = &zDetail[i+13];
         while( zIdx[nIdx]!='\0' && zIdx[nIdx]!=' ' ) nIdx++;
         sqlite3_bind_text(pSelect, 1, zIdx, nIdx, SQLITE_STATIC);
         if( SQLITE_ROW==sqlite3_step(pSelect) ){
@@ -856,6 +878,7 @@ int shellIndexesCommand(
   sqlite3_stmt *pStmt = 0;        /* Statement compiled from zSql */
 
   memset(&ctx, 0, sizeof(IdxContext));
+  ctx.pzErrmsg = pzErrmsg;
 
   /* Open an in-memory database to work with. The main in-memory 
   ** database schema contains tables similar to those in the users 

From 14479504381b65251f332d819e7f03a36ad08ea1 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 22 Feb 2016 19:51:08 +0000
Subject: [PATCH 015/488] Add test script shell6.test, containing tests for
 schemalint.

FossilOrigin-Name: 0b73406595c9a077399b0f4c17af3a826cf3612f
---
 manifest            |  15 ++--
 manifest.uuid       |   2 +-
 src/shell_indexes.c |  21 ++++-
 src/where.c         |   6 +-
 test/shell6.test    | 191 ++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 221 insertions(+), 14 deletions(-)
 create mode 100644 test/shell6.test

diff --git a/manifest b/manifest
index 549a6f5bf1..1ea78374dd 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\scouple\sof\sbugs\sin\sthe\sschemalint\scode.
-D 2016-02-19T07:53:43.883
+C Add\stest\sscript\sshell6.test,\scontaining\stests\sfor\sschemalint.
+D 2016-02-22T19:51:08.971
 F Makefile.in dac2776c84e0d533b158a9af6e57e05c4a6b19f3
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc b0493f10caddb8adf992a4e6f1943141fc7c6816
@@ -350,7 +350,7 @@ F src/resolve.c 9f7ce3a3c087afb7597b7c916c99126ff3f12f0c
 F src/rowset.c 9fe4b3ad7cc00944386bb600233d8f523de07a6e
 F src/select.c ff80004a9a6ece891a8d9327a88e7b6e2588ee6d
 F src/shell.c 2cde87e03712204231167c4a6c61b0eb5129e105
-F src/shell_indexes.c c0099c01c7af01038ab4315621814df535b700a3
+F src/shell_indexes.c e10b3c2c4bc9a87f7bff1cf622473717b1a00698
 F src/sqlite.h.in c7db059d3b810b70b83d9ed1436fa813eba22462
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h dfbe62ffd95b99afe2140d8c35b180d11924072d
@@ -428,7 +428,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c d21b99fd1458159d0b1ecdccc8ee6ada4fdc4c54
 F src/wal.h 2f7c831cf3b071fa548bf2d5cac640846a7ff19c
 F src/walker.c 0f142b5bd3ed2041fc52d773880748b212e63354
-F src/where.c 89d5845353fe6d2e77bce52a2c8bea0781c69dad
+F src/where.c 0ecce6da62ad521cac83dfa1b60a37936b36b6a1
 F src/whereInt.h 78b6b4de94db84aecbdc07fe3e38f648eb391e9a
 F src/wherecode.c 791a784bbf8749d560fdb0b990b607bc4f44a38d
 F src/whereexpr.c de117970b29471177a6901d60ad83a194671dc03
@@ -1025,6 +1025,7 @@ F test/shell2.test 12b8bf901b0e3a8ac58cf5c0c63a0a388d4d1862
 F test/shell3.test 5e8545ec72c4413a0e8d4c6be56496e3c257ca29
 F test/shell4.test ddf0a99044e2245a87fc17423e3aaa1445b3243b
 F test/shell5.test c04e9f9f948305706b88377c464c7f08ce7479f9
+F test/shell6.test dc93ef3f42c5c385e66b97729374fa1c017ea5ed
 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3
 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5
 F test/shrink.test 1b4330b1fd9e818c04726d45cb28db73087535ce
@@ -1430,7 +1431,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 4ab3df25f1fee7c8fea19d0c64b3e0e4d3b9c3cf
-R 0c9cae22954d1f32b48770c565f201e2
+P 02fbf699c07286f842d9617755f071b0fffc5d40
+R a01d8436d914e04ca0b643f78b3bf6ab
 U dan
-Z cecc60af647c055f880ee4bdf2018855
+Z 48d2e83e55b4a0cbf023823a82d88ee7
diff --git a/manifest.uuid b/manifest.uuid
index 675cfe73ee..a71d648d17 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-02fbf699c07286f842d9617755f071b0fffc5d40
\ No newline at end of file
+0b73406595c9a077399b0f4c17af3a826cf3612f
\ No newline at end of file
diff --git a/src/shell_indexes.c b/src/shell_indexes.c
index cd8a5e7288..738a035f02 100644
--- a/src/shell_indexes.c
+++ b/src/shell_indexes.c
@@ -34,6 +34,7 @@ struct IdxConstraint {
   int iCol;                       /* Constrained table column */
   i64 depmask;                    /* Dependency mask */
   int bFlag;                      /* Used by idxFindCompatible() */
+  int bDesc;                      /* True if ORDER BY <expr> DESC */
   IdxConstraint *pNext;           /* Next constraint in pEq or pRange list */
   IdxConstraint *pLink;           /* See above */
 };
@@ -198,11 +199,17 @@ static void idxWhereInfo(
 
       case SQLITE_WHEREINFO_ORDERBY: {
         IdxConstraint *pNew = idxNewConstraint(&p->rc, zVal);
-        IdxConstraint **pp;
         if( pNew==0 ) return;
         pNew->iCol = iVal;
-        for(pp=&p->pScan->pOrder; *pp; pp=&(*pp)->pNext);
-        *pp = pNew;
+        pNew->bDesc = (int)mask;
+        if( p->pScan->pOrder==0 ){
+          p->pScan->pOrder = pNew;
+        }else{
+          IdxConstraint *pIter;
+          for(pIter=p->pScan->pOrder; pIter->pNext; pIter=pIter->pNext);
+          pIter->pNext = pNew;
+          pIter->pLink = pNew;
+        }
         break;
       }
 
@@ -508,6 +515,10 @@ static char *idxAppendColDefn(
       zRet = idxAppendText(pRc, zRet, " COLLATE %s", pCons->zColl);
     }
   }
+
+  if( pCons->bDesc ){
+    zRet = idxAppendText(pRc, zRet, " DESC");
+  }
   return zRet;
 }
 
@@ -812,7 +823,9 @@ int idxFindIndexes(
       }
       if( zIdx ){
         int nIdx = 0;
-        while( zIdx[nIdx]!='\0' && zIdx[nIdx]!=' ' ) nIdx++;
+        while( zIdx[nIdx]!='\0' && (zIdx[nIdx]!=' ' || zIdx[nIdx+1]!='(') ){
+          nIdx++;
+        }
         sqlite3_bind_text(pSelect, 1, zIdx, nIdx, SQLITE_STATIC);
         if( SQLITE_ROW==sqlite3_step(pSelect) ){
           i64 iRowid = sqlite3_column_int64(pSelect, 0);
diff --git a/src/where.c b/src/where.c
index 3724fa33e0..6275fc8d1e 100644
--- a/src/where.c
+++ b/src/where.c
@@ -3990,11 +3990,13 @@ static void whereTraceBuilder(
           for(i=0; i<p->pOrderBy->nExpr; i++){
             Expr *pExpr = p->pOrderBy->a[i].pExpr; 
             CollSeq *pColl = sqlite3ExprCollSeq(pParse, pExpr);
+            assert( pColl || pParse->rc );
             pExpr = sqlite3ExprSkipCollate(pExpr);
             if( pExpr->op==TK_COLUMN && pExpr->iTable==pItem->iCursor ){
               int iCol = pExpr->iColumn;
-              if( iCol>=0 ){
-                x(pCtx, SQLITE_WHEREINFO_ORDERBY, pColl->zName, iCol, 0); 
+              if( pColl && iCol>=0 ){
+                int bDesc = p->pOrderBy->a[i].sortOrder;
+                x(pCtx, SQLITE_WHEREINFO_ORDERBY, pColl->zName, iCol, bDesc); 
               }
             }
           }
diff --git a/test/shell6.test b/test/shell6.test
new file mode 100644
index 0000000000..2b200a2757
--- /dev/null
+++ b/test/shell6.test
@@ -0,0 +1,191 @@
+# 2009 Nov 11
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# The focus of this file is testing the CLI shell tool. Specifically,
+# the ".recommend" command.
+#
+#
+
+# Test plan:
+#
+#   shell1-1.*: Basic command line option handling.
+#   shell1-2.*: Basic "dot" command token parsing.
+#   shell1-3.*: Basic test that "dot" command can be called.
+#
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix shell6
+
+if {$tcl_platform(platform)=="windows"} {
+  set CLI "sqlite3.exe"
+} else {
+  set CLI "./sqlite3"
+}
+if {![file executable $CLI]} {
+  finish_test
+  return
+}
+
+
+proc squish {txt} {
+  regsub -all {[[:space:]]+} $txt { }
+}
+
+proc do_rec_test {tn sql res} {
+  set res [squish [string trim $res]]
+  set tst [subst -nocommands { 
+    squish [lindex [catchcmd [list -rec test.db {$sql;}]] 1]
+  }]
+  uplevel [list do_test $tn $tst $res]
+}
+
+proc do_setup_rec_test {tn setup sql res} {
+  reset_db
+  db eval $setup
+  uplevel [list do_rec_test $tn $sql $res]
+}
+
+
+do_setup_rec_test 1.1 { CREATE TABLE t1(a, b, c) } {
+  SELECT * FROM t1
+} {
+  (no new indexes)
+  0|0|0|SCAN TABLE t1
+}
+
+do_setup_rec_test 1.2 {
+  CREATE TABLE t1(a, b, c);
+} {
+  SELECT * FROM t1 WHERE b>?;
+} {
+  CREATE INDEX t1_idx_00000062 ON t1(b) 
+  0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_00000062 (b>?)
+}
+
+do_setup_rec_test 1.3 {
+  CREATE TABLE t1(a, b, c);
+} {
+  SELECT * FROM t1 WHERE b COLLATE nocase BETWEEN ? AND ?
+} {
+  CREATE INDEX t1_idx_3e094c27 ON t1(b COLLATE NOCASE) 
+  0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_3e094c27 (b>? AND b<?)
+}
+
+do_setup_rec_test 1.4 {
+  CREATE TABLE t1(a, b, c);
+} {
+  SELECT a FROM t1 ORDER BY b;
+} {
+  CREATE INDEX t1_idx_00000062 ON t1(b) 
+  0|0|0|SCAN TABLE t1 USING INDEX t1_idx_00000062
+}
+
+do_setup_rec_test 1.5 {
+  CREATE TABLE t1(a, b, c);
+} {
+  SELECT a FROM t1 WHERE a=? ORDER BY b;
+} {
+  CREATE INDEX t1_idx_000123a7 ON t1(a, b) 
+  0|0|0|SEARCH TABLE t1 USING COVERING INDEX t1_idx_000123a7 (a=?)
+}
+
+do_setup_rec_test 1.6 {
+  CREATE TABLE t1(a, b, c);
+} {
+  SELECT min(a) FROM t1
+} {
+  CREATE INDEX t1_idx_00000061 ON t1(a) 
+  0|0|0|SEARCH TABLE t1 USING COVERING INDEX t1_idx_00000061
+}
+
+do_setup_rec_test 1.7 {
+  CREATE TABLE t1(a, b, c);
+} {
+  SELECT * FROM t1 ORDER BY a, b, c;
+} {
+  CREATE INDEX t1_idx_033e95fe ON t1(a, b, c) 
+  0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_033e95fe
+}
+
+do_setup_rec_test 1.8 {
+  CREATE TABLE t1(a, b, c);
+} {
+  SELECT * FROM t1 ORDER BY a ASC, b COLLATE nocase DESC, c ASC;
+} {
+  CREATE INDEX t1_idx_5be6e222 ON t1(a, b COLLATE NOCASE DESC, c)
+  0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_5be6e222
+}
+
+do_setup_rec_test 1.9 {
+  CREATE TABLE t1(a COLLATE NOCase, b, c);
+} {
+  SELECT * FROM t1 WHERE a=?
+} {
+  CREATE INDEX t1_idx_00000061 ON t1(a) 
+  0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_00000061 (a=?)
+}
+
+
+# Tables with names that require quotes.
+#
+do_setup_rec_test 8.1 {
+  CREATE TABLE "t t"(a, b, c);
+} {
+  SELECT * FROM "t t" WHERE a=?
+} {
+  CREATE INDEX 't t_idx_00000061' ON 't t'(a)
+  0|0|0|SEARCH TABLE t t USING INDEX t t_idx_00000061 (a=?) 
+}
+
+do_setup_rec_test 8.2 {
+  CREATE TABLE "t t"(a, b, c);
+} {
+  SELECT * FROM "t t" WHERE b BETWEEN ? AND ?
+} {
+  CREATE INDEX 't t_idx_00000062' ON 't t'(b) 
+  0|0|0|SEARCH TABLE t t USING INDEX t t_idx_00000062 (b>? AND b<?)
+}
+
+# Columns with names that require quotes.
+#
+do_setup_rec_test 9.1 {
+  CREATE TABLE t3(a, "b b", c);
+} {
+  SELECT * FROM t3 WHERE "b b" = ?
+} {
+  CREATE INDEX t3_idx_00050c52 ON t3('b b') 
+  0|0|0|SEARCH TABLE t3 USING INDEX t3_idx_00050c52 (b b=?)
+}
+
+do_setup_rec_test 9.2 {
+  CREATE TABLE t3(a, "b b", c);
+} {
+  SELECT * FROM t3 ORDER BY "b b"
+} {
+  CREATE INDEX t3_idx_00050c52 ON t3('b b') 
+  0|0|0|SCAN TABLE t3 USING INDEX t3_idx_00050c52
+}
+
+# Transitive constraints
+#
+do_setup_rec_test 10.1 {
+  CREATE TABLE t5(a, b);
+  CREATE TABLE t6(c, d);
+} {
+  SELECT * FROM t5, t6 WHERE a=? AND b=c AND c=?
+} {
+  CREATE INDEX t6_idx_00000063 ON t6(c) 
+  CREATE INDEX t5_idx_000123a7 ON t5(a, b) 
+  0|0|1|SEARCH TABLE t6 USING INDEX t6_idx_00000063 (c=?) 
+  0|1|0|SEARCH TABLE t5 USING COVERING INDEX t5_idx_000123a7 (a=? AND b=?)
+}
+
+finish_test

From 94c67eca890b703f6f14218553adc89d4781ac95 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 9 Mar 2016 08:07:31 +0000
Subject: [PATCH 016/488] Fix a problem in the schemalint code that comes up
 when a sub-query uses one or more of the same tables as its parent.

FossilOrigin-Name: fc18cc9293fb0080b7152c16baac49f44e2db7b3
---
 manifest             |  17 ++++----
 manifest.uuid        |   2 +-
 src/shell_indexes.c  |  48 ++++++++++++++++++---
 src/where.c          |   1 -
 test/schemalint.test | 100 -------------------------------------------
 test/shell6.test     |   2 +-
 6 files changed, 52 insertions(+), 118 deletions(-)
 delete mode 100644 test/schemalint.test

diff --git a/manifest b/manifest
index 1ea78374dd..7b7e824272 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\stest\sscript\sshell6.test,\scontaining\stests\sfor\sschemalint.
-D 2016-02-22T19:51:08.971
+C Fix\sa\sproblem\sin\sthe\sschemalint\scode\sthat\scomes\sup\swhen\sa\ssub-query\suses\sone\sor\smore\sof\sthe\ssame\stables\sas\sits\sparent.
+D 2016-03-09T08:07:31.577
 F Makefile.in dac2776c84e0d533b158a9af6e57e05c4a6b19f3
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc b0493f10caddb8adf992a4e6f1943141fc7c6816
@@ -350,7 +350,7 @@ F src/resolve.c 9f7ce3a3c087afb7597b7c916c99126ff3f12f0c
 F src/rowset.c 9fe4b3ad7cc00944386bb600233d8f523de07a6e
 F src/select.c ff80004a9a6ece891a8d9327a88e7b6e2588ee6d
 F src/shell.c 2cde87e03712204231167c4a6c61b0eb5129e105
-F src/shell_indexes.c e10b3c2c4bc9a87f7bff1cf622473717b1a00698
+F src/shell_indexes.c 23c4b7f7bb8e9b0a8b912c389a85922939bb0769
 F src/sqlite.h.in c7db059d3b810b70b83d9ed1436fa813eba22462
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h dfbe62ffd95b99afe2140d8c35b180d11924072d
@@ -428,7 +428,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c d21b99fd1458159d0b1ecdccc8ee6ada4fdc4c54
 F src/wal.h 2f7c831cf3b071fa548bf2d5cac640846a7ff19c
 F src/walker.c 0f142b5bd3ed2041fc52d773880748b212e63354
-F src/where.c 0ecce6da62ad521cac83dfa1b60a37936b36b6a1
+F src/where.c 59f267a53d29db13880597011323745b69e14f24
 F src/whereInt.h 78b6b4de94db84aecbdc07fe3e38f648eb391e9a
 F src/wherecode.c 791a784bbf8749d560fdb0b990b607bc4f44a38d
 F src/whereexpr.c de117970b29471177a6901d60ad83a194671dc03
@@ -988,7 +988,6 @@ F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5
 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38
 F test/schema4.test e6a66e20cc69f0e306667c08be7fda3d11707dc5
 F test/schema5.test 29699b4421f183c8f0e88bd28ce7d75d13ea653e
-F test/schemalint.test cee9f375637fcc8c6e77d971abe044445c23e024
 F test/securedel.test 21749c32ccc30f1ea9e4b9f33295a6521ec20fa0
 F test/securedel2.test 2d54c28e46eb1fd6902089958b20b1b056c6f1c5
 F test/select1.test be62204d2bd9a5a8a149e9974cfddce893d8f686
@@ -1025,7 +1024,7 @@ F test/shell2.test 12b8bf901b0e3a8ac58cf5c0c63a0a388d4d1862
 F test/shell3.test 5e8545ec72c4413a0e8d4c6be56496e3c257ca29
 F test/shell4.test ddf0a99044e2245a87fc17423e3aaa1445b3243b
 F test/shell5.test c04e9f9f948305706b88377c464c7f08ce7479f9
-F test/shell6.test dc93ef3f42c5c385e66b97729374fa1c017ea5ed
+F test/shell6.test 4ce53ce53e1206742e003b5ae609b6c2be438df1
 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3
 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5
 F test/shrink.test 1b4330b1fd9e818c04726d45cb28db73087535ce
@@ -1431,7 +1430,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 02fbf699c07286f842d9617755f071b0fffc5d40
-R a01d8436d914e04ca0b643f78b3bf6ab
+P 0b73406595c9a077399b0f4c17af3a826cf3612f
+R d77c52a71510d468f62d262501df9c9f
 U dan
-Z 48d2e83e55b4a0cbf023823a82d88ee7
+Z c51c58b805618dc47dc45d8f0cd33733
diff --git a/manifest.uuid b/manifest.uuid
index a71d648d17..e9344a7af6 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0b73406595c9a077399b0f4c17af3a826cf3612f
\ No newline at end of file
+fc18cc9293fb0080b7152c16baac49f44e2db7b3
\ No newline at end of file
diff --git a/src/shell_indexes.c b/src/shell_indexes.c
index 738a035f02..c5e172cb97 100644
--- a/src/shell_indexes.c
+++ b/src/shell_indexes.c
@@ -413,7 +413,6 @@ static int idxGetTableInfo(
   return rc;
 }
 
-
 static int idxCreateTables(
   sqlite3 *db,                    /* User database */
   sqlite3 *dbm,                   /* In-memory database to create tables in */
@@ -424,6 +423,22 @@ static int idxCreateTables(
   IdxScan *pIter;
   for(pIter=pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
     rc = idxGetTableInfo(db, pIter, pzErrmsg);
+
+    /* Test if table has already been created. If so, jump to the next
+    ** iteration of the loop.  */
+    if( rc==SQLITE_OK ){
+      sqlite3_stmt *pSql = 0;
+      rc = idxPrintfPrepareStmt(dbm, &pSql, pzErrmsg, 
+          "SELECT 1 FROM sqlite_master WHERE tbl_name = %Q", pIter->zTable
+      );
+      if( rc==SQLITE_OK ){
+        int bSkip = 0;
+        if( sqlite3_step(pSql)==SQLITE_ROW ) bSkip = 1;
+        rc = sqlite3_finalize(pSql);
+        if( bSkip ) continue;
+      }
+    }
+
     if( rc==SQLITE_OK ){
       int rc2;
       sqlite3_stmt *pSql = 0;
@@ -799,6 +814,7 @@ int idxFindIndexes(
   sqlite3 *dbm = pCtx->dbm;
   sqlite3_stmt *pExplain = 0;
   sqlite3_stmt *pSelect = 0;
+  sqlite3_stmt *pInsert = 0;
   int rc, rc2;
   int bFound = 0;
 
@@ -808,6 +824,11 @@ int idxFindIndexes(
         "SELECT rowid, sql FROM sqlite_master WHERE name = ?"
     );
   }
+  if( rc==SQLITE_OK ){
+    rc = idxPrepareStmt(dbm, &pInsert, pzErr,
+        "INSERT OR IGNORE INTO aux.indexes VALUES(?)"
+    );
+  }
 
   while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){
     int i;
@@ -831,8 +852,10 @@ int idxFindIndexes(
           i64 iRowid = sqlite3_column_int64(pSelect, 0);
           const char *zSql = (const char*)sqlite3_column_text(pSelect, 1);
           if( iRowid>=pCtx->iIdxRowid ){
-            xOut(pOutCtx, zSql);
-            bFound = 1;
+            sqlite3_bind_text(pInsert, 1, zSql, -1, SQLITE_STATIC);
+            sqlite3_step(pInsert);
+            rc = sqlite3_reset(pInsert);
+            if( rc ) goto find_indexes_out;
           }
         }
         rc = sqlite3_reset(pSelect);
@@ -843,8 +866,19 @@ int idxFindIndexes(
   rc2 = sqlite3_reset(pExplain);
   if( rc==SQLITE_OK ) rc = rc2;
   if( rc==SQLITE_OK ){
-    if( bFound==0 ) xOut(pOutCtx, "(no new indexes)");
-    xOut(pOutCtx, "");
+    sqlite3_stmt *pLoop = 0;
+    rc = idxPrepareStmt(dbm, &pLoop, pzErr, "SELECT name FROM aux.indexes");
+    if( rc==SQLITE_OK ){
+      while( SQLITE_ROW==sqlite3_step(pLoop) ){
+        bFound = 1;
+        xOut(pOutCtx, sqlite3_column_text(pLoop, 0));
+      }
+      rc = sqlite3_finalize(pLoop);
+    }
+    if( rc==SQLITE_OK ){
+      if( bFound==0 ) xOut(pOutCtx, "(no new indexes)");
+      xOut(pOutCtx, "");
+    }
   }
 
   while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){
@@ -868,6 +902,8 @@ int idxFindIndexes(
   if( rc==SQLITE_OK ) rc = rc2;
   rc2 = sqlite3_finalize(pSelect);
   if( rc==SQLITE_OK ) rc = rc2;
+  rc2 = sqlite3_finalize(pInsert);
+  if( rc==SQLITE_OK ) rc = rc2;
 
   return rc;
 }
@@ -911,7 +947,7 @@ int shellIndexesCommand(
   /* Prepare an INSERT statement for writing to aux.depmask */
   if( rc==SQLITE_OK ){
     rc = idxPrepareStmt(dbm, &ctx.pInsertMask, pzErrmsg,
-        "INSERT OR IGNORE INTO depmask SELECT mask | ?1 FROM depmask;"
+        "INSERT OR IGNORE INTO aux.depmask SELECT mask | ?1 FROM aux.depmask;"
     );
   }
 
diff --git a/src/where.c b/src/where.c
index 6275fc8d1e..abe1015b95 100644
--- a/src/where.c
+++ b/src/where.c
@@ -3990,7 +3990,6 @@ static void whereTraceBuilder(
           for(i=0; i<p->pOrderBy->nExpr; i++){
             Expr *pExpr = p->pOrderBy->a[i].pExpr; 
             CollSeq *pColl = sqlite3ExprCollSeq(pParse, pExpr);
-            assert( pColl || pParse->rc );
             pExpr = sqlite3ExprSkipCollate(pExpr);
             if( pExpr->op==TK_COLUMN && pExpr->iTable==pItem->iCursor ){
               int iCol = pExpr->iColumn;
diff --git a/test/schemalint.test b/test/schemalint.test
deleted file mode 100644
index 7e0eba494d..0000000000
--- a/test/schemalint.test
+++ /dev/null
@@ -1,100 +0,0 @@
-
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-set testprefix schemalint
-
-proc xTrace {zMsg} {
-  lappend ::trace_out $zMsg
-}
-db trace xTrace
-
-proc do_trace_test {tn sql res} {
-  uplevel [list do_test $tn [subst -nocommands {
-    set ::trace_out [list]
-    set stmt [sqlite3_prepare db "$sql" -1 x]
-    sqlite3_finalize [set stmt]
-    set ::trace_out
-  }] [list {*}$res]]
-}
-
-
-do_execsql_test 1.0 {
-  CREATE TABLE t1(a, b, c);
-  CREATE TABLE t2(x, y, z);
-}
-
-do_trace_test 1.1 {
-  SELECT b, c, y, z FROM t1, t2 WHERE c=? AND z=?
-} {
-  {"t1" {cols "b" "c"} {eq "c" "BINARY" 0}}
-  {"t2" {cols "y" "z"} {eq "z" "BINARY" 0}}
-}
-
-do_trace_test 1.2 {
-  SELECT a FROM t1 WHERE b>10
-} {
-  {"t1" {cols "a" "b"} {range "b" "BINARY" 0}}
-}
-
-do_trace_test 1.3 {
-  SELECT b FROM t1 WHERE b IN (10, 20, 30)
-} {
-  {"t1" {cols "b"} {eq "b" "BINARY" 0}}
-}
-
-do_trace_test 1.4 {
-  SELECT * FROM t1, t2 WHERE x=a
-} {
-  {"t1" {cols "a" "b" "c"} {eq "a" "BINARY" 2}} 
-  {"t2" {cols "x" "y" "z"} {eq "x" "BINARY" 1}}
-}
-
-do_trace_test 1.5 {
-  SELECT * FROM t1 WHERE a IN (1, 2, 3)
-} {
-  {"t1" {cols "a" "b" "c"} {eq "a" "BINARY" 0}}
-}
-
-#-----------------------------------------------------------------------
-# Cases involving OR clauses in the WHERE clause.
-#
-do_trace_test 2.1 {
-  SELECT * FROM t1 WHERE a=? OR b=?
-} {
-  {"t1" {cols "a" "b" "c"} {or {{eq "a" "BINARY" 0}} {{eq "b" "BINARY" 0}}}}
-}
-
-do_trace_test 2.2 {
-  SELECT * FROM t1 WHERE a=? OR (b=? AND c=?)
-} {
-  {"t1" {cols "a" "b" "c"} {or {{eq "a" "BINARY" 0}} {{eq "b" "BINARY" 0} {eq "c" "BINARY" 0}}}}
-}
-
-do_trace_test 2.3 {
-  SELECT * FROM t1 WHERE (a=? AND b=?) OR c=?
-} {
-  {"t1" {cols "a" "b" "c"} {or {{eq "c" "BINARY" 0}} {{eq "a" "BINARY" 0} {eq "b" "BINARY" 0}}}}
-}
-
-#-----------------------------------------------------------------------
-# Cases involving ORDER BY.
-#
-do_trace_test 3.1 {
-  SELECT * FROM t1 ORDER BY a;
-} {{"t1" {cols "a" "b" "c"} {orderby "a" "BINARY" ASC}}}
-
-do_trace_test 3.2 {
-  SELECT * FROM t1 WHERE a=? ORDER BY b;
-} {{"t1" {cols "a" "b" "c"} {eq "a" "BINARY" 0} {orderby "b" "BINARY" ASC}}}
-
-do_trace_test 3.3 {
-  SELECT min(a) FROM t1;
-} {{"t1" {cols "a"} {orderby "a" "BINARY" ASC}}}
-
-do_trace_test 3.4 {
-  SELECT max(a) FROM t1;
-} {{"t1" {cols "a"} {orderby "a" "BINARY" DESC}}}
-
-finish_test
-
diff --git a/test/shell6.test b/test/shell6.test
index 2b200a2757..ce8c24ceba 100644
--- a/test/shell6.test
+++ b/test/shell6.test
@@ -182,8 +182,8 @@ do_setup_rec_test 10.1 {
 } {
   SELECT * FROM t5, t6 WHERE a=? AND b=c AND c=?
 } {
-  CREATE INDEX t6_idx_00000063 ON t6(c) 
   CREATE INDEX t5_idx_000123a7 ON t5(a, b) 
+  CREATE INDEX t6_idx_00000063 ON t6(c) 
   0|0|1|SEARCH TABLE t6 USING INDEX t6_idx_00000063 (c=?) 
   0|1|0|SEARCH TABLE t5 USING COVERING INDEX t5_idx_000123a7 (a=? AND b=?)
 }

From b5c39ac0c916f03c85f3b1e328c554632969bf45 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 31 Mar 2017 08:00:03 +0000
Subject: [PATCH 017/488] Update shell6.test to account for the fact that tests
 are now run in a separate directory.

FossilOrigin-Name: 1e3622de8b51fdb219eb1934533a95fe6dc5158f44d358bc2efdcb53a4a9fdab
---
 manifest         | 12 ++++++------
 manifest.uuid    |  2 +-
 test/shell6.test |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 4540ae7f75..b8f39b0c2a 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\sthis\sbranch\swith\slatest\strunk\schanges.
-D 2016-03-15T09:42:39.387
+C Update\sshell6.test\sto\saccount\sfor\sthe\sfact\sthat\stests\sare\snow\srun\sin\sa\nseparate\sdirectory.
+D 2017-03-31T08:00:03.133
 F Makefile.in f53429fb2f313c099283659d0df6f20f932c861f
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc df0bf9ff7f8b3f4dd9fb4cc43f92fe58f6ec5c66
@@ -1031,7 +1031,7 @@ F test/shell2.test 12b8bf901b0e3a8ac58cf5c0c63a0a388d4d1862
 F test/shell3.test 5e8545ec72c4413a0e8d4c6be56496e3c257ca29
 F test/shell4.test f43e250139dc5dc5f0f2ec1752c50284a1ede102
 F test/shell5.test c04e9f9f948305706b88377c464c7f08ce7479f9
-F test/shell6.test 4ce53ce53e1206742e003b5ae609b6c2be438df1
+F test/shell6.test 05c2488ff2bb11ac300b9170e8f5b6e03d58982c77035cdca5dbf036f0d3c5b7
 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3
 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5
 F test/shrink.test 1b4330b1fd9e818c04726d45cb28db73087535ce
@@ -1459,7 +1459,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 59caca4387e97231834a3bffdbed8636be4e8e19 9f194f90c07d5415b1d6bb10ec049f74999d6885
-R 5b90dca0c628375a48db229132bb19b2
+P 249cd361b840913794b7cd2f2d42777dcd547a60
+R b5c9ea0357d4d69dec4a40a72a5670a9
 U dan
-Z dd17403945ec5cf2087723aff15fb7cd
+Z 84e2a466cf9b475b6898bdfe1581ffe0
diff --git a/manifest.uuid b/manifest.uuid
index 234da9ff63..8cfc3a9b80 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-249cd361b840913794b7cd2f2d42777dcd547a60
\ No newline at end of file
+1e3622de8b51fdb219eb1934533a95fe6dc5158f44d358bc2efdcb53a4a9fdab
\ No newline at end of file
diff --git a/test/shell6.test b/test/shell6.test
index ce8c24ceba..19b094a80d 100644
--- a/test/shell6.test
+++ b/test/shell6.test
@@ -27,7 +27,7 @@ set testprefix shell6
 if {$tcl_platform(platform)=="windows"} {
   set CLI "sqlite3.exe"
 } else {
-  set CLI "./sqlite3"
+  set CLI ".././sqlite3"
 }
 if {![file executable $CLI]} {
   finish_test

From e86573fa512a2e06e7ae0b737c0b839974355598 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 4 Apr 2017 04:23:06 +0000
Subject: [PATCH 018/488] Add the sqlite3_whereinfo_hook() API - an
 experimental API replacing the DBCONFIG_WHEREINFO hack on this branch.

FossilOrigin-Name: a54aef35da11f7508a8888a159372036a362fc52afa1df752dc835db334c4330
---
 manifest            |  22 +++----
 manifest.uuid       |   2 +-
 src/main.c          |  27 ++++++--
 src/shell_indexes.c |  33 ++--------
 src/sqlite.h.in     | 152 ++++++++++++++++++++++++++++++++++++++++----
 src/sqliteInt.h     |   4 +-
 src/where.c         |  38 +++--------
 7 files changed, 189 insertions(+), 89 deletions(-)

diff --git a/manifest b/manifest
index 90f504d2aa..fc73fb3e18 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\slatest\strunk\sinto\sthis\sbranch.
-D 2017-03-31T11:20:20.468
+C Add\sthe\ssqlite3_whereinfo_hook()\sAPI\s-\san\sexperimental\sAPI\sreplacing\sthe\nDBCONFIG_WHEREINFO\shack\son\sthis\sbranch.
+D 2017-04-04T04:23:06.108
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a
@@ -36,7 +36,7 @@ F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad
 F doc/lemon.html b5a3c07d33ecb8e019ce8f7660fe2dbbad9d7977
 F doc/pager-invariants.txt 27fed9a70ddad2088750c4a2b493b63853da2710
 F doc/vfs-shm.txt e101f27ea02a8387ce46a05be2b1a902a021d37a
-F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd w ext/README.txt
+F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd
 F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
@@ -366,7 +366,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c d4bb3a135948553d18cf992f76f7ed7b18aa0327f250607b5a6671e55d9947d5
 F src/legacy.c e88ed13c2d531decde75d42c2e35623fb9ce3cb0
 F src/loadext.c a72909474dadce771d3669bf84bf689424f6f87d471fee898589c3ef9b2acfd9
-F src/main.c 36974edf7461f708b337389c59b30ec91257420c129c1f5ee022bc2c2784d320
+F src/main.c 4b93bda0f1f916f4cb618a6fce358f7e01ab060971c586c4cea90ad0c83a83f5
 F src/malloc.c 89c98e3619d362dcffa5c1c639b364b65b474751
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -403,11 +403,11 @@ F src/resolve.c 3e518b962d932a997fae373366880fc028c75706
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 2496d0cc6368dabe7ad2e4c7f5ed3ad9aa3b4d11cd90f33fa1d1ef72493f43aa
 F src/shell.c e524688c2544167f835ba43e24309f8707ca60c8ab6eb5c263a12c8618a233b8
-F src/shell_indexes.c 23c4b7f7bb8e9b0a8b912c389a85922939bb0769
-F src/sqlite.h.in 3db120c750634c0acc1ca291140a4cae75cb41cb0507f53c31b838c040954366
+F src/shell_indexes.c 12cc58b62492bddc88f879086b4cd1045dbfd4e52040ecb6c39b971a6aac80ba
+F src/sqlite.h.in ae5c9cbf2e77492c319fca08769575d9695e64718a16d32324944d24e291bcf7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 58fd0676d3111d02e62e5a35992a7d3da5d3f88753acc174f2d37b774fbbdd28
-F src/sqliteInt.h 22a0a101c51a0fae4156088f7c613b99a73d20ba6153a1400d1ae3011cf1082c
+F src/sqliteInt.h 9af1569ac748f72169b4ad0004d03ea14c75b59eb955a2c0ecca75a16ea2015f
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -483,7 +483,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 40c543f0a2195d1b0dc88ef12142bea690009344
 F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71
 F src/walker.c b71a992b413b3a022572eccf29ef4b4890223791
-F src/where.c 30c790ab98b759ac3d7fa564e1321489151fac7d1587525bc5b9698394a41ecf
+F src/where.c cbd1e8562fa37de359963749e1dc3d0dd1a42a894d10a62928bffc709490fde6
 F src/whereInt.h 2d50c2b74a33be44cb68fdecee30b4d93552f1f4
 F src/wherecode.c 677e95413c472c0b413023b6b69a47f40fce1b04
 F src/whereexpr.c 130cdd1a43af71b19755270fb1224874cf55158c
@@ -1571,7 +1571,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 1e3622de8b51fdb219eb1934533a95fe6dc5158f44d358bc2efdcb53a4a9fdab c8000e94cca59dabf83d6cb75b40441aaf793d29880582dc4baa17246449b5fe
-R a9131615092ed98b863a64f66624daea
+P cb721d0b36268a7b0ef493fa4d7f6bcbaa9ead8b1990e3c3fae015fa1d545226
+R 061e66f6528edc67cfbe5cb82be7b02d
 U dan
-Z 924aec6608b5903c59e65e7507c45eb8
+Z d08fd2524fbe07ca7104a150ba1edcbe
diff --git a/manifest.uuid b/manifest.uuid
index 1b590b1f1f..64b0731eb5 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-cb721d0b36268a7b0ef493fa4d7f6bcbaa9ead8b1990e3c3fae015fa1d545226
\ No newline at end of file
+a54aef35da11f7508a8888a159372036a362fc52afa1df752dc835db334c4330
\ No newline at end of file
diff --git a/src/main.c b/src/main.c
index eed82b0e66..df715852ae 100644
--- a/src/main.c
+++ b/src/main.c
@@ -801,13 +801,6 @@ int sqlite3_db_config(sqlite3 *db, int op, ...){
       rc = setupLookaside(db, pBuf, sz, cnt);
       break;
     }
-#ifdef SQLITE_SCHEMA_LINT
-    case SQLITE_DBCONFIG_WHEREINFO: {
-      db->xWhereInfo = va_arg(ap, void(*)(void*, int, const char*, int, i64));
-      db->pWhereInfoCtx = va_arg(ap, void*);
-      break;
-    }
-#endif
     default: {
       static const struct {
         int op;      /* The opcode */
@@ -2004,6 +1997,26 @@ void *sqlite3_preupdate_hook(
 }
 #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */
 
+#ifdef SQLITE_ENABLE_WHEREINFO_HOOK
+/*
+** Register a where-info hook.
+*/
+void *sqlite3_whereinfo_hook(
+  sqlite3 *db,                    /* Register callback with this db handle */
+  void (*xWhereInfo)(void*, int, const char*, int, sqlite3_uint64),
+  void *pCtx                      /* User pointer passed to callback */
+){
+  void *pRet;
+  sqlite3_mutex_enter(db->mutex);
+  pRet = db->pWhereInfoCtx;
+  db->xWhereInfo = xWhereInfo;
+  db->pWhereInfoCtx = pCtx;
+  sqlite3_mutex_leave(db->mutex);
+  return pRet;
+}
+#endif /* SQLITE_ENABLE_WHEREINFO_HOOK */
+
+
 #ifndef SQLITE_OMIT_WAL
 /*
 ** The sqlite3_wal_hook() callback registered by sqlite3_wal_autocheckpoint().
diff --git a/src/shell_indexes.c b/src/shell_indexes.c
index c5e172cb97..efdc24fac3 100644
--- a/src/shell_indexes.c
+++ b/src/shell_indexes.c
@@ -12,6 +12,7 @@
 */
 
 typedef sqlite3_int64 i64;
+typedef sqlite3_uint64 u64;
 
 typedef struct IdxConstraint IdxConstraint;
 typedef struct IdxContext IdxContext;
@@ -164,7 +165,7 @@ static void idxWhereInfo(
   int eOp, 
   const char *zVal, 
   int iVal, 
-  i64 mask
+  u64 mask
 ){
   IdxContext *p = (IdxContext*)pCtx;
 
@@ -233,32 +234,6 @@ static void idxWhereInfo(
         p->rc = sqlite3_reset(p->pInsertMask);
         break;
       }
-
-      case SQLITE_WHEREINFO_BEGINOR: {
-        IdxWhere *pNew = (IdxWhere*)idxMalloc(&p->rc, sizeof(IdxWhere));
-        if( pNew==0 ) return;
-        pNew->pParent = p->pCurrent;
-        pNew->pNextOr = p->pCurrent->pOr;
-        p->pCurrent->pOr = pNew;
-        p->pCurrent = pNew;
-        break;
-      }
-
-      case SQLITE_WHEREINFO_NEXTOR: {
-        IdxWhere *pNew = (IdxWhere*)idxMalloc(&p->rc, sizeof(IdxWhere));
-        if( pNew==0 ) return;
-        pNew->pParent = p->pCurrent->pParent;
-        assert( p->pCurrent->pSibling==0 );
-        p->pCurrent->pSibling = pNew;
-        p->pCurrent = pNew;
-        break;
-      }
-
-      case SQLITE_WHEREINFO_ENDOR: {
-        assert( p->pCurrent->pParent );
-        p->pCurrent = p->pCurrent->pParent;
-        break;
-      }
     }
   }
 }
@@ -954,9 +929,9 @@ int shellIndexesCommand(
   /* Analyze the SELECT statement in zSql. */
   if( rc==SQLITE_OK ){
     ctx.dbm = dbm;
-    sqlite3_db_config(db, SQLITE_DBCONFIG_WHEREINFO, idxWhereInfo, (void*)&ctx);
+    sqlite3_whereinfo_hook(db, idxWhereInfo, (void*)&ctx);
     rc = idxPrepareStmt(db, &pStmt, pzErrmsg, zSql);
-    sqlite3_db_config(db, SQLITE_DBCONFIG_WHEREINFO, (void*)0, (void*)0);
+    sqlite3_whereinfo_hook(db, 0, 0);
     sqlite3_finalize(pStmt);
   }
 
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index f3067b64eb..3f9fb2fccb 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -2017,17 +2017,6 @@ struct sqlite3_mem_methods {
 #define SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION 1005 /* int int* */
 #define SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE      1006 /* int int* */
 
-#define SQLITE_DBCONFIG_WHEREINFO             1007  /* xWhereInfo void* */
-
-#define SQLITE_WHEREINFO_TABLE   1
-#define SQLITE_WHEREINFO_EQUALS  2
-#define SQLITE_WHEREINFO_RANGE   3
-#define SQLITE_WHEREINFO_ORDERBY 4
-#define SQLITE_WHEREINFO_BEGINOR 5
-#define SQLITE_WHEREINFO_ENDOR   6
-#define SQLITE_WHEREINFO_NEXTOR  7
-
-
 /*
 ** CAPI3REF: Enable Or Disable Extended Result Codes
 ** METHOD: sqlite3
@@ -8496,6 +8485,147 @@ SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp(
 */
 SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb);
 
+/*
+** This function is used to register a whereinfo hook with the database
+** handle passed as the first argument. Once registered, the whereinfo hook
+** is invoked zero or more times while preparing a query to provide
+** information to the application. It is intended to be used by expert 
+** systems to recommend indexes that could be added to the database in order
+** to improve query response time.
+**
+** An SQLite query plan describes the way data is read from zero or more
+** database tables. For each table read, the data required may be
+** constrained by equality or range constraints on one or more columns, 
+** and it may or may not be required in order sorted by one or more 
+** columns. For example, the following query:
+**
+** <pre>
+**     SELECT * FROM t1 WHERE t1.a = ? ORDER BY t1.b;
+** </pre>
+**
+** reads data from table t1. It requires only those rows for which t1.a
+** is set to a specific value, and requires them sorted in order of
+** column t1.b. Internally, SQLite determines this and attempts to locate
+** an index that can be used to efficiently find the required subset of
+** rows and/or allows the rows to be read from the database in the 
+** required order. In this case, ideally an index of the form:
+**
+** <pre>
+**     CREATE INDEX i1 ON t1(a, b);
+** </pre>
+**
+** The data passed to the whereinfo hook during query preparation do
+** not describe the actual query plan to the application. Instead, it
+** describes the parts of the query that SQLite could use an index to
+** optimize if a suitable index existed. In this case, that only
+** rows with t1.a=? are required, and that they are required sorted
+** in order by t1.b.
+**
+** Each time the whereinfo hook is invoked, the first argument is a
+** copy of the (void*) pointer passed as the second argument to this
+** API function. The second is always one of the SQLITE_WHEREINFO_XXX
+** constants defined below.
+**
+** For each table read by a query, the whereinfo hook is invoked as follows:
+**
+** <ul>
+**   <li> Once with SQLITE_WHEREINFO_TABLE as the second argument. This
+**        indicates the table that subsequent callbacks (until the next
+**        SQLITE_WHEREINFO_TABLE) apply to.
+**
+**   <li> If SQLite requires rows in a specific order, once with
+**        SQLITE_WHEREINFO_ORDERBY for each column of the table that is
+**        one of the sort keys.
+**
+**   <li> If there are any "<col> = ?" constraints that restrict the rows
+**        required by SQLite, once with SQLITE_WHEREINFO_EQUALS for each
+**        such constraint.
+**
+**   <li> If there are any "<col> > ?" constraints (or any other range
+**        constraint) that restrict the rows required by SQLite, once with 
+**        SQLITE_WHEREINFO_RANGE for each such constraint.
+** </ul>
+**
+** The third, fourth and fifth arguments passed to the whereinfo callback
+** are interpreted differently, depending on the SQLITE_WHEREINFO_XXX value
+** as follows:
+**
+** <dl>
+**   <dt> SQLITE_WHEREINFO_TABLE
+**   <dd> The third argument passed in this case is the name of the table.
+**        The fourth is the index of the database in which the table is
+**        located (0 for "main", 1 for "temp" or higher for an attached
+**        database). The fifth argument is a bitmask that indicates which
+**        of the tables columns may be required by the query. If the leftmost
+**        column of the table is used in some way, bit 0 of the bitmask is
+**        set. If the next-to-leftmost is used, bit 1 etc. Bit 63 is used to
+**        represent all columns with an index of 63 or higher. If bit 63
+**        is set, the application should assume that the query requires all
+**        columns from the table with an index of 63 or greater.
+**
+**   <dt> SQLITE_WHEREINFO_ORDERBY
+**   <dd> The third argument passed in this case is the name of the collation
+**        sequence to sort by. The fourth is the index of the table column to
+**        sort by (0 for the leftmost column, 1 for the next-to-leftmost
+**        etc.). The fifth argument is a boolean flag - true for a DESC sort
+**        or false for ASC.
+**
+**   <dt> SQLITE_WHEREINFO_EQUALS
+**   <dd> The third argument passed in this case is the name of the collation
+**        sequence used by the constraint. The fourth is the index of the 
+**        table column in the constraint. If the current table is not part
+**        of a join, then the value passed as the fifth argument is always
+**        zero. Or, if it is part of a join, then the fifth parameter passed
+**        to this callback is a mask of other tables that the current
+**        constraint depends on. For example, in the query:
+**
+**   <pre>
+**      SELECT * FROM t1, t2 WHERE t1.a = (t2.b+1);
+**   </pre>
+**
+**        the fifth parameter passed to the the SQLITE_WHEREINFO_EQUALS
+**        whereinfo callback would have the bit assigned to table "t2"
+**        set to true. There is no way for the application to determine 
+**        the specific bit in the mask assigned to any table, but the bit
+**        assignments are consistent while parsing a single query.
+**
+**   <dt> SQLITE_WHEREINFO_RANGE
+**   <dd> As for SQLITE_WHEREINFO_EQUALS.
+** </dl>
+**
+** Note that if a WHERE clause includes an OR expression, then there may be
+** more than one set of callbacks for a single table. For example, the
+** following SQL:
+**
+** <pre>
+**    SELECT * FROM t1 WHERE t1.a=? OR t1.b=?
+** </pre>
+**
+** Provokes the same callbacks as the following two queries executed in
+** series.
+**
+** <pre>
+**    SELECT * FROM t1 WHERE t1.a=?;
+**    SELECT * FROM t1 WHERE t1.b=?;
+** </pre>
+*/
+SQLITE_EXPERIMENTAL void *sqlite3_whereinfo_hook(
+  sqlite3 *db,                    /* Register callback with this db handle */
+  void (*xWhereInfo)(
+    void*,                        /* Copy of pCtx */
+    int,                          /* SQLITE_WHEREINFO_XXX constant */
+    const char*,
+    int,
+    sqlite3_uint64
+  ),
+  void *pCtx                      /* User pointer passed to callback */
+);
+
+#define SQLITE_WHEREINFO_TABLE   1
+#define SQLITE_WHEREINFO_EQUALS  2
+#define SQLITE_WHEREINFO_RANGE   3
+#define SQLITE_WHEREINFO_ORDERBY 4
+
 /*
 ** Undo the hack that converts floating point types to integer for
 ** builds on processors without floating point support.
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 0f6ffd5175..c5f5ccf0da 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1431,8 +1431,8 @@ struct sqlite3 {
 #ifdef SQLITE_USER_AUTHENTICATION
   sqlite3_userauth auth;        /* User authentication information */
 #endif
-#ifdef SQLITE_SCHEMA_LINT
-  void (*xWhereInfo)(void*, int, const char*, int, i64);
+#ifdef SQLITE_ENABLE_WHEREINFO_HOOK
+  void (*xWhereInfo)(void*, int, const char*, int, u64);
   void *pWhereInfoCtx;
 #endif
 };
diff --git a/src/where.c b/src/where.c
index c7e4ba200a..ba2ccfe65e 100644
--- a/src/where.c
+++ b/src/where.c
@@ -3277,7 +3277,7 @@ static int whereLoopAddOr(
   WhereLoopBuilder sSubBuild;
   WhereOrSet sSum, sCur;
   struct SrcList_item *pItem;
-  
+
   pWC = pBuilder->pWC;
   pWCEnd = pWC->a + pWC->nTerm;
   pNew = pBuilder->pNew;
@@ -3294,7 +3294,7 @@ static int whereLoopAddOr(
       WhereTerm *pOrTerm;
       int once = 1;
       int i, j;
-    
+
       sSubBuild = *pBuilder;
       sSubBuild.pOrderBy = 0;
       sSubBuild.pOrSet = &sCur;
@@ -4277,18 +4277,16 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){
   return 0;
 }
 
-#ifdef SQLITE_SCHEMA_LINT
+#ifdef SQLITE_ENABLE_WHEREINFO_HOOK
 static void whereTraceWC(
   Parse *pParse, 
   struct SrcList_item *pItem,
-  WhereClause *pWC,
-  int bOr
+  WhereClause *pWC
 ){
   sqlite3 *db = pParse->db;
   Table *pTab = pItem->pTab;
   void (*x)(void*, int, const char*, int, i64) = db->xWhereInfo;
   void *pCtx = db->pWhereInfoCtx;
-  int bFirst = 1;                 /* True until first callback is made */
   int ii;
 
   /* Issue callbacks for WO_SINGLE constraints */
@@ -4308,30 +4306,15 @@ static void whereTraceWC(
       }else{
         eOp = SQLITE_WHEREINFO_RANGE;
       }
-      if( bOr && !bFirst ) x(pCtx, SQLITE_WHEREINFO_NEXTOR, 0, 0, 0);
       x(pCtx, eOp, (pC ? pC->zName : "BINARY"), ii, pTerm->prereqRight);
-      bFirst = 0;
-    }
-  }
-
-  /* Callbacks for composite - (WO_OR|WO_AND) - constraints */
-  for(ii=0; ii<pWC->nTerm; ii++){
-    WhereTerm *pTerm = &pWC->a[ii];
-    if( pTerm->eOperator & WO_OR ){
-      assert( bOr==0 );
-      x(pCtx, SQLITE_WHEREINFO_BEGINOR, 0, 0, 0);
-      whereTraceWC(pParse, pItem, &pTerm->u.pOrInfo->wc, 1);
-      x(pCtx, SQLITE_WHEREINFO_ENDOR, 0, 0, 0);
-    }
-    if( pTerm->eOperator & WO_AND ){
-      if( bOr && !bFirst ) x(pCtx, SQLITE_WHEREINFO_NEXTOR, 0, 0, 0);
-      whereTraceWC(pParse, pItem, &pTerm->u.pAndInfo->wc, 0);
-      bFirst = 0;
     }
   }
 }
 
-
+/*
+** If there is a where-info hook attached to the database handle, issue all
+** required callbacks for the current sqlite3WhereBegin() call.
+*/
 static void whereTraceBuilder(
   Parse *pParse,
   WhereLoopBuilder *p
@@ -4358,7 +4341,6 @@ static void whereTraceBuilder(
         /* ORDER BY callbacks */
         if( p->pOrderBy ){
           int i;
-          int bFirst = 1;
           for(i=0; i<p->pOrderBy->nExpr; i++){
             Expr *pExpr = p->pOrderBy->a[i].pExpr; 
             CollSeq *pColl = sqlite3ExprCollSeq(pParse, pExpr);
@@ -4374,7 +4356,7 @@ static void whereTraceBuilder(
         }
 
         /* WHERE callbacks */
-        whereTraceWC(pParse, pItem, p->pWC, 0);
+        whereTraceWC(pParse, pItem, p->pWC);
       }
     }
   }
@@ -4654,7 +4636,7 @@ WhereInfo *sqlite3WhereBegin(
   }
 #endif
 
-  /* Schema-lint xTrace callback */
+  /* Invoke the where-info hook, if one has been registered. */
   whereTraceBuilder(pParse, &sWLB);
 
   if( nTabList!=1 || whereShortCut(&sWLB)==0 ){

From 53761f4d521be3b719a96106eaf89627abf20694 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 4 Apr 2017 17:50:31 +0000
Subject: [PATCH 019/488] Add some support for OR terms to
 sqlite3_whereinfo_hook().

FossilOrigin-Name: 5cd070000da1d9e399090677b4db75dc5639c33211385d6eb84f14a4d0b617cd
---
 manifest            |  16 +++---
 manifest.uuid       |   2 +-
 src/shell_indexes.c |   7 ++-
 src/where.c         | 117 ++++++++++++++++++++++++++++++++++++++++++--
 test/shell6.test    |  14 ++++++
 5 files changed, 138 insertions(+), 18 deletions(-)

diff --git a/manifest b/manifest
index fc73fb3e18..0e690d166d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\ssqlite3_whereinfo_hook()\sAPI\s-\san\sexperimental\sAPI\sreplacing\sthe\nDBCONFIG_WHEREINFO\shack\son\sthis\sbranch.
-D 2017-04-04T04:23:06.108
+C Add\ssome\ssupport\sfor\sOR\sterms\sto\ssqlite3_whereinfo_hook().
+D 2017-04-04T17:50:31.913
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a
@@ -403,7 +403,7 @@ F src/resolve.c 3e518b962d932a997fae373366880fc028c75706
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 2496d0cc6368dabe7ad2e4c7f5ed3ad9aa3b4d11cd90f33fa1d1ef72493f43aa
 F src/shell.c e524688c2544167f835ba43e24309f8707ca60c8ab6eb5c263a12c8618a233b8
-F src/shell_indexes.c 12cc58b62492bddc88f879086b4cd1045dbfd4e52040ecb6c39b971a6aac80ba
+F src/shell_indexes.c d40ea0a81112df7bdccd7232238bee0bbb39699085ea78cc08fd863bf052a63b
 F src/sqlite.h.in ae5c9cbf2e77492c319fca08769575d9695e64718a16d32324944d24e291bcf7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 58fd0676d3111d02e62e5a35992a7d3da5d3f88753acc174f2d37b774fbbdd28
@@ -483,7 +483,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 40c543f0a2195d1b0dc88ef12142bea690009344
 F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71
 F src/walker.c b71a992b413b3a022572eccf29ef4b4890223791
-F src/where.c cbd1e8562fa37de359963749e1dc3d0dd1a42a894d10a62928bffc709490fde6
+F src/where.c be352441558b55e21964ff53f0483776594789d0ede12dd9c58b022873c858f8
 F src/whereInt.h 2d50c2b74a33be44cb68fdecee30b4d93552f1f4
 F src/wherecode.c 677e95413c472c0b413023b6b69a47f40fce1b04
 F src/whereexpr.c 130cdd1a43af71b19755270fb1224874cf55158c
@@ -1120,7 +1120,7 @@ F test/shell2.test e242a9912f44f4c23c3d1d802a83e934e84c853b
 F test/shell3.test 9b95ba643eaa228376f06a898fb410ee9b6e57c1
 F test/shell4.test 89ad573879a745974ff2df20ff97c5d6ffffbd5d
 F test/shell5.test 50a732c1c2158b1cd62cf53975ce1ea7ce6b9dc9
-F test/shell6.test 05c2488ff2bb11ac300b9170e8f5b6e03d58982c77035cdca5dbf036f0d3c5b7
+F test/shell6.test 081067c1afcb38da50134ffd5ccc0a59ede14f41959486f733ffbba689c0ccfa
 F test/shell7.test 07751911b294698e0c5df67bcbd29e7d2f0f2907
 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3
 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5
@@ -1571,7 +1571,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P cb721d0b36268a7b0ef493fa4d7f6bcbaa9ead8b1990e3c3fae015fa1d545226
-R 061e66f6528edc67cfbe5cb82be7b02d
+P a54aef35da11f7508a8888a159372036a362fc52afa1df752dc835db334c4330
+R 51e547bc1c25ea79dd375d8f020224f3
 U dan
-Z d08fd2524fbe07ca7104a150ba1edcbe
+Z fe378068686ea107c39185021a17b4ae
diff --git a/manifest.uuid b/manifest.uuid
index 64b0731eb5..7669c54b05 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a54aef35da11f7508a8888a159372036a362fc52afa1df752dc835db334c4330
\ No newline at end of file
+5cd070000da1d9e399090677b4db75dc5639c33211385d6eb84f14a4d0b617cd
\ No newline at end of file
diff --git a/src/shell_indexes.c b/src/shell_indexes.c
index efdc24fac3..e061f04d3f 100644
--- a/src/shell_indexes.c
+++ b/src/shell_indexes.c
@@ -175,9 +175,6 @@ static void idxWhereInfo(
     eOp==SQLITE_WHEREINFO_EQUALS ? "EQUALS" :
     eOp==SQLITE_WHEREINFO_RANGE ? "RANGE" :
     eOp==SQLITE_WHEREINFO_ORDERBY ? "ORDERBY" :
-    eOp==SQLITE_WHEREINFO_NEXTOR ? "NEXTOR" :
-    eOp==SQLITE_WHEREINFO_ENDOR ? "ENDOR" :
-    eOp==SQLITE_WHEREINFO_BEGINOR ? "BEGINOR" :
     "!error!";
   printf("op=%s zVal=%s iVal=%d mask=%llx\n", zOp, zVal, iVal, mask);
 #endif
@@ -630,7 +627,9 @@ static int idxCreateFromCons(
         rc = SQLITE_NOMEM;
       }else{
         rc = sqlite3_exec(dbm, zIdx, 0, 0, pCtx->pzErrmsg);
-        /* printf("%s\n", zIdx); */
+#if 0
+        printf("CANDIDATE: %s\n", zIdx);
+#endif
       }
     }
     if( rc==SQLITE_OK && pCtx->iIdxRowid==0 ){
diff --git a/src/where.c b/src/where.c
index ba2ccfe65e..a423fa1839 100644
--- a/src/where.c
+++ b/src/where.c
@@ -4278,6 +4278,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){
 }
 
 #ifdef SQLITE_ENABLE_WHEREINFO_HOOK
+
 static void whereTraceWC(
   Parse *pParse, 
   struct SrcList_item *pItem,
@@ -4285,7 +4286,7 @@ static void whereTraceWC(
 ){
   sqlite3 *db = pParse->db;
   Table *pTab = pItem->pTab;
-  void (*x)(void*, int, const char*, int, i64) = db->xWhereInfo;
+  void (*x)(void*, int, const char*, int, u64) = db->xWhereInfo;
   void *pCtx = db->pWhereInfoCtx;
   int ii;
 
@@ -4311,6 +4312,109 @@ static void whereTraceWC(
   }
 }
 
+/*
+** If there are any OR terms in WHERE clause pWC, make the associated
+** where-info hook callbacks.
+*/
+static void whereTraceOR(
+  Parse *pParse, 
+  struct SrcList_item *pItem,
+  WhereClause *pWC
+){
+  sqlite3 *db = pParse->db;
+  WhereClause tempWC;
+  struct TermAndIdx {
+    WhereTerm *pTerm;
+    int iIdx;
+  } aOr[4];
+  int nOr = 0;
+  Table *pTab = pItem->pTab;
+  int iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
+  int ii;
+
+  memset(aOr, 0, sizeof(aOr));
+
+  /* Iterate through OR nodes */
+  for(ii=0; ii<pWC->nTerm; ii++){
+    WhereTerm *pTerm = &pWC->a[ii];
+    if( pTerm->eOperator & WO_OR ){
+      /* Check that each branch of this OR term contains at least
+      ** one reference to the table currently being processed. If that
+      ** is not the case, this term can be ignored.  */
+      WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc;
+      WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm];
+      WhereTerm *pOrTerm;
+      WhereClause *pTermWC;
+      WhereScan scan;
+
+      for(pOrTerm=pOrWC->a; pOrTerm<pOrWCEnd; pOrTerm++){
+        int iCol;
+        if( (pOrTerm->eOperator & WO_AND)!=0 ){
+          pTermWC = &pOrTerm->u.pAndInfo->wc;
+        }else{
+          tempWC.pWInfo = pWC->pWInfo;
+          tempWC.pOuter = pWC;
+          tempWC.op = TK_AND;
+          tempWC.nTerm = 1;
+          tempWC.a = pOrTerm;
+          pTermWC = &tempWC;
+        }
+
+        for(iCol=0; iCol<pTab->nCol; iCol++){
+          int iCsr = pItem->iCursor;
+          if( !whereScanInit(&scan, pTermWC, iCsr, iCol, WO_SINGLE, 0) ){
+            break;
+          }
+        }
+        if( iCol==pTab->nCol ) break;
+      }
+
+      if( pOrTerm==pOrWCEnd ){
+        aOr[nOr].pTerm = pTerm;
+        aOr[nOr].iIdx = pOrWC->nTerm;
+        nOr++;
+        if( nOr==ArraySize(aOr) ) break;
+      }
+    }
+  }
+
+  while( 1 ){
+    for(ii=0; ii<nOr; ii++){
+      if( aOr[ii].iIdx==0 ){
+        aOr[ii].iIdx = aOr[ii].pTerm->u.pOrInfo->wc.nTerm;
+      }else{
+        aOr[ii].iIdx--;
+        break;
+      }
+    }
+    if( ii==nOr ) break;
+
+    /* Table name callback */
+    db->xWhereInfo(db->pWhereInfoCtx, 
+        SQLITE_WHEREINFO_TABLE, pTab->zName, iDb, pItem->colUsed
+    );
+    /* whereTraceWC(pParse, pItem, pWC); */
+    for(ii=0; ii<nOr; ii++){
+      WhereClause * const pOrWC = &aOr[ii].pTerm->u.pOrInfo->wc;
+      if( aOr[ii].iIdx<pOrWC->nTerm ){
+        WhereClause *pTermWC;
+        WhereTerm *pOrTerm = &pOrWC->a[aOr[ii].iIdx];
+        if( (pOrTerm->eOperator & WO_AND)!=0 ){
+          pTermWC = &pOrTerm->u.pAndInfo->wc;
+        }else{
+          tempWC.pWInfo = pWC->pWInfo;
+          tempWC.pOuter = pWC;
+          tempWC.op = TK_AND;
+          tempWC.nTerm = 1;
+          tempWC.a = pOrTerm;
+          pTermWC = &tempWC;
+        }
+        whereTraceWC(pParse, pItem, pTermWC);
+      }
+    }
+  }
+}
+
 /*
 ** If there is a where-info hook attached to the database handle, issue all
 ** required callbacks for the current sqlite3WhereBegin() call.
@@ -4321,16 +4425,16 @@ static void whereTraceBuilder(
 ){
   sqlite3 *db = pParse->db;
   if( db->xWhereInfo && db->init.busy==0 ){
-    void (*x)(void*, int, const char*, int, i64) = db->xWhereInfo;
+    void (*x)(void*, int, const char*, int, u64) = db->xWhereInfo;
     void *pCtx = db->pWhereInfoCtx;
     int ii;
-    int nTab = p->pWInfo->pTabList->nSrc;
+    SrcList *pTabList = p->pWInfo->pTabList;
 
     /* Loop through each element of the FROM clause. Ignore any sub-selects
     ** or views. Invoke the xWhereInfo() callback multiple times for each
     ** real table.  */
-    for(ii=0; ii<p->pWInfo->pTabList->nSrc; ii++){
-      struct SrcList_item *pItem = &p->pWInfo->pTabList->a[ii];
+    for(ii=0; ii<pTabList->nSrc; ii++){
+      struct SrcList_item *pItem = &pTabList->a[ii];
       if( pItem->pSelect==0 ){
         Table *pTab = pItem->pTab;
         int iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
@@ -4357,6 +4461,9 @@ static void whereTraceBuilder(
 
         /* WHERE callbacks */
         whereTraceWC(pParse, pItem, p->pWC);
+
+        /* OR-clause processing */
+        whereTraceOR(pParse, pItem, p->pWC);
       }
     }
   }
diff --git a/test/shell6.test b/test/shell6.test
index 19b094a80d..c44ccf0bd1 100644
--- a/test/shell6.test
+++ b/test/shell6.test
@@ -188,4 +188,18 @@ do_setup_rec_test 10.1 {
   0|1|0|SEARCH TABLE t5 USING COVERING INDEX t5_idx_000123a7 (a=? AND b=?)
 }
 
+# OR terms.
+#
+do_setup_rec_test 11.1 {
+  CREATE TABLE t7(a, b);
+} {
+  SELECT * FROM t7 WHERE a=? OR b=?
+} {
+  CREATE INDEX t7_idx_00000061 ON t7(a) 
+  CREATE INDEX t7_idx_00000062 ON t7(b)
+  0|0|0|SEARCH TABLE t7 USING INDEX t7_idx_00000061 (a=?) 
+  0|0|0|SEARCH TABLE t7 USING INDEX t7_idx_00000062 (b=?)
+}
+
 finish_test
+

From d8ac297233b9af6cc74f92a14834eaacda781676 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 6 Apr 2017 18:44:18 +0000
Subject: [PATCH 020/488] Changes to allow indexes to be recommended for
 queries on SQL views.

FossilOrigin-Name: 0884ff1da2e27b146c764b73cf299a1f2cfe213c4a79bde34dec02d1fc946e70
---
 manifest            |  14 ++---
 manifest.uuid       |   2 +-
 src/shell_indexes.c | 148 +++++++-------------------------------------
 test/shell6.test    |  32 +++++-----
 4 files changed, 47 insertions(+), 149 deletions(-)

diff --git a/manifest b/manifest
index 0e690d166d..ce8fc15893 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\ssome\ssupport\sfor\sOR\sterms\sto\ssqlite3_whereinfo_hook().
-D 2017-04-04T17:50:31.913
+C Changes\sto\sallow\sindexes\sto\sbe\srecommended\sfor\squeries\son\sSQL\sviews.
+D 2017-04-06T18:44:18.391
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a
@@ -403,7 +403,7 @@ F src/resolve.c 3e518b962d932a997fae373366880fc028c75706
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 2496d0cc6368dabe7ad2e4c7f5ed3ad9aa3b4d11cd90f33fa1d1ef72493f43aa
 F src/shell.c e524688c2544167f835ba43e24309f8707ca60c8ab6eb5c263a12c8618a233b8
-F src/shell_indexes.c d40ea0a81112df7bdccd7232238bee0bbb39699085ea78cc08fd863bf052a63b
+F src/shell_indexes.c 1f5ab036ec189411aeea27e6e74ab0009d831764d5d8517455dcb6b6a734beb7
 F src/sqlite.h.in ae5c9cbf2e77492c319fca08769575d9695e64718a16d32324944d24e291bcf7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 58fd0676d3111d02e62e5a35992a7d3da5d3f88753acc174f2d37b774fbbdd28
@@ -1120,7 +1120,7 @@ F test/shell2.test e242a9912f44f4c23c3d1d802a83e934e84c853b
 F test/shell3.test 9b95ba643eaa228376f06a898fb410ee9b6e57c1
 F test/shell4.test 89ad573879a745974ff2df20ff97c5d6ffffbd5d
 F test/shell5.test 50a732c1c2158b1cd62cf53975ce1ea7ce6b9dc9
-F test/shell6.test 081067c1afcb38da50134ffd5ccc0a59ede14f41959486f733ffbba689c0ccfa
+F test/shell6.test f37998b26dfde19beaaf06a4cb60c476f66a7b54affff3870a2a011402c13efc
 F test/shell7.test 07751911b294698e0c5df67bcbd29e7d2f0f2907
 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3
 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5
@@ -1571,7 +1571,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a54aef35da11f7508a8888a159372036a362fc52afa1df752dc835db334c4330
-R 51e547bc1c25ea79dd375d8f020224f3
+P 5cd070000da1d9e399090677b4db75dc5639c33211385d6eb84f14a4d0b617cd
+R aa13547c61fd31f43629472eaf9b4008
 U dan
-Z fe378068686ea107c39185021a17b4ae
+Z 2f3b5e2a515dd18e4123d763066c9a33
diff --git a/manifest.uuid b/manifest.uuid
index 7669c54b05..3b9f56b1a1 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5cd070000da1d9e399090677b4db75dc5639c33211385d6eb84f14a4d0b617cd
\ No newline at end of file
+0884ff1da2e27b146c764b73cf299a1f2cfe213c4a79bde34dec02d1fc946e70
\ No newline at end of file
diff --git a/src/shell_indexes.c b/src/shell_indexes.c
index e061f04d3f..c05672e9e0 100644
--- a/src/shell_indexes.c
+++ b/src/shell_indexes.c
@@ -43,44 +43,18 @@ struct IdxConstraint {
 /*
 ** A WHERE clause. Made up of IdxConstraint objects. Example WHERE clause:
 **
-**   a=? AND b=? AND ((c=? AND d=?) OR e=?) AND (f=? OR g=?) AND h>?
+**   a=? AND b=? AND c=? AND d=? AND e>? AND f<?
 **
-** The above is decomposed into 5 AND connected clauses. The first two are
+** The above is decomposed into 6 AND connected clauses. The first four are
 ** added to the IdxWhere.pEq linked list, the following two into 
-** IdxWhere.pOr and the last into IdxWhere.pRange.
+** IdxWhere.pRange.
 **
 ** IdxWhere.pEq and IdxWhere.pRange are simple linked lists of IdxConstraint
 ** objects linked by the IdxConstraint.pNext field.
-**
-** The list headed at IdxWhere.pOr and linked by IdxWhere.pNextOr contains
-** all "OR" terms that belong to the current WHERE clause. In the example
-** above, there are two OR terms:
-**
-**   ((c=? AND d=?) OR e=?)
-**   (f=? OR g=?)
-**
-** Within an OR term, the OR connected sub-expressions are termed siblings.
-** These are connected into a linked list by the pSibling pointers. Each OR
-** term above consists of two siblings.
-**
-**   pOr -> (c=? AND d=?) -> pNextOr -> (f=?)
-**               |                        |
-**            pSibling                 pSibling
-**               |                        |
-**               V                        V
-**             (e=?)                    (g=?)
-**
-** IdxWhere.pParent is only used while constructing a tree of IdxWhere 
-** structures. It is NULL for the root IdxWhere. For all others, the parent
-** WHERE clause.
 */
 struct IdxWhere {
   IdxConstraint *pEq;             /* List of == constraints */
   IdxConstraint *pRange;          /* List of < constraints */
-  IdxWhere *pOr;                  /* List of OR constraints */
-  IdxWhere *pNextOr;              /* Next in OR constraints of same IdxWhere */
-  IdxWhere *pSibling;             /* Next branch in single OR constraint */
-  IdxWhere *pParent;              /* Parent object (or NULL) */
 };
 
 /*
@@ -246,40 +220,6 @@ static void idxDatabaseError(
   *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
 }
 
-static char *idxQueryToList(
-  sqlite3 *db, 
-  const char *zBind,
-  int *pRc,
-  char **pzErrmsg,
-  const char *zSql
-){
-  char *zRet = 0;
-  if( *pRc==SQLITE_OK ){
-    sqlite3_stmt *pStmt = 0;
-    int rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
-    if( rc==SQLITE_OK ){
-      sqlite3_bind_text(pStmt, 1, zBind, -1, SQLITE_TRANSIENT);
-      while( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){
-        const char *z = (const char*)sqlite3_column_text(pStmt, 0);
-        zRet = sqlite3_mprintf("%z%s%Q", zRet, zRet?", ":"", z);
-        if( zRet==0 ){
-          rc = SQLITE_NOMEM;
-        }
-      }
-      rc = sqlite3_finalize(pStmt);
-    }
-
-    if( rc ){
-      idxDatabaseError(db, pzErrmsg);
-      sqlite3_free(zRet);
-      zRet = 0;
-    }
-    *pRc = rc;
-  }
-
-  return zRet;
-}
-
 static int idxPrepareStmt(
   sqlite3 *db,                    /* Database handle to compile against */
   sqlite3_stmt **ppStmt,          /* OUT: Compiled SQL statement */
@@ -331,7 +271,7 @@ static int idxGetTableInfo(
 
   rc = idxPrintfPrepareStmt(db, &p1, pzErrmsg, "PRAGMA table_info=%Q", zTbl);
   while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){
-    const char *zCol = sqlite3_column_text(p1, 1);
+    const char *zCol = (const char*)sqlite3_column_text(p1, 1);
     nByte += 1 + strlen(zCol);
     rc = sqlite3_table_column_metadata(
         db, "main", zTbl, zCol, 0, &zCol, 0, 0, 0
@@ -354,7 +294,7 @@ static int idxGetTableInfo(
 
   nCol = 0;
   while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){
-    const char *zCol = sqlite3_column_text(p1, 1);
+    const char *zCol = (const char*)sqlite3_column_text(p1, 1);
     int nCopy = strlen(zCol) + 1;
     pNew->aCol[nCol].zName = pCsr;
     pNew->aCol[nCol].iPk = sqlite3_column_int(p1, 5);
@@ -392,38 +332,24 @@ static int idxCreateTables(
   char **pzErrmsg                 /* OUT: Error message */
 ){
   int rc = SQLITE_OK;
+  int rc2;
   IdxScan *pIter;
+  sqlite3_stmt *pSql = 0;
+
+  /* Copy the entire schema of database [db] into [dbm]. */
+  rc = idxPrintfPrepareStmt(db, &pSql, pzErrmsg, 
+      "SELECT sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%%'"
+  );
+  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
+    const char *zSql = (const char*)sqlite3_column_text(pSql, 0);
+    rc = sqlite3_exec(dbm, zSql, 0, 0, pzErrmsg);
+  }
+  rc2 = sqlite3_finalize(pSql);
+  if( rc==SQLITE_OK ) rc = rc2;
+
+  /* Load IdxTable objects */
   for(pIter=pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
     rc = idxGetTableInfo(db, pIter, pzErrmsg);
-
-    /* Test if table has already been created. If so, jump to the next
-    ** iteration of the loop.  */
-    if( rc==SQLITE_OK ){
-      sqlite3_stmt *pSql = 0;
-      rc = idxPrintfPrepareStmt(dbm, &pSql, pzErrmsg, 
-          "SELECT 1 FROM sqlite_master WHERE tbl_name = %Q", pIter->zTable
-      );
-      if( rc==SQLITE_OK ){
-        int bSkip = 0;
-        if( sqlite3_step(pSql)==SQLITE_ROW ) bSkip = 1;
-        rc = sqlite3_finalize(pSql);
-        if( bSkip ) continue;
-      }
-    }
-
-    if( rc==SQLITE_OK ){
-      int rc2;
-      sqlite3_stmt *pSql = 0;
-      rc = idxPrintfPrepareStmt(db, &pSql, pzErrmsg, 
-          "SELECT sql FROM sqlite_master WHERE tbl_name = %Q", pIter->zTable
-      );
-      while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
-        const char *zSql = (const char*)sqlite3_column_text(pSql, 0);
-        rc = sqlite3_exec(dbm, zSql, 0, 0, pzErrmsg);
-      }
-      rc2 = sqlite3_finalize(pSql);
-      if( rc==SQLITE_OK ) rc = rc2;
-    }
   }
   return rc;
 }
@@ -655,26 +581,6 @@ static int idxCreateFromWhere(
     IdxContext*, i64, IdxScan*, IdxWhere*, IdxConstraint*, IdxConstraint*
 );
 
-static int idxCreateForeachOr(
-  IdxContext *pCtx, 
-  i64 mask,                       /* Consider only these constraints */
-  IdxScan *pScan,                 /* Create indexes for this scan */
-  IdxWhere *pWhere,               /* Read constraints from here */
-  IdxConstraint *pEq,             /* == constraints for inclusion */
-  IdxConstraint *pTail            /* range/ORDER BY constraints for inclusion */
-){
-  int rc = SQLITE_OK;
-  IdxWhere *p1;
-  IdxWhere *p2;
-  for(p1=pWhere->pOr; p1 && rc==SQLITE_OK; p1=p1->pNextOr){
-    rc = idxCreateFromWhere(pCtx, mask, pScan, p1, pEq, pTail);
-    for(p2=p1->pSibling; p2 && rc==SQLITE_OK; p2=p2->pSibling){
-      rc = idxCreateFromWhere(pCtx, mask, pScan, p2, pEq, pTail);
-    }
-  }
-  return rc;
-}
-
 /*
 ** Return true if list pList (linked by IdxConstraint.pLink) contains
 ** a constraint compatible with *p. Otherwise return false.
@@ -695,7 +601,6 @@ static int idxCreateFromWhere(
   IdxConstraint *pEq,             /* == constraints for inclusion */
   IdxConstraint *pTail            /* range/ORDER BY constraints for inclusion */
 ){
-  sqlite3 *dbm = pCtx->dbm;
   IdxConstraint *p1 = pEq;
   IdxConstraint *pCon;
   int rc;
@@ -714,9 +619,6 @@ static int idxCreateFromWhere(
   /* Create an index using the == constraints collected above. And the
   ** range constraint/ORDER BY terms passed in by the caller, if any. */
   rc = idxCreateFromCons(pCtx, pScan, p1, pTail);
-  if( rc==SQLITE_OK ){
-    rc = idxCreateForeachOr(pCtx, mask, pScan, pWhere, p1, pTail);
-  }
 
   /* If no range/ORDER BY passed by the caller, create a version of the
   ** index for each range constraint that matches the mask. */
@@ -728,9 +630,6 @@ static int idxCreateFromWhere(
         && idxFindConstraint(pTail, pCon)==0
       ){
         rc = idxCreateFromCons(pCtx, pScan, p1, pCon);
-        if( rc==SQLITE_OK ){
-          rc = idxCreateForeachOr(pCtx, mask, pScan, pWhere, p1, pCon);
-        }
       }
     }
   }
@@ -749,7 +648,7 @@ static int idxCreateCandidates(IdxContext *pCtx){
   sqlite3_stmt *pDepmask;         /* Foreach depmask */
   IdxScan *pIter;
 
-  rc = idxPrepareStmt(pCtx->dbm, &pDepmask, pCtx->pzErrmsg, 
+  rc = idxPrepareStmt(dbm, &pDepmask, pCtx->pzErrmsg, 
       "SELECT mask FROM depmask"
   );
 
@@ -774,7 +673,6 @@ static void idxScanFree(IdxScan *pScan){
   IdxScan *pNext;
   for(pIter=pScan; pIter; pIter=pNext){
     pNext = pIter->pNextScan;
-
   }
 }
 
@@ -841,11 +739,11 @@ int idxFindIndexes(
   if( rc==SQLITE_OK ) rc = rc2;
   if( rc==SQLITE_OK ){
     sqlite3_stmt *pLoop = 0;
-    rc = idxPrepareStmt(dbm, &pLoop, pzErr, "SELECT name FROM aux.indexes");
+    rc = idxPrepareStmt(dbm, &pLoop, pzErr,"SELECT name||';' FROM aux.indexes");
     if( rc==SQLITE_OK ){
       while( SQLITE_ROW==sqlite3_step(pLoop) ){
         bFound = 1;
-        xOut(pOutCtx, sqlite3_column_text(pLoop, 0));
+        xOut(pOutCtx, (const char*)sqlite3_column_text(pLoop, 0));
       }
       rc = sqlite3_finalize(pLoop);
     }
diff --git a/test/shell6.test b/test/shell6.test
index c44ccf0bd1..2f7c251827 100644
--- a/test/shell6.test
+++ b/test/shell6.test
@@ -66,7 +66,7 @@ do_setup_rec_test 1.2 {
 } {
   SELECT * FROM t1 WHERE b>?;
 } {
-  CREATE INDEX t1_idx_00000062 ON t1(b) 
+  CREATE INDEX t1_idx_00000062 ON t1(b);
   0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_00000062 (b>?)
 }
 
@@ -75,7 +75,7 @@ do_setup_rec_test 1.3 {
 } {
   SELECT * FROM t1 WHERE b COLLATE nocase BETWEEN ? AND ?
 } {
-  CREATE INDEX t1_idx_3e094c27 ON t1(b COLLATE NOCASE) 
+  CREATE INDEX t1_idx_3e094c27 ON t1(b COLLATE NOCASE);
   0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_3e094c27 (b>? AND b<?)
 }
 
@@ -84,7 +84,7 @@ do_setup_rec_test 1.4 {
 } {
   SELECT a FROM t1 ORDER BY b;
 } {
-  CREATE INDEX t1_idx_00000062 ON t1(b) 
+  CREATE INDEX t1_idx_00000062 ON t1(b);
   0|0|0|SCAN TABLE t1 USING INDEX t1_idx_00000062
 }
 
@@ -93,7 +93,7 @@ do_setup_rec_test 1.5 {
 } {
   SELECT a FROM t1 WHERE a=? ORDER BY b;
 } {
-  CREATE INDEX t1_idx_000123a7 ON t1(a, b) 
+  CREATE INDEX t1_idx_000123a7 ON t1(a, b);
   0|0|0|SEARCH TABLE t1 USING COVERING INDEX t1_idx_000123a7 (a=?)
 }
 
@@ -102,7 +102,7 @@ do_setup_rec_test 1.6 {
 } {
   SELECT min(a) FROM t1
 } {
-  CREATE INDEX t1_idx_00000061 ON t1(a) 
+  CREATE INDEX t1_idx_00000061 ON t1(a);
   0|0|0|SEARCH TABLE t1 USING COVERING INDEX t1_idx_00000061
 }
 
@@ -111,7 +111,7 @@ do_setup_rec_test 1.7 {
 } {
   SELECT * FROM t1 ORDER BY a, b, c;
 } {
-  CREATE INDEX t1_idx_033e95fe ON t1(a, b, c) 
+  CREATE INDEX t1_idx_033e95fe ON t1(a, b, c);
   0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_033e95fe
 }
 
@@ -120,7 +120,7 @@ do_setup_rec_test 1.8 {
 } {
   SELECT * FROM t1 ORDER BY a ASC, b COLLATE nocase DESC, c ASC;
 } {
-  CREATE INDEX t1_idx_5be6e222 ON t1(a, b COLLATE NOCASE DESC, c)
+  CREATE INDEX t1_idx_5be6e222 ON t1(a, b COLLATE NOCASE DESC, c);
   0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_5be6e222
 }
 
@@ -129,7 +129,7 @@ do_setup_rec_test 1.9 {
 } {
   SELECT * FROM t1 WHERE a=?
 } {
-  CREATE INDEX t1_idx_00000061 ON t1(a) 
+  CREATE INDEX t1_idx_00000061 ON t1(a);
   0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_00000061 (a=?)
 }
 
@@ -141,7 +141,7 @@ do_setup_rec_test 8.1 {
 } {
   SELECT * FROM "t t" WHERE a=?
 } {
-  CREATE INDEX 't t_idx_00000061' ON 't t'(a)
+  CREATE INDEX 't t_idx_00000061' ON 't t'(a);
   0|0|0|SEARCH TABLE t t USING INDEX t t_idx_00000061 (a=?) 
 }
 
@@ -150,7 +150,7 @@ do_setup_rec_test 8.2 {
 } {
   SELECT * FROM "t t" WHERE b BETWEEN ? AND ?
 } {
-  CREATE INDEX 't t_idx_00000062' ON 't t'(b) 
+  CREATE INDEX 't t_idx_00000062' ON 't t'(b);
   0|0|0|SEARCH TABLE t t USING INDEX t t_idx_00000062 (b>? AND b<?)
 }
 
@@ -161,7 +161,7 @@ do_setup_rec_test 9.1 {
 } {
   SELECT * FROM t3 WHERE "b b" = ?
 } {
-  CREATE INDEX t3_idx_00050c52 ON t3('b b') 
+  CREATE INDEX t3_idx_00050c52 ON t3('b b');
   0|0|0|SEARCH TABLE t3 USING INDEX t3_idx_00050c52 (b b=?)
 }
 
@@ -170,7 +170,7 @@ do_setup_rec_test 9.2 {
 } {
   SELECT * FROM t3 ORDER BY "b b"
 } {
-  CREATE INDEX t3_idx_00050c52 ON t3('b b') 
+  CREATE INDEX t3_idx_00050c52 ON t3('b b');
   0|0|0|SCAN TABLE t3 USING INDEX t3_idx_00050c52
 }
 
@@ -182,8 +182,8 @@ do_setup_rec_test 10.1 {
 } {
   SELECT * FROM t5, t6 WHERE a=? AND b=c AND c=?
 } {
-  CREATE INDEX t5_idx_000123a7 ON t5(a, b) 
-  CREATE INDEX t6_idx_00000063 ON t6(c) 
+  CREATE INDEX t5_idx_000123a7 ON t5(a, b);
+  CREATE INDEX t6_idx_00000063 ON t6(c);
   0|0|1|SEARCH TABLE t6 USING INDEX t6_idx_00000063 (c=?) 
   0|1|0|SEARCH TABLE t5 USING COVERING INDEX t5_idx_000123a7 (a=? AND b=?)
 }
@@ -195,8 +195,8 @@ do_setup_rec_test 11.1 {
 } {
   SELECT * FROM t7 WHERE a=? OR b=?
 } {
-  CREATE INDEX t7_idx_00000061 ON t7(a) 
-  CREATE INDEX t7_idx_00000062 ON t7(b)
+  CREATE INDEX t7_idx_00000061 ON t7(a);
+  CREATE INDEX t7_idx_00000062 ON t7(b);
   0|0|0|SEARCH TABLE t7 USING INDEX t7_idx_00000061 (a=?) 
   0|0|0|SEARCH TABLE t7 USING INDEX t7_idx_00000062 (b=?)
 }

From 8e0b8e06884c56eb0e275d3f3cfb18510de84db8 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 7 Apr 2017 20:14:22 +0000
Subject: [PATCH 021/488] Refactor code to suggest indexes from the shell tool
 into an extension in ext/expert. Unfinished.

FossilOrigin-Name: 305e19f976ca064638a294e0817bf547ea745e1eb74746c5855514e6ced9c5fa
---
 ext/expert/expert.c                           | 104 ++++
 .../expert/sqlite3expert.c                    | 476 +++++++++++++-----
 ext/expert/sqlite3expert.h                    |  57 +++
 main.mk                                       |   6 +-
 manifest                                      |  18 +-
 manifest.uuid                                 |   2 +-
 src/shell.c                                   |  34 --
 7 files changed, 518 insertions(+), 179 deletions(-)
 create mode 100644 ext/expert/expert.c
 rename src/shell_indexes.c => ext/expert/sqlite3expert.c (67%)
 create mode 100644 ext/expert/sqlite3expert.h

diff --git a/ext/expert/expert.c b/ext/expert/expert.c
new file mode 100644
index 0000000000..5a5cda185f
--- /dev/null
+++ b/ext/expert/expert.c
@@ -0,0 +1,104 @@
+/*
+** 2017 April 07
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+*/
+
+
+#include <sqlite3.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "sqlite3expert.h"
+
+
+static void option_requires_argument(const char *zOpt){
+  fprintf(stderr, "Option requires an argument: %s\n", zOpt);
+  exit(-3);
+}
+
+static void usage(char **argv){
+  fprintf(stderr, "\n");
+  fprintf(stderr, "Usage %s ?OPTIONS? DATABASE\n", argv[0]);
+  fprintf(stderr, "\n");
+  fprintf(stderr, "Options are:\n");
+  fprintf(stderr, "  -sql SQL   (analyze SQL statements passed as argument)\n");
+  fprintf(stderr, "  -file FILE (read SQL statements from file FILE)\n");
+  exit(-1);
+}
+
+static int readSqlFromFile(sqlite3expert *p, const char *zFile, char **pzErr){
+  return SQLITE_OK;
+}
+
+int main(int argc, char **argv){
+  const char *zDb;
+  int rc = 0;
+  char *zErr = 0;
+  int i;
+
+  sqlite3 *db = 0;
+  sqlite3expert *p = 0;
+
+  if( argc<2 ) usage(argv);
+  zDb = argv[argc-1];
+  rc = sqlite3_open(zDb, &db);
+  if( rc!=SQLITE_OK ){
+    fprintf(stderr, "Cannot open db file: %s - %s\n", zDb, sqlite3_errmsg(db));
+    exit(-2);
+  }
+
+  p = sqlite3_expert_new(db, &zErr);
+  if( p==0 ){
+    fprintf(stderr, "Cannot run analysis: %s\n", zErr);
+    rc = 1;
+  }else{
+    for(i=1; i<(argc-1); i++){
+      char *zArg = argv[i];
+      int nArg = strlen(zArg);
+      if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-file", nArg) ){
+        if( ++i==(argc-1) ) option_requires_argument("-file");
+        rc = readSqlFromFile(p, argv[i], &zErr);
+      }
+
+      else if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-sql", nArg) ){
+        if( ++i==(argc-1) ) option_requires_argument("-sql");
+        rc = sqlite3_expert_sql(p, argv[i], &zErr);
+      }
+
+      else{
+        usage(argv);
+      }
+    }
+  }
+
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_expert_analyze(p, &zErr);
+  }
+
+  if( rc==SQLITE_OK ){
+    int nQuery = sqlite3_expert_count(p);
+    for(i=0; i<nQuery; i++){
+      const char *zSql = sqlite3_expert_report(p, i, EXPERT_REPORT_SQL);
+      const char *zIdx = sqlite3_expert_report(p, i, EXPERT_REPORT_INDEXES);
+      const char *zEQP = sqlite3_expert_report(p, i, EXPERT_REPORT_PLAN);
+      fprintf(stdout, "-- query %d ----------------------------------\n", i+1);
+      fprintf(stdout, "%s\n\n%s\n%s\n", zSql, zIdx, zEQP);
+    }
+  }else if( zErr ){
+    fprintf(stderr, "Error: %s\n", zErr);
+  }
+
+  sqlite3_expert_destroy(p);
+  sqlite3_free(zErr);
+  return rc;
+}
+
+
diff --git a/src/shell_indexes.c b/ext/expert/sqlite3expert.c
similarity index 67%
rename from src/shell_indexes.c
rename to ext/expert/sqlite3expert.c
index c05672e9e0..3bcfbbc6e8 100644
--- a/src/shell_indexes.c
+++ b/ext/expert/sqlite3expert.c
@@ -11,12 +11,18 @@
 *************************************************************************
 */
 
+#include "sqlite3expert.h"
+#include <assert.h>
+#include <string.h>
+#include <stdio.h>
+
 typedef sqlite3_int64 i64;
 typedef sqlite3_uint64 u64;
 
 typedef struct IdxConstraint IdxConstraint;
 typedef struct IdxContext IdxContext;
 typedef struct IdxScan IdxScan;
+typedef struct IdxStatement IdxStatement;
 typedef struct IdxWhere IdxWhere;
 
 typedef struct IdxColumn IdxColumn;
@@ -70,6 +76,19 @@ struct IdxScan {
   IdxScan *pNextScan;             /* Next IdxScan object for same query */
 };
 
+/*
+** Data regarding a database table. Extracted from "PRAGMA table_info"
+*/
+struct IdxColumn {
+  char *zName;
+  char *zColl;
+  int iPk;
+};
+struct IdxTable {
+  int nCol;
+  IdxColumn *aCol;
+};
+
 /*
 ** Context object passed to idxWhereInfo() and other functions.
 */
@@ -84,19 +103,31 @@ struct IdxContext {
   i64 iIdxRowid;                  /* Rowid of first index created */
 };
 
+struct IdxStatement {
+  int iId;                        /* Statement number */
+  char *zSql;                     /* SQL statement */
+  char *zIdx;                     /* Indexes */
+  char *zEQP;                     /* Plan */
+  IdxStatement *pNext;
+};
+
 /*
-** Data regarding a database table. Extracted from "PRAGMA table_info"
+** sqlite3expert object.
 */
-struct IdxColumn {
-  char *zName;
-  char *zColl;
-  int iPk;
-};
-struct IdxTable {
-  int nCol;
-  IdxColumn *aCol;
+struct sqlite3expert {
+  sqlite3 *db;                    /* Users database */
+  sqlite3 *dbm;                   /* In-memory db for this analysis */
+
+  int bRun;                       /* True once analysis has run */
+  char **pzErrmsg;
+
+  IdxScan *pScan;                 /* List of scan objects */
+  IdxStatement *pStatement;       /* List of IdxStatement objects */
+  int rc;                         /* Error code from whereinfo hook */
+  i64 iIdxRowid;                  /* Rowid of first index created */
 };
 
+
 /*
 ** Allocate and return nByte bytes of zeroed memory using sqlite3_malloc(). 
 ** If the allocation fails, set *pRc to SQLITE_NOMEM and return NULL.
@@ -132,7 +163,7 @@ static IdxConstraint *idxNewConstraint(int *pRc, const char *zColl){
 }
 
 /*
-** SQLITE_DBCONFIG_WHEREINFO callback.
+** sqlite3_whereinfo_hook() callback.
 */
 static void idxWhereInfo(
   void *pCtx,                     /* Pointer to IdxContext structure */
@@ -141,7 +172,7 @@ static void idxWhereInfo(
   int iVal, 
   u64 mask
 ){
-  IdxContext *p = (IdxContext*)pCtx;
+  sqlite3expert *p = (sqlite3expert*)pCtx;
 
 #if 0
   const char *zOp = 
@@ -165,7 +196,6 @@ static void idxWhereInfo(
         pNew->pNextScan = p->pScan;
         pNew->covering = mask;
         p->pScan = pNew;
-        p->pCurrent = &pNew->where;
         break;
       }
 
@@ -193,16 +223,17 @@ static void idxWhereInfo(
         pNew->depmask = mask;
 
         if( eOp==SQLITE_WHEREINFO_RANGE ){
-          pNew->pNext = p->pCurrent->pRange;
-          p->pCurrent->pRange = pNew;
+          pNew->pNext = p->pScan->where.pRange;
+          p->pScan->where.pRange = pNew;
         }else{
-          pNew->pNext = p->pCurrent->pEq;
-          p->pCurrent->pEq = pNew;
+          pNew->pNext = p->pScan->where.pEq;
+          p->pScan->where.pEq = pNew;
         }
-
+#if 0
         sqlite3_bind_int64(p->pInsertMask, 1, mask);
         sqlite3_step(p->pInsertMask);
         p->rc = sqlite3_reset(p->pInsertMask);
+#endif
         break;
       }
     }
@@ -325,35 +356,6 @@ static int idxGetTableInfo(
   return rc;
 }
 
-static int idxCreateTables(
-  sqlite3 *db,                    /* User database */
-  sqlite3 *dbm,                   /* In-memory database to create tables in */
-  IdxScan *pScan,                 /* List of scans */
-  char **pzErrmsg                 /* OUT: Error message */
-){
-  int rc = SQLITE_OK;
-  int rc2;
-  IdxScan *pIter;
-  sqlite3_stmt *pSql = 0;
-
-  /* Copy the entire schema of database [db] into [dbm]. */
-  rc = idxPrintfPrepareStmt(db, &pSql, pzErrmsg, 
-      "SELECT sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%%'"
-  );
-  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
-    const char *zSql = (const char*)sqlite3_column_text(pSql, 0);
-    rc = sqlite3_exec(dbm, zSql, 0, 0, pzErrmsg);
-  }
-  rc2 = sqlite3_finalize(pSql);
-  if( rc==SQLITE_OK ) rc = rc2;
-
-  /* Load IdxTable objects */
-  for(pIter=pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
-    rc = idxGetTableInfo(db, pIter, pzErrmsg);
-  }
-  return rc;
-}
-
 /*
 ** This function is a no-op if *pRc is set to anything other than 
 ** SQLITE_OK when it is called.
@@ -456,7 +458,6 @@ static int idxFindCompatible(
   int nEq = 0;                    /* Number of elements in pEq */
   int rc, rc2;
 
-
   /* Count the elements in list pEq */
   for(pIter=pEq; pIter; pIter=pIter->pLink) nEq++;
 
@@ -514,12 +515,12 @@ static int idxFindCompatible(
 }
 
 static int idxCreateFromCons(
-  IdxContext *pCtx,
+  sqlite3expert *p,
   IdxScan *pScan,
   IdxConstraint *pEq, 
   IdxConstraint *pTail
 ){
-  sqlite3 *dbm = pCtx->dbm;
+  sqlite3 *dbm = p->dbm;
   int rc = SQLITE_OK;
   if( (pEq || pTail) && 0==idxFindCompatible(&rc, dbm, pScan, pEq, pTail) ){
     IdxTable *pTab = pScan->pTable;
@@ -552,20 +553,20 @@ static int idxCreateFromCons(
       if( !zIdx ){
         rc = SQLITE_NOMEM;
       }else{
-        rc = sqlite3_exec(dbm, zIdx, 0, 0, pCtx->pzErrmsg);
-#if 0
+        rc = sqlite3_exec(dbm, zIdx, 0, 0, p->pzErrmsg);
+#if 1
         printf("CANDIDATE: %s\n", zIdx);
 #endif
       }
     }
-    if( rc==SQLITE_OK && pCtx->iIdxRowid==0 ){
+    if( rc==SQLITE_OK && p->iIdxRowid==0 ){
       int rc2;
       sqlite3_stmt *pLast = 0;
-      rc = idxPrepareStmt(dbm, &pLast, pCtx->pzErrmsg, 
+      rc = idxPrepareStmt(dbm, &pLast, p->pzErrmsg, 
           "SELECT max(rowid) FROM sqlite_master"
       );
       if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLast) ){
-        pCtx->iIdxRowid = sqlite3_column_int64(pLast, 0);
+        p->iIdxRowid = sqlite3_column_int64(pLast, 0);
       }
       rc2 = sqlite3_finalize(pLast);
       if( rc==SQLITE_OK ) rc = rc2;
@@ -578,7 +579,7 @@ static int idxCreateFromCons(
 }
 
 static int idxCreateFromWhere(
-    IdxContext*, i64, IdxScan*, IdxWhere*, IdxConstraint*, IdxConstraint*
+    sqlite3expert*, i64, IdxScan*, IdxWhere*, IdxConstraint*, IdxConstraint*
 );
 
 /*
@@ -594,7 +595,7 @@ static int idxFindConstraint(IdxConstraint *pList, IdxConstraint *p){
 }
 
 static int idxCreateFromWhere(
-  IdxContext *pCtx, 
+  sqlite3expert *p, 
   i64 mask,                       /* Consider only these constraints */
   IdxScan *pScan,                 /* Create indexes for this scan */
   IdxWhere *pWhere,               /* Read constraints from here */
@@ -618,7 +619,7 @@ static int idxCreateFromWhere(
 
   /* Create an index using the == constraints collected above. And the
   ** range constraint/ORDER BY terms passed in by the caller, if any. */
-  rc = idxCreateFromCons(pCtx, pScan, p1, pTail);
+  rc = idxCreateFromCons(p, pScan, p1, pTail);
 
   /* If no range/ORDER BY passed by the caller, create a version of the
   ** index for each range constraint that matches the mask. */
@@ -629,7 +630,7 @@ static int idxCreateFromWhere(
         && idxFindConstraint(pEq, pCon)==0
         && idxFindConstraint(pTail, pCon)==0
       ){
-        rc = idxCreateFromCons(pCtx, pScan, p1, pCon);
+        rc = idxCreateFromCons(p, pScan, p1, pCon);
       }
     }
   }
@@ -641,56 +642,90 @@ static int idxCreateFromWhere(
 ** Create candidate indexes in database [dbm] based on the data in 
 ** linked-list pScan.
 */
-static int idxCreateCandidates(IdxContext *pCtx){
-  sqlite3 *dbm = pCtx->dbm;
+static int idxCreateCandidates(sqlite3expert *p, char **pzErr){
+  sqlite3 *dbm = p->dbm;
   int rc2;
   int rc = SQLITE_OK;
-  sqlite3_stmt *pDepmask;         /* Foreach depmask */
+  sqlite3_stmt *pDepmask = 0;     /* Foreach depmask */
+  sqlite3_stmt *pInsert = 0;      /* insert */
   IdxScan *pIter;
 
-  rc = idxPrepareStmt(dbm, &pDepmask, pCtx->pzErrmsg, 
-      "SELECT mask FROM depmask"
+  rc = idxPrepareStmt(dbm, &pInsert, pzErr, 
+      "INSERT OR IGNORE INTO aux.depmask SELECT mask | ?1 FROM aux.depmask;"
   );
+  if( rc==SQLITE_OK ){
+    rc = idxPrepareStmt(dbm, &pDepmask, pzErr, "SELECT mask FROM depmask");
+  }
 
-  for(pIter=pCtx->pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
+  for(pIter=p->pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
     IdxWhere *pWhere = &pIter->where;
+    IdxConstraint *pCons;
+    rc = sqlite3_exec(dbm, 
+        "DELETE FROM aux.depmask;"
+        "INSERT INTO aux.depmask VALUES(0);"
+        , 0, 0, pzErr
+    );
+    for(pCons=pIter->where.pEq; pCons; pCons=pCons->pNext){
+      sqlite3_bind_int64(pInsert, 1, pCons->depmask);
+      sqlite3_step(pInsert);
+      rc = sqlite3_reset(pInsert);
+    }
+
     while( SQLITE_ROW==sqlite3_step(pDepmask) && rc==SQLITE_OK ){
       i64 mask = sqlite3_column_int64(pDepmask, 0);
-      rc = idxCreateFromWhere(pCtx, mask, pIter, pWhere, 0, 0);
+      rc = idxCreateFromWhere(p, mask, pIter, pWhere, 0, 0);
       if( rc==SQLITE_OK && pIter->pOrder ){
-        rc = idxCreateFromWhere(pCtx, mask, pIter, pWhere, 0, pIter->pOrder);
+        rc = idxCreateFromWhere(p, mask, pIter, pWhere, 0, pIter->pOrder);
       }
     }
+    rc2 = sqlite3_reset(pDepmask);
+    if( rc==SQLITE_OK ) rc = rc2;
   }
 
   rc2 = sqlite3_finalize(pDepmask);
   if( rc==SQLITE_OK ) rc = rc2;
+  rc2 = sqlite3_finalize(pInsert);
+  if( rc==SQLITE_OK ) rc = rc2;
   return rc;
 }
 
-static void idxScanFree(IdxScan *pScan){
-  IdxScan *pIter;
-  IdxScan *pNext;
-  for(pIter=pScan; pIter; pIter=pNext){
-    pNext = pIter->pNextScan;
-  }
+/*
+** Free all elements of the linked list starting from pScan up until pLast
+** (pLast is not freed).
+*/
+static void idxScanFree(IdxScan *pScan, IdxScan *pLast){
+  /* TODO! */
 }
 
+/*
+** Free all elements of the linked list starting from pStatement up 
+** until pLast (pLast is not freed).
+*/
+static void idxStatementFree(IdxStatement *pStatement, IdxStatement *pLast){
+  /* TODO! */
+}
+
+static void idxFinalize(int *pRc, sqlite3_stmt *pStmt){
+  int rc = sqlite3_finalize(pStmt);
+  if( *pRc==SQLITE_OK ) *pRc = rc;
+}
+static void idxReset(int *pRc, sqlite3_stmt *pStmt){
+  int rc = sqlite3_reset(pStmt);
+  if( *pRc==SQLITE_OK ) *pRc = rc;
+}
+
+
 int idxFindIndexes(
-  IdxContext *pCtx,
-  const char *zSql,                    /* SQL to find indexes for */
-  void (*xOut)(void*, const char*),    /* Output callback */
-  void *pOutCtx,                       /* Context for xOut() */
+  sqlite3expert *p,
   char **pzErr                         /* OUT: Error message (sqlite3_malloc) */
 ){
-  sqlite3 *dbm = pCtx->dbm;
-  sqlite3_stmt *pExplain = 0;
+  IdxStatement *pStmt;
+  sqlite3 *dbm = p->dbm;
   sqlite3_stmt *pSelect = 0;
   sqlite3_stmt *pInsert = 0;
   int rc, rc2;
   int bFound = 0;
 
-  rc = idxPrintfPrepareStmt(dbm, &pExplain, pzErr,"EXPLAIN QUERY PLAN %s",zSql);
   if( rc==SQLITE_OK ){
     rc = idxPrepareStmt(dbm, &pSelect, pzErr, 
         "SELECT rowid, sql FROM sqlite_master WHERE name = ?"
@@ -702,76 +737,81 @@ int idxFindIndexes(
     );
   }
 
-  while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){
-    int i;
-    const char *zDetail = (const char*)sqlite3_column_text(pExplain, 3);
-    int nDetail = strlen(zDetail);
+  for(pStmt=p->pStatement; rc==SQLITE_OK && pStmt; pStmt=pStmt->pNext){
+    sqlite3_stmt *pExplain = 0;
+    rc = sqlite3_exec(dbm, "DELETE FROM aux.indexes", 0, 0, 0);
+    if( rc==SQLITE_OK ){
+      rc = idxPrintfPrepareStmt(dbm, &pExplain, pzErr,
+          "EXPLAIN QUERY PLAN %s", pStmt->zSql
+      );
+    }
+    while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){
+      int i;
+      const char *zDetail = (const char*)sqlite3_column_text(pExplain, 3);
+      int nDetail = strlen(zDetail);
 
-    for(i=0; i<nDetail; i++){
-      const char *zIdx = 0;
-      if( memcmp(&zDetail[i], " USING INDEX ", 13)==0 ){
-        zIdx = &zDetail[i+13];
-      }else if( memcmp(&zDetail[i], " USING COVERING INDEX ", 22)==0 ){
-        zIdx = &zDetail[i+22];
-      }
-      if( zIdx ){
-        int nIdx = 0;
-        while( zIdx[nIdx]!='\0' && (zIdx[nIdx]!=' ' || zIdx[nIdx+1]!='(') ){
-          nIdx++;
+      for(i=0; i<nDetail; i++){
+        const char *zIdx = 0;
+        if( memcmp(&zDetail[i], " USING INDEX ", 13)==0 ){
+          zIdx = &zDetail[i+13];
+        }else if( memcmp(&zDetail[i], " USING COVERING INDEX ", 22)==0 ){
+          zIdx = &zDetail[i+22];
         }
-        sqlite3_bind_text(pSelect, 1, zIdx, nIdx, SQLITE_STATIC);
-        if( SQLITE_ROW==sqlite3_step(pSelect) ){
-          i64 iRowid = sqlite3_column_int64(pSelect, 0);
-          const char *zSql = (const char*)sqlite3_column_text(pSelect, 1);
-          if( iRowid>=pCtx->iIdxRowid ){
-            sqlite3_bind_text(pInsert, 1, zSql, -1, SQLITE_STATIC);
-            sqlite3_step(pInsert);
-            rc = sqlite3_reset(pInsert);
-            if( rc ) goto find_indexes_out;
+        if( zIdx ){
+          int nIdx = 0;
+          while( zIdx[nIdx]!='\0' && (zIdx[nIdx]!=' ' || zIdx[nIdx+1]!='(') ){
+            nIdx++;
           }
+          sqlite3_bind_text(pSelect, 1, zIdx, nIdx, SQLITE_STATIC);
+          if( SQLITE_ROW==sqlite3_step(pSelect) ){
+            i64 iRowid = sqlite3_column_int64(pSelect, 0);
+            const char *zSql = (const char*)sqlite3_column_text(pSelect, 1);
+            if( iRowid>=p->iIdxRowid ){
+              sqlite3_bind_text(pInsert, 1, zSql, -1, SQLITE_STATIC);
+              sqlite3_step(pInsert);
+              rc = sqlite3_reset(pInsert);
+              if( rc ) goto find_indexes_out;
+            }
+          }
+          rc = sqlite3_reset(pSelect);
+          break;
         }
-        rc = sqlite3_reset(pSelect);
-        break;
       }
     }
-  }
-  rc2 = sqlite3_reset(pExplain);
-  if( rc==SQLITE_OK ) rc = rc2;
-  if( rc==SQLITE_OK ){
-    sqlite3_stmt *pLoop = 0;
-    rc = idxPrepareStmt(dbm, &pLoop, pzErr,"SELECT name||';' FROM aux.indexes");
+    idxReset(&rc, pExplain);
     if( rc==SQLITE_OK ){
-      while( SQLITE_ROW==sqlite3_step(pLoop) ){
-        bFound = 1;
-        xOut(pOutCtx, (const char*)sqlite3_column_text(pLoop, 0));
+      sqlite3_stmt *pLoop = 0;
+      rc = idxPrepareStmt(dbm,&pLoop,pzErr,"SELECT name||';' FROM aux.indexes");
+      if( rc==SQLITE_OK ){
+        while( SQLITE_ROW==sqlite3_step(pLoop) ){
+          bFound = 1;
+          pStmt->zIdx = idxAppendText(&rc, pStmt->zIdx, "%s\n",
+              (const char*)sqlite3_column_text(pLoop, 0)
+          );
+        }
+        idxFinalize(&rc, pLoop);
+      }
+      if( bFound==0 ){
+        pStmt->zIdx = idxAppendText(&rc, pStmt->zIdx, "(no new indexes)\n");
       }
-      rc = sqlite3_finalize(pLoop);
     }
-    if( rc==SQLITE_OK ){
-      if( bFound==0 ) xOut(pOutCtx, "(no new indexes)");
-      xOut(pOutCtx, "");
-    }
-  }
 
-  while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){
-    int iSelectid = sqlite3_column_int(pExplain, 0);
-    int iOrder = sqlite3_column_int(pExplain, 1);
-    int iFrom = sqlite3_column_int(pExplain, 2);
-    const char *zDetail = (const char*)sqlite3_column_text(pExplain, 3);
-    char *zOut;
+    while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){
+      int iSelectid = sqlite3_column_int(pExplain, 0);
+      int iOrder = sqlite3_column_int(pExplain, 1);
+      int iFrom = sqlite3_column_int(pExplain, 2);
+      const char *zDetail = (const char*)sqlite3_column_text(pExplain, 3);
 
-    zOut = sqlite3_mprintf("%d|%d|%d|%s", iSelectid, iOrder, iFrom, zDetail);
-    if( zOut==0 ){
-      rc = SQLITE_NOMEM;
-    }else{
-      xOut(pOutCtx, zOut);
-      sqlite3_free(zOut);
+      pStmt->zEQP = idxAppendText(&rc, pStmt->zEQP, "%d|%d|%d|%s\n", 
+          iSelectid, iOrder, iFrom, zDetail
+      );
     }
+
+    rc2 = sqlite3_finalize(pExplain);
+    if( rc==SQLITE_OK ) rc = rc2;
   }
 
  find_indexes_out:
-  rc2 = sqlite3_finalize(pExplain);
-  if( rc==SQLITE_OK ) rc = rc2;
   rc2 = sqlite3_finalize(pSelect);
   if( rc==SQLITE_OK ) rc = rc2;
   rc2 = sqlite3_finalize(pInsert);
@@ -794,6 +834,7 @@ int shellIndexesCommand(
   char **pzErrmsg                      /* OUT: Error message (sqlite3_malloc) */
 ){
   int rc = SQLITE_OK;
+#if 0
   sqlite3 *dbm = 0;
   IdxContext ctx;
   sqlite3_stmt *pStmt = 0;        /* Statement compiled from zSql */
@@ -851,10 +892,175 @@ int shellIndexesCommand(
     rc = idxFindIndexes(&ctx, zSql, xOut, pOutCtx, pzErrmsg);
   }
 
-  idxScanFree(ctx.pScan);
+  idxScanFree(ctx.pScan, 0);
   sqlite3_finalize(ctx.pInsertMask);
   sqlite3_close(dbm);
+#endif
   return rc;
 }
 
+/*************************************************************************/
+
+/*
+** Allocate a new sqlite3expert object.
+*/
+sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){
+  int rc = SQLITE_OK;
+  sqlite3expert *pNew;
+
+  pNew = (sqlite3expert*)idxMalloc(&rc, sizeof(sqlite3expert));
+  pNew->db = db;
+
+  /* Open an in-memory database to work with. The main in-memory 
+  ** database schema contains tables similar to those in the users 
+  ** database (handle db). The attached in-memory db (aux) contains
+  ** application tables used by the code in this file.  */
+  rc = sqlite3_open(":memory:", &pNew->dbm);
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_exec(pNew->dbm,
+        "ATTACH ':memory:' AS aux;"
+        "CREATE TABLE aux.depmask(mask PRIMARY KEY) WITHOUT ROWID;"
+        "CREATE TABLE aux.indexes(name PRIMARY KEY) WITHOUT ROWID;"
+        , 0, 0, pzErrmsg
+    );
+  }
+
+  /* Copy the entire schema of database [db] into [dbm]. */
+  if( rc==SQLITE_OK ){
+    sqlite3_stmt *pSql;
+    int rc2;
+    rc = idxPrintfPrepareStmt(pNew->db, &pSql, pzErrmsg, 
+        "SELECT sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%%'"
+    );
+    while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
+      const char *zSql = (const char*)sqlite3_column_text(pSql, 0);
+      rc = sqlite3_exec(pNew->dbm, zSql, 0, 0, pzErrmsg);
+    }
+    rc2 = sqlite3_finalize(pSql);
+    if( rc==SQLITE_OK ) rc = rc2;
+  }
+
+  /* If an error has occurred, free the new object and reutrn NULL. Otherwise,
+  ** return the new sqlite3expert handle.  */
+  if( rc!=SQLITE_OK ){
+    sqlite3_expert_destroy(pNew);
+    pNew = 0;
+  }
+  return pNew;
+}
+
+/*
+** Add an SQL statement to the analysis.
+*/
+int sqlite3_expert_sql(
+  sqlite3expert *p,               /* From sqlite3_expert_new() */
+  const char *zSql,               /* SQL statement to add */
+  char **pzErr                    /* OUT: Error message (if any) */
+){
+  IdxScan *pScanOrig = p->pScan;
+  IdxStatement *pStmtOrig = p->pStatement;
+  int rc = SQLITE_OK;
+  const char *zStmt = zSql;
+
+  if( p->bRun ) return SQLITE_MISUSE;
+
+  sqlite3_whereinfo_hook(p->db, idxWhereInfo, p);
+  while( rc==SQLITE_OK && zStmt && zStmt[0] ){
+    sqlite3_stmt *pStmt = 0;
+    rc = sqlite3_prepare_v2(p->db, zStmt, -1, &pStmt, &zStmt);
+    if( rc==SQLITE_OK ){
+      if( pStmt ){
+        IdxStatement *pNew;
+        const char *z = sqlite3_sql(pStmt);
+        int n = strlen(z);
+        pNew = (IdxStatement*)idxMalloc(&rc, sizeof(IdxStatement) + n+1);
+        if( rc==SQLITE_OK ){
+          pNew->zSql = (char*)&pNew[1];
+          memcpy(pNew->zSql, z, n+1);
+          pNew->pNext = p->pStatement;
+          if( p->pStatement ) pNew->iId = p->pStatement->iId+1;
+          p->pStatement = pNew;
+        }
+        sqlite3_finalize(pStmt);
+      }
+    }else{
+      idxDatabaseError(p->db, pzErr);
+    }
+  }
+  sqlite3_whereinfo_hook(p->db, 0, 0);
+
+  if( rc!=SQLITE_OK ){
+    idxScanFree(p->pScan, pScanOrig);
+    idxStatementFree(p->pStatement, pStmtOrig);
+    p->pScan = pScanOrig;
+    p->pStatement = pStmtOrig;
+  }
+
+  return rc;
+}
+
+int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr){
+  int rc = SQLITE_OK;
+  IdxScan *pIter;
+
+  /* Load IdxTable objects */
+  for(pIter=p->pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
+    rc = idxGetTableInfo(p->dbm, pIter, pzErr);
+  }
+
+
+  /* Create candidate indexes within the in-memory database file */
+  if( rc==SQLITE_OK ){
+    rc = idxCreateCandidates(p, pzErr);
+  }
+
+  /* Figure out which of the candidate indexes are preferred by the query
+  ** planner and report the results to the user.  */
+  if( rc==SQLITE_OK ){
+    rc = idxFindIndexes(p, pzErr);
+  }
+
+  if( rc==SQLITE_OK ){
+    p->bRun = 1;
+  }
+  return rc;
+}
+
+int sqlite3_expert_count(sqlite3expert *p){
+  int nRet = 0;
+  if( p->pStatement ) nRet = p->pStatement->iId+1;
+  return nRet;
+}
+
+const char *sqlite3_expert_report(sqlite3expert *p, int iStmt, int eReport){
+  const char *zRet = 0;
+  IdxStatement *pStmt;
+
+  if( p->bRun==0 ) return 0;
+  for(pStmt=p->pStatement; pStmt && pStmt->iId!=iStmt; pStmt=pStmt->pNext);
+  if( pStmt ){
+    switch( eReport ){
+      case EXPERT_REPORT_SQL:
+        zRet = pStmt->zSql;
+        break;
+      case EXPERT_REPORT_INDEXES:
+        zRet = pStmt->zIdx;
+        break;
+      case EXPERT_REPORT_PLAN:
+        zRet = pStmt->zEQP;
+        break;
+    }
+  }
+  return zRet;
+}
+
+/*
+** Free an sqlite3expert object.
+*/
+void sqlite3_expert_destroy(sqlite3expert *p){
+  sqlite3_close(p->dbm);
+  idxScanFree(p->pScan, 0);
+  idxStatementFree(p->pStatement, 0);
+  sqlite3_free(p);
+}
 
diff --git a/ext/expert/sqlite3expert.h b/ext/expert/sqlite3expert.h
new file mode 100644
index 0000000000..6384f8e556
--- /dev/null
+++ b/ext/expert/sqlite3expert.h
@@ -0,0 +1,57 @@
+/*
+** 2017 April 07
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+*/
+
+
+#include "sqlite3.h"
+
+typedef struct sqlite3expert sqlite3expert;
+
+/*
+** Create a new sqlite3expert object.
+*/
+sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErr);
+
+/*
+** Add an SQL statement to the analysis.
+*/
+int sqlite3_expert_sql(
+  sqlite3expert *p,               /* From sqlite3_expert_new() */
+  const char *zSql,               /* SQL statement to add */
+  char **pzErr                    /* OUT: Error message (if any) */
+);
+
+int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr);
+
+/*
+** Return the total number of SQL queries loaded via sqlite3_expert_sql().
+*/
+int sqlite3_expert_count(sqlite3expert*);
+
+/*
+** Return a component of the report.
+*/
+const char *sqlite3_expert_report(sqlite3expert*, int iStmt, int eReport);
+
+/*
+** Values for the third argument passed to sqlite3_expert_report().
+*/
+#define EXPERT_REPORT_SQL        1
+#define EXPERT_REPORT_INDEXES    2
+#define EXPERT_REPORT_PLAN       3
+
+/*
+** Free an (sqlite3expert*) handle allocated by sqlite3-expert_new().
+*/
+void sqlite3_expert_destroy(sqlite3expert*);
+
+
diff --git a/main.mk b/main.mk
index b22d07d5a2..cb41cb7e00 100644
--- a/main.mk
+++ b/main.mk
@@ -491,7 +491,7 @@ libsqlite3.a:	$(LIBOBJ)
 	$(AR) libsqlite3.a $(LIBOBJ)
 	$(RANLIB) libsqlite3.a
 
-sqlite3$(EXE):	$(TOP)/src/shell.c libsqlite3.a sqlite3.h $(TOP)/src/shell_indexes.c
+sqlite3$(EXE):	$(TOP)/src/shell.c libsqlite3.a sqlite3.h
 	$(TCCX) $(READLINE_FLAGS) -o sqlite3$(EXE) $(SHELL_OPT) \
 		$(TOP)/src/shell.c libsqlite3.a $(LIBREADLINE) $(TLIBS) $(THREADLIB)
 
@@ -761,6 +761,9 @@ sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl
 sqlite3_analyzer$(EXE): sqlite3_analyzer.c
 	$(TCCX) $(TCL_FLAGS) sqlite3_analyzer.c -o $@ $(LIBTCL) $(THREADLIB) 
 
+sqlite3_expert$(EXE): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c
+	$(TCCX) -DSQLITE_ENABLE_WHEREINFO_HOOK $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert $(THREADLIB)
+
 sqlite3_schemalint.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/schemalint.tcl
 	echo "#define TCLSH 2" > $@
 	echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@
@@ -979,6 +982,7 @@ clean:
 	rm -f sqlite3rc.h
 	rm -f shell.c sqlite3ext.h
 	rm -f sqlite3_analyzer sqlite3_analyzer.exe sqlite3_analyzer.c
+	rm -f sqlite3_expert sqlite3_expert.exe 
 	rm -f sqlite-*-output.vsix
 	rm -f mptester mptester.exe
 	rm -f fuzzershell fuzzershell.exe
diff --git a/manifest b/manifest
index ce8fc15893..8a7b97520c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Changes\sto\sallow\sindexes\sto\sbe\srecommended\sfor\squeries\son\sSQL\sviews.
-D 2017-04-06T18:44:18.391
+C Refactor\scode\sto\ssuggest\sindexes\sfrom\sthe\sshell\stool\sinto\san\sextension\sin\next/expert.\sUnfinished.
+D 2017-04-07T20:14:22.417
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a
@@ -40,6 +40,9 @@ F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd
 F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
+F ext/expert/expert.c c73a0da702a2e9f5fd48046ab182683a73ee0318cefa3f45826536d015f39021
+F ext/expert/sqlite3expert.c 75ee320cf38b50df331232cbb60ec3b7703dd2770cc8df4ebbb445f664f6827d
+F ext/expert/sqlite3expert.h feeaee4ab73ba52426329781bbb28032ce18cf5abd2bf6221bac2df4c32b3013
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
 F ext/fts1/ft_hash.c 3927bd880e65329bdc6f506555b228b28924921b
 F ext/fts1/ft_hash.h 06df7bba40dadd19597aa400a875dbc2fed705ea
@@ -325,7 +328,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk e6f8f6895b75e4615a284b546933ca88fd56f3b85b560f4fb0da731abaa14f80
+F main.mk 6d4f1f1f78a6ac453d35f18c4e696fdefbe65dfec9530a41c5579ef8ec076072
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -402,8 +405,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 3e518b962d932a997fae373366880fc028c75706
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 2496d0cc6368dabe7ad2e4c7f5ed3ad9aa3b4d11cd90f33fa1d1ef72493f43aa
-F src/shell.c e524688c2544167f835ba43e24309f8707ca60c8ab6eb5c263a12c8618a233b8
-F src/shell_indexes.c 1f5ab036ec189411aeea27e6e74ab0009d831764d5d8517455dcb6b6a734beb7
+F src/shell.c ceb2b2f1f958ea2c47a7f37972d0f715fbf9dcf6a34a5e98c886b85e3ce6a238
 F src/sqlite.h.in ae5c9cbf2e77492c319fca08769575d9695e64718a16d32324944d24e291bcf7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 58fd0676d3111d02e62e5a35992a7d3da5d3f88753acc174f2d37b774fbbdd28
@@ -1571,7 +1573,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5cd070000da1d9e399090677b4db75dc5639c33211385d6eb84f14a4d0b617cd
-R aa13547c61fd31f43629472eaf9b4008
+P 0884ff1da2e27b146c764b73cf299a1f2cfe213c4a79bde34dec02d1fc946e70
+R 2f1dd4cb0b48a4b54a46f30181a94d12
 U dan
-Z 2f3b5e2a515dd18e4123d763066c9a33
+Z bc46ca2174a3c856c4daa1344a9d88d3
diff --git a/manifest.uuid b/manifest.uuid
index 3b9f56b1a1..9b195887ca 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0884ff1da2e27b146c764b73cf299a1f2cfe213c4a79bde34dec02d1fc946e70
\ No newline at end of file
+305e19f976ca064638a294e0817bf547ea745e1eb74746c5855514e6ced9c5fa
\ No newline at end of file
diff --git a/src/shell.c b/src/shell.c
index e51694bc76..1236b12a0e 100644
--- a/src/shell.c
+++ b/src/shell.c
@@ -166,7 +166,6 @@ static void setTextMode(FILE *file, int isOutput){
 # define setTextMode(X,Y)
 #endif
 
-#include "shell_indexes.c"
 
 /* True if the timer is enabled */
 static int enableTimer = 0;
@@ -1364,7 +1363,6 @@ struct ShellState {
   sqlite3 *db;           /* The database */
   int autoExplain;       /* Automatically turn on .explain mode */
   int autoEQP;           /* Run EXPLAIN QUERY PLAN prior to seach SQL stmt */
-  int bRecommend;        /* Instead of sqlite3_exec(), recommend indexes */
   int statsOn;           /* True to display memory stats before each finalize */
   int scanstatsOn;       /* True to display scan stats before each finalize */
   int outCount;          /* Revert to stdout when reaching zero */
@@ -2586,19 +2584,6 @@ static void explain_data_delete(ShellState *p){
   p->iIndent = 0;
 }
 
-typedef struct RecCommandCtx RecCommandCtx;
-struct RecCommandCtx {
-  int (*xCallback)(void*,int,char**,char**,int*);
-  ShellState *pArg;
-};
-
-static void recCommandOut(void *pCtx, const char *zLine){
-  const char *zCol = "output";
-  RecCommandCtx *p = (RecCommandCtx*)pCtx;
-  int t = SQLITE_TEXT;
-  p->xCallback(p->pArg, 1, (char**)&zLine, (char**)&zCol, &t);
-}
-
 /*
 ** Disable and restore .wheretrace and .selecttrace settings.
 */
@@ -2723,13 +2708,6 @@ static int shell_exec(
     *pzErrMsg = NULL;
   }
 
-  if( pArg->bRecommend ){
-    RecCommandCtx ctx;
-    ctx.xCallback = xCallback;
-    ctx.pArg = pArg;
-    rc = shellIndexesCommand(db, zSql, recCommandOut, &ctx, pzErrMsg);
-  }else
-
   while( zSql[0] && (SQLITE_OK == rc) ){
     static const char *zStmtSql;
     rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zLeftover);
@@ -5522,15 +5500,6 @@ static int do_meta_command(char *zLine, ShellState *p){
     sqlite3_close(pSrc);
   }else
 
-  if( c=='r' && n>=2 && strncmp(azArg[0], "recommend", n)==0 ){
-    if( nArg==2 ){
-      p->bRecommend = booleanValue(azArg[1]);
-    }else{
-      raw_printf(stderr, "Usage: .recommend on|off\n");
-      rc = 1;
-    }
-  }else
-
 
   if( c=='s' && strncmp(azArg[0], "scanstats", n)==0 ){
     if( nArg==2 ){
@@ -7337,9 +7306,6 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
           if( bail_on_error ) return rc;
         }
       }
-
-    }else if( strcmp(z, "-recommend") ){
-      data.bRecommend = 1;
     }else{
       utf8_printf(stderr,"%s: Error: unknown option: %s\n", Argv0, z);
       raw_printf(stderr,"Use -help for a list of options.\n");

From 65e67ed1b234403355c6f06404c4162493d5f6c7 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 8 Apr 2017 17:41:24 +0000
Subject: [PATCH 022/488] Use hash tables instead of in-memory database tables
 for a few purposes in sqlite3expert.c.

FossilOrigin-Name: bf10e68d9e4d5eae7ae6148a7ad64c9596f2ed8ccd36065adb09a1f9e7dae50b
---
 ext/expert/sqlite3expert.c | 438 +++++++++++++++++--------------------
 manifest                   |  12 +-
 manifest.uuid              |   2 +-
 3 files changed, 209 insertions(+), 243 deletions(-)

diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index 3bcfbbc6e8..fb442a0b3c 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -100,7 +100,6 @@ struct IdxContext {
   sqlite3 *dbm;                   /* In-memory db for this analysis */
   sqlite3 *db;                    /* User database under analysis */
   sqlite3_stmt *pInsertMask;      /* To write to aux.depmask */
-  i64 iIdxRowid;                  /* Rowid of first index created */
 };
 
 struct IdxStatement {
@@ -111,6 +110,21 @@ struct IdxStatement {
   IdxStatement *pNext;
 };
 
+
+#define IDX_HASH_SIZE 1023
+typedef struct IdxHashEntry IdxHashEntry;
+typedef struct IdxHash IdxHash;
+struct IdxHashEntry {
+  char *zKey;                     /* nul-terminated key */
+  char *zVal;                     /* nul-terminated value string */
+  IdxHashEntry *pHashNext;        /* Next entry in same hash bucket */
+  IdxHashEntry *pNext;            /* Next entry in hash */
+};
+struct IdxHash {
+  IdxHashEntry *pFirst;
+  IdxHashEntry *aHash[IDX_HASH_SIZE];
+};
+
 /*
 ** sqlite3expert object.
 */
@@ -124,7 +138,8 @@ struct sqlite3expert {
   IdxScan *pScan;                 /* List of scan objects */
   IdxStatement *pStatement;       /* List of IdxStatement objects */
   int rc;                         /* Error code from whereinfo hook */
-  i64 iIdxRowid;                  /* Rowid of first index created */
+
+  IdxHash hIdx;                   /* Hash containing all candidate indexes */
 };
 
 
@@ -145,6 +160,126 @@ static void *idxMalloc(int *pRc, int nByte){
   return pRet;
 }
 
+/*************************************************************************
+** Start of hash table implementations.
+*/
+typedef struct IdxHash64Entry IdxHash64Entry;
+typedef struct IdxHash64 IdxHash64;
+struct IdxHash64Entry {
+  u64 iVal;
+  IdxHash64Entry *pNext;          /* Next entry in hash table */
+  IdxHash64Entry *pHashNext;      /* Next entry in same hash bucket */
+};
+struct IdxHash64 {
+  IdxHash64Entry *pFirst;         /* Most recently added entry in hash table */
+  IdxHash64Entry *aHash[IDX_HASH_SIZE];
+};
+
+static void idxHash64Init(IdxHash64 *pHash){
+  memset(pHash, 0, sizeof(IdxHash64));
+}
+static void idxHash64Clear(IdxHash64 *pHash){
+  IdxHash64Entry *pEntry;
+  IdxHash64Entry *pNext;
+  for(pEntry=pHash->pFirst; pEntry; pEntry=pNext){
+    pNext = pEntry->pNext;
+    sqlite3_free(pEntry);
+  }
+  memset(pHash, 0, sizeof(IdxHash64));
+}
+static void idxHash64Add(int *pRc, IdxHash64 *pHash, u64 iVal){
+  int iHash = (int)((iVal*7) % IDX_HASH_SIZE);
+  IdxHash64Entry *pEntry;
+  assert( iHash>=0 );
+
+  for(pEntry=pHash->aHash[iHash]; pEntry; pEntry=pEntry->pHashNext){
+    if( pEntry->iVal==iVal ) return;
+  }
+  pEntry = idxMalloc(pRc, sizeof(IdxHash64Entry));
+  if( pEntry ){
+    pEntry->iVal = iVal;
+    pEntry->pHashNext = pHash->aHash[iHash];
+    pHash->aHash[iHash] = pEntry;
+    pEntry->pNext = pHash->pFirst;
+    pHash->pFirst = pEntry;
+  }
+}
+
+static void idxHashInit(IdxHash *pHash){
+  memset(pHash, 0, sizeof(IdxHash));
+}
+static void idxHashClear(IdxHash *pHash){
+  int i;
+  for(i=0; i<IDX_HASH_SIZE; i++){
+    IdxHashEntry *pEntry;
+    IdxHashEntry *pNext;
+    for(pEntry=pHash->aHash[i]; pEntry; pEntry=pNext){
+      pNext = pEntry->pHashNext;
+      sqlite3_free(pEntry);
+    }
+  }
+  memset(pHash, 0, sizeof(IdxHash));
+}
+static int idxHashString(const char *z, int n){
+  unsigned int ret = 0;
+  int i;
+  for(i=0; i<n; i++){
+    ret += (ret<<3) + (unsigned char)(z[i]);
+  }
+  return (int)(ret % IDX_HASH_SIZE);
+}
+
+static int idxHashAdd(
+  int *pRc, 
+  IdxHash *pHash, 
+  const char *zKey,
+  const char *zVal
+){
+  int nKey = strlen(zKey);
+  int iHash = idxHashString(zKey, nKey);
+  int nVal = (zVal ? strlen(zVal) : 0);
+  IdxHashEntry *pEntry;
+  assert( iHash>=0 );
+  for(pEntry=pHash->aHash[iHash]; pEntry; pEntry=pEntry->pHashNext){
+    if( strlen(pEntry->zKey)==nKey && 0==memcmp(pEntry->zKey, zKey, nKey) ){
+      return 1;
+    }
+  }
+  pEntry = idxMalloc(pRc, sizeof(IdxHashEntry) + nKey+1 + nVal+1);
+  if( pEntry ){
+    pEntry->zKey = (char*)&pEntry[1];
+    memcpy(pEntry->zKey, zKey, nKey);
+    if( zVal ){
+      pEntry->zVal = &pEntry->zKey[nKey+1];
+      memcpy(pEntry->zVal, zVal, nVal);
+    }
+    pEntry->pHashNext = pHash->aHash[iHash];
+    pHash->aHash[iHash] = pEntry;
+
+    pEntry->pNext = pHash->pFirst;
+    pHash->pFirst = pEntry;
+  }
+  return 0;
+}
+
+static const char *idxHashSearch(IdxHash *pHash, const char *zKey, int nKey){
+  int iHash;
+  IdxHashEntry *pEntry;
+  if( nKey<0 ) nKey = strlen(zKey);
+  iHash = idxHashString(zKey, nKey);
+  assert( iHash>=0 );
+  for(pEntry=pHash->aHash[iHash]; pEntry; pEntry=pEntry->pHashNext){
+    if( strlen(pEntry->zKey)==nKey && 0==memcmp(pEntry->zKey, zKey, nKey) ){
+      return pEntry->zVal;
+    }
+  }
+  return 0;
+}
+
+/*
+** End of hash table implementations.
+**************************************************************************/
+
 /*
 ** Allocate and return a new IdxConstraint object. Set the IdxConstraint.zColl
 ** variable to point to a copy of nul-terminated string zColl.
@@ -229,11 +364,6 @@ static void idxWhereInfo(
           pNew->pNext = p->pScan->where.pEq;
           p->pScan->where.pEq = pNew;
         }
-#if 0
-        sqlite3_bind_int64(p->pInsertMask, 1, mask);
-        sqlite3_step(p->pInsertMask);
-        p->rc = sqlite3_reset(p->pInsertMask);
-#endif
         break;
       }
     }
@@ -287,6 +417,11 @@ static int idxPrintfPrepareStmt(
   return rc;
 }
 
+static void idxFinalize(int *pRc, sqlite3_stmt *pStmt){
+  int rc = sqlite3_finalize(pStmt);
+  if( *pRc==SQLITE_OK ) *pRc = rc;
+}
+
 static int idxGetTableInfo(
   sqlite3 *db,
   IdxScan *pScan,
@@ -344,8 +479,7 @@ static int idxGetTableInfo(
 
     nCol++;
   }
-  rc2 = sqlite3_finalize(p1);
-  if( rc==SQLITE_OK ) rc = rc2;
+  idxFinalize(&rc, p1);
 
   if( rc==SQLITE_OK ){
     pScan->pTable = pNew;
@@ -456,7 +590,7 @@ static int idxFindCompatible(
   sqlite3_stmt *pIdxList = 0;
   IdxConstraint *pIter;
   int nEq = 0;                    /* Number of elements in pEq */
-  int rc, rc2;
+  int rc;
 
   /* Count the elements in list pEq */
   for(pIter=pEq; pIter; pIter=pIter->pLink) nEq++;
@@ -499,16 +633,14 @@ static int idxFindCompatible(
         }
       }
     }
-    rc2 = sqlite3_finalize(pInfo);
-    if( rc==SQLITE_OK ) rc = rc2;
+    idxFinalize(&rc, pInfo);
 
     if( rc==SQLITE_OK && bMatch ){
       sqlite3_finalize(pIdxList);
       return 1;
     }
   }
-  rc2 = sqlite3_finalize(pIdxList);
-  if( rc==SQLITE_OK ) rc = rc2;
+  idxFinalize(&rc, pIdxList);
 
   *pRc = rc;
   return 0;
@@ -539,49 +671,37 @@ static int idxCreateFromCons(
 
     if( rc==SQLITE_OK ){
       /* Hash the list of columns to come up with a name for the index */
+      char *zName;                /* Index name */
       int i;
       for(i=0; zCols[i]; i++){
         h += ((h<<3) + zCols[i]);
       }
-
-      if( idxIdentifierRequiresQuotes(pScan->zTable) ){
-        zFmt = "CREATE INDEX '%q_idx_%08x' ON %Q(%s)";
-      }else{
-        zFmt = "CREATE INDEX %s_idx_%08x ON %s(%s)";
-      }
-      zIdx = sqlite3_mprintf(zFmt, pScan->zTable, h, pScan->zTable, zCols);
-      if( !zIdx ){
+      zName = sqlite3_mprintf("%s_idx_%08x", pScan->zTable, h);
+      if( zName==0 ){ 
         rc = SQLITE_NOMEM;
       }else{
-        rc = sqlite3_exec(dbm, zIdx, 0, 0, p->pzErrmsg);
-#if 1
-        printf("CANDIDATE: %s\n", zIdx);
-#endif
+        if( idxIdentifierRequiresQuotes(pScan->zTable) ){
+          zFmt = "CREATE INDEX '%q' ON %Q(%s)";
+        }else{
+          zFmt = "CREATE INDEX %s ON %s(%s)";
+        }
+        zIdx = sqlite3_mprintf(zFmt, zName, pScan->zTable, zCols);
+        if( !zIdx ){
+          rc = SQLITE_NOMEM;
+        }else{
+          rc = sqlite3_exec(dbm, zIdx, 0, 0, p->pzErrmsg);
+          idxHashAdd(&rc, &p->hIdx, zName, zIdx);
+        }
+        sqlite3_free(zName);
+        sqlite3_free(zIdx);
       }
     }
-    if( rc==SQLITE_OK && p->iIdxRowid==0 ){
-      int rc2;
-      sqlite3_stmt *pLast = 0;
-      rc = idxPrepareStmt(dbm, &pLast, p->pzErrmsg, 
-          "SELECT max(rowid) FROM sqlite_master"
-      );
-      if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLast) ){
-        p->iIdxRowid = sqlite3_column_int64(pLast, 0);
-      }
-      rc2 = sqlite3_finalize(pLast);
-      if( rc==SQLITE_OK ) rc = rc2;
-    }
 
-    sqlite3_free(zIdx);
     sqlite3_free(zCols);
   }
   return rc;
 }
 
-static int idxCreateFromWhere(
-    sqlite3expert*, i64, IdxScan*, IdxWhere*, IdxConstraint*, IdxConstraint*
-);
-
 /*
 ** Return true if list pList (linked by IdxConstraint.pLink) contains
 ** a constraint compatible with *p. Otherwise return false.
@@ -643,49 +763,34 @@ static int idxCreateFromWhere(
 ** linked-list pScan.
 */
 static int idxCreateCandidates(sqlite3expert *p, char **pzErr){
-  sqlite3 *dbm = p->dbm;
-  int rc2;
   int rc = SQLITE_OK;
-  sqlite3_stmt *pDepmask = 0;     /* Foreach depmask */
-  sqlite3_stmt *pInsert = 0;      /* insert */
   IdxScan *pIter;
-
-  rc = idxPrepareStmt(dbm, &pInsert, pzErr, 
-      "INSERT OR IGNORE INTO aux.depmask SELECT mask | ?1 FROM aux.depmask;"
-  );
-  if( rc==SQLITE_OK ){
-    rc = idxPrepareStmt(dbm, &pDepmask, pzErr, "SELECT mask FROM depmask");
-  }
+  IdxHash64 hMask;
+  idxHash64Init(&hMask);
 
   for(pIter=p->pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
+    IdxHash64Entry *pEntry;
     IdxWhere *pWhere = &pIter->where;
     IdxConstraint *pCons;
-    rc = sqlite3_exec(dbm, 
-        "DELETE FROM aux.depmask;"
-        "INSERT INTO aux.depmask VALUES(0);"
-        , 0, 0, pzErr
-    );
+
+    idxHash64Add(&rc, &hMask, 0);
     for(pCons=pIter->where.pEq; pCons; pCons=pCons->pNext){
-      sqlite3_bind_int64(pInsert, 1, pCons->depmask);
-      sqlite3_step(pInsert);
-      rc = sqlite3_reset(pInsert);
+      for(pEntry=hMask.pFirst; pEntry; pEntry=pEntry->pNext){
+        idxHash64Add(&rc, &hMask, pEntry->iVal | (u64)pCons->depmask);
+      }
     }
 
-    while( SQLITE_ROW==sqlite3_step(pDepmask) && rc==SQLITE_OK ){
-      i64 mask = sqlite3_column_int64(pDepmask, 0);
+    for(pEntry=hMask.pFirst; pEntry; pEntry=pEntry->pNext){
+      i64 mask = (i64)pEntry->iVal;
       rc = idxCreateFromWhere(p, mask, pIter, pWhere, 0, 0);
       if( rc==SQLITE_OK && pIter->pOrder ){
         rc = idxCreateFromWhere(p, mask, pIter, pWhere, 0, pIter->pOrder);
       }
     }
-    rc2 = sqlite3_reset(pDepmask);
-    if( rc==SQLITE_OK ) rc = rc2;
+
+    idxHash64Clear(&hMask);
   }
 
-  rc2 = sqlite3_finalize(pDepmask);
-  if( rc==SQLITE_OK ) rc = rc2;
-  rc2 = sqlite3_finalize(pInsert);
-  if( rc==SQLITE_OK ) rc = rc2;
   return rc;
 }
 
@@ -705,15 +810,6 @@ static void idxStatementFree(IdxStatement *pStatement, IdxStatement *pLast){
   /* TODO! */
 }
 
-static void idxFinalize(int *pRc, sqlite3_stmt *pStmt){
-  int rc = sqlite3_finalize(pStmt);
-  if( *pRc==SQLITE_OK ) *pRc = rc;
-}
-static void idxReset(int *pRc, sqlite3_stmt *pStmt){
-  int rc = sqlite3_reset(pStmt);
-  if( *pRc==SQLITE_OK ) *pRc = rc;
-}
-
 
 int idxFindIndexes(
   sqlite3expert *p,
@@ -721,34 +817,25 @@ int idxFindIndexes(
 ){
   IdxStatement *pStmt;
   sqlite3 *dbm = p->dbm;
-  sqlite3_stmt *pSelect = 0;
-  sqlite3_stmt *pInsert = 0;
-  int rc, rc2;
-  int bFound = 0;
+  int rc = SQLITE_OK;
 
-  if( rc==SQLITE_OK ){
-    rc = idxPrepareStmt(dbm, &pSelect, pzErr, 
-        "SELECT rowid, sql FROM sqlite_master WHERE name = ?"
-    );
-  }
-  if( rc==SQLITE_OK ){
-    rc = idxPrepareStmt(dbm, &pInsert, pzErr,
-        "INSERT OR IGNORE INTO aux.indexes VALUES(?)"
-    );
-  }
+  IdxHash hIdx;
+  idxHashInit(&hIdx);
 
   for(pStmt=p->pStatement; rc==SQLITE_OK && pStmt; pStmt=pStmt->pNext){
+    IdxHashEntry *pEntry;
     sqlite3_stmt *pExplain = 0;
-    rc = sqlite3_exec(dbm, "DELETE FROM aux.indexes", 0, 0, 0);
-    if( rc==SQLITE_OK ){
-      rc = idxPrintfPrepareStmt(dbm, &pExplain, pzErr,
-          "EXPLAIN QUERY PLAN %s", pStmt->zSql
-      );
-    }
+    idxHashClear(&hIdx);
+    rc = idxPrintfPrepareStmt(dbm, &pExplain, pzErr,
+        "EXPLAIN QUERY PLAN %s", pStmt->zSql
+    );
     while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){
-      int i;
+      int iSelectid = sqlite3_column_int(pExplain, 0);
+      int iOrder = sqlite3_column_int(pExplain, 1);
+      int iFrom = sqlite3_column_int(pExplain, 2);
       const char *zDetail = (const char*)sqlite3_column_text(pExplain, 3);
       int nDetail = strlen(zDetail);
+      int i;
 
       for(i=0; i<nDetail; i++){
         const char *zIdx = 0;
@@ -758,149 +845,39 @@ int idxFindIndexes(
           zIdx = &zDetail[i+22];
         }
         if( zIdx ){
+          const char *zSql;
           int nIdx = 0;
           while( zIdx[nIdx]!='\0' && (zIdx[nIdx]!=' ' || zIdx[nIdx+1]!='(') ){
             nIdx++;
           }
-          sqlite3_bind_text(pSelect, 1, zIdx, nIdx, SQLITE_STATIC);
-          if( SQLITE_ROW==sqlite3_step(pSelect) ){
-            i64 iRowid = sqlite3_column_int64(pSelect, 0);
-            const char *zSql = (const char*)sqlite3_column_text(pSelect, 1);
-            if( iRowid>=p->iIdxRowid ){
-              sqlite3_bind_text(pInsert, 1, zSql, -1, SQLITE_STATIC);
-              sqlite3_step(pInsert);
-              rc = sqlite3_reset(pInsert);
-              if( rc ) goto find_indexes_out;
-            }
+          zSql = idxHashSearch(&p->hIdx, zIdx, nIdx);
+          if( zSql ){
+            idxHashAdd(&rc, &hIdx, zSql, 0);
+            if( rc ) goto find_indexes_out;
           }
-          rc = sqlite3_reset(pSelect);
           break;
         }
       }
-    }
-    idxReset(&rc, pExplain);
-    if( rc==SQLITE_OK ){
-      sqlite3_stmt *pLoop = 0;
-      rc = idxPrepareStmt(dbm,&pLoop,pzErr,"SELECT name||';' FROM aux.indexes");
-      if( rc==SQLITE_OK ){
-        while( SQLITE_ROW==sqlite3_step(pLoop) ){
-          bFound = 1;
-          pStmt->zIdx = idxAppendText(&rc, pStmt->zIdx, "%s\n",
-              (const char*)sqlite3_column_text(pLoop, 0)
-          );
-        }
-        idxFinalize(&rc, pLoop);
-      }
-      if( bFound==0 ){
-        pStmt->zIdx = idxAppendText(&rc, pStmt->zIdx, "(no new indexes)\n");
-      }
-    }
-
-    while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){
-      int iSelectid = sqlite3_column_int(pExplain, 0);
-      int iOrder = sqlite3_column_int(pExplain, 1);
-      int iFrom = sqlite3_column_int(pExplain, 2);
-      const char *zDetail = (const char*)sqlite3_column_text(pExplain, 3);
 
       pStmt->zEQP = idxAppendText(&rc, pStmt->zEQP, "%d|%d|%d|%s\n", 
           iSelectid, iOrder, iFrom, zDetail
       );
     }
 
-    rc2 = sqlite3_finalize(pExplain);
-    if( rc==SQLITE_OK ) rc = rc2;
+    for(pEntry=hIdx.pFirst; pEntry; pEntry=pEntry->pNext){
+      pStmt->zIdx = idxAppendText(&rc, pStmt->zIdx, "%s\n", pEntry->zKey);
+    }
+    if( pStmt->zIdx==0 ){
+      pStmt->zIdx = idxAppendText(&rc, 0, "(no new indexes)\n");
+    }
+
+    idxFinalize(&rc, pExplain);
   }
 
  find_indexes_out:
-  rc2 = sqlite3_finalize(pSelect);
-  if( rc==SQLITE_OK ) rc = rc2;
-  rc2 = sqlite3_finalize(pInsert);
-  if( rc==SQLITE_OK ) rc = rc2;
-
   return rc;
 }
 
-/*
-** The xOut callback is invoked to return command output to the user. The
-** second argument is always a nul-terminated string. The first argument is
-** passed zero if the string contains normal output or non-zero if it is an
-** error message.
-*/
-int shellIndexesCommand(
-  sqlite3 *db,                         /* Database handle */
-  const char *zSql,                    /* SQL to find indexes for */
-  void (*xOut)(void*, const char*),    /* Output callback */
-  void *pOutCtx,                       /* Context for xOut() */
-  char **pzErrmsg                      /* OUT: Error message (sqlite3_malloc) */
-){
-  int rc = SQLITE_OK;
-#if 0
-  sqlite3 *dbm = 0;
-  IdxContext ctx;
-  sqlite3_stmt *pStmt = 0;        /* Statement compiled from zSql */
-
-  memset(&ctx, 0, sizeof(IdxContext));
-  ctx.pzErrmsg = pzErrmsg;
-
-  /* Open an in-memory database to work with. The main in-memory 
-  ** database schema contains tables similar to those in the users 
-  ** database (handle db). The attached in-memory db (aux) contains
-  ** application tables used by the code in this file.  */
-  rc = sqlite3_open(":memory:", &dbm);
-  if( rc==SQLITE_OK ){
-    rc = sqlite3_exec(dbm, 
-        "ATTACH ':memory:' AS aux;"
-        "CREATE TABLE aux.depmask(mask PRIMARY KEY) WITHOUT ROWID;"
-        "CREATE TABLE aux.indexes(name PRIMARY KEY) WITHOUT ROWID;"
-        "INSERT INTO aux.depmask VALUES(0);"
-        , 0, 0, pzErrmsg
-    );
-  }
-
-  /* Prepare an INSERT statement for writing to aux.depmask */
-  if( rc==SQLITE_OK ){
-    rc = idxPrepareStmt(dbm, &ctx.pInsertMask, pzErrmsg,
-        "INSERT OR IGNORE INTO aux.depmask SELECT mask | ?1 FROM aux.depmask;"
-    );
-  }
-
-  /* Analyze the SELECT statement in zSql. */
-  if( rc==SQLITE_OK ){
-    ctx.dbm = dbm;
-    sqlite3_whereinfo_hook(db, idxWhereInfo, (void*)&ctx);
-    rc = idxPrepareStmt(db, &pStmt, pzErrmsg, zSql);
-    sqlite3_whereinfo_hook(db, 0, 0);
-    sqlite3_finalize(pStmt);
-  }
-
-  /* Create tables within the main in-memory database. These tables
-  ** have the same names, columns and declared types as the tables in
-  ** the user database. All constraints except for PRIMARY KEY are
-  ** removed. */
-  if( rc==SQLITE_OK ){
-    rc = idxCreateTables(db, dbm, ctx.pScan, pzErrmsg);
-  }
-
-  /* Create candidate indexes within the in-memory database file */
-  if( rc==SQLITE_OK ){
-    rc = idxCreateCandidates(&ctx);
-  }
-
-  /* Figure out which of the candidate indexes are preferred by the query
-  ** planner and report the results to the user.  */
-  if( rc==SQLITE_OK ){
-    rc = idxFindIndexes(&ctx, zSql, xOut, pOutCtx, pzErrmsg);
-  }
-
-  idxScanFree(ctx.pScan, 0);
-  sqlite3_finalize(ctx.pInsertMask);
-  sqlite3_close(dbm);
-#endif
-  return rc;
-}
-
-/*************************************************************************/
-
 /*
 ** Allocate a new sqlite3expert object.
 */
@@ -913,22 +890,12 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){
 
   /* Open an in-memory database to work with. The main in-memory 
   ** database schema contains tables similar to those in the users 
-  ** database (handle db). The attached in-memory db (aux) contains
-  ** application tables used by the code in this file.  */
+  ** database (handle db).  */
   rc = sqlite3_open(":memory:", &pNew->dbm);
-  if( rc==SQLITE_OK ){
-    rc = sqlite3_exec(pNew->dbm,
-        "ATTACH ':memory:' AS aux;"
-        "CREATE TABLE aux.depmask(mask PRIMARY KEY) WITHOUT ROWID;"
-        "CREATE TABLE aux.indexes(name PRIMARY KEY) WITHOUT ROWID;"
-        , 0, 0, pzErrmsg
-    );
-  }
 
   /* Copy the entire schema of database [db] into [dbm]. */
   if( rc==SQLITE_OK ){
     sqlite3_stmt *pSql;
-    int rc2;
     rc = idxPrintfPrepareStmt(pNew->db, &pSql, pzErrmsg, 
         "SELECT sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%%'"
     );
@@ -936,8 +903,7 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){
       const char *zSql = (const char*)sqlite3_column_text(pSql, 0);
       rc = sqlite3_exec(pNew->dbm, zSql, 0, 0, pzErrmsg);
     }
-    rc2 = sqlite3_finalize(pSql);
-    if( rc==SQLITE_OK ) rc = rc2;
+    idxFinalize(&rc, pSql);
   }
 
   /* If an error has occurred, free the new object and reutrn NULL. Otherwise,
diff --git a/manifest b/manifest
index 8a7b97520c..83ba986696 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Refactor\scode\sto\ssuggest\sindexes\sfrom\sthe\sshell\stool\sinto\san\sextension\sin\next/expert.\sUnfinished.
-D 2017-04-07T20:14:22.417
+C Use\shash\stables\sinstead\sof\sin-memory\sdatabase\stables\sfor\sa\sfew\spurposes\sin\nsqlite3expert.c.
+D 2017-04-08T17:41:24.364
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a
@@ -41,7 +41,7 @@ F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/expert.c c73a0da702a2e9f5fd48046ab182683a73ee0318cefa3f45826536d015f39021
-F ext/expert/sqlite3expert.c 75ee320cf38b50df331232cbb60ec3b7703dd2770cc8df4ebbb445f664f6827d
+F ext/expert/sqlite3expert.c c0e3ee75550fe73c3f6a536801071b97536e0907e898993086ab478853139327
 F ext/expert/sqlite3expert.h feeaee4ab73ba52426329781bbb28032ce18cf5abd2bf6221bac2df4c32b3013
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
 F ext/fts1/ft_hash.c 3927bd880e65329bdc6f506555b228b28924921b
@@ -1573,7 +1573,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0884ff1da2e27b146c764b73cf299a1f2cfe213c4a79bde34dec02d1fc946e70
-R 2f1dd4cb0b48a4b54a46f30181a94d12
+P 305e19f976ca064638a294e0817bf547ea745e1eb74746c5855514e6ced9c5fa
+R 628a1adb0870c895045a02cfe74a8f4c
 U dan
-Z bc46ca2174a3c856c4daa1344a9d88d3
+Z c1b7449477bbba31dce4f7f111235b67
diff --git a/manifest.uuid b/manifest.uuid
index 9b195887ca..04abc2e7d4 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-305e19f976ca064638a294e0817bf547ea745e1eb74746c5855514e6ced9c5fa
\ No newline at end of file
+bf10e68d9e4d5eae7ae6148a7ad64c9596f2ed8ccd36065adb09a1f9e7dae50b
\ No newline at end of file

From e7c3aca60161d6537e4375c2a94442439e5c7a40 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 8 Apr 2017 18:56:32 +0000
Subject: [PATCH 023/488] Rename shell6.test to expert1.test. Have it invoke
 the sqlite3_expert binary if it is present.

FossilOrigin-Name: be0deff940bea4f653d644113f42529a32ff7039d1d168119233aaf000a22f40
---
 ext/expert/expert.c                | 18 ++++++++++++++++--
 ext/expert/sqlite3expert.c         |  2 +-
 manifest                           | 16 ++++++++--------
 manifest.uuid                      |  2 +-
 test/{shell6.test => expert1.test} | 15 ++++++---------
 5 files changed, 32 insertions(+), 21 deletions(-)
 rename test/{shell6.test => expert1.test} (93%)

diff --git a/ext/expert/expert.c b/ext/expert/expert.c
index 5a5cda185f..e211faa08e 100644
--- a/ext/expert/expert.c
+++ b/ext/expert/expert.c
@@ -24,6 +24,10 @@ static void option_requires_argument(const char *zOpt){
   exit(-3);
 }
 
+static int option_integer_arg(const char *zVal){
+  return atoi(zVal);
+}
+
 static void usage(char **argv){
   fprintf(stderr, "\n");
   fprintf(stderr, "Usage %s ?OPTIONS? DATABASE\n", argv[0]);
@@ -31,6 +35,7 @@ static void usage(char **argv){
   fprintf(stderr, "Options are:\n");
   fprintf(stderr, "  -sql SQL   (analyze SQL statements passed as argument)\n");
   fprintf(stderr, "  -file FILE (read SQL statements from file FILE)\n");
+  fprintf(stderr, "  -verbose LEVEL (integer verbosity level. default 1)\n");
   exit(-1);
 }
 
@@ -43,6 +48,7 @@ int main(int argc, char **argv){
   int rc = 0;
   char *zErr = 0;
   int i;
+  int iVerbose = 1;               /* -verbose option */
 
   sqlite3 *db = 0;
   sqlite3expert *p = 0;
@@ -73,6 +79,11 @@ int main(int argc, char **argv){
         rc = sqlite3_expert_sql(p, argv[i], &zErr);
       }
 
+      else if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-verbose", nArg) ){
+        if( ++i==(argc-1) ) option_requires_argument("-verbose");
+        iVerbose = option_integer_arg(argv[i]);
+      }
+
       else{
         usage(argv);
       }
@@ -89,8 +100,11 @@ int main(int argc, char **argv){
       const char *zSql = sqlite3_expert_report(p, i, EXPERT_REPORT_SQL);
       const char *zIdx = sqlite3_expert_report(p, i, EXPERT_REPORT_INDEXES);
       const char *zEQP = sqlite3_expert_report(p, i, EXPERT_REPORT_PLAN);
-      fprintf(stdout, "-- query %d ----------------------------------\n", i+1);
-      fprintf(stdout, "%s\n\n%s\n%s\n", zSql, zIdx, zEQP);
+      if( iVerbose>0 ){
+        fprintf(stdout, "-- query %d ----------------------------------\n",i+1);
+        fprintf(stdout, "%s\n\n", zSql);
+      }
+      fprintf(stdout, "%s\n%s\n", zIdx, zEQP);
     }
   }else if( zErr ){
     fprintf(stderr, "Error: %s\n", zErr);
diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index fb442a0b3c..6d674a748b 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -865,7 +865,7 @@ int idxFindIndexes(
     }
 
     for(pEntry=hIdx.pFirst; pEntry; pEntry=pEntry->pNext){
-      pStmt->zIdx = idxAppendText(&rc, pStmt->zIdx, "%s\n", pEntry->zKey);
+      pStmt->zIdx = idxAppendText(&rc, pStmt->zIdx, "%s;\n", pEntry->zKey);
     }
     if( pStmt->zIdx==0 ){
       pStmt->zIdx = idxAppendText(&rc, 0, "(no new indexes)\n");
diff --git a/manifest b/manifest
index 83ba986696..5deeb9c637 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Use\shash\stables\sinstead\sof\sin-memory\sdatabase\stables\sfor\sa\sfew\spurposes\sin\nsqlite3expert.c.
-D 2017-04-08T17:41:24.364
+C Rename\sshell6.test\sto\sexpert1.test.\sHave\sit\sinvoke\sthe\ssqlite3_expert\sbinary\nif\sit\sis\spresent.
+D 2017-04-08T18:56:32.081
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a
@@ -40,8 +40,8 @@ F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd
 F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
-F ext/expert/expert.c c73a0da702a2e9f5fd48046ab182683a73ee0318cefa3f45826536d015f39021
-F ext/expert/sqlite3expert.c c0e3ee75550fe73c3f6a536801071b97536e0907e898993086ab478853139327
+F ext/expert/expert.c 861735e296c68cd442c20d5f6e75d07c957227009b79b31581a0971481e35d97
+F ext/expert/sqlite3expert.c 864ebebab5fd12cad8fb5debd26ff083dffe87ade15c1b6dcc7a7c3ae0afdd1b
 F ext/expert/sqlite3expert.h feeaee4ab73ba52426329781bbb28032ce18cf5abd2bf6221bac2df4c32b3013
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
 F ext/fts1/ft_hash.c 3927bd880e65329bdc6f506555b228b28924921b
@@ -699,6 +699,7 @@ F test/exclusive.test 9a57bd66e39144b888ca75c309914fcdefb4e3f9
 F test/exclusive2.test 984090e8e9d1b331d2e8111daf6e5d61dda0bef7
 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7
 F test/exists.test 79a75323c78f02bbe9c251ea502a092f9ef63dac
+F test/expert1.test 4108ab79ee454679e3d46e3703943ac49676b6174703a13ed81f9ee8cd794e68
 F test/expr.test 66a2c9ac34f74f036faa4092f5402c7d3162fc93
 F test/extension01.test 00d13cec817f331a687a243e0e5a2d87b0e358c9
 F test/extraquick.test cb254400bd42bfb777ff675356aabf3287978f79
@@ -1122,7 +1123,6 @@ F test/shell2.test e242a9912f44f4c23c3d1d802a83e934e84c853b
 F test/shell3.test 9b95ba643eaa228376f06a898fb410ee9b6e57c1
 F test/shell4.test 89ad573879a745974ff2df20ff97c5d6ffffbd5d
 F test/shell5.test 50a732c1c2158b1cd62cf53975ce1ea7ce6b9dc9
-F test/shell6.test f37998b26dfde19beaaf06a4cb60c476f66a7b54affff3870a2a011402c13efc
 F test/shell7.test 07751911b294698e0c5df67bcbd29e7d2f0f2907
 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3
 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5
@@ -1573,7 +1573,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 305e19f976ca064638a294e0817bf547ea745e1eb74746c5855514e6ced9c5fa
-R 628a1adb0870c895045a02cfe74a8f4c
+P bf10e68d9e4d5eae7ae6148a7ad64c9596f2ed8ccd36065adb09a1f9e7dae50b
+R e14200231c0812aa1e853be6a898ba2f
 U dan
-Z c1b7449477bbba31dce4f7f111235b67
+Z b04d01a3056a5795f1f1bb4407f0bf9d
diff --git a/manifest.uuid b/manifest.uuid
index 04abc2e7d4..6f644ebe90 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-bf10e68d9e4d5eae7ae6148a7ad64c9596f2ed8ccd36065adb09a1f9e7dae50b
\ No newline at end of file
+be0deff940bea4f653d644113f42529a32ff7039d1d168119233aaf000a22f40
\ No newline at end of file
diff --git a/test/shell6.test b/test/expert1.test
similarity index 93%
rename from test/shell6.test
rename to test/expert1.test
index 2f7c251827..b5651040d8 100644
--- a/test/shell6.test
+++ b/test/expert1.test
@@ -16,20 +16,17 @@
 
 # Test plan:
 #
-#   shell1-1.*: Basic command line option handling.
-#   shell1-2.*: Basic "dot" command token parsing.
-#   shell1-3.*: Basic test that "dot" command can be called.
 #
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
-set testprefix shell6
+set testprefix expert1
 
 if {$tcl_platform(platform)=="windows"} {
-  set CLI "sqlite3.exe"
+  set CMD "sqlite3_expert.exe"
 } else {
-  set CLI ".././sqlite3"
+  set CMD ".././sqlite3_expert"
 }
-if {![file executable $CLI]} {
+if {![file executable $CMD]} {
   finish_test
   return
 }
@@ -42,7 +39,7 @@ proc squish {txt} {
 proc do_rec_test {tn sql res} {
   set res [squish [string trim $res]]
   set tst [subst -nocommands { 
-    squish [lindex [catchcmd [list -rec test.db {$sql;}]] 1]
+    squish [string trim [exec $::CMD -verbose 0 -sql {$sql;} test.db]]
   }]
   uplevel [list do_test $tn $tst $res]
 }
@@ -195,8 +192,8 @@ do_setup_rec_test 11.1 {
 } {
   SELECT * FROM t7 WHERE a=? OR b=?
 } {
-  CREATE INDEX t7_idx_00000061 ON t7(a);
   CREATE INDEX t7_idx_00000062 ON t7(b);
+  CREATE INDEX t7_idx_00000061 ON t7(a);
   0|0|0|SEARCH TABLE t7 USING INDEX t7_idx_00000061 (a=?) 
   0|0|0|SEARCH TABLE t7 USING INDEX t7_idx_00000062 (b=?)
 }

From 958151b00571ee62c626875a2c30cc099155f041 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sun, 9 Apr 2017 08:38:37 +0000
Subject: [PATCH 024/488] Fix the -file option on the sqlite3_expert program.

FossilOrigin-Name: 0857c48e02a76490fc623364f77363165dea94ec254f93d8f0fd0bac2968c572
---
 ext/expert/expert.c | 25 ++++++++++++++++++++++++-
 manifest            | 12 ++++++------
 manifest.uuid       |  2 +-
 3 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/ext/expert/expert.c b/ext/expert/expert.c
index e211faa08e..fca2d05194 100644
--- a/ext/expert/expert.c
+++ b/ext/expert/expert.c
@@ -40,7 +40,30 @@ static void usage(char **argv){
 }
 
 static int readSqlFromFile(sqlite3expert *p, const char *zFile, char **pzErr){
-  return SQLITE_OK;
+  FILE *in = fopen(zFile, "rb");
+  long nIn;
+  size_t nRead;
+  char *pBuf;
+  int rc;
+  if( in==0 ){
+    *pzErr = sqlite3_mprintf("failed to open file %s\n", zFile);
+    return SQLITE_ERROR;
+  }
+  fseek(in, 0, SEEK_END);
+  nIn = ftell(in);
+  rewind(in);
+  pBuf = sqlite3_malloc64( nIn+1 );
+  nRead = fread(pBuf, nIn, 1, in);
+  fclose(in);
+  if( nRead!=1 ){
+    sqlite3_free(pBuf);
+    *pzErr = sqlite3_mprintf("failed to read file %s\n", zFile);
+    return SQLITE_ERROR;
+  }
+  pBuf[nIn] = 0;
+  rc = sqlite3_expert_sql(p, pBuf, pzErr);
+  sqlite3_free(pBuf);
+  return rc;
 }
 
 int main(int argc, char **argv){
diff --git a/manifest b/manifest
index bcb6036c74..cfcf08aa4c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\slatest\strunk\swith\sthis\sbranch.
-D 2017-04-08T19:00:37.586
+C Fix\sthe\s-file\soption\son\sthe\ssqlite3_expert\sprogram.
+D 2017-04-09T08:38:37.326
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a
@@ -40,7 +40,7 @@ F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd
 F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
-F ext/expert/expert.c 861735e296c68cd442c20d5f6e75d07c957227009b79b31581a0971481e35d97
+F ext/expert/expert.c bf0fd71921cb7b807cda9a76fb380e3d6e6b980d6167093b2952b41ec9ad8f46
 F ext/expert/sqlite3expert.c 864ebebab5fd12cad8fb5debd26ff083dffe87ade15c1b6dcc7a7c3ae0afdd1b
 F ext/expert/sqlite3expert.h feeaee4ab73ba52426329781bbb28032ce18cf5abd2bf6221bac2df4c32b3013
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
@@ -1575,7 +1575,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P be0deff940bea4f653d644113f42529a32ff7039d1d168119233aaf000a22f40 204e72f0080e8f08f99978870bd3cb9d59b068ecffee82192d707c650548b43b
-R 6e0d9b80ca81b8d3b8a7b8f8dc2a80a0
+P e051e8f21b909c5e89a7203eef6501fecd64f82a634c4e10398b5ef7d716eb37
+R c2522cb6c6416b4473f74ba522229374
 U dan
-Z 4063bdd74acb02b7df335075f419d3f7
+Z 7f4db390eaebfa3a6aedc09b4bf9a775
diff --git a/manifest.uuid b/manifest.uuid
index 581e5de589..53bb31e126 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e051e8f21b909c5e89a7203eef6501fecd64f82a634c4e10398b5ef7d716eb37
\ No newline at end of file
+0857c48e02a76490fc623364f77363165dea94ec254f93d8f0fd0bac2968c572
\ No newline at end of file

From cd84474ece326bf56480b6cc9e118f6bb4e7b7d2 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 10 Apr 2017 16:13:20 +0000
Subject: [PATCH 025/488] Changes to allow the code in sqlite3expert.c to be
 tested directly (via the API in sqlite3expert.h) instead of by invoking the
 sqlite3_expert application. Fix memory leaks and other problems.

FossilOrigin-Name: 5dd9831721b70a89a26728adcd49c7f6103ef8266891a79c2db34d913702709e
---
 {test => ext/expert}/expert1.test |  81 +++++++----
 ext/expert/sqlite3expert.c        |  60 +++++---
 ext/expert/test_expert.c          | 218 ++++++++++++++++++++++++++++++
 main.mk                           |   2 +
 manifest                          |  21 +--
 manifest.uuid                     |   2 +-
 src/tclsqlite.c                   |   2 +
 test/permutations.test            |   1 +
 8 files changed, 330 insertions(+), 57 deletions(-)
 rename {test => ext/expert}/expert1.test (74%)
 create mode 100644 ext/expert/test_expert.c

diff --git a/test/expert1.test b/ext/expert/expert1.test
similarity index 74%
rename from test/expert1.test
rename to ext/expert/expert1.test
index b5651040d8..684c618780 100644
--- a/test/expert1.test
+++ b/ext/expert/expert1.test
@@ -26,39 +26,62 @@ if {$tcl_platform(platform)=="windows"} {
 } else {
   set CMD ".././sqlite3_expert"
 }
-if {![file executable $CMD]} {
-  finish_test
-  return
-}
-
 
 proc squish {txt} {
   regsub -all {[[:space:]]+} $txt { }
 }
 
-proc do_rec_test {tn sql res} {
-  set res [squish [string trim $res]]
-  set tst [subst -nocommands { 
-    squish [string trim [exec $::CMD -verbose 0 -sql {$sql;} test.db]]
-  }]
-  uplevel [list do_test $tn $tst $res]
-}
-
 proc do_setup_rec_test {tn setup sql res} {
   reset_db
   db eval $setup
   uplevel [list do_rec_test $tn $sql $res]
 }
 
+foreach {tn setup} {
+  1 {
+    if {![file executable $CMD]} { continue }
 
-do_setup_rec_test 1.1 { CREATE TABLE t1(a, b, c) } {
+    proc do_rec_test {tn sql res} {
+      set res [squish [string trim $res]]
+      set tst [subst -nocommands { 
+        squish [string trim [exec $::CMD -verbose 0 -sql {$sql;} test.db]]
+      }]
+      uplevel [list do_test $tn $tst $res]
+    }
+  }
+  2 {
+    if {[info commands sqlite3_expert_new]==""} { continue }
+
+    proc do_rec_test {tn sql res} {
+      set expert [sqlite3_expert_new db]
+      $expert sql $sql
+      $expert analyze
+
+      set result [list]
+      for {set i 0} {$i < [$expert count]} {incr i} {
+        lappend result [string trim [$expert report $i indexes]]
+        lappend result [string trim [$expert report $i plan]]
+      }
+
+      $expert destroy
+
+      set tst [subst -nocommands {set {} [squish [join {$result}]]}]
+      uplevel [list do_test $tn $tst [string trim [squish $res]]]
+    }
+  }
+} {
+
+  eval $setup
+
+
+do_setup_rec_test $tn.1.1 { CREATE TABLE t1(a, b, c) } {
   SELECT * FROM t1
 } {
   (no new indexes)
   0|0|0|SCAN TABLE t1
 }
 
-do_setup_rec_test 1.2 {
+do_setup_rec_test $tn.1.2 {
   CREATE TABLE t1(a, b, c);
 } {
   SELECT * FROM t1 WHERE b>?;
@@ -67,7 +90,7 @@ do_setup_rec_test 1.2 {
   0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_00000062 (b>?)
 }
 
-do_setup_rec_test 1.3 {
+do_setup_rec_test $tn.1.3 {
   CREATE TABLE t1(a, b, c);
 } {
   SELECT * FROM t1 WHERE b COLLATE nocase BETWEEN ? AND ?
@@ -76,7 +99,7 @@ do_setup_rec_test 1.3 {
   0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_3e094c27 (b>? AND b<?)
 }
 
-do_setup_rec_test 1.4 {
+do_setup_rec_test $tn.1.4 {
   CREATE TABLE t1(a, b, c);
 } {
   SELECT a FROM t1 ORDER BY b;
@@ -85,7 +108,7 @@ do_setup_rec_test 1.4 {
   0|0|0|SCAN TABLE t1 USING INDEX t1_idx_00000062
 }
 
-do_setup_rec_test 1.5 {
+do_setup_rec_test $tn.1.5 {
   CREATE TABLE t1(a, b, c);
 } {
   SELECT a FROM t1 WHERE a=? ORDER BY b;
@@ -94,7 +117,7 @@ do_setup_rec_test 1.5 {
   0|0|0|SEARCH TABLE t1 USING COVERING INDEX t1_idx_000123a7 (a=?)
 }
 
-do_setup_rec_test 1.6 {
+do_setup_rec_test $tn.1.6 {
   CREATE TABLE t1(a, b, c);
 } {
   SELECT min(a) FROM t1
@@ -103,7 +126,7 @@ do_setup_rec_test 1.6 {
   0|0|0|SEARCH TABLE t1 USING COVERING INDEX t1_idx_00000061
 }
 
-do_setup_rec_test 1.7 {
+do_setup_rec_test $tn.1.7 {
   CREATE TABLE t1(a, b, c);
 } {
   SELECT * FROM t1 ORDER BY a, b, c;
@@ -112,7 +135,7 @@ do_setup_rec_test 1.7 {
   0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_033e95fe
 }
 
-do_setup_rec_test 1.8 {
+do_setup_rec_test $tn.1.8 {
   CREATE TABLE t1(a, b, c);
 } {
   SELECT * FROM t1 ORDER BY a ASC, b COLLATE nocase DESC, c ASC;
@@ -121,7 +144,7 @@ do_setup_rec_test 1.8 {
   0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_5be6e222
 }
 
-do_setup_rec_test 1.9 {
+do_setup_rec_test $tn.1.9 {
   CREATE TABLE t1(a COLLATE NOCase, b, c);
 } {
   SELECT * FROM t1 WHERE a=?
@@ -133,7 +156,7 @@ do_setup_rec_test 1.9 {
 
 # Tables with names that require quotes.
 #
-do_setup_rec_test 8.1 {
+do_setup_rec_test $tn.8.1 {
   CREATE TABLE "t t"(a, b, c);
 } {
   SELECT * FROM "t t" WHERE a=?
@@ -142,7 +165,7 @@ do_setup_rec_test 8.1 {
   0|0|0|SEARCH TABLE t t USING INDEX t t_idx_00000061 (a=?) 
 }
 
-do_setup_rec_test 8.2 {
+do_setup_rec_test $tn.8.2 {
   CREATE TABLE "t t"(a, b, c);
 } {
   SELECT * FROM "t t" WHERE b BETWEEN ? AND ?
@@ -153,7 +176,7 @@ do_setup_rec_test 8.2 {
 
 # Columns with names that require quotes.
 #
-do_setup_rec_test 9.1 {
+do_setup_rec_test $tn.9.1 {
   CREATE TABLE t3(a, "b b", c);
 } {
   SELECT * FROM t3 WHERE "b b" = ?
@@ -162,7 +185,7 @@ do_setup_rec_test 9.1 {
   0|0|0|SEARCH TABLE t3 USING INDEX t3_idx_00050c52 (b b=?)
 }
 
-do_setup_rec_test 9.2 {
+do_setup_rec_test $tn.9.2 {
   CREATE TABLE t3(a, "b b", c);
 } {
   SELECT * FROM t3 ORDER BY "b b"
@@ -173,7 +196,7 @@ do_setup_rec_test 9.2 {
 
 # Transitive constraints
 #
-do_setup_rec_test 10.1 {
+do_setup_rec_test $tn.10.1 {
   CREATE TABLE t5(a, b);
   CREATE TABLE t6(c, d);
 } {
@@ -187,7 +210,7 @@ do_setup_rec_test 10.1 {
 
 # OR terms.
 #
-do_setup_rec_test 11.1 {
+do_setup_rec_test $tn.11.1 {
   CREATE TABLE t7(a, b);
 } {
   SELECT * FROM t7 WHERE a=? OR b=?
@@ -198,5 +221,7 @@ do_setup_rec_test 11.1 {
   0|0|0|SEARCH TABLE t7 USING INDEX t7_idx_00000062 (b=?)
 }
 
+}
+
 finish_test
 
diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index 6d674a748b..6bcc697fcf 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -11,6 +11,8 @@
 *************************************************************************
 */
 
+#if !defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHEREINFO_HOOK)
+
 #include "sqlite3expert.h"
 #include <assert.h>
 #include <string.h>
@@ -20,7 +22,6 @@ typedef sqlite3_int64 i64;
 typedef sqlite3_uint64 u64;
 
 typedef struct IdxConstraint IdxConstraint;
-typedef struct IdxContext IdxContext;
 typedef struct IdxScan IdxScan;
 typedef struct IdxStatement IdxStatement;
 typedef struct IdxWhere IdxWhere;
@@ -89,19 +90,6 @@ struct IdxTable {
   IdxColumn *aCol;
 };
 
-/*
-** Context object passed to idxWhereInfo() and other functions.
-*/
-struct IdxContext {
-  char **pzErrmsg;
-  IdxWhere *pCurrent;             /* Current where clause */
-  int rc;                         /* Error code (if error has occurred) */
-  IdxScan *pScan;                 /* List of scan objects */
-  sqlite3 *dbm;                   /* In-memory db for this analysis */
-  sqlite3 *db;                    /* User database under analysis */
-  sqlite3_stmt *pInsertMask;      /* To write to aux.depmask */
-};
-
 struct IdxStatement {
   int iId;                        /* Statement number */
   char *zSql;                     /* SQL statement */
@@ -301,7 +289,7 @@ static IdxConstraint *idxNewConstraint(int *pRc, const char *zColl){
 ** sqlite3_whereinfo_hook() callback.
 */
 static void idxWhereInfo(
-  void *pCtx,                     /* Pointer to IdxContext structure */
+  void *pCtx,                     /* Pointer to sqlite3expert structure */
   int eOp, 
   const char *zVal, 
   int iVal, 
@@ -794,12 +782,31 @@ static int idxCreateCandidates(sqlite3expert *p, char **pzErr){
   return rc;
 }
 
+static void idxConstraintFree(IdxConstraint *pConstraint){
+  IdxConstraint *pNext;
+  IdxConstraint *p;
+
+  for(p=pConstraint; p; p=pNext){
+    pNext = p->pNext;
+    sqlite3_free(p);
+  }
+}
+
 /*
 ** Free all elements of the linked list starting from pScan up until pLast
 ** (pLast is not freed).
 */
 static void idxScanFree(IdxScan *pScan, IdxScan *pLast){
-  /* TODO! */
+  IdxScan *p;
+  IdxScan *pNext;
+  for(p=pScan; p!=pLast; p=pNext){
+    pNext = p->pNextScan;
+    idxConstraintFree(p->pOrder);
+    idxConstraintFree(p->where.pEq);
+    idxConstraintFree(p->where.pRange);
+    sqlite3_free(p->pTable);
+    sqlite3_free(p);
+  }
 }
 
 /*
@@ -807,7 +814,14 @@ static void idxScanFree(IdxScan *pScan, IdxScan *pLast){
 ** until pLast (pLast is not freed).
 */
 static void idxStatementFree(IdxStatement *pStatement, IdxStatement *pLast){
-  /* TODO! */
+  IdxStatement *p;
+  IdxStatement *pNext;
+  for(p=pStatement; p!=pLast; p=pNext){
+    pNext = p->pNext;
+    sqlite3_free(p->zEQP);
+    sqlite3_free(p->zIdx);
+    sqlite3_free(p);
+  }
 }
 
 
@@ -875,6 +889,7 @@ int idxFindIndexes(
   }
 
  find_indexes_out:
+  idxHashClear(&hIdx);
   return rc;
 }
 
@@ -974,7 +989,6 @@ int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr){
     rc = idxGetTableInfo(p->dbm, pIter, pzErr);
   }
 
-
   /* Create candidate indexes within the in-memory database file */
   if( rc==SQLITE_OK ){
     rc = idxCreateCandidates(p, pzErr);
@@ -992,12 +1006,19 @@ int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr){
   return rc;
 }
 
+/*
+** Return the total number of statements that have been added to this
+** sqlite3expert using sqlite3_expert_sql().
+*/
 int sqlite3_expert_count(sqlite3expert *p){
   int nRet = 0;
   if( p->pStatement ) nRet = p->pStatement->iId+1;
   return nRet;
 }
 
+/*
+** Return a component of the report.
+*/
 const char *sqlite3_expert_report(sqlite3expert *p, int iStmt, int eReport){
   const char *zRet = 0;
   IdxStatement *pStmt;
@@ -1027,6 +1048,9 @@ void sqlite3_expert_destroy(sqlite3expert *p){
   sqlite3_close(p->dbm);
   idxScanFree(p->pScan, 0);
   idxStatementFree(p->pStatement, 0);
+  idxHashClear(&p->hIdx);
   sqlite3_free(p);
 }
 
+#endif /* !defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHEREINFO_HOOK) */
+
diff --git a/ext/expert/test_expert.c b/ext/expert/test_expert.c
new file mode 100644
index 0000000000..a37887e587
--- /dev/null
+++ b/ext/expert/test_expert.c
@@ -0,0 +1,218 @@
+/*
+** 2017 April 07
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+*/
+
+#if defined(SQLITE_TEST) && defined(SQLITE_ENABLE_WHEREINFO_HOOK)
+
+#include "sqlite3expert.h"
+#include <assert.h>
+#include <string.h>
+
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
+
+/*
+** Extract an sqlite3* db handle from the object passed as the second
+** argument. If successful, set *pDb to point to the db handle and return
+** TCL_OK. Otherwise, return TCL_ERROR.
+*/
+static int dbHandleFromObj(Tcl_Interp *interp, Tcl_Obj *pObj, sqlite3 **pDb){
+  Tcl_CmdInfo info;
+  if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(pObj), &info) ){
+    Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(pObj), 0);
+    return TCL_ERROR;
+  }
+
+  *pDb = *(sqlite3 **)info.objClientData;
+  return TCL_OK;
+}
+
+
+/*
+** Tclcmd:  $expert sql SQL
+**          $expert analyze
+**          $expert count
+**          $expert report STMT EREPORT
+**          $expert destroy
+*/
+static int SQLITE_TCLAPI testExpertCmd(
+  void *clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  sqlite3expert *pExpert = (sqlite3expert*)clientData;
+  struct Subcmd {
+    const char *zSub;
+    int nArg;
+    const char *zMsg;
+  } aSub[] = {
+    { "sql",       1, "TABLE",        }, /* 0 */
+    { "analyze",   0, "",             }, /* 1 */
+    { "count",     0, "",             }, /* 2 */
+    { "report",    2, "STMT EREPORT", }, /* 3 */
+    { "destroy",   0, "",             }, /* 4 */
+    { 0 }
+  };
+  int iSub;
+  int rc = TCL_OK;
+  char *zErr = 0;
+
+  if( objc<2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "SUBCOMMAND ...");
+    return TCL_ERROR;
+  }
+  rc = Tcl_GetIndexFromObjStruct(interp, 
+      objv[1], aSub, sizeof(aSub[0]), "sub-command", 0, &iSub
+  );
+  if( rc!=TCL_OK ) return rc;
+  if( objc!=2+aSub[iSub].nArg ){
+    Tcl_WrongNumArgs(interp, 2, objv, aSub[iSub].zMsg);
+    return TCL_ERROR;
+  }
+
+  switch( iSub ){
+    case 0: {      /* sql */
+      char *zArg = Tcl_GetString(objv[2]);
+      rc = sqlite3_expert_sql(pExpert, zArg, &zErr);
+      break;
+    }
+
+    case 1: {      /* analyze */
+      rc = sqlite3_expert_analyze(pExpert, &zErr);
+      break;
+    }
+
+    case 2: {      /* count */
+      int n = sqlite3_expert_count(pExpert);
+      Tcl_SetObjResult(interp, Tcl_NewIntObj(n));
+      break;
+    }
+
+    case 3: {      /* report */
+      const char *aEnum[] = {
+        "sql", "indexes", "plan", 0
+      };
+      int iEnum;
+      int iStmt;
+      const char *zReport;
+
+      if( Tcl_GetIntFromObj(interp, objv[2], &iStmt) 
+       || Tcl_GetIndexFromObj(interp, objv[3], aEnum, "report", 0, &iEnum)
+      ){
+        return TCL_ERROR;
+      }
+
+      assert( EXPERT_REPORT_SQL==1 );
+      assert( EXPERT_REPORT_INDEXES==2 );
+      assert( EXPERT_REPORT_PLAN==3 );
+      zReport = sqlite3_expert_report(pExpert, iStmt, 1+iEnum);
+      Tcl_SetObjResult(interp, Tcl_NewStringObj(zReport, -1));
+      break;
+    }
+
+    default:       /* destroy */
+      assert( iSub==4 );     
+      Tcl_DeleteCommand(interp, Tcl_GetString(objv[0]));
+      break;
+  }
+
+  if( rc!=TCL_OK ){
+    if( zErr ){
+      Tcl_SetObjResult(interp, Tcl_NewStringObj(zErr, -1));
+    }else{
+      extern const char *sqlite3ErrName(int);
+      Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
+    }
+  }
+  sqlite3_free(zErr);
+  return rc;
+}
+
+static void SQLITE_TCLAPI testExpertDel(void *clientData){
+  sqlite3expert *pExpert = (sqlite3expert*)clientData;
+  sqlite3_expert_destroy(pExpert);
+}
+
+/*
+** sqlite3_expert_new DB
+*/
+static int SQLITE_TCLAPI test_sqlite3_expert_new(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  static int iCmd = 0;
+  sqlite3 *db;
+  char *zCmd = 0;
+  char *zErr = 0;
+  sqlite3expert *pExpert;
+  int rc = TCL_OK;
+
+  if( objc!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "DB");
+    return TCL_ERROR;
+  }
+  if( dbHandleFromObj(interp, objv[1], &db) ){
+    return TCL_ERROR;
+  }
+
+  zCmd = sqlite3_mprintf("sqlite3expert%d", ++iCmd);
+  if( zCmd==0 ){
+    Tcl_AppendResult(interp, "out of memory", (char*)0);
+    return TCL_ERROR;
+  }
+
+  pExpert = sqlite3_expert_new(db, &zErr);
+  if( pExpert==0 ){
+    Tcl_AppendResult(interp, zErr, (char*)0);
+    rc = TCL_ERROR;
+  }else{
+    void *p = (void*)pExpert;
+    Tcl_CreateObjCommand(interp, zCmd, testExpertCmd, p, testExpertDel);
+    Tcl_SetObjResult(interp, Tcl_NewStringObj(zCmd, -1));
+  }
+
+  sqlite3_free(zCmd);
+  sqlite3_free(zErr);
+  return rc;
+}
+
+int TestExpert_Init(Tcl_Interp *interp){
+  struct Cmd {
+    const char *zCmd;
+    Tcl_ObjCmdProc *xProc;
+  } aCmd[] = {
+    { "sqlite3_expert_new", test_sqlite3_expert_new },
+  };
+  int i;
+
+  for(i=0; i<sizeof(aCmd)/sizeof(struct Cmd); i++){
+    struct Cmd *p = &aCmd[i];
+    Tcl_CreateObjCommand(interp, p->zCmd, p->xProc, 0, 0);
+  }
+
+  return TCL_OK;
+}
+
+#else /* defined(SQLITE_TEST) && defined(SQLITE_ENABLE_WHEREINFO_HOOK) */
+int TestExpert_Init(Tcl_Interp *interp){
+  return TCL_OK;
+}
+#endif
diff --git a/main.mk b/main.mk
index cb41cb7e00..ea4f9891f7 100644
--- a/main.mk
+++ b/main.mk
@@ -274,6 +274,8 @@ SRC += \
 # Source code to the test files.
 #
 TESTSRC = \
+  $(TOP)/ext/expert/sqlite3expert.c \
+  $(TOP)/ext/expert/test_expert.c \
   $(TOP)/ext/fts3/fts3_term.c \
   $(TOP)/ext/fts3/fts3_test.c \
   $(TOP)/ext/rbu/test_rbu.c \
diff --git a/manifest b/manifest
index cfcf08aa4c..7660557a22 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sthe\s-file\soption\son\sthe\ssqlite3_expert\sprogram.
-D 2017-04-09T08:38:37.326
+C Changes\sto\sallow\sthe\scode\sin\ssqlite3expert.c\sto\sbe\stested\sdirectly\s(via\sthe\nAPI\sin\ssqlite3expert.h)\sinstead\sof\sby\sinvoking\sthe\ssqlite3_expert\sapplication.\nFix\smemory\sleaks\sand\sother\sproblems.
+D 2017-04-10T16:13:20.707
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a
@@ -41,8 +41,10 @@ F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/expert.c bf0fd71921cb7b807cda9a76fb380e3d6e6b980d6167093b2952b41ec9ad8f46
-F ext/expert/sqlite3expert.c 864ebebab5fd12cad8fb5debd26ff083dffe87ade15c1b6dcc7a7c3ae0afdd1b
+F ext/expert/expert1.test c1b1405f3ac20e9f71dacdf7bd68ff22e273b249a419260b123ebe385daf2db5 w test/expert1.test
+F ext/expert/sqlite3expert.c b87f13e90b999b5b10c4ec004b6a935150c00d3af1a16944e262172b9b831b8c
 F ext/expert/sqlite3expert.h feeaee4ab73ba52426329781bbb28032ce18cf5abd2bf6221bac2df4c32b3013
+F ext/expert/test_expert.c bad0611732d07180d586bd589cbb7713dc3ab0338c52bff29680eb2007678c05
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
 F ext/fts1/ft_hash.c 3927bd880e65329bdc6f506555b228b28924921b
 F ext/fts1/ft_hash.h 06df7bba40dadd19597aa400a875dbc2fed705ea
@@ -328,7 +330,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 6d4f1f1f78a6ac453d35f18c4e696fdefbe65dfec9530a41c5579ef8ec076072
+F main.mk ef818c7b1bb21f657e3bfb363cc7167264d688ca404a666e6ddda6029e94c43b
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -413,7 +415,7 @@ F src/sqliteInt.h 36eec0868d25b50e64a05bc7355557a0aa9bd14330c16b7d7568520b2bf2bb
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
-F src/tclsqlite.c 6c2151b6d8d98e183a04466d40df8889c0574d79
+F src/tclsqlite.c 2e0f7f63de8329526fbcb14fa5261d3574b2a06dd330f4df680120a3e6156133
 F src/test1.c 8a98191a1da8e100f77cdb5cc716df67d405028d
 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5
 F src/test3.c d03f5b5da9a2410b7a91c64b0d3306ed28ab6fee
@@ -699,7 +701,6 @@ F test/exclusive.test 9a57bd66e39144b888ca75c309914fcdefb4e3f9
 F test/exclusive2.test 984090e8e9d1b331d2e8111daf6e5d61dda0bef7
 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7
 F test/exists.test 79a75323c78f02bbe9c251ea502a092f9ef63dac
-F test/expert1.test 4108ab79ee454679e3d46e3703943ac49676b6174703a13ed81f9ee8cd794e68
 F test/expr.test 66a2c9ac34f74f036faa4092f5402c7d3162fc93
 F test/extension01.test 00d13cec817f331a687a243e0e5a2d87b0e358c9
 F test/extraquick.test cb254400bd42bfb777ff675356aabf3287978f79
@@ -1031,7 +1032,7 @@ F test/parser1.test 391b9bf9a229547a129c61ac345ed1a6f5eb1854
 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b
 F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442
 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff
-F test/permutations.test af720e7d139e7e5417341d0f0eef2b911c0b067852138dc2f5b6a451b5725118
+F test/permutations.test 9c0da2079fa37e7509957c9efbbdc282dea4ed0e732d19e6f216d53ae431a67d
 F test/pragma.test 1e94755164a3a3264cd39836de4bebcb7809e5f8
 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f
 F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed
@@ -1575,7 +1576,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e051e8f21b909c5e89a7203eef6501fecd64f82a634c4e10398b5ef7d716eb37
-R c2522cb6c6416b4473f74ba522229374
+P 0857c48e02a76490fc623364f77363165dea94ec254f93d8f0fd0bac2968c572
+R edcb3d0fdfbb918bffbc8d72b5429335
 U dan
-Z 7f4db390eaebfa3a6aedc09b4bf9a775
+Z 0d7d47356215948792ad81449f48b82b
diff --git a/manifest.uuid b/manifest.uuid
index 53bb31e126..a059662f4d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0857c48e02a76490fc623364f77363165dea94ec254f93d8f0fd0bac2968c572
\ No newline at end of file
+5dd9831721b70a89a26728adcd49c7f6103ef8266891a79c2db34d913702709e
\ No newline at end of file
diff --git a/src/tclsqlite.c b/src/tclsqlite.c
index 9df023b45c..a4a0ed3f30 100644
--- a/src/tclsqlite.c
+++ b/src/tclsqlite.c
@@ -4129,6 +4129,7 @@ static void init_all(Tcl_Interp *interp){
 #if defined(SQLITE_ENABLE_SESSION) && defined(SQLITE_ENABLE_PREUPDATE_HOOK)
     extern int TestSession_Init(Tcl_Interp*);
 #endif
+    extern int TestExpert_Init(Tcl_Interp*);
     extern int Fts5tcl_Init(Tcl_Interp *);
     extern int SqliteRbu_Init(Tcl_Interp*);
     extern int Sqlitetesttcl_Init(Tcl_Interp*);
@@ -4177,6 +4178,7 @@ static void init_all(Tcl_Interp *interp){
 #if defined(SQLITE_ENABLE_SESSION) && defined(SQLITE_ENABLE_PREUPDATE_HOOK)
     TestSession_Init(interp);
 #endif
+    TestExpert_Init(interp);
     Fts5tcl_Init(interp);
     SqliteRbu_Init(interp);
     Sqlitetesttcl_Init(interp);
diff --git a/test/permutations.test b/test/permutations.test
index 628e7ba44e..7e47155154 100644
--- a/test/permutations.test
+++ b/test/permutations.test
@@ -89,6 +89,7 @@ foreach f [glob $testdir/*.test] { lappend alltests [file tail $f] }
 foreach f [glob -nocomplain       \
     $testdir/../ext/rtree/*.test  \
     $testdir/../ext/fts5/test/*.test   \
+    $testdir/../ext/expert/*.test   \
 ] { 
   lappend alltests $f 
 }

From 96d43a05ecbc611a27751446c808d1710b2d8e6e Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 10 Apr 2017 20:00:26 +0000
Subject: [PATCH 026/488] Add ext/expert/README.md.

FossilOrigin-Name: 9318f1b9ed2d8da3a82ea69179e2d56a99d326c7721642665f87f6a4534e7bf0
---
 ext/expert/README.md       |  68 +++++++++++
 ext/expert/expert1.test    |   4 +-
 ext/expert/sqlite3expert.c | 237 +++++++++++++++++++++++--------------
 manifest                   |  15 +--
 manifest.uuid              |   2 +-
 5 files changed, 228 insertions(+), 98 deletions(-)
 create mode 100644 ext/expert/README.md

diff --git a/ext/expert/README.md b/ext/expert/README.md
new file mode 100644
index 0000000000..3cede6c09d
--- /dev/null
+++ b/ext/expert/README.md
@@ -0,0 +1,68 @@
+## SQLite Expert Extension
+
+This folder contains code for a simple system to propose useful indexes
+given a database and a set of SQL queries. It works as follows:
+
+  1. The user database schema is copied to a temporary database.
+
+  1. All SQL queries are prepared against the temporary database. The
+     **sqlite3\_whereinfo\_hook()** API is used to record information regarding
+     the WHERE and ORDER BY clauses attached to each query.
+
+  1. The information gathered in step 2 is used to create (possibly a large
+     number of) candidate indexes.
+
+  1. The SQL queries are prepared a second time. If the planner uses any
+     of the indexes created in step 3, they are recommended to the user.
+
+No ANALYZE data is available to the planner in step 4 above. This can lead to sub-optimal results.
+
+This extension requires that SQLite be built with the 
+SQLITE\_ENABLE\_WHEREINFO\_HOOK pre-processor symbol defined.
+
+# C API
+
+The SQLite expert C API is defined in sqlite3expert.h. Most uses will proceed
+as follows:
+
+  1. An sqlite3expert object is created by calling **sqlite3\_expert\_new()**.
+     A database handle opened by the user is passed as an argument.
+
+  1. The sqlite3expert object is configured with one or more SQL statements
+     by making one or more calls to **sqlite3\_expert\_sql()**. Each call may
+     specify a single SQL statement, or multiple statements separated by
+     semi-colons.
+
+  1. **sqlite3\_expert\_analyze()** is called to run the analysis.
+
+  1. One or more calls are made to **sqlite3\_expert\_report()** to extract
+     components of the results of the analysis.
+
+  1. **sqlite3\_expert\_destroy()** is called to free all resources.
+
+Refer to comments in sqlite3expert.h for further details.
+
+# sqlite3_expert application
+
+The file "expert.c" contains the code for a command line application that
+uses the API described above. It can be compiled with (for example):
+
+<pre>
+  gcc -O2 -DSQLITE_ENABLE_WHEREINFO_HOOK sqlite3.c expert.c sqlite3expert.c -o sqlite3_expert
+</pre>
+
+Assuming the database is "test.db", it can then be run to analyze a single query:
+
+<pre>
+  ./sqlite3_expert -sql &lt;sql-query&gt; test.db
+</pre>
+
+Or an entire text file worth of queries with:
+
+<pre>
+  ./sqlite3_expert -file &lt;text-file&gt; test.db
+</pre>
+
+
+
+
diff --git a/ext/expert/expert1.test b/ext/expert/expert1.test
index 684c618780..2e8c1c7e76 100644
--- a/ext/expert/expert1.test
+++ b/ext/expert/expert1.test
@@ -17,7 +17,9 @@
 # Test plan:
 #
 #
-set testdir [file dirname $argv0]
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+}
 source $testdir/tester.tcl
 set testprefix expert1
 
diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index 6bcc697fcf..faa6966932 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -1,5 +1,5 @@
 /*
-** 2016 February 10
+** 2017 April 09
 **
 ** The author disclaims copyright to this source code.  In place of
 ** a legal notice, here is a blessing:
@@ -21,16 +21,15 @@
 typedef sqlite3_int64 i64;
 typedef sqlite3_uint64 u64;
 
+typedef struct IdxColumn IdxColumn;
 typedef struct IdxConstraint IdxConstraint;
 typedef struct IdxScan IdxScan;
 typedef struct IdxStatement IdxStatement;
-typedef struct IdxWhere IdxWhere;
-
-typedef struct IdxColumn IdxColumn;
 typedef struct IdxTable IdxTable;
 
 /*
-** A single constraint. Equivalent to either "col = ?" or "col < ?".
+** A single constraint. Equivalent to either "col = ?" or "col < ?" (or
+** any other type of single-ended range constraint on a column).
 **
 ** pLink:
 **   Used to temporarily link IdxConstraint objects into lists while
@@ -47,38 +46,22 @@ struct IdxConstraint {
   IdxConstraint *pLink;           /* See above */
 };
 
-/*
-** A WHERE clause. Made up of IdxConstraint objects. Example WHERE clause:
-**
-**   a=? AND b=? AND c=? AND d=? AND e>? AND f<?
-**
-** The above is decomposed into 6 AND connected clauses. The first four are
-** added to the IdxWhere.pEq linked list, the following two into 
-** IdxWhere.pRange.
-**
-** IdxWhere.pEq and IdxWhere.pRange are simple linked lists of IdxConstraint
-** objects linked by the IdxConstraint.pNext field.
-*/
-struct IdxWhere {
-  IdxConstraint *pEq;             /* List of == constraints */
-  IdxConstraint *pRange;          /* List of < constraints */
-};
-
 /*
 ** A single scan of a single table.
 */
 struct IdxScan {
-  IdxTable *pTable;               /* Table-info */
   char *zTable;                   /* Name of table to scan */
   int iDb;                        /* Database containing table zTable */
   i64 covering;                   /* Mask of columns required for cov. index */
   IdxConstraint *pOrder;          /* ORDER BY columns */
-  IdxWhere where;                 /* WHERE Constraints */
-  IdxScan *pNextScan;             /* Next IdxScan object for same query */
+  IdxConstraint *pEq;             /* List of == constraints */
+  IdxConstraint *pRange;          /* List of < constraints */
+  IdxScan *pNextScan;             /* Next IdxScan object for same analysis */
 };
 
 /*
-** Data regarding a database table. Extracted from "PRAGMA table_info"
+** Information regarding a single database table. Extracted from 
+** "PRAGMA table_info" by function idxGetTableInfo().
 */
 struct IdxColumn {
   char *zName;
@@ -90,6 +73,10 @@ struct IdxTable {
   IdxColumn *aCol;
 };
 
+/*
+** Each statement being analyzed is represented by an instance of this
+** structure.
+*/
 struct IdxStatement {
   int iId;                        /* Statement number */
   char *zSql;                     /* SQL statement */
@@ -99,6 +86,15 @@ struct IdxStatement {
 };
 
 
+/*
+** A hash table for storing strings. With space for a payload string
+** with each entry. Methods are:
+**
+**   idxHashInit()
+**   idxHashClear()
+**   idxHashAdd()
+**   idxHashSearch()
+*/
 #define IDX_HASH_SIZE 1023
 typedef struct IdxHashEntry IdxHashEntry;
 typedef struct IdxHash IdxHash;
@@ -113,20 +109,36 @@ struct IdxHash {
   IdxHashEntry *aHash[IDX_HASH_SIZE];
 };
 
+/*
+** A hash table for storing a set of 64-bit values. Methods are:
+**
+**   idxHash64Init()
+**   idxHash64Clear()
+**   idxHash64Add()
+*/
+typedef struct IdxHash64Entry IdxHash64Entry;
+typedef struct IdxHash64 IdxHash64;
+struct IdxHash64Entry {
+  u64 iVal;
+  IdxHash64Entry *pNext;          /* Next entry in hash table */
+  IdxHash64Entry *pHashNext;      /* Next entry in same hash bucket */
+};
+struct IdxHash64 {
+  IdxHash64Entry *pFirst;         /* Most recently added entry in hash table */
+  IdxHash64Entry *aHash[IDX_HASH_SIZE];
+};
+
 /*
 ** sqlite3expert object.
 */
 struct sqlite3expert {
-  sqlite3 *db;                    /* Users database */
+  sqlite3 *db;                    /* User database */
   sqlite3 *dbm;                   /* In-memory db for this analysis */
-
   int bRun;                       /* True once analysis has run */
   char **pzErrmsg;
-
   IdxScan *pScan;                 /* List of scan objects */
   IdxStatement *pStatement;       /* List of IdxStatement objects */
   int rc;                         /* Error code from whereinfo hook */
-
   IdxHash hIdx;                   /* Hash containing all candidate indexes */
 };
 
@@ -148,24 +160,16 @@ static void *idxMalloc(int *pRc, int nByte){
   return pRet;
 }
 
-/*************************************************************************
-** Start of hash table implementations.
+/*
+** Initialize an IdxHash64 hash table.
 */
-typedef struct IdxHash64Entry IdxHash64Entry;
-typedef struct IdxHash64 IdxHash64;
-struct IdxHash64Entry {
-  u64 iVal;
-  IdxHash64Entry *pNext;          /* Next entry in hash table */
-  IdxHash64Entry *pHashNext;      /* Next entry in same hash bucket */
-};
-struct IdxHash64 {
-  IdxHash64Entry *pFirst;         /* Most recently added entry in hash table */
-  IdxHash64Entry *aHash[IDX_HASH_SIZE];
-};
-
 static void idxHash64Init(IdxHash64 *pHash){
   memset(pHash, 0, sizeof(IdxHash64));
 }
+
+/*
+** Reset an IdxHash64 hash table.
+*/
 static void idxHash64Clear(IdxHash64 *pHash){
   IdxHash64Entry *pEntry;
   IdxHash64Entry *pNext;
@@ -175,6 +179,11 @@ static void idxHash64Clear(IdxHash64 *pHash){
   }
   memset(pHash, 0, sizeof(IdxHash64));
 }
+
+/*
+** Add iVal to the IdxHash64 hash table passed as the second argument. This
+** function is a no-op if iVal is already present in the hash table.
+*/
 static void idxHash64Add(int *pRc, IdxHash64 *pHash, u64 iVal){
   int iHash = (int)((iVal*7) % IDX_HASH_SIZE);
   IdxHash64Entry *pEntry;
@@ -193,9 +202,16 @@ static void idxHash64Add(int *pRc, IdxHash64 *pHash, u64 iVal){
   }
 }
 
+/*
+** Initialize an IdxHash hash table.
+*/
 static void idxHashInit(IdxHash *pHash){
   memset(pHash, 0, sizeof(IdxHash));
 }
+
+/*
+** Reset an IdxHash hash table.
+*/
 static void idxHashClear(IdxHash *pHash){
   int i;
   for(i=0; i<IDX_HASH_SIZE; i++){
@@ -208,6 +224,11 @@ static void idxHashClear(IdxHash *pHash){
   }
   memset(pHash, 0, sizeof(IdxHash));
 }
+
+/*
+** Return the index of the hash bucket that the string specified by the
+** arguments to this function belongs.
+*/
 static int idxHashString(const char *z, int n){
   unsigned int ret = 0;
   int i;
@@ -217,6 +238,11 @@ static int idxHashString(const char *z, int n){
   return (int)(ret % IDX_HASH_SIZE);
 }
 
+/*
+** If zKey is already present in the hash table, return non-zero and do
+** nothing. Otherwise, add an entry with key zKey and payload string zVal to
+** the hash table passed as the second argument. 
+*/
 static int idxHashAdd(
   int *pRc, 
   IdxHash *pHash, 
@@ -250,6 +276,12 @@ static int idxHashAdd(
   return 0;
 }
 
+/*
+** If the hash table contains an entry with a key equal to the string
+** passed as the final two arguments to this function, return a pointer
+** to the payload string. Otherwise, if zKey/nKey is not present in the
+** hash table, return NULL.
+*/
 static const char *idxHashSearch(IdxHash *pHash, const char *zKey, int nKey){
   int iHash;
   IdxHashEntry *pEntry;
@@ -264,10 +296,6 @@ static const char *idxHashSearch(IdxHash *pHash, const char *zKey, int nKey){
   return 0;
 }
 
-/*
-** End of hash table implementations.
-**************************************************************************/
-
 /*
 ** Allocate and return a new IdxConstraint object. Set the IdxConstraint.zColl
 ** variable to point to a copy of nul-terminated string zColl.
@@ -346,11 +374,11 @@ static void idxWhereInfo(
         pNew->depmask = mask;
 
         if( eOp==SQLITE_WHEREINFO_RANGE ){
-          pNew->pNext = p->pScan->where.pRange;
-          p->pScan->where.pRange = pNew;
+          pNew->pNext = p->pScan->pRange;
+          p->pScan->pRange = pNew;
         }else{
-          pNew->pNext = p->pScan->where.pEq;
-          p->pScan->where.pEq = pNew;
+          pNew->pNext = p->pScan->pEq;
+          p->pScan->pEq = pNew;
         }
         break;
       }
@@ -369,6 +397,9 @@ static void idxDatabaseError(
   *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
 }
 
+/*
+** Prepare an SQL statement.
+*/
 static int idxPrepareStmt(
   sqlite3 *db,                    /* Database handle to compile against */
   sqlite3_stmt **ppStmt,          /* OUT: Compiled SQL statement */
@@ -383,6 +414,9 @@ static int idxPrepareStmt(
   return rc;
 }
 
+/*
+** Prepare an SQL statement using the results of a printf() formatting.
+*/
 static int idxPrintfPrepareStmt(
   sqlite3 *db,                    /* Database handle to compile against */
   sqlite3_stmt **ppStmt,          /* OUT: Compiled SQL statement */
@@ -405,17 +439,32 @@ static int idxPrintfPrepareStmt(
   return rc;
 }
 
+/*
+** Finalize SQL statement pStmt. If (*pRc) is SQLITE_OK when this function
+** is called, set it to the return value of sqlite3_finalize() before
+** returning. Otherwise, discard the sqlite3_finalize() return value.
+*/
 static void idxFinalize(int *pRc, sqlite3_stmt *pStmt){
   int rc = sqlite3_finalize(pStmt);
   if( *pRc==SQLITE_OK ) *pRc = rc;
 }
 
+/*
+** Attempt to allocate an IdxTable structure corresponding to table zTab
+** in the main database of connection db. If successful, set (*ppOut) to
+** point to the new object and return SQLITE_OK. Otherwise, return an
+** SQLite error code and set (*ppOut) to NULL. In this case *pzErrmsg may be
+** set to point to an error string.
+**
+** It is the responsibility of the caller to eventually free either the
+** IdxTable object or error message using sqlite3_free().
+*/
 static int idxGetTableInfo(
-  sqlite3 *db,
-  IdxScan *pScan,
-  char **pzErrmsg
+  sqlite3 *db,                    /* Database connection to read details from */
+  const char *zTab,               /* Table name */
+  IdxTable **ppOut,               /* OUT: New object (if successful) */
+  char **pzErrmsg                 /* OUT: Error message (if not) */
 ){
-  const char *zTbl = pScan->zTable;
   sqlite3_stmt *p1 = 0;
   int nCol = 0;
   int nByte = sizeof(IdxTable);
@@ -423,12 +472,12 @@ static int idxGetTableInfo(
   int rc, rc2;
   char *pCsr;
 
-  rc = idxPrintfPrepareStmt(db, &p1, pzErrmsg, "PRAGMA table_info=%Q", zTbl);
+  rc = idxPrintfPrepareStmt(db, &p1, pzErrmsg, "PRAGMA table_info=%Q", zTab);
   while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){
     const char *zCol = (const char*)sqlite3_column_text(p1, 1);
     nByte += 1 + strlen(zCol);
     rc = sqlite3_table_column_metadata(
-        db, "main", zTbl, zCol, 0, &zCol, 0, 0, 0
+        db, "main", zTab, zCol, 0, &zCol, 0, 0, 0
     );
     nByte += 1 + strlen(zCol);
     nCol++;
@@ -456,7 +505,7 @@ static int idxGetTableInfo(
     pCsr += nCopy;
 
     rc = sqlite3_table_column_metadata(
-        db, "main", zTbl, zCol, 0, &zCol, 0, 0, 0
+        db, "main", zTab, zCol, 0, &zCol, 0, 0, 0
     );
     if( rc==SQLITE_OK ){
       nCopy = strlen(zCol) + 1;
@@ -469,12 +518,12 @@ static int idxGetTableInfo(
   }
   idxFinalize(&rc, p1);
 
-  if( rc==SQLITE_OK ){
-    pScan->pTable = pNew;
-  }else{
+  if( rc!=SQLITE_OK ){
     sqlite3_free(pNew);
+    pNew = 0;
   }
 
+  *ppOut = pNew;
   return rc;
 }
 
@@ -515,6 +564,10 @@ static char *idxAppendText(int *pRc, char *zIn, const char *zFmt, ...){
   return zRet;
 }
 
+/*
+** Return true if zId must be quoted in order to use it as an SQL
+** identifier, or false otherwise.
+*/
 static int idxIdentifierRequiresQuotes(const char *zId){
   int i;
   for(i=0; zId[i]; i++){
@@ -529,10 +582,14 @@ static int idxIdentifierRequiresQuotes(const char *zId){
   return 0;
 }
 
+/*
+** This function appends an index column definition suitable for constraint
+** pCons to the string passed as zIn and returns the result.
+*/
 static char *idxAppendColDefn(
-  int *pRc, 
-  char *zIn, 
-  IdxTable *pTab, 
+  int *pRc,                       /* IN/OUT: Error code */
+  char *zIn,                      /* Column defn accumulated so far */
+  IdxTable *pTab,                 /* Table index will be created on */
   IdxConstraint *pCons
 ){
   char *zRet = zIn;
@@ -636,6 +693,7 @@ static int idxFindCompatible(
 
 static int idxCreateFromCons(
   sqlite3expert *p,
+  IdxTable *pTab,
   IdxScan *pScan,
   IdxConstraint *pEq, 
   IdxConstraint *pTail
@@ -643,7 +701,6 @@ static int idxCreateFromCons(
   sqlite3 *dbm = p->dbm;
   int rc = SQLITE_OK;
   if( (pEq || pTail) && 0==idxFindCompatible(&rc, dbm, pScan, pEq, pTail) ){
-    IdxTable *pTab = pScan->pTable;
     char *zCols = 0;
     char *zIdx = 0;
     IdxConstraint *pCons;
@@ -704,9 +761,9 @@ static int idxFindConstraint(IdxConstraint *pList, IdxConstraint *p){
 
 static int idxCreateFromWhere(
   sqlite3expert *p, 
+  IdxTable *pTab,
   i64 mask,                       /* Consider only these constraints */
   IdxScan *pScan,                 /* Create indexes for this scan */
-  IdxWhere *pWhere,               /* Read constraints from here */
   IdxConstraint *pEq,             /* == constraints for inclusion */
   IdxConstraint *pTail            /* range/ORDER BY constraints for inclusion */
 ){
@@ -715,7 +772,7 @@ static int idxCreateFromWhere(
   int rc;
 
   /* Gather up all the == constraints that match the mask. */
-  for(pCon=pWhere->pEq; pCon; pCon=pCon->pNext){
+  for(pCon=pScan->pEq; pCon; pCon=pCon->pNext){
     if( (mask & pCon->depmask)==pCon->depmask 
      && idxFindConstraint(p1, pCon)==0
      && idxFindConstraint(pTail, pCon)==0
@@ -727,18 +784,18 @@ static int idxCreateFromWhere(
 
   /* Create an index using the == constraints collected above. And the
   ** range constraint/ORDER BY terms passed in by the caller, if any. */
-  rc = idxCreateFromCons(p, pScan, p1, pTail);
+  rc = idxCreateFromCons(p, pTab, pScan, p1, pTail);
 
   /* If no range/ORDER BY passed by the caller, create a version of the
   ** index for each range constraint that matches the mask. */
   if( pTail==0 ){
-    for(pCon=pWhere->pRange; rc==SQLITE_OK && pCon; pCon=pCon->pNext){
+    for(pCon=pScan->pRange; rc==SQLITE_OK && pCon; pCon=pCon->pNext){
       assert( pCon->pLink==0 );
       if( (mask & pCon->depmask)==pCon->depmask
         && idxFindConstraint(pEq, pCon)==0
         && idxFindConstraint(pTail, pCon)==0
       ){
-        rc = idxCreateFromCons(p, pScan, p1, pCon);
+        rc = idxCreateFromCons(p, pTab, pScan, p1, pCon);
       }
     }
   }
@@ -758,30 +815,36 @@ static int idxCreateCandidates(sqlite3expert *p, char **pzErr){
 
   for(pIter=p->pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
     IdxHash64Entry *pEntry;
-    IdxWhere *pWhere = &pIter->where;
     IdxConstraint *pCons;
+    IdxTable *pTab = 0;
+
+    rc = idxGetTableInfo(p->dbm, pIter->zTable, &pTab, pzErr);
 
     idxHash64Add(&rc, &hMask, 0);
-    for(pCons=pIter->where.pEq; pCons; pCons=pCons->pNext){
+    for(pCons=pIter->pEq; pCons; pCons=pCons->pNext){
       for(pEntry=hMask.pFirst; pEntry; pEntry=pEntry->pNext){
         idxHash64Add(&rc, &hMask, pEntry->iVal | (u64)pCons->depmask);
       }
     }
 
-    for(pEntry=hMask.pFirst; pEntry; pEntry=pEntry->pNext){
+    for(pEntry=hMask.pFirst; rc==SQLITE_OK && pEntry; pEntry=pEntry->pNext){
       i64 mask = (i64)pEntry->iVal;
-      rc = idxCreateFromWhere(p, mask, pIter, pWhere, 0, 0);
+      rc = idxCreateFromWhere(p, pTab, mask, pIter, 0, 0);
       if( rc==SQLITE_OK && pIter->pOrder ){
-        rc = idxCreateFromWhere(p, mask, pIter, pWhere, 0, pIter->pOrder);
+        rc = idxCreateFromWhere(p, pTab, mask, pIter, 0, pIter->pOrder);
       }
     }
 
+    sqlite3_free(pTab);
     idxHash64Clear(&hMask);
   }
 
   return rc;
 }
 
+/*
+** Free all elements of the linked list starting at pConstraint.
+*/
 static void idxConstraintFree(IdxConstraint *pConstraint){
   IdxConstraint *pNext;
   IdxConstraint *p;
@@ -802,9 +865,8 @@ static void idxScanFree(IdxScan *pScan, IdxScan *pLast){
   for(p=pScan; p!=pLast; p=pNext){
     pNext = p->pNextScan;
     idxConstraintFree(p->pOrder);
-    idxConstraintFree(p->where.pEq);
-    idxConstraintFree(p->where.pRange);
-    sqlite3_free(p->pTable);
+    idxConstraintFree(p->pEq);
+    idxConstraintFree(p->pRange);
     sqlite3_free(p);
   }
 }
@@ -825,6 +887,11 @@ static void idxStatementFree(IdxStatement *pStatement, IdxStatement *pLast){
 }
 
 
+/*
+** This function is called after candidate indexes have been created. It
+** runs all the queries to see which indexes they prefer, and populates
+** IdxStatement.zIdx and IdxStatement.zEQP with the results.
+*/
 int idxFindIndexes(
   sqlite3expert *p,
   char **pzErr                         /* OUT: Error message (sqlite3_malloc) */
@@ -981,18 +1048,10 @@ int sqlite3_expert_sql(
 }
 
 int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr){
-  int rc = SQLITE_OK;
-  IdxScan *pIter;
-
-  /* Load IdxTable objects */
-  for(pIter=p->pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
-    rc = idxGetTableInfo(p->dbm, pIter, pzErr);
-  }
+  int rc;
 
   /* Create candidate indexes within the in-memory database file */
-  if( rc==SQLITE_OK ){
-    rc = idxCreateCandidates(p, pzErr);
-  }
+  rc = idxCreateCandidates(p, pzErr);
 
   /* Figure out which of the candidate indexes are preferred by the query
   ** planner and report the results to the user.  */
diff --git a/manifest b/manifest
index 7660557a22..ab763c7f7c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Changes\sto\sallow\sthe\scode\sin\ssqlite3expert.c\sto\sbe\stested\sdirectly\s(via\sthe\nAPI\sin\ssqlite3expert.h)\sinstead\sof\sby\sinvoking\sthe\ssqlite3_expert\sapplication.\nFix\smemory\sleaks\sand\sother\sproblems.
-D 2017-04-10T16:13:20.707
+C Add\sext/expert/README.md.
+D 2017-04-10T20:00:26.414
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a
@@ -40,9 +40,10 @@ F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd
 F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
+F ext/expert/README.md 9f15075ec5ad772808eff55ef044c31140fd1146aa0a3c47eafd155e71851b01
 F ext/expert/expert.c bf0fd71921cb7b807cda9a76fb380e3d6e6b980d6167093b2952b41ec9ad8f46
-F ext/expert/expert1.test c1b1405f3ac20e9f71dacdf7bd68ff22e273b249a419260b123ebe385daf2db5 w test/expert1.test
-F ext/expert/sqlite3expert.c b87f13e90b999b5b10c4ec004b6a935150c00d3af1a16944e262172b9b831b8c
+F ext/expert/expert1.test d4451ccdca910ec3652a7a1994210fe2b7bc8f56431ed0cf2ff3dee798c89084
+F ext/expert/sqlite3expert.c 8bcb83b3723239dc6a2a199e7a030741b7ecf47814828a1da7ea559aa42f094a
 F ext/expert/sqlite3expert.h feeaee4ab73ba52426329781bbb28032ce18cf5abd2bf6221bac2df4c32b3013
 F ext/expert/test_expert.c bad0611732d07180d586bd589cbb7713dc3ab0338c52bff29680eb2007678c05
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
@@ -1576,7 +1577,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0857c48e02a76490fc623364f77363165dea94ec254f93d8f0fd0bac2968c572
-R edcb3d0fdfbb918bffbc8d72b5429335
+P 5dd9831721b70a89a26728adcd49c7f6103ef8266891a79c2db34d913702709e
+R 119e273740189bcdef018d8609f3a756
 U dan
-Z 0d7d47356215948792ad81449f48b82b
+Z 664c10ad907cc8fb129a4f4feaf498a1
diff --git a/manifest.uuid b/manifest.uuid
index a059662f4d..935458cfd1 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5dd9831721b70a89a26728adcd49c7f6103ef8266891a79c2db34d913702709e
\ No newline at end of file
+9318f1b9ed2d8da3a82ea69179e2d56a99d326c7721642665f87f6a4534e7bf0
\ No newline at end of file

From a4e61024d75cd1ac531553cee0a0dae13b460474 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 11 Apr 2017 17:43:12 +0000
Subject: [PATCH 027/488] Add header comments to the API functions in
 sqlite3expert.h. Include a list of all candidate indexes in the report output
 by the sqlite3_expert program.

FossilOrigin-Name: 0c45c5eb9f0f171b8d7c5f0d2973f9f59915467506cdff1450f3e4b2134a01ca
---
 ext/expert/expert.c        |  8 +++-
 ext/expert/expert1.test    |  4 +-
 ext/expert/sqlite3expert.c | 38 ++++++++--------
 ext/expert/sqlite3expert.h | 88 +++++++++++++++++++++++++++++++++++---
 manifest                   | 18 ++++----
 manifest.uuid              |  2 +-
 6 files changed, 124 insertions(+), 34 deletions(-)

diff --git a/ext/expert/expert.c b/ext/expert/expert.c
index fca2d05194..04be60b9d1 100644
--- a/ext/expert/expert.c
+++ b/ext/expert/expert.c
@@ -119,12 +119,18 @@ int main(int argc, char **argv){
 
   if( rc==SQLITE_OK ){
     int nQuery = sqlite3_expert_count(p);
+    if( iVerbose>0 ){
+      const char *zCand = sqlite3_expert_report(p,0,EXPERT_REPORT_CANDIDATES);
+      fprintf(stdout, "-- Candidates -------------------------------\n");
+      fprintf(stdout, "%s\n", zCand);
+    }
     for(i=0; i<nQuery; i++){
       const char *zSql = sqlite3_expert_report(p, i, EXPERT_REPORT_SQL);
       const char *zIdx = sqlite3_expert_report(p, i, EXPERT_REPORT_INDEXES);
       const char *zEQP = sqlite3_expert_report(p, i, EXPERT_REPORT_PLAN);
+      if( zIdx==0 ) zIdx = "(no new indexes)";
       if( iVerbose>0 ){
-        fprintf(stdout, "-- query %d ----------------------------------\n",i+1);
+        fprintf(stdout, "-- Query %d ----------------------------------\n",i+1);
         fprintf(stdout, "%s\n\n", zSql);
       }
       fprintf(stdout, "%s\n%s\n", zIdx, zEQP);
diff --git a/ext/expert/expert1.test b/ext/expert/expert1.test
index 2e8c1c7e76..11d5bd66d4 100644
--- a/ext/expert/expert1.test
+++ b/ext/expert/expert1.test
@@ -61,7 +61,9 @@ foreach {tn setup} {
 
       set result [list]
       for {set i 0} {$i < [$expert count]} {incr i} {
-        lappend result [string trim [$expert report $i indexes]]
+        set idx [string trim [$expert report $i indexes]]
+        if {$idx==""} {set idx "(no new indexes)"}
+        lappend result $idx
         lappend result [string trim [$expert report $i plan]]
       }
 
diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index faa6966932..1f77a15bc3 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -134,12 +134,13 @@ struct IdxHash64 {
 struct sqlite3expert {
   sqlite3 *db;                    /* User database */
   sqlite3 *dbm;                   /* In-memory db for this analysis */
-  int bRun;                       /* True once analysis has run */
-  char **pzErrmsg;
   IdxScan *pScan;                 /* List of scan objects */
   IdxStatement *pStatement;       /* List of IdxStatement objects */
+  int bRun;                       /* True once analysis has run */
+  char **pzErrmsg;
   int rc;                         /* Error code from whereinfo hook */
   IdxHash hIdx;                   /* Hash containing all candidate indexes */
+  char *zCandidates;              /* For EXPERT_REPORT_CANDIDATES */
 };
 
 
@@ -948,9 +949,6 @@ int idxFindIndexes(
     for(pEntry=hIdx.pFirst; pEntry; pEntry=pEntry->pNext){
       pStmt->zIdx = idxAppendText(&rc, pStmt->zIdx, "%s;\n", pEntry->zKey);
     }
-    if( pStmt->zIdx==0 ){
-      pStmt->zIdx = idxAppendText(&rc, 0, "(no new indexes)\n");
-    }
 
     idxFinalize(&rc, pExplain);
   }
@@ -1049,10 +1047,15 @@ int sqlite3_expert_sql(
 
 int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr){
   int rc;
+  IdxHashEntry *pEntry;
 
   /* Create candidate indexes within the in-memory database file */
   rc = idxCreateCandidates(p, pzErr);
 
+  for(pEntry=p->hIdx.pFirst; pEntry; pEntry=pEntry->pNext){
+    p->zCandidates = idxAppendText(&rc, p->zCandidates, "%s;\n", pEntry->zVal);
+  }
+
   /* Figure out which of the candidate indexes are preferred by the query
   ** planner and report the results to the user.  */
   if( rc==SQLITE_OK ){
@@ -1084,18 +1087,19 @@ const char *sqlite3_expert_report(sqlite3expert *p, int iStmt, int eReport){
 
   if( p->bRun==0 ) return 0;
   for(pStmt=p->pStatement; pStmt && pStmt->iId!=iStmt; pStmt=pStmt->pNext);
-  if( pStmt ){
-    switch( eReport ){
-      case EXPERT_REPORT_SQL:
-        zRet = pStmt->zSql;
-        break;
-      case EXPERT_REPORT_INDEXES:
-        zRet = pStmt->zIdx;
-        break;
-      case EXPERT_REPORT_PLAN:
-        zRet = pStmt->zEQP;
-        break;
-    }
+  switch( eReport ){
+    case EXPERT_REPORT_SQL:
+      if( pStmt ) zRet = pStmt->zSql;
+      break;
+    case EXPERT_REPORT_INDEXES:
+      if( pStmt ) zRet = pStmt->zIdx;
+      break;
+    case EXPERT_REPORT_PLAN:
+      if( pStmt ) zRet = pStmt->zEQP;
+      break;
+    case EXPERT_REPORT_CANDIDATES:
+      zRet = p->zCandidates;
+      break;
   }
   return zRet;
 }
diff --git a/ext/expert/sqlite3expert.h b/ext/expert/sqlite3expert.h
index 6384f8e556..455f41d991 100644
--- a/ext/expert/sqlite3expert.h
+++ b/ext/expert/sqlite3expert.h
@@ -18,27 +18,102 @@ typedef struct sqlite3expert sqlite3expert;
 
 /*
 ** Create a new sqlite3expert object.
+**
+** If successful, a pointer to the new object is returned and (*pzErr) set
+** to NULL. Or, if an error occurs, NULL is returned and (*pzErr) set to
+** an English-language error message. In this case it is the responsibility
+** of the caller to eventually free the error message buffer using
+** sqlite3_free().
 */
 sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErr);
 
 /*
-** Add an SQL statement to the analysis.
+** Specify zero or more SQL statements to be included in the analysis.
+**
+** Buffer zSql must contain zero or more complete SQL statements. This
+** function parses all statements contained in the buffer and adds them
+** to the internal list of statements to analyze. If successful, SQLITE_OK
+** is returned and (*pzErr) set to NULL. Or, if an error occurs - for example
+** due to a error in the SQL - an SQLite error code is returned and (*pzErr)
+** may be set to point to an English language error message. In this case
+** the caller is responsible for eventually freeing the error message buffer
+** using sqlite3_free().
+**
+** If an error does occur while processing one of the statements in the
+** buffer passed as the second argument, none of the statements in the
+** buffer are added to the analysis.
+**
+** This function must be called before sqlite3_expert_analyze(). If a call
+** to this function is made on an sqlite3expert object that has already
+** been passed to sqlite3_expert_analyze() SQLITE_MISUSE is returned
+** immediately and no statements are added to the analysis.
 */
 int sqlite3_expert_sql(
-  sqlite3expert *p,               /* From sqlite3_expert_new() */
-  const char *zSql,               /* SQL statement to add */
+  sqlite3expert *p,               /* From a successful sqlite3_expert_new() */
+  const char *zSql,               /* SQL statement(s) to add */
   char **pzErr                    /* OUT: Error message (if any) */
 );
 
+/*
+** This function is called after the sqlite3expert object has been configured
+** with all SQL statements using sqlite3_expert_sql() to actually perform
+** the analysis. Once this function has been called, it is not possible to
+** add further SQL statements to the analysis.
+**
+** If successful, SQLITE_OK is returned and (*pzErr) is set to NULL. Or, if
+** an error occurs, an SQLite error code is returned and (*pzErr) set to 
+** point to a buffer containing an English language error message. In this
+** case it is the responsibility of the caller to eventually free the buffer
+** using sqlite3_free().
+**
+** If an error does occur within this function, the sqlite3expert object
+** is no longer useful for any purpose. At that point it is no longer
+** possible to add further SQL statements to the object or to re-attempt
+** the analysis. The sqlite3expert object must still be freed using a call
+** sqlite3_expert_destroy().
+*/
 int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr);
 
 /*
-** Return the total number of SQL queries loaded via sqlite3_expert_sql().
+** Return the total number of statements loaded using sqlite3_expert_sql().
+** The total number of SQL statements may be different from the total number
+** to calls to sqlite3_expert_sql().
 */
 int sqlite3_expert_count(sqlite3expert*);
 
 /*
 ** Return a component of the report.
+**
+** This function is called after sqlite3_expert_analyze() to extract the
+** results of the analysis. Each call to this function returns either a
+** NULL pointer or a pointer to a buffer containing a nul-terminated string.
+** The value passed as the third argument must be one of the EXPERT_REPORT_*
+** #define constants defined below.
+**
+** For some EXPERT_REPORT_* parameters, the buffer returned contains 
+** information relating to a specific SQL statement. In these cases that
+** SQL statement is identified by the value passed as the second argument.
+** SQL statements are numbered from 0 in the order in which they are parsed.
+** If an out-of-range value (less than zero or equal to or greater than the
+** value returned by sqlite3_expert_count()) is passed as the second argument
+** along with such an EXPERT_REPORT_* parameter, NULL is always returned.
+**
+** EXPERT_REPORT_SQL:
+**   Return the text of SQL statement iStmt.
+**
+** EXPERT_REPORT_INDEXES:
+**   Return a buffer containing the CREATE INDEX statements for all recommended
+**   indexes for statement iStmt. If there are no new recommeded indexes, NULL 
+**   is returned.
+**
+** EXPERT_REPORT_PLAN:
+**   Return a buffer containing the EXPLAIN QUERY PLAN output for SQL query
+**   iStmt after the proposed indexes have been added to the database schema.
+**
+** EXPERT_REPORT_CANDIDATES:
+**   Return a pointer to a buffer containing the CREATE INDEX statements 
+**   for all indexes that were tested (for all SQL statements). The iStmt
+**   parameter is ignored for EXPERT_REPORT_CANDIDATES calls.
 */
 const char *sqlite3_expert_report(sqlite3expert*, int iStmt, int eReport);
 
@@ -48,9 +123,12 @@ const char *sqlite3_expert_report(sqlite3expert*, int iStmt, int eReport);
 #define EXPERT_REPORT_SQL        1
 #define EXPERT_REPORT_INDEXES    2
 #define EXPERT_REPORT_PLAN       3
+#define EXPERT_REPORT_CANDIDATES 4
 
 /*
-** Free an (sqlite3expert*) handle allocated by sqlite3-expert_new().
+** Free an (sqlite3expert*) handle and all associated resources. There 
+** should be one call to this function for each successful call to 
+** sqlite3-expert_new().
 */
 void sqlite3_expert_destroy(sqlite3expert*);
 
diff --git a/manifest b/manifest
index ab763c7f7c..8f08e742b8 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sext/expert/README.md.
-D 2017-04-10T20:00:26.414
+C Add\sheader\scomments\sto\sthe\sAPI\sfunctions\sin\ssqlite3expert.h.\sInclude\sa\slist\sof\nall\scandidate\sindexes\sin\sthe\sreport\soutput\sby\sthe\ssqlite3_expert\sprogram.
+D 2017-04-11T17:43:12.842
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a
@@ -41,10 +41,10 @@ F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md 9f15075ec5ad772808eff55ef044c31140fd1146aa0a3c47eafd155e71851b01
-F ext/expert/expert.c bf0fd71921cb7b807cda9a76fb380e3d6e6b980d6167093b2952b41ec9ad8f46
-F ext/expert/expert1.test d4451ccdca910ec3652a7a1994210fe2b7bc8f56431ed0cf2ff3dee798c89084
-F ext/expert/sqlite3expert.c 8bcb83b3723239dc6a2a199e7a030741b7ecf47814828a1da7ea559aa42f094a
-F ext/expert/sqlite3expert.h feeaee4ab73ba52426329781bbb28032ce18cf5abd2bf6221bac2df4c32b3013
+F ext/expert/expert.c 5bcc2fba8509fd1ddeb755df9273e9667ce82cda97b0dd6240792d9b9e86a873
+F ext/expert/expert1.test cc33f9390f205bfeb6a30c7618b24a5675f4b9cb844c9154c11398a7f1477e81
+F ext/expert/sqlite3expert.c 2b22a5fbd093a7020ea9625928292265f31b8e78f6feabb987e71a79b2a29089
+F ext/expert/sqlite3expert.h b1c9eedeb647fd734c4206ae6851635284cfbfa5fb688eff74c3265c9f949b4d
 F ext/expert/test_expert.c bad0611732d07180d586bd589cbb7713dc3ab0338c52bff29680eb2007678c05
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
 F ext/fts1/ft_hash.c 3927bd880e65329bdc6f506555b228b28924921b
@@ -1577,7 +1577,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5dd9831721b70a89a26728adcd49c7f6103ef8266891a79c2db34d913702709e
-R 119e273740189bcdef018d8609f3a756
+P 9318f1b9ed2d8da3a82ea69179e2d56a99d326c7721642665f87f6a4534e7bf0
+R 15f16c821c74aae47eac0f21fc6788f2
 U dan
-Z 664c10ad907cc8fb129a4f4feaf498a1
+Z 914bc7396f378320666a7fa77328351c
diff --git a/manifest.uuid b/manifest.uuid
index 935458cfd1..8933394847 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-9318f1b9ed2d8da3a82ea69179e2d56a99d326c7721642665f87f6a4534e7bf0
\ No newline at end of file
+0c45c5eb9f0f171b8d7c5f0d2973f9f59915467506cdff1450f3e4b2134a01ca
\ No newline at end of file

From c2309693b98d2b45a1629b28537507edf35d5e88 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 11 Apr 2017 18:29:14 +0000
Subject: [PATCH 028/488] Fix a formatting issue in the output of the
 sqlite3_expert program.

FossilOrigin-Name: cc8c3581060ffef02290b680183e6f6bc9837ba3550e74c8aaabdc7c45edc223
---
 ext/expert/expert.c |  2 +-
 manifest            | 12 ++++++------
 manifest.uuid       |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/ext/expert/expert.c b/ext/expert/expert.c
index 04be60b9d1..19b52c867d 100644
--- a/ext/expert/expert.c
+++ b/ext/expert/expert.c
@@ -128,7 +128,7 @@ int main(int argc, char **argv){
       const char *zSql = sqlite3_expert_report(p, i, EXPERT_REPORT_SQL);
       const char *zIdx = sqlite3_expert_report(p, i, EXPERT_REPORT_INDEXES);
       const char *zEQP = sqlite3_expert_report(p, i, EXPERT_REPORT_PLAN);
-      if( zIdx==0 ) zIdx = "(no new indexes)";
+      if( zIdx==0 ) zIdx = "(no new indexes)\n";
       if( iVerbose>0 ){
         fprintf(stdout, "-- Query %d ----------------------------------\n",i+1);
         fprintf(stdout, "%s\n\n", zSql);
diff --git a/manifest b/manifest
index 8f08e742b8..dd5849b713 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sheader\scomments\sto\sthe\sAPI\sfunctions\sin\ssqlite3expert.h.\sInclude\sa\slist\sof\nall\scandidate\sindexes\sin\sthe\sreport\soutput\sby\sthe\ssqlite3_expert\sprogram.
-D 2017-04-11T17:43:12.842
+C Fix\sa\sformatting\sissue\sin\sthe\soutput\sof\sthe\ssqlite3_expert\sprogram.
+D 2017-04-11T18:29:14.206
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a
@@ -41,7 +41,7 @@ F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md 9f15075ec5ad772808eff55ef044c31140fd1146aa0a3c47eafd155e71851b01
-F ext/expert/expert.c 5bcc2fba8509fd1ddeb755df9273e9667ce82cda97b0dd6240792d9b9e86a873
+F ext/expert/expert.c 6349cf8d26c847f5f0fa7e25772b614c67f60f3c850dca0d75d55eb27cf3f69b
 F ext/expert/expert1.test cc33f9390f205bfeb6a30c7618b24a5675f4b9cb844c9154c11398a7f1477e81
 F ext/expert/sqlite3expert.c 2b22a5fbd093a7020ea9625928292265f31b8e78f6feabb987e71a79b2a29089
 F ext/expert/sqlite3expert.h b1c9eedeb647fd734c4206ae6851635284cfbfa5fb688eff74c3265c9f949b4d
@@ -1577,7 +1577,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 9318f1b9ed2d8da3a82ea69179e2d56a99d326c7721642665f87f6a4534e7bf0
-R 15f16c821c74aae47eac0f21fc6788f2
+P 0c45c5eb9f0f171b8d7c5f0d2973f9f59915467506cdff1450f3e4b2134a01ca
+R 33f15deae23049b95df8ad5185895aff
 U dan
-Z 914bc7396f378320666a7fa77328351c
+Z 2806083003fa51636f951574e2a01b7d
diff --git a/manifest.uuid b/manifest.uuid
index 8933394847..70191b5b97 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0c45c5eb9f0f171b8d7c5f0d2973f9f59915467506cdff1450f3e4b2134a01ca
\ No newline at end of file
+cc8c3581060ffef02290b680183e6f6bc9837ba3550e74c8aaabdc7c45edc223
\ No newline at end of file

From 0824ccf29bb9f5c72261d69a7b43740ae6a27735 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 14 Apr 2017 19:41:37 +0000
Subject: [PATCH 029/488] Modify the code in ext/expert/ to use the vtab
 interface instead of sqlite3_whereinfo_hook(). Remove
 sqlite3_whereinfo_hook().

FossilOrigin-Name: 3bb6585004090dbf92dd5e9abdf0fd2c921e64b5b3121c4fb7446db764ab59e5
---
 ext/expert/expert1.test    |  25 ++-
 ext/expert/sqlite3expert.c | 352 ++++++++++++++++++++++++++++---------
 manifest                   |  22 +--
 manifest.uuid              |   2 +-
 src/main.c                 |  20 ---
 src/sqlite.h.in            | 144 +--------------
 src/sqliteInt.h            |   5 +-
 src/where.c                | 243 +++++--------------------
 8 files changed, 344 insertions(+), 469 deletions(-)

diff --git a/ext/expert/expert1.test b/ext/expert/expert1.test
index 11d5bd66d4..437ba45600 100644
--- a/ext/expert/expert1.test
+++ b/ext/expert/expert1.test
@@ -139,14 +139,14 @@ do_setup_rec_test $tn.1.7 {
   0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_033e95fe
 }
 
-do_setup_rec_test $tn.1.8 {
-  CREATE TABLE t1(a, b, c);
-} {
-  SELECT * FROM t1 ORDER BY a ASC, b COLLATE nocase DESC, c ASC;
-} {
-  CREATE INDEX t1_idx_5be6e222 ON t1(a, b COLLATE NOCASE DESC, c);
-  0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_5be6e222
-}
+#do_setup_rec_test $tn.1.8 {
+#  CREATE TABLE t1(a, b, c);
+#} {
+#  SELECT * FROM t1 ORDER BY a ASC, b COLLATE nocase DESC, c ASC;
+#} {
+#  CREATE INDEX t1_idx_5be6e222 ON t1(a, b COLLATE NOCASE DESC, c);
+#  0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_5be6e222
+#}
 
 do_setup_rec_test $tn.1.9 {
   CREATE TABLE t1(a COLLATE NOCase, b, c);
@@ -157,6 +157,15 @@ do_setup_rec_test $tn.1.9 {
   0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_00000061 (a=?)
 }
 
+do_setup_rec_test $tn.1.10 {
+  CREATE TABLE t1(a, b COLLATE nocase, c);
+} {
+  SELECT * FROM t1 ORDER BY a ASC, b DESC, c ASC;
+} {
+  CREATE INDEX t1_idx_5cb97285 ON t1(a, b DESC, c);
+  0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_5cb97285
+}
+
 
 # Tables with names that require quotes.
 #
diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index 1f77a15bc3..3cd110c367 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -50,7 +50,7 @@ struct IdxConstraint {
 ** A single scan of a single table.
 */
 struct IdxScan {
-  char *zTable;                   /* Name of table to scan */
+  IdxTable *pTab;                 /* Associated table object */
   int iDb;                        /* Database containing table zTable */
   i64 covering;                   /* Mask of columns required for cov. index */
   IdxConstraint *pOrder;          /* ORDER BY columns */
@@ -70,7 +70,9 @@ struct IdxColumn {
 };
 struct IdxTable {
   int nCol;
+  char *zName;                    /* Table name */
   IdxColumn *aCol;
+  IdxTable *pNext;                /* Next table in linked list of all tables */
 };
 
 /*
@@ -134,6 +136,9 @@ struct IdxHash64 {
 struct sqlite3expert {
   sqlite3 *db;                    /* User database */
   sqlite3 *dbm;                   /* In-memory db for this analysis */
+  sqlite3 *dbv;                   /* Vtab schema for this analysis */
+  IdxTable *pTable;               /* List of all IdxTable objects */
+
   IdxScan *pScan;                 /* List of scan objects */
   IdxStatement *pStatement;       /* List of IdxStatement objects */
   int bRun;                       /* True once analysis has run */
@@ -314,79 +319,189 @@ static IdxConstraint *idxNewConstraint(int *pRc, const char *zColl){
   return pNew;
 }
 
-/*
-** sqlite3_whereinfo_hook() callback.
+
+/*************************************************************************
+** Beginning of virtual table implementation.
 */
-static void idxWhereInfo(
-  void *pCtx,                     /* Pointer to sqlite3expert structure */
-  int eOp, 
-  const char *zVal, 
-  int iVal, 
-  u64 mask
+typedef struct ExpertVtab ExpertVtab;
+struct ExpertVtab {
+  sqlite3_vtab base;
+  IdxTable *pTab;
+  sqlite3expert *pExpert;
+};
+
+static char *expertDequote(const char *zIn){
+  int n = strlen(zIn);
+  char *zRet = sqlite3_malloc(n);
+
+  assert( zIn[0]=='\'' );
+  assert( zIn[n-1]=='\'' );
+
+  if( zRet ){
+    int iOut = 0;
+    int iIn = 0;
+    for(iIn=1; iIn<(n-1); iIn++){
+      if( zIn[iIn]=='\'' ){
+        assert( zIn[iIn+1]=='\'' );
+        iIn++;
+      }
+      zRet[iOut++] = zIn[iIn];
+    }
+    zRet[iOut] = '\0';
+  }
+
+  return zRet;
+}
+
+/* 
+** This function is the implementation of both the xConnect and xCreate
+** methods of the r-tree virtual table.
+**
+**   argv[0]   -> module name
+**   argv[1]   -> database name
+**   argv[2]   -> table name
+**   argv[...] -> column names...
+*/
+static int expertConnect(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
 ){
-  sqlite3expert *p = (sqlite3expert*)pCtx;
+  sqlite3expert *pExpert = (sqlite3expert*)pAux;
+  ExpertVtab *p = 0;
+  int rc;
 
-#if 0
-  const char *zOp = 
-    eOp==SQLITE_WHEREINFO_TABLE ? "TABLE" :
-    eOp==SQLITE_WHEREINFO_EQUALS ? "EQUALS" :
-    eOp==SQLITE_WHEREINFO_RANGE ? "RANGE" :
-    eOp==SQLITE_WHEREINFO_ORDERBY ? "ORDERBY" :
-    "!error!";
-  printf("op=%s zVal=%s iVal=%d mask=%llx\n", zOp, zVal, iVal, mask);
-#endif
-
-  if( p->rc==SQLITE_OK ){
-    assert( eOp==SQLITE_WHEREINFO_TABLE || p->pScan!=0 );
-    switch( eOp ){
-      case SQLITE_WHEREINFO_TABLE: {
-        int nVal = strlen(zVal);
-        IdxScan *pNew = (IdxScan*)idxMalloc(&p->rc, sizeof(IdxScan) + nVal + 1);
-        if( !pNew ) return;
-        pNew->zTable = (char*)&pNew[1];
-        memcpy(pNew->zTable, zVal, nVal+1);
-        pNew->pNextScan = p->pScan;
-        pNew->covering = mask;
-        p->pScan = pNew;
-        break;
+  if( argc!=4 ){
+    *pzErr = sqlite3_mprintf("internal error!");
+    rc = SQLITE_ERROR;
+  }else{
+    char *zCreateTable = expertDequote(argv[3]);
+    if( zCreateTable ){
+      rc = sqlite3_declare_vtab(db, zCreateTable);
+      if( rc==SQLITE_OK ){
+        p = idxMalloc(&rc, sizeof(ExpertVtab));
       }
-
-      case SQLITE_WHEREINFO_ORDERBY: {
-        IdxConstraint *pNew = idxNewConstraint(&p->rc, zVal);
-        if( pNew==0 ) return;
-        pNew->iCol = iVal;
-        pNew->bDesc = (int)mask;
-        if( p->pScan->pOrder==0 ){
-          p->pScan->pOrder = pNew;
-        }else{
-          IdxConstraint *pIter;
-          for(pIter=p->pScan->pOrder; pIter->pNext; pIter=pIter->pNext);
-          pIter->pNext = pNew;
-          pIter->pLink = pNew;
-        }
-        break;
+      if( rc==SQLITE_OK ){
+        p->pExpert = pExpert;
+        p->pTab = pExpert->pTable;
+        assert( sqlite3_stricmp(p->pTab->zName, argv[2])==0 );
       }
+      sqlite3_free(zCreateTable);
+    }else{
+      rc = SQLITE_NOMEM;
+    }
+  }
 
-      case SQLITE_WHEREINFO_EQUALS:
-      case SQLITE_WHEREINFO_RANGE: {
-        IdxConstraint *pNew = idxNewConstraint(&p->rc, zVal);
-        if( pNew==0 ) return;
-        pNew->iCol = iVal;
-        pNew->depmask = mask;
+  *ppVtab = (sqlite3_vtab*)p;
+  return rc;
+}
 
-        if( eOp==SQLITE_WHEREINFO_RANGE ){
-          pNew->pNext = p->pScan->pRange;
-          p->pScan->pRange = pNew;
-        }else{
-          pNew->pNext = p->pScan->pEq;
-          p->pScan->pEq = pNew;
+static int expertDisconnect(sqlite3_vtab *pVtab){
+  ExpertVtab *p = (ExpertVtab*)pVtab;
+  sqlite3_free(p);
+  return SQLITE_OK;
+}
+
+static int expertBestIndex(sqlite3_vtab *pVtab, sqlite3_index_info *pIdxInfo){
+  ExpertVtab *p = (ExpertVtab*)pVtab;
+  sqlite3 *dbv = p->pExpert->dbv;
+  int rc = SQLITE_OK;
+  int n = 0;
+  IdxScan *pScan;
+  const int opmask = 
+    SQLITE_INDEX_CONSTRAINT_EQ | SQLITE_INDEX_CONSTRAINT_GT |
+    SQLITE_INDEX_CONSTRAINT_LT | SQLITE_INDEX_CONSTRAINT_GE |
+    SQLITE_INDEX_CONSTRAINT_LE;
+
+  pScan = idxMalloc(&rc, sizeof(IdxScan));
+  if( pScan ){
+    int i;
+
+    /* Link the new scan object into the list */
+    pScan->pTab = p->pTab;
+    pScan->pNextScan = p->pExpert->pScan;
+    p->pExpert->pScan = pScan;
+
+    /* Add the constraints to the IdxScan object */
+    for(i=0; i<pIdxInfo->nConstraint; i++){
+      int op = pIdxInfo->aConstraint[i].op;
+      if( op&opmask ){
+        IdxConstraint *pNew;
+        const char *zColl = sqlite3_vtab_collation(dbv, i);
+        pNew = idxNewConstraint(&rc, zColl);
+        if( pNew ){
+          pNew->iCol = pIdxInfo->aConstraint[i].iColumn;
+          if( op==SQLITE_INDEX_CONSTRAINT_EQ ){
+            pNew->pNext = pScan->pEq;
+            pScan->pEq = pNew;
+          }else{
+            pNew->bRange = 1;
+            pNew->pNext = pScan->pRange;
+            pScan->pRange = pNew;
+          }
         }
-        break;
+        if( pIdxInfo->aConstraint[i].usable ){
+          n++;
+          pIdxInfo->aConstraintUsage[i].argvIndex = n;
+        }
+      }
+    }
+
+    /* Add the ORDER BY to the IdxScan object */
+    for(i=pIdxInfo->nOrderBy-1; i>=0; i--){
+      IdxConstraint *pNew;
+      const char *zColl = sqlite3_vtab_collation(dbv, i+pIdxInfo->nConstraint);
+      pNew = idxNewConstraint(&rc, zColl);
+      if( pNew ){
+        pNew->iCol = pIdxInfo->aOrderBy[i].iColumn;
+        pNew->bDesc = pIdxInfo->aOrderBy[i].desc;
+        pNew->pNext = pScan->pOrder;
+        pNew->pLink = pScan->pOrder;
+        pScan->pOrder = pNew;
+        n++;
       }
     }
   }
+
+  pIdxInfo->estimatedCost = 1000000.0 / n;
+  return rc;
 }
 
+static int idxRegisterVtab(sqlite3expert *p){
+  static sqlite3_module expertModule = {
+    2,                            /* iVersion */
+    expertConnect,                /* xCreate - create a table */
+    expertConnect,                /* xConnect - connect to an existing table */
+    expertBestIndex,              /* xBestIndex - Determine search strategy */
+    expertDisconnect,             /* xDisconnect - Disconnect from a table */
+    expertDisconnect,             /* xDestroy - Drop a table */
+    0,                            /* xOpen - open a cursor */
+    0,                            /* xClose - close a cursor */
+    0,                            /* xFilter - configure scan constraints */
+    0,                            /* xNext - advance a cursor */
+    0,                            /* xEof */
+    0,                            /* xColumn - read data */
+    0,                            /* xRowid - read data */
+    0,                            /* xUpdate - write data */
+    0,                            /* xBegin - begin transaction */
+    0,                            /* xSync - sync transaction */
+    0,                            /* xCommit - commit transaction */
+    0,                            /* xRollback - rollback transaction */
+    0,                            /* xFindFunction - function overloading */
+    0,                            /* xRename - rename the table */
+    0,                            /* xSavepoint */
+    0,                            /* xRelease */
+    0,                            /* xRollbackTo */
+  };
+
+  return sqlite3_create_module(p->dbv, "expert", &expertModule, (void*)p);
+}
+/*
+** End of virtual table implementation.
+*************************************************************************/
+
 /*
 ** An error associated with database handle db has just occurred. Pass
 ** the error message to callback function xOut.
@@ -468,7 +583,8 @@ static int idxGetTableInfo(
 ){
   sqlite3_stmt *p1 = 0;
   int nCol = 0;
-  int nByte = sizeof(IdxTable);
+  int nTab = strlen(zTab);
+  int nByte = sizeof(IdxTable) + nTab + 1;
   IdxTable *pNew = 0;
   int rc, rc2;
   char *pCsr;
@@ -522,6 +638,9 @@ static int idxGetTableInfo(
   if( rc!=SQLITE_OK ){
     sqlite3_free(pNew);
     pNew = 0;
+  }else{
+    pNew->zName = pCsr;
+    memcpy(pNew->zName, zTab, nTab+1);
   }
 
   *ppOut = pNew;
@@ -632,7 +751,7 @@ static int idxFindCompatible(
   IdxConstraint *pEq,             /* List of == constraints */
   IdxConstraint *pTail            /* List of range constraints */
 ){
-  const char *zTbl = pScan->zTable;
+  const char *zTbl = pScan->pTab->zName;
   sqlite3_stmt *pIdxList = 0;
   IdxConstraint *pIter;
   int nEq = 0;                    /* Number of elements in pEq */
@@ -717,21 +836,22 @@ static int idxCreateFromCons(
 
     if( rc==SQLITE_OK ){
       /* Hash the list of columns to come up with a name for the index */
+      const char *zTable = pScan->pTab->zName;
       char *zName;                /* Index name */
       int i;
       for(i=0; zCols[i]; i++){
         h += ((h<<3) + zCols[i]);
       }
-      zName = sqlite3_mprintf("%s_idx_%08x", pScan->zTable, h);
+      zName = sqlite3_mprintf("%s_idx_%08x", zTable, h);
       if( zName==0 ){ 
         rc = SQLITE_NOMEM;
       }else{
-        if( idxIdentifierRequiresQuotes(pScan->zTable) ){
+        if( idxIdentifierRequiresQuotes(zTable) ){
           zFmt = "CREATE INDEX '%q' ON %Q(%s)";
         }else{
           zFmt = "CREATE INDEX %s ON %s(%s)";
         }
-        zIdx = sqlite3_mprintf(zFmt, zName, pScan->zTable, zCols);
+        zIdx = sqlite3_mprintf(zFmt, zName, zTable, zCols);
         if( !zIdx ){
           rc = SQLITE_NOMEM;
         }else{
@@ -817,9 +937,7 @@ static int idxCreateCandidates(sqlite3expert *p, char **pzErr){
   for(pIter=p->pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
     IdxHash64Entry *pEntry;
     IdxConstraint *pCons;
-    IdxTable *pTab = 0;
-
-    rc = idxGetTableInfo(p->dbm, pIter->zTable, &pTab, pzErr);
+    IdxTable *pTab = pIter->pTab;
 
     idxHash64Add(&rc, &hMask, 0);
     for(pCons=pIter->pEq; pCons; pCons=pCons->pNext){
@@ -836,7 +954,6 @@ static int idxCreateCandidates(sqlite3expert *p, char **pzErr){
       }
     }
 
-    sqlite3_free(pTab);
     idxHash64Clear(&hMask);
   }
 
@@ -958,6 +1075,60 @@ int idxFindIndexes(
   return rc;
 }
 
+static int idxCreateVtabSchema(sqlite3expert *p, char **pzErrmsg){
+  int rc = idxRegisterVtab(p);
+  sqlite3_stmt *pSchema = 0;
+
+  /* For each table in the main db schema:
+  **
+  **   1) Add an entry to the p->pTable list, and
+  **   2) Create the equivalent virtual table in dbv.
+  */
+  rc = idxPrepareStmt(p->db, &pSchema, pzErrmsg,
+      "SELECT type, name, sql FROM sqlite_master "
+      "WHERE type IN ('table','view') ORDER BY 1"
+  );
+  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSchema) ){
+    const char *zType = (const char*)sqlite3_column_text(pSchema, 0);
+    const char *zName = (const char*)sqlite3_column_text(pSchema, 1);
+    const char *zSql = (const char*)sqlite3_column_text(pSchema, 2);
+
+    if( zType[0]=='v' ){
+      rc = sqlite3_exec(p->dbv, zSql, 0, 0, pzErrmsg);
+    }else{
+      IdxTable *pTab;
+      rc = idxGetTableInfo(p->db, zName, &pTab, pzErrmsg);
+      if( rc==SQLITE_OK ){
+        int i;
+        char *zInner = 0;
+        char *zOuter = 0;
+        pTab->pNext = p->pTable;
+        p->pTable = pTab;
+
+        /* The statement the vtab will pass to sqlite3_declare_vtab() */
+        zInner = idxAppendText(&rc, 0, "CREATE TABLE x(");
+        for(i=0; i<pTab->nCol; i++){
+          zInner = idxAppendText(&rc, zInner, "%s%Q COLLATE %s", 
+              (i==0 ? "" : ", "), pTab->aCol[i].zName, pTab->aCol[i].zColl
+          );
+        }
+        zInner = idxAppendText(&rc, zInner, ")");
+
+        /* The CVT statement to create the vtab */
+        zOuter = idxAppendText(&rc, 0, 
+            "CREATE VIRTUAL TABLE %Q USING expert(%Q)", zName, zInner
+        );
+        if( rc==SQLITE_OK ){
+          rc = sqlite3_exec(p->dbv, zOuter, 0, 0, pzErrmsg);
+        }
+        sqlite3_free(zInner);
+        sqlite3_free(zOuter);
+      }
+    }
+  }
+  return rc;
+}
+
 /*
 ** Allocate a new sqlite3expert object.
 */
@@ -966,12 +1137,20 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){
   sqlite3expert *pNew;
 
   pNew = (sqlite3expert*)idxMalloc(&rc, sizeof(sqlite3expert));
-  pNew->db = db;
 
-  /* Open an in-memory database to work with. The main in-memory 
-  ** database schema contains tables similar to those in the users 
-  ** database (handle db).  */
-  rc = sqlite3_open(":memory:", &pNew->dbm);
+  /* Open two in-memory databases to work with. The "vtab database" (dbv)
+  ** will contain a virtual table corresponding to each real table in
+  ** the user database schema, and a copy of each view. It is used to
+  ** collect information regarding the WHERE, ORDER BY and other clauses
+  ** of the user's query.
+  */
+  if( rc==SQLITE_OK ){
+    pNew->db = db;
+    rc = sqlite3_open(":memory:", &pNew->dbv);
+  }
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_open(":memory:", &pNew->dbm);
+  }
 
   /* Copy the entire schema of database [db] into [dbm]. */
   if( rc==SQLITE_OK ){
@@ -986,6 +1165,11 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){
     idxFinalize(&rc, pSql);
   }
 
+  /* Create the vtab schema */
+  if( rc==SQLITE_OK ){
+    rc = idxCreateVtabSchema(pNew, pzErrmsg);
+  }
+
   /* If an error has occurred, free the new object and reutrn NULL. Otherwise,
   ** return the new sqlite3expert handle.  */
   if( rc!=SQLITE_OK ){
@@ -1010,10 +1194,9 @@ int sqlite3_expert_sql(
 
   if( p->bRun ) return SQLITE_MISUSE;
 
-  sqlite3_whereinfo_hook(p->db, idxWhereInfo, p);
   while( rc==SQLITE_OK && zStmt && zStmt[0] ){
     sqlite3_stmt *pStmt = 0;
-    rc = sqlite3_prepare_v2(p->db, zStmt, -1, &pStmt, &zStmt);
+    rc = sqlite3_prepare_v2(p->dbv, zStmt, -1, &pStmt, &zStmt);
     if( rc==SQLITE_OK ){
       if( pStmt ){
         IdxStatement *pNew;
@@ -1033,7 +1216,6 @@ int sqlite3_expert_sql(
       idxDatabaseError(p->db, pzErr);
     }
   }
-  sqlite3_whereinfo_hook(p->db, 0, 0);
 
   if( rc!=SQLITE_OK ){
     idxScanFree(p->pScan, pScanOrig);
@@ -1052,6 +1234,7 @@ int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr){
   /* Create candidate indexes within the in-memory database file */
   rc = idxCreateCandidates(p, pzErr);
 
+  /* Formulate the EXPERT_REPORT_CANDIDATES text */
   for(pEntry=p->hIdx.pFirst; pEntry; pEntry=pEntry->pNext){
     p->zCandidates = idxAppendText(&rc, p->zCandidates, "%s;\n", pEntry->zVal);
   }
@@ -1108,11 +1291,14 @@ const char *sqlite3_expert_report(sqlite3expert *p, int iStmt, int eReport){
 ** Free an sqlite3expert object.
 */
 void sqlite3_expert_destroy(sqlite3expert *p){
-  sqlite3_close(p->dbm);
-  idxScanFree(p->pScan, 0);
-  idxStatementFree(p->pStatement, 0);
-  idxHashClear(&p->hIdx);
-  sqlite3_free(p);
+  if( p ){
+    sqlite3_close(p->dbm);
+    sqlite3_close(p->dbv);
+    idxScanFree(p->pScan, 0);
+    idxStatementFree(p->pStatement, 0);
+    idxHashClear(&p->hIdx);
+    sqlite3_free(p);
+  }
 }
 
 #endif /* !defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHEREINFO_HOOK) */
diff --git a/manifest b/manifest
index 44bc9ecdb4..89f2ca08db 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\sthis\sbranch\swith\slatest\schanges\sfrom\strunk.
-D 2017-04-13T16:19:40.114
+C Modify\sthe\scode\sin\sext/expert/\sto\suse\sthe\svtab\sinterface\sinstead\sof\nsqlite3_whereinfo_hook().\sRemove\ssqlite3_whereinfo_hook().
+D 2017-04-14T19:41:37.126
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a
@@ -42,8 +42,8 @@ F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md 9f15075ec5ad772808eff55ef044c31140fd1146aa0a3c47eafd155e71851b01
 F ext/expert/expert.c 6349cf8d26c847f5f0fa7e25772b614c67f60f3c850dca0d75d55eb27cf3f69b
-F ext/expert/expert1.test cc33f9390f205bfeb6a30c7618b24a5675f4b9cb844c9154c11398a7f1477e81
-F ext/expert/sqlite3expert.c 2b22a5fbd093a7020ea9625928292265f31b8e78f6feabb987e71a79b2a29089
+F ext/expert/expert1.test cd630eda18a2508eade4c39a1eafe32e7437a33973391e5dddfc7fd0f3163684
+F ext/expert/sqlite3expert.c 9473b011d7e0be5b52157f3b1fc153d7e5f7d2b43af110180843e7a03972439f
 F ext/expert/sqlite3expert.h b1c9eedeb647fd734c4206ae6851635284cfbfa5fb688eff74c3265c9f949b4d
 F ext/expert/test_expert.c bad0611732d07180d586bd589cbb7713dc3ab0338c52bff29680eb2007678c05
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
@@ -372,7 +372,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c d4bb3a135948553d18cf992f76f7ed7b18aa0327f250607b5a6671e55d9947d5
 F src/legacy.c e88ed13c2d531decde75d42c2e35623fb9ce3cb0
 F src/loadext.c a72909474dadce771d3669bf84bf689424f6f87d471fee898589c3ef9b2acfd9
-F src/main.c 4b93bda0f1f916f4cb618a6fce358f7e01ab060971c586c4cea90ad0c83a83f5
+F src/main.c 158326243c5ddc8b98a1e983fa488650cf76d760
 F src/malloc.c e20bb2b48abec52d3faf01cce12e8b4f95973755fafec98d45162dfdab111978
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -409,10 +409,10 @@ F src/resolve.c 3e518b962d932a997fae373366880fc028c75706
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c e6f9afd8a5ef35bd186e51a6bea6d3d46bc93a530f26a21fe8a0a43dbeca9415
 F src/shell.c 70f4957b988572315e97c56941fdc81fd35907fee36b7b2e7be5ec4c7e9d065d
-F src/sqlite.h.in cf20591fa0eb09e435db647ab28b61159262cbebac69ddad3c8c01accfb6c856
+F src/sqlite.h.in fa0a24af458b49de8d5654fd1f3b61422ef513cee9b17f39ce3f5e5731ce4f6a
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 58fd0676d3111d02e62e5a35992a7d3da5d3f88753acc174f2d37b774fbbdd28
-F src/sqliteInt.h 5bcafb7c36f7f8765ed1e7031b7eb5f5e84cfdfe5ea4b3af01178528bde259c8
+F src/sqliteInt.h 8ac5d9b92084de271b913387968720f4dfe17302eb48a51c6ae161b6149057ad
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -488,7 +488,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 40c543f0a2195d1b0dc88ef12142bea690009344
 F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71
 F src/walker.c b71a992b413b3a022572eccf29ef4b4890223791
-F src/where.c f5acfb6fbac65e7da7b0e718fa6c6e784dee37eb29dad6efd42880ca117e7277
+F src/where.c 64f2c18c72e06bc935c64a53fd5cea6446ffedeba87c45905408bea124c201b6
 F src/whereInt.h 7a21ef633e26acbf46df04add2eba6e0a2100c78dc5879049e93f981fc3344df
 F src/wherecode.c 943e32e9dccd0af802e0683ae11071c8bd808364e5908a5fb66758bd404c8681
 F src/whereexpr.c e913aaa7b73ffcce66abcea5f197e2c538d48b5df78d0b7bba8ff4d73cc2e745
@@ -1578,7 +1578,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0f66a093935100efd731e14aa63b57360ddd517c1ac97edd1ea9a9de95e1f3cc 59c70108fd179932ccdd860f93e1cd68b77476d3b1af1af00cf6e378c9984862
-R 67cf9bfa549dda32e4c23afbb89f9c17
+P 5fcd840cf9b6a5c3ee4ef1e8f92f6c30f96a7899a3d774ee9be8a816916f2c3b
+R 754b00e271f01cdc7a8ee810736e1b12
 U dan
-Z 6319da725d84003b27573110707fc385
+Z bc14af8147812d1c0312a546deaaf0cc
diff --git a/manifest.uuid b/manifest.uuid
index 123a103b07..7b25e887aa 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5fcd840cf9b6a5c3ee4ef1e8f92f6c30f96a7899a3d774ee9be8a816916f2c3b
\ No newline at end of file
+3bb6585004090dbf92dd5e9abdf0fd2c921e64b5b3121c4fb7446db764ab59e5
\ No newline at end of file
diff --git a/src/main.c b/src/main.c
index df715852ae..4ac5327e4f 100644
--- a/src/main.c
+++ b/src/main.c
@@ -1997,26 +1997,6 @@ void *sqlite3_preupdate_hook(
 }
 #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */
 
-#ifdef SQLITE_ENABLE_WHEREINFO_HOOK
-/*
-** Register a where-info hook.
-*/
-void *sqlite3_whereinfo_hook(
-  sqlite3 *db,                    /* Register callback with this db handle */
-  void (*xWhereInfo)(void*, int, const char*, int, sqlite3_uint64),
-  void *pCtx                      /* User pointer passed to callback */
-){
-  void *pRet;
-  sqlite3_mutex_enter(db->mutex);
-  pRet = db->pWhereInfoCtx;
-  db->xWhereInfo = xWhereInfo;
-  db->pWhereInfoCtx = pCtx;
-  sqlite3_mutex_leave(db->mutex);
-  return pRet;
-}
-#endif /* SQLITE_ENABLE_WHEREINFO_HOOK */
-
-
 #ifndef SQLITE_OMIT_WAL
 /*
 ** The sqlite3_wal_hook() callback registered by sqlite3_wal_autocheckpoint().
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 923e7974e5..3c54efd36a 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -2017,6 +2017,7 @@ struct sqlite3_mem_methods {
 #define SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION 1005 /* int int* */
 #define SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE      1006 /* int int* */
 
+
 /*
 ** CAPI3REF: Enable Or Disable Extended Result Codes
 ** METHOD: sqlite3
@@ -8036,6 +8037,8 @@ int sqlite3_vtab_config(sqlite3*, int op, ...);
 */
 int sqlite3_vtab_on_conflict(sqlite3 *);
 
+SQLITE_EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3*, int);
+
 /*
 ** CAPI3REF: Conflict resolution modes
 ** KEYWORDS: {conflict resolution mode}
@@ -8482,147 +8485,6 @@ SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp(
 */
 SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb);
 
-/*
-** This function is used to register a whereinfo hook with the database
-** handle passed as the first argument. Once registered, the whereinfo hook
-** is invoked zero or more times while preparing a query to provide
-** information to the application. It is intended to be used by expert 
-** systems to recommend indexes that could be added to the database in order
-** to improve query response time.
-**
-** An SQLite query plan describes the way data is read from zero or more
-** database tables. For each table read, the data required may be
-** constrained by equality or range constraints on one or more columns, 
-** and it may or may not be required in order sorted by one or more 
-** columns. For example, the following query:
-**
-** <pre>
-**     SELECT * FROM t1 WHERE t1.a = ? ORDER BY t1.b;
-** </pre>
-**
-** reads data from table t1. It requires only those rows for which t1.a
-** is set to a specific value, and requires them sorted in order of
-** column t1.b. Internally, SQLite determines this and attempts to locate
-** an index that can be used to efficiently find the required subset of
-** rows and/or allows the rows to be read from the database in the 
-** required order. In this case, ideally an index of the form:
-**
-** <pre>
-**     CREATE INDEX i1 ON t1(a, b);
-** </pre>
-**
-** The data passed to the whereinfo hook during query preparation do
-** not describe the actual query plan to the application. Instead, it
-** describes the parts of the query that SQLite could use an index to
-** optimize if a suitable index existed. In this case, that only
-** rows with t1.a=? are required, and that they are required sorted
-** in order by t1.b.
-**
-** Each time the whereinfo hook is invoked, the first argument is a
-** copy of the (void*) pointer passed as the second argument to this
-** API function. The second is always one of the SQLITE_WHEREINFO_XXX
-** constants defined below.
-**
-** For each table read by a query, the whereinfo hook is invoked as follows:
-**
-** <ul>
-**   <li> Once with SQLITE_WHEREINFO_TABLE as the second argument. This
-**        indicates the table that subsequent callbacks (until the next
-**        SQLITE_WHEREINFO_TABLE) apply to.
-**
-**   <li> If SQLite requires rows in a specific order, once with
-**        SQLITE_WHEREINFO_ORDERBY for each column of the table that is
-**        one of the sort keys.
-**
-**   <li> If there are any "<col> = ?" constraints that restrict the rows
-**        required by SQLite, once with SQLITE_WHEREINFO_EQUALS for each
-**        such constraint.
-**
-**   <li> If there are any "<col> > ?" constraints (or any other range
-**        constraint) that restrict the rows required by SQLite, once with 
-**        SQLITE_WHEREINFO_RANGE for each such constraint.
-** </ul>
-**
-** The third, fourth and fifth arguments passed to the whereinfo callback
-** are interpreted differently, depending on the SQLITE_WHEREINFO_XXX value
-** as follows:
-**
-** <dl>
-**   <dt> SQLITE_WHEREINFO_TABLE
-**   <dd> The third argument passed in this case is the name of the table.
-**        The fourth is the index of the database in which the table is
-**        located (0 for "main", 1 for "temp" or higher for an attached
-**        database). The fifth argument is a bitmask that indicates which
-**        of the tables columns may be required by the query. If the leftmost
-**        column of the table is used in some way, bit 0 of the bitmask is
-**        set. If the next-to-leftmost is used, bit 1 etc. Bit 63 is used to
-**        represent all columns with an index of 63 or higher. If bit 63
-**        is set, the application should assume that the query requires all
-**        columns from the table with an index of 63 or greater.
-**
-**   <dt> SQLITE_WHEREINFO_ORDERBY
-**   <dd> The third argument passed in this case is the name of the collation
-**        sequence to sort by. The fourth is the index of the table column to
-**        sort by (0 for the leftmost column, 1 for the next-to-leftmost
-**        etc.). The fifth argument is a boolean flag - true for a DESC sort
-**        or false for ASC.
-**
-**   <dt> SQLITE_WHEREINFO_EQUALS
-**   <dd> The third argument passed in this case is the name of the collation
-**        sequence used by the constraint. The fourth is the index of the 
-**        table column in the constraint. If the current table is not part
-**        of a join, then the value passed as the fifth argument is always
-**        zero. Or, if it is part of a join, then the fifth parameter passed
-**        to this callback is a mask of other tables that the current
-**        constraint depends on. For example, in the query:
-**
-**   <pre>
-**      SELECT * FROM t1, t2 WHERE t1.a = (t2.b+1);
-**   </pre>
-**
-**        the fifth parameter passed to the the SQLITE_WHEREINFO_EQUALS
-**        whereinfo callback would have the bit assigned to table "t2"
-**        set to true. There is no way for the application to determine 
-**        the specific bit in the mask assigned to any table, but the bit
-**        assignments are consistent while parsing a single query.
-**
-**   <dt> SQLITE_WHEREINFO_RANGE
-**   <dd> As for SQLITE_WHEREINFO_EQUALS.
-** </dl>
-**
-** Note that if a WHERE clause includes an OR expression, then there may be
-** more than one set of callbacks for a single table. For example, the
-** following SQL:
-**
-** <pre>
-**    SELECT * FROM t1 WHERE t1.a=? OR t1.b=?
-** </pre>
-**
-** Provokes the same callbacks as the following two queries executed in
-** series.
-**
-** <pre>
-**    SELECT * FROM t1 WHERE t1.a=?;
-**    SELECT * FROM t1 WHERE t1.b=?;
-** </pre>
-*/
-SQLITE_EXPERIMENTAL void *sqlite3_whereinfo_hook(
-  sqlite3 *db,                    /* Register callback with this db handle */
-  void (*xWhereInfo)(
-    void*,                        /* Copy of pCtx */
-    int,                          /* SQLITE_WHEREINFO_XXX constant */
-    const char*,
-    int,
-    sqlite3_uint64
-  ),
-  void *pCtx                      /* User pointer passed to callback */
-);
-
-#define SQLITE_WHEREINFO_TABLE   1
-#define SQLITE_WHEREINFO_EQUALS  2
-#define SQLITE_WHEREINFO_RANGE   3
-#define SQLITE_WHEREINFO_ORDERBY 4
-
 /*
 ** Undo the hack that converts floating point types to integer for
 ** builds on processors without floating point support.
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 0c5d4b74d7..22892c967f 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1399,6 +1399,7 @@ struct sqlite3 {
   VtabCtx *pVtabCtx;            /* Context for active vtab connect/create */
   VTable **aVTrans;             /* Virtual tables with open transactions */
   VTable *pDisconnect;    /* Disconnect these in next sqlite3_prepare() */
+  void *pVtabWC;                /* For sqlite3_vtab_collation() */
 #endif
   Hash aFunc;                   /* Hash table of connection functions */
   Hash aCollSeq;                /* All collating sequences */
@@ -1431,10 +1432,6 @@ struct sqlite3 {
 #ifdef SQLITE_USER_AUTHENTICATION
   sqlite3_userauth auth;        /* User authentication information */
 #endif
-#ifdef SQLITE_ENABLE_WHEREINFO_HOOK
-  void (*xWhereInfo)(void*, int, const char*, int, u64);
-  void *pWhereInfoCtx;
-#endif
 };
 
 /*
diff --git a/src/where.c b/src/where.c
index 3cbd959771..df7582832a 100644
--- a/src/where.c
+++ b/src/where.c
@@ -885,7 +885,8 @@ static sqlite3_index_info *allocateIndexInfo(
   */
   pIdxInfo = sqlite3DbMallocZero(pParse->db, sizeof(*pIdxInfo)
                            + (sizeof(*pIdxCons) + sizeof(*pUsage))*nTerm
-                           + sizeof(*pIdxOrderBy)*nOrderBy );
+                           + sizeof(*pIdxOrderBy)*nOrderBy 
+                           );
   if( pIdxInfo==0 ){
     sqlite3ErrorMsg(pParse, "out of memory");
     return 0;
@@ -3116,6 +3117,34 @@ static int whereLoopAddVirtualOne(
 }
 
 
+struct BestIndexCtx {
+  WhereClause *pWC;
+  sqlite3_index_info *pIdxInfo;
+  ExprList *pOrderBy;
+  Parse *pParse;
+};
+
+const char *sqlite3_vtab_collation(sqlite3 *db, int iCons){
+  struct BestIndexCtx *p = (struct BestIndexCtx*)db->pVtabWC;
+  const char *zRet = 0;
+  if( p && iCons>=0 ){
+    if( iCons<p->pIdxInfo->nConstraint ){
+      int iTerm = p->pIdxInfo->aConstraint[iCons].iTermOffset;
+      Expr *pX = p->pWC->a[iTerm].pExpr;
+      CollSeq *pC = sqlite3BinaryCompareCollSeq(p->pParse,pX->pLeft,pX->pRight);
+      zRet = (pC ? pC->zName : "BINARY");
+    }else{
+      iCons -= p->pIdxInfo->nConstraint;
+      if( iCons<p->pIdxInfo->nOrderBy ){
+        Expr *pX = p->pOrderBy->a[iCons].pExpr;
+        CollSeq *pC = sqlite3ExprCollSeq(p->pParse, pX);
+        zRet = (pC ? pC->zName : "BINARY");
+      }
+    }
+  }
+  return zRet;
+}
+
 /*
 ** Add all WhereLoop objects for a table of the join identified by
 ** pBuilder->pNew->iTab.  That table is guaranteed to be a virtual table.
@@ -3157,6 +3186,8 @@ static int whereLoopAddVirtual(
   WhereLoop *pNew;
   Bitmask mBest;               /* Tables used by best possible plan */
   u16 mNoOmit;
+  struct BestIndexCtx bic;
+  void *pSaved;
 
   assert( (mPrereq & mUnusable)==0 );
   pWInfo = pBuilder->pWInfo;
@@ -3178,6 +3209,13 @@ static int whereLoopAddVirtual(
     return SQLITE_NOMEM_BKPT;
   }
 
+  bic.pWC = pWC;
+  bic.pIdxInfo = p;
+  bic.pParse = pParse;
+  bic.pOrderBy = pBuilder->pOrderBy;
+  pSaved = pParse->db->pVtabWC;
+  pParse->db->pVtabWC = (void*)&bic;
+
   /* First call xBestIndex() with all constraints usable. */
   WHERETRACE(0x40, ("  VirtualOne: all usable\n"));
   rc = whereLoopAddVirtualOne(pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn);
@@ -3254,6 +3292,7 @@ static int whereLoopAddVirtual(
 
   if( p->needToFreeIdxStr ) sqlite3_free(p->idxStr);
   sqlite3DbFreeNN(pParse->db, p);
+  pParse->db->pVtabWC = pSaved;
   return rc;
 }
 #endif /* SQLITE_OMIT_VIRTUALTABLE */
@@ -3277,7 +3316,7 @@ static int whereLoopAddOr(
   WhereLoopBuilder sSubBuild;
   WhereOrSet sSum, sCur;
   struct SrcList_item *pItem;
-
+  
   pWC = pBuilder->pWC;
   pWCEnd = pWC->a + pWC->nTerm;
   pNew = pBuilder->pNew;
@@ -3294,7 +3333,7 @@ static int whereLoopAddOr(
       WhereTerm *pOrTerm;
       int once = 1;
       int i, j;
-
+    
       sSubBuild = *pBuilder;
       sSubBuild.pOrderBy = 0;
       sSubBuild.pOrSet = &sCur;
@@ -4278,201 +4317,6 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){
   return 0;
 }
 
-#ifdef SQLITE_ENABLE_WHEREINFO_HOOK
-
-static void whereTraceWC(
-  Parse *pParse, 
-  struct SrcList_item *pItem,
-  WhereClause *pWC
-){
-  sqlite3 *db = pParse->db;
-  Table *pTab = pItem->pTab;
-  void (*x)(void*, int, const char*, int, u64) = db->xWhereInfo;
-  void *pCtx = db->pWhereInfoCtx;
-  int ii;
-
-  /* Issue callbacks for WO_SINGLE constraints */
-  for(ii=0; ii<pTab->nCol; ii++){
-    int opMask = WO_SINGLE; 
-    WhereScan scan;
-    WhereTerm *pTerm;
-    for(pTerm=whereScanInit(&scan, pWC, pItem->iCursor, ii, opMask, 0);
-        pTerm;
-        pTerm=whereScanNext(&scan)
-    ){
-      int eOp;
-      Expr *pX = pTerm->pExpr;
-      CollSeq *pC = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight);
-      if( pTerm->eOperator & (WO_IS|WO_EQ|WO_IN) ){
-        eOp = SQLITE_WHEREINFO_EQUALS;
-      }else{
-        eOp = SQLITE_WHEREINFO_RANGE;
-      }
-      x(pCtx, eOp, (pC ? pC->zName : "BINARY"), ii, pTerm->prereqRight);
-    }
-  }
-}
-
-/*
-** If there are any OR terms in WHERE clause pWC, make the associated
-** where-info hook callbacks.
-*/
-static void whereTraceOR(
-  Parse *pParse, 
-  struct SrcList_item *pItem,
-  WhereClause *pWC
-){
-  sqlite3 *db = pParse->db;
-  WhereClause tempWC;
-  struct TermAndIdx {
-    WhereTerm *pTerm;
-    int iIdx;
-  } aOr[4];
-  int nOr = 0;
-  Table *pTab = pItem->pTab;
-  int iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
-  int ii;
-
-  memset(aOr, 0, sizeof(aOr));
-
-  /* Iterate through OR nodes */
-  for(ii=0; ii<pWC->nTerm; ii++){
-    WhereTerm *pTerm = &pWC->a[ii];
-    if( pTerm->eOperator & WO_OR ){
-      /* Check that each branch of this OR term contains at least
-      ** one reference to the table currently being processed. If that
-      ** is not the case, this term can be ignored.  */
-      WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc;
-      WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm];
-      WhereTerm *pOrTerm;
-      WhereClause *pTermWC;
-      WhereScan scan;
-
-      for(pOrTerm=pOrWC->a; pOrTerm<pOrWCEnd; pOrTerm++){
-        int iCol;
-        if( (pOrTerm->eOperator & WO_AND)!=0 ){
-          pTermWC = &pOrTerm->u.pAndInfo->wc;
-        }else{
-          tempWC.pWInfo = pWC->pWInfo;
-          tempWC.pOuter = pWC;
-          tempWC.op = TK_AND;
-          tempWC.nTerm = 1;
-          tempWC.a = pOrTerm;
-          pTermWC = &tempWC;
-        }
-
-        for(iCol=0; iCol<pTab->nCol; iCol++){
-          int iCsr = pItem->iCursor;
-          if( !whereScanInit(&scan, pTermWC, iCsr, iCol, WO_SINGLE, 0) ){
-            break;
-          }
-        }
-        if( iCol==pTab->nCol ) break;
-      }
-
-      if( pOrTerm==pOrWCEnd ){
-        aOr[nOr].pTerm = pTerm;
-        aOr[nOr].iIdx = pOrWC->nTerm;
-        nOr++;
-        if( nOr==ArraySize(aOr) ) break;
-      }
-    }
-  }
-
-  while( 1 ){
-    for(ii=0; ii<nOr; ii++){
-      if( aOr[ii].iIdx==0 ){
-        aOr[ii].iIdx = aOr[ii].pTerm->u.pOrInfo->wc.nTerm;
-      }else{
-        aOr[ii].iIdx--;
-        break;
-      }
-    }
-    if( ii==nOr ) break;
-
-    /* Table name callback */
-    db->xWhereInfo(db->pWhereInfoCtx, 
-        SQLITE_WHEREINFO_TABLE, pTab->zName, iDb, pItem->colUsed
-    );
-    /* whereTraceWC(pParse, pItem, pWC); */
-    for(ii=0; ii<nOr; ii++){
-      WhereClause * const pOrWC = &aOr[ii].pTerm->u.pOrInfo->wc;
-      if( aOr[ii].iIdx<pOrWC->nTerm ){
-        WhereClause *pTermWC;
-        WhereTerm *pOrTerm = &pOrWC->a[aOr[ii].iIdx];
-        if( (pOrTerm->eOperator & WO_AND)!=0 ){
-          pTermWC = &pOrTerm->u.pAndInfo->wc;
-        }else{
-          tempWC.pWInfo = pWC->pWInfo;
-          tempWC.pOuter = pWC;
-          tempWC.op = TK_AND;
-          tempWC.nTerm = 1;
-          tempWC.a = pOrTerm;
-          pTermWC = &tempWC;
-        }
-        whereTraceWC(pParse, pItem, pTermWC);
-      }
-    }
-  }
-}
-
-/*
-** If there is a where-info hook attached to the database handle, issue all
-** required callbacks for the current sqlite3WhereBegin() call.
-*/
-static void whereTraceBuilder(
-  Parse *pParse,
-  WhereLoopBuilder *p
-){
-  sqlite3 *db = pParse->db;
-  if( db->xWhereInfo && db->init.busy==0 ){
-    void (*x)(void*, int, const char*, int, u64) = db->xWhereInfo;
-    void *pCtx = db->pWhereInfoCtx;
-    int ii;
-    SrcList *pTabList = p->pWInfo->pTabList;
-
-    /* Loop through each element of the FROM clause. Ignore any sub-selects
-    ** or views. Invoke the xWhereInfo() callback multiple times for each
-    ** real table.  */
-    for(ii=0; ii<pTabList->nSrc; ii++){
-      struct SrcList_item *pItem = &pTabList->a[ii];
-      if( pItem->pSelect==0 ){
-        Table *pTab = pItem->pTab;
-        int iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
-
-        /* Table name callback */
-        x(pCtx, SQLITE_WHEREINFO_TABLE, pTab->zName, iDb, pItem->colUsed);
-
-        /* ORDER BY callbacks */
-        if( p->pOrderBy ){
-          int i;
-          for(i=0; i<p->pOrderBy->nExpr; i++){
-            Expr *pExpr = p->pOrderBy->a[i].pExpr; 
-            CollSeq *pColl = sqlite3ExprCollSeq(pParse, pExpr);
-            pExpr = sqlite3ExprSkipCollate(pExpr);
-            if( pExpr->op==TK_COLUMN && pExpr->iTable==pItem->iCursor ){
-              int iCol = pExpr->iColumn;
-              if( pColl && iCol>=0 ){
-                int bDesc = p->pOrderBy->a[i].sortOrder;
-                x(pCtx, SQLITE_WHEREINFO_ORDERBY, pColl->zName, iCol, bDesc); 
-              }
-            }
-          }
-        }
-
-        /* WHERE callbacks */
-        whereTraceWC(pParse, pItem, p->pWC);
-
-        /* OR-clause processing */
-        whereTraceOR(pParse, pItem, p->pWC);
-      }
-    }
-  }
-}
-#else
-# define whereTraceBuilder(x,y)
-#endif
-
 /*
 ** Generate the beginning of the loop used for WHERE clause processing.
 ** The return value is a pointer to an opaque structure that contains
@@ -4745,9 +4589,6 @@ WhereInfo *sqlite3WhereBegin(
   }
 #endif
 
-  /* Invoke the where-info hook, if one has been registered. */
-  whereTraceBuilder(pParse, &sWLB);
-
   if( nTabList!=1 || whereShortCut(&sWLB)==0 ){
     rc = whereLoopAddAll(&sWLB);
     if( rc ) goto whereBeginError;

From 79610f5d090160c0df4d80f3badd9b50f8e959e2 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 15 Apr 2017 14:16:04 +0000
Subject: [PATCH 030/488] Fix memory leaks in the code on this branch. Make use
 of the sqlite3_index_constraint.usage field. Do not try to handle ORDER BY
 terms with explicit COLLATE clauses - they don't get passed to the vtab layer
 anyway.

FossilOrigin-Name: 0cd75a872c89958a7f418720a0e8c6f638f8284c488f666015c19136feae6be8
---
 ext/expert/sqlite3expert.c | 147 +++++++++----------------------------
 manifest                   |  14 ++--
 manifest.uuid              |   2 +-
 src/where.c                |  21 ++----
 4 files changed, 48 insertions(+), 136 deletions(-)

diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index 3cd110c367..153159fbbf 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -39,7 +39,6 @@ struct IdxConstraint {
   char *zColl;                    /* Collation sequence */
   int bRange;                     /* True for range, false for eq */
   int iCol;                       /* Constrained table column */
-  i64 depmask;                    /* Dependency mask */
   int bFlag;                      /* Used by idxFindCompatible() */
   int bDesc;                      /* True if ORDER BY <expr> DESC */
   IdxConstraint *pNext;           /* Next constraint in pEq or pRange list */
@@ -111,25 +110,6 @@ struct IdxHash {
   IdxHashEntry *aHash[IDX_HASH_SIZE];
 };
 
-/*
-** A hash table for storing a set of 64-bit values. Methods are:
-**
-**   idxHash64Init()
-**   idxHash64Clear()
-**   idxHash64Add()
-*/
-typedef struct IdxHash64Entry IdxHash64Entry;
-typedef struct IdxHash64 IdxHash64;
-struct IdxHash64Entry {
-  u64 iVal;
-  IdxHash64Entry *pNext;          /* Next entry in hash table */
-  IdxHash64Entry *pHashNext;      /* Next entry in same hash bucket */
-};
-struct IdxHash64 {
-  IdxHash64Entry *pFirst;         /* Most recently added entry in hash table */
-  IdxHash64Entry *aHash[IDX_HASH_SIZE];
-};
-
 /*
 ** sqlite3expert object.
 */
@@ -166,48 +146,6 @@ static void *idxMalloc(int *pRc, int nByte){
   return pRet;
 }
 
-/*
-** Initialize an IdxHash64 hash table.
-*/
-static void idxHash64Init(IdxHash64 *pHash){
-  memset(pHash, 0, sizeof(IdxHash64));
-}
-
-/*
-** Reset an IdxHash64 hash table.
-*/
-static void idxHash64Clear(IdxHash64 *pHash){
-  IdxHash64Entry *pEntry;
-  IdxHash64Entry *pNext;
-  for(pEntry=pHash->pFirst; pEntry; pEntry=pNext){
-    pNext = pEntry->pNext;
-    sqlite3_free(pEntry);
-  }
-  memset(pHash, 0, sizeof(IdxHash64));
-}
-
-/*
-** Add iVal to the IdxHash64 hash table passed as the second argument. This
-** function is a no-op if iVal is already present in the hash table.
-*/
-static void idxHash64Add(int *pRc, IdxHash64 *pHash, u64 iVal){
-  int iHash = (int)((iVal*7) % IDX_HASH_SIZE);
-  IdxHash64Entry *pEntry;
-  assert( iHash>=0 );
-
-  for(pEntry=pHash->aHash[iHash]; pEntry; pEntry=pEntry->pHashNext){
-    if( pEntry->iVal==iVal ) return;
-  }
-  pEntry = idxMalloc(pRc, sizeof(IdxHash64Entry));
-  if( pEntry ){
-    pEntry->iVal = iVal;
-    pEntry->pHashNext = pHash->aHash[iHash];
-    pHash->aHash[iHash] = pEntry;
-    pEntry->pNext = pHash->pFirst;
-    pHash->pFirst = pEntry;
-  }
-}
-
 /*
 ** Initialize an IdxHash hash table.
 */
@@ -426,14 +364,14 @@ static int expertBestIndex(sqlite3_vtab *pVtab, sqlite3_index_info *pIdxInfo){
 
     /* Add the constraints to the IdxScan object */
     for(i=0; i<pIdxInfo->nConstraint; i++){
-      int op = pIdxInfo->aConstraint[i].op;
-      if( op&opmask ){
+      struct sqlite3_index_constraint *pCons = &pIdxInfo->aConstraint[i];
+      if( pCons->usable && (pCons->op & opmask) ){
         IdxConstraint *pNew;
         const char *zColl = sqlite3_vtab_collation(dbv, i);
         pNew = idxNewConstraint(&rc, zColl);
         if( pNew ){
-          pNew->iCol = pIdxInfo->aConstraint[i].iColumn;
-          if( op==SQLITE_INDEX_CONSTRAINT_EQ ){
+          pNew->iCol = pCons->iColumn;
+          if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ ){
             pNew->pNext = pScan->pEq;
             pScan->pEq = pNew;
           }else{
@@ -442,20 +380,17 @@ static int expertBestIndex(sqlite3_vtab *pVtab, sqlite3_index_info *pIdxInfo){
             pScan->pRange = pNew;
           }
         }
-        if( pIdxInfo->aConstraint[i].usable ){
-          n++;
-          pIdxInfo->aConstraintUsage[i].argvIndex = n;
-        }
+        n++;
+        pIdxInfo->aConstraintUsage[i].argvIndex = n;
       }
     }
 
     /* Add the ORDER BY to the IdxScan object */
     for(i=pIdxInfo->nOrderBy-1; i>=0; i--){
-      IdxConstraint *pNew;
-      const char *zColl = sqlite3_vtab_collation(dbv, i+pIdxInfo->nConstraint);
-      pNew = idxNewConstraint(&rc, zColl);
+      int iCol = pIdxInfo->aOrderBy[i].iColumn;
+      IdxConstraint *pNew = idxNewConstraint(&rc, p->pTab->aCol[iCol].zColl);
       if( pNew ){
-        pNew->iCol = pIdxInfo->aOrderBy[i].iColumn;
+        pNew->iCol = iCol;
         pNew->bDesc = pIdxInfo->aOrderBy[i].desc;
         pNew->pNext = pScan->pOrder;
         pNew->pLink = pScan->pOrder;
@@ -813,7 +748,6 @@ static int idxFindCompatible(
 
 static int idxCreateFromCons(
   sqlite3expert *p,
-  IdxTable *pTab,
   IdxScan *pScan,
   IdxConstraint *pEq, 
   IdxConstraint *pTail
@@ -821,6 +755,7 @@ static int idxCreateFromCons(
   sqlite3 *dbm = p->dbm;
   int rc = SQLITE_OK;
   if( (pEq || pTail) && 0==idxFindCompatible(&rc, dbm, pScan, pEq, pTail) ){
+    IdxTable *pTab = pScan->pTab;
     char *zCols = 0;
     char *zIdx = 0;
     IdxConstraint *pCons;
@@ -882,22 +817,16 @@ static int idxFindConstraint(IdxConstraint *pList, IdxConstraint *p){
 
 static int idxCreateFromWhere(
   sqlite3expert *p, 
-  IdxTable *pTab,
-  i64 mask,                       /* Consider only these constraints */
   IdxScan *pScan,                 /* Create indexes for this scan */
-  IdxConstraint *pEq,             /* == constraints for inclusion */
   IdxConstraint *pTail            /* range/ORDER BY constraints for inclusion */
 ){
-  IdxConstraint *p1 = pEq;
+  IdxConstraint *p1 = 0;
   IdxConstraint *pCon;
   int rc;
 
-  /* Gather up all the == constraints that match the mask. */
+  /* Gather up all the == constraints. */
   for(pCon=pScan->pEq; pCon; pCon=pCon->pNext){
-    if( (mask & pCon->depmask)==pCon->depmask 
-     && idxFindConstraint(p1, pCon)==0
-     && idxFindConstraint(pTail, pCon)==0
-    ){
+    if( !idxFindConstraint(p1, pCon) && !idxFindConstraint(pTail, pCon) ){
       pCon->pLink = p1;
       p1 = pCon;
     }
@@ -905,18 +834,15 @@ static int idxCreateFromWhere(
 
   /* Create an index using the == constraints collected above. And the
   ** range constraint/ORDER BY terms passed in by the caller, if any. */
-  rc = idxCreateFromCons(p, pTab, pScan, p1, pTail);
+  rc = idxCreateFromCons(p, pScan, p1, pTail);
 
   /* If no range/ORDER BY passed by the caller, create a version of the
-  ** index for each range constraint that matches the mask. */
+  ** index for each range constraint.  */
   if( pTail==0 ){
     for(pCon=pScan->pRange; rc==SQLITE_OK && pCon; pCon=pCon->pNext){
       assert( pCon->pLink==0 );
-      if( (mask & pCon->depmask)==pCon->depmask
-        && idxFindConstraint(pEq, pCon)==0
-        && idxFindConstraint(pTail, pCon)==0
-      ){
-        rc = idxCreateFromCons(p, pTab, pScan, p1, pCon);
+      if( !idxFindConstraint(p1, pCon) && !idxFindConstraint(pTail, pCon) ){
+        rc = idxCreateFromCons(p, pScan, p1, pCon);
       }
     }
   }
@@ -931,30 +857,12 @@ static int idxCreateFromWhere(
 static int idxCreateCandidates(sqlite3expert *p, char **pzErr){
   int rc = SQLITE_OK;
   IdxScan *pIter;
-  IdxHash64 hMask;
-  idxHash64Init(&hMask);
 
   for(pIter=p->pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
-    IdxHash64Entry *pEntry;
-    IdxConstraint *pCons;
-    IdxTable *pTab = pIter->pTab;
-
-    idxHash64Add(&rc, &hMask, 0);
-    for(pCons=pIter->pEq; pCons; pCons=pCons->pNext){
-      for(pEntry=hMask.pFirst; pEntry; pEntry=pEntry->pNext){
-        idxHash64Add(&rc, &hMask, pEntry->iVal | (u64)pCons->depmask);
-      }
+    rc = idxCreateFromWhere(p, pIter, 0);
+    if( rc==SQLITE_OK && pIter->pOrder ){
+      rc = idxCreateFromWhere(p, pIter, pIter->pOrder);
     }
-
-    for(pEntry=hMask.pFirst; rc==SQLITE_OK && pEntry; pEntry=pEntry->pNext){
-      i64 mask = (i64)pEntry->iVal;
-      rc = idxCreateFromWhere(p, pTab, mask, pIter, 0, 0);
-      if( rc==SQLITE_OK && pIter->pOrder ){
-        rc = idxCreateFromWhere(p, pTab, mask, pIter, 0, pIter->pOrder);
-      }
-    }
-
-    idxHash64Clear(&hMask);
   }
 
   return rc;
@@ -1004,6 +912,18 @@ static void idxStatementFree(IdxStatement *pStatement, IdxStatement *pLast){
   }
 }
 
+/*
+** Free the linked list of IdxTable objects starting at pTab.
+*/
+static void idxTableFree(IdxTable *pTab){
+  IdxTable *pIter;
+  IdxTable *pNext;
+  for(pIter=pTab; pIter; pIter=pNext){
+    pNext = pIter->pNext;
+    sqlite3_free(pIter);
+  }
+}
+
 
 /*
 ** This function is called after candidate indexes have been created. It
@@ -1126,6 +1046,7 @@ static int idxCreateVtabSchema(sqlite3expert *p, char **pzErrmsg){
       }
     }
   }
+  idxFinalize(&rc, pSchema);
   return rc;
 }
 
@@ -1296,7 +1217,9 @@ void sqlite3_expert_destroy(sqlite3expert *p){
     sqlite3_close(p->dbv);
     idxScanFree(p->pScan, 0);
     idxStatementFree(p->pStatement, 0);
+    idxTableFree(p->pTable);
     idxHashClear(&p->hIdx);
+    sqlite3_free(p->zCandidates);
     sqlite3_free(p);
   }
 }
diff --git a/manifest b/manifest
index 89f2ca08db..6db1d15746 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Modify\sthe\scode\sin\sext/expert/\sto\suse\sthe\svtab\sinterface\sinstead\sof\nsqlite3_whereinfo_hook().\sRemove\ssqlite3_whereinfo_hook().
-D 2017-04-14T19:41:37.126
+C Fix\smemory\sleaks\sin\sthe\scode\son\sthis\sbranch.\sMake\suse\sof\sthe\nsqlite3_index_constraint.usage\sfield.\sDo\snot\stry\sto\shandle\sORDER\sBY\sterms\swith\nexplicit\sCOLLATE\sclauses\s-\sthey\sdon't\sget\spassed\sto\sthe\svtab\slayer\sanyway.
+D 2017-04-15T14:16:04.093
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a
@@ -43,7 +43,7 @@ F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md 9f15075ec5ad772808eff55ef044c31140fd1146aa0a3c47eafd155e71851b01
 F ext/expert/expert.c 6349cf8d26c847f5f0fa7e25772b614c67f60f3c850dca0d75d55eb27cf3f69b
 F ext/expert/expert1.test cd630eda18a2508eade4c39a1eafe32e7437a33973391e5dddfc7fd0f3163684
-F ext/expert/sqlite3expert.c 9473b011d7e0be5b52157f3b1fc153d7e5f7d2b43af110180843e7a03972439f
+F ext/expert/sqlite3expert.c d4a0a45be58874b4cf54316e5d776726489967140399b1e77a17dbb66558cb38
 F ext/expert/sqlite3expert.h b1c9eedeb647fd734c4206ae6851635284cfbfa5fb688eff74c3265c9f949b4d
 F ext/expert/test_expert.c bad0611732d07180d586bd589cbb7713dc3ab0338c52bff29680eb2007678c05
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
@@ -488,7 +488,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 40c543f0a2195d1b0dc88ef12142bea690009344
 F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71
 F src/walker.c b71a992b413b3a022572eccf29ef4b4890223791
-F src/where.c 64f2c18c72e06bc935c64a53fd5cea6446ffedeba87c45905408bea124c201b6
+F src/where.c 7b4d39ffdb82a6c5abd8678c66a4cd16d8bdba3bbe158a9e2f181a78e05b07ef
 F src/whereInt.h 7a21ef633e26acbf46df04add2eba6e0a2100c78dc5879049e93f981fc3344df
 F src/wherecode.c 943e32e9dccd0af802e0683ae11071c8bd808364e5908a5fb66758bd404c8681
 F src/whereexpr.c e913aaa7b73ffcce66abcea5f197e2c538d48b5df78d0b7bba8ff4d73cc2e745
@@ -1578,7 +1578,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5fcd840cf9b6a5c3ee4ef1e8f92f6c30f96a7899a3d774ee9be8a816916f2c3b
-R 754b00e271f01cdc7a8ee810736e1b12
+P 3bb6585004090dbf92dd5e9abdf0fd2c921e64b5b3121c4fb7446db764ab59e5
+R 039cfbfa3ee0c0fcda94663d31b151d9
 U dan
-Z bc14af8147812d1c0312a546deaaf0cc
+Z 80d209b7df04543ba8ed8a1a58dd2c4a
diff --git a/manifest.uuid b/manifest.uuid
index 7b25e887aa..de896ff6d4 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-3bb6585004090dbf92dd5e9abdf0fd2c921e64b5b3121c4fb7446db764ab59e5
\ No newline at end of file
+0cd75a872c89958a7f418720a0e8c6f638f8284c488f666015c19136feae6be8
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index df7582832a..d49c3d42f2 100644
--- a/src/where.c
+++ b/src/where.c
@@ -3120,27 +3120,17 @@ static int whereLoopAddVirtualOne(
 struct BestIndexCtx {
   WhereClause *pWC;
   sqlite3_index_info *pIdxInfo;
-  ExprList *pOrderBy;
   Parse *pParse;
 };
 
 const char *sqlite3_vtab_collation(sqlite3 *db, int iCons){
   struct BestIndexCtx *p = (struct BestIndexCtx*)db->pVtabWC;
   const char *zRet = 0;
-  if( p && iCons>=0 ){
-    if( iCons<p->pIdxInfo->nConstraint ){
-      int iTerm = p->pIdxInfo->aConstraint[iCons].iTermOffset;
-      Expr *pX = p->pWC->a[iTerm].pExpr;
-      CollSeq *pC = sqlite3BinaryCompareCollSeq(p->pParse,pX->pLeft,pX->pRight);
-      zRet = (pC ? pC->zName : "BINARY");
-    }else{
-      iCons -= p->pIdxInfo->nConstraint;
-      if( iCons<p->pIdxInfo->nOrderBy ){
-        Expr *pX = p->pOrderBy->a[iCons].pExpr;
-        CollSeq *pC = sqlite3ExprCollSeq(p->pParse, pX);
-        zRet = (pC ? pC->zName : "BINARY");
-      }
-    }
+  if( p && iCons>=0 && iCons<p->pIdxInfo->nConstraint ){
+    int iTerm = p->pIdxInfo->aConstraint[iCons].iTermOffset;
+    Expr *pX = p->pWC->a[iTerm].pExpr;
+    CollSeq *pC = sqlite3BinaryCompareCollSeq(p->pParse,pX->pLeft,pX->pRight);
+    zRet = (pC ? pC->zName : "BINARY");
   }
   return zRet;
 }
@@ -3212,7 +3202,6 @@ static int whereLoopAddVirtual(
   bic.pWC = pWC;
   bic.pIdxInfo = p;
   bic.pParse = pParse;
-  bic.pOrderBy = pBuilder->pOrderBy;
   pSaved = pParse->db->pVtabWC;
   pParse->db->pVtabWC = (void*)&bic;
 

From e01b9281fc5b47a6052cc3fcaa7a36c0bb1f550d Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 15 Apr 2017 14:30:01 +0000
Subject: [PATCH 031/488] Add header comment for sqlite3_vtab_collation().

FossilOrigin-Name: d238694ca445ccb4eeb3e3269a5f872f998f795945d0f9dd95c11d0e42d4d538
---
 manifest        | 16 ++++++++--------
 manifest.uuid   |  2 +-
 src/sqlite.h.in | 14 ++++++++++++++
 src/sqliteInt.h |  2 +-
 src/where.c     | 19 +++++++++++++++----
 5 files changed, 39 insertions(+), 14 deletions(-)

diff --git a/manifest b/manifest
index 6db1d15746..38a861353e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\smemory\sleaks\sin\sthe\scode\son\sthis\sbranch.\sMake\suse\sof\sthe\nsqlite3_index_constraint.usage\sfield.\sDo\snot\stry\sto\shandle\sORDER\sBY\sterms\swith\nexplicit\sCOLLATE\sclauses\s-\sthey\sdon't\sget\spassed\sto\sthe\svtab\slayer\sanyway.
-D 2017-04-15T14:16:04.093
+C Add\sheader\scomment\sfor\ssqlite3_vtab_collation().
+D 2017-04-15T14:30:01.495
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a
@@ -409,10 +409,10 @@ F src/resolve.c 3e518b962d932a997fae373366880fc028c75706
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c e6f9afd8a5ef35bd186e51a6bea6d3d46bc93a530f26a21fe8a0a43dbeca9415
 F src/shell.c 70f4957b988572315e97c56941fdc81fd35907fee36b7b2e7be5ec4c7e9d065d
-F src/sqlite.h.in fa0a24af458b49de8d5654fd1f3b61422ef513cee9b17f39ce3f5e5731ce4f6a
+F src/sqlite.h.in 18b4f1367bb80c07a6883c75c18267de2b977960adf1f30c783a731691eae3b0
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 58fd0676d3111d02e62e5a35992a7d3da5d3f88753acc174f2d37b774fbbdd28
-F src/sqliteInt.h 8ac5d9b92084de271b913387968720f4dfe17302eb48a51c6ae161b6149057ad
+F src/sqliteInt.h bd00872a45aa8eaa262301436bb63dc0afa685ceab7361790b065d5802269650
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -488,7 +488,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 40c543f0a2195d1b0dc88ef12142bea690009344
 F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71
 F src/walker.c b71a992b413b3a022572eccf29ef4b4890223791
-F src/where.c 7b4d39ffdb82a6c5abd8678c66a4cd16d8bdba3bbe158a9e2f181a78e05b07ef
+F src/where.c 4b0d7b6696f2de64b3bdd2bc600f51ef6cd832fe269f2c0ad330974885c30652
 F src/whereInt.h 7a21ef633e26acbf46df04add2eba6e0a2100c78dc5879049e93f981fc3344df
 F src/wherecode.c 943e32e9dccd0af802e0683ae11071c8bd808364e5908a5fb66758bd404c8681
 F src/whereexpr.c e913aaa7b73ffcce66abcea5f197e2c538d48b5df78d0b7bba8ff4d73cc2e745
@@ -1578,7 +1578,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3bb6585004090dbf92dd5e9abdf0fd2c921e64b5b3121c4fb7446db764ab59e5
-R 039cfbfa3ee0c0fcda94663d31b151d9
+P 0cd75a872c89958a7f418720a0e8c6f638f8284c488f666015c19136feae6be8
+R c9114bbd4adb78d55a86a8431d68fb8b
 U dan
-Z 80d209b7df04543ba8ed8a1a58dd2c4a
+Z d3545883deff75eff19284d6064a11b5
diff --git a/manifest.uuid b/manifest.uuid
index de896ff6d4..409e30cf73 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0cd75a872c89958a7f418720a0e8c6f638f8284c488f666015c19136feae6be8
\ No newline at end of file
+d238694ca445ccb4eeb3e3269a5f872f998f795945d0f9dd95c11d0e42d4d538
\ No newline at end of file
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 3c54efd36a..219e17b9fe 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -8037,6 +8037,20 @@ int sqlite3_vtab_config(sqlite3*, int op, ...);
 */
 int sqlite3_vtab_on_conflict(sqlite3 *);
 
+/*
+** CAPI3REF: Determine The Collation For a Virtual Table Constraint
+**
+** This function may only be called from within a call to the [xBestIndex]
+** method of a [virtual table implementation]. 
+**
+** The first argument must be the database handle with which the virtual 
+** table is associated (the one passed to the [xConnect] or [xCreate] method 
+** to create the sqlite3_vtab object. The second argument must be an index
+** into the aConstraint[] array belonging to the sqlite3_index_info structure
+** passed to xBestIndex. This function returns a pointer to a buffer 
+** containing the name of the collation sequence for the corresponding
+** constraint.
+*/
 SQLITE_EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3*, int);
 
 /*
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 22892c967f..259c020fd6 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1399,7 +1399,7 @@ struct sqlite3 {
   VtabCtx *pVtabCtx;            /* Context for active vtab connect/create */
   VTable **aVTrans;             /* Virtual tables with open transactions */
   VTable *pDisconnect;    /* Disconnect these in next sqlite3_prepare() */
-  void *pVtabWC;                /* For sqlite3_vtab_collation() */
+  void *pBestIndexCtx;          /* For sqlite3_vtab_collation() */
 #endif
   Hash aFunc;                   /* Hash table of connection functions */
   Hash aCollSeq;                /* All collating sequences */
diff --git a/src/where.c b/src/where.c
index d49c3d42f2..b1926a2d40 100644
--- a/src/where.c
+++ b/src/where.c
@@ -3117,14 +3117,25 @@ static int whereLoopAddVirtualOne(
 }
 
 
+/*
+** Context object used to pass information from whereLoopAddVirtual()
+** to sqlite3_vtab_collation().
+*/
 struct BestIndexCtx {
   WhereClause *pWC;
   sqlite3_index_info *pIdxInfo;
   Parse *pParse;
 };
 
+/*
+** If this function is invoked from within an xBestIndex() callback, it
+** returns a pointer to a buffer containing the name of the collation
+** sequence associated with element iCons of the sqlite3_index_info.aConstraint
+** array. Or, if iCons is out of range or there is no active xBestIndex
+** call, return NULL.
+*/
 const char *sqlite3_vtab_collation(sqlite3 *db, int iCons){
-  struct BestIndexCtx *p = (struct BestIndexCtx*)db->pVtabWC;
+  struct BestIndexCtx *p = (struct BestIndexCtx*)db->pBestIndexCtx;
   const char *zRet = 0;
   if( p && iCons>=0 && iCons<p->pIdxInfo->nConstraint ){
     int iTerm = p->pIdxInfo->aConstraint[iCons].iTermOffset;
@@ -3202,8 +3213,8 @@ static int whereLoopAddVirtual(
   bic.pWC = pWC;
   bic.pIdxInfo = p;
   bic.pParse = pParse;
-  pSaved = pParse->db->pVtabWC;
-  pParse->db->pVtabWC = (void*)&bic;
+  pSaved = pParse->db->pBestIndexCtx;
+  pParse->db->pBestIndexCtx = (void*)&bic;
 
   /* First call xBestIndex() with all constraints usable. */
   WHERETRACE(0x40, ("  VirtualOne: all usable\n"));
@@ -3281,7 +3292,7 @@ static int whereLoopAddVirtual(
 
   if( p->needToFreeIdxStr ) sqlite3_free(p->idxStr);
   sqlite3DbFreeNN(pParse->db, p);
-  pParse->db->pVtabWC = pSaved;
+  pParse->db->pBestIndexCtx = pSaved;
   return rc;
 }
 #endif /* SQLITE_OMIT_VIRTUALTABLE */

From 2abf90096ff64257a12f602e5754d3c33e97a41a Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 15 Apr 2017 16:52:12 +0000
Subject: [PATCH 032/488] Fix problems with handling constraints on the rowid
 column in sqlite3expert.c.

FossilOrigin-Name: 2e6308798ae2db30564deb35ba3896597448edabbcac6efc4ff084552e42de30
---
 ext/expert/expert1.test    | 58 +++++++++++++++++++++++++++-----------
 ext/expert/sqlite3expert.c | 20 +++++++------
 manifest                   | 14 ++++-----
 manifest.uuid              |  2 +-
 4 files changed, 61 insertions(+), 33 deletions(-)

diff --git a/ext/expert/expert1.test b/ext/expert/expert1.test
index 437ba45600..bcde9b2dfd 100644
--- a/ext/expert/expert1.test
+++ b/ext/expert/expert1.test
@@ -78,14 +78,14 @@ foreach {tn setup} {
   eval $setup
 
 
-do_setup_rec_test $tn.1.1 { CREATE TABLE t1(a, b, c) } {
+do_setup_rec_test $tn.1 { CREATE TABLE t1(a, b, c) } {
   SELECT * FROM t1
 } {
   (no new indexes)
   0|0|0|SCAN TABLE t1
 }
 
-do_setup_rec_test $tn.1.2 {
+do_setup_rec_test $tn.2 {
   CREATE TABLE t1(a, b, c);
 } {
   SELECT * FROM t1 WHERE b>?;
@@ -94,7 +94,7 @@ do_setup_rec_test $tn.1.2 {
   0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_00000062 (b>?)
 }
 
-do_setup_rec_test $tn.1.3 {
+do_setup_rec_test $tn.3 {
   CREATE TABLE t1(a, b, c);
 } {
   SELECT * FROM t1 WHERE b COLLATE nocase BETWEEN ? AND ?
@@ -103,7 +103,7 @@ do_setup_rec_test $tn.1.3 {
   0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_3e094c27 (b>? AND b<?)
 }
 
-do_setup_rec_test $tn.1.4 {
+do_setup_rec_test $tn.4 {
   CREATE TABLE t1(a, b, c);
 } {
   SELECT a FROM t1 ORDER BY b;
@@ -112,7 +112,7 @@ do_setup_rec_test $tn.1.4 {
   0|0|0|SCAN TABLE t1 USING INDEX t1_idx_00000062
 }
 
-do_setup_rec_test $tn.1.5 {
+do_setup_rec_test $tn.5 {
   CREATE TABLE t1(a, b, c);
 } {
   SELECT a FROM t1 WHERE a=? ORDER BY b;
@@ -121,7 +121,7 @@ do_setup_rec_test $tn.1.5 {
   0|0|0|SEARCH TABLE t1 USING COVERING INDEX t1_idx_000123a7 (a=?)
 }
 
-do_setup_rec_test $tn.1.6 {
+do_setup_rec_test $tn.6 {
   CREATE TABLE t1(a, b, c);
 } {
   SELECT min(a) FROM t1
@@ -130,7 +130,7 @@ do_setup_rec_test $tn.1.6 {
   0|0|0|SEARCH TABLE t1 USING COVERING INDEX t1_idx_00000061
 }
 
-do_setup_rec_test $tn.1.7 {
+do_setup_rec_test $tn.7 {
   CREATE TABLE t1(a, b, c);
 } {
   SELECT * FROM t1 ORDER BY a, b, c;
@@ -148,7 +148,7 @@ do_setup_rec_test $tn.1.7 {
 #  0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_5be6e222
 #}
 
-do_setup_rec_test $tn.1.9 {
+do_setup_rec_test $tn.8.1 {
   CREATE TABLE t1(a COLLATE NOCase, b, c);
 } {
   SELECT * FROM t1 WHERE a=?
@@ -156,8 +156,7 @@ do_setup_rec_test $tn.1.9 {
   CREATE INDEX t1_idx_00000061 ON t1(a);
   0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_00000061 (a=?)
 }
-
-do_setup_rec_test $tn.1.10 {
+do_setup_rec_test $tn.8.2 {
   CREATE TABLE t1(a, b COLLATE nocase, c);
 } {
   SELECT * FROM t1 ORDER BY a ASC, b DESC, c ASC;
@@ -169,7 +168,7 @@ do_setup_rec_test $tn.1.10 {
 
 # Tables with names that require quotes.
 #
-do_setup_rec_test $tn.8.1 {
+do_setup_rec_test $tn.9.1 {
   CREATE TABLE "t t"(a, b, c);
 } {
   SELECT * FROM "t t" WHERE a=?
@@ -178,7 +177,7 @@ do_setup_rec_test $tn.8.1 {
   0|0|0|SEARCH TABLE t t USING INDEX t t_idx_00000061 (a=?) 
 }
 
-do_setup_rec_test $tn.8.2 {
+do_setup_rec_test $tn.9.2 {
   CREATE TABLE "t t"(a, b, c);
 } {
   SELECT * FROM "t t" WHERE b BETWEEN ? AND ?
@@ -189,7 +188,7 @@ do_setup_rec_test $tn.8.2 {
 
 # Columns with names that require quotes.
 #
-do_setup_rec_test $tn.9.1 {
+do_setup_rec_test $tn.10.1 {
   CREATE TABLE t3(a, "b b", c);
 } {
   SELECT * FROM t3 WHERE "b b" = ?
@@ -198,7 +197,7 @@ do_setup_rec_test $tn.9.1 {
   0|0|0|SEARCH TABLE t3 USING INDEX t3_idx_00050c52 (b b=?)
 }
 
-do_setup_rec_test $tn.9.2 {
+do_setup_rec_test $tn.10.2 {
   CREATE TABLE t3(a, "b b", c);
 } {
   SELECT * FROM t3 ORDER BY "b b"
@@ -209,7 +208,7 @@ do_setup_rec_test $tn.9.2 {
 
 # Transitive constraints
 #
-do_setup_rec_test $tn.10.1 {
+do_setup_rec_test $tn.11.1 {
   CREATE TABLE t5(a, b);
   CREATE TABLE t6(c, d);
 } {
@@ -223,7 +222,7 @@ do_setup_rec_test $tn.10.1 {
 
 # OR terms.
 #
-do_setup_rec_test $tn.11.1 {
+do_setup_rec_test $tn.12.1 {
   CREATE TABLE t7(a, b);
 } {
   SELECT * FROM t7 WHERE a=? OR b=?
@@ -234,6 +233,33 @@ do_setup_rec_test $tn.11.1 {
   0|0|0|SEARCH TABLE t7 USING INDEX t7_idx_00000062 (b=?)
 }
 
+# rowid terms.
+#
+do_setup_rec_test $tn.13.1 {
+  CREATE TABLE t8(a, b);
+} {
+  SELECT * FROM t8 WHERE rowid=?
+} {
+  (no new indexes)
+  0|0|0|SEARCH TABLE t8 USING INTEGER PRIMARY KEY (rowid=?)
+}
+do_setup_rec_test $tn.13.2 {
+  CREATE TABLE t8(a, b);
+} {
+  SELECT * FROM t8 ORDER BY rowid
+} {
+  (no new indexes)
+  0|0|0|SCAN TABLE t8
+}
+do_setup_rec_test $tn.13.3 {
+  CREATE TABLE t8(a, b);
+} {
+  SELECT * FROM t8 WHERE a=? ORDER BY rowid
+} {
+  CREATE INDEX t8_idx_00000061 ON t8(a); 
+  0|0|0|SEARCH TABLE t8 USING INDEX t8_idx_00000061 (a=?)
+}
+
 }
 
 finish_test
diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index 153159fbbf..ab419d3f08 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -365,7 +365,7 @@ static int expertBestIndex(sqlite3_vtab *pVtab, sqlite3_index_info *pIdxInfo){
     /* Add the constraints to the IdxScan object */
     for(i=0; i<pIdxInfo->nConstraint; i++){
       struct sqlite3_index_constraint *pCons = &pIdxInfo->aConstraint[i];
-      if( pCons->usable && (pCons->op & opmask) ){
+      if( pCons->usable && pCons->iColumn>=0 && (pCons->op & opmask) ){
         IdxConstraint *pNew;
         const char *zColl = sqlite3_vtab_collation(dbv, i);
         pNew = idxNewConstraint(&rc, zColl);
@@ -388,14 +388,16 @@ static int expertBestIndex(sqlite3_vtab *pVtab, sqlite3_index_info *pIdxInfo){
     /* Add the ORDER BY to the IdxScan object */
     for(i=pIdxInfo->nOrderBy-1; i>=0; i--){
       int iCol = pIdxInfo->aOrderBy[i].iColumn;
-      IdxConstraint *pNew = idxNewConstraint(&rc, p->pTab->aCol[iCol].zColl);
-      if( pNew ){
-        pNew->iCol = iCol;
-        pNew->bDesc = pIdxInfo->aOrderBy[i].desc;
-        pNew->pNext = pScan->pOrder;
-        pNew->pLink = pScan->pOrder;
-        pScan->pOrder = pNew;
-        n++;
+      if( iCol>=0 ){
+        IdxConstraint *pNew = idxNewConstraint(&rc, p->pTab->aCol[iCol].zColl);
+        if( pNew ){
+          pNew->iCol = iCol;
+          pNew->bDesc = pIdxInfo->aOrderBy[i].desc;
+          pNew->pNext = pScan->pOrder;
+          pNew->pLink = pScan->pOrder;
+          pScan->pOrder = pNew;
+          n++;
+        }
       }
     }
   }
diff --git a/manifest b/manifest
index a544938625..8f2777fd7d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\slatest\strunk\schanges\sinto\sthis\sbranch.
-D 2017-04-15T15:47:19.800
+C Fix\sproblems\swith\shandling\sconstraints\son\sthe\srowid\scolumn\sin\ssqlite3expert.c.
+D 2017-04-15T16:52:12.782
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6
@@ -42,8 +42,8 @@ F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md 9f15075ec5ad772808eff55ef044c31140fd1146aa0a3c47eafd155e71851b01
 F ext/expert/expert.c 6349cf8d26c847f5f0fa7e25772b614c67f60f3c850dca0d75d55eb27cf3f69b
-F ext/expert/expert1.test cd630eda18a2508eade4c39a1eafe32e7437a33973391e5dddfc7fd0f3163684
-F ext/expert/sqlite3expert.c d4a0a45be58874b4cf54316e5d776726489967140399b1e77a17dbb66558cb38
+F ext/expert/expert1.test c08c95fd81e80073def8bdbf30b67934a9c20193b3632e5f27565ef88f964809
+F ext/expert/sqlite3expert.c d73a4813af4c8097bb574113c91abf9c0ec06a7af47c80227a160a7c413265b5
 F ext/expert/sqlite3expert.h b1c9eedeb647fd734c4206ae6851635284cfbfa5fb688eff74c3265c9f949b4d
 F ext/expert/test_expert.c bad0611732d07180d586bd589cbb7713dc3ab0338c52bff29680eb2007678c05
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
@@ -1579,7 +1579,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d238694ca445ccb4eeb3e3269a5f872f998f795945d0f9dd95c11d0e42d4d538 89f9e4363aa19f306e55f749c442eae2f8994f6a47c65e645a79b308b450d5e5
-R ee849516fe355a9533f6086faf663944
+P 2d0c458e013cb2d02fbeabed8dabd66f55141aac194611f0e599b3c95af1964f
+R d323d2b5ffa5632b2c495cb6bacbb016
 U dan
-Z 67dd241955788961a5e3c1b5613b9072
+Z 2344908fdbfea0ffd0d694cf678615cc
diff --git a/manifest.uuid b/manifest.uuid
index 69d4418f08..afc4462a6b 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2d0c458e013cb2d02fbeabed8dabd66f55141aac194611f0e599b3c95af1964f
\ No newline at end of file
+2e6308798ae2db30564deb35ba3896597448edabbcac6efc4ff084552e42de30
\ No newline at end of file

From 280db65e2cc87e8e842ed7fe3800de72e8bb8521 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 17 Apr 2017 17:03:08 +0000
Subject: [PATCH 033/488] Add support for analyzing trigger programs to the
 sqlite3_expert code.

FossilOrigin-Name: 159e8022a9d6701532b8b60e0c41154bc434c1bbdb107c8c97a78fb1140fa745
---
 ext/expert/expert1.test    |  15 +++
 ext/expert/sqlite3expert.c | 189 ++++++++++++++++++++++++++++++++++++-
 manifest                   |  30 +++---
 manifest.uuid              |   2 +-
 src/build.c                |  12 +--
 src/main.c                 |   5 +-
 src/prepare.c              |   2 +-
 src/sqlite.h.in            |  11 +++
 src/sqliteInt.h            |   3 +-
 src/trigger.c              |   2 +-
 src/vdbe.c                 |   6 +-
 src/vdbeaux.c              | 185 +++++++++++++++++++-----------------
 12 files changed, 340 insertions(+), 122 deletions(-)

diff --git a/ext/expert/expert1.test b/ext/expert/expert1.test
index bcde9b2dfd..a817865229 100644
--- a/ext/expert/expert1.test
+++ b/ext/expert/expert1.test
@@ -260,6 +260,21 @@ do_setup_rec_test $tn.13.3 {
   0|0|0|SEARCH TABLE t8 USING INDEX t8_idx_00000061 (a=?)
 }
 
+# Triggers
+#
+do_setup_rec_test $tn.14 {
+  CREATE TABLE t9(a, b, c);
+  CREATE TABLE t10(a, b, c);
+  CREATE TRIGGER t9t AFTER INSERT ON t9 BEGIN
+    UPDATE t10 SET a=new.a WHERE b = new.b;
+  END;
+} {
+  INSERT INTO t9 VALUES(?, ?, ?);
+} {
+  CREATE INDEX t10_idx_00000062 ON t10(b); 
+  0|0|0|SEARCH TABLE t10 USING INDEX t10_idx_00000062 (b=?)
+}
+
 }
 
 finish_test
diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index ab419d3f08..8316a18ae3 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -26,6 +26,7 @@ typedef struct IdxConstraint IdxConstraint;
 typedef struct IdxScan IdxScan;
 typedef struct IdxStatement IdxStatement;
 typedef struct IdxTable IdxTable;
+typedef struct IdxWrite IdxWrite;
 
 /*
 ** A single constraint. Equivalent to either "col = ?" or "col < ?" (or
@@ -74,6 +75,17 @@ struct IdxTable {
   IdxTable *pNext;                /* Next table in linked list of all tables */
 };
 
+/*
+** An object of the following type is created for each unique table/write-op
+** seen. The objects are stored in a singly-linked list beginning at
+** sqlite3expert.pWrite.
+*/
+struct IdxWrite {
+  IdxTable *pTab;
+  int eOp;                        /* SQLITE_UPDATE, DELETE or INSERT */
+  IdxWrite *pNext;
+};
+
 /*
 ** Each statement being analyzed is represented by an instance of this
 ** structure.
@@ -118,8 +130,8 @@ struct sqlite3expert {
   sqlite3 *dbm;                   /* In-memory db for this analysis */
   sqlite3 *dbv;                   /* Vtab schema for this analysis */
   IdxTable *pTable;               /* List of all IdxTable objects */
-
   IdxScan *pScan;                 /* List of scan objects */
+  IdxWrite *pWrite;               /* List of write objects */
   IdxStatement *pStatement;       /* List of IdxStatement objects */
   int bRun;                       /* True once analysis has run */
   char **pzErrmsg;
@@ -406,6 +418,15 @@ static int expertBestIndex(sqlite3_vtab *pVtab, sqlite3_index_info *pIdxInfo){
   return rc;
 }
 
+static int expertUpdate(
+  sqlite3_vtab *pVtab, 
+  int nData, 
+  sqlite3_value **azData, 
+  sqlite_int64 *pRowid
+){
+  return SQLITE_OK;
+}
+
 static int idxRegisterVtab(sqlite3expert *p){
   static sqlite3_module expertModule = {
     2,                            /* iVersion */
@@ -421,7 +442,7 @@ static int idxRegisterVtab(sqlite3expert *p){
     0,                            /* xEof */
     0,                            /* xColumn - read data */
     0,                            /* xRowid - read data */
-    0,                            /* xUpdate - write data */
+    expertUpdate,                 /* xUpdate - write data */
     0,                            /* xBegin - begin transaction */
     0,                            /* xSync - sync transaction */
     0,                            /* xCommit - commit transaction */
@@ -926,6 +947,19 @@ static void idxTableFree(IdxTable *pTab){
   }
 }
 
+/*
+** Free the linked list of IdxWrite objects starting at pTab.
+*/
+static void idxWriteFree(IdxWrite *pTab){
+  IdxWrite *pIter;
+  IdxWrite *pNext;
+  for(pIter=pTab; pIter; pIter=pNext){
+    pNext = pIter->pNext;
+    sqlite3_free(pIter);
+  }
+}
+
+
 
 /*
 ** This function is called after candidate indexes have been created. It
@@ -997,6 +1031,139 @@ int idxFindIndexes(
   return rc;
 }
 
+static int idxAuthCallback(
+  void *pCtx,
+  int eOp,
+  const char *z3,
+  const char *z4,
+  const char *zDb,
+  const char *zTrigger
+){
+  int rc = SQLITE_OK;
+  if( eOp==SQLITE_INSERT || eOp==SQLITE_UPDATE || eOp==SQLITE_DELETE ){
+    if( sqlite3_stricmp(zDb, "main")==0 ){
+      sqlite3expert *p = (sqlite3expert*)pCtx;
+      IdxTable *pTab;
+      for(pTab=p->pTable; pTab; pTab=pTab->pNext){
+        if( 0==sqlite3_stricmp(z3, pTab->zName) ) break;
+      }
+      if( pTab ){
+        IdxWrite *pWrite;
+        for(pWrite=p->pWrite; pWrite; pWrite=pWrite->pNext){
+          if( pWrite->pTab==pTab && pWrite->eOp==eOp ) break;
+        }
+        if( pWrite==0 ){
+          pWrite = idxMalloc(&rc, sizeof(IdxWrite));
+          if( rc==SQLITE_OK ){
+            pWrite->pTab = pTab;
+            pWrite->eOp = eOp;
+            pWrite->pNext = p->pWrite;
+            p->pWrite = pWrite;
+          }
+        }
+      }
+    }
+  }
+  return rc;
+}
+
+static int idxProcessOneTrigger(
+  sqlite3expert *p, 
+  IdxWrite *pWrite, 
+  char **pzErr
+){
+  static const char *zInt = "t592690916721053953805701627921227776";
+  static const char *zDrop = "DROP TABLE t592690916721053953805701627921227776";
+  IdxTable *pTab = pWrite->pTab;
+  const char *zTab = pTab->zName;
+  const char *zSql = 
+    "SELECT 'CREATE TEMP' || substr(sql, 7) FROM sqlite_master "
+    "WHERE tbl_name = %Q AND type IN ('table', 'trigger') "
+    "ORDER BY type;";
+  sqlite3_stmt *pSelect = 0;
+  int rc = SQLITE_OK;
+  char *zWrite = 0;
+
+  /* Create the table and its triggers in the temp schema */
+  rc = idxPrintfPrepareStmt(p->db, &pSelect, pzErr, zSql, zTab, zTab);
+  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSelect) ){
+    const char *zCreate = (const char*)sqlite3_column_text(pSelect, 0);
+    rc = sqlite3_exec(p->dbv, zCreate, 0, 0, pzErr);
+  }
+  idxFinalize(&rc, pSelect);
+
+  /* Rename the table in the temp schema to zInt */
+  if( rc==SQLITE_OK ){
+    char *z = sqlite3_mprintf("ALTER TABLE temp.%Q RENAME TO %Q", zTab, zInt);
+    if( z==0 ){
+      rc = SQLITE_NOMEM;
+    }else{
+      rc = sqlite3_exec(p->dbv, z, 0, 0, pzErr);
+      sqlite3_free(z);
+    }
+  }
+
+  switch( pWrite->eOp ){
+    case SQLITE_INSERT: {
+      int i;
+      zWrite = idxAppendText(&rc, zWrite, "INSERT INTO %Q VALUES(", zInt);
+      for(i=0; i<pTab->nCol; i++){
+        zWrite = idxAppendText(&rc, zWrite, "%s?", i==0 ? "" : ", ");
+      }
+      zWrite = idxAppendText(&rc, zWrite, ")");
+      break;
+    }
+    case SQLITE_UPDATE: {
+      int i;
+      zWrite = idxAppendText(&rc, zWrite, "UPDATE %Q SET ", zInt);
+      for(i=0; i<pTab->nCol; i++){
+        zWrite = idxAppendText(&rc, zWrite, "%s%Q=?", i==0 ? "" : ", ", 
+            pTab->aCol[i].zName
+        );
+      }
+      break;
+    }
+    default: {
+      assert( pWrite->eOp==SQLITE_DELETE );
+      if( rc==SQLITE_OK ){
+        zWrite = sqlite3_mprintf("DELETE FROM %Q", zInt);
+        if( zWrite==0 ) rc = SQLITE_NOMEM;
+      }
+    }
+  }
+
+  if( rc==SQLITE_OK ){
+    sqlite3_stmt *pX = 0;
+    rc = sqlite3_prepare_v2(p->dbv, zWrite, -1, &pX, 0);
+    idxFinalize(&rc, pX);
+  }
+  sqlite3_free(zWrite);
+
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_exec(p->dbv, zDrop, 0, 0, pzErr);
+  }
+
+  return rc;
+}
+
+static int idxProcessTriggers(sqlite3expert *p, char **pzErr){
+  int rc = SQLITE_OK;
+  IdxWrite *pEnd = 0;
+  IdxWrite *pFirst = p->pWrite;
+
+  while( rc==SQLITE_OK && pFirst!=pEnd ){
+    IdxWrite *pIter;
+    for(pIter=pFirst; rc==SQLITE_OK && pIter!=pEnd; pIter=pIter->pNext){
+      rc = idxProcessOneTrigger(p, pIter, pzErr);
+    }
+    pEnd = pFirst;
+    pFirst = p->pWrite;
+  }
+
+  return rc;
+}
+
+
 static int idxCreateVtabSchema(sqlite3expert *p, char **pzErrmsg){
   int rc = idxRegisterVtab(p);
   sqlite3_stmt *pSchema = 0;
@@ -1073,7 +1240,11 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){
   }
   if( rc==SQLITE_OK ){
     rc = sqlite3_open(":memory:", &pNew->dbm);
+    if( rc==SQLITE_OK ){
+      sqlite3_db_config(pNew->dbm, SQLITE_DBCONFIG_FULL_EQP, 1, (int*)0);
+    }
   }
+  
 
   /* Copy the entire schema of database [db] into [dbm]. */
   if( rc==SQLITE_OK ){
@@ -1093,6 +1264,11 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){
     rc = idxCreateVtabSchema(pNew, pzErrmsg);
   }
 
+  /* Register the auth callback with dbv */
+  if( rc==SQLITE_OK ){
+    sqlite3_set_authorizer(pNew->dbv, idxAuthCallback, (void*)pNew);
+  }
+
   /* If an error has occurred, free the new object and reutrn NULL. Otherwise,
   ** return the new sqlite3expert handle.  */
   if( rc!=SQLITE_OK ){
@@ -1136,7 +1312,7 @@ int sqlite3_expert_sql(
         sqlite3_finalize(pStmt);
       }
     }else{
-      idxDatabaseError(p->db, pzErr);
+      idxDatabaseError(p->dbv, pzErr);
     }
   }
 
@@ -1154,8 +1330,12 @@ int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr){
   int rc;
   IdxHashEntry *pEntry;
 
+  rc = idxProcessTriggers(p, pzErr);
+
   /* Create candidate indexes within the in-memory database file */
-  rc = idxCreateCandidates(p, pzErr);
+  if( rc==SQLITE_OK ){
+    rc = idxCreateCandidates(p, pzErr);
+  }
 
   /* Formulate the EXPERT_REPORT_CANDIDATES text */
   for(pEntry=p->hIdx.pFirst; pEntry; pEntry=pEntry->pNext){
@@ -1220,6 +1400,7 @@ void sqlite3_expert_destroy(sqlite3expert *p){
     idxScanFree(p->pScan, 0);
     idxStatementFree(p->pStatement, 0);
     idxTableFree(p->pTable);
+    idxWriteFree(p->pWrite);
     idxHashClear(&p->hIdx);
     sqlite3_free(p->zCandidates);
     sqlite3_free(p);
diff --git a/manifest b/manifest
index 8f2777fd7d..7bc6efbb2d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sproblems\swith\shandling\sconstraints\son\sthe\srowid\scolumn\sin\ssqlite3expert.c.
-D 2017-04-15T16:52:12.782
+C Add\ssupport\sfor\sanalyzing\strigger\sprograms\sto\sthe\ssqlite3_expert\scode.
+D 2017-04-17T17:03:08.723
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6
@@ -42,8 +42,8 @@ F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md 9f15075ec5ad772808eff55ef044c31140fd1146aa0a3c47eafd155e71851b01
 F ext/expert/expert.c 6349cf8d26c847f5f0fa7e25772b614c67f60f3c850dca0d75d55eb27cf3f69b
-F ext/expert/expert1.test c08c95fd81e80073def8bdbf30b67934a9c20193b3632e5f27565ef88f964809
-F ext/expert/sqlite3expert.c d73a4813af4c8097bb574113c91abf9c0ec06a7af47c80227a160a7c413265b5
+F ext/expert/expert1.test 6a50a1538dc9e4ff360fb117298aa3b085beed030cbe15dd36803da1a9f70702
+F ext/expert/sqlite3expert.c f9f41caf6d941b52ef76043f3191ab05294f4d286425e34923a7d74876014f0b
 F ext/expert/sqlite3expert.h b1c9eedeb647fd734c4206ae6851635284cfbfa5fb688eff74c3265c9f949b4d
 F ext/expert/test_expert.c bad0611732d07180d586bd589cbb7713dc3ab0338c52bff29680eb2007678c05
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
@@ -353,7 +353,7 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
 F src/btree.c 24ae5472bd0b53b4130ecdda389deb621af721d1fcb50890b878102b00bd10fa
 F src/btree.h 80f518c0788be6cec8d9f8e13bd8e380df299d2b5e4ac340dc887b0642647cfc
 F src/btreeInt.h a392d353104b4add58b4a59cb185f5d5693dde832c565b77d8d4c343ed98f610
-F src/build.c 4026a9c554b233e50c5e9ad46963e676cf54dd2306d952aa1eaa07a1bc9ce14f
+F src/build.c 3fd46781483b527ee18508e7854e87e60a259211bb9bbf16b6fafaf08a043a64
 F src/callback.c 2e76147783386374bf01b227f752c81ec872d730
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c 47d91a25ad8f199a71a5b1b7b169d6dd0d6e98c5719eca801568798743d1161c
@@ -372,7 +372,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c d4bb3a135948553d18cf992f76f7ed7b18aa0327f250607b5a6671e55d9947d5
 F src/legacy.c e88ed13c2d531decde75d42c2e35623fb9ce3cb0
 F src/loadext.c a72909474dadce771d3669bf84bf689424f6f87d471fee898589c3ef9b2acfd9
-F src/main.c 158326243c5ddc8b98a1e983fa488650cf76d760
+F src/main.c ffb658429483c0d1c604014c631871f64b8db19666d8b70f75c7512d003ac1ad
 F src/malloc.c e20bb2b48abec52d3faf01cce12e8b4f95973755fafec98d45162dfdab111978
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -402,17 +402,17 @@ F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490
 F src/pcache1.c 1195a21fe28e223e024f900b2011e80df53793f0356a24caace4188b098540dc
 F src/pragma.c 150821702fc90694b46c3432c1402fc970a4c5b8595cb13c21aeb568f9a78fc3
 F src/pragma.h 37a1311d0388db480388d7ec09054f7103045eff20d4971f8a433b77f40b9921
-F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a
+F src/prepare.c 7c46b5c7be9e19a1bf87777f0b7f9fb257b5ff9856c46de49f2354acfbeb4c86
 F src/printf.c 8757834f1b54dae512fb25eb1acc8e94a0d15dd2290b58f2563f65973265adb2
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 3e518b962d932a997fae373366880fc028c75706
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 4588dcfb0fa430012247a209ba08e17904dd32ec7690e9cb6c85e0ef012b0518
 F src/shell.c 70f4957b988572315e97c56941fdc81fd35907fee36b7b2e7be5ec4c7e9d065d
-F src/sqlite.h.in 18b4f1367bb80c07a6883c75c18267de2b977960adf1f30c783a731691eae3b0
+F src/sqlite.h.in 900a07463a87be50b9954817f4c24a0660b4c4ddc1bfe83dedea484c6ac98425
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 58fd0676d3111d02e62e5a35992a7d3da5d3f88753acc174f2d37b774fbbdd28
-F src/sqliteInt.h bd00872a45aa8eaa262301436bb63dc0afa685ceab7361790b065d5802269650
+F src/sqliteInt.h 0e520ab49f019221dd5a17b6e4006523ce4f33d88b20bcf9115d11952a487c39
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -469,16 +469,16 @@ F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
 F src/tokenize.c 1003d6d90c6783206c711f0a9397656fa5b055209f4d092caa43bb3bf5215db5
 F src/treeview.c b92d57c1ac59f4a3f6b189506921a2b48098f6f4d6afd0b715bc2815ef6af092
-F src/trigger.c c9f0810043b265724fdb1bdd466894f984dfc182
+F src/trigger.c 134b8e7b61317ab7b2a2dd12eb1b9aa2e23ac5bc4a05e63e35b3609b6b30a7c0
 F src/update.c c443935c652af9365e033f756550b5032d02e1b06eb2cb890ed7511ae0c051dc
 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c
 F src/util.c ca8440ede81e155d15cff7c101654f60b55a9ae6
 F src/vacuum.c 1fe4555cd8c9b263afb85b5b4ee3a4a4181ad569
-F src/vdbe.c 808fda3d50f544120d27c731449b524b4ec8f8b0f734b228831078f0ba53ecb9
+F src/vdbe.c 314f0a70ddc29e63f131dfbe6f53c277875d3028cdf5654a709e21cab39fe0c9
 F src/vdbe.h f7d1456e28875c2dcb964056589b5b7149ab7edf39edeca801596a39bb3d3848
 F src/vdbeInt.h c070bc5c8b913bda0ceaa995cd4d939ded5e4fc96cf7c3c1c602d41b871f8ade
 F src/vdbeapi.c 5b08d82592bcff4470601fe78aaabebd50837860
-F src/vdbeaux.c 6b3f6ce909e206d4c918988b13b7fa687e92b4471d137e0f2a37edac80ec60be
+F src/vdbeaux.c 526b617ac6b5e167a6bd581e067f1ee1dbcb06e7802cff46b76fb1c02ed7d34e
 F src/vdbeblob.c 359891617358deefc85bef7bcf787fa6b77facb9
 F src/vdbemem.c 3122f5a21064198c10ee1b4686937aab27d5395712d9af905b7fa1affc47a453
 F src/vdbesort.c e72fe02a2121386ba767ede8942e9450878b8fc873abf3d1b6824485f092570c
@@ -1579,7 +1579,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2d0c458e013cb2d02fbeabed8dabd66f55141aac194611f0e599b3c95af1964f
-R d323d2b5ffa5632b2c495cb6bacbb016
+P 2e6308798ae2db30564deb35ba3896597448edabbcac6efc4ff084552e42de30
+R 3935ba65365bf244da478b55268b3a1f
 U dan
-Z 2344908fdbfea0ffd0d694cf678615cc
+Z 871de5d26699f20ec603c0a41beb3b11
diff --git a/manifest.uuid b/manifest.uuid
index afc4462a6b..8158ffa2b0 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2e6308798ae2db30564deb35ba3896597448edabbcac6efc4ff084552e42de30
\ No newline at end of file
+159e8022a9d6701532b8b60e0c41154bc434c1bbdb107c8c97a78fb1140fa745
\ No newline at end of file
diff --git a/src/build.c b/src/build.c
index e04406d857..47339e82b6 100644
--- a/src/build.c
+++ b/src/build.c
@@ -479,7 +479,7 @@ void sqlite3UnlinkAndDeleteIndex(sqlite3 *db, int iDb, const char *zIdxName){
     }
     freeIndex(db, pIndex);
   }
-  db->flags |= SQLITE_InternChanges;
+  db->bInternChanges = 1;
 }
 
 /*
@@ -551,7 +551,7 @@ void sqlite3ResetAllSchemasOfConnection(sqlite3 *db){
       sqlite3SchemaClear(pDb->pSchema);
     }
   }
-  db->flags &= ~SQLITE_InternChanges;
+  db->bInternChanges = 0;
   sqlite3VtabUnlockList(db);
   sqlite3BtreeLeaveAll(db);
   sqlite3CollapseDatabaseArray(db);
@@ -561,7 +561,7 @@ void sqlite3ResetAllSchemasOfConnection(sqlite3 *db){
 ** This routine is called when a commit occurs.
 */
 void sqlite3CommitInternalChanges(sqlite3 *db){
-  db->flags &= ~SQLITE_InternChanges;
+  db->bInternChanges = 0;
 }
 
 /*
@@ -665,7 +665,7 @@ void sqlite3UnlinkAndDeleteTable(sqlite3 *db, int iDb, const char *zTabName){
   pDb = &db->aDb[iDb];
   p = sqlite3HashInsert(&pDb->pSchema->tblHash, zTabName, 0);
   sqlite3DeleteTable(db, p);
-  db->flags |= SQLITE_InternChanges;
+  db->bInternChanges = 1;
 }
 
 /*
@@ -2051,7 +2051,7 @@ void sqlite3EndTable(
       return;
     }
     pParse->pNewTable = 0;
-    db->flags |= SQLITE_InternChanges;
+    db->bInternChanges = 1;
 
 #ifndef SQLITE_OMIT_ALTERTABLE
     if( !p->pSelect ){
@@ -3320,7 +3320,7 @@ void sqlite3CreateIndex(
       sqlite3OomFault(db);
       goto exit_create_index;
     }
-    db->flags |= SQLITE_InternChanges;
+    db->bInternChanges = 1;
     if( pTblName!=0 ){
       pIndex->tnum = db->init.newTnum;
     }
diff --git a/src/main.c b/src/main.c
index 4ac5327e4f..db508abc57 100644
--- a/src/main.c
+++ b/src/main.c
@@ -811,6 +811,7 @@ int sqlite3_db_config(sqlite3 *db, int op, ...){
         { SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, SQLITE_Fts3Tokenizer  },
         { SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, SQLITE_LoadExtension  },
         { SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE,      SQLITE_NoCkptOnClose  },
+        { SQLITE_DBCONFIG_FULL_EQP,              SQLITE_FullEQP  },
       };
       unsigned int i;
       rc = SQLITE_ERROR; /* IMP: R-42790-23372 */
@@ -1252,7 +1253,7 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){
   ** the database rollback and schema reset, which can cause false
   ** corruption reports in some cases.  */
   sqlite3BtreeEnterAll(db);
-  schemaChange = (db->flags & SQLITE_InternChanges)!=0 && db->init.busy==0;
+  schemaChange = db->bInternChanges && db->init.busy==0;
 
   for(i=0; i<db->nDb; i++){
     Btree *p = db->aDb[i].pBt;
@@ -1266,7 +1267,7 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){
   sqlite3VtabRollback(db);
   sqlite3EndBenignMalloc();
 
-  if( (db->flags&SQLITE_InternChanges)!=0 && db->init.busy==0 ){
+  if( db->bInternChanges && db->init.busy==0 ){
     sqlite3ExpirePreparedStatements(db);
     sqlite3ResetAllSchemasOfConnection(db);
   }
diff --git a/src/prepare.c b/src/prepare.c
index 74127bc76b..aa36cca166 100644
--- a/src/prepare.c
+++ b/src/prepare.c
@@ -354,7 +354,7 @@ error_out:
 */
 int sqlite3Init(sqlite3 *db, char **pzErrMsg){
   int i, rc;
-  int commit_internal = !(db->flags&SQLITE_InternChanges);
+  int commit_internal = db->bInternChanges==0;
   
   assert( sqlite3_mutex_held(db->mutex) );
   assert( sqlite3BtreeHoldsMutex(db->aDb[0].pBt) );
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 219e17b9fe..6a5de5f5b9 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -2007,6 +2007,16 @@ struct sqlite3_mem_methods {
 ** have been disabled - 0 if they are not disabled, 1 if they are.
 ** </dd>
 **
+** <dt>SQLITE_DBCONFIG_FULL_EQP</dt>
+** <dd> By default, the output of EXPLAIN QUERY PLAN commands does not 
+** include output for any operations performed by trigger programs. This
+** option is used to set or clear (the default) a flag that governs this
+** behavior. The first parameter passed to this operation is an integer -
+** non-zero to enable output for trigger programs, or zero to disable it.
+** The second parameter is a pointer to an integer into which is written 
+** 0 or 1 to indicate whether output-for-triggers has been disabled - 0 if 
+** it is not disabled, 1 if it is.  
+** </dd>
 ** </dl>
 */
 #define SQLITE_DBCONFIG_MAINDBNAME            1000 /* const char* */
@@ -2016,6 +2026,7 @@ struct sqlite3_mem_methods {
 #define SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 1004 /* int int* */
 #define SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION 1005 /* int int* */
 #define SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE      1006 /* int int* */
+#define SQLITE_DBCONFIG_FULL_EQP              1007 /* int int* */
 
 
 /*
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 259c020fd6..3ce77bf7d5 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1334,6 +1334,7 @@ struct sqlite3 {
   u8 mTrace;                    /* zero or more SQLITE_TRACE flags */
   u8 skipBtreeMutex;            /* True if no shared-cache backends */
   u8 nSqlExec;                  /* Number of pending OP_SqlExec opcodes */
+  u8 bInternChanges;            /* There are uncommited schema changes */
   int nextPagesize;             /* Pagesize after VACUUM if >0 */
   u32 magic;                    /* Magic number for detect library misuse */
   int nChange;                  /* Value returned by sqlite3_changes() */
@@ -1449,7 +1450,7 @@ struct sqlite3 {
 **      SQLITE_CacheSpill    == PAGER_CACHE_SPILL
 */
 #define SQLITE_VdbeTrace      0x00000001  /* True to trace VDBE execution */
-#define SQLITE_InternChanges  0x00000002  /* Uncommitted Hash table changes */
+#define SQLITE_FullEQP        0x00000002  /* Include triggers in EQP output */
 #define SQLITE_FullColNames   0x00000004  /* Show full column names on SELECT */
 #define SQLITE_FullFSync      0x00000008  /* Use full fsync on the backend */
 #define SQLITE_CkptFullFSync  0x00000010  /* Use full fsync for checkpoint */
diff --git a/src/trigger.c b/src/trigger.c
index bdf964084b..197dcda6c3 100644
--- a/src/trigger.c
+++ b/src/trigger.c
@@ -584,7 +584,7 @@ void sqlite3UnlinkAndDeleteTrigger(sqlite3 *db, int iDb, const char *zName){
       *pp = (*pp)->pNext;
     }
     sqlite3DeleteTrigger(db, pTrigger);
-    db->flags |= SQLITE_InternChanges;
+    db->bInternChanges = 1;
   }
 }
 
diff --git a/src/vdbe.c b/src/vdbe.c
index a990afb11d..4f8657829f 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -3037,7 +3037,7 @@ case OP_Savepoint: {
         int isSchemaChange;
         iSavepoint = db->nSavepoint - iSavepoint - 1;
         if( p1==SAVEPOINT_ROLLBACK ){
-          isSchemaChange = (db->flags & SQLITE_InternChanges)!=0;
+          isSchemaChange = db->bInternChanges;
           for(ii=0; ii<db->nDb; ii++){
             rc = sqlite3BtreeTripAllCursors(db->aDb[ii].pBt,
                                        SQLITE_ABORT_ROLLBACK,
@@ -3056,7 +3056,7 @@ case OP_Savepoint: {
         if( isSchemaChange ){
           sqlite3ExpirePreparedStatements(db);
           sqlite3ResetAllSchemasOfConnection(db);
-          db->flags = (db->flags | SQLITE_InternChanges);
+          db->bInternChanges = 1;
         }
       }
   
@@ -3336,7 +3336,7 @@ case OP_SetCookie: {
   if( pOp->p2==BTREE_SCHEMA_VERSION ){
     /* When the schema cookie changes, record the new cookie internally */
     pDb->pSchema->schema_cookie = pOp->p3;
-    db->flags |= SQLITE_InternChanges;
+    db->bInternChanges = 1;
   }else if( pOp->p2==BTREE_FILE_FORMAT ){
     /* Record changes in the file format */
     pDb->pSchema->file_format = pOp->p3;
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index 00a5ec91a9..5de43f42ce 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -1611,6 +1611,8 @@ int sqlite3VdbeList(
   int i;                               /* Loop counter */
   int rc = SQLITE_OK;                  /* Return code */
   Mem *pMem = &p->aMem[1];             /* First Mem of result set */
+  int bFull = (p->explain==1 || (db->flags & SQLITE_FullEQP));
+  Op *pOp;
 
   assert( p->explain );
   assert( p->magic==VDBE_MAGIC_RUN );
@@ -1638,7 +1640,7 @@ int sqlite3VdbeList(
   ** encountered, but p->pc will eventually catch up to nRow.
   */
   nRow = p->nOp;
-  if( p->explain==1 ){
+  if( bFull ){
     /* The first 8 memory cells are used for the result set.  So we will
     ** commandeer the 9th cell to use as storage for an array of pointers
     ** to trigger subprograms.  The VDBE is guaranteed to have at least 9
@@ -1658,17 +1660,11 @@ int sqlite3VdbeList(
 
   do{
     i = p->pc++;
-  }while( i<nRow && p->explain==2 && p->aOp[i].opcode!=OP_Explain );
-  if( i>=nRow ){
-    p->rc = SQLITE_OK;
-    rc = SQLITE_DONE;
-  }else if( db->u1.isInterrupted ){
-    p->rc = SQLITE_INTERRUPT;
-    rc = SQLITE_ERROR;
-    sqlite3VdbeError(p, sqlite3ErrStr(p->rc));
-  }else{
-    char *zP4;
-    Op *pOp;
+    if( i>=nRow ){
+      p->rc = SQLITE_OK;
+      rc = SQLITE_DONE;
+      break;
+    }
     if( i<p->nOp ){
       /* The output line number is small enough that we are still in the
       ** main program. */
@@ -1683,94 +1679,107 @@ int sqlite3VdbeList(
       }
       pOp = &apSub[j]->aOp[i];
     }
-    if( p->explain==1 ){
-      pMem->flags = MEM_Int;
-      pMem->u.i = i;                                /* Program counter */
-      pMem++;
-  
-      pMem->flags = MEM_Static|MEM_Str|MEM_Term;
-      pMem->z = (char*)sqlite3OpcodeName(pOp->opcode); /* Opcode */
-      assert( pMem->z!=0 );
-      pMem->n = sqlite3Strlen30(pMem->z);
-      pMem->enc = SQLITE_UTF8;
-      pMem++;
 
-      /* When an OP_Program opcode is encounter (the only opcode that has
-      ** a P4_SUBPROGRAM argument), expand the size of the array of subprograms
-      ** kept in p->aMem[9].z to hold the new program - assuming this subprogram
-      ** has not already been seen.
-      */
-      if( pOp->p4type==P4_SUBPROGRAM ){
-        int nByte = (nSub+1)*sizeof(SubProgram*);
-        int j;
-        for(j=0; j<nSub; j++){
-          if( apSub[j]==pOp->p4.pProgram ) break;
-        }
-        if( j==nSub && SQLITE_OK==sqlite3VdbeMemGrow(pSub, nByte, nSub!=0) ){
-          apSub = (SubProgram **)pSub->z;
-          apSub[nSub++] = pOp->p4.pProgram;
-          pSub->flags |= MEM_Blob;
-          pSub->n = nSub*sizeof(SubProgram*);
-        }
+    /* When an OP_Program opcode is encounter (the only opcode that has
+    ** a P4_SUBPROGRAM argument), expand the size of the array of subprograms
+    ** kept in p->aMem[9].z to hold the new program - assuming this subprogram
+    ** has not already been seen.
+    */
+    if( bFull && pOp->p4type==P4_SUBPROGRAM ){
+      int nByte = (nSub+1)*sizeof(SubProgram*);
+      int j;
+      for(j=0; j<nSub; j++){
+        if( apSub[j]==pOp->p4.pProgram ) break;
+      }
+      if( j==nSub ){
+        rc = sqlite3VdbeMemGrow(pSub, nByte, nSub!=0);
+        if( rc!=SQLITE_OK ) break;
+        apSub = (SubProgram **)pSub->z;
+        apSub[nSub++] = pOp->p4.pProgram;
+        pSub->flags |= MEM_Blob;
+        pSub->n = nSub*sizeof(SubProgram*);
+        nRow += pOp->p4.pProgram->nOp;
       }
     }
+  }while( p->explain==2 && pOp->opcode!=OP_Explain );
 
-    pMem->flags = MEM_Int;
-    pMem->u.i = pOp->p1;                          /* P1 */
-    pMem++;
-
-    pMem->flags = MEM_Int;
-    pMem->u.i = pOp->p2;                          /* P2 */
-    pMem++;
-
-    pMem->flags = MEM_Int;
-    pMem->u.i = pOp->p3;                          /* P3 */
-    pMem++;
-
-    if( sqlite3VdbeMemClearAndResize(pMem, 100) ){ /* P4 */
-      assert( p->db->mallocFailed );
-      return SQLITE_ERROR;
-    }
-    pMem->flags = MEM_Str|MEM_Term;
-    zP4 = displayP4(pOp, pMem->z, pMem->szMalloc);
-    if( zP4!=pMem->z ){
-      pMem->n = 0;
-      sqlite3VdbeMemSetStr(pMem, zP4, -1, SQLITE_UTF8, 0);
+  if( rc==SQLITE_OK ){
+    if( db->u1.isInterrupted ){
+      p->rc = SQLITE_INTERRUPT;
+      rc = SQLITE_ERROR;
+      sqlite3VdbeError(p, sqlite3ErrStr(p->rc));
     }else{
-      assert( pMem->z!=0 );
-      pMem->n = sqlite3Strlen30(pMem->z);
-      pMem->enc = SQLITE_UTF8;
-    }
-    pMem++;
-
-    if( p->explain==1 ){
-      if( sqlite3VdbeMemClearAndResize(pMem, 4) ){
-        assert( p->db->mallocFailed );
-        return SQLITE_ERROR;
+      char *zP4;
+      if( p->explain==1 ){
+        pMem->flags = MEM_Int;
+        pMem->u.i = i;                                /* Program counter */
+        pMem++;
+    
+        pMem->flags = MEM_Static|MEM_Str|MEM_Term;
+        pMem->z = (char*)sqlite3OpcodeName(pOp->opcode); /* Opcode */
+        assert( pMem->z!=0 );
+        pMem->n = sqlite3Strlen30(pMem->z);
+        pMem->enc = SQLITE_UTF8;
+        pMem++;
       }
-      pMem->flags = MEM_Str|MEM_Term;
-      pMem->n = 2;
-      sqlite3_snprintf(3, pMem->z, "%.2x", pOp->p5);   /* P5 */
-      pMem->enc = SQLITE_UTF8;
+
+      pMem->flags = MEM_Int;
+      pMem->u.i = pOp->p1;                          /* P1 */
       pMem++;
-  
-#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS
-      if( sqlite3VdbeMemClearAndResize(pMem, 500) ){
+
+      pMem->flags = MEM_Int;
+      pMem->u.i = pOp->p2;                          /* P2 */
+      pMem++;
+
+      pMem->flags = MEM_Int;
+      pMem->u.i = pOp->p3;                          /* P3 */
+      pMem++;
+
+      if( sqlite3VdbeMemClearAndResize(pMem, 100) ){ /* P4 */
         assert( p->db->mallocFailed );
         return SQLITE_ERROR;
       }
       pMem->flags = MEM_Str|MEM_Term;
-      pMem->n = displayComment(pOp, zP4, pMem->z, 500);
-      pMem->enc = SQLITE_UTF8;
-#else
-      pMem->flags = MEM_Null;                       /* Comment */
-#endif
-    }
+      zP4 = displayP4(pOp, pMem->z, pMem->szMalloc);
+      if( zP4!=pMem->z ){
+        pMem->n = 0;
+        sqlite3VdbeMemSetStr(pMem, zP4, -1, SQLITE_UTF8, 0);
+      }else{
+        assert( pMem->z!=0 );
+        pMem->n = sqlite3Strlen30(pMem->z);
+        pMem->enc = SQLITE_UTF8;
+      }
+      pMem++;
 
-    p->nResColumn = 8 - 4*(p->explain-1);
-    p->pResultSet = &p->aMem[1];
-    p->rc = SQLITE_OK;
-    rc = SQLITE_ROW;
+      if( p->explain==1 ){
+        if( sqlite3VdbeMemClearAndResize(pMem, 4) ){
+          assert( p->db->mallocFailed );
+          return SQLITE_ERROR;
+        }
+        pMem->flags = MEM_Str|MEM_Term;
+        pMem->n = 2;
+        sqlite3_snprintf(3, pMem->z, "%.2x", pOp->p5);   /* P5 */
+        pMem->enc = SQLITE_UTF8;
+        pMem++;
+    
+#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS
+        if( sqlite3VdbeMemClearAndResize(pMem, 500) ){
+          assert( p->db->mallocFailed );
+          return SQLITE_ERROR;
+        }
+        pMem->flags = MEM_Str|MEM_Term;
+        pMem->n = displayComment(pOp, zP4, pMem->z, 500);
+        pMem->enc = SQLITE_UTF8;
+#else
+        pMem->flags = MEM_Null;                       /* Comment */
+#endif
+      }
+
+      p->nResColumn = 8 - 4*(p->explain-1);
+      p->pResultSet = &p->aMem[1];
+      p->rc = SQLITE_OK;
+      rc = SQLITE_ROW;
+    }
   }
   return rc;
 }

From 7853002c712e439537276d6e820b8b9f56b505c8 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 18 Apr 2017 09:04:48 +0000
Subject: [PATCH 034/488] Fix sqlite3_expert handling of triggers on views.

FossilOrigin-Name: ff4976da667872614331d88e68fb67d347874f164a1c7950dd738c7c2320b954
---
 ext/expert/expert.c        |  4 ++--
 ext/expert/sqlite3expert.c | 14 +++++++++++---
 manifest                   | 14 +++++++-------
 manifest.uuid              |  2 +-
 4 files changed, 21 insertions(+), 13 deletions(-)

diff --git a/ext/expert/expert.c b/ext/expert/expert.c
index 19b52c867d..3aafde1d3f 100644
--- a/ext/expert/expert.c
+++ b/ext/expert/expert.c
@@ -135,8 +135,8 @@ int main(int argc, char **argv){
       }
       fprintf(stdout, "%s\n%s\n", zIdx, zEQP);
     }
-  }else if( zErr ){
-    fprintf(stderr, "Error: %s\n", zErr);
+  }else{
+    fprintf(stderr, "Error: %s\n", zErr ? zErr : "?");
   }
 
   sqlite3_expert_destroy(p);
diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index 8316a18ae3..093e4438f5 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -1136,6 +1136,9 @@ static int idxProcessOneTrigger(
     sqlite3_stmt *pX = 0;
     rc = sqlite3_prepare_v2(p->dbv, zWrite, -1, &pX, 0);
     idxFinalize(&rc, pX);
+    if( rc!=SQLITE_OK ){
+      idxDatabaseError(p->dbv, pzErr);
+    }
   }
   sqlite3_free(zWrite);
 
@@ -1174,15 +1177,20 @@ static int idxCreateVtabSchema(sqlite3expert *p, char **pzErrmsg){
   **   2) Create the equivalent virtual table in dbv.
   */
   rc = idxPrepareStmt(p->db, &pSchema, pzErrmsg,
-      "SELECT type, name, sql FROM sqlite_master "
-      "WHERE type IN ('table','view') ORDER BY 1"
+      "SELECT type, name, sql, 1 FROM sqlite_master "
+      "WHERE type IN ('table','view') AND name NOT LIKE 'sqlite_%%' "
+      " UNION ALL "
+      "SELECT type, name, sql, 2 FROM sqlite_master "
+      "WHERE type = 'trigger'"
+      "  AND tbl_name IN(SELECT name FROM sqlite_master WHERE type = 'view') "
+      "ORDER BY 4, 1"
   );
   while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSchema) ){
     const char *zType = (const char*)sqlite3_column_text(pSchema, 0);
     const char *zName = (const char*)sqlite3_column_text(pSchema, 1);
     const char *zSql = (const char*)sqlite3_column_text(pSchema, 2);
 
-    if( zType[0]=='v' ){
+    if( zType[0]=='v' || zType[1]=='r' ){
       rc = sqlite3_exec(p->dbv, zSql, 0, 0, pzErrmsg);
     }else{
       IdxTable *pTab;
diff --git a/manifest b/manifest
index 7bc6efbb2d..cde195ed9d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\ssupport\sfor\sanalyzing\strigger\sprograms\sto\sthe\ssqlite3_expert\scode.
-D 2017-04-17T17:03:08.723
+C Fix\ssqlite3_expert\shandling\sof\striggers\son\sviews.
+D 2017-04-18T09:04:48.670
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6
@@ -41,9 +41,9 @@ F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md 9f15075ec5ad772808eff55ef044c31140fd1146aa0a3c47eafd155e71851b01
-F ext/expert/expert.c 6349cf8d26c847f5f0fa7e25772b614c67f60f3c850dca0d75d55eb27cf3f69b
+F ext/expert/expert.c 22d2dd096d479049bc332506fc8c0294bf53b7ebfe60af99635d8c87839bb40b
 F ext/expert/expert1.test 6a50a1538dc9e4ff360fb117298aa3b085beed030cbe15dd36803da1a9f70702
-F ext/expert/sqlite3expert.c f9f41caf6d941b52ef76043f3191ab05294f4d286425e34923a7d74876014f0b
+F ext/expert/sqlite3expert.c 8befe20906cf8f4928c754763cbf0a81f42fc1ee242fe8e5869c3bb4adba7300
 F ext/expert/sqlite3expert.h b1c9eedeb647fd734c4206ae6851635284cfbfa5fb688eff74c3265c9f949b4d
 F ext/expert/test_expert.c bad0611732d07180d586bd589cbb7713dc3ab0338c52bff29680eb2007678c05
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
@@ -1579,7 +1579,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2e6308798ae2db30564deb35ba3896597448edabbcac6efc4ff084552e42de30
-R 3935ba65365bf244da478b55268b3a1f
+P 159e8022a9d6701532b8b60e0c41154bc434c1bbdb107c8c97a78fb1140fa745
+R 52d2320865ff7653f66201ecd375f3d6
 U dan
-Z 871de5d26699f20ec603c0a41beb3b11
+Z e02fbf739534a0fe797908906bbc0c24
diff --git a/manifest.uuid b/manifest.uuid
index 8158ffa2b0..6969132da7 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-159e8022a9d6701532b8b60e0c41154bc434c1bbdb107c8c97a78fb1140fa745
\ No newline at end of file
+ff4976da667872614331d88e68fb67d347874f164a1c7950dd738c7c2320b954
\ No newline at end of file

From a6ed5a4f397a97ca92d392fb5481da0f2058c41f Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 18 Apr 2017 20:10:16 +0000
Subject: [PATCH 035/488] Have sqlite3_expert_analyze() populate the
 sqlite_stat1 table before running queries through the planner for the second
 time.

FossilOrigin-Name: a157fcfde5afc27ae38e7cf4669fcc8e60e23d9d301ffe2e541dd69f895b493b
---
 ext/expert/expert1.test    |  79 ++++++++++
 ext/expert/sqlite3expert.c | 306 +++++++++++++++++++++++++++++++++++--
 ext/expert/test_expert.c   |   3 +-
 manifest                   |  16 +-
 manifest.uuid              |   2 +-
 5 files changed, 387 insertions(+), 19 deletions(-)

diff --git a/ext/expert/expert1.test b/ext/expert/expert1.test
index a817865229..7995327557 100644
--- a/ext/expert/expert1.test
+++ b/ext/expert/expert1.test
@@ -275,7 +275,86 @@ do_setup_rec_test $tn.14 {
   0|0|0|SEARCH TABLE t10 USING INDEX t10_idx_00000062 (b=?)
 }
 
+do_setup_rec_test $tn.15 {
+  CREATE TABLE t1(a, b);
+  CREATE TABLE t2(c, d);
+
+  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
+  INSERT INTO t1 SELECT (i-1)/50, (i-1)/20 FROM s;
+
+  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
+  INSERT INTO t2 SELECT (i-1)/20, (i-1)/5 FROM s;
+} {
+  SELECT * FROM t2, t1 WHERE b=? AND d=? AND t2.rowid=t1.rowid
+} {
+  CREATE INDEX t2_idx_00000064 ON t2(d);
+  0|0|0|SEARCH TABLE t2 USING INDEX t2_idx_00000064 (d=?) 
+  0|1|1|SEARCH TABLE t1 USING INTEGER PRIMARY KEY (rowid=?)
 }
 
+}
+
+proc do_candidates_test {tn sql res} {
+  set res [squish [string trim $res]]
+
+  set expert [sqlite3_expert_new db]
+  $expert sql $sql
+  $expert analyze
+
+  set candidates [squish [string trim [$expert report 0 candidates]]]
+  $expert destroy
+
+  uplevel [list do_test $tn [list set {} $candidates] $res]
+}
+
+
+reset_db
+do_execsql_test 3.0 {
+  CREATE TABLE t1(a, b);
+  CREATE TABLE t2(c, d);
+
+  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
+  INSERT INTO t1 SELECT (i-1)/50, (i-1)/20 FROM s;
+
+  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
+  INSERT INTO t2 SELECT (i-1)/20, (i-1)/5 FROM s;
+}
+do_candidates_test 3.1 {
+  SELECT * FROM t1,t2 WHERE (b=? OR a=?) AND (c=? OR d=?)
+} {
+  CREATE INDEX t1_idx_00000062 ON t1(b); -- stat1: 100 20 
+  CREATE INDEX t1_idx_00000061 ON t1(a); -- stat1: 100 50 
+  CREATE INDEX t2_idx_00000063 ON t2(c); -- stat1: 100 20 
+  CREATE INDEX t2_idx_00000064 ON t2(d); -- stat1: 100 5
+}
+
+do_candidates_test 3.2 {
+  SELECT * FROM t1,t2 WHERE a=? AND b=? AND c=? AND d=?
+} {
+  CREATE INDEX t1_idx_000123a7 ON t1(a, b); -- stat1: 100 50 17
+  CREATE INDEX t2_idx_0001295b ON t2(c, d); -- stat1: 100 20 5
+}
+
+do_execsql_test 3.2 {
+  CREATE INDEX t1_idx_00000061 ON t1(a); -- stat1: 100 50 
+  CREATE INDEX t1_idx_00000062 ON t1(b); -- stat1: 100 20 
+  CREATE INDEX t1_idx_000123a7 ON t1(a, b); -- stat1: 100 50 16
+
+  CREATE INDEX t2_idx_00000063 ON t2(c); -- stat1: 100 20 
+  CREATE INDEX t2_idx_00000064 ON t2(d); -- stat1: 100 5
+  CREATE INDEX t2_idx_0001295b ON t2(c, d); -- stat1: 100 20 5
+
+  ANALYZE;
+  SELECT * FROM sqlite_stat1 ORDER BY 1, 2;
+} {
+  t1 t1_idx_00000061 {100 50} 
+  t1 t1_idx_00000062 {100 20}
+  t1 t1_idx_000123a7 {100 50 17}
+  t2 t2_idx_00000063 {100 20} 
+  t2 t2_idx_00000064 {100 5} 
+  t2 t2_idx_0001295b {100 20 5}
+}
+
+
 finish_test
 
diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index 093e4438f5..bba5190d5b 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -114,6 +114,7 @@ typedef struct IdxHash IdxHash;
 struct IdxHashEntry {
   char *zKey;                     /* nul-terminated key */
   char *zVal;                     /* nul-terminated value string */
+  char *zVal2;                    /* nul-terminated value string 2 */
   IdxHashEntry *pHashNext;        /* Next entry in same hash bucket */
   IdxHashEntry *pNext;            /* Next entry in hash */
 };
@@ -175,6 +176,7 @@ static void idxHashClear(IdxHash *pHash){
     IdxHashEntry *pNext;
     for(pEntry=pHash->aHash[i]; pEntry; pEntry=pNext){
       pNext = pEntry->pHashNext;
+      sqlite3_free(pEntry->zVal2);
       sqlite3_free(pEntry);
     }
   }
@@ -232,13 +234,7 @@ static int idxHashAdd(
   return 0;
 }
 
-/*
-** If the hash table contains an entry with a key equal to the string
-** passed as the final two arguments to this function, return a pointer
-** to the payload string. Otherwise, if zKey/nKey is not present in the
-** hash table, return NULL.
-*/
-static const char *idxHashSearch(IdxHash *pHash, const char *zKey, int nKey){
+static IdxHashEntry *idxHashFind(IdxHash *pHash, const char *zKey, int nKey){
   int iHash;
   IdxHashEntry *pEntry;
   if( nKey<0 ) nKey = strlen(zKey);
@@ -246,12 +242,24 @@ static const char *idxHashSearch(IdxHash *pHash, const char *zKey, int nKey){
   assert( iHash>=0 );
   for(pEntry=pHash->aHash[iHash]; pEntry; pEntry=pEntry->pHashNext){
     if( strlen(pEntry->zKey)==nKey && 0==memcmp(pEntry->zKey, zKey, nKey) ){
-      return pEntry->zVal;
+      return pEntry;
     }
   }
   return 0;
 }
 
+/*
+** If the hash table contains an entry with a key equal to the string
+** passed as the final two arguments to this function, return a pointer
+** to the payload string. Otherwise, if zKey/nKey is not present in the
+** hash table, return NULL.
+*/
+static const char *idxHashSearch(IdxHash *pHash, const char *zKey, int nKey){
+  IdxHashEntry *pEntry = idxHashFind(pHash, zKey, nKey);
+  if( pEntry ) return pEntry->zVal;
+  return 0;
+}
+
 /*
 ** Allocate and return a new IdxConstraint object. Set the IdxConstraint.zColl
 ** variable to point to a copy of nul-terminated string zColl.
@@ -1227,6 +1235,277 @@ static int idxCreateVtabSchema(sqlite3expert *p, char **pzErrmsg){
   return rc;
 }
 
+struct IdxRemCtx {
+  int nSlot;
+  struct IdxRemSlot {
+    int eType;                    /* SQLITE_NULL, INTEGER, REAL, TEXT, BLOB */
+    i64 iVal;                     /* SQLITE_INTEGER value */
+    double rVal;                  /* SQLITE_FLOAT value */
+    int nByte;                    /* Bytes of space allocated at z */
+    int n;                        /* Size of buffer z */
+    char *z;                      /* SQLITE_TEXT/BLOB value */
+  } aSlot[1];
+};
+
+/*
+** Implementation of scalar function rem().
+*/
+static void idxRemFunc(
+  sqlite3_context *pCtx,
+  int argc,
+  sqlite3_value **argv
+){
+  struct IdxRemCtx *p = (struct IdxRemCtx*)sqlite3_user_data(pCtx);
+  struct IdxRemSlot *pSlot;
+  int iSlot;
+  assert( argc==2 );
+
+  iSlot = sqlite3_value_int(argv[0]);
+  assert( iSlot<=p->nSlot );
+  pSlot = &p->aSlot[iSlot];
+
+  switch( pSlot->eType ){
+    case SQLITE_NULL:
+      /* no-op */
+      break;
+
+    case SQLITE_INTEGER:
+      sqlite3_result_int64(pCtx, pSlot->iVal);
+      break;
+
+    case SQLITE_FLOAT:
+      sqlite3_result_double(pCtx, pSlot->rVal);
+      break;
+
+    case SQLITE_BLOB:
+      sqlite3_result_blob(pCtx, pSlot->z, pSlot->n, SQLITE_TRANSIENT);
+      break;
+
+    case SQLITE_TEXT:
+      sqlite3_result_text(pCtx, pSlot->z, pSlot->n, SQLITE_TRANSIENT);
+      break;
+  }
+
+  pSlot->eType = sqlite3_value_type(argv[1]);
+  switch( pSlot->eType ){
+    case SQLITE_NULL:
+      /* no-op */
+      break;
+
+    case SQLITE_INTEGER:
+      pSlot->iVal = sqlite3_value_int64(argv[1]);
+      break;
+
+    case SQLITE_FLOAT:
+      pSlot->rVal = sqlite3_value_double(argv[1]);
+      break;
+
+    case SQLITE_BLOB:
+    case SQLITE_TEXT: {
+      int nByte = sqlite3_value_bytes(argv[1]);
+      if( nByte>pSlot->nByte ){
+        char *zNew = (char*)sqlite3_realloc(pSlot->z, nByte*2);
+        if( zNew==0 ){
+          sqlite3_result_error_nomem(pCtx);
+          return;
+        }
+        pSlot->nByte = nByte*2;
+        pSlot->z = zNew;
+      }
+      pSlot->n = nByte;
+      if( pSlot->eType==SQLITE_BLOB ){
+        memcpy(pSlot->z, sqlite3_value_blob(argv[1]), nByte);
+      }else{
+        memcpy(pSlot->z, sqlite3_value_text(argv[1]), nByte);
+      }
+      break;
+    }
+  }
+}
+
+static int idxLargestIndex(sqlite3 *db, int *pnMax, char **pzErr){
+  int rc = SQLITE_OK;
+  const char *zMax = 
+    "SELECT max(i.seqno) FROM "
+    "  sqlite_master AS s, "
+    "  pragma_index_list(s.name) AS l, "
+    "  pragma_index_info(l.name) AS i "
+    "WHERE s.type = 'table'";
+  sqlite3_stmt *pMax = 0;
+
+  *pnMax = 0;
+  rc = idxPrepareStmt(db, &pMax, pzErr, zMax);
+  if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pMax) ){
+    *pnMax = sqlite3_column_int(pMax, 0) + 1;
+  }
+  idxFinalize(&rc, pMax);
+
+  return rc;
+}
+
+static int idxPopulateOneStat1(
+  sqlite3expert *p,
+  sqlite3_stmt *pIndexXInfo,
+  sqlite3_stmt *pWriteStat,
+  const char *zTab,
+  const char *zIdx,
+  char **pzErr
+){
+  char *zCols = 0;
+  char *zOrder = 0;
+  char *zQuery = 0;
+  int nCol = 0;
+  int i;
+  sqlite3_stmt *pQuery = 0;
+  int *aStat = 0;
+  int rc = SQLITE_OK;
+
+  /* Formulate the query text */
+  sqlite3_bind_text(pIndexXInfo, 1, zIdx, -1, SQLITE_STATIC);
+  while( SQLITE_OK==rc && SQLITE_ROW==sqlite3_step(pIndexXInfo) ){
+    const char *zComma = zCols==0 ? "" : ", ";
+    const char *zName = (const char*)sqlite3_column_text(pIndexXInfo, 0);
+    const char *zColl = (const char*)sqlite3_column_text(pIndexXInfo, 1);
+    zCols = idxAppendText(&rc, zCols, 
+        "%sx.%Q IS rem(%d, x.%Q) COLLATE %s", zComma, zName, nCol, zName, zColl
+    );
+    zOrder = idxAppendText(&rc, zOrder, "%s%d", zComma, ++nCol);
+  }
+  if( rc==SQLITE_OK ){
+    zQuery = sqlite3_mprintf(
+        "SELECT %s FROM %Q x ORDER BY %s", zCols, zTab, zOrder
+    );
+  }
+  sqlite3_free(zCols);
+  sqlite3_free(zOrder);
+
+  /* Formulate the query text */
+  if( rc==SQLITE_OK ){
+    rc = idxPrepareStmt(p->db, &pQuery, pzErr, zQuery);
+  }
+  sqlite3_free(zQuery);
+
+  if( rc==SQLITE_OK ){
+    aStat = (int*)idxMalloc(&rc, sizeof(int)*(nCol+1));
+  }
+  if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pQuery) ){
+    IdxHashEntry *pEntry;
+    char *zStat = 0;
+    for(i=0; i<=nCol; i++) aStat[i] = 1;
+    while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pQuery) ){
+      aStat[0]++;
+      for(i=0; i<nCol; i++){
+        if( sqlite3_column_int(pQuery, i)==0 ) break;
+      }
+      for(/*no-op*/; i<nCol; i++){
+        aStat[i+1]++;
+      }
+    }
+
+    if( rc==SQLITE_OK ){
+      int s0 = aStat[0];
+      zStat = sqlite3_mprintf("%d", s0);
+      if( zStat==0 ) rc = SQLITE_NOMEM;
+      for(i=1; rc==SQLITE_OK && i<=nCol; i++){
+        zStat = idxAppendText(&rc, zStat, " %d", (s0+aStat[i]/2) / aStat[i]);
+      }
+    }
+
+    if( rc==SQLITE_OK ){
+      sqlite3_bind_text(pWriteStat, 1, zTab, -1, SQLITE_STATIC);
+      sqlite3_bind_text(pWriteStat, 2, zIdx, -1, SQLITE_STATIC);
+      sqlite3_bind_text(pWriteStat, 3, zStat, -1, SQLITE_STATIC);
+      sqlite3_step(pWriteStat);
+      rc = sqlite3_reset(pWriteStat);
+    }
+
+    pEntry = idxHashFind(&p->hIdx, zIdx, strlen(zIdx));
+    if( pEntry ){
+      assert( pEntry->zVal2==0 );
+      pEntry->zVal2 = zStat;
+    }else{
+      sqlite3_free(zStat);
+    }
+  }
+  sqlite3_free(aStat);
+  idxFinalize(&rc, pQuery);
+
+  return rc;
+}
+
+/*
+** This function is called as part of sqlite3_expert_analyze(). Candidate
+** indexes have already been created in database sqlite3expert.dbm, this
+** function populates sqlite_stat1 table in the same database.
+**
+** The stat1 data is generated by querying the 
+*/
+static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
+  int rc = SQLITE_OK;
+  int nMax =0;
+  struct IdxRemCtx *pCtx = 0;
+  int i;
+  sqlite3_stmt *pAllIndex = 0;
+  sqlite3_stmt *pIndexXInfo = 0;
+  sqlite3_stmt *pWrite = 0;
+
+  const char *zAllIndex = 
+    "SELECT s.name, l.name FROM "
+    "  sqlite_master AS s, "
+    "  pragma_index_list(s.name) AS l "
+    "WHERE s.type = 'table'";
+  const char *zIndexXInfo = 
+    "SELECT name, coll FROM pragma_index_xinfo(?) WHERE key";
+  const char *zWrite = "INSERT INTO sqlite_stat1 VALUES(?, ?, ?)";
+
+  rc = idxLargestIndex(p->dbm, &nMax, pzErr);
+  if( nMax<=0 || rc!=SQLITE_OK ) return rc;
+
+  rc = sqlite3_exec(p->dbm, "ANALYZE; PRAGMA writable_schema=1", 0, 0, 0);
+
+  if( rc==SQLITE_OK ){
+    int nByte = sizeof(struct IdxRemCtx) + (sizeof(struct IdxRemSlot) * nMax);
+    pCtx = (struct IdxRemCtx*)idxMalloc(&rc, nByte);
+  }
+
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_create_function(
+        p->db, "rem", 2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0
+    );
+  }
+
+  if( rc==SQLITE_OK ){
+    pCtx->nSlot = nMax+1;
+    rc = idxPrepareStmt(p->dbm, &pAllIndex, pzErr, zAllIndex);
+  }
+  if( rc==SQLITE_OK ){
+    rc = idxPrepareStmt(p->dbm, &pIndexXInfo, pzErr, zIndexXInfo);
+  }
+  if( rc==SQLITE_OK ){
+    rc = idxPrepareStmt(p->dbm, &pWrite, pzErr, zWrite);
+  }
+
+  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pAllIndex) ){
+    const char *zTab = (const char*)sqlite3_column_text(pAllIndex, 0);
+    const char *zIdx = (const char*)sqlite3_column_text(pAllIndex, 1);
+    rc = idxPopulateOneStat1(p, pIndexXInfo, pWrite, zTab, zIdx, pzErr);
+  }
+
+  idxFinalize(&rc, pAllIndex);
+  idxFinalize(&rc, pIndexXInfo);
+  idxFinalize(&rc, pWrite);
+
+  for(i=0; i<pCtx->nSlot; i++){
+    sqlite3_free(pCtx->aSlot[i].z);
+  }
+  sqlite3_free(pCtx);
+
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_exec(p->dbm, "ANALYZE sqlite_master", 0, 0, 0);
+  }
+  return rc;
+}
+
 /*
 ** Allocate a new sqlite3expert object.
 */
@@ -1338,6 +1617,7 @@ int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr){
   int rc;
   IdxHashEntry *pEntry;
 
+  /* Do trigger processing to collect any extra IdxScan structures */
   rc = idxProcessTriggers(p, pzErr);
 
   /* Create candidate indexes within the in-memory database file */
@@ -1345,9 +1625,17 @@ int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr){
     rc = idxCreateCandidates(p, pzErr);
   }
 
+  /* Generate the stat1 data */
+  if( rc==SQLITE_OK ){
+    rc = idxPopulateStat1(p, pzErr);
+  }
+
   /* Formulate the EXPERT_REPORT_CANDIDATES text */
   for(pEntry=p->hIdx.pFirst; pEntry; pEntry=pEntry->pNext){
-    p->zCandidates = idxAppendText(&rc, p->zCandidates, "%s;\n", pEntry->zVal);
+    p->zCandidates = idxAppendText(&rc, p->zCandidates, 
+        "%s;%s%s\n", pEntry->zVal, 
+        pEntry->zVal2 ? " -- stat1: " : "", pEntry->zVal2
+    );
   }
 
   /* Figure out which of the candidate indexes are preferred by the query
diff --git a/ext/expert/test_expert.c b/ext/expert/test_expert.c
index a37887e587..587f95c91b 100644
--- a/ext/expert/test_expert.c
+++ b/ext/expert/test_expert.c
@@ -106,7 +106,7 @@ static int SQLITE_TCLAPI testExpertCmd(
 
     case 3: {      /* report */
       const char *aEnum[] = {
-        "sql", "indexes", "plan", 0
+        "sql", "indexes", "plan", "candidates", 0
       };
       int iEnum;
       int iStmt;
@@ -121,6 +121,7 @@ static int SQLITE_TCLAPI testExpertCmd(
       assert( EXPERT_REPORT_SQL==1 );
       assert( EXPERT_REPORT_INDEXES==2 );
       assert( EXPERT_REPORT_PLAN==3 );
+      assert( EXPERT_REPORT_CANDIDATES==4 );
       zReport = sqlite3_expert_report(pExpert, iStmt, 1+iEnum);
       Tcl_SetObjResult(interp, Tcl_NewStringObj(zReport, -1));
       break;
diff --git a/manifest b/manifest
index cde195ed9d..5fa07a1f45 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\ssqlite3_expert\shandling\sof\striggers\son\sviews.
-D 2017-04-18T09:04:48.670
+C Have\ssqlite3_expert_analyze()\spopulate\sthe\ssqlite_stat1\stable\sbefore\srunning\nqueries\sthrough\sthe\splanner\sfor\sthe\ssecond\stime.
+D 2017-04-18T20:10:16.786
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6
@@ -42,10 +42,10 @@ F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md 9f15075ec5ad772808eff55ef044c31140fd1146aa0a3c47eafd155e71851b01
 F ext/expert/expert.c 22d2dd096d479049bc332506fc8c0294bf53b7ebfe60af99635d8c87839bb40b
-F ext/expert/expert1.test 6a50a1538dc9e4ff360fb117298aa3b085beed030cbe15dd36803da1a9f70702
-F ext/expert/sqlite3expert.c 8befe20906cf8f4928c754763cbf0a81f42fc1ee242fe8e5869c3bb4adba7300
+F ext/expert/expert1.test 1033e43071b69dc2f4e88fbf03fc7f18846c9865cac14f28c80f581437f09acb
+F ext/expert/sqlite3expert.c 713388c6c440c6759a1e0898c7936a014dc9791237e62780412229e4a79b0035
 F ext/expert/sqlite3expert.h b1c9eedeb647fd734c4206ae6851635284cfbfa5fb688eff74c3265c9f949b4d
-F ext/expert/test_expert.c bad0611732d07180d586bd589cbb7713dc3ab0338c52bff29680eb2007678c05
+F ext/expert/test_expert.c b01a5115f9444a9b416582c985138f5dfdb279848ce8b7452be383530be27f01
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
 F ext/fts1/ft_hash.c 3927bd880e65329bdc6f506555b228b28924921b
 F ext/fts1/ft_hash.h 06df7bba40dadd19597aa400a875dbc2fed705ea
@@ -1579,7 +1579,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 159e8022a9d6701532b8b60e0c41154bc434c1bbdb107c8c97a78fb1140fa745
-R 52d2320865ff7653f66201ecd375f3d6
+P ff4976da667872614331d88e68fb67d347874f164a1c7950dd738c7c2320b954
+R d621eb0093377628a3811751fb5daa84
 U dan
-Z e02fbf739534a0fe797908906bbc0c24
+Z 02b917b4563ed84ab0581f0a345c83b3
diff --git a/manifest.uuid b/manifest.uuid
index 6969132da7..68ccc22188 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ff4976da667872614331d88e68fb67d347874f164a1c7950dd738c7c2320b954
\ No newline at end of file
+a157fcfde5afc27ae38e7cf4669fcc8e60e23d9d301ffe2e541dd69f895b493b
\ No newline at end of file

From e53b4f977484247b9b893d191252e6ce5bfd8e87 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 20 Apr 2017 09:54:04 +0000
Subject: [PATCH 036/488] Add an option to generate stat1 data based on a
 subset of the user database table contents to sqlite3_expert.

FossilOrigin-Name: c69c3e21db6e141f7e24226c6432f2ed31fe5f177bd23781915871f8600ee56a
---
 ext/expert/expert.c        |  10 ++-
 ext/expert/sqlite3expert.c | 133 ++++++++++++++++++++++++++++++++++---
 ext/expert/sqlite3expert.h |  33 +++++++++
 manifest                   |  16 ++---
 manifest.uuid              |   2 +-
 5 files changed, 175 insertions(+), 19 deletions(-)

diff --git a/ext/expert/expert.c b/ext/expert/expert.c
index 3aafde1d3f..baa7be940f 100644
--- a/ext/expert/expert.c
+++ b/ext/expert/expert.c
@@ -36,6 +36,7 @@ static void usage(char **argv){
   fprintf(stderr, "  -sql SQL   (analyze SQL statements passed as argument)\n");
   fprintf(stderr, "  -file FILE (read SQL statements from file FILE)\n");
   fprintf(stderr, "  -verbose LEVEL (integer verbosity level. default 1)\n");
+  fprintf(stderr, "  -sample PERCENT (percent of db to sample. default 100)\n");
   exit(-1);
 }
 
@@ -97,11 +98,18 @@ int main(int argc, char **argv){
         rc = readSqlFromFile(p, argv[i], &zErr);
       }
 
-      else if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-sql", nArg) ){
+      else if( nArg>=3 && 0==sqlite3_strnicmp(zArg, "-sql", nArg) ){
         if( ++i==(argc-1) ) option_requires_argument("-sql");
         rc = sqlite3_expert_sql(p, argv[i], &zErr);
       }
 
+      else if( nArg>=3 && 0==sqlite3_strnicmp(zArg, "-sample", nArg) ){
+        int iSample;
+        if( ++i==(argc-1) ) option_requires_argument("-sample");
+        iSample = option_integer_arg(argv[i]);
+        sqlite3_expert_config(p, EXPERT_CONFIG_SAMPLE, iSample);
+      }
+
       else if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-verbose", nArg) ){
         if( ++i==(argc-1) ) option_requires_argument("-verbose");
         iVerbose = option_integer_arg(argv[i]);
diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index bba5190d5b..03963e1f7e 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -28,6 +28,9 @@ typedef struct IdxStatement IdxStatement;
 typedef struct IdxTable IdxTable;
 typedef struct IdxWrite IdxWrite;
 
+#define UNIQUE_TABLE_NAME "t592690916721053953805701627921227776"
+
+
 /*
 ** A single constraint. Equivalent to either "col = ?" or "col < ?" (or
 ** any other type of single-ended range constraint on a column).
@@ -127,6 +130,7 @@ struct IdxHash {
 ** sqlite3expert object.
 */
 struct sqlite3expert {
+  int iSample;                    /* Percentage of tables to sample for stat1 */
   sqlite3 *db;                    /* User database */
   sqlite3 *dbm;                   /* In-memory db for this analysis */
   sqlite3 *dbv;                   /* Vtab schema for this analysis */
@@ -1080,8 +1084,8 @@ static int idxProcessOneTrigger(
   IdxWrite *pWrite, 
   char **pzErr
 ){
-  static const char *zInt = "t592690916721053953805701627921227776";
-  static const char *zDrop = "DROP TABLE t592690916721053953805701627921227776";
+  static const char *zInt = UNIQUE_TABLE_NAME;
+  static const char *zDrop = "DROP TABLE " UNIQUE_TABLE_NAME;
   IdxTable *pTab = pWrite->pTab;
   const char *zTab = pTab->zName;
   const char *zSql = 
@@ -1235,6 +1239,38 @@ static int idxCreateVtabSchema(sqlite3expert *p, char **pzErrmsg){
   return rc;
 }
 
+struct IdxSampleCtx {
+  int iTarget;
+  double target;                  /* Target nRet/nRow value */
+  double nRow;                    /* Number of rows seen */
+  double nRet;                    /* Number of rows returned */
+};
+
+static void idxSampleFunc(
+  sqlite3_context *pCtx,
+  int argc,
+  sqlite3_value **argv
+){
+  struct IdxSampleCtx *p = (struct IdxSampleCtx*)sqlite3_user_data(pCtx);
+  int bRet;
+
+  assert( argc==0 );
+  if( p->nRow==0.0 ){
+    bRet = 1;
+  }else{
+    bRet = (p->nRet / p->nRow) <= p->target;
+    if( bRet==0 ){
+      unsigned short rnd;
+      sqlite3_randomness(2, (void*)&rnd);
+      bRet = ((int)rnd % 100) <= p->iTarget;
+    }
+  }
+
+  sqlite3_result_int(pCtx, bRet);
+  p->nRow += 1.0;
+  p->nRet += (double)bRet;
+}
+
 struct IdxRemCtx {
   int nSlot;
   struct IdxRemSlot {
@@ -1360,6 +1396,8 @@ static int idxPopulateOneStat1(
   int *aStat = 0;
   int rc = SQLITE_OK;
 
+  assert( p->iSample>0 );
+
   /* Formulate the query text */
   sqlite3_bind_text(pIndexXInfo, 1, zIdx, -1, SQLITE_STATIC);
   while( SQLITE_OK==rc && SQLITE_ROW==sqlite3_step(pIndexXInfo) ){
@@ -1372,9 +1410,15 @@ static int idxPopulateOneStat1(
     zOrder = idxAppendText(&rc, zOrder, "%s%d", zComma, ++nCol);
   }
   if( rc==SQLITE_OK ){
-    zQuery = sqlite3_mprintf(
-        "SELECT %s FROM %Q x ORDER BY %s", zCols, zTab, zOrder
-    );
+    if( p->iSample==100 ){
+      zQuery = sqlite3_mprintf(
+          "SELECT %s FROM %Q x ORDER BY %s", zCols, zTab, zOrder
+      );
+    }else{
+      zQuery = sqlite3_mprintf(
+          "SELECT %s FROM temp."UNIQUE_TABLE_NAME" x ORDER BY %s", zCols, zOrder
+      );
+    }
   }
   sqlite3_free(zCols);
   sqlite3_free(zOrder);
@@ -1433,6 +1477,25 @@ static int idxPopulateOneStat1(
   return rc;
 }
 
+static int idxBuildSampleTable(sqlite3expert *p, const char *zTab){
+  int rc;
+  char *zSql;
+
+  rc = sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0);
+  if( rc!=SQLITE_OK ) return rc;
+
+  zSql = sqlite3_mprintf(
+      "CREATE TABLE temp." UNIQUE_TABLE_NAME 
+      "  AS SELECT * FROM %Q WHERE sample()"
+      , zTab
+  );
+  if( zSql==0 ) return SQLITE_NOMEM;
+  rc = sqlite3_exec(p->db, zSql, 0, 0, 0);
+  sqlite3_free(zSql);
+
+  return rc;
+}
+
 /*
 ** This function is called as part of sqlite3_expert_analyze(). Candidate
 ** indexes have already been created in database sqlite3expert.dbm, this
@@ -1444,13 +1507,15 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
   int rc = SQLITE_OK;
   int nMax =0;
   struct IdxRemCtx *pCtx = 0;
+  struct IdxSampleCtx samplectx; 
   int i;
+  i64 iPrev = -100000;
   sqlite3_stmt *pAllIndex = 0;
   sqlite3_stmt *pIndexXInfo = 0;
   sqlite3_stmt *pWrite = 0;
 
-  const char *zAllIndex = 
-    "SELECT s.name, l.name FROM "
+  const char *zAllIndex =
+    "SELECT s.rowid, s.name, l.name FROM "
     "  sqlite_master AS s, "
     "  pragma_index_list(s.name) AS l "
     "WHERE s.type = 'table'";
@@ -1458,6 +1523,9 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
     "SELECT name, coll FROM pragma_index_xinfo(?) WHERE key";
   const char *zWrite = "INSERT INTO sqlite_stat1 VALUES(?, ?, ?)";
 
+  /* If iSample==0, no sqlite_stat1 data is required. */
+  if( p->iSample==0 ) return SQLITE_OK;
+
   rc = idxLargestIndex(p->dbm, &nMax, pzErr);
   if( nMax<=0 || rc!=SQLITE_OK ) return rc;
 
@@ -1473,6 +1541,11 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
         p->db, "rem", 2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0
     );
   }
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_create_function(
+        p->db, "sample", 0, SQLITE_UTF8, (void*)&samplectx, idxSampleFunc, 0, 0
+    );
+  }
 
   if( rc==SQLITE_OK ){
     pCtx->nSlot = nMax+1;
@@ -1486,9 +1559,24 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
   }
 
   while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pAllIndex) ){
-    const char *zTab = (const char*)sqlite3_column_text(pAllIndex, 0);
-    const char *zIdx = (const char*)sqlite3_column_text(pAllIndex, 1);
+    i64 iRowid = sqlite3_column_int64(pAllIndex, 0);
+    const char *zTab = (const char*)sqlite3_column_text(pAllIndex, 1);
+    const char *zIdx = (const char*)sqlite3_column_text(pAllIndex, 2);
+    if( p->iSample<100 && iPrev!=iRowid ){
+      samplectx.target = (double)p->iSample / 100.0;
+      samplectx.iTarget = p->iSample;
+      samplectx.nRow = 0.0;
+      samplectx.nRet = 0.0;
+      rc = idxBuildSampleTable(p, zTab);
+      if( rc!=SQLITE_OK ) break;
+    }
     rc = idxPopulateOneStat1(p, pIndexXInfo, pWrite, zTab, zIdx, pzErr);
+    iPrev = iRowid;
+  }
+  if( p->iSample<100 ){
+    rc = sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp." UNIQUE_TABLE_NAME,
+        0,0,0
+    );
   }
 
   idxFinalize(&rc, pAllIndex);
@@ -1503,6 +1591,8 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
   if( rc==SQLITE_OK ){
     rc = sqlite3_exec(p->dbm, "ANALYZE sqlite_master", 0, 0, 0);
   }
+
+  sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0);
   return rc;
 }
 
@@ -1523,6 +1613,7 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){
   */
   if( rc==SQLITE_OK ){
     pNew->db = db;
+    pNew->iSample = 100;
     rc = sqlite3_open(":memory:", &pNew->dbv);
   }
   if( rc==SQLITE_OK ){
@@ -1565,6 +1656,30 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){
   return pNew;
 }
 
+/*
+** Configure an sqlite3expert object.
+*/
+int sqlite3_expert_config(sqlite3expert *p, int op, ...){
+  int rc = SQLITE_OK;
+  va_list ap;
+  va_start(ap, op);
+  switch( op ){
+    case EXPERT_CONFIG_SAMPLE: {
+      int iVal = va_arg(ap, int);
+      if( iVal<0 ) iVal = 0;
+      if( iVal>100 ) iVal = 100;
+      p->iSample = iVal;
+      break;
+    }
+    default:
+      rc = SQLITE_NOTFOUND;
+      break;
+  }
+
+  va_end(ap);
+  return rc;
+}
+
 /*
 ** Add an SQL statement to the analysis.
 */
diff --git a/ext/expert/sqlite3expert.h b/ext/expert/sqlite3expert.h
index 455f41d991..39135dc274 100644
--- a/ext/expert/sqlite3expert.h
+++ b/ext/expert/sqlite3expert.h
@@ -27,6 +27,38 @@ typedef struct sqlite3expert sqlite3expert;
 */
 sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErr);
 
+/*
+** Configure an sqlite3expert object.
+**
+** EXPERT_CONFIG_SAMPLE:
+**   By default, sqlite3_expert_analyze() generates sqlite_stat1 data for
+**   each candidate index. This involves scanning and sorting the entire
+**   contents of each user database table once for each candidate index
+**   associated with the table. For large databases, this can be 
+**   prohibitively slow. This option allows the sqlite3expert object to
+**   be configured so that sqlite_stat1 data is instead generated based on a
+**   subset of each table, or so that no sqlite_stat1 data is used at all.
+**
+**   A single integer argument is passed to this option. If the value is less
+**   than or equal to zero, then no sqlite_stat1 data is generated or used by
+**   the analysis - indexes are recommended based on the database schema only.
+**   Or, if the value is 100 or greater, complete sqlite_stat1 data is
+**   generated for each candidate index (this is the default). Finally, if the
+**   value falls between 0 and 100, then it represents the percentage of user
+**   table rows that should be considered when generating sqlite_stat1 data.
+**
+**   Examples:
+**
+**     // Do not generate any sqlite_stat1 data
+**     sqlite3_expert_config(pExpert, EXPERT_CONFIG_SAMPLE, 0);
+**
+**     // Generate sqlite_stat1 data based on 10% of the rows in each table.
+**     sqlite3_expert_config(pExpert, EXPERT_CONFIG_SAMPLE, 10);
+*/
+int sqlite3_expert_config(sqlite3expert *p, int op, ...);
+
+#define EXPERT_CONFIG_SAMPLE 1    /* int */
+
 /*
 ** Specify zero or more SQL statements to be included in the analysis.
 **
@@ -54,6 +86,7 @@ int sqlite3_expert_sql(
   char **pzErr                    /* OUT: Error message (if any) */
 );
 
+
 /*
 ** This function is called after the sqlite3expert object has been configured
 ** with all SQL statements using sqlite3_expert_sql() to actually perform
diff --git a/manifest b/manifest
index 5fa07a1f45..5ae9923e9e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Have\ssqlite3_expert_analyze()\spopulate\sthe\ssqlite_stat1\stable\sbefore\srunning\nqueries\sthrough\sthe\splanner\sfor\sthe\ssecond\stime.
-D 2017-04-18T20:10:16.786
+C Add\san\soption\sto\sgenerate\sstat1\sdata\sbased\son\sa\ssubset\sof\sthe\suser\sdatabase\ntable\scontents\sto\ssqlite3_expert.
+D 2017-04-20T09:54:04.700
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6
@@ -41,10 +41,10 @@ F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md 9f15075ec5ad772808eff55ef044c31140fd1146aa0a3c47eafd155e71851b01
-F ext/expert/expert.c 22d2dd096d479049bc332506fc8c0294bf53b7ebfe60af99635d8c87839bb40b
+F ext/expert/expert.c 33842ef151d84c5f8000f9c7b938998c6b999eaef7ce1f4eeb0df8ffe6739496
 F ext/expert/expert1.test 1033e43071b69dc2f4e88fbf03fc7f18846c9865cac14f28c80f581437f09acb
-F ext/expert/sqlite3expert.c 713388c6c440c6759a1e0898c7936a014dc9791237e62780412229e4a79b0035
-F ext/expert/sqlite3expert.h b1c9eedeb647fd734c4206ae6851635284cfbfa5fb688eff74c3265c9f949b4d
+F ext/expert/sqlite3expert.c af3b336f83bcd2a586f6119d4040ac36ccf45162c48e3780ed63ab119fd04fe1
+F ext/expert/sqlite3expert.h af6354f8ee5c9e025024e63fec3bd640a802afcc3099a44d804752cf0791d811
 F ext/expert/test_expert.c b01a5115f9444a9b416582c985138f5dfdb279848ce8b7452be383530be27f01
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
 F ext/fts1/ft_hash.c 3927bd880e65329bdc6f506555b228b28924921b
@@ -1579,7 +1579,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ff4976da667872614331d88e68fb67d347874f164a1c7950dd738c7c2320b954
-R d621eb0093377628a3811751fb5daa84
+P a157fcfde5afc27ae38e7cf4669fcc8e60e23d9d301ffe2e541dd69f895b493b
+R 2a75643c384b85cff6c5d9d5615ac2a2
 U dan
-Z 02b917b4563ed84ab0581f0a345c83b3
+Z 7876a034c88841c26664b8a83eb3817f
diff --git a/manifest.uuid b/manifest.uuid
index 68ccc22188..d816bcc63e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a157fcfde5afc27ae38e7cf4669fcc8e60e23d9d301ffe2e541dd69f895b493b
\ No newline at end of file
+c69c3e21db6e141f7e24226c6432f2ed31fe5f177bd23781915871f8600ee56a
\ No newline at end of file

From 2ed99def5b67ab87c79255368bf939afa5fdb22b Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 20 Apr 2017 16:08:33 +0000
Subject: [PATCH 037/488] Avoid creating a temp table in the user database in
 the sqlite3_expert code. Trouble is, this makes sampling for stat1 data much
 slower.

FossilOrigin-Name: c62e358243d96cb38a7ce2aa679fc640b62bf46080eab4bd5fc2acf5997d6cd5
---
 ext/expert/sqlite3expert.c | 270 +++++++++++++++++++++++++++----------
 manifest                   |  15 ++-
 manifest.uuid              |   2 +-
 3 files changed, 210 insertions(+), 77 deletions(-)

diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index 03963e1f7e..d578973101 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -281,6 +281,59 @@ static IdxConstraint *idxNewConstraint(int *pRc, const char *zColl){
   return pNew;
 }
 
+/*
+** An error associated with database handle db has just occurred. Pass
+** the error message to callback function xOut.
+*/
+static void idxDatabaseError(
+  sqlite3 *db,                    /* Database handle */
+  char **pzErrmsg                 /* Write error here */
+){
+  *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
+}
+
+/*
+** Prepare an SQL statement.
+*/
+static int idxPrepareStmt(
+  sqlite3 *db,                    /* Database handle to compile against */
+  sqlite3_stmt **ppStmt,          /* OUT: Compiled SQL statement */
+  char **pzErrmsg,                /* OUT: sqlite3_malloc()ed error message */
+  const char *zSql                /* SQL statement to compile */
+){
+  int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0);
+  if( rc!=SQLITE_OK ){
+    *ppStmt = 0;
+    idxDatabaseError(db, pzErrmsg);
+  }
+  return rc;
+}
+
+/*
+** Prepare an SQL statement using the results of a printf() formatting.
+*/
+static int idxPrintfPrepareStmt(
+  sqlite3 *db,                    /* Database handle to compile against */
+  sqlite3_stmt **ppStmt,          /* OUT: Compiled SQL statement */
+  char **pzErrmsg,                /* OUT: sqlite3_malloc()ed error message */
+  const char *zFmt,               /* printf() format of SQL statement */
+  ...                             /* Trailing printf() arguments */
+){
+  va_list ap;
+  int rc;
+  char *zSql;
+  va_start(ap, zFmt);
+  zSql = sqlite3_vmprintf(zFmt, ap);
+  if( zSql==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    rc = idxPrepareStmt(db, ppStmt, pzErrmsg, zSql);
+    sqlite3_free(zSql);
+  }
+  va_end(ap);
+  return rc;
+}
+
 
 /*************************************************************************
 ** Beginning of virtual table implementation.
@@ -292,6 +345,17 @@ struct ExpertVtab {
   sqlite3expert *pExpert;
 };
 
+typedef struct ExpertCsr ExpertCsr;
+struct ExpertCsr {
+  sqlite3_vtab_cursor base;
+  sqlite3_stmt *pData;
+
+  int iTarget;                    /* Target as a percentage */
+  double target;                  /* Target nRet/nRow value */
+  double nRow;                    /* Rows seen */
+  double nRet;                    /* Rows returned */
+};
+
 static char *expertDequote(const char *zIn){
   int n = strlen(zIn);
   char *zRet = sqlite3_malloc(n);
@@ -389,7 +453,11 @@ static int expertBestIndex(sqlite3_vtab *pVtab, sqlite3_index_info *pIdxInfo){
     /* Add the constraints to the IdxScan object */
     for(i=0; i<pIdxInfo->nConstraint; i++){
       struct sqlite3_index_constraint *pCons = &pIdxInfo->aConstraint[i];
-      if( pCons->usable && pCons->iColumn>=0 && (pCons->op & opmask) ){
+      if( pCons->usable 
+       && pCons->iColumn>=0 
+       && p->pTab->aCol[pCons->iColumn].iPk==0
+       && (pCons->op & opmask) 
+      ){
         IdxConstraint *pNew;
         const char *zColl = sqlite3_vtab_collation(dbv, i);
         pNew = idxNewConstraint(&rc, zColl);
@@ -439,6 +507,122 @@ static int expertUpdate(
   return SQLITE_OK;
 }
 
+/* 
+** Virtual table module xOpen method.
+*/
+static int expertOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
+  int rc = SQLITE_OK;
+  ExpertCsr *pCsr;
+  pCsr = idxMalloc(&rc, sizeof(ExpertCsr));
+  *ppCursor = (sqlite3_vtab_cursor*)pCsr;
+  return rc;
+}
+
+/* 
+** Virtual table module xClose method.
+*/
+static int expertClose(sqlite3_vtab_cursor *cur){
+  ExpertCsr *pCsr = (ExpertCsr*)cur;
+  sqlite3_finalize(pCsr->pData);
+  sqlite3_free(pCsr);
+  return SQLITE_OK;
+}
+
+/*
+** Virtual table module xEof method.
+**
+** Return non-zero if the cursor does not currently point to a valid 
+** record (i.e if the scan has finished), or zero otherwise.
+*/
+static int expertEof(sqlite3_vtab_cursor *cur){
+  ExpertCsr *pCsr = (ExpertCsr*)cur;
+  return pCsr->pData==0;
+}
+
+/* 
+** Virtual table module xNext method.
+*/
+static int expertNext(sqlite3_vtab_cursor *cur){
+  ExpertCsr *pCsr = (ExpertCsr*)cur;
+  int rc = SQLITE_OK;
+  int bRet;
+  assert( pCsr->pData );
+
+  do {
+    rc = sqlite3_step(pCsr->pData);
+    if( rc!=SQLITE_ROW ){
+      rc = sqlite3_finalize(pCsr->pData);
+      pCsr->pData = 0;
+      bRet = 1;
+    }else{
+      rc = SQLITE_OK;
+      bRet = (pCsr->nRow==0.0 || pCsr->nRow/pCsr->nRet < pCsr->target);
+      if( bRet==0 ){
+        unsigned short rnd;
+        sqlite3_randomness(2, (void*)&rnd);
+        bRet = ((int)rnd % 100) <= pCsr->iTarget;
+      }
+    }
+    pCsr->nRow += 1.0;
+  }while( bRet==0 );
+
+  pCsr->nRet += 1.0;
+  return rc;
+}
+
+/* 
+** Virtual table module xRowid method.
+*/
+static int expertRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
+  *pRowid = 0;
+  return SQLITE_OK;
+}
+
+/* 
+** Virtual table module xColumn method.
+*/
+static int expertColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){
+  ExpertCsr *pCsr = (ExpertCsr*)cur;
+  sqlite3_value *pVal;
+  pVal = sqlite3_column_value(pCsr->pData, i);
+  if( pVal ){
+    sqlite3_result_value(ctx, pVal);
+  }
+  return SQLITE_OK;
+}
+
+/* 
+** Virtual table module xFilter method.
+*/
+static int expertFilter(
+  sqlite3_vtab_cursor *cur, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  ExpertCsr *pCsr = (ExpertCsr*)cur;
+  ExpertVtab *pVtab = (ExpertVtab*)(cur->pVtab);
+  sqlite3expert *pExpert = pVtab->pExpert;
+  int rc;
+
+  rc = sqlite3_finalize(pCsr->pData);
+  pCsr->pData = 0;
+  pCsr->nRow = 0.0;
+  pCsr->nRet = 0.0;
+  pCsr->iTarget = pExpert->iSample;
+  pCsr->target = (double)pExpert->iSample / 100.0;
+
+  if( rc==SQLITE_OK ){
+    rc = idxPrintfPrepareStmt(pExpert->db, &pCsr->pData, &pVtab->base.zErrMsg,
+        "SELECT * FROM main.%Q", pVtab->pTab->zName
+    );
+  }
+
+  if( rc==SQLITE_OK ){
+    rc = expertNext(cur);
+  }
+  return rc;
+}
+
 static int idxRegisterVtab(sqlite3expert *p){
   static sqlite3_module expertModule = {
     2,                            /* iVersion */
@@ -447,13 +631,13 @@ static int idxRegisterVtab(sqlite3expert *p){
     expertBestIndex,              /* xBestIndex - Determine search strategy */
     expertDisconnect,             /* xDisconnect - Disconnect from a table */
     expertDisconnect,             /* xDestroy - Drop a table */
-    0,                            /* xOpen - open a cursor */
-    0,                            /* xClose - close a cursor */
-    0,                            /* xFilter - configure scan constraints */
-    0,                            /* xNext - advance a cursor */
-    0,                            /* xEof */
-    0,                            /* xColumn - read data */
-    0,                            /* xRowid - read data */
+    expertOpen,                   /* xOpen - open a cursor */
+    expertClose,                  /* xClose - close a cursor */
+    expertFilter,                 /* xFilter - configure scan constraints */
+    expertNext,                   /* xNext - advance a cursor */
+    expertEof,                    /* xEof */
+    expertColumn,                 /* xColumn - read data */
+    expertRowid,                  /* xRowid - read data */
     expertUpdate,                 /* xUpdate - write data */
     0,                            /* xBegin - begin transaction */
     0,                            /* xSync - sync transaction */
@@ -471,60 +655,6 @@ static int idxRegisterVtab(sqlite3expert *p){
 /*
 ** End of virtual table implementation.
 *************************************************************************/
-
-/*
-** An error associated with database handle db has just occurred. Pass
-** the error message to callback function xOut.
-*/
-static void idxDatabaseError(
-  sqlite3 *db,                    /* Database handle */
-  char **pzErrmsg                 /* Write error here */
-){
-  *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
-}
-
-/*
-** Prepare an SQL statement.
-*/
-static int idxPrepareStmt(
-  sqlite3 *db,                    /* Database handle to compile against */
-  sqlite3_stmt **ppStmt,          /* OUT: Compiled SQL statement */
-  char **pzErrmsg,                /* OUT: sqlite3_malloc()ed error message */
-  const char *zSql                /* SQL statement to compile */
-){
-  int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0);
-  if( rc!=SQLITE_OK ){
-    *ppStmt = 0;
-    idxDatabaseError(db, pzErrmsg);
-  }
-  return rc;
-}
-
-/*
-** Prepare an SQL statement using the results of a printf() formatting.
-*/
-static int idxPrintfPrepareStmt(
-  sqlite3 *db,                    /* Database handle to compile against */
-  sqlite3_stmt **ppStmt,          /* OUT: Compiled SQL statement */
-  char **pzErrmsg,                /* OUT: sqlite3_malloc()ed error message */
-  const char *zFmt,               /* printf() format of SQL statement */
-  ...                             /* Trailing printf() arguments */
-){
-  va_list ap;
-  int rc;
-  char *zSql;
-  va_start(ap, zFmt);
-  zSql = sqlite3_vmprintf(zFmt, ap);
-  if( zSql==0 ){
-    rc = SQLITE_NOMEM;
-  }else{
-    rc = idxPrepareStmt(db, ppStmt, pzErrmsg, zSql);
-    sqlite3_free(zSql);
-  }
-  va_end(ap);
-  return rc;
-}
-
 /*
 ** Finalize SQL statement pStmt. If (*pRc) is SQLITE_OK when this function
 ** is called, set it to the return value of sqlite3_finalize() before
@@ -557,7 +687,7 @@ static int idxGetTableInfo(
   int nByte = sizeof(IdxTable) + nTab + 1;
   IdxTable *pNew = 0;
   int rc, rc2;
-  char *pCsr;
+  char *pCsr = 0;
 
   rc = idxPrintfPrepareStmt(db, &p1, pzErrmsg, "PRAGMA table_info=%Q", zTab);
   while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){
@@ -1425,7 +1555,8 @@ static int idxPopulateOneStat1(
 
   /* Formulate the query text */
   if( rc==SQLITE_OK ){
-    rc = idxPrepareStmt(p->db, &pQuery, pzErr, zQuery);
+    sqlite3 *dbrem = (p->iSample==100 ? p->db : p->dbv);
+    rc = idxPrepareStmt(dbrem, &pQuery, pzErr, zQuery);
   }
   sqlite3_free(zQuery);
 
@@ -1481,16 +1612,14 @@ static int idxBuildSampleTable(sqlite3expert *p, const char *zTab){
   int rc;
   char *zSql;
 
-  rc = sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0);
+  rc = sqlite3_exec(p->dbv,"DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0);
   if( rc!=SQLITE_OK ) return rc;
 
   zSql = sqlite3_mprintf(
-      "CREATE TABLE temp." UNIQUE_TABLE_NAME 
-      "  AS SELECT * FROM %Q WHERE sample()"
-      , zTab
+      "CREATE TABLE temp." UNIQUE_TABLE_NAME " AS SELECT * FROM %Q", zTab
   );
   if( zSql==0 ) return SQLITE_NOMEM;
-  rc = sqlite3_exec(p->db, zSql, 0, 0, 0);
+  rc = sqlite3_exec(p->dbv, zSql, 0, 0, 0);
   sqlite3_free(zSql);
 
   return rc;
@@ -1537,8 +1666,9 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
   }
 
   if( rc==SQLITE_OK ){
+    sqlite3 *dbrem = (p->iSample==100 ? p->db : p->dbv);
     rc = sqlite3_create_function(
-        p->db, "rem", 2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0
+        dbrem, "rem", 2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0
     );
   }
   if( rc==SQLITE_OK ){
diff --git a/manifest b/manifest
index 5ae9923e9e..86d64841b6 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\san\soption\sto\sgenerate\sstat1\sdata\sbased\son\sa\ssubset\sof\sthe\suser\sdatabase\ntable\scontents\sto\ssqlite3_expert.
-D 2017-04-20T09:54:04.700
+C Avoid\screating\sa\stemp\stable\sin\sthe\suser\sdatabase\sin\sthe\ssqlite3_expert\scode.\nTrouble\sis,\sthis\smakes\ssampling\sfor\sstat1\sdata\smuch\sslower.
+D 2017-04-20T16:08:33.333
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6
@@ -43,7 +43,7 @@ F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md 9f15075ec5ad772808eff55ef044c31140fd1146aa0a3c47eafd155e71851b01
 F ext/expert/expert.c 33842ef151d84c5f8000f9c7b938998c6b999eaef7ce1f4eeb0df8ffe6739496
 F ext/expert/expert1.test 1033e43071b69dc2f4e88fbf03fc7f18846c9865cac14f28c80f581437f09acb
-F ext/expert/sqlite3expert.c af3b336f83bcd2a586f6119d4040ac36ccf45162c48e3780ed63ab119fd04fe1
+F ext/expert/sqlite3expert.c 68acd26b0520408f0c5e76bd9ddaf70d776d397f8aa1bd6eda91f42f34c0a8cf
 F ext/expert/sqlite3expert.h af6354f8ee5c9e025024e63fec3bd640a802afcc3099a44d804752cf0791d811
 F ext/expert/test_expert.c b01a5115f9444a9b416582c985138f5dfdb279848ce8b7452be383530be27f01
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
@@ -1579,7 +1579,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a157fcfde5afc27ae38e7cf4669fcc8e60e23d9d301ffe2e541dd69f895b493b
-R 2a75643c384b85cff6c5d9d5615ac2a2
+P c69c3e21db6e141f7e24226c6432f2ed31fe5f177bd23781915871f8600ee56a
+R 02fc06fd99e34f4a0d95661da624b768
+T *branch * schemalint-failure
+T *sym-schemalint-failure *
+T -sym-schemalint *
 U dan
-Z 7876a034c88841c26664b8a83eb3817f
+Z 8a7b9cb83c65b8cfdb4d171254abdcee
diff --git a/manifest.uuid b/manifest.uuid
index d816bcc63e..1472955024 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c69c3e21db6e141f7e24226c6432f2ed31fe5f177bd23781915871f8600ee56a
\ No newline at end of file
+c62e358243d96cb38a7ce2aa679fc640b62bf46080eab4bd5fc2acf5997d6cd5
\ No newline at end of file

From 6ebd7ff4827d2902f1e5bec3eba7d9c7d34564e3 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 20 Apr 2017 16:18:43 +0000
Subject: [PATCH 038/488] Avoid adding INTEGER PRIMARY KEY columns to candidate
 indexes.

FossilOrigin-Name: 4577fea5cd9d91ea241e9be82797ca1a4447f536e1e4b78a4a569aeb52e78fcb
---
 ext/expert/sqlite3expert.c |  8 ++++++--
 manifest                   | 12 ++++++------
 manifest.uuid              |  2 +-
 3 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index 03963e1f7e..1966ab700f 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -389,7 +389,11 @@ static int expertBestIndex(sqlite3_vtab *pVtab, sqlite3_index_info *pIdxInfo){
     /* Add the constraints to the IdxScan object */
     for(i=0; i<pIdxInfo->nConstraint; i++){
       struct sqlite3_index_constraint *pCons = &pIdxInfo->aConstraint[i];
-      if( pCons->usable && pCons->iColumn>=0 && (pCons->op & opmask) ){
+      if( pCons->usable 
+       && pCons->iColumn>=0 
+       && p->pTab->aCol[pCons->iColumn].iPk==0
+       && (pCons->op & opmask) 
+      ){
         IdxConstraint *pNew;
         const char *zColl = sqlite3_vtab_collation(dbv, i);
         pNew = idxNewConstraint(&rc, zColl);
@@ -557,7 +561,7 @@ static int idxGetTableInfo(
   int nByte = sizeof(IdxTable) + nTab + 1;
   IdxTable *pNew = 0;
   int rc, rc2;
-  char *pCsr;
+  char *pCsr = 0;
 
   rc = idxPrintfPrepareStmt(db, &p1, pzErrmsg, "PRAGMA table_info=%Q", zTab);
   while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){
diff --git a/manifest b/manifest
index 5ae9923e9e..e63525e54c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\san\soption\sto\sgenerate\sstat1\sdata\sbased\son\sa\ssubset\sof\sthe\suser\sdatabase\ntable\scontents\sto\ssqlite3_expert.
-D 2017-04-20T09:54:04.700
+C Avoid\sadding\sINTEGER\sPRIMARY\sKEY\scolumns\sto\scandidate\sindexes.
+D 2017-04-20T16:18:43.967
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6
@@ -43,7 +43,7 @@ F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md 9f15075ec5ad772808eff55ef044c31140fd1146aa0a3c47eafd155e71851b01
 F ext/expert/expert.c 33842ef151d84c5f8000f9c7b938998c6b999eaef7ce1f4eeb0df8ffe6739496
 F ext/expert/expert1.test 1033e43071b69dc2f4e88fbf03fc7f18846c9865cac14f28c80f581437f09acb
-F ext/expert/sqlite3expert.c af3b336f83bcd2a586f6119d4040ac36ccf45162c48e3780ed63ab119fd04fe1
+F ext/expert/sqlite3expert.c 403b261dbea217f1b221dd4ab19774d1acfc4eba44c1ed35ab38060168f81b6d
 F ext/expert/sqlite3expert.h af6354f8ee5c9e025024e63fec3bd640a802afcc3099a44d804752cf0791d811
 F ext/expert/test_expert.c b01a5115f9444a9b416582c985138f5dfdb279848ce8b7452be383530be27f01
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
@@ -1579,7 +1579,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a157fcfde5afc27ae38e7cf4669fcc8e60e23d9d301ffe2e541dd69f895b493b
-R 2a75643c384b85cff6c5d9d5615ac2a2
+P c69c3e21db6e141f7e24226c6432f2ed31fe5f177bd23781915871f8600ee56a
+R 755161800d19d245e3f0050f91b54ff6
 U dan
-Z 7876a034c88841c26664b8a83eb3817f
+Z c9b10ac09e4b6010a00208af302face2
diff --git a/manifest.uuid b/manifest.uuid
index d816bcc63e..a34fd3239d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c69c3e21db6e141f7e24226c6432f2ed31fe5f177bd23781915871f8600ee56a
\ No newline at end of file
+4577fea5cd9d91ea241e9be82797ca1a4447f536e1e4b78a4a569aeb52e78fcb
\ No newline at end of file

From bc3f784cd2695c5d7c6cc4958088b93662a9597a Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 20 Apr 2017 16:43:32 +0000
Subject: [PATCH 039/488] Speed this branch up a bit by filtering before the
 virtual table layer when sampling user data.

FossilOrigin-Name: 8e57c31340dd9ffc457da63c5996fb1b573f8154f864ec2b52c15f399906ac8b
---
 ext/expert/sqlite3expert.c | 46 ++++++++++----------------------------
 manifest                   | 15 +++++--------
 manifest.uuid              |  2 +-
 3 files changed, 19 insertions(+), 44 deletions(-)

diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index d578973101..3ee74b5710 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -349,11 +349,6 @@ typedef struct ExpertCsr ExpertCsr;
 struct ExpertCsr {
   sqlite3_vtab_cursor base;
   sqlite3_stmt *pData;
-
-  int iTarget;                    /* Target as a percentage */
-  double target;                  /* Target nRet/nRow value */
-  double nRow;                    /* Rows seen */
-  double nRet;                    /* Rows returned */
 };
 
 static char *expertDequote(const char *zIn){
@@ -545,28 +540,16 @@ static int expertEof(sqlite3_vtab_cursor *cur){
 static int expertNext(sqlite3_vtab_cursor *cur){
   ExpertCsr *pCsr = (ExpertCsr*)cur;
   int rc = SQLITE_OK;
-  int bRet;
+
   assert( pCsr->pData );
+  rc = sqlite3_step(pCsr->pData);
+  if( rc!=SQLITE_ROW ){
+    rc = sqlite3_finalize(pCsr->pData);
+    pCsr->pData = 0;
+  }else{
+    rc = SQLITE_OK;
+  }
 
-  do {
-    rc = sqlite3_step(pCsr->pData);
-    if( rc!=SQLITE_ROW ){
-      rc = sqlite3_finalize(pCsr->pData);
-      pCsr->pData = 0;
-      bRet = 1;
-    }else{
-      rc = SQLITE_OK;
-      bRet = (pCsr->nRow==0.0 || pCsr->nRow/pCsr->nRet < pCsr->target);
-      if( bRet==0 ){
-        unsigned short rnd;
-        sqlite3_randomness(2, (void*)&rnd);
-        bRet = ((int)rnd % 100) <= pCsr->iTarget;
-      }
-    }
-    pCsr->nRow += 1.0;
-  }while( bRet==0 );
-
-  pCsr->nRet += 1.0;
   return rc;
 }
 
@@ -606,14 +589,9 @@ static int expertFilter(
 
   rc = sqlite3_finalize(pCsr->pData);
   pCsr->pData = 0;
-  pCsr->nRow = 0.0;
-  pCsr->nRet = 0.0;
-  pCsr->iTarget = pExpert->iSample;
-  pCsr->target = (double)pExpert->iSample / 100.0;
-
   if( rc==SQLITE_OK ){
     rc = idxPrintfPrepareStmt(pExpert->db, &pCsr->pData, &pVtab->base.zErrMsg,
-        "SELECT * FROM main.%Q", pVtab->pTab->zName
+        "SELECT * FROM main.%Q WHERE sample()", pVtab->pTab->zName
     );
   }
 
@@ -1703,9 +1681,9 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
     rc = idxPopulateOneStat1(p, pIndexXInfo, pWrite, zTab, zIdx, pzErr);
     iPrev = iRowid;
   }
-  if( p->iSample<100 ){
-    rc = sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp." UNIQUE_TABLE_NAME,
-        0,0,0
+  if( rc==SQLITE_OK && p->iSample<100 ){
+    rc = sqlite3_exec(p->dbv, 
+        "DROP TABLE IF EXISTS temp." UNIQUE_TABLE_NAME, 0,0,0
     );
   }
 
diff --git a/manifest b/manifest
index 86d64841b6..78e4264cc9 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Avoid\screating\sa\stemp\stable\sin\sthe\suser\sdatabase\sin\sthe\ssqlite3_expert\scode.\nTrouble\sis,\sthis\smakes\ssampling\sfor\sstat1\sdata\smuch\sslower.
-D 2017-04-20T16:08:33.333
+C Speed\sthis\sbranch\sup\sa\sbit\sby\sfiltering\sbefore\sthe\svirtual\stable\slayer\swhen\nsampling\suser\sdata.
+D 2017-04-20T16:43:32.927
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6
@@ -43,7 +43,7 @@ F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md 9f15075ec5ad772808eff55ef044c31140fd1146aa0a3c47eafd155e71851b01
 F ext/expert/expert.c 33842ef151d84c5f8000f9c7b938998c6b999eaef7ce1f4eeb0df8ffe6739496
 F ext/expert/expert1.test 1033e43071b69dc2f4e88fbf03fc7f18846c9865cac14f28c80f581437f09acb
-F ext/expert/sqlite3expert.c 68acd26b0520408f0c5e76bd9ddaf70d776d397f8aa1bd6eda91f42f34c0a8cf
+F ext/expert/sqlite3expert.c fde366d8c1b1970b2c18196ca2e64d01c2106bd9431c371a26e8d5b79f37f90b
 F ext/expert/sqlite3expert.h af6354f8ee5c9e025024e63fec3bd640a802afcc3099a44d804752cf0791d811
 F ext/expert/test_expert.c b01a5115f9444a9b416582c985138f5dfdb279848ce8b7452be383530be27f01
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
@@ -1579,10 +1579,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c69c3e21db6e141f7e24226c6432f2ed31fe5f177bd23781915871f8600ee56a
-R 02fc06fd99e34f4a0d95661da624b768
-T *branch * schemalint-failure
-T *sym-schemalint-failure *
-T -sym-schemalint *
+P c62e358243d96cb38a7ce2aa679fc640b62bf46080eab4bd5fc2acf5997d6cd5
+R 73307a63b82b2606f65f28fc989fd3ae
 U dan
-Z 8a7b9cb83c65b8cfdb4d171254abdcee
+Z b05e3391f67ce73794363163fdd05070
diff --git a/manifest.uuid b/manifest.uuid
index 1472955024..a17ccd88b9 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c62e358243d96cb38a7ce2aa679fc640b62bf46080eab4bd5fc2acf5997d6cd5
\ No newline at end of file
+8e57c31340dd9ffc457da63c5996fb1b573f8154f864ec2b52c15f399906ac8b
\ No newline at end of file

From 5b9dbd1ed0a8470aa6536b0b9d28f85ea53fccbf Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 21 Apr 2017 19:53:39 +0000
Subject: [PATCH 040/488] Update the README.md file in the ext/expert/
 directory.

FossilOrigin-Name: 3b2ff4e0692dfca395d4523b7d5cd0dfd5c319c1072a2a873631fa477cee0b79
---
 ext/expert/README.md | 36 +++++++++++++++++++++++++-----------
 manifest             | 12 ++++++------
 manifest.uuid        |  2 +-
 3 files changed, 32 insertions(+), 18 deletions(-)

diff --git a/ext/expert/README.md b/ext/expert/README.md
index 3cede6c09d..b3023f2c1a 100644
--- a/ext/expert/README.md
+++ b/ext/expert/README.md
@@ -5,21 +5,20 @@ given a database and a set of SQL queries. It works as follows:
 
   1. The user database schema is copied to a temporary database.
 
-  1. All SQL queries are prepared against the temporary database. The
-     **sqlite3\_whereinfo\_hook()** API is used to record information regarding
-     the WHERE and ORDER BY clauses attached to each query.
+  1. All SQL queries are prepared against the temporary database. I
+     Information regarding the WHERE and ORDER BY clauses, and other query
+     features that affect index selection, are recorded.
 
   1. The information gathered in step 2 is used to create (possibly a large
      number of) candidate indexes.
 
+  1. A subset of the data in the user database is used to generate statistics
+     for all existing indexes and the candidate indexes generated in step 3
+     above.
+
   1. The SQL queries are prepared a second time. If the planner uses any
      of the indexes created in step 3, they are recommended to the user.
 
-No ANALYZE data is available to the planner in step 4 above. This can lead to sub-optimal results.
-
-This extension requires that SQLite be built with the 
-SQLITE\_ENABLE\_WHEREINFO\_HOOK pre-processor symbol defined.
-
 # C API
 
 The SQLite expert C API is defined in sqlite3expert.h. Most uses will proceed
@@ -32,6 +31,10 @@ as follows:
      by making one or more calls to **sqlite3\_expert\_sql()**. Each call may
      specify a single SQL statement, or multiple statements separated by
      semi-colons.
+  
+  1. Optionally, the **sqlite3\_expert\_config()** API may be used to 
+     configure the size of the data subset used to generate index statistics.
+     Using a smaller subset of the data can speed up the analysis.
 
   1. **sqlite3\_expert\_analyze()** is called to run the analysis.
 
@@ -48,10 +51,11 @@ The file "expert.c" contains the code for a command line application that
 uses the API described above. It can be compiled with (for example):
 
 <pre>
-  gcc -O2 -DSQLITE_ENABLE_WHEREINFO_HOOK sqlite3.c expert.c sqlite3expert.c -o sqlite3_expert
+  gcc -O2 sqlite3.c expert.c sqlite3expert.c -o sqlite3_expert
 </pre>
 
-Assuming the database is "test.db", it can then be run to analyze a single query:
+Assuming the database is named "test.db", it can then be run to analyze a
+single query:
 
 <pre>
   ./sqlite3_expert -sql &lt;sql-query&gt; test.db
@@ -63,6 +67,16 @@ Or an entire text file worth of queries with:
   ./sqlite3_expert -file &lt;text-file&gt; test.db
 </pre>
 
+By default, sqlite3_expert generates index statistics using all the data in
+the user database. For a large database, this may be prohibitively time
+consuming. The "-sample" option may be used to configure sqlite3_expert to
+generate statistics based on an integer percentage of the user database as
+follows:
 
+<pre>
+  # Generate statistics based on 25% of the user database rows:
+  ./sqlite3_expert -sample 25 -sql &lt;sql-query&gt; test.db
 
-
+  # Do not generate any statistics at all:
+  ./sqlite3_expert -sample 0 -sql &lt;sql-query&gt; test.db
+</pre>
diff --git a/manifest b/manifest
index 66b5295d63..a85be5f925 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\slatest\strunk\schanges\sinto\sthis\sbranch.
-D 2017-04-20T17:35:46.403
+C Update\sthe\sREADME.md\sfile\sin\sthe\sext/expert/\sdirectory.
+D 2017-04-21T19:53:39.781
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6
@@ -40,7 +40,7 @@ F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd
 F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
-F ext/expert/README.md 9f15075ec5ad772808eff55ef044c31140fd1146aa0a3c47eafd155e71851b01
+F ext/expert/README.md 58a5af07981d6946464eb4b069f36a18dff1d594bcf331a02b4485dac5cd5207
 F ext/expert/expert.c 33842ef151d84c5f8000f9c7b938998c6b999eaef7ce1f4eeb0df8ffe6739496
 F ext/expert/expert1.test 1033e43071b69dc2f4e88fbf03fc7f18846c9865cac14f28c80f581437f09acb
 F ext/expert/sqlite3expert.c 4bc1820a70d68b478884a26a2215df8c1f2d44fb40d9cd8c47d2046c8ce0c8bc
@@ -1582,7 +1582,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 4e366996434e63f06cc451d2011f1f1464360f03430b8260e48f78a612b4e9d6 6b21d0fdebdccfaf63590d9ca9a279c22b8baec07c1a669b9f617f25bd857384
-R 3ff876810132a5a27eb9c4a943446c06
+P b1533bc455f52f570c0f4b8aaa0da802757dc89b0e45b9a9b31aa591a44bf7bd
+R 1bfa4bacff37532498e779dfe65e0523
 U dan
-Z 9bcc0d98d05bcef127f4de36c5b9d364
+Z 425c38161b659ffbe58ff987543a301c
diff --git a/manifest.uuid b/manifest.uuid
index 10e306f449..e42f6c368b 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b1533bc455f52f570c0f4b8aaa0da802757dc89b0e45b9a9b31aa591a44bf7bd
\ No newline at end of file
+3b2ff4e0692dfca395d4523b7d5cd0dfd5c319c1072a2a873631fa477cee0b79
\ No newline at end of file

From 6b0fa916f570dd80b462c924d05048d52c5651f5 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 21 Apr 2017 19:56:53 +0000
Subject: [PATCH 041/488] Fix formatting errors in the previous commit.

FossilOrigin-Name: da9a2e5aa977f7e8e9e4365f7b34bb4f482029a3d44646100773cedc8ea9b959
---
 ext/expert/README.md | 13 +++++++------
 manifest             | 12 ++++++------
 manifest.uuid        |  2 +-
 3 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/ext/expert/README.md b/ext/expert/README.md
index b3023f2c1a..3c2e0a65e0 100644
--- a/ext/expert/README.md
+++ b/ext/expert/README.md
@@ -5,12 +5,13 @@ given a database and a set of SQL queries. It works as follows:
 
   1. The user database schema is copied to a temporary database.
 
-  1. All SQL queries are prepared against the temporary database. I
+  1. All SQL queries are prepared against the temporary database.
      Information regarding the WHERE and ORDER BY clauses, and other query
-     features that affect index selection, are recorded.
+     features that affect index selection are recorded.
 
-  1. The information gathered in step 2 is used to create (possibly a large
-     number of) candidate indexes.
+  1. The information gathered in step 2 is used to create candidate indexes
+     - indexes that the planner might have made use of in the previous step,
+     had they been available.
 
   1. A subset of the data in the user database is used to generate statistics
      for all existing indexes and the candidate indexes generated in step 3
@@ -67,9 +68,9 @@ Or an entire text file worth of queries with:
   ./sqlite3_expert -file &lt;text-file&gt; test.db
 </pre>
 
-By default, sqlite3_expert generates index statistics using all the data in
+By default, sqlite3\_expert generates index statistics using all the data in
 the user database. For a large database, this may be prohibitively time
-consuming. The "-sample" option may be used to configure sqlite3_expert to
+consuming. The "-sample" option may be used to configure sqlite3\_expert to
 generate statistics based on an integer percentage of the user database as
 follows:
 
diff --git a/manifest b/manifest
index a85be5f925..f684b3a0fc 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\sthe\sREADME.md\sfile\sin\sthe\sext/expert/\sdirectory.
-D 2017-04-21T19:53:39.781
+C Fix\sformatting\serrors\sin\sthe\sprevious\scommit.
+D 2017-04-21T19:56:53.579
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6
@@ -40,7 +40,7 @@ F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd
 F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
-F ext/expert/README.md 58a5af07981d6946464eb4b069f36a18dff1d594bcf331a02b4485dac5cd5207
+F ext/expert/README.md ffa4899855d90ae1a2a29b29754c7abad9ed8c42f187c72e913cb671264dbd5d
 F ext/expert/expert.c 33842ef151d84c5f8000f9c7b938998c6b999eaef7ce1f4eeb0df8ffe6739496
 F ext/expert/expert1.test 1033e43071b69dc2f4e88fbf03fc7f18846c9865cac14f28c80f581437f09acb
 F ext/expert/sqlite3expert.c 4bc1820a70d68b478884a26a2215df8c1f2d44fb40d9cd8c47d2046c8ce0c8bc
@@ -1582,7 +1582,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b1533bc455f52f570c0f4b8aaa0da802757dc89b0e45b9a9b31aa591a44bf7bd
-R 1bfa4bacff37532498e779dfe65e0523
+P 3b2ff4e0692dfca395d4523b7d5cd0dfd5c319c1072a2a873631fa477cee0b79
+R 95ece7093798132a04553def12c0836e
 U dan
-Z 425c38161b659ffbe58ff987543a301c
+Z 360220848d9d809f77758310da99ccdb
diff --git a/manifest.uuid b/manifest.uuid
index e42f6c368b..77c9b27734 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-3b2ff4e0692dfca395d4523b7d5cd0dfd5c319c1072a2a873631fa477cee0b79
\ No newline at end of file
+da9a2e5aa977f7e8e9e4365f7b34bb4f482029a3d44646100773cedc8ea9b959
\ No newline at end of file

From f3ac8ba569f51cf2946511471f121aefcf449406 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 21 Apr 2017 19:58:35 +0000
Subject: [PATCH 042/488] Another minor formatting fix.

FossilOrigin-Name: 9fa2ce3c2b75408594d387684984cd946765776d30bc622a1f4e64d6fe1856d5
---
 ext/expert/README.md |  6 +++---
 manifest             | 12 ++++++------
 manifest.uuid        |  2 +-
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/ext/expert/README.md b/ext/expert/README.md
index 3c2e0a65e0..28886fd1f2 100644
--- a/ext/expert/README.md
+++ b/ext/expert/README.md
@@ -9,9 +9,9 @@ given a database and a set of SQL queries. It works as follows:
      Information regarding the WHERE and ORDER BY clauses, and other query
      features that affect index selection are recorded.
 
-  1. The information gathered in step 2 is used to create candidate indexes
-     - indexes that the planner might have made use of in the previous step,
-     had they been available.
+  1. The information gathered in step 2 is used to create candidate 
+     indexes - indexes that the planner might have made use of in the previous
+     step, had they been available.
 
   1. A subset of the data in the user database is used to generate statistics
      for all existing indexes and the candidate indexes generated in step 3
diff --git a/manifest b/manifest
index f684b3a0fc..02667e9490 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sformatting\serrors\sin\sthe\sprevious\scommit.
-D 2017-04-21T19:56:53.579
+C Another\sminor\sformatting\sfix.
+D 2017-04-21T19:58:35.957
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6
@@ -40,7 +40,7 @@ F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd
 F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
-F ext/expert/README.md ffa4899855d90ae1a2a29b29754c7abad9ed8c42f187c72e913cb671264dbd5d
+F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3
 F ext/expert/expert.c 33842ef151d84c5f8000f9c7b938998c6b999eaef7ce1f4eeb0df8ffe6739496
 F ext/expert/expert1.test 1033e43071b69dc2f4e88fbf03fc7f18846c9865cac14f28c80f581437f09acb
 F ext/expert/sqlite3expert.c 4bc1820a70d68b478884a26a2215df8c1f2d44fb40d9cd8c47d2046c8ce0c8bc
@@ -1582,7 +1582,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3b2ff4e0692dfca395d4523b7d5cd0dfd5c319c1072a2a873631fa477cee0b79
-R 95ece7093798132a04553def12c0836e
+P da9a2e5aa977f7e8e9e4365f7b34bb4f482029a3d44646100773cedc8ea9b959
+R 5d55911452a98d7b9a566802f4defda8
 U dan
-Z 360220848d9d809f77758310da99ccdb
+Z bc8b305bc880d9cdf8a4b644dabd4fac
diff --git a/manifest.uuid b/manifest.uuid
index 77c9b27734..d9908b5ae2 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-da9a2e5aa977f7e8e9e4365f7b34bb4f482029a3d44646100773cedc8ea9b959
\ No newline at end of file
+9fa2ce3c2b75408594d387684984cd946765776d30bc622a1f4e64d6fe1856d5
\ No newline at end of file

From 382bce037cad6c70635b23f0ee63669e326fad99 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 2 May 2017 20:42:30 +0000
Subject: [PATCH 043/488] In the sqlite3_expert command-line tool, allow
 two-dash options.  Do not accept the database name if it begins with "-".

FossilOrigin-Name: af7d1596044980e0a18baa3ab6674779724dffbf18a152c72e53f11a08999e68
---
 ext/expert/expert.c |  4 ++--
 manifest            | 12 ++++++------
 manifest.uuid       |  2 +-
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/ext/expert/expert.c b/ext/expert/expert.c
index baa7be940f..1c4b758d8f 100644
--- a/ext/expert/expert.c
+++ b/ext/expert/expert.c
@@ -79,6 +79,7 @@ int main(int argc, char **argv){
 
   if( argc<2 ) usage(argv);
   zDb = argv[argc-1];
+  if( zDb[0]=='-' ) usage(argv);
   rc = sqlite3_open(zDb, &db);
   if( rc!=SQLITE_OK ){
     fprintf(stderr, "Cannot open db file: %s - %s\n", zDb, sqlite3_errmsg(db));
@@ -92,6 +93,7 @@ int main(int argc, char **argv){
   }else{
     for(i=1; i<(argc-1); i++){
       char *zArg = argv[i];
+      if( zArg[0]=='-' && zArg[1]=='-' && zArg[2]!=0 ) zArg++;
       int nArg = strlen(zArg);
       if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-file", nArg) ){
         if( ++i==(argc-1) ) option_requires_argument("-file");
@@ -151,5 +153,3 @@ int main(int argc, char **argv){
   sqlite3_free(zErr);
   return rc;
 }
-
-
diff --git a/manifest b/manifest
index 9ca3ce06e1..7ad30c6537 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\sthe\slatest\senhancements\sfrom\strunk.
-D 2017-05-02T19:45:14.234
+C In\sthe\ssqlite3_expert\scommand-line\stool,\sallow\stwo-dash\soptions.\s\sDo\snot\naccept\sthe\sdatabase\sname\sif\sit\sbegins\swith\s"-".
+D 2017-05-02T20:42:30.619
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6
@@ -41,7 +41,7 @@ F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3
-F ext/expert/expert.c 33842ef151d84c5f8000f9c7b938998c6b999eaef7ce1f4eeb0df8ffe6739496
+F ext/expert/expert.c dc88a3e73fdeb0d21e7d60b791120961853af931f3c509b508f4ac4a0233438e
 F ext/expert/expert1.test 1033e43071b69dc2f4e88fbf03fc7f18846c9865cac14f28c80f581437f09acb
 F ext/expert/sqlite3expert.c 4bc1820a70d68b478884a26a2215df8c1f2d44fb40d9cd8c47d2046c8ce0c8bc
 F ext/expert/sqlite3expert.h af6354f8ee5c9e025024e63fec3bd640a802afcc3099a44d804752cf0791d811
@@ -1585,7 +1585,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 11f4761c3a84e2cc9df62f117a003af8c57f3d226eec5a40d6241b121e78d002 430f539cbb3f806fb89191e0b759a5f8b49d9e5b6c95fe9a4b55a1aa0875762a
-R c37204de6c890801155672b7d23b8abd
+P a7dcf6a79f7e1c5884baee2909a4bf3174ae06d561dae87b390856e573f81b49
+R 84109358bc8c579f4172a289a09c559d
 U drh
-Z 640e08616c5ac7ee4ced91411da47a09
+Z 3bebc4378f2040a907bd12dcd9347c44
diff --git a/manifest.uuid b/manifest.uuid
index 17b09356de..c25e85551e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a7dcf6a79f7e1c5884baee2909a4bf3174ae06d561dae87b390856e573f81b49
\ No newline at end of file
+af7d1596044980e0a18baa3ab6674779724dffbf18a152c72e53f11a08999e68
\ No newline at end of file

From 138bd6df4199ecf7e3d75c9ea9bc8ce86c3055c4 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 3 May 2017 12:15:20 +0000
Subject: [PATCH 044/488] In sqlite3expert.c, do not copy the schema for
 virtual tables.  Updates to makefiles to make building easier.

FossilOrigin-Name: da15752dccf6090e40ec825db89048eca2b30185882225bf81f1891e914c2e7f
---
 Makefile.in                | 19 +++++++++++++++++++
 ext/expert/sqlite3expert.c |  7 +------
 main.mk                    |  6 +++---
 manifest                   | 16 ++++++++--------
 manifest.uuid              |  2 +-
 5 files changed, 32 insertions(+), 18 deletions(-)

diff --git a/Makefile.in b/Makefile.in
index 9076ffc085..0cde7a64f6 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -416,6 +416,8 @@ TESTSRC = \
 # Statically linked extensions
 #
 TESTSRC += \
+  $(TOP)/ext/expert/sqlite3expert.c \
+  $(TOP)/ext/expert/test_expert.c \
   $(TOP)/ext/misc/amatch.c \
   $(TOP)/ext/misc/carray.c \
   $(TOP)/ext/misc/closure.c \
@@ -1155,6 +1157,23 @@ sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl
 sqlite3_analyzer$(TEXE): sqlite3_analyzer.c
 	$(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS)
 
+sqlite3_expert$(TEXE): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c
+	$(LTLINK)	$(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert $(TLIBS)
+
+sqlite3_schemalint.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/schemalint.tcl
+	echo "#define TCLSH 2" > $@
+	echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@
+	cat sqlite3.c $(TOP)/src/tclsqlite.c >> $@
+	echo "static const char *tclsh_main_loop(void){" >> $@
+	echo "static const char *zMainloop = " >> $@
+	$(TCLSH_CMD) $(TOP)/tool/tostr.tcl $(TOP)/tool/schemalint.tcl >> $@
+	echo "; return zMainloop; }" >> $@
+
+sqlite3_schemalint$(TEXE): $(TESTSRC) sqlite3_schemalint.c
+	$(LTLINK) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS)                  \
+		sqlite3_schemalint.c $(TESTSRC)               \
+		-o sqlite3_schemalint$(EXE) $(LIBTCL) $(TLIBS)
+
 dbdump$(TEXE): $(TOP)/ext/misc/dbdump.c sqlite3.lo
 	$(LTLINK) -DDBDUMP_STANDALONE -o $@ \
            $(TOP)/ext/misc/dbdump.c sqlite3.lo $(TLIBS)
diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index 8cacc9d706..1e02328211 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -10,9 +10,6 @@
 **
 *************************************************************************
 */
-
-#if !defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHEREINFO_HOOK)
-
 #include "sqlite3expert.h"
 #include <assert.h>
 #include <string.h>
@@ -1745,6 +1742,7 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){
     sqlite3_stmt *pSql;
     rc = idxPrintfPrepareStmt(pNew->db, &pSql, pzErrmsg, 
         "SELECT sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%%'"
+        " AND sql NOT LIKE 'CREATE VIRTUAL %%'"
     );
     while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
       const char *zSql = (const char*)sqlite3_column_text(pSql, 0);
@@ -1933,6 +1931,3 @@ void sqlite3_expert_destroy(sqlite3expert *p){
     sqlite3_free(p);
   }
 }
-
-#endif /* !defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHEREINFO_HOOK) */
-
diff --git a/main.mk b/main.mk
index ea4f9891f7..218598b549 100644
--- a/main.mk
+++ b/main.mk
@@ -764,7 +764,7 @@ sqlite3_analyzer$(EXE): sqlite3_analyzer.c
 	$(TCCX) $(TCL_FLAGS) sqlite3_analyzer.c -o $@ $(LIBTCL) $(THREADLIB) 
 
 sqlite3_expert$(EXE): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c
-	$(TCCX) -DSQLITE_ENABLE_WHEREINFO_HOOK $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert $(THREADLIB)
+	$(TCCX) $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert $(THREADLIB)
 
 sqlite3_schemalint.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/schemalint.tcl
 	echo "#define TCLSH 2" > $@
@@ -775,9 +775,9 @@ sqlite3_schemalint.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/schemalint.tc
 	tclsh $(TOP)/tool/tostr.tcl $(TOP)/tool/schemalint.tcl >> $@
 	echo "; return zMainloop; }" >> $@
 
-sqlite3_schemalint$(EXE): $(TESTSRC) sqlite3_schemalint.c
+sqlite3_schemalint$(EXE): $(TESTSRC) sqlite3_schemalint.c $(TOP)/ext/session/test_session.c
 	$(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS)                  \
-		sqlite3_schemalint.c $(TESTSRC)               \
+		sqlite3_schemalint.c $(TESTSRC) $(TOP)/ext/session/test_session.c  \
 		-o sqlite3_schemalint$(EXE) $(LIBTCL) $(THREADLIB)
 
 dbdump$(EXE):	$(TOP)/ext/misc/dbdump.c sqlite3.o
diff --git a/manifest b/manifest
index 7ad30c6537..ff6e6888b8 100644
--- a/manifest
+++ b/manifest
@@ -1,6 +1,6 @@
-C In\sthe\ssqlite3_expert\scommand-line\stool,\sallow\stwo-dash\soptions.\s\sDo\snot\naccept\sthe\sdatabase\sname\sif\sit\sbegins\swith\s"-".
-D 2017-05-02T20:42:30.619
-F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
+C In\ssqlite3expert.c,\sdo\snot\scopy\sthe\sschema\sfor\svirtual\stables.\s\sUpdates\sto\nmakefiles\sto\smake\sbuilding\seasier.
+D 2017-05-03T12:15:20.951
+F Makefile.in 2c991e7b1a2bb23c147406c3630b54d99c4931ae1fa0e8c6b6bf40a7a2fd02a3
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6
 F README.md 2b15fae33852f2f53996774c21fb41e1d94181c4401a0e43ac93e11f2cc901b9
@@ -43,7 +43,7 @@ F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3
 F ext/expert/expert.c dc88a3e73fdeb0d21e7d60b791120961853af931f3c509b508f4ac4a0233438e
 F ext/expert/expert1.test 1033e43071b69dc2f4e88fbf03fc7f18846c9865cac14f28c80f581437f09acb
-F ext/expert/sqlite3expert.c 4bc1820a70d68b478884a26a2215df8c1f2d44fb40d9cd8c47d2046c8ce0c8bc
+F ext/expert/sqlite3expert.c 87bac32f90492adfd99d3dc142d5c9af38a3bd961637202285077968051bc6e8
 F ext/expert/sqlite3expert.h af6354f8ee5c9e025024e63fec3bd640a802afcc3099a44d804752cf0791d811
 F ext/expert/test_expert.c 85f5c743a899063fa48296d21de2f32c26d09a21c8582b2a0bc482e8de183e7a
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
@@ -332,7 +332,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk ef818c7b1bb21f657e3bfb363cc7167264d688ca404a666e6ddda6029e94c43b
+F main.mk 7fcb8db53d07ea8124c2c43dac5fcee8095e4abeab1acdf114833b8805aa72c9
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -1585,7 +1585,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a7dcf6a79f7e1c5884baee2909a4bf3174ae06d561dae87b390856e573f81b49
-R 84109358bc8c579f4172a289a09c559d
+P af7d1596044980e0a18baa3ab6674779724dffbf18a152c72e53f11a08999e68
+R 446fb26db86c2ead652773ba0b926ac7
 U drh
-Z 3bebc4378f2040a907bd12dcd9347c44
+Z 183bc096a41586435af9a58068b8a076
diff --git a/manifest.uuid b/manifest.uuid
index c25e85551e..388a18ee4f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-af7d1596044980e0a18baa3ab6674779724dffbf18a152c72e53f11a08999e68
\ No newline at end of file
+da15752dccf6090e40ec825db89048eca2b30185882225bf81f1891e914c2e7f
\ No newline at end of file

From bd0f1dbd06baae25459d270633300f3dd7ec5484 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 3 May 2017 12:50:46 +0000
Subject: [PATCH 045/488] Get sqlite3_expert building on Windows.

FossilOrigin-Name: d8254047b30f7c1be486bf39d4420678604573b951b5cc83c19ebf74aba0864c
---
 Makefile.msc               | 18 ++++++++++++++++++
 ext/expert/expert.c        |  2 +-
 ext/expert/sqlite3expert.c | 36 +++++++++++++++++++-----------------
 manifest                   | 16 ++++++++--------
 manifest.uuid              |  2 +-
 5 files changed, 47 insertions(+), 27 deletions(-)

diff --git a/Makefile.msc b/Makefile.msc
index 5ef8decc38..143a90246f 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -1399,6 +1399,8 @@ TESTSRC = \
 # Statically linked extensions.
 #
 TESTEXT = \
+  $(TOP)\ext\expert\sqlite3expert.c \
+  $(TOP)\ext\expert\test_expert.c \
   $(TOP)\ext\misc\amatch.c \
   $(TOP)\ext\misc\carray.c \
   $(TOP)\ext\misc\closure.c \
@@ -2181,6 +2183,22 @@ sqlite3_analyzer.exe:	sqlite3_analyzer.c $(LIBRESOBJS)
 	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \
 		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
 
+sqlite3_expert.exe: $(SQLITE3C) $(TOP)\ext\expert\sqlite3expert.h $(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c
+	$(LTLINK) $(NO_WARN)	$(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c $(SQLITE3C) $(TLIBS)
+
+sqlite3_schemalint.c: $(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\schemalint.tcl $(SQLITE_TCL_DEP)
+	echo "#define TCLSH 2" > $@
+	echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@
+	copy $@ + $(SQLITE3C) + $(TOP)\src\tclsqlite.c >> $@
+	echo "static const char *tclsh_main_loop(void){" >> $@
+	echo "static const char *zMainloop = " >> $@
+	$(TCLSH_CMD) $(TOP)\tool\tostr.tcl $(TOP)\tool\schemalint.tcl >> $@
+	echo "; return zMainloop; }" >> $@
+
+sqlite3_schemalint.exe: $(TESTSRC) sqlite3_schemalint.c
+	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_schemalint.c \
+		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
+
 dbdump.exe:	$(TOP)\ext\misc\dbdump.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DDBDUMP_STANDALONE $(TOP)\ext\misc\dbdump.c $(SQLITE3C) \
 		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS)
diff --git a/ext/expert/expert.c b/ext/expert/expert.c
index 1c4b758d8f..13fc87ea89 100644
--- a/ext/expert/expert.c
+++ b/ext/expert/expert.c
@@ -94,7 +94,7 @@ int main(int argc, char **argv){
     for(i=1; i<(argc-1); i++){
       char *zArg = argv[i];
       if( zArg[0]=='-' && zArg[1]=='-' && zArg[2]!=0 ) zArg++;
-      int nArg = strlen(zArg);
+      int nArg = (int)strlen(zArg);
       if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-file", nArg) ){
         if( ++i==(argc-1) ) option_requires_argument("-file");
         rc = readSqlFromFile(p, argv[i], &zErr);
diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index 1e02328211..8627d196f0 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -25,6 +25,8 @@ typedef struct IdxStatement IdxStatement;
 typedef struct IdxTable IdxTable;
 typedef struct IdxWrite IdxWrite;
 
+#define STRLEN  (int)strlen
+
 /*
 ** A temp table name that we assume no user database will actually use.
 ** If this assumption proves incorrect triggers on the table with the
@@ -212,13 +214,13 @@ static int idxHashAdd(
   const char *zKey,
   const char *zVal
 ){
-  int nKey = strlen(zKey);
+  int nKey = STRLEN(zKey);
   int iHash = idxHashString(zKey, nKey);
-  int nVal = (zVal ? strlen(zVal) : 0);
+  int nVal = (zVal ? STRLEN(zVal) : 0);
   IdxHashEntry *pEntry;
   assert( iHash>=0 );
   for(pEntry=pHash->aHash[iHash]; pEntry; pEntry=pEntry->pHashNext){
-    if( strlen(pEntry->zKey)==nKey && 0==memcmp(pEntry->zKey, zKey, nKey) ){
+    if( STRLEN(pEntry->zKey)==nKey && 0==memcmp(pEntry->zKey, zKey, nKey) ){
       return 1;
     }
   }
@@ -246,11 +248,11 @@ static int idxHashAdd(
 static IdxHashEntry *idxHashFind(IdxHash *pHash, const char *zKey, int nKey){
   int iHash;
   IdxHashEntry *pEntry;
-  if( nKey<0 ) nKey = strlen(zKey);
+  if( nKey<0 ) nKey = STRLEN(zKey);
   iHash = idxHashString(zKey, nKey);
   assert( iHash>=0 );
   for(pEntry=pHash->aHash[iHash]; pEntry; pEntry=pEntry->pHashNext){
-    if( strlen(pEntry->zKey)==nKey && 0==memcmp(pEntry->zKey, zKey, nKey) ){
+    if( STRLEN(pEntry->zKey)==nKey && 0==memcmp(pEntry->zKey, zKey, nKey) ){
       return pEntry;
     }
   }
@@ -275,7 +277,7 @@ static const char *idxHashSearch(IdxHash *pHash, const char *zKey, int nKey){
 */
 static IdxConstraint *idxNewConstraint(int *pRc, const char *zColl){
   IdxConstraint *pNew;
-  int nColl = strlen(zColl);
+  int nColl = STRLEN(zColl);
 
   assert( *pRc==SQLITE_OK );
   pNew = (IdxConstraint*)idxMalloc(pRc, sizeof(IdxConstraint) * nColl + 1);
@@ -357,7 +359,7 @@ struct ExpertCsr {
 };
 
 static char *expertDequote(const char *zIn){
-  int n = strlen(zIn);
+  int n = STRLEN(zIn);
   char *zRet = sqlite3_malloc(n);
 
   assert( zIn[0]=='\'' );
@@ -666,7 +668,7 @@ static int idxGetTableInfo(
 ){
   sqlite3_stmt *p1 = 0;
   int nCol = 0;
-  int nTab = strlen(zTab);
+  int nTab = STRLEN(zTab);
   int nByte = sizeof(IdxTable) + nTab + 1;
   IdxTable *pNew = 0;
   int rc, rc2;
@@ -675,11 +677,11 @@ static int idxGetTableInfo(
   rc = idxPrintfPrepareStmt(db, &p1, pzErrmsg, "PRAGMA table_info=%Q", zTab);
   while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){
     const char *zCol = (const char*)sqlite3_column_text(p1, 1);
-    nByte += 1 + strlen(zCol);
+    nByte += 1 + STRLEN(zCol);
     rc = sqlite3_table_column_metadata(
         db, "main", zTab, zCol, 0, &zCol, 0, 0, 0
     );
-    nByte += 1 + strlen(zCol);
+    nByte += 1 + STRLEN(zCol);
     nCol++;
   }
   rc2 = sqlite3_reset(p1);
@@ -698,7 +700,7 @@ static int idxGetTableInfo(
   nCol = 0;
   while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){
     const char *zCol = (const char*)sqlite3_column_text(p1, 1);
-    int nCopy = strlen(zCol) + 1;
+    int nCopy = STRLEN(zCol) + 1;
     pNew->aCol[nCol].zName = pCsr;
     pNew->aCol[nCol].iPk = sqlite3_column_int(p1, 5);
     memcpy(pCsr, zCol, nCopy);
@@ -708,7 +710,7 @@ static int idxGetTableInfo(
         db, "main", zTab, zCol, 0, &zCol, 0, 0, 0
     );
     if( rc==SQLITE_OK ){
-      nCopy = strlen(zCol) + 1;
+      nCopy = STRLEN(zCol) + 1;
       pNew->aCol[nCol].zColl = pCsr;
       memcpy(pCsr, zCol, nCopy);
       pCsr += nCopy;
@@ -743,13 +745,13 @@ static char *idxAppendText(int *pRc, char *zIn, const char *zFmt, ...){
   va_list ap;
   char *zAppend = 0;
   char *zRet = 0;
-  int nIn = zIn ? strlen(zIn) : 0;
+  int nIn = zIn ? STRLEN(zIn) : 0;
   int nAppend = 0;
   va_start(ap, zFmt);
   if( *pRc==SQLITE_OK ){
     zAppend = sqlite3_vmprintf(zFmt, ap);
     if( zAppend ){
-      nAppend = strlen(zAppend);
+      nAppend = STRLEN(zAppend);
       zRet = (char*)sqlite3_malloc(nIn + nAppend + 1);
     }
     if( zAppend && zRet ){
@@ -1114,7 +1116,7 @@ int idxFindIndexes(
       int iOrder = sqlite3_column_int(pExplain, 1);
       int iFrom = sqlite3_column_int(pExplain, 2);
       const char *zDetail = (const char*)sqlite3_column_text(pExplain, 3);
-      int nDetail = strlen(zDetail);
+      int nDetail = STRLEN(zDetail);
       int i;
 
       for(i=0; i<nDetail; i++){
@@ -1577,7 +1579,7 @@ static int idxPopulateOneStat1(
       rc = sqlite3_reset(pWriteStat);
     }
 
-    pEntry = idxHashFind(&p->hIdx, zIdx, strlen(zIdx));
+    pEntry = idxHashFind(&p->hIdx, zIdx, STRLEN(zIdx));
     if( pEntry ){
       assert( pEntry->zVal2==0 );
       pEntry->zVal2 = zStat;
@@ -1816,7 +1818,7 @@ int sqlite3_expert_sql(
       if( pStmt ){
         IdxStatement *pNew;
         const char *z = sqlite3_sql(pStmt);
-        int n = strlen(z);
+        int n = STRLEN(z);
         pNew = (IdxStatement*)idxMalloc(&rc, sizeof(IdxStatement) + n+1);
         if( rc==SQLITE_OK ){
           pNew->zSql = (char*)&pNew[1];
diff --git a/manifest b/manifest
index ff6e6888b8..770ed52d6a 100644
--- a/manifest
+++ b/manifest
@@ -1,8 +1,8 @@
-C In\ssqlite3expert.c,\sdo\snot\scopy\sthe\sschema\sfor\svirtual\stables.\s\sUpdates\sto\nmakefiles\sto\smake\sbuilding\seasier.
-D 2017-05-03T12:15:20.951
+C Get\ssqlite3_expert\sbuilding\son\sWindows.
+D 2017-05-03T12:50:46.615
 F Makefile.in 2c991e7b1a2bb23c147406c3630b54d99c4931ae1fa0e8c6b6bf40a7a2fd02a3
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6
+F Makefile.msc 2b4876693e76420704bd2defe9f957b902ccb35ebe1fd071b80f70e862b7d444
 F README.md 2b15fae33852f2f53996774c21fb41e1d94181c4401a0e43ac93e11f2cc901b9
 F VERSION 0a0e02e16b44ea735b40118fc844311b2ab0d35b25fbeda5120aee62f973f663
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -41,9 +41,9 @@ F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91
 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3
-F ext/expert/expert.c dc88a3e73fdeb0d21e7d60b791120961853af931f3c509b508f4ac4a0233438e
+F ext/expert/expert.c 4791c5e064aea81b2b829fa95228b22283380ee370ea88a1e580103b75516ebf
 F ext/expert/expert1.test 1033e43071b69dc2f4e88fbf03fc7f18846c9865cac14f28c80f581437f09acb
-F ext/expert/sqlite3expert.c 87bac32f90492adfd99d3dc142d5c9af38a3bd961637202285077968051bc6e8
+F ext/expert/sqlite3expert.c 6ed4e84a06d1a29b2cf3009c0266573b88602d098055caa46c467154a64e0959
 F ext/expert/sqlite3expert.h af6354f8ee5c9e025024e63fec3bd640a802afcc3099a44d804752cf0791d811
 F ext/expert/test_expert.c 85f5c743a899063fa48296d21de2f32c26d09a21c8582b2a0bc482e8de183e7a
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
@@ -1585,7 +1585,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P af7d1596044980e0a18baa3ab6674779724dffbf18a152c72e53f11a08999e68
-R 446fb26db86c2ead652773ba0b926ac7
+P da15752dccf6090e40ec825db89048eca2b30185882225bf81f1891e914c2e7f
+R d4fb7977c87e361df2272c02e0401fc7
 U drh
-Z 183bc096a41586435af9a58068b8a076
+Z 1174ddb23c34b04fd900b7d4a8068d04
diff --git a/manifest.uuid b/manifest.uuid
index 388a18ee4f..8d1a6daaf2 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-da15752dccf6090e40ec825db89048eca2b30185882225bf81f1891e914c2e7f
\ No newline at end of file
+d8254047b30f7c1be486bf39d4420678604573b951b5cc83c19ebf74aba0864c
\ No newline at end of file

From bd72749f7d9c387e28a93b3877f5d416a85b7014 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 3 May 2017 13:05:08 +0000
Subject: [PATCH 046/488] Fix a harmless compiler warning on Windows.

FossilOrigin-Name: 593e5dd00cdf8fbc680951d68b53b38262c61c467703f6a8eb477226cf6beedd
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/vdbeaux.c |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 770ed52d6a..77a8c112c8 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Get\ssqlite3_expert\sbuilding\son\sWindows.
-D 2017-05-03T12:50:46.615
+C Fix\sa\sharmless\scompiler\swarning\son\sWindows.
+D 2017-05-03T13:05:08.456
 F Makefile.in 2c991e7b1a2bb23c147406c3630b54d99c4931ae1fa0e8c6b6bf40a7a2fd02a3
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 2b4876693e76420704bd2defe9f957b902ccb35ebe1fd071b80f70e862b7d444
@@ -479,7 +479,7 @@ F src/vdbe.c 356042d11e05064c43242020e8de97acef9fc8931cfc39ae7cf4cf91d6e42c19
 F src/vdbe.h f7d1456e28875c2dcb964056589b5b7149ab7edf39edeca801596a39bb3d3848
 F src/vdbeInt.h c070bc5c8b913bda0ceaa995cd4d939ded5e4fc96cf7c3c1c602d41b871f8ade
 F src/vdbeapi.c 5b08d82592bcff4470601fe78aaabebd50837860
-F src/vdbeaux.c 98ced78bb4d8f1c66a4519591804cbf34530f19c295a8589833aaa6004ea8731
+F src/vdbeaux.c dd4106907d8a52f80876c7cec030511220146056d0bb5d274f8528e251ffaed9
 F src/vdbeblob.c 359891617358deefc85bef7bcf787fa6b77facb9
 F src/vdbemem.c 2c70f8f5de6c71fb99a22c5b83be9fab5c47cdd8e279fa44a8c00cfed06d7e89
 F src/vdbesort.c e72fe02a2121386ba767ede8942e9450878b8fc873abf3d1b6824485f092570c
@@ -1585,7 +1585,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P da15752dccf6090e40ec825db89048eca2b30185882225bf81f1891e914c2e7f
-R d4fb7977c87e361df2272c02e0401fc7
+P d8254047b30f7c1be486bf39d4420678604573b951b5cc83c19ebf74aba0864c
+R efc1806e3149882e34f308cbcfce19a8
 U drh
-Z 1174ddb23c34b04fd900b7d4a8068d04
+Z 5eec4984a758352eb665dd5f89e1a5b7
diff --git a/manifest.uuid b/manifest.uuid
index 8d1a6daaf2..03edeee37c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d8254047b30f7c1be486bf39d4420678604573b951b5cc83c19ebf74aba0864c
\ No newline at end of file
+593e5dd00cdf8fbc680951d68b53b38262c61c467703f6a8eb477226cf6beedd
\ No newline at end of file
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index bde66dc1ba..de9ea10cbb 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -1612,7 +1612,7 @@ int sqlite3VdbeList(
   int rc = SQLITE_OK;                  /* Return code */
   Mem *pMem = &p->aMem[1];             /* First Mem of result set */
   int bFull = (p->explain==1 || (db->flags & SQLITE_FullEQP));
-  Op *pOp;
+  Op *pOp = 0;
 
   assert( p->explain );
   assert( p->magic==VDBE_MAGIC_RUN );

From e064d572e79203edb90e56f226ea1da792cb5ff3 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 4 May 2017 14:02:05 +0000
Subject: [PATCH 047/488] Remove the tool/schemalint.tcl script. And related
 Makefile entries. It is superseded by sqlite3_expert.

FossilOrigin-Name: 269bf52e27611cd00fa7f73ee98b395a30dec215232cf1c34f6b741112ba530b
---
 Makefile.in         |  14 --
 Makefile.msc        |  13 -
 main.mk             |  14 --
 manifest            |  19 +-
 manifest.uuid       |   2 +-
 tool/schemalint.tcl | 594 --------------------------------------------
 6 files changed, 10 insertions(+), 646 deletions(-)
 delete mode 100644 tool/schemalint.tcl

diff --git a/Makefile.in b/Makefile.in
index 0cde7a64f6..ac7062d5d4 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -1160,20 +1160,6 @@ sqlite3_analyzer$(TEXE): sqlite3_analyzer.c
 sqlite3_expert$(TEXE): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c
 	$(LTLINK)	$(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert $(TLIBS)
 
-sqlite3_schemalint.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/schemalint.tcl
-	echo "#define TCLSH 2" > $@
-	echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@
-	cat sqlite3.c $(TOP)/src/tclsqlite.c >> $@
-	echo "static const char *tclsh_main_loop(void){" >> $@
-	echo "static const char *zMainloop = " >> $@
-	$(TCLSH_CMD) $(TOP)/tool/tostr.tcl $(TOP)/tool/schemalint.tcl >> $@
-	echo "; return zMainloop; }" >> $@
-
-sqlite3_schemalint$(TEXE): $(TESTSRC) sqlite3_schemalint.c
-	$(LTLINK) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS)                  \
-		sqlite3_schemalint.c $(TESTSRC)               \
-		-o sqlite3_schemalint$(EXE) $(LIBTCL) $(TLIBS)
-
 dbdump$(TEXE): $(TOP)/ext/misc/dbdump.c sqlite3.lo
 	$(LTLINK) -DDBDUMP_STANDALONE -o $@ \
            $(TOP)/ext/misc/dbdump.c sqlite3.lo $(TLIBS)
diff --git a/Makefile.msc b/Makefile.msc
index 143a90246f..c8658ed15e 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -2186,19 +2186,6 @@ sqlite3_analyzer.exe:	sqlite3_analyzer.c $(LIBRESOBJS)
 sqlite3_expert.exe: $(SQLITE3C) $(TOP)\ext\expert\sqlite3expert.h $(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c
 	$(LTLINK) $(NO_WARN)	$(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c $(SQLITE3C) $(TLIBS)
 
-sqlite3_schemalint.c: $(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\schemalint.tcl $(SQLITE_TCL_DEP)
-	echo "#define TCLSH 2" > $@
-	echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@
-	copy $@ + $(SQLITE3C) + $(TOP)\src\tclsqlite.c >> $@
-	echo "static const char *tclsh_main_loop(void){" >> $@
-	echo "static const char *zMainloop = " >> $@
-	$(TCLSH_CMD) $(TOP)\tool\tostr.tcl $(TOP)\tool\schemalint.tcl >> $@
-	echo "; return zMainloop; }" >> $@
-
-sqlite3_schemalint.exe: $(TESTSRC) sqlite3_schemalint.c
-	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_schemalint.c \
-		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
-
 dbdump.exe:	$(TOP)\ext\misc\dbdump.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DDBDUMP_STANDALONE $(TOP)\ext\misc\dbdump.c $(SQLITE3C) \
 		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS)
diff --git a/main.mk b/main.mk
index 218598b549..8a0ab0674a 100644
--- a/main.mk
+++ b/main.mk
@@ -766,20 +766,6 @@ sqlite3_analyzer$(EXE): sqlite3_analyzer.c
 sqlite3_expert$(EXE): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c
 	$(TCCX) $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert $(THREADLIB)
 
-sqlite3_schemalint.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/schemalint.tcl
-	echo "#define TCLSH 2" > $@
-	echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@
-	cat sqlite3.c $(TOP)/src/tclsqlite.c >> $@
-	echo "static const char *tclsh_main_loop(void){" >> $@
-	echo "static const char *zMainloop = " >> $@
-	tclsh $(TOP)/tool/tostr.tcl $(TOP)/tool/schemalint.tcl >> $@
-	echo "; return zMainloop; }" >> $@
-
-sqlite3_schemalint$(EXE): $(TESTSRC) sqlite3_schemalint.c $(TOP)/ext/session/test_session.c
-	$(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS)                  \
-		sqlite3_schemalint.c $(TESTSRC) $(TOP)/ext/session/test_session.c  \
-		-o sqlite3_schemalint$(EXE) $(LIBTCL) $(THREADLIB)
-
 dbdump$(EXE):	$(TOP)/ext/misc/dbdump.c sqlite3.o
 	$(TCCX) -DDBDUMP_STANDALONE -o dbdump$(EXE) \
             $(TOP)/ext/misc/dbdump.c sqlite3.o $(THREADLIB)
diff --git a/manifest b/manifest
index 77a8c112c8..c5b72ddd17 100644
--- a/manifest
+++ b/manifest
@@ -1,8 +1,8 @@
-C Fix\sa\sharmless\scompiler\swarning\son\sWindows.
-D 2017-05-03T13:05:08.456
-F Makefile.in 2c991e7b1a2bb23c147406c3630b54d99c4931ae1fa0e8c6b6bf40a7a2fd02a3
+C Remove\sthe\stool/schemalint.tcl\sscript.\sAnd\srelated\sMakefile\sentries.\sIt\sis\nsuperseded\sby\ssqlite3_expert.
+D 2017-05-04T14:02:05.547
+F Makefile.in 2594c46dc86cb8cf0bd397c89c16b946ba45cd8c3459471a8634a9a9412a4724
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 2b4876693e76420704bd2defe9f957b902ccb35ebe1fd071b80f70e862b7d444
+F Makefile.msc 35879b76bc9136af684cc54b4178ebc6ab231ac0f24990bef703ad80a9330641
 F README.md 2b15fae33852f2f53996774c21fb41e1d94181c4401a0e43ac93e11f2cc901b9
 F VERSION 0a0e02e16b44ea735b40118fc844311b2ab0d35b25fbeda5120aee62f973f663
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -332,7 +332,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 7fcb8db53d07ea8124c2c43dac5fcee8095e4abeab1acdf114833b8805aa72c9
+F main.mk e916822e73ef2cf567d9fb079735d0bd1fb7afc89f75252a62c72d50730a49f3
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -1537,7 +1537,6 @@ F tool/replace.tcl 60f91e8dd06ab81f74d213ecbd9c9945f32ac048
 F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a
 F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5
 F tool/run-speed-test.sh f95d19fd669b68c4c38b6b475242841d47c66076
-F tool/schemalint.tcl 49690356702d6cac07e2bb1790eac73862e92926
 F tool/showdb.c e6bc9dba233bf1b57ca0a525a2bba762db4e223de84990739db3f09c46151b1e
 F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818
 F tool/showlocks.c 9920bcc64f58378ff1118caead34147201f48c68
@@ -1585,7 +1584,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d8254047b30f7c1be486bf39d4420678604573b951b5cc83c19ebf74aba0864c
-R efc1806e3149882e34f308cbcfce19a8
-U drh
-Z 5eec4984a758352eb665dd5f89e1a5b7
+P 593e5dd00cdf8fbc680951d68b53b38262c61c467703f6a8eb477226cf6beedd
+R bd679d9017e39a4713f5824e1924f07c
+U dan
+Z f03ecb4affa26d84072c6437b5574775
diff --git a/manifest.uuid b/manifest.uuid
index 03edeee37c..fc8dafbb37 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-593e5dd00cdf8fbc680951d68b53b38262c61c467703f6a8eb477226cf6beedd
\ No newline at end of file
+269bf52e27611cd00fa7f73ee98b395a30dec215232cf1c34f6b741112ba530b
\ No newline at end of file
diff --git a/tool/schemalint.tcl b/tool/schemalint.tcl
deleted file mode 100644
index 1c3f2e27c3..0000000000
--- a/tool/schemalint.tcl
+++ /dev/null
@@ -1,594 +0,0 @@
-if {[catch {
-
-set ::VERBOSE 0
-
-proc usage {} {
-  puts stderr "Usage: $::argv0 ?SWITCHES? DATABASE/SCHEMA"
-  puts stderr "  Switches are:"
-  puts stderr "  -select SQL     (recommend indexes for SQL statement)"
-  puts stderr "  -verbose        (increase verbosity of output)"
-  puts stderr "  -test           (run internal tests and then exit)"
-  puts stderr ""
-  exit
-}
-
-# Return the quoted version of identfier $id. Quotes are only added if 
-# they are required by SQLite.
-#
-# This command currently assumes that quotes are required if the 
-# identifier contains any ASCII-range characters that are not 
-# alpha-numeric or underscores.
-#
-proc quote {id} {
-  if {[requires_quote $id]} {
-    set x [string map {\" \"\"} $id]
-    return "\"$x\""
-  }
-  return $id
-}
-proc requires_quote {id} {
-  foreach c [split $id {}] {
-    if {[string is alnum $c]==0 && $c!="_"} {
-      return 1
-    }
-  }
-  return 0
-}
-
-# The argument passed to this command is a Tcl list of identifiers. The
-# value returned is the same list, except with each item quoted and the
-# elements comma-separated.
-#
-proc list_to_sql {L} {
-  set ret [list]
-  foreach l $L {
-    lappend ret [quote $l]
-  }
-  join $ret ", "
-}
-
-proc readfile {zFile} {
-  set fd [open $zFile]
-  set data [read $fd]
-  close $fd
-  return $data
-}
-
-proc process_cmdline_args {ctxvar argv} {
-  upvar $ctxvar G
-  set nArg [llength $argv]
-  set G(database) [lindex $argv end]
-
-  for {set i 0} {$i < [llength $argv]-1} {incr i} {
-    set k [lindex $argv $i]
-    switch -- $k {
-      -select {
-        incr i
-        if {$i>=[llength $argv]-1} usage
-        set zSelect [lindex $argv $i]
-        if {[file readable $zSelect]} {
-          lappend G(lSelect) [readfile $zSelect]
-        } else {
-          lappend G(lSelect) $zSelect
-        }
-      }
-      -verbose {
-        set ::VERBOSE 1
-      }
-      -test {
-        sqlidx_internal_tests
-      }
-      default {
-        usage
-      }
-    }
-  }
-
-  if {$G(database)=="-test"} {
-    sqlidx_internal_tests
-  }
-}
-
-proc open_database {ctxvar} {
-  upvar $ctxvar G
-  sqlite3 db ""
-
-  # Check if the "database" file is really an SQLite database. If so, copy
-  # it into the temp db just opened. Otherwise, assume that it is an SQL
-  # schema and execute it directly.
-  set fd [open $G(database)]
-  set hdr [read $fd 16]
-  if {$hdr == "SQLite format 3\000"} {
-    close $fd
-    sqlite3 db2 $G(database)
-    sqlite3_backup B db main db2 main
-    B step 2000000000
-    set rc [B finish]
-    db2 close
-    if {$rc != "SQLITE_OK"} { error "Failed to load database $G(database)" }
-  } else {
-    append hdr [read $fd]
-    db eval $hdr
-    close $fd
-  }
-}
-
-proc analyze_selects {ctxvar} {
-  upvar $ctxvar G
-  set G(trace) ""
-
-  # Collect a line of xTrace output for each loop in the set of SELECT
-  # statements.
-  proc xTrace {zMsg} { 
-    upvar G G
-    lappend G(trace) $zMsg 
-  }
-  db trace xTrace
-  foreach s $G(lSelect) {
-    set stmt [sqlite3_prepare_v2 db $s -1 dummy]
-    set rc [sqlite3_finalize $stmt]
-    if {$rc!="SQLITE_OK"} {
-      error "Failed to compile SQL: [sqlite3_errmsg db]"
-    }
-  }
-
-  db trace ""
-  if {$::VERBOSE} {
-    foreach t $G(trace) { puts "trace: $t" }
-  }
-
-  # puts $G(trace)
-}
-
-# The argument is a list of the form:
-#
-#    key1 {value1.1 value1.2} key2 {value2.1 value 2.2...}
-#
-# Values lists may be of any length greater than zero. This function returns
-# a list of lists created by pivoting on each values list. i.e. a list
-# consisting of the elements:
-#
-#   {{key1 value1.1} {key2 value2.1}}
-#   {{key1 value1.2} {key2 value2.1}}
-#   {{key1 value1.1} {key2 value2.2}}
-#   {{key1 value1.2} {key2 value2.2}}
-#
-proc expand_eq_list {L} {
-  set ll [list {}]
-  for {set i 0} {$i < [llength $L]} {incr i 2} {
-    set key [lindex $L $i]
-    set new [list]
-    foreach piv [lindex $L $i+1] {
-      foreach l $ll {
-        lappend new [concat $l [list [list $key $piv]]]
-      }
-    }
-    set ll $new
-  }
-
-  return $ll
-}
-
-#--------------------------------------------------------------------------
-# Formulate a CREATE INDEX statement that creates an index on table $tname.
-#
-proc eqset_to_index {ctxvar aCollVar tname eqset {range {}}} {
-  upvar $ctxvar G
-  upvar $aCollVar aColl
-
-  set rangeset [list]
-  foreach e [lsort $eqset] {
-    lappend rangeset [lindex $e 0] [lindex $e 1] ASC
-  }
-  set rangeset [concat $rangeset $range]
-
-  set lCols [list]
-  set idxname $tname
-
-  foreach {c collate dir} $rangeset {
-    append idxname "_$c"
-    set coldef [quote $c]
-
-    if {[string compare -nocase $collate $aColl($c)]!=0} {
-      append idxname [string tolower $collate]
-      append coldef " COLLATE [quote $collate]"
-    }
-
-    if {$dir=="DESC"} {
-      append coldef " DESC"
-      append idxname "desc"
-    }
-    lappend lCols $coldef
-  }
-
-  set create_index "CREATE INDEX [quote $idxname] ON [quote $tname]("
-  append create_index [join $lCols ", "]
-  append create_index ");"
-
-  set G(trial.$idxname) $create_index
-}
-
-proc expand_or_cons {L} {
-  set lRet [list [list]]
-  foreach elem $L {
-    set type [lindex $elem 0]
-    if {$type=="eq" || $type=="range"} {
-      set lNew [list]
-      for {set i 0} {$i < [llength $lRet]} {incr i} {
-        lappend lNew [concat [lindex $lRet $i] [list $elem]]
-      }
-      set lRet $lNew
-    } elseif {$type=="or"} {
-      set lNew [list]
-      foreach branch [lrange $elem 1 end] {
-        foreach b [expand_or_cons $branch] {
-          for {set i 0} {$i < [llength $lRet]} {incr i} {
-            lappend lNew [concat [lindex $lRet $i] $b]
-          }
-        }
-      }
-      set lRet $lNew
-    } 
-  }
-  return $lRet
-}
-
-#--------------------------------------------------------------------------
-# Argument $tname is the name of a table in the main database opened by
-# database handle [db]. $arrayvar is the name of an array variable in the
-# caller's context. This command populates the array with an entry mapping 
-# from column name to default collation sequence for each column of table
-# $tname. For example, if a table is declared:
-#
-#   CREATE TABLE t1(a COLLATE nocase, b, c COLLATE binary)
-#
-# the mapping is populated with:
-#
-#   map(a) -> "nocase"
-#   map(b) -> "binary"
-#   map(c) -> "binary"
-#
-proc sqlidx_get_coll_map {tname arrayvar} {
-  upvar $arrayvar aColl
-  set colnames [list]
-  set qname [quote $tname]
-  db eval "PRAGMA table_info = $qname" x { lappend colnames $x(name) }
-  db eval "CREATE INDEX schemalint_test ON ${qname}([list_to_sql $colnames])"
-  db eval "PRAGMA index_xinfo = schemalint_test" x { 
-    set aColl($x(name)) $x(coll)
-  }
-  db eval "DROP INDEX schemalint_test"
-}
-
-proc find_trial_indexes {ctxvar} {
-  upvar $ctxvar G
-  foreach t $G(trace) {
-    set tname [lindex $t 0]
-    catch { array unset mask }
-
-    # Invoke "PRAGMA table_info" on the table. Use the results to create
-    # an array mapping from column name to collation sequence. Store the
-    # array in local variable aColl.
-    #
-    sqlidx_get_coll_map $tname aColl
-
-    set orderby [list]
-    if {[lindex $t end 0]=="orderby"} {
-      set orderby [lrange [lindex $t end] 1 end]
-    }
-
-    foreach lCons [expand_or_cons [lrange $t 2 end]] {
-
-      # Populate the array mask() so that it contains an entry for each
-      # combination of prerequisite scans that may lead to distinct sets
-      # of constraints being usable.
-      #
-      catch { array unset mask }
-      set mask(0) 1
-      foreach a $lCons {
-        set type [lindex $a 0]
-        if {$type=="eq" || $type=="range"} {
-          set m [lindex $a 3]
-          foreach k [array names mask] { set mask([expr ($k & $m)]) 1 }
-          set mask($m) 1
-        }
-      }
-
-      # Loop once for each distinct prerequisite scan mask identified in
-      # the previous block.
-      #
-      foreach k [array names mask] {
-
-        # Identify the constraints available for prerequisite mask $k. For
-        # each == constraint, set an entry in the eq() array as follows:
-        # 
-        #   set eq(<col>) <collation>
-        #
-        # If there is more than one == constraint for a column, and they use
-        # different collation sequences, <collation> is replaced with a list
-        # of the possible collation sequences. For example, for:
-        #
-        #   SELECT * FROM t1 WHERE a=? COLLATE BINARY AND a=? COLLATE NOCASE
-        #
-        # Set the following entry in the eq() array:
-        #
-        #   set eq(a) {binary nocase}
-        #
-        # For each range constraint found an entry is appended to the $ranges
-        # list. The entry is itself a list of the form {<col> <collation>}.
-        #
-        catch {array unset eq}
-        set ranges [list]
-        foreach a $lCons {
-          set type [lindex $a 0]
-          if {$type=="eq" || $type=="range"} {
-            foreach {type col collate m} $a {
-              if {($m & $k)==$m} {
-                if {$type=="eq"} {
-                  lappend eq($col) $collate
-                } else {
-                  lappend ranges [list $col $collate ASC]
-                }
-              }
-            }
-          }
-        }
-        set ranges [lsort -unique $ranges]
-        if {$orderby != ""} {
-          lappend ranges $orderby
-        }
-
-        foreach eqset [expand_eq_list [array get eq]] {
-          if {$eqset != ""} {
-            eqset_to_index G aColl $tname $eqset
-          }
-
-          foreach r $ranges {
-            set tail [list]
-            foreach {c collate dir} $r {
-              set bSeen 0
-              foreach e $eqset {
-                if {[lindex $e 0] == $c} {
-                  set bSeen 1
-                  break
-                }
-              }
-              if {$bSeen==0} { lappend tail {*}$r }
-            }
-            if {[llength $tail]} {
-              eqset_to_index G aColl $tname $eqset $r
-            }
-          }
-        }
-      }
-    }
-  }
-
-  if {$::VERBOSE} {
-    foreach k [array names G trial.*] { puts "index: $G($k)" }
-  }
-}
-
-proc run_trials {ctxvar} {
-  upvar $ctxvar G
-  set ret [list]
-
-  foreach k [array names G trial.*] {
-    set idxname [lindex [split $k .] 1]
-    db eval $G($k)
-    set pgno [db one {SELECT rootpage FROM sqlite_master WHERE name = $idxname}]
-    set IDX($pgno) $idxname
-  }
-  db eval ANALYZE
-
-  catch { array unset used }
-  foreach s $G(lSelect) {
-    db eval "EXPLAIN $s" x {
-      if {($x(opcode)=="OpenRead" || $x(opcode)=="ReopenIdx")} {
-        if {[info exists IDX($x(p2))]} { set used($IDX($x(p2))) 1 }
-      }
-    }
-    foreach idx [array names used] {
-      lappend ret $G(trial.$idx)
-    }
-  }
-
-  set ret
-}
-
-proc sqlidx_init_context {varname} {
-  upvar $varname G
-  set G(lSelect)  [list]           ;# List of SELECT statements to analyze
-  set G(database) ""               ;# Name of database or SQL schema file
-  set G(trace)    [list]           ;# List of data from xTrace()
-}
-
-#-------------------------------------------------------------------------
-# The following is test code only.
-#
-proc sqlidx_one_test {tn schema select expected} {
-#  if {$tn!=2} return
-  sqlidx_init_context C
-
-  sqlite3 db ""
-  db collate "a b c" [list string compare]
-  db eval $schema
-  lappend C(lSelect) $select
-  analyze_selects C
-  find_trial_indexes C
-
-  set idxlist [run_trials C]
-  if {$idxlist != [list {*}$expected]} {
-    puts stderr "Test $tn failed"
-    puts stderr "Expected: $expected"
-    puts stderr "Got: $idxlist"
-    exit -1
-  }
-
-  db close
-
-  upvar nTest nTest
-  incr nTest
-}
-
-proc sqlidx_internal_tests {} {
-  set nTest 0
-
-
-  # No indexes for a query with no constraints.
-  sqlidx_one_test 0 {
-    CREATE TABLE t1(a, b, c);
-  } {
-    SELECT * FROM t1;
-  } {
-  }
-
-  sqlidx_one_test 1 {
-    CREATE TABLE t1(a, b, c);
-    CREATE TABLE t2(x, y, z);
-  } {
-    SELECT a FROM t1, t2 WHERE a=? AND x=c
-  } {
-    {CREATE INDEX t2_x ON t2(x);}
-    {CREATE INDEX t1_a_c ON t1(a, c);}
-  }
-
-  sqlidx_one_test 2 {
-    CREATE TABLE t1(a, b, c);
-  } {
-    SELECT * FROM t1 WHERE b>?;
-  } {
-    {CREATE INDEX t1_b ON t1(b);}
-  }
-
-  sqlidx_one_test 3 {
-    CREATE TABLE t1(a, b, c);
-  } {
-    SELECT * FROM t1 WHERE b COLLATE nocase BETWEEN ? AND ?
-  } {
-    {CREATE INDEX t1_bnocase ON t1(b COLLATE NOCASE);}
-  }
-
-  sqlidx_one_test 4 {
-    CREATE TABLE t1(a, b, c);
-  } {
-    SELECT a FROM t1 ORDER BY b;
-  } {
-    {CREATE INDEX t1_b ON t1(b);}
-  }
-
-  sqlidx_one_test 5 {
-    CREATE TABLE t1(a, b, c);
-  } {
-    SELECT a FROM t1 WHERE a=? ORDER BY b;
-  } {
-    {CREATE INDEX t1_a_b ON t1(a, b);}
-  }
-
-  sqlidx_one_test 5 {
-    CREATE TABLE t1(a, b, c);
-  } {
-    SELECT min(a) FROM t1
-  } {
-    {CREATE INDEX t1_a ON t1(a);}
-  }
-
-  sqlidx_one_test 6 {
-    CREATE TABLE t1(a, b, c);
-  } {
-    SELECT * FROM t1 ORDER BY a ASC, b COLLATE nocase DESC, c ASC;
-  } {
-    {CREATE INDEX t1_a_bnocasedesc_c ON t1(a, b COLLATE NOCASE DESC, c);}
-  }
-
-  sqlidx_one_test 7 {
-    CREATE TABLE t1(a COLLATE NOCase, b, c);
-  } {
-    SELECT * FROM t1 WHERE a=?
-  } {
-    {CREATE INDEX t1_a ON t1(a);}
-  }
-
-  # Tables with names that require quotes.
-  #
-  sqlidx_one_test 8.1 {
-    CREATE TABLE "t t"(a, b, c);
-  } {
-    SELECT * FROM "t t" WHERE a=?
-  } {
-    {CREATE INDEX "t t_a" ON "t t"(a);}
-  }
-  sqlidx_one_test 8.2 {
-    CREATE TABLE "t t"(a, b, c);
-  } {
-    SELECT * FROM "t t" WHERE b BETWEEN ? AND ?
-  } {
-    {CREATE INDEX "t t_b" ON "t t"(b);}
-  }
-  
-  # Columns with names that require quotes.
-  #
-  sqlidx_one_test 9.1 {
-    CREATE TABLE t3(a, "b b", c);
-  } {
-    SELECT * FROM t3 WHERE "b b" = ?
-  } {
-    {CREATE INDEX "t3_b b" ON t3("b b");}
-  }
-  sqlidx_one_test 9.2 {
-    CREATE TABLE t3(a, "b b", c);
-  } {
-    SELECT * FROM t3 ORDER BY "b b"
-  } {
-    {CREATE INDEX "t3_b b" ON t3("b b");}
-  }
-
-  # Collations with names that require quotes.
-  #
-  sqlidx_one_test 10.1 {
-    CREATE TABLE t4(a, b, c);
-  } {
-    SELECT * FROM t4 ORDER BY c COLLATE "a b c"
-  } {
-    {CREATE INDEX "t4_ca b c" ON t4(c COLLATE "a b c");}
-  }
-  sqlidx_one_test 10.2 {
-    CREATE TABLE t4(a, b, c);
-  } {
-    SELECT * FROM t4 WHERE c = ? COLLATE "a b c"
-  } {
-    {CREATE INDEX "t4_ca b c" ON t4(c COLLATE "a b c");}
-  }
-
-  # Transitive constraints
-  #
-  sqlidx_one_test 11.1 {
-    CREATE TABLE t5(a, b);
-    CREATE TABLE t6(c, d);
-  } {
-    SELECT * FROM t5, t6 WHERE a=? AND b=c AND c=?
-  } {
-    {CREATE INDEX t6_c ON t6(c);} 
-    {CREATE INDEX t5_a_b ON t5(a, b);}
-  }
-
-  puts "All $nTest tests passed"
-  exit
-}
-# End of internal test code.
-#-------------------------------------------------------------------------
-
-if {[info exists ::argv0]==0} { set ::argv0 [info nameofexec] }
-if {[info exists ::argv]==0} usage
-sqlidx_init_context D
-process_cmdline_args D $::argv
-open_database D
-analyze_selects D
-find_trial_indexes D
-foreach idx [run_trials D] { puts $idx }
-
-} err]} {
-  puts "ERROR: $err"
-  puts $errorInfo
-  exit 1
-}

From 21f2bafd9ba4cba4b6a837166cc673424dabe3f1 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 23 Sep 2017 07:46:54 +0000
Subject: [PATCH 048/488] Experimental change so that snapshot transactions
 always lock the wal file - preventing writers or truncate-checkpointers from
 wrapping it.

FossilOrigin-Name: d71eeaab9ecdeed772047498b781be1f0be0655af284b94cf676bb408ceea8b1
---
 manifest            |  18 ++++----
 manifest.uuid       |   2 +-
 src/wal.c           |   3 +-
 test/snapshot3.test | 100 ++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 113 insertions(+), 10 deletions(-)
 create mode 100644 test/snapshot3.test

diff --git a/manifest b/manifest
index 1938515870..c550d72b06 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Use\sthe\supdated\sWin32\sVFS\ssemantics\sfor\swinOpen\sfrom\scheck-in\s[5d03c738e9]\sfor\sWinRT,\set\sal,\sas\swell.
-D 2017-09-22T16:23:23.946
+C Experimental\schange\sso\sthat\ssnapshot\stransactions\salways\slock\sthe\swal\sfile\s-\npreventing\swriters\sor\struncate-checkpointers\sfrom\swrapping\sit.
+D 2017-09-23T07:46:54.932
 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f
@@ -537,7 +537,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c f1d5c23132fb0247af3e86146404112283ddedb6c518de0d4edc91cfb36970ef
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c 839db09792fead5052bb35e533fa485e134913d547d05b5f42e537b73e63f07a
+F src/wal.c 0ceefa9f2a0cb00033d312a04c2d5c2005a6404a6fccab22743de9b3c363154b
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6
 F src/where.c 21eea981920a13fd3c0ac3d6c128d0a34b22cbec064e4f0603375fe1ffe26ca6
@@ -1206,6 +1206,7 @@ F test/skipscan5.test 67817a4b6857c47e0e33ba3e506da6f23ef68de2
 F test/skipscan6.test 5866039d03a56f5bd0b3d172a012074a1d90a15b
 F test/snapshot.test 85735bd997a4f6d710140c28fd860519a299649f
 F test/snapshot2.test 867652ed4a13282dce218723535fad1c7b44c3c4
+F test/snapshot3.test 9719443594a04778861bd20d12596c5f880af177d6cd62f111da3198cafc6096
 F test/snapshot_fault.test 52c5e97ebd218846a8ae2da4d147d3e77d71f963
 F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f
 F test/softheap1.test 843cd84db9891b2d01b9ab64cef3e9020f98d087
@@ -1655,7 +1656,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 403b88a894d877b85bcc33133abad06c3c576e4928de4a4b0c091f74c4015355
-R 7f08f0df04c939d1dd5a187c9ae3cc49
-U mistachkin
-Z 321f82ff3326ba269549cfe661409f34
+P 2c03d8b8f028b6a736aaf2cf8b28a51b3434cf341c95cf3a80469e0a24acdd98
+R 44e3d692cd4dca7384d5a55f548b9b05
+T *branch * snapshots-lock-wal
+T *sym-snapshots-lock-wal *
+T -sym-trunk *
+U dan
+Z 91f0de7cf02d66dc9c74cc874f9821f7
diff --git a/manifest.uuid b/manifest.uuid
index 04153e1aac..b33987e64e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2c03d8b8f028b6a736aaf2cf8b28a51b3434cf341c95cf3a80469e0a24acdd98
\ No newline at end of file
+d71eeaab9ecdeed772047498b781be1f0be0655af284b94cf676bb408ceea8b1
\ No newline at end of file
diff --git a/src/wal.c b/src/wal.c
index 9930b84421..73a6db0e02 100644
--- a/src/wal.c
+++ b/src/wal.c
@@ -2250,8 +2250,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){
   pInfo = walCkptInfo(pWal);
   if( !useWal && pInfo->nBackfill==pWal->hdr.mxFrame 
 #ifdef SQLITE_ENABLE_SNAPSHOT
-   && (pWal->pSnapshot==0 || pWal->hdr.mxFrame==0
-     || 0==memcmp(&pWal->hdr, pWal->pSnapshot, sizeof(WalIndexHdr)))
+   && (pWal->pSnapshot==0 || pWal->hdr.mxFrame==0)
 #endif
   ){
     /* The WAL has been completely backfilled (or it is empty).
diff --git a/test/snapshot3.test b/test/snapshot3.test
new file mode 100644
index 0000000000..266f43dfd4
--- /dev/null
+++ b/test/snapshot3.test
@@ -0,0 +1,100 @@
+# 2016 September 23
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library. The focus
+# of this file is the sqlite3_snapshot_xxx() APIs.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+ifcapable !snapshot {finish_test; return}
+set testprefix snapshot3
+
+# This test does not work with the inmemory_journal permutation. The reason
+# is that each connection opened as part of this permutation executes
+# "PRAGMA journal_mode=memory", which fails if the database is in wal mode
+# and there are one or more existing connections.
+if {[permutation]=="inmemory_journal"} {
+  finish_test
+  return
+}
+
+#-------------------------------------------------------------------------
+# This block of tests verifies that it is not possible to wrap the wal
+# file - using a writer or a "PRAGMA wal_checkpoint = TRUNCATE" - while
+# there is an open snapshot transaction (transaction opened using
+# sqlite3_snapshot_open()).
+#
+do_execsql_test 1.0 {
+  CREATE TABLE t1(y);
+  PRAGMA journal_mode = wal;
+  INSERT INTO t1 VALUES(1);
+  INSERT INTO t1 VALUES(2);
+  INSERT INTO t1 VALUES(3);
+  INSERT INTO t1 VALUES(4);
+} {wal}
+
+do_test 1.1 {
+  sqlite3 db2 test.db
+  sqlite3 db3 test.db
+
+  execsql {SELECT * FROM sqlite_master} db2
+  execsql {SELECT * FROM sqlite_master} db3
+
+  db2 trans { set snap [sqlite3_snapshot_get_blob db2 main] }
+  db2 eval { SELECT * FROM t1 }
+} {1 2 3 4}
+
+do_test 1.2 {
+  execsql BEGIN db2
+  sqlite3_snapshot_open_blob db2 main $snap
+  db2 eval { SELECT * FROM t1 }
+} {1 2 3 4}
+
+do_test 1.2 {
+  execsql END db2
+  execsql { PRAGMA wal_checkpoint }
+
+  execsql BEGIN db2
+  sqlite3_snapshot_open_blob db2 main $snap
+  db2 eval { SELECT * FROM t1 }
+} {1 2 3 4}
+
+set sz [file size test.db-wal]
+do_test 1.3 {
+  execsql { PRAGMA wal_checkpoint = truncate }
+  file size test.db-wal
+} $sz
+
+do_test 1.4 {
+  execsql BEGIN db3
+  list [catch { sqlite3_snapshot_open_blob db3 main $snap } msg] $msg
+} {0 {}}
+
+do_test 1.5 {
+  db3 eval { SELECT * FROM t1; END }
+} {1 2 3 4}
+
+do_test 1.6 {
+  db2 eval { SELECT * FROM t1; END }
+} {1 2 3 4}
+
+do_test 1.7 {
+  execsql { PRAGMA wal_checkpoint = truncate }
+  file size test.db-wal
+} 0
+
+do_test 1.8 {
+  execsql BEGIN db3
+  list [catch { sqlite3_snapshot_open_blob db3 main $snap } msg] $msg
+} {1 SQLITE_BUSY_SNAPSHOT}
+
+finish_test
+

From 7fdb9a7edcd0f17f17c62127ee3912c655aee173 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 11 Oct 2017 20:10:22 +0000
Subject: [PATCH 049/488] Change some internal details to bring this branch
 closer to the code on trunk.

FossilOrigin-Name: 58e42cfd7d4af2279dab8dd895dca7afbdf9035445bc81bd4a7461ea24e846fc
---
 manifest        | 26 +++++++++++++-------------
 manifest.uuid   |  2 +-
 src/build.c     | 12 ++++++------
 src/main.c      | 20 +++++++++++++++++---
 src/prepare.c   |  2 +-
 src/sqliteInt.h |  4 ++--
 src/trigger.c   |  2 +-
 src/vdbe.c      |  6 +++---
 src/vdbeaux.c   |  2 +-
 9 files changed, 45 insertions(+), 31 deletions(-)

diff --git a/manifest b/manifest
index fd41b85331..c1b89c6095 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\schanges\sfrom\strunk.
-D 2017-05-15T17:56:48.105
+C Change\ssome\sinternal\sdetails\sto\sbring\sthis\sbranch\scloser\sto\sthe\scode\son\strunk.
+D 2017-10-11T20:10:22.514
 F Makefile.in 2594c46dc86cb8cf0bd397c89c16b946ba45cd8c3459471a8634a9a9412a4724
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 19636d6f00b600680edca5e437cd895599cffc11ceaa6ddac866373b98a9e644
@@ -355,7 +355,7 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
 F src/btree.c 8c1fd4cfa2b0bf021386e0a1f4e30b64eea7a2c1bc2e0c3e5901a626b1ab6aa9
 F src/btree.h 80f518c0788be6cec8d9f8e13bd8e380df299d2b5e4ac340dc887b0642647cfc
 F src/btreeInt.h a392d353104b4add58b4a59cb185f5d5693dde832c565b77d8d4c343ed98f610
-F src/build.c 3fd46781483b527ee18508e7854e87e60a259211bb9bbf16b6fafaf08a043a64
+F src/build.c 4026a9c554b233e50c5e9ad46963e676cf54dd2306d952aa1eaa07a1bc9ce14f
 F src/callback.c 2e76147783386374bf01b227f752c81ec872d730
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c 47d91a25ad8f199a71a5b1b7b169d6dd0d6e98c5719eca801568798743d1161c
@@ -374,7 +374,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c d4bb3a135948553d18cf992f76f7ed7b18aa0327f250607b5a6671e55d9947d5
 F src/legacy.c e88ed13c2d531decde75d42c2e35623fb9ce3cb0
 F src/loadext.c a72909474dadce771d3669bf84bf689424f6f87d471fee898589c3ef9b2acfd9
-F src/main.c 253de9f00311bd2b9898c93f22556a741e1a63310785c70356141326765fb64a
+F src/main.c d88cf6d3a281c983f4e00a9fbe8be011504e5d98f06237639b3f546da57d197b
 F src/malloc.c e20bb2b48abec52d3faf01cce12e8b4f95973755fafec98d45162dfdab111978
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -404,7 +404,7 @@ F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490
 F src/pcache1.c 1195a21fe28e223e024f900b2011e80df53793f0356a24caace4188b098540dc
 F src/pragma.c 7fef375edafdb7ae9ba938b992aa726e18bf07b0599cfed040a088a262744b7a
 F src/pragma.h 37a1311d0388db480388d7ec09054f7103045eff20d4971f8a433b77f40b9921
-F src/prepare.c 7c46b5c7be9e19a1bf87777f0b7f9fb257b5ff9856c46de49f2354acfbeb4c86
+F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a
 F src/printf.c 8757834f1b54dae512fb25eb1acc8e94a0d15dd2290b58f2563f65973265adb2
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 3e518b962d932a997fae373366880fc028c75706
@@ -414,7 +414,7 @@ F src/shell.c a37d96b20b3644d0eb905df5aa7a0fcf9f6e73c15898337230c760a24a8df794
 F src/sqlite.h.in 1672a3a22b877d5fbd5d3d243591e11e241c5abc321dbf969429c4bf5656acd9
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 58fd0676d3111d02e62e5a35992a7d3da5d3f88753acc174f2d37b774fbbdd28
-F src/sqliteInt.h a8be6c63ce04fc759e3d8ca2dee2fa2d4128b0a4bf2031c3f6e482fd5c5abdfe
+F src/sqliteInt.h c4d3d3902d7beea52c4fdeb8fda1b9e80bcff0f51230751ec98e2def7cbac801
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -471,16 +471,16 @@ F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
 F src/tokenize.c 1003d6d90c6783206c711f0a9397656fa5b055209f4d092caa43bb3bf5215db5
 F src/treeview.c 6cf8d7fe9e63fae57dad1bb57f6615e14eac0c527e43d868e805042cae8ed1f7
-F src/trigger.c 134b8e7b61317ab7b2a2dd12eb1b9aa2e23ac5bc4a05e63e35b3609b6b30a7c0
+F src/trigger.c c9f0810043b265724fdb1bdd466894f984dfc182
 F src/update.c c443935c652af9365e033f756550b5032d02e1b06eb2cb890ed7511ae0c051dc
 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c
 F src/util.c fc081ec6f63448dcd80d3dfad35baecfa104823254a815b081a4d9fe76e1db23
 F src/vacuum.c 1fe4555cd8c9b263afb85b5b4ee3a4a4181ad569
-F src/vdbe.c 356042d11e05064c43242020e8de97acef9fc8931cfc39ae7cf4cf91d6e42c19
+F src/vdbe.c 9bac2bc2313ed682e6f48ccff6644d3263341885bfcbb3cdea7b720c722be2d5
 F src/vdbe.h f7d1456e28875c2dcb964056589b5b7149ab7edf39edeca801596a39bb3d3848
 F src/vdbeInt.h 1ecdacc1322fdd3241ec30c32a480e328a6f864e532dc53fae8e0ab68121aebf
 F src/vdbeapi.c dc904b3c5e459727993c2421e653e29d63223846d129fae98adc782b0a996481
-F src/vdbeaux.c 73f10e298ed1e0565a30d8f8c909534ded5a32075a27542cc624d711ddcdc6b4
+F src/vdbeaux.c 4f54ba67ffc192e06b3b470c09c5db69044673da4e16f371670b60094794ad99
 F src/vdbeblob.c 359891617358deefc85bef7bcf787fa6b77facb9
 F src/vdbemem.c 2c70f8f5de6c71fb99a22c5b83be9fab5c47cdd8e279fa44a8c00cfed06d7e89
 F src/vdbesort.c e72fe02a2121386ba767ede8942e9450878b8fc873abf3d1b6824485f092570c
@@ -1586,7 +1586,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 269bf52e27611cd00fa7f73ee98b395a30dec215232cf1c34f6b741112ba530b bb0d9281588b8cc24bf2f1f10d0c56277004226adaa2ce5037782503b283b45d
-R 7d3a8849d7369ade14c69fc7df3fcb28
-U drh
-Z 969ec953bff5414f5fad1a53a1834b03
+P 6e0f64ab5eafda5d9e61e00c89af3c1ea2c5aa29821da1bdbcab040957b12403
+R ae231083620f188faca2d9c880ca5095
+U dan
+Z b2966b026a12abf27911ccd1f8c99303
diff --git a/manifest.uuid b/manifest.uuid
index 92a71c9011..8fd1e9c5ad 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6e0f64ab5eafda5d9e61e00c89af3c1ea2c5aa29821da1bdbcab040957b12403
\ No newline at end of file
+58e42cfd7d4af2279dab8dd895dca7afbdf9035445bc81bd4a7461ea24e846fc
\ No newline at end of file
diff --git a/src/build.c b/src/build.c
index 47339e82b6..e04406d857 100644
--- a/src/build.c
+++ b/src/build.c
@@ -479,7 +479,7 @@ void sqlite3UnlinkAndDeleteIndex(sqlite3 *db, int iDb, const char *zIdxName){
     }
     freeIndex(db, pIndex);
   }
-  db->bInternChanges = 1;
+  db->flags |= SQLITE_InternChanges;
 }
 
 /*
@@ -551,7 +551,7 @@ void sqlite3ResetAllSchemasOfConnection(sqlite3 *db){
       sqlite3SchemaClear(pDb->pSchema);
     }
   }
-  db->bInternChanges = 0;
+  db->flags &= ~SQLITE_InternChanges;
   sqlite3VtabUnlockList(db);
   sqlite3BtreeLeaveAll(db);
   sqlite3CollapseDatabaseArray(db);
@@ -561,7 +561,7 @@ void sqlite3ResetAllSchemasOfConnection(sqlite3 *db){
 ** This routine is called when a commit occurs.
 */
 void sqlite3CommitInternalChanges(sqlite3 *db){
-  db->bInternChanges = 0;
+  db->flags &= ~SQLITE_InternChanges;
 }
 
 /*
@@ -665,7 +665,7 @@ void sqlite3UnlinkAndDeleteTable(sqlite3 *db, int iDb, const char *zTabName){
   pDb = &db->aDb[iDb];
   p = sqlite3HashInsert(&pDb->pSchema->tblHash, zTabName, 0);
   sqlite3DeleteTable(db, p);
-  db->bInternChanges = 1;
+  db->flags |= SQLITE_InternChanges;
 }
 
 /*
@@ -2051,7 +2051,7 @@ void sqlite3EndTable(
       return;
     }
     pParse->pNewTable = 0;
-    db->bInternChanges = 1;
+    db->flags |= SQLITE_InternChanges;
 
 #ifndef SQLITE_OMIT_ALTERTABLE
     if( !p->pSelect ){
@@ -3320,7 +3320,7 @@ void sqlite3CreateIndex(
       sqlite3OomFault(db);
       goto exit_create_index;
     }
-    db->bInternChanges = 1;
+    db->flags |= SQLITE_InternChanges;
     if( pTblName!=0 ){
       pIndex->tnum = db->init.newTnum;
     }
diff --git a/src/main.c b/src/main.c
index d39c49a73d..2212224741 100644
--- a/src/main.c
+++ b/src/main.c
@@ -801,6 +801,21 @@ int sqlite3_db_config(sqlite3 *db, int op, ...){
       rc = setupLookaside(db, pBuf, sz, cnt);
       break;
     }
+    case SQLITE_DBCONFIG_FULL_EQP: {
+      int onoff = va_arg(ap, int);
+      int *pRes = va_arg(ap, int*);
+      if( onoff>0 ){
+        db->bFullEQP = 1;
+      }else if( onoff==0 ){
+        db->bFullEQP = 0;
+      }
+      sqlite3ExpirePreparedStatements(db);
+      if( pRes ){
+        *pRes = db->bFullEQP;
+      }
+      rc = SQLITE_OK;
+      break;
+    }
     default: {
       static const struct {
         int op;      /* The opcode */
@@ -811,7 +826,6 @@ int sqlite3_db_config(sqlite3 *db, int op, ...){
         { SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, SQLITE_Fts3Tokenizer  },
         { SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, SQLITE_LoadExtension  },
         { SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE,      SQLITE_NoCkptOnClose  },
-        { SQLITE_DBCONFIG_FULL_EQP,              SQLITE_FullEQP  },
       };
       unsigned int i;
       rc = SQLITE_ERROR; /* IMP: R-42790-23372 */
@@ -1253,7 +1267,7 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){
   ** the database rollback and schema reset, which can cause false
   ** corruption reports in some cases.  */
   sqlite3BtreeEnterAll(db);
-  schemaChange = db->bInternChanges && db->init.busy==0;
+  schemaChange = (db->flags & SQLITE_InternChanges)!=0 && db->init.busy==0;
 
   for(i=0; i<db->nDb; i++){
     Btree *p = db->aDb[i].pBt;
@@ -1267,7 +1281,7 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){
   sqlite3VtabRollback(db);
   sqlite3EndBenignMalloc();
 
-  if( db->bInternChanges && db->init.busy==0 ){
+  if( (db->flags&SQLITE_InternChanges)!=0 && db->init.busy==0 ){
     sqlite3ExpirePreparedStatements(db);
     sqlite3ResetAllSchemasOfConnection(db);
   }
diff --git a/src/prepare.c b/src/prepare.c
index aa36cca166..74127bc76b 100644
--- a/src/prepare.c
+++ b/src/prepare.c
@@ -354,7 +354,7 @@ error_out:
 */
 int sqlite3Init(sqlite3 *db, char **pzErrMsg){
   int i, rc;
-  int commit_internal = db->bInternChanges==0;
+  int commit_internal = !(db->flags&SQLITE_InternChanges);
   
   assert( sqlite3_mutex_held(db->mutex) );
   assert( sqlite3BtreeHoldsMutex(db->aDb[0].pBt) );
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index d1c80d5618..af99b6c085 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1334,7 +1334,7 @@ struct sqlite3 {
   u8 mTrace;                    /* zero or more SQLITE_TRACE flags */
   u8 skipBtreeMutex;            /* True if no shared-cache backends */
   u8 nSqlExec;                  /* Number of pending OP_SqlExec opcodes */
-  u8 bInternChanges;            /* There are uncommited schema changes */
+  u8 bFullEQP;                  /* Include triggers in EQP output */
   int nextPagesize;             /* Pagesize after VACUUM if >0 */
   u32 magic;                    /* Magic number for detect library misuse */
   int nChange;                  /* Value returned by sqlite3_changes() */
@@ -1450,7 +1450,7 @@ struct sqlite3 {
 **      SQLITE_CacheSpill    == PAGER_CACHE_SPILL
 */
 #define SQLITE_VdbeTrace      0x00000001  /* True to trace VDBE execution */
-#define SQLITE_FullEQP        0x00000002  /* Include triggers in EQP output */
+#define SQLITE_InternChanges  0x00000002  /* Uncommitted Hash table changes */
 #define SQLITE_FullColNames   0x00000004  /* Show full column names on SELECT */
 #define SQLITE_FullFSync      0x00000008  /* Use full fsync on the backend */
 #define SQLITE_CkptFullFSync  0x00000010  /* Use full fsync for checkpoint */
diff --git a/src/trigger.c b/src/trigger.c
index 197dcda6c3..bdf964084b 100644
--- a/src/trigger.c
+++ b/src/trigger.c
@@ -584,7 +584,7 @@ void sqlite3UnlinkAndDeleteTrigger(sqlite3 *db, int iDb, const char *zName){
       *pp = (*pp)->pNext;
     }
     sqlite3DeleteTrigger(db, pTrigger);
-    db->bInternChanges = 1;
+    db->flags |= SQLITE_InternChanges;
   }
 }
 
diff --git a/src/vdbe.c b/src/vdbe.c
index 6d1b334cdf..e6c964245f 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -3056,7 +3056,7 @@ case OP_Savepoint: {
         int isSchemaChange;
         iSavepoint = db->nSavepoint - iSavepoint - 1;
         if( p1==SAVEPOINT_ROLLBACK ){
-          isSchemaChange = db->bInternChanges;
+          isSchemaChange = (db->flags & SQLITE_InternChanges)!=0;
           for(ii=0; ii<db->nDb; ii++){
             rc = sqlite3BtreeTripAllCursors(db->aDb[ii].pBt,
                                        SQLITE_ABORT_ROLLBACK,
@@ -3075,7 +3075,7 @@ case OP_Savepoint: {
         if( isSchemaChange ){
           sqlite3ExpirePreparedStatements(db);
           sqlite3ResetAllSchemasOfConnection(db);
-          db->bInternChanges = 1;
+          db->flags = (db->flags | SQLITE_InternChanges);
         }
       }
   
@@ -3355,7 +3355,7 @@ case OP_SetCookie: {
   if( pOp->p2==BTREE_SCHEMA_VERSION ){
     /* When the schema cookie changes, record the new cookie internally */
     pDb->pSchema->schema_cookie = pOp->p3;
-    db->bInternChanges = 1;
+    db->flags |= SQLITE_InternChanges;
   }else if( pOp->p2==BTREE_FILE_FORMAT ){
     /* Record changes in the file format */
     pDb->pSchema->file_format = pOp->p3;
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index 87b1d16316..97da4628bd 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -1611,7 +1611,7 @@ int sqlite3VdbeList(
   int i;                               /* Loop counter */
   int rc = SQLITE_OK;                  /* Return code */
   Mem *pMem = &p->aMem[1];             /* First Mem of result set */
-  int bFull = (p->explain==1 || (db->flags & SQLITE_FullEQP));
+  int bFull = (p->explain==1 || db->bFullEQP);
   Op *pOp = 0;
 
   assert( p->explain );

From 85b76a284e63192324782f953bf3df053189def2 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 12 Oct 2017 20:24:09 +0000
Subject: [PATCH 050/488] Fix the EXPLAIN processing so that it returns
 SQLITE_ERROR on an OOM, as it should.

FossilOrigin-Name: 4ec63ef233c9239d3d383a99c234ef6604d55a9838c16f57e56c2a120aeb63c4
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/vdbeaux.c |  9 ++++++---
 3 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index 57a59acd5c..6df14defa2 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\sthis\sbranch\sto\smatch\slatest\strunk.
-D 2017-10-12T14:13:31.376
+C Fix\sthe\sEXPLAIN\sprocessing\sso\sthat\sit\sreturns\sSQLITE_ERROR\son\san\sOOM,\sas\sit\nshould.
+D 2017-10-12T20:24:09.195
 F Makefile.in 51259a193f348a96b0ebc60ad84cfe83fdeecdd1d0ecb24f0e278a35ac1449da
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e3fd2bfdf18211892e3f09db7e2841753fe61cc0747a0b0e87d77406840c3f6e
@@ -535,7 +535,7 @@ F src/vdbe.c 176c0897af0aedecd3abc9afaf7fa80eaa7cf5eaf62583de256a9961df474373
 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97
 F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9
 F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1
-F src/vdbeaux.c 32530e6e635649244399e2230b41f7dcfc95c40b5a4dcf81611b8aeec6b9d064
+F src/vdbeaux.c 2b10f70d6479f990e66d0e7bbd3c58b85cd8388c1dba5b89698b7a42c185d0f2
 F src/vdbeblob.c 635a79b60340a6a14a622ea8dcb081f0a66b1ac3836870c587f232eec08c0286
 F src/vdbemem.c 5c1533bf756918b4e46b2ed2bb82c29c7c651e1e37bbd0a0d8731a68787598ff
 F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f
@@ -1662,7 +1662,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2719cf5c5bbe8e31d18368d54d968af3878ad2e15f0666e18d7b567d7439c451 36acc0a97fdcc6f54f29c68c4e131702f69c3e59e58237ff4e5c647928699956
-R 7d4d36d7e390bddaee9373964ffc0b0e
-U dan
-Z 45c234430da48e38efb9390ff8797b6d
+P d325da6c50f2f9c10f50b3839f87c155876c339879bb8bdb82567b5ff34ef7ba
+R 6202e14e92fa1dfd47a763d4d423b320
+U drh
+Z 6985a8a5847e041f83568bb74bfda40b
diff --git a/manifest.uuid b/manifest.uuid
index 6530f4e47e..ca99924a85 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d325da6c50f2f9c10f50b3839f87c155876c339879bb8bdb82567b5ff34ef7ba
\ No newline at end of file
+4ec63ef233c9239d3d383a99c234ef6604d55a9838c16f57e56c2a120aeb63c4
\ No newline at end of file
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index 727a43ef97..399981f480 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -1653,7 +1653,7 @@ int sqlite3VdbeList(
   releaseMemArray(pMem, 8);
   p->pResultSet = 0;
 
-  if( p->rc==SQLITE_NOMEM_BKPT ){
+  if( p->rc==SQLITE_NOMEM ){
     /* This happens if a malloc() inside a call to sqlite3_column_text() or
     ** sqlite3_column_text16() failed.  */
     sqlite3OomFault(db);
@@ -1720,8 +1720,11 @@ int sqlite3VdbeList(
         if( apSub[j]==pOp->p4.pProgram ) break;
       }
       if( j==nSub ){
-        rc = sqlite3VdbeMemGrow(pSub, nByte, nSub!=0);
-        if( rc!=SQLITE_OK ) break;
+        p->rc = sqlite3VdbeMemGrow(pSub, nByte, nSub!=0);
+        if( p->rc!=SQLITE_OK ){
+          rc = SQLITE_ERROR;
+          break;
+        }
         apSub = (SubProgram **)pSub->z;
         apSub[nSub++] = pOp->p4.pProgram;
         pSub->flags |= MEM_Blob;

From ae542401d801b95b5805f3f291a2f64ee074df90 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 13 Oct 2017 14:20:15 +0000
Subject: [PATCH 051/488] Update main.mk to build the sqlite3_expert program
 with -DSQLITE_THREADSAFE=0 and -DSQLITE_OMIT_LOAD_EXTENSION. To minimize
 dependencies.

FossilOrigin-Name: 4c68ad7da48c941eb1ca0210a85d95467f3f3640bdc55a02686166471806ca4e
---
 main.mk       |  2 +-
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/main.mk b/main.mk
index bfbfee1a4e..a92a8137f5 100644
--- a/main.mk
+++ b/main.mk
@@ -788,7 +788,7 @@ sqlite3_analyzer$(EXE): sqlite3_analyzer.c
 	$(TCCX) $(TCL_FLAGS) sqlite3_analyzer.c -o $@ $(LIBTCL) $(THREADLIB) 
 
 sqlite3_expert$(EXE): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c
-	$(TCCX) $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert $(THREADLIB)
+	$(TCCX) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert $(THREADLIB)
 
 dbdump$(EXE):	$(TOP)/ext/misc/dbdump.c sqlite3.o
 	$(TCCX) -DDBDUMP_STANDALONE -o dbdump$(EXE) \
diff --git a/manifest b/manifest
index 6df14defa2..bf28bea373 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sthe\sEXPLAIN\sprocessing\sso\sthat\sit\sreturns\sSQLITE_ERROR\son\san\sOOM,\sas\sit\nshould.
-D 2017-10-12T20:24:09.195
+C Update\smain.mk\sto\sbuild\sthe\ssqlite3_expert\sprogram\swith\s-DSQLITE_THREADSAFE=0\nand\s-DSQLITE_OMIT_LOAD_EXTENSION.\sTo\sminimize\sdependencies.
+D 2017-10-13T14:20:15.203
 F Makefile.in 51259a193f348a96b0ebc60ad84cfe83fdeecdd1d0ecb24f0e278a35ac1449da
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e3fd2bfdf18211892e3f09db7e2841753fe61cc0747a0b0e87d77406840c3f6e
@@ -388,7 +388,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 4ba2640f924e022ec66cadcb629e7e1d01aeb1932a004f92abc6238eac1c6a56
+F main.mk 6ec9f289c46fdc66928782d7ff0d1b07c2e4c6ca553a3fe4f94e793ac53e8db6
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -1662,7 +1662,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d325da6c50f2f9c10f50b3839f87c155876c339879bb8bdb82567b5ff34ef7ba
-R 6202e14e92fa1dfd47a763d4d423b320
-U drh
-Z 6985a8a5847e041f83568bb74bfda40b
+P 4ec63ef233c9239d3d383a99c234ef6604d55a9838c16f57e56c2a120aeb63c4
+R d11eafd1958fe2d9bdec327b55b3ebe5
+U dan
+Z 3e15c410e04c5e53e4b05cff4437d5be
diff --git a/manifest.uuid b/manifest.uuid
index ca99924a85..424d577e4a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4ec63ef233c9239d3d383a99c234ef6604d55a9838c16f57e56c2a120aeb63c4
\ No newline at end of file
+4c68ad7da48c941eb1ca0210a85d95467f3f3640bdc55a02686166471806ca4e
\ No newline at end of file

From 2da1504432a7e0efb4ccd119db29eb3dea2d6ad8 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 13 Oct 2017 16:24:32 +0000
Subject: [PATCH 052/488] Fix main.mk to name the win32 executable
 "sqlite3_expert.exe", not "sqlite3_expert".

FossilOrigin-Name: e38571d518ff399d1033b33843dae60bc61181cdf692dcd8e2e14746bf0d1e61
---
 main.mk       |  2 +-
 manifest      | 13 +++++++------
 manifest.uuid |  2 +-
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/main.mk b/main.mk
index a92a8137f5..b1bdf7a21c 100644
--- a/main.mk
+++ b/main.mk
@@ -788,7 +788,7 @@ sqlite3_analyzer$(EXE): sqlite3_analyzer.c
 	$(TCCX) $(TCL_FLAGS) sqlite3_analyzer.c -o $@ $(LIBTCL) $(THREADLIB) 
 
 sqlite3_expert$(EXE): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c
-	$(TCCX) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert $(THREADLIB)
+	$(TCCX) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert$(EXE) $(THREADLIB)
 
 dbdump$(EXE):	$(TOP)/ext/misc/dbdump.c sqlite3.o
 	$(TCCX) -DDBDUMP_STANDALONE -o dbdump$(EXE) \
diff --git a/manifest b/manifest
index bf28bea373..b371ad7c22 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\smain.mk\sto\sbuild\sthe\ssqlite3_expert\sprogram\swith\s-DSQLITE_THREADSAFE=0\nand\s-DSQLITE_OMIT_LOAD_EXTENSION.\sTo\sminimize\sdependencies.
-D 2017-10-13T14:20:15.203
+C Fix\smain.mk\sto\sname\sthe\swin32\sexecutable\s"sqlite3_expert.exe",\snot\n"sqlite3_expert".
+D 2017-10-13T16:24:32.830
 F Makefile.in 51259a193f348a96b0ebc60ad84cfe83fdeecdd1d0ecb24f0e278a35ac1449da
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e3fd2bfdf18211892e3f09db7e2841753fe61cc0747a0b0e87d77406840c3f6e
@@ -388,7 +388,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 6ec9f289c46fdc66928782d7ff0d1b07c2e4c6ca553a3fe4f94e793ac53e8db6
+F main.mk 88c1647059bb48a7314285794b2090e7c931d665d526fa4d88952f5040bee9b3
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -1662,7 +1662,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 4ec63ef233c9239d3d383a99c234ef6604d55a9838c16f57e56c2a120aeb63c4
-R d11eafd1958fe2d9bdec327b55b3ebe5
+P 4c68ad7da48c941eb1ca0210a85d95467f3f3640bdc55a02686166471806ca4e
+R 549f733ecdac0d819a10aad7a58032a8
+T +closed 5981969cad5afebb8f8098489d0419cc9ead560a0f22a484230f1886011cd57c
 U dan
-Z 3e15c410e04c5e53e4b05cff4437d5be
+Z b4137401f555133414b7d9121a7d2894
diff --git a/manifest.uuid b/manifest.uuid
index 424d577e4a..2f5abff820 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4c68ad7da48c941eb1ca0210a85d95467f3f3640bdc55a02686166471806ca4e
\ No newline at end of file
+e38571d518ff399d1033b33843dae60bc61181cdf692dcd8e2e14746bf0d1e61
\ No newline at end of file

From 3be8b1a4f7848c1d67324893f4ac9cace8c06eb0 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 21 Oct 2017 12:59:27 +0000
Subject: [PATCH 053/488] Initial implementation of the appendvfs extension. 
 Untested.

FossilOrigin-Name: 063a03a3779e8c032dd006712facaaa6d60964425701ea10c753ff981a8f2bd9
---
 ext/misc/appendvfs.c | 453 +++++++++++++++++++++++++++++++++++++++++++
 manifest             |  17 +-
 manifest.uuid        |   2 +-
 3 files changed, 464 insertions(+), 8 deletions(-)
 create mode 100644 ext/misc/appendvfs.c

diff --git a/ext/misc/appendvfs.c b/ext/misc/appendvfs.c
new file mode 100644
index 0000000000..b650b80be3
--- /dev/null
+++ b/ext/misc/appendvfs.c
@@ -0,0 +1,453 @@
+/*
+** 2017-10-20
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file implements a VFS shim that allows an SQLite database to be
+** appended onto the end of some other file, such as an executable.
+**
+** A special record must appear at the end of the file that identifies the
+** file as an appended database and provides an offset to page 1.  For
+** best performance page 1 should be located at a disk page boundary, though
+** that is not required.
+**
+** An appended database is considered immutable.  It is read-only and no
+** locks are ever taken.
+**
+** If the file being opened is not an appended database, then this shim is
+** a pass-through into the default underlying VFS.
+**/
+#include <sqlite3ext.h>
+SQLITE_EXTENSION_INIT1
+#include <string.h>
+#include <assert.h>
+
+/* The append mark at the end of the database is:
+**
+**     Start-Of-SQLite3-NNNNNNNN
+**     123456789 123456789 12345
+**
+** The NNNNNNNN represents a 64-bit big-endian unsigned integer which is
+** the offset to page 1.
+*/
+#define APND_MARK_PREFIX     "Start-Of-SQLite3-"
+#define APND_MARK_PREFIX_SZ  17
+#define APND_MARK_SIZE       25
+
+/*
+** Forward declaration of objects used by this utility
+*/
+typedef struct sqlite3_vfs ApndVfs;
+typedef struct ApndFile ApndFile;
+
+/* Access to a lower-level VFS that (might) implement dynamic loading,
+** access to randomness, etc.
+*/
+#define ORIGVFS(p)  ((sqlite3_vfs*)((p)->pAppData))
+#define ORIGFILE(p) ((sqlite3_file*)(((ApndFile*)(p))+1))
+
+/* An open file */
+struct ApndFile {
+  sqlite3_file base;              /* IO methods */
+  sqlite3_int64 iPgOne;           /* File offset to page 1 */
+};
+
+/*
+** Methods for ApndFile
+*/
+static int apndClose(sqlite3_file*);
+static int apndRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst);
+static int apndWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst);
+static int apndTruncate(sqlite3_file*, sqlite3_int64 size);
+static int apndSync(sqlite3_file*, int flags);
+static int apndFileSize(sqlite3_file*, sqlite3_int64 *pSize);
+static int apndLock(sqlite3_file*, int);
+static int apndUnlock(sqlite3_file*, int);
+static int apndCheckReservedLock(sqlite3_file*, int *pResOut);
+static int apndFileControl(sqlite3_file*, int op, void *pArg);
+static int apndSectorSize(sqlite3_file*);
+static int apndDeviceCharacteristics(sqlite3_file*);
+static int apndShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**);
+static int apndShmLock(sqlite3_file*, int offset, int n, int flags);
+static void apndShmBarrier(sqlite3_file*);
+static int apndShmUnmap(sqlite3_file*, int deleteFlag);
+static int apndFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp);
+static int apndUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p);
+
+/*
+** Methods for ApndVfs
+*/
+static int apndOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *);
+static int apndDelete(sqlite3_vfs*, const char *zName, int syncDir);
+static int apndAccess(sqlite3_vfs*, const char *zName, int flags, int *);
+static int apndFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut);
+static void *apndDlOpen(sqlite3_vfs*, const char *zFilename);
+static void apndDlError(sqlite3_vfs*, int nByte, char *zErrMsg);
+static void (*apndDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void);
+static void apndDlClose(sqlite3_vfs*, void*);
+static int apndRandomness(sqlite3_vfs*, int nByte, char *zOut);
+static int apndSleep(sqlite3_vfs*, int microseconds);
+static int apndCurrentTime(sqlite3_vfs*, double*);
+static int apndGetLastError(sqlite3_vfs*, int, char *);
+static int apndCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*);
+static int apndSetSystemCall(sqlite3_vfs*, const char*,sqlite3_syscall_ptr);
+static sqlite3_syscall_ptr apndGetSystemCall(sqlite3_vfs*, const char *z);
+static const char *apndNextSystemCall(sqlite3_vfs*, const char *zName);
+
+static sqlite3_vfs apnd_vfs = {
+  3,                            /* iVersion (set when registered) */
+  0,                            /* szOsFile (set when registered) */
+  1024,                         /* mxPathname */
+  0,                            /* pNext */
+  "apndvfs",                    /* zName */
+  0,                            /* pAppData (set when registered) */ 
+  apndOpen,                     /* xOpen */
+  apndDelete,                   /* xDelete */
+  apndAccess,                   /* xAccess */
+  apndFullPathname,             /* xFullPathname */
+  apndDlOpen,                   /* xDlOpen */
+  apndDlError,                  /* xDlError */
+  apndDlSym,                    /* xDlSym */
+  apndDlClose,                  /* xDlClose */
+  apndRandomness,               /* xRandomness */
+  apndSleep,                    /* xSleep */
+  apndCurrentTime,              /* xCurrentTime */
+  apndGetLastError,             /* xGetLastError */
+  apndCurrentTimeInt64,         /* xCurrentTimeInt64 */
+  apndSetSystemCall,            /* xSetSystemCall */
+  apndGetSystemCall,            /* xGetSystemCall */
+  apndNextSystemCall            /* xNextSystemCall */
+};
+
+static const sqlite3_io_methods apnd_io_methods = {
+  3,                              /* iVersion */
+  apndClose,                      /* xClose */
+  apndRead,                       /* xRead */
+  apndWrite,                      /* xWrite */
+  apndTruncate,                   /* xTruncate */
+  apndSync,                       /* xSync */
+  apndFileSize,                   /* xFileSize */
+  apndLock,                       /* xLock */
+  apndUnlock,                     /* xUnlock */
+  apndCheckReservedLock,          /* xCheckReservedLock */
+  apndFileControl,                /* xFileControl */
+  apndSectorSize,                 /* xSectorSize */
+  apndDeviceCharacteristics,      /* xDeviceCharacteristics */
+  apndShmMap,                     /* xShmMap */
+  apndShmLock,                    /* xShmLock */
+  apndShmBarrier,                 /* xShmBarrier */
+  apndShmUnmap,                   /* xShmUnmap */
+  apndFetch,                      /* xFetch */
+  apndUnfetch                     /* xUnfetch */
+};
+
+
+
+/*
+** Close an apnd-file.
+*/
+static int apndClose(sqlite3_file *pFile){
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xClose(pFile);
+}
+
+/*
+** Read data from an apnd-file.
+*/
+static int apndRead(
+  sqlite3_file *pFile, 
+  void *zBuf, 
+  int iAmt, 
+  sqlite_int64 iOfst
+){
+  ApndFile *p = (ApndFile *)pFile;
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xRead(pFile, zBuf, iAmt, iOfst+p->iPgOne);
+}
+
+/*
+** Write data to an apnd-file.
+*/
+static int apndWrite(
+  sqlite3_file *pFile,
+  const void *z,
+  int iAmt,
+  sqlite_int64 iOfst
+){
+  return SQLITE_READONLY;
+}
+
+/*
+** Truncate an apnd-file.
+*/
+static int apndTruncate(sqlite3_file *pFile, sqlite_int64 size){
+  return SQLITE_READONLY;
+}
+
+/*
+** Sync an apnd-file.
+*/
+static int apndSync(sqlite3_file *pFile, int flags){
+  return SQLITE_READONLY;
+}
+
+/*
+** Return the current file-size of an apnd-file.
+*/
+static int apndFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
+  ApndFile *p = (ApndFile*)pFile;
+  int rc;
+  pFile = ORIGFILE(p);
+  rc = pFile->pMethods->xFileSize(pFile, pSize);
+  if( rc==SQLITE_OK && p->iPgOne ){
+    *pSize -= p->iPgOne + APND_MARK_SIZE;
+  }
+  return rc;
+}
+
+/*
+** Lock an apnd-file.
+*/
+static int apndLock(sqlite3_file *pFile, int eLock){
+  return SQLITE_READONLY;
+}
+
+/*
+** Unlock an apnd-file.
+*/
+static int apndUnlock(sqlite3_file *pFile, int eLock){
+  return SQLITE_OK;
+}
+
+/*
+** Check if another file-handle holds a RESERVED lock on an apnd-file.
+*/
+static int apndCheckReservedLock(sqlite3_file *pFile, int *pResOut){
+  *pResOut = 0;
+  return SQLITE_OK;
+}
+
+/*
+** File control method. For custom operations on an apnd-file.
+*/
+static int apndFileControl(sqlite3_file *pFile, int op, void *pArg){
+  ApndFile *p = (ApndFile *)pFile;
+  int rc;
+  pFile = ORIGFILE(pFile);
+  rc = pFile->pMethods->xFileControl(pFile, op, pArg);
+  if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){
+    *(char**)pArg = sqlite3_mprintf("apnd(%lld)/%z", p->iPgOne, *(char**)pArg);
+  }
+  return rc;
+}
+
+/*
+** Return the sector-size in bytes for an apnd-file.
+*/
+static int apndSectorSize(sqlite3_file *pFile){
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xSectorSize(pFile);
+}
+
+/*
+** Return the device characteristic flags supported by an apnd-file.
+*/
+static int apndDeviceCharacteristics(sqlite3_file *pFile){
+  pFile = ORIGFILE(pFile);
+  return SQLITE_IOCAP_IMMUTABLE |
+           pFile->pMethods->xDeviceCharacteristics(pFile);
+}
+
+/* Create a shared memory file mapping */
+static int apndShmMap(
+  sqlite3_file *pFile,
+  int iPg,
+  int pgsz,
+  int bExtend,
+  void volatile **pp
+){
+  return SQLITE_READONLY;
+}
+
+/* Perform locking on a shared-memory segment */
+static int apndShmLock(sqlite3_file *pFile, int offset, int n, int flags){
+  return SQLITE_READONLY;
+}
+
+/* Memory barrier operation on shared memory */
+static void apndShmBarrier(sqlite3_file *pFile){
+  return;
+}
+
+/* Unmap a shared memory segment */
+static int apndShmUnmap(sqlite3_file *pFile, int deleteFlag){
+  return SQLITE_OK;
+}
+
+/* Fetch a page of a memory-mapped file */
+static int apndFetch(
+  sqlite3_file *pFile,
+  sqlite3_int64 iOfst,
+  int iAmt,
+  void **pp
+){
+  ApndFile *p = (ApndFile *)pFile;
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xFetch(pFile, iOfst+p->iPgOne, iAmt, pp);
+}
+
+/* Release a memory-mapped page */
+static int apndUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){
+  ApndFile *p = (ApndFile *)pFile;
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xUnfetch(pFile, iOfst+p->iPgOne, pPage);
+}
+
+/*
+** Open an apnd file handle.
+*/
+static int apndOpen(
+  sqlite3_vfs *pVfs,
+  const char *zName,
+  sqlite3_file *pFile,
+  int flags,
+  int *pOutFlags
+){
+  ApndFile *p;
+  int rc;
+  sqlite3_int64 sz;
+  pVfs = ORIGVFS(pVfs);
+  if( (flags & SQLITE_OPEN_MAIN_DB)==0 ){
+    return pVfs->xOpen(pVfs, zName, pFile, flags, pOutFlags);
+  }
+  p = (ApndFile*)pFile;
+  memset(p, 0, sizeof(*p));
+  p->base.pMethods = &apnd_io_methods;
+  pFile = ORIGFILE(pFile);
+  flags &= ~(SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE);
+  flags |= SQLITE_OPEN_READONLY;
+  rc = pVfs->xOpen(pVfs, zName, pFile, flags, pOutFlags);
+  if( rc ) return rc;
+  rc = pFile->pMethods->xFileSize(pFile, &sz);
+  if( rc==SQLITE_OK && sz>512 ){
+    unsigned char a[APND_MARK_SIZE];
+    rc = pFile->pMethods->xRead(pFile, a, APND_MARK_SIZE, sz-APND_MARK_SIZE);
+    if( rc==SQLITE_OK
+     && memcmp(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ)==0
+    ){
+      p->iPgOne =
+         ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ]<<56) +
+         ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+1]<<48) +
+         ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+2]<<40) +
+         ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+3]<<32) +
+         ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+4]<<24) +
+         ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+5]<<16) +
+         ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+6]<<8) +
+         ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+7]);
+    }
+  }
+  return SQLITE_OK;
+}
+
+/*
+** All other VFS methods are pass-thrus.
+*/
+static int apndDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
+  return ORIGVFS(pVfs)->xDelete(ORIGVFS(pVfs), zPath, dirSync);
+}
+static int apndAccess(
+  sqlite3_vfs *pVfs, 
+  const char *zPath, 
+  int flags, 
+  int *pResOut
+){
+  return ORIGVFS(pVfs)->xAccess(ORIGVFS(pVfs), zPath, flags, pResOut);
+}
+static int apndFullPathname(
+  sqlite3_vfs *pVfs, 
+  const char *zPath, 
+  int nOut, 
+  char *zOut
+){
+  return ORIGVFS(pVfs)->xFullPathname(ORIGVFS(pVfs),zPath,nOut,zOut);
+}
+static void *apndDlOpen(sqlite3_vfs *pVfs, const char *zPath){
+  return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath);
+}
+static void apndDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){
+  ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg);
+}
+static void (*apndDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){
+  return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym);
+}
+static void apndDlClose(sqlite3_vfs *pVfs, void *pHandle){
+  ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle);
+}
+static int apndRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){
+  return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut);
+}
+static int apndSleep(sqlite3_vfs *pVfs, int nMicro){
+  return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro);
+}
+static int apndCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){
+  return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut);
+}
+static int apndGetLastError(sqlite3_vfs *pVfs, int a, char *b){
+  return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b);
+}
+static int apndCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){
+  return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p);
+}
+static int apndSetSystemCall(
+  sqlite3_vfs *pVfs,
+  const char *zName,
+  sqlite3_syscall_ptr pCall
+){
+  return ORIGVFS(pVfs)->xSetSystemCall(ORIGVFS(pVfs),zName,pCall);
+}
+static sqlite3_syscall_ptr apndGetSystemCall(
+  sqlite3_vfs *pVfs,
+  const char *zName
+){
+  return ORIGVFS(pVfs)->xGetSystemCall(ORIGVFS(pVfs),zName);
+}
+static const char *apndNextSystemCall(sqlite3_vfs *pVfs, const char *zName){
+  return ORIGVFS(pVfs)->xNextSystemCall(ORIGVFS(pVfs), zName);
+}
+
+  
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+/* 
+** This routine is called when the extension is loaded.
+** Register the new VFS.
+*/
+int sqlite3_appendvfs_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  int rc = SQLITE_OK;
+  sqlite3_vfs *pOrig;
+  SQLITE_EXTENSION_INIT2(pApi);
+  pOrig = sqlite3_vfs_find(0);
+  apnd_vfs.iVersion = pOrig->iVersion;
+  apnd_vfs.pAppData = pOrig;
+  apnd_vfs.szOsFile = sizeof(ApndFile);
+  rc = sqlite3_vfs_register(&apnd_vfs, 1);
+#ifdef APPENDVFS_TEST
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_auto_extension((void(*)(void))apndvfsRegister);
+  }
+#endif
+  if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY;
+  return rc;
+}
diff --git a/manifest b/manifest
index 5c8ae4d13c..e9413c8f80 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Take\sextra\scare\sto\savoid\san\sOOB\sread\scaused\sby\sa\scorrupt\sb-tree\spage.
-D 2017-10-19T15:17:38.752
+C Initial\simplementation\sof\sthe\sappendvfs\sextension.\s\sUntested.
+D 2017-10-21T12:59:27.233
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -258,6 +258,7 @@ F ext/lsm1/test/lsm1_simple.test ca949efefa102f4644231dcd9291d8cda7699a4ce1006b2
 F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601d90b2
 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87
 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb
+F ext/misc/appendvfs.c 487a5db4427d118413346780c4c90d0256315e8de3fb528ee54daef3bb73276f
 F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005
 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704
 F ext/misc/completion.c 52c3f01523e3e387eb321b4739a89d1fe47cbe6025aa1f2d8d3685e9e365df0f
@@ -1664,8 +1665,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P cd0471ca9f75e7c8be74536ff4ec85b5d70f0d7994b680ed5f45b9f12a46cf46
-R 14641e8822b464d39bf472f899e19240
-T +closed 9dd591ef24b302a5fe2af0619d0cda6733348bacc541b3c0a134ac25981d4b2a
-U dan
-Z 576e55698c218b4128bbad5de39dc1b0
+P 04925dee41a21ffca9a9f9df27d8165431668c42c2b33d08b077fdb28011170b
+R b5d8b4b601b02e4902be4e639107c749
+T *branch * appendvfs
+T *sym-appendvfs *
+T -sym-trunk *
+U drh
+Z 11816656e34dd3c798cb5d3f2287708a
diff --git a/manifest.uuid b/manifest.uuid
index 26a2375f60..69ef83190d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-04925dee41a21ffca9a9f9df27d8165431668c42c2b33d08b077fdb28011170b
\ No newline at end of file
+063a03a3779e8c032dd006712facaaa6d60964425701ea10c753ff981a8f2bd9
\ No newline at end of file

From b5039fb7d3944049da6c48e83a6d7685b5cd6da2 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 25 Oct 2017 23:28:13 +0000
Subject: [PATCH 054/488] Use extra locks to prevent a readonly_shm=1 process
 from connecting to a WAL-mode database if there are no writers.

FossilOrigin-Name: 35d979082b4ab36d6a8975f8f15a50e69f46b72a173164d2b353377b9f758bd8
---
 manifest        | 17 +++++----
 manifest.uuid   |  2 +-
 src/os_unix.c   | 96 ++++++++++++++++++++++++++++++-------------------
 src/sqlite.h.in |  1 +
 4 files changed, 72 insertions(+), 44 deletions(-)

diff --git a/manifest b/manifest
index a593818258..bf7be72e5e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sthe\ssqlite3_dbpage\svirtual\stable\sso\sthat\sit\scan\sread\sand\swrite\sfrom\nany\sattached\sdatabase.
-D 2017-10-25T19:18:33.009
+C Use\sextra\slocks\sto\sprevent\sa\sreadonly_shm=1\sprocess\sfrom\sconnecting\sto\sa\nWAL-mode\sdatabase\sif\sthere\sare\sno\swriters.
+D 2017-10-25T23:28:13.771
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -447,7 +447,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104
 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
-F src/os_unix.c 7edc872747feaa3016bd04e5e4389743bacafc0fee3444b0ecdec5d8f45049df
+F src/os_unix.c bec568f6f89e57b987f5ca38ae90087687d8e18669cd37eceb48ca6b56a8fbfb
 F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903
@@ -465,7 +465,7 @@ F src/resolve.c 5a461643f294ec510ca615b67256fc3861e4c8eff5f29e5940491e70553b1955
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c e6a068d9ea54417d625578086d3d482284af8d5a449bb3593d40c257080806a8
 F src/shell.c.in f13262c8778f0cd76bf8d9c01bbf5ef66842e6b14e1705cd60d86ab32a6ce69f
-F src/sqlite.h.in ab4f8a29d1580dfaeb6891fa1b83cff8229ba0daa56994707ceaca71495d9ab7
+F src/sqlite.h.in ba9029e71a605bc5f236cd693abeb7920285785f2e1a92313ecf8fb1c9f52a86
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
 F src/sqliteInt.h 6f93fd6fde862410ac26b930f70752c38ad99ea78c3fc28356bac78049c53bd9
@@ -1666,7 +1666,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d6130cd226c0ca95e02f0cbabfdc27071acdcf83e0d0cb0eaa47d992479ed9a1
-R de06ad2d264283eba96aba95f64ee6f5
+P d4f893e1ae53a0445939ea2920af87d21dd36270494381028b2eaebe5c188f18
+R 7c17d54c5cff309a6f13f49dd620b485
+T *branch * readonly-wal-recovery
+T *sym-readonly-wal-recovery *
+T -sym-trunk *
 U drh
-Z 3b4d77a61659a96790923a30bda00b59
+Z 26f9af4fe6668c97b92039b43092c035
diff --git a/manifest.uuid b/manifest.uuid
index ce16a7caf0..44570bf71c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d4f893e1ae53a0445939ea2920af87d21dd36270494381028b2eaebe5c188f18
\ No newline at end of file
+35d979082b4ab36d6a8975f8f15a50e69f46b72a173164d2b353377b9f758bd8
\ No newline at end of file
diff --git a/src/os_unix.c b/src/os_unix.c
index c33d21fd9b..dadfaf9908 100644
--- a/src/os_unix.c
+++ b/src/os_unix.c
@@ -4145,6 +4145,7 @@ struct unixShm {
 */
 #define UNIX_SHM_BASE   ((22+SQLITE_SHM_NLOCK)*4)         /* first lock byte */
 #define UNIX_SHM_DMS    (UNIX_SHM_BASE+SQLITE_SHM_NLOCK)  /* deadman switch */
+#define UNIX_SHM_N_DMS  1000000000                        /* Size of the DMS */
 
 /*
 ** Apply posix advisory locks for all bytes from ofst through ofst+n-1.
@@ -4166,12 +4167,6 @@ static int unixShmSystemLock(
   pShmNode = pFile->pInode->pShmNode;
   assert( sqlite3_mutex_held(pShmNode->mutex) || pShmNode->nRef==0 );
 
-  /* Shared locks never span more than one byte */
-  assert( n==1 || lockType!=F_RDLCK );
-
-  /* Locks are within range */
-  assert( n>=1 && n<=SQLITE_SHM_NLOCK );
-
   if( pShmNode->h>=0 ){
     /* Initialize the locking parameters */
     memset(&f, 0, sizeof(f));
@@ -4186,36 +4181,44 @@ static int unixShmSystemLock(
 
   /* Update the global lock state and do debug tracing */
 #ifdef SQLITE_DEBUG
-  { u16 mask;
-  OSTRACE(("SHM-LOCK "));
-  mask = ofst>31 ? 0xffff : (1<<(ofst+n)) - (1<<ofst);
-  if( rc==SQLITE_OK ){
-    if( lockType==F_UNLCK ){
-      OSTRACE(("unlock %d ok", ofst));
-      pShmNode->exclMask &= ~mask;
-      pShmNode->sharedMask &= ~mask;
-    }else if( lockType==F_RDLCK ){
-      OSTRACE(("read-lock %d ok", ofst));
-      pShmNode->exclMask &= ~mask;
-      pShmNode->sharedMask |= mask;
+  if( ofst<UNIX_SHM_DMS ){
+    u16 mask;
+
+    /* Shared locks never span more than one byte */
+    assert( n==1 || lockType!=F_RDLCK );
+
+    /* Locks are within range */
+    assert( n>=1 && n<=SQLITE_SHM_NLOCK );
+
+    OSTRACE(("SHM-LOCK "));
+    mask = ofst>31 ? 0xffff : (1<<(ofst+n)) - (1<<ofst);
+    if( rc==SQLITE_OK ){
+      if( lockType==F_UNLCK ){
+        OSTRACE(("unlock %d ok", ofst));
+        pShmNode->exclMask &= ~mask;
+        pShmNode->sharedMask &= ~mask;
+      }else if( lockType==F_RDLCK ){
+        OSTRACE(("read-lock %d ok", ofst));
+        pShmNode->exclMask &= ~mask;
+        pShmNode->sharedMask |= mask;
+      }else{
+        assert( lockType==F_WRLCK );
+        OSTRACE(("write-lock %d ok", ofst));
+        pShmNode->exclMask |= mask;
+        pShmNode->sharedMask &= ~mask;
+      }
     }else{
-      assert( lockType==F_WRLCK );
-      OSTRACE(("write-lock %d ok", ofst));
-      pShmNode->exclMask |= mask;
-      pShmNode->sharedMask &= ~mask;
+      if( lockType==F_UNLCK ){
+        OSTRACE(("unlock %d failed", ofst));
+      }else if( lockType==F_RDLCK ){
+        OSTRACE(("read-lock failed"));
+      }else{
+        assert( lockType==F_WRLCK );
+        OSTRACE(("write-lock %d failed", ofst));
+      }
     }
-  }else{
-    if( lockType==F_UNLCK ){
-      OSTRACE(("unlock %d failed", ofst));
-    }else if( lockType==F_RDLCK ){
-      OSTRACE(("read-lock failed"));
-    }else{
-      assert( lockType==F_WRLCK );
-      OSTRACE(("write-lock %d failed", ofst));
-    }
-  }
-  OSTRACE((" - afterwards %03x,%03x\n",
-           pShmNode->sharedMask, pShmNode->exclMask));
+    OSTRACE((" - afterwards %03x,%03x\n",
+             pShmNode->sharedMask, pShmNode->exclMask));
   }
 #endif
 
@@ -4389,17 +4392,38 @@ static int unixOpenSharedMemory(unixFile *pDbFd){
       */
       robustFchown(pShmNode->h, sStat.st_uid, sStat.st_gid);
   
-      /* Check to see if another process is holding the dead-man switch.
-      ** If not, truncate the file to zero length. 
+      /* Do not allow a read-only process to connect if there are no
+      ** writers, because a read-only process is unable to recover the
+      ** shm file following a system crash.
       */
       rc = SQLITE_OK;
+      if( pShmNode->isReadonly ){
+        if( !unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, UNIX_SHM_N_DMS) ){
+          rc = SQLITE_CANTOPEN_DIRTYWAL;
+        }
+      }
+
+      /* If we are able to grab the dead-man switch, that means this is the
+      ** first (write-enable) process to connect to the database.  In that
+      ** case, truncate the shm file because the contents found on disk might
+      ** be invalid leftovers from a system crash.  The shm will be rebuilt
+      */
       if( unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1)==SQLITE_OK ){
         if( robust_ftruncate(pShmNode->h, 0) ){
           rc = unixLogError(SQLITE_IOERR_SHMOPEN, "ftruncate", zShmFilename);
         }
       }
+
+      /* Acquires locks to tell other processes that a this process is
+      ** running and therefore the shm is valid they do not need to run
+      ** recovery.
+      */
       if( rc==SQLITE_OK ){
+        unsigned r;
         rc = unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, 1);
+        sqlite3_randomness(sizeof(r), &r);
+        r = 1 + (r%(UNIX_SHM_N_DMS-1));
+        (void)unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS+r, 1);
       }
       if( rc ) goto shm_open_err;
     }
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 9ff366304f..0c0ed25b30 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -508,6 +508,7 @@ int sqlite3_exec(
 #define SQLITE_CANTOPEN_ISDIR          (SQLITE_CANTOPEN | (2<<8))
 #define SQLITE_CANTOPEN_FULLPATH       (SQLITE_CANTOPEN | (3<<8))
 #define SQLITE_CANTOPEN_CONVPATH       (SQLITE_CANTOPEN | (4<<8))
+#define SQLITE_CANTOPEN_DIRTYWAL       (SQLITE_CANTOPEN | (5<<8))
 #define SQLITE_CORRUPT_VTAB            (SQLITE_CORRUPT | (1<<8))
 #define SQLITE_READONLY_RECOVERY       (SQLITE_READONLY | (1<<8))
 #define SQLITE_READONLY_CANTLOCK       (SQLITE_READONLY | (2<<8))

From 9181ae990a3b8623d27813209e6649bf76565545 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 26 Oct 2017 17:05:22 +0000
Subject: [PATCH 055/488] Instead of extra locks, use F_GETLK to ensure that
 readonly_shm clients cannot connect to a wal-mode database if there are no
 writers.

FossilOrigin-Name: 5492f457dc7cc5c416de4b4e61e84bd2f10b4e6ce54011b7a60feb47f629c923
---
 manifest      |  17 ++++----
 manifest.uuid |   2 +-
 src/os_unix.c | 107 ++++++++++++++++++++++----------------------------
 3 files changed, 56 insertions(+), 70 deletions(-)

diff --git a/manifest b/manifest
index bf7be72e5e..c648e5a856 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Use\sextra\slocks\sto\sprevent\sa\sreadonly_shm=1\sprocess\sfrom\sconnecting\sto\sa\nWAL-mode\sdatabase\sif\sthere\sare\sno\swriters.
-D 2017-10-25T23:28:13.771
+C Instead\sof\sextra\slocks,\suse\sF_GETLK\sto\sensure\sthat\sreadonly_shm\sclients\scannot\nconnect\sto\sa\swal-mode\sdatabase\sif\sthere\sare\sno\swriters.
+D 2017-10-26T17:05:22.656
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -447,7 +447,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104
 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
-F src/os_unix.c bec568f6f89e57b987f5ca38ae90087687d8e18669cd37eceb48ca6b56a8fbfb
+F src/os_unix.c 8103f60342c65d501b4e58b381796648d6584b4814ffee79cd3a6e0c12fb6545
 F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903
@@ -1666,10 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d4f893e1ae53a0445939ea2920af87d21dd36270494381028b2eaebe5c188f18
-R 7c17d54c5cff309a6f13f49dd620b485
-T *branch * readonly-wal-recovery
-T *sym-readonly-wal-recovery *
-T -sym-trunk *
-U drh
-Z 26f9af4fe6668c97b92039b43092c035
+P 35d979082b4ab36d6a8975f8f15a50e69f46b72a173164d2b353377b9f758bd8
+R 87551e9df4f45e3704d812374bd0d192
+U dan
+Z a113fb74c5589c985f30184211e03e42
diff --git a/manifest.uuid b/manifest.uuid
index 44570bf71c..1815e9a333 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-35d979082b4ab36d6a8975f8f15a50e69f46b72a173164d2b353377b9f758bd8
\ No newline at end of file
+5492f457dc7cc5c416de4b4e61e84bd2f10b4e6ce54011b7a60feb47f629c923
\ No newline at end of file
diff --git a/src/os_unix.c b/src/os_unix.c
index dadfaf9908..f689ff75fa 100644
--- a/src/os_unix.c
+++ b/src/os_unix.c
@@ -4145,7 +4145,6 @@ struct unixShm {
 */
 #define UNIX_SHM_BASE   ((22+SQLITE_SHM_NLOCK)*4)         /* first lock byte */
 #define UNIX_SHM_DMS    (UNIX_SHM_BASE+SQLITE_SHM_NLOCK)  /* deadman switch */
-#define UNIX_SHM_N_DMS  1000000000                        /* Size of the DMS */
 
 /*
 ** Apply posix advisory locks for all bytes from ofst through ofst+n-1.
@@ -4167,6 +4166,12 @@ static int unixShmSystemLock(
   pShmNode = pFile->pInode->pShmNode;
   assert( sqlite3_mutex_held(pShmNode->mutex) || pShmNode->nRef==0 );
 
+  /* Shared locks never span more than one byte */
+  assert( n==1 || lockType!=F_RDLCK );
+
+  /* Locks are within range */
+  assert( n>=1 && n<=SQLITE_SHM_NLOCK );
+
   if( pShmNode->h>=0 ){
     /* Initialize the locking parameters */
     memset(&f, 0, sizeof(f));
@@ -4181,44 +4186,36 @@ static int unixShmSystemLock(
 
   /* Update the global lock state and do debug tracing */
 #ifdef SQLITE_DEBUG
-  if( ofst<UNIX_SHM_DMS ){
-    u16 mask;
-
-    /* Shared locks never span more than one byte */
-    assert( n==1 || lockType!=F_RDLCK );
-
-    /* Locks are within range */
-    assert( n>=1 && n<=SQLITE_SHM_NLOCK );
-
-    OSTRACE(("SHM-LOCK "));
-    mask = ofst>31 ? 0xffff : (1<<(ofst+n)) - (1<<ofst);
-    if( rc==SQLITE_OK ){
-      if( lockType==F_UNLCK ){
-        OSTRACE(("unlock %d ok", ofst));
-        pShmNode->exclMask &= ~mask;
-        pShmNode->sharedMask &= ~mask;
-      }else if( lockType==F_RDLCK ){
-        OSTRACE(("read-lock %d ok", ofst));
-        pShmNode->exclMask &= ~mask;
-        pShmNode->sharedMask |= mask;
-      }else{
-        assert( lockType==F_WRLCK );
-        OSTRACE(("write-lock %d ok", ofst));
-        pShmNode->exclMask |= mask;
-        pShmNode->sharedMask &= ~mask;
-      }
+  { u16 mask;
+  OSTRACE(("SHM-LOCK "));
+  mask = ofst>31 ? 0xffff : (1<<(ofst+n)) - (1<<ofst);
+  if( rc==SQLITE_OK ){
+    if( lockType==F_UNLCK ){
+      OSTRACE(("unlock %d ok", ofst));
+      pShmNode->exclMask &= ~mask;
+      pShmNode->sharedMask &= ~mask;
+    }else if( lockType==F_RDLCK ){
+      OSTRACE(("read-lock %d ok", ofst));
+      pShmNode->exclMask &= ~mask;
+      pShmNode->sharedMask |= mask;
     }else{
-      if( lockType==F_UNLCK ){
-        OSTRACE(("unlock %d failed", ofst));
-      }else if( lockType==F_RDLCK ){
-        OSTRACE(("read-lock failed"));
-      }else{
-        assert( lockType==F_WRLCK );
-        OSTRACE(("write-lock %d failed", ofst));
-      }
+      assert( lockType==F_WRLCK );
+      OSTRACE(("write-lock %d ok", ofst));
+      pShmNode->exclMask |= mask;
+      pShmNode->sharedMask &= ~mask;
     }
-    OSTRACE((" - afterwards %03x,%03x\n",
-             pShmNode->sharedMask, pShmNode->exclMask));
+  }else{
+    if( lockType==F_UNLCK ){
+      OSTRACE(("unlock %d failed", ofst));
+    }else if( lockType==F_RDLCK ){
+      OSTRACE(("read-lock failed"));
+    }else{
+      assert( lockType==F_WRLCK );
+      OSTRACE(("write-lock %d failed", ofst));
+    }
+  }
+  OSTRACE((" - afterwards %03x,%03x\n",
+           pShmNode->sharedMask, pShmNode->exclMask));
   }
 #endif
 
@@ -4392,38 +4389,30 @@ static int unixOpenSharedMemory(unixFile *pDbFd){
       */
       robustFchown(pShmNode->h, sStat.st_uid, sStat.st_gid);
   
-      /* Do not allow a read-only process to connect if there are no
-      ** writers, because a read-only process is unable to recover the
-      ** shm file following a system crash.
-      */
+      /* Check to see if another process is holding the dead-man switch.
+      ** For a readonly_shm client, if no other process holds the DMS lock,
+      ** the file cannot be opened and SQLITE_CANTOPEN_DIRTYWAL is returned.
+      ** Or, for a read-write connection, if no other process holds a
+      ** DMS lock the file is truncated to zero bytes in size.  */
       rc = SQLITE_OK;
       if( pShmNode->isReadonly ){
-        if( !unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, UNIX_SHM_N_DMS) ){
+        struct flock lock;
+        lock.l_whence = SEEK_SET;
+        lock.l_start = UNIX_SHM_DMS;
+        lock.l_len = 1;
+        lock.l_type = F_WRLCK;
+        if( osFcntl(pShmNode->h, F_GETLK, &lockInfo)!=0 ) {
+          rc = SQLITE_IOERR_LOCK;
+        }else if( lock.l_type==F_UNLCK ){
           rc = SQLITE_CANTOPEN_DIRTYWAL;
         }
-      }
-
-      /* If we are able to grab the dead-man switch, that means this is the
-      ** first (write-enable) process to connect to the database.  In that
-      ** case, truncate the shm file because the contents found on disk might
-      ** be invalid leftovers from a system crash.  The shm will be rebuilt
-      */
-      if( unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1)==SQLITE_OK ){
+      }else if( unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1)==SQLITE_OK ){
         if( robust_ftruncate(pShmNode->h, 0) ){
           rc = unixLogError(SQLITE_IOERR_SHMOPEN, "ftruncate", zShmFilename);
         }
       }
-
-      /* Acquires locks to tell other processes that a this process is
-      ** running and therefore the shm is valid they do not need to run
-      ** recovery.
-      */
       if( rc==SQLITE_OK ){
-        unsigned r;
         rc = unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, 1);
-        sqlite3_randomness(sizeof(r), &r);
-        r = 1 + (r%(UNIX_SHM_N_DMS-1));
-        (void)unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS+r, 1);
       }
       if( rc ) goto shm_open_err;
     }

From ab04eff809c0ee16a913e469ce51acdfcaf20c5d Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 26 Oct 2017 17:34:50 +0000
Subject: [PATCH 056/488] Fix an error in the previous commit on this branch.

FossilOrigin-Name: f71dfee06ce1e0eee760cfca19482bdec7729d6c7d28f10f4cfd21e1f92a04b0
---
 manifest        | 14 +++++++-------
 manifest.uuid   |  2 +-
 src/os_unix.c   |  2 +-
 test/walro.test | 13 +++++++------
 4 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/manifest b/manifest
index c648e5a856..ebf38789b5 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Instead\sof\sextra\slocks,\suse\sF_GETLK\sto\sensure\sthat\sreadonly_shm\sclients\scannot\nconnect\sto\sa\swal-mode\sdatabase\sif\sthere\sare\sno\swriters.
-D 2017-10-26T17:05:22.656
+C Fix\san\serror\sin\sthe\sprevious\scommit\son\sthis\sbranch.
+D 2017-10-26T17:34:50.823
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -447,7 +447,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104
 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
-F src/os_unix.c 8103f60342c65d501b4e58b381796648d6584b4814ffee79cd3a6e0c12fb6545
+F src/os_unix.c 9bf0c1b7156cbcd2ec32e557cffa319251e1ffb1515d923a2dd2d8eee69b4ee4
 F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903
@@ -1526,7 +1526,7 @@ F test/walnoshm.test 84ca10c544632a756467336b7c3b864d493ee496
 F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03
 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6
 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20
-F test/walro.test 4ab7ac01b77c2f894235c699d59e3e3c7f15a160
+F test/walro.test e492598baa8cd7777fef6203f6fe922c20cd691cc19e60ccd0dd0dbc68394d0a
 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417
 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f
 F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e
@@ -1666,7 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 35d979082b4ab36d6a8975f8f15a50e69f46b72a173164d2b353377b9f758bd8
-R 87551e9df4f45e3704d812374bd0d192
+P 5492f457dc7cc5c416de4b4e61e84bd2f10b4e6ce54011b7a60feb47f629c923
+R 49c1c99b22f8e84ff20a7afe5d1d547b
 U dan
-Z a113fb74c5589c985f30184211e03e42
+Z 07348874af35473af5ead94c9a86c9df
diff --git a/manifest.uuid b/manifest.uuid
index 1815e9a333..3da837219d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5492f457dc7cc5c416de4b4e61e84bd2f10b4e6ce54011b7a60feb47f629c923
\ No newline at end of file
+f71dfee06ce1e0eee760cfca19482bdec7729d6c7d28f10f4cfd21e1f92a04b0
\ No newline at end of file
diff --git a/src/os_unix.c b/src/os_unix.c
index f689ff75fa..7720a255ac 100644
--- a/src/os_unix.c
+++ b/src/os_unix.c
@@ -4401,7 +4401,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){
         lock.l_start = UNIX_SHM_DMS;
         lock.l_len = 1;
         lock.l_type = F_WRLCK;
-        if( osFcntl(pShmNode->h, F_GETLK, &lockInfo)!=0 ) {
+        if( osFcntl(pShmNode->h, F_GETLK, &lock)!=0 ) {
           rc = SQLITE_IOERR_LOCK;
         }else if( lock.l_type==F_UNLCK ){
           rc = SQLITE_CANTOPEN_DIRTYWAL;
diff --git a/test/walro.test b/test/walro.test
index f46e44d4cb..09a33bbd3e 100644
--- a/test/walro.test
+++ b/test/walro.test
@@ -101,10 +101,11 @@ do_multiclient_test tn {
     code1 { db close }
     list [file exists test.db-wal] [file exists test.db-shm]
   } {1 1}
+
   do_test 1.2.2 {
     code1 { sqlite3 db file:test.db?readonly_shm=1 }
-    sql1 { SELECT * FROM t1 }
-  } {a b c d e f g h i j}
+    list [catch { sql1 { SELECT * FROM t1 } } msg] $msg
+  } {1 {unable to open database file}}
 
   do_test 1.2.3 {
     code1 { db close }
@@ -113,10 +114,10 @@ do_multiclient_test tn {
     file attributes test.db-shm -permissions r--r--r--
     code1 { sqlite3 db file:test.db?readonly_shm=1 }
     csql1 { SELECT * FROM t1 }
-  } {1 {attempt to write a readonly database}}
+  } {1 {unable to open database file}}
   do_test 1.2.4 {
     code1 { sqlite3_extended_errcode db } 
-  } {SQLITE_READONLY_RECOVERY}
+  } {SQLITE_CANTOPEN}
 
   do_test 1.2.5 {
     file attributes test.db-shm -permissions rw-r--r--
@@ -161,10 +162,10 @@ do_multiclient_test tn {
     file attributes test.db-shm -permissions r--r--r--
     code1 { sqlite3 db file:test.db?readonly_shm=1 }
     csql1 { SELECT * FROM t1 }
-  } {1 {attempt to write a readonly database}}
+  } {1 {unable to open database file}}
   do_test 1.3.2.4 {
     code1 { sqlite3_extended_errcode db } 
-  } {SQLITE_READONLY_RECOVERY}
+  } {SQLITE_CANTOPEN}
 
   #-----------------------------------------------------------------------
   # Test cases 1.4.* check that checkpoints and log wraps don't prevent

From 176b2a916bfa2d6577638713d4e3e3264132f122 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 1 Nov 2017 06:59:19 +0000
Subject: [PATCH 057/488] Fix a race condition in os_unix.c that might allow a
 client to use a *-shm file corrupted by a power failure if another client
 fails between locking the *-shm file and truncating it to zero bytes.

FossilOrigin-Name: d655bfabd110999b6808073c334869c5b6a8334df56811df883e47e56d3f1cbb
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/os_unix.c | 54 ++++++++++++++++++++++++++++++++++-----------------
 3 files changed, 43 insertions(+), 25 deletions(-)

diff --git a/manifest b/manifest
index ebf38789b5..91f071548a 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\san\serror\sin\sthe\sprevious\scommit\son\sthis\sbranch.
-D 2017-10-26T17:34:50.823
+C Fix\sa\srace\scondition\sin\sos_unix.c\sthat\smight\sallow\sa\sclient\sto\suse\sa\s*-shm\nfile\scorrupted\sby\sa\spower\sfailure\sif\sanother\sclient\sfails\sbetween\slocking\sthe\n*-shm\sfile\sand\struncating\sit\sto\szero\sbytes.
+D 2017-11-01T06:59:19.745
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -447,7 +447,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104
 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
-F src/os_unix.c 9bf0c1b7156cbcd2ec32e557cffa319251e1ffb1515d923a2dd2d8eee69b4ee4
+F src/os_unix.c 9137cfdb42e83f4fb599aa2b1c6996f368aacbb410bde53b534e766a61ba65ca
 F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903
@@ -1666,7 +1666,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5492f457dc7cc5c416de4b4e61e84bd2f10b4e6ce54011b7a60feb47f629c923
-R 49c1c99b22f8e84ff20a7afe5d1d547b
+P f71dfee06ce1e0eee760cfca19482bdec7729d6c7d28f10f4cfd21e1f92a04b0
+R 37c523032e6b14675e1621a01cbce4c1
 U dan
-Z 07348874af35473af5ead94c9a86c9df
+Z e56573f2b7ef574fffc6cf13c6dfb22b
diff --git a/manifest.uuid b/manifest.uuid
index 3da837219d..46c47f2d96 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f71dfee06ce1e0eee760cfca19482bdec7729d6c7d28f10f4cfd21e1f92a04b0
\ No newline at end of file
+d655bfabd110999b6808073c334869c5b6a8334df56811df883e47e56d3f1cbb
\ No newline at end of file
diff --git a/src/os_unix.c b/src/os_unix.c
index 7720a255ac..b94d417f49 100644
--- a/src/os_unix.c
+++ b/src/os_unix.c
@@ -4372,6 +4372,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){
     }
 
     if( pInode->bProcessLock==0 ){
+      struct flock lock;
       int openFlags = O_RDWR | O_CREAT;
       if( sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){
         openFlags = O_RDONLY;
@@ -4389,29 +4390,46 @@ static int unixOpenSharedMemory(unixFile *pDbFd){
       */
       robustFchown(pShmNode->h, sStat.st_uid, sStat.st_gid);
   
-      /* Check to see if another process is holding the dead-man switch.
-      ** For a readonly_shm client, if no other process holds the DMS lock,
-      ** the file cannot be opened and SQLITE_CANTOPEN_DIRTYWAL is returned.
-      ** Or, for a read-write connection, if no other process holds a
-      ** DMS lock the file is truncated to zero bytes in size.  */
+      /* Use F_GETLK to determine the locks other processes are holding
+      ** on the DMS byte. If it indicates that another process is holding
+      ** a SHARED lock, then this process may also take a SHARED lock
+      ** and proceed with opening the *-shm file. 
+      **
+      ** Or, if no other process is holding any lock, then this process
+      ** is the first to open it. In this case take an EXCLUSIVE lock on the
+      ** DMS byte and truncate the *-shm file to zero bytes in size. Then
+      ** downgrade to a SHARED lock on the DMS byte.
+      **
+      ** If another process is holding an EXCLUSIVE lock on the DMS byte,
+      ** return SQLITE_BUSY to the caller (it will try again). An earlier
+      ** version of this code attempted the SHARED lock at this point. But
+      ** this introduced a subtle race condition: if the process holding
+      ** EXCLUSIVE failed just before truncating the *-shm file, then this
+      ** process might open and use the *-shm file without truncating it.
+      ** And if the *-shm file has been corrupted by a power failure or
+      ** system crash, the database itself may also become corrupt.  */
       rc = SQLITE_OK;
-      if( pShmNode->isReadonly ){
-        struct flock lock;
-        lock.l_whence = SEEK_SET;
-        lock.l_start = UNIX_SHM_DMS;
-        lock.l_len = 1;
-        lock.l_type = F_WRLCK;
-        if( osFcntl(pShmNode->h, F_GETLK, &lock)!=0 ) {
-          rc = SQLITE_IOERR_LOCK;
-        }else if( lock.l_type==F_UNLCK ){
+      lock.l_whence = SEEK_SET;
+      lock.l_start = UNIX_SHM_DMS;
+      lock.l_len = 1;
+      lock.l_type = F_WRLCK;
+      if( osFcntl(pShmNode->h, F_GETLK, &lock)!=0 ) {
+        rc = SQLITE_IOERR_LOCK;
+      }else if( lock.l_type==F_UNLCK ){
+        if( pShmNode->isReadonly ){
           rc = SQLITE_CANTOPEN_DIRTYWAL;
+        }else{
+          rc = unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1);
+          if( rc==SQLITE_OK && robust_ftruncate(pShmNode->h, 0) ){
+            rc = unixLogError(SQLITE_IOERR_SHMOPEN, "ftruncate", zShmFilename);
+          }
         }
-      }else if( unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1)==SQLITE_OK ){
-        if( robust_ftruncate(pShmNode->h, 0) ){
-          rc = unixLogError(SQLITE_IOERR_SHMOPEN, "ftruncate", zShmFilename);
-        }
+      }else if( lock.l_type==F_WRLCK ){
+        rc = SQLITE_BUSY;
       }
+
       if( rc==SQLITE_OK ){
+        assert( lock.l_type==F_UNLCK || lock.l_type==F_RDLCK );
         rc = unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, 1);
       }
       if( rc ) goto shm_open_err;

From 92c02da33ec2717828eba13400355d43a7a6395f Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 1 Nov 2017 20:59:28 +0000
Subject: [PATCH 058/488] If a readonly_shm connection cannot map the *-shm
 file because no other process is holding the DMS lock, have it read from the
 database file only, ignoring any content in the wal file.

FossilOrigin-Name: ce5d13c2de69b73378637d4f7e109714f7cd17bf1d1ad995e0be442d517ed1b3
---
 manifest         |  15 ++++---
 manifest.uuid    |   2 +-
 src/os_unix.c    | 114 ++++++++++++++++++++++++++++-------------------
 src/wal.c        |  24 +++++++---
 test/walro2.test |  87 ++++++++++++++++++++++++++++++++++++
 5 files changed, 183 insertions(+), 59 deletions(-)
 create mode 100644 test/walro2.test

diff --git a/manifest b/manifest
index a6e4cc1fc1..45c7b16d67 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\slatest\strunk\schanges\sinto\sthis\sbranch.
-D 2017-11-01T07:06:41.244
+C If\sa\sreadonly_shm\sconnection\scannot\smap\sthe\s*-shm\sfile\sbecause\sno\sother\nprocess\sis\sholding\sthe\sDMS\slock,\shave\sit\sread\sfrom\sthe\sdatabase\sfile\sonly,\nignoring\sany\scontent\sin\sthe\swal\sfile.
+D 2017-11-01T20:59:28.295
 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2
@@ -447,7 +447,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104
 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
-F src/os_unix.c 9137cfdb42e83f4fb599aa2b1c6996f368aacbb410bde53b534e766a61ba65ca
+F src/os_unix.c e376adf6014df7d1a73faaaa6c87e6eb9b299b157a58cccff02fad8abc943fe7
 F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903
@@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c cc9b1120f1955b66af425630c9893acd537a39d967fd39d404417f0a1b4c1579
+F src/wal.c 1521bdcfe9a536752a339c91b63ca0d226d8d266636391aac93537fdea8657fc
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f
 F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4
@@ -1527,6 +1527,7 @@ F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03
 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6
 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20
 F test/walro.test e492598baa8cd7777fef6203f6fe922c20cd691cc19e60ccd0dd0dbc68394d0a
+F test/walro2.test e2cd102cafceafaf19c56d55e55222fdd26d7621d7ef42b0eaf35c06c5bb3d19
 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417
 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f
 F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e
@@ -1667,7 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d655bfabd110999b6808073c334869c5b6a8334df56811df883e47e56d3f1cbb bb39744f4b2b25c10d293e85db7579e2a99c639fdab45e93d1de75952b68b2de
-R 46264ab14606a934f99609a277e3ff11
+P 985bfc992950625a45a7521bf4c8438cd0170de974dff976968be158ac5922a9
+R fcc128b567671c56015745eca2c28340
 U dan
-Z 7bfe9089058e9a41a8fafe2fd8ba1cb5
+Z e9ea9ea441f189aa52edada39fcf2f83
diff --git a/manifest.uuid b/manifest.uuid
index 48096a84ae..ebf42c65a3 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-985bfc992950625a45a7521bf4c8438cd0170de974dff976968be158ac5922a9
\ No newline at end of file
+ce5d13c2de69b73378637d4f7e109714f7cd17bf1d1ad995e0be442d517ed1b3
\ No newline at end of file
diff --git a/src/os_unix.c b/src/os_unix.c
index b94d417f49..e90e335cb7 100644
--- a/src/os_unix.c
+++ b/src/os_unix.c
@@ -4108,6 +4108,7 @@ struct unixShmNode {
   int szRegion;              /* Size of shared-memory regions */
   u16 nRegion;               /* Size of array apRegion */
   u8 isReadonly;             /* True if read-only */
+  u8 isUnlocked;             /* True if no DMS lock held */
   char **apRegion;           /* Array of mapped shared-memory regions */
   int nRef;                  /* Number of unixShm objects pointing to this */
   unixShm *pFirst;           /* All unixShm objects pointing to this */
@@ -4270,6 +4271,64 @@ static void unixShmPurge(unixFile *pFd){
   }
 }
 
+/*
+** The DMS lock has not yet been taken on shm file pShmNode. Attempt to
+** take it now. Return SQLITE_OK if successful, or an SQLite error
+** code otherwise.
+**
+** If the DMS cannot be locked because this is a readonly_shm=1 
+** connection and no other process already holds a lock, return
+** SQLITE_READONLY_CANTLOCK and set pShmNode->isUnlocked=1.
+*/
+static int unixLockSharedMemory(unixFile *pDbFd, unixShmNode *pShmNode){
+  struct flock lock;
+  int rc = SQLITE_OK;
+
+  /* Use F_GETLK to determine the locks other processes are holding
+  ** on the DMS byte. If it indicates that another process is holding
+  ** a SHARED lock, then this process may also take a SHARED lock
+  ** and proceed with opening the *-shm file. 
+  **
+  ** Or, if no other process is holding any lock, then this process
+  ** is the first to open it. In this case take an EXCLUSIVE lock on the
+  ** DMS byte and truncate the *-shm file to zero bytes in size. Then
+  ** downgrade to a SHARED lock on the DMS byte.
+  **
+  ** If another process is holding an EXCLUSIVE lock on the DMS byte,
+  ** return SQLITE_BUSY to the caller (it will try again). An earlier
+  ** version of this code attempted the SHARED lock at this point. But
+  ** this introduced a subtle race condition: if the process holding
+  ** EXCLUSIVE failed just before truncating the *-shm file, then this
+  ** process might open and use the *-shm file without truncating it.
+  ** And if the *-shm file has been corrupted by a power failure or
+  ** system crash, the database itself may also become corrupt.  */
+  lock.l_whence = SEEK_SET;
+  lock.l_start = UNIX_SHM_DMS;
+  lock.l_len = 1;
+  lock.l_type = F_WRLCK;
+  if( osFcntl(pShmNode->h, F_GETLK, &lock)!=0 ) {
+    rc = SQLITE_IOERR_LOCK;
+  }else if( lock.l_type==F_UNLCK ){
+    if( pShmNode->isReadonly ){
+      pShmNode->isUnlocked = 1;
+      rc = SQLITE_READONLY_CANTLOCK;
+    }else{
+      rc = unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1);
+      if( rc==SQLITE_OK && robust_ftruncate(pShmNode->h, 0) ){
+        rc = unixLogError(SQLITE_IOERR_SHMOPEN,"ftruncate",pShmNode->zFilename);
+      }
+    }
+  }else if( lock.l_type==F_WRLCK ){
+    rc = SQLITE_BUSY;
+  }
+
+  if( rc==SQLITE_OK ){
+    assert( lock.l_type==F_UNLCK || lock.l_type==F_RDLCK );
+    rc = unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, 1);
+  }
+  return rc;
+}
+
 /*
 ** Open a shared-memory area associated with open database file pDbFd.  
 ** This particular implementation uses mmapped files.
@@ -4308,7 +4367,7 @@ static void unixShmPurge(unixFile *pFd){
 static int unixOpenSharedMemory(unixFile *pDbFd){
   struct unixShm *p = 0;          /* The connection to be opened */
   struct unixShmNode *pShmNode;   /* The underlying mmapped file */
-  int rc;                         /* Result code */
+  int rc = SQLITE_OK;             /* Result code */
   unixInodeInfo *pInode;          /* The inode of fd */
   char *zShmFilename;             /* Name of the file used for SHM */
   int nShmFilename;               /* Size of the SHM filename in bytes */
@@ -4372,7 +4431,6 @@ static int unixOpenSharedMemory(unixFile *pDbFd){
     }
 
     if( pInode->bProcessLock==0 ){
-      struct flock lock;
       int openFlags = O_RDWR | O_CREAT;
       if( sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){
         openFlags = O_RDONLY;
@@ -4389,50 +4447,9 @@ static int unixOpenSharedMemory(unixFile *pDbFd){
       ** the original owner will not be able to connect.
       */
       robustFchown(pShmNode->h, sStat.st_uid, sStat.st_gid);
-  
-      /* Use F_GETLK to determine the locks other processes are holding
-      ** on the DMS byte. If it indicates that another process is holding
-      ** a SHARED lock, then this process may also take a SHARED lock
-      ** and proceed with opening the *-shm file. 
-      **
-      ** Or, if no other process is holding any lock, then this process
-      ** is the first to open it. In this case take an EXCLUSIVE lock on the
-      ** DMS byte and truncate the *-shm file to zero bytes in size. Then
-      ** downgrade to a SHARED lock on the DMS byte.
-      **
-      ** If another process is holding an EXCLUSIVE lock on the DMS byte,
-      ** return SQLITE_BUSY to the caller (it will try again). An earlier
-      ** version of this code attempted the SHARED lock at this point. But
-      ** this introduced a subtle race condition: if the process holding
-      ** EXCLUSIVE failed just before truncating the *-shm file, then this
-      ** process might open and use the *-shm file without truncating it.
-      ** And if the *-shm file has been corrupted by a power failure or
-      ** system crash, the database itself may also become corrupt.  */
-      rc = SQLITE_OK;
-      lock.l_whence = SEEK_SET;
-      lock.l_start = UNIX_SHM_DMS;
-      lock.l_len = 1;
-      lock.l_type = F_WRLCK;
-      if( osFcntl(pShmNode->h, F_GETLK, &lock)!=0 ) {
-        rc = SQLITE_IOERR_LOCK;
-      }else if( lock.l_type==F_UNLCK ){
-        if( pShmNode->isReadonly ){
-          rc = SQLITE_CANTOPEN_DIRTYWAL;
-        }else{
-          rc = unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1);
-          if( rc==SQLITE_OK && robust_ftruncate(pShmNode->h, 0) ){
-            rc = unixLogError(SQLITE_IOERR_SHMOPEN, "ftruncate", zShmFilename);
-          }
-        }
-      }else if( lock.l_type==F_WRLCK ){
-        rc = SQLITE_BUSY;
-      }
 
-      if( rc==SQLITE_OK ){
-        assert( lock.l_type==F_UNLCK || lock.l_type==F_RDLCK );
-        rc = unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, 1);
-      }
-      if( rc ) goto shm_open_err;
+      rc = unixLockSharedMemory(pDbFd, pShmNode);
+      if( rc!=SQLITE_OK && rc!=SQLITE_READONLY_CANTLOCK ) goto shm_open_err;
     }
   }
 
@@ -4456,7 +4473,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){
   p->pNext = pShmNode->pFirst;
   pShmNode->pFirst = p;
   sqlite3_mutex_leave(pShmNode->mutex);
-  return SQLITE_OK;
+  return rc;
 
   /* Jump here on any error */
 shm_open_err:
@@ -4508,6 +4525,11 @@ static int unixShmMap(
   p = pDbFd->pShm;
   pShmNode = p->pShmNode;
   sqlite3_mutex_enter(pShmNode->mutex);
+  if( pShmNode->isUnlocked ){
+    rc = unixLockSharedMemory(pDbFd, pShmNode);
+    if( rc!=SQLITE_OK ) goto shmpage_out;
+    pShmNode->isUnlocked = 0;
+  }
   assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 );
   assert( pShmNode->pInode==pDbFd->pInode );
   assert( pShmNode->h>=0 || pDbFd->pInode->bProcessLock==1 );
diff --git a/src/wal.c b/src/wal.c
index 19c9ea0a08..1a11eb31e0 100644
--- a/src/wal.c
+++ b/src/wal.c
@@ -575,9 +575,11 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){
       rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, 
           pWal->writeLock, (void volatile **)&pWal->apWiData[iPage]
       );
-      if( rc==SQLITE_READONLY ){
+      if( (rc&0xff)==SQLITE_READONLY ){
         pWal->readOnly |= WAL_SHM_RDONLY;
-        rc = SQLITE_OK;
+        if( rc==SQLITE_READONLY ){
+          rc = SQLITE_OK;
+        }
       }
     }
   }
@@ -2084,6 +2086,14 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){
   assert( pChanged );
   rc = walIndexPage(pWal, 0, &page0);
   if( rc!=SQLITE_OK ){
+    if( rc==SQLITE_READONLY_CANTLOCK 
+#ifdef SQLITE_ENABLE_SNAPSHOT
+        && pWal->pSnapshot==0 
+#endif
+    ){
+      memset(&pWal->hdr, 0, sizeof(WalIndexHdr));
+      rc = SQLITE_OK;
+    }
     return rc;
   };
   assert( page0 || pWal->writeLock==0 );
@@ -2259,8 +2269,10 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){
     }
   }
 
-  pInfo = walCkptInfo(pWal);
-  if( !useWal && pInfo->nBackfill==pWal->hdr.mxFrame 
+  assert( pWal->nWiData>0 );
+  assert( pWal->apWiData[0] || (pWal->readOnly & WAL_SHM_RDONLY) );
+  pInfo = pWal->apWiData[0] ? walCkptInfo(pWal) : 0;
+  if( !useWal && (pInfo==0 || pInfo->nBackfill==pWal->hdr.mxFrame)
 #ifdef SQLITE_ENABLE_SNAPSHOT
    && (pWal->pSnapshot==0 || pWal->hdr.mxFrame==0
      || 0==memcmp(&pWal->hdr, pWal->pSnapshot, sizeof(WalIndexHdr)))
@@ -2272,7 +2284,9 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){
     rc = walLockShared(pWal, WAL_READ_LOCK(0));
     walShmBarrier(pWal);
     if( rc==SQLITE_OK ){
-      if( memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){
+      if( pInfo
+       && memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) 
+      ){
         /* It is not safe to allow the reader to continue here if frames
         ** may have been appended to the log before READ_LOCK(0) was obtained.
         ** When holding READ_LOCK(0), the reader ignores the entire log file,
diff --git a/test/walro2.test b/test/walro2.test
new file mode 100644
index 0000000000..2337776f27
--- /dev/null
+++ b/test/walro2.test
@@ -0,0 +1,87 @@
+# 2011 May 09
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains tests for using WAL databases in read-only mode.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+set ::testprefix walro
+
+# These tests are only going to work on unix.
+#
+if {$::tcl_platform(platform) != "unix"} {
+  finish_test
+  return
+}
+
+# And only if the build is WAL-capable.
+#
+ifcapable !wal {
+  finish_test
+  return
+}
+
+do_multiclient_test tn {
+  
+  # Close all connections and delete the database.
+  #
+  code1 { db close  }
+  code2 { db2 close }
+  code3 { db3 close }
+  forcedelete test.db
+  
+  # Do not run tests with the connections in the same process.
+  #
+  if {$tn==2} continue
+
+  foreach c {code1 code2 code3} {
+    $c {
+      sqlite3_shutdown
+      sqlite3_config_uri 1
+    }
+  }
+
+  do_test 1.1 {
+    code2 { sqlite3 db2 test.db }
+    sql2 { 
+      CREATE TABLE t1(x, y);
+      PRAGMA journal_mode = WAL;
+      INSERT INTO t1 VALUES('a', 'b');
+      INSERT INTO t1 VALUES('c', 'd');
+    }
+    file exists test.db-shm
+  } {1}
+
+  do_test 1.2 {
+    forcecopy test.db test.db2
+    forcecopy test.db-wal test.db2-wal
+    forcecopy test.db-shm test.db2-shm
+    code1 {
+      sqlite3 db file:test.db2?readonly_shm=1
+    }
+
+    sql1 { SELECT * FROM t1 }
+  } {}
+
+  do_test 1.3.1 {
+    code3 { sqlite3 db3 test.db2 }
+    sql3 { SELECT * FROM t1 }
+  } {a b c d}
+
+  do_test 1.3.2 {
+    sql1 { SELECT * FROM t1 }
+  } {a b c d}
+
+}
+
+finish_test

From dea5ce36f57f83b4c6c78154c088d67ec8c5c594 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 2 Nov 2017 11:12:03 +0000
Subject: [PATCH 059/488] Avoid locking shm-lock WAL_READ_LOCK(0) during
 recovery. Doing this allows recovery to proceed while a readonly_shm
 connection in unlocked mode has an ongoing read transaction.

FossilOrigin-Name: 5190d84a296b7cf716ef43bf7b6d4d351ef1a4d650de37dc01a5ab333da7c05d
---
 manifest         | 14 +++++++-------
 manifest.uuid    |  2 +-
 src/wal.c        | 14 ++++++++++----
 test/walro2.test | 31 +++++++++++++++++++++++++++++++
 4 files changed, 49 insertions(+), 12 deletions(-)

diff --git a/manifest b/manifest
index 45c7b16d67..f3b7f8abe0 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C If\sa\sreadonly_shm\sconnection\scannot\smap\sthe\s*-shm\sfile\sbecause\sno\sother\nprocess\sis\sholding\sthe\sDMS\slock,\shave\sit\sread\sfrom\sthe\sdatabase\sfile\sonly,\nignoring\sany\scontent\sin\sthe\swal\sfile.
-D 2017-11-01T20:59:28.295
+C Avoid\slocking\sshm-lock\sWAL_READ_LOCK(0)\sduring\srecovery.\sDoing\sthis\sallows\nrecovery\sto\sproceed\swhile\sa\sreadonly_shm\sconnection\sin\sunlocked\smode\shas\san\nongoing\sread\stransaction.
+D 2017-11-02T11:12:03.045
 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2
@@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c 1521bdcfe9a536752a339c91b63ca0d226d8d266636391aac93537fdea8657fc
+F src/wal.c 38480e7bdc697cf88a13a22ffe60f7bd761bc02b45f7a323f1bb9e61a136b3ae
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f
 F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4
@@ -1527,7 +1527,7 @@ F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03
 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6
 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20
 F test/walro.test e492598baa8cd7777fef6203f6fe922c20cd691cc19e60ccd0dd0dbc68394d0a
-F test/walro2.test e2cd102cafceafaf19c56d55e55222fdd26d7621d7ef42b0eaf35c06c5bb3d19
+F test/walro2.test 23fea1e7abae13072b0640ef846d32080b2fc435658d4c4eb9db266b07b33776
 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417
 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f
 F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e
@@ -1668,7 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 985bfc992950625a45a7521bf4c8438cd0170de974dff976968be158ac5922a9
-R fcc128b567671c56015745eca2c28340
+P ce5d13c2de69b73378637d4f7e109714f7cd17bf1d1ad995e0be442d517ed1b3
+R 5519605a9af9008f6aa3aade00e8ea39
 U dan
-Z e9ea9ea441f189aa52edada39fcf2f83
+Z aa82fa216e4468716131c3fe0543b69b
diff --git a/manifest.uuid b/manifest.uuid
index ebf42c65a3..08fbfd4ed2 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ce5d13c2de69b73378637d4f7e109714f7cd17bf1d1ad995e0be442d517ed1b3
\ No newline at end of file
+5190d84a296b7cf716ef43bf7b6d4d351ef1a4d650de37dc01a5ab333da7c05d
\ No newline at end of file
diff --git a/src/wal.c b/src/wal.c
index 1a11eb31e0..28fade96b0 100644
--- a/src/wal.c
+++ b/src/wal.c
@@ -1101,7 +1101,6 @@ static int walIndexRecover(Wal *pWal){
   i64 nSize;                      /* Size of log file */
   u32 aFrameCksum[2] = {0, 0};
   int iLock;                      /* Lock offset to lock for checkpoint */
-  int nLock;                      /* Number of locks to hold */
 
   /* Obtain an exclusive lock on all byte in the locking range not already
   ** locked by the caller. The caller is guaranteed to have locked the
@@ -1114,11 +1113,17 @@ static int walIndexRecover(Wal *pWal){
   assert( WAL_CKPT_LOCK==WAL_ALL_BUT_WRITE );
   assert( pWal->writeLock );
   iLock = WAL_ALL_BUT_WRITE + pWal->ckptLock;
-  nLock = SQLITE_SHM_NLOCK - iLock;
-  rc = walLockExclusive(pWal, iLock, nLock);
+  rc = walLockExclusive(pWal, iLock, WAL_READ_LOCK(0)-iLock);
+  if( rc==SQLITE_OK ){
+    rc = walLockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1);
+    if( rc!=SQLITE_OK ){
+      walUnlockExclusive(pWal, iLock, WAL_READ_LOCK(0)-iLock);
+    }
+  }
   if( rc ){
     return rc;
   }
+
   WALTRACE(("WAL%p: recovery begin...\n", pWal));
 
   memset(&pWal->hdr, 0, sizeof(WalIndexHdr));
@@ -1256,7 +1261,8 @@ finished:
 
 recovery_error:
   WALTRACE(("WAL%p: recovery %s\n", pWal, rc ? "failed" : "ok"));
-  walUnlockExclusive(pWal, iLock, nLock);
+  walUnlockExclusive(pWal, iLock, WAL_READ_LOCK(0)-iLock);
+  walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1);
   return rc;
 }
 
diff --git a/test/walro2.test b/test/walro2.test
index 2337776f27..fb41d17f79 100644
--- a/test/walro2.test
+++ b/test/walro2.test
@@ -82,6 +82,37 @@ do_multiclient_test tn {
     sql1 { SELECT * FROM t1 }
   } {a b c d}
 
+  code1 { db close  }
+  code2 { db2 close }
+  code3 { db3 close }
+
+  do_test 2.1 {
+    code2 { sqlite3 db2 test.db }
+    sql2 { 
+      INSERT INTO t1 VALUES('e', 'f');
+      INSERT INTO t1 VALUES('g', 'h');
+    }
+    file exists test.db-shm
+  } {1}
+
+  do_test 2.2 {
+    forcecopy test.db test.db2
+    forcecopy test.db-wal test.db2-wal
+    forcecopy test.db-shm test.db2-shm
+    code1 {
+      sqlite3 db file:test.db2?readonly_shm=1
+    }
+    sql1 { 
+      BEGIN;
+      SELECT * FROM t1;
+    }
+  } {a b c d}
+
+  do_test 2.3.1 {
+    code3 { sqlite3 db3 test.db2 }
+    sql3 { SELECT * FROM t1 }
+  } {a b c d e f g h}
+
 }
 
 finish_test

From 44c8a97e019bfb791fd9b330212e2375d7516849 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 2 Nov 2017 18:57:46 +0000
Subject: [PATCH 060/488] Fix test cases in wal2.test broken by the locking
 change in the previous commit.

FossilOrigin-Name: f569c3517234881f9425075aab65a32ffd0deb8e793f421a241d8cca881da33f
---
 manifest       | 12 ++++++------
 manifest.uuid  |  2 +-
 test/wal2.test | 14 ++++++++------
 3 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/manifest b/manifest
index f3b7f8abe0..5ad595579c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Avoid\slocking\sshm-lock\sWAL_READ_LOCK(0)\sduring\srecovery.\sDoing\sthis\sallows\nrecovery\sto\sproceed\swhile\sa\sreadonly_shm\sconnection\sin\sunlocked\smode\shas\san\nongoing\sread\stransaction.
-D 2017-11-02T11:12:03.045
+C Fix\stest\scases\sin\swal2.test\sbroken\sby\sthe\slocking\schange\sin\sthe\sprevious\ncommit.
+D 2017-11-02T18:57:46.036
 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2
@@ -1501,7 +1501,7 @@ F test/vtab_alter.test 736e66fb5ec7b4fee58229aa3ada2f27ec58bc58c00edae4836890c37
 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8
 F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad
 F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477
-F test/wal2.test 6ac39b94a284ebac6efb6be93b0cdfe73ee6083f129555e3144d8a615e9900ef
+F test/wal2.test 4c44bbe447959638e5163631a1fe95c9dbc01a06eff6eb34449be06b6e0ed64c
 F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2
 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c
 F test/wal5.test 9c11da7aeccd83a46d79a556ad11a18d3cb15aa9
@@ -1668,7 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ce5d13c2de69b73378637d4f7e109714f7cd17bf1d1ad995e0be442d517ed1b3
-R 5519605a9af9008f6aa3aade00e8ea39
+P 5190d84a296b7cf716ef43bf7b6d4d351ef1a4d650de37dc01a5ab333da7c05d
+R 30693e487c23aaa3c080ebb269e4e38f
 U dan
-Z aa82fa216e4468716131c3fe0543b69b
+Z a5242a9fca30f5b5641106b18cdb01ae
diff --git a/manifest.uuid b/manifest.uuid
index 08fbfd4ed2..a8bcfdb1fe 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5190d84a296b7cf716ef43bf7b6d4d351ef1a4d650de37dc01a5ab333da7c05d
\ No newline at end of file
+f569c3517234881f9425075aab65a32ffd0deb8e793f421a241d8cca881da33f
\ No newline at end of file
diff --git a/test/wal2.test b/test/wal2.test
index bc170ad3ff..ddff71aed4 100644
--- a/test/wal2.test
+++ b/test/wal2.test
@@ -122,8 +122,8 @@ do_test wal2-1.1 {
 } {4 10}
 
 set RECOVER [list                                      \
-  {0 1 lock exclusive}   {1 7 lock exclusive}          \
-  {1 7 unlock exclusive} {0 1 unlock exclusive}        \
+  {0 1 lock exclusive}   {1 2 lock exclusive} {4 4 lock exclusive} \
+  {1 2 unlock exclusive} {4 4 unlock exclusive} {0 1 unlock exclusive}  \
 ]
 set READ [list                                         \
   {4 1 lock shared}    {4 1 unlock shared}             \
@@ -393,8 +393,10 @@ tvfs delete
 set expected_locks [list]
 lappend expected_locks {1 1 lock exclusive}   ;# Lock checkpoint
 lappend expected_locks {0 1 lock exclusive}   ;# Lock writer
-lappend expected_locks {2 6 lock exclusive}   ;# Lock recovery & all aReadMark[]
-lappend expected_locks {2 6 unlock exclusive} ;# Unlock recovery & aReadMark[]
+lappend expected_locks {2 1 lock exclusive}   ;# Lock recovery
+lappend expected_locks {4 4 lock exclusive}   ;# Lock all aReadMark[]
+lappend expected_locks {2 1 unlock exclusive} ;# Unlock recovery 
+lappend expected_locks {4 4 unlock exclusive} ;# Unlock all aReadMark[] 
 lappend expected_locks {0 1 unlock exclusive} ;# Unlock writer
 lappend expected_locks {3 1 lock exclusive}   ;# Lock aReadMark[0]
 lappend expected_locks {3 1 unlock exclusive} ;# Unlock aReadMark[0]
@@ -615,8 +617,8 @@ do_test wal2-6.4.1 {
 } {}
 
 set RECOVERY {
-  {0 1 lock exclusive} {1 7 lock exclusive} 
-  {1 7 unlock exclusive} {0 1 unlock exclusive}
+  {0 1 lock exclusive} {1 2 lock exclusive} {4 4 lock exclusive}
+  {1 2 unlock exclusive} {4 4 unlock exclusive} {0 1 unlock exclusive}
 }
 set READMARK0_READ {
   {3 1 lock shared} {3 1 unlock shared}

From 11caf4f4b73736f6b209d591d652c7e0398aa024 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 4 Nov 2017 18:10:03 +0000
Subject: [PATCH 061/488] In cases where a readonly_shm client cannot take the
 DMS lock on the *-shm file, have it parse the wal file and create a wal-index
 to access it in heap memory.

FossilOrigin-Name: 18b268433d739486eac1b04947bd418655e4bc56e8dc63ffa558aa4552a32e30
---
 manifest         |  16 ++---
 manifest.uuid    |   2 +-
 src/wal.c        | 176 +++++++++++++++++++++++++++++++++++++++++++----
 test/walro.test  |  10 +--
 test/walro2.test |  19 +++--
 5 files changed, 192 insertions(+), 31 deletions(-)

diff --git a/manifest b/manifest
index 5ad595579c..7e5b9e5a86 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\stest\scases\sin\swal2.test\sbroken\sby\sthe\slocking\schange\sin\sthe\sprevious\ncommit.
-D 2017-11-02T18:57:46.036
+C In\scases\swhere\sa\sreadonly_shm\sclient\scannot\stake\sthe\sDMS\slock\son\sthe\s*-shm\nfile,\shave\sit\sparse\sthe\swal\sfile\sand\screate\sa\swal-index\sto\saccess\sit\sin\sheap\nmemory.
+D 2017-11-04T18:10:03.528
 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2
@@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c 38480e7bdc697cf88a13a22ffe60f7bd761bc02b45f7a323f1bb9e61a136b3ae
+F src/wal.c 2b287b5250e89d548c6bbd1d204d0db41046bb3984b9b4a79fc84e22359f1beb
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f
 F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4
@@ -1526,8 +1526,8 @@ F test/walnoshm.test 84ca10c544632a756467336b7c3b864d493ee496
 F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03
 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6
 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20
-F test/walro.test e492598baa8cd7777fef6203f6fe922c20cd691cc19e60ccd0dd0dbc68394d0a
-F test/walro2.test 23fea1e7abae13072b0640ef846d32080b2fc435658d4c4eb9db266b07b33776
+F test/walro.test 906586c3ae7a991d8c840ceed92400aee21a0a3e4155ce7c4220399777311552
+F test/walro2.test 611ceebd190edeca9bf39e5068cbc864f15294371b4acf9ee837db477840af54
 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417
 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f
 F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e
@@ -1668,7 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5190d84a296b7cf716ef43bf7b6d4d351ef1a4d650de37dc01a5ab333da7c05d
-R 30693e487c23aaa3c080ebb269e4e38f
+P f569c3517234881f9425075aab65a32ffd0deb8e793f421a241d8cca881da33f
+R 10fc0a1645ce290a2a283e11360fe887
 U dan
-Z a5242a9fca30f5b5641106b18cdb01ae
+Z 79ea7f3203670e23f752126d91be88fd
diff --git a/manifest.uuid b/manifest.uuid
index a8bcfdb1fe..e8b4f82532 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f569c3517234881f9425075aab65a32ffd0deb8e793f421a241d8cca881da33f
\ No newline at end of file
+18b268433d739486eac1b04947bd418655e4bc56e8dc63ffa558aa4552a32e30
\ No newline at end of file
diff --git a/src/wal.c b/src/wal.c
index 28fade96b0..17e253a8b6 100644
--- a/src/wal.c
+++ b/src/wal.c
@@ -455,6 +455,7 @@ struct Wal {
   u8 truncateOnCommit;       /* True to truncate WAL file on commit */
   u8 syncHeader;             /* Fsync the WAL header if true */
   u8 padToSectorBoundary;    /* Pad transactions out to the next sector */
+  u8 bUnlocked;
   WalIndexHdr hdr;           /* Wal-index header for current transaction */
   u32 minFrame;              /* Ignore wal frames before this one */
   u32 iReCksum;              /* On commit, recalculate checksums from here */
@@ -1270,13 +1271,14 @@ recovery_error:
 ** Close an open wal-index.
 */
 static void walIndexClose(Wal *pWal, int isDelete){
-  if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){
+  if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE || pWal->bUnlocked ){
     int i;
     for(i=0; i<pWal->nWiData; i++){
       sqlite3_free((void *)pWal->apWiData[i]);
       pWal->apWiData[i] = 0;
     }
-  }else{
+  }
+  if( pWal->exclusiveMode!=WAL_HEAPMEMORY_MODE ){
     sqlite3OsShmUnmap(pWal->pDbFd, isDelete);
   }
 }
@@ -2091,15 +2093,13 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){
   */
   assert( pChanged );
   rc = walIndexPage(pWal, 0, &page0);
+  if( rc==SQLITE_READONLY_CANTLOCK ){
+    assert( page0==0 && pWal->writeLock==0 );
+    pWal->bUnlocked = 1;
+    pWal->exclusiveMode = WAL_HEAPMEMORY_MODE;
+    *pChanged = 1;
+  }else
   if( rc!=SQLITE_OK ){
-    if( rc==SQLITE_READONLY_CANTLOCK 
-#ifdef SQLITE_ENABLE_SNAPSHOT
-        && pWal->pSnapshot==0 
-#endif
-    ){
-      memset(&pWal->hdr, 0, sizeof(WalIndexHdr));
-      rc = SQLITE_OK;
-    }
     return rc;
   };
   assert( page0 || pWal->writeLock==0 );
@@ -2116,7 +2116,7 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){
   */
   assert( badHdr==0 || pWal->writeLock==0 );
   if( badHdr ){
-    if( pWal->readOnly & WAL_SHM_RDONLY ){
+    if( pWal->bUnlocked==0 && (pWal->readOnly & WAL_SHM_RDONLY) ){
       if( SQLITE_OK==(rc = walLockShared(pWal, WAL_WRITE_LOCK)) ){
         walUnlockShared(pWal, WAL_WRITE_LOCK);
         rc = SQLITE_READONLY_RECOVERY;
@@ -2146,6 +2146,12 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){
   if( badHdr==0 && pWal->hdr.iVersion!=WALINDEX_MAX_VERSION ){
     rc = SQLITE_CANTOPEN_BKPT;
   }
+  if( pWal->bUnlocked ){
+    if( rc!=SQLITE_OK ){
+      walIndexClose(pWal, 0);
+    }
+    pWal->exclusiveMode = WAL_NORMAL_MODE;
+  }
 
   return rc;
 }
@@ -2156,6 +2162,144 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){
 */
 #define WAL_RETRY  (-1)
 
+/*
+** Open an "unlocked" transaction. An unlocked transaction is a read 
+** transaction used by a read-only client in cases where the *-shm
+** file cannot be mapped and its contents cannot be trusted. It is
+** assumed that the *-wal file has been read and that a wal-index 
+** constructed in heap memory is currently available in Wal.apWiData[].
+**
+** If this function returns SQLITE_OK, then the read transaction has
+** been successfully opened. In this case output variable (*pChanged) 
+** is set to true before returning if the caller should discard the
+** contents of the page cache before proceeding. Or, if it returns 
+** WAL_RETRY, then the heap memory wal-index has been discarded and 
+** the caller should retry opening the read transaction from the 
+** beginning (including attempting to map the *-shm file). 
+**
+** If an error occurs, an SQLite error code is returned.
+*/
+static int walBeginUnlocked(Wal *pWal, int *pChanged){
+  i64 szWal;                      /* Size of wal file on disk in bytes */
+  i64 iOffset;                    /* Current offset when reading wal file */
+  u8 aBuf[WAL_HDRSIZE];           /* Buffer to load WAL header into */
+  u8 *aFrame = 0;                 /* Malloc'd buffer to load entire frame */
+  int szFrame;                    /* Number of bytes in buffer aFrame[] */
+  u8 *aData;                      /* Pointer to data part of aFrame buffer */
+  volatile void *pDummy;          /* Dummy argument for xShmMap */
+  int rc;                         /* Return code */
+  u32 aSaveCksum[2];              /* Saved copy of pWal->hdr.aFrameCksum */
+
+  assert( pWal->bUnlocked );
+  assert( pWal->readOnly & WAL_SHM_RDONLY );
+  assert( pWal->nWiData>0 && pWal->apWiData[0] );
+
+  /* Take WAL_READ_LOCK(0). This has the effect of preventing any
+  ** live clients from running a checkpoint, but does not stop them
+  ** from running recovery.  */
+  rc = walLockShared(pWal, WAL_READ_LOCK(0));
+  if( rc!=SQLITE_OK ){
+    return (rc==SQLITE_BUSY ? WAL_RETRY : rc);
+  }
+  pWal->readLock = 0;
+
+  /* Try to map the *-shm file again. If it succeeds this time, then 
+  ** a non-readonly_shm connection has already connected to the database.
+  ** In this case, start over with opening the transaction.
+  ** 
+  ** The WAL_READ_LOCK(0) lock held by this client prevents a checkpoint
+  ** from taking place. But it does not prevent the wal from being wrapped
+  ** if a checkpoint has already taken place. This means that if another
+  ** client is connected at this point, it may have already checkpointed 
+  ** the entire wal. In that case it would not be safe to continue with
+  ** the unlocked transaction, as the other client may overwrite wal 
+  ** frames that this client is still using.  */
+  rc = sqlite3OsShmMap(pWal->pDbFd, 0, WALINDEX_PGSZ, 0, &pDummy);
+  if( rc!=SQLITE_READONLY_CANTLOCK ){
+    assert( rc!=SQLITE_OK );
+    rc = (rc==SQLITE_READONLY ? WAL_RETRY : rc);
+    goto begin_unlocked_out;
+  }
+
+  memcpy(&pWal->hdr, (void*)walIndexHdr(pWal), sizeof(WalIndexHdr));
+  rc = sqlite3OsFileSize(pWal->pWalFd, &szWal);
+  if( rc!=SQLITE_OK || (szWal<WAL_HDRSIZE && pWal->hdr.mxFrame==0) ){
+    /* If the wal file is too small to contain a wal-header and the
+    ** wal-index header has mxFrame==0, then it must be safe to proceed
+    ** reading the database file only. However, the page cache cannot
+    ** be trusted, as a read/write connection may have connected, written
+    ** the db, run a checkpoint, truncated the wal file and disconnected
+    ** since this client's last read transaction.  */
+    *pChanged = 1;
+    goto begin_unlocked_out;
+  }
+
+  /* Check the salt keys at the start of the wal file still match. */
+  rc = sqlite3OsRead(pWal->pWalFd, aBuf, WAL_HDRSIZE, 0);
+  if( rc!=SQLITE_OK ){
+    goto begin_unlocked_out;
+  }
+  if( memcmp(&pWal->hdr.aSalt, &aBuf[16], 8) ){
+    rc = WAL_RETRY;
+    goto begin_unlocked_out;
+  }
+
+  /* Allocate a buffer to read frames into */
+  szFrame = pWal->hdr.szPage + WAL_FRAME_HDRSIZE;
+  aFrame = (u8 *)sqlite3_malloc64(szFrame);
+  if( aFrame==0 ){
+    rc = SQLITE_NOMEM_BKPT;
+    goto begin_unlocked_out;
+  }
+  aData = &aFrame[WAL_FRAME_HDRSIZE];
+
+  aSaveCksum[0] = pWal->hdr.aFrameCksum[0];
+  aSaveCksum[1] = pWal->hdr.aFrameCksum[1];
+  for(iOffset=walFrameOffset(pWal->hdr.mxFrame+1, pWal->hdr.szPage); 
+      iOffset+szFrame<=szWal; 
+      iOffset+=szFrame
+  ){
+    u32 pgno;                   /* Database page number for frame */
+    u32 nTruncate;              /* dbsize field from frame header */
+
+    /* Read and decode the next log frame. */
+    rc = sqlite3OsRead(pWal->pWalFd, aFrame, szFrame, iOffset);
+    if( rc!=SQLITE_OK ){
+      if( rc==SQLITE_IOERR_SHORT_READ ){
+        /* If this branch is taken, some other client has truncated the
+        ** *-wal file since the call to sqlite3OsFileSize() above. This
+        ** indicates that a read-write client has connected to the system.
+        ** So retry opening this read transaction.  */
+        rc = WAL_RETRY;
+      }
+      break;
+    }
+    if( !walDecodeFrame(pWal, &pgno, &nTruncate, aData, aFrame) ) break;
+
+    /* If nTruncate is non-zero, this is a commit record. */
+    if( nTruncate ){
+      rc = WAL_RETRY;
+      break;
+    }
+  }
+  pWal->hdr.aFrameCksum[0] = aSaveCksum[0];
+  pWal->hdr.aFrameCksum[1] = aSaveCksum[1];
+
+ begin_unlocked_out:
+  sqlite3_free(aFrame);
+  if( rc!=SQLITE_OK ){
+    int i;
+    for(i=0; i<pWal->nWiData; i++){
+      sqlite3_free((void*)pWal->apWiData[i]);
+      pWal->apWiData[i] = 0;
+    }
+    pWal->bUnlocked = 0;
+    sqlite3WalEndReadTransaction(pWal);
+    *pChanged = 1;
+  }
+  return rc;
+}
+
 /*
 ** Attempt to start a read transaction.  This might fail due to a race or
 ** other transient condition.  When that happens, it returns WAL_RETRY to
@@ -2244,7 +2388,10 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){
   }
 
   if( !useWal ){
-    rc = walIndexReadHdr(pWal, pChanged);
+    assert( rc==SQLITE_OK );
+    if( pWal->bUnlocked==0 ){
+      rc = walIndexReadHdr(pWal, pChanged);
+    }
     if( rc==SQLITE_BUSY ){
       /* If there is not a recovery running in another thread or process
       ** then convert BUSY errors to WAL_RETRY.  If recovery is known to
@@ -2273,6 +2420,9 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){
     if( rc!=SQLITE_OK ){
       return rc;
     }
+    else if( pWal->bUnlocked ){
+      return walBeginUnlocked(pWal, pChanged);
+    }
   }
 
   assert( pWal->nWiData>0 );
@@ -2626,7 +2776,7 @@ int sqlite3WalFindFrame(
   ** then the WAL is ignored by the reader so return early, as if the 
   ** WAL were empty.
   */
-  if( iLast==0 || pWal->readLock==0 ){
+  if( iLast==0 || (pWal->readLock==0 && pWal->bUnlocked==0) ){
     *piRead = 0;
     return SQLITE_OK;
   }
diff --git a/test/walro.test b/test/walro.test
index 09a33bbd3e..150344e151 100644
--- a/test/walro.test
+++ b/test/walro.test
@@ -105,7 +105,7 @@ do_multiclient_test tn {
   do_test 1.2.2 {
     code1 { sqlite3 db file:test.db?readonly_shm=1 }
     list [catch { sql1 { SELECT * FROM t1 } } msg] $msg
-  } {1 {unable to open database file}}
+  } {0 {a b c d e f g h i j}}
 
   do_test 1.2.3 {
     code1 { db close }
@@ -114,10 +114,10 @@ do_multiclient_test tn {
     file attributes test.db-shm -permissions r--r--r--
     code1 { sqlite3 db file:test.db?readonly_shm=1 }
     csql1 { SELECT * FROM t1 }
-  } {1 {unable to open database file}}
+  } {0 {a b c d e f g h i j}}
   do_test 1.2.4 {
     code1 { sqlite3_extended_errcode db } 
-  } {SQLITE_CANTOPEN}
+  } {SQLITE_OK}
 
   do_test 1.2.5 {
     file attributes test.db-shm -permissions rw-r--r--
@@ -162,10 +162,10 @@ do_multiclient_test tn {
     file attributes test.db-shm -permissions r--r--r--
     code1 { sqlite3 db file:test.db?readonly_shm=1 }
     csql1 { SELECT * FROM t1 }
-  } {1 {unable to open database file}}
+  } {0 {a b c d e f g h i j k l}}
   do_test 1.3.2.4 {
     code1 { sqlite3_extended_errcode db } 
-  } {SQLITE_CANTOPEN}
+  } {SQLITE_OK}
 
   #-----------------------------------------------------------------------
   # Test cases 1.4.* check that checkpoints and log wraps don't prevent
diff --git a/test/walro2.test b/test/walro2.test
index fb41d17f79..ee4a341e24 100644
--- a/test/walro2.test
+++ b/test/walro2.test
@@ -62,7 +62,7 @@ do_multiclient_test tn {
     file exists test.db-shm
   } {1}
 
-  do_test 1.2 {
+  do_test 1.2.1 {
     forcecopy test.db test.db2
     forcecopy test.db-wal test.db2-wal
     forcecopy test.db-shm test.db2-shm
@@ -71,7 +71,10 @@ do_multiclient_test tn {
     }
 
     sql1 { SELECT * FROM t1 }
-  } {}
+  } {a b c d}
+  do_test 1.2.2 {
+    sql1 { SELECT * FROM t1 }
+  } {a b c d}
 
   do_test 1.3.1 {
     code3 { sqlite3 db3 test.db2 }
@@ -106,13 +109,21 @@ do_multiclient_test tn {
       BEGIN;
       SELECT * FROM t1;
     }
-  } {a b c d}
+  } {a b c d e f g h}
 
   do_test 2.3.1 {
     code3 { sqlite3 db3 test.db2 }
     sql3 { SELECT * FROM t1 }
   } {a b c d e f g h}
-
+  do_test 2.3.2 {
+    sql3 { INSERT INTO t1 VALUES('i', 'j') }
+    code3 { db3 close }
+    sql1 { COMMIT } 
+  } {}
+  breakpoint
+  do_test 2.3.3 {
+    sql1 { SELECT * FROM t1 }
+  } {a b c d e f g h i j}
 }
 
 finish_test

From cbd3321978bd66bb53f87abe4e25c1f57b087652 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 4 Nov 2017 21:06:35 +0000
Subject: [PATCH 062/488] Add further tests for the code added on this branch.

FossilOrigin-Name: a6716fcde38b28b8a03b40f9d16f78a57ec20f60cf391ff553692641cb7f0d3f
---
 manifest         | 14 ++++-----
 manifest.uuid    |  2 +-
 src/wal.c        |  8 ++++-
 test/walro2.test | 76 ++++++++++++++++++++++++++++++++++++++++++++++--
 4 files changed, 89 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index 7e5b9e5a86..84f5558cad 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\scases\swhere\sa\sreadonly_shm\sclient\scannot\stake\sthe\sDMS\slock\son\sthe\s*-shm\nfile,\shave\sit\sparse\sthe\swal\sfile\sand\screate\sa\swal-index\sto\saccess\sit\sin\sheap\nmemory.
-D 2017-11-04T18:10:03.528
+C Add\sfurther\stests\sfor\sthe\scode\sadded\son\sthis\sbranch.
+D 2017-11-04T21:06:35.734
 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2
@@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c 2b287b5250e89d548c6bbd1d204d0db41046bb3984b9b4a79fc84e22359f1beb
+F src/wal.c 0b3c6b805fc1cf288a7c63b1ac0f78dcc0ad6a54b5b0d72fb0992b16360e7647
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f
 F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4
@@ -1527,7 +1527,7 @@ F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03
 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6
 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20
 F test/walro.test 906586c3ae7a991d8c840ceed92400aee21a0a3e4155ce7c4220399777311552
-F test/walro2.test 611ceebd190edeca9bf39e5068cbc864f15294371b4acf9ee837db477840af54
+F test/walro2.test 811ab176ab8571c59c2aac889fdacc7cff8d7a1ceb083796540c2886620a599f
 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417
 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f
 F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e
@@ -1668,7 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P f569c3517234881f9425075aab65a32ffd0deb8e793f421a241d8cca881da33f
-R 10fc0a1645ce290a2a283e11360fe887
+P 18b268433d739486eac1b04947bd418655e4bc56e8dc63ffa558aa4552a32e30
+R 5345a5078e9be9411c860b392d7c8c77
 U dan
-Z 79ea7f3203670e23f752126d91be88fd
+Z 9992407e85299b3993fc8148a1281ba1
diff --git a/manifest.uuid b/manifest.uuid
index e8b4f82532..ce4b2816dd 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-18b268433d739486eac1b04947bd418655e4bc56e8dc63ffa558aa4552a32e30
\ No newline at end of file
+a6716fcde38b28b8a03b40f9d16f78a57ec20f60cf391ff553692641cb7f0d3f
\ No newline at end of file
diff --git a/src/wal.c b/src/wal.c
index 17e253a8b6..d8612ec850 100644
--- a/src/wal.c
+++ b/src/wal.c
@@ -2253,6 +2253,10 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){
   }
   aData = &aFrame[WAL_FRAME_HDRSIZE];
 
+  /* Check to see if a complete transaction has been appended to the
+  ** wal file since the heap-memory wal-index was created. If so, the
+  ** heap-memory wal-index is discarded and WAL_RETRY returned to
+  ** the caller.  */
   aSaveCksum[0] = pWal->hdr.aFrameCksum[0];
   aSaveCksum[1] = pWal->hdr.aFrameCksum[1];
   for(iOffset=walFrameOffset(pWal->hdr.mxFrame+1, pWal->hdr.szPage); 
@@ -2276,7 +2280,9 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){
     }
     if( !walDecodeFrame(pWal, &pgno, &nTruncate, aData, aFrame) ) break;
 
-    /* If nTruncate is non-zero, this is a commit record. */
+    /* If nTruncate is non-zero, then a complete transaction has been
+    ** appended to this wal file. Set rc to WAL_RETRY and break out of
+    ** the loop.  */
     if( nTruncate ){
       rc = WAL_RETRY;
       break;
diff --git a/test/walro2.test b/test/walro2.test
index ee4a341e24..9a6b1a1b5c 100644
--- a/test/walro2.test
+++ b/test/walro2.test
@@ -15,7 +15,8 @@
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 source $testdir/lock_common.tcl
-set ::testprefix walro
+source $testdir/wal_common.tcl
+set ::testprefix walro2
 
 # These tests are only going to work on unix.
 #
@@ -120,10 +121,81 @@ do_multiclient_test tn {
     code3 { db3 close }
     sql1 { COMMIT } 
   } {}
-  breakpoint
   do_test 2.3.3 {
     sql1 { SELECT * FROM t1 }
   } {a b c d e f g h i j}
+
+
+  #-----------------------------------------------------------------------
+  # 3.1.*: That a readonly_shm connection can read a database file if both
+  #        the *-wal and *-shm files are zero bytes in size.
+  #
+  # 3.2.*: That it flushes the cache if, between transactions on a db with a
+  #        zero byte *-wal file, some other connection modifies the db, then
+  #        does "PRAGMA wal_checkpoint=truncate" to truncate the wal file
+  #        back to zero bytes in size.
+  #
+  # 3.3.*: That, if between transactions some other process wraps the wal
+  #        file, the readonly_shm client reruns recovery.
+  #
+  catch { code1 { db close } }
+  catch { code2 { db2 close } }
+  catch { code3 { db3 close } }
+  do_test 3.1.0 {
+    list [file exists test.db-wal] [file exists test.db-shm]
+  } {0 0}
+  do_test 3.1.1 {
+    close [open test.db-wal w]
+    close [open test.db-shm w]
+    code1 {
+      sqlite3 db file:test.db?readonly_shm=1
+    }
+    sql1 { SELECT * FROM t1 }
+  } {a b c d e f g h}
+
+  do_test 3.2.0 {
+    list [file size test.db-wal] [file size test.db-shm]
+  } {0 0}
+  do_test 3.2.1 {
+    code2 { sqlite3 db2 test.db }
+    sql2 { INSERT INTO t1 VALUES(1, 2) ; PRAGMA wal_checkpoint=truncate }
+    code2 { db2 close }
+    sql1 { SELECT * FROM t1 }
+  } {a b c d e f g h 1 2}
+  do_test 3.2.2 {
+    list [file size test.db-wal] [file size test.db-shm]
+  } {0 32768}
+
+  do_test 3.3.0 {
+    code2 { sqlite3 db2 test.db }
+    sql2 { 
+      INSERT INTO t1 VALUES(3, 4);
+      INSERT INTO t1 VALUES(5, 6);
+      INSERT INTO t1 VALUES(7, 8);
+      INSERT INTO t1 VALUES(9, 10);
+    }
+    code2 { db2 close }
+    code1 { db close }
+    list [file size test.db-wal] [file size test.db-shm]
+  } [list [wal_file_size 4 1024] 32768]
+  do_test 3.3.1 {
+    code1 { sqlite3 db file:test.db?readonly_shm=1 }
+    sql1 { SELECT * FROM t1 }
+  } {a b c d e f g h 1 2 3 4 5 6 7 8 9 10}
+  do_test 3.3.2 {
+    code2 { sqlite3 db2 test.db }
+    sql2 { 
+      PRAGMA wal_checkpoint; 
+      DELETE FROM t1;
+      INSERT INTO t1 VALUES('i', 'ii');
+    }
+    code2 { db2 close }
+    list [file size test.db-wal] [file size test.db-shm]
+  } [list [wal_file_size 4 1024] 32768]
+  do_test 3.3.3 {
+    sql1 { SELECT * FROM t1 }
+  } {i ii}
+
 }
 
 finish_test

From ab548384525574b953fa7cec2a21b44b10d145e9 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 6 Nov 2017 19:49:34 +0000
Subject: [PATCH 063/488] Add further test cases for the new code on this
 branch. And a couple of fixes.

FossilOrigin-Name: 71af9acb227a91d9ad8798c9d0b12d6967e863d050f5cb1fddb45f25ee1f47db
---
 manifest         |  14 ++---
 manifest.uuid    |   2 +-
 src/wal.c        |  20 +++----
 test/walro2.test | 140 +++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 156 insertions(+), 20 deletions(-)

diff --git a/manifest b/manifest
index 84f5558cad..1cc173595f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sfurther\stests\sfor\sthe\scode\sadded\son\sthis\sbranch.
-D 2017-11-04T21:06:35.734
+C Add\sfurther\stest\scases\sfor\sthe\snew\scode\son\sthis\sbranch.\sAnd\sa\scouple\sof\sfixes.
+D 2017-11-06T19:49:34.916
 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2
@@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c 0b3c6b805fc1cf288a7c63b1ac0f78dcc0ad6a54b5b0d72fb0992b16360e7647
+F src/wal.c 32ee6550804a27c155bdeeddd9bf9bc6ca5331a901c763105bccd0b408049d20
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f
 F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4
@@ -1527,7 +1527,7 @@ F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03
 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6
 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20
 F test/walro.test 906586c3ae7a991d8c840ceed92400aee21a0a3e4155ce7c4220399777311552
-F test/walro2.test 811ab176ab8571c59c2aac889fdacc7cff8d7a1ceb083796540c2886620a599f
+F test/walro2.test 2f0f662f880580d6ecadda9d7cc647d90b1f9e0fb1d487c2a723bcea07eb17dd
 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417
 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f
 F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e
@@ -1668,7 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 18b268433d739486eac1b04947bd418655e4bc56e8dc63ffa558aa4552a32e30
-R 5345a5078e9be9411c860b392d7c8c77
+P a6716fcde38b28b8a03b40f9d16f78a57ec20f60cf391ff553692641cb7f0d3f
+R fb9f39269dd42d0221893523e00b9010
 U dan
-Z 9992407e85299b3993fc8148a1281ba1
+Z ae1d97ffe1b3072414f0d45a94e66fe3
diff --git a/manifest.uuid b/manifest.uuid
index ce4b2816dd..d96dff9523 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a6716fcde38b28b8a03b40f9d16f78a57ec20f60cf391ff553692641cb7f0d3f
\ No newline at end of file
+71af9acb227a91d9ad8798c9d0b12d6967e863d050f5cb1fddb45f25ee1f47db
\ No newline at end of file
diff --git a/src/wal.c b/src/wal.c
index d8612ec850..017e5e865c 100644
--- a/src/wal.c
+++ b/src/wal.c
@@ -2199,7 +2199,8 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){
   ** from running recovery.  */
   rc = walLockShared(pWal, WAL_READ_LOCK(0));
   if( rc!=SQLITE_OK ){
-    return (rc==SQLITE_BUSY ? WAL_RETRY : rc);
+    if( rc==SQLITE_BUSY ) rc = WAL_RETRY;
+    goto begin_unlocked_out;
   }
   pWal->readLock = 0;
 
@@ -2223,7 +2224,10 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){
 
   memcpy(&pWal->hdr, (void*)walIndexHdr(pWal), sizeof(WalIndexHdr));
   rc = sqlite3OsFileSize(pWal->pWalFd, &szWal);
-  if( rc!=SQLITE_OK || (szWal<WAL_HDRSIZE && pWal->hdr.mxFrame==0) ){
+  if( rc!=SQLITE_OK ){
+    goto begin_unlocked_out;
+  }
+  if( szWal<WAL_HDRSIZE ){
     /* If the wal file is too small to contain a wal-header and the
     ** wal-index header has mxFrame==0, then it must be safe to proceed
     ** reading the database file only. However, the page cache cannot
@@ -2231,6 +2235,7 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){
     ** the db, run a checkpoint, truncated the wal file and disconnected
     ** since this client's last read transaction.  */
     *pChanged = 1;
+    rc = (pWal->hdr.mxFrame==0 ? SQLITE_OK : WAL_RETRY);
     goto begin_unlocked_out;
   }
 
@@ -2268,16 +2273,7 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){
 
     /* Read and decode the next log frame. */
     rc = sqlite3OsRead(pWal->pWalFd, aFrame, szFrame, iOffset);
-    if( rc!=SQLITE_OK ){
-      if( rc==SQLITE_IOERR_SHORT_READ ){
-        /* If this branch is taken, some other client has truncated the
-        ** *-wal file since the call to sqlite3OsFileSize() above. This
-        ** indicates that a read-write client has connected to the system.
-        ** So retry opening this read transaction.  */
-        rc = WAL_RETRY;
-      }
-      break;
-    }
+    if( rc!=SQLITE_OK ) break;
     if( !walDecodeFrame(pWal, &pgno, &nTruncate, aData, aFrame) ) break;
 
     /* If nTruncate is non-zero, then a complete transaction has been
diff --git a/test/walro2.test b/test/walro2.test
index 9a6b1a1b5c..1c59b70ee6 100644
--- a/test/walro2.test
+++ b/test/walro2.test
@@ -196,6 +196,146 @@ do_multiclient_test tn {
     sql1 { SELECT * FROM t1 }
   } {i ii}
 
+  #-----------------------------------------------------------------------
+  #
+  #
+  catch { code1 { db close } }
+  catch { code2 { db2 close } }
+  catch { code3 { db3 close } }
+
+  do_test 4.0 {
+    code1 { forcedelete test.db }
+    code1 { sqlite3 db test.db }
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(x);
+      INSERT INTO t1 VALUES('hello');
+      INSERT INTO t1 VALUES('world');
+    }
+
+    forcecopy test.db test.db2
+    forcecopy test.db-wal test.db2-wal
+    forcecopy test.db-shm test.db2-shm
+    
+    code1 { db close }
+  } {}
+
+  do_test 4.1.1 {
+    code2 { sqlite3 db2 file:test.db2?readonly_shm=1 }
+    sql2 { SELECT * FROM t1 }
+  } {hello world}
+
+  do_test 4.1.2 {
+    code3 { sqlite3 db3 test.db2 }
+    sql3 {
+      INSERT INTO t1 VALUES('!');
+      PRAGMA wal_checkpoint = truncate;
+    }
+    code3 { db3 close }
+  } {}
+  do_test 4.1.3 {
+    sql2 { SELECT * FROM t1 }
+  } {hello world !}
+
+  catch { code1 { db close } }
+  catch { code2 { db2 close } }
+  catch { code3 { db3 close } }
+
+  do_test 4.2.1 {
+    code1 { sqlite3 db test.db }
+    sql1 {
+      INSERT INTO t1 VALUES('!');
+      INSERT INTO t1 VALUES('!');
+
+      PRAGMA cache_size = 10;
+      CREATE TABLE t2(x);
+
+      BEGIN;
+        WITH s(i) AS (
+          SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<500
+          )
+        INSERT INTO t2 SELECT randomblob(500) FROM s;
+        SELECT count(*) FROM t2;
+    } 
+  } {500}
+  do_test 4.2.2 {
+    file size test.db-wal
+  } {461152}
+  do_test 4.2.4 {
+    forcecopy test.db test.db2
+    forcecopy test.db-wal test.db2-wal
+    forcecopy test.db-shm test.db2-shm
+
+    code2 { sqlite3 db2 file:test.db2?readonly_shm=1 }
+    sql2 {
+      SELECT * FROM t1;
+      SELECT count(*) FROM t2;
+    }
+  } {hello world ! ! 0}
+
+  #-----------------------------------------------------------------------
+  #
+  #
+  catch { code1 { db close } }
+  catch { code2 { db2 close } }
+  catch { code3 { db3 close } }
+
+  do_test 5.0 {
+    code1 { forcedelete test.db }
+    code1 { sqlite3 db test.db }
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(x);
+      INSERT INTO t1 VALUES('hello');
+      INSERT INTO t1 VALUES('world');
+      INSERT INTO t1 VALUES('!');
+      INSERT INTO t1 VALUES('world');
+      INSERT INTO t1 VALUES('hello');
+    }
+
+    forcecopy test.db test.db2
+    forcecopy test.db-wal test.db2-wal
+    forcecopy test.db-shm test.db2-shm
+    
+    code1 { db close }
+  } {}
+
+  do_test 5.1 {
+    code2 { sqlite3 db2 file:test.db2?readonly_shm=1 }
+    sql2 {
+      SELECT * FROM t1;
+    }
+  } {hello world ! world hello}
+
+  do_test 5.2 {
+    code1 {
+      proc handle_read {op args} {
+        if {$op=="xRead" && [file tail [lindex $args 0]]=="test.db2-wal"} {
+          set ::res2 [sql2 { SELECT * FROM t1 }]
+        }
+        puts "$msg xRead $args"
+        return "SQLITE_OK"
+      }
+      testvfs tvfs -fullshm 1
+
+      sqlite3 db file:test.db2?vfs=tvfs
+      db eval { SELECT * FROM sqlite_master }
+
+      tvfs filter xRead
+      tvfs script handle_read
+    }
+    sql1 {
+      PRAGMA wal_checkpoint = truncate;
+    }
+    code1 { set ::res2 }
+  } {hello world ! world hello}
+
+  do_test 5.3 {
+    code1 { db close }
+    code1 { tvfs delete }
+  } {}
+
+
 }
 
 finish_test

From d24c5b1c9834e0bb0cbbc9f8731ef4a9332c1415 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 7 Nov 2017 09:08:43 +0000
Subject: [PATCH 064/488] Add fault-injection tests for the code on this
 branch.

FossilOrigin-Name: a7d949fb735f60c19e7257a1a7a12568a9c15be9cd980c018f3a0d6bf112c339
---
 manifest             | 11 ++++----
 manifest.uuid        |  2 +-
 test/walrofault.test | 66 ++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 73 insertions(+), 6 deletions(-)
 create mode 100644 test/walrofault.test

diff --git a/manifest b/manifest
index 1cc173595f..dc34f6b3d8 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sfurther\stest\scases\sfor\sthe\snew\scode\son\sthis\sbranch.\sAnd\sa\scouple\sof\sfixes.
-D 2017-11-06T19:49:34.916
+C Add\sfault-injection\stests\sfor\sthe\scode\son\sthis\sbranch.
+D 2017-11-07T09:08:43.757
 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2
@@ -1528,6 +1528,7 @@ F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6
 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20
 F test/walro.test 906586c3ae7a991d8c840ceed92400aee21a0a3e4155ce7c4220399777311552
 F test/walro2.test 2f0f662f880580d6ecadda9d7cc647d90b1f9e0fb1d487c2a723bcea07eb17dd
+F test/walrofault.test befa889648b2f779e2886f8434d8b44c05c49c130048305977da3e97c33dcb8d
 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417
 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f
 F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e
@@ -1668,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a6716fcde38b28b8a03b40f9d16f78a57ec20f60cf391ff553692641cb7f0d3f
-R fb9f39269dd42d0221893523e00b9010
+P 71af9acb227a91d9ad8798c9d0b12d6967e863d050f5cb1fddb45f25ee1f47db
+R 2894b2184d60284fd42b25cb64811e1a
 U dan
-Z ae1d97ffe1b3072414f0d45a94e66fe3
+Z 3f08ac70c434ed263499224338c46e3a
diff --git a/manifest.uuid b/manifest.uuid
index d96dff9523..881ec0ad62 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-71af9acb227a91d9ad8798c9d0b12d6967e863d050f5cb1fddb45f25ee1f47db
\ No newline at end of file
+a7d949fb735f60c19e7257a1a7a12568a9c15be9cd980c018f3a0d6bf112c339
\ No newline at end of file
diff --git a/test/walrofault.test b/test/walrofault.test
new file mode 100644
index 0000000000..b396be218c
--- /dev/null
+++ b/test/walrofault.test
@@ -0,0 +1,66 @@
+# 2011 May 09
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains tests for using WAL databases in read-only mode.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/malloc_common.tcl
+set ::testprefix walro2
+
+# These tests are only going to work on unix.
+#
+if {$::tcl_platform(platform) != "unix"} {
+  finish_test
+  return
+}
+
+# And only if the build is WAL-capable.
+#
+ifcapable !wal {
+  finish_test
+  return
+}
+
+db close
+sqlite3_shutdown
+sqlite3_config_uri 1
+sqlite3 db test.db
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(b);
+  PRAGMA journal_mode = wal;
+  INSERT INTO t1 VALUES('hello');
+  INSERT INTO t1 VALUES('world');
+  INSERT INTO t1 VALUES('!');
+  INSERT INTO t1 VALUES('world');
+  INSERT INTO t1 VALUES('hello');
+  PRAGMA cache_size = 10;
+  BEGIN;
+    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<30 ) 
+    INSERT INTO t1(b) SELECT randomblob(800) FROM s;
+} {wal}
+faultsim_save_and_close
+
+do_faultsim_test 1 -faults oom* -prep {
+  catch { db close }
+  faultsim_restore
+  sqlite3 db file:test.db?readonly_shm=1
+} -body {
+  execsql { SELECT * FROM t1 }
+} -test {
+  faultsim_test_result {0 {hello world ! world hello}}
+}
+
+
+
+finish_test

From f12ba66cf8f6158965b22033c51703c0f6e1fde0 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 7 Nov 2017 15:43:52 +0000
Subject: [PATCH 065/488] On unix, if the *-shm file cannot be opened for
 read/write access, open it read-only and proceed as if the readonly_shm=1 URI
 option were specified.

FossilOrigin-Name: ba718754fa5ab8596cb84b751051de98afa2706fe6c5df39ad6d925d790719ee
---
 manifest        | 16 ++++++++--------
 manifest.uuid   |  2 +-
 src/os_unix.c   | 25 +++++++++++++------------
 test/wal2.test  |  2 +-
 test/walro.test |  6 +++++-
 5 files changed, 28 insertions(+), 23 deletions(-)

diff --git a/manifest b/manifest
index dc34f6b3d8..d0bf050785 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sfault-injection\stests\sfor\sthe\scode\son\sthis\sbranch.
-D 2017-11-07T09:08:43.757
+C On\sunix,\sif\sthe\s*-shm\sfile\scannot\sbe\sopened\sfor\sread/write\saccess,\sopen\sit\nread-only\sand\sproceed\sas\sif\sthe\sreadonly_shm=1\sURI\soption\swere\sspecified.
+D 2017-11-07T15:43:52.117
 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2
@@ -447,7 +447,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104
 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
-F src/os_unix.c e376adf6014df7d1a73faaaa6c87e6eb9b299b157a58cccff02fad8abc943fe7
+F src/os_unix.c 7de9280173d58b557c602d2a47c2893e536ef9547fa43df92bce0bc2c0d47b0e
 F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903
@@ -1501,7 +1501,7 @@ F test/vtab_alter.test 736e66fb5ec7b4fee58229aa3ada2f27ec58bc58c00edae4836890c37
 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8
 F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad
 F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477
-F test/wal2.test 4c44bbe447959638e5163631a1fe95c9dbc01a06eff6eb34449be06b6e0ed64c
+F test/wal2.test 2d81ffe2a02d9e5c7447b266f7153716cfcba7aecda5ed832db4544617399e29
 F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2
 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c
 F test/wal5.test 9c11da7aeccd83a46d79a556ad11a18d3cb15aa9
@@ -1526,7 +1526,7 @@ F test/walnoshm.test 84ca10c544632a756467336b7c3b864d493ee496
 F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03
 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6
 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20
-F test/walro.test 906586c3ae7a991d8c840ceed92400aee21a0a3e4155ce7c4220399777311552
+F test/walro.test cb438d05ba0d191f10b688e39c4f0cd5b71569a1d1f4440e5bdf3c6880e08c20
 F test/walro2.test 2f0f662f880580d6ecadda9d7cc647d90b1f9e0fb1d487c2a723bcea07eb17dd
 F test/walrofault.test befa889648b2f779e2886f8434d8b44c05c49c130048305977da3e97c33dcb8d
 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417
@@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 71af9acb227a91d9ad8798c9d0b12d6967e863d050f5cb1fddb45f25ee1f47db
-R 2894b2184d60284fd42b25cb64811e1a
+P a7d949fb735f60c19e7257a1a7a12568a9c15be9cd980c018f3a0d6bf112c339
+R 7a4e57959f170e8ae96924256d8168d4
 U dan
-Z 3f08ac70c434ed263499224338c46e3a
+Z fa20e28121f5e7127ed54b089c80ff82
diff --git a/manifest.uuid b/manifest.uuid
index 881ec0ad62..6168e89f1d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a7d949fb735f60c19e7257a1a7a12568a9c15be9cd980c018f3a0d6bf112c339
\ No newline at end of file
+ba718754fa5ab8596cb84b751051de98afa2706fe6c5df39ad6d925d790719ee
\ No newline at end of file
diff --git a/src/os_unix.c b/src/os_unix.c
index e90e335cb7..c656215c4a 100644
--- a/src/os_unix.c
+++ b/src/os_unix.c
@@ -4369,7 +4369,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){
   struct unixShmNode *pShmNode;   /* The underlying mmapped file */
   int rc = SQLITE_OK;             /* Result code */
   unixInodeInfo *pInode;          /* The inode of fd */
-  char *zShmFilename;             /* Name of the file used for SHM */
+  char *zShm;             /* Name of the file used for SHM */
   int nShmFilename;               /* Size of the SHM filename in bytes */
 
   /* Allocate space for the new unixShm object. */
@@ -4410,14 +4410,14 @@ static int unixOpenSharedMemory(unixFile *pDbFd){
       goto shm_open_err;
     }
     memset(pShmNode, 0, sizeof(*pShmNode)+nShmFilename);
-    zShmFilename = pShmNode->zFilename = (char*)&pShmNode[1];
+    zShm = pShmNode->zFilename = (char*)&pShmNode[1];
 #ifdef SQLITE_SHM_DIRECTORY
-    sqlite3_snprintf(nShmFilename, zShmFilename, 
+    sqlite3_snprintf(nShmFilename, zShm, 
                      SQLITE_SHM_DIRECTORY "/sqlite-shm-%x-%x",
                      (u32)sStat.st_ino, (u32)sStat.st_dev);
 #else
-    sqlite3_snprintf(nShmFilename, zShmFilename, "%s-shm", zBasePath);
-    sqlite3FileSuffix3(pDbFd->zPath, zShmFilename);
+    sqlite3_snprintf(nShmFilename, zShm, "%s-shm", zBasePath);
+    sqlite3FileSuffix3(pDbFd->zPath, zShm);
 #endif
     pShmNode->h = -1;
     pDbFd->pInode->pShmNode = pShmNode;
@@ -4431,15 +4431,16 @@ static int unixOpenSharedMemory(unixFile *pDbFd){
     }
 
     if( pInode->bProcessLock==0 ){
-      int openFlags = O_RDWR | O_CREAT;
-      if( sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){
-        openFlags = O_RDONLY;
-        pShmNode->isReadonly = 1;
+      if( 0==sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){
+        pShmNode->h = robust_open(zShm, O_RDWR|O_CREAT, (sStat.st_mode&0777));
       }
-      pShmNode->h = robust_open(zShmFilename, openFlags, (sStat.st_mode&0777));
       if( pShmNode->h<0 ){
-        rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShmFilename);
-        goto shm_open_err;
+        pShmNode->h = robust_open(zShm, O_RDONLY, (sStat.st_mode&0777));
+        if( pShmNode->h<0 ){
+          rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShm);
+          goto shm_open_err;
+        }
+        pShmNode->isReadonly = 1;
       }
 
       /* If this process is running as root, make sure that the SHM file
diff --git a/test/wal2.test b/test/wal2.test
index ddff71aed4..a9cafec66d 100644
--- a/test/wal2.test
+++ b/test/wal2.test
@@ -1130,7 +1130,7 @@ if {$::tcl_platform(platform) == "unix"} {
   foreach {tn db_perm wal_perm shm_perm can_open can_read can_write} {
     2   00644   00644   00644   1   1   1
     3   00644   00400   00644   1   1   0
-    4   00644   00644   00400   1   0   0
+    4   00644   00644   00400   1   1   0
     5   00400   00644   00644   1   1   0
 
     7   00644   00000   00644   1   0   0
diff --git a/test/walro.test b/test/walro.test
index 150344e151..cae52db6d3 100644
--- a/test/walro.test
+++ b/test/walro.test
@@ -139,11 +139,15 @@ do_multiclient_test tn {
   # Now check that if the readonly_shm option is not supplied, or if it
   # is set to zero, it is not possible to connect to the database without
   # read-write access to the shm.
+  # 
+  # UPDATE: os_unix.c now opens the *-shm file in readonly mode 
+  # automatically.
+  #
   do_test 1.3.1 {
     code1 { db close }
     code1 { sqlite3 db test.db }
     csql1 { SELECT * FROM t1 }
-  } {1 {unable to open database file}}
+  } {0 {a b c d e f g h i j k l}}
 
   # Also test that if the -shm file can be opened for read/write access,
   # it is not if readonly_shm=1 is present in the URI.

From 08ecefc5b1bbcbcb3d04989daf454e8fa3007f82 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 7 Nov 2017 21:15:07 +0000
Subject: [PATCH 066/488] Handle the race condition that may occur if another
 process connects and then checkpoints and truncates the wal file while a
 readonly-shm client is building its heap-memory wal-index.

FossilOrigin-Name: 5a6703fc3f2174b3e9a624c7272ae013b73c42d6c97ffa62b58553efdb54e3bc
---
 manifest         | 14 ++++++-------
 manifest.uuid    |  2 +-
 src/wal.c        | 15 ++++++++------
 test/walro2.test | 53 ++++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 70 insertions(+), 14 deletions(-)

diff --git a/manifest b/manifest
index d0bf050785..bb8c19984a 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C On\sunix,\sif\sthe\s*-shm\sfile\scannot\sbe\sopened\sfor\sread/write\saccess,\sopen\sit\nread-only\sand\sproceed\sas\sif\sthe\sreadonly_shm=1\sURI\soption\swere\sspecified.
-D 2017-11-07T15:43:52.117
+C Handle\sthe\srace\scondition\sthat\smay\soccur\sif\sanother\sprocess\sconnects\sand\sthen\ncheckpoints\sand\struncates\sthe\swal\sfile\swhile\sa\sreadonly-shm\sclient\sis\sbuilding\nits\sheap-memory\swal-index.
+D 2017-11-07T21:15:07.949
 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2
@@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c 32ee6550804a27c155bdeeddd9bf9bc6ca5331a901c763105bccd0b408049d20
+F src/wal.c b3cd00a165e63ffa91f803fd678f5fc9b7e82745db79bb6c8c220678d5c7ec2f
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f
 F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4
@@ -1527,7 +1527,7 @@ F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03
 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6
 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20
 F test/walro.test cb438d05ba0d191f10b688e39c4f0cd5b71569a1d1f4440e5bdf3c6880e08c20
-F test/walro2.test 2f0f662f880580d6ecadda9d7cc647d90b1f9e0fb1d487c2a723bcea07eb17dd
+F test/walro2.test 2e499d89fa825c9d23b53ed4da8e4dcc7017ea16212d6a4f3aec56d1861eaf8e
 F test/walrofault.test befa889648b2f779e2886f8434d8b44c05c49c130048305977da3e97c33dcb8d
 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417
 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f
@@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a7d949fb735f60c19e7257a1a7a12568a9c15be9cd980c018f3a0d6bf112c339
-R 7a4e57959f170e8ae96924256d8168d4
+P ba718754fa5ab8596cb84b751051de98afa2706fe6c5df39ad6d925d790719ee
+R e391f2931828dcd8c6e04450ea4ce5f1
 U dan
-Z fa20e28121f5e7127ed54b089c80ff82
+Z a1807096877d0d55e25dd1ebf350b50a
diff --git a/manifest.uuid b/manifest.uuid
index 6168e89f1d..011649cacf 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ba718754fa5ab8596cb84b751051de98afa2706fe6c5df39ad6d925d790719ee
\ No newline at end of file
+5a6703fc3f2174b3e9a624c7272ae013b73c42d6c97ffa62b58553efdb54e3bc
\ No newline at end of file
diff --git a/src/wal.c b/src/wal.c
index 017e5e865c..1a8bd74b02 100644
--- a/src/wal.c
+++ b/src/wal.c
@@ -2071,6 +2071,12 @@ static int walIndexTryHdr(Wal *pWal, int *pChanged){
   return 0;
 }
 
+/*
+** This is the value that walTryBeginRead returns when it needs to
+** be retried.
+*/
+#define WAL_RETRY  (-1)
+
 /*
 ** Read the wal-index header from the wal-index and into pWal->hdr.
 ** If the wal-header appears to be corrupt, try to reconstruct the
@@ -2149,6 +2155,9 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){
   if( pWal->bUnlocked ){
     if( rc!=SQLITE_OK ){
       walIndexClose(pWal, 0);
+      pWal->bUnlocked = 0;
+      assert( pWal->nWiData>0 && pWal->apWiData[0]==0 );
+      if( rc==SQLITE_IOERR_SHORT_READ ) rc = WAL_RETRY;
     }
     pWal->exclusiveMode = WAL_NORMAL_MODE;
   }
@@ -2156,12 +2165,6 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){
   return rc;
 }
 
-/*
-** This is the value that walTryBeginRead returns when it needs to
-** be retried.
-*/
-#define WAL_RETRY  (-1)
-
 /*
 ** Open an "unlocked" transaction. An unlocked transaction is a read 
 ** transaction used by a read-only client in cases where the *-shm
diff --git a/test/walro2.test b/test/walro2.test
index 1c59b70ee6..5a8ce023eb 100644
--- a/test/walro2.test
+++ b/test/walro2.test
@@ -335,7 +335,60 @@ do_multiclient_test tn {
     code1 { tvfs delete }
   } {}
 
+  #-----------------------------------------------------------------------
+  #
+  #
+  catch { code1 { db close } }
+  catch { code2 { db2 close } }
+  catch { code3 { db3 close } }
 
+  do_test 6.1 {
+    code1 { forcedelete test.db }
+    code1 { sqlite3 db test.db }
+    sql1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(x);
+      INSERT INTO t1 VALUES('hello');
+      INSERT INTO t1 VALUES('world');
+      INSERT INTO t1 VALUES('!');
+      INSERT INTO t1 VALUES('world');
+      INSERT INTO t1 VALUES('hello');
+    }
+
+    forcecopy test.db test.db2
+    forcecopy test.db-wal test.db2-wal
+    forcecopy test.db-shm test.db2-shm
+    
+    code1 { db close }
+  } {}
+
+  do_test 6.2 {
+    code1 {
+      set ::nRem 5
+      proc handle_read {op args} {
+        if {$op=="xRead" && [file tail [lindex $args 0]]=="test.db2-wal"} {
+          incr ::nRem -1
+          if {$::nRem==0} {
+            code2 { sqlite3 db2 test.db2 }
+            sql2  { PRAGMA wal_checkpoint = truncate }
+          }
+        }
+        return "SQLITE_OK"
+      }
+      testvfs tvfs -fullshm 1
+
+      tvfs filter xRead
+      tvfs script handle_read
+
+      sqlite3 db file:test.db2?readonly_shm=1&vfs=tvfs
+      db eval { SELECT * FROM t1 }
+    }
+  } {hello world ! world hello}
+
+  do_test 6.3 {
+    code1 { db close }
+    code1 { tvfs delete }
+  } {}
 }
 
 finish_test

From 6c9d8f640b5d2397e0f1338ae160f11220146d44 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 7 Nov 2017 21:25:15 +0000
Subject: [PATCH 067/488] Update an assert in wal.c.

FossilOrigin-Name: 94527b897bac66d100ca92161f18b6f0e0768dd77ebcb06e3fb106f0e0e380ee
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/wal.c     |  4 ++--
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index bb8c19984a..0f9d93ff2b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Handle\sthe\srace\scondition\sthat\smay\soccur\sif\sanother\sprocess\sconnects\sand\sthen\ncheckpoints\sand\struncates\sthe\swal\sfile\swhile\sa\sreadonly-shm\sclient\sis\sbuilding\nits\sheap-memory\swal-index.
-D 2017-11-07T21:15:07.949
+C Update\san\sassert\sin\swal.c.
+D 2017-11-07T21:25:15.188
 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2
@@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c b3cd00a165e63ffa91f803fd678f5fc9b7e82745db79bb6c8c220678d5c7ec2f
+F src/wal.c 6227c952d86fb9cdc380ffda2413ca4f32a44309c61d10c55dec7dc7bc135ba2
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f
 F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4
@@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ba718754fa5ab8596cb84b751051de98afa2706fe6c5df39ad6d925d790719ee
-R e391f2931828dcd8c6e04450ea4ce5f1
+P 5a6703fc3f2174b3e9a624c7272ae013b73c42d6c97ffa62b58553efdb54e3bc
+R 598ffeddb9515e7bdc3f854533f484ab
 U dan
-Z a1807096877d0d55e25dd1ebf350b50a
+Z 7844b4272e27592e14850347bc28da72
diff --git a/manifest.uuid b/manifest.uuid
index 011649cacf..4a3a422204 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5a6703fc3f2174b3e9a624c7272ae013b73c42d6c97ffa62b58553efdb54e3bc
\ No newline at end of file
+94527b897bac66d100ca92161f18b6f0e0768dd77ebcb06e3fb106f0e0e380ee
\ No newline at end of file
diff --git a/src/wal.c b/src/wal.c
index 1a8bd74b02..4f8d5eadd9 100644
--- a/src/wal.c
+++ b/src/wal.c
@@ -2844,8 +2844,8 @@ int sqlite3WalFindFrame(
   {
     u32 iRead2 = 0;
     u32 iTest;
-    assert( pWal->minFrame>0 );
-    for(iTest=iLast; iTest>=pWal->minFrame; iTest--){
+    assert( pWal->bUnlocked || pWal->minFrame>0 );
+    for(iTest=iLast; iTest>=pWal->minFrame && iTest>0; iTest--){
       if( walFramePgno(pWal, iTest)==pgno ){
         iRead2 = iTest;
         break;

From 7e45e3a5d76678356b88ceef773cdd6824008651 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 8 Nov 2017 17:32:12 +0000
Subject: [PATCH 068/488] Change the name of SQLITE_READONLY_CANTLOCK to
 SQLITE_READONLY_CANTINIT.

FossilOrigin-Name: 6d7f94faa7e6de62f82bc6cac019508a9c1ffd6fa1d14f52fa93e9c06afdd32f
---
 manifest        | 20 ++++++++++----------
 manifest.uuid   |  2 +-
 src/main.c      |  2 +-
 src/os_unix.c   |  6 +++---
 src/sqlite.h.in |  2 +-
 src/wal.c       |  6 +++---
 6 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/manifest b/manifest
index 0f9d93ff2b..7b0825f535 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\san\sassert\sin\swal.c.
-D 2017-11-07T21:25:15.188
+C Change\sthe\sname\sof\sSQLITE_READONLY_CANTLOCK\sto\sSQLITE_READONLY_CANTINIT.
+D 2017-11-08T17:32:12.655
 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2
@@ -428,7 +428,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c c7f333547211b8efbac8a72f71adad736b91e655d7bcdfacc737351ecf3c8df2
 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e
 F src/loadext.c 20865b183bb8a3723d59cf1efffc3c50217eb452c1021d077b908c94da26b0b2
-F src/main.c 54637b9e7f91de6d281e577cd1a997762a4613f51a0509790027ca9865185d7c
+F src/main.c c1965ee8159cee5fba3f590cc4767515a690504455a03e4817b1accfe0ba95a5
 F src/malloc.c a02c9e69bc76bee0f639416b947a946412890b606301454727feadcb313536d6
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -447,7 +447,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104
 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
-F src/os_unix.c 7de9280173d58b557c602d2a47c2893e536ef9547fa43df92bce0bc2c0d47b0e
+F src/os_unix.c e87cef0bb894b94d96ee3af210be669549d111c580817d14818101b992640767
 F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903
@@ -465,7 +465,7 @@ F src/resolve.c 5a461643f294ec510ca615b67256fc3861e4c8eff5f29e5940491e70553b1955
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 660ef7977841fb462f24c8561e4212615bb6e5c9835fd3556257ce8316c50fee
 F src/shell.c.in 08cbffc31900359fea85896342a46147e9772c370d8a5079b7be26e3a1f50e8a
-F src/sqlite.h.in ba9029e71a605bc5f236cd693abeb7920285785f2e1a92313ecf8fb1c9f52a86
+F src/sqlite.h.in 49b42d0376e7ed94fe71a7338ea19b20185897abaf6b87fd79ec7126c9541c24
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
 F src/sqliteInt.h f5377febf86654c975e1d4e4353a5ad2fbaa5bc86b584ba3761ed33e24ce2c0e
@@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c 6227c952d86fb9cdc380ffda2413ca4f32a44309c61d10c55dec7dc7bc135ba2
+F src/wal.c b1fef64a242ab30b5fdeeaba49150b420608681673b08833acd17024a179cb1e
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f
 F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4
@@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5a6703fc3f2174b3e9a624c7272ae013b73c42d6c97ffa62b58553efdb54e3bc
-R 598ffeddb9515e7bdc3f854533f484ab
-U dan
-Z 7844b4272e27592e14850347bc28da72
+P 94527b897bac66d100ca92161f18b6f0e0768dd77ebcb06e3fb106f0e0e380ee
+R a029a6995bf9493f5c27e7086cc8ad9a
+U drh
+Z d606e5b93c29b98849932178fdbf9f8a
diff --git a/manifest.uuid b/manifest.uuid
index 4a3a422204..3e854635d3 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-94527b897bac66d100ca92161f18b6f0e0768dd77ebcb06e3fb106f0e0e380ee
\ No newline at end of file
+6d7f94faa7e6de62f82bc6cac019508a9c1ffd6fa1d14f52fa93e9c06afdd32f
\ No newline at end of file
diff --git a/src/main.c b/src/main.c
index 49613f6c74..d090845ae3 100644
--- a/src/main.c
+++ b/src/main.c
@@ -1314,7 +1314,7 @@ const char *sqlite3ErrName(int rc){
       case SQLITE_NOMEM:              zName = "SQLITE_NOMEM";             break;
       case SQLITE_READONLY:           zName = "SQLITE_READONLY";          break;
       case SQLITE_READONLY_RECOVERY:  zName = "SQLITE_READONLY_RECOVERY"; break;
-      case SQLITE_READONLY_CANTLOCK:  zName = "SQLITE_READONLY_CANTLOCK"; break;
+      case SQLITE_READONLY_CANTINIT:  zName = "SQLITE_READONLY_CANTINIT"; break;
       case SQLITE_READONLY_ROLLBACK:  zName = "SQLITE_READONLY_ROLLBACK"; break;
       case SQLITE_READONLY_DBMOVED:   zName = "SQLITE_READONLY_DBMOVED";  break;
       case SQLITE_INTERRUPT:          zName = "SQLITE_INTERRUPT";         break;
diff --git a/src/os_unix.c b/src/os_unix.c
index c656215c4a..c5d9aca2c6 100644
--- a/src/os_unix.c
+++ b/src/os_unix.c
@@ -4278,7 +4278,7 @@ static void unixShmPurge(unixFile *pFd){
 **
 ** If the DMS cannot be locked because this is a readonly_shm=1 
 ** connection and no other process already holds a lock, return
-** SQLITE_READONLY_CANTLOCK and set pShmNode->isUnlocked=1.
+** SQLITE_READONLY_CANTINIT and set pShmNode->isUnlocked=1.
 */
 static int unixLockSharedMemory(unixFile *pDbFd, unixShmNode *pShmNode){
   struct flock lock;
@@ -4311,7 +4311,7 @@ static int unixLockSharedMemory(unixFile *pDbFd, unixShmNode *pShmNode){
   }else if( lock.l_type==F_UNLCK ){
     if( pShmNode->isReadonly ){
       pShmNode->isUnlocked = 1;
-      rc = SQLITE_READONLY_CANTLOCK;
+      rc = SQLITE_READONLY_CANTINIT;
     }else{
       rc = unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1);
       if( rc==SQLITE_OK && robust_ftruncate(pShmNode->h, 0) ){
@@ -4450,7 +4450,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){
       robustFchown(pShmNode->h, sStat.st_uid, sStat.st_gid);
 
       rc = unixLockSharedMemory(pDbFd, pShmNode);
-      if( rc!=SQLITE_OK && rc!=SQLITE_READONLY_CANTLOCK ) goto shm_open_err;
+      if( rc!=SQLITE_OK && rc!=SQLITE_READONLY_CANTINIT ) goto shm_open_err;
     }
   }
 
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 0c0ed25b30..1d083cf06f 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -511,7 +511,7 @@ int sqlite3_exec(
 #define SQLITE_CANTOPEN_DIRTYWAL       (SQLITE_CANTOPEN | (5<<8))
 #define SQLITE_CORRUPT_VTAB            (SQLITE_CORRUPT | (1<<8))
 #define SQLITE_READONLY_RECOVERY       (SQLITE_READONLY | (1<<8))
-#define SQLITE_READONLY_CANTLOCK       (SQLITE_READONLY | (2<<8))
+#define SQLITE_READONLY_CANTINIT       (SQLITE_READONLY | (2<<8))
 #define SQLITE_READONLY_ROLLBACK       (SQLITE_READONLY | (3<<8))
 #define SQLITE_READONLY_DBMOVED        (SQLITE_READONLY | (4<<8))
 #define SQLITE_ABORT_ROLLBACK          (SQLITE_ABORT | (2<<8))
diff --git a/src/wal.c b/src/wal.c
index 4f8d5eadd9..11cdeb8440 100644
--- a/src/wal.c
+++ b/src/wal.c
@@ -2099,7 +2099,7 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){
   */
   assert( pChanged );
   rc = walIndexPage(pWal, 0, &page0);
-  if( rc==SQLITE_READONLY_CANTLOCK ){
+  if( rc==SQLITE_READONLY_CANTINIT ){
     assert( page0==0 && pWal->writeLock==0 );
     pWal->bUnlocked = 1;
     pWal->exclusiveMode = WAL_HEAPMEMORY_MODE;
@@ -2219,7 +2219,7 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){
   ** the unlocked transaction, as the other client may overwrite wal 
   ** frames that this client is still using.  */
   rc = sqlite3OsShmMap(pWal->pDbFd, 0, WALINDEX_PGSZ, 0, &pDummy);
-  if( rc!=SQLITE_READONLY_CANTLOCK ){
+  if( rc!=SQLITE_READONLY_CANTINIT ){
     assert( rc!=SQLITE_OK );
     rc = (rc==SQLITE_READONLY ? WAL_RETRY : rc);
     goto begin_unlocked_out;
@@ -2509,7 +2509,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){
   }
   if( mxI==0 ){
     assert( rc==SQLITE_BUSY || (pWal->readOnly & WAL_SHM_RDONLY)!=0 );
-    return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTLOCK;
+    return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTINIT;
   }
 
   rc = walLockShared(pWal, WAL_READ_LOCK(mxI));

From 7316960cec0ea2b411946964f4c02538ff7e3e1c Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 8 Nov 2017 17:51:10 +0000
Subject: [PATCH 069/488] Turns out that SQLITE_READONLY_CANTLOCK is an
 historical name that must be preserved.  So make a new
 SQLITE_READLOCK_CANTINIT name instead.

FossilOrigin-Name: 04974a8b5c0e6748216226006ca9125529c8bb2a7a9df4641217eb1413426a14
---
 manifest        | 12 ++++++------
 manifest.uuid   |  2 +-
 src/sqlite.h.in |  3 ++-
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 7b0825f535..3f6afa398d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Change\sthe\sname\sof\sSQLITE_READONLY_CANTLOCK\sto\sSQLITE_READONLY_CANTINIT.
-D 2017-11-08T17:32:12.655
+C Turns\sout\sthat\sSQLITE_READONLY_CANTLOCK\sis\san\shistorical\sname\sthat\smust\nbe\spreserved.\s\sSo\smake\sa\snew\sSQLITE_READLOCK_CANTINIT\sname\sinstead.
+D 2017-11-08T17:51:10.682
 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2
@@ -465,7 +465,7 @@ F src/resolve.c 5a461643f294ec510ca615b67256fc3861e4c8eff5f29e5940491e70553b1955
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 660ef7977841fb462f24c8561e4212615bb6e5c9835fd3556257ce8316c50fee
 F src/shell.c.in 08cbffc31900359fea85896342a46147e9772c370d8a5079b7be26e3a1f50e8a
-F src/sqlite.h.in 49b42d0376e7ed94fe71a7338ea19b20185897abaf6b87fd79ec7126c9541c24
+F src/sqlite.h.in df0f3e18bd7210a562dfa4dc7c5b8f57058dfd0f51d771feb5fcfbe3d6ccae5b
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
 F src/sqliteInt.h f5377febf86654c975e1d4e4353a5ad2fbaa5bc86b584ba3761ed33e24ce2c0e
@@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 94527b897bac66d100ca92161f18b6f0e0768dd77ebcb06e3fb106f0e0e380ee
-R a029a6995bf9493f5c27e7086cc8ad9a
+P 6d7f94faa7e6de62f82bc6cac019508a9c1ffd6fa1d14f52fa93e9c06afdd32f
+R 1af2a9a9afceb6f159be2ed742a0108b
 U drh
-Z d606e5b93c29b98849932178fdbf9f8a
+Z 83682bfec24bdede404f3ec79bae717f
diff --git a/manifest.uuid b/manifest.uuid
index 3e854635d3..70823ee687 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6d7f94faa7e6de62f82bc6cac019508a9c1ffd6fa1d14f52fa93e9c06afdd32f
\ No newline at end of file
+04974a8b5c0e6748216226006ca9125529c8bb2a7a9df4641217eb1413426a14
\ No newline at end of file
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 1d083cf06f..db6e0cbae3 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -511,9 +511,10 @@ int sqlite3_exec(
 #define SQLITE_CANTOPEN_DIRTYWAL       (SQLITE_CANTOPEN | (5<<8))
 #define SQLITE_CORRUPT_VTAB            (SQLITE_CORRUPT | (1<<8))
 #define SQLITE_READONLY_RECOVERY       (SQLITE_READONLY | (1<<8))
-#define SQLITE_READONLY_CANTINIT       (SQLITE_READONLY | (2<<8))
+#define SQLITE_READONLY_CANTLOCK       (SQLITE_READONLY | (2<<8))
 #define SQLITE_READONLY_ROLLBACK       (SQLITE_READONLY | (3<<8))
 #define SQLITE_READONLY_DBMOVED        (SQLITE_READONLY | (4<<8))
+#define SQLITE_READONLY_CANTINIT       (SQLITE_READONLY | (5<<8))
 #define SQLITE_ABORT_ROLLBACK          (SQLITE_ABORT | (2<<8))
 #define SQLITE_CONSTRAINT_CHECK        (SQLITE_CONSTRAINT | (1<<8))
 #define SQLITE_CONSTRAINT_COMMITHOOK   (SQLITE_CONSTRAINT | (2<<8))

From 9214c1efe831ca60df04f0a06bb6b1cfd5ca6d38 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 8 Nov 2017 19:26:27 +0000
Subject: [PATCH 070/488] Extra comments on the sqlite3OsShmMap() call in
 walBeginUnlocked().  No changes to code.

FossilOrigin-Name: 033ee92bf4d5dc57f5cb8fd02d1154ae06f2d3261d214e7191a82c70c8ffebf7
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/wal.c     | 10 +++++++++-
 3 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 3f6afa398d..a22cee602e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Turns\sout\sthat\sSQLITE_READONLY_CANTLOCK\sis\san\shistorical\sname\sthat\smust\nbe\spreserved.\s\sSo\smake\sa\snew\sSQLITE_READLOCK_CANTINIT\sname\sinstead.
-D 2017-11-08T17:51:10.682
+C Extra\scomments\son\sthe\ssqlite3OsShmMap()\scall\sin\swalBeginUnlocked().\s\sNo\nchanges\sto\scode.
+D 2017-11-08T19:26:27.278
 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2
@@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c b1fef64a242ab30b5fdeeaba49150b420608681673b08833acd17024a179cb1e
+F src/wal.c 6903d391b6c224e45910795d66f97151428662c5f459d3b69997209dbcadc6f5
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f
 F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4
@@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6d7f94faa7e6de62f82bc6cac019508a9c1ffd6fa1d14f52fa93e9c06afdd32f
-R 1af2a9a9afceb6f159be2ed742a0108b
+P 04974a8b5c0e6748216226006ca9125529c8bb2a7a9df4641217eb1413426a14
+R eba2a72214a6da7d512bda6426cfe5de
 U drh
-Z 83682bfec24bdede404f3ec79bae717f
+Z 493ea33d970c37f0e10aaccf35cd6d47
diff --git a/manifest.uuid b/manifest.uuid
index 70823ee687..a866afde61 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-04974a8b5c0e6748216226006ca9125529c8bb2a7a9df4641217eb1413426a14
\ No newline at end of file
+033ee92bf4d5dc57f5cb8fd02d1154ae06f2d3261d214e7191a82c70c8ffebf7
\ No newline at end of file
diff --git a/src/wal.c b/src/wal.c
index 11cdeb8440..afd0e3f7b6 100644
--- a/src/wal.c
+++ b/src/wal.c
@@ -2210,6 +2210,14 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){
   /* Try to map the *-shm file again. If it succeeds this time, then 
   ** a non-readonly_shm connection has already connected to the database.
   ** In this case, start over with opening the transaction.
+  **
+  ** The *-shm file was opened read-only, so sqlite3OsShmMap() can never
+  ** return SQLITE_OK here, as that would imply that it had established
+  ** a read/write mapping.  A return of SQLITE_READONLY means success - that
+  ** a mapping has been established to a shared-memory segment that is actively
+  ** maintained by a writer.  SQLITE_READONLY_CANTINIT means that all
+  ** all connections to the -shm file are read-only and hence the content
+  ** of the -shm file might be out-of-date.
   ** 
   ** The WAL_READ_LOCK(0) lock held by this client prevents a checkpoint
   ** from taking place. But it does not prevent the wal from being wrapped
@@ -2219,8 +2227,8 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){
   ** the unlocked transaction, as the other client may overwrite wal 
   ** frames that this client is still using.  */
   rc = sqlite3OsShmMap(pWal->pDbFd, 0, WALINDEX_PGSZ, 0, &pDummy);
+  assert( rc!=SQLITE_OK ); /* SQLITE_OK not possible for read-only connection */
   if( rc!=SQLITE_READONLY_CANTINIT ){
-    assert( rc!=SQLITE_OK );
     rc = (rc==SQLITE_READONLY ? WAL_RETRY : rc);
     goto begin_unlocked_out;
   }

From 4ff8431fd161d4d55fed8ba4b88a1c37a4357b76 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Thu, 9 Nov 2017 16:30:55 +0000
Subject: [PATCH 071/488] Initial work on porting the changes on this branch to
 Win32.

FossilOrigin-Name: 3738bfd0c0eadb10eea58954af5052cb6ce164059f3aacfe65d7da6a400c63c7
---
 manifest             |  18 ++--
 manifest.uuid        |   2 +-
 src/os_win.c         | 201 ++++++++++++++++++++++++++++++++++++-------
 test/walro2.test     |   7 --
 test/walrofault.test |   7 --
 5 files changed, 179 insertions(+), 56 deletions(-)

diff --git a/manifest b/manifest
index a22cee602e..dbc8286837 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Extra\scomments\son\sthe\ssqlite3OsShmMap()\scall\sin\swalBeginUnlocked().\s\sNo\nchanges\sto\scode.
-D 2017-11-08T19:26:27.278
+C Initial\swork\son\sporting\sthe\schanges\son\sthis\sbranch\sto\sWin32.
+D 2017-11-09T16:30:55.245
 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2
@@ -448,7 +448,7 @@ F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
 F src/os_unix.c e87cef0bb894b94d96ee3af210be669549d111c580817d14818101b992640767
-F src/os_win.c 6892c3ff23b7886577e47f13d827ca220c0831bae3ce00eea8c258352692f8c6
+F src/os_win.c 47687775641c97743c228f99813fbcffe7d53602da5cdcf0fe52f6810341a46c
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
@@ -1527,8 +1527,8 @@ F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03
 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6
 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20
 F test/walro.test cb438d05ba0d191f10b688e39c4f0cd5b71569a1d1f4440e5bdf3c6880e08c20
-F test/walro2.test 2e499d89fa825c9d23b53ed4da8e4dcc7017ea16212d6a4f3aec56d1861eaf8e
-F test/walrofault.test befa889648b2f779e2886f8434d8b44c05c49c130048305977da3e97c33dcb8d
+F test/walro2.test 2c01a3c38ca731df4690cc901e3c0bee21bf9d231fa3d47d81162be10462c40b
+F test/walrofault.test f1c9c361a73faab98ce1bb4588333e42f7dd330f2b4987e962aa5fe68e7982de
 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417
 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f
 F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e
@@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 04974a8b5c0e6748216226006ca9125529c8bb2a7a9df4641217eb1413426a14
-R eba2a72214a6da7d512bda6426cfe5de
-U drh
-Z 493ea33d970c37f0e10aaccf35cd6d47
+P 033ee92bf4d5dc57f5cb8fd02d1154ae06f2d3261d214e7191a82c70c8ffebf7
+R a3370fa9345068426fa8edd8924927fc
+U mistachkin
+Z fbbc6e55aa870fcd83aef8acf0e01375
diff --git a/manifest.uuid b/manifest.uuid
index a866afde61..f5c23f13f2 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-033ee92bf4d5dc57f5cb8fd02d1154ae06f2d3261d214e7191a82c70c8ffebf7
\ No newline at end of file
+3738bfd0c0eadb10eea58954af5052cb6ce164059f3aacfe65d7da6a400c63c7
\ No newline at end of file
diff --git a/src/os_win.c b/src/os_win.c
index 245a86045b..9bfcfe1c81 100644
--- a/src/os_win.c
+++ b/src/os_win.c
@@ -2098,6 +2098,17 @@ static int winLogErrorAtLine(
 static int winIoerrRetry = SQLITE_WIN32_IOERR_RETRY;
 static int winIoerrRetryDelay = SQLITE_WIN32_IOERR_RETRY_DELAY;
 
+/*
+** The "winIsLockingError" macro is used to determine if a particular I/O
+** error code is due to file locking.  It must accept the error code DWORD
+** as its only argument and should return non-zero if the error code is due
+** to file locking.
+*/
+#if !defined(winIsLockingError)
+#define winIsLockingError(a) (((a)==ERROR_LOCK_VIOLATION)       || \
+                              ((a)==ERROR_IO_PENDING))
+#endif
+
 /*
 ** The "winIoerrCanRetry1" macro is used to determine if a particular I/O
 ** error code obtained via GetLastError() is eligible to be retried.  It
@@ -3673,6 +3684,9 @@ struct winShmNode {
 
   int szRegion;              /* Size of shared-memory regions */
   int nRegion;               /* Size of array apRegion */
+  u8 isReadonly;             /* True if read-only */
+  u8 isUnlocked;             /* True if no DMS lock held */
+
   struct ShmRegion {
     HANDLE hMap;             /* File handle from CreateFileMapping */
     void *pMap;
@@ -3820,6 +3834,116 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){
   }
 }
 
+/*
+** Query the status of the DMS lock for the specified file.  Returns
+** SQLITE_OK upon success.  Upon success, the integer pointed to by
+** the pLockType argument will be set to the lock type held by the
+** other process, as follows:
+**
+**       WINSHM_UNLCK -- No locks are held on the DMS.
+**       WINSHM_RDLCK -- A SHARED lock is held on the DMS.
+**       WINSHM_WRLCK -- An EXCLUSIVE lock is held on the DMS.
+*/
+static int winGetShmDmsLockType(
+  winFile *pFile, /* File handle object */
+  int *pLockType  /* WINSHM_UNLCK, WINSHM_RDLCK, or WINSHM_WRLCK */
+){
+#if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_OVERLAPPED)
+  OVERLAPPED overlapped; /* The offset for ReadFile/WriteFile. */
+#endif
+  LPVOID pOverlapped = 0;
+  sqlite3_int64 offset = WIN_SHM_DMS;
+  BYTE notUsed1 = 0;
+  DWORD notUsed2 = 0;
+
+#if SQLITE_OS_WINCE || defined(SQLITE_WIN32_NO_OVERLAPPED)
+  if( winSeekFile(pFile, offset) ){
+    return SQLITE_IOERR_SEEK;
+  }
+#else
+  memset(&overlapped, 0, sizeof(OVERLAPPED));
+  overlapped.Offset = (LONG)(offset & 0xffffffff);
+  overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff);
+  pOverlapped = &overlapped;
+#endif
+  if( !osWriteFile(pFile->h, &notUsed1, 1, &notUsed2, pOverlapped) ){
+    if( !osReadFile(pFile->h, &notUsed1, 1, &notUsed2, pOverlapped) ){
+      if( winIsLockingError(osGetLastError()) ){
+        *pLockType = WINSHM_WRLCK;
+      }else{
+        return SQLITE_IOERR_READ;
+      }
+    }else{
+      if( winIsLockingError(osGetLastError()) ){
+        *pLockType = WINSHM_RDLCK;
+      }else{
+        return SQLITE_IOERR_WRITE;
+      }
+    }
+  }else{
+    *pLockType = WINSHM_UNLCK;
+  }
+  return SQLITE_OK;
+}
+
+/*
+** The DMS lock has not yet been taken on shm file pShmNode. Attempt to
+** take it now. Return SQLITE_OK if successful, or an SQLite error
+** code otherwise.
+**
+** If the DMS cannot be locked because this is a readonly_shm=1
+** connection and no other process already holds a lock, return
+** SQLITE_READONLY_CANTINIT and set pShmNode->isUnlocked=1.
+*/
+static int winLockSharedMemory(winFile *pDbFd, winShmNode *pShmNode){
+  int lockType;
+  int rc = SQLITE_OK;
+
+  /* Use ReadFile/WriteFile to determine the locks other processes are
+  ** holding on the DMS byte. If it indicates that another process is
+  ** holding a SHARED lock, then this process may also take a SHARED
+  ** lock and proceed with opening the *-shm file.
+  **
+  ** Or, if no other process is holding any lock, then this process
+  ** is the first to open it. In this case take an EXCLUSIVE lock on the
+  ** DMS byte and truncate the *-shm file to zero bytes in size. Then
+  ** downgrade to a SHARED lock on the DMS byte.
+  **
+  ** If another process is holding an EXCLUSIVE lock on the DMS byte,
+  ** return SQLITE_BUSY to the caller (it will try again). An earlier
+  ** version of this code attempted the SHARED lock at this point. But
+  ** this introduced a subtle race condition: if the process holding
+  ** EXCLUSIVE failed just before truncating the *-shm file, then this
+  ** process might open and use the *-shm file without truncating it.
+  ** And if the *-shm file has been corrupted by a power failure or
+  ** system crash, the database itself may also become corrupt.  */
+  if( winGetShmDmsLockType(&pShmNode->hFile, &lockType)!=SQLITE_OK ){
+    rc = SQLITE_IOERR_LOCK;
+  }else if( lockType==WINSHM_UNLCK ){
+    if( pShmNode->isReadonly ){
+      pShmNode->isUnlocked = 1;
+      rc = SQLITE_READONLY_CANTINIT;
+    }else{
+      winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1);
+      rc = winShmSystemLock(pShmNode, WINSHM_WRLCK, WIN_SHM_DMS, 1);
+      if( rc==SQLITE_OK && winTruncate((sqlite3_file*)&pShmNode->hFile, 0) ){
+        rc = winLogError(SQLITE_IOERR_SHMOPEN, osGetLastError(),
+                         "winLockSharedMemory", pShmNode->zFilename);
+      }
+    }
+  }else if( lockType==WINSHM_WRLCK ){
+    rc = SQLITE_BUSY;
+  }
+
+  if( rc==SQLITE_OK ){
+    assert( lockType==WINSHM_UNLCK || lockType==WINSHM_RDLCK );
+    winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1);
+    rc = winShmSystemLock(pShmNode, WINSHM_RDLCK, WIN_SHM_DMS, 1);
+  }
+
+  return rc;
+}
+
 /*
 ** Open the shared-memory area associated with database file pDbFd.
 **
@@ -3828,11 +3952,12 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){
 ** the file must be truncated to zero length or have its header cleared.
 */
 static int winOpenSharedMemory(winFile *pDbFd){
-  struct winShm *p;                  /* The connection to be opened */
-  struct winShmNode *pShmNode = 0;   /* The underlying mmapped file */
-  int rc;                            /* Result code */
-  struct winShmNode *pNew;           /* Newly allocated winShmNode */
-  int nName;                         /* Size of zName in bytes */
+  struct winShm *p;         /* The connection to be opened */
+  winShmNode *pShmNode = 0; /* The underlying mmapped file */
+  int rc = SQLITE_OK;       /* Result code */
+  int rc2 = SQLITE_ERROR;   /* winOpen result code */
+  winShmNode *pNew;         /* Newly allocated winShmNode */
+  int nName;                /* Size of zName in bytes */
 
   assert( pDbFd->pShm==0 );    /* Not previously opened */
 
@@ -3878,30 +4003,29 @@ static int winOpenSharedMemory(winFile *pDbFd){
       }
     }
 
-    rc = winOpen(pDbFd->pVfs,
-                 pShmNode->zFilename,             /* Name of the file (UTF-8) */
-                 (sqlite3_file*)&pShmNode->hFile,  /* File handle here */
-                 SQLITE_OPEN_WAL | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE,
-                 0);
-    if( SQLITE_OK!=rc ){
-      goto shm_open_err;
+    if( 0==sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){
+      rc2 = winOpen(pDbFd->pVfs,
+                    pShmNode->zFilename,
+                    (sqlite3_file*)&pShmNode->hFile,
+                    SQLITE_OPEN_WAL|SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE,
+                    0);
+    }
+    if( rc2!=SQLITE_OK ){
+      rc2 = winOpen(pDbFd->pVfs,
+                    pShmNode->zFilename,
+                    (sqlite3_file*)&pShmNode->hFile,
+                    SQLITE_OPEN_WAL|SQLITE_OPEN_READONLY|SQLITE_OPEN_CREATE,
+                    0);
+      if( rc2!=SQLITE_OK ){
+        rc = winLogError(SQLITE_CANTOPEN_BKPT, osGetLastError(),
+                         "winOpenShm", pShmNode->zFilename);
+        goto shm_open_err;
+      }
+      pShmNode->isReadonly = 1;
     }
 
-    /* Check to see if another process is holding the dead-man switch.
-    ** If not, truncate the file to zero length.
-    */
-    if( winShmSystemLock(pShmNode, WINSHM_WRLCK, WIN_SHM_DMS, 1)==SQLITE_OK ){
-      rc = winTruncate((sqlite3_file *)&pShmNode->hFile, 0);
-      if( rc!=SQLITE_OK ){
-        rc = winLogError(SQLITE_IOERR_SHMOPEN, osGetLastError(),
-                         "winOpenShm", pDbFd->zPath);
-      }
-    }
-    if( rc==SQLITE_OK ){
-      winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1);
-      rc = winShmSystemLock(pShmNode, WINSHM_RDLCK, WIN_SHM_DMS, 1);
-    }
-    if( rc ) goto shm_open_err;
+    rc = winLockSharedMemory(pDbFd, pShmNode);
+    if( rc!=SQLITE_OK && rc!=SQLITE_READONLY_CANTINIT ) goto shm_open_err;
   }
 
   /* Make the new connection a child of the winShmNode */
@@ -4128,6 +4252,8 @@ static int winShmMap(
   winFile *pDbFd = (winFile*)fd;
   winShm *pShm = pDbFd->pShm;
   winShmNode *pShmNode;
+  DWORD protect = PAGE_READWRITE;
+  DWORD flags = FILE_MAP_WRITE | FILE_MAP_READ;
   int rc = SQLITE_OK;
 
   if( !pShm ){
@@ -4138,6 +4264,11 @@ static int winShmMap(
   pShmNode = pShm->pShmNode;
 
   sqlite3_mutex_enter(pShmNode->mutex);
+  if( pShmNode->isUnlocked ){
+    rc = winLockSharedMemory(pDbFd, pShmNode);
+    if( rc!=SQLITE_OK ) goto shmpage_out;
+    pShmNode->isUnlocked = 0;
+  }
   assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 );
 
   if( pShmNode->nRegion<=iRegion ){
@@ -4184,21 +4315,26 @@ static int winShmMap(
     }
     pShmNode->aRegion = apNew;
 
+    if( pShmNode->isReadonly ){
+      protect = PAGE_READONLY;
+      flags = FILE_MAP_READ;
+    }
+
     while( pShmNode->nRegion<=iRegion ){
       HANDLE hMap = NULL;         /* file-mapping handle */
       void *pMap = 0;             /* Mapped memory region */
 
 #if SQLITE_OS_WINRT
       hMap = osCreateFileMappingFromApp(pShmNode->hFile.h,
-          NULL, PAGE_READWRITE, nByte, NULL
+          NULL, protect, nByte, NULL
       );
 #elif defined(SQLITE_WIN32_HAS_WIDE)
       hMap = osCreateFileMappingW(pShmNode->hFile.h,
-          NULL, PAGE_READWRITE, 0, nByte, NULL
+          NULL, protect, 0, nByte, NULL
       );
 #elif defined(SQLITE_WIN32_HAS_ANSI) && SQLITE_WIN32_CREATEFILEMAPPINGA
       hMap = osCreateFileMappingA(pShmNode->hFile.h,
-          NULL, PAGE_READWRITE, 0, nByte, NULL
+          NULL, protect, 0, nByte, NULL
       );
 #endif
       OSTRACE(("SHM-MAP-CREATE pid=%lu, region=%d, size=%d, rc=%s\n",
@@ -4208,11 +4344,11 @@ static int winShmMap(
         int iOffset = pShmNode->nRegion*szRegion;
         int iOffsetShift = iOffset % winSysInfo.dwAllocationGranularity;
 #if SQLITE_OS_WINRT
-        pMap = osMapViewOfFileFromApp(hMap, FILE_MAP_WRITE | FILE_MAP_READ,
+        pMap = osMapViewOfFileFromApp(hMap, flags,
             iOffset - iOffsetShift, szRegion + iOffsetShift
         );
 #else
-        pMap = osMapViewOfFile(hMap, FILE_MAP_WRITE | FILE_MAP_READ,
+        pMap = osMapViewOfFile(hMap, flags,
             0, iOffset - iOffsetShift, szRegion + iOffsetShift
         );
 #endif
@@ -4243,6 +4379,7 @@ shmpage_out:
   }else{
     *pp = 0;
   }
+  if( pShmNode->isReadonly && rc==SQLITE_OK ) rc = SQLITE_READONLY;
   sqlite3_mutex_leave(pShmNode->mutex);
   return rc;
 }
diff --git a/test/walro2.test b/test/walro2.test
index 5a8ce023eb..4b90618fc6 100644
--- a/test/walro2.test
+++ b/test/walro2.test
@@ -18,13 +18,6 @@ source $testdir/lock_common.tcl
 source $testdir/wal_common.tcl
 set ::testprefix walro2
 
-# These tests are only going to work on unix.
-#
-if {$::tcl_platform(platform) != "unix"} {
-  finish_test
-  return
-}
-
 # And only if the build is WAL-capable.
 #
 ifcapable !wal {
diff --git a/test/walrofault.test b/test/walrofault.test
index b396be218c..22d4f96bbc 100644
--- a/test/walrofault.test
+++ b/test/walrofault.test
@@ -17,13 +17,6 @@ source $testdir/tester.tcl
 source $testdir/malloc_common.tcl
 set ::testprefix walro2
 
-# These tests are only going to work on unix.
-#
-if {$::tcl_platform(platform) != "unix"} {
-  finish_test
-  return
-}
-
 # And only if the build is WAL-capable.
 #
 ifcapable !wal {

From 4a9ff91852c8bc23a7a1157d05a789b1a1211dd4 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Thu, 9 Nov 2017 17:29:04 +0000
Subject: [PATCH 072/488] Make it possible to use OSTRACE for multi-process
 testing.

FossilOrigin-Name: 0a7d416c4c43632725dc89cda8667cd9726b5152ee4692e4d0c9e2031e60cfb4
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/printf.c  |  7 +++++++
 3 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index dbc8286837..d11e831d0c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Initial\swork\son\sporting\sthe\schanges\son\sthis\sbranch\sto\sWin32.
-D 2017-11-09T16:30:55.245
+C Make\sit\spossible\sto\suse\sOSTRACE\sfor\smulti-process\stesting.
+D 2017-11-09T17:29:04.731
 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2
@@ -459,7 +459,7 @@ F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
 F src/pragma.c d04725ac25387d9638919e197fb009f378e13af7bf899516979e54b3164e3602
 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
 F src/prepare.c 7cf451f903ad92a14e22de415a13e7a7d30f1bd23b3d21eeb0dc7264723244c5
-F src/printf.c 40aee47ae9be4bd3dbdc8968bd07fddc027be8edec8daddf24d3391d36698a1c
+F src/printf.c 9506b4b96e59c0467047155f09015750cb2878aeda3d39e5610c1192ddc3c41c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 5a461643f294ec510ca615b67256fc3861e4c8eff5f29e5940491e70553b1955
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
@@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 033ee92bf4d5dc57f5cb8fd02d1154ae06f2d3261d214e7191a82c70c8ffebf7
-R a3370fa9345068426fa8edd8924927fc
+P 3738bfd0c0eadb10eea58954af5052cb6ce164059f3aacfe65d7da6a400c63c7
+R b588bb75e731a71d38f9b889268c4e35
 U mistachkin
-Z fbbc6e55aa870fcd83aef8acf0e01375
+Z 3f860c7c58efb09a7e9d6ae32c9bc1c6
diff --git a/manifest.uuid b/manifest.uuid
index f5c23f13f2..866cdfe91f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-3738bfd0c0eadb10eea58954af5052cb6ce164059f3aacfe65d7da6a400c63c7
\ No newline at end of file
+0a7d416c4c43632725dc89cda8667cd9726b5152ee4692e4d0c9e2031e60cfb4
\ No newline at end of file
diff --git a/src/printf.c b/src/printf.c
index 9427844e09..ca8d26e4f9 100644
--- a/src/printf.c
+++ b/src/printf.c
@@ -1092,8 +1092,15 @@ void sqlite3DebugPrintf(const char *zFormat, ...){
   sqlite3VXPrintf(&acc, zFormat, ap);
   va_end(ap);
   sqlite3StrAccumFinish(&acc);
+#ifdef SQLITE_OS_TRACE_PROC
+  {
+    extern void SQLITE_OS_TRACE_PROC(const char *zBuf, int nBuf);
+    SQLITE_OS_TRACE_PROC(zBuf, sizeof(zBuf));
+  }
+#else
   fprintf(stdout,"%s", zBuf);
   fflush(stdout);
+#endif
 }
 #endif
 

From 8a6fa5d41c3188c93a0b3e01d1395ab88e384a82 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Thu, 9 Nov 2017 17:29:18 +0000
Subject: [PATCH 073/488] Corrections to the Win32 porting changes on this
 branch.

FossilOrigin-Name: 0b26a5a26d700e20eea5ebbd620af0af6f2d61c652cfca5b8563267588cb2be6
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/os_win.c  | 10 +++++-----
 3 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/manifest b/manifest
index d11e831d0c..021c050212 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Make\sit\spossible\sto\suse\sOSTRACE\sfor\smulti-process\stesting.
-D 2017-11-09T17:29:04.731
+C Corrections\sto\sthe\sWin32\sporting\schanges\son\sthis\sbranch.
+D 2017-11-09T17:29:18.701
 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2
@@ -448,7 +448,7 @@ F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
 F src/os_unix.c e87cef0bb894b94d96ee3af210be669549d111c580817d14818101b992640767
-F src/os_win.c 47687775641c97743c228f99813fbcffe7d53602da5cdcf0fe52f6810341a46c
+F src/os_win.c ce0b1da0d41a1cc78e33431c72fb4b53f3247b3884841ca0a988761565488824
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
@@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3738bfd0c0eadb10eea58954af5052cb6ce164059f3aacfe65d7da6a400c63c7
-R b588bb75e731a71d38f9b889268c4e35
+P 0a7d416c4c43632725dc89cda8667cd9726b5152ee4692e4d0c9e2031e60cfb4
+R 83920a29a292bf4d0aadf8e0cc58f982
 U mistachkin
-Z 3f860c7c58efb09a7e9d6ae32c9bc1c6
+Z f20b1547624a9a7c878450951251f899
diff --git a/manifest.uuid b/manifest.uuid
index 866cdfe91f..f22324c47b 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0a7d416c4c43632725dc89cda8667cd9726b5152ee4692e4d0c9e2031e60cfb4
\ No newline at end of file
+0b26a5a26d700e20eea5ebbd620af0af6f2d61c652cfca5b8563267588cb2be6
\ No newline at end of file
diff --git a/src/os_win.c b/src/os_win.c
index 9bfcfe1c81..d434cfe927 100644
--- a/src/os_win.c
+++ b/src/os_win.c
@@ -3895,7 +3895,7 @@ static int winGetShmDmsLockType(
 ** connection and no other process already holds a lock, return
 ** SQLITE_READONLY_CANTINIT and set pShmNode->isUnlocked=1.
 */
-static int winLockSharedMemory(winFile *pDbFd, winShmNode *pShmNode){
+static int winLockSharedMemory(winShmNode *pShmNode){
   int lockType;
   int rc = SQLITE_OK;
 
@@ -4014,7 +4014,7 @@ static int winOpenSharedMemory(winFile *pDbFd){
       rc2 = winOpen(pDbFd->pVfs,
                     pShmNode->zFilename,
                     (sqlite3_file*)&pShmNode->hFile,
-                    SQLITE_OPEN_WAL|SQLITE_OPEN_READONLY|SQLITE_OPEN_CREATE,
+                    SQLITE_OPEN_WAL|SQLITE_OPEN_READONLY,
                     0);
       if( rc2!=SQLITE_OK ){
         rc = winLogError(SQLITE_CANTOPEN_BKPT, osGetLastError(),
@@ -4024,7 +4024,7 @@ static int winOpenSharedMemory(winFile *pDbFd){
       pShmNode->isReadonly = 1;
     }
 
-    rc = winLockSharedMemory(pDbFd, pShmNode);
+    rc = winLockSharedMemory(pShmNode);
     if( rc!=SQLITE_OK && rc!=SQLITE_READONLY_CANTINIT ) goto shm_open_err;
   }
 
@@ -4048,7 +4048,7 @@ static int winOpenSharedMemory(winFile *pDbFd){
   p->pNext = pShmNode->pFirst;
   pShmNode->pFirst = p;
   sqlite3_mutex_leave(pShmNode->mutex);
-  return SQLITE_OK;
+  return rc;
 
   /* Jump here on any error */
 shm_open_err:
@@ -4265,7 +4265,7 @@ static int winShmMap(
 
   sqlite3_mutex_enter(pShmNode->mutex);
   if( pShmNode->isUnlocked ){
-    rc = winLockSharedMemory(pDbFd, pShmNode);
+    rc = winLockSharedMemory(pShmNode);
     if( rc!=SQLITE_OK ) goto shmpage_out;
     pShmNode->isUnlocked = 0;
   }

From a5150656494e915b675498f8b6b888a336997bec Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Thu, 9 Nov 2017 18:21:51 +0000
Subject: [PATCH 074/488] Corrections to Win32 lock detection for SHM files.

FossilOrigin-Name: 3a91be975daee65c3e1199855613066015d5df8ad44ababdef31d1c698b5e746
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/os_win.c  |  9 ++++++---
 3 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index 021c050212..eb409a8a6a 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Corrections\sto\sthe\sWin32\sporting\schanges\son\sthis\sbranch.
-D 2017-11-09T17:29:18.701
+C Corrections\sto\sWin32\slock\sdetection\sfor\sSHM\sfiles.
+D 2017-11-09T18:21:51.933
 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2
@@ -448,7 +448,7 @@ F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
 F src/os_unix.c e87cef0bb894b94d96ee3af210be669549d111c580817d14818101b992640767
-F src/os_win.c ce0b1da0d41a1cc78e33431c72fb4b53f3247b3884841ca0a988761565488824
+F src/os_win.c eac2f14343eaf9ff8c02d7025ce459f64dc1334c1f6739be5b6595aabed32ca2
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
@@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0a7d416c4c43632725dc89cda8667cd9726b5152ee4692e4d0c9e2031e60cfb4
-R 83920a29a292bf4d0aadf8e0cc58f982
+P 0b26a5a26d700e20eea5ebbd620af0af6f2d61c652cfca5b8563267588cb2be6
+R e599061360fdeb01478e7f6faea2d0b3
 U mistachkin
-Z f20b1547624a9a7c878450951251f899
+Z 310f1e71824409858101a715db0b42e2
diff --git a/manifest.uuid b/manifest.uuid
index f22324c47b..238938df65 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0b26a5a26d700e20eea5ebbd620af0af6f2d61c652cfca5b8563267588cb2be6
\ No newline at end of file
+3a91be975daee65c3e1199855613066015d5df8ad44ababdef31d1c698b5e746
\ No newline at end of file
diff --git a/src/os_win.c b/src/os_win.c
index d434cfe927..34069351df 100644
--- a/src/os_win.c
+++ b/src/os_win.c
@@ -2105,7 +2105,8 @@ static int winIoerrRetryDelay = SQLITE_WIN32_IOERR_RETRY_DELAY;
 ** to file locking.
 */
 #if !defined(winIsLockingError)
-#define winIsLockingError(a) (((a)==ERROR_LOCK_VIOLATION)       || \
+#define winIsLockingError(a) (((a)==ERROR_ACCESS_DENIED)        || \
+                              ((a)==ERROR_LOCK_VIOLATION)       || \
                               ((a)==ERROR_IO_PENDING))
 #endif
 
@@ -3867,14 +3868,16 @@ static int winGetShmDmsLockType(
   pOverlapped = &overlapped;
 #endif
   if( !osWriteFile(pFile->h, &notUsed1, 1, &notUsed2, pOverlapped) ){
+    DWORD lastErrno = osGetLastError();
     if( !osReadFile(pFile->h, &notUsed1, 1, &notUsed2, pOverlapped) ){
-      if( winIsLockingError(osGetLastError()) ){
+      lastErrno = osGetLastError();
+      if( winIsLockingError(lastErrno) ){
         *pLockType = WINSHM_WRLCK;
       }else{
         return SQLITE_IOERR_READ;
       }
     }else{
-      if( winIsLockingError(osGetLastError()) ){
+      if( winIsLockingError(lastErrno) ){
         *pLockType = WINSHM_RDLCK;
       }else{
         return SQLITE_IOERR_WRITE;

From 9f4aad446b5ed8b0006c323f4df2db1e5743bf90 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Thu, 9 Nov 2017 18:53:51 +0000
Subject: [PATCH 075/488] Further corrections to read-only SHM file handling on
 Win32.

FossilOrigin-Name: 43c311701bdf1202918cd46fa6133a11458e0ef8ddb09e46290a231083f395ce
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/os_win.c  | 18 +++++++++++-------
 3 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/manifest b/manifest
index eb409a8a6a..0ecda71659 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Corrections\sto\sWin32\slock\sdetection\sfor\sSHM\sfiles.
-D 2017-11-09T18:21:51.933
+C Further\scorrections\sto\sread-only\sSHM\sfile\shandling\son\sWin32.
+D 2017-11-09T18:53:51.070
 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2
@@ -448,7 +448,7 @@ F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
 F src/os_unix.c e87cef0bb894b94d96ee3af210be669549d111c580817d14818101b992640767
-F src/os_win.c eac2f14343eaf9ff8c02d7025ce459f64dc1334c1f6739be5b6595aabed32ca2
+F src/os_win.c f55a1ae65702e1762dcc175c1b3b32818bcb4d5faee83d2159adafbac94770c4
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
@@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0b26a5a26d700e20eea5ebbd620af0af6f2d61c652cfca5b8563267588cb2be6
-R e599061360fdeb01478e7f6faea2d0b3
+P 3a91be975daee65c3e1199855613066015d5df8ad44ababdef31d1c698b5e746
+R b94c408b8a8461b2b6bed04445c4e88e
 U mistachkin
-Z 310f1e71824409858101a715db0b42e2
+Z 2f572d772e98ad83ecab51d538357fde
diff --git a/manifest.uuid b/manifest.uuid
index 238938df65..3ece53ff59 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-3a91be975daee65c3e1199855613066015d5df8ad44ababdef31d1c698b5e746
\ No newline at end of file
+43c311701bdf1202918cd46fa6133a11458e0ef8ddb09e46290a231083f395ce
\ No newline at end of file
diff --git a/src/os_win.c b/src/os_win.c
index 34069351df..6b0bb3dbd0 100644
--- a/src/os_win.c
+++ b/src/os_win.c
@@ -2105,8 +2105,9 @@ static int winIoerrRetryDelay = SQLITE_WIN32_IOERR_RETRY_DELAY;
 ** to file locking.
 */
 #if !defined(winIsLockingError)
-#define winIsLockingError(a) (((a)==ERROR_ACCESS_DENIED)        || \
+#define winIsLockingError(a) (((a)==NO_ERROR)                   || \
                               ((a)==ERROR_LOCK_VIOLATION)       || \
+                              ((a)==ERROR_HANDLE_EOF)           || \
                               ((a)==ERROR_IO_PENDING))
 #endif
 
@@ -3847,6 +3848,7 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){
 */
 static int winGetShmDmsLockType(
   winFile *pFile, /* File handle object */
+  int bReadOnly,  /* Non-zero if the SHM was opened read-only */
   int *pLockType  /* WINSHM_UNLCK, WINSHM_RDLCK, or WINSHM_WRLCK */
 ){
 #if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_OVERLAPPED)
@@ -3867,24 +3869,25 @@ static int winGetShmDmsLockType(
   overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff);
   pOverlapped = &overlapped;
 #endif
-  if( !osWriteFile(pFile->h, &notUsed1, 1, &notUsed2, pOverlapped) ){
-    DWORD lastErrno = osGetLastError();
+  if( bReadOnly ||
+      !osWriteFile(pFile->h, &notUsed1, 1, &notUsed2, pOverlapped) ){
+    DWORD lastErrno = bReadOnly ? NO_ERROR : osGetLastError();
     if( !osReadFile(pFile->h, &notUsed1, 1, &notUsed2, pOverlapped) ){
       lastErrno = osGetLastError();
       if( winIsLockingError(lastErrno) ){
-        *pLockType = WINSHM_WRLCK;
+        if( pLockType ) *pLockType = WINSHM_WRLCK;
       }else{
         return SQLITE_IOERR_READ;
       }
     }else{
       if( winIsLockingError(lastErrno) ){
-        *pLockType = WINSHM_RDLCK;
+        if( pLockType ) *pLockType = WINSHM_RDLCK;
       }else{
         return SQLITE_IOERR_WRITE;
       }
     }
   }else{
-    *pLockType = WINSHM_UNLCK;
+    if( pLockType ) *pLockType = WINSHM_UNLCK;
   }
   return SQLITE_OK;
 }
@@ -3920,7 +3923,8 @@ static int winLockSharedMemory(winShmNode *pShmNode){
   ** process might open and use the *-shm file without truncating it.
   ** And if the *-shm file has been corrupted by a power failure or
   ** system crash, the database itself may also become corrupt.  */
-  if( winGetShmDmsLockType(&pShmNode->hFile, &lockType)!=SQLITE_OK ){
+  if( winGetShmDmsLockType(&pShmNode->hFile, pShmNode->isReadonly,
+                           &lockType)!=SQLITE_OK ){
     rc = SQLITE_IOERR_LOCK;
   }else if( lockType==WINSHM_UNLCK ){
     if( pShmNode->isReadonly ){

From bcb416a9ff954bbc22bd226615edc335a6989666 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Thu, 9 Nov 2017 20:02:44 +0000
Subject: [PATCH 076/488] Get read-only SHM file tests passing on Win32.

FossilOrigin-Name: abef05353554e72f4d08aff562b87ff8530e8537a79e58d831205ea8c46eed07
---
 manifest             | 16 ++++++++--------
 manifest.uuid        |  2 +-
 src/os_win.c         | 32 +++++++++++++++++++++-----------
 test/walro2.test     |  2 ++
 test/walrofault.test |  1 +
 5 files changed, 33 insertions(+), 20 deletions(-)

diff --git a/manifest b/manifest
index 0ecda71659..05976da356 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Further\scorrections\sto\sread-only\sSHM\sfile\shandling\son\sWin32.
-D 2017-11-09T18:53:51.070
+C Get\sread-only\sSHM\sfile\stests\spassing\son\sWin32.
+D 2017-11-09T20:02:44.785
 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2
@@ -448,7 +448,7 @@ F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
 F src/os_unix.c e87cef0bb894b94d96ee3af210be669549d111c580817d14818101b992640767
-F src/os_win.c f55a1ae65702e1762dcc175c1b3b32818bcb4d5faee83d2159adafbac94770c4
+F src/os_win.c cf4099958dcc72a9e36ce161638aa369811012fa1e2592415fdc1fdafdf26c8c
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
@@ -1527,8 +1527,8 @@ F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03
 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6
 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20
 F test/walro.test cb438d05ba0d191f10b688e39c4f0cd5b71569a1d1f4440e5bdf3c6880e08c20
-F test/walro2.test 2c01a3c38ca731df4690cc901e3c0bee21bf9d231fa3d47d81162be10462c40b
-F test/walrofault.test f1c9c361a73faab98ce1bb4588333e42f7dd330f2b4987e962aa5fe68e7982de
+F test/walro2.test bde4b25b701be452ba1436409d9ee418513f77bf4c11229d558fd14282330e00
+F test/walrofault.test c70cb6e308c443867701856cce92ad8288cd99488fa52afab77cca6cfd51af68
 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417
 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f
 F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e
@@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3a91be975daee65c3e1199855613066015d5df8ad44ababdef31d1c698b5e746
-R b94c408b8a8461b2b6bed04445c4e88e
+P 43c311701bdf1202918cd46fa6133a11458e0ef8ddb09e46290a231083f395ce
+R b51af4203460b2806284a868a4615565
 U mistachkin
-Z 2f572d772e98ad83ecab51d538357fde
+Z 2f0a43c33871392e0df8028f39e6b334
diff --git a/manifest.uuid b/manifest.uuid
index 3ece53ff59..dd49a781d5 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-43c311701bdf1202918cd46fa6133a11458e0ef8ddb09e46290a231083f395ce
\ No newline at end of file
+abef05353554e72f4d08aff562b87ff8530e8537a79e58d831205ea8c46eed07
\ No newline at end of file
diff --git a/src/os_win.c b/src/os_win.c
index 6b0bb3dbd0..05ae7e67db 100644
--- a/src/os_win.c
+++ b/src/os_win.c
@@ -2099,18 +2099,26 @@ static int winIoerrRetry = SQLITE_WIN32_IOERR_RETRY;
 static int winIoerrRetryDelay = SQLITE_WIN32_IOERR_RETRY_DELAY;
 
 /*
-** The "winIsLockingError" macro is used to determine if a particular I/O
-** error code is due to file locking.  It must accept the error code DWORD
-** as its only argument and should return non-zero if the error code is due
-** to file locking.
+** The "winIsLockConflict" macro is used to determine if a particular I/O
+** error code is due to a file locking conflict.  It must accept the error
+** code DWORD as its only argument.
 */
-#if !defined(winIsLockingError)
-#define winIsLockingError(a) (((a)==NO_ERROR)                   || \
+#if !defined(winIsLockConflict)
+#define winIsLockConflict(a) (((a)==NO_ERROR)                   || \
                               ((a)==ERROR_LOCK_VIOLATION)       || \
-                              ((a)==ERROR_HANDLE_EOF)           || \
                               ((a)==ERROR_IO_PENDING))
 #endif
 
+/*
+** The "winIsLockMissing" macro is used to determine if a particular I/O
+** error code is due to being unable to obtain a file lock because all or
+** part of the range requested within the file is missing.  It must accept
+** the error code DWORD as its only argument.
+*/
+#if !defined(winIsLockMissing)
+#define winIsLockMissing(a) (((a)==ERROR_HANDLE_EOF))
+#endif
+
 /*
 ** The "winIoerrCanRetry1" macro is used to determine if a particular I/O
 ** error code obtained via GetLastError() is eligible to be retried.  It
@@ -3874,13 +3882,15 @@ static int winGetShmDmsLockType(
     DWORD lastErrno = bReadOnly ? NO_ERROR : osGetLastError();
     if( !osReadFile(pFile->h, &notUsed1, 1, &notUsed2, pOverlapped) ){
       lastErrno = osGetLastError();
-      if( winIsLockingError(lastErrno) ){
+      if( winIsLockConflict(lastErrno) ){
         if( pLockType ) *pLockType = WINSHM_WRLCK;
+      }else if( winIsLockMissing(lastErrno) ){
+        if( pLockType ) *pLockType = WINSHM_UNLCK;
       }else{
         return SQLITE_IOERR_READ;
       }
     }else{
-      if( winIsLockingError(lastErrno) ){
+      if( winIsLockConflict(lastErrno) ){
         if( pLockType ) *pLockType = WINSHM_RDLCK;
       }else{
         return SQLITE_IOERR_WRITE;
@@ -4024,8 +4034,8 @@ static int winOpenSharedMemory(winFile *pDbFd){
                     SQLITE_OPEN_WAL|SQLITE_OPEN_READONLY,
                     0);
       if( rc2!=SQLITE_OK ){
-        rc = winLogError(SQLITE_CANTOPEN_BKPT, osGetLastError(),
-                         "winOpenShm", pShmNode->zFilename);
+        rc = winLogError(rc2, osGetLastError(), "winOpenShm",
+                         pShmNode->zFilename);
         goto shm_open_err;
       }
       pShmNode->isReadonly = 1;
diff --git a/test/walro2.test b/test/walro2.test
index 4b90618fc6..67023de1a2 100644
--- a/test/walro2.test
+++ b/test/walro2.test
@@ -255,6 +255,8 @@ do_multiclient_test tn {
     file size test.db-wal
   } {461152}
   do_test 4.2.4 {
+    file_control_persist_wal db 1; db close
+
     forcecopy test.db test.db2
     forcecopy test.db-wal test.db2-wal
     forcecopy test.db-shm test.db2-shm
diff --git a/test/walrofault.test b/test/walrofault.test
index 22d4f96bbc..3e66e2d920 100644
--- a/test/walrofault.test
+++ b/test/walrofault.test
@@ -42,6 +42,7 @@ do_execsql_test 1.0 {
     WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<30 ) 
     INSERT INTO t1(b) SELECT randomblob(800) FROM s;
 } {wal}
+file_control_persist_wal db 1; db close
 faultsim_save_and_close
 
 do_faultsim_test 1 -faults oom* -prep {

From e94187780620d5c1b6ceafe3e90ac15f17fe428b Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 9 Nov 2017 20:34:35 +0000
Subject: [PATCH 077/488] Enhance walro2.test to better ensure that
 readonly_shm clients are not using invalid *-shm files.

FossilOrigin-Name: ff630b66714b20c09888ead0a45f344d63e0d9a5208867d6266e74f79187076c
---
 manifest         |  14 +++---
 manifest.uuid    |   2 +-
 test/walro2.test | 110 ++++++++++++++++++++++++-----------------------
 3 files changed, 65 insertions(+), 61 deletions(-)

diff --git a/manifest b/manifest
index 05976da356..e214aee04c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Get\sread-only\sSHM\sfile\stests\spassing\son\sWin32.
-D 2017-11-09T20:02:44.785
+C Enhance\swalro2.test\sto\sbetter\sensure\sthat\sreadonly_shm\sclients\sare\snot\susing\ninvalid\s*-shm\sfiles.
+D 2017-11-09T20:34:35.406
 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2
@@ -1527,7 +1527,7 @@ F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03
 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6
 F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20
 F test/walro.test cb438d05ba0d191f10b688e39c4f0cd5b71569a1d1f4440e5bdf3c6880e08c20
-F test/walro2.test bde4b25b701be452ba1436409d9ee418513f77bf4c11229d558fd14282330e00
+F test/walro2.test 8812e514c968bf4ee317571fafedac43443360ae23edd7d0f4ef1eae0c13e8e8
 F test/walrofault.test c70cb6e308c443867701856cce92ad8288cd99488fa52afab77cca6cfd51af68
 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417
 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f
@@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 43c311701bdf1202918cd46fa6133a11458e0ef8ddb09e46290a231083f395ce
-R b51af4203460b2806284a868a4615565
-U mistachkin
-Z 2f0a43c33871392e0df8028f39e6b334
+P abef05353554e72f4d08aff562b87ff8530e8537a79e58d831205ea8c46eed07
+R 43c8def15803418ff2b6211daf889aff
+U dan
+Z b374f730e5d2822de9c4c678f7c53c0c
diff --git a/manifest.uuid b/manifest.uuid
index dd49a781d5..d156b1b802 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-abef05353554e72f4d08aff562b87ff8530e8537a79e58d831205ea8c46eed07
\ No newline at end of file
+ff630b66714b20c09888ead0a45f344d63e0d9a5208867d6266e74f79187076c
\ No newline at end of file
diff --git a/test/walro2.test b/test/walro2.test
index 67023de1a2..9d0a6b4a19 100644
--- a/test/walro2.test
+++ b/test/walro2.test
@@ -25,6 +25,22 @@ ifcapable !wal {
   return
 }
 
+proc copy_to_test2 {bZeroShm} {
+  forcecopy test.db test.db2
+  forcecopy test.db-wal test.db2-wal
+  if {$bZeroShm} {
+    forcedelete test.db2-shm
+    set fd [open test.db2-shm w]
+    seek $fd [expr [file size test.db-shm]-1]
+    puts -nonewline $fd "\0"
+    close $fd
+  } else {
+    forcecopy test.db-shm test.db2-shm
+  }
+}
+
+foreach bZeroShm {0 1} {
+set TN [expr $bZeroShm+1]
 do_multiclient_test tn {
   
   # Close all connections and delete the database.
@@ -45,7 +61,7 @@ do_multiclient_test tn {
     }
   }
 
-  do_test 1.1 {
+  do_test $TN.1.1 {
     code2 { sqlite3 db2 test.db }
     sql2 { 
       CREATE TABLE t1(x, y);
@@ -56,26 +72,24 @@ do_multiclient_test tn {
     file exists test.db-shm
   } {1}
 
-  do_test 1.2.1 {
-    forcecopy test.db test.db2
-    forcecopy test.db-wal test.db2-wal
-    forcecopy test.db-shm test.db2-shm
+  do_test $TN.1.2.1 {
+    copy_to_test2 $bZeroShm
     code1 {
       sqlite3 db file:test.db2?readonly_shm=1
     }
 
     sql1 { SELECT * FROM t1 }
   } {a b c d}
-  do_test 1.2.2 {
+  do_test $TN.1.2.2 {
     sql1 { SELECT * FROM t1 }
   } {a b c d}
 
-  do_test 1.3.1 {
+  do_test $TN.1.3.1 {
     code3 { sqlite3 db3 test.db2 }
     sql3 { SELECT * FROM t1 }
   } {a b c d}
 
-  do_test 1.3.2 {
+  do_test $TN.1.3.2 {
     sql1 { SELECT * FROM t1 }
   } {a b c d}
 
@@ -83,7 +97,7 @@ do_multiclient_test tn {
   code2 { db2 close }
   code3 { db3 close }
 
-  do_test 2.1 {
+  do_test $TN.2.1 {
     code2 { sqlite3 db2 test.db }
     sql2 { 
       INSERT INTO t1 VALUES('e', 'f');
@@ -92,10 +106,8 @@ do_multiclient_test tn {
     file exists test.db-shm
   } {1}
 
-  do_test 2.2 {
-    forcecopy test.db test.db2
-    forcecopy test.db-wal test.db2-wal
-    forcecopy test.db-shm test.db2-shm
+  do_test $TN.2.2 {
+    copy_to_test2 $bZeroShm
     code1 {
       sqlite3 db file:test.db2?readonly_shm=1
     }
@@ -105,16 +117,16 @@ do_multiclient_test tn {
     }
   } {a b c d e f g h}
 
-  do_test 2.3.1 {
+  do_test $TN.2.3.1 {
     code3 { sqlite3 db3 test.db2 }
     sql3 { SELECT * FROM t1 }
   } {a b c d e f g h}
-  do_test 2.3.2 {
+  do_test $TN.2.3.2 {
     sql3 { INSERT INTO t1 VALUES('i', 'j') }
     code3 { db3 close }
     sql1 { COMMIT } 
   } {}
-  do_test 2.3.3 {
+  do_test $TN.2.3.3 {
     sql1 { SELECT * FROM t1 }
   } {a b c d e f g h i j}
 
@@ -134,10 +146,10 @@ do_multiclient_test tn {
   catch { code1 { db close } }
   catch { code2 { db2 close } }
   catch { code3 { db3 close } }
-  do_test 3.1.0 {
+  do_test $TN.3.1.0 {
     list [file exists test.db-wal] [file exists test.db-shm]
   } {0 0}
-  do_test 3.1.1 {
+  do_test $TN.3.1.1 {
     close [open test.db-wal w]
     close [open test.db-shm w]
     code1 {
@@ -146,20 +158,20 @@ do_multiclient_test tn {
     sql1 { SELECT * FROM t1 }
   } {a b c d e f g h}
 
-  do_test 3.2.0 {
+  do_test $TN.3.2.0 {
     list [file size test.db-wal] [file size test.db-shm]
   } {0 0}
-  do_test 3.2.1 {
+  do_test $TN.3.2.1 {
     code2 { sqlite3 db2 test.db }
     sql2 { INSERT INTO t1 VALUES(1, 2) ; PRAGMA wal_checkpoint=truncate }
     code2 { db2 close }
     sql1 { SELECT * FROM t1 }
   } {a b c d e f g h 1 2}
-  do_test 3.2.2 {
+  do_test $TN.3.2.2 {
     list [file size test.db-wal] [file size test.db-shm]
   } {0 32768}
 
-  do_test 3.3.0 {
+  do_test $TN.3.3.0 {
     code2 { sqlite3 db2 test.db }
     sql2 { 
       INSERT INTO t1 VALUES(3, 4);
@@ -171,11 +183,11 @@ do_multiclient_test tn {
     code1 { db close }
     list [file size test.db-wal] [file size test.db-shm]
   } [list [wal_file_size 4 1024] 32768]
-  do_test 3.3.1 {
+  do_test $TN.3.3.1 {
     code1 { sqlite3 db file:test.db?readonly_shm=1 }
     sql1 { SELECT * FROM t1 }
   } {a b c d e f g h 1 2 3 4 5 6 7 8 9 10}
-  do_test 3.3.2 {
+  do_test $TN.3.3.2 {
     code2 { sqlite3 db2 test.db }
     sql2 { 
       PRAGMA wal_checkpoint; 
@@ -185,7 +197,7 @@ do_multiclient_test tn {
     code2 { db2 close }
     list [file size test.db-wal] [file size test.db-shm]
   } [list [wal_file_size 4 1024] 32768]
-  do_test 3.3.3 {
+  do_test $TN.3.3.3 {
     sql1 { SELECT * FROM t1 }
   } {i ii}
 
@@ -196,7 +208,7 @@ do_multiclient_test tn {
   catch { code2 { db2 close } }
   catch { code3 { db3 close } }
 
-  do_test 4.0 {
+  do_test $TN.4.0 {
     code1 { forcedelete test.db }
     code1 { sqlite3 db test.db }
     sql1 {
@@ -206,19 +218,17 @@ do_multiclient_test tn {
       INSERT INTO t1 VALUES('world');
     }
 
-    forcecopy test.db test.db2
-    forcecopy test.db-wal test.db2-wal
-    forcecopy test.db-shm test.db2-shm
-    
+    copy_to_test2 $bZeroShm
+
     code1 { db close }
   } {}
 
-  do_test 4.1.1 {
+  do_test $TN.4.1.1 {
     code2 { sqlite3 db2 file:test.db2?readonly_shm=1 }
     sql2 { SELECT * FROM t1 }
   } {hello world}
 
-  do_test 4.1.2 {
+  do_test $TN.4.1.2 {
     code3 { sqlite3 db3 test.db2 }
     sql3 {
       INSERT INTO t1 VALUES('!');
@@ -226,7 +236,7 @@ do_multiclient_test tn {
     }
     code3 { db3 close }
   } {}
-  do_test 4.1.3 {
+  do_test $TN.4.1.3 {
     sql2 { SELECT * FROM t1 }
   } {hello world !}
 
@@ -234,7 +244,7 @@ do_multiclient_test tn {
   catch { code2 { db2 close } }
   catch { code3 { db3 close } }
 
-  do_test 4.2.1 {
+  do_test $TN.4.2.1 {
     code1 { sqlite3 db test.db }
     sql1 {
       INSERT INTO t1 VALUES('!');
@@ -251,16 +261,13 @@ do_multiclient_test tn {
         SELECT count(*) FROM t2;
     } 
   } {500}
-  do_test 4.2.2 {
+  do_test $TN.4.2.2 {
     file size test.db-wal
   } {461152}
-  do_test 4.2.4 {
+  do_test $TN.4.2.4 {
     file_control_persist_wal db 1; db close
 
-    forcecopy test.db test.db2
-    forcecopy test.db-wal test.db2-wal
-    forcecopy test.db-shm test.db2-shm
-
+    copy_to_test2 $bZeroShm
     code2 { sqlite3 db2 file:test.db2?readonly_shm=1 }
     sql2 {
       SELECT * FROM t1;
@@ -275,7 +282,7 @@ do_multiclient_test tn {
   catch { code2 { db2 close } }
   catch { code3 { db3 close } }
 
-  do_test 5.0 {
+  do_test $TN.5.0 {
     code1 { forcedelete test.db }
     code1 { sqlite3 db test.db }
     sql1 {
@@ -288,21 +295,19 @@ do_multiclient_test tn {
       INSERT INTO t1 VALUES('hello');
     }
 
-    forcecopy test.db test.db2
-    forcecopy test.db-wal test.db2-wal
-    forcecopy test.db-shm test.db2-shm
+    copy_to_test2 $bZeroShm
     
     code1 { db close }
   } {}
 
-  do_test 5.1 {
+  do_test $TN.5.1 {
     code2 { sqlite3 db2 file:test.db2?readonly_shm=1 }
     sql2 {
       SELECT * FROM t1;
     }
   } {hello world ! world hello}
 
-  do_test 5.2 {
+  do_test $TN.5.2 {
     code1 {
       proc handle_read {op args} {
         if {$op=="xRead" && [file tail [lindex $args 0]]=="test.db2-wal"} {
@@ -325,7 +330,7 @@ do_multiclient_test tn {
     code1 { set ::res2 }
   } {hello world ! world hello}
 
-  do_test 5.3 {
+  do_test $TN.5.3 {
     code1 { db close }
     code1 { tvfs delete }
   } {}
@@ -337,7 +342,7 @@ do_multiclient_test tn {
   catch { code2 { db2 close } }
   catch { code3 { db3 close } }
 
-  do_test 6.1 {
+  do_test $TN.6.1 {
     code1 { forcedelete test.db }
     code1 { sqlite3 db test.db }
     sql1 {
@@ -350,14 +355,12 @@ do_multiclient_test tn {
       INSERT INTO t1 VALUES('hello');
     }
 
-    forcecopy test.db test.db2
-    forcecopy test.db-wal test.db2-wal
-    forcecopy test.db-shm test.db2-shm
+    copy_to_test2 $bZeroShm
     
     code1 { db close }
   } {}
 
-  do_test 6.2 {
+  do_test $TN.6.2 {
     code1 {
       set ::nRem 5
       proc handle_read {op args} {
@@ -380,10 +383,11 @@ do_multiclient_test tn {
     }
   } {hello world ! world hello}
 
-  do_test 6.3 {
+  do_test $TN.6.3 {
     code1 { db close }
     code1 { tvfs delete }
   } {}
 }
+} ;# foreach bZeroShm
 
 finish_test

From 95a05aae679bb79bd697ab082fc6e6b50356f2c5 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Thu, 9 Nov 2017 20:37:37 +0000
Subject: [PATCH 078/488] Add an assert() in the Win32 VFS.

FossilOrigin-Name: 22e58330461736ca22d6f4d7eab897a3597de2e7434a6f4a474f0f0d7f964281
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/os_win.c  |  1 +
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index e214aee04c..68233db669 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Enhance\swalro2.test\sto\sbetter\sensure\sthat\sreadonly_shm\sclients\sare\snot\susing\ninvalid\s*-shm\sfiles.
-D 2017-11-09T20:34:35.406
+C Add\san\sassert()\sin\sthe\sWin32\sVFS.
+D 2017-11-09T20:37:37.876
 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2
@@ -448,7 +448,7 @@ F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
 F src/os_unix.c e87cef0bb894b94d96ee3af210be669549d111c580817d14818101b992640767
-F src/os_win.c cf4099958dcc72a9e36ce161638aa369811012fa1e2592415fdc1fdafdf26c8c
+F src/os_win.c 64bc61821f75b37ca213da93aef84557c8730be6e0ca93943223b5e57fe6e5a3
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
@@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P abef05353554e72f4d08aff562b87ff8530e8537a79e58d831205ea8c46eed07
-R 43c8def15803418ff2b6211daf889aff
-U dan
-Z b374f730e5d2822de9c4c678f7c53c0c
+P ff630b66714b20c09888ead0a45f344d63e0d9a5208867d6266e74f79187076c
+R 9f94324b020c3efbc23aa89a0c6b1ac2
+U mistachkin
+Z b9496fb62e9a99521b89e7f9003a99d8
diff --git a/manifest.uuid b/manifest.uuid
index d156b1b802..da346528d8 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ff630b66714b20c09888ead0a45f344d63e0d9a5208867d6266e74f79187076c
\ No newline at end of file
+22e58330461736ca22d6f4d7eab897a3597de2e7434a6f4a474f0f0d7f964281
\ No newline at end of file
diff --git a/src/os_win.c b/src/os_win.c
index 05ae7e67db..aff48e25e9 100644
--- a/src/os_win.c
+++ b/src/os_win.c
@@ -3885,6 +3885,7 @@ static int winGetShmDmsLockType(
       if( winIsLockConflict(lastErrno) ){
         if( pLockType ) *pLockType = WINSHM_WRLCK;
       }else if( winIsLockMissing(lastErrno) ){
+        assert( bReadOnly );
         if( pLockType ) *pLockType = WINSHM_UNLCK;
       }else{
         return SQLITE_IOERR_READ;

From 7b7f224c6242f032e2e1d84e4aeb78ede1cbcae1 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Thu, 9 Nov 2017 22:23:50 +0000
Subject: [PATCH 079/488] Revise and vastly simplify the Win32 SHM file locking
 semantics, allowing all new tests to pass.

FossilOrigin-Name: d0997b0f5bc9a9869684e39a17a01c430d6383c8b31d6c00ea17a5eac15bc6f0
---
 manifest      |  12 ++---
 manifest.uuid |   2 +-
 src/os_win.c  | 128 ++++----------------------------------------------
 3 files changed, 15 insertions(+), 127 deletions(-)

diff --git a/manifest b/manifest
index 68233db669..bf3dc40e6b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\san\sassert()\sin\sthe\sWin32\sVFS.
-D 2017-11-09T20:37:37.876
+C Revise\sand\svastly\ssimplify\sthe\sWin32\sSHM\sfile\slocking\ssemantics,\sallowing\sall\snew\stests\sto\spass.
+D 2017-11-09T22:23:50.758
 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2
@@ -448,7 +448,7 @@ F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
 F src/os_unix.c e87cef0bb894b94d96ee3af210be669549d111c580817d14818101b992640767
-F src/os_win.c 64bc61821f75b37ca213da93aef84557c8730be6e0ca93943223b5e57fe6e5a3
+F src/os_win.c b40d4f98562048b1d084b71bb08367903b937424427ee7b866902dca7b79cb88
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
@@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ff630b66714b20c09888ead0a45f344d63e0d9a5208867d6266e74f79187076c
-R 9f94324b020c3efbc23aa89a0c6b1ac2
+P 22e58330461736ca22d6f4d7eab897a3597de2e7434a6f4a474f0f0d7f964281
+R 0c5d22f8af57ea73c927075712f52c29
 U mistachkin
-Z b9496fb62e9a99521b89e7f9003a99d8
+Z e19a1ec884226a6dec0cb1fab52ceb52
diff --git a/manifest.uuid b/manifest.uuid
index da346528d8..2503e60869 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-22e58330461736ca22d6f4d7eab897a3597de2e7434a6f4a474f0f0d7f964281
\ No newline at end of file
+d0997b0f5bc9a9869684e39a17a01c430d6383c8b31d6c00ea17a5eac15bc6f0
\ No newline at end of file
diff --git a/src/os_win.c b/src/os_win.c
index aff48e25e9..975df6154b 100644
--- a/src/os_win.c
+++ b/src/os_win.c
@@ -2098,27 +2098,6 @@ static int winLogErrorAtLine(
 static int winIoerrRetry = SQLITE_WIN32_IOERR_RETRY;
 static int winIoerrRetryDelay = SQLITE_WIN32_IOERR_RETRY_DELAY;
 
-/*
-** The "winIsLockConflict" macro is used to determine if a particular I/O
-** error code is due to a file locking conflict.  It must accept the error
-** code DWORD as its only argument.
-*/
-#if !defined(winIsLockConflict)
-#define winIsLockConflict(a) (((a)==NO_ERROR)                   || \
-                              ((a)==ERROR_LOCK_VIOLATION)       || \
-                              ((a)==ERROR_IO_PENDING))
-#endif
-
-/*
-** The "winIsLockMissing" macro is used to determine if a particular I/O
-** error code is due to being unable to obtain a file lock because all or
-** part of the range requested within the file is missing.  It must accept
-** the error code DWORD as its only argument.
-*/
-#if !defined(winIsLockMissing)
-#define winIsLockMissing(a) (((a)==ERROR_HANDLE_EOF))
-#endif
-
 /*
 ** The "winIoerrCanRetry1" macro is used to determine if a particular I/O
 ** error code obtained via GetLastError() is eligible to be retried.  It
@@ -3844,65 +3823,6 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){
   }
 }
 
-/*
-** Query the status of the DMS lock for the specified file.  Returns
-** SQLITE_OK upon success.  Upon success, the integer pointed to by
-** the pLockType argument will be set to the lock type held by the
-** other process, as follows:
-**
-**       WINSHM_UNLCK -- No locks are held on the DMS.
-**       WINSHM_RDLCK -- A SHARED lock is held on the DMS.
-**       WINSHM_WRLCK -- An EXCLUSIVE lock is held on the DMS.
-*/
-static int winGetShmDmsLockType(
-  winFile *pFile, /* File handle object */
-  int bReadOnly,  /* Non-zero if the SHM was opened read-only */
-  int *pLockType  /* WINSHM_UNLCK, WINSHM_RDLCK, or WINSHM_WRLCK */
-){
-#if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_OVERLAPPED)
-  OVERLAPPED overlapped; /* The offset for ReadFile/WriteFile. */
-#endif
-  LPVOID pOverlapped = 0;
-  sqlite3_int64 offset = WIN_SHM_DMS;
-  BYTE notUsed1 = 0;
-  DWORD notUsed2 = 0;
-
-#if SQLITE_OS_WINCE || defined(SQLITE_WIN32_NO_OVERLAPPED)
-  if( winSeekFile(pFile, offset) ){
-    return SQLITE_IOERR_SEEK;
-  }
-#else
-  memset(&overlapped, 0, sizeof(OVERLAPPED));
-  overlapped.Offset = (LONG)(offset & 0xffffffff);
-  overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff);
-  pOverlapped = &overlapped;
-#endif
-  if( bReadOnly ||
-      !osWriteFile(pFile->h, &notUsed1, 1, &notUsed2, pOverlapped) ){
-    DWORD lastErrno = bReadOnly ? NO_ERROR : osGetLastError();
-    if( !osReadFile(pFile->h, &notUsed1, 1, &notUsed2, pOverlapped) ){
-      lastErrno = osGetLastError();
-      if( winIsLockConflict(lastErrno) ){
-        if( pLockType ) *pLockType = WINSHM_WRLCK;
-      }else if( winIsLockMissing(lastErrno) ){
-        assert( bReadOnly );
-        if( pLockType ) *pLockType = WINSHM_UNLCK;
-      }else{
-        return SQLITE_IOERR_READ;
-      }
-    }else{
-      if( winIsLockConflict(lastErrno) ){
-        if( pLockType ) *pLockType = WINSHM_RDLCK;
-      }else{
-        return SQLITE_IOERR_WRITE;
-      }
-    }
-  }else{
-    if( pLockType ) *pLockType = WINSHM_UNLCK;
-  }
-  return SQLITE_OK;
-}
-
 /*
 ** The DMS lock has not yet been taken on shm file pShmNode. Attempt to
 ** take it now. Return SQLITE_OK if successful, or an SQLite error
@@ -3913,53 +3833,21 @@ static int winGetShmDmsLockType(
 ** SQLITE_READONLY_CANTINIT and set pShmNode->isUnlocked=1.
 */
 static int winLockSharedMemory(winShmNode *pShmNode){
-  int lockType;
-  int rc = SQLITE_OK;
-
-  /* Use ReadFile/WriteFile to determine the locks other processes are
-  ** holding on the DMS byte. If it indicates that another process is
-  ** holding a SHARED lock, then this process may also take a SHARED
-  ** lock and proceed with opening the *-shm file.
-  **
-  ** Or, if no other process is holding any lock, then this process
-  ** is the first to open it. In this case take an EXCLUSIVE lock on the
-  ** DMS byte and truncate the *-shm file to zero bytes in size. Then
-  ** downgrade to a SHARED lock on the DMS byte.
-  **
-  ** If another process is holding an EXCLUSIVE lock on the DMS byte,
-  ** return SQLITE_BUSY to the caller (it will try again). An earlier
-  ** version of this code attempted the SHARED lock at this point. But
-  ** this introduced a subtle race condition: if the process holding
-  ** EXCLUSIVE failed just before truncating the *-shm file, then this
-  ** process might open and use the *-shm file without truncating it.
-  ** And if the *-shm file has been corrupted by a power failure or
-  ** system crash, the database itself may also become corrupt.  */
-  if( winGetShmDmsLockType(&pShmNode->hFile, pShmNode->isReadonly,
-                           &lockType)!=SQLITE_OK ){
-    rc = SQLITE_IOERR_LOCK;
-  }else if( lockType==WINSHM_UNLCK ){
+  int rc = winShmSystemLock(pShmNode, WINSHM_WRLCK, WIN_SHM_DMS, 1);
+  if( rc==SQLITE_OK ){
     if( pShmNode->isReadonly ){
       pShmNode->isUnlocked = 1;
-      rc = SQLITE_READONLY_CANTINIT;
-    }else{
       winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1);
-      rc = winShmSystemLock(pShmNode, WINSHM_WRLCK, WIN_SHM_DMS, 1);
-      if( rc==SQLITE_OK && winTruncate((sqlite3_file*)&pShmNode->hFile, 0) ){
-        rc = winLogError(SQLITE_IOERR_SHMOPEN, osGetLastError(),
+      return SQLITE_READONLY_CANTINIT;
+    }else if( winTruncate((sqlite3_file*)&pShmNode->hFile, 0) ){
+      winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1);
+      return winLogError(SQLITE_IOERR_SHMOPEN, osGetLastError(),
                          "winLockSharedMemory", pShmNode->zFilename);
-      }
     }
-  }else if( lockType==WINSHM_WRLCK ){
-    rc = SQLITE_BUSY;
   }
 
-  if( rc==SQLITE_OK ){
-    assert( lockType==WINSHM_UNLCK || lockType==WINSHM_RDLCK );
-    winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1);
-    rc = winShmSystemLock(pShmNode, WINSHM_RDLCK, WIN_SHM_DMS, 1);
-  }
-
-  return rc;
+  winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1);
+  return winShmSystemLock(pShmNode, WINSHM_RDLCK, WIN_SHM_DMS, 1);
 }
 
 /*

From 10813713baec7d78f700c19e51107a19bb698d98 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Thu, 9 Nov 2017 22:25:58 +0000
Subject: [PATCH 080/488] Cleanup superfluous whitespace changes.

FossilOrigin-Name: a2908e2c88f7a30638a7e791fc7ad0325b663097c12cecd1f4726b0d60a9a3ed
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/os_win.c  | 12 ++++++------
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/manifest b/manifest
index bf3dc40e6b..125e07f68d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Revise\sand\svastly\ssimplify\sthe\sWin32\sSHM\sfile\slocking\ssemantics,\sallowing\sall\snew\stests\sto\spass.
-D 2017-11-09T22:23:50.758
+C Cleanup\ssuperfluous\swhitespace\schanges.
+D 2017-11-09T22:25:58.319
 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2
@@ -448,7 +448,7 @@ F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
 F src/os_unix.c e87cef0bb894b94d96ee3af210be669549d111c580817d14818101b992640767
-F src/os_win.c b40d4f98562048b1d084b71bb08367903b937424427ee7b866902dca7b79cb88
+F src/os_win.c 670d296c06f8bfce01887baacf52ec19ec2d682b169a6a0ad8b85a830c2b4db0
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
@@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 22e58330461736ca22d6f4d7eab897a3597de2e7434a6f4a474f0f0d7f964281
-R 0c5d22f8af57ea73c927075712f52c29
+P d0997b0f5bc9a9869684e39a17a01c430d6383c8b31d6c00ea17a5eac15bc6f0
+R d62fe3d1eed9309359251a90225ada34
 U mistachkin
-Z e19a1ec884226a6dec0cb1fab52ceb52
+Z 112940bd575d10def1c3f99e15024bb0
diff --git a/manifest.uuid b/manifest.uuid
index 2503e60869..1dbf5a7d5c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d0997b0f5bc9a9869684e39a17a01c430d6383c8b31d6c00ea17a5eac15bc6f0
\ No newline at end of file
+a2908e2c88f7a30638a7e791fc7ad0325b663097c12cecd1f4726b0d60a9a3ed
\ No newline at end of file
diff --git a/src/os_win.c b/src/os_win.c
index 975df6154b..2ee4ad1acf 100644
--- a/src/os_win.c
+++ b/src/os_win.c
@@ -3858,12 +3858,12 @@ static int winLockSharedMemory(winShmNode *pShmNode){
 ** the file must be truncated to zero length or have its header cleared.
 */
 static int winOpenSharedMemory(winFile *pDbFd){
-  struct winShm *p;         /* The connection to be opened */
-  winShmNode *pShmNode = 0; /* The underlying mmapped file */
-  int rc = SQLITE_OK;       /* Result code */
-  int rc2 = SQLITE_ERROR;   /* winOpen result code */
-  winShmNode *pNew;         /* Newly allocated winShmNode */
-  int nName;                /* Size of zName in bytes */
+  struct winShm *p;                  /* The connection to be opened */
+  winShmNode *pShmNode = 0;          /* The underlying mmapped file */
+  int rc = SQLITE_OK;                /* Result code */
+  int rc2 = SQLITE_ERROR;            /* winOpen result code */
+  winShmNode *pNew;                  /* Newly allocated winShmNode */
+  int nName;                         /* Size of zName in bytes */
 
   assert( pDbFd->pShm==0 );    /* Not previously opened */
 

From 0e026f403b9900351cfa11c67f9a4682f2911381 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Thu, 9 Nov 2017 23:24:29 +0000
Subject: [PATCH 081/488] Avoid superfluous SHM unlock call in the Win32 VFS.

FossilOrigin-Name: 5a384be6979b783d1f3af2ac6307e7e731c415d052f9405f04c0216f59414633
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/os_win.c  |  6 +++++-
 3 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 125e07f68d..453bed54e5 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Cleanup\ssuperfluous\swhitespace\schanges.
-D 2017-11-09T22:25:58.319
+C Avoid\ssuperfluous\sSHM\sunlock\scall\sin\sthe\sWin32\sVFS.
+D 2017-11-09T23:24:29.716
 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2
@@ -448,7 +448,7 @@ F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
 F src/os_unix.c e87cef0bb894b94d96ee3af210be669549d111c580817d14818101b992640767
-F src/os_win.c 670d296c06f8bfce01887baacf52ec19ec2d682b169a6a0ad8b85a830c2b4db0
+F src/os_win.c 7f36120492e4a23c48d1dd685edf29ae459c6d555660c61f1323cea3e5a1191d
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
@@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d0997b0f5bc9a9869684e39a17a01c430d6383c8b31d6c00ea17a5eac15bc6f0
-R d62fe3d1eed9309359251a90225ada34
+P a2908e2c88f7a30638a7e791fc7ad0325b663097c12cecd1f4726b0d60a9a3ed
+R 9d6be609689f3743af6adbbc5cf401b2
 U mistachkin
-Z 112940bd575d10def1c3f99e15024bb0
+Z 39f42e393c5183ce2049496b416ae681
diff --git a/manifest.uuid b/manifest.uuid
index 1dbf5a7d5c..ba222182ca 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a2908e2c88f7a30638a7e791fc7ad0325b663097c12cecd1f4726b0d60a9a3ed
\ No newline at end of file
+5a384be6979b783d1f3af2ac6307e7e731c415d052f9405f04c0216f59414633
\ No newline at end of file
diff --git a/src/os_win.c b/src/os_win.c
index 2ee4ad1acf..086bbf90aa 100644
--- a/src/os_win.c
+++ b/src/os_win.c
@@ -3834,6 +3834,7 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){
 */
 static int winLockSharedMemory(winShmNode *pShmNode){
   int rc = winShmSystemLock(pShmNode, WINSHM_WRLCK, WIN_SHM_DMS, 1);
+
   if( rc==SQLITE_OK ){
     if( pShmNode->isReadonly ){
       pShmNode->isUnlocked = 1;
@@ -3846,7 +3847,10 @@ static int winLockSharedMemory(winShmNode *pShmNode){
     }
   }
 
-  winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1);
+  if( rc==SQLITE_OK ){
+    winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1);
+  }
+
   return winShmSystemLock(pShmNode, WINSHM_RDLCK, WIN_SHM_DMS, 1);
 }
 

From 85bc6df2f197c3f5f040b8c17c567b1cf5b20e12 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 10 Nov 2017 20:00:50 +0000
Subject: [PATCH 082/488] Improved comments and variable names in the read-only
 WAL logic.

FossilOrigin-Name: d3c25740eec9a2a41c29e6e488fcf6587c1fb821147a442c29439b25a92154a5
---
 manifest      |  14 +++---
 manifest.uuid |   2 +-
 src/wal.c     | 131 ++++++++++++++++++++++++++++++--------------------
 3 files changed, 87 insertions(+), 60 deletions(-)

diff --git a/manifest b/manifest
index 453bed54e5..c2392bfdac 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Avoid\ssuperfluous\sSHM\sunlock\scall\sin\sthe\sWin32\sVFS.
-D 2017-11-09T23:24:29.716
+C Improved\scomments\sand\svariable\snames\sin\sthe\sread-only\sWAL\slogic.
+D 2017-11-10T20:00:50.239
 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2
@@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c 6903d391b6c224e45910795d66f97151428662c5f459d3b69997209dbcadc6f5
+F src/wal.c 47f8a4493e077b312399b62fd36fd4e8b70535dcd8d9a5caf18b8be019fcf420
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f
 F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4
@@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a2908e2c88f7a30638a7e791fc7ad0325b663097c12cecd1f4726b0d60a9a3ed
-R 9d6be609689f3743af6adbbc5cf401b2
-U mistachkin
-Z 39f42e393c5183ce2049496b416ae681
+P 5a384be6979b783d1f3af2ac6307e7e731c415d052f9405f04c0216f59414633
+R 4efdec1d3561b29c7744e3cfcf4d5f7b
+U drh
+Z df395ed2f6beb277e98c44e2de48326a
diff --git a/manifest.uuid b/manifest.uuid
index ba222182ca..354097e8fa 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5a384be6979b783d1f3af2ac6307e7e731c415d052f9405f04c0216f59414633
\ No newline at end of file
+d3c25740eec9a2a41c29e6e488fcf6587c1fb821147a442c29439b25a92154a5
\ No newline at end of file
diff --git a/src/wal.c b/src/wal.c
index afd0e3f7b6..aff0e97fa2 100644
--- a/src/wal.c
+++ b/src/wal.c
@@ -455,7 +455,7 @@ struct Wal {
   u8 truncateOnCommit;       /* True to truncate WAL file on commit */
   u8 syncHeader;             /* Fsync the WAL header if true */
   u8 padToSectorBoundary;    /* Pad transactions out to the next sector */
-  u8 bUnlocked;
+  u8 bShmUnreliable;         /* SHM content is read-only and unreliable */
   WalIndexHdr hdr;           /* Wal-index header for current transaction */
   u32 minFrame;              /* Ignore wal frames before this one */
   u32 iReCksum;              /* On commit, recalculate checksums from here */
@@ -1271,7 +1271,7 @@ recovery_error:
 ** Close an open wal-index.
 */
 static void walIndexClose(Wal *pWal, int isDelete){
-  if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE || pWal->bUnlocked ){
+  if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE || pWal->bShmUnreliable ){
     int i;
     for(i=0; i<pWal->nWiData; i++){
       sqlite3_free((void *)pWal->apWiData[i]);
@@ -2099,14 +2099,22 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){
   */
   assert( pChanged );
   rc = walIndexPage(pWal, 0, &page0);
-  if( rc==SQLITE_READONLY_CANTINIT ){
-    assert( page0==0 && pWal->writeLock==0 );
-    pWal->bUnlocked = 1;
-    pWal->exclusiveMode = WAL_HEAPMEMORY_MODE;
-    *pChanged = 1;
-  }else
   if( rc!=SQLITE_OK ){
-    return rc;
+    assert( rc!=SQLITE_READONLY ); /* READONLY changed to OK in walIndexPage */
+    if( rc==SQLITE_READONLY_CANTINIT ){
+      /* The SQLITE_READONLY_CANTINIT return means that the shared-memory
+      ** was openable but is not writable, and this thread is unable to
+      ** confirm that another write-capable connection has the shared-memory
+      ** open, and hence the content of the shared-memory is unreliable,
+      ** since the shared-memory might be inconsistent with the WAL file
+      ** and there is no writer on hand to fix it. */
+      assert( page0==0 && pWal->writeLock==0 );
+      pWal->bShmUnreliable = 1;
+      pWal->exclusiveMode = WAL_HEAPMEMORY_MODE;
+      *pChanged = 1;
+    }else{
+      return rc; /* Any other non-OK return is just an error */
+    }
   };
   assert( page0 || pWal->writeLock==0 );
 
@@ -2122,7 +2130,7 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){
   */
   assert( badHdr==0 || pWal->writeLock==0 );
   if( badHdr ){
-    if( pWal->bUnlocked==0 && (pWal->readOnly & WAL_SHM_RDONLY) ){
+    if( pWal->bShmUnreliable==0 && (pWal->readOnly & WAL_SHM_RDONLY) ){
       if( SQLITE_OK==(rc = walLockShared(pWal, WAL_WRITE_LOCK)) ){
         walUnlockShared(pWal, WAL_WRITE_LOCK);
         rc = SQLITE_READONLY_RECOVERY;
@@ -2152,10 +2160,10 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){
   if( badHdr==0 && pWal->hdr.iVersion!=WALINDEX_MAX_VERSION ){
     rc = SQLITE_CANTOPEN_BKPT;
   }
-  if( pWal->bUnlocked ){
+  if( pWal->bShmUnreliable ){
     if( rc!=SQLITE_OK ){
       walIndexClose(pWal, 0);
-      pWal->bUnlocked = 0;
+      pWal->bShmUnreliable = 0;
       assert( pWal->nWiData>0 && pWal->apWiData[0]==0 );
       if( rc==SQLITE_IOERR_SHORT_READ ) rc = WAL_RETRY;
     }
@@ -2166,11 +2174,21 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){
 }
 
 /*
-** Open an "unlocked" transaction. An unlocked transaction is a read 
-** transaction used by a read-only client in cases where the *-shm
-** file cannot be mapped and its contents cannot be trusted. It is
-** assumed that the *-wal file has been read and that a wal-index 
-** constructed in heap memory is currently available in Wal.apWiData[].
+** Open a transaction in a connection where the shared-memory is read-only
+** and where we cannot verify that there is a separate write-capable connection
+** on hand to keep the shared-memory up-to-date with the WAL file.
+**
+** This can happen, for example, when the shared-memory is implemented by
+** memory-mapping a *-shm file, where a prior writer has shut down and
+** left the *-shm file on disk, and now the present connection is trying
+** to use that database but lacks write permission on the *-shm file.
+** Other scenarios are also possible, depending on the VFS implementation.
+**
+** Precondition:
+**
+**    The *-wal file has been read and an appropriate wal-index has been
+**    constructed in pWal->apWiData[] using heap memory instead of shared
+**    memory. 
 **
 ** If this function returns SQLITE_OK, then the read transaction has
 ** been successfully opened. In this case output variable (*pChanged) 
@@ -2182,7 +2200,7 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){
 **
 ** If an error occurs, an SQLite error code is returned.
 */
-static int walBeginUnlocked(Wal *pWal, int *pChanged){
+static int walBeginShmUnreliable(Wal *pWal, int *pChanged){
   i64 szWal;                      /* Size of wal file on disk in bytes */
   i64 iOffset;                    /* Current offset when reading wal file */
   u8 aBuf[WAL_HDRSIZE];           /* Buffer to load WAL header into */
@@ -2193,50 +2211,59 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){
   int rc;                         /* Return code */
   u32 aSaveCksum[2];              /* Saved copy of pWal->hdr.aFrameCksum */
 
-  assert( pWal->bUnlocked );
+  assert( pWal->bShmUnreliable );
   assert( pWal->readOnly & WAL_SHM_RDONLY );
   assert( pWal->nWiData>0 && pWal->apWiData[0] );
 
   /* Take WAL_READ_LOCK(0). This has the effect of preventing any
-  ** live clients from running a checkpoint, but does not stop them
+  ** writers from running a checkpoint, but does not stop them
   ** from running recovery.  */
   rc = walLockShared(pWal, WAL_READ_LOCK(0));
   if( rc!=SQLITE_OK ){
     if( rc==SQLITE_BUSY ) rc = WAL_RETRY;
-    goto begin_unlocked_out;
+    goto begin_unreliable_shm_out;
   }
   pWal->readLock = 0;
 
-  /* Try to map the *-shm file again. If it succeeds this time, then 
-  ** a non-readonly_shm connection has already connected to the database.
-  ** In this case, start over with opening the transaction.
+  /* Check to see if a separate writer has attached to the shared-memory area,
+  ** thus making the shared-memory "reliable" again.  Do this by invoking
+  ** the xShmMap() routine of the VFS and looking to see if the return
+  ** is SQLITE_READONLY instead of SQLITE_READONLY_CANTINIT.
   **
-  ** The *-shm file was opened read-only, so sqlite3OsShmMap() can never
-  ** return SQLITE_OK here, as that would imply that it had established
-  ** a read/write mapping.  A return of SQLITE_READONLY means success - that
-  ** a mapping has been established to a shared-memory segment that is actively
-  ** maintained by a writer.  SQLITE_READONLY_CANTINIT means that all
-  ** all connections to the -shm file are read-only and hence the content
-  ** of the -shm file might be out-of-date.
-  ** 
-  ** The WAL_READ_LOCK(0) lock held by this client prevents a checkpoint
-  ** from taking place. But it does not prevent the wal from being wrapped
-  ** if a checkpoint has already taken place. This means that if another
-  ** client is connected at this point, it may have already checkpointed 
-  ** the entire wal. In that case it would not be safe to continue with
-  ** the unlocked transaction, as the other client may overwrite wal 
-  ** frames that this client is still using.  */
+  ** Once sqlite3OsShmMap() has been called for a file and has returned
+  ** any SQLITE_READONLY value, it must SQLITE_READONLY or
+  ** SQLITE_READONLY_CANTINIT or some error for all subsequent invocations,
+  ** until sqlite3OsShmUnmap() has been called.  This is a requirement
+  ** on the VFS implementation.
+  **
+  ** If the shared-memory is now "reliable" return WAL_RETRY, which will
+  ** cause the heap-memory WAL-index to be discarded and the actual
+  ** shared memory to be used in its place.
+  */
   rc = sqlite3OsShmMap(pWal->pDbFd, 0, WALINDEX_PGSZ, 0, &pDummy);
   assert( rc!=SQLITE_OK ); /* SQLITE_OK not possible for read-only connection */
   if( rc!=SQLITE_READONLY_CANTINIT ){
     rc = (rc==SQLITE_READONLY ? WAL_RETRY : rc);
-    goto begin_unlocked_out;
+    goto begin_unreliable_shm_out;
   }
 
+  /* Reach this point only if the real shared-memory is still unreliable.
+  ** Assume the in-memory WAL-index substitute is correct and load it
+  ** into pWal->hdr.
+  */
   memcpy(&pWal->hdr, (void*)walIndexHdr(pWal), sizeof(WalIndexHdr));
+
+  /* The WAL_READ_LOCK(0) lock held by this client prevents a checkpoint
+  ** from taking place. But it does not prevent the wal from being wrapped
+  ** if a checkpoint has already taken place. This means that if another
+  ** client is connected at this point, it may have already checkpointed 
+  ** the entire wal. In that case it would not be safe to continue with
+  ** the this transaction, as the other client may overwrite wal 
+  ** frames that this client is still using.
+  */
   rc = sqlite3OsFileSize(pWal->pWalFd, &szWal);
   if( rc!=SQLITE_OK ){
-    goto begin_unlocked_out;
+    goto begin_unreliable_shm_out;
   }
   if( szWal<WAL_HDRSIZE ){
     /* If the wal file is too small to contain a wal-header and the
@@ -2247,17 +2274,17 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){
     ** since this client's last read transaction.  */
     *pChanged = 1;
     rc = (pWal->hdr.mxFrame==0 ? SQLITE_OK : WAL_RETRY);
-    goto begin_unlocked_out;
+    goto begin_unreliable_shm_out;
   }
 
   /* Check the salt keys at the start of the wal file still match. */
   rc = sqlite3OsRead(pWal->pWalFd, aBuf, WAL_HDRSIZE, 0);
   if( rc!=SQLITE_OK ){
-    goto begin_unlocked_out;
+    goto begin_unreliable_shm_out;
   }
   if( memcmp(&pWal->hdr.aSalt, &aBuf[16], 8) ){
     rc = WAL_RETRY;
-    goto begin_unlocked_out;
+    goto begin_unreliable_shm_out;
   }
 
   /* Allocate a buffer to read frames into */
@@ -2265,7 +2292,7 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){
   aFrame = (u8 *)sqlite3_malloc64(szFrame);
   if( aFrame==0 ){
     rc = SQLITE_NOMEM_BKPT;
-    goto begin_unlocked_out;
+    goto begin_unreliable_shm_out;
   }
   aData = &aFrame[WAL_FRAME_HDRSIZE];
 
@@ -2298,7 +2325,7 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){
   pWal->hdr.aFrameCksum[0] = aSaveCksum[0];
   pWal->hdr.aFrameCksum[1] = aSaveCksum[1];
 
- begin_unlocked_out:
+ begin_unreliable_shm_out:
   sqlite3_free(aFrame);
   if( rc!=SQLITE_OK ){
     int i;
@@ -2306,7 +2333,7 @@ static int walBeginUnlocked(Wal *pWal, int *pChanged){
       sqlite3_free((void*)pWal->apWiData[i]);
       pWal->apWiData[i] = 0;
     }
-    pWal->bUnlocked = 0;
+    pWal->bShmUnreliable = 0;
     sqlite3WalEndReadTransaction(pWal);
     *pChanged = 1;
   }
@@ -2402,7 +2429,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){
 
   if( !useWal ){
     assert( rc==SQLITE_OK );
-    if( pWal->bUnlocked==0 ){
+    if( pWal->bShmUnreliable==0 ){
       rc = walIndexReadHdr(pWal, pChanged);
     }
     if( rc==SQLITE_BUSY ){
@@ -2433,8 +2460,8 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){
     if( rc!=SQLITE_OK ){
       return rc;
     }
-    else if( pWal->bUnlocked ){
-      return walBeginUnlocked(pWal, pChanged);
+    else if( pWal->bShmUnreliable ){
+      return walBeginShmUnreliable(pWal, pChanged);
     }
   }
 
@@ -2789,7 +2816,7 @@ int sqlite3WalFindFrame(
   ** then the WAL is ignored by the reader so return early, as if the 
   ** WAL were empty.
   */
-  if( iLast==0 || (pWal->readLock==0 && pWal->bUnlocked==0) ){
+  if( iLast==0 || (pWal->readLock==0 && pWal->bShmUnreliable==0) ){
     *piRead = 0;
     return SQLITE_OK;
   }
@@ -2852,7 +2879,7 @@ int sqlite3WalFindFrame(
   {
     u32 iRead2 = 0;
     u32 iTest;
-    assert( pWal->bUnlocked || pWal->minFrame>0 );
+    assert( pWal->bShmUnreliable || pWal->minFrame>0 );
     for(iTest=iLast; iTest>=pWal->minFrame && iTest>0; iTest--){
       if( walFramePgno(pWal, iTest)==pgno ){
         iRead2 = iTest;

From 870655bb9e7ef261995c2f3dc6fa5f6c9b115997 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 11 Nov 2017 13:30:44 +0000
Subject: [PATCH 083/488] Further comment improvements in wal.c.  No code
 changes.

FossilOrigin-Name: 346388007de585083dc67ad865b91db7c7d7b78c10a06f8bb7c48767c326c47e
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/wal.c     | 35 ++++++++++++++++++++---------------
 3 files changed, 27 insertions(+), 22 deletions(-)

diff --git a/manifest b/manifest
index c2392bfdac..e067bdd554 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improved\scomments\sand\svariable\snames\sin\sthe\sread-only\sWAL\slogic.
-D 2017-11-10T20:00:50.239
+C Further\scomment\simprovements\sin\swal.c.\s\sNo\scode\schanges.
+D 2017-11-11T13:30:44.595
 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2
@@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c 47f8a4493e077b312399b62fd36fd4e8b70535dcd8d9a5caf18b8be019fcf420
+F src/wal.c 0e19d4fbb52085697509f9407176e732d440f372f49ca08510a65d1aa976bd07
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f
 F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4
@@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5a384be6979b783d1f3af2ac6307e7e731c415d052f9405f04c0216f59414633
-R 4efdec1d3561b29c7744e3cfcf4d5f7b
+P d3c25740eec9a2a41c29e6e488fcf6587c1fb821147a442c29439b25a92154a5
+R cc050d490699d35569ec56a8515bac08
 U drh
-Z df395ed2f6beb277e98c44e2de48326a
+Z 1e870a391397f9beedb8e100937bac44
diff --git a/manifest.uuid b/manifest.uuid
index 354097e8fa..3a4ec03265 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d3c25740eec9a2a41c29e6e488fcf6587c1fb821147a442c29439b25a92154a5
\ No newline at end of file
+346388007de585083dc67ad865b91db7c7d7b78c10a06f8bb7c48767c326c47e
\ No newline at end of file
diff --git a/src/wal.c b/src/wal.c
index aff0e97fa2..87fbda12d6 100644
--- a/src/wal.c
+++ b/src/wal.c
@@ -2230,16 +2230,23 @@ static int walBeginShmUnreliable(Wal *pWal, int *pChanged){
   ** the xShmMap() routine of the VFS and looking to see if the return
   ** is SQLITE_READONLY instead of SQLITE_READONLY_CANTINIT.
   **
-  ** Once sqlite3OsShmMap() has been called for a file and has returned
-  ** any SQLITE_READONLY value, it must SQLITE_READONLY or
-  ** SQLITE_READONLY_CANTINIT or some error for all subsequent invocations,
-  ** until sqlite3OsShmUnmap() has been called.  This is a requirement
-  ** on the VFS implementation.
-  **
   ** If the shared-memory is now "reliable" return WAL_RETRY, which will
   ** cause the heap-memory WAL-index to be discarded and the actual
   ** shared memory to be used in its place.
-  */
+  **
+  ** This step is important because, even though this connection is holding
+  ** the WAL_READ_LOCK(0) which prevents a checkpoint, a writer might
+  ** have already checkpointed the WAL file and, while the current
+  ** is active, wrap the WAL and start overwriting frames that this
+  ** process wants to use.
+  **
+  ** Once sqlite3OsShmMap() has been called for an sqlite3_file and has
+  ** returned any SQLITE_READONLY value, it must return only SQLITE_READONLY
+  ** or SQLITE_READONLY_CANTINIT or some error for all subsequent invocations,
+  ** even if some external agent does a "chmod" to make the shared-memory
+  ** writable by us, until sqlite3OsShmUnmap() has been called.
+  ** This is a requirement on the VFS implementation.
+   */
   rc = sqlite3OsShmMap(pWal->pDbFd, 0, WALINDEX_PGSZ, 0, &pDummy);
   assert( rc!=SQLITE_OK ); /* SQLITE_OK not possible for read-only connection */
   if( rc!=SQLITE_READONLY_CANTINIT ){
@@ -2247,19 +2254,14 @@ static int walBeginShmUnreliable(Wal *pWal, int *pChanged){
     goto begin_unreliable_shm_out;
   }
 
-  /* Reach this point only if the real shared-memory is still unreliable.
+  /* We reach this point only if the real shared-memory is still unreliable.
   ** Assume the in-memory WAL-index substitute is correct and load it
   ** into pWal->hdr.
   */
   memcpy(&pWal->hdr, (void*)walIndexHdr(pWal), sizeof(WalIndexHdr));
 
-  /* The WAL_READ_LOCK(0) lock held by this client prevents a checkpoint
-  ** from taking place. But it does not prevent the wal from being wrapped
-  ** if a checkpoint has already taken place. This means that if another
-  ** client is connected at this point, it may have already checkpointed 
-  ** the entire wal. In that case it would not be safe to continue with
-  ** the this transaction, as the other client may overwrite wal 
-  ** frames that this client is still using.
+  /* Make sure some writer hasn't come in and changed the WAL file out
+  ** from under us, then disconnected, while we were not looking.
   */
   rc = sqlite3OsFileSize(pWal->pWalFd, &szWal);
   if( rc!=SQLITE_OK ){
@@ -2283,6 +2285,9 @@ static int walBeginShmUnreliable(Wal *pWal, int *pChanged){
     goto begin_unreliable_shm_out;
   }
   if( memcmp(&pWal->hdr.aSalt, &aBuf[16], 8) ){
+    /* Some writer has wrapped the WAL file while we were not looking.
+    ** Return WAL_RETRY which will cause the in-memory WAL-index to be
+    ** rebuilt. */
     rc = WAL_RETRY;
     goto begin_unreliable_shm_out;
   }

From c05a063c680af17386b221b977bc4eda1cf56e2b Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 11 Nov 2017 20:11:01 +0000
Subject: [PATCH 084/488] In wal.c: improved comments, new assert() and
 testcase() macros, and replace some magic numbers with appropriate symbolic
 constants.

FossilOrigin-Name: 13ec8a77a47aa471af587459f4094da0d06674c5960f0d34777bcb3d38bc413b
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/wal.c     | 33 +++++++++++++++++++++++----------
 3 files changed, 30 insertions(+), 17 deletions(-)

diff --git a/manifest b/manifest
index e067bdd554..2fbd04081b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Further\scomment\simprovements\sin\swal.c.\s\sNo\scode\schanges.
-D 2017-11-11T13:30:44.595
+C In\swal.c:\simproved\scomments,\snew\sassert()\sand\stestcase()\smacros,\sand\nreplace\ssome\smagic\snumbers\swith\sappropriate\ssymbolic\sconstants.
+D 2017-11-11T20:11:01.646
 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2
@@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c 0e19d4fbb52085697509f9407176e732d440f372f49ca08510a65d1aa976bd07
+F src/wal.c 213ddce034c354e640294e77b28097db2a7e10e16c12f6a1158bd7fbcc8e730c
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f
 F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4
@@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d3c25740eec9a2a41c29e6e488fcf6587c1fb821147a442c29439b25a92154a5
-R cc050d490699d35569ec56a8515bac08
+P 346388007de585083dc67ad865b91db7c7d7b78c10a06f8bb7c48767c326c47e
+R cc8d47b78b14e89aa0bda73b44dcaa29
 U drh
-Z 1e870a391397f9beedb8e100937bac44
+Z f012da86d6540243e580ab3866be51e7
diff --git a/manifest.uuid b/manifest.uuid
index 3a4ec03265..cbdfc58fda 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-346388007de585083dc67ad865b91db7c7d7b78c10a06f8bb7c48767c326c47e
\ No newline at end of file
+13ec8a77a47aa471af587459f4094da0d06674c5960f0d34777bcb3d38bc413b
\ No newline at end of file
diff --git a/src/wal.c b/src/wal.c
index 87fbda12d6..6fddf39076 100644
--- a/src/wal.c
+++ b/src/wal.c
@@ -545,6 +545,11 @@ struct WalIterator {
 ** is broken into pages of WALINDEX_PGSZ bytes. Wal-index pages are
 ** numbered from zero.
 **
+** If the wal-index is currently smaller the iPage pages then the size
+** of the wal-index might be increased, but only if it is safe to do
+** so.  It is safe to enlarge the wal-index if pWal->writeLock is true
+** or pWal->exclusiveMode==WAL_HEAPMEMORY_MODE.
+**
 ** If this call is successful, *ppPage is set to point to the wal-index
 ** page and SQLITE_OK is returned. If an error (an OOM or VFS error) occurs,
 ** then an SQLite error code is returned and *ppPage is set to 0.
@@ -576,6 +581,8 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){
       rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, 
           pWal->writeLock, (void volatile **)&pWal->apWiData[iPage]
       );
+      assert( pWal->apWiData[iPage]!=0 || rc!=SQLITE_OK || pWal->writeLock==0 );
+      testcase( pWal->apWiData[iPage]==0 && rc==SQLITE_OK );
       if( (rc&0xff)==SQLITE_READONLY ){
         pWal->readOnly |= WAL_SHM_RDONLY;
         if( rc==SQLITE_READONLY ){
@@ -2108,15 +2115,21 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){
       ** open, and hence the content of the shared-memory is unreliable,
       ** since the shared-memory might be inconsistent with the WAL file
       ** and there is no writer on hand to fix it. */
-      assert( page0==0 && pWal->writeLock==0 );
+      assert( page0==0 );
+      assert( pWal->writeLock==0 );
+      assert( pWal->readOnly & WAL_SHM_RDONLY );
       pWal->bShmUnreliable = 1;
       pWal->exclusiveMode = WAL_HEAPMEMORY_MODE;
       *pChanged = 1;
     }else{
       return rc; /* Any other non-OK return is just an error */
     }
-  };
-  assert( page0 || pWal->writeLock==0 );
+  }else{
+    /* page0 can be NULL if the SHM is zero bytes in size and pWal->writeLock
+    ** is zero, which prevents the SHM from growing */
+    testcase( page0!=0 );
+  }
+  assert( page0!=0 || pWal->writeLock==0 );
 
   /* If the first page of the wal-index has been mapped, try to read the
   ** wal-index header immediately, without holding any lock. This usually
@@ -3661,24 +3674,24 @@ int sqlite3WalExclusiveMode(Wal *pWal, int op){
   assert( pWal->readLock>=0 || (op<=0 && pWal->exclusiveMode==0) );
 
   if( op==0 ){
-    if( pWal->exclusiveMode ){
-      pWal->exclusiveMode = 0;
+    if( pWal->exclusiveMode!=WAL_NORMAL_MODE ){
+      pWal->exclusiveMode = WAL_NORMAL_MODE;
       if( walLockShared(pWal, WAL_READ_LOCK(pWal->readLock))!=SQLITE_OK ){
-        pWal->exclusiveMode = 1;
+        pWal->exclusiveMode = WAL_EXCLUSIVE_MODE;
       }
-      rc = pWal->exclusiveMode==0;
+      rc = pWal->exclusiveMode==WAL_NORMAL_MODE;
     }else{
       /* Already in locking_mode=NORMAL */
       rc = 0;
     }
   }else if( op>0 ){
-    assert( pWal->exclusiveMode==0 );
+    assert( pWal->exclusiveMode==WAL_NORMAL_MODE );
     assert( pWal->readLock>=0 );
     walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock));
-    pWal->exclusiveMode = 1;
+    pWal->exclusiveMode = WAL_EXCLUSIVE_MODE;
     rc = 1;
   }else{
-    rc = pWal->exclusiveMode==0;
+    rc = pWal->exclusiveMode==WAL_NORMAL_MODE;
   }
   return rc;
 }

From 2e9b0923aca34aa5d18a3be65f7e082c45758cc3 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 13 Nov 2017 05:51:37 +0000
Subject: [PATCH 085/488] Remove some branches in walTryBeginRead() that were
 added by check-in [ce5d13c2de] but became unreachable with the addition of
 logic in check-in [18b26843] that enabled read-only clients to parse the WAL
 file into a heap-memory WAL-index, thus guaranteeing that the WAL-index
 header is always available.

FossilOrigin-Name: 9c6b38b9a96c11bdf9db4ea025720a4f49dcb723fa2e2776edc8453bce85c7e3
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/wal.c     | 13 +++++++------
 3 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/manifest b/manifest
index 2fbd04081b..fc71ea63a6 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\swal.c:\simproved\scomments,\snew\sassert()\sand\stestcase()\smacros,\sand\nreplace\ssome\smagic\snumbers\swith\sappropriate\ssymbolic\sconstants.
-D 2017-11-11T20:11:01.646
+C Remove\ssome\sbranches\sin\swalTryBeginRead()\sthat\swere\s\nadded\sby\scheck-in\s[ce5d13c2de]\sbut\sbecame\sunreachable\swith\sthe\saddition\nof\slogic\sin\scheck-in\s[18b26843]\sthat\senabled\sread-only\sclients\sto\sparse\nthe\sWAL\sfile\sinto\sa\sheap-memory\sWAL-index,\sthus\sguaranteeing\sthat\sthe\s\nWAL-index\sheader\sis\salways\savailable.
+D 2017-11-13T05:51:37.627
 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2
@@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c 213ddce034c354e640294e77b28097db2a7e10e16c12f6a1158bd7fbcc8e730c
+F src/wal.c 31ae1a82afccf02e2ff220add690d057dd7e25d9d7ec0b53fe57f2d62a4ba510
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f
 F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4
@@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 346388007de585083dc67ad865b91db7c7d7b78c10a06f8bb7c48767c326c47e
-R cc8d47b78b14e89aa0bda73b44dcaa29
+P 13ec8a77a47aa471af587459f4094da0d06674c5960f0d34777bcb3d38bc413b
+R be755a48b3ce9c3ee36556946fccdf2c
 U drh
-Z f012da86d6540243e580ab3866be51e7
+Z a296aa345abf23e8a36cfa010f746e55
diff --git a/manifest.uuid b/manifest.uuid
index cbdfc58fda..0dbf02629b 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-13ec8a77a47aa471af587459f4094da0d06674c5960f0d34777bcb3d38bc413b
\ No newline at end of file
+9c6b38b9a96c11bdf9db4ea025720a4f49dcb723fa2e2776edc8453bce85c7e3
\ No newline at end of file
diff --git a/src/wal.c b/src/wal.c
index 6fddf39076..382be4e0c8 100644
--- a/src/wal.c
+++ b/src/wal.c
@@ -2418,6 +2418,9 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){
 
   assert( pWal->readLock<0 );     /* Not currently locked */
 
+  /* useWal may only be set for read/write connections */
+  assert( (pWal->readOnly & WAL_SHM_RDONLY)==0 || useWal==0 );
+
   /* Take steps to avoid spinning forever if there is a protocol error.
   **
   ** Circumstances that cause a RETRY should only last for the briefest
@@ -2484,9 +2487,9 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){
   }
 
   assert( pWal->nWiData>0 );
-  assert( pWal->apWiData[0] || (pWal->readOnly & WAL_SHM_RDONLY) );
-  pInfo = pWal->apWiData[0] ? walCkptInfo(pWal) : 0;
-  if( !useWal && (pInfo==0 || pInfo->nBackfill==pWal->hdr.mxFrame)
+  assert( pWal->apWiData[0]!=0 );
+  pInfo = walCkptInfo(pWal);
+  if( !useWal && pInfo->nBackfill==pWal->hdr.mxFrame
 #ifdef SQLITE_ENABLE_SNAPSHOT
    && (pWal->pSnapshot==0 || pWal->hdr.mxFrame==0
      || 0==memcmp(&pWal->hdr, pWal->pSnapshot, sizeof(WalIndexHdr)))
@@ -2498,9 +2501,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){
     rc = walLockShared(pWal, WAL_READ_LOCK(0));
     walShmBarrier(pWal);
     if( rc==SQLITE_OK ){
-      if( pInfo
-       && memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) 
-      ){
+      if( memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){
         /* It is not safe to allow the reader to continue here if frames
         ** may have been appended to the log before READ_LOCK(0) was obtained.
         ** When holding READ_LOCK(0), the reader ignores the entire log file,

From 8b17ac1919afa4f2c73869e30f6938db775ea779 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 14 Nov 2017 03:42:52 +0000
Subject: [PATCH 086/488] Improvement to a comment.  No changes to code.

FossilOrigin-Name: 486949fc03706e0056439b52ce60931ea4ce0a65e391da7f6287fe13862de251
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/wal.c     |  3 +++
 3 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index fc71ea63a6..50ac558bb2 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\ssome\sbranches\sin\swalTryBeginRead()\sthat\swere\s\nadded\sby\scheck-in\s[ce5d13c2de]\sbut\sbecame\sunreachable\swith\sthe\saddition\nof\slogic\sin\scheck-in\s[18b26843]\sthat\senabled\sread-only\sclients\sto\sparse\nthe\sWAL\sfile\sinto\sa\sheap-memory\sWAL-index,\sthus\sguaranteeing\sthat\sthe\s\nWAL-index\sheader\sis\salways\savailable.
-D 2017-11-13T05:51:37.627
+C Improvement\sto\sa\scomment.\s\sNo\schanges\sto\scode.
+D 2017-11-14T03:42:52.056
 F Makefile.in 5bae3f2f3d42f2ad52b141562d74872c97ac0fca6c54953c91bb150a0e6427a8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 3a5cb477ec3ce5274663b693164e349db63348667cd45bad78cc13d580b691e2
@@ -543,7 +543,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c 31ae1a82afccf02e2ff220add690d057dd7e25d9d7ec0b53fe57f2d62a4ba510
+F src/wal.c beeb71e4eab65dbf0d95f2717efc6ca3c0f5b3090ce67f3de63828f39a6ff053
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f
 F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4
@@ -1669,7 +1669,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 13ec8a77a47aa471af587459f4094da0d06674c5960f0d34777bcb3d38bc413b
-R be755a48b3ce9c3ee36556946fccdf2c
+P 9c6b38b9a96c11bdf9db4ea025720a4f49dcb723fa2e2776edc8453bce85c7e3
+R 696f834341714d48762ee99cfc8345fc
 U drh
-Z a296aa345abf23e8a36cfa010f746e55
+Z 27bc2b0f1009e1afc7034f11c759006b
diff --git a/manifest.uuid b/manifest.uuid
index 0dbf02629b..b0b16a61f7 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-9c6b38b9a96c11bdf9db4ea025720a4f49dcb723fa2e2776edc8453bce85c7e3
\ No newline at end of file
+486949fc03706e0056439b52ce60931ea4ce0a65e391da7f6287fe13862de251
\ No newline at end of file
diff --git a/src/wal.c b/src/wal.c
index 382be4e0c8..376151e1a8 100644
--- a/src/wal.c
+++ b/src/wal.c
@@ -2178,6 +2178,9 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){
       walIndexClose(pWal, 0);
       pWal->bShmUnreliable = 0;
       assert( pWal->nWiData>0 && pWal->apWiData[0]==0 );
+      /* walIndexRecover() might have returned SHORT_READ if a concurrent
+      ** writer truncated the WAL out from under it.  If that happens, it
+      ** indicates that a writer has fixed the SHM file for us, so retry */
       if( rc==SQLITE_IOERR_SHORT_READ ) rc = WAL_RETRY;
     }
     pWal->exclusiveMode = WAL_NORMAL_MODE;

From 8c0833fb218ef0b3fb89bef2b9600834e74cf8b7 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 14 Nov 2017 23:48:23 +0000
Subject: [PATCH 087/488] In the parse tree, combine LIMIT and OFFSET into a
 single expression rooted on a TK_LIMIT node, for a small code size reduction
 and performance increase, and a reduction in code complexity.

FossilOrigin-Name: 3925facd942c9df663f9b29b1e6f94f6be14af8c2b99eb691bfc836b4c220826
---
 manifest        | 37 +++++++++++++++---------------
 manifest.uuid   |  2 +-
 src/attach.c    |  3 ---
 src/delete.c    | 22 ++++++------------
 src/expr.c      | 14 +++++++-----
 src/fkey.c      |  4 ++--
 src/insert.c    |  1 -
 src/parse.y     | 45 ++++++++++++++-----------------------
 src/resolve.c   |  3 +--
 src/select.c    | 60 +++++++++++++++++--------------------------------
 src/sqliteInt.h | 11 +++++----
 src/treeview.c  | 13 +++++------
 src/trigger.c   |  4 ++--
 src/update.c    | 12 +++++-----
 src/walker.c    |  1 -
 15 files changed, 92 insertions(+), 140 deletions(-)

diff --git a/manifest b/manifest
index f1fb27c3db..1475fea8d1 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\sability\sto\sread\sfrom\sread-only\sWAL-mode\sdatabase\sfiles\sas\slong\sas\nthe\s-wal\sand\s-shm\sfiles\sare\spresent\son\sdisk.
-D 2017-11-14T19:34:22.764
+C In\sthe\sparse\stree,\scombine\sLIMIT\sand\sOFFSET\sinto\sa\ssingle\sexpression\srooted\non\sa\sTK_LIMIT\snode,\sfor\sa\ssmall\scode\ssize\sreduction\sand\sperformance\sincrease,\nand\sa\sreduction\sin\scode\scomplexity.
+D 2017-11-14T23:48:23.099
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -407,7 +407,7 @@ F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786
 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a
 F src/alter.c cf7a8af45cb0ace672f47a1b29ab24092a9e8cd8d945a9974e3b5d925f548594
 F src/analyze.c 0d0ccf7520a201d8747ea2f02c92c26e26f801bc161f714f27b9f7630dde0421
-F src/attach.c 07b706e336fd3cedbd855e1f8266d10e82fecae07daf86717b5760cd7784c584
+F src/attach.c 84c477e856b24c2b9a0983b438a707c0cf4d616cee7a425401d418e58afec24c
 F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
@@ -422,17 +422,17 @@ F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74
 F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
-F src/delete.c f97cc2a9a30b4119ec5ed0da4e41335060cbb8510a5f1a756be48c065cb824f0
-F src/expr.c 0016b95aed1df47ebc0eb7415bd621841d72dbffd3bfb62210e50e71d83b4691
+F src/delete.c e6a70fb58f6628f0ffc6d7221a6702c0d7b342c82520385b3996b364c22e0cb3
+F src/expr.c 5257a9157f22f048ddcce5cd494d39633e89c2a4769671311b3e7875d262f746
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
-F src/fkey.c 821f70b6c43ba4542ffb5fdf11d3ac5f79e4cbae4750ee3de379afbc5e1f7781
+F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331
 F src/func.c 0fb9a2d678d3c8aba89b46468b309cd7e8fa9806a369a30aa89024660845bb13
 F src/global.c ac3094f1dc59fbeb919aef7cc0cc827a8459d1fb1adb7972ef75bd9e0c10b75b
 F src/hash.c a12580e143f10301ed5166ea4964ae2853d3905a511d4e0c44497245c7ce1f7a
 F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4
 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
-F src/insert.c c7f333547211b8efbac8a72f71adad736b91e655d7bcdfacc737351ecf3c8df2
+F src/insert.c cb67cc56ef2ddd13e6944b2c0dd08a920bcd9503230adef8b9928d338097c722
 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e
 F src/loadext.c 20865b183bb8a3723d59cf1efffc3c50217eb452c1021d077b908c94da26b0b2
 F src/main.c c1965ee8159cee5fba3f590cc4767515a690504455a03e4817b1accfe0ba95a5
@@ -459,7 +459,7 @@ F src/os_win.c 7f36120492e4a23c48d1dd685edf29ae459c6d555660c61f1323cea3e5a1191d
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
-F src/parse.y f5f02ef39444982af36545bd52ae2921fc4ba1bac24d9b11efd8ec1f52c5b4dc
+F src/parse.y d79001da275bfe344b409006b85e81e486a0f6afc3762fdf0944f000f4aa0111
 F src/pcache.c 7ae91a4557a43d77d449accbfdc68846e6516f8e2eda46e8bbe4536fb669b201
 F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170
 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
@@ -468,14 +468,14 @@ F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
 F src/prepare.c 7cf451f903ad92a14e22de415a13e7a7d30f1bd23b3d21eeb0dc7264723244c5
 F src/printf.c 9506b4b96e59c0467047155f09015750cb2878aeda3d39e5610c1192ddc3c41c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
-F src/resolve.c 5b1e89ba279f4a4ab2f0975a7100d75be71e1a43a2df75a9c909d45bdd18c6ed
+F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c 660ef7977841fb462f24c8561e4212615bb6e5c9835fd3556257ce8316c50fee
+F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
 F src/shell.c.in 08cbffc31900359fea85896342a46147e9772c370d8a5079b7be26e3a1f50e8a
 F src/sqlite.h.in 8fd97993d48b50b9bade38c52f12d175942c9497c960905610c7b03a3e4b5818
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
-F src/sqliteInt.h abd4e64bc72906449d801d0e211265525239bc021bd9b7a72143c281fc24fa03
+F src/sqliteInt.h fb297e4b891608057e857d583e30a261d905a3f41493f351fc91bae7d22008ff
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -533,9 +533,9 @@ F src/test_windirent.h 5d67483a55442e31e1bde0f4a230e6e932ad5906
 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
 F src/tokenize.c 1003d6d90c6783206c711f0a9397656fa5b055209f4d092caa43bb3bf5215db5
-F src/treeview.c 2ee4a5dada213d5ab08a742af5c876cee6f1aaae65f10a61923f3fb63846afef
-F src/trigger.c 929b12bb5f9ab3b10c589f71fa176effe6ea8bddc9eb1bbd2439755fa0acdfdf
-F src/update.c 2e7ef3839eb620e9140a41bbfd05e29d7d216441c5d97221b75635dea4e49672
+F src/treeview.c 08a83195de8fad3f00542e3c8b3c1eb1222c999817c9e301ffb7f332882b96dd
+F src/trigger.c fc6be2a6e103d9e38b161e07d7db0ffb1f2218bd2f27ccdc0a3d1cc89e9cea0f
+F src/update.c 961bd1265d4d1e5cd65c9a54fa5122fb7aefcb003fcf2de0c092fceb7e58972c
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c d01fa6f45bfad3b65fb2490513aa2e0676412c61b4b094340b513cf72c3704a4
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
@@ -552,7 +552,7 @@ F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c beeb71e4eab65dbf0d95f2717efc6ca3c0f5b3090ce67f3de63828f39a6ff053
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
-F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f
+F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f
 F src/where.c 031a80bcafe93934fd7052f3031c9e7eb36b61754c6c84d6bf0833184abad3db
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
 F src/wherecode.c 4a117dd5886616d074f7b6589c23bf742f5a9858d6ffdaf8b9d1f76ab06245d2
@@ -1677,8 +1677,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P dae4a97a483bee1e6ac0271ddd28a0dffcebf7522edaf12eb5e0eba5fc62516a 486949fc03706e0056439b52ce60931ea4ce0a65e391da7f6287fe13862de251
-R 4dd08ae7170c8d270fe307a98bb5bfb2
-T +closed 486949fc03706e0056439b52ce60931ea4ce0a65e391da7f6287fe13862de251
+P 00ec95fcd02bb415dabd7f25fee24856d45d6916c18b2728e97e9bb9b8322ba3
+R 9fe4e3d4453a36374c8a196662e2c5b0
 U drh
-Z 1fe131254421255cb83ffd7abf325441
+Z 3ab594fe26a5b432ce6d9e25b1b1040f
diff --git a/manifest.uuid b/manifest.uuid
index 1aae26850e..41cb7f1619 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-00ec95fcd02bb415dabd7f25fee24856d45d6916c18b2728e97e9bb9b8322ba3
\ No newline at end of file
+3925facd942c9df663f9b29b1e6f94f6be14af8c2b99eb691bfc836b4c220826
\ No newline at end of file
diff --git a/src/attach.c b/src/attach.c
index 68a9a77c66..fa38e84159 100644
--- a/src/attach.c
+++ b/src/attach.c
@@ -504,9 +504,6 @@ int sqlite3FixSelect(
     if( sqlite3FixExpr(pFix, pSelect->pLimit) ){
       return 1;
     }
-    if( sqlite3FixExpr(pFix, pSelect->pOffset) ){
-      return 1;
-    }
     pSelect = pSelect->pPrior;
   }
   return 0;
diff --git a/src/delete.c b/src/delete.c
index 5c96fdfb8d..8e25b36f5a 100644
--- a/src/delete.c
+++ b/src/delete.c
@@ -92,7 +92,6 @@ void sqlite3MaterializeView(
   Expr *pWhere,        /* Optional WHERE clause to be added */
   ExprList *pOrderBy,  /* Optional ORDER BY clause */
   Expr *pLimit,        /* Optional LIMIT clause */
-  Expr *pOffset,       /* Optional OFFSET clause */
   int iCur             /* Cursor number for ephemeral table */
 ){
   SelectDest dest;
@@ -110,7 +109,7 @@ void sqlite3MaterializeView(
     assert( pFrom->a[0].pUsing==0 );
   }
   pSel = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, pOrderBy, 
-                          SF_IncludeHidden, pLimit, pOffset);
+                          SF_IncludeHidden, pLimit);
   sqlite3SelectDestInit(&dest, SRT_EphemTab, iCur);
   sqlite3Select(pParse, pSel, &dest);
   sqlite3SelectDelete(db, pSel);
@@ -132,7 +131,6 @@ Expr *sqlite3LimitWhere(
   Expr *pWhere,                /* The WHERE clause.  May be null */
   ExprList *pOrderBy,          /* The ORDER BY clause.  May be null */
   Expr *pLimit,                /* The LIMIT clause.  May be null */
-  Expr *pOffset,               /* The OFFSET clause.  May be null */
   char *zStmtType              /* Either DELETE or UPDATE.  For err msgs. */
 ){
   sqlite3 *db = pParse->db;
@@ -149,8 +147,6 @@ Expr *sqlite3LimitWhere(
     sqlite3ErrorMsg(pParse, "ORDER BY without LIMIT on %s", zStmtType);
     sqlite3ExprDelete(pParse->db, pWhere);
     sqlite3ExprListDelete(pParse->db, pOrderBy);
-    sqlite3ExprDelete(pParse->db, pLimit);
-    sqlite3ExprDelete(pParse->db, pOffset);
     return 0;
   }
 
@@ -158,8 +154,6 @@ Expr *sqlite3LimitWhere(
   ** is a limit/offset term to enforce.
   */
   if( pLimit == 0 ) {
-    /* if pLimit is null, pOffset will always be null as well. */
-    assert( pOffset == 0 );
     return pWhere;
   }
 
@@ -206,7 +200,7 @@ Expr *sqlite3LimitWhere(
 
   /* generate the SELECT expression tree. */
   pSelect = sqlite3SelectNew(pParse, pEList, pSelectSrc, pWhere, 0 ,0, 
-      pOrderBy,0,pLimit,pOffset
+      pOrderBy,0,pLimit
   );
 
   /* now generate the new WHERE rowid IN clause for the DELETE/UDPATE */
@@ -229,8 +223,7 @@ void sqlite3DeleteFrom(
   SrcList *pTabList,     /* The table from which we should delete things */
   Expr *pWhere,          /* The WHERE clause.  May be null */
   ExprList *pOrderBy,    /* ORDER BY clause. May be null */
-  Expr *pLimit,          /* LIMIT clause. May be null */
-  Expr *pOffset          /* OFFSET clause. May be null */
+  Expr *pLimit           /* LIMIT clause. May be null */
 ){
   Vdbe *v;               /* The virtual database engine */
   Table *pTab;           /* The table from which records will be deleted */
@@ -303,10 +296,10 @@ void sqlite3DeleteFrom(
 #ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
   if( !isView ){
     pWhere = sqlite3LimitWhere(
-        pParse, pTabList, pWhere, pOrderBy, pLimit, pOffset, "DELETE"
+        pParse, pTabList, pWhere, pOrderBy, pLimit, "DELETE"
     );
     pOrderBy = 0;
-    pLimit = pOffset = 0;
+    pLimit = 0;
   }
 #endif
 
@@ -358,11 +351,11 @@ void sqlite3DeleteFrom(
 #if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER)
   if( isView ){
     sqlite3MaterializeView(pParse, pTab, 
-        pWhere, pOrderBy, pLimit, pOffset, iTabCur
+        pWhere, pOrderBy, pLimit, iTabCur
     );
     iDataCur = iIdxCur = iTabCur;
     pOrderBy = 0;
-    pLimit = pOffset = 0;
+    pLimit = 0;
   }
 #endif
 
@@ -609,7 +602,6 @@ delete_from_cleanup:
 #if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) 
   sqlite3ExprListDelete(db, pOrderBy);
   sqlite3ExprDelete(db, pLimit);
-  sqlite3ExprDelete(db, pOffset);
 #endif
   sqlite3DbFree(db, aToOpen);
   return;
diff --git a/src/expr.c b/src/expr.c
index 0a5dc913f6..5027f994db 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -663,7 +663,6 @@ static void heightOfSelect(Select *p, int *pnHeight){
     heightOfExpr(p->pWhere, pnHeight);
     heightOfExpr(p->pHaving, pnHeight);
     heightOfExpr(p->pLimit, pnHeight);
-    heightOfExpr(p->pOffset, pnHeight);
     heightOfExprList(p->pEList, pnHeight);
     heightOfExprList(p->pGroupBy, pnHeight);
     heightOfExprList(p->pOrderBy, pnHeight);
@@ -1462,7 +1461,6 @@ Select *sqlite3SelectDup(sqlite3 *db, Select *pDup, int flags){
     pNew->pNext = pNext;
     pNew->pPrior = 0;
     pNew->pLimit = sqlite3ExprDup(db, p->pLimit, flags);
-    pNew->pOffset = sqlite3ExprDup(db, p->pOffset, flags);
     pNew->iLimit = 0;
     pNew->iOffset = 0;
     pNew->selFlags = p->selFlags & ~SF_UsesEphemeral;
@@ -2099,7 +2097,6 @@ static Select *isCandidateForInOpt(Expr *pX){
   }
   assert( p->pGroupBy==0 );              /* Has no GROUP BY clause */
   if( p->pLimit ) return 0;              /* Has no LIMIT clause */
-  assert( p->pOffset==0 );               /* No LIMIT means no OFFSET */
   if( p->pWhere ) return 0;              /* Has no WHERE clause */
   pSrc = p->pSrc;
   assert( pSrc!=0 );
@@ -2739,6 +2736,7 @@ int sqlite3CodeSubselect(
       Select *pSel;                         /* SELECT statement to encode */
       SelectDest dest;                      /* How to deal with SELECT result */
       int nReg;                             /* Registers to allocate */
+      Expr *pLimit;                         /* New limit expression */
 
       testcase( pExpr->op==TK_EXISTS );
       testcase( pExpr->op==TK_SELECT );
@@ -2760,9 +2758,13 @@ int sqlite3CodeSubselect(
         sqlite3VdbeAddOp2(v, OP_Integer, 0, dest.iSDParm);
         VdbeComment((v, "Init EXISTS result"));
       }
-      sqlite3ExprDelete(pParse->db, pSel->pLimit);
-      pSel->pLimit = sqlite3ExprAlloc(pParse->db, TK_INTEGER,
-                                  &sqlite3IntTokens[1], 0);
+      pLimit = sqlite3ExprAlloc(pParse->db, TK_INTEGER,&sqlite3IntTokens[1], 0);
+      if( pSel->pLimit ){
+        sqlite3ExprDelete(pParse->db, pSel->pLimit->pLeft);
+        pSel->pLimit->pLeft = pLimit;
+      }else{
+        pSel->pLimit = sqlite3PExpr(pParse, TK_LIMIT, pLimit, 0);
+      }
       pSel->iLimit = 0;
       pSel->selFlags &= ~SF_MultiValue;
       if( sqlite3Select(pParse, pSel, &dest) ){
diff --git a/src/fkey.c b/src/fkey.c
index 44ef3c7c7b..c366c1b3aa 100644
--- a/src/fkey.c
+++ b/src/fkey.c
@@ -725,7 +725,7 @@ void sqlite3FkDropTable(Parse *pParse, SrcList *pName, Table *pTab){
     }
 
     pParse->disableTriggers = 1;
-    sqlite3DeleteFrom(pParse, sqlite3SrcListDup(db, pName, 0), 0, 0, 0, 0);
+    sqlite3DeleteFrom(pParse, sqlite3SrcListDup(db, pName, 0), 0, 0, 0);
     pParse->disableTriggers = 0;
 
     /* If the DELETE has generated immediate foreign key constraint 
@@ -1283,7 +1283,7 @@ static Trigger *fkActionTrigger(
           sqlite3ExprListAppend(pParse, 0, pRaise),
           sqlite3SrcListAppend(db, 0, &tFrom, 0),
           pWhere,
-          0, 0, 0, 0, 0, 0
+          0, 0, 0, 0, 0
       );
       pWhere = 0;
     }
diff --git a/src/insert.c b/src/insert.c
index 3a531495cf..f0af0fbd10 100644
--- a/src/insert.c
+++ b/src/insert.c
@@ -2007,7 +2007,6 @@ static int xferOptimization(
   if( pSelect->pLimit ){
     return 0;   /* SELECT may not have a LIMIT clause */
   }
-  assert( pSelect->pOffset==0 );  /* Must be so if pLimit==0 */
   if( pSelect->pPrior ){
     return 0;   /* SELECT may not be a compound query */
   }
diff --git a/src/parse.y b/src/parse.y
index 18f587c05d..0867d1e0c8 100644
--- a/src/parse.y
+++ b/src/parse.y
@@ -84,15 +84,6 @@
 */
 #define YYMALLOCARGTYPE  u64
 
-/*
-** An instance of this structure holds information about the
-** LIMIT clause of a SELECT statement.
-*/
-struct LimitVal {
-  Expr *pLimit;    /* The LIMIT expression.  NULL if there is no limit */
-  Expr *pOffset;   /* The OFFSET expression.  NULL if there is none */
-};
-
 /*
 ** An instance of the following structure describes the event of a
 ** TRIGGER.  "a" is the event type, one of TK_UPDATE, TK_INSERT,
@@ -470,7 +461,7 @@ selectnowith(A) ::= selectnowith(A) multiselect_op(Y) oneselect(Z).  {
     x.n = 0;
     parserDoubleLinkSelect(pParse, pRhs);
     pFrom = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&x,pRhs,0,0);
-    pRhs = sqlite3SelectNew(pParse,0,pFrom,0,0,0,0,0,0,0);
+    pRhs = sqlite3SelectNew(pParse,0,pFrom,0,0,0,0,0,0);
   }
   if( pRhs ){
     pRhs->op = (u8)Y;
@@ -493,7 +484,7 @@ oneselect(A) ::= SELECT(S) distinct(D) selcollist(W) from(X) where_opt(Y)
 #if SELECTTRACE_ENABLED
   Token s = S; /*A-overwrites-S*/
 #endif
-  A = sqlite3SelectNew(pParse,W,X,Y,P,Q,Z,D,L.pLimit,L.pOffset);
+  A = sqlite3SelectNew(pParse,W,X,Y,P,Q,Z,D,L);
 #if SELECTTRACE_ENABLED
   /* Populate the Select.zSelName[] string that is used to help with
   ** query planner debugging, to differentiate between multiple Select
@@ -524,11 +515,11 @@ oneselect(A) ::= values(A).
 %type values {Select*}
 %destructor values {sqlite3SelectDelete(pParse->db, $$);}
 values(A) ::= VALUES LP nexprlist(X) RP. {
-  A = sqlite3SelectNew(pParse,X,0,0,0,0,0,SF_Values,0,0);
+  A = sqlite3SelectNew(pParse,X,0,0,0,0,0,SF_Values,0);
 }
 values(A) ::= values(A) COMMA LP exprlist(Y) RP. {
   Select *pRight, *pLeft = A;
-  pRight = sqlite3SelectNew(pParse,Y,0,0,0,0,0,SF_Values|SF_MultiValue,0,0);
+  pRight = sqlite3SelectNew(pParse,Y,0,0,0,0,0,SF_Values|SF_MultiValue,0);
   if( ALWAYS(pLeft) ) pLeft->selFlags &= ~SF_MultiValue;
   if( pRight ){
     pRight->op = TK_ALL;
@@ -639,7 +630,7 @@ seltablist(A) ::= stl_prefix(A) nm(Y) dbnm(D) LP exprlist(E) RP as(Z)
     }else{
       Select *pSubquery;
       sqlite3SrcListShiftJoinType(F);
-      pSubquery = sqlite3SelectNew(pParse,0,F,0,0,0,0,SF_NestedFrom,0,0);
+      pSubquery = sqlite3SelectNew(pParse,0,F,0,0,0,0,SF_NestedFrom,0);
       A = sqlite3SrcListAppendFromTerm(pParse,A,0,0,&Z,pSubquery,N,U);
     }
   }
@@ -726,7 +717,7 @@ groupby_opt(A) ::= GROUP BY nexprlist(X). {A = X;}
 having_opt(A) ::= .                {A = 0;}
 having_opt(A) ::= HAVING expr(X).  {A = X.pExpr;}
 
-%type limit_opt {struct LimitVal}
+%type limit_opt {Expr*}
 
 // The destructor for limit_opt will never fire in the current grammar.
 // The limit_opt non-terminal only occurs at the end of a single production
@@ -735,16 +726,14 @@ having_opt(A) ::= HAVING expr(X).  {A = X.pExpr;}
 // reduce.  So there is never a limit_opt non-terminal on the stack 
 // except as a transient.  So there is never anything to destroy.
 //
-//%destructor limit_opt {
-//  sqlite3ExprDelete(pParse->db, $$.pLimit);
-//  sqlite3ExprDelete(pParse->db, $$.pOffset);
-//}
-limit_opt(A) ::= .                    {A.pLimit = 0; A.pOffset = 0;}
-limit_opt(A) ::= LIMIT expr(X).       {A.pLimit = X.pExpr; A.pOffset = 0;}
+//%destructor limit_opt {sqlite3ExprDelete(pParse->db, $$);}
+limit_opt(A) ::= .       {A = 0;}
+limit_opt(A) ::= LIMIT expr(X).
+                         {A = sqlite3PExpr(pParse,TK_LIMIT,X.pExpr,0);}
 limit_opt(A) ::= LIMIT expr(X) OFFSET expr(Y). 
-                                      {A.pLimit = X.pExpr; A.pOffset = Y.pExpr;}
+                         {A = sqlite3PExpr(pParse,TK_LIMIT,X.pExpr,Y.pExpr);}
 limit_opt(A) ::= LIMIT expr(X) COMMA expr(Y). 
-                                      {A.pOffset = X.pExpr; A.pLimit = Y.pExpr;}
+                         {A = sqlite3PExpr(pParse,TK_LIMIT,Y.pExpr,X.pExpr);}
 
 /////////////////////////// The DELETE statement /////////////////////////////
 //
@@ -753,14 +742,14 @@ cmd ::= with(C) DELETE FROM fullname(X) indexed_opt(I) where_opt(W)
         orderby_opt(O) limit_opt(L). {
   sqlite3WithPush(pParse, C, 1);
   sqlite3SrcListIndexedBy(pParse, X, &I);
-  sqlite3DeleteFrom(pParse,X,W,O,L.pLimit,L.pOffset); 
+  sqlite3DeleteFrom(pParse,X,W,O,L);
 }
 %endif
 %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
 cmd ::= with(C) DELETE FROM fullname(X) indexed_opt(I) where_opt(W). {
   sqlite3WithPush(pParse, C, 1);
   sqlite3SrcListIndexedBy(pParse, X, &I);
-  sqlite3DeleteFrom(pParse,X,W,0,0,0);
+  sqlite3DeleteFrom(pParse,X,W,0,0);
 }
 %endif
 
@@ -778,7 +767,7 @@ cmd ::= with(C) UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y)
   sqlite3WithPush(pParse, C, 1);
   sqlite3SrcListIndexedBy(pParse, X, &I);
   sqlite3ExprListCheckLength(pParse,Y,"set list"); 
-  sqlite3Update(pParse,X,Y,W,R,O,L.pLimit,L.pOffset);
+  sqlite3Update(pParse,X,Y,W,R,O,L);
 }
 %endif
 %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
@@ -787,7 +776,7 @@ cmd ::= with(C) UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y)
   sqlite3WithPush(pParse, C, 1);
   sqlite3SrcListIndexedBy(pParse, X, &I);
   sqlite3ExprListCheckLength(pParse,Y,"set list"); 
-  sqlite3Update(pParse,X,Y,W,R,0,0,0);
+  sqlite3Update(pParse,X,Y,W,R,0,0);
 }
 %endif
 
@@ -1189,7 +1178,7 @@ expr(A) ::= expr(A) between_op(N) expr(X) AND expr(Y). [BETWEEN] {
   }
   expr(A) ::= expr(A) in_op(N) nm(Y) dbnm(Z) paren_exprlist(E). [IN] {
     SrcList *pSrc = sqlite3SrcListAppend(pParse->db, 0,&Y,&Z);
-    Select *pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0,0);
+    Select *pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0);
     if( E )  sqlite3SrcListFuncArgs(pParse, pSelect ? pSrc : 0, E);
     A.pExpr = sqlite3PExpr(pParse, TK_IN, A.pExpr, 0);
     sqlite3PExprAddSelect(pParse, A.pExpr, pSelect);
diff --git a/src/resolve.c b/src/resolve.c
index 17dbbccfc5..f735fffa02 100644
--- a/src/resolve.c
+++ b/src/resolve.c
@@ -1196,8 +1196,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){
     */
     memset(&sNC, 0, sizeof(sNC));
     sNC.pParse = pParse;
-    if( sqlite3ResolveExprNames(&sNC, p->pLimit) ||
-        sqlite3ResolveExprNames(&sNC, p->pOffset) ){
+    if( sqlite3ResolveExprNames(&sNC, p->pLimit) ){
       return WRC_Abort;
     }
 
diff --git a/src/select.c b/src/select.c
index 46c69d71fb..97eaf21b4d 100644
--- a/src/select.c
+++ b/src/select.c
@@ -74,7 +74,6 @@ static void clearSelect(sqlite3 *db, Select *p, int bFree){
     sqlite3ExprDelete(db, p->pHaving);
     sqlite3ExprListDelete(db, p->pOrderBy);
     sqlite3ExprDelete(db, p->pLimit);
-    sqlite3ExprDelete(db, p->pOffset);
     if( OK_IF_ALWAYS_TRUE(p->pWith) ) sqlite3WithDelete(db, p->pWith);
     if( bFree ) sqlite3DbFreeNN(db, p);
     p = pPrior;
@@ -107,8 +106,7 @@ Select *sqlite3SelectNew(
   Expr *pHaving,        /* the HAVING clause */
   ExprList *pOrderBy,   /* the ORDER BY clause */
   u32 selFlags,         /* Flag parameters, such as SF_Distinct */
-  Expr *pLimit,         /* LIMIT value.  NULL means not used */
-  Expr *pOffset         /* OFFSET value.  NULL means no offset */
+  Expr *pLimit          /* LIMIT value.  NULL means not used */
 ){
   Select *pNew;
   Select standin;
@@ -141,10 +139,7 @@ Select *sqlite3SelectNew(
   pNew->pPrior = 0;
   pNew->pNext = 0;
   pNew->pLimit = pLimit;
-  pNew->pOffset = pOffset;
   pNew->pWith = 0;
-  assert( pOffset==0 || pLimit!=0 || pParse->nErr>0
-                     || pParse->db->mallocFailed!=0 );
   if( pParse->db->mallocFailed ) {
     clearSelect(pParse->db, pNew, pNew!=&standin);
     pNew = 0;
@@ -1874,7 +1869,7 @@ Vdbe *sqlite3GetVdbe(Parse *pParse){
 
 /*
 ** Compute the iLimit and iOffset fields of the SELECT based on the
-** pLimit and pOffset expressions.  pLimit and pOffset hold the expressions
+** pLimit expressions.  pLimit->pLeft and pLimit->pRight hold the expressions
 ** that appear in the original SQL statement after the LIMIT and OFFSET
 ** keywords.  Or NULL if those keywords are omitted. iLimit and iOffset 
 ** are the integer memory register numbers for counters used to compute 
@@ -1882,15 +1877,15 @@ Vdbe *sqlite3GetVdbe(Parse *pParse){
 ** iLimit and iOffset are negative.
 **
 ** This routine changes the values of iLimit and iOffset only if
-** a limit or offset is defined by pLimit and pOffset.  iLimit and
-** iOffset should have been preset to appropriate default values (zero)
+** a limit or offset is defined by pLimit->pLeft and pLimit->pRight.  iLimit
+** and iOffset should have been preset to appropriate default values (zero)
 ** prior to calling this routine.
 **
 ** The iOffset register (if it exists) is initialized to the value
 ** of the OFFSET.  The iLimit register is initialized to LIMIT.  Register
 ** iOffset+1 is initialized to LIMIT+OFFSET.
 **
-** Only if pLimit!=0 or pOffset!=0 do the limit registers get
+** Only if pLimit->pLeft!=0 do the limit registers get
 ** redefined.  The UNION ALL operator uses this property to force
 ** the reuse of the same limit and offset registers across multiple
 ** SELECT statements.
@@ -1900,6 +1895,8 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){
   int iLimit = 0;
   int iOffset;
   int n;
+  Expr *pLimit = p->pLimit;
+
   if( p->iLimit ) return;
 
   /* 
@@ -1909,12 +1906,13 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){
   ** no rows.
   */
   sqlite3ExprCacheClear(pParse);
-  assert( p->pOffset==0 || p->pLimit!=0 );
-  if( p->pLimit ){
+  if( pLimit ){
+    assert( pLimit->op==TK_LIMIT );
+    assert( pLimit->pLeft!=0 );
     p->iLimit = iLimit = ++pParse->nMem;
     v = sqlite3GetVdbe(pParse);
     assert( v!=0 );
-    if( sqlite3ExprIsInteger(p->pLimit, &n) ){
+    if( sqlite3ExprIsInteger(pLimit->pLeft, &n) ){
       sqlite3VdbeAddOp2(v, OP_Integer, n, iLimit);
       VdbeComment((v, "LIMIT counter"));
       if( n==0 ){
@@ -1924,15 +1922,15 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){
         p->selFlags |= SF_FixedLimit;
       }
     }else{
-      sqlite3ExprCode(pParse, p->pLimit, iLimit);
+      sqlite3ExprCode(pParse, pLimit->pLeft, iLimit);
       sqlite3VdbeAddOp1(v, OP_MustBeInt, iLimit); VdbeCoverage(v);
       VdbeComment((v, "LIMIT counter"));
       sqlite3VdbeAddOp2(v, OP_IfNot, iLimit, iBreak); VdbeCoverage(v);
     }
-    if( p->pOffset ){
+    if( pLimit->pRight ){
       p->iOffset = iOffset = ++pParse->nMem;
       pParse->nMem++;   /* Allocate an extra register for limit+offset */
-      sqlite3ExprCode(pParse, p->pOffset, iOffset);
+      sqlite3ExprCode(pParse, pLimit->pRight, iOffset);
       sqlite3VdbeAddOp1(v, OP_MustBeInt, iOffset); VdbeCoverage(v);
       VdbeComment((v, "OFFSET counter"));
       sqlite3VdbeAddOp3(v, OP_OffsetLimit, iLimit, iOffset+1, iOffset);
@@ -2062,7 +2060,7 @@ static void generateWithRecursiveQuery(
   int i;                        /* Loop counter */
   int rc;                       /* Result code */
   ExprList *pOrderBy;           /* The ORDER BY clause */
-  Expr *pLimit, *pOffset;       /* Saved LIMIT and OFFSET */
+  Expr *pLimit;                 /* Saved LIMIT and OFFSET */
   int regLimit, regOffset;      /* Registers used by LIMIT and OFFSET */
 
   /* Obtain authorization to do a recursive query */
@@ -2073,10 +2071,9 @@ static void generateWithRecursiveQuery(
   p->nSelectRow = 320;  /* 4 billion rows */
   computeLimitRegisters(pParse, p, addrBreak);
   pLimit = p->pLimit;
-  pOffset = p->pOffset;
   regLimit = p->iLimit;
   regOffset = p->iOffset;
-  p->pLimit = p->pOffset = 0;
+  p->pLimit = 0;
   p->iLimit = p->iOffset = 0;
   pOrderBy = p->pOrderBy;
 
@@ -2169,7 +2166,6 @@ end_of_recursive_query:
   sqlite3ExprListDelete(pParse->db, p->pOrderBy);
   p->pOrderBy = pOrderBy;
   p->pLimit = pLimit;
-  p->pOffset = pOffset;
   return;
 }
 #endif /* SQLITE_OMIT_CTE */
@@ -2205,7 +2201,6 @@ static int multiSelectValues(
     assert( p->selFlags & SF_Values );
     assert( p->op==TK_ALL || (p->op==TK_SELECT && p->pPrior==0) );
     assert( p->pLimit==0 );
-    assert( p->pOffset==0 );
     assert( p->pNext==0 || p->pEList->nExpr==p->pNext->pEList->nExpr );
     if( p->pPrior==0 ) break;
     assert( p->pPrior->pNext==p );
@@ -2332,11 +2327,9 @@ static int multiSelect(
       pPrior->iLimit = p->iLimit;
       pPrior->iOffset = p->iOffset;
       pPrior->pLimit = p->pLimit;
-      pPrior->pOffset = p->pOffset;
       explainSetInteger(iSub1, pParse->iNextSelectId);
       rc = sqlite3Select(pParse, pPrior, &dest);
       p->pLimit = 0;
-      p->pOffset = 0;
       if( rc ){
         goto multi_select_end;
       }
@@ -2358,7 +2351,7 @@ static int multiSelect(
       p->pPrior = pPrior;
       p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow);
       if( pPrior->pLimit
-       && sqlite3ExprIsInteger(pPrior->pLimit, &nLimit)
+       && sqlite3ExprIsInteger(pPrior->pLimit->pLeft, &nLimit)
        && nLimit>0 && p->nSelectRow > sqlite3LogEst((u64)nLimit) 
       ){
         p->nSelectRow = sqlite3LogEst((u64)nLimit);
@@ -2373,7 +2366,7 @@ static int multiSelect(
       int unionTab;    /* Cursor number of the temporary table holding result */
       u8 op = 0;       /* One of the SRT_ operations to apply to self */
       int priorOp;     /* The SRT_ operation to apply to prior selects */
-      Expr *pLimit, *pOffset; /* Saved values of p->nLimit and p->nOffset */
+      Expr *pLimit;    /* Saved values of p->nLimit  */
       int addr;
       SelectDest uniondest;
 
@@ -2385,7 +2378,6 @@ static int multiSelect(
         ** right.
         */
         assert( p->pLimit==0 );      /* Not allowed on leftward elements */
-        assert( p->pOffset==0 );     /* Not allowed on leftward elements */
         unionTab = dest.iSDParm;
       }else{
         /* We will need to create our own temporary table to hold the
@@ -2421,8 +2413,6 @@ static int multiSelect(
       p->pPrior = 0;
       pLimit = p->pLimit;
       p->pLimit = 0;
-      pOffset = p->pOffset;
-      p->pOffset = 0;
       uniondest.eDest = op;
       explainSetInteger(iSub2, pParse->iNextSelectId);
       rc = sqlite3Select(pParse, p, &uniondest);
@@ -2438,7 +2428,6 @@ static int multiSelect(
       }
       sqlite3ExprDelete(db, p->pLimit);
       p->pLimit = pLimit;
-      p->pOffset = pOffset;
       p->iLimit = 0;
       p->iOffset = 0;
 
@@ -2466,7 +2455,7 @@ static int multiSelect(
     default: assert( p->op==TK_INTERSECT ); {
       int tab1, tab2;
       int iCont, iBreak, iStart;
-      Expr *pLimit, *pOffset;
+      Expr *pLimit;
       int addr;
       SelectDest intersectdest;
       int r1;
@@ -2502,8 +2491,6 @@ static int multiSelect(
       p->pPrior = 0;
       pLimit = p->pLimit;
       p->pLimit = 0;
-      pOffset = p->pOffset;
-      p->pOffset = 0;
       intersectdest.iSDParm = tab2;
       explainSetInteger(iSub2, pParse->iNextSelectId);
       rc = sqlite3Select(pParse, p, &intersectdest);
@@ -2513,7 +2500,6 @@ static int multiSelect(
       if( p->nSelectRow>pPrior->nSelectRow ) p->nSelectRow = pPrior->nSelectRow;
       sqlite3ExprDelete(db, p->pLimit);
       p->pLimit = pLimit;
-      p->pOffset = pOffset;
 
       /* Generate code to take the intersection of the two temporary
       ** tables.
@@ -2992,8 +2978,6 @@ static int multiSelectOrderBy(
   }
   sqlite3ExprDelete(db, p->pLimit);
   p->pLimit = 0;
-  sqlite3ExprDelete(db, p->pOffset);
-  p->pOffset = 0;
 
   regAddrA = ++pParse->nMem;
   regAddrB = ++pParse->nMem;
@@ -3457,7 +3441,7 @@ static int flattenSubquery(
   ** became arbitrary expressions, we were forced to add restrictions (13)
   ** and (14). */
   if( pSub->pLimit && p->pLimit ) return 0;              /* Restriction (13) */
-  if( pSub->pOffset ) return 0;                          /* Restriction (14) */
+  if( pSub->pLimit && pSub->pLimit->pRight ) return 0;   /* Restriction (14) */
   if( (p->selFlags & SF_Compound)!=0 && pSub->pLimit ){
     return 0;                                            /* Restriction (15) */
   }
@@ -3605,16 +3589,13 @@ static int flattenSubquery(
     Select *pNew;
     ExprList *pOrderBy = p->pOrderBy;
     Expr *pLimit = p->pLimit;
-    Expr *pOffset = p->pOffset;
     Select *pPrior = p->pPrior;
     p->pOrderBy = 0;
     p->pSrc = 0;
     p->pPrior = 0;
     p->pLimit = 0;
-    p->pOffset = 0;
     pNew = sqlite3SelectDup(db, p, 0);
     sqlite3SelectSetName(pNew, pSub->zSelName);
-    p->pOffset = pOffset;
     p->pLimit = pLimit;
     p->pOrderBy = pOrderBy;
     p->pSrc = pSrc;
@@ -4080,7 +4061,6 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){
   assert( pNew->pPrior!=0 );
   pNew->pPrior->pNext = pNew;
   pNew->pLimit = 0;
-  pNew->pOffset = 0;
   return WRC_Continue;
 }
 
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 95a66b4fe7..058fee1461 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -2749,7 +2749,6 @@ struct Select {
   Select *pPrior;        /* Prior select in a compound select statement */
   Select *pNext;         /* Next select to the left in a compound */
   Expr *pLimit;          /* LIMIT expression. NULL means not used. */
-  Expr *pOffset;         /* OFFSET expression. NULL means not used. */
   With *pWith;           /* WITH clause attached to this select. Or NULL. */
 };
 
@@ -3756,16 +3755,16 @@ void sqlite3CreateIndex(Parse*,Token*,Token*,SrcList*,ExprList*,int,Token*,
 void sqlite3DropIndex(Parse*, SrcList*, int);
 int sqlite3Select(Parse*, Select*, SelectDest*);
 Select *sqlite3SelectNew(Parse*,ExprList*,SrcList*,Expr*,ExprList*,
-                         Expr*,ExprList*,u32,Expr*,Expr*);
+                         Expr*,ExprList*,u32,Expr*);
 void sqlite3SelectDelete(sqlite3*, Select*);
 Table *sqlite3SrcListLookup(Parse*, SrcList*);
 int sqlite3IsReadOnly(Parse*, Table*, int);
 void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int);
 #if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY)
-Expr *sqlite3LimitWhere(Parse*,SrcList*,Expr*,ExprList*,Expr*,Expr*,char*);
+Expr *sqlite3LimitWhere(Parse*,SrcList*,Expr*,ExprList*,Expr*,char*);
 #endif
-void sqlite3DeleteFrom(Parse*, SrcList*, Expr*, ExprList*, Expr*, Expr*);
-void sqlite3Update(Parse*, SrcList*, ExprList*,Expr*,int,ExprList*,Expr*,Expr*);
+void sqlite3DeleteFrom(Parse*, SrcList*, Expr*, ExprList*, Expr*);
+void sqlite3Update(Parse*, SrcList*, ExprList*,Expr*,int,ExprList*,Expr*);
 WhereInfo *sqlite3WhereBegin(Parse*,SrcList*,Expr*,ExprList*,ExprList*,u16,int);
 void sqlite3WhereEnd(WhereInfo*);
 LogEst sqlite3WhereOutputRowCount(WhereInfo*);
@@ -3889,7 +3888,7 @@ int sqlite3SafetyCheckSickOrOk(sqlite3*);
 void sqlite3ChangeCookie(Parse*, int);
 
 #if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER)
-void sqlite3MaterializeView(Parse*, Table*, Expr*, ExprList*,Expr*,Expr*,int);
+void sqlite3MaterializeView(Parse*, Table*, Expr*, ExprList*,Expr*,int);
 #endif
 
 #ifndef SQLITE_OMIT_TRIGGER
diff --git a/src/treeview.c b/src/treeview.c
index ba9fa7b2f0..8d171a4733 100644
--- a/src/treeview.c
+++ b/src/treeview.c
@@ -153,7 +153,6 @@ void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){
       if( p->pHaving ) n++;
       if( p->pOrderBy ) n++;
       if( p->pLimit ) n++;
-      if( p->pOffset ) n++;
     }
     sqlite3TreeViewExprList(pView, p->pEList, (n--)>0, "result-set");
     if( p->pSrc && p->pSrc->nSrc ){
@@ -210,12 +209,12 @@ void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){
     }
     if( p->pLimit ){
       sqlite3TreeViewItem(pView, "LIMIT", (n--)>0);
-      sqlite3TreeViewExpr(pView, p->pLimit, 0);
-      sqlite3TreeViewPop(pView);
-    }
-    if( p->pOffset ){
-      sqlite3TreeViewItem(pView, "OFFSET", (n--)>0);
-      sqlite3TreeViewExpr(pView, p->pOffset, 0);
+      sqlite3TreeViewExpr(pView, p->pLimit->pLeft, p->pLimit->pRight!=0);
+      if( p->pLimit->pRight ){
+        sqlite3TreeViewItem(pView, "OFFSET", (n--)>0);
+        sqlite3TreeViewExpr(pView, p->pLimit->pRight, 0);
+        sqlite3TreeViewPop(pView);
+      }
       sqlite3TreeViewPop(pView);
     }
     if( p->pPrior ){
diff --git a/src/trigger.c b/src/trigger.c
index a17769ae98..a64fb291a0 100644
--- a/src/trigger.c
+++ b/src/trigger.c
@@ -711,7 +711,7 @@ static int codeTriggerProgram(
           targetSrcList(pParse, pStep),
           sqlite3ExprListDup(db, pStep->pExprList, 0), 
           sqlite3ExprDup(db, pStep->pWhere, 0), 
-          pParse->eOrconf, 0, 0, 0
+          pParse->eOrconf, 0, 0
         );
         break;
       }
@@ -727,7 +727,7 @@ static int codeTriggerProgram(
       case TK_DELETE: {
         sqlite3DeleteFrom(pParse, 
           targetSrcList(pParse, pStep),
-          sqlite3ExprDup(db, pStep->pWhere, 0), 0, 0, 0
+          sqlite3ExprDup(db, pStep->pWhere, 0), 0, 0
         );
         break;
       }
diff --git a/src/update.c b/src/update.c
index 4cfa4019eb..15e8f4a6ce 100644
--- a/src/update.c
+++ b/src/update.c
@@ -93,8 +93,7 @@ void sqlite3Update(
   Expr *pWhere,          /* The WHERE clause.  May be null */
   int onError,           /* How to handle constraint errors */
   ExprList *pOrderBy,    /* ORDER BY clause. May be null */
-  Expr *pLimit,          /* LIMIT clause. May be null */
-  Expr *pOffset          /* OFFSET clause. May be null */
+  Expr *pLimit           /* LIMIT clause. May be null */
 ){
   int i, j;              /* Loop counters */
   Table *pTab;           /* The table to be updated */
@@ -182,10 +181,10 @@ void sqlite3Update(
 #ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
   if( !isView ){
     pWhere = sqlite3LimitWhere(
-        pParse, pTabList, pWhere, pOrderBy, pLimit, pOffset, "UPDATE"
+        pParse, pTabList, pWhere, pOrderBy, pLimit, "UPDATE"
     );
     pOrderBy = 0;
-    pLimit = pOffset = 0;
+    pLimit = 0;
   }
 #endif
 
@@ -358,10 +357,10 @@ void sqlite3Update(
 #if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER)
   if( isView ){
     sqlite3MaterializeView(pParse, pTab, 
-        pWhere, pOrderBy, pLimit, pOffset, iDataCur
+        pWhere, pOrderBy, pLimit, iDataCur
     );
     pOrderBy = 0;
-    pLimit = pOffset = 0;
+    pLimit = 0;
   }
 #endif
 
@@ -748,7 +747,6 @@ update_cleanup:
 #if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) 
   sqlite3ExprListDelete(db, pOrderBy);
   sqlite3ExprDelete(db, pLimit);
-  sqlite3ExprDelete(db, pOffset);
 #endif
   return;
 }
diff --git a/src/walker.c b/src/walker.c
index ae7545bf5f..60bf8226fe 100644
--- a/src/walker.c
+++ b/src/walker.c
@@ -91,7 +91,6 @@ int sqlite3WalkSelectExpr(Walker *pWalker, Select *p){
   if( sqlite3WalkExpr(pWalker, p->pHaving) ) return WRC_Abort;
   if( sqlite3WalkExprList(pWalker, p->pOrderBy) ) return WRC_Abort;
   if( sqlite3WalkExpr(pWalker, p->pLimit) ) return WRC_Abort;
-  if( sqlite3WalkExpr(pWalker, p->pOffset) ) return WRC_Abort;
   return WRC_Continue;
 }
 

From 35f51a4460f8548021ef1c709197622172a9e496 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 15 Nov 2017 17:07:22 +0000
Subject: [PATCH 088/488] UI improvements on the .testctrl dot-command in the
 shell.

FossilOrigin-Name: 5d66219c4706e11a9c2a1290224e7c17b19f74657e8239accfb4c6f6be6af9b5
---
 manifest       | 12 +++++-----
 manifest.uuid  |  2 +-
 src/shell.c.in | 59 +++++++++++++++++++++++++++++++++-----------------
 3 files changed, 46 insertions(+), 27 deletions(-)

diff --git a/manifest b/manifest
index 1475fea8d1..959866cf88 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\sthe\sparse\stree,\scombine\sLIMIT\sand\sOFFSET\sinto\sa\ssingle\sexpression\srooted\non\sa\sTK_LIMIT\snode,\sfor\sa\ssmall\scode\ssize\sreduction\sand\sperformance\sincrease,\nand\sa\sreduction\sin\scode\scomplexity.
-D 2017-11-14T23:48:23.099
+C UI\simprovements\son\sthe\s.testctrl\sdot-command\sin\sthe\sshell.
+D 2017-11-15T17:07:22.982
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -471,7 +471,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
-F src/shell.c.in 08cbffc31900359fea85896342a46147e9772c370d8a5079b7be26e3a1f50e8a
+F src/shell.c.in e261e3c2f39edbabfdea00ef8f50adf7b2fd7d84fc6d78d8c2a06b335bd1916f
 F src/sqlite.h.in 8fd97993d48b50b9bade38c52f12d175942c9497c960905610c7b03a3e4b5818
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1677,7 +1677,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 00ec95fcd02bb415dabd7f25fee24856d45d6916c18b2728e97e9bb9b8322ba3
-R 9fe4e3d4453a36374c8a196662e2c5b0
+P 3925facd942c9df663f9b29b1e6f94f6be14af8c2b99eb691bfc836b4c220826
+R 837aef86d6c701c4be924bf38f742114
 U drh
-Z 3ab594fe26a5b432ce6d9e25b1b1040f
+Z 657eeee0b34954f0ac1c7bdc4eb3dc1b
diff --git a/manifest.uuid b/manifest.uuid
index 41cb7f1619..7a1cc65c2e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-3925facd942c9df663f9b29b1e6f94f6be14af8c2b99eb691bfc836b4c220826
\ No newline at end of file
+5d66219c4706e11a9c2a1290224e7c17b19f74657e8239accfb4c6f6be6af9b5
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index e85ebd3c93..1b5adc0734 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -5922,49 +5922,68 @@ static int do_meta_command(char *zLine, ShellState *p){
   }else
 
 #ifndef SQLITE_UNTESTABLE
-  if( c=='t' && n>=8 && strncmp(azArg[0], "testctrl", n)==0 && nArg>=2 ){
+  if( c=='t' && n>=8 && strncmp(azArg[0], "testctrl", n)==0 ){
     static const struct {
        const char *zCtrlName;   /* Name of a test-control option */
        int ctrlCode;            /* Integer code for that option */
     } aCtrl[] = {
-      { "prng_save",             SQLITE_TESTCTRL_PRNG_SAVE              },
-      { "prng_restore",          SQLITE_TESTCTRL_PRNG_RESTORE           },
-      { "prng_reset",            SQLITE_TESTCTRL_PRNG_RESET             },
-      { "bitvec_test",           SQLITE_TESTCTRL_BITVEC_TEST            },
-      { "fault_install",         SQLITE_TESTCTRL_FAULT_INSTALL          },
-      { "benign_malloc_hooks",   SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS    },
-      { "pending_byte",          SQLITE_TESTCTRL_PENDING_BYTE           },
-      { "assert",                SQLITE_TESTCTRL_ASSERT                 },
       { "always",                SQLITE_TESTCTRL_ALWAYS                 },
-      { "reserve",               SQLITE_TESTCTRL_RESERVE                },
-      { "optimizations",         SQLITE_TESTCTRL_OPTIMIZATIONS          },
-      { "iskeyword",             SQLITE_TESTCTRL_ISKEYWORD              },
+      { "assert",                SQLITE_TESTCTRL_ASSERT                 },
+      { "benign_malloc_hooks",   SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS    },
+      { "bitvec_test",           SQLITE_TESTCTRL_BITVEC_TEST            },
       { "byteorder",             SQLITE_TESTCTRL_BYTEORDER              },
-      { "never_corrupt",         SQLITE_TESTCTRL_NEVER_CORRUPT          },
+      { "fault_install",         SQLITE_TESTCTRL_FAULT_INSTALL          },
       { "imposter",              SQLITE_TESTCTRL_IMPOSTER               },
+      { "iskeyword",             SQLITE_TESTCTRL_ISKEYWORD              },
+      { "never_corrupt",         SQLITE_TESTCTRL_NEVER_CORRUPT          },
+      { "optimizations",         SQLITE_TESTCTRL_OPTIMIZATIONS          },
+      { "pending_byte",          SQLITE_TESTCTRL_PENDING_BYTE           },
+      { "prng_reset",            SQLITE_TESTCTRL_PRNG_RESET             },
+      { "prng_restore",          SQLITE_TESTCTRL_PRNG_RESTORE           },
+      { "prng_save",             SQLITE_TESTCTRL_PRNG_SAVE              },
+      { "reserve",               SQLITE_TESTCTRL_RESERVE                },
     };
     int testctrl = -1;
     int rc2 = 0;
     int i, n2;
     open_db(p, 0);
+    const char *zCmd = nArg>=2 ? azArg[1] : "help";
+
+    /* The argument can optionally begin with "-" or "--" */
+    if( zCmd[0]=='-' && zCmd[1] ){
+      zCmd++;
+      if( zCmd[0]=='-' && zCmd[1] ) zCmd++;
+    }
+
+    /* --help lists all test-controls */
+    if( strcmp(zCmd,"help")==0 ){
+      utf8_printf(p->out, "Available test-controls:\n");
+      for(i=0; i<ArraySize(aCtrl); i++){
+        utf8_printf(p->out, "  .testctrl %s\n", aCtrl[i].zCtrlName);
+      }
+      rc = 1;
+      goto meta_command_exit;
+    }
 
     /* convert testctrl text option to value. allow any unique prefix
     ** of the option name, or a numerical value. */
-    n2 = strlen30(azArg[1]);
+    n2 = strlen30(zCmd);
     for(i=0; i<ArraySize(aCtrl); i++){
-      if( strncmp(azArg[1], aCtrl[i].zCtrlName, n2)==0 ){
+      if( strncmp(zCmd, aCtrl[i].zCtrlName, n2)==0 ){
         if( testctrl<0 ){
           testctrl = aCtrl[i].ctrlCode;
         }else{
-          utf8_printf(stderr, "ambiguous option name: \"%s\"\n", azArg[1]);
-          testctrl = -1;
-          break;
+          utf8_printf(stderr, "Error: ambiguous test-control: \"%s\"\n"
+                              "Use \".testctrl --help\" for help\n", zCmd);
+          rc = 1;
+          goto meta_command_exit;
         }
       }
     }
-    if( testctrl<0 ) testctrl = (int)integerValue(azArg[1]);
+    if( testctrl<0 ) testctrl = (int)integerValue(zCmd);
     if( (testctrl<SQLITE_TESTCTRL_FIRST) || (testctrl>SQLITE_TESTCTRL_LAST) ){
-      utf8_printf(stderr,"Error: invalid testctrl option: %s\n", azArg[1]);
+      utf8_printf(stderr,"Error: unknown test-control: %s\n"
+                         "Use \".testctrl --help\" for help\n", zCmd);
     }else{
       switch(testctrl){
 

From ef302e8a676b42f061aadcf1e12c9f9bceb53570 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 15 Nov 2017 19:14:08 +0000
Subject: [PATCH 089/488] Futher improvements to the .testctrl interface in the
 shell.

FossilOrigin-Name: 1ca3b8cce93e83fabe35c22cb726b8ce0883e0448afa3301ad0b53073055fa8d
---
 manifest       |  12 +++---
 manifest.uuid  |   2 +-
 src/shell.c.in | 101 ++++++++++++++++++++++++-------------------------
 3 files changed, 56 insertions(+), 59 deletions(-)

diff --git a/manifest b/manifest
index 959866cf88..31f6458b7c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C UI\simprovements\son\sthe\s.testctrl\sdot-command\sin\sthe\sshell.
-D 2017-11-15T17:07:22.982
+C Futher\simprovements\sto\sthe\s.testctrl\sinterface\sin\sthe\sshell.
+D 2017-11-15T19:14:08.134
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -471,7 +471,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
-F src/shell.c.in e261e3c2f39edbabfdea00ef8f50adf7b2fd7d84fc6d78d8c2a06b335bd1916f
+F src/shell.c.in a87f3094c1d2a07ed3f731409b33dbf36faf99157bfedb17855a664c60e1a909
 F src/sqlite.h.in 8fd97993d48b50b9bade38c52f12d175942c9497c960905610c7b03a3e4b5818
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1677,7 +1677,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3925facd942c9df663f9b29b1e6f94f6be14af8c2b99eb691bfc836b4c220826
-R 837aef86d6c701c4be924bf38f742114
+P 5d66219c4706e11a9c2a1290224e7c17b19f74657e8239accfb4c6f6be6af9b5
+R bb554ed3e0e111141682146f45553c3a
 U drh
-Z 657eeee0b34954f0ac1c7bdc4eb3dc1b
+Z e5620a9dcff0b704be870774526d47b0
diff --git a/manifest.uuid b/manifest.uuid
index 7a1cc65c2e..659585bc39 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5d66219c4706e11a9c2a1290224e7c17b19f74657e8239accfb4c6f6be6af9b5
\ No newline at end of file
+1ca3b8cce93e83fabe35c22cb726b8ce0883e0448afa3301ad0b53073055fa8d
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 1b5adc0734..68a337db11 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -5926,25 +5926,31 @@ static int do_meta_command(char *zLine, ShellState *p){
     static const struct {
        const char *zCtrlName;   /* Name of a test-control option */
        int ctrlCode;            /* Integer code for that option */
+       const char *zUsage;      /* Usage notes */
     } aCtrl[] = {
-      { "always",                SQLITE_TESTCTRL_ALWAYS                 },
-      { "assert",                SQLITE_TESTCTRL_ASSERT                 },
-      { "benign_malloc_hooks",   SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS    },
-      { "bitvec_test",           SQLITE_TESTCTRL_BITVEC_TEST            },
-      { "byteorder",             SQLITE_TESTCTRL_BYTEORDER              },
-      { "fault_install",         SQLITE_TESTCTRL_FAULT_INSTALL          },
-      { "imposter",              SQLITE_TESTCTRL_IMPOSTER               },
-      { "iskeyword",             SQLITE_TESTCTRL_ISKEYWORD              },
-      { "never_corrupt",         SQLITE_TESTCTRL_NEVER_CORRUPT          },
-      { "optimizations",         SQLITE_TESTCTRL_OPTIMIZATIONS          },
-      { "pending_byte",          SQLITE_TESTCTRL_PENDING_BYTE           },
-      { "prng_reset",            SQLITE_TESTCTRL_PRNG_RESET             },
-      { "prng_restore",          SQLITE_TESTCTRL_PRNG_RESTORE           },
-      { "prng_save",             SQLITE_TESTCTRL_PRNG_SAVE              },
-      { "reserve",               SQLITE_TESTCTRL_RESERVE                },
+      { "always",             SQLITE_TESTCTRL_ALWAYS,        "BOOLEAN"            },
+      { "assert",             SQLITE_TESTCTRL_ASSERT,        "BOOLEAN"            },
+    /*{ "benign_malloc_hooks",SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS, ""          },*/
+    /*{ "bitvec_test",        SQLITE_TESTCTRL_BITVEC_TEST,   ""                },*/
+      { "byteorder",          SQLITE_TESTCTRL_BYTEORDER,     ""                   },
+    /*{ "fault_install",      SQLITE_TESTCTRL_FAULT_INSTALL, ""                }, */
+      { "imposter",           SQLITE_TESTCTRL_IMPOSTER,   "SCHEMA ON/OFF ROOTPAGE"},
+#ifdef SQLITE_N_KEYWORD
+      { "iskeyword",          SQLITE_TESTCTRL_ISKEYWORD,     "IDENTIFIER"         },
+#endif
+      { "localtime_fault",    SQLITE_TESTCTRL_LOCALTIME_FAULT,"BOOLEAN"           },
+      { "never_corrupt",      SQLITE_TESTCTRL_NEVER_CORRUPT, "BOOLEAN"            },
+      { "optimizations",      SQLITE_TESTCTRL_OPTIMIZATIONS, "DISABLE-MASK"       },
+      { "pending_byte",       SQLITE_TESTCTRL_PENDING_BYTE,  "OFFSET  "           },
+      { "prng_reset",         SQLITE_TESTCTRL_PRNG_RESET,    ""                   },
+      { "prng_restore",       SQLITE_TESTCTRL_PRNG_RESTORE,  ""                   },
+      { "prng_save",          SQLITE_TESTCTRL_PRNG_SAVE,     ""                   },
+      { "reserve",            SQLITE_TESTCTRL_RESERVE,       "BYTES-OF-RESERVE"   },
     };
     int testctrl = -1;
-    int rc2 = 0;
+    int iCtrl = -1;
+    int rc2 = 0;    /* 0: usage.  1: %d  2: %x  3: no-output */
+    int isOk = 0;
     int i, n2;
     open_db(p, 0);
     const char *zCmd = nArg>=2 ? azArg[1] : "help";
@@ -5959,7 +5965,8 @@ static int do_meta_command(char *zLine, ShellState *p){
     if( strcmp(zCmd,"help")==0 ){
       utf8_printf(p->out, "Available test-controls:\n");
       for(i=0; i<ArraySize(aCtrl); i++){
-        utf8_printf(p->out, "  .testctrl %s\n", aCtrl[i].zCtrlName);
+        utf8_printf(p->out, "  .testctrl %s %s\n",
+                    aCtrl[i].zCtrlName, aCtrl[i].zUsage);
       }
       rc = 1;
       goto meta_command_exit;
@@ -5972,6 +5979,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       if( strncmp(zCmd, aCtrl[i].zCtrlName, n2)==0 ){
         if( testctrl<0 ){
           testctrl = aCtrl[i].ctrlCode;
+          iCtrl = i;
         }else{
           utf8_printf(stderr, "Error: ambiguous test-control: \"%s\"\n"
                               "Use \".testctrl --help\" for help\n", zCmd);
@@ -5980,8 +5988,7 @@ static int do_meta_command(char *zLine, ShellState *p){
         }
       }
     }
-    if( testctrl<0 ) testctrl = (int)integerValue(zCmd);
-    if( (testctrl<SQLITE_TESTCTRL_FIRST) || (testctrl>SQLITE_TESTCTRL_LAST) ){
+    if( testctrl<0 ){
       utf8_printf(stderr,"Error: unknown test-control: %s\n"
                          "Use \".testctrl --help\" for help\n", zCmd);
     }else{
@@ -5993,10 +6000,7 @@ static int do_meta_command(char *zLine, ShellState *p){
           if( nArg==3 ){
             int opt = (int)strtol(azArg[2], 0, 0);
             rc2 = sqlite3_test_control(testctrl, p->db, opt);
-            raw_printf(p->out, "%d (0x%08x)\n", rc2, rc2);
-          } else {
-            utf8_printf(stderr,"Error: testctrl %s takes a single int option\n",
-                    azArg[1]);
+            isOk = 3;
           }
           break;
 
@@ -6007,10 +6011,7 @@ static int do_meta_command(char *zLine, ShellState *p){
         case SQLITE_TESTCTRL_BYTEORDER:
           if( nArg==2 ){
             rc2 = sqlite3_test_control(testctrl);
-            raw_printf(p->out, "%d (0x%08x)\n", rc2, rc2);
-          } else {
-            utf8_printf(stderr,"Error: testctrl %s takes no options\n",
-                        azArg[1]);
+            isOk = testctrl==SQLITE_TESTCTRL_BYTEORDER ? 1 : 3;
           }
           break;
 
@@ -6019,24 +6020,27 @@ static int do_meta_command(char *zLine, ShellState *p){
           if( nArg==3 ){
             unsigned int opt = (unsigned int)integerValue(azArg[2]);
             rc2 = sqlite3_test_control(testctrl, opt);
-            raw_printf(p->out, "%d (0x%08x)\n", rc2, rc2);
-          } else {
-            utf8_printf(stderr,"Error: testctrl %s takes a single unsigned"
-                           " int option\n", azArg[1]);
+            isOk = 3;
           }
           break;
 
         /* sqlite3_test_control(int, int) */
         case SQLITE_TESTCTRL_ASSERT:
         case SQLITE_TESTCTRL_ALWAYS:
+          if( nArg==3 ){
+            int opt = booleanValue(azArg[2]);
+            rc2 = sqlite3_test_control(testctrl, opt);
+            isOk = 1;
+          }
+          break;
+
+        /* sqlite3_test_control(int, int) */
+        case SQLITE_TESTCTRL_LOCALTIME_FAULT:
         case SQLITE_TESTCTRL_NEVER_CORRUPT:
           if( nArg==3 ){
             int opt = booleanValue(azArg[2]);
             rc2 = sqlite3_test_control(testctrl, opt);
-            raw_printf(p->out, "%d (0x%08x)\n", rc2, rc2);
-          } else {
-            utf8_printf(stderr,"Error: testctrl %s takes a single int option\n",
-                            azArg[1]);
+            isOk = 3;
           }
           break;
 
@@ -6046,11 +6050,7 @@ static int do_meta_command(char *zLine, ShellState *p){
           if( nArg==3 ){
             const char *opt = azArg[2];
             rc2 = sqlite3_test_control(testctrl, opt);
-            raw_printf(p->out, "%d (0x%08x)\n", rc2, rc2);
-          } else {
-            utf8_printf(stderr,
-                        "Error: testctrl %s takes a single char * option\n",
-                        azArg[1]);
+            isOk = 1;
           }
           break;
 #endif
@@ -6061,22 +6061,19 @@ static int do_meta_command(char *zLine, ShellState *p){
                           azArg[2],
                           integerValue(azArg[3]),
                           integerValue(azArg[4]));
-            raw_printf(p->out, "%d (0x%08x)\n", rc2, rc2);
-          }else{
-            raw_printf(stderr,"Usage: .testctrl imposter dbName onoff tnum\n");
+            isOk = 3;
           }
           break;
-
-        case SQLITE_TESTCTRL_BITVEC_TEST:
-        case SQLITE_TESTCTRL_FAULT_INSTALL:
-        case SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS:
-        default:
-          utf8_printf(stderr,
-                      "Error: CLI support for testctrl %s not implemented\n",
-                      azArg[1]);
-          break;
       }
     }
+    if( isOk==0 && iCtrl>=0 ){
+      utf8_printf(p->out, "Usage: .testctrl %s %s\n", zCmd, aCtrl[iCtrl].zUsage);
+      rc = 1;
+    }else if( isOk==1 ){
+      raw_printf(p->out, "%d\n", rc2);
+    }else if( isOk==2 ){
+      raw_printf(p->out, "0x%08x\n", rc2);
+    }
   }else
 #endif /* !defined(SQLITE_UNTESTABLE) */
 

From 3dbb4acf0715336c867758d6de81e57a2a1b8e96 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 16 Nov 2017 19:04:33 +0000
Subject: [PATCH 090/488] Add a missing "finish_test" to the end of the
 stmtvtab1.test script.

FossilOrigin-Name: e0b5c0585e8530bd516b340093a46d8e358a731facb78da22d3208633c852804
---
 manifest            | 12 ++++++------
 manifest.uuid       |  2 +-
 test/stmtvtab1.test |  2 ++
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index 31f6458b7c..df822c015e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Futher\simprovements\sto\sthe\s.testctrl\sinterface\sin\sthe\sshell.
-D 2017-11-15T19:14:08.134
+C Add\sa\smissing\s"finish_test"\sto\sthe\send\sof\sthe\sstmtvtab1.test\sscript.
+D 2017-11-16T19:04:33.787
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -1249,7 +1249,7 @@ F test/sqllog.test 6af6cb0b09f4e44e1917e06ce85be7670302517a
 F test/stat.test f8f1279ffffabe6df825723af18cc6e0ae70a893
 F test/statfault.test f525a7bf633e50afd027700e9a486090684b1ac1
 F test/stmt.test 64844332db69cf1a735fcb3e11548557fc95392f
-F test/stmtvtab1.test acc3c40f484f2c4922e270724383d715abb9d69676da907a9c64f31c54f2ef9f
+F test/stmtvtab1.test 6873dfb24f8e79cbb5b799b95c2e4349060eb7a3b811982749a84b359468e2d5
 F test/subjournal.test 8d4e2572c0ee9a15549f0d8e40863161295107e52f07a3e8012a2e1fdd093c49
 F test/subquery.test d7268d193dd33d5505df965399d3a594e76ae13f
 F test/subquery2.test 438f8a7da1457277b22e4176510f7659b286995f
@@ -1677,7 +1677,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5d66219c4706e11a9c2a1290224e7c17b19f74657e8239accfb4c6f6be6af9b5
-R bb554ed3e0e111141682146f45553c3a
+P 1ca3b8cce93e83fabe35c22cb726b8ce0883e0448afa3301ad0b53073055fa8d
+R 43f6376592b4d26341e49684bbb727c1
 U drh
-Z e5620a9dcff0b704be870774526d47b0
+Z 7e47c726c44355ca56f3a93b39a4aaed
diff --git a/manifest.uuid b/manifest.uuid
index 659585bc39..4216e3c295 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-1ca3b8cce93e83fabe35c22cb726b8ce0883e0448afa3301ad0b53073055fa8d
\ No newline at end of file
+e0b5c0585e8530bd516b340093a46d8e358a731facb78da22d3208633c852804
\ No newline at end of file
diff --git a/test/stmtvtab1.test b/test/stmtvtab1.test
index 4937102091..e8a79fdacd 100644
--- a/test/stmtvtab1.test
+++ b/test/stmtvtab1.test
@@ -78,3 +78,5 @@ db cache flush
 do_execsql_test stmtvtab1-160 {
   SELECT * FROM sqlite_stmt WHERE NOT busy;
 } {}
+
+finish_test

From cda185d3a268f17faf85968b036070d45bd030fb Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 16 Nov 2017 20:48:47 +0000
Subject: [PATCH 091/488] Ensure that empty space on pages is zeroed before
 they are written to disk. This helps with compressed databases, and stops
 valgrind complaining about uninitialized bytes and write().

FossilOrigin-Name: 5ffec5db33137251090d45e2ca7e3e7823e3215ae89822d905923424ceba2ac2
---
 ext/lsm1/lsm_sorted.c | 66 +++++++++++++++++++++++++++----------------
 manifest              | 14 ++++-----
 manifest.uuid         |  2 +-
 3 files changed, 50 insertions(+), 32 deletions(-)

diff --git a/ext/lsm1/lsm_sorted.c b/ext/lsm1/lsm_sorted.c
index f479f4ce8c..9eff2d08cc 100644
--- a/ext/lsm1/lsm_sorted.c
+++ b/ext/lsm1/lsm_sorted.c
@@ -92,7 +92,7 @@
 #define SEGMENT_POINTER_OFFSET(pgsz)        ((pgsz) - 2 - 2 - 8)
 #define SEGMENT_CELLPTR_OFFSET(pgsz, iCell) ((pgsz) - 2 - 2 - 8 - 2 - (iCell)*2)
 
-#define SEGMENT_EOF(pgsz, nEntry) SEGMENT_CELLPTR_OFFSET(pgsz, nEntry)
+#define SEGMENT_EOF(pgsz, nEntry) SEGMENT_CELLPTR_OFFSET(pgsz, nEntry-1)
 
 #define SEGMENT_BTREE_FLAG     0x0001
 #define PGFTR_SKIP_NEXT_FLAG   0x0002
@@ -3999,6 +3999,11 @@ static int mergeWorkerWrite(
     ** marked read-only, advance to the next page of the output run. */
     iOff = pMerge->iOutputOff;
     if( iOff<0 || pPg==0 || iOff+nHdr > SEGMENT_EOF(nData, nRec+1) ){
+      if( iOff>=0 && pPg ){
+        /* Zero any free space on the page */
+        assert( aData );
+        memset(&aData[iOff], 0, SEGMENT_EOF(nData, nRec)-iOff);
+      }
       iFPtr = (int)*pMW->pCsr->pPrevMergePtr;
       iRPtr = iPtr - iFPtr;
       iOff = 0;
@@ -4069,36 +4074,49 @@ static void mergeWorkerShutdown(MergeWorker *pMW, int *pRc){
   /* Unless the merge has finished, save the cursor position in the
   ** Merge.aInput[] array. See function mergeWorkerInit() for the 
   ** code to restore a cursor position based on aInput[].  */
-  if( rc==LSM_OK && pCsr && lsmMCursorValid(pCsr) ){
+  if( rc==LSM_OK && pCsr ){
     Merge *pMerge = pMW->pLevel->pMerge;
-    int bBtree = (pCsr->pBtCsr!=0);
-    int iPtr;
+    if( lsmMCursorValid(pCsr) ){
+      int bBtree = (pCsr->pBtCsr!=0);
+      int iPtr;
 
-    /* pMerge->nInput==0 indicates that this is a FlushTree() operation. */
-    assert( pMerge->nInput==0 || pMW->pLevel->nRight>0 );
-    assert( pMerge->nInput==0 || pMerge->nInput==(pCsr->nPtr+bBtree) );
+      /* pMerge->nInput==0 indicates that this is a FlushTree() operation. */
+      assert( pMerge->nInput==0 || pMW->pLevel->nRight>0 );
+      assert( pMerge->nInput==0 || pMerge->nInput==(pCsr->nPtr+bBtree) );
 
-    for(i=0; i<(pMerge->nInput-bBtree); i++){
-      SegmentPtr *pPtr = &pCsr->aPtr[i];
-      if( pPtr->pPg ){
-        pMerge->aInput[i].iPg = lsmFsPageNumber(pPtr->pPg);
-        pMerge->aInput[i].iCell = pPtr->iCell;
+      for(i=0; i<(pMerge->nInput-bBtree); i++){
+        SegmentPtr *pPtr = &pCsr->aPtr[i];
+        if( pPtr->pPg ){
+          pMerge->aInput[i].iPg = lsmFsPageNumber(pPtr->pPg);
+          pMerge->aInput[i].iCell = pPtr->iCell;
+        }else{
+          pMerge->aInput[i].iPg = 0;
+          pMerge->aInput[i].iCell = 0;
+        }
+      }
+      if( bBtree && pMerge->nInput ){
+        assert( i==pCsr->nPtr );
+        btreeCursorPosition(pCsr->pBtCsr, &pMerge->aInput[i]);
+      }
+
+      /* Store the location of the split-key */
+      iPtr = pCsr->aTree[1] - CURSOR_DATA_SEGMENT;
+      if( iPtr<pCsr->nPtr ){
+        pMerge->splitkey = pMerge->aInput[iPtr];
       }else{
-        pMerge->aInput[i].iPg = 0;
-        pMerge->aInput[i].iCell = 0;
+        btreeCursorSplitkey(pCsr->pBtCsr, &pMerge->splitkey);
       }
     }
-    if( bBtree && pMerge->nInput ){
-      assert( i==pCsr->nPtr );
-      btreeCursorPosition(pCsr->pBtCsr, &pMerge->aInput[i]);
-    }
 
-    /* Store the location of the split-key */
-    iPtr = pCsr->aTree[1] - CURSOR_DATA_SEGMENT;
-    if( iPtr<pCsr->nPtr ){
-      pMerge->splitkey = pMerge->aInput[iPtr];
-    }else{
-      btreeCursorSplitkey(pCsr->pBtCsr, &pMerge->splitkey);
+    /* Zero any free space left on the final page. This helps with
+    ** compression if using a compression hook. And prevents valgrind
+    ** from complaining about uninitialized byte passed to write(). */
+    if( pMW->pPage ){
+      int nData;
+      u8 *aData = fsPageData(pMW->pPage, &nData);
+      int iOff = pMerge->iOutputOff;
+      int iEof = SEGMENT_EOF(nData, pageGetNRec(aData, nData));
+      memset(&aData[iOff], 0, iEof - iOff);
     }
     
     pMerge->iOutputOff = -1;
diff --git a/manifest b/manifest
index df822c015e..d601661e29 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sa\smissing\s"finish_test"\sto\sthe\send\sof\sthe\sstmtvtab1.test\sscript.
-D 2017-11-16T19:04:33.787
+C Ensure\sthat\sempty\sspace\son\spages\sis\szeroed\sbefore\sthey\sare\swritten\sto\sdisk.\nThis\shelps\swith\scompressed\sdatabases,\sand\sstops\svalgrind\scomplaining\sabout\nuninitialized\sbytes\sand\swrite().
+D 2017-11-16T20:48:47.980
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -246,7 +246,7 @@ F ext/lsm1/lsm_main.c 15e73ccdafdd44ddeefc29e332079d88ba8f00c12c797b3c2b63d3171b
 F ext/lsm1/lsm_mem.c 4c51ea9fa285ee6e35301b33491642d071740a0a
 F ext/lsm1/lsm_mutex.c 378edf0a2b142b4f7640ee982df06d50b98788ea
 F ext/lsm1/lsm_shared.c 76adfc1ed9ffebaf92746dde4b370ccc48143ca8b05b563816eadd2aadf1c525
-F ext/lsm1/lsm_sorted.c a04518dfbfff0171fafb152a46e9fe9f45e1edbf3570e4533dd58ddb6567f0c9
+F ext/lsm1/lsm_sorted.c 8f899fb64a4c736ff3c27d5126c7ce181129ddffde947fe5fb657a7a413f470b
 F ext/lsm1/lsm_str.c 65e361b488c87b10bf3e5c0070b14ffc602cf84f094880bece77bbf6678bca82
 F ext/lsm1/lsm_tree.c 682679d7ef2b8b6f2fe77aeb532c8d29695bca671c220b0abac77069de5fb9fb
 F ext/lsm1/lsm_unix.c 57361bcf5b1a1a028f5d66571ee490e9064d2cfb145a2cc9e5ddade467bb551b
@@ -1677,7 +1677,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 1ca3b8cce93e83fabe35c22cb726b8ce0883e0448afa3301ad0b53073055fa8d
-R 43f6376592b4d26341e49684bbb727c1
-U drh
-Z 7e47c726c44355ca56f3a93b39a4aaed
+P e0b5c0585e8530bd516b340093a46d8e358a731facb78da22d3208633c852804
+R a54da776e28f1875d13b67b86731a30b
+U dan
+Z 4bb1ae38bdfadd3dc2e110f3406150e8
diff --git a/manifest.uuid b/manifest.uuid
index 4216e3c295..39825a308f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e0b5c0585e8530bd516b340093a46d8e358a731facb78da22d3208633c852804
\ No newline at end of file
+5ffec5db33137251090d45e2ca7e3e7823e3215ae89822d905923424ceba2ac2
\ No newline at end of file

From 6cbc5074e8ff6ec0b4953573c3cbfe015ba1f779 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 17 Nov 2017 08:20:10 +0000
Subject: [PATCH 092/488] Avoid creating a master journal if all or all but one
 of the databases in the transaction is a temp file.

FossilOrigin-Name: 355d1232fd7314723afaa8d6b8b73506b09cbba1113a88b10204ba89be993508
---
 manifest           | 17 +++++-----
 manifest.uuid      |  2 +-
 src/vdbeaux.c      |  1 +
 test/mjournal.test | 77 +++++++++++++++++++++++++++++++++++++++++++++-
 4 files changed, 88 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index d601661e29..8fa35294e4 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Ensure\sthat\sempty\sspace\son\spages\sis\szeroed\sbefore\sthey\sare\swritten\sto\sdisk.\nThis\shelps\swith\scompressed\sdatabases,\sand\sstops\svalgrind\scomplaining\sabout\nuninitialized\sbytes\sand\swrite().
-D 2017-11-16T20:48:47.980
+C Avoid\screating\sa\smaster\sjournal\sif\sall\sor\sall\sbut\sone\sof\sthe\sdatabases\sin\sthe\ntransaction\sis\sa\stemp\sfile.
+D 2017-11-17T08:20:10.452
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -543,7 +543,7 @@ F src/vdbe.c 3393b508d9ad084ffce232a7c53e375ef5ac99b50b685c5131fcdfce97a9d534
 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97
 F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9
 F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1
-F src/vdbeaux.c 9521a9364e68edad3c8d05ae63395d076724bed1c878c9b13fab61ada54e7d2a
+F src/vdbeaux.c b02a1f842c0e916285643b8475b7189f10b76f9e7edb5e2353a913c7980f90b5
 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191
 F src/vdbemem.c 2ef9e66b301a1e575e32966c4c0fd4844e8eea37a2f02bae78c4f68f50a6ab30
 F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f
@@ -1062,7 +1062,7 @@ F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91
 F test/misc7.test edd0b63e2ee29a256900b0514f6fff27e19e9bb2
 F test/misc8.test ba03aaa08f02d62fbb8d3b2f5595c1b33aa9bbc5
 F test/misuse.test 9e7f78402005e833af71dcab32d048003869eca5abcaccc985d4f8dc1d86bcc7
-F test/mjournal.test 68b749956f9a179e7e633a3958b48a5a905d28d30c7ec88f3f26dc6f220129db
+F test/mjournal.test 946d7161cabbc45ecc818d7fa38c1925bbf36e5534a08a81fa0203ee7ced7d54
 F test/mmap1.test d2cfc1635171c434dcff0ece2f1c8e0a658807ce
 F test/mmap2.test 9d6dd9ddb4ad2379f29cc78f38ce1e63ed418022
 F test/mmap3.test b3c297e78e6a8520aafcc1a8f140535594c9086e
@@ -1677,7 +1677,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e0b5c0585e8530bd516b340093a46d8e358a731facb78da22d3208633c852804
-R a54da776e28f1875d13b67b86731a30b
+P 5ffec5db33137251090d45e2ca7e3e7823e3215ae89822d905923424ceba2ac2
+R 549acd152a27430f44fc7371aab0d342
+T *branch * master-journal-temp-files
+T *sym-master-journal-temp-files *
+T -sym-trunk *
 U dan
-Z 4bb1ae38bdfadd3dc2e110f3406150e8
+Z adbdc7db93ebd9a4eb2724ff9604ae84
diff --git a/manifest.uuid b/manifest.uuid
index 39825a308f..52e9077fed 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5ffec5db33137251090d45e2ca7e3e7823e3215ae89822d905923424ceba2ac2
\ No newline at end of file
+355d1232fd7314723afaa8d6b8b73506b09cbba1113a88b10204ba89be993508
\ No newline at end of file
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index e057a5df5f..e73a339f81 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -2268,6 +2268,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){
       pPager = sqlite3BtreePager(pBt);
       if( db->aDb[i].safety_level!=PAGER_SYNCHRONOUS_OFF
        && aMJNeeded[sqlite3PagerGetJournalMode(pPager)]
+       && sqlite3PagerIsMemdb(pPager)==0
       ){ 
         assert( i!=1 );
         nTrans++;
diff --git a/test/mjournal.test b/test/mjournal.test
index aab2c08b51..cf6ef5f7b5 100644
--- a/test/mjournal.test
+++ b/test/mjournal.test
@@ -79,5 +79,80 @@ do_execsql_test 1.6 {
   SELECT * FROM t1;
 }
 
-  
+#-------------------------------------------------------------------------
+# Check that master journals are not created if the transaction involves
+# multiple temp files.
+#
+db close
+testvfs tvfs
+tvfs filter xOpen
+tvfs script open_cb
+set ::open ""
+proc open_cb {method file arglist} {
+  lappend ::open $file
+}
+
+proc contains_mj {} {
+  foreach f $::open {
+    set t [file tail $f]
+    if {[string match *mj* $t]} { return 1 }
+  }
+  return 0
+}
+
+# Like [do_execsql_test], except that a boolean indicating whether or
+# not a master journal file was opened ([file tail] contains "mj") or
+# not. Example:
+#
+#   do_hasmj_test 1.0 { SELECT 'a', 'b' } {0 a b}
+#
+proc do_hasmj_test {tn sql expected} {
+  set ::open [list]
+  uplevel [list do_test $tn [subst -nocommands {
+    set res [execsql "$sql"]
+    concat [contains_mj] [set res]
+  }] [list {*}$expected]]
+}
+
+forcedelete test.db
+forcedelete test.db2
+forcedelete test.db3
+sqlite3 db test.db -vfs tvfs
+
+do_execsql_test 2.0 {
+  ATTACH 'test.db2' AS dbfile;
+  ATTACH ''         AS dbtemp;
+  ATTACH ':memory:'  AS dbmem;
+
+  CREATE TABLE t1(x);
+  CREATE TABLE dbfile.t2(x);
+  CREATE TABLE dbtemp.t3(x);
+  CREATE TABLE dbmem.t4(x);
+}
+
+# Two real files.
+do_hasmj_test 2.1 {
+  BEGIN;
+    INSERT INTO t1 VALUES(1);
+    INSERT INTO t2 VALUES(1);
+  COMMIT;
+} {1}
+
+# One real, one temp file.
+do_hasmj_test 2.2 {
+  BEGIN;
+    INSERT INTO t1 VALUES(1);
+    INSERT INTO t3 VALUES(1);
+  COMMIT;
+} {0}
+
+# One file, one :memory: db.
+do_hasmj_test 2.3 {
+  BEGIN;
+    INSERT INTO t1 VALUES(1);
+    INSERT INTO t4 VALUES(1);
+  COMMIT;
+} {0}
+
 finish_test
+

From 7fc0ba0f4cb864cfb518aae35b6fb4adbe7a12f4 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 17 Nov 2017 15:02:00 +0000
Subject: [PATCH 093/488] Clarification of comments on sqlite3FindInIndex(). 
 No changes to code.

FossilOrigin-Name: 071cabd23cd010180711a138f891a0e358031dd128532def4f62c5764651bace
---
 manifest      | 15 +++++++--------
 manifest.uuid |  2 +-
 src/expr.c    | 13 ++++++-------
 3 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/manifest b/manifest
index 103854a9cb..a59ee6a275 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Do\snot\scount\stemporary\sdatabases\sthat\shave\sbeen\sattached\susing\sATTACH\swhen\nfiguring\sout\sif\sa\smaster-journal\sfile\sis\srequired\sby\sa\stransaction.
-D 2017-11-17T13:21:12.384
+C Clarification\sof\scomments\son\ssqlite3FindInIndex().\s\sNo\schanges\sto\scode.
+D 2017-11-17T15:02:00.058
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -423,7 +423,7 @@ F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74
 F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
 F src/delete.c e6a70fb58f6628f0ffc6d7221a6702c0d7b342c82520385b3996b364c22e0cb3
-F src/expr.c 5257a9157f22f048ddcce5cd494d39633e89c2a4769671311b3e7875d262f746
+F src/expr.c fe11b91bb65b869143bd42023427c4429778ae42c0a0db7762f68f75b347a958
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331
 F src/func.c 0fb9a2d678d3c8aba89b46468b309cd7e8fa9806a369a30aa89024660845bb13
@@ -1677,8 +1677,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5ffec5db33137251090d45e2ca7e3e7823e3215ae89822d905923424ceba2ac2 355d1232fd7314723afaa8d6b8b73506b09cbba1113a88b10204ba89be993508
-R 549acd152a27430f44fc7371aab0d342
-T +closed 355d1232fd7314723afaa8d6b8b73506b09cbba1113a88b10204ba89be993508
-U dan
-Z eb504c2f876fdf46c1053782d9bac493
+P 93e012a317c8a4bfb84327616a597acabfcb24417197eefdccb8031bcf64e0c0
+R 9e3518c1c08ebf61892be3d9ca3ce745
+U drh
+Z 183368b52c0a93e109b0b85758c70894
diff --git a/manifest.uuid b/manifest.uuid
index 39dbcda631..cea62a5a5c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-93e012a317c8a4bfb84327616a597acabfcb24417197eefdccb8031bcf64e0c0
\ No newline at end of file
+071cabd23cd010180711a138f891a0e358031dd128532def4f62c5764651bace
\ No newline at end of file
diff --git a/src/expr.c b/src/expr.c
index 5027f994db..524e539349 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -2186,16 +2186,15 @@ static int sqlite3InRhsIsConstant(Expr *pIn){
 ** pX->iTable made to point to the ephemeral table instead of an
 ** existing table.
 **
-** The inFlags parameter must contain exactly one of the bits
-** IN_INDEX_MEMBERSHIP or IN_INDEX_LOOP.  If inFlags contains
-** IN_INDEX_MEMBERSHIP, then the generated table will be used for a
-** fast membership test.  When the IN_INDEX_LOOP bit is set, the
-** IN index will be used to loop over all values of the RHS of the
-** IN operator.
+** The inFlags parameter must contain, at a minimum, one of the bits
+** IN_INDEX_MEMBERSHIP or IN_INDEX_LOOP but not both.  If inFlags contains
+** IN_INDEX_MEMBERSHIP, then the generated table will be used for a fast
+** membership test.  When the IN_INDEX_LOOP bit is set, the IN index will
+** be used to loop over all values of the RHS of the IN operator.
 **
 ** When IN_INDEX_LOOP is used (and the b-tree will be used to iterate
 ** through the set members) then the b-tree must not contain duplicates.
-** An epheremal table must be used unless the selected columns are guaranteed
+** An epheremal table will be created unless the selected columns are guaranteed
 ** to be unique - either because it is an INTEGER PRIMARY KEY or due to
 ** a UNIQUE constraint or index.
 **

From 9b1ecb67c3a5bd5a049b4e2c0d027405e991febd Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 17 Nov 2017 17:32:40 +0000
Subject: [PATCH 094/488] New assert() statements in the rowvalue IN expression
 processing.

FossilOrigin-Name: 00c328317473cee8fd7bd92c58409a356337b727cfa562bd8de59350d978769c
---
 manifest        | 12 ++++++------
 manifest.uuid   |  2 +-
 src/wherecode.c | 29 +++++++++++++++++++++++++++++
 3 files changed, 36 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index a59ee6a275..860e60f5e2 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Clarification\sof\scomments\son\ssqlite3FindInIndex().\s\sNo\schanges\sto\scode.
-D 2017-11-17T15:02:00.058
+C New\sassert()\sstatements\sin\sthe\srowvalue\sIN\sexpression\sprocessing.
+D 2017-11-17T17:32:40.141
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -555,7 +555,7 @@ F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f
 F src/where.c 031a80bcafe93934fd7052f3031c9e7eb36b61754c6c84d6bf0833184abad3db
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
-F src/wherecode.c 4a117dd5886616d074f7b6589c23bf742f5a9858d6ffdaf8b9d1f76ab06245d2
+F src/wherecode.c 8605c0ca0c34d4692011cf68a5f4cfc85352c1df917dc6eada320cecc4f5ea73
 F src/whereexpr.c 427ea8e96ec24f2a7814c67b8024ad664a9c7656264c4566c34743cb23186e46
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd
@@ -1677,7 +1677,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 93e012a317c8a4bfb84327616a597acabfcb24417197eefdccb8031bcf64e0c0
-R 9e3518c1c08ebf61892be3d9ca3ce745
+P 071cabd23cd010180711a138f891a0e358031dd128532def4f62c5764651bace
+R e07fbdd249f84df3df0af0935ae0894c
 U drh
-Z 183368b52c0a93e109b0b85758c70894
+Z dfffcc7482c8c9b6b4a114214fd236e3
diff --git a/manifest.uuid b/manifest.uuid
index cea62a5a5c..e60558e452 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-071cabd23cd010180711a138f891a0e358031dd128532def4f62c5764651bace
\ No newline at end of file
+00c328317473cee8fd7bd92c58409a356337b727cfa562bd8de59350d978769c
\ No newline at end of file
diff --git a/src/wherecode.c b/src/wherecode.c
index da5c686a95..cc2759eeaf 100644
--- a/src/wherecode.c
+++ b/src/wherecode.c
@@ -377,6 +377,27 @@ static void updateRangeAffinityStr(
   }
 }
 
+#ifdef SQLITE_DEBUG
+/* Return true if the pSub ExprList is a subset of pMain.  The terms
+** of pSub can be in a different order from pMain.  The only requirement
+** is that every term in pSub must exist somewhere in pMain.
+**
+** Return false if pSub contains any term that is not found in pMain.
+*/
+static int exprListSubset(ExprList *pSub, ExprList *pMain){
+  int i, j;
+  for(i=0; i<pSub->nExpr; i++){
+    Expr *p = pSub->a[i].pExpr;
+    for(j=0; j<pMain->nExpr; j++){
+      if( sqlite3ExprCompare(0, p, pMain->a[j].pExpr, 0)==0 ) break;
+    }
+    if( j>=pMain->nExpr ) return 0;
+  }
+  return 1;
+}
+#endif /* SQLITE_DEBUG */
+
+
 /*
 ** Generate code for a single equality term of the WHERE clause.  An equality
 ** term can be either X=expr or X IN (...).   pTerm is the term to be 
@@ -463,6 +484,14 @@ static int codeEqualityTerm(
           pLhs = sqlite3ExprListAppend(pParse, pLhs, pNewLhs);
         }
       }
+ 
+      /* pRhs should be a subset of pOrigRhs (though possibly in a different
+      ** order).  And pLhs should be a subset of pOrigLhs.  To put it 
+      ** another way:  Every term of pRhs should exist in pOrigRhs and
+      ** every term of pLhs should exist in pOrigLhs. */
+      assert( db->mallocFailed || exprListSubset(pRhs, pOrigRhs) );
+      assert( db->mallocFailed || exprListSubset(pLhs, pOrigLhs) );
+
       if( !db->mallocFailed ){
         Expr *pLeft = pX->pLeft;
 

From 6bcabfe1a38ae090c276bb99ff73c8f6e4e70b3d Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 17 Nov 2017 20:07:19 +0000
Subject: [PATCH 095/488] Add some missing "finish_test" lines to the end of
 test scripts.

FossilOrigin-Name: c21406ab3281480d3eddca0cdf5aea3abc224425ee52c10eed3ff702a0ae5c26
---
 manifest                  | 24 ++++++++++++------------
 manifest.uuid             |  2 +-
 test/tkt-26ff0c2d1e.test  |  2 ++
 test/tkt-7a31705a7e6.test |  2 ++
 test/tkt-a8a0d2996a.test  |  2 ++
 test/tkt3334.test         |  2 ++
 test/vacuum4.test         |  2 ++
 test/varint.test          |  2 ++
 8 files changed, 25 insertions(+), 13 deletions(-)

diff --git a/manifest b/manifest
index 860e60f5e2..5615a01c67 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C New\sassert()\sstatements\sin\sthe\srowvalue\sIN\sexpression\sprocessing.
-D 2017-11-17T17:32:40.141
+C Add\ssome\smissing\s"finish_test"\slines\sto\sthe\send\sof\stest\sscripts.
+D 2017-11-17T20:07:19.718
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -1292,7 +1292,7 @@ F test/threadtest3.c 38a612ea62854349ed66372f330a40d73c5cf956
 F test/threadtest4.c c1e67136ceb6c7ec8184e56ac61db28f96bd2925
 F test/time-wordcount.sh 8e0b0f8109367827ad5d58f5cc849705731e4b90
 F test/tkt-02a8e81d44.test 6c80d9c7514e2a42d4918bf87bf6bc54f379110c
-F test/tkt-26ff0c2d1e.test 888324e751512972c6e0d1a09df740d8f5aaf660
+F test/tkt-26ff0c2d1e.test c15bec890c4d226c0da2f35ff30f9e84c169cfef90e73a8cb5cec11d723dfa96
 F test/tkt-2a5629202f.test 0521bd25658428baa26665aa53ffed9367d33af2
 F test/tkt-2d1a5c67d.test be1326f3061caec85085f4c9ee4490561ca037c0
 F test/tkt-2ea2425d34.test 1cf13e6f75d149b3209a0cb32927a82d3d79fb28
@@ -1314,7 +1314,7 @@ F test/tkt-5ee23731f.test 9db6e1d7209dc0794948b260d6f82b2b1de83a9f
 F test/tkt-6bfb98dfc0.test 24780633627b5cfc0635a5500c2389ebfb563336
 F test/tkt-752e1646fc.test ea78d88d14fe9866bdd991c634483334639e13bf
 F test/tkt-78e04e52ea.test 1b2e6bf4f1d9887b216b6da774e5f25915ec8118
-F test/tkt-7a31705a7e6.test e75a2bba4eec801b92c8040eb22096ac6d35e844
+F test/tkt-7a31705a7e6.test 9e9c057b6a9497c8f7ba7b16871029414ccf6550e7345d9085d6d71c9a56bb6f
 F test/tkt-7bbfb7d442.test 7b2cd79c7a17ae6750e75ec1a7846712a69c9d18
 F test/tkt-80ba201079.test 105a721e6aad0ae3c5946d7615d1e4d03f6145b8
 F test/tkt-80e031a00f.test 9ee36348b761bf7c14261e002b75a4c0d5a04d4c
@@ -1327,7 +1327,7 @@ F test/tkt-9a8b09f8e6.test b2ef151d0984b2ebf237760dbeaa50724e5a0667
 F test/tkt-9d68c883.test 16f7cb96781ba579bc2e19bb14b4ad609d9774b6
 F test/tkt-9f2eb3abac.test cb6123ac695a08b4454c3792fbe85108f67fabf8
 F test/tkt-a7b7803e.test 159ef554234fa1f9fb318c751b284bd1cf858da4
-F test/tkt-a8a0d2996a.test eb597379dbcefa24765763d7f682c00cb5924fa9
+F test/tkt-a8a0d2996a.test 76662ff0622c90e7ce7bbcb4d9e1129acddf877d17c3489f2da7f17ddfaad1f4
 F test/tkt-b1d3a2e531.test 8f7576e41ca179289ee1a8fee28386fd8e4b0550
 F test/tkt-b351d95f9.test d14a503c414c5c58fdde3e80f9a3cfef986498c0
 F test/tkt-b72787b1.test a95e8cdad0b98af1853ac7f0afd4ab27b77bf5f3
@@ -1391,7 +1391,7 @@ F test/tkt3121.test 536df66a02838c26a12fe98639354ca1290ca68b
 F test/tkt3201.test f1500ccecc0d578dc4cde7d3242008297c4d59b3
 F test/tkt3292.test 962465a0984a3b8c757efe59c2c59144871ee1dd
 F test/tkt3298.test 20fd8773b825cb602e033aa04f8602e1ebdcd93c
-F test/tkt3334.test ea13a53cb176e90571a76c86605b14a09efe366d
+F test/tkt3334.test 9756631e3c4aa3c416362c279e3c0953a83b7ca8274cb81a13264bb56296d8b0
 F test/tkt3346.test 6f67c3ed7db94dfc5df4f5f0b63809a1f611e01a
 F test/tkt3357.test 77c37c6482b526fe89941ce951c22d011f5922ed
 F test/tkt3419.test 1bbf36d7ea03b638c15804251287c2391f5c1f6b
@@ -1479,10 +1479,10 @@ F test/utf16align.test 54cd35a27c005a9b6e7815d887718780b6a462ae
 F test/vacuum.test ce91c39f7f91a4273bf620efad21086b5aa6ef1d
 F test/vacuum2.test aa048abee196c16c9ba308465494009057b79f9b
 F test/vacuum3.test 77ecdd54592b45a0bcb133339f99f1ae0ae94d0d
-F test/vacuum4.test d3f8ecff345f166911568f397d2432c16d2867d9
+F test/vacuum4.test 7ea76b769fffeb41f925303b04cbcf5a5bbeabe55e4c60ae754ff24eeeb7c010
 F test/vacuum5.test c87234e8ca4107f349da4edbeda3e4ea5adc93f3
 F test/vacuummem.test 7b42abb3208bd82dd23a7536588396f295a314f2
-F test/varint.test ab7b110089a08b9926ed7390e7e97bdefeb74102
+F test/varint.test bbce22cda8fc4d135bcc2b589574be8410614e62
 F test/veryquick.test 57ab846bacf7b90cf4e9a672721ea5c5b669b661
 F test/view.test 765802c7a66d37fabd5ac8e2f2dbe572b43eb9ab
 F test/vtab1.test 8f91b9538d1404c3932293a588c4344218a0c94792d4289bb55e41020e7b3fff
@@ -1677,7 +1677,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 071cabd23cd010180711a138f891a0e358031dd128532def4f62c5764651bace
-R e07fbdd249f84df3df0af0935ae0894c
-U drh
-Z dfffcc7482c8c9b6b4a114214fd236e3
+P 00c328317473cee8fd7bd92c58409a356337b727cfa562bd8de59350d978769c
+R 7da8d3fa9a019a6fbc5c0bb38be51e2d
+U dan
+Z 96ccfa676c7df4dbf371fdb6ce8e1bc3
diff --git a/manifest.uuid b/manifest.uuid
index e60558e452..de5b268438 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-00c328317473cee8fd7bd92c58409a356337b727cfa562bd8de59350d978769c
\ No newline at end of file
+c21406ab3281480d3eddca0cdf5aea3abc224425ee52c10eed3ff702a0ae5c26
\ No newline at end of file
diff --git a/test/tkt-26ff0c2d1e.test b/test/tkt-26ff0c2d1e.test
index 83a4f3d674..d7c5c6bae3 100644
--- a/test/tkt-26ff0c2d1e.test
+++ b/test/tkt-26ff0c2d1e.test
@@ -31,3 +31,5 @@ do_test bug-20100512-3 {
   sqlite3_column_int $STMT 0
 } {555}
 sqlite3_finalize $STMT
+
+finish_test
diff --git a/test/tkt-7a31705a7e6.test b/test/tkt-7a31705a7e6.test
index e3e402917d..fd862484ea 100644
--- a/test/tkt-7a31705a7e6.test
+++ b/test/tkt-7a31705a7e6.test
@@ -23,3 +23,5 @@ do_execsql_test tkt-7a31705a7e6-1.1 {
   CREATE TABLE t2x (b INTEGER PRIMARY KEY);
   SELECT t1.a FROM ((t1 JOIN t2 ON t1.a=t2.a) AS x JOIN t2x ON x.b=t2x.b) as y;
 } {}
+
+finish_test
diff --git a/test/tkt-a8a0d2996a.test b/test/tkt-a8a0d2996a.test
index 6b15e410e7..03c2ee9448 100644
--- a/test/tkt-a8a0d2996a.test
+++ b/test/tkt-a8a0d2996a.test
@@ -91,3 +91,5 @@ do_execsql_test 4.5 {
 do_execsql_test 4.6 {
   SELECT '1234x'/'10y';
 } {123.4}
+
+finish_test
diff --git a/test/tkt3334.test b/test/tkt3334.test
index 5473ab4cf8..3527932de3 100644
--- a/test/tkt3334.test
+++ b/test/tkt3334.test
@@ -82,3 +82,5 @@ do_test tkt3334-1.10 {
     SELECT count(*) FROM (SELECT a FROM t1) WHERE a=1;
   }
 } {3}
+
+finish_test
diff --git a/test/vacuum4.test b/test/vacuum4.test
index 326d037276..bbf0dd4379 100644
--- a/test/vacuum4.test
+++ b/test/vacuum4.test
@@ -65,3 +65,5 @@ do_test vacuum4-1.1 {
     VACUUM;
   }
 } {}
+
+finish_test
diff --git a/test/varint.test b/test/varint.test
index 974e88f2a6..fd0ec0d415 100644
--- a/test/varint.test
+++ b/test/varint.test
@@ -30,3 +30,5 @@ foreach start {0 100 10000 1000000 0x10000000} {
     }
   }
 }
+
+finish_test

From 2410243e9d27b5f1f5862368eb52ded8898250d8 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 17 Nov 2017 21:01:04 +0000
Subject: [PATCH 096/488] Improved fix for ticket [da78413751863] that does not
 require disabling the query flattener as was done in [005d5b870625].  This
 also makes the code generator for vector IN operators a little easier to
 understand.

FossilOrigin-Name: 723f1be3d4a905a6a16333f8ef3e1067dcd4944497b303033c49946fc37c780f
---
 manifest        |  14 ++--
 manifest.uuid   |   2 +-
 src/wherecode.c | 170 +++++++++++++++++++++++++++---------------------
 3 files changed, 104 insertions(+), 82 deletions(-)

diff --git a/manifest b/manifest
index 5615a01c67..b8aae7e011 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\ssome\smissing\s"finish_test"\slines\sto\sthe\send\sof\stest\sscripts.
-D 2017-11-17T20:07:19.718
+C Improved\sfix\sfor\sticket\s[da78413751863]\sthat\sdoes\snot\srequire\sdisabling\sthe\nquery\sflattener\sas\swas\sdone\sin\s[005d5b870625].\s\sThis\salso\smakes\sthe\scode\ngenerator\sfor\svector\sIN\soperators\sa\slittle\seasier\sto\sunderstand.
+D 2017-11-17T21:01:04.980
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -555,7 +555,7 @@ F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f
 F src/where.c 031a80bcafe93934fd7052f3031c9e7eb36b61754c6c84d6bf0833184abad3db
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
-F src/wherecode.c 8605c0ca0c34d4692011cf68a5f4cfc85352c1df917dc6eada320cecc4f5ea73
+F src/wherecode.c 611fcabd05592ed2febd7d182f9621425b0466c5232d70e0981c842d429356d5
 F src/whereexpr.c 427ea8e96ec24f2a7814c67b8024ad664a9c7656264c4566c34743cb23186e46
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd
@@ -1677,7 +1677,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 00c328317473cee8fd7bd92c58409a356337b727cfa562bd8de59350d978769c
-R 7da8d3fa9a019a6fbc5c0bb38be51e2d
-U dan
-Z 96ccfa676c7df4dbf371fdb6ce8e1bc3
+P c21406ab3281480d3eddca0cdf5aea3abc224425ee52c10eed3ff702a0ae5c26
+R 04b5e48559de23ef7902e501e8dcaac8
+U drh
+Z cc87a690f1d098ebb337ab16a1f38777
diff --git a/manifest.uuid b/manifest.uuid
index de5b268438..99c0990a4c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c21406ab3281480d3eddca0cdf5aea3abc224425ee52c10eed3ff702a0ae5c26
\ No newline at end of file
+723f1be3d4a905a6a16333f8ef3e1067dcd4944497b303033c49946fc37c780f
\ No newline at end of file
diff --git a/src/wherecode.c b/src/wherecode.c
index cc2759eeaf..40c5f41182 100644
--- a/src/wherecode.c
+++ b/src/wherecode.c
@@ -377,25 +377,100 @@ static void updateRangeAffinityStr(
   }
 }
 
-#ifdef SQLITE_DEBUG
-/* Return true if the pSub ExprList is a subset of pMain.  The terms
-** of pSub can be in a different order from pMain.  The only requirement
-** is that every term in pSub must exist somewhere in pMain.
+
+/*
+** pX is an expression of the form:  (vector) IN (SELECT ...)
+** In other words, it is a vector IN operator with a SELECT clause on the
+** LHS.  But not all terms in the vector are indexable and the terms might
+** not be in the correct order for indexing.
 **
-** Return false if pSub contains any term that is not found in pMain.
+** This routine makes a copy of the input pX expression and then adjusts
+** the vector on the LHS with corresponding changes to the SELECT so that
+** the vector contains only index terms and those terms are in the correct
+** order.  The modified IN expression is returned.  The caller is responsible
+** for deleting the returned expression.
+**
+** Example:
+**
+**    CREATE TABLE t1(a,b,c,d,e,f);
+**    CREATE INDEX t1x1 ON t1(e,c);
+**    SELECT * FROM t1 WHERE (a,b,c,d,e) IN (SELECT v,w,x,y,z FROM t2)
+**                           \_______________________________________/
+**                                     The pX expression
+**
+** Since only columns e and c can be used with the index, in that order,
+** the modified IN expression that is returned will be:
+**
+**        (e,c) IN (SELECT z,x FROM t2)
+**
+** The reduced pX is different from the original (obviously) and thus is
+** only used for indexing, to improve performance.  The original unaltered
+** IN expression must also be run on each output row for correctness.
 */
-static int exprListSubset(ExprList *pSub, ExprList *pMain){
-  int i, j;
-  for(i=0; i<pSub->nExpr; i++){
-    Expr *p = pSub->a[i].pExpr;
-    for(j=0; j<pMain->nExpr; j++){
-      if( sqlite3ExprCompare(0, p, pMain->a[j].pExpr, 0)==0 ) break;
+static Expr *removeUnindexableInClauseTerms(
+  Parse *pParse,        /* The parsing context */
+  int iEq,              /* Look at loop terms starting here */
+  WhereLoop *pLoop,     /* The current loop */
+  Expr *pX              /* The IN expression to be reduced */
+){
+  sqlite3 *db = pParse->db;
+  Expr *pNew = sqlite3ExprDup(db, pX, 0);
+  if( db->mallocFailed==0 ){
+    ExprList *pOrigRhs = pNew->x.pSelect->pEList;  /* Original unmodified RHS */
+    ExprList *pOrigLhs = pNew->pLeft->x.pList;     /* Original unmodified LHS */
+    ExprList *pRhs = 0;         /* New RHS after modifications */
+    ExprList *pLhs = 0;         /* New LHS after mods */
+    int i;                      /* Loop counter */
+    Select *pSelect;            /* Pointer to the SELECT on the RHS */
+
+    for(i=iEq; i<pLoop->nLTerm; i++){
+      if( pLoop->aLTerm[i]->pExpr==pX ){
+        int iField = pLoop->aLTerm[i]->iField - 1;
+        assert( pOrigRhs->a[iField].pExpr!=0 );
+        pRhs = sqlite3ExprListAppend(pParse, pRhs, pOrigRhs->a[iField].pExpr);
+        pOrigRhs->a[iField].pExpr = 0;
+        assert( pOrigLhs->a[iField].pExpr!=0 );
+        pLhs = sqlite3ExprListAppend(pParse, pLhs, pOrigLhs->a[iField].pExpr);
+        pOrigLhs->a[iField].pExpr = 0;
+      }
     }
-    if( j>=pMain->nExpr ) return 0;
+    sqlite3ExprListDelete(db, pOrigRhs);
+    sqlite3ExprListDelete(db, pOrigLhs);
+    pNew->pLeft->x.pList = pLhs;
+    pNew->x.pSelect->pEList = pRhs;
+    if( pLhs && pLhs->nExpr==1 ){
+      /* Take care here not to generate a TK_VECTOR containing only a
+      ** single value. Since the parser never creates such a vector, some
+      ** of the subroutines do not handle this case.  */
+      Expr *p = pLhs->a[0].pExpr;
+      pLhs->a[0].pExpr = 0;
+      sqlite3ExprDelete(db, pNew->pLeft);
+      pNew->pLeft = p;
+    }
+    pSelect = pNew->x.pSelect;
+    if( pSelect->pOrderBy ){
+      /* If the SELECT statement has an ORDER BY clause, zero the 
+      ** iOrderByCol variables. These are set to non-zero when an 
+      ** ORDER BY term exactly matches one of the terms of the 
+      ** result-set. Since the result-set of the SELECT statement may
+      ** have been modified or reordered, these variables are no longer 
+      ** set correctly.  Since setting them is just an optimization, 
+      ** it's easiest just to zero them here.  */
+      ExprList *pOrderBy = pSelect->pOrderBy;
+      for(i=0; i<pOrderBy->nExpr; i++){
+        pOrderBy->a[i].u.x.iOrderByCol = 0;
+      }
+    }
+
+#if 0
+    printf("For indexing, change the IN expr:\n");
+    sqlite3TreeViewExpr(0, pX, 0);
+    printf("Into:\n");
+    sqlite3TreeViewExpr(0, pNew, 0);
+#endif
   }
-  return 1;
+  return pNew;
 }
-#endif /* SQLITE_DEBUG */
 
 
 /*
@@ -460,76 +535,23 @@ static int codeEqualityTerm(
       }
     }
     for(i=iEq;i<pLoop->nLTerm; i++){
-      if( ALWAYS(pLoop->aLTerm[i]) && pLoop->aLTerm[i]->pExpr==pX ) nEq++;
+      assert( pLoop->aLTerm[i]!=0 );
+      if( pLoop->aLTerm[i]->pExpr==pX ) nEq++;
     }
 
     if( (pX->flags & EP_xIsSelect)==0 || pX->x.pSelect->pEList->nExpr==1 ){
       eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, 0);
     }else{
-      Select *pSelect = pX->x.pSelect;
       sqlite3 *db = pParse->db;
-      u16 savedDbOptFlags = db->dbOptFlags;
-      ExprList *pOrigRhs = pSelect->pEList;
-      ExprList *pOrigLhs = pX->pLeft->x.pList;
-      ExprList *pRhs = 0;         /* New Select.pEList for RHS */
-      ExprList *pLhs = 0;         /* New pX->pLeft vector */
-
-      for(i=iEq;i<pLoop->nLTerm; i++){
-        if( pLoop->aLTerm[i]->pExpr==pX ){
-          int iField = pLoop->aLTerm[i]->iField - 1;
-          Expr *pNewRhs = sqlite3ExprDup(db, pOrigRhs->a[iField].pExpr, 0);
-          Expr *pNewLhs = sqlite3ExprDup(db, pOrigLhs->a[iField].pExpr, 0);
-
-          pRhs = sqlite3ExprListAppend(pParse, pRhs, pNewRhs);
-          pLhs = sqlite3ExprListAppend(pParse, pLhs, pNewLhs);
-        }
-      }
- 
-      /* pRhs should be a subset of pOrigRhs (though possibly in a different
-      ** order).  And pLhs should be a subset of pOrigLhs.  To put it 
-      ** another way:  Every term of pRhs should exist in pOrigRhs and
-      ** every term of pLhs should exist in pOrigLhs. */
-      assert( db->mallocFailed || exprListSubset(pRhs, pOrigRhs) );
-      assert( db->mallocFailed || exprListSubset(pLhs, pOrigLhs) );
+      pX = removeUnindexableInClauseTerms(pParse, iEq, pLoop, pX);
 
       if( !db->mallocFailed ){
-        Expr *pLeft = pX->pLeft;
-
-        if( pSelect->pOrderBy ){
-          /* If the SELECT statement has an ORDER BY clause, zero the 
-          ** iOrderByCol variables. These are set to non-zero when an 
-          ** ORDER BY term exactly matches one of the terms of the 
-          ** result-set. Since the result-set of the SELECT statement may
-          ** have been modified or reordered, these variables are no longer 
-          ** set correctly.  Since setting them is just an optimization, 
-          ** it's easiest just to zero them here.  */
-          ExprList *pOrderBy = pSelect->pOrderBy;
-          for(i=0; i<pOrderBy->nExpr; i++){
-            pOrderBy->a[i].u.x.iOrderByCol = 0;
-          }
-        }
-
-        /* Take care here not to generate a TK_VECTOR containing only a
-        ** single value. Since the parser never creates such a vector, some
-        ** of the subroutines do not handle this case.  */
-        if( pLhs->nExpr==1 ){
-          pX->pLeft = pLhs->a[0].pExpr;
-        }else{
-          pLeft->x.pList = pLhs;
-          aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int) * nEq);
-          testcase( aiMap==0 );
-        }
-        pSelect->pEList = pRhs;
-        db->dbOptFlags |= SQLITE_QueryFlattener;
+        aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*nEq);
         eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, aiMap);
-        db->dbOptFlags = savedDbOptFlags;
-        testcase( aiMap!=0 && aiMap[0]!=0 );
-        pSelect->pEList = pOrigRhs;
-        pLeft->x.pList = pOrigLhs;
-        pX->pLeft = pLeft;
+        pTerm->pExpr->iTable = pX->iTable;
       }
-      sqlite3ExprListDelete(pParse->db, pLhs);
-      sqlite3ExprListDelete(pParse->db, pRhs);
+      sqlite3ExprDelete(db, pX);
+      pX = pTerm->pExpr;
     }
 
     if( eType==IN_INDEX_INDEX_DESC ){

From eebf2f5747182c8a46e4eb58fc9e8dc2567ed2cf Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 18 Nov 2017 17:30:08 +0000
Subject: [PATCH 097/488] Enhance the log messages produced in some cases if
 database corruption is encountered by an SQLITE_DEBUG build.

FossilOrigin-Name: 23a3128083834b5fc80adf45448f7dc65587c52761fb3c9483b80313b369406f
---
 manifest        | 21 +++++++-----
 manifest.uuid   |  2 +-
 src/btree.c     | 91 +++++++++++++++++++++++++++++++------------------
 src/main.c      | 14 ++++----
 src/sqliteInt.h |  1 +
 5 files changed, 79 insertions(+), 50 deletions(-)

diff --git a/manifest b/manifest
index b8aae7e011..344babb1fb 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improved\sfix\sfor\sticket\s[da78413751863]\sthat\sdoes\snot\srequire\sdisabling\sthe\nquery\sflattener\sas\swas\sdone\sin\s[005d5b870625].\s\sThis\salso\smakes\sthe\scode\ngenerator\sfor\svector\sIN\soperators\sa\slittle\seasier\sto\sunderstand.
-D 2017-11-17T21:01:04.980
+C Enhance\sthe\slog\smessages\sproduced\sin\ssome\scases\sif\sdatabase\scorruption\sis\nencountered\sby\san\sSQLITE_DEBUG\sbuild.
+D 2017-11-18T17:30:08.436
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -412,7 +412,7 @@ F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
-F src/btree.c 75229a5a47985997f861b428552acd14fe42b657f755cba5e0b1a007bd77b2ea
+F src/btree.c b83a6b03f160528020bb965f0c3a40af5286cd4923c3870fd218177f03a120a7
 F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09
 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc
 F src/build.c 514db9d494ed29155e552f2ec2fa7c55c0241f847c683156b7c017f4b0bad9fa
@@ -435,7 +435,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c cb67cc56ef2ddd13e6944b2c0dd08a920bcd9503230adef8b9928d338097c722
 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e
 F src/loadext.c 20865b183bb8a3723d59cf1efffc3c50217eb452c1021d077b908c94da26b0b2
-F src/main.c c1965ee8159cee5fba3f590cc4767515a690504455a03e4817b1accfe0ba95a5
+F src/main.c 99ed3d45e315afb2ada049991db7944b1210663bb30bfd0b63103537c1ac25d0
 F src/malloc.c a02c9e69bc76bee0f639416b947a946412890b606301454727feadcb313536d6
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -475,7 +475,7 @@ F src/shell.c.in a87f3094c1d2a07ed3f731409b33dbf36faf99157bfedb17855a664c60e1a90
 F src/sqlite.h.in 8fd97993d48b50b9bade38c52f12d175942c9497c960905610c7b03a3e4b5818
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
-F src/sqliteInt.h fb297e4b891608057e857d583e30a261d905a3f41493f351fc91bae7d22008ff
+F src/sqliteInt.h 9b26fbab75ef426efae70d88ab535844d59de8954542b122c1d49af580a76f58
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -1677,7 +1677,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c21406ab3281480d3eddca0cdf5aea3abc224425ee52c10eed3ff702a0ae5c26
-R 04b5e48559de23ef7902e501e8dcaac8
-U drh
-Z cc87a690f1d098ebb337ab16a1f38777
+P 723f1be3d4a905a6a16333f8ef3e1067dcd4944497b303033c49946fc37c780f
+R 05158c3570d45bd02cc2ba4cbb719bc9
+T *branch * sqlite-corrupt-page
+T *sym-sqlite-corrupt-page *
+T -sym-trunk *
+U dan
+Z b072798007848ebbd313378cc8741324
diff --git a/manifest.uuid b/manifest.uuid
index 99c0990a4c..dfb35df8dd 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-723f1be3d4a905a6a16333f8ef3e1067dcd4944497b303033c49946fc37c780f
\ No newline at end of file
+23a3128083834b5fc80adf45448f7dc65587c52761fb3c9483b80313b369406f
\ No newline at end of file
diff --git a/src/btree.c b/src/btree.c
index ddcb6cfd35..0fa00a2e12 100644
--- a/src/btree.c
+++ b/src/btree.c
@@ -112,6 +112,31 @@ int sqlite3_enable_shared_cache(int enable){
   #define hasReadConflicts(a, b) 0
 #endif
 
+/*
+** Implementation of the SQLITE_CORRUPT_PAGE() macro. Takes a single
+** (MemPage*) as an argument. The (MemPage*) must not be NULL.
+**
+** If SQLITE_DEBUG is not defined, then this macro is equivalent to
+** SQLITE_CORRUPT_BKPT. Or, if SQLITE_DEBUG is set, then the log message
+** normally produced as a side-effect of SQLITE_CORRUPT_BKPT is augmented
+** with the page number and filename associated with the (MemPage*).
+*/
+#ifdef SQLITE_DEBUG
+int corruptPageError(int lineno, MemPage *p){
+  char *zMsg = sqlite3_mprintf("database corruption page %d of %s",
+      (int)p->pgno, sqlite3PagerFilename(p->pBt->pPager, 0)
+  );
+  if( zMsg ){
+    sqlite3ReportError(SQLITE_CORRUPT, lineno, zMsg);
+  }
+  sqlite3_free(zMsg);
+  return SQLITE_CORRUPT_BKPT;
+}
+# define SQLITE_CORRUPT_PAGE(pMemPage) corruptPageError(__LINE__, pMemPage)
+#else
+# define SQLITE_CORRUPT_PAGE(pMemPage) SQLITE_CORRUPT_PGNO(pMemPage->pgno)
+#endif
+
 #ifndef SQLITE_OMIT_SHARED_CACHE
 
 #ifdef SQLITE_DEBUG
@@ -1400,7 +1425,7 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){
         int sz = get2byte(&data[iFree+2]);
         int top = get2byte(&data[hdr+5]);
         if( top>=iFree ){
-          return SQLITE_CORRUPT_PGNO(pPage->pgno);
+          return SQLITE_CORRUPT_PAGE(pPage);
         }
         if( iFree2 ){
           assert( iFree+sz<=iFree2 ); /* Verified by pageFindSlot() */
@@ -1434,13 +1459,13 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){
     ** if PRAGMA cell_size_check=ON.
     */
     if( pc<iCellFirst || pc>iCellLast ){
-      return SQLITE_CORRUPT_PGNO(pPage->pgno);
+      return SQLITE_CORRUPT_PAGE(pPage);
     }
     assert( pc>=iCellFirst && pc<=iCellLast );
     size = pPage->xCellSize(pPage, &src[pc]);
     cbrk -= size;
     if( cbrk<iCellFirst || pc+size>usableSize ){
-      return SQLITE_CORRUPT_PGNO(pPage->pgno);
+      return SQLITE_CORRUPT_PAGE(pPage);
     }
     assert( cbrk+size<=usableSize && cbrk>=iCellFirst );
     testcase( cbrk+size==usableSize );
@@ -1460,7 +1485,7 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){
 
  defragment_out:
   if( data[hdr+7]+cbrk-iCellFirst!=pPage->nFree ){
-    return SQLITE_CORRUPT_PGNO(pPage->pgno);
+    return SQLITE_CORRUPT_PAGE(pPage);
   }
   assert( cbrk>=iCellFirst );
   put2byte(&data[hdr+5], cbrk);
@@ -1504,7 +1529,7 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){
       testcase( x==4 );
       testcase( x==3 );
       if( size+pc > usableSize ){
-        *pRc = SQLITE_CORRUPT_PGNO(pPg->pgno);
+        *pRc = SQLITE_CORRUPT_PAGE(pPg);
         return 0;
       }else if( x<4 ){
         /* EVIDENCE-OF: R-11498-58022 In a well-formed b-tree page, the total
@@ -1527,7 +1552,7 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){
     if( pc<iAddr+size ) break;
   }
   if( pc ){
-    *pRc = SQLITE_CORRUPT_PGNO(pPg->pgno);
+    *pRc = SQLITE_CORRUPT_PAGE(pPg);
   }
 
   return 0;
@@ -1575,7 +1600,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){
     if( top==0 && pPage->pBt->usableSize==65536 ){
       top = 65536;
     }else{
-      return SQLITE_CORRUPT_PGNO(pPage->pgno);
+      return SQLITE_CORRUPT_PAGE(pPage);
     }
   }
 
@@ -1665,12 +1690,12 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){
     while( (iFreeBlk = get2byte(&data[iPtr]))<iStart ){
       if( iFreeBlk<iPtr+4 ){
         if( iFreeBlk==0 ) break;
-        return SQLITE_CORRUPT_PGNO(pPage->pgno);
+        return SQLITE_CORRUPT_PAGE(pPage);
       }
       iPtr = iFreeBlk;
     }
     if( iFreeBlk>pPage->pBt->usableSize-4 ){
-      return SQLITE_CORRUPT_PGNO(pPage->pgno);
+      return SQLITE_CORRUPT_PAGE(pPage);
     }
     assert( iFreeBlk>iPtr || iFreeBlk==0 );
   
@@ -1682,10 +1707,10 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){
     */
     if( iFreeBlk && iEnd+3>=iFreeBlk ){
       nFrag = iFreeBlk - iEnd;
-      if( iEnd>iFreeBlk ) return SQLITE_CORRUPT_PGNO(pPage->pgno);
+      if( iEnd>iFreeBlk ) return SQLITE_CORRUPT_PAGE(pPage);
       iEnd = iFreeBlk + get2byte(&data[iFreeBlk+2]);
       if( iEnd > pPage->pBt->usableSize ){
-        return SQLITE_CORRUPT_PGNO(pPage->pgno);
+        return SQLITE_CORRUPT_PAGE(pPage);
       }
       iSize = iEnd - iStart;
       iFreeBlk = get2byte(&data[iFreeBlk]);
@@ -1698,13 +1723,13 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){
     if( iPtr>hdr+1 ){
       int iPtrEnd = iPtr + get2byte(&data[iPtr+2]);
       if( iPtrEnd+3>=iStart ){
-        if( iPtrEnd>iStart ) return SQLITE_CORRUPT_PGNO(pPage->pgno);
+        if( iPtrEnd>iStart ) return SQLITE_CORRUPT_PAGE(pPage);
         nFrag += iStart - iPtrEnd;
         iSize = iEnd - iPtr;
         iStart = iPtr;
       }
     }
-    if( nFrag>data[hdr+7] ) return SQLITE_CORRUPT_PGNO(pPage->pgno);
+    if( nFrag>data[hdr+7] ) return SQLITE_CORRUPT_PAGE(pPage);
     data[hdr+7] -= nFrag;
   }
   x = get2byte(&data[hdr+5]);
@@ -1712,7 +1737,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){
     /* The new freeblock is at the beginning of the cell content area,
     ** so just extend the cell content area rather than create another
     ** freelist entry */
-    if( iStart<x || iPtr!=hdr+1 ) return SQLITE_CORRUPT_PGNO(pPage->pgno);
+    if( iStart<x || iPtr!=hdr+1 ) return SQLITE_CORRUPT_PAGE(pPage);
     put2byte(&data[hdr+1], iFreeBlk);
     put2byte(&data[hdr+5], iEnd);
   }else{
@@ -1785,7 +1810,7 @@ static int decodeFlags(MemPage *pPage, int flagByte){
   }else{
     /* EVIDENCE-OF: R-47608-56469 Any other value for the b-tree page type is
     ** an error. */
-    return SQLITE_CORRUPT_PGNO(pPage->pgno);
+    return SQLITE_CORRUPT_PAGE(pPage);
   }
   pPage->max1bytePayload = pBt->max1bytePayload;
   return SQLITE_OK;
@@ -1826,7 +1851,7 @@ static int btreeInitPage(MemPage *pPage){
   /* EVIDENCE-OF: R-28594-02890 The one-byte flag at offset 0 indicating
   ** the b-tree page type. */
   if( decodeFlags(pPage, data[hdr]) ){
-    return SQLITE_CORRUPT_PGNO(pPage->pgno);
+    return SQLITE_CORRUPT_PAGE(pPage);
   }
   assert( pBt->pageSize>=512 && pBt->pageSize<=65536 );
   pPage->maskPage = (u16)(pBt->pageSize - 1);
@@ -1845,7 +1870,7 @@ static int btreeInitPage(MemPage *pPage){
   pPage->nCell = get2byte(&data[hdr+3]);
   if( pPage->nCell>MX_CELL(pBt) ){
     /* To many cells for a single page.  The page must be corrupt */
-    return SQLITE_CORRUPT_PGNO(pPage->pgno);
+    return SQLITE_CORRUPT_PAGE(pPage);
   }
   testcase( pPage->nCell==MX_CELL(pBt) );
   /* EVIDENCE-OF: R-24089-57979 If a page contains no cells (which is only
@@ -1873,12 +1898,12 @@ static int btreeInitPage(MemPage *pPage){
       testcase( pc==iCellFirst );
       testcase( pc==iCellLast );
       if( pc<iCellFirst || pc>iCellLast ){
-        return SQLITE_CORRUPT_PGNO(pPage->pgno);
+        return SQLITE_CORRUPT_PAGE(pPage);
       }
       sz = pPage->xCellSize(pPage, &data[pc]);
       testcase( pc+sz==usableSize );
       if( pc+sz>usableSize ){
-        return SQLITE_CORRUPT_PGNO(pPage->pgno);
+        return SQLITE_CORRUPT_PAGE(pPage);
       }
     }
     if( !pPage->leaf ) iCellLast++;
@@ -1896,12 +1921,12 @@ static int btreeInitPage(MemPage *pPage){
       /* EVIDENCE-OF: R-55530-52930 In a well-formed b-tree page, there will
       ** always be at least one cell before the first freeblock.
       */
-      return SQLITE_CORRUPT_PGNO(pPage->pgno); 
+      return SQLITE_CORRUPT_PAGE(pPage); 
     }
     while( 1 ){
       if( pc>iCellLast ){
         /* Freeblock off the end of the page */
-        return SQLITE_CORRUPT_PGNO(pPage->pgno);
+        return SQLITE_CORRUPT_PAGE(pPage);
       }
       next = get2byte(&data[pc]);
       size = get2byte(&data[pc+2]);
@@ -1911,11 +1936,11 @@ static int btreeInitPage(MemPage *pPage){
     }
     if( next>0 ){
       /* Freeblock not in ascending order */
-      return SQLITE_CORRUPT_PGNO(pPage->pgno);
+      return SQLITE_CORRUPT_PAGE(pPage);
     }
     if( pc+size>(unsigned int)usableSize ){
       /* Last freeblock extends past page end */
-      return SQLITE_CORRUPT_PGNO(pPage->pgno);
+      return SQLITE_CORRUPT_PAGE(pPage);
     }
   }
 
@@ -1927,7 +1952,7 @@ static int btreeInitPage(MemPage *pPage){
   ** area, according to the page header, lies within the page.
   */
   if( nFree>usableSize ){
-    return SQLITE_CORRUPT_PGNO(pPage->pgno);
+    return SQLITE_CORRUPT_PAGE(pPage);
   }
   pPage->nFree = (u16)(nFree - iCellFirst);
   pPage->isInit = 1;
@@ -3458,7 +3483,7 @@ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){
   if( eType==PTRMAP_OVERFLOW2 ){
     /* The pointer is always the first 4 bytes of the page in this case.  */
     if( get4byte(pPage->aData)!=iFrom ){
-      return SQLITE_CORRUPT_PGNO(pPage->pgno);
+      return SQLITE_CORRUPT_PAGE(pPage);
     }
     put4byte(pPage->aData, iTo);
   }else{
@@ -3477,7 +3502,7 @@ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){
         pPage->xParseCell(pPage, pCell, &info);
         if( info.nLocal<info.nPayload ){
           if( pCell+info.nSize > pPage->aData+pPage->pBt->usableSize ){
-            return SQLITE_CORRUPT_PGNO(pPage->pgno);
+            return SQLITE_CORRUPT_PAGE(pPage);
           }
           if( iFrom==get4byte(pCell+info.nSize-4) ){
             put4byte(pCell+info.nSize-4, iTo);
@@ -3495,7 +3520,7 @@ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){
     if( i==nCell ){
       if( eType!=PTRMAP_BTREE || 
           get4byte(&pPage->aData[pPage->hdrOffset+8])!=iFrom ){
-        return SQLITE_CORRUPT_PGNO(pPage->pgno);
+        return SQLITE_CORRUPT_PAGE(pPage);
       }
       put4byte(&pPage->aData[pPage->hdrOffset+8], iTo);
     }
@@ -4593,7 +4618,7 @@ static int accessPayload(
     **    &aPayload[pCur->info.nLocal] > &pPage->aData[pBt->usableSize]
     ** but is recast into its current form to avoid integer overflow problems
     */
-    return SQLITE_CORRUPT_PGNO(pPage->pgno);
+    return SQLITE_CORRUPT_PAGE(pPage);
   }
 
   /* Check if data must be read/written to/from the btree page itself. */
@@ -4741,7 +4766,7 @@ static int accessPayload(
 
   if( rc==SQLITE_OK && amt>0 ){
     /* Overflow chain ends prematurely */
-    return SQLITE_CORRUPT_PGNO(pPage->pgno);
+    return SQLITE_CORRUPT_PAGE(pPage);
   }
   return rc;
 }
@@ -5019,7 +5044,7 @@ static int moveToRoot(BtCursor *pCur){
   ** (or the freelist).  */
   assert( pRoot->intKey==1 || pRoot->intKey==0 );
   if( pRoot->isInit==0 || (pCur->pKeyInfo==0)!=pRoot->intKey ){
-    return SQLITE_CORRUPT_PGNO(pCur->pPage->pgno);
+    return SQLITE_CORRUPT_PAGE(pCur->pPage);
   }
 
 skip_init:  
@@ -5292,7 +5317,7 @@ int sqlite3BtreeMovetoUnpacked(
         if( pPage->intKeyLeaf ){
           while( 0x80 <= *(pCell++) ){
             if( pCell>=pPage->aDataEnd ){
-              return SQLITE_CORRUPT_PGNO(pPage->pgno);
+              return SQLITE_CORRUPT_PAGE(pPage);
             }
           }
         }
@@ -5366,7 +5391,7 @@ int sqlite3BtreeMovetoUnpacked(
           testcase( nCell==1 );  /* Invalid key size:  0x80 0x80 0x01 */
           testcase( nCell==2 );  /* Minimum legal index key size */
           if( nCell<2 ){
-            rc = SQLITE_CORRUPT_PGNO(pPage->pgno);
+            rc = SQLITE_CORRUPT_PAGE(pPage);
             goto moveto_finish;
           }
           pCellKey = sqlite3Malloc( nCell+18 );
@@ -6169,7 +6194,7 @@ static int clearCell(
   }
   if( pCell+pInfo->nSize-1 > pPage->aData+pPage->maskPage ){
     /* Cell extends past end of page */
-    return SQLITE_CORRUPT_PGNO(pPage->pgno);
+    return SQLITE_CORRUPT_PAGE(pPage);
   }
   ovflPgno = get4byte(pCell + pInfo->nSize - 4);
   pBt = pPage->pBt;
diff --git a/src/main.c b/src/main.c
index d090845ae3..f9b34f80e8 100644
--- a/src/main.c
+++ b/src/main.c
@@ -3344,37 +3344,37 @@ int sqlite3_get_autocommit(sqlite3 *db){
 **   2.  Invoke sqlite3_log() to provide the source code location where
 **       a low-level error is first detected.
 */
-static int reportError(int iErr, int lineno, const char *zType){
+int sqlite3ReportError(int iErr, int lineno, const char *zType){
   sqlite3_log(iErr, "%s at line %d of [%.10s]",
               zType, lineno, 20+sqlite3_sourceid());
   return iErr;
 }
 int sqlite3CorruptError(int lineno){
   testcase( sqlite3GlobalConfig.xLog!=0 );
-  return reportError(SQLITE_CORRUPT, lineno, "database corruption");
+  return sqlite3ReportError(SQLITE_CORRUPT, lineno, "database corruption");
 }
 int sqlite3MisuseError(int lineno){
   testcase( sqlite3GlobalConfig.xLog!=0 );
-  return reportError(SQLITE_MISUSE, lineno, "misuse");
+  return sqlite3ReportError(SQLITE_MISUSE, lineno, "misuse");
 }
 int sqlite3CantopenError(int lineno){
   testcase( sqlite3GlobalConfig.xLog!=0 );
-  return reportError(SQLITE_CANTOPEN, lineno, "cannot open file");
+  return sqlite3ReportError(SQLITE_CANTOPEN, lineno, "cannot open file");
 }
 #ifdef SQLITE_DEBUG
 int sqlite3CorruptPgnoError(int lineno, Pgno pgno){
   char zMsg[100];
   sqlite3_snprintf(sizeof(zMsg), zMsg, "database corruption page %d", pgno);
   testcase( sqlite3GlobalConfig.xLog!=0 );
-  return reportError(SQLITE_CORRUPT, lineno, zMsg);
+  return sqlite3ReportError(SQLITE_CORRUPT, lineno, zMsg);
 }
 int sqlite3NomemError(int lineno){
   testcase( sqlite3GlobalConfig.xLog!=0 );
-  return reportError(SQLITE_NOMEM, lineno, "OOM");
+  return sqlite3ReportError(SQLITE_NOMEM, lineno, "OOM");
 }
 int sqlite3IoerrnomemError(int lineno){
   testcase( sqlite3GlobalConfig.xLog!=0 );
-  return reportError(SQLITE_IOERR_NOMEM, lineno, "I/O OOM error");
+  return sqlite3ReportError(SQLITE_IOERR_NOMEM, lineno, "I/O OOM error");
 }
 #endif
 
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 058fee1461..c4e3a7dabc 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -3431,6 +3431,7 @@ struct TreeView {
 ** using sqlite3_log().  The routines also provide a convenient place
 ** to set a debugger breakpoint.
 */
+int sqlite3ReportError(int iErr, int lineno, const char *zType);
 int sqlite3CorruptError(int);
 int sqlite3MisuseError(int);
 int sqlite3CantopenError(int);

From 75dbf68b5b8b91d209702b17db48b70af43c5f96 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 20 Nov 2017 14:40:03 +0000
Subject: [PATCH 098/488] Fix a problem preventing the planner from identifying
 scans that visit at most one row in cases where that property is guaranteed
 by a unique, not-null, non-IPK column that is the leftmost in its table.

FossilOrigin-Name: 299d7ca52fec32f04ffd2b8561dd4b839e891792162f8b00259368683436b02d
---
 manifest        | 15 +++++++--------
 manifest.uuid   |  2 +-
 src/where.c     |  2 +-
 test/join2.test | 31 +++++++++++++++++++++++++++++++
 4 files changed, 40 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index b35a1be8bc..12b73310c0 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Enhance\sthe\slog\smessages\sproduced\sin\ssome\scases\sif\sdatabase\scorruption\sis\nencountered\sby\san\sSQLITE_DEBUG\sbuild.
-D 2017-11-18T18:07:17.406
+C Fix\sa\sproblem\spreventing\sthe\splanner\sfrom\sidentifying\sscans\sthat\svisit\sat\smost\none\srow\sin\scases\swhere\sthat\sproperty\sis\sguaranteed\sby\sa\sunique,\snot-null,\nnon-IPK\scolumn\sthat\sis\sthe\sleftmost\sin\sits\stable.
+D 2017-11-20T14:40:03.587
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -553,7 +553,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c beeb71e4eab65dbf0d95f2717efc6ca3c0f5b3090ce67f3de63828f39a6ff053
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f
-F src/where.c 031a80bcafe93934fd7052f3031c9e7eb36b61754c6c84d6bf0833184abad3db
+F src/where.c e729f477523e3394892951347f57da1471a94a601133c19efd2573903e60a4a2
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
 F src/wherecode.c 611fcabd05592ed2febd7d182f9621425b0466c5232d70e0981c842d429356d5
 F src/whereexpr.c 427ea8e96ec24f2a7814c67b8024ad664a9c7656264c4566c34743cb23186e46
@@ -983,7 +983,7 @@ F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c
 F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4
 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b
 F test/join.test 442c462eea85cf065d70a663c626b780a95af6e11585d909bb63b87598afe678
-F test/join2.test a48f723c5692e2cbb23a9297ac2720cb77d51a70
+F test/join2.test 8e3085b352e9f6d89247059ffd55c572b517f00295c8c0643a5c20cabd1e3848
 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0
 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916
 F test/join5.test bc98ea4b4e5003f5b1453701ebb8cd7d1c01a550
@@ -1677,8 +1677,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 723f1be3d4a905a6a16333f8ef3e1067dcd4944497b303033c49946fc37c780f 23a3128083834b5fc80adf45448f7dc65587c52761fb3c9483b80313b369406f
-R 05158c3570d45bd02cc2ba4cbb719bc9
-T +closed 23a3128083834b5fc80adf45448f7dc65587c52761fb3c9483b80313b369406f
+P ee840a7669dd462af072625232ea4238198c9b94e1873f361c45f3b0985456f3
+R d6874f2de8f8cac9d7779bbf432e0b3c
 U dan
-Z ce33cab931a12d704bb3e13109bc56c2
+Z 3df96851db88f42b898e0f724b99ed03
diff --git a/manifest.uuid b/manifest.uuid
index f4e48d0118..64fa4f8dcf 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ee840a7669dd462af072625232ea4238198c9b94e1873f361c45f3b0985456f3
\ No newline at end of file
+299d7ca52fec32f04ffd2b8561dd4b839e891792162f8b00259368683436b02d
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index 5545a45e87..955b9450ef 100644
--- a/src/where.c
+++ b/src/where.c
@@ -2460,7 +2460,7 @@ static int whereLoopAddBtreeIndex(
       pNew->wsFlags |= WHERE_COLUMN_EQ;
       assert( saved_nEq==pNew->u.btree.nEq );
       if( iCol==XN_ROWID 
-       || (iCol>0 && nInMul==0 && saved_nEq==pProbe->nKeyCol-1)
+       || (iCol>=0 && nInMul==0 && saved_nEq==pProbe->nKeyCol-1)
       ){
         if( iCol>=0 && pProbe->uniqNotNull==0 ){
           pNew->wsFlags |= WHERE_UNQ_WANTED;
diff --git a/test/join2.test b/test/join2.test
index 9372e770c3..850338955f 100644
--- a/test/join2.test
+++ b/test/join2.test
@@ -92,4 +92,35 @@ do_catchsql_test 2.2 {
   SELECT * FROM aa JOIN cc ON (a=b) JOIN bb ON (b=c);
 } {0 {one one one}}
 
+#-------------------------------------------------------------------------
+# Test that a problem causing where.c to overlook opportunities to
+# omit unnecessary tables from a LEFT JOIN when UNIQUE, NOT NULL column 
+# that makes this possible happens to be the leftmost in its table.
+#
+reset_db
+do_execsql_test 3.0 {
+  CREATE TABLE t1(k1 INTEGER PRIMARY KEY, k2, k3);
+  CREATE TABLE t2(k2 INTEGER PRIMARY KEY, v2);
+
+  # Prior to this problem being fixed, table t3_2 would be omitted from
+  # the join queries below, but if t3_1 were used in its place it would
+  # not.
+  CREATE TABLE t3_1(k3 PRIMARY KEY, v3) WITHOUT ROWID;
+  CREATE TABLE t3_2(v3, k3 PRIMARY KEY) WITHOUT ROWID;
+}
+
+do_eqp_test 3.1 {
+  SELECT v2 FROM t1 LEFT JOIN t2 USING (k2) LEFT JOIN t3_1 USING (k3);
+} {
+  0 0 0 {SCAN TABLE t1} 
+  0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?)}
+}
+
+do_eqp_test 3.2 {
+  SELECT v2 FROM t1 LEFT JOIN t2 USING (k2) LEFT JOIN t3_2 USING (k3);
+} {
+  0 0 0 {SCAN TABLE t1} 
+  0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?)}
+}
+
 finish_test

From 17f188e3cfcba13a96bf0fddbe61e3c9d16e35fa Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 20 Nov 2017 15:45:03 +0000
Subject: [PATCH 099/488] Fix a typo in a test script on this branch.

FossilOrigin-Name: bff5dcfd2b29ee4834258914410a5dee69ec2727dd254053e3ebaf5090937694
---
 manifest        | 12 ++++++------
 manifest.uuid   |  2 +-
 test/join2.test |  6 +++---
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index 12b73310c0..b3ae26712d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sproblem\spreventing\sthe\splanner\sfrom\sidentifying\sscans\sthat\svisit\sat\smost\none\srow\sin\scases\swhere\sthat\sproperty\sis\sguaranteed\sby\sa\sunique,\snot-null,\nnon-IPK\scolumn\sthat\sis\sthe\sleftmost\sin\sits\stable.
-D 2017-11-20T14:40:03.587
+C Fix\sa\stypo\sin\sa\stest\sscript\son\sthis\sbranch.
+D 2017-11-20T15:45:03.103
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -983,7 +983,7 @@ F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c
 F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4
 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b
 F test/join.test 442c462eea85cf065d70a663c626b780a95af6e11585d909bb63b87598afe678
-F test/join2.test 8e3085b352e9f6d89247059ffd55c572b517f00295c8c0643a5c20cabd1e3848
+F test/join2.test 770a9b6c9f0e91590a102bae707727bce6103b8ad478cea077b2ac126d0698e8
 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0
 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916
 F test/join5.test bc98ea4b4e5003f5b1453701ebb8cd7d1c01a550
@@ -1677,7 +1677,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ee840a7669dd462af072625232ea4238198c9b94e1873f361c45f3b0985456f3
-R d6874f2de8f8cac9d7779bbf432e0b3c
+P 299d7ca52fec32f04ffd2b8561dd4b839e891792162f8b00259368683436b02d
+R e3d55f8947932d938e221f1f7209d770
 U dan
-Z 3df96851db88f42b898e0f724b99ed03
+Z 5b68c810792c127ff29194bdbf236929
diff --git a/manifest.uuid b/manifest.uuid
index 64fa4f8dcf..f9d78fd1e6 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-299d7ca52fec32f04ffd2b8561dd4b839e891792162f8b00259368683436b02d
\ No newline at end of file
+bff5dcfd2b29ee4834258914410a5dee69ec2727dd254053e3ebaf5090937694
\ No newline at end of file
diff --git a/test/join2.test b/test/join2.test
index 850338955f..65a0ca266b 100644
--- a/test/join2.test
+++ b/test/join2.test
@@ -102,9 +102,9 @@ do_execsql_test 3.0 {
   CREATE TABLE t1(k1 INTEGER PRIMARY KEY, k2, k3);
   CREATE TABLE t2(k2 INTEGER PRIMARY KEY, v2);
 
-  # Prior to this problem being fixed, table t3_2 would be omitted from
-  # the join queries below, but if t3_1 were used in its place it would
-  # not.
+  -- Prior to this problem being fixed, table t3_2 would be omitted from
+  -- the join queries below, but if t3_1 were used in its place it would
+  -- not.
   CREATE TABLE t3_1(k3 PRIMARY KEY, v3) WITHOUT ROWID;
   CREATE TABLE t3_2(v3, k3 PRIMARY KEY) WITHOUT ROWID;
 }

From 41203c6cf1e62bed4a39e72cb34e9128bf5108d0 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 21 Nov 2017 19:22:45 +0000
Subject: [PATCH 100/488] Update the omit-table-from-left-join optimization so
 that it can omit tables from the middle of the join as well as the end.

FossilOrigin-Name: 618ca9fe53d8d2d7b4f368e6ee404d5fceeecac0d689f32ab62af8a6cbb37401
---
 manifest        | 18 ++++++++------
 manifest.uuid   |  2 +-
 src/where.c     | 65 +++++++++++++++++++++++++++++++++++++++----------
 test/join2.test | 48 ++++++++++++++++++++++++++++++++++++
 4 files changed, 111 insertions(+), 22 deletions(-)

diff --git a/manifest b/manifest
index ace7d54144..cd6879f39b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sproblem\spreventing\sthe\splanner\sfrom\sidentifying\sscans\sthat\svisit\sat\smost\none\srow\sin\scases\swhere\sthat\sproperty\sis\sguaranteed\sby\sa\sunique,\snot-null,\nnon-IPK\scolumn\sthat\sis\sthe\sleftmost\sin\sits\stable.
-D 2017-11-20T15:46:10.484
+C Update\sthe\somit-table-from-left-join\soptimization\sso\sthat\sit\scan\somit\stables\nfrom\sthe\smiddle\sof\sthe\sjoin\sas\swell\sas\sthe\send.
+D 2017-11-21T19:22:45.416
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -553,7 +553,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c beeb71e4eab65dbf0d95f2717efc6ca3c0f5b3090ce67f3de63828f39a6ff053
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f
-F src/where.c e729f477523e3394892951347f57da1471a94a601133c19efd2573903e60a4a2
+F src/where.c 9742731e325768332d67df217eb636556c8605f895d518ee61e4d13029f8aed8
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
 F src/wherecode.c 611fcabd05592ed2febd7d182f9621425b0466c5232d70e0981c842d429356d5
 F src/whereexpr.c 427ea8e96ec24f2a7814c67b8024ad664a9c7656264c4566c34743cb23186e46
@@ -983,7 +983,7 @@ F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c
 F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4
 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b
 F test/join.test 442c462eea85cf065d70a663c626b780a95af6e11585d909bb63b87598afe678
-F test/join2.test 770a9b6c9f0e91590a102bae707727bce6103b8ad478cea077b2ac126d0698e8
+F test/join2.test ac70b2b79ac593550d1d6f15e9bb3693dd71826b496e84f15166c5053fa518e6
 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0
 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916
 F test/join5.test bc98ea4b4e5003f5b1453701ebb8cd7d1c01a550
@@ -1677,8 +1677,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ee840a7669dd462af072625232ea4238198c9b94e1873f361c45f3b0985456f3 bff5dcfd2b29ee4834258914410a5dee69ec2727dd254053e3ebaf5090937694
-R e3d55f8947932d938e221f1f7209d770
-T +closed bff5dcfd2b29ee4834258914410a5dee69ec2727dd254053e3ebaf5090937694
+P 7fdb1e2ac2040dc47800a224d33a5c95d55200c480d46fedec1e97fb4f089ef7
+R 77239b01fe37a025cc689347a6c626c1
+T *branch * left-join-optimization
+T *sym-left-join-optimization *
+T -sym-trunk *
 U dan
-Z 0d562170c07312a10559bcbda1122c53
+Z 55e1c0e690d530335519db505e442adf
diff --git a/manifest.uuid b/manifest.uuid
index 258ed7d3a3..ea81b36077 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7fdb1e2ac2040dc47800a224d33a5c95d55200c480d46fedec1e97fb4f089ef7
\ No newline at end of file
+618ca9fe53d8d2d7b4f368e6ee404d5fceeecac0d689f32ab62af8a6cbb37401
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index 955b9450ef..0ced477da4 100644
--- a/src/where.c
+++ b/src/where.c
@@ -4677,35 +4677,73 @@ WhereInfo *sqlite3WhereBegin(
     }
   }
 #endif
-  /* Attempt to omit tables from the join that do not effect the result */
+
+  /* Attempt to omit tables from the join that do not affect the result.
+  ** For a table to not affect the result, the following must be true:
+  **
+  **   1) The query must not be an aggregate.
+  **   2) The table must be the RHS of a LEFT JOIN.
+  **   3) Either the query must be DISTINCT, or else the ON or USING clause
+  **      must contain a constraint that limits the scan of the table to 
+  **      at most a single row.
+  **   4) The table must not be referenced by any part of the query apart
+  **      from its own USING or ON clause.
+  **
+  ** For example, given:
+  **
+  **     CREATE TABLE t1(ipk INTEGER PRIMARY KEY, v1);
+  **     CREATE TABLE t2(ipk INTEGER PRIMARY KEY, v2);
+  **     CREATE TABLE t3(ipk INTEGER PRIMARY KEY, v3);
+  **
+  ** then table t2 can be omitted from the following:
+  **
+  **     SELECT v1, v3 FROM t1 
+  **       LEFT JOIN t2 USING (t1.ipk=t2.ipk)
+  **       LEFT JOIN t3 USING (t1.ipk=t3.ipk)
+  **
+  ** or from:
+  **
+  **     SELECT DISTINCT v1, v3 FROM t1 
+  **       LEFT JOIN t2
+  **       LEFT JOIN t3 USING (t1.ipk=t3.ipk)
+  */
   if( pWInfo->nLevel>=2
-   && pResultSet!=0
+   && pResultSet!=0               /* guarantees condition (1) above */
    && OptimizationEnabled(db, SQLITE_OmitNoopJoin)
   ){
+    int i;
     Bitmask tabUsed = sqlite3WhereExprListUsage(pMaskSet, pResultSet);
     if( sWLB.pOrderBy ){
       tabUsed |= sqlite3WhereExprListUsage(pMaskSet, sWLB.pOrderBy);
     }
-    while( pWInfo->nLevel>=2 ){
+    for(i=pWInfo->nLevel-1; i>=1; i--){
       WhereTerm *pTerm, *pEnd;
-      pLoop = pWInfo->a[pWInfo->nLevel-1].pWLoop;
-      if( (pWInfo->pTabList->a[pLoop->iTab].fg.jointype & JT_LEFT)==0 ) break;
+      struct SrcList_item *pItem;
+      pLoop = pWInfo->a[i].pWLoop;
+      pItem = &pWInfo->pTabList->a[pLoop->iTab];
+      if( (pItem->fg.jointype & JT_LEFT)==0 ) continue;
       if( (wctrlFlags & WHERE_WANT_DISTINCT)==0
        && (pLoop->wsFlags & WHERE_ONEROW)==0
       ){
-        break;
+        continue;
       }
-      if( (tabUsed & pLoop->maskSelf)!=0 ) break;
+      if( (tabUsed & pLoop->maskSelf)!=0 ) continue;
       pEnd = sWLB.pWC->a + sWLB.pWC->nTerm;
       for(pTerm=sWLB.pWC->a; pTerm<pEnd; pTerm++){
-        if( (pTerm->prereqAll & pLoop->maskSelf)!=0
-         && !ExprHasProperty(pTerm->pExpr, EP_FromJoin)
-        ){
-          break;
+        if( (pTerm->prereqAll & pLoop->maskSelf)!=0 ){
+          if( !ExprHasProperty(pTerm->pExpr, EP_FromJoin)
+           || pTerm->pExpr->iRightJoinTable!=pItem->iCursor
+          ){
+            break;
+          }
         }
       }
-      if( pTerm<pEnd ) break;
+      if( pTerm<pEnd ) continue;
       WHERETRACE(0xffff, ("-> drop loop %c not used\n", pLoop->cId));
+      if( i!=pWInfo->nLevel-1 ){
+        int nByte = (pWInfo->nLevel-1-i) * sizeof(WhereLevel);
+        memmove(&pWInfo->a[i], &pWInfo->a[i+1], nByte);
+      }
       pWInfo->nLevel--;
       nTabList--;
     }
@@ -4990,7 +5028,8 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
       addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); VdbeCoverage(v);
       assert( (ws & WHERE_IDX_ONLY)==0 || (ws & WHERE_INDEXED)!=0 );
       if( (ws & WHERE_IDX_ONLY)==0 ){
-        sqlite3VdbeAddOp1(v, OP_NullRow, pTabList->a[i].iCursor);
+        assert( pLevel->iTabCur==pTabList->a[pLevel->iFrom].iCursor );
+        sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iTabCur);
       }
       if( (ws & WHERE_INDEXED) 
        || ((ws & WHERE_MULTI_OR) && pLevel->u.pCovidx) 
diff --git a/test/join2.test b/test/join2.test
index 65a0ca266b..9fdcef1783 100644
--- a/test/join2.test
+++ b/test/join2.test
@@ -123,4 +123,52 @@ do_eqp_test 3.2 {
   0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?)}
 }
 
+#-------------------------------------------------------------------------
+# Test that tables other than the rightmost can be omitted from a
+# LEFT JOIN query.
+#
+do_execsql_test 4.0 {
+  CREATE TABLE c1(k INTEGER PRIMARY KEY, v1);
+  CREATE TABLE c2(k INTEGER PRIMARY KEY, v2);
+  CREATE TABLE c3(k INTEGER PRIMARY KEY, v3);
+
+  INSERT INTO c1 VALUES(1, 2);
+  INSERT INTO c2 VALUES(2, 3);
+  INSERT INTO c3 VALUES(3, 'v3');
+
+  INSERT INTO c1 VALUES(111, 1112);
+  INSERT INTO c2 VALUES(112, 1113);
+  INSERT INTO c3 VALUES(113, 'v1113');
+}
+do_execsql_test 4.1.1 {
+  SELECT v1, v3 FROM c1 LEFT JOIN c2 ON (c2.k=v1) LEFT JOIN c3 ON (c3.k=v2);
+} {2 v3 1112 {}}
+do_execsql_test 4.1.2 {
+  SELECT v1, v3 FROM c1 LEFT JOIN c2 ON (c2.k=v1) LEFT JOIN c3 ON (c3.k=v1+1);
+} {2 v3 1112 {}}
+
+do_execsql_test 4.1.3 {
+  SELECT DISTINCT v1, v3 FROM c1 LEFT JOIN c2 LEFT JOIN c3 ON (c3.k=v1+1);
+} {2 v3 1112 {}}
+
+do_execsql_test 4.1.4 {
+  SELECT v1, v3 FROM c1 LEFT JOIN c2 LEFT JOIN c3 ON (c3.k=v1+1);
+} {2 v3 2 v3 1112 {} 1112 {}}
+
+do_eqp_test 4.2.1 {
+  SELECT v1, v3 FROM c1 LEFT JOIN c2 ON (c2.k=v1) LEFT JOIN c3 ON (c3.k=v2);
+} {
+  0 0 0 {SCAN TABLE c1} 
+  0 1 1 {SEARCH TABLE c2 USING INTEGER PRIMARY KEY (rowid=?)}
+  0 2 2 {SEARCH TABLE c3 USING INTEGER PRIMARY KEY (rowid=?)}
+}
+do_eqp_test 4.2.2 {
+  SELECT v1, v3 FROM c1 LEFT JOIN c2 ON (c2.k=v1) LEFT JOIN c3 ON (c3.k=v1+1);
+} {
+  0 0 0 {SCAN TABLE c1} 
+  0 1 2 {SEARCH TABLE c3 USING INTEGER PRIMARY KEY (rowid=?)}
+}
+
+
+
 finish_test

From c6bc15a804eb5a37a1159d2ef7b0e8c12844e44e Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Tue, 21 Nov 2017 21:14:32 +0000
Subject: [PATCH 101/488] Fix compilation issue (C99-ism) in the shell seen
 with MSVC.

FossilOrigin-Name: 9cb47430553f00216e9b06e3d8226b903da536152fac80a5b6f615508c933252
---
 manifest       | 15 +++++++--------
 manifest.uuid  |  2 +-
 src/shell.c.in |  4 +++-
 3 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index 3cff6827c1..6b501fa59f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\sthe\somit-table-from-left-join\soptimization\sso\sthat\sit\scan\somit\stables\nfrom\sthe\smiddle\sof\sthe\sjoin\sas\swell\sas\sthe\send.
-D 2017-11-21T20:53:14.147
+C Fix\scompilation\sissue\s(C99-ism)\sin\sthe\sshell\sseen\swith\sMSVC.
+D 2017-11-21T21:14:32.355
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
@@ -471,7 +471,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
-F src/shell.c.in a87f3094c1d2a07ed3f731409b33dbf36faf99157bfedb17855a664c60e1a909
+F src/shell.c.in cb1b5e41ef9c081b2b8927ae32c9c384a9ec110ada808ebfe083ba7c8a19bbb8
 F src/sqlite.h.in 8fd97993d48b50b9bade38c52f12d175942c9497c960905610c7b03a3e4b5818
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1677,8 +1677,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 7fdb1e2ac2040dc47800a224d33a5c95d55200c480d46fedec1e97fb4f089ef7 618ca9fe53d8d2d7b4f368e6ee404d5fceeecac0d689f32ab62af8a6cbb37401
-R 77239b01fe37a025cc689347a6c626c1
-T +closed 618ca9fe53d8d2d7b4f368e6ee404d5fceeecac0d689f32ab62af8a6cbb37401
-U dan
-Z 33c2b988566a650eaa7cbc98ff0c3943
+P 0cd82ee9a8413cf127b5ca65770e3f363bd579941cd592298d3b0c27715583f3
+R 4fa7a7c4266f9577164091bee69af1d6
+U mistachkin
+Z 932ce0ecb18fdaa178b1c4aecbb344e5
diff --git a/manifest.uuid b/manifest.uuid
index 2aded3bcbc..7486e65b91 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0cd82ee9a8413cf127b5ca65770e3f363bd579941cd592298d3b0c27715583f3
\ No newline at end of file
+9cb47430553f00216e9b06e3d8226b903da536152fac80a5b6f615508c933252
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 68a337db11..f41e7438ed 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -5952,8 +5952,10 @@ static int do_meta_command(char *zLine, ShellState *p){
     int rc2 = 0;    /* 0: usage.  1: %d  2: %x  3: no-output */
     int isOk = 0;
     int i, n2;
+    const char *zCmd = 0;
+
     open_db(p, 0);
-    const char *zCmd = nArg>=2 ? azArg[1] : "help";
+    zCmd = nArg>=2 ? azArg[1] : "help";
 
     /* The argument can optionally begin with "-" or "--" */
     if( zCmd[0]=='-' && zCmd[1] ){

From 041343dea141b51974a465ed2fb3412edadd7bee Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Tue, 21 Nov 2017 21:15:17 +0000
Subject: [PATCH 102/488] Fix typo in the Makefiles for MSVC.

FossilOrigin-Name: 17dd2f7314e7eb124e0a2a7a6cf475850e87fe3041e6ce1e1fd71b38d54852a8
---
 Makefile.msc          |  4 ++--
 autoconf/Makefile.msc |  2 +-
 manifest              | 14 +++++++-------
 manifest.uuid         |  2 +-
 4 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/Makefile.msc b/Makefile.msc
index 9c834931d7..b16f08adee 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -974,7 +974,7 @@ LTLINK = $(TCC) -Fe$@
 # If requested, link to the RPCRT4 library.
 #
 !IF $(USE_RPCRT4_LIB)!=0
-LTLINK = $(LTLINK) rpcrt4.lib
+LTLIBS = $(LTLIBS) rpcrt4.lib
 !ENDIF
 
 # If a platform was set, force the linker to target that.
@@ -1072,7 +1072,7 @@ LDFLAGS = $(LDOPTS)
 #
 !IF $(NO_TCL)==0
 LTLIBPATHS = /LIBPATH:$(TCLLIBDIR)
-LTLIBS = $(LIBTCL)
+LTLIBS = $(LTLIBS) $(LIBTCL)
 !ENDIF
 
 # If ICU support is enabled, add the linker options for it.
diff --git a/autoconf/Makefile.msc b/autoconf/Makefile.msc
index 39831da03e..199af21ea9 100644
--- a/autoconf/Makefile.msc
+++ b/autoconf/Makefile.msc
@@ -808,7 +808,7 @@ LTLINK = $(TCC) -Fe$@
 # If requested, link to the RPCRT4 library.
 #
 !IF $(USE_RPCRT4_LIB)!=0
-LTLINK = $(LTLINK) rpcrt4.lib
+LTLIBS = $(LTLIBS) rpcrt4.lib
 !ENDIF
 
 # If a platform was set, force the linker to target that.
diff --git a/manifest b/manifest
index 6b501fa59f..333b9dce22 100644
--- a/manifest
+++ b/manifest
@@ -1,8 +1,8 @@
-C Fix\scompilation\sissue\s(C99-ism)\sin\sthe\sshell\sseen\swith\sMSVC.
-D 2017-11-21T21:14:32.355
+C Fix\stypo\sin\sthe\sMakefiles\sfor\sMSVC.
+D 2017-11-21T21:15:17.379
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b
+F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd
 F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -11,7 +11,7 @@ F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90
 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2
 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903
 F autoconf/Makefile.am 66c0befa511f0d95ba229e180067cf0357a9ebf8b3201b06d683c5ba6220fb39
-F autoconf/Makefile.msc 6143fe5b571cfeb0159702931d3ade664a00edc0c03814c7f6d825ae73eeffac
+F autoconf/Makefile.msc b88a70dee8453cc353e5d6df172d60a11a0af905710a24b1e6be80f8fea6e96b
 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7
 F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1
 F autoconf/configure.ac 8dd08ca564279fff091c9bfdd2599d8f992c9f1f70c5396de2126ad2bd1b3bed
@@ -1677,7 +1677,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0cd82ee9a8413cf127b5ca65770e3f363bd579941cd592298d3b0c27715583f3
-R 4fa7a7c4266f9577164091bee69af1d6
+P 9cb47430553f00216e9b06e3d8226b903da536152fac80a5b6f615508c933252
+R a6bc597bdf966513d392563d9213ee42
 U mistachkin
-Z 932ce0ecb18fdaa178b1c4aecbb344e5
+Z c9f491fa6730dc4831f9db1bff90ef98
diff --git a/manifest.uuid b/manifest.uuid
index 7486e65b91..33f6abfa72 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-9cb47430553f00216e9b06e3d8226b903da536152fac80a5b6f615508c933252
\ No newline at end of file
+17dd2f7314e7eb124e0a2a7a6cf475850e87fe3041e6ce1e1fd71b38d54852a8
\ No newline at end of file

From 065b34f6e54d420a6a3a73e1394cc4560b3d64aa Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 21 Nov 2017 23:38:48 +0000
Subject: [PATCH 103/488] Fix the skip-ahead-distinct optimization on joins for
 cases there the table in the inner loop of the join does not contribute any
 columns to the result set.  Proposed fix for ticket [ef9318757b152e3a2]

FossilOrigin-Name: 2dcef5a9ae7f347da65207bf6bf612fb12e18e1a6704799322f0cf2a86154cfd
---
 manifest            | 16 ++++++++--------
 manifest.uuid       |  2 +-
 src/where.c         |  7 +++++++
 test/distinct2.test | 19 +++++++++++++++++++
 4 files changed, 35 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index 333b9dce22..a96176b69b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\stypo\sin\sthe\sMakefiles\sfor\sMSVC.
-D 2017-11-21T21:15:17.379
+C Fix\sthe\sskip-ahead-distinct\soptimization\son\sjoins\sfor\scases\sthere\sthe\stable\nin\sthe\sinner\sloop\sof\sthe\sjoin\sdoes\snot\scontribute\sany\scolumns\sto\sthe\nresult\sset.\s\sProposed\sfix\sfor\sticket\s[ef9318757b152e3a2]
+D 2017-11-21T23:38:48.349
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
@@ -553,7 +553,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c beeb71e4eab65dbf0d95f2717efc6ca3c0f5b3090ce67f3de63828f39a6ff053
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f
-F src/where.c 9742731e325768332d67df217eb636556c8605f895d518ee61e4d13029f8aed8
+F src/where.c 1c0ab20720f1a5e76fb4f78d2c45b4525b59167a20d44baf4467fa5b106679ad
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
 F src/wherecode.c 611fcabd05592ed2febd7d182f9621425b0466c5232d70e0981c842d429356d5
 F src/whereexpr.c 427ea8e96ec24f2a7814c67b8024ad664a9c7656264c4566c34743cb23186e46
@@ -735,7 +735,7 @@ F test/descidx2.test 9f1a0c83fd57f8667c82310ca21b30a350888b5d
 F test/descidx3.test 09ddbe3f5295f482d2f8b687cf6db8bad7acd9a2
 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e
 F test/distinct.test a1783b960ad8c15a77cd9f207be072898db1026c
-F test/distinct2.test faef8a3f27424e2cfbe19b2a40752294de3b5d957e049e3336be53ec0476cb58
+F test/distinct2.test ba239e9674495eda8e130cb5d498036b781dbd65c5868b58e094e36279859e83
 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376
 F test/e_blobbytes.test 439a945953b35cb6948a552edaec4dc31fd70a05
 F test/e_blobclose.test 4b3c8c60c2171164d472059c73e9f3c1844bb66d
@@ -1677,7 +1677,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 9cb47430553f00216e9b06e3d8226b903da536152fac80a5b6f615508c933252
-R a6bc597bdf966513d392563d9213ee42
-U mistachkin
-Z c9f491fa6730dc4831f9db1bff90ef98
+P 17dd2f7314e7eb124e0a2a7a6cf475850e87fe3041e6ce1e1fd71b38d54852a8
+R c2dbc2aeaa2d6cfa4070ebdfc171fa89
+U drh
+Z 8287b3f25cb0100d8a846549bf308df5
diff --git a/manifest.uuid b/manifest.uuid
index 33f6abfa72..bc87796a6f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-17dd2f7314e7eb124e0a2a7a6cf475850e87fe3041e6ce1e1fd71b38d54852a8
\ No newline at end of file
+2dcef5a9ae7f347da65207bf6bf612fb12e18e1a6704799322f0cf2a86154cfd
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index 0ced477da4..517d69fc1b 100644
--- a/src/where.c
+++ b/src/where.c
@@ -4978,6 +4978,13 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
         VdbeCoverageIf(v, op==OP_SeekLT);
         VdbeCoverageIf(v, op==OP_SeekGT);
         sqlite3VdbeAddOp2(v, OP_Goto, 1, pLevel->p2);
+        if( i<pWInfo->nLevel-1 ){
+          /* Ticket https://sqlite.org/src/info/ef9318757b152e3 2017-11-21
+          ** The break location for the next inner loop is above the code
+          ** generated here, but it should be afterwards.  So call re-resolve
+          ** the break location to be afterwards. */
+          sqlite3VdbeResolveLabel(v, pWInfo->a[i+1].addrBrk);
+        }
       }
 #endif /* SQLITE_DISABLE_SKIPAHEAD_DISTINCT */
       /* The common case: Advance to the next row */
diff --git a/test/distinct2.test b/test/distinct2.test
index a7d59db705..ae7a14cb1a 100644
--- a/test/distinct2.test
+++ b/test/distinct2.test
@@ -179,5 +179,24 @@ do_execsql_test 920 {
   wxYZ wxYz wxYz wxyZ wxyZ wxyz wxyz
 }
 
+# Ticket https://sqlite.org/src/info/ef9318757b152e3a on 2017-11-21
+# Incorrect result due to a skip-ahead-distinct optimization on a
+# join where no rows of the inner loop appear in the result set.
+#
+db close
+sqlite3 db :memory:
+do_execsql_test 1000 {
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b INTEGER);
+  CREATE INDEX t1b ON t1(b);
+  CREATE TABLE t2(x INTEGER PRIMARY KEY, y INTEGER);
+  CREATE INDEX t2y ON t2(y);
+  WITH RECURSIVE c(x) AS (VALUES(0) UNION ALL SELECT x+1 FROM c WHERE x<49)
+    INSERT INTO t1(b) SELECT x/10 - 1 FROM c;
+  WITH RECURSIVE c(x) AS (VALUES(-1) UNION ALL SELECT x+1 FROM c WHERE x<19)
+    INSERT INTO t2(x,y) SELECT x, 1 FROM c;
+  SELECT DISTINCT y FROM t1, t2 WHERE b=x AND b<>-1;
+  ANALYZE;
+  SELECT DISTINCT y FROM t1, t2 WHERE b=x AND b<>-1;
+} {1 1}
 
 finish_test

From fa337cc115c3151175607054607510f777a525d9 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 23 Nov 2017 00:45:21 +0000
Subject: [PATCH 104/488] Completely disable the skip-ahead-distinct
 optimization for all but the inner-most loop of a DISTINCT join.  Fix for
 ticket [ef9318757b152e3a26e9592]

FossilOrigin-Name: b7595cf2cadcba486e60b3c230ccc412a7cf449c1d56cbe65869e6d5d9f1374d
---
 manifest            | 14 +++++++-------
 manifest.uuid       |  2 +-
 src/where.c         |  8 +-------
 test/distinct2.test | 31 +++++++++++++++++++++++++++++++
 4 files changed, 40 insertions(+), 15 deletions(-)

diff --git a/manifest b/manifest
index a96176b69b..4044f32416 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sthe\sskip-ahead-distinct\soptimization\son\sjoins\sfor\scases\sthere\sthe\stable\nin\sthe\sinner\sloop\sof\sthe\sjoin\sdoes\snot\scontribute\sany\scolumns\sto\sthe\nresult\sset.\s\sProposed\sfix\sfor\sticket\s[ef9318757b152e3a2]
-D 2017-11-21T23:38:48.349
+C Completely\sdisable\sthe\sskip-ahead-distinct\soptimization\sfor\sall\sbut\sthe\ninner-most\sloop\sof\sa\sDISTINCT\sjoin.\s\sFix\sfor\sticket\s[ef9318757b152e3a26e9592]
+D 2017-11-23T00:45:21.475
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
@@ -553,7 +553,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c beeb71e4eab65dbf0d95f2717efc6ca3c0f5b3090ce67f3de63828f39a6ff053
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f
-F src/where.c 1c0ab20720f1a5e76fb4f78d2c45b4525b59167a20d44baf4467fa5b106679ad
+F src/where.c 1b85f0676c6b0e07bf815ce752e2f1e9968a4d506bfb1e9cb35dd95d18254fcf
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
 F src/wherecode.c 611fcabd05592ed2febd7d182f9621425b0466c5232d70e0981c842d429356d5
 F src/whereexpr.c 427ea8e96ec24f2a7814c67b8024ad664a9c7656264c4566c34743cb23186e46
@@ -735,7 +735,7 @@ F test/descidx2.test 9f1a0c83fd57f8667c82310ca21b30a350888b5d
 F test/descidx3.test 09ddbe3f5295f482d2f8b687cf6db8bad7acd9a2
 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e
 F test/distinct.test a1783b960ad8c15a77cd9f207be072898db1026c
-F test/distinct2.test ba239e9674495eda8e130cb5d498036b781dbd65c5868b58e094e36279859e83
+F test/distinct2.test df0bb52b754661ea84ec9ff488d48913c97bd31d83ca17ce0bf1334645e660cf
 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376
 F test/e_blobbytes.test 439a945953b35cb6948a552edaec4dc31fd70a05
 F test/e_blobclose.test 4b3c8c60c2171164d472059c73e9f3c1844bb66d
@@ -1677,7 +1677,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 17dd2f7314e7eb124e0a2a7a6cf475850e87fe3041e6ce1e1fd71b38d54852a8
-R c2dbc2aeaa2d6cfa4070ebdfc171fa89
+P 2dcef5a9ae7f347da65207bf6bf612fb12e18e1a6704799322f0cf2a86154cfd
+R 5a77744cb1beed3c66e0137b7a88aa75
 U drh
-Z 8287b3f25cb0100d8a846549bf308df5
+Z 8adc99178c09900ff045958fd53e1035
diff --git a/manifest.uuid b/manifest.uuid
index bc87796a6f..0aaff2a8b0 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2dcef5a9ae7f347da65207bf6bf612fb12e18e1a6704799322f0cf2a86154cfd
\ No newline at end of file
+b7595cf2cadcba486e60b3c230ccc412a7cf449c1d56cbe65869e6d5d9f1374d
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index 517d69fc1b..1a9f74dba0 100644
--- a/src/where.c
+++ b/src/where.c
@@ -4962,6 +4962,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
       Index *pIdx;
       int n;
       if( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED
+       && i==pWInfo->nLevel-1  /* Ticket [ef9318757b152e3] 2017-10-21 */
        && (pLoop->wsFlags & WHERE_INDEXED)!=0
        && (pIdx = pLoop->u.btree.pIndex)->hasStat1
        && (n = pLoop->u.btree.nIdxCol)>0
@@ -4978,13 +4979,6 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
         VdbeCoverageIf(v, op==OP_SeekLT);
         VdbeCoverageIf(v, op==OP_SeekGT);
         sqlite3VdbeAddOp2(v, OP_Goto, 1, pLevel->p2);
-        if( i<pWInfo->nLevel-1 ){
-          /* Ticket https://sqlite.org/src/info/ef9318757b152e3 2017-11-21
-          ** The break location for the next inner loop is above the code
-          ** generated here, but it should be afterwards.  So call re-resolve
-          ** the break location to be afterwards. */
-          sqlite3VdbeResolveLabel(v, pWInfo->a[i+1].addrBrk);
-        }
       }
 #endif /* SQLITE_DISABLE_SKIPAHEAD_DISTINCT */
       /* The common case: Advance to the next row */
diff --git a/test/distinct2.test b/test/distinct2.test
index ae7a14cb1a..31ab355132 100644
--- a/test/distinct2.test
+++ b/test/distinct2.test
@@ -198,5 +198,36 @@ do_execsql_test 1000 {
   ANALYZE;
   SELECT DISTINCT y FROM t1, t2 WHERE b=x AND b<>-1;
 } {1 1}
+db close
+sqlite3 db :memory:
+do_execsql_test 1010 {
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b INTEGER);
+  CREATE INDEX t1b ON t1(b);
+  CREATE TABLE t2(x INTEGER PRIMARY KEY, y INTEGER);
+  CREATE INDEX t2y ON t2(y);
+  WITH RECURSIVE c(x) AS (VALUES(0) UNION ALL SELECT x+1 FROM c WHERE x<49)
+    INSERT INTO t1(b) SELECT -(x/10 - 1) FROM c;
+  WITH RECURSIVE c(x) AS (VALUES(-1) UNION ALL SELECT x+1 FROM c WHERE x<19)
+    INSERT INTO t2(x,y) SELECT -x, 1 FROM c;
+  SELECT DISTINCT y FROM t1, t2 WHERE b=x AND b<>1 ORDER BY y DESC;
+  ANALYZE;
+  SELECT DISTINCT y FROM t1, t2 WHERE b=x AND b<>1 ORDER BY y DESC;
+} {1 1}
+db close
+sqlite3 db :memory:
+do_execsql_test 1020 {
+  CREATE TABLE t1(a, b);
+  CREATE INDEX t1a ON t1(a, b);
+  -- Lots of rows of (1, 'no'), followed by a single (1, 'yes').
+  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100)
+    INSERT INTO t1(a, b) SELECT 1, 'no' FROM c;
+  INSERT INTO t1(a, b) VALUES(1, 'yes');
+  CREATE TABLE t2(x PRIMARY KEY);
+  INSERT INTO t2 VALUES('yes');
+  SELECT DISTINCT a FROM t1, t2 WHERE x=b;
+  ANALYZE;
+  SELECT DISTINCT a FROM t1, t2 WHERE x=b;
+} {1 1}
+
 
 finish_test

From 53bf7175afd5cc963f2f863c7ee62e160bd9f26d Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 23 Nov 2017 04:45:35 +0000
Subject: [PATCH 105/488] Fix a problem in the omit-table-from-left-join
 optimization from check-in [0cd82ee9a8413cf] that was discovered by OSSFuzz.

FossilOrigin-Name: b016c28fa5617a20ad34c005372e738d28f7fc4388d19ee0cb7add4ed19d74aa
---
 manifest        | 14 +++++++-------
 manifest.uuid   |  2 +-
 src/where.c     |  8 +++++++-
 test/join2.test | 23 ++++++++++++++++++++++-
 4 files changed, 37 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index 4044f32416..d89d6d2180 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Completely\sdisable\sthe\sskip-ahead-distinct\soptimization\sfor\sall\sbut\sthe\ninner-most\sloop\sof\sa\sDISTINCT\sjoin.\s\sFix\sfor\sticket\s[ef9318757b152e3a26e9592]
-D 2017-11-23T00:45:21.475
+C Fix\sa\sproblem\sin\sthe\somit-table-from-left-join\soptimization\s\nfrom\scheck-in\s[0cd82ee9a8413cf]\sthat\swas\sdiscovered\sby\sOSSFuzz.
+D 2017-11-23T04:45:35.235
 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
@@ -553,7 +553,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c beeb71e4eab65dbf0d95f2717efc6ca3c0f5b3090ce67f3de63828f39a6ff053
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f
-F src/where.c 1b85f0676c6b0e07bf815ce752e2f1e9968a4d506bfb1e9cb35dd95d18254fcf
+F src/where.c 9752b68e03e2044f0faa4708fabb0189769067b660bffa931e1fd65736269659
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
 F src/wherecode.c 611fcabd05592ed2febd7d182f9621425b0466c5232d70e0981c842d429356d5
 F src/whereexpr.c 427ea8e96ec24f2a7814c67b8024ad664a9c7656264c4566c34743cb23186e46
@@ -983,7 +983,7 @@ F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c
 F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4
 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b
 F test/join.test 442c462eea85cf065d70a663c626b780a95af6e11585d909bb63b87598afe678
-F test/join2.test ac70b2b79ac593550d1d6f15e9bb3693dd71826b496e84f15166c5053fa518e6
+F test/join2.test 1a0c26399910b015d9f8f95b884e9a079fd2cfdccd65f7b1603846508cae0dc6
 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0
 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916
 F test/join5.test bc98ea4b4e5003f5b1453701ebb8cd7d1c01a550
@@ -1677,7 +1677,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2dcef5a9ae7f347da65207bf6bf612fb12e18e1a6704799322f0cf2a86154cfd
-R 5a77744cb1beed3c66e0137b7a88aa75
+P b7595cf2cadcba486e60b3c230ccc412a7cf449c1d56cbe65869e6d5d9f1374d
+R 9669b3f605458d14cc31b00c90e7e717
 U drh
-Z 8adc99178c09900ff045958fd53e1035
+Z 772d565646149360b17fb446b36f96b2
diff --git a/manifest.uuid b/manifest.uuid
index 0aaff2a8b0..adb6476bf6 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b7595cf2cadcba486e60b3c230ccc412a7cf449c1d56cbe65869e6d5d9f1374d
\ No newline at end of file
+b016c28fa5617a20ad34c005372e738d28f7fc4388d19ee0cb7add4ed19d74aa
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index 1a9f74dba0..27979efbc4 100644
--- a/src/where.c
+++ b/src/where.c
@@ -4707,6 +4707,7 @@ WhereInfo *sqlite3WhereBegin(
   **       LEFT JOIN t2
   **       LEFT JOIN t3 USING (t1.ipk=t3.ipk)
   */
+  notReady = ~(Bitmask)0;
   if( pWInfo->nLevel>=2
    && pResultSet!=0               /* guarantees condition (1) above */
    && OptimizationEnabled(db, SQLITE_OmitNoopJoin)
@@ -4740,6 +4741,12 @@ WhereInfo *sqlite3WhereBegin(
       }
       if( pTerm<pEnd ) continue;
       WHERETRACE(0xffff, ("-> drop loop %c not used\n", pLoop->cId));
+      notReady &= ~pLoop->maskSelf;
+      for(pTerm=sWLB.pWC->a; pTerm<pEnd; pTerm++){
+        if( (pTerm->prereqAll & pLoop->maskSelf)!=0 ){
+          pTerm->wtFlags |= TERM_CODED;
+        }
+      }
       if( i!=pWInfo->nLevel-1 ){
         int nByte = (pWInfo->nLevel-1-i) * sizeof(WhereLevel);
         memmove(&pWInfo->a[i], &pWInfo->a[i+1], nByte);
@@ -4898,7 +4905,6 @@ WhereInfo *sqlite3WhereBegin(
   ** loop below generates code for a single nested loop of the VM
   ** program.
   */
-  notReady = ~(Bitmask)0;
   for(ii=0; ii<nTabList; ii++){
     int addrExplain;
     int wsFlags;
diff --git a/test/join2.test b/test/join2.test
index 9fdcef1783..76006f1ecd 100644
--- a/test/join2.test
+++ b/test/join2.test
@@ -169,6 +169,27 @@ do_eqp_test 4.2.2 {
   0 1 2 {SEARCH TABLE c3 USING INTEGER PRIMARY KEY (rowid=?)}
 }
 
-
+# 2017-11-23 (Thanksgiving day)
+# OSSFuzz found an assertion fault in the new LEFT JOIN eliminator code.
+#
+do_execsql_test 4.3.0 {
+  DROP TABLE IF EXISTS t1;
+  DROP TABLE IF EXISTS t2;
+  CREATE TABLE t1(x PRIMARY KEY) WITHOUT ROWID;
+  CREATE TABLE t2(x);
+  SELECT a.x
+    FROM t1 AS a
+    LEFT JOIN t1 AS b ON (a.x=b.x)
+    LEFT JOIN t2 AS c ON (a.x=c.x);
+} {}
+do_execsql_test 4.3.1 {
+  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<10)
+    INSERT INTO t1(x) SELECT x FROM c;
+  INSERT INTO t2(x) SELECT x+9 FROM t1;
+  SELECT a.x, c.x
+    FROM t1 AS a
+    LEFT JOIN t1 AS b ON (a.x=b.x)
+    LEFT JOIN t2 AS c ON (a.x=c.x);
+} {1 {} 2 {} 3 {} 4 {} 5 {} 6 {} 7 {} 8 {} 9 {} 10 10}
 
 finish_test

From 56c9311c638f44c7b6e0522f82dc799b9cccdfb3 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 24 Nov 2017 16:55:48 +0000
Subject: [PATCH 106/488] Enhance the configure script to detect zLib.

FossilOrigin-Name: e3b6e22049caf78bc4153ded8dc295fe30ad320323d921f16bd794ef30f1b115
---
 Makefile.in   |   3 ++
 configure     | 103 +++++++++++++++++++++++++++++++++++++++++++-------
 configure.ac  |   6 +++
 manifest      |  16 ++++----
 manifest.uuid |   2 +-
 5 files changed, 107 insertions(+), 23 deletions(-)

diff --git a/Makefile.in b/Makefile.in
index 567f31215f..4883c1ac6b 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -89,6 +89,9 @@ TCC += $(OPT_FEATURE_FLAGS)
 # ie.  make "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1".
 TCC += $(OPTS)
 
+# Add in compile-time options for some libraries used by extensions
+TCC += @HAVE_ZLIB@
+
 # Version numbers and release number for the SQLite being compiled.
 #
 VERSION = @VERSION@
diff --git a/configure b/configure
index 9e7aa526a2..8e735c6b12 100755
--- a/configure
+++ b/configure
@@ -772,6 +772,7 @@ LIBOBJS
 BUILD_CFLAGS
 USE_GCOV
 OPT_FEATURE_FLAGS
+HAVE_ZLIB
 USE_AMALGAMATION
 TARGET_DEBUG
 TARGET_HAVE_EDITLINE
@@ -3931,13 +3932,13 @@ if ${lt_cv_nm_interface+:} false; then :
 else
   lt_cv_nm_interface="BSD nm"
   echo "int some_variable = 0;" > conftest.$ac_ext
-  (eval echo "\"\$as_me:3934: $ac_compile\"" >&5)
+  (eval echo "\"\$as_me:3935: $ac_compile\"" >&5)
   (eval "$ac_compile" 2>conftest.err)
   cat conftest.err >&5
-  (eval echo "\"\$as_me:3937: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
+  (eval echo "\"\$as_me:3938: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
   (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out)
   cat conftest.err >&5
-  (eval echo "\"\$as_me:3940: output\"" >&5)
+  (eval echo "\"\$as_me:3941: output\"" >&5)
   cat conftest.out >&5
   if $GREP 'External.*some_variable' conftest.out > /dev/null; then
     lt_cv_nm_interface="MS dumpbin"
@@ -5143,7 +5144,7 @@ ia64-*-hpux*)
   ;;
 *-*-irix6*)
   # Find out which ABI we are using.
-  echo '#line 5146 "configure"' > conftest.$ac_ext
+  echo '#line 5147 "configure"' > conftest.$ac_ext
   if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5
   (eval $ac_compile) 2>&5
   ac_status=$?
@@ -6668,11 +6669,11 @@ else
    -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
    -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
    -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:6671: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:6672: $lt_compile\"" >&5)
    (eval "$lt_compile" 2>conftest.err)
    ac_status=$?
    cat conftest.err >&5
-   echo "$as_me:6675: \$? = $ac_status" >&5
+   echo "$as_me:6676: \$? = $ac_status" >&5
    if (exit $ac_status) && test -s "$ac_outfile"; then
      # The compiler can only warn and ignore the option if not recognized
      # So say no if there are warnings other than the usual output.
@@ -7007,11 +7008,11 @@ else
    -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
    -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
    -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7010: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7011: $lt_compile\"" >&5)
    (eval "$lt_compile" 2>conftest.err)
    ac_status=$?
    cat conftest.err >&5
-   echo "$as_me:7014: \$? = $ac_status" >&5
+   echo "$as_me:7015: \$? = $ac_status" >&5
    if (exit $ac_status) && test -s "$ac_outfile"; then
      # The compiler can only warn and ignore the option if not recognized
      # So say no if there are warnings other than the usual output.
@@ -7112,11 +7113,11 @@ else
    -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
    -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
    -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7115: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7116: $lt_compile\"" >&5)
    (eval "$lt_compile" 2>out/conftest.err)
    ac_status=$?
    cat out/conftest.err >&5
-   echo "$as_me:7119: \$? = $ac_status" >&5
+   echo "$as_me:7120: \$? = $ac_status" >&5
    if (exit $ac_status) && test -s out/conftest2.$ac_objext
    then
      # The compiler can only warn and ignore the option if not recognized
@@ -7167,11 +7168,11 @@ else
    -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
    -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
    -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7170: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7171: $lt_compile\"" >&5)
    (eval "$lt_compile" 2>out/conftest.err)
    ac_status=$?
    cat out/conftest.err >&5
-   echo "$as_me:7174: \$? = $ac_status" >&5
+   echo "$as_me:7175: \$? = $ac_status" >&5
    if (exit $ac_status) && test -s out/conftest2.$ac_objext
    then
      # The compiler can only warn and ignore the option if not recognized
@@ -9547,7 +9548,7 @@ else
   lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
   lt_status=$lt_dlunknown
   cat > conftest.$ac_ext <<_LT_EOF
-#line 9550 "configure"
+#line 9551 "configure"
 #include "confdefs.h"
 
 #if HAVE_DLFCN_H
@@ -9643,7 +9644,7 @@ else
   lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
   lt_status=$lt_dlunknown
   cat > conftest.$ac_ext <<_LT_EOF
-#line 9646 "configure"
+#line 9647 "configure"
 #include "confdefs.h"
 
 #if HAVE_DLFCN_H
@@ -11274,6 +11275,80 @@ if test "${use_amalgamation}" != "yes" ; then
 fi
 
 
+#########
+# Look for zlib.  Only needed by extensions and by the sqlite3.exe shell
+for ac_header in zlib.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "zlib.h" "ac_cv_header_zlib_h" "$ac_includes_default"
+if test "x$ac_cv_header_zlib_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_ZLIB_H 1
+_ACEOF
+
+fi
+
+done
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing deflate" >&5
+$as_echo_n "checking for library containing deflate... " >&6; }
+if ${ac_cv_search_deflate+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  ac_func_search_save_LIBS=$LIBS
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+/* Override any GCC internal prototype to avoid an error.
+   Use char because int might match the return type of a GCC
+   builtin and then its argument prototype would still apply.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+char deflate ();
+int
+main ()
+{
+return deflate ();
+  ;
+  return 0;
+}
+_ACEOF
+for ac_lib in '' z; do
+  if test -z "$ac_lib"; then
+    ac_res="none required"
+  else
+    ac_res=-l$ac_lib
+    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
+  fi
+  if ac_fn_c_try_link "$LINENO"; then :
+  ac_cv_search_deflate=$ac_res
+fi
+rm -f core conftest.err conftest.$ac_objext \
+    conftest$ac_exeext
+  if ${ac_cv_search_deflate+:} false; then :
+  break
+fi
+done
+if ${ac_cv_search_deflate+:} false; then :
+
+else
+  ac_cv_search_deflate=no
+fi
+rm conftest.$ac_ext
+LIBS=$ac_func_search_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_deflate" >&5
+$as_echo "$ac_cv_search_deflate" >&6; }
+ac_res=$ac_cv_search_deflate
+if test "$ac_res" != no; then :
+  test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
+  HAVE_ZLIB="-DSQLITE_HAVE_ZLIB=1"
+else
+  HAVE_ZLIB=""
+fi
+
+
+
 #########
 # See whether we should allow loadable extensions
 # Check whether --enable-load-extension was given.
diff --git a/configure.ac b/configure.ac
index ace5d5d667..7089772d19 100644
--- a/configure.ac
+++ b/configure.ac
@@ -576,6 +576,12 @@ if test "${use_amalgamation}" != "yes" ; then
 fi
 AC_SUBST(USE_AMALGAMATION)
 
+#########
+# Look for zlib.  Only needed by extensions and by the sqlite3.exe shell
+AC_CHECK_HEADERS(zlib.h)
+AC_SEARCH_LIBS(deflate, z, [HAVE_ZLIB="-DSQLITE_HAVE_ZLIB=1"], [HAVE_ZLIB=""])
+AC_SUBST(HAVE_ZLIB)
+
 #########
 # See whether we should allow loadable extensions
 AC_ARG_ENABLE(load-extension, AC_HELP_STRING([--disable-load-extension],
diff --git a/manifest b/manifest
index d89d6d2180..d56b60ea65 100644
--- a/manifest
+++ b/manifest
@@ -1,6 +1,6 @@
-C Fix\sa\sproblem\sin\sthe\somit-table-from-left-join\soptimization\s\nfrom\scheck-in\s[0cd82ee9a8413cf]\sthat\swas\sdiscovered\sby\sOSSFuzz.
-D 2017-11-23T04:45:35.235
-F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2
+C Enhance\sthe\sconfigure\sscript\sto\sdetect\szLib.
+D 2017-11-24T16:55:48.156
+F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd
@@ -30,8 +30,8 @@ F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63
 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977
 F config.h.in 6376abec766e9a0785178b1823b5a587e9f1ccbc
 F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55
-F configure bdc49e9f0b0ced903ebdb2850362dd3391eeb88585e0429d12b94928d2873b6b x
-F configure.ac 369ebae6c04d9d2de5064e21d300f2f42f2fbf13235cabff9d1a54f2b2c4d05d
+F configure 9af547be0e0e1a8fca8553b82599b5a3be1528a3d78deb68cb49d3b611215cb7 x
+F configure.ac d4529ebb26ae046269334f1dac65f2b1d6927c2efe22b2ec24dce24dfe4f83dd
 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad
 F doc/lemon.html 278113807f49d12d04179a93fab92b5b917a08771152ca7949d34e928efa3941
 F doc/pager-invariants.txt 27fed9a70ddad2088750c4a2b493b63853da2710
@@ -1677,7 +1677,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b7595cf2cadcba486e60b3c230ccc412a7cf449c1d56cbe65869e6d5d9f1374d
-R 9669b3f605458d14cc31b00c90e7e717
+P b016c28fa5617a20ad34c005372e738d28f7fc4388d19ee0cb7add4ed19d74aa
+R 32193c0299a7059451a8ff25bb895cce
 U drh
-Z 772d565646149360b17fb446b36f96b2
+Z 88a3edc8b736c2bf1f1a3f732d1a86ca
diff --git a/manifest.uuid b/manifest.uuid
index adb6476bf6..bdb0f396fc 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b016c28fa5617a20ad34c005372e738d28f7fc4388d19ee0cb7add4ed19d74aa
\ No newline at end of file
+e3b6e22049caf78bc4153ded8dc295fe30ad320323d921f16bd794ef30f1b115
\ No newline at end of file

From d37ce8396a820cb01105d3c88b63e3922ff77c98 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 24 Nov 2017 19:24:44 +0000
Subject: [PATCH 107/488] Add the "^" syntax from fts3/4 to fts5.

FossilOrigin-Name: 24d7058e2799133dd681d2fef341025ca50554861bb4cd39e93ee87ae1d8a605
---
 ext/fts5/fts5Int.h            |  2 +
 ext/fts5/fts5_expr.c          | 62 +++++++++++++++-------
 ext/fts5/fts5parse.y          |  7 ++-
 ext/fts5/test/fts5faultB.test | 17 +++++++
 ext/fts5/test/fts5first.test  | 96 +++++++++++++++++++++++++++++++++++
 manifest                      | 21 ++++----
 manifest.uuid                 |  2 +-
 7 files changed, 176 insertions(+), 31 deletions(-)
 create mode 100644 ext/fts5/test/fts5first.test

diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index 63dc082687..c87583676b 100644
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -722,6 +722,8 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(
   int bPrefix
 );
 
+void sqlite3Fts5ParseSetCaret(Fts5ExprPhrase*);
+
 Fts5ExprNearset *sqlite3Fts5ParseNearset(
   Fts5Parse*, 
   Fts5ExprNearset*,
diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c
index aa7141cfee..a86dbebdf9 100644
--- a/ext/fts5/fts5_expr.c
+++ b/ext/fts5/fts5_expr.c
@@ -87,7 +87,8 @@ struct Fts5ExprNode {
 ** or term prefix.
 */
 struct Fts5ExprTerm {
-  int bPrefix;                    /* True for a prefix term */
+  u8 bPrefix;                     /* True for a prefix term */
+  u8 bFirst;                      /* True if token must be first in column */
   char *zTerm;                    /* nul-terminated term */
   Fts5IndexIter *pIter;           /* Iterator for this term */
   Fts5ExprTerm *pSynonym;         /* Pointer to first in list of synonyms */
@@ -168,6 +169,7 @@ static int fts5ExprGetToken(
     case '+':  tok = FTS5_PLUS;  break;
     case '*':  tok = FTS5_STAR;  break;
     case '-':  tok = FTS5_MINUS; break;
+    case '^':  tok = FTS5_CARET; break;
     case '\0': tok = FTS5_EOF;   break;
 
     case '"': {
@@ -427,6 +429,7 @@ static int fts5ExprPhraseIsMatch(
   Fts5PoslistReader *aIter = aStatic;
   int i;
   int rc = SQLITE_OK;
+  int bFirst = pPhrase->aTerm[0].bFirst;
   
   fts5BufferZero(&pPhrase->poslist);
 
@@ -481,8 +484,10 @@ static int fts5ExprPhraseIsMatch(
     }while( bMatch==0 );
 
     /* Append position iPos to the output */
-    rc = sqlite3Fts5PoslistWriterAppend(&pPhrase->poslist, &writer, iPos);
-    if( rc!=SQLITE_OK ) goto ismatch_out;
+    if( bFirst==0 || FTS5_POS2OFFSET(iPos)==0 ){
+      rc = sqlite3Fts5PoslistWriterAppend(&pPhrase->poslist, &writer, iPos);
+      if( rc!=SQLITE_OK ) goto ismatch_out;
+    }
 
     for(i=0; i<pPhrase->nTerm; i++){
       if( sqlite3Fts5PoslistReaderNext(&aIter[i]) ) goto ismatch_out;
@@ -736,7 +741,9 @@ static int fts5ExprNearTest(
     ** phrase is not a match, break out of the loop early.  */
     for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
       Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
-      if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){
+      if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym 
+       || pNear->pColset || pPhrase->aTerm[0].bFirst
+      ){
         int bMatch = 0;
         rc = fts5ExprPhraseIsMatch(pNode, pPhrase, &bMatch);
         if( bMatch==0 ) break;
@@ -917,6 +924,7 @@ static int fts5ExprNodeTest_STRING(
   assert( pNear->nPhrase>1 
        || pNear->apPhrase[0]->nTerm>1 
        || pNear->apPhrase[0]->aTerm[0].pSynonym
+       || pNear->apPhrase[0]->aTerm[0].bFirst
   );
 
   /* Initialize iLast, the "lastest" rowid any iterator points to. If the
@@ -1441,6 +1449,16 @@ static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){
   }
 }
 
+/*
+** Set the "bFirst" flag on the first token of the phrase passed as the
+** only argument.
+*/
+void sqlite3Fts5ParseSetCaret(Fts5ExprPhrase *pPhrase){
+  if( pPhrase && pPhrase->nTerm ){
+    pPhrase->aTerm[0].bFirst = 1;
+  }
+}
+
 /*
 ** If argument pNear is NULL, then a new Fts5ExprNearset object is allocated
 ** and populated with pPhrase. Or, if pNear is not NULL, phrase pPhrase is
@@ -1719,6 +1737,7 @@ int sqlite3Fts5ExprClonePhrase(
       }
       if( rc==SQLITE_OK ){
         sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
+        sCtx.pPhrase->aTerm[i].bFirst = pOrig->aTerm[i].bFirst;
       }
     }
   }else{
@@ -1737,7 +1756,10 @@ int sqlite3Fts5ExprClonePhrase(
     pNew->pRoot->pNear->nPhrase = 1;
     sCtx.pPhrase->pNode = pNew->pRoot;
 
-    if( pOrig->nTerm==1 && pOrig->aTerm[0].pSynonym==0 ){
+    if( pOrig->nTerm==1 
+     && pOrig->aTerm[0].pSynonym==0 
+     && pOrig->aTerm[0].bFirst==0 
+    ){
       pNew->pRoot->eType = FTS5_TERM;
       pNew->pRoot->xNext = fts5ExprNodeNext_TERM;
     }else{
@@ -2011,6 +2033,7 @@ static void fts5ExprAssignXNext(Fts5ExprNode *pNode){
       Fts5ExprNearset *pNear = pNode->pNear;
       if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 
        && pNear->apPhrase[0]->aTerm[0].pSynonym==0
+       && pNear->apPhrase[0]->aTerm[0].bFirst==0
       ){
         pNode->eType = FTS5_TERM;
         pNode->xNext = fts5ExprNodeNext_TERM;
@@ -2097,20 +2120,23 @@ Fts5ExprNode *sqlite3Fts5ParseNode(
           }
         }
 
-        if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL 
-         && (pNear->nPhrase!=1 || pNear->apPhrase[0]->nTerm>1)
-        ){
-          assert( pParse->rc==SQLITE_OK );
-          pParse->rc = SQLITE_ERROR;
-          assert( pParse->zErr==0 );
-          pParse->zErr = sqlite3_mprintf(
-              "fts5: %s queries are not supported (detail!=full)", 
-              pNear->nPhrase==1 ? "phrase": "NEAR"
-          );
-          sqlite3_free(pRet);
-          pRet = 0;
+        if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL ){
+          Fts5ExprPhrase *pPhrase = pNear->apPhrase[0];
+          if( pNear->nPhrase!=1 
+           || pPhrase->nTerm>1
+           || (pPhrase->nTerm>0 && pPhrase->aTerm[0].bFirst)
+          ){
+            assert( pParse->rc==SQLITE_OK );
+            pParse->rc = SQLITE_ERROR;
+            assert( pParse->zErr==0 );
+            pParse->zErr = sqlite3_mprintf(
+                "fts5: %s queries are not supported (detail!=full)", 
+                pNear->nPhrase==1 ? "phrase": "NEAR"
+                );
+            sqlite3_free(pRet);
+            pRet = 0;
+          }
         }
-
       }else{
         fts5ExprAddChildren(pRet, pLeft);
         fts5ExprAddChildren(pRet, pRight);
diff --git a/ext/fts5/fts5parse.y b/ext/fts5/fts5parse.y
index 1582909aa8..134125db1f 100644
--- a/ext/fts5/fts5parse.y
+++ b/ext/fts5/fts5parse.y
@@ -148,7 +148,11 @@ cnearset(A) ::= colset(X) COLON nearset(Y). {
 %destructor nearset { sqlite3Fts5ParseNearsetFree($$); }
 %destructor nearphrases { sqlite3Fts5ParseNearsetFree($$); }
 
-nearset(A) ::= phrase(X). { A = sqlite3Fts5ParseNearset(pParse, 0, X); }
+nearset(A) ::= phrase(Y). { A = sqlite3Fts5ParseNearset(pParse, 0, Y); }
+nearset(A) ::= CARET phrase(Y). { 
+  sqlite3Fts5ParseSetCaret(Y);
+  A = sqlite3Fts5ParseNearset(pParse, 0, Y); 
+}
 nearset(A) ::= STRING(X) LP nearphrases(Y) neardist_opt(Z) RP. {
   sqlite3Fts5ParseNear(pParse, &X);
   sqlite3Fts5ParseSetDistance(pParse, Y, &Z);
@@ -189,6 +193,5 @@ phrase(A) ::= STRING(Y) star_opt(Z). {
 ** Optional "*" character.
 */
 %type star_opt {int}
-
 star_opt(A) ::= STAR. { A = 1; }
 star_opt(A) ::= . { A = 0; }
diff --git a/ext/fts5/test/fts5faultB.test b/ext/fts5/test/fts5faultB.test
index a4fef523f5..2faec706d5 100644
--- a/ext/fts5/test/fts5faultB.test
+++ b/ext/fts5/test/fts5faultB.test
@@ -130,5 +130,22 @@ do_faultsim_test 4.2 -faults oom* -body {
   faultsim_test_result {0 {2 3}}
 }
 
+#-------------------------------------------------------------------------
+# Test OOM injection while parsing a CARET expression
+#
+reset_db
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a);
+  INSERT INTO t1 VALUES('a b c d');  -- 1
+  INSERT INTO t1 VALUES('d a b c');  -- 2
+  INSERT INTO t1 VALUES('c d a b');  -- 3
+  INSERT INTO t1 VALUES('b c d a');  -- 4
+}
+do_faultsim_test 5.1 -faults oom* -body {
+  execsql { SELECT rowid FROM t1('^a OR ^b') }
+} -test {
+  faultsim_test_result {0 {1 4}}
+}
+
 
 finish_test
diff --git a/ext/fts5/test/fts5first.test b/ext/fts5/test/fts5first.test
new file mode 100644
index 0000000000..b2cac1fdf1
--- /dev/null
+++ b/ext/fts5/test/fts5first.test
@@ -0,0 +1,96 @@
+# 2017 November 25
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5first
+
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(a, b);
+}
+
+foreach {tn expr ok} {
+  1 {^abc}           1
+  2 {^abc + def}     1
+  3 {^ "abc def"}    1
+  4 {^"abc def"}     1
+  5 {abc ^def}       1
+  6 {abc + ^def}     0
+  7 {abc ^+ def}     0
+  8 {"^abc"}         1
+  9 {NEAR(^abc def)} 0
+} {
+  set res(0) {/1 {fts5: syntax error near .*}/}
+  set res(1) {0 {}}
+
+  do_catchsql_test 1.$tn { SELECT * FROM x1($expr) } $res($ok)
+}
+
+#-------------------------------------------------------------------------
+# 
+do_execsql_test 2.0 {
+  INSERT INTO x1 VALUES('a b c', 'b c a');
+}
+
+foreach {tn expr match} {
+  1 {^a} 1
+  2 {^b} 1
+  3 {^c} 0
+  4 {^a + b} 1
+  5 {^b + c} 1
+  6 {^c + a} 0
+  7 {^"c a"} 0
+  8 {a:^a} 1
+  9 {a:^b} 0
+  10 {a:^"a b"} 1
+} {
+  do_execsql_test 2.$tn { SELECT EXISTS (SELECT rowid FROM x1($expr)) } $match
+}
+
+#-------------------------------------------------------------------------
+# 
+do_execsql_test 3.0 {
+  DELETE FROM x1;
+  INSERT INTO x1 VALUES('b a', 'c a');
+  INSERT INTO x1 VALUES('a a', 'c c');
+  INSERT INTO x1 VALUES('a b', 'a a');
+}
+fts5_aux_test_functions db
+
+foreach {tn expr expect} {
+  1 {^a} {{2 1}}
+  2 {^c AND ^b} {{0 2} {1 0}}
+} {
+  do_execsql_test 3.$tn {
+    SELECT fts5_test_queryphrase(x1) FROM x1($expr) LIMIT 1
+  } [list $expect]
+}
+
+#-------------------------------------------------------------------------
+# 
+do_execsql_test 3.1 {
+  CREATE VIRTUAL TABLE x2 USING fts5(a, b, c, detail=column);
+}
+
+do_catchsql_test 3.2 {
+  SELECT * FROM x2('a + b');
+} {1 {fts5: phrase queries are not supported (detail!=full)}}
+
+do_catchsql_test 3.3 {
+  SELECT * FROM x2('^a');
+} {1 {fts5: phrase queries are not supported (detail!=full)}}
+finish_test
+
diff --git a/manifest b/manifest
index d56b60ea65..fa07b8e660 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Enhance\sthe\sconfigure\sscript\sto\sdetect\szLib.
-D 2017-11-24T16:55:48.156
+C Add\sthe\s"^"\ssyntax\sfrom\sfts3/4\sto\sfts5.
+D 2017-11-24T19:24:44.918
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
@@ -99,11 +99,11 @@ F ext/fts3/unicode/mkunicode.tcl ab0543a3b2399092ea2dd75df1bef333405b0d7f6b8c495
 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h 62f3e33ceeb9a428db139f9c012186b371da1cc7
-F ext/fts5/fts5Int.h 15e7514b46a845937d7c62e5c69e935091f0dbb72eb61aa4c8bcfbd39fdea158
+F ext/fts5/fts5Int.h eda28e3a0a5d87c412e8355fe35da875b04cb389908c8eb0d867ad662adbc491
 F ext/fts5/fts5_aux.c 67acf8d51723cf28ffc3828210ba662df4b8d267
 F ext/fts5/fts5_buffer.c 1dd1ec0446b3acfc2d7d407eb894762a461613e2695273f48e449bfd13e973ff
 F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857
-F ext/fts5/fts5_expr.c f2825f714d91bbe62ab5820aee9ad12e0c94205b2a01725eaa9072415ae9ff1c
+F ext/fts5/fts5_expr.c 01048018d21524e2c302b063ff5c3cdcf546e03297215e577205d85b47499deb
 F ext/fts5/fts5_hash.c 32be400cf761868c9db33efe81a06eb19a17c5402ad477ee9efb51301546dd55
 F ext/fts5/fts5_index.c 2ce9d50ec5508b8205615aad69e1c9b2c77f017f21d4479e1fb2079c01fdd017
 F ext/fts5/fts5_main.c 24868f88ab2a865defbba7a92eebeb726cc991eb092b71b5f5508f180c72605b
@@ -115,7 +115,7 @@ F ext/fts5/fts5_tokenize.c 2ce7b44183538ec46b7907726262ee43ffdd39a8
 F ext/fts5/fts5_unicode2.c b450b209b157d598f7b9df9f837afb75a14c24bf
 F ext/fts5/fts5_varint.c a5aceacda04dafcbae725413d7a16818ecd65738
 F ext/fts5/fts5_vocab.c 1cd79854cb21543e66507b25b0578bc1b20aa6a1349b7feceb8e8fed0e7a77a6
-F ext/fts5/fts5parse.y a070b538e08ae9e2177d15c337ed2a3464408f0f886e746307098f746efd94ca
+F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05
 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
 F ext/fts5/test/fts5_common.tcl b01c584144b5064f30e6c648145a2dd6bc440841
 F ext/fts5/test/fts5aa.test cba3fae6466446980caf1b9f5f26df77f95a999d35db7d932d6e82ae7ba0ede9
@@ -162,8 +162,9 @@ F ext/fts5/test/fts5fault7.test 0acbec416edb24b8881f154e99c31e9ccf73f539cfcd1640
 F ext/fts5/test/fts5fault8.test 318238659d35f82ad215ecb57ca4c87486ea85d45dbeedaee42f148ff5105ee2
 F ext/fts5/test/fts5fault9.test 0111b229388bdf251b91cfead68580227801dd30960a19aa8fe9021a1e73cb6d
 F ext/fts5/test/fts5faultA.test be4487576bff8c22cee6597d1893b312f306504a8c6ccd3c53ca85af12290c8c
-F ext/fts5/test/fts5faultB.test 28810d93d37b59ebd5cf9502897f4dc9e6adb8ea6a5f64e125d3088597199d0d
+F ext/fts5/test/fts5faultB.test e6d04f9ea7b21be1d89abb8df2cb4baf65b0453b744d5a805fcd3ef45ff86a7e
 F ext/fts5/test/fts5faultD.test cc5d1225556e356615e719c612e845d41bff7d5a
+F ext/fts5/test/fts5first.test 707a591b1b7d893fcfcb2366cbfe56aefab5d9c7cfa58bef35eba73a1dbf3b29
 F ext/fts5/test/fts5full.test 49b565da02918c06e58f51f0b953b0302b96f155aa68baba24782b81570685e2
 F ext/fts5/test/fts5fuzz1.test 238d8c45f3b81342aa384de3e581ff2fa330bf922a7b69e484bbc06051a1080e
 F ext/fts5/test/fts5hash.test a4cf51acad99bfc43c16fb74f9d22495dc221ae0701fc5e908ca963a9b26a02b
@@ -1677,7 +1678,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b016c28fa5617a20ad34c005372e738d28f7fc4388d19ee0cb7add4ed19d74aa
-R 32193c0299a7059451a8ff25bb895cce
-U drh
-Z 88a3edc8b736c2bf1f1a3f732d1a86ca
+P e3b6e22049caf78bc4153ded8dc295fe30ad320323d921f16bd794ef30f1b115
+R dd0642a6cc4628ee924c556518e62685
+U dan
+Z b13dcd2f6b4a5ff1701c0fa74c45ed53
diff --git a/manifest.uuid b/manifest.uuid
index bdb0f396fc..45ca11ffc8 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e3b6e22049caf78bc4153ded8dc295fe30ad320323d921f16bd794ef30f1b115
\ No newline at end of file
+24d7058e2799133dd681d2fef341025ca50554861bb4cd39e93ee87ae1d8a605
\ No newline at end of file

From 8385becfe153283312468feb5bf32219e6973940 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 25 Nov 2017 17:51:01 +0000
Subject: [PATCH 108/488] Add experimental feature to detect threading bugs in
 apps that use SQLITE_CONFIG_MULTITHREADED. Enabled at compile time using
 SQLITE_ENABLE_MULTITHREADED_CHECKS.

FossilOrigin-Name: a66886ac13aa6d8ccbb6d673ddd00267c93e3ee1fbc158236fce3157d150868d
---
 manifest        |  19 ++---
 manifest.uuid   |   2 +-
 src/main.c      |  10 ++-
 src/mutex.c     | 181 ++++++++++++++++++++++++++++++++++++++++++++++++
 src/sqliteInt.h |   6 ++
 5 files changed, 208 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index fa07b8e660..a55dbc7aa2 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\s"^"\ssyntax\sfrom\sfts3/4\sto\sfts5.
-D 2017-11-24T19:24:44.918
+C Add\sexperimental\sfeature\sto\sdetect\sthreading\sbugs\sin\sapps\sthat\suse\nSQLITE_CONFIG_MULTITHREADED.\sEnabled\sat\scompile\stime\susing\nSQLITE_ENABLE_MULTITHREADED_CHECKS.
+D 2017-11-25T17:51:01.072
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
@@ -436,7 +436,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c cb67cc56ef2ddd13e6944b2c0dd08a920bcd9503230adef8b9928d338097c722
 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e
 F src/loadext.c 20865b183bb8a3723d59cf1efffc3c50217eb452c1021d077b908c94da26b0b2
-F src/main.c 99ed3d45e315afb2ada049991db7944b1210663bb30bfd0b63103537c1ac25d0
+F src/main.c 6a0cc1c7b8ab92374effecdd7b92792b3273a255c70575b7d67bd9a4315e6d3a
 F src/malloc.c a02c9e69bc76bee0f639416b947a946412890b606301454727feadcb313536d6
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -445,7 +445,7 @@ F src/mem3.c 8768ac94694f31ffaf8b4d0ea5dc08af7010a35a
 F src/mem5.c 9bf955937b07f8c32541c8a9991f33ce3173d944
 F src/memjournal.c 6f3d36a0a8f72f48f6c3c722f04301ac64f2515435fa42924293e46fc7994661
 F src/msvc.h 4942752b6a253116baaa8de75256c51a459a5e81
-F src/mutex.c 8e45800ee78e0cd1f1f3fe8e398853307f4a085c
+F src/mutex.c 38addb10f90641b5f88521f7099e729ef10e3f0ac50bd6b9196183fb313a0378
 F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85
 F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4
 F src/mutex_unix.c 27bb6cc49485ee46711a6580ab7b3f1402211d23
@@ -476,7 +476,7 @@ F src/shell.c.in cb1b5e41ef9c081b2b8927ae32c9c384a9ec110ada808ebfe083ba7c8a19bbb
 F src/sqlite.h.in 8fd97993d48b50b9bade38c52f12d175942c9497c960905610c7b03a3e4b5818
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
-F src/sqliteInt.h 9b26fbab75ef426efae70d88ab535844d59de8954542b122c1d49af580a76f58
+F src/sqliteInt.h 6b084f4b7c0ea0cde24f4400a88fca7108f1a304cb567338495bba46b0fd68c6
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -1678,7 +1678,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e3b6e22049caf78bc4153ded8dc295fe30ad320323d921f16bd794ef30f1b115
-R dd0642a6cc4628ee924c556518e62685
+P 24d7058e2799133dd681d2fef341025ca50554861bb4cd39e93ee87ae1d8a605
+R 1e88d93b9907cb6e6661756c6df7c284
+T *branch * multithreaded-checks
+T *sym-multithreaded-checks *
+T -sym-trunk *
 U dan
-Z b13dcd2f6b4a5ff1701c0fa74c45ed53
+Z 7380701e6ba5cd3f46b8984142176402
diff --git a/manifest.uuid b/manifest.uuid
index 45ca11ffc8..99d6d7ea4c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-24d7058e2799133dd681d2fef341025ca50554861bb4cd39e93ee87ae1d8a605
\ No newline at end of file
+a66886ac13aa6d8ccbb6d673ddd00267c93e3ee1fbc158236fce3157d150868d
\ No newline at end of file
diff --git a/src/main.c b/src/main.c
index f9b34f80e8..a7b631270e 100644
--- a/src/main.c
+++ b/src/main.c
@@ -2822,6 +2822,7 @@ static int openDatabase(
   }else{
     isThreadsafe = sqlite3GlobalConfig.bFullMutex;
   }
+
   if( flags & SQLITE_OPEN_PRIVATECACHE ){
     flags &= ~SQLITE_OPEN_SHAREDCACHE;
   }else if( sqlite3GlobalConfig.sharedCacheEnabled ){
@@ -2854,13 +2855,20 @@ static int openDatabase(
   /* Allocate the sqlite data structure */
   db = sqlite3MallocZero( sizeof(sqlite3) );
   if( db==0 ) goto opendb_out;
-  if( isThreadsafe ){
+  if( isThreadsafe 
+#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS
+   || sqlite3GlobalConfig.bCoreMutex
+#endif
+  ){
     db->mutex = sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE);
     if( db->mutex==0 ){
       sqlite3_free(db);
       db = 0;
       goto opendb_out;
     }
+    if( isThreadsafe==0 ){
+      sqlite3MutexWarnOnContention(db->mutex);
+    }
   }
   sqlite3_mutex_enter(db->mutex);
   db->errMask = 0xff;
diff --git a/src/mutex.c b/src/mutex.c
index 6f1bc9767d..dd24e9be8c 100644
--- a/src/mutex.c
+++ b/src/mutex.c
@@ -26,6 +26,182 @@ static SQLITE_WSD int mutexIsInit = 0;
 
 
 #ifndef SQLITE_MUTEX_OMIT
+
+#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS
+/*
+** This block (enclosed by SQLITE_ENABLE_MULTITHREADED_CHECKS) contains
+** the implementation of a wrapper around the system default mutex
+** implementation (sqlite3DefaultMutex()). 
+**
+** Most calls are passed directly through to the underlying default
+** mutex implementation. Except, if a mutex is configured by calling
+** sqlite3MutexWarnOnContention() on it, then if contention is ever
+** encountered within xMutexEnter() a warning is emitted via sqlite3_log().
+**
+** This type of mutex is used as the database handle mutex when testing
+** apps that usually use SQLITE_CONFIG_MULTITHREAD mode.
+*/
+
+/* 
+** Type for all mutexes used when SQLITE_ENABLE_MULTITHREADED_CHECKS
+** is defined.
+*/
+typedef struct CheckMutex CheckMutex;
+struct CheckMutex {
+  int iType;
+  sqlite3_mutex *mutex;
+};
+
+/* 
+** Pointer to real mutex methods object used by the CheckMutex
+** implementation. Set by checkMutexInit(). 
+*/
+static SQLITE_WSD const sqlite3_mutex_methods *pGlobalMutexMethods;
+
+#ifdef SQLITE_DEBUG
+static int checkMutexHeld(sqlite3_mutex *p){
+  return pGlobalMutexMethods->xMutexHeld(((CheckMutex*)p)->mutex);
+}
+static int checkMutexNotheld(sqlite3_mutex *p){
+  return pGlobalMutexMethods->xMutexNotheld(((CheckMutex*)p)->mutex);
+}
+#endif
+
+/*
+** Initialize and deinitialize the mutex subsystem.
+*/
+static int checkMutexInit(void){ 
+  pGlobalMutexMethods = sqlite3DefaultMutex();
+  return SQLITE_OK; 
+}
+static int checkMutexEnd(void){ 
+  pGlobalMutexMethods = 0;
+  return SQLITE_OK; 
+}
+
+/*
+** Allocate a mutex.
+*/
+static sqlite3_mutex *checkMutexAlloc(int iType){
+  static CheckMutex staticMutexes[] = {
+    {2, 0}, {3, 0}, {4, 0}, {5, 0},
+    {6, 0}, {7, 0}, {8, 0}, {9, 0},
+    {10, 0}, {11, 0}, {12, 0}, {13, 0}
+  };
+  CheckMutex *p = 0;
+
+  assert( SQLITE_MUTEX_RECURSIVE==1 && SQLITE_MUTEX_FAST==0 );
+  if( iType<2 ){
+    p = sqlite3MallocZero(sizeof(CheckMutex));
+    if( p==0 ) return 0;
+    p->iType = iType;
+  }else{
+#ifdef SQLITE_ENABLE_API_ARMOR
+    if( iType-2>=ArraySize(staticMutexes) ){
+      (void)SQLITE_MISUSE_BKPT;
+      return 0;
+    }
+#endif
+    p = &staticMutexes[iType-2];
+  }
+
+  if( p->mutex==0 ){
+    p->mutex = pGlobalMutexMethods->xMutexAlloc(iType);
+    if( p->mutex==0 ){
+      if( iType<2 ){
+        sqlite3_free(p);
+      }
+      p = 0;
+    }
+  }
+
+  return (sqlite3_mutex*)p;
+}
+
+/*
+** Free a mutex.
+*/
+static void checkMutexFree(sqlite3_mutex *p){
+#if SQLITE_ENABLE_API_ARMOR
+  if( p->iType<2 ){
+#endif
+  {
+    CheckMutex *pCheck = (CheckMutex*)p;
+    pGlobalMutexMethods->xMutexFree(pCheck->mutex);
+    sqlite3_free(pCheck);
+  }
+#ifdef SQLITE_ENABLE_API_ARMOR
+  else{
+    (void)SQLITE_MISUSE_BKPT;
+  }
+#endif
+}
+
+/*
+** Enter the mutex.
+*/
+static void checkMutexEnter(sqlite3_mutex *p){
+  CheckMutex *pCheck = (CheckMutex*)p;
+  if( pCheck->iType<0 ){
+    if( SQLITE_OK==pGlobalMutexMethods->xMutexTry(pCheck->mutex) ){
+      return;
+    }
+    sqlite3_log(SQLITE_MISUSE, 
+        "illegal multi-threaded access to database connection"
+    );
+  }
+  pGlobalMutexMethods->xMutexEnter(pCheck->mutex);
+}
+
+/*
+** Enter the mutex (do not block).
+*/
+static int checkMutexTry(sqlite3_mutex *p){
+  CheckMutex *pCheck = (CheckMutex*)p;
+  return pGlobalMutexMethods->xMutexTry(pCheck->mutex);
+}
+
+/*
+** Leave the mutex.
+*/
+static void checkMutexLeave(sqlite3_mutex *p){
+  CheckMutex *pCheck = (CheckMutex*)p;
+  pGlobalMutexMethods->xMutexLeave(pCheck->mutex);
+}
+
+sqlite3_mutex_methods const *multiThreadedCheckMutex(void){
+  static const sqlite3_mutex_methods sMutex = {
+    checkMutexInit,
+    checkMutexEnd,
+    checkMutexAlloc,
+    checkMutexFree,
+    checkMutexEnter,
+    checkMutexTry,
+    checkMutexLeave,
+#ifdef SQLITE_DEBUG
+    checkMutexHeld,
+    checkMutexNotheld
+#else
+    0,
+    0
+#endif
+  };
+  return &sMutex;
+}
+
+/*
+** Mark the SQLITE_MUTEX_RECURSIVE mutex passed as the only argument as
+** one on which there should be no contention.
+*/
+void sqlite3MutexWarnOnContention(sqlite3_mutex *p){
+  if( sqlite3GlobalConfig.mutex.xMutexAlloc==checkMutexAlloc ){
+    CheckMutex *pCheck = (CheckMutex*)p;
+    assert( pCheck->iType==SQLITE_MUTEX_RECURSIVE );
+    pCheck->iType = -1;
+  }
+}
+#endif   /* ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS */
+
 /*
 ** Initialize the mutex system.
 */
@@ -41,7 +217,11 @@ int sqlite3MutexInit(void){
     sqlite3_mutex_methods *pTo = &sqlite3GlobalConfig.mutex;
 
     if( sqlite3GlobalConfig.bCoreMutex ){
+#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS
+      pFrom = multiThreadedCheckMutex();
+#else
       pFrom = sqlite3DefaultMutex();
+#endif
     }else{
       pFrom = sqlite3NoopMutex();
     }
@@ -167,3 +347,4 @@ int sqlite3_mutex_notheld(sqlite3_mutex *p){
 #endif
 
 #endif /* !defined(SQLITE_MUTEX_OMIT) */
+
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index c4e3a7dabc..c416754eed 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -3590,6 +3590,12 @@ int sqlite3LookasideUsed(sqlite3*,int*);
 sqlite3_mutex *sqlite3Pcache1Mutex(void);
 sqlite3_mutex *sqlite3MallocMutex(void);
 
+#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS
+void sqlite3MutexWarnOnContention(sqlite3_mutex*);
+#else
+# define sqlite3MutexWarnOnContention(x)
+#endif
+
 #ifndef SQLITE_OMIT_FLOATING_POINT
   int sqlite3IsNaN(double);
 #else

From 61f8e86f93ae0410bb93b27dfad4368539e5d67c Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 25 Nov 2017 21:09:29 +0000
Subject: [PATCH 109/488] Fix builds with both
 SQLITE_ENABLE_MULTITHREADED_CHECKS and SQLITE_THREADSAFE=0 defined.

FossilOrigin-Name: 7d0b12fcb58353b883ffce77df824a5cc8b1e913a21ec3f22fb73481a398c916
---
 manifest        | 15 ++++++---------
 manifest.uuid   |  2 +-
 src/sqliteInt.h |  2 +-
 3 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index a55dbc7aa2..48f178e1ea 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sexperimental\sfeature\sto\sdetect\sthreading\sbugs\sin\sapps\sthat\suse\nSQLITE_CONFIG_MULTITHREADED.\sEnabled\sat\scompile\stime\susing\nSQLITE_ENABLE_MULTITHREADED_CHECKS.
-D 2017-11-25T17:51:01.072
+C Fix\sbuilds\swith\sboth\sSQLITE_ENABLE_MULTITHREADED_CHECKS\sand\nSQLITE_THREADSAFE=0\sdefined.
+D 2017-11-25T21:09:29.272
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
@@ -476,7 +476,7 @@ F src/shell.c.in cb1b5e41ef9c081b2b8927ae32c9c384a9ec110ada808ebfe083ba7c8a19bbb
 F src/sqlite.h.in 8fd97993d48b50b9bade38c52f12d175942c9497c960905610c7b03a3e4b5818
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
-F src/sqliteInt.h 6b084f4b7c0ea0cde24f4400a88fca7108f1a304cb567338495bba46b0fd68c6
+F src/sqliteInt.h 4c910d9c0d88a90e8639a4f83ef05f701ccfe731cf593e757444074f01df4964
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -1678,10 +1678,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 24d7058e2799133dd681d2fef341025ca50554861bb4cd39e93ee87ae1d8a605
-R 1e88d93b9907cb6e6661756c6df7c284
-T *branch * multithreaded-checks
-T *sym-multithreaded-checks *
-T -sym-trunk *
+P a66886ac13aa6d8ccbb6d673ddd00267c93e3ee1fbc158236fce3157d150868d
+R e1844ccd94c28116d72c961404a2fb64
 U dan
-Z 7380701e6ba5cd3f46b8984142176402
+Z 03bc6023c96e908c306e6b560dfce5a2
diff --git a/manifest.uuid b/manifest.uuid
index 99d6d7ea4c..ec64aba6fd 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a66886ac13aa6d8ccbb6d673ddd00267c93e3ee1fbc158236fce3157d150868d
\ No newline at end of file
+7d0b12fcb58353b883ffce77df824a5cc8b1e913a21ec3f22fb73481a398c916
\ No newline at end of file
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index c416754eed..27b896d67c 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -3590,7 +3590,7 @@ int sqlite3LookasideUsed(sqlite3*,int*);
 sqlite3_mutex *sqlite3Pcache1Mutex(void);
 sqlite3_mutex *sqlite3MallocMutex(void);
 
-#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS
+#if defined(SQLITE_ENABLE_MULTITHREADED_CHECKS) && !defined(SQLITE_MUTEX_OMIT)
 void sqlite3MutexWarnOnContention(sqlite3_mutex*);
 #else
 # define sqlite3MutexWarnOnContention(x)

From 9b7affc451d41afd060dd98b7b76605e92787142 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sun, 26 Nov 2017 02:14:18 +0000
Subject: [PATCH 110/488] Fix an CSV output quoting problem in the command-line
 shell on Windows.

FossilOrigin-Name: 6500cdbd36b0169f37fe363c49182b3e30477a51bb521c2a964555cd086a5589
---
 manifest       | 14 +++++++-------
 manifest.uuid  |  2 +-
 src/shell.c.in |  9 +++------
 3 files changed, 11 insertions(+), 14 deletions(-)

diff --git a/manifest b/manifest
index fa07b8e660..d1e9d5f842 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\s"^"\ssyntax\sfrom\sfts3/4\sto\sfts5.
-D 2017-11-24T19:24:44.918
+C Fix\san\sCSV\soutput\squoting\sproblem\sin\sthe\scommand-line\sshell\son\sWindows.
+D 2017-11-26T02:14:18.907
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
@@ -472,7 +472,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
-F src/shell.c.in cb1b5e41ef9c081b2b8927ae32c9c384a9ec110ada808ebfe083ba7c8a19bbb8
+F src/shell.c.in c441d7ddfbb8120cd8a7cde838ca5c9167311a7e400b1077c3ae6090aa420be8
 F src/sqlite.h.in 8fd97993d48b50b9bade38c52f12d175942c9497c960905610c7b03a3e4b5818
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1678,7 +1678,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e3b6e22049caf78bc4153ded8dc295fe30ad320323d921f16bd794ef30f1b115
-R dd0642a6cc4628ee924c556518e62685
-U dan
-Z b13dcd2f6b4a5ff1701c0fa74c45ed53
+P 24d7058e2799133dd681d2fef341025ca50554861bb4cd39e93ee87ae1d8a605
+R 8e5988afe15fc17639c16180584a05a2
+U drh
+Z 78b26e27b3171f1fcd46be47e5730c56
diff --git a/manifest.uuid b/manifest.uuid
index 45ca11ffc8..9e0d368055 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-24d7058e2799133dd681d2fef341025ca50554861bb4cd39e93ee87ae1d8a605
\ No newline at end of file
+6500cdbd36b0169f37fe363c49182b3e30477a51bb521c2a964555cd086a5589
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index f41e7438ed..daa887ef5d 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -1190,12 +1190,9 @@ static void output_csv(ShellState *p, const char *z, int bSep){
       }
     }
     if( i==0 ){
-      putc('"', out);
-      for(i=0; z[i]; i++){
-        if( z[i]=='"' ) putc('"', out);
-        putc(z[i], out);
-      }
-      putc('"', out);
+      char *zQuoted = sqlite3_mprintf("\"%w\"", z);
+      utf8_printf(out, "%s", zQuoted);
+      sqlite3_free(zQuoted);
     }else{
       utf8_printf(out, "%s", z);
     }

From 7ac2d48eba5973fea8998ad80523ea5537b2b53f Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 27 Nov 2017 17:56:14 +0000
Subject: [PATCH 111/488] Fix a faulty NEVER assert() that could fail for
 SQLITE_ENABLE_STAT4 builds that use foreign keys.

FossilOrigin-Name: 465350e55ddaf30cfba7874653301de7238a9dc2bc5e1f800fc95de9360679f6
---
 manifest        | 16 ++++++++--------
 manifest.uuid   |  2 +-
 src/vdbemem.c   |  2 +-
 test/fkey7.test | 14 ++++++++++++++
 4 files changed, 24 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index d1e9d5f842..24f6855872 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\san\sCSV\soutput\squoting\sproblem\sin\sthe\scommand-line\sshell\son\sWindows.
-D 2017-11-26T02:14:18.907
+C Fix\sa\sfaulty\sNEVER\sassert()\sthat\scould\sfail\sfor\sSQLITE_ENABLE_STAT4\sbuilds\nthat\suse\sforeign\skeys.
+D 2017-11-27T17:56:14.310
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
@@ -546,7 +546,7 @@ F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9
 F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1
 F src/vdbeaux.c b02a1f842c0e916285643b8475b7189f10b76f9e7edb5e2353a913c7980f90b5
 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191
-F src/vdbemem.c 2ef9e66b301a1e575e32966c4c0fd4844e8eea37a2f02bae78c4f68f50a6ab30
+F src/vdbemem.c 6e6082377183b8fd8d6f2bce45250b4c2e89bfeb534a094cd8612dae8aaa2fda
 F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a
@@ -787,7 +787,7 @@ F test/fkey3.test 76d475c80b84ee7a5d062e56ccb6ea68882e2b49
 F test/fkey4.test 86446017011273aad8f9a99c1a65019e7bd9ca9d
 F test/fkey5.test 24dd28eb3d9f1b5a174f47e9899ace5facb08373a4223593c8c631e6cf9f7d5a
 F test/fkey6.test d078a1e323a740062bed38df32b8a736fd320dc0
-F test/fkey7.test 72e915890ee4a005daaf3002cb208e8fe973ac13
+F test/fkey7.test 24076d43d3449f12f25503909ca4bfb5fc5fefd5af1f930723a496343eb28454
 F test/fkey8.test e5372e32cdb4481f121ec3550703eeb7b4e0762c
 F test/fkey_malloc.test 594a7ea1fbab553c036c70813cd8bd9407d63749
 F test/fordelete.test eb93a2f34137bb87bdab88fcab06c0bd92719aff
@@ -1678,7 +1678,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 24d7058e2799133dd681d2fef341025ca50554861bb4cd39e93ee87ae1d8a605
-R 8e5988afe15fc17639c16180584a05a2
-U drh
-Z 78b26e27b3171f1fcd46be47e5730c56
+P 6500cdbd36b0169f37fe363c49182b3e30477a51bb521c2a964555cd086a5589
+R 0247a4bb86c66f994a19293f6b9d8594
+U dan
+Z a556263a76630f97a74cefb44456d7c1
diff --git a/manifest.uuid b/manifest.uuid
index 9e0d368055..51e3848821 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6500cdbd36b0169f37fe363c49182b3e30477a51bb521c2a964555cd086a5589
\ No newline at end of file
+465350e55ddaf30cfba7874653301de7238a9dc2bc5e1f800fc95de9360679f6
\ No newline at end of file
diff --git a/src/vdbemem.c b/src/vdbemem.c
index f9f58c43f4..3ec7394a45 100644
--- a/src/vdbemem.c
+++ b/src/vdbemem.c
@@ -1321,7 +1321,7 @@ static int valueFromExpr(
 
   assert( pExpr!=0 );
   while( (op = pExpr->op)==TK_UPLUS || op==TK_SPAN ) pExpr = pExpr->pLeft;
-  if( NEVER(op==TK_REGISTER) ) op = pExpr->op2;
+  if( op==TK_REGISTER ) op = pExpr->op2;
 
   /* Compressed expressions only appear when parsing the DEFAULT clause
   ** on a table column definition, and hence only when pCtx==0.  This
diff --git a/test/fkey7.test b/test/fkey7.test
index 6c646a9a7f..e86fc5c57b 100644
--- a/test/fkey7.test
+++ b/test/fkey7.test
@@ -68,4 +68,18 @@ ifcapable incrblob {
   } {SQLITE_CONSTRAINT}
 }
 
+ifcapable stat4 {
+  do_execsql_test 3.0 {
+    CREATE TABLE p4 (id INTEGER NOT NULL PRIMARY KEY);
+    INSERT INTO p4 VALUES(1), (2), (3);
+
+    CREATE TABLE c4(x INTEGER REFERENCES p4(id) DEFERRABLE INITIALLY DEFERRED);
+    CREATE INDEX c4_x ON c4(x);
+    INSERT INTO c4 VALUES(1), (2), (3);
+
+    ANALYZE;
+    INSERT INTO p4(id) VALUES(4);
+  }
+}
+
 finish_test

From cb45eef4d1de63685812dbbad1fc1d1dc1b76401 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 28 Nov 2017 00:52:14 +0000
Subject: [PATCH 112/488] Update tests to deal with SQLITE_FAST_SECURE_DELETE.

FossilOrigin-Name: e6b89304695be371978e65dddd710c8bd563c66b9c94d23165142b6c235c82e1
---
 manifest            | 16 ++++++++--------
 manifest.uuid       |  2 +-
 src/test_config.c   |  6 ++++++
 test/securedel.test |  8 ++++++--
 4 files changed, 21 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index 24f6855872..9d698eeef1 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sfaulty\sNEVER\sassert()\sthat\scould\sfail\sfor\sSQLITE_ENABLE_STAT4\sbuilds\nthat\suse\sforeign\skeys.
-D 2017-11-27T17:56:14.310
+C Update\stests\sto\sdeal\swith\sSQLITE_FAST_SECURE_DELETE.
+D 2017-11-28T00:52:14.148
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
@@ -496,7 +496,7 @@ F src/test_backup.c bf5da90c9926df0a4b941f2d92825a01bbe090a0
 F src/test_bestindex.c 78809f11026f18a93fcfd798d9479cba37e1201c830260bf1edc674b2fa9b857
 F src/test_blob.c ae4a0620b478548afb67963095a7417cd06a4ec0a56adb453542203bfdcb31ce
 F src/test_btree.c 8b2dc8b8848cf3a4db93f11578f075e82252a274
-F src/test_config.c 3000f00b9b47b149d842059904c3fcab5f3871fb6aee7d7cc5756f0c64779ae3
+F src/test_config.c 2dad654eb81e90160f764f485bf7248e87b6d251e484c959de1aa04935acae8e
 F src/test_delete.c e2fe07646dff6300b48d49b2fee2fe192ed389e834dd635e3b3bac0ce0bf9f8f
 F src/test_demovfs.c a0c3bdd45ed044115c2c9f7779e56eafff18741e
 F src/test_devsym.c 1960abbb234b97e9b920f07e99503fc04b443f62bbc3c6ff2c2cea2133e3b8a2
@@ -1173,7 +1173,7 @@ F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38
 F test/schema4.test 3b26c9fa916abb6dadf894137adcf41b7796f7b9
 F test/schema5.test 29699b4421f183c8f0e88bd28ce7d75d13ea653e
 F test/schema6.test e4bd1f23d368695eb9e7b51ef6e02ca0642ea2ab4a52579959826b5e7dce1f9b
-F test/securedel.test 5f997cb6bd38727b81e0985f53ec386c99db6441b2b9e6357240649d29017239
+F test/securedel.test 2f70b2449186a1921bd01ec9da407fbfa98c3a7a5521854c300c194b2ff09384
 F test/securedel2.test 2d54c28e46eb1fd6902089958b20b1b056c6f1c5
 F test/select1.test 460a5824df01575b18f7fa4bd8e40d09de20c542e90c1543e164bc7d3b0a0bb7
 F test/select2.test 352480e0e9c66eda9c3044e412abdf5be0215b56
@@ -1678,7 +1678,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6500cdbd36b0169f37fe363c49182b3e30477a51bb521c2a964555cd086a5589
-R 0247a4bb86c66f994a19293f6b9d8594
-U dan
-Z a556263a76630f97a74cefb44456d7c1
+P 465350e55ddaf30cfba7874653301de7238a9dc2bc5e1f800fc95de9360679f6
+R 5c3698b136ad7029a7e22749e9f1a061
+U drh
+Z 455c99a49ae132932748abfe00c111c4
diff --git a/manifest.uuid b/manifest.uuid
index 51e3848821..289fbf5515 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-465350e55ddaf30cfba7874653301de7238a9dc2bc5e1f800fc95de9360679f6
\ No newline at end of file
+e6b89304695be371978e65dddd710c8bd563c66b9c94d23165142b6c235c82e1
\ No newline at end of file
diff --git a/src/test_config.c b/src/test_config.c
index 193f95edf9..8756876d19 100644
--- a/src/test_config.c
+++ b/src/test_config.c
@@ -696,6 +696,12 @@ Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY);
   Tcl_SetVar2(interp, "sqlite_options", "unlock_notify", "0", TCL_GLOBAL_ONLY);
 #endif
 
+#ifdef SQLITE_FAST_SECURE_DELETE
+  Tcl_SetVar2(interp, "sqlite_options", "fast_secure_delete", "1", TCL_GLOBAL_ONLY);
+#else
+  Tcl_SetVar2(interp, "sqlite_options", "fast_secure_delete", "0", TCL_GLOBAL_ONLY);
+#endif
+
 #ifdef SQLITE_SECURE_DELETE
   Tcl_SetVar2(interp, "sqlite_options", "secure_delete", "1", TCL_GLOBAL_ONLY);
 #else
diff --git a/test/securedel.test b/test/securedel.test
index a78f466031..8323a30497 100644
--- a/test/securedel.test
+++ b/test/securedel.test
@@ -17,8 +17,12 @@ source $testdir/tester.tcl
 
 unset -nocomplain DEFAULT_SECDEL
 set DEFAULT_SECDEL 0
-ifcapable secure_delete {
-  set DEFAULT_SECDEL 1
+ifcapable fast_secure_delete {
+  set DEFAULT_SECDEL 2
+} else {
+  ifcapable secure_delete {
+    set DEFAULT_SECDEL 1
+  }
 }
 
 

From a90a2d58a450538fee710f6958c2a19fdbfb033a Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 28 Nov 2017 07:47:57 +0000
Subject: [PATCH 113/488] Use a symbolic constant instead of a literal (-1) to
 identify a warn-on-contention mutex.

FossilOrigin-Name: 12a23c0a66fac5c9674120b390f6abaeaba3f7ff04693b281af1eefb93d6f47c
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/mutex.c   | 17 ++++++++++++++---
 3 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index 48f178e1ea..6f5326f179 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sbuilds\swith\sboth\sSQLITE_ENABLE_MULTITHREADED_CHECKS\sand\nSQLITE_THREADSAFE=0\sdefined.
-D 2017-11-25T21:09:29.272
+C Use\sa\ssymbolic\sconstant\sinstead\sof\sa\sliteral\s(-1)\sto\sidentify\sa\nwarn-on-contention\smutex.
+D 2017-11-28T07:47:57.520
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
@@ -445,7 +445,7 @@ F src/mem3.c 8768ac94694f31ffaf8b4d0ea5dc08af7010a35a
 F src/mem5.c 9bf955937b07f8c32541c8a9991f33ce3173d944
 F src/memjournal.c 6f3d36a0a8f72f48f6c3c722f04301ac64f2515435fa42924293e46fc7994661
 F src/msvc.h 4942752b6a253116baaa8de75256c51a459a5e81
-F src/mutex.c 38addb10f90641b5f88521f7099e729ef10e3f0ac50bd6b9196183fb313a0378
+F src/mutex.c 20172f2cc43c4542f7860ab6bafdd16965822c4e56650d62628059a48c7f43c4
 F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85
 F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4
 F src/mutex_unix.c 27bb6cc49485ee46711a6580ab7b3f1402211d23
@@ -1678,7 +1678,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a66886ac13aa6d8ccbb6d673ddd00267c93e3ee1fbc158236fce3157d150868d
-R e1844ccd94c28116d72c961404a2fb64
+P 7d0b12fcb58353b883ffce77df824a5cc8b1e913a21ec3f22fb73481a398c916
+R a940a7b7e2663c64f019728def2f503f
 U dan
-Z 03bc6023c96e908c306e6b560dfce5a2
+Z d7c77380d4461a6101913d9af19a0151
diff --git a/manifest.uuid b/manifest.uuid
index ec64aba6fd..60dae67939 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7d0b12fcb58353b883ffce77df824a5cc8b1e913a21ec3f22fb73481a398c916
\ No newline at end of file
+12a23c0a66fac5c9674120b390f6abaeaba3f7ff04693b281af1eefb93d6f47c
\ No newline at end of file
diff --git a/src/mutex.c b/src/mutex.c
index dd24e9be8c..9e1bb65553 100644
--- a/src/mutex.c
+++ b/src/mutex.c
@@ -44,7 +44,12 @@ static SQLITE_WSD int mutexIsInit = 0;
 
 /* 
 ** Type for all mutexes used when SQLITE_ENABLE_MULTITHREADED_CHECKS
-** is defined.
+** is defined. Variable CheckMutex.mutex is a pointer to the real mutex
+** allocated by the system mutex implementation. Variable iType is usually set
+** to the type of mutex requested - SQLITE_MUTEX_RECURSIVE, SQLITE_MUTEX_FAST
+** or one of the static mutex identifiers. Or, if this is a recursive mutex
+** that has been configured using sqlite3MutexWarnOnContention(), it is
+** set to SQLITE_MUTEX_WARNONCONTENTION.
 */
 typedef struct CheckMutex CheckMutex;
 struct CheckMutex {
@@ -52,6 +57,8 @@ struct CheckMutex {
   sqlite3_mutex *mutex;
 };
 
+#define SQLITE_MUTEX_WARNONCONTENTION  (-1)
+
 /* 
 ** Pointer to real mutex methods object used by the CheckMutex
 ** implementation. Set by checkMutexInit(). 
@@ -122,6 +129,10 @@ static sqlite3_mutex *checkMutexAlloc(int iType){
 ** Free a mutex.
 */
 static void checkMutexFree(sqlite3_mutex *p){
+  assert( SQLITE_MUTEX_RECURSIVE<2 );
+  assert( SQLITE_MUTEX_FAST<2 );
+  assert( SQLITE_MUTEX_WARNONCONTENTION<2 );
+
 #if SQLITE_ENABLE_API_ARMOR
   if( p->iType<2 ){
 #endif
@@ -142,7 +153,7 @@ static void checkMutexFree(sqlite3_mutex *p){
 */
 static void checkMutexEnter(sqlite3_mutex *p){
   CheckMutex *pCheck = (CheckMutex*)p;
-  if( pCheck->iType<0 ){
+  if( pCheck->iType==SQLITE_MUTEX_WARNONCONTENTION ){
     if( SQLITE_OK==pGlobalMutexMethods->xMutexTry(pCheck->mutex) ){
       return;
     }
@@ -197,7 +208,7 @@ void sqlite3MutexWarnOnContention(sqlite3_mutex *p){
   if( sqlite3GlobalConfig.mutex.xMutexAlloc==checkMutexAlloc ){
     CheckMutex *pCheck = (CheckMutex*)p;
     assert( pCheck->iType==SQLITE_MUTEX_RECURSIVE );
-    pCheck->iType = -1;
+    pCheck->iType = SQLITE_MUTEX_WARNONCONTENTION;
   }
 }
 #endif   /* ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS */

From 0895117512bd461ac97f432f20a2dbd91095e1d5 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 28 Nov 2017 20:43:40 +0000
Subject: [PATCH 114/488] Fix an assertion fault found by OSSFuzz.

FossilOrigin-Name: 75d699877fa7d06d30285ecf008fbedfdf68cc7965bb328c96f5a931d1f13f04
---
 manifest           | 15 +++++++--------
 manifest.uuid      |  2 +-
 src/trigger.c      |  2 +-
 test/triggerG.test | 13 +++++++++++++
 4 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index 0d6ebb2ca6..3a1692e2ea 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Lock\sthe\swal\sfile\sfor\sall\ssnapshot\stransactions,\seven\sif\sthey\swould\snot\notherwise\srequire\sthis,\spreventing\scheckpointers\sand\swriters\sfrom\swrapping\sthe\nwal\sfile.\sThis\smeans\sthat\sif\sone\sconnection\shas\san\sopen\ssnapshot\stransaction\nit\sis\sguaranteed\sthat\sa\ssecond\sconnection\scan\sopen\sa\stransaction\son\sthe\ssame\nsnapshot.
-D 2017-11-28T13:39:41.135
+C Fix\san\sassertion\sfault\sfound\sby\sOSSFuzz.
+D 2017-11-28T20:43:40.167
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
@@ -535,7 +535,7 @@ F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
 F src/tokenize.c 1003d6d90c6783206c711f0a9397656fa5b055209f4d092caa43bb3bf5215db5
 F src/treeview.c 08a83195de8fad3f00542e3c8b3c1eb1222c999817c9e301ffb7f332882b96dd
-F src/trigger.c fc6be2a6e103d9e38b161e07d7db0ffb1f2218bd2f27ccdc0a3d1cc89e9cea0f
+F src/trigger.c 775053eecf6b73062e243404b56f5064446254d5cce17d8704d5cdffd72a546a
 F src/update.c 961bd1265d4d1e5cd65c9a54fa5122fb7aefcb003fcf2de0c092fceb7e58972c
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c d01fa6f45bfad3b65fb2490513aa2e0676412c61b4b094340b513cf72c3704a4
@@ -1457,7 +1457,7 @@ F test/triggerC.test 302d8995f5ffe63bbc15053abb3ef7a39cf5a092
 F test/triggerD.test 8e7f3921a92a5797d472732108109e44575fa650
 F test/triggerE.test 15fa63f1097db1f83dd62d121616006978063d1f
 F test/triggerF.test 6a8c22bd058cf467f0c7d112afe87f7a8c579c0c4681b914b8f19020f48528a4
-F test/triggerG.test 175cafdc6399d85231a09e82e051b0e45a2fd1f23dd08ae715bc359716149ab6
+F test/triggerG.test d5caeef6144ede2426dd13211fd72248241ff2ebc68e12a4c0bf30f5faa21499
 F test/tt3_checkpoint.c 9e75cf7c1c364f52e1c47fd0f14c4340a9db0fe1
 F test/tt3_index.c 39eec10a35f57672225be4d182862152896dee4a
 F test/tt3_lookaside1.c 0377e202c3c2a50d688cb65ba203afeda6fafeb9
@@ -1679,8 +1679,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 40b598c8392f030f6ed8c63ce81cb0426bb3984397c19c756215f6a569a40164 d71eeaab9ecdeed772047498b781be1f0be0655af284b94cf676bb408ceea8b1
-R e37ca38ceb47340e2d23476ad044f736
-T +closed d71eeaab9ecdeed772047498b781be1f0be0655af284b94cf676bb408ceea8b1
+P b81a31495bd27c1d96f7df653da3502054240cb5acf66b860da7f0f9b422a524
+R 419b1183c51a17454670ae5c0f948f4f
 U dan
-Z a9d291c094a58941f666434dce6709fb
+Z af229f4b2717b4f314275fdbf907e896
diff --git a/manifest.uuid b/manifest.uuid
index 1c080520c0..3f2f6eee60 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b81a31495bd27c1d96f7df653da3502054240cb5acf66b860da7f0f9b422a524
\ No newline at end of file
+75d699877fa7d06d30285ecf008fbedfdf68cc7965bb328c96f5a931d1f13f04
\ No newline at end of file
diff --git a/src/trigger.c b/src/trigger.c
index a64fb291a0..d8aac2cc22 100644
--- a/src/trigger.c
+++ b/src/trigger.c
@@ -875,7 +875,7 @@ static TriggerPrg *codeRowTrigger(
     VdbeComment((v, "End: %s.%s", pTrigger->zName, onErrorText(orconf)));
 
     transferParseError(pParse, pSubParse);
-    if( db->mallocFailed==0 ){
+    if( db->mallocFailed==0 && pParse->nErr==0 ){
       pProgram->aOp = sqlite3VdbeTakeOpArray(v, &pProgram->nOp, &pTop->nMaxArg);
     }
     pProgram->nMem = pSubParse->nMem;
diff --git a/test/triggerG.test b/test/triggerG.test
index c770b5bd76..f5965e1707 100644
--- a/test/triggerG.test
+++ b/test/triggerG.test
@@ -62,4 +62,17 @@ do_execsql_test 200 {
   SELECT b FROM t2 ORDER BY b;
 } {20202 20203 20302 20303 30202 30203 30302 30303 40202 40203 40302 40303 50202 50203 50302 50303}
 
+# At one point the following was causing an assert() to fail.
+#
+do_execsql_test 300 {
+  CREATE TABLE t4(x);
+  CREATE TRIGGER tr4 AFTER INSERT ON t4 BEGIN
+    SELECT 0x2147483648e0e0099 AS y WHERE y;
+  END;
+}
+
+do_catchsql_test 310 {
+  INSERT INTO t4 VALUES(1);
+} {1 {hex literal too big: 0x2147483648e0e0099}}
+
 finish_test

From 63025906777c6af4f056031f32df3fd656650593 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 30 Nov 2017 07:55:15 +0000
Subject: [PATCH 115/488] Update test file walprotocol.test to account for the
 changes in the wal locking protocol.

FossilOrigin-Name: 64e567009dd56ef595850fe460925bc15fa875163541527638b654aa2b2cf785
---
 manifest              | 12 ++++++------
 manifest.uuid         |  2 +-
 test/walprotocol.test | 32 +++++++++++++++++++++++---------
 3 files changed, 30 insertions(+), 16 deletions(-)

diff --git a/manifest b/manifest
index 3a1692e2ea..6f7fbcd4c6 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\san\sassertion\sfault\sfound\sby\sOSSFuzz.
-D 2017-11-28T20:43:40.167
+C Update\stest\sfile\swalprotocol.test\sto\saccount\sfor\sthe\schanges\sin\sthe\swal\nlocking\sprotocol.
+D 2017-11-30T07:55:15.935
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
@@ -1533,7 +1533,7 @@ F test/walmode.test 4022fe03ae6e830583672caa101f046438a0473c
 F test/walnoshm.test 84ca10c544632a756467336b7c3b864d493ee496
 F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03
 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6
-F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20
+F test/walprotocol.test a112aba0b79e3adeaa485fed09484b32c654e97df58e454aa8489ac2cd57bf84
 F test/walro.test cb438d05ba0d191f10b688e39c4f0cd5b71569a1d1f4440e5bdf3c6880e08c20
 F test/walro2.test 8812e514c968bf4ee317571fafedac43443360ae23edd7d0f4ef1eae0c13e8e8
 F test/walrofault.test c70cb6e308c443867701856cce92ad8288cd99488fa52afab77cca6cfd51af68
@@ -1679,7 +1679,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b81a31495bd27c1d96f7df653da3502054240cb5acf66b860da7f0f9b422a524
-R 419b1183c51a17454670ae5c0f948f4f
+P 75d699877fa7d06d30285ecf008fbedfdf68cc7965bb328c96f5a931d1f13f04
+R fc8bdb8d48da4660663e6bac79ba45ec
 U dan
-Z af229f4b2717b4f314275fdbf907e896
+Z d8a8a814dbff1ab540fb1916cfe46931
diff --git a/manifest.uuid b/manifest.uuid
index 3f2f6eee60..86aba2434a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-75d699877fa7d06d30285ecf008fbedfdf68cc7965bb328c96f5a931d1f13f04
\ No newline at end of file
+64e567009dd56ef595850fe460925bc15fa875163541527638b654aa2b2cf785
\ No newline at end of file
diff --git a/test/walprotocol.test b/test/walprotocol.test
index ee8d0b72a5..b1d9e8c01f 100644
--- a/test/walprotocol.test
+++ b/test/walprotocol.test
@@ -52,21 +52,21 @@ do_test 1.1 {
   set ::locks [list]
   sqlite3 db test.db -vfs T
   execsql { SELECT * FROM x }
-  lrange $::locks 0 3
-} [list {0 1 lock exclusive} {1 7 lock exclusive}      \
-        {1 7 unlock exclusive} {0 1 unlock exclusive}  \
+  lrange $::locks 0 5
+} [list {0 1 lock exclusive} {1 2 lock exclusive} {4 4 lock exclusive} \
+        {1 2 unlock exclusive} {4 4 unlock exclusive} {0 1 unlock exclusive}  \
 ]
 do_test 1.2 {
   db close
   set ::locks [list]
   sqlite3 db test.db -vfs T
   execsql { SELECT * FROM x }
-  lrange $::locks 0 3
-} [list {0 1 lock exclusive} {1 7 lock exclusive}      \
-        {1 7 unlock exclusive} {0 1 unlock exclusive}  \
+  lrange $::locks 0 5
+} [list {0 1 lock exclusive} {1 2 lock exclusive} {4 4 lock exclusive} \
+        {1 2 unlock exclusive} {4 4 unlock exclusive} {0 1 unlock exclusive}  \
 ]
 proc lock_callback {method filename handle lock} {
-  if {$lock == "1 7 lock exclusive"} { return SQLITE_BUSY }
+  if {$lock == "1 2 lock exclusive"} { return SQLITE_BUSY }
   return SQLITE_OK
 }
 puts "# Warning: This next test case causes SQLite to call xSleep(1) 100 times."
@@ -90,6 +90,18 @@ do_test 1.4 {
   sqlite3 db test.db -vfs T
   catchsql { SELECT * FROM x }
 } {1 {locking protocol}}
+
+puts "# Warning: Third time!"
+proc lock_callback {method filename handle lock} {
+  if {$lock == "4 4 lock exclusive"} { return SQLITE_BUSY }
+  return SQLITE_OK
+}
+do_test 1.5 {
+  db close
+  set ::locks [list]
+  sqlite3 db test.db -vfs T
+  catchsql { SELECT * FROM x }
+} {1 {locking protocol}}
 db close
 T delete
 
@@ -135,13 +147,14 @@ T filter xShmLock
 T script lock_callback
 
 proc lock_callback {method file handle spec} {
-  if {$spec == "1 7 unlock exclusive"} {
+  if {$spec == "1 2 unlock exclusive"} {
     T filter {}
     set ::r [catchsql { SELECT * FROM b } db2]
   }
 }
 sqlite3 db test.db
 sqlite3 db2 test.db
+puts "# Warning: Another slow test!"
 do_test 2.5 {
   execsql { SELECT * FROM b }
 } {Tehran Qom Markazi Qazvin Gilan Ardabil}
@@ -157,12 +170,13 @@ sqlite3 db2 test.db
 T filter xShmLock
 T script lock_callback
 proc lock_callback {method file handle spec} {
-  if {$spec == "1 7 unlock exclusive"} {
+  if {$spec == "1 2 unlock exclusive"} {
     T filter {}
     set ::r [catchsql { SELECT * FROM b } db2]
   }
 }
 unset ::r
+puts "# Warning: Last one!"
 do_test 2.7 {
   execsql { SELECT * FROM b }
 } {Tehran Qom Markazi Qazvin Gilan Ardabil}

From 92b67025d450c3f20a9a6f55bd4b63a65f8cbc24 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 30 Nov 2017 11:21:59 +0000
Subject: [PATCH 116/488] Fix some minor problems in test scripts.

FossilOrigin-Name: 4c551fdebc7feda3dcfeec719387d879cd5e2cbe213c0c1aac0a965b3f9e882d
---
 manifest           | 14 +++++++-------
 manifest.uuid      |  2 +-
 test/mjournal.test |  5 +++++
 test/walro2.test   |  5 +++--
 4 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index 6f7fbcd4c6..d7cb47fa84 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\stest\sfile\swalprotocol.test\sto\saccount\sfor\sthe\schanges\sin\sthe\swal\nlocking\sprotocol.
-D 2017-11-30T07:55:15.935
+C Fix\ssome\sminor\sproblems\sin\stest\sscripts.
+D 2017-11-30T11:21:59.935
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
@@ -1063,7 +1063,7 @@ F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91
 F test/misc7.test edd0b63e2ee29a256900b0514f6fff27e19e9bb2
 F test/misc8.test ba03aaa08f02d62fbb8d3b2f5595c1b33aa9bbc5
 F test/misuse.test 9e7f78402005e833af71dcab32d048003869eca5abcaccc985d4f8dc1d86bcc7
-F test/mjournal.test 946d7161cabbc45ecc818d7fa38c1925bbf36e5534a08a81fa0203ee7ced7d54
+F test/mjournal.test 9d86e697dcbc5da2c4e8caba9b176b5765fe65e80c88c278b8c09a917e436795
 F test/mmap1.test d2cfc1635171c434dcff0ece2f1c8e0a658807ce
 F test/mmap2.test 9d6dd9ddb4ad2379f29cc78f38ce1e63ed418022
 F test/mmap3.test b3c297e78e6a8520aafcc1a8f140535594c9086e
@@ -1535,7 +1535,7 @@ F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03
 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6
 F test/walprotocol.test a112aba0b79e3adeaa485fed09484b32c654e97df58e454aa8489ac2cd57bf84
 F test/walro.test cb438d05ba0d191f10b688e39c4f0cd5b71569a1d1f4440e5bdf3c6880e08c20
-F test/walro2.test 8812e514c968bf4ee317571fafedac43443360ae23edd7d0f4ef1eae0c13e8e8
+F test/walro2.test 5cd57d192ee334c3894330303b5f8cb6789fef49b2c83ad1b50b9b132d0f7ae1
 F test/walrofault.test c70cb6e308c443867701856cce92ad8288cd99488fa52afab77cca6cfd51af68
 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417
 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f
@@ -1679,7 +1679,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 75d699877fa7d06d30285ecf008fbedfdf68cc7965bb328c96f5a931d1f13f04
-R fc8bdb8d48da4660663e6bac79ba45ec
+P 64e567009dd56ef595850fe460925bc15fa875163541527638b654aa2b2cf785
+R bcb5dee584d5b6628891c4aa47c39d94
 U dan
-Z d8a8a814dbff1ab540fb1916cfe46931
+Z 7f9d49097c1baf2fffb084b71e5e24bc
diff --git a/manifest.uuid b/manifest.uuid
index 86aba2434a..42170c4583 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-64e567009dd56ef595850fe460925bc15fa875163541527638b654aa2b2cf785
\ No newline at end of file
+4c551fdebc7feda3dcfeec719387d879cd5e2cbe213c0c1aac0a965b3f9e882d
\ No newline at end of file
diff --git a/test/mjournal.test b/test/mjournal.test
index cf6ef5f7b5..539ebc6946 100644
--- a/test/mjournal.test
+++ b/test/mjournal.test
@@ -15,6 +15,11 @@ set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 set testprefix mjournal
 
+if {[permutation]=="inmemory_journal"} {
+  finish_test
+  return
+}
+
 # Test that nothing bad happens if a journal file contains a pointer to
 # a master journal file that does not have a "-" in the name. At one point
 # this was causing a segfault on unix.
diff --git a/test/walro2.test b/test/walro2.test
index 9d0a6b4a19..876a03196b 100644
--- a/test/walro2.test
+++ b/test/walro2.test
@@ -262,8 +262,9 @@ do_multiclient_test tn {
     } 
   } {500}
   do_test $TN.4.2.2 {
-    file size test.db-wal
-  } {461152}
+    set sz [file size test.db-wal]
+    expr {$sz>400000 && $sz<500000}
+  } {1}
   do_test $TN.4.2.4 {
     file_control_persist_wal db 1; db close
 

From f9679318f23286b2c90e38b281711619eb047e98 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 1 Dec 2017 18:40:18 +0000
Subject: [PATCH 117/488] Fix the ".lint fkey-indexes" shell command so that it
 works with WITHOUT ROWID tables.

FossilOrigin-Name: 5771b1d611b3562ea6c040f3f893073c4d0ee58c35b37ae211520d9aee8ed547
---
 manifest         | 14 +++++++-------
 manifest.uuid    |  2 +-
 src/shell.c.in   |  8 ++++----
 test/shell6.test |  8 ++++++++
 4 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/manifest b/manifest
index d7cb47fa84..59176fbb68 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\ssome\sminor\sproblems\sin\stest\sscripts.
-D 2017-11-30T11:21:59.935
+C Fix\sthe\s".lint\sfkey-indexes"\sshell\scommand\sso\sthat\sit\sworks\swith\sWITHOUT\sROWID\ntables.
+D 2017-12-01T18:40:18.535
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
@@ -472,7 +472,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
-F src/shell.c.in c441d7ddfbb8120cd8a7cde838ca5c9167311a7e400b1077c3ae6090aa420be8
+F src/shell.c.in 6ffed0c589f5aff180789a8c8abf5b2d3e2eea7470c86b30e797887cb0c9d0e5
 F src/sqlite.h.in 8fd97993d48b50b9bade38c52f12d175942c9497c960905610c7b03a3e4b5818
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1210,7 +1210,7 @@ F test/shell2.test e242a9912f44f4c23c3d1d802a83e934e84c853b
 F test/shell3.test 9b95ba643eaa228376f06a898fb410ee9b6e57c1
 F test/shell4.test 89ad573879a745974ff2df20ff97c5d6ffffbd5d
 F test/shell5.test 23939a4c51f0421330ea61dbd3c74f9c215f5f8d3d1a94846da6ffc777a35458
-F test/shell6.test ab1592ebe881371f651f18ee6a0df21cbfb5310f88cb832ab642d4038f679772
+F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3
 F test/shell7.test 115132f66d0463417f408562cc2cf534f6bbc6d83a6d50f0072a9eb171bae97f
 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3
 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5
@@ -1679,7 +1679,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 64e567009dd56ef595850fe460925bc15fa875163541527638b654aa2b2cf785
-R bcb5dee584d5b6628891c4aa47c39d94
+P 4c551fdebc7feda3dcfeec719387d879cd5e2cbe213c0c1aac0a965b3f9e882d
+R 16fbfec98373820063ea860cc1009c10
 U dan
-Z 7f9d49097c1baf2fffb084b71e5e24bc
+Z b7aa606a2d2373a4fc095d067f4d9e36
diff --git a/manifest.uuid b/manifest.uuid
index 42170c4583..01389c8854 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4c551fdebc7feda3dcfeec719387d879cd5e2cbe213c0c1aac0a965b3f9e882d
\ No newline at end of file
+5771b1d611b3562ea6c040f3f893073c4d0ee58c35b37ae211520d9aee8ed547
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index daa887ef5d..062f76e475 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -3895,10 +3895,10 @@ static int lintFkeyIndexes(
   **
   ** 0. The text of an SQL statement similar to:
   **
-  **      "EXPLAIN QUERY PLAN SELECT rowid FROM child_table WHERE child_key=?"
+  **      "EXPLAIN QUERY PLAN SELECT 1 FROM child_table WHERE child_key=?"
   **
-  **    This is the same SELECT that the foreign keys implementation needs
-  **    to run internally on child tables. If there is an index that can
+  **    This SELECT is similar to the one that the foreign keys implementation
+  **    needs to run internally on child tables. If there is an index that can
   **    be used to optimize this query, then it can also be used by the FK
   **    implementation to optimize DELETE or UPDATE statements on the parent
   **    table.
@@ -3926,7 +3926,7 @@ static int lintFkeyIndexes(
   */
   const char *zSql =
   "SELECT "
-    "     'EXPLAIN QUERY PLAN SELECT rowid FROM ' || quote(s.name) || ' WHERE '"
+    "     'EXPLAIN QUERY PLAN SELECT 1 FROM ' || quote(s.name) || ' WHERE '"
     "  || group_concat(quote(s.name) || '.' || quote(f.[from]) || '=?' "
     "  || fkey_collate_clause("
     "       f.[table], COALESCE(f.[to], p.[name]), s.name, f.[from]),' AND ')"
diff --git a/test/shell6.test b/test/shell6.test
index 25bfa32338..49b4cc3344 100644
--- a/test/shell6.test
+++ b/test/shell6.test
@@ -93,6 +93,14 @@ foreach {tn schema output} {
   } {
   }
 
+  10 {
+    CREATE TABLE parent (id INTEGER PRIMARY KEY); 
+    CREATE TABLE child2 (id INT PRIMARY KEY, parentID INT REFERENCES parent) 
+      WITHOUT ROWID;
+  } {
+    CREATE INDEX 'child2_parentID' ON 'child2'('parentID'); --> parent(id)
+  }
+
 } {
   forcedelete test.db
   sqlite3 db test.db

From 8adb25ab63825eef00552fd7a0b52979dd81413d Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 5 Dec 2017 14:58:59 +0000
Subject: [PATCH 118/488] Fix a compilation problem when both
 SQLITE_ENABLE_MULTITHREADED_CHECKS and SQLITE_ENABLE_API_ARMOUR are defined.

FossilOrigin-Name: 0342ce510d2063a63839399a2cfa25b7fc02f4fde17764082676b332d6136241
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/mutex.c   |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 59176fbb68..f5ffa7a07d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sthe\s".lint\sfkey-indexes"\sshell\scommand\sso\sthat\sit\sworks\swith\sWITHOUT\sROWID\ntables.
-D 2017-12-01T18:40:18.535
+C Fix\sa\scompilation\sproblem\swhen\sboth\sSQLITE_ENABLE_MULTITHREADED_CHECKS\sand\nSQLITE_ENABLE_API_ARMOUR\sare\sdefined.
+D 2017-12-05T14:58:59.468
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
@@ -445,7 +445,7 @@ F src/mem3.c 8768ac94694f31ffaf8b4d0ea5dc08af7010a35a
 F src/mem5.c 9bf955937b07f8c32541c8a9991f33ce3173d944
 F src/memjournal.c 6f3d36a0a8f72f48f6c3c722f04301ac64f2515435fa42924293e46fc7994661
 F src/msvc.h 4942752b6a253116baaa8de75256c51a459a5e81
-F src/mutex.c 20172f2cc43c4542f7860ab6bafdd16965822c4e56650d62628059a48c7f43c4
+F src/mutex.c b021263554c8a3995e9d53193b8194b96d1ed28e06c3b532dd7f7d29cf0c7d53
 F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85
 F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4
 F src/mutex_unix.c 27bb6cc49485ee46711a6580ab7b3f1402211d23
@@ -1679,7 +1679,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 4c551fdebc7feda3dcfeec719387d879cd5e2cbe213c0c1aac0a965b3f9e882d
-R 16fbfec98373820063ea860cc1009c10
+P 5771b1d611b3562ea6c040f3f893073c4d0ee58c35b37ae211520d9aee8ed547
+R 926398ae5d668c18adb3c9f09b2dd707
 U dan
-Z b7aa606a2d2373a4fc095d067f4d9e36
+Z 60a10472a4a400f76cfacb5adca001dc
diff --git a/manifest.uuid b/manifest.uuid
index 01389c8854..3ccf62c280 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5771b1d611b3562ea6c040f3f893073c4d0ee58c35b37ae211520d9aee8ed547
\ No newline at end of file
+0342ce510d2063a63839399a2cfa25b7fc02f4fde17764082676b332d6136241
\ No newline at end of file
diff --git a/src/mutex.c b/src/mutex.c
index 9e1bb65553..3699f91e38 100644
--- a/src/mutex.c
+++ b/src/mutex.c
@@ -134,7 +134,7 @@ static void checkMutexFree(sqlite3_mutex *p){
   assert( SQLITE_MUTEX_WARNONCONTENTION<2 );
 
 #if SQLITE_ENABLE_API_ARMOR
-  if( p->iType<2 ){
+  if( ((CheckMutex*)p)->iType<2 )
 #endif
   {
     CheckMutex *pCheck = (CheckMutex*)p;

From 72afc3c5de1a4c125b919fe086f83dfe27a43e03 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 5 Dec 2017 18:32:40 +0000
Subject: [PATCH 119/488] If SQLITE_HAVE_ZLIB is defined at build-time, include
 the functions in ext/misc/compress.c in the shell.

FossilOrigin-Name: 0296286a9963bbaab60ee30d8700703f5ccb382380c9bfc0eb12c4bcb6f2accd
---
 manifest       | 15 +++++++++------
 manifest.uuid  |  2 +-
 src/shell.c.in |  6 ++++++
 3 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index f5ffa7a07d..d6e9390bd0 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\scompilation\sproblem\swhen\sboth\sSQLITE_ENABLE_MULTITHREADED_CHECKS\sand\nSQLITE_ENABLE_API_ARMOUR\sare\sdefined.
-D 2017-12-05T14:58:59.468
+C If\sSQLITE_HAVE_ZLIB\sis\sdefined\sat\sbuild-time,\sinclude\sthe\sfunctions\sin\next/misc/compress.c\sin\sthe\sshell.
+D 2017-12-05T18:32:40.201
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
@@ -472,7 +472,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
-F src/shell.c.in 6ffed0c589f5aff180789a8c8abf5b2d3e2eea7470c86b30e797887cb0c9d0e5
+F src/shell.c.in ab727c09b4c87c0c1db32d2fe0a910c0a8e468a0209233328753f5526d6c6c73
 F src/sqlite.h.in 8fd97993d48b50b9bade38c52f12d175942c9497c960905610c7b03a3e4b5818
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1679,7 +1679,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5771b1d611b3562ea6c040f3f893073c4d0ee58c35b37ae211520d9aee8ed547
-R 926398ae5d668c18adb3c9f09b2dd707
+P 0342ce510d2063a63839399a2cfa25b7fc02f4fde17764082676b332d6136241
+R eddb6c6e05812e7a9afe7b2909e80007
+T *branch * sqlar-shell-support
+T *sym-sqlar-shell-support *
+T -sym-trunk *
 U dan
-Z 60a10472a4a400f76cfacb5adca001dc
+Z e3b1a40cd391e9a775fce7f501c26e90
diff --git a/manifest.uuid b/manifest.uuid
index 3ccf62c280..6f484dbf3c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0342ce510d2063a63839399a2cfa25b7fc02f4fde17764082676b332d6136241
\ No newline at end of file
+0296286a9963bbaab60ee30d8700703f5ccb382380c9bfc0eb12c4bcb6f2accd
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 062f76e475..edd75b078c 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -796,6 +796,9 @@ static void shellAddSchemaName(
 INCLUDE ../ext/misc/shathree.c
 INCLUDE ../ext/misc/fileio.c
 INCLUDE ../ext/misc/completion.c
+#ifdef SQLITE_HAVE_ZLIB
+INCLUDE ../ext/misc/compress.c
+#endif
 
 #if defined(SQLITE_ENABLE_SESSION)
 /*
@@ -2897,6 +2900,9 @@ static void open_db(ShellState *p, int keepAlive){
     sqlite3_fileio_init(p->db, 0, 0);
     sqlite3_shathree_init(p->db, 0, 0);
     sqlite3_completion_init(p->db, 0, 0);
+#ifdef SQLITE_HAVE_ZLIB
+    sqlite3_compress_init(p->db, 0, 0);
+#endif
     sqlite3_create_function(p->db, "shell_add_schema", 2, SQLITE_UTF8, 0,
                             shellAddSchemaName, 0, 0);
   }

From cb6acda902b5a1ad02fa888d7b95884ba1bc94d1 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Tue, 5 Dec 2017 18:54:12 +0000
Subject: [PATCH 120/488] Add support for including zlib in the shell when
 compiling with MSVC.

FossilOrigin-Name: e1838cee3847301ef491467dc75d9c4e1e3b12599596c058bdb14319a52fd8a0
---
 .fossil-settings/empty-dirs  |  1 +
 .fossil-settings/ignore-glob |  1 +
 Makefile.msc                 | 83 +++++++++++++++++++++++++++++++++---
 autoconf/Makefile.msc        |  1 +
 manifest                     | 21 +++++----
 manifest.uuid                |  2 +-
 6 files changed, 92 insertions(+), 17 deletions(-)
 create mode 100644 .fossil-settings/empty-dirs
 create mode 100644 .fossil-settings/ignore-glob

diff --git a/.fossil-settings/empty-dirs b/.fossil-settings/empty-dirs
new file mode 100644
index 0000000000..64fb6839df
--- /dev/null
+++ b/.fossil-settings/empty-dirs
@@ -0,0 +1 @@
+compat
diff --git a/.fossil-settings/ignore-glob b/.fossil-settings/ignore-glob
new file mode 100644
index 0000000000..5282ca9c3d
--- /dev/null
+++ b/.fossil-settings/ignore-glob
@@ -0,0 +1 @@
+compat/*
diff --git a/Makefile.msc b/Makefile.msc
index b16f08adee..e924db395a 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -92,6 +92,20 @@ SPLIT_AMALGAMATION = 0
 !ENDIF
 
 # <<mark>>
+# Set this non-0 to use zlib, possibly compiling it from source code.
+#
+!IFNDEF USE_ZLIB
+USE_ZLIB = 0
+!ENDIF
+
+# Set this non-0 to build zlib from source code.  This is enabled by
+# default and in that case it will be assumed that the ZLIBDIR macro
+# points to the top-level source code directory for zlib.
+#
+!IFNDEF BUILD_ZLIB
+BUILD_ZLIB = 1
+!ENDIF
+
 # Set this non-0 to use the International Components for Unicode (ICU).
 #
 !IFNDEF USE_ICU
@@ -612,6 +626,14 @@ SHELL_CORE_DEP =
 !ENDIF
 !ENDIF
 
+# <<mark>>
+# If zlib support is enabled, add the shell dependency for it.
+#
+!IF $(USE_ZLIB)!=0 && $(BUILD_ZLIB)!=0
+SHELL_CORE_DEP = zlib $(SHELL_CORE_DEP)
+!ENDIF
+# <</mark>>
+
 # This is the core library that the shell executable should link with.
 #
 !IFNDEF SHELL_CORE_LIB
@@ -802,12 +824,16 @@ RCC = $(RCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1
 # prior to running nmake in order to match the actual installed location and
 # version on this machine.
 #
+!IFNDEF TCLDIR
+TCLDIR = c:\tcl
+!ENDIF
+
 !IFNDEF TCLINCDIR
-TCLINCDIR = c:\tcl\include
+TCLINCDIR = $(TCLDIR)\include
 !ENDIF
 
 !IFNDEF TCLLIBDIR
-TCLLIBDIR = c:\tcl\lib
+TCLLIBDIR = $(TCLDIR)\lib
 !ENDIF
 
 !IFNDEF LIBTCL
@@ -819,7 +845,32 @@ LIBTCLSTUB = tclstub86.lib
 !ENDIF
 
 !IFNDEF LIBTCLPATH
-LIBTCLPATH = c:\tcl\bin
+LIBTCLPATH = $(TCLDIR)\bin
+!ENDIF
+
+# The locations of the zlib header and library files.  These variables
+# (ZLIBINCDIR, ZLIBLIBDIR, and ZLIBLIB) may be overridden via the environment
+# prior to running nmake in order to match the actual installed (or source
+# code) location on this machine.
+#
+!IFNDEF ZLIBDIR
+ZLIBDIR = $(TOP)\compat\zlib
+!ENDIF
+
+!IFNDEF ZLIBINCDIR
+ZLIBINCDIR = $(ZLIBDIR)
+!ENDIF
+
+!IFNDEF ZLIBLIBDIR
+ZLIBLIBDIR = $(ZLIBDIR)
+!ENDIF
+
+!IFNDEF ZLIBLIB
+!IF $(DYNAMIC_SHELL)!=0
+ZLIBLIB = zdll.lib
+!ELSE
+ZLIBLIB = zlib.lib
+!ENDIF
 !ENDIF
 
 # The locations of the ICU header and library files.  These variables
@@ -827,12 +878,16 @@ LIBTCLPATH = c:\tcl\bin
 # prior to running nmake in order to match the actual installed location on
 # this machine.
 #
+!IFNDEF ICUDIR
+ICUDIR = C:\icu
+!ENDIF
+
 !IFNDEF ICUINCDIR
-ICUINCDIR = c:\icu\include
+ICUINCDIR = $(ICUDIR)\include
 !ENDIF
 
 !IFNDEF ICULIBDIR
-ICULIBDIR = c:\icu\lib
+ICULIBDIR = $(ICUDIR)\lib
 !ENDIF
 
 !IFNDEF LIBICU
@@ -951,6 +1006,15 @@ BCC = $(BCC) -Zi
 !ENDIF
 
 # <<mark>>
+# If zlib support is enabled, add the compiler options for it.
+#
+!IF $(USE_ZLIB)!=0
+TCC = $(TCC) -DSQLITE_HAVE_ZLIB=1
+RCC = $(RCC) -DSQLITE_HAVE_ZLIB=1
+TCC = $(TCC) -I$(ZLIBINCDIR)
+RCC = $(RCC) -I$(ZLIBINCDIR)
+!ENDIF
+
 # If ICU support is enabled, add the compiler options for it.
 #
 !IF $(USE_ICU)!=0
@@ -1075,6 +1139,13 @@ LTLIBPATHS = /LIBPATH:$(TCLLIBDIR)
 LTLIBS = $(LTLIBS) $(LIBTCL)
 !ENDIF
 
+# If zlib support is enabled, add the linker options for it.
+#
+!IF $(USE_ZLIB)!=0
+LTLIBPATHS = $(LTLIBPATHS) /LIBPATH:$(ZLIBLIBDIR)
+LTLIBS = $(LTLIBS) $(ZLIBLIB)
+!ENDIF
+
 # If ICU support is enabled, add the linker options for it.
 #
 !IF $(USE_ICU)!=0
@@ -1987,6 +2058,8 @@ SHELL_SRC = \
 shell.c:	$(SHELL_SRC) $(TOP)\tool\mkshellc.tcl
 	$(TCLSH_CMD) $(TOP)\tool\mkshellc.tcl > shell.c
 
+zlib:
+	pushd $(ZLIBDIR) && $(MAKE) /f win32\Makefile.msc $(ZLIBLIB) && popd
 
 # Rules to build the extension objects.
 #
diff --git a/autoconf/Makefile.msc b/autoconf/Makefile.msc
index 199af21ea9..5b5133c8b9 100644
--- a/autoconf/Makefile.msc
+++ b/autoconf/Makefile.msc
@@ -561,6 +561,7 @@ SHELL_CORE_DEP =
 !ENDIF
 !ENDIF
 
+
 # This is the core library that the shell executable should link with.
 #
 !IFNDEF SHELL_CORE_LIB
diff --git a/manifest b/manifest
index d6e9390bd0..f406bec75d 100644
--- a/manifest
+++ b/manifest
@@ -1,8 +1,10 @@
-C If\sSQLITE_HAVE_ZLIB\sis\sdefined\sat\sbuild-time,\sinclude\sthe\sfunctions\sin\next/misc/compress.c\sin\sthe\sshell.
-D 2017-12-05T18:32:40.201
+C Add\ssupport\sfor\sincluding\szlib\sin\sthe\sshell\swhen\scompiling\swith\sMSVC.
+D 2017-12-05T18:54:12.861
+F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
+F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
+F Makefile.msc 2cc781176ca1a58bfb7c2741a192726e2f08db0b473daba3d6c3388d44698470
 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd
 F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -11,7 +13,7 @@ F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90
 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2
 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903
 F autoconf/Makefile.am 66c0befa511f0d95ba229e180067cf0357a9ebf8b3201b06d683c5ba6220fb39
-F autoconf/Makefile.msc b88a70dee8453cc353e5d6df172d60a11a0af905710a24b1e6be80f8fea6e96b
+F autoconf/Makefile.msc 2b4b5e5ff7e7a9806ebdd4b441f8fb54695a3628701a97ae53860e67e872acc3
 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7
 F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1
 F autoconf/configure.ac 8dd08ca564279fff091c9bfdd2599d8f992c9f1f70c5396de2126ad2bd1b3bed
@@ -1679,10 +1681,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0342ce510d2063a63839399a2cfa25b7fc02f4fde17764082676b332d6136241
-R eddb6c6e05812e7a9afe7b2909e80007
-T *branch * sqlar-shell-support
-T *sym-sqlar-shell-support *
-T -sym-trunk *
-U dan
-Z e3b1a40cd391e9a775fce7f501c26e90
+P 0296286a9963bbaab60ee30d8700703f5ccb382380c9bfc0eb12c4bcb6f2accd
+R 8fbbfa59bd11832619e2c9584cf54899
+U mistachkin
+Z 4e0794167698ff60a9b30c96ee6b24e0
diff --git a/manifest.uuid b/manifest.uuid
index 6f484dbf3c..c2f98dfc4d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0296286a9963bbaab60ee30d8700703f5ccb382380c9bfc0eb12c4bcb6f2accd
\ No newline at end of file
+e1838cee3847301ef491467dc75d9c4e1e3b12599596c058bdb14319a52fd8a0
\ No newline at end of file

From a0fcafe7623efb2bbef52e15b8dbc6ae82e71c0c Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Tue, 5 Dec 2017 19:07:30 +0000
Subject: [PATCH 121/488] For MSVC, simplify default locations for Tcl and ICU
 by using directories inside 'compat'.

FossilOrigin-Name: 8155b5ac850327ea76aba2adf624132f3e05024c973afd218b12f186fc7630e8
---
 Makefile.msc  | 17 +++++++++++++++--
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 3 files changed, 22 insertions(+), 9 deletions(-)

diff --git a/Makefile.msc b/Makefile.msc
index e924db395a..c40d195404 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -92,6 +92,15 @@ SPLIT_AMALGAMATION = 0
 !ENDIF
 
 # <<mark>>
+# Set this non-0 to have this makefile assume the Tcl shell executable
+# (tclsh*.exe) is available in the PATH.  By default, this is enabled
+# for compatibility with older build environments.  This setting only
+# applies if TCLSH_CMD is not set manually.
+#
+!IFNDEF USE_TCLSH_IN_PATH
+USE_TCLSH_IN_PATH = 1
+!ENDIF
+
 # Set this non-0 to use zlib, possibly compiling it from source code.
 #
 !IFNDEF USE_ZLIB
@@ -825,7 +834,7 @@ RCC = $(RCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1
 # version on this machine.
 #
 !IFNDEF TCLDIR
-TCLDIR = c:\tcl
+TCLDIR = $(TOP)\compat\tcl
 !ENDIF
 
 !IFNDEF TCLINCDIR
@@ -879,7 +888,7 @@ ZLIBLIB = zlib.lib
 # this machine.
 #
 !IFNDEF ICUDIR
-ICUDIR = C:\icu
+ICUDIR = $(TOP)\compat\icu
 !ENDIF
 
 !IFNDEF ICUINCDIR
@@ -900,7 +909,11 @@ LIBICU = icuuc.lib icuin.lib
 # specific Tcl shell to use.
 #
 !IFNDEF TCLSH_CMD
+!IF $(USE_TCLSH_IN_PATH)!=0
 TCLSH_CMD = tclsh
+!ELSE
+TCLSH_CMD = $(TCLDIR)\bin\tclsh.exe
+!ENDIF
 !ENDIF
 # <</mark>>
 
diff --git a/manifest b/manifest
index f406bec75d..3e24dc86b3 100644
--- a/manifest
+++ b/manifest
@@ -1,10 +1,10 @@
-C Add\ssupport\sfor\sincluding\szlib\sin\sthe\sshell\swhen\scompiling\swith\sMSVC.
-D 2017-12-05T18:54:12.861
+C For\sMSVC,\ssimplify\sdefault\slocations\sfor\sTcl\sand\sICU\sby\susing\sdirectories\sinside\s'compat'.
+D 2017-12-05T19:07:30.651
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 2cc781176ca1a58bfb7c2741a192726e2f08db0b473daba3d6c3388d44698470
+F Makefile.msc 2eb985916ed36fb4ba73033d030090fbf22d4d972e06caee19e7d89fb305a8ab
 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd
 F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -1681,7 +1681,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0296286a9963bbaab60ee30d8700703f5ccb382380c9bfc0eb12c4bcb6f2accd
-R 8fbbfa59bd11832619e2c9584cf54899
+P e1838cee3847301ef491467dc75d9c4e1e3b12599596c058bdb14319a52fd8a0
+R a792968384e617f4a37df83169716b24
 U mistachkin
-Z 4e0794167698ff60a9b30c96ee6b24e0
+Z 06008f15f266edb3571035a6d698db2a
diff --git a/manifest.uuid b/manifest.uuid
index c2f98dfc4d..b50deeb9bc 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e1838cee3847301ef491467dc75d9c4e1e3b12599596c058bdb14319a52fd8a0
\ No newline at end of file
+8155b5ac850327ea76aba2adf624132f3e05024c973afd218b12f186fc7630e8
\ No newline at end of file

From 01f6b2dc2dd66708c362043252aa33fb49adbf36 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 6 Dec 2017 20:50:08 +0000
Subject: [PATCH 122/488] Fix an unreachable branch in cases when
 SQLITE_ENABLE_STAT4 is not defined.

FossilOrigin-Name: 6a55bd67720451cdf316155cf348103bfce1056a78bcddf0029b45ff0fdbcc71
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/vdbemem.c |  4 ++++
 3 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index f5ffa7a07d..976bea8394 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\scompilation\sproblem\swhen\sboth\sSQLITE_ENABLE_MULTITHREADED_CHECKS\sand\nSQLITE_ENABLE_API_ARMOUR\sare\sdefined.
-D 2017-12-05T14:58:59.468
+C Fix\san\sunreachable\sbranch\sin\scases\swhen\sSQLITE_ENABLE_STAT4\sis\snot\sdefined.
+D 2017-12-06T20:50:08.379
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
@@ -546,7 +546,7 @@ F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9
 F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1
 F src/vdbeaux.c b02a1f842c0e916285643b8475b7189f10b76f9e7edb5e2353a913c7980f90b5
 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191
-F src/vdbemem.c 6e6082377183b8fd8d6f2bce45250b4c2e89bfeb534a094cd8612dae8aaa2fda
+F src/vdbemem.c 3e8c795e0874b2848658b7ca96930a7bcda3f80dc93be2a3422cd30967961e4c
 F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a
@@ -1679,7 +1679,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5771b1d611b3562ea6c040f3f893073c4d0ee58c35b37ae211520d9aee8ed547
-R 926398ae5d668c18adb3c9f09b2dd707
-U dan
-Z 60a10472a4a400f76cfacb5adca001dc
+P 0342ce510d2063a63839399a2cfa25b7fc02f4fde17764082676b332d6136241
+R dd71a721334f8be84f1b50254aaa4af8
+U drh
+Z e071d47cd491bbadd8e0c9ee301f9ca9
diff --git a/manifest.uuid b/manifest.uuid
index 3ccf62c280..fc1d483ded 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0342ce510d2063a63839399a2cfa25b7fc02f4fde17764082676b332d6136241
\ No newline at end of file
+6a55bd67720451cdf316155cf348103bfce1056a78bcddf0029b45ff0fdbcc71
\ No newline at end of file
diff --git a/src/vdbemem.c b/src/vdbemem.c
index 3ec7394a45..17d57bbfa9 100644
--- a/src/vdbemem.c
+++ b/src/vdbemem.c
@@ -1321,7 +1321,11 @@ static int valueFromExpr(
 
   assert( pExpr!=0 );
   while( (op = pExpr->op)==TK_UPLUS || op==TK_SPAN ) pExpr = pExpr->pLeft;
+#if defined(SQLITE_ENABLE_STAT3_OR_STAT4)
   if( op==TK_REGISTER ) op = pExpr->op2;
+#else
+  if( NEVER(op==TK_REGISTER) ) op = pExpr->op2;
+#endif
 
   /* Compressed expressions only appear when parsing the DEFAULT clause
   ** on a table column definition, and hence only when pCtx==0.  This

From b5008252b3dbfa1a049c28b8638bc1d8ed0a37be Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 7 Dec 2017 13:15:48 +0000
Subject: [PATCH 123/488] Updates to the main README.md file.

FossilOrigin-Name: 6bfafc35d1fa0415dcc85edbbd1b273f84b7a6ab75f979b01d90b18ce6aa5703
---
 README.md     | 32 ++++++++++++++++++++++++--------
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 3 files changed, 31 insertions(+), 15 deletions(-)

diff --git a/README.md b/README.md
index a7b8701106..f70b512d41 100644
--- a/README.md
+++ b/README.md
@@ -207,8 +207,8 @@ The amalgamation source file is more than 200K lines long.  Some symbolic
 debuggers (most notably MSVC) are unable to deal with files longer than 64K
 lines.  To work around this, a separate Tcl script, tool/split-sqlite3c.tcl,
 can be run on the amalgamation to break it up into a single small C file
-called **sqlite3-all.c** that does #include on about five other files
-named **sqlite3-1.c**, **sqlite3-2.c**, ..., **sqlite3-5.c**.  In this way,
+called **sqlite3-all.c** that does #include on about seven other files
+named **sqlite3-1.c**, **sqlite3-2.c**, ..., **sqlite3-7.c**.  In this way,
 all of the source code is contained within a single translation unit so
 that the compiler can do extra cross-procedure optimization, but no
 individual source file exceeds 32K lines in length.
@@ -237,7 +237,8 @@ Key files:
      trying to understand how the library works internally.
 
   *  **sqliteInt.h** - this header file defines many of the data objects
-     used internally by SQLite.
+     used internally by SQLite.  In addition to "sqliteInt.h", some
+     subsystems have their own header files.
 
   *  **parse.y** - This file describes the LALR(1) grammar that SQLite uses
      to parse SQL statements, and the actions that are taken at each step
@@ -249,29 +250,44 @@ Key files:
      which defines internal data objects.  The rest of SQLite interacts
      with the VDBE through an interface defined by vdbe.h.
 
-  *  **where.c** - This file analyzes the WHERE clause and generates
+  *  **where.c** - This file (together with its helper files named
+     by "where*.c") analyzes the WHERE clause and generates
      virtual machine code to run queries efficiently.  This file is
      sometimes called the "query optimizer".  It has its own private
      header file, whereInt.h, that defines data objects used internally.
 
   *  **btree.c** - This file contains the implementation of the B-Tree
-     storage engine used by SQLite.
+     storage engine used by SQLite.  The interface to the rest of the system
+     is defined by "btree.h".  The "btreeInt.h" header defines objects
+     used internally by btree.c and not published to the rest of the system.
 
   *  **pager.c** - This file contains the "pager" implementation, the
-     module that implements transactions.
+     module that implements transactions.  The "pager.h" header file
+     defines the interface between pager.c and the rest of the system.
 
   *  **os_unix.c** and **os_win.c** - These two files implement the interface
      between SQLite and the underlying operating system using the run-time
      pluggable VFS interface.
 
-  *  **shell.c** - This file is not part of the core SQLite library.  This
+  *  **shell.c.in** - This file is not part of the core SQLite library.  This
      is the file that, when linked against sqlite3.a, generates the
-     "sqlite3.exe" command-line shell.
+     "sqlite3.exe" command-line shell.  The "shell.c.in" file is transformed
+     into "shell.c" as part of the build process.
 
   *  **tclsqlite.c** - This file implements the Tcl bindings for SQLite.  It
      is not part of the core SQLite library.  But as most of the tests in this
      repository are written in Tcl, the Tcl language bindings are important.
 
+  *  **test*.c** - Files in the src/ folder that begin with "test" go into
+     building the "testfixture.exe" program.  The testfixture.exe program is
+     an enhanced TCL shell.  The testfixture.exe program runs scripts in the
+     test/ folder to validate the core SQLite code.  The testfixture program
+     (and some other test programs too) is build and run when you type
+     "make test".
+
+  *  **ext/misc/json1.c** - This file implements the various JSON functions
+     that are build into SQLite.
+
 There are many other source files.  Each has a succinct header comment that
 describes its purpose and role within the larger system.
 
diff --git a/manifest b/manifest
index 976bea8394..10ef87f007 100644
--- a/manifest
+++ b/manifest
@@ -1,9 +1,9 @@
-C Fix\san\sunreachable\sbranch\sin\scases\swhen\sSQLITE_ENABLE_STAT4\sis\snot\sdefined.
-D 2017-12-06T20:50:08.379
+C Updates\sto\sthe\smain\sREADME.md\sfile.
+D 2017-12-07T13:15:48.698
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
-F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd
+F README.md eeae1e552f93ef72ef7c5b8f6647b368a001c28820ad1df179d3dae602bef681
 F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2
@@ -1679,7 +1679,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0342ce510d2063a63839399a2cfa25b7fc02f4fde17764082676b332d6136241
-R dd71a721334f8be84f1b50254aaa4af8
+P 6a55bd67720451cdf316155cf348103bfce1056a78bcddf0029b45ff0fdbcc71
+R 5db6dab0cbf463182c8a99af07dcc6eb
 U drh
-Z e071d47cd491bbadd8e0c9ee301f9ca9
+Z 889514808bfb806c866b9adca4b0ed44
diff --git a/manifest.uuid b/manifest.uuid
index fc1d483ded..266f892dfb 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6a55bd67720451cdf316155cf348103bfce1056a78bcddf0029b45ff0fdbcc71
\ No newline at end of file
+6bfafc35d1fa0415dcc85edbbd1b273f84b7a6ab75f979b01d90b18ce6aa5703
\ No newline at end of file

From fd0245d771542a60f17fb7aa5c1ab13990d0a92f Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 7 Dec 2017 15:44:29 +0000
Subject: [PATCH 124/488] Begin adding support for the sqlar archive format to
 shell.c. There is no "extract" command so far, only "create".

FossilOrigin-Name: c9827a01a6e107f38f85c2b2c1c7a599e443067b106217e965b6936441ca619d
---
 ext/misc/fileio.c | 403 ++++++++++++++++++++++++++++++++++++++++++++--
 manifest          |  16 +-
 manifest.uuid     |   2 +-
 src/shell.c.in    | 215 +++++++++++++++++++++++++
 4 files changed, 611 insertions(+), 25 deletions(-)

diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c
index 2c00ad971d..475f3713ce 100644
--- a/ext/misc/fileio.c
+++ b/ext/misc/fileio.c
@@ -12,11 +12,46 @@
 **
 ** This SQLite extension implements SQL functions readfile() and
 ** writefile().
+**
+** Also, an eponymous virtual table type "fsdir". Used as follows:
+**
+**   SELECT * FROM fsdir($dirname);
+**
+** Returns one row for each entry in the directory $dirname. No row is
+** returned for "." or "..". Row columns are as follows:
+**
+**   name:  Name of directory entry.
+**   mode:  Value of stat.st_mode for directory entry.
+**   mtime: Value of stat.st_mtime for directory entry.
+**   data:  For a regular file, a blob containing the file data. For a
+**          symlink, a text value containing the text of the link. For a
+**          directory, NULL.
 */
 #include "sqlite3ext.h"
 SQLITE_EXTENSION_INIT1
 #include <stdio.h>
 
+#define FSDIR_SCHEMA "CREATE TABLE x(name,mode,mtime,data,dir HIDDEN)"
+
+static void readFileContents(sqlite3_context *ctx, const char *zName){
+  FILE *in;
+  long nIn;
+  void *pBuf;
+
+  in = fopen(zName, "rb");
+  if( in==0 ) return;
+  fseek(in, 0, SEEK_END);
+  nIn = ftell(in);
+  rewind(in);
+  pBuf = sqlite3_malloc( nIn );
+  if( pBuf && 1==fread(pBuf, nIn, 1, in) ){
+    sqlite3_result_blob(ctx, pBuf, nIn, sqlite3_free);
+  }else{
+    sqlite3_free(pBuf);
+  }
+  fclose(in);
+}
+
 /*
 ** Implementation of the "readfile(X)" SQL function.  The entire content
 ** of the file named X is read and returned as a BLOB.  NULL is returned
@@ -28,25 +63,10 @@ static void readfileFunc(
   sqlite3_value **argv
 ){
   const char *zName;
-  FILE *in;
-  long nIn;
-  void *pBuf;
-
   (void)(argc);  /* Unused parameter */
   zName = (const char*)sqlite3_value_text(argv[0]);
   if( zName==0 ) return;
-  in = fopen(zName, "rb");
-  if( in==0 ) return;
-  fseek(in, 0, SEEK_END);
-  nIn = ftell(in);
-  rewind(in);
-  pBuf = sqlite3_malloc( nIn );
-  if( pBuf && 1==fread(pBuf, nIn, 1, in) ){
-    sqlite3_result_blob(context, pBuf, nIn, sqlite3_free);
-  }else{
-    sqlite3_free(pBuf);
-  }
-  fclose(in);
+  readFileContents(context, zName);
 }
 
 /*
@@ -80,6 +100,354 @@ static void writefileFunc(
   sqlite3_result_int64(context, rc);
 }
 
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <dirent.h>
+
+/* 
+*/
+typedef struct fsdir_cursor fsdir_cursor;
+struct fsdir_cursor {
+  sqlite3_vtab_cursor base;  /* Base class - must be first */
+  int eType;                 /* One of FSDIR_DIR or FSDIR_ENTRY */
+  DIR *pDir;                 /* From opendir() */
+  struct stat sStat;         /* Current lstat() results */
+  char *zDir;                /* Directory to read */
+  int nDir;                  /* Value of strlen(zDir) */
+  char *zPath;               /* Path to current entry */
+  int bEof;
+  sqlite3_int64 iRowid;      /* Current rowid */
+};
+
+typedef struct fsdir_tab fsdir_tab;
+struct fsdir_tab {
+  sqlite3_vtab base;         /* Base class - must be first */
+  int eType;                 /* One of FSDIR_DIR or FSDIR_ENTRY */
+};
+
+#define FSDIR_DIR   0
+#define FSDIR_ENTRY 1
+
+/*
+** Construct a new fsdir virtual table object.
+*/
+static int fsdirConnect(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  fsdir_tab *pNew = 0;
+  int rc;
+
+  rc = sqlite3_declare_vtab(db, FSDIR_SCHEMA);
+  if( rc==SQLITE_OK ){
+    pNew = (fsdir_tab*)sqlite3_malloc( sizeof(*pNew) );
+    if( pNew==0 ) return SQLITE_NOMEM;
+    memset(pNew, 0, sizeof(*pNew));
+    pNew->eType = (pAux==0 ? FSDIR_DIR : FSDIR_ENTRY);
+  }
+  *ppVtab = (sqlite3_vtab*)pNew;
+  return rc;
+}
+
+/*
+** This method is the destructor for fsdir vtab objects.
+*/
+static int fsdirDisconnect(sqlite3_vtab *pVtab){
+  sqlite3_free(pVtab);
+  return SQLITE_OK;
+}
+
+/*
+** Constructor for a new fsdir_cursor object.
+*/
+static int fsdirOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
+  fsdir_cursor *pCur;
+  pCur = sqlite3_malloc( sizeof(*pCur) );
+  if( pCur==0 ) return SQLITE_NOMEM;
+  memset(pCur, 0, sizeof(*pCur));
+  pCur->eType = ((fsdir_tab*)p)->eType;
+  *ppCursor = &pCur->base;
+  return SQLITE_OK;
+}
+
+/*
+** Destructor for an fsdir_cursor.
+*/
+static int fsdirClose(sqlite3_vtab_cursor *cur){
+  fsdir_cursor *pCur = (fsdir_cursor*)cur;
+  if( pCur->pDir ) closedir(pCur->pDir);
+  sqlite3_free(pCur->zDir);
+  sqlite3_free(pCur->zPath);
+  sqlite3_free(pCur);
+  return SQLITE_OK;
+}
+
+/*
+** Advance an fsdir_cursor to its next row of output.
+*/
+static int fsdirNext(sqlite3_vtab_cursor *cur){
+  fsdir_cursor *pCur = (fsdir_cursor*)cur;
+  struct dirent *pEntry;
+
+  if( pCur->eType==FSDIR_ENTRY ){
+    pCur->bEof = 1;
+    return SQLITE_OK;
+  }
+
+  sqlite3_free(pCur->zPath);
+  pCur->zPath = 0;
+
+  while( 1 ){
+    pEntry = readdir(pCur->pDir);
+    if( pEntry ){
+      if( strcmp(pEntry->d_name, ".") 
+       && strcmp(pEntry->d_name, "..") 
+      ){
+        pCur->zPath = sqlite3_mprintf("%s/%s", pCur->zDir, pEntry->d_name);
+        if( pCur->zPath==0 ) return SQLITE_NOMEM;
+        lstat(pCur->zPath, &pCur->sStat);
+        break;
+      }
+    }else{
+      pCur->bEof = 1;
+      break;
+    }
+  }
+
+  pCur->iRowid++;
+  return SQLITE_OK;
+}
+
+/*
+** Return values of columns for the row at which the series_cursor
+** is currently pointing.
+*/
+static int fsdirColumn(
+  sqlite3_vtab_cursor *cur,   /* The cursor */
+  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
+  int i                       /* Which column to return */
+){
+  fsdir_cursor *pCur = (fsdir_cursor*)cur;
+  switch( i ){
+    case 0: { /* name */
+      const char *zName;
+      if( pCur->eType==FSDIR_DIR ){
+        zName = &pCur->zPath[pCur->nDir+1];
+      }else{
+        zName = pCur->zPath;
+      }
+      sqlite3_result_text(ctx, zName, -1, SQLITE_TRANSIENT);
+      break;
+    }
+
+    case 1: /* mode */
+      sqlite3_result_int64(ctx, pCur->sStat.st_mode);
+      break;
+
+    case 2: /* mode */
+      sqlite3_result_int64(ctx, pCur->sStat.st_mtime);
+      break;
+
+    case 3: {
+      mode_t m = pCur->sStat.st_mode;
+      if( S_ISDIR(m) ){
+        sqlite3_result_null(ctx);
+      }else if( S_ISLNK(m) ){
+        char aStatic[64];
+        char *aBuf = aStatic;
+        int nBuf = 64;
+        int n;
+
+        while( 1 ){
+          n = readlink(pCur->zPath, aBuf, nBuf);
+          if( n<nBuf ) break;
+          if( aBuf!=aStatic ) sqlite3_free(aBuf);
+          nBuf = nBuf*2;
+          aBuf = sqlite3_malloc(nBuf);
+          if( aBuf==0 ){
+            sqlite3_result_error_nomem(ctx);
+            return SQLITE_NOMEM;
+          }
+        }
+
+        sqlite3_result_text(ctx, aBuf, n, SQLITE_TRANSIENT);
+        if( aBuf!=aStatic ) sqlite3_free(aBuf);
+      }else{
+        readFileContents(ctx, pCur->zPath);
+      }
+    }
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Return the rowid for the current row. In this implementation, the
+** first row returned is assigned rowid value 1, and each subsequent
+** row a value 1 more than that of the previous.
+*/
+static int fsdirRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
+  fsdir_cursor *pCur = (fsdir_cursor*)cur;
+  *pRowid = pCur->iRowid;
+  return SQLITE_OK;
+}
+
+/*
+** Return TRUE if the cursor has been moved off of the last
+** row of output.
+*/
+static int fsdirEof(sqlite3_vtab_cursor *cur){
+  fsdir_cursor *pCur = (fsdir_cursor*)cur;
+  return pCur->bEof;
+}
+
+static void fsdirSetErrmsg(fsdir_cursor *pCur, const char *zFmt, ...){
+  va_list ap;
+  va_start(ap, zFmt);
+  pCur->base.pVtab->zErrMsg = sqlite3_vmprintf(zFmt, ap);
+  va_end(ap);
+}
+
+/*
+** xFilter callback.
+*/
+static int fsdirFilter(
+  sqlite3_vtab_cursor *cur, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  const char *zDir = 0;
+  fsdir_cursor *pCur = (fsdir_cursor*)cur;
+
+  sqlite3_free(pCur->zDir);
+  pCur->iRowid = 0;
+  pCur->zDir = 0;
+  pCur->bEof = 0;
+  if( pCur->pDir ){
+    closedir(pCur->pDir);
+    pCur->pDir = 0;
+  }
+
+  if( idxNum==0 ){
+    fsdirSetErrmsg(pCur, "table function fsdir requires an argument");
+    return SQLITE_ERROR;
+  }
+
+  assert( argc==1 );
+  zDir = (const char*)sqlite3_value_text(argv[0]);
+  if( zDir==0 ){
+    fsdirSetErrmsg(pCur, "table function fsdir requires a non-NULL argument");
+    return SQLITE_ERROR;
+  }
+
+  pCur->zDir = sqlite3_mprintf("%s", zDir);
+  if( pCur->zDir==0 ){
+    return SQLITE_NOMEM;
+  }
+
+  if( pCur->eType==FSDIR_ENTRY ){
+    int rc = lstat(pCur->zDir, &pCur->sStat);
+    if( rc ){
+      fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zDir);
+    }else{
+      pCur->zPath = sqlite3_mprintf("%s", pCur->zDir);
+      if( pCur->zPath==0 ) return SQLITE_NOMEM;
+    }
+    return SQLITE_OK;
+  }else{
+    pCur->nDir = strlen(pCur->zDir);
+    pCur->pDir = opendir(zDir);
+    if( pCur->pDir==0 ){
+      fsdirSetErrmsg(pCur, "error in opendir(\"%s\")", zDir);
+      return SQLITE_ERROR;
+    }
+
+    return fsdirNext(cur);
+  }
+}
+
+/*
+** SQLite will invoke this method one or more times while planning a query
+** that uses the generate_series virtual table.  This routine needs to create
+** a query plan for each invocation and compute an estimated cost for that
+** plan.
+**
+** In this implementation idxNum is used to represent the
+** query plan.  idxStr is unused.
+**
+** The query plan is represented by bits in idxNum:
+**
+**  (1)  start = $value  -- constraint exists
+**  (2)  stop = $value   -- constraint exists
+**  (4)  step = $value   -- constraint exists
+**  (8)  output in descending order
+*/
+static int fsdirBestIndex(
+  sqlite3_vtab *tab,
+  sqlite3_index_info *pIdxInfo
+){
+  int i;                 /* Loop over constraints */
+
+  const struct sqlite3_index_constraint *pConstraint;
+  pConstraint = pIdxInfo->aConstraint;
+  for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){
+    if( pConstraint->usable==0 ) continue;
+    if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
+    if( pConstraint->iColumn!=4 ) continue;
+    break;
+  }
+
+  if( i<pIdxInfo->nConstraint ){
+    pIdxInfo->aConstraintUsage[i].omit = 1;
+    pIdxInfo->aConstraintUsage[i].argvIndex = 1;
+    pIdxInfo->idxNum = 1;
+    pIdxInfo->estimatedCost = 10.0;
+  }else{
+    pIdxInfo->idxNum = 0;
+    pIdxInfo->estimatedCost = (double)(((sqlite3_int64)1) << 50);
+  }
+
+  return SQLITE_OK;
+}
+
+static int fsdirRegister(sqlite3 *db){
+  static sqlite3_module fsdirModule = {
+    0,                         /* iVersion */
+    0,                         /* xCreate */
+    fsdirConnect,              /* xConnect */
+    fsdirBestIndex,            /* xBestIndex */
+    fsdirDisconnect,           /* xDisconnect */
+    0,                         /* xDestroy */
+    fsdirOpen,                 /* xOpen - open a cursor */
+    fsdirClose,                /* xClose - close a cursor */
+    fsdirFilter,               /* xFilter - configure scan constraints */
+    fsdirNext,                 /* xNext - advance a cursor */
+    fsdirEof,                  /* xEof - check for end of scan */
+    fsdirColumn,               /* xColumn - read data */
+    fsdirRowid,                /* xRowid - read data */
+    0,                         /* xUpdate */
+    0,                         /* xBegin */
+    0,                         /* xSync */
+    0,                         /* xCommit */
+    0,                         /* xRollback */
+    0,                         /* xFindMethod */
+    0,                         /* xRename */
+  };
+
+  int rc = sqlite3_create_module(db, "fsdir", &fsdirModule, 0);
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_create_module(db, "fsentry", &fsdirModule, (void*)1);
+  }
+  return rc;
+}
+#else         /* SQLITE_OMIT_VIRTUALTABLE */
+# define fsdirRegister(x) SQLITE_OK
+#endif
 
 #ifdef _WIN32
 __declspec(dllexport)
@@ -98,5 +466,8 @@ int sqlite3_fileio_init(
     rc = sqlite3_create_function(db, "writefile", 2, SQLITE_UTF8, 0,
                                  writefileFunc, 0, 0);
   }
+  if( rc==SQLITE_OK ){
+    rc = fsdirRegister(db);
+  }
   return rc;
 }
diff --git a/manifest b/manifest
index 3e24dc86b3..dc93367667 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C For\sMSVC,\ssimplify\sdefault\slocations\sfor\sTcl\sand\sICU\sby\susing\sdirectories\sinside\s'compat'.
-D 2017-12-05T19:07:30.651
+C Begin\sadding\ssupport\sfor\sthe\ssqlar\sarchive\sformat\sto\sshell.c.\sThere\sis\sno\n"extract"\scommand\sso\sfar,\sonly\s"create".
+D 2017-12-07T15:44:29.604
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
@@ -269,7 +269,7 @@ F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83
 F ext/misc/csv.c 1a009b93650732e22334edc92459c4630b9fa703397cbb3c8ca279921a36ca11
 F ext/misc/dbdump.c 3509fa6b8932d04e932d6b6b827b6a82ca362781b8e8f3c77336f416793e215e
 F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2
-F ext/misc/fileio.c b1aa06c0f1dac277695d4529e5e976c65ab5678dcbb53a0304deaa8adc44b332
+F ext/misc/fileio.c bd2dd9bd22a509f972a4658be18bbfef80aec3cbc3e18948c5e8c5e29ece9939
 F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25
 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c
 F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281984
@@ -474,7 +474,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
-F src/shell.c.in ab727c09b4c87c0c1db32d2fe0a910c0a8e468a0209233328753f5526d6c6c73
+F src/shell.c.in 56c4c091c74af2c7858f2d8af962caa632889596435b17bffbc308058fb305cc
 F src/sqlite.h.in 8fd97993d48b50b9bade38c52f12d175942c9497c960905610c7b03a3e4b5818
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1681,7 +1681,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e1838cee3847301ef491467dc75d9c4e1e3b12599596c058bdb14319a52fd8a0
-R a792968384e617f4a37df83169716b24
-U mistachkin
-Z 06008f15f266edb3571035a6d698db2a
+P 8155b5ac850327ea76aba2adf624132f3e05024c973afd218b12f186fc7630e8
+R 1ffe38726fc0bf4dcb6cd1bf88405c9b
+U dan
+Z f98960131c8a51264c03e5b1a40bd1b7
diff --git a/manifest.uuid b/manifest.uuid
index b50deeb9bc..3b745e1dd5 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-8155b5ac850327ea76aba2adf624132f3e05024c973afd218b12f186fc7630e8
\ No newline at end of file
+c9827a01a6e107f38f85c2b2c1c7a599e443067b106217e965b6936441ca619d
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index edd75b078c..9015a4330a 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -4074,6 +4074,214 @@ static int lintDotCommand(
   return SQLITE_ERROR;
 }
 
+static void shellPrepare(
+  ShellState *p, 
+  int *pRc, 
+  const char *zSql, 
+  sqlite3_stmt **ppStmt
+){
+  *ppStmt = 0;
+  if( *pRc==SQLITE_OK ){
+    int rc = sqlite3_prepare_v2(p->db, zSql, -1, ppStmt, 0);
+    if( rc!=SQLITE_OK ){
+      raw_printf(stderr, "sql error: %s (%d)\n", 
+          sqlite3_errmsg(p->db), sqlite3_errcode(p->db)
+      );
+      *pRc = rc;
+    }
+  }
+}
+
+static void shellFinalize(
+  int *pRc, 
+  sqlite3_stmt *pStmt
+){
+  int rc = sqlite3_finalize(pStmt);
+  if( *pRc==SQLITE_OK ) *pRc = rc;
+}
+
+static void shellReset(
+  int *pRc, 
+  sqlite3_stmt *pStmt
+){
+  int rc = sqlite3_reset(pStmt);
+  if( *pRc==SQLITE_OK ) *pRc = rc;
+}
+
+/*
+** Implementation of .ar "eXtract" command. 
+*/
+static int arExtractCommand(ShellState *p, int bVerbose){
+  const char *zSql = 
+    "SELECT name, mode, mtime, sz, data FROM sqlar";
+  sqlite3_stmt *pSql = 0;
+  int rc = SQLITE_OK;
+
+  shellPrepare(p, &rc, zSql, &pSql);
+  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
+  }
+
+  shellFinalize(&rc, pSql);
+  return rc;
+}
+
+/*
+** Implementation of .ar "Create" command. 
+**
+** Create the "sqlar" table in the database if it does not already exist.
+** Then add each file in the azFile[] array to the archive. Directories
+** are added recursively. If argument bVerbose is non-zero, a message is
+** printed on stdout for each file archived.
+*/
+static int arCreateCommand(
+  ShellState *p,                  /* Shell state pointer */
+  char **azFile,                  /* Array of files to add to archive */
+  int nFile,                      /* Number of entries in azFile[] */
+  int bVerbose                    /* True to be verbose on stdout */
+){
+  const char *zSql = 
+    "WITH f(n, m, t, d) AS ("
+    "  SELECT name, mode, mtime, data FROM fsentry(:1) UNION ALL "
+    "  SELECT n || '/' || name, mode, mtime, data "
+    "      FROM f, fsdir(n) WHERE (m&?)"
+    ") SELECT * FROM f";
+
+  const char *zSqlar = 
+      "CREATE TABLE IF NOT EXISTS sqlar("
+      "name TEXT PRIMARY KEY,  -- name of the file\n"
+      "mode INT,               -- access permissions\n"
+      "mtime INT,              -- last modification time\n"
+      "sz INT,                 -- original file size\n"
+      "data BLOB               -- compressed content\n"
+      ")";
+
+  const char *zInsert = "REPLACE INTO sqlar VALUES(?, ?, ?, ?, ?)";
+
+  sqlite3_stmt *pStmt = 0;        /* Directory traverser */
+  sqlite3_stmt *pInsert = 0;      /* Compilation of zInsert */
+  int i;                          /* For iterating through azFile[] */
+  int rc;                         /* Return code */
+
+  Bytef *aCompress = 0;           /* Compression buffer */
+  int nCompress = 0;              /* Size of compression buffer */
+  
+  rc = sqlite3_exec(p->db, "SAVEPOINT ar;", 0, 0, 0);
+  if( rc!=SQLITE_OK ) return rc;
+
+  rc = sqlite3_exec(p->db, zSqlar, 0, 0, 0);
+  shellPrepare(p, &rc, zInsert, &pInsert);
+  shellPrepare(p, &rc, zSql, &pStmt);
+
+  for(i=0; i<nFile && rc==SQLITE_OK; i++){
+    sqlite3_bind_text(pStmt, 1, azFile[i], -1, SQLITE_STATIC);
+    sqlite3_bind_int(pStmt, 2, S_IFDIR);
+    while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
+      int sz;
+      const char *zName = (const char*)sqlite3_column_text(pStmt, 0);
+      int mode = sqlite3_column_int(pStmt, 1);
+      unsigned int mtime = sqlite3_column_int(pStmt, 2);
+
+      if( bVerbose ){
+        raw_printf(stdout, "%s\n", zName);
+      }
+
+      sqlite3_bind_text(pInsert, 1, zName, -1, SQLITE_STATIC);
+      sqlite3_bind_int(pInsert, 2, mode);
+      sqlite3_bind_int64(pInsert, 3, (sqlite3_int64)mtime);
+
+      if( S_ISDIR(mode) ){
+        sz = 0;
+        sqlite3_bind_null(pInsert, 5);
+      }else if( S_ISLNK(mode) ){
+        sz = -1;
+        sqlite3_bind_value(pInsert, 5, sqlite3_column_value(pStmt, 3));
+      }else{
+        uLongf nReq;              /* Required size of compression buffer */
+        const Bytef *pData = (const Bytef*)sqlite3_column_blob(pStmt, 3);
+        sz = sqlite3_column_bytes(pStmt, 3);
+        nReq = compressBound(sz);
+        if( aCompress==0 || nReq>nCompress ){
+          Bytef *aNew = sqlite3_realloc(aCompress, nReq);
+          if( aNew==0 ){
+            rc = SQLITE_NOMEM;
+          }else{
+            aCompress = aNew;
+            nCompress = nReq;
+          }
+        }
+
+        if( Z_OK!=compress(aCompress, &nReq, pData, sz) ){
+          rc = SQLITE_ERROR;
+        }
+        if( nReq<sz ){
+          sqlite3_bind_blob(pInsert, 5, aCompress, nReq, SQLITE_STATIC);
+        }else{
+          sqlite3_bind_blob(pInsert, 5, pData, sz, SQLITE_STATIC);
+        }
+      }
+
+      if( rc==SQLITE_OK ){
+        sqlite3_bind_int(pInsert, 4, sz);
+        sqlite3_step(pInsert);
+        rc = sqlite3_reset(pInsert);
+      }
+    }
+    shellReset(&rc, pStmt);
+  }
+
+  if( rc!=SQLITE_OK ){
+    sqlite3_exec(p->db, "ROLLBACK TO ar; RELEASE ar;", 0, 0, 0);
+  }else{
+    rc = sqlite3_exec(p->db, "RELEASE ar;", 0, 0, 0);
+  }
+  shellFinalize(&rc, pStmt);
+  shellFinalize(&rc, pInsert);
+  sqlite3_free(aCompress);
+  return rc;
+}
+
+/*
+** Implementation of ".ar" dot command.
+*/
+static int arDotCommand(
+  ShellState *pState,             /* Current shell tool state */
+  char **azArg,                   /* Array of arguments passed to dot command */
+  int nArg                        /* Number of entries in azArg[] */
+){
+  int bVerbose = 0;
+  char cmd = 0;
+  int i;
+  int n1;
+  if( nArg<=1 ) goto usage;
+
+  n1 = strlen(azArg[1]);
+  for(i=0; i<n1; i++){
+    char c = azArg[1][i];
+    if( c=='c' || c=='x' ){
+      if( cmd ) goto usage;
+      cmd = c;
+    }
+    else if( c=='v' ){
+      bVerbose = 1;
+    }else{
+      goto usage;
+    }
+  }
+
+  if( cmd=='c' ){
+    return arCreateCommand(pState, &azArg[2], nArg-2, bVerbose);
+  }
+
+  if( cmd=='x' ){
+    if( nArg!=2 ) goto usage;
+    return arExtractCommand(pState, bVerbose);
+  }
+
+ usage:
+  raw_printf(stderr, "Usage %s sub-command ?args...?\n", azArg[0]);
+  return SQLITE_ERROR;
+}
+
 
 /*
 ** If an input line begins with "." then invoke this routine to
@@ -4134,6 +4342,13 @@ static int do_meta_command(char *zLine, ShellState *p){
   }else
 #endif
 
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+  if( c=='a' && strncmp(azArg[0], "ar", n)==0 ){
+    open_db(p, 0);
+    rc = arDotCommand(p, azArg, nArg);
+  }else
+#endif
+
   if( (c=='b' && n>=3 && strncmp(azArg[0], "backup", n)==0)
    || (c=='s' && n>=3 && strncmp(azArg[0], "save", n)==0)
   ){

From 70fedc584d602c22da460fd97d9f257b742f3913 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 7 Dec 2017 16:51:25 +0000
Subject: [PATCH 125/488] Fix harmless compiler warnings in the rot13
 extension.

FossilOrigin-Name: 113470772b04210b9300803124c06af2c0b33278313b311fefe6f80594b24e4a
---
 ext/misc/rot13.c |  6 +++---
 manifest         | 12 ++++++------
 manifest.uuid    |  2 +-
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/ext/misc/rot13.c b/ext/misc/rot13.c
index 68fdf60b0f..2e9dd21c60 100644
--- a/ext/misc/rot13.c
+++ b/ext/misc/rot13.c
@@ -47,9 +47,9 @@ static void rot13func(
   const unsigned char *zIn;
   int nIn;
   unsigned char *zOut;
-  char *zToFree = 0;
+  unsigned char *zToFree = 0;
   int i;
-  char zTemp[100];
+  unsigned char zTemp[100];
   assert( argc==1 );
   if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return;
   zIn = (const unsigned char*)sqlite3_value_text(argv[0]);
@@ -57,7 +57,7 @@ static void rot13func(
   if( nIn<sizeof(zTemp)-1 ){
     zOut = zTemp;
   }else{
-    zOut = zToFree = sqlite3_malloc( nIn+1 );
+    zOut = zToFree = (unsigned char*)sqlite3_malloc64( nIn+1 );
     if( zOut==0 ){
       sqlite3_result_error_nomem(context);
       return;
diff --git a/manifest b/manifest
index 10ef87f007..65cef35537 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Updates\sto\sthe\smain\sREADME.md\sfile.
-D 2017-12-07T13:15:48.698
+C Fix\sharmless\scompiler\swarnings\sin\sthe\srot13\sextension.
+D 2017-12-07T16:51:25.777
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
@@ -277,7 +277,7 @@ F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342
 F ext/misc/percentile.c 92699c8cd7d517ff610e6037e56506f8904dae2e
 F ext/misc/regexp.c a68d25c659bd2d893cd1215667bbf75ecb9dc7d4
 F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c
-F ext/misc/rot13.c 1ac6f95f99b575907b9b09c81a349114cf9be45a
+F ext/misc/rot13.c 540a169cb0d74f15522a8930b0cccdcb37a4fd071d219a5a083a319fc6e8db77
 F ext/misc/scrub.c 1c5bfb8b0cd18b602fcb55755e84abf0023ac2fb
 F ext/misc/series.c f3c0dba5c5c749ce1782b53076108f87cf0b71041eb6023f727a9c50681da564
 F ext/misc/sha1.c 0b9e9b855354910d3ca467bf39099d570e73db56
@@ -1679,7 +1679,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6a55bd67720451cdf316155cf348103bfce1056a78bcddf0029b45ff0fdbcc71
-R 5db6dab0cbf463182c8a99af07dcc6eb
+P 6bfafc35d1fa0415dcc85edbbd1b273f84b7a6ab75f979b01d90b18ce6aa5703
+R 6e8dc804e858af71ad2b8ec1737b4689
 U drh
-Z 889514808bfb806c866b9adca4b0ed44
+Z 5dfea499ba027d378d355b994a19e393
diff --git a/manifest.uuid b/manifest.uuid
index 266f892dfb..3895727b02 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6bfafc35d1fa0415dcc85edbbd1b273f84b7a6ab75f979b01d90b18ce6aa5703
\ No newline at end of file
+113470772b04210b9300803124c06af2c0b33278313b311fefe6f80594b24e4a
\ No newline at end of file

From 25c1218e73b918d27af79245136c9efe17c6f5f9 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 7 Dec 2017 21:03:33 +0000
Subject: [PATCH 126/488] Add the ".ar x" command to the shell. For extracting
 the contents of sqlar archives.

FossilOrigin-Name: 0cc699d14adfe8c7b7be50c180186562861806c47425c80c935bce43ee5c5c12
---
 ext/misc/fileio.c | 104 ++++++++++++++++++++++++++++++++++++----------
 manifest          |  14 +++----
 manifest.uuid     |   2 +-
 src/shell.c.in    |  46 ++++++++++++++++----
 4 files changed, 128 insertions(+), 38 deletions(-)

diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c
index 475f3713ce..db69357008 100644
--- a/ext/misc/fileio.c
+++ b/ext/misc/fileio.c
@@ -30,6 +30,17 @@
 #include "sqlite3ext.h"
 SQLITE_EXTENSION_INIT1
 #include <stdio.h>
+#include <string.h>
+#include <assert.h>
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <dirent.h>
+#include <time.h>
+#include <utime.h>
+
 
 #define FSDIR_SCHEMA "CREATE TABLE x(name,mode,mtime,data,dir HIDDEN)"
 
@@ -69,44 +80,91 @@ static void readfileFunc(
   readFileContents(context, zName);
 }
 
+static void ctxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){
+  char *zMsg = 0;
+  va_list ap;
+  va_start(ap, zFmt);
+  zMsg = sqlite3_vmprintf(zFmt, ap);
+  sqlite3_result_error(ctx, zMsg, -1);
+  sqlite3_free(zMsg);
+  va_end(ap);
+}
+
 /*
-** Implementation of the "writefile(X,Y)" SQL function.  The argument Y
-** is written into file X.  The number of bytes written is returned.  Or
-** NULL is returned if something goes wrong, such as being unable to open
-** file X for writing.
+** Implementation of the "writefile(W,X[,Y]])" SQL function.  
+**
+** The argument X is written into file W.  The number of bytes written is
+** returned. Or NULL is returned if something goes wrong, such as being unable
+** to open file X for writing.
 */
 static void writefileFunc(
   sqlite3_context *context,
   int argc,
   sqlite3_value **argv
 ){
-  FILE *out;
-  const char *z;
-  sqlite3_int64 rc;
   const char *zFile;
+  mode_t mode = 0;
+
+  if( argc<2 || argc>3 ){
+    sqlite3_result_error(context, 
+        "wrong number of arguments to function writefile()", -1
+    );
+    return;
+  }
 
-  (void)(argc);  /* Unused parameter */
   zFile = (const char*)sqlite3_value_text(argv[0]);
   if( zFile==0 ) return;
-  out = fopen(zFile, "wb");
-  if( out==0 ) return;
-  z = (const char*)sqlite3_value_blob(argv[1]);
-  if( z==0 ){
-    rc = 0;
-  }else{
-    rc = fwrite(z, 1, sqlite3_value_bytes(argv[1]), out);
+  if( argc>=3 ){
+    sqlite3_result_int(context, 0);
+    mode = sqlite3_value_int(argv[2]);
+  }
+
+  if( S_ISLNK(mode) ){
+    const char *zTo = (const char*)sqlite3_value_text(argv[1]);
+    if( symlink(zTo, zFile)<0 ){
+      ctxErrorMsg(context, "failed to create symlink: %s", zFile);
+      return;
+    }
+  }else{
+    if( S_ISDIR(mode) ){
+      if( mkdir(zFile, mode) ){
+        ctxErrorMsg(context, "failed to create directory: %s", zFile);
+        return;
+      }
+    }else{
+      sqlite3_int64 nWrite = 0;
+      const char *z;
+      int rc = 0;
+      FILE *out = fopen(zFile, "wb");
+      if( out==0 ){
+        if( argc>2 ){
+          ctxErrorMsg(context, "failed to open file for writing: %s", zFile);
+        }
+        return;
+      }
+      z = (const char*)sqlite3_value_blob(argv[1]);
+      if( z ){
+        sqlite3_int64 n = fwrite(z, 1, sqlite3_value_bytes(argv[1]), out);
+        nWrite = sqlite3_value_bytes(argv[1]);
+        if( nWrite!=n ){
+          ctxErrorMsg(context, "failed to write file: %s", zFile);
+          rc = 1;
+        }
+      }
+      fclose(out);
+      if( rc ) return;
+      sqlite3_result_int64(context, nWrite);
+    }
+
+    if( argc>2 && chmod(zFile, mode & 0777) ){
+      ctxErrorMsg(context, "failed to chmod file: %s", zFile);
+      return;
+    }
   }
-  fclose(out);
-  sqlite3_result_int64(context, rc);
 }
 
 #ifndef SQLITE_OMIT_VIRTUALTABLE
 
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <unistd.h>
-#include <dirent.h>
-
 /* 
 */
 typedef struct fsdir_cursor fsdir_cursor;
@@ -463,7 +521,7 @@ int sqlite3_fileio_init(
   rc = sqlite3_create_function(db, "readfile", 1, SQLITE_UTF8, 0,
                                readfileFunc, 0, 0);
   if( rc==SQLITE_OK ){
-    rc = sqlite3_create_function(db, "writefile", 2, SQLITE_UTF8, 0,
+    rc = sqlite3_create_function(db, "writefile", -1, SQLITE_UTF8, 0,
                                  writefileFunc, 0, 0);
   }
   if( rc==SQLITE_OK ){
diff --git a/manifest b/manifest
index dc93367667..ab01f7195c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Begin\sadding\ssupport\sfor\sthe\ssqlar\sarchive\sformat\sto\sshell.c.\sThere\sis\sno\n"extract"\scommand\sso\sfar,\sonly\s"create".
-D 2017-12-07T15:44:29.604
+C Add\sthe\s".ar\sx"\scommand\sto\sthe\sshell.\sFor\sextracting\sthe\scontents\sof\ssqlar\narchives.
+D 2017-12-07T21:03:33.903
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
@@ -269,7 +269,7 @@ F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83
 F ext/misc/csv.c 1a009b93650732e22334edc92459c4630b9fa703397cbb3c8ca279921a36ca11
 F ext/misc/dbdump.c 3509fa6b8932d04e932d6b6b827b6a82ca362781b8e8f3c77336f416793e215e
 F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2
-F ext/misc/fileio.c bd2dd9bd22a509f972a4658be18bbfef80aec3cbc3e18948c5e8c5e29ece9939
+F ext/misc/fileio.c c84ec9c399657bd95914e7c4e9aefcc148cb86fe3ab7f2102e9557c861dd0d51
 F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25
 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c
 F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281984
@@ -474,7 +474,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
-F src/shell.c.in 56c4c091c74af2c7858f2d8af962caa632889596435b17bffbc308058fb305cc
+F src/shell.c.in 2f9ae0bee09bdd35922ab7ed264d88e1d7fb34d39d37fc633e6a3a1af60036be
 F src/sqlite.h.in 8fd97993d48b50b9bade38c52f12d175942c9497c960905610c7b03a3e4b5818
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1681,7 +1681,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 8155b5ac850327ea76aba2adf624132f3e05024c973afd218b12f186fc7630e8
-R 1ffe38726fc0bf4dcb6cd1bf88405c9b
+P c9827a01a6e107f38f85c2b2c1c7a599e443067b106217e965b6936441ca619d
+R 4b2bb930e9456062f8914cba7a04cf63
 U dan
-Z f98960131c8a51264c03e5b1a40bd1b7
+Z 4726613d4458219c808274490c55a044
diff --git a/manifest.uuid b/manifest.uuid
index 3b745e1dd5..245ee49a8f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c9827a01a6e107f38f85c2b2c1c7a599e443067b106217e965b6936441ca619d
\ No newline at end of file
+0cc699d14adfe8c7b7be50c180186562861806c47425c80c935bce43ee5c5c12
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 9015a4330a..fe58460fe1 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -4096,8 +4096,16 @@ static void shellFinalize(
   int *pRc, 
   sqlite3_stmt *pStmt
 ){
-  int rc = sqlite3_finalize(pStmt);
-  if( *pRc==SQLITE_OK ) *pRc = rc;
+  if( pStmt ){
+    sqlite3 *db = sqlite3_db_handle(pStmt);
+    int rc = sqlite3_finalize(pStmt);
+    if( *pRc==SQLITE_OK ){
+      if( rc!=SQLITE_OK ){
+        raw_printf(stderr, "SQL error: %s\n", sqlite3_errmsg(db));
+      }
+      *pRc = rc;
+    }
+  }
 }
 
 static void shellReset(
@@ -4112,16 +4120,40 @@ static void shellReset(
 ** Implementation of .ar "eXtract" command. 
 */
 static int arExtractCommand(ShellState *p, int bVerbose){
-  const char *zSql = 
-    "SELECT name, mode, mtime, sz, data FROM sqlar";
+  const char *zSql1 = 
+    "SELECT name, writefile(name, "
+    "CASE WHEN (data AND sz>=0 AND sz!=length(data)) THEN uncompress(data) "
+    "   ELSE data END, "
+    "mode) FROM sqlar";
+  const char *zSql2 = "SELECT name, mtime FROM sqlar"; 
+
+  struct timespec times[2];
   sqlite3_stmt *pSql = 0;
   int rc = SQLITE_OK;
 
-  shellPrepare(p, &rc, zSql, &pSql);
-  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
-  }
+  memset(times, 0, sizeof(times));
+  times[0].tv_sec = time(0);
 
+  shellPrepare(p, &rc, zSql1, &pSql);
+  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
+    if( bVerbose ){
+      raw_printf(stdout, "%s\n", sqlite3_column_text(pSql, 0));
+    }
+  }
   shellFinalize(&rc, pSql);
+
+  shellPrepare(p, &rc, zSql2, &pSql);
+  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
+    const char *zPath = (const char*)sqlite3_column_text(pSql, 0);
+    times[1].tv_sec = (time_t)sqlite3_column_int64(pSql, 1);
+    if( utimensat(AT_FDCWD, zPath, times, AT_SYMLINK_NOFOLLOW) ){
+      raw_printf(stderr, "failed to set timestamp for %s\n", zPath);
+      rc = SQLITE_ERROR;
+      break;
+    }
+  }
+  shellFinalize(&rc, pSql);
+
   return rc;
 }
 

From a0951d875bd55ea6dbb103a89acf264e52858a8f Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Thu, 7 Dec 2017 22:04:53 +0000
Subject: [PATCH 127/488] Fix typo in comment.  No changes to code.

FossilOrigin-Name: 95958b60f9282384cac8f29c519d0fa1c32d7c0366c01ce681f6691e5bbf8438
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/date.c    |  2 +-
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index 65cef35537..d31e28b549 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sharmless\scompiler\swarnings\sin\sthe\srot13\sextension.
-D 2017-12-07T16:51:25.777
+C Fix\stypo\sin\scomment.\s\sNo\schanges\sto\scode.
+D 2017-12-07T22:04:53.501
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
@@ -420,7 +420,7 @@ F src/build.c 514db9d494ed29155e552f2ec2fa7c55c0241f847c683156b7c017f4b0bad9fa
 F src/callback.c 28a8ede982fde4129b828350f78f2c01fe7d12c74d1a0a05d7108ab36f308688
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
-F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74
+F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957
 F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
 F src/delete.c e6a70fb58f6628f0ffc6d7221a6702c0d7b342c82520385b3996b364c22e0cb3
@@ -1679,7 +1679,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6bfafc35d1fa0415dcc85edbbd1b273f84b7a6ab75f979b01d90b18ce6aa5703
-R 6e8dc804e858af71ad2b8ec1737b4689
-U drh
-Z 5dfea499ba027d378d355b994a19e393
+P 113470772b04210b9300803124c06af2c0b33278313b311fefe6f80594b24e4a
+R a3d7a4858f784ab2683858753b638ca6
+U mistachkin
+Z 94edf6faf2cefb2eca29e53557ee84d1
diff --git a/manifest.uuid b/manifest.uuid
index 3895727b02..3bb72446e5 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-113470772b04210b9300803124c06af2c0b33278313b311fefe6f80594b24e4a
\ No newline at end of file
+95958b60f9282384cac8f29c519d0fa1c32d7c0366c01ce681f6691e5bbf8438
\ No newline at end of file
diff --git a/src/date.c b/src/date.c
index c0ca4c9904..313c7f9137 100644
--- a/src/date.c
+++ b/src/date.c
@@ -39,7 +39,7 @@
 **
 **      Jean Meeus
 **      Astronomical Algorithms, 2nd Edition, 1998
-**      ISBM 0-943396-61-1
+**      ISBN 0-943396-61-1
 **      Willmann-Bell, Inc
 **      Richmond, Virginia (USA)
 */

From b7571b4ecb705a8728717249b25022d4d60bb081 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 7 Dec 2017 22:10:16 +0000
Subject: [PATCH 128/488] When doing a table scan using an index, do not error
 out if collating functions used by that index are unavailable, since they
 will not be used.

FossilOrigin-Name: bbd69fa6fa7f3c86fb5cd1b69e1abbe1bbad61aa281e6c073a402d1b202e42ec
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/where.c   |  9 +++++++++
 3 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index d31e28b549..bdfc8161a8 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\stypo\sin\scomment.\s\sNo\schanges\sto\scode.
-D 2017-12-07T22:04:53.501
+C When\sdoing\sa\stable\sscan\susing\san\sindex,\sdo\snot\serror\sout\sif\scollating\nfunctions\sused\sby\sthat\sindex\sare\sunavailable,\ssince\sthey\swill\snot\sbe\sused.
+D 2017-12-07T22:10:16.830
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
@@ -554,7 +554,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 5a3f464edd64596f601683ed321d12e6fd93c5fb9afdfb3653d6ffd0fee9c48f
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f
-F src/where.c 9752b68e03e2044f0faa4708fabb0189769067b660bffa931e1fd65736269659
+F src/where.c ea27cbc78ccc8c1f054adc2ebced57a883620b77605699ccf6e865ea871bf17b
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
 F src/wherecode.c 611fcabd05592ed2febd7d182f9621425b0466c5232d70e0981c842d429356d5
 F src/whereexpr.c 427ea8e96ec24f2a7814c67b8024ad664a9c7656264c4566c34743cb23186e46
@@ -1679,7 +1679,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 113470772b04210b9300803124c06af2c0b33278313b311fefe6f80594b24e4a
-R a3d7a4858f784ab2683858753b638ca6
-U mistachkin
-Z 94edf6faf2cefb2eca29e53557ee84d1
+P 95958b60f9282384cac8f29c519d0fa1c32d7c0366c01ce681f6691e5bbf8438
+R bf4e2e41467697978aeb7f6536c24c13
+U drh
+Z c50a2b3f457141d0934c9b148b713f31
diff --git a/manifest.uuid b/manifest.uuid
index 3bb72446e5..7dd4d26d71 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-95958b60f9282384cac8f29c519d0fa1c32d7c0366c01ce681f6691e5bbf8438
\ No newline at end of file
+bbd69fa6fa7f3c86fb5cd1b69e1abbe1bbad61aa281e6c073a402d1b202e42ec
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index 27979efbc4..e11194bdd4 100644
--- a/src/where.c
+++ b/src/where.c
@@ -4870,7 +4870,16 @@ WhereInfo *sqlite3WhereBegin(
       assert( iIndexCur>=0 );
       if( op ){
         sqlite3VdbeAddOp3(v, op, iIndexCur, pIx->tnum, iDb);
+
+        /* If the index is only being scanned - if there is no searching -
+        ** then no collating functions are required.  Set db->init.busy in that
+        ** case, to prevent sqlite3VdbeSetP4KeyInfo() from raising needless
+        ** errors about the missing collating functions. */
+        assert( db->init.busy==0 );
+        db->init.busy = (pLoop->wsFlags & ~(WHERE_IDX_ONLY|WHERE_INDEXED))==0;
         sqlite3VdbeSetP4KeyInfo(pParse, pIx);
+        db->init.busy = 0;  /* Restore db->init.busy */
+
         if( (pLoop->wsFlags & WHERE_CONSTRAINT)!=0
          && (pLoop->wsFlags & (WHERE_COLUMN_RANGE|WHERE_SKIPSCAN))==0
          && (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0

From ee15962dd049b1f0113453525143fd1d32b12830 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 8 Dec 2017 14:07:14 +0000
Subject: [PATCH 129/488] Make sure the bComplex variable in
 sqlite3DeleteFrom() is initialized when compiling with -DSQLITE_OMIT_TRIGGER.

FossilOrigin-Name: e526d0c40b971d970367a52a57d3920cb64e6a98920114bfac46ba009f0b1eb5
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/delete.c  |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index bdfc8161a8..4dc605677d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C When\sdoing\sa\stable\sscan\susing\san\sindex,\sdo\snot\serror\sout\sif\scollating\nfunctions\sused\sby\sthat\sindex\sare\sunavailable,\ssince\sthey\swill\snot\sbe\sused.
-D 2017-12-07T22:10:16.830
+C Make\ssure\sthe\sbComplex\svariable\sin\ssqlite3DeleteFrom()\sis\sinitialized\swhen\ncompiling\swith\s-DSQLITE_OMIT_TRIGGER.
+D 2017-12-08T14:07:14.466
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
@@ -423,7 +423,7 @@ F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
 F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957
 F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
-F src/delete.c e6a70fb58f6628f0ffc6d7221a6702c0d7b342c82520385b3996b364c22e0cb3
+F src/delete.c 74667ad914ac143731a444a1bacf29ceb18f6eded8a0dd17aafae80baa07f8bb
 F src/expr.c fe11b91bb65b869143bd42023427c4429778ae42c0a0db7762f68f75b347a958
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331
@@ -1679,7 +1679,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 95958b60f9282384cac8f29c519d0fa1c32d7c0366c01ce681f6691e5bbf8438
-R bf4e2e41467697978aeb7f6536c24c13
+P bbd69fa6fa7f3c86fb5cd1b69e1abbe1bbad61aa281e6c073a402d1b202e42ec
+R ca151348414bf89661535a75ec0c12be
 U drh
-Z c50a2b3f457141d0934c9b148b713f31
+Z 7fa850481121f6987d8c1258ea8eefc5
diff --git a/manifest.uuid b/manifest.uuid
index 7dd4d26d71..ac6bcf5c02 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-bbd69fa6fa7f3c86fb5cd1b69e1abbe1bbad61aa281e6c073a402d1b202e42ec
\ No newline at end of file
+e526d0c40b971d970367a52a57d3920cb64e6a98920114bfac46ba009f0b1eb5
\ No newline at end of file
diff --git a/src/delete.c b/src/delete.c
index 8e25b36f5a..5808ac51d4 100644
--- a/src/delete.c
+++ b/src/delete.c
@@ -283,11 +283,11 @@ void sqlite3DeleteFrom(
 #ifndef SQLITE_OMIT_TRIGGER
   pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0);
   isView = pTab->pSelect!=0;
-  bComplex = pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0);
 #else
 # define pTrigger 0
 # define isView 0
 #endif
+  bComplex = pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0);
 #ifdef SQLITE_OMIT_VIEW
 # undef isView
 # define isView 0

From 21540ae47984b44a70dbc653e2edca27c3ee4735 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 8 Dec 2017 16:23:38 +0000
Subject: [PATCH 130/488] Add compile time switch SQLITE_ENABLE_ICU_COLLATIONS.
 For enabling ICU collations without also enabling the tokenizer, the LIKE
 operator, the REGEXP operator, or the unicode aware upper()/lower() scalar
 functions.

FossilOrigin-Name: a079f914522d7bc4b3d27d70114eb09adedfac936a64883e6ed8d382c428dd0e
---
 ext/icu/icu.c     |  47 ++++++++++--------
 manifest          |  20 ++++----
 manifest.uuid     |   2 +-
 src/main.c        |   4 +-
 src/test_config.c |   6 +++
 test/icu.test     | 120 ++++++++++++++++++++++++----------------------
 6 files changed, 108 insertions(+), 91 deletions(-)

diff --git a/ext/icu/icu.c b/ext/icu/icu.c
index 33c4aa76f3..13524ebc2a 100644
--- a/ext/icu/icu.c
+++ b/ext/icu/icu.c
@@ -28,7 +28,9 @@
 **     provide case-independent matching.
 */
 
-#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU)
+#if !defined(SQLITE_CORE)                  \
+ || defined(SQLITE_ENABLE_ICU)             \
+ || defined(SQLITE_ENABLE_ICU_COLLATIONS)
 
 /* Include ICU headers */
 #include <unicode/utypes.h>
@@ -45,6 +47,26 @@
   #include "sqlite3.h"
 #endif
 
+/*
+** This function is called when an ICU function called from within
+** the implementation of an SQL scalar function returns an error.
+**
+** The scalar function context passed as the first argument is 
+** loaded with an error message based on the following two args.
+*/
+static void icuFunctionError(
+  sqlite3_context *pCtx,       /* SQLite scalar function context */
+  const char *zName,           /* Name of ICU function that failed */
+  UErrorCode e                 /* Error code returned by ICU function */
+){
+  char zBuf[128];
+  sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e));
+  zBuf[127] = '\0';
+  sqlite3_result_error(pCtx, zBuf, -1);
+}
+
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU)
+
 /*
 ** Maximum length (in bytes) of the pattern in a LIKE or GLOB
 ** operator.
@@ -224,24 +246,6 @@ static void icuLikeFunc(
   }
 }
 
-/*
-** This function is called when an ICU function called from within
-** the implementation of an SQL scalar function returns an error.
-**
-** The scalar function context passed as the first argument is 
-** loaded with an error message based on the following two args.
-*/
-static void icuFunctionError(
-  sqlite3_context *pCtx,       /* SQLite scalar function context */
-  const char *zName,           /* Name of ICU function that failed */
-  UErrorCode e                 /* Error code returned by ICU function */
-){
-  char zBuf[128];
-  sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e));
-  zBuf[127] = '\0';
-  sqlite3_result_error(pCtx, zBuf, -1);
-}
-
 /*
 ** Function to delete compiled regexp objects. Registered as
 ** a destructor function with sqlite3_set_auxdata().
@@ -407,6 +411,8 @@ static void icuCaseFunc16(sqlite3_context *p, int nArg, sqlite3_value **apArg){
   assert( 0 );     /* Unreachable */
 }
 
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) */
+
 /*
 ** Collation sequence destructor function. The pCtx argument points to
 ** a UCollator structure previously allocated using ucol_open().
@@ -501,6 +507,7 @@ int sqlite3IcuInit(sqlite3 *db){
     void (*xFunc)(sqlite3_context*,int,sqlite3_value**);
   } scalars[] = {
     {"icu_load_collation",  2, SQLITE_UTF8,                1, icuLoadCollation},
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU)
     {"regexp", 2, SQLITE_ANY|SQLITE_DETERMINISTIC,         0, icuRegexpFunc},
     {"lower",  1, SQLITE_UTF16|SQLITE_DETERMINISTIC,       0, icuCaseFunc16},
     {"lower",  2, SQLITE_UTF16|SQLITE_DETERMINISTIC,       0, icuCaseFunc16},
@@ -512,10 +519,10 @@ int sqlite3IcuInit(sqlite3 *db){
     {"upper",  2, SQLITE_UTF8|SQLITE_DETERMINISTIC,        1, icuCaseFunc16},
     {"like",   2, SQLITE_UTF8|SQLITE_DETERMINISTIC,        0, icuLikeFunc},
     {"like",   3, SQLITE_UTF8|SQLITE_DETERMINISTIC,        0, icuLikeFunc},
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) */
   };
   int rc = SQLITE_OK;
   int i;
-
   
   for(i=0; rc==SQLITE_OK && i<(int)(sizeof(scalars)/sizeof(scalars[0])); i++){
     const struct IcuScalar *p = &scalars[i];
diff --git a/manifest b/manifest
index 4dc605677d..7872cf0b11 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Make\ssure\sthe\sbComplex\svariable\sin\ssqlite3DeleteFrom()\sis\sinitialized\swhen\ncompiling\swith\s-DSQLITE_OMIT_TRIGGER.
-D 2017-12-08T14:07:14.466
+C Add\scompile\stime\sswitch\sSQLITE_ENABLE_ICU_COLLATIONS.\sFor\senabling\sICU\ncollations\swithout\salso\senabling\sthe\stokenizer,\sthe\sLIKE\soperator,\sthe\nREGEXP\soperator,\sor\sthe\sunicode\saware\supper()/lower()\sscalar\sfunctions.
+D 2017-12-08T16:23:38.347
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
@@ -210,7 +210,7 @@ F ext/fts5/tool/loadfts5.tcl 95b03429ee6b138645703c6ca192c3ac96eaf093
 F ext/fts5/tool/mkfts5c.tcl d1c2a9ab8e0ec690a52316f33dd9b1d379942f45
 F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c
 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43
-F ext/icu/icu.c 635775226d07c743c770888a9dd5175afc6e67d3e28a4032b7fedc3bcaa92e65
+F ext/icu/icu.c c2c7592574c08cd1270d909b8fb8797f6ea1f49e931e71dbcc25506b9b224580
 F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37
 F ext/lsm1/Makefile 98b0a24b45e248283d6bea4b6cb3e58d7b394edd8e96a0ac28c5fa5104813bad
 F ext/lsm1/Makefile.msc f8c878b467232226de288da320e1ac71c131f5ec91e08b21f502303347260013
@@ -436,7 +436,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c cb67cc56ef2ddd13e6944b2c0dd08a920bcd9503230adef8b9928d338097c722
 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e
 F src/loadext.c 20865b183bb8a3723d59cf1efffc3c50217eb452c1021d077b908c94da26b0b2
-F src/main.c 6a0cc1c7b8ab92374effecdd7b92792b3273a255c70575b7d67bd9a4315e6d3a
+F src/main.c 4f94536a61dc77477e1cee7ecfae2896778c1a3d3de4f978db28f8b9220f6e52
 F src/malloc.c a02c9e69bc76bee0f639416b947a946412890b606301454727feadcb313536d6
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -496,7 +496,7 @@ F src/test_backup.c bf5da90c9926df0a4b941f2d92825a01bbe090a0
 F src/test_bestindex.c 78809f11026f18a93fcfd798d9479cba37e1201c830260bf1edc674b2fa9b857
 F src/test_blob.c ae4a0620b478548afb67963095a7417cd06a4ec0a56adb453542203bfdcb31ce
 F src/test_btree.c 8b2dc8b8848cf3a4db93f11578f075e82252a274
-F src/test_config.c 2dad654eb81e90160f764f485bf7248e87b6d251e484c959de1aa04935acae8e
+F src/test_config.c 3904a8682aac58b77d20ca236face2b11e50781be4116004ba1ba79f69896ec9
 F src/test_delete.c e2fe07646dff6300b48d49b2fee2fe192ed389e834dd635e3b3bac0ce0bf9f8f
 F src/test_demovfs.c a0c3bdd45ed044115c2c9f7779e56eafff18741e
 F src/test_devsym.c 1960abbb234b97e9b920f07e99503fc04b443f62bbc3c6ff2c2cea2133e3b8a2
@@ -932,7 +932,7 @@ F test/hexlit.test 4a6a5f46e3c65c4bf1fa06f5dd5a9507a5627751
 F test/hidden.test 23c1393a79e846d68fd902d72c85d5e5dcf98711
 F test/hook.test dbc0b87756e1e20e7497b56889c9e9cd2f8cc2b5
 F test/hook2.test b9ff3b8c6519fb67f33192f1afe86e7782ee4ac8
-F test/icu.test 73956798bace8982909c00476b216714a6d0559a
+F test/icu.test 7fb00edc09e05d51e36be12b33e0af04e3394e3b02dbdcb2eefcb901203e28c4
 F test/ieee754.test 806fc0ce7f305f57e3331eaceeddcfec9339e607
 F test/imposter1.test c3f1db2d3db2c24611a6596a3fc0ffc14f1466c8
 F test/in.test 2fa2dfba1afe30eb830f327e7131dfadaa7a701d677de0eb65f9303d99e18fe0
@@ -1679,7 +1679,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P bbd69fa6fa7f3c86fb5cd1b69e1abbe1bbad61aa281e6c073a402d1b202e42ec
-R ca151348414bf89661535a75ec0c12be
-U drh
-Z 7fa850481121f6987d8c1258ea8eefc5
+P e526d0c40b971d970367a52a57d3920cb64e6a98920114bfac46ba009f0b1eb5
+R e5fd041d6b7727f71627ca57ebea4f83
+U dan
+Z b6d953d578629eaa29d8e731d0cdeaa8
diff --git a/manifest.uuid b/manifest.uuid
index ac6bcf5c02..8ccd338cb9 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e526d0c40b971d970367a52a57d3920cb64e6a98920114bfac46ba009f0b1eb5
\ No newline at end of file
+a079f914522d7bc4b3d27d70114eb09adedfac936a64883e6ed8d382c428dd0e
\ No newline at end of file
diff --git a/src/main.c b/src/main.c
index a7b631270e..c25d3a4480 100644
--- a/src/main.c
+++ b/src/main.c
@@ -22,7 +22,7 @@
 #ifdef SQLITE_ENABLE_RTREE
 # include "rtree.h"
 #endif
-#ifdef SQLITE_ENABLE_ICU
+#if defined(SQLITE_ENABLE_ICU) || defined(SQLITE_ENABLE_ICU_COLLATIONS)
 # include "sqliteicu.h"
 #endif
 #ifdef SQLITE_ENABLE_JSON1
@@ -3050,7 +3050,7 @@ static int openDatabase(
   }
 #endif
 
-#ifdef SQLITE_ENABLE_ICU
+#if defined(SQLITE_ENABLE_ICU) || defined(SQLITE_ENABLE_ICU_COLLATIONS)
   if( !db->mallocFailed && rc==SQLITE_OK ){
     rc = sqlite3IcuInit(db);
   }
diff --git a/src/test_config.c b/src/test_config.c
index 8756876d19..24a7287da2 100644
--- a/src/test_config.c
+++ b/src/test_config.c
@@ -429,6 +429,12 @@ static void set_options(Tcl_Interp *interp){
   Tcl_SetVar2(interp, "sqlite_options", "icu", "0", TCL_GLOBAL_ONLY);
 #endif
 
+#ifdef SQLITE_ENABLE_ICU_COLLATIONS
+  Tcl_SetVar2(interp, "sqlite_options", "icu_collations", "1", TCL_GLOBAL_ONLY);
+#else
+  Tcl_SetVar2(interp, "sqlite_options", "icu_collations", "0", TCL_GLOBAL_ONLY);
+#endif
+
 #ifdef SQLITE_OMIT_INCRBLOB
   Tcl_SetVar2(interp, "sqlite_options", "incrblob", "0", TCL_GLOBAL_ONLY);
 #else
diff --git a/test/icu.test b/test/icu.test
index 743bcfaea1..b6d3b7f847 100644
--- a/test/icu.test
+++ b/test/icu.test
@@ -15,7 +15,7 @@
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 
-ifcapable !icu {
+ifcapable !icu&&!icu_collations {
   finish_test
   return
 }
@@ -35,54 +35,57 @@ proc test_expr {name settings expr result} {
   } $settings $expr] $result
 }
 
-# Tests of the REGEXP operator.
-#
-test_expr icu-1.1 {i1='hello'} {i1 REGEXP 'hello'}  1
-test_expr icu-1.2 {i1='hello'} {i1 REGEXP '.ello'}  1
-test_expr icu-1.3 {i1='hello'} {i1 REGEXP '.ell'}   0
-test_expr icu-1.4 {i1='hello'} {i1 REGEXP '.ell.*'} 1
-test_expr icu-1.5 {i1=NULL}    {i1 REGEXP '.ell.*'} {}
+ifcapable icu {
 
-# Some non-ascii characters with defined case mappings
-#
-set ::EGRAVE "\xC8"
-set ::egrave "\xE8"
+  # Tests of the REGEXP operator.
+  #
+  test_expr icu-1.1 {i1='hello'} {i1 REGEXP 'hello'}  1
+  test_expr icu-1.2 {i1='hello'} {i1 REGEXP '.ello'}  1
+  test_expr icu-1.3 {i1='hello'} {i1 REGEXP '.ell'}   0
+  test_expr icu-1.4 {i1='hello'} {i1 REGEXP '.ell.*'} 1
+  test_expr icu-1.5 {i1=NULL}    {i1 REGEXP '.ell.*'} {}
 
-set ::OGRAVE "\xD2"
-set ::ograve "\xF2"
+  # Some non-ascii characters with defined case mappings
+  #
+  set ::EGRAVE "\xC8"
+  set ::egrave "\xE8"
 
-# That German letter that looks a bit like a B. The
-# upper-case version of which is "SS" (two characters).
-#
-set ::szlig "\xDF" 
+  set ::OGRAVE "\xD2"
+  set ::ograve "\xF2"
 
-# Tests of the upper()/lower() functions.
-#
-test_expr icu-2.1 {i1='HellO WorlD'} {upper(i1)} {HELLO WORLD}
-test_expr icu-2.2 {i1='HellO WorlD'} {lower(i1)} {hello world}
-test_expr icu-2.3 {i1=$::egrave} {lower(i1)}     $::egrave
-test_expr icu-2.4 {i1=$::egrave} {upper(i1)}     $::EGRAVE
-test_expr icu-2.5 {i1=$::ograve} {lower(i1)}     $::ograve
-test_expr icu-2.6 {i1=$::ograve} {upper(i1)}     $::OGRAVE
-test_expr icu-2.3 {i1=$::EGRAVE} {lower(i1)}     $::egrave
-test_expr icu-2.4 {i1=$::EGRAVE} {upper(i1)}     $::EGRAVE
-test_expr icu-2.5 {i1=$::OGRAVE} {lower(i1)}     $::ograve
-test_expr icu-2.6 {i1=$::OGRAVE} {upper(i1)}     $::OGRAVE
+  # That German letter that looks a bit like a B. The
+  # upper-case version of which is "SS" (two characters).
+  #
+  set ::szlig "\xDF" 
 
-test_expr icu-2.7 {i1=$::szlig} {upper(i1)}      "SS"
-test_expr icu-2.8 {i1='SS'} {lower(i1)}          "ss"
+  # Tests of the upper()/lower() functions.
+  #
+  test_expr icu-2.1 {i1='HellO WorlD'} {upper(i1)} {HELLO WORLD}
+  test_expr icu-2.2 {i1='HellO WorlD'} {lower(i1)} {hello world}
+  test_expr icu-2.3 {i1=$::egrave} {lower(i1)}     $::egrave
+  test_expr icu-2.4 {i1=$::egrave} {upper(i1)}     $::EGRAVE
+  test_expr icu-2.5 {i1=$::ograve} {lower(i1)}     $::ograve
+  test_expr icu-2.6 {i1=$::ograve} {upper(i1)}     $::OGRAVE
+  test_expr icu-2.3 {i1=$::EGRAVE} {lower(i1)}     $::egrave
+  test_expr icu-2.4 {i1=$::EGRAVE} {upper(i1)}     $::EGRAVE
+  test_expr icu-2.5 {i1=$::OGRAVE} {lower(i1)}     $::ograve
+  test_expr icu-2.6 {i1=$::OGRAVE} {upper(i1)}     $::OGRAVE
 
-do_execsql_test icu-2.9 {
-  SELECT upper(char(0xfb04,0xfb04,0xfb04,0xfb04));
-} {FFLFFLFFLFFL}
+  test_expr icu-2.7 {i1=$::szlig} {upper(i1)}      "SS"
+  test_expr icu-2.8 {i1='SS'} {lower(i1)}          "ss"
 
-# In turkish (locale="tr_TR"), the lower case version of I
-# is "small dotless i" (code point 0x131 (decimal 305)).
-#
-set ::small_dotless_i "\u0131"
-test_expr icu-3.1 {i1='I'} {lower(i1)}           "i"
-test_expr icu-3.2 {i1='I'} {lower(i1, 'tr_tr')}  $::small_dotless_i
-test_expr icu-3.3 {i1='I'} {lower(i1, 'en_AU')}  "i"
+  do_execsql_test icu-2.9 {
+    SELECT upper(char(0xfb04,0xfb04,0xfb04,0xfb04));
+  } {FFLFFLFFLFFL}
+
+  # In turkish (locale="tr_TR"), the lower case version of I
+  # is "small dotless i" (code point 0x131 (decimal 305)).
+  #
+  set ::small_dotless_i "\u0131"
+  test_expr icu-3.1 {i1='I'} {lower(i1)}           "i"
+  test_expr icu-3.2 {i1='I'} {lower(i1, 'tr_tr')}  $::small_dotless_i
+  test_expr icu-3.3 {i1='I'} {lower(i1, 'en_AU')}  "i"
+}
 
 #--------------------------------------------------------------------
 # Test the collation sequence function.
@@ -124,22 +127,23 @@ do_test icu-4.3 {
 #
 #   http://src.chromium.org/viewvc/chrome/trunk/src/third_party/sqlite/icu-regexp.patch?revision=34807&view=markup
 #
-do_catchsql_test icu-5.1 { SELECT regexp('a[abc]c.*', 'abc') } {0 1}
-do_catchsql_test icu-5.2 { 
-  SELECT regexp('a[abc]c.*') 
-} {1 {wrong number of arguments to function regexp()}}
-do_catchsql_test icu-5.3 { 
-  SELECT regexp('a[abc]c.*', 'abc', 'c') 
-} {1 {wrong number of arguments to function regexp()}}
-do_catchsql_test icu-5.4 { 
-  SELECT 'abc' REGEXP 'a[abc]c.*'
-} {0 1}
-do_catchsql_test icu-5.4 { SELECT 'abc' REGEXP }    {1 {near " ": syntax error}}
-do_catchsql_test icu-5.5 { SELECT 'abc' REGEXP, 1 } {1 {near ",": syntax error}}
-
-
-do_malloc_test icu-6.10 -sqlbody {
-  SELECT upper(char(0xfb04,0xdf,0xfb04,0xe8,0xfb04));
+ifcapable icu {
+  do_catchsql_test icu-5.1 { SELECT regexp('a[abc]c.*', 'abc') } {0 1}
+  do_catchsql_test icu-5.2 { 
+    SELECT regexp('a[abc]c.*') 
+  } {1 {wrong number of arguments to function regexp()}}
+  do_catchsql_test icu-5.3 { 
+    SELECT regexp('a[abc]c.*', 'abc', 'c') 
+  } {1 {wrong number of arguments to function regexp()}}
+  do_catchsql_test icu-5.4 { 
+    SELECT 'abc' REGEXP 'a[abc]c.*'
+  } {0 1}
+  do_catchsql_test icu-5.4 {SELECT 'abc' REGEXP }   {1 {near " ": syntax error}}
+  do_catchsql_test icu-5.5 {SELECT 'abc' REGEXP, 1} {1 {near ",": syntax error}}
+ 
+  do_malloc_test icu-6.10 -sqlbody {
+    SELECT upper(char(0xfb04,0xdf,0xfb04,0xe8,0xfb04));
+  }
 }
 
 finish_test

From 7e8515d8bec405f8d17a91a94c9d0d690b0a9ff5 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 8 Dec 2017 19:37:04 +0000
Subject: [PATCH 131/488] The query planner tries to avoids using indexes that
 use unknown collating functions.

FossilOrigin-Name: 02013fc120bf71a8be3550c696a588af8c92f2209f8e5db530624878ddc8aa7e
---
 manifest        | 24 ++++++++++++------------
 manifest.uuid   |  2 +-
 src/build.c     | 12 ++++++++++++
 src/callback.c  |  1 +
 src/prepare.c   | 18 ++++++++++--------
 src/sqlite.h.in |  2 ++
 src/sqliteInt.h |  3 ++-
 src/where.c     | 10 +---------
 8 files changed, 41 insertions(+), 31 deletions(-)

diff --git a/manifest b/manifest
index 7872cf0b11..d824fdaf09 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\scompile\stime\sswitch\sSQLITE_ENABLE_ICU_COLLATIONS.\sFor\senabling\sICU\ncollations\swithout\salso\senabling\sthe\stokenizer,\sthe\sLIKE\soperator,\sthe\nREGEXP\soperator,\sor\sthe\sunicode\saware\supper()/lower()\sscalar\sfunctions.
-D 2017-12-08T16:23:38.347
+C The\squery\splanner\stries\sto\savoids\susing\sindexes\sthat\suse\sunknown\scollating\nfunctions.
+D 2017-12-08T19:37:04.216
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
@@ -416,8 +416,8 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
 F src/btree.c b83a6b03f160528020bb965f0c3a40af5286cd4923c3870fd218177f03a120a7
 F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09
 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc
-F src/build.c 514db9d494ed29155e552f2ec2fa7c55c0241f847c683156b7c017f4b0bad9fa
-F src/callback.c 28a8ede982fde4129b828350f78f2c01fe7d12c74d1a0a05d7108ab36f308688
+F src/build.c 87b68e3b45559ec404b12f095f0ba5f06f91a6dd2d21bd8443e41d8ac2e67196
+F src/callback.c fe677cb5f5abb02f7a772a62a98c2f516426081df68856e8f2d5f950929b966a
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
 F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957
@@ -466,17 +466,17 @@ F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170
 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
 F src/pragma.c d04725ac25387d9638919e197fb009f378e13af7bf899516979e54b3164e3602
 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
-F src/prepare.c 7cf451f903ad92a14e22de415a13e7a7d30f1bd23b3d21eeb0dc7264723244c5
+F src/prepare.c 259f4e7960c47082c9653f3d5f0c294abd68bb9c3aab86de7630700cba1c20fb
 F src/printf.c 9506b4b96e59c0467047155f09015750cb2878aeda3d39e5610c1192ddc3c41c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
 F src/shell.c.in 6ffed0c589f5aff180789a8c8abf5b2d3e2eea7470c86b30e797887cb0c9d0e5
-F src/sqlite.h.in 8fd97993d48b50b9bade38c52f12d175942c9497c960905610c7b03a3e4b5818
+F src/sqlite.h.in 4622dbb3bb94119795fd926c2a10cacfb86d59900fa9b037e8678acd9830d147
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
-F src/sqliteInt.h 4c910d9c0d88a90e8639a4f83ef05f701ccfe731cf593e757444074f01df4964
+F src/sqliteInt.h 55b8e7da85947eb61b13d4d2523ccdda7800a13e987c3fc4ca73d8518bbf02fa
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -554,7 +554,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 5a3f464edd64596f601683ed321d12e6fd93c5fb9afdfb3653d6ffd0fee9c48f
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f
-F src/where.c ea27cbc78ccc8c1f054adc2ebced57a883620b77605699ccf6e865ea871bf17b
+F src/where.c ee9dd4a438a07cd364c8449e834db4c4d6163a2576a69e937d7a4c37685612a2
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
 F src/wherecode.c 611fcabd05592ed2febd7d182f9621425b0466c5232d70e0981c842d429356d5
 F src/whereexpr.c 427ea8e96ec24f2a7814c67b8024ad664a9c7656264c4566c34743cb23186e46
@@ -1679,7 +1679,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e526d0c40b971d970367a52a57d3920cb64e6a98920114bfac46ba009f0b1eb5
-R e5fd041d6b7727f71627ca57ebea4f83
-U dan
-Z b6d953d578629eaa29d8e731d0cdeaa8
+P a079f914522d7bc4b3d27d70114eb09adedfac936a64883e6ed8d382c428dd0e
+R 4ebf4200d512dd6155db0a93b95e863f
+U drh
+Z 0f53fc2b441082d70fbf7ea067c767d4
diff --git a/manifest.uuid b/manifest.uuid
index 8ccd338cb9..16a4e71f56 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a079f914522d7bc4b3d27d70114eb09adedfac936a64883e6ed8d382c428dd0e
\ No newline at end of file
+02013fc120bf71a8be3550c696a588af8c92f2209f8e5db530624878ddc8aa7e
\ No newline at end of file
diff --git a/src/build.c b/src/build.c
index fc4dbcdba8..9582f136c8 100644
--- a/src/build.c
+++ b/src/build.c
@@ -4364,6 +4364,18 @@ KeyInfo *sqlite3KeyInfoOfIndex(Parse *pParse, Index *pIdx){
       pKey->aSortOrder[i] = pIdx->aSortOrder[i];
     }
     if( pParse->nErr ){
+      assert( pParse->rc==SQLITE_ERROR_MISSING_COLLSEQ );
+      if( pIdx->bNoQuery==0 ){
+        /* Deactivate the index because it contains an unknown collating
+        ** sequence.  The only way to reactive the index is to reload the
+        ** schema.  Adding the missing collating sequence later does not
+        ** reactive the index.  The application had the chance to register
+        ** the missing index using the collation-needed callback.  For
+        ** simplicity, SQLite will not give the application a second chance.
+        */
+        pIdx->bNoQuery = 1;
+        pParse->rc = SQLITE_ERROR_RETRY;
+      }
       sqlite3KeyInfoUnref(pKey);
       pKey = 0;
     }
diff --git a/src/callback.c b/src/callback.c
index e08924b2c2..0396df7a02 100644
--- a/src/callback.c
+++ b/src/callback.c
@@ -105,6 +105,7 @@ CollSeq *sqlite3GetCollSeq(
   assert( !p || p->xCmp );
   if( p==0 ){
     sqlite3ErrorMsg(pParse, "no such collation sequence: %s", zName);
+    pParse->rc = SQLITE_ERROR_MISSING_COLLSEQ;
   }
   return p;
 }
diff --git a/src/prepare.c b/src/prepare.c
index 9141cb8a78..65a4afcbbd 100644
--- a/src/prepare.c
+++ b/src/prepare.c
@@ -655,8 +655,6 @@ static int sqlite3Prepare(
 end_prepare:
 
   sqlite3ParserReset(&sParse);
-  rc = sqlite3ApiExit(db, rc);
-  assert( (rc&db->errMask)==rc );
   return rc;
 }
 static int sqlite3LockAndPrepare(
@@ -669,6 +667,7 @@ static int sqlite3LockAndPrepare(
   const char **pzTail       /* OUT: End of parsed string */
 ){
   int rc;
+  int cnt = 0;
 
 #ifdef SQLITE_ENABLE_API_ARMOR
   if( ppStmt==0 ) return SQLITE_MISUSE_BKPT;
@@ -679,15 +678,18 @@ static int sqlite3LockAndPrepare(
   }
   sqlite3_mutex_enter(db->mutex);
   sqlite3BtreeEnterAll(db);
-  rc = sqlite3Prepare(db, zSql, nBytes, prepFlags, pOld, ppStmt, pzTail);
-  if( rc==SQLITE_SCHEMA ){
-    sqlite3ResetOneSchema(db, -1);
-    sqlite3_finalize(*ppStmt);
+  do{
+    /* Make multiple attempts to compile the SQL, until it either succeeds
+    ** or encounters a permanent error.  A schema problem after one schema
+    ** reset is considered a permanent error. */
     rc = sqlite3Prepare(db, zSql, nBytes, prepFlags, pOld, ppStmt, pzTail);
-  }
+    assert( rc==SQLITE_OK || *ppStmt==0 );
+  }while( rc==SQLITE_ERROR_RETRY
+       || (rc==SQLITE_SCHEMA && (sqlite3ResetOneSchema(db,-1), cnt++)==0) );
   sqlite3BtreeLeaveAll(db);
+  rc = sqlite3ApiExit(db, rc);
+  assert( (rc&db->errMask)==rc );
   sqlite3_mutex_leave(db->mutex);
-  assert( rc==SQLITE_OK || *ppStmt==0 );
   return rc;
 }
 
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 4434064421..29b0458d76 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -470,6 +470,8 @@ int sqlite3_exec(
 ** the most recent error can be obtained using
 ** [sqlite3_extended_errcode()].
 */
+#define SQLITE_ERROR_MISSING_COLLSEQ   (SQLITE_ERROR | (1<<8))
+#define SQLITE_ERROR_RETRY             (SQLITE_ERROR | (2<<8))
 #define SQLITE_IOERR_READ              (SQLITE_IOERR | (1<<8))
 #define SQLITE_IOERR_SHORT_READ        (SQLITE_IOERR | (2<<8))
 #define SQLITE_IOERR_WRITE             (SQLITE_IOERR | (3<<8))
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 27b896d67c..47d9fc6953 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -2172,6 +2172,7 @@ struct Index {
   unsigned isCovering:1;   /* True if this is a covering index */
   unsigned noSkipScan:1;   /* Do not try to use skip-scan if true */
   unsigned hasStat1:1;     /* aiRowLogEst values come from sqlite_stat1 */
+  unsigned bNoQuery:1;     /* Do not use this index to optimize queries */
 #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
   int nSample;             /* Number of elements in aSample[] */
   int nSampleCol;          /* Size of IndexSample.anEq[] and so on */
@@ -2984,7 +2985,7 @@ struct Parse {
   int nMem;            /* Number of memory cells used so far */
   int nOpAlloc;        /* Number of slots allocated for Vdbe.aOp[] */
   int szOpAlloc;       /* Bytes of memory space allocated for Vdbe.aOp[] */
-  int iSelfTab;        /* Table for associated with an index on expr, or negative
+  int iSelfTab;        /* Table associated with an index on expr, or negative
                        ** of the base register during check-constraint eval */
   int iCacheLevel;     /* ColCache valid when aColCache[].iLevel<=iCacheLevel */
   int iCacheCnt;       /* Counter used to generate aColCache[].lru values */
diff --git a/src/where.c b/src/where.c
index e11194bdd4..5b77b76625 100644
--- a/src/where.c
+++ b/src/where.c
@@ -2879,6 +2879,7 @@ static int whereLoopAddBtree(
       testcase( pNew->iTab!=pSrc->iCursor );  /* See ticket [98d973b8f5] */
       continue;  /* Partial index inappropriate for this query */
     }
+    if( pProbe->bNoQuery ) continue;
     rSize = pProbe->aiRowLogEst[0];
     pNew->u.btree.nEq = 0;
     pNew->u.btree.nBtm = 0;
@@ -4870,16 +4871,7 @@ WhereInfo *sqlite3WhereBegin(
       assert( iIndexCur>=0 );
       if( op ){
         sqlite3VdbeAddOp3(v, op, iIndexCur, pIx->tnum, iDb);
-
-        /* If the index is only being scanned - if there is no searching -
-        ** then no collating functions are required.  Set db->init.busy in that
-        ** case, to prevent sqlite3VdbeSetP4KeyInfo() from raising needless
-        ** errors about the missing collating functions. */
-        assert( db->init.busy==0 );
-        db->init.busy = (pLoop->wsFlags & ~(WHERE_IDX_ONLY|WHERE_INDEXED))==0;
         sqlite3VdbeSetP4KeyInfo(pParse, pIx);
-        db->init.busy = 0;  /* Restore db->init.busy */
-
         if( (pLoop->wsFlags & WHERE_CONSTRAINT)!=0
          && (pLoop->wsFlags & (WHERE_COLUMN_RANGE|WHERE_SKIPSCAN))==0
          && (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0

From 62be1fab6aa8e04265fd47012dcad8cd0868cdcc Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 9 Dec 2017 01:02:33 +0000
Subject: [PATCH 132/488] Fix a harmless API signature mismatch in the unix
 VFS.

FossilOrigin-Name: bab9de7fdda20a724f7c21ec2c25b488ece08b685f1e4fd15c9e73b6b4a0133a
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/os_unix.c |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index d824fdaf09..bdd8ba7d82 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C The\squery\splanner\stries\sto\savoids\susing\sindexes\sthat\suse\sunknown\scollating\nfunctions.
-D 2017-12-08T19:37:04.216
+C Fix\sa\sharmless\sAPI\ssignature\smismatch\sin\sthe\sunix\sVFS.
+D 2017-12-09T01:02:33.171
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
@@ -455,7 +455,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104
 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
-F src/os_unix.c e87cef0bb894b94d96ee3af210be669549d111c580817d14818101b992640767
+F src/os_unix.c 888afdeab00cd5df5e1c2cefe26926127664d73416e60804576d661cb1c11f52
 F src/os_win.c 7f36120492e4a23c48d1dd685edf29ae459c6d555660c61f1323cea3e5a1191d
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903
@@ -1679,7 +1679,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a079f914522d7bc4b3d27d70114eb09adedfac936a64883e6ed8d382c428dd0e
-R 4ebf4200d512dd6155db0a93b95e863f
+P 02013fc120bf71a8be3550c696a588af8c92f2209f8e5db530624878ddc8aa7e
+R 70ec80a62b754f49e015c4fa7a36d748
 U drh
-Z 0f53fc2b441082d70fbf7ea067c767d4
+Z d8e2fb990ec2e3ae449b802623df526d
diff --git a/manifest.uuid b/manifest.uuid
index 16a4e71f56..ae71661621 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-02013fc120bf71a8be3550c696a588af8c92f2209f8e5db530624878ddc8aa7e
\ No newline at end of file
+bab9de7fdda20a724f7c21ec2c25b488ece08b685f1e4fd15c9e73b6b4a0133a
\ No newline at end of file
diff --git a/src/os_unix.c b/src/os_unix.c
index c5d9aca2c6..6ba4a5645a 100644
--- a/src/os_unix.c
+++ b/src/os_unix.c
@@ -483,7 +483,7 @@ static struct unix_syscall {
 #else
   { "munmap",       (sqlite3_syscall_ptr)0,               0 },
 #endif
-#define osMunmap ((void*(*)(void*,size_t))aSyscall[23].pCurrent)
+#define osMunmap ((int(*)(void*,size_t))aSyscall[23].pCurrent)
 
 #if HAVE_MREMAP && (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0)
   { "mremap",       (sqlite3_syscall_ptr)mremap,          0 },

From 88be0209168e1625ef5a5b0ef7f33f14e8895b1b Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 9 Dec 2017 17:58:02 +0000
Subject: [PATCH 133/488] Improve parsing of ".ar" commands. Add new test file
 for the same.

FossilOrigin-Name: 840401cc8ce3a09e0663b46973ecd2856d9607be71d2d1e9b21f7df7a82dcbe5
---
 manifest         |  13 ++--
 manifest.uuid    |   2 +-
 src/shell.c.in   | 177 +++++++++++++++++++++++++++++++++++++----------
 test/shell8.test |  83 ++++++++++++++++++++++
 4 files changed, 232 insertions(+), 43 deletions(-)
 create mode 100644 test/shell8.test

diff --git a/manifest b/manifest
index ab01f7195c..98adac2007 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\s".ar\sx"\scommand\sto\sthe\sshell.\sFor\sextracting\sthe\scontents\sof\ssqlar\narchives.
-D 2017-12-07T21:03:33.903
+C Improve\sparsing\sof\s".ar"\scommands.\sAdd\snew\stest\sfile\sfor\sthe\ssame.
+D 2017-12-09T17:58:02.648
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
@@ -474,7 +474,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
-F src/shell.c.in 2f9ae0bee09bdd35922ab7ed264d88e1d7fb34d39d37fc633e6a3a1af60036be
+F src/shell.c.in 907661eeab82949420270b24f5989a399242cb8721e6140f73b3a46939fc4820
 F src/sqlite.h.in 8fd97993d48b50b9bade38c52f12d175942c9497c960905610c7b03a3e4b5818
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1214,6 +1214,7 @@ F test/shell4.test 89ad573879a745974ff2df20ff97c5d6ffffbd5d
 F test/shell5.test 23939a4c51f0421330ea61dbd3c74f9c215f5f8d3d1a94846da6ffc777a35458
 F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3
 F test/shell7.test 115132f66d0463417f408562cc2cf534f6bbc6d83a6d50f0072a9eb171bae97f
+F test/shell8.test 98b1d7b218060e557b3a789f3396635a0c03873ea652b3154c7f3f238d4a1a8f
 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3
 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5
 F test/shrink.test 1b4330b1fd9e818c04726d45cb28db73087535ce
@@ -1681,7 +1682,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c9827a01a6e107f38f85c2b2c1c7a599e443067b106217e965b6936441ca619d
-R 4b2bb930e9456062f8914cba7a04cf63
+P 0cc699d14adfe8c7b7be50c180186562861806c47425c80c935bce43ee5c5c12
+R 0b3e6167ae82d64c1e021b537a83d40a
 U dan
-Z 4726613d4458219c808274490c55a044
+Z 9b95380c27ad603c463b4469d523a6d2
diff --git a/manifest.uuid b/manifest.uuid
index 245ee49a8f..d1e1c772f3 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0cc699d14adfe8c7b7be50c180186562861806c47425c80c935bce43ee5c5c12
\ No newline at end of file
+840401cc8ce3a09e0663b46973ecd2856d9607be71d2d1e9b21f7df7a82dcbe5
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index fe58460fe1..b8471fea9f 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -4116,10 +4116,124 @@ static void shellReset(
   if( *pRc==SQLITE_OK ) *pRc = rc;
 }
 
+/* 
+** Structure representing a single ".ar" command.
+*/
+typedef struct ArCommand ArCommand;
+struct ArCommand {
+  int eCmd;                       /* An AR_CMD_* value */
+  const char *zFile;              /* --file argument, or NULL */
+  const char *zDir;               /* --directory argument, or NULL */
+  int bVerbose;                   /* True if --verbose */
+  int nArg;                       /* Number of command arguments */
+  char **azArg;                   /* Array of command arguments */
+};
+
+/*
+** Print a usage message for the .ar command to stderr and return SQLITE_ERROR.
+*/
+static int arUsage(void){
+  /* todo */
+  raw_printf(stderr, "error in .ar command line\n");
+  return SQLITE_ERROR;
+}
+
+/*
+** Values for ArCommand.eCmd.
+*/
+#define AR_CMD_CREATE  1
+#define AR_CMD_EXTRACT 2
+#define AR_CMD_LIST    3
+#define AR_CMD_UPDATE  4
+
+/*
+** Parse the command line for an ".ar" command. The results are written into
+** structure (*pAr). SQLITE_OK is returned if the command line is parsed
+** successfully, otherwise an error message is written to stderr and 
+** SQLITE_ERROR returned.
+*/
+static int arParseCommand(
+  char **azArg,                   /* Array of arguments passed to dot command */
+  int nArg,                       /* Number of entries in azArg[] */
+  ArCommand *pAr                  /* Populate this object */
+){
+  if( nArg<=1 ){
+    return arUsage();
+  }else{
+    char *z = azArg[1];
+    memset(pAr, 0, sizeof(ArCommand));
+
+    if( z[0]!='-' ){
+      /* Traditional style [tar] invocation */
+      int i;
+      int iArg = 2;
+      for(i=0; z[i]; i++){
+        switch( z[i] ){
+          case 'c':
+            if( pAr->eCmd ) return arUsage();
+            pAr->eCmd = AR_CMD_CREATE;
+            break;
+          case 'x':
+            if( pAr->eCmd ) return arUsage();
+            pAr->eCmd = AR_CMD_EXTRACT;
+            break;
+          case 't':
+            if( pAr->eCmd ) return arUsage();
+            pAr->eCmd = AR_CMD_LIST;
+            break;
+          case 'u':
+            if( pAr->eCmd ) return arUsage();
+            pAr->eCmd = AR_CMD_UPDATE;
+            break;
+
+          case 'v':
+            pAr->bVerbose = 1;
+            break;
+          case 'f':
+            if( iArg>=nArg ) return arUsage();
+            pAr->zFile = azArg[iArg++];
+            break;
+          case 'C':
+            if( iArg>=nArg ) return arUsage();
+            pAr->zDir = azArg[iArg++];
+            break;
+
+          default:
+            return arUsage();
+        }
+      }
+
+      pAr->nArg = nArg-iArg;
+      if( pAr->nArg>0 ){
+        pAr->azArg = &azArg[iArg];
+      }
+    }
+  }
+
+  return SQLITE_OK;
+}
+
+/*
+** Implementation of .ar "Update" command. 
+*/
+static int arUpdateCmd(ShellState *p, ArCommand *pAr){
+  raw_printf(stderr, "todo...\n");
+  return SQLITE_OK;
+}
+
+/*
+** Implementation of .ar "lisT" command. 
+*/
+static int arListCommand(ShellState *p, ArCommand *pAr){
+  raw_printf(stderr, "todo...\n");
+  return SQLITE_OK;
+}
+
+
 /*
 ** Implementation of .ar "eXtract" command. 
 */
-static int arExtractCommand(ShellState *p, int bVerbose){
+static int arExtractCommand(ShellState *p, ArCommand *pAr){
   const char *zSql1 = 
     "SELECT name, writefile(name, "
     "CASE WHEN (data AND sz>=0 AND sz!=length(data)) THEN uncompress(data) "
@@ -4136,7 +4250,7 @@ static int arExtractCommand(ShellState *p, int bVerbose){
 
   shellPrepare(p, &rc, zSql1, &pSql);
   while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
-    if( bVerbose ){
+    if( pAr->bVerbose ){
       raw_printf(stdout, "%s\n", sqlite3_column_text(pSql, 0));
     }
   }
@@ -4167,9 +4281,7 @@ static int arExtractCommand(ShellState *p, int bVerbose){
 */
 static int arCreateCommand(
   ShellState *p,                  /* Shell state pointer */
-  char **azFile,                  /* Array of files to add to archive */
-  int nFile,                      /* Number of entries in azFile[] */
-  int bVerbose                    /* True to be verbose on stdout */
+  ArCommand *pAr                  /* Command arguments and options */
 ){
   const char *zSql = 
     "WITH f(n, m, t, d) AS ("
@@ -4204,8 +4316,8 @@ static int arCreateCommand(
   shellPrepare(p, &rc, zInsert, &pInsert);
   shellPrepare(p, &rc, zSql, &pStmt);
 
-  for(i=0; i<nFile && rc==SQLITE_OK; i++){
-    sqlite3_bind_text(pStmt, 1, azFile[i], -1, SQLITE_STATIC);
+  for(i=0; i<pAr->nArg && rc==SQLITE_OK; i++){
+    sqlite3_bind_text(pStmt, 1, pAr->azArg[i], -1, SQLITE_STATIC);
     sqlite3_bind_int(pStmt, 2, S_IFDIR);
     while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
       int sz;
@@ -4213,7 +4325,7 @@ static int arCreateCommand(
       int mode = sqlite3_column_int(pStmt, 1);
       unsigned int mtime = sqlite3_column_int(pStmt, 2);
 
-      if( bVerbose ){
+      if( pAr->bVerbose ){
         raw_printf(stdout, "%s\n", zName);
       }
 
@@ -4280,38 +4392,31 @@ static int arDotCommand(
   char **azArg,                   /* Array of arguments passed to dot command */
   int nArg                        /* Number of entries in azArg[] */
 ){
-  int bVerbose = 0;
-  char cmd = 0;
-  int i;
-  int n1;
-  if( nArg<=1 ) goto usage;
+  ArCommand cmd;
+  int rc;
+  rc = arParseCommand(azArg, nArg, &cmd);
+  if( rc==SQLITE_OK ){
+    switch( cmd.eCmd ){
+      case AR_CMD_CREATE:
+        rc = arCreateCommand(pState, &cmd);
+        break;
 
-  n1 = strlen(azArg[1]);
-  for(i=0; i<n1; i++){
-    char c = azArg[1][i];
-    if( c=='c' || c=='x' ){
-      if( cmd ) goto usage;
-      cmd = c;
-    }
-    else if( c=='v' ){
-      bVerbose = 1;
-    }else{
-      goto usage;
+      case AR_CMD_EXTRACT:
+        rc = arExtractCommand(pState, &cmd);
+        break;
+
+      case AR_CMD_LIST:
+        rc = arListCommand(pState, &cmd);
+        break;
+
+      default:
+        assert( cmd.eCmd==AR_CMD_UPDATE );
+        rc = arUpdateCmd(pState, &cmd);
+        break;
     }
   }
 
-  if( cmd=='c' ){
-    return arCreateCommand(pState, &azArg[2], nArg-2, bVerbose);
-  }
-
-  if( cmd=='x' ){
-    if( nArg!=2 ) goto usage;
-    return arExtractCommand(pState, bVerbose);
-  }
-
- usage:
-  raw_printf(stderr, "Usage %s sub-command ?args...?\n", azArg[0]);
-  return SQLITE_ERROR;
+  return rc;
 }
 
 
diff --git a/test/shell8.test b/test/shell8.test
new file mode 100644
index 0000000000..4c412b41dc
--- /dev/null
+++ b/test/shell8.test
@@ -0,0 +1,83 @@
+# 2017 December 9
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Test the shell tool ".ar" command.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix shell8
+set CLI [test_find_cli]
+
+proc populate_dir {dirname spec} {
+  # First delete the current tree, if one exists.
+  file delete -force $dirname
+  
+  # Recreate the root of the new tree.
+  file mkdir $dirname
+
+  # Add each file to the new tree.
+  foreach {f d} $spec {
+    set path [file join $dirname $f]
+    file mkdir [file dirname $path]
+    set fd [open $path w]
+    puts -nonewline $fd $d
+    close $fd
+  }
+}
+
+proc dir_to_list {dirname} {
+  set res [list]
+  foreach f [glob -nocomplain $dirname/*] {
+    set mtime [file mtime $f]
+    set perm [file attributes $f -perm]
+    set relpath [file join {*}[lrange [file split $f] 1 end]]
+    lappend res 
+    if {[file isdirectory $f]} {
+      lappend res [list $relpath / $mtime $perm]
+      lappend res {*}[dir_to_list $f]
+    } else {
+      set fd [open $f]
+      set data [read $fd]
+      close $fd
+      lappend res [list $relpath $data $mtime $perm]
+    }
+  }
+  lsort $res
+}
+
+proc dir_compare {d1 d2} {
+  set l1 [dir_to_list $d1]
+  set l2 [dir_to_list $d1]
+  string compare $l1 $l2
+}
+
+populate_dir ar1 {
+  file1 "abcd" 
+  file2 "efgh"
+  dir1/file3 "ijkl"
+}
+
+set expected [dir_to_list ar1]
+# puts "# $expected"
+do_test 1.1 {
+  forcedelete test_ar.db
+
+  catchcmd test_ar.db ".ar c ar1"
+  file delete -force ar1
+  catchcmd test_ar.db ".ar x"
+
+  dir_to_list ar1
+} $expected
+
+
+
+finish_test

From 2ad0949db4fbaca1a92a6fdf9957842cb6492366 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 9 Dec 2017 18:28:22 +0000
Subject: [PATCH 134/488] Add support for -C to ".ar x".

FossilOrigin-Name: 8cd70960c5ddf0d0b2c40b8b6af4ce6b0277ffdaf04f33fcb33227d2b99ad515
---
 manifest         | 14 +++++++-------
 manifest.uuid    |  2 +-
 src/shell.c.in   | 18 ++++++++++++++++--
 test/shell8.test | 13 +++++++++++--
 4 files changed, 35 insertions(+), 12 deletions(-)

diff --git a/manifest b/manifest
index 98adac2007..1916921b89 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improve\sparsing\sof\s".ar"\scommands.\sAdd\snew\stest\sfile\sfor\sthe\ssame.
-D 2017-12-09T17:58:02.648
+C Add\ssupport\sfor\s-C\sto\s".ar\sx".
+D 2017-12-09T18:28:22.916
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
@@ -474,7 +474,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
-F src/shell.c.in 907661eeab82949420270b24f5989a399242cb8721e6140f73b3a46939fc4820
+F src/shell.c.in 8e57abbd26d7d2344ba752be0f86b2cbf93ad73ca28c651786392fbfd3b512ba
 F src/sqlite.h.in 8fd97993d48b50b9bade38c52f12d175942c9497c960905610c7b03a3e4b5818
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1214,7 +1214,7 @@ F test/shell4.test 89ad573879a745974ff2df20ff97c5d6ffffbd5d
 F test/shell5.test 23939a4c51f0421330ea61dbd3c74f9c215f5f8d3d1a94846da6ffc777a35458
 F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3
 F test/shell7.test 115132f66d0463417f408562cc2cf534f6bbc6d83a6d50f0072a9eb171bae97f
-F test/shell8.test 98b1d7b218060e557b3a789f3396635a0c03873ea652b3154c7f3f238d4a1a8f
+F test/shell8.test 0e8e064da50c92df8eb514202dcfd0020f71762250066bf41ed098e0ff5f0e3d
 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3
 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5
 F test/shrink.test 1b4330b1fd9e818c04726d45cb28db73087535ce
@@ -1682,7 +1682,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0cc699d14adfe8c7b7be50c180186562861806c47425c80c935bce43ee5c5c12
-R 0b3e6167ae82d64c1e021b537a83d40a
+P 840401cc8ce3a09e0663b46973ecd2856d9607be71d2d1e9b21f7df7a82dcbe5
+R e8ce05071b1dec8c5193e0b4d7b226fe
 U dan
-Z 9b95380c27ad603c463b4469d523a6d2
+Z f3cf78b2d0eddbe511b3f9a244339aaa
diff --git a/manifest.uuid b/manifest.uuid
index d1e1c772f3..b240d7b3d5 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-840401cc8ce3a09e0663b46973ecd2856d9607be71d2d1e9b21f7df7a82dcbe5
\ No newline at end of file
+8cd70960c5ddf0d0b2c40b8b6af4ce6b0277ffdaf04f33fcb33227d2b99ad515
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index b8471fea9f..431329579e 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -4235,20 +4235,30 @@ static int arListCommand(ShellState *p, ArCommand *pAr){
 */
 static int arExtractCommand(ShellState *p, ArCommand *pAr){
   const char *zSql1 = 
-    "SELECT name, writefile(name, "
+    "SELECT :1 || name, writefile(:1 || name, "
     "CASE WHEN (data AND sz>=0 AND sz!=length(data)) THEN uncompress(data) "
     "   ELSE data END, "
     "mode) FROM sqlar";
-  const char *zSql2 = "SELECT name, mtime FROM sqlar"; 
+  const char *zSql2 = "SELECT :1 || name, mtime FROM sqlar"; 
 
   struct timespec times[2];
   sqlite3_stmt *pSql = 0;
   int rc = SQLITE_OK;
+  char *zDir = 0;
+
+  if( pAr->zDir ){
+    zDir = sqlite3_mprintf("%s/", pAr->zDir);
+  }else{
+    zDir = sqlite3_mprintf("");
+  }
 
   memset(times, 0, sizeof(times));
   times[0].tv_sec = time(0);
 
   shellPrepare(p, &rc, zSql1, &pSql);
+  if( rc==SQLITE_OK ){
+    sqlite3_bind_text(pSql, 1, zDir, -1, SQLITE_STATIC);
+  }
   while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
     if( pAr->bVerbose ){
       raw_printf(stdout, "%s\n", sqlite3_column_text(pSql, 0));
@@ -4257,6 +4267,9 @@ static int arExtractCommand(ShellState *p, ArCommand *pAr){
   shellFinalize(&rc, pSql);
 
   shellPrepare(p, &rc, zSql2, &pSql);
+  if( rc==SQLITE_OK ){
+    sqlite3_bind_text(pSql, 1, zDir, -1, SQLITE_STATIC);
+  }
   while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
     const char *zPath = (const char*)sqlite3_column_text(pSql, 0);
     times[1].tv_sec = (time_t)sqlite3_column_int64(pSql, 1);
@@ -4268,6 +4281,7 @@ static int arExtractCommand(ShellState *p, ArCommand *pAr){
   }
   shellFinalize(&rc, pSql);
 
+  sqlite3_free(zDir);
   return rc;
 }
 
diff --git a/test/shell8.test b/test/shell8.test
index 4c412b41dc..2d2fc7b94d 100644
--- a/test/shell8.test
+++ b/test/shell8.test
@@ -34,12 +34,14 @@ proc populate_dir {dirname spec} {
   }
 }
 
-proc dir_to_list {dirname} {
+proc dir_to_list {dirname {n -1}} {
+  if {$n<0} {set n [llength [file split $dirname]]}
+
   set res [list]
   foreach f [glob -nocomplain $dirname/*] {
     set mtime [file mtime $f]
     set perm [file attributes $f -perm]
-    set relpath [file join {*}[lrange [file split $f] 1 end]]
+    set relpath [file join {*}[lrange [file split $f] $n end]]
     lappend res 
     if {[file isdirectory $f]} {
       lappend res [list $relpath / $mtime $perm]
@@ -78,6 +80,13 @@ do_test 1.1 {
   dir_to_list ar1
 } $expected
 
+do_test 1.2 {
+  file delete -force ar3
+  file mkdir ar3
+  catchcmd test_ar.db ".ar xvC ar3"
+  dir_to_list ar3/ar1
+} $expected
+
 
 
 finish_test

From 2e6ca188c49e54bd8ffb6917b055c93644437ff9 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 11 Dec 2017 17:20:37 +0000
Subject: [PATCH 135/488] Fix a buffer overwrite in fts5 that could occur when
 processing a prefix query.

FossilOrigin-Name: 92fc146bc2b781e7e2d7138b00e5ea649c6fee1c2b8449420460a1b3e5c9661b
---
 ext/fts5/fts5_index.c        |  9 ++++++++-
 ext/fts5/test/fts5query.test | 12 +++++++++++-
 manifest                     | 16 ++++++++--------
 manifest.uuid                |  2 +-
 4 files changed, 28 insertions(+), 11 deletions(-)

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index c94122838d..a75bf0fd42 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -4909,7 +4909,13 @@ static void fts5MergePrefixLists(
     Fts5Buffer out = {0, 0, 0};
     Fts5Buffer tmp = {0, 0, 0};
 
-    if( sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n) ) return;
+    /* The maximum size of the output is equal to the sum of the two 
+    ** input sizes + 1 varint (9 bytes). The extra varint is because if the
+    ** first rowid in one input is a large negative number, and the first in
+    ** the other a non-negative number, the delta for the non-negative
+    ** number will be larger on disk than the literal integer value
+    ** was.  */
+    if( sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n + 9) ) return;
     fts5DoclistIterInit(p1, &i1);
     fts5DoclistIterInit(p2, &i2);
 
@@ -5003,6 +5009,7 @@ static void fts5MergePrefixLists(
       fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
       fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.aEof - i2.aPoslist);
     }
+    assert( out.n<=(p1->n+p2->n+9) );
 
     fts5BufferSet(&p->rc, p1, out.n, out.p);
     fts5BufferFree(&tmp);
diff --git a/ext/fts5/test/fts5query.test b/ext/fts5/test/fts5query.test
index 854651ef4f..5237e8e250 100644
--- a/ext/fts5/test/fts5query.test
+++ b/ext/fts5/test/fts5query.test
@@ -64,7 +64,7 @@ for {set tn 1 ; set pgsz 64} {$tn<32} {incr tn; incr pgsz 16} {
     execsql COMMIT
   } {}
 
-  do_execsql_test 1.$tn.2 {
+  do_execsql_test 2.$tn.2 {
     INSERT INTO t1(t1) VALUES('integrity-check');
   }
 
@@ -77,5 +77,15 @@ for {set tn 1 ; set pgsz 64} {$tn<32} {incr tn; incr pgsz 16} {
   }
 }
 
+reset_db
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(a);
+  INSERT INTO x1(rowid, a) VALUES(-1000000000000, 'toyota');
+  INSERT INTO x1(rowid, a) VALUES(1, 'tarago');
+}
+do_execsql_test 3.1 {
+  SELECT rowid FROM x1('t*');
+} {-1000000000000 1}
+
 
 finish_test
diff --git a/manifest b/manifest
index bdd8ba7d82..9bd5c38054 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sharmless\sAPI\ssignature\smismatch\sin\sthe\sunix\sVFS.
-D 2017-12-09T01:02:33.171
+C Fix\sa\sbuffer\soverwrite\sin\sfts5\sthat\scould\soccur\swhen\sprocessing\sa\sprefix\nquery.
+D 2017-12-11T17:20:37.958
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
@@ -105,7 +105,7 @@ F ext/fts5/fts5_buffer.c 1dd1ec0446b3acfc2d7d407eb894762a461613e2695273f48e449bf
 F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857
 F ext/fts5/fts5_expr.c 01048018d21524e2c302b063ff5c3cdcf546e03297215e577205d85b47499deb
 F ext/fts5/fts5_hash.c 32be400cf761868c9db33efe81a06eb19a17c5402ad477ee9efb51301546dd55
-F ext/fts5/fts5_index.c 2ce9d50ec5508b8205615aad69e1c9b2c77f017f21d4479e1fb2079c01fdd017
+F ext/fts5/fts5_index.c 5fe14375a29e8a7aa8f3e863babe180a19269206c254c8f47b216821d4ac1e15
 F ext/fts5/fts5_main.c 24868f88ab2a865defbba7a92eebeb726cc991eb092b71b5f5508f180c72605b
 F ext/fts5/fts5_storage.c fb5ef3c27073f67ade2e1bea08405f9e43f68f5f3676ed0ab7013bce5ba10be6
 F ext/fts5/fts5_tcl.c a7df39442ae674dde877cf06fe02ebb7658e69c179a4d223241c90df4f14b54e
@@ -183,7 +183,7 @@ F ext/fts5/test/fts5plan.test e30e8378441114ef6977a3dc24ecd203caa670d782124dfc9a
 F ext/fts5/test/fts5porter.test 8d08010c28527db66bc3feebd2b8767504aaeb9b101a986342fa7833d49d0d15
 F ext/fts5/test/fts5porter2.test 0d251a673f02fa13ca7f011654873b3add20745f7402f108600a23e52d8c7457
 F ext/fts5/test/fts5prefix.test a0fa67b06650f2deaa7bf27745899d94e0fb547ad9ecbd08bfad98c04912c056
-F ext/fts5/test/fts5query.test bdb6fd9e73268cfc07f789f1448cd71ea78acb02e481c619f286289ea18ca518
+F ext/fts5/test/fts5query.test ac363b17a442620bb0780e93c24f16a5f963dfe2f23dc85647b869efcfada728
 F ext/fts5/test/fts5rank.test 6e149da77a269923a8439aaa52366e49b85be4721902662da39a5ded16ed85d9
 F ext/fts5/test/fts5rebuild.test 6d09fd54b1170a1e54fe17b808bbf17fba3154956cc2f065dd94bf1e3d254f63
 F ext/fts5/test/fts5restart.test 835ecc8f449e3919f72509ab58056d0cedca40d1fe04108ccf8ac4c2ba41f415
@@ -1679,7 +1679,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 02013fc120bf71a8be3550c696a588af8c92f2209f8e5db530624878ddc8aa7e
-R 70ec80a62b754f49e015c4fa7a36d748
-U drh
-Z d8e2fb990ec2e3ae449b802623df526d
+P bab9de7fdda20a724f7c21ec2c25b488ece08b685f1e4fd15c9e73b6b4a0133a
+R 693906cf332f161a7ecc56eb7becf7aa
+U dan
+Z fd6217bb3206aa8caa41c580e04a37c2
diff --git a/manifest.uuid b/manifest.uuid
index ae71661621..88730c3b6a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-bab9de7fdda20a724f7c21ec2c25b488ece08b685f1e4fd15c9e73b6b4a0133a
\ No newline at end of file
+92fc146bc2b781e7e2d7138b00e5ea649c6fee1c2b8449420460a1b3e5c9661b
\ No newline at end of file

From 1ad3f61b810e48ad8d6f0b59152689fc10bdfb28 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 11 Dec 2017 20:22:02 +0000
Subject: [PATCH 136/488] Enhance virtual table "fsdir" in ext/misc/fileio.c.
 Add support for "-C" to the shell command's ".ar c" command.

FossilOrigin-Name: 0394889afed2479773af594e2d9659cf58b8959004ebcdeaff8e08e5dae684ef
---
 ext/misc/fileio.c | 230 +++++++++++++++++++++++++++-------------------
 manifest          |  16 ++--
 manifest.uuid     |   2 +-
 src/shell.c.in    |  11 +--
 test/shell8.test  |  10 ++
 5 files changed, 158 insertions(+), 111 deletions(-)

diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c
index db69357008..d3c00b4b1f 100644
--- a/ext/misc/fileio.c
+++ b/ext/misc/fileio.c
@@ -42,7 +42,7 @@ SQLITE_EXTENSION_INIT1
 #include <utime.h>
 
 
-#define FSDIR_SCHEMA "CREATE TABLE x(name,mode,mtime,data,dir HIDDEN)"
+#define FSDIR_SCHEMA "(name,mode,mtime,data,path HIDDEN,dir HIDDEN)"
 
 static void readFileContents(sqlite3_context *ctx, const char *zName){
   FILE *in;
@@ -166,29 +166,36 @@ static void writefileFunc(
 #ifndef SQLITE_OMIT_VIRTUALTABLE
 
 /* 
+** Cursor type for recursively iterating through a directory structure.
 */
 typedef struct fsdir_cursor fsdir_cursor;
+typedef struct FsdirLevel FsdirLevel;
+
+struct FsdirLevel {
+  DIR *pDir;                 /* From opendir() */
+  char *zDir;                /* Name of directory (nul-terminated) */
+};
+
 struct fsdir_cursor {
   sqlite3_vtab_cursor base;  /* Base class - must be first */
-  int eType;                 /* One of FSDIR_DIR or FSDIR_ENTRY */
-  DIR *pDir;                 /* From opendir() */
+
+  int nLvl;                  /* Number of entries in aLvl[] array */
+  int iLvl;                  /* Index of current entry */
+  FsdirLevel *aLvl;          /* Hierarchy of directories being traversed */
+
+  const char *zBase;
+  int nBase;
+
   struct stat sStat;         /* Current lstat() results */
-  char *zDir;                /* Directory to read */
-  int nDir;                  /* Value of strlen(zDir) */
   char *zPath;               /* Path to current entry */
-  int bEof;
   sqlite3_int64 iRowid;      /* Current rowid */
 };
 
 typedef struct fsdir_tab fsdir_tab;
 struct fsdir_tab {
   sqlite3_vtab base;         /* Base class - must be first */
-  int eType;                 /* One of FSDIR_DIR or FSDIR_ENTRY */
 };
 
-#define FSDIR_DIR   0
-#define FSDIR_ENTRY 1
-
 /*
 ** Construct a new fsdir virtual table object.
 */
@@ -202,12 +209,11 @@ static int fsdirConnect(
   fsdir_tab *pNew = 0;
   int rc;
 
-  rc = sqlite3_declare_vtab(db, FSDIR_SCHEMA);
+  rc = sqlite3_declare_vtab(db, "CREATE TABLE x" FSDIR_SCHEMA);
   if( rc==SQLITE_OK ){
     pNew = (fsdir_tab*)sqlite3_malloc( sizeof(*pNew) );
     if( pNew==0 ) return SQLITE_NOMEM;
     memset(pNew, 0, sizeof(*pNew));
-    pNew->eType = (pAux==0 ? FSDIR_DIR : FSDIR_ENTRY);
   }
   *ppVtab = (sqlite3_vtab*)pNew;
   return rc;
@@ -229,56 +235,107 @@ static int fsdirOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
   pCur = sqlite3_malloc( sizeof(*pCur) );
   if( pCur==0 ) return SQLITE_NOMEM;
   memset(pCur, 0, sizeof(*pCur));
-  pCur->eType = ((fsdir_tab*)p)->eType;
+  pCur->iLvl = -1;
   *ppCursor = &pCur->base;
   return SQLITE_OK;
 }
 
+static void fsdirResetCursor(fsdir_cursor *pCur){
+  int i;
+  for(i=0; i<=pCur->iLvl; i++){
+    FsdirLevel *pLvl = &pCur->aLvl[i];
+    if( pLvl->pDir ) closedir(pLvl->pDir);
+    sqlite3_free(pLvl->zDir);
+  }
+  sqlite3_free(pCur->zPath);
+  pCur->aLvl = 0;
+  pCur->zPath = 0;
+  pCur->zBase = 0;
+  pCur->nBase = 0;
+  pCur->iLvl = -1;
+  pCur->iRowid = 1;
+}
+
 /*
 ** Destructor for an fsdir_cursor.
 */
 static int fsdirClose(sqlite3_vtab_cursor *cur){
   fsdir_cursor *pCur = (fsdir_cursor*)cur;
-  if( pCur->pDir ) closedir(pCur->pDir);
-  sqlite3_free(pCur->zDir);
-  sqlite3_free(pCur->zPath);
+
+  fsdirResetCursor(pCur);
+  sqlite3_free(pCur->aLvl);
   sqlite3_free(pCur);
   return SQLITE_OK;
 }
 
+static void fsdirSetErrmsg(fsdir_cursor *pCur, const char *zFmt, ...){
+  va_list ap;
+  va_start(ap, zFmt);
+  pCur->base.pVtab->zErrMsg = sqlite3_vmprintf(zFmt, ap);
+  va_end(ap);
+}
+
+
 /*
 ** Advance an fsdir_cursor to its next row of output.
 */
 static int fsdirNext(sqlite3_vtab_cursor *cur){
   fsdir_cursor *pCur = (fsdir_cursor*)cur;
-  struct dirent *pEntry;
+  mode_t m = pCur->sStat.st_mode;
 
-  if( pCur->eType==FSDIR_ENTRY ){
-    pCur->bEof = 1;
-    return SQLITE_OK;
-  }
-
-  sqlite3_free(pCur->zPath);
-  pCur->zPath = 0;
-
-  while( 1 ){
-    pEntry = readdir(pCur->pDir);
-    if( pEntry ){
-      if( strcmp(pEntry->d_name, ".") 
-       && strcmp(pEntry->d_name, "..") 
-      ){
-        pCur->zPath = sqlite3_mprintf("%s/%s", pCur->zDir, pEntry->d_name);
-        if( pCur->zPath==0 ) return SQLITE_NOMEM;
-        lstat(pCur->zPath, &pCur->sStat);
-        break;
-      }
-    }else{
-      pCur->bEof = 1;
-      break;
+  pCur->iRowid++;
+  if( S_ISDIR(m) ){
+    /* Descend into this directory */
+    int iNew = pCur->iLvl + 1;
+    FsdirLevel *pLvl;
+    if( iNew>=pCur->nLvl ){
+      int nNew = iNew+1;
+      int nByte = nNew*sizeof(FsdirLevel);
+      FsdirLevel *aNew = (FsdirLevel*)sqlite3_realloc(pCur->aLvl, nByte);
+      if( aNew==0 ) return SQLITE_NOMEM;
+      memset(&aNew[pCur->nLvl], 0, sizeof(FsdirLevel)*(nNew-pCur->nLvl));
+      pCur->aLvl = aNew;
+      pCur->nLvl = nNew;
+    }
+    pCur->iLvl = iNew;
+    pLvl = &pCur->aLvl[iNew];
+    
+    pLvl->zDir = pCur->zPath;
+    pCur->zPath = 0;
+    pLvl->pDir = opendir(pLvl->zDir);
+    if( pLvl->pDir==0 ){
+      fsdirSetErrmsg(pCur, "cannot read directory: %s", pCur->zPath);
+      return SQLITE_ERROR;
     }
   }
 
-  pCur->iRowid++;
+  while( pCur->iLvl>=0 ){
+    FsdirLevel *pLvl = &pCur->aLvl[pCur->iLvl];
+    struct dirent *pEntry = readdir(pLvl->pDir);
+    if( pEntry ){
+      if( pEntry->d_name[0]=='.' ){
+       if( pEntry->d_name[1]=='.' && pEntry->d_name[2]=='\0' ) continue;
+       if( pEntry->d_name[1]=='\0' ) continue;
+      }
+      sqlite3_free(pCur->zPath);
+      pCur->zPath = sqlite3_mprintf("%s/%s", pLvl->zDir, pEntry->d_name);
+      if( pCur->zPath==0 ) return SQLITE_NOMEM;
+      if( lstat(pCur->zPath, &pCur->sStat) ){
+        fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zPath);
+        return SQLITE_ERROR;
+      }
+      return SQLITE_OK;
+    }
+    closedir(pLvl->pDir);
+    sqlite3_free(pLvl->zDir);
+    pLvl->pDir = 0;
+    pLvl->zDir = 0;
+    pCur->iLvl--;
+  }
+
+  /* EOF */
+  sqlite3_free(pCur->zPath);
+  pCur->zPath = 0;
   return SQLITE_OK;
 }
 
@@ -294,13 +351,7 @@ static int fsdirColumn(
   fsdir_cursor *pCur = (fsdir_cursor*)cur;
   switch( i ){
     case 0: { /* name */
-      const char *zName;
-      if( pCur->eType==FSDIR_DIR ){
-        zName = &pCur->zPath[pCur->nDir+1];
-      }else{
-        zName = pCur->zPath;
-      }
-      sqlite3_result_text(ctx, zName, -1, SQLITE_TRANSIENT);
+      sqlite3_result_text(ctx, &pCur->zPath[pCur->nBase], -1, SQLITE_TRANSIENT);
       break;
     }
 
@@ -308,11 +359,11 @@ static int fsdirColumn(
       sqlite3_result_int64(ctx, pCur->sStat.st_mode);
       break;
 
-    case 2: /* mode */
+    case 2: /* mtime */
       sqlite3_result_int64(ctx, pCur->sStat.st_mtime);
       break;
 
-    case 3: {
+    case 3: { /* data */
       mode_t m = pCur->sStat.st_mode;
       if( S_ISDIR(m) ){
         sqlite3_result_null(ctx);
@@ -361,14 +412,7 @@ static int fsdirRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
 */
 static int fsdirEof(sqlite3_vtab_cursor *cur){
   fsdir_cursor *pCur = (fsdir_cursor*)cur;
-  return pCur->bEof;
-}
-
-static void fsdirSetErrmsg(fsdir_cursor *pCur, const char *zFmt, ...){
-  va_list ap;
-  va_start(ap, zFmt);
-  pCur->base.pVtab->zErrMsg = sqlite3_vmprintf(zFmt, ap);
-  va_end(ap);
+  return (pCur->zPath==0);
 }
 
 /*
@@ -382,51 +426,38 @@ static int fsdirFilter(
   const char *zDir = 0;
   fsdir_cursor *pCur = (fsdir_cursor*)cur;
 
-  sqlite3_free(pCur->zDir);
-  pCur->iRowid = 0;
-  pCur->zDir = 0;
-  pCur->bEof = 0;
-  if( pCur->pDir ){
-    closedir(pCur->pDir);
-    pCur->pDir = 0;
-  }
+  fsdirResetCursor(pCur);
 
   if( idxNum==0 ){
     fsdirSetErrmsg(pCur, "table function fsdir requires an argument");
     return SQLITE_ERROR;
   }
 
-  assert( argc==1 );
+  assert( argc==idxNum && (argc==1 || argc==2) );
   zDir = (const char*)sqlite3_value_text(argv[0]);
   if( zDir==0 ){
     fsdirSetErrmsg(pCur, "table function fsdir requires a non-NULL argument");
     return SQLITE_ERROR;
   }
+  if( argc==2 ){
+    pCur->zBase = (const char*)sqlite3_value_text(argv[1]);
+  }
+  if( pCur->zBase ){
+    pCur->nBase = strlen(pCur->zBase)+1;
+    pCur->zPath = sqlite3_mprintf("%s/%s", pCur->zBase, zDir);
+  }else{
+    pCur->zPath = sqlite3_mprintf("%s", zDir);
+  }
 
-  pCur->zDir = sqlite3_mprintf("%s", zDir);
-  if( pCur->zDir==0 ){
+  if( pCur->zPath==0 ){
     return SQLITE_NOMEM;
   }
-
-  if( pCur->eType==FSDIR_ENTRY ){
-    int rc = lstat(pCur->zDir, &pCur->sStat);
-    if( rc ){
-      fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zDir);
-    }else{
-      pCur->zPath = sqlite3_mprintf("%s", pCur->zDir);
-      if( pCur->zPath==0 ) return SQLITE_NOMEM;
-    }
-    return SQLITE_OK;
-  }else{
-    pCur->nDir = strlen(pCur->zDir);
-    pCur->pDir = opendir(zDir);
-    if( pCur->pDir==0 ){
-      fsdirSetErrmsg(pCur, "error in opendir(\"%s\")", zDir);
-      return SQLITE_ERROR;
-    }
-
-    return fsdirNext(cur);
+  if( lstat(pCur->zPath, &pCur->sStat) ){
+    fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zPath);
+    return SQLITE_ERROR;
   }
+
+  return SQLITE_OK;
 }
 
 /*
@@ -450,24 +481,33 @@ static int fsdirBestIndex(
   sqlite3_index_info *pIdxInfo
 ){
   int i;                 /* Loop over constraints */
+  int idx4 = -1;
+  int idx5 = -1;
 
   const struct sqlite3_index_constraint *pConstraint;
   pConstraint = pIdxInfo->aConstraint;
   for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){
     if( pConstraint->usable==0 ) continue;
     if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
-    if( pConstraint->iColumn!=4 ) continue;
-    break;
+    if( pConstraint->iColumn==4 ) idx4 = i;
+    if( pConstraint->iColumn==5 ) idx5 = i;
   }
 
-  if( i<pIdxInfo->nConstraint ){
-    pIdxInfo->aConstraintUsage[i].omit = 1;
-    pIdxInfo->aConstraintUsage[i].argvIndex = 1;
-    pIdxInfo->idxNum = 1;
-    pIdxInfo->estimatedCost = 10.0;
-  }else{
+  if( idx4<0 ){
     pIdxInfo->idxNum = 0;
     pIdxInfo->estimatedCost = (double)(((sqlite3_int64)1) << 50);
+  }else{
+    pIdxInfo->aConstraintUsage[idx4].omit = 1;
+    pIdxInfo->aConstraintUsage[idx4].argvIndex = 1;
+    if( idx5>=0 ){
+      pIdxInfo->aConstraintUsage[idx5].omit = 1;
+      pIdxInfo->aConstraintUsage[idx5].argvIndex = 2;
+      pIdxInfo->idxNum = 2;
+      pIdxInfo->estimatedCost = 10.0;
+    }else{
+      pIdxInfo->idxNum = 1;
+      pIdxInfo->estimatedCost = 100.0;
+    }
   }
 
   return SQLITE_OK;
diff --git a/manifest b/manifest
index 1916921b89..d95b1b4be9 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\ssupport\sfor\s-C\sto\s".ar\sx".
-D 2017-12-09T18:28:22.916
+C Enhance\svirtual\stable\s"fsdir"\sin\sext/misc/fileio.c.\sAdd\ssupport\sfor\s"-C"\sto\nthe\sshell\scommand's\s".ar\sc"\scommand.
+D 2017-12-11T20:22:02.045
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
@@ -269,7 +269,7 @@ F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83
 F ext/misc/csv.c 1a009b93650732e22334edc92459c4630b9fa703397cbb3c8ca279921a36ca11
 F ext/misc/dbdump.c 3509fa6b8932d04e932d6b6b827b6a82ca362781b8e8f3c77336f416793e215e
 F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2
-F ext/misc/fileio.c c84ec9c399657bd95914e7c4e9aefcc148cb86fe3ab7f2102e9557c861dd0d51
+F ext/misc/fileio.c 238d0c65a14bda99748285527543273e0248bc428c01c3955ec45acb8738db3b
 F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25
 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c
 F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281984
@@ -474,7 +474,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
-F src/shell.c.in 8e57abbd26d7d2344ba752be0f86b2cbf93ad73ca28c651786392fbfd3b512ba
+F src/shell.c.in 4c7a2c1226216dfad7978642c958143f0fdfee4f09aef2add1e73e3601dd7d33
 F src/sqlite.h.in 8fd97993d48b50b9bade38c52f12d175942c9497c960905610c7b03a3e4b5818
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1214,7 +1214,7 @@ F test/shell4.test 89ad573879a745974ff2df20ff97c5d6ffffbd5d
 F test/shell5.test 23939a4c51f0421330ea61dbd3c74f9c215f5f8d3d1a94846da6ffc777a35458
 F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3
 F test/shell7.test 115132f66d0463417f408562cc2cf534f6bbc6d83a6d50f0072a9eb171bae97f
-F test/shell8.test 0e8e064da50c92df8eb514202dcfd0020f71762250066bf41ed098e0ff5f0e3d
+F test/shell8.test d04773f4e8cdad62d42a1b13bf4c1182109880441689a2f5d5001e190ec04831
 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3
 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5
 F test/shrink.test 1b4330b1fd9e818c04726d45cb28db73087535ce
@@ -1682,7 +1682,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 840401cc8ce3a09e0663b46973ecd2856d9607be71d2d1e9b21f7df7a82dcbe5
-R e8ce05071b1dec8c5193e0b4d7b226fe
+P 8cd70960c5ddf0d0b2c40b8b6af4ce6b0277ffdaf04f33fcb33227d2b99ad515
+R ddf59f7efc51e054de53889e9f879e48
 U dan
-Z f3cf78b2d0eddbe511b3f9a244339aaa
+Z c8a2667de38f55deb8cffcf1545f4d0c
diff --git a/manifest.uuid b/manifest.uuid
index b240d7b3d5..1e0e922cc2 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-8cd70960c5ddf0d0b2c40b8b6af4ce6b0277ffdaf04f33fcb33227d2b99ad515
\ No newline at end of file
+0394889afed2479773af594e2d9659cf58b8959004ebcdeaff8e08e5dae684ef
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 431329579e..0e32cbe1e2 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -4285,6 +4285,7 @@ static int arExtractCommand(ShellState *p, ArCommand *pAr){
   return rc;
 }
 
+
 /*
 ** Implementation of .ar "Create" command. 
 **
@@ -4298,11 +4299,7 @@ static int arCreateCommand(
   ArCommand *pAr                  /* Command arguments and options */
 ){
   const char *zSql = 
-    "WITH f(n, m, t, d) AS ("
-    "  SELECT name, mode, mtime, data FROM fsentry(:1) UNION ALL "
-    "  SELECT n || '/' || name, mode, mtime, data "
-    "      FROM f, fsdir(n) WHERE (m&?)"
-    ") SELECT * FROM f";
+    "SELECT name, mode, mtime, data FROM fsdir(?, ?)";
 
   const char *zSqlar = 
       "CREATE TABLE IF NOT EXISTS sqlar("
@@ -4322,17 +4319,17 @@ static int arCreateCommand(
 
   Bytef *aCompress = 0;           /* Compression buffer */
   int nCompress = 0;              /* Size of compression buffer */
-  
+
   rc = sqlite3_exec(p->db, "SAVEPOINT ar;", 0, 0, 0);
   if( rc!=SQLITE_OK ) return rc;
 
   rc = sqlite3_exec(p->db, zSqlar, 0, 0, 0);
   shellPrepare(p, &rc, zInsert, &pInsert);
   shellPrepare(p, &rc, zSql, &pStmt);
+  sqlite3_bind_text(pStmt, 2, pAr->zDir, -1, SQLITE_STATIC);
 
   for(i=0; i<pAr->nArg && rc==SQLITE_OK; i++){
     sqlite3_bind_text(pStmt, 1, pAr->azArg[i], -1, SQLITE_STATIC);
-    sqlite3_bind_int(pStmt, 2, S_IFDIR);
     while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
       int sz;
       const char *zName = (const char*)sqlite3_column_text(pStmt, 0);
diff --git a/test/shell8.test b/test/shell8.test
index 2d2fc7b94d..b3189d4215 100644
--- a/test/shell8.test
+++ b/test/shell8.test
@@ -87,6 +87,16 @@ do_test 1.2 {
   dir_to_list ar3/ar1
 } $expected
 
+do_test 1.3 {
+  file delete -force ar3
+  file mkdir ar3
+
+  forcedelete test_ar.db
+  catchcmd test_ar.db ".ar cC ar1 file1 file2 dir1"
+  catchcmd test_ar.db ".ar xC ar3"
+  dir_to_list ar3
+} $expected
+
 
 
 finish_test

From d4b56e59017d3f57afdc7c5f3d0b84904eca920d Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 12 Dec 2017 20:04:59 +0000
Subject: [PATCH 137/488] Add support for parsing options in non-traditional
 tar form to the ".ar" command. Have writefile() attempt to create any missing
 path components. And not to throw an exception if it is called to create a
 directory that already exists.

FossilOrigin-Name: 38dbeb1e777aa7ec742aa27002ad4dcee28af520dc43de96e5c56c39f16574ff
---
 ext/misc/fileio.c | 149 ++++++++++++++++++++--------
 manifest          |  16 +--
 manifest.uuid     |   2 +-
 src/shell.c.in    | 244 ++++++++++++++++++++++++++++++++++------------
 test/shell8.test  |  91 +++++++++++------
 5 files changed, 362 insertions(+), 140 deletions(-)

diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c
index d3c00b4b1f..2320173997 100644
--- a/ext/misc/fileio.c
+++ b/ext/misc/fileio.c
@@ -40,6 +40,7 @@ SQLITE_EXTENSION_INIT1
 #include <dirent.h>
 #include <time.h>
 #include <utime.h>
+#include <errno.h>
 
 
 #define FSDIR_SCHEMA "(name,mode,mtime,data,path HIDDEN,dir HIDDEN)"
@@ -90,6 +91,103 @@ static void ctxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){
   va_end(ap);
 }
 
+/*
+** Argument zFile is the name of a file that will be created and/or written
+** by SQL function writefile(). This function ensures that the directory
+** zFile will be written to exists, creating it if required. The permissions
+** for any path components created by this function are set to (mode&0777).
+**
+** If an OOM condition is encountered, SQLITE_NOMEM is returned. Otherwise,
+** SQLITE_OK is returned if the directory is successfully created, or
+** SQLITE_ERROR otherwise.
+*/
+static int makeDirectory(
+  const char *zFile,
+  mode_t mode
+){
+  char *zCopy = sqlite3_mprintf("%s", zFile);
+  int rc = SQLITE_OK;
+
+  if( zCopy==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    int nCopy = strlen(zCopy);
+    int i = 1;
+
+    while( rc==SQLITE_OK ){
+      struct stat sStat;
+      int rc;
+
+      for(; zCopy[i]!='/' && i<nCopy; i++);
+      if( i==nCopy ) break;
+      zCopy[i] = '\0';
+
+      rc = stat(zCopy, &sStat);
+      if( rc!=0 ){
+        if( mkdir(zCopy, mode & 0777) ) rc = SQLITE_ERROR;
+      }else{
+        if( !S_ISDIR(sStat.st_mode) ) rc = SQLITE_ERROR;
+      }
+      zCopy[i] = '/';
+      i++;
+    }
+
+    sqlite3_free(zCopy);
+  }
+
+  return rc;
+}
+
+static int writeFile(
+  sqlite3_context *pCtx, 
+  const char *zFile, 
+  mode_t mode, 
+  sqlite3_value *pData
+){
+  if( S_ISLNK(mode) ){
+    const char *zTo = (const char*)sqlite3_value_text(pData);
+    if( symlink(zTo, zFile)<0 ) return 1;
+  }else{
+    if( S_ISDIR(mode) ){
+      if( mkdir(zFile, mode) ){
+        /* The mkdir() call to create the directory failed. This might not
+        ** be an error though - if there is already a directory at the same
+        ** path and either the permissions already match or can be changed
+        ** to do so using chmod(), it is not an error.  */
+        struct stat sStat;
+        if( errno!=EEXIST
+         || 0!=stat(zFile, &sStat)
+         || !S_ISDIR(sStat.st_mode)
+         || ((sStat.st_mode&0777)!=(mode&0777) && 0!=chmod(zFile, mode&0777))
+        ){
+          return 1;
+        }
+      }
+    }else{
+      sqlite3_int64 nWrite = 0;
+      const char *z;
+      int rc = 0;
+      FILE *out = fopen(zFile, "wb");
+      if( out==0 ) return 1;
+      z = (const char*)sqlite3_value_blob(pData);
+      if( z ){
+        sqlite3_int64 n = fwrite(z, 1, sqlite3_value_bytes(pData), out);
+        nWrite = sqlite3_value_bytes(pData);
+        if( nWrite!=n ){
+          rc = 1;
+        }
+      }
+      fclose(out);
+      if( rc==0 && chmod(zFile, mode & 0777) ){
+        rc = 1;
+      }
+      if( rc ) return 2;
+      sqlite3_result_int64(pCtx, nWrite);
+    }
+  }
+  return 0;
+}
+
 /*
 ** Implementation of the "writefile(W,X[,Y]])" SQL function.  
 **
@@ -104,6 +202,7 @@ static void writefileFunc(
 ){
   const char *zFile;
   mode_t mode = 0;
+  int res;
 
   if( argc<2 || argc>3 ){
     sqlite3_result_error(context, 
@@ -119,46 +218,20 @@ static void writefileFunc(
     mode = sqlite3_value_int(argv[2]);
   }
 
-  if( S_ISLNK(mode) ){
-    const char *zTo = (const char*)sqlite3_value_text(argv[1]);
-    if( symlink(zTo, zFile)<0 ){
-      ctxErrorMsg(context, "failed to create symlink: %s", zFile);
-      return;
-    }
-  }else{
-    if( S_ISDIR(mode) ){
-      if( mkdir(zFile, mode) ){
-        ctxErrorMsg(context, "failed to create directory: %s", zFile);
-        return;
-      }
-    }else{
-      sqlite3_int64 nWrite = 0;
-      const char *z;
-      int rc = 0;
-      FILE *out = fopen(zFile, "wb");
-      if( out==0 ){
-        if( argc>2 ){
-          ctxErrorMsg(context, "failed to open file for writing: %s", zFile);
-        }
-        return;
-      }
-      z = (const char*)sqlite3_value_blob(argv[1]);
-      if( z ){
-        sqlite3_int64 n = fwrite(z, 1, sqlite3_value_bytes(argv[1]), out);
-        nWrite = sqlite3_value_bytes(argv[1]);
-        if( nWrite!=n ){
-          ctxErrorMsg(context, "failed to write file: %s", zFile);
-          rc = 1;
-        }
-      }
-      fclose(out);
-      if( rc ) return;
-      sqlite3_result_int64(context, nWrite);
+  res = writeFile(context, zFile, mode, argv[1]);
+  if( res==1 && errno==ENOENT ){
+    if( makeDirectory(zFile, mode)==SQLITE_OK ){
+      res = writeFile(context, zFile, mode, argv[1]);
     }
+  }
 
-    if( argc>2 && chmod(zFile, mode & 0777) ){
-      ctxErrorMsg(context, "failed to chmod file: %s", zFile);
-      return;
+  if( res!=0 ){
+    if( S_ISLNK(mode) ){
+      ctxErrorMsg(context, "failed to create symlink: %s", zFile);
+    }else if( S_ISDIR(mode) ){
+      ctxErrorMsg(context, "failed to create directory: %s", zFile);
+    }else{
+      ctxErrorMsg(context, "failed to write file: %s", zFile);
     }
   }
 }
diff --git a/manifest b/manifest
index d95b1b4be9..af08ef11da 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Enhance\svirtual\stable\s"fsdir"\sin\sext/misc/fileio.c.\sAdd\ssupport\sfor\s"-C"\sto\nthe\sshell\scommand's\s".ar\sc"\scommand.
-D 2017-12-11T20:22:02.045
+C Add\ssupport\sfor\sparsing\soptions\sin\snon-traditional\star\sform\sto\sthe\s".ar"\ncommand.\sHave\swritefile()\sattempt\sto\screate\sany\smissing\spath\scomponents.\sAnd\nnot\sto\sthrow\san\sexception\sif\sit\sis\scalled\sto\screate\sa\sdirectory\sthat\salready\nexists.
+D 2017-12-12T20:04:59.331
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
@@ -269,7 +269,7 @@ F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83
 F ext/misc/csv.c 1a009b93650732e22334edc92459c4630b9fa703397cbb3c8ca279921a36ca11
 F ext/misc/dbdump.c 3509fa6b8932d04e932d6b6b827b6a82ca362781b8e8f3c77336f416793e215e
 F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2
-F ext/misc/fileio.c 238d0c65a14bda99748285527543273e0248bc428c01c3955ec45acb8738db3b
+F ext/misc/fileio.c 29b7fc94752fff6245cf4a81455f98cf6778ec1102ca7e67bf693d41a7db4307
 F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25
 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c
 F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281984
@@ -474,7 +474,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
-F src/shell.c.in 4c7a2c1226216dfad7978642c958143f0fdfee4f09aef2add1e73e3601dd7d33
+F src/shell.c.in 0ab6e3c1fa09e420e643628d55929422867ca053f05df67a4cae4a67e2a6cfc5
 F src/sqlite.h.in 8fd97993d48b50b9bade38c52f12d175942c9497c960905610c7b03a3e4b5818
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1214,7 +1214,7 @@ F test/shell4.test 89ad573879a745974ff2df20ff97c5d6ffffbd5d
 F test/shell5.test 23939a4c51f0421330ea61dbd3c74f9c215f5f8d3d1a94846da6ffc777a35458
 F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3
 F test/shell7.test 115132f66d0463417f408562cc2cf534f6bbc6d83a6d50f0072a9eb171bae97f
-F test/shell8.test d04773f4e8cdad62d42a1b13bf4c1182109880441689a2f5d5001e190ec04831
+F test/shell8.test 5c5a9d100d34b125e0f46d259ea76cf074ac60719b722b2a2c63d759c63fc113
 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3
 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5
 F test/shrink.test 1b4330b1fd9e818c04726d45cb28db73087535ce
@@ -1682,7 +1682,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 8cd70960c5ddf0d0b2c40b8b6af4ce6b0277ffdaf04f33fcb33227d2b99ad515
-R ddf59f7efc51e054de53889e9f879e48
+P 0394889afed2479773af594e2d9659cf58b8959004ebcdeaff8e08e5dae684ef
+R e0d6dbcefc8608142fe03f9ef6d2dee7
 U dan
-Z c8a2667de38f55deb8cffcf1545f4d0c
+Z 5813e7d404a813775aab77275468b917
diff --git a/manifest.uuid b/manifest.uuid
index 1e0e922cc2..9e345298fb 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0394889afed2479773af594e2d9659cf58b8959004ebcdeaff8e08e5dae684ef
\ No newline at end of file
+38dbeb1e777aa7ec742aa27002ad4dcee28af520dc43de96e5c56c39f16574ff
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 0e32cbe1e2..fd8659297c 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -4075,17 +4075,17 @@ static int lintDotCommand(
 }
 
 static void shellPrepare(
-  ShellState *p, 
+  sqlite3 *db, 
   int *pRc, 
   const char *zSql, 
   sqlite3_stmt **ppStmt
 ){
   *ppStmt = 0;
   if( *pRc==SQLITE_OK ){
-    int rc = sqlite3_prepare_v2(p->db, zSql, -1, ppStmt, 0);
+    int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0);
     if( rc!=SQLITE_OK ){
       raw_printf(stderr, "sql error: %s (%d)\n", 
-          sqlite3_errmsg(p->db), sqlite3_errcode(p->db)
+          sqlite3_errmsg(db), sqlite3_errcode(db)
       );
       *pRc = rc;
     }
@@ -4141,10 +4141,42 @@ static int arUsage(void){
 /*
 ** Values for ArCommand.eCmd.
 */
-#define AR_CMD_CREATE  1
-#define AR_CMD_EXTRACT 2
-#define AR_CMD_LIST    3
-#define AR_CMD_UPDATE  4
+#define AR_CMD_CREATE       1
+#define AR_CMD_EXTRACT      2
+#define AR_CMD_LIST         3
+#define AR_CMD_UPDATE       4
+
+/*
+** Other (non-command) switches.
+*/
+#define AR_SWITCH_VERBOSE   5
+#define AR_SWITCH_FILE      6
+#define AR_SWITCH_DIRECTORY 7
+
+static int arProcessSwitch(ArCommand *pAr, int eSwitch, const char *zArg){
+  switch( eSwitch ){
+    case AR_CMD_CREATE:
+    case AR_CMD_EXTRACT:
+    case AR_CMD_LIST:
+    case AR_CMD_UPDATE:
+      if( pAr->eCmd ) return arUsage();
+      pAr->eCmd = eSwitch;
+      break;
+
+    case AR_SWITCH_VERBOSE:
+      pAr->bVerbose = 1;
+      break;
+
+    case AR_SWITCH_FILE:
+      pAr->zFile = zArg;
+      break;
+    case AR_SWITCH_DIRECTORY:
+      pAr->zDir = zArg;
+      break;
+  }
+
+  return SQLITE_OK;
+}
 
 /*
 ** Parse the command line for an ".ar" command. The results are written into
@@ -4157,6 +4189,23 @@ static int arParseCommand(
   int nArg,                       /* Number of entries in azArg[] */
   ArCommand *pAr                  /* Populate this object */
 ){
+  struct ArSwitch {
+    char cShort;
+    const char *zLong;
+    int eSwitch;
+    int bArg;
+  } aSwitch[] = {
+    { 'c', "create",    AR_CMD_CREATE, 0 },
+    { 'x', "extract",   AR_CMD_EXTRACT, 0 },
+    { 't', "list",      AR_CMD_LIST, 0 },
+    { 'u', "update",    AR_CMD_UPDATE, 0 },
+    { 'v', "verbose",   AR_SWITCH_VERBOSE, 0 },
+    { 'f', "file",      AR_SWITCH_FILE, 1 },
+    { 'C', "directory", AR_SWITCH_DIRECTORY, 1 }
+  };
+  int nSwitch = sizeof(aSwitch) / sizeof(struct ArSwitch);
+  struct ArSwitch *pEnd = &aSwitch[nSwitch];
+
   if( nArg<=1 ){
     return arUsage();
   }else{
@@ -4168,45 +4217,91 @@ static int arParseCommand(
       int i;
       int iArg = 2;
       for(i=0; z[i]; i++){
-        switch( z[i] ){
-          case 'c':
-            if( pAr->eCmd ) return arUsage();
-            pAr->eCmd = AR_CMD_CREATE;
-            break;
-          case 'x':
-            if( pAr->eCmd ) return arUsage();
-            pAr->eCmd = AR_CMD_EXTRACT;
-            break;
-          case 't':
-            if( pAr->eCmd ) return arUsage();
-            pAr->eCmd = AR_CMD_LIST;
-            break;
-          case 'u':
-            if( pAr->eCmd ) return arUsage();
-            pAr->eCmd = AR_CMD_UPDATE;
-            break;
-
-          case 'v':
-            pAr->bVerbose = 1;
-            break;
-          case 'f':
-            if( iArg>=nArg ) return arUsage();
-            pAr->zFile = azArg[iArg++];
-            break;
-          case 'C':
-            if( iArg>=nArg ) return arUsage();
-            pAr->zDir = azArg[iArg++];
-            break;
-
-          default:
-            return arUsage();
+        const char *zArg = 0;
+        struct ArSwitch *pOpt;
+        for(pOpt=&aSwitch[0]; pOpt<pEnd; pOpt++){
+          if( z[i]==pOpt->cShort ) break;
         }
+        if( pOpt==pEnd ) return arUsage();
+        if( pOpt->bArg ){
+          if( iArg>=nArg ) return arUsage();
+          zArg = azArg[iArg++];
+        }
+        if( arProcessSwitch(pAr, pOpt->eSwitch, zArg) ) return SQLITE_ERROR;
       }
-
       pAr->nArg = nArg-iArg;
       if( pAr->nArg>0 ){
         pAr->azArg = &azArg[iArg];
       }
+    }else{
+      /* Non-traditional invocation */
+      int iArg;
+      for(iArg=1; iArg<nArg; iArg++){
+        int n;
+        z = azArg[iArg];
+        if( z[0]!='-' ){
+          /* All remaining command line words are command arguments. */
+          pAr->azArg = &azArg[iArg];
+          pAr->nArg = nArg-iArg;
+          break;
+        }
+        n = strlen(z);
+
+        if( z[1]!='-' ){
+          int i;
+          /* One or more short options */
+          for(i=1; i<n; i++){
+            const char *zArg = 0;
+            struct ArSwitch *pOpt;
+            for(pOpt=&aSwitch[0]; pOpt<pEnd; pOpt++){
+              if( z[i]==pOpt->cShort ) break;
+            }
+            if( pOpt==pEnd ) return arUsage();
+            if( pOpt->bArg ){
+              if( i<(n-1) ){
+                zArg = &z[i+1];
+                i = n;
+              }else{
+                if( iArg>=(nArg-1) ) return arUsage();
+                zArg = azArg[++iArg];
+              }
+            }
+            if( arProcessSwitch(pAr, pOpt->eSwitch, zArg) ) return SQLITE_ERROR;
+          }
+        }else if( z[2]=='\0' ){
+          /* A -- option, indicating that all remaining command line words
+          ** are command arguments.  */
+          pAr->azArg = &azArg[iArg+1];
+          pAr->nArg = nArg-iArg-1;
+          break;
+        }else{
+          /* A long option */
+          const char *zArg = 0;             /* Argument for option, if any */
+          struct ArSwitch *pMatch = 0;      /* Matching option */
+          struct ArSwitch *pOpt;            /* Iterator */
+          for(pOpt=&aSwitch[0]; pOpt<pEnd; pOpt++){
+            const char *zLong = pOpt->zLong;
+            if( (n-2)<=strlen(zLong) && 0==memcmp(&z[2], zLong, n-2) ){
+              if( pMatch ){
+                /* ambiguous option */
+                return arUsage();
+              }else{
+                pMatch = pOpt;
+              }
+            }
+          }
+
+          if( pMatch==0 ){
+            /* no such option. */
+            return arUsage();
+          }
+          if( pMatch->bArg ){
+            if( iArg>=(nArg-1) ) return arUsage();
+            zArg = azArg[++iArg];
+          }
+          if( arProcessSwitch(pAr, pMatch->eSwitch, zArg) ) return SQLITE_ERROR;
+        }
+      }
     }
   }
 
@@ -4216,7 +4311,7 @@ static int arParseCommand(
 /*
 ** Implementation of .ar "Update" command. 
 */
-static int arUpdateCmd(ShellState *p, ArCommand *pAr){
+static int arUpdateCmd(ShellState *p, sqlite3 *db, ArCommand *pAr){
   raw_printf(stderr, "todo...\n");
   return SQLITE_OK;
 }
@@ -4224,7 +4319,7 @@ static int arUpdateCmd(ShellState *p, ArCommand *pAr){
 /*
 ** Implementation of .ar "lisT" command. 
 */
-static int arListCommand(ShellState *p, ArCommand *pAr){
+static int arListCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
   raw_printf(stderr, "todo...\n");
   return SQLITE_OK;
 }
@@ -4233,7 +4328,7 @@ static int arListCommand(ShellState *p, ArCommand *pAr){
 /*
 ** Implementation of .ar "eXtract" command. 
 */
-static int arExtractCommand(ShellState *p, ArCommand *pAr){
+static int arExtractCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
   const char *zSql1 = 
     "SELECT :1 || name, writefile(:1 || name, "
     "CASE WHEN (data AND sz>=0 AND sz!=length(data)) THEN uncompress(data) "
@@ -4255,7 +4350,7 @@ static int arExtractCommand(ShellState *p, ArCommand *pAr){
   memset(times, 0, sizeof(times));
   times[0].tv_sec = time(0);
 
-  shellPrepare(p, &rc, zSql1, &pSql);
+  shellPrepare(db, &rc, zSql1, &pSql);
   if( rc==SQLITE_OK ){
     sqlite3_bind_text(pSql, 1, zDir, -1, SQLITE_STATIC);
   }
@@ -4266,7 +4361,7 @@ static int arExtractCommand(ShellState *p, ArCommand *pAr){
   }
   shellFinalize(&rc, pSql);
 
-  shellPrepare(p, &rc, zSql2, &pSql);
+  shellPrepare(db, &rc, zSql2, &pSql);
   if( rc==SQLITE_OK ){
     sqlite3_bind_text(pSql, 1, zDir, -1, SQLITE_STATIC);
   }
@@ -4296,20 +4391,19 @@ static int arExtractCommand(ShellState *p, ArCommand *pAr){
 */
 static int arCreateCommand(
   ShellState *p,                  /* Shell state pointer */
+  sqlite3 *db,
   ArCommand *pAr                  /* Command arguments and options */
 ){
-  const char *zSql = 
-    "SELECT name, mode, mtime, data FROM fsdir(?, ?)";
-
-  const char *zSqlar = 
+  const char *zSql = "SELECT name, mode, mtime, data FROM fsdir(?, ?)";
+  const char *zCreate = 
       "CREATE TABLE IF NOT EXISTS sqlar("
       "name TEXT PRIMARY KEY,  -- name of the file\n"
       "mode INT,               -- access permissions\n"
       "mtime INT,              -- last modification time\n"
       "sz INT,                 -- original file size\n"
       "data BLOB               -- compressed content\n"
-      ")";
-
+  ")";
+  const char *zDrop = "DROP TABLE IF EXISTS sqlar";
   const char *zInsert = "REPLACE INTO sqlar VALUES(?, ?, ?, ?, ?)";
 
   sqlite3_stmt *pStmt = 0;        /* Directory traverser */
@@ -4320,12 +4414,15 @@ static int arCreateCommand(
   Bytef *aCompress = 0;           /* Compression buffer */
   int nCompress = 0;              /* Size of compression buffer */
 
-  rc = sqlite3_exec(p->db, "SAVEPOINT ar;", 0, 0, 0);
+  rc = sqlite3_exec(db, "SAVEPOINT ar;", 0, 0, 0);
   if( rc!=SQLITE_OK ) return rc;
 
-  rc = sqlite3_exec(p->db, zSqlar, 0, 0, 0);
-  shellPrepare(p, &rc, zInsert, &pInsert);
-  shellPrepare(p, &rc, zSql, &pStmt);
+  rc = sqlite3_exec(db, zDrop, 0, 0, 0);
+  if( rc!=SQLITE_OK ) return rc;
+
+  rc = sqlite3_exec(db, zCreate, 0, 0, 0);
+  shellPrepare(db, &rc, zInsert, &pInsert);
+  shellPrepare(db, &rc, zSql, &pStmt);
   sqlite3_bind_text(pStmt, 2, pAr->zDir, -1, SQLITE_STATIC);
 
   for(i=0; i<pAr->nArg && rc==SQLITE_OK; i++){
@@ -4385,9 +4482,9 @@ static int arCreateCommand(
   }
 
   if( rc!=SQLITE_OK ){
-    sqlite3_exec(p->db, "ROLLBACK TO ar; RELEASE ar;", 0, 0, 0);
+    sqlite3_exec(db, "ROLLBACK TO ar; RELEASE ar;", 0, 0, 0);
   }else{
-    rc = sqlite3_exec(p->db, "RELEASE ar;", 0, 0, 0);
+    rc = sqlite3_exec(db, "RELEASE ar;", 0, 0, 0);
   }
   shellFinalize(&rc, pStmt);
   shellFinalize(&rc, pInsert);
@@ -4407,24 +4504,49 @@ static int arDotCommand(
   int rc;
   rc = arParseCommand(azArg, nArg, &cmd);
   if( rc==SQLITE_OK ){
+    sqlite3 *db = 0;              /* Database handle to use as archive */
+
+    if( cmd.zFile ){
+      int flags;
+      if( cmd.eCmd==AR_CMD_CREATE || cmd.eCmd==AR_CMD_UPDATE ){
+        flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE;
+      }else{
+        flags = SQLITE_OPEN_READONLY;
+      }
+      rc = sqlite3_open_v2(cmd.zFile, &db, flags, 0);
+      if( rc!=SQLITE_OK ){
+        raw_printf(stderr, "cannot open file: %s (%s)\n", 
+            cmd.zFile, sqlite3_errmsg(db)
+        );
+        sqlite3_close(db);
+        return rc;
+      }
+    }else{
+      db = pState->db;
+    }
+
     switch( cmd.eCmd ){
       case AR_CMD_CREATE:
-        rc = arCreateCommand(pState, &cmd);
+        rc = arCreateCommand(pState, db, &cmd);
         break;
 
       case AR_CMD_EXTRACT:
-        rc = arExtractCommand(pState, &cmd);
+        rc = arExtractCommand(pState, db, &cmd);
         break;
 
       case AR_CMD_LIST:
-        rc = arListCommand(pState, &cmd);
+        rc = arListCommand(pState, db, &cmd);
         break;
 
       default:
         assert( cmd.eCmd==AR_CMD_UPDATE );
-        rc = arUpdateCmd(pState, &cmd);
+        rc = arUpdateCmd(pState, db, &cmd);
         break;
     }
+
+    if( cmd.zFile ){
+      sqlite3_close(db);
+    }
   }
 
   return rc;
diff --git a/test/shell8.test b/test/shell8.test
index b3189d4215..577452511a 100644
--- a/test/shell8.test
+++ b/test/shell8.test
@@ -62,41 +62,68 @@ proc dir_compare {d1 d2} {
   string compare $l1 $l2
 }
 
-populate_dir ar1 {
-  file1 "abcd" 
-  file2 "efgh"
-  dir1/file3 "ijkl"
+foreach {tn tcl} {
+  1 {
+    set c1 ".ar c ar1"
+    set x1 ".ar x"
+
+    set c2 ".ar cC ar1 ."
+    set x2 ".ar Cx ar3"
+  }
+
+  2 {
+    set c1 ".ar -c ar1"
+    set x1 ".ar -x"
+
+    set c2 ".ar -cC ar1 ."
+    set x2 ".ar -xC ar3"
+  }
+
+  3 {
+    set c1 ".ar --create ar1"
+    set x1 ".ar --extract"
+
+    set c2 ".ar --directory ar1 --create ."
+    set x2 ".ar --extract --dir ar3"
+  }
+
+  4 {
+    set c1 ".ar --cr ar1"
+    set x1 ".ar --e"
+
+    set c2 ".ar -C ar1 -c ."
+    set x2 ".ar -x -C ar3"
+  }
+} {
+  eval $tcl
+
+  # Populate directory "ar1" with some files.
+  #
+  populate_dir ar1 {
+    file1 "abcd" 
+    file2 "efgh"
+    dir1/file3 "ijkl"
+  }
+  set expected [dir_to_list ar1]
+
+  do_test 1.$tn.1 {
+    catchcmd test_ar.db $c1
+    file delete -force ar1
+    catchcmd test_ar.db $x1
+    dir_to_list ar1
+  } $expected
+
+  do_test 1.$tn.2 {
+    file delete -force ar3
+    catchcmd test_ar.db $c2
+    catchcmd test_ar.db $x2
+    dir_to_list ar3
+  } $expected
 }
 
-set expected [dir_to_list ar1]
-# puts "# $expected"
-do_test 1.1 {
-  forcedelete test_ar.db
-
-  catchcmd test_ar.db ".ar c ar1"
-  file delete -force ar1
-  catchcmd test_ar.db ".ar x"
-
-  dir_to_list ar1
-} $expected
-
-do_test 1.2 {
-  file delete -force ar3
-  file mkdir ar3
-  catchcmd test_ar.db ".ar xvC ar3"
-  dir_to_list ar3/ar1
-} $expected
-
-do_test 1.3 {
-  file delete -force ar3
-  file mkdir ar3
-
-  forcedelete test_ar.db
-  catchcmd test_ar.db ".ar cC ar1 file1 file2 dir1"
-  catchcmd test_ar.db ".ar xC ar3"
-  dir_to_list ar3
-} $expected
+finish_test
 
 
 
 finish_test
+

From ece4b0c17201699beb3df4e073fda7b6287cde64 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 12 Dec 2017 20:28:36 +0000
Subject: [PATCH 138/488] Add tests and fixes for the shell ".ar" command -f
 option.

FossilOrigin-Name: 1a9867973c9d6675fa5254fdd74f36004707a98a91593a188033cf5a49cc7a0b
---
 manifest         | 14 +++++++-------
 manifest.uuid    |  2 +-
 src/shell.c.in   |  2 ++
 test/shell8.test | 20 ++++++++++++++++++++
 4 files changed, 30 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index af08ef11da..61046afc09 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\ssupport\sfor\sparsing\soptions\sin\snon-traditional\star\sform\sto\sthe\s".ar"\ncommand.\sHave\swritefile()\sattempt\sto\screate\sany\smissing\spath\scomponents.\sAnd\nnot\sto\sthrow\san\sexception\sif\sit\sis\scalled\sto\screate\sa\sdirectory\sthat\salready\nexists.
-D 2017-12-12T20:04:59.331
+C Add\stests\sand\sfixes\sfor\sthe\sshell\s".ar"\scommand\s-f\soption.
+D 2017-12-12T20:28:36.588
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
@@ -474,7 +474,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
-F src/shell.c.in 0ab6e3c1fa09e420e643628d55929422867ca053f05df67a4cae4a67e2a6cfc5
+F src/shell.c.in a09773c80a647f6ba4ef8dd9ce88840d52dbede5a9fa318333843deb8c8548b7
 F src/sqlite.h.in 8fd97993d48b50b9bade38c52f12d175942c9497c960905610c7b03a3e4b5818
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1214,7 +1214,7 @@ F test/shell4.test 89ad573879a745974ff2df20ff97c5d6ffffbd5d
 F test/shell5.test 23939a4c51f0421330ea61dbd3c74f9c215f5f8d3d1a94846da6ffc777a35458
 F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3
 F test/shell7.test 115132f66d0463417f408562cc2cf534f6bbc6d83a6d50f0072a9eb171bae97f
-F test/shell8.test 5c5a9d100d34b125e0f46d259ea76cf074ac60719b722b2a2c63d759c63fc113
+F test/shell8.test 0f7dfc5b33bde7143df8e37cbb4ae6ccc7e91f87232dc8e5e02be03117cdebb8
 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3
 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5
 F test/shrink.test 1b4330b1fd9e818c04726d45cb28db73087535ce
@@ -1682,7 +1682,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0394889afed2479773af594e2d9659cf58b8959004ebcdeaff8e08e5dae684ef
-R e0d6dbcefc8608142fe03f9ef6d2dee7
+P 38dbeb1e777aa7ec742aa27002ad4dcee28af520dc43de96e5c56c39f16574ff
+R e6fcdae437270c2329ef751955f65f0e
 U dan
-Z 5813e7d404a813775aab77275468b917
+Z 1b5cf819c180e1535b22a7c0cf2ece4d
diff --git a/manifest.uuid b/manifest.uuid
index 9e345298fb..7640d02900 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-38dbeb1e777aa7ec742aa27002ad4dcee28af520dc43de96e5c56c39f16574ff
\ No newline at end of file
+1a9867973c9d6675fa5254fdd74f36004707a98a91593a188033cf5a49cc7a0b
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index fd8659297c..ee9ccd6e76 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -4521,6 +4521,8 @@ static int arDotCommand(
         sqlite3_close(db);
         return rc;
       }
+      sqlite3_fileio_init(db, 0, 0);
+      sqlite3_compress_init(db, 0, 0);
     }else{
       db = pState->db;
     }
diff --git a/test/shell8.test b/test/shell8.test
index 577452511a..07065b56bd 100644
--- a/test/shell8.test
+++ b/test/shell8.test
@@ -69,6 +69,9 @@ foreach {tn tcl} {
 
     set c2 ".ar cC ar1 ."
     set x2 ".ar Cx ar3"
+
+    set c3 ".ar cCf ar1 test_xyz.db ."
+    set x3 ".ar Cfx ar3 test_xyz.db"
   }
 
   2 {
@@ -77,6 +80,9 @@ foreach {tn tcl} {
 
     set c2 ".ar -cC ar1 ."
     set x2 ".ar -xC ar3"
+
+    set c3 ".ar -cCar1 -ftest_xyz.db ."
+    set x3 ".ar -x -C ar3 -f test_xyz.db"
   }
 
   3 {
@@ -85,6 +91,9 @@ foreach {tn tcl} {
 
     set c2 ".ar --directory ar1 --create ."
     set x2 ".ar --extract --dir ar3"
+
+    set c3 ".ar --creat --dir ar1 --file test_xyz.db ."
+    set x3 ".ar --e  --d ar3 --f test_xyz.db"
   }
 
   4 {
@@ -93,6 +102,9 @@ foreach {tn tcl} {
 
     set c2 ".ar -C ar1 -c ."
     set x2 ".ar -x -C ar3"
+
+    set c3 ".ar -c --directory ar1 --file test_xyz.db ."
+    set x3 ".ar -x --directory ar3 --file test_xyz.db"
   }
 } {
   eval $tcl
@@ -119,6 +131,14 @@ foreach {tn tcl} {
     catchcmd test_ar.db $x2
     dir_to_list ar3
   } $expected
+
+  do_test 1.$tn.3 {
+    file delete -force ar3
+    file delete -force test_xyz.db
+    catchcmd ":memory:" $c3
+    catchcmd ":memory:" $x3
+    dir_to_list ar3
+  } $expected
 }
 
 finish_test

From 37874b5f7466564081adbfa146caa3337062477a Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 13 Dec 2017 10:11:09 +0000
Subject: [PATCH 139/488] Minor enhancement to two assert() statements in the
 default VFSes.

FossilOrigin-Name: 9cede8a83ca4cd88d504050115e1e89e7b3d3cd3cb2ffb5b8961e311a23ff5e2
---
 manifest      | 16 ++++++++--------
 manifest.uuid |  2 +-
 src/os_unix.c |  2 +-
 src/os_win.c  |  2 +-
 4 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index 9bd5c38054..cdf085dc66 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sbuffer\soverwrite\sin\sfts5\sthat\scould\soccur\swhen\sprocessing\sa\sprefix\nquery.
-D 2017-12-11T17:20:37.958
+C Minor\senhancement\sto\stwo\sassert()\sstatements\sin\sthe\sdefault\sVFSes.
+D 2017-12-13T10:11:09.727
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
@@ -455,8 +455,8 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104
 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
-F src/os_unix.c 888afdeab00cd5df5e1c2cefe26926127664d73416e60804576d661cb1c11f52
-F src/os_win.c 7f36120492e4a23c48d1dd685edf29ae459c6d555660c61f1323cea3e5a1191d
+F src/os_unix.c a40e2926d2cee18f5bd919356f560ff30f0189dd03cf298662e70ece89c23088
+F src/os_win.c 0a4afa35cc8e812000df3ea2f64b476131b39e29e75d8007d0504726e4761de4
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
@@ -1679,7 +1679,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P bab9de7fdda20a724f7c21ec2c25b488ece08b685f1e4fd15c9e73b6b4a0133a
-R 693906cf332f161a7ecc56eb7becf7aa
-U dan
-Z fd6217bb3206aa8caa41c580e04a37c2
+P 92fc146bc2b781e7e2d7138b00e5ea649c6fee1c2b8449420460a1b3e5c9661b
+R 001c7e37a1fb8cc134a2537ef8c0ad1e
+U drh
+Z d178c4b2ec00ed03598e2a437f7153cc
diff --git a/manifest.uuid b/manifest.uuid
index 88730c3b6a..a972d08bb5 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-92fc146bc2b781e7e2d7138b00e5ea649c6fee1c2b8449420460a1b3e5c9661b
\ No newline at end of file
+9cede8a83ca4cd88d504050115e1e89e7b3d3cd3cb2ffb5b8961e311a23ff5e2
\ No newline at end of file
diff --git a/src/os_unix.c b/src/os_unix.c
index 6ba4a5645a..3c2f15ed05 100644
--- a/src/os_unix.c
+++ b/src/os_unix.c
@@ -4165,7 +4165,7 @@ static int unixShmSystemLock(
 
   /* Access to the unixShmNode object is serialized by the caller */
   pShmNode = pFile->pInode->pShmNode;
-  assert( sqlite3_mutex_held(pShmNode->mutex) || pShmNode->nRef==0 );
+  assert( pShmNode->nRef==0 || sqlite3_mutex_held(pShmNode->mutex) );
 
   /* Shared locks never span more than one byte */
   assert( n==1 || lockType!=F_RDLCK );
diff --git a/src/os_win.c b/src/os_win.c
index 086bbf90aa..e3243a9c68 100644
--- a/src/os_win.c
+++ b/src/os_win.c
@@ -3742,7 +3742,7 @@ static int winShmSystemLock(
   int rc = 0;           /* Result code form Lock/UnlockFileEx() */
 
   /* Access to the winShmNode object is serialized by the caller */
-  assert( sqlite3_mutex_held(pFile->mutex) || pFile->nRef==0 );
+  assert( pFile->nRef==0 || sqlite3_mutex_held(pFile->mutex) );
 
   OSTRACE(("SHM-LOCK file=%p, lock=%d, offset=%d, size=%d\n",
            pFile->hFile.h, lockType, ofst, nByte));

From 62128540ef1746f03902280c8ad04ce59f727a4d Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 13 Dec 2017 14:22:48 +0000
Subject: [PATCH 140/488] Add script to amalgamate all LSM files into "lsm1.c".

FossilOrigin-Name: e32b69d73062e233b0ac853611d10b24546a346a603289ab0e339a3604ae2af4
---
 ext/lsm1/tool/mklsm1c.tcl | 88 +++++++++++++++++++++++++++++++++++++++
 main.mk                   | 23 ++++++++++
 manifest                  | 15 +++----
 manifest.uuid             |  2 +-
 4 files changed, 120 insertions(+), 8 deletions(-)
 create mode 100644 ext/lsm1/tool/mklsm1c.tcl

diff --git a/ext/lsm1/tool/mklsm1c.tcl b/ext/lsm1/tool/mklsm1c.tcl
new file mode 100644
index 0000000000..d4a317b700
--- /dev/null
+++ b/ext/lsm1/tool/mklsm1c.tcl
@@ -0,0 +1,88 @@
+#!/bin/sh
+# restart with tclsh \
+exec tclsh "$0" "$@"
+
+set srcdir [file dirname [file dirname [info script]]]
+set G(src) [string map [list %dir% $srcdir] {
+  %dir%/lsm.h
+  %dir%/lsmInt.h
+  %dir%/lsm_vtab.c
+  %dir%/lsm_ckpt.c
+  %dir%/lsm_file.c
+  %dir%/lsm_log.c
+  %dir%/lsm_main.c
+  %dir%/lsm_mem.c
+  %dir%/lsm_mutex.c
+  %dir%/lsm_shared.c
+  %dir%/lsm_sorted.c
+  %dir%/lsm_str.c
+  %dir%/lsm_tree.c
+  %dir%/lsm_unix.c
+  %dir%/lsm_varint.c
+  %dir%/lsm_win32.c
+}]
+
+set G(hdr) {
+
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_LSM1) 
+
+#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) 
+# define NDEBUG 1
+#endif
+#if defined(NDEBUG) && defined(SQLITE_DEBUG)
+# undef NDEBUG
+#endif
+
+}
+
+set G(footer) {
+    
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_LSM1) */
+}
+
+#-------------------------------------------------------------------------
+# Read and return the entire contents of text file $zFile from disk.
+#
+proc readfile {zFile} {
+  set fd [open $zFile]
+  set data [read $fd]
+  close $fd
+  return $data
+}
+
+proc lsm1c_init {zOut} {
+  global G
+  set G(fd) stdout
+  set G(fd) [open $zOut w]
+
+  puts -nonewline $G(fd) $G(hdr)
+}
+
+proc lsm1c_printfile {zIn} {
+  global G
+  set data [readfile $zIn]
+  set zTail [file tail $zIn]
+  puts $G(fd) "#line 1 \"$zTail\""
+
+  foreach line [split $data "\n"] {
+    if {[regexp {^# *include.*lsm} $line]} {
+      set line "/* $line */"
+    } elseif { [regexp {^(const )?[a-zA-Z][a-zA-Z0-9]* [*]?lsm[^_]} $line] } {
+      set line "static $line"
+    }
+    puts $G(fd) $line
+  }
+}
+
+proc lsm1c_close {} {
+  global G
+  puts -nonewline $G(fd) $G(footer)
+  if {$G(fd)!="stdout"} {
+    close $G(fd)
+  }
+}
+
+
+lsm1c_init lsm1.c
+foreach f $G(src) { lsm1c_printfile $f }
+lsm1c_close
diff --git a/main.mk b/main.mk
index d8660fc17b..e978b23a58 100644
--- a/main.mk
+++ b/main.mk
@@ -263,6 +263,24 @@ FTS5_SRC = \
    $(TOP)/ext/fts5/fts5_varint.c \
    $(TOP)/ext/fts5/fts5_vocab.c  \
 
+LSM1_SRC = \
+   $(TOP)/ext/lsm1/lsm.h \
+   $(TOP)/ext/lsm1/lsmInt.h \
+   $(TOP)/ext/lsm1/lsm_ckpt.c \
+   $(TOP)/ext/lsm1/lsm_file.c \
+   $(TOP)/ext/lsm1/lsm_log.c \
+   $(TOP)/ext/lsm1/lsm_main.c \
+   $(TOP)/ext/lsm1/lsm_mem.c \
+   $(TOP)/ext/lsm1/lsm_mutex.c \
+   $(TOP)/ext/lsm1/lsm_shared.c \
+   $(TOP)/ext/lsm1/lsm_sorted.c \
+   $(TOP)/ext/lsm1/lsm_str.c \
+   $(TOP)/ext/lsm1/lsm_tree.c \
+   $(TOP)/ext/lsm1/lsm_unix.c \
+   $(TOP)/ext/lsm1/lsm_varint.c \
+   $(TOP)/ext/lsm1/lsm_vtab.c \
+   $(TOP)/ext/lsm1/lsm_win32.c
+
 
 # Generated source code files
 #
@@ -766,6 +784,10 @@ fts5.c: $(FTS5_SRC) $(FTS5_HDR)
 	tclsh $(TOP)/ext/fts5/tool/mkfts5c.tcl
 	cp $(TOP)/ext/fts5/fts5.h .
 
+lsm1.c: $(LSM1_SRC)
+	tclsh $(TOP)/ext/lsm1/tool/mklsm1c.tcl
+	cp $(TOP)/ext/lsm1/lsm.h .
+
 userauth.o:	$(TOP)/ext/userauth/userauth.c $(HDR) $(EXTHDR)
 	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/userauth/userauth.c
 
@@ -1019,3 +1041,4 @@ clean:
 	rm -f fuzzcheck fuzzcheck.exe
 	rm -f sqldiff sqldiff.exe
 	rm -f fts5.* fts5parse.*
+	rm -f lsm.h lsm1.c
diff --git a/manifest b/manifest
index cdf085dc66..cad9609c6b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Minor\senhancement\sto\stwo\sassert()\sstatements\sin\sthe\sdefault\sVFSes.
-D 2017-12-13T10:11:09.727
+C Add\sscript\sto\samalgamate\sall\sLSM\sfiles\sinto\s"lsm1.c".
+D 2017-12-13T14:22:48.449
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
@@ -256,6 +256,7 @@ F ext/lsm1/lsm_vtab.c 529255dc704289001b225d97e57e0cfa14b29c3f281c7349cfa8fdb655
 F ext/lsm1/lsm_win32.c 0a4acbd7e8d136dd3a5753f0a9e7a9802263a9d96cef3278cf120bcaa724db7c
 F ext/lsm1/test/lsm1_common.tcl 5ed4bab07c93be2e4f300ebe46007ecf4b3e20bc5fbe1dedaf04a8774a6d8d82
 F ext/lsm1/test/lsm1_simple.test ca949efefa102f4644231dcd9291d8cda7699a4ce1006b26e0e3fcb72233f422
+F ext/lsm1/tool/mklsm1c.tcl f31561bbee5349f0a554d1ad7236ac1991fc09176626f529f6078e07335398b0
 F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601d90b2
 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87
 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb
@@ -394,7 +395,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk fbe15be384ec172be0cc30efc91cda61ca16bd5d833e8b812cf653ccb0c74977
+F main.mk 6123b0b2db806ddb482c24786ad6603a289df720382a3bce8f532d76a94c84b1
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -1679,7 +1680,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 92fc146bc2b781e7e2d7138b00e5ea649c6fee1c2b8449420460a1b3e5c9661b
-R 001c7e37a1fb8cc134a2537ef8c0ad1e
-U drh
-Z d178c4b2ec00ed03598e2a437f7153cc
+P 9cede8a83ca4cd88d504050115e1e89e7b3d3cd3cb2ffb5b8961e311a23ff5e2
+R 0bb1b86efd302c3523a2137056206a8a
+U dan
+Z 0df44b61ec124ac51afd1eb16b579be7
diff --git a/manifest.uuid b/manifest.uuid
index a972d08bb5..f9c2f8f2b4 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-9cede8a83ca4cd88d504050115e1e89e7b3d3cd3cb2ffb5b8961e311a23ff5e2
\ No newline at end of file
+e32b69d73062e233b0ac853611d10b24546a346a603289ab0e339a3604ae2af4
\ No newline at end of file

From 34aebb88da49697c411b64c8e81e17486f26a513 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 13 Dec 2017 15:32:33 +0000
Subject: [PATCH 141/488] Change some LSM code internal typenames from "Blob"
 and "Pgno" to "LsmBlob" and "LsmPgno". This allows the LSM code to be
 compiled as a single unit with sqlite3.c.

FossilOrigin-Name: 30bf38d589adf0b2eb613e184ffb03ed7e625736cb04fd5c341328f72bc5ef4d
---
 ext/lsm1/lsmInt.h     |  36 +++++-----
 ext/lsm1/lsm_ckpt.c   |   2 +-
 ext/lsm1/lsm_file.c   | 141 +++++++++++++++++++------------------
 ext/lsm1/lsm_main.c   |   6 +-
 ext/lsm1/lsm_sorted.c | 160 +++++++++++++++++++++---------------------
 manifest              |  20 +++---
 manifest.uuid         |   2 +-
 7 files changed, 186 insertions(+), 181 deletions(-)

diff --git a/ext/lsm1/lsmInt.h b/ext/lsm1/lsmInt.h
index b346d8dded..0f822e4793 100644
--- a/ext/lsm1/lsmInt.h
+++ b/ext/lsm1/lsmInt.h
@@ -110,7 +110,7 @@ typedef unsigned long long int u64;
 #endif
 
 /* A page number is a 64-bit integer. */
-typedef i64 Pgno;
+typedef i64 LsmPgno;
 
 #ifdef LSM_DEBUG
 int lsmErrorBkpt(int);
@@ -402,9 +402,9 @@ struct lsm_db {
 };
 
 struct Segment {
-  Pgno iFirst;                     /* First page of this run */
-  Pgno iLastPg;                    /* Last page of this run */
-  Pgno iRoot;                      /* Root page number (if any) */
+  LsmPgno iFirst;                  /* First page of this run */
+  LsmPgno iLastPg;                 /* Last page of this run */
+  LsmPgno iRoot;                   /* Root page number (if any) */
   int nSize;                       /* Size of this run in pages */
 
   Redirect *pRedirect;             /* Block redirects (or NULL) */
@@ -456,7 +456,7 @@ struct Level {
 **   output segment.
 */
 struct MergeInput {
-  Pgno iPg;                       /* Page on which next input is stored */
+  LsmPgno iPg;                    /* Page on which next input is stored */
   int iCell;                      /* Cell containing next input to merge */
 };
 struct Merge {
@@ -465,7 +465,7 @@ struct Merge {
   MergeInput splitkey;            /* Location in file of current splitkey */
   int nSkip;                      /* Number of separators entries to skip */
   int iOutputOff;                 /* Write offset on output page */
-  Pgno iCurrentPtr;               /* Current pointer value */
+  LsmPgno iCurrentPtr;            /* Current pointer value */
 };
 
 /* 
@@ -579,10 +579,10 @@ struct Snapshot {
   Redirect redirect;              /* Block redirection array */
 
   /* Used by worker snapshots only */
-  int nBlock;                     /* Number of blocks in database file */
-  Pgno aiAppend[LSM_APPLIST_SZ];  /* Append point list */
-  Freelist freelist;              /* Free block list */
-  u32 nWrite;                     /* Total number of pages written to disk */
+  int nBlock;                        /* Number of blocks in database file */
+  LsmPgno aiAppend[LSM_APPLIST_SZ];  /* Append point list */
+  Freelist freelist;                 /* Free block list */
+  u32 nWrite;                        /* Total number of pages written to disk */
 };
 #define LSM_INITIAL_SNAPSHOT_ID 11
 
@@ -710,7 +710,7 @@ void lsmFsSetPageSize(FileSystem *, int);
 int lsmFsFileid(lsm_db *pDb, void **ppId, int *pnId);
 
 /* Creating, populating, gobbling and deleting sorted runs. */
-void lsmFsGobble(lsm_db *, Segment *, Pgno *, int);
+void lsmFsGobble(lsm_db *, Segment *, LsmPgno *, int);
 int lsmFsSortedDelete(FileSystem *, Snapshot *, int, Segment *);
 int lsmFsSortedFinish(FileSystem *, Segment *);
 int lsmFsSortedAppend(FileSystem *, Snapshot *, Level *, int, Page **);
@@ -727,14 +727,14 @@ void lsmSortedSplitkey(lsm_db *, Level *, int *);
 
 /* Reading sorted run content. */
 int lsmFsDbPageLast(FileSystem *pFS, Segment *pSeg, Page **ppPg);
-int lsmFsDbPageGet(FileSystem *, Segment *, Pgno, Page **);
+int lsmFsDbPageGet(FileSystem *, Segment *, LsmPgno, Page **);
 int lsmFsDbPageNext(Segment *, Page *, int eDir, Page **);
 
 u8 *lsmFsPageData(Page *, int *);
 int lsmFsPageRelease(Page *);
 int lsmFsPagePersist(Page *);
 void lsmFsPageRef(Page *);
-Pgno lsmFsPageNumber(Page *);
+LsmPgno lsmFsPageNumber(Page *);
 
 int lsmFsNRead(FileSystem *);
 int lsmFsNWrite(FileSystem *);
@@ -748,7 +748,7 @@ int lsmFsDbPageIsLast(Segment *pSeg, Page *pPg);
 int lsmFsIntegrityCheck(lsm_db *);
 #endif
 
-Pgno lsmFsRedirectPage(FileSystem *, Redirect *, Pgno);
+LsmPgno lsmFsRedirectPage(FileSystem *, Redirect *, LsmPgno);
 
 int lsmFsPageWritable(Page *);
 
@@ -768,8 +768,8 @@ int lsmFsSyncDb(FileSystem *, int);
 void lsmFsFlushWaiting(FileSystem *, int *);
 
 /* Used by lsm_info(ARRAY_STRUCTURE) and lsm_config(MMAP) */
-int lsmInfoArrayStructure(lsm_db *pDb, int bBlock, Pgno iFirst, char **pzOut);
-int lsmInfoArrayPages(lsm_db *pDb, Pgno iFirst, char **pzOut);
+int lsmInfoArrayStructure(lsm_db *pDb, int bBlock, LsmPgno iFirst, char **pz);
+int lsmInfoArrayPages(lsm_db *pDb, LsmPgno iFirst, char **pzOut);
 int lsmConfigMmap(lsm_db *pDb, int *piParam);
 
 int lsmEnvOpen(lsm_env *, const char *, int, lsm_file **);
@@ -785,7 +785,7 @@ void lsmEnvSleep(lsm_env *, int);
 
 int lsmFsReadSyncedId(lsm_db *db, int, i64 *piVal);
 
-int lsmFsSegmentContainsPg(FileSystem *pFS, Segment *, Pgno, int *);
+int lsmFsSegmentContainsPg(FileSystem *pFS, Segment *, LsmPgno, int *);
 
 void lsmFsPurgeCache(FileSystem *);
 
@@ -796,7 +796,7 @@ void lsmFsPurgeCache(FileSystem *);
 /* 
 ** Functions from file "lsm_sorted.c".
 */
-int lsmInfoPageDump(lsm_db *, Pgno, int, char **);
+int lsmInfoPageDump(lsm_db *, LsmPgno, int, char **);
 void lsmSortedCleanup(lsm_db *);
 int lsmSortedAutoWork(lsm_db *, int nUnit);
 
diff --git a/ext/lsm1/lsm_ckpt.c b/ext/lsm1/lsm_ckpt.c
index cf4c55bf84..ba92a823cf 100644
--- a/ext/lsm1/lsm_ckpt.c
+++ b/ext/lsm1/lsm_ckpt.c
@@ -389,7 +389,7 @@ static void ckptExportAppendlist(
   int *pRc                        /* IN/OUT: Error code */
 ){
   int i;
-  Pgno *aiAppend = db->pWorker->aiAppend;
+  LsmPgno *aiAppend = db->pWorker->aiAppend;
 
   for(i=0; i<LSM_APPLIST_SZ; i++){
     ckptAppend64(p, piOut, aiAppend[i], pRc);
diff --git a/ext/lsm1/lsm_file.c b/ext/lsm1/lsm_file.c
index 4001aac978..1dcdd05d99 100644
--- a/ext/lsm1/lsm_file.c
+++ b/ext/lsm1/lsm_file.c
@@ -269,7 +269,7 @@ struct FileSystem {
 struct Page {
   u8 *aData;                      /* Buffer containing page data */
   int nData;                      /* Bytes of usable data at aData[] */
-  Pgno iPg;                       /* Page number */
+  LsmPgno iPg;                    /* Page number */
   int nRef;                       /* Number of outstanding references */
   int flags;                      /* Combination of PAGE_XXX flags */
   Page *pHashNext;                /* Next page in hash table slot */
@@ -332,7 +332,7 @@ static int IOERR_WRAPPER(int rc){
 #ifdef NDEBUG
 # define assert_lists_are_ok(x)
 #else
-static Page *fsPageFindInHash(FileSystem *pFS, Pgno iPg, int *piHash);
+static Page *fsPageFindInHash(FileSystem *pFS, LsmPgno iPg, int *piHash);
 
 static void assert_lists_are_ok(FileSystem *pFS){
 #if 0
@@ -532,7 +532,7 @@ int lsmFsCloseAndDeleteLog(FileSystem *pFS){
 ** Return true if page iReal of the database should be accessed using mmap.
 ** False otherwise.
 */
-static int fsMmapPage(FileSystem *pFS, Pgno iReal){
+static int fsMmapPage(FileSystem *pFS, LsmPgno iReal){
   return ((i64)iReal*pFS->nPagesize <= pFS->nMapLimit);
 }
 
@@ -540,7 +540,7 @@ static int fsMmapPage(FileSystem *pFS, Pgno iReal){
 ** Given that there are currently nHash slots in the hash table, return 
 ** the hash key for file iFile, page iPg.
 */
-static int fsHashKey(int nHash, Pgno iPg){
+static int fsHashKey(int nHash, LsmPgno iPg){
   return (iPg % nHash);
 }
 
@@ -880,13 +880,13 @@ void lsmFsSetBlockSize(FileSystem *pFS, int nBlocksize){
 ** page on each block is the byte offset immediately following the 4-byte
 ** "previous block" pointer at the start of each block.
 */
-static Pgno fsFirstPageOnBlock(FileSystem *pFS, int iBlock){
-  Pgno iPg;
+static LsmPgno fsFirstPageOnBlock(FileSystem *pFS, int iBlock){
+  LsmPgno iPg;
   if( pFS->pCompress ){
     if( iBlock==1 ){
       iPg = pFS->nMetasize * 2 + 4;
     }else{
-      iPg = pFS->nBlocksize * (Pgno)(iBlock-1) + 4;
+      iPg = pFS->nBlocksize * (LsmPgno)(iBlock-1) + 4;
     }
   }else{
     const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize);
@@ -907,9 +907,9 @@ static Pgno fsFirstPageOnBlock(FileSystem *pFS, int iBlock){
 ** page on each block is the byte offset of the byte immediately before 
 ** the 4-byte "next block" pointer at the end of each block.
 */
-static Pgno fsLastPageOnBlock(FileSystem *pFS, int iBlock){
+static LsmPgno fsLastPageOnBlock(FileSystem *pFS, int iBlock){
   if( pFS->pCompress ){
-    return pFS->nBlocksize * (Pgno)iBlock - 1 - 4;
+    return pFS->nBlocksize * (LsmPgno)iBlock - 1 - 4;
   }else{
     const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize);
     return iBlock * nPagePerBlock;
@@ -920,7 +920,7 @@ static Pgno fsLastPageOnBlock(FileSystem *pFS, int iBlock){
 ** Return the block number of the block that page iPg is located on. 
 ** Blocks are numbered starting from 1.
 */
-static int fsPageToBlock(FileSystem *pFS, Pgno iPg){
+static int fsPageToBlock(FileSystem *pFS, LsmPgno iPg){
   if( pFS->pCompress ){
     return (int)((iPg / pFS->nBlocksize) + 1);
   }else{
@@ -933,7 +933,7 @@ static int fsPageToBlock(FileSystem *pFS, Pgno iPg){
 **
 ** This function is only called in non-compressed database mode.
 */
-static int fsIsLast(FileSystem *pFS, Pgno iPg){
+static int fsIsLast(FileSystem *pFS, LsmPgno iPg){
   const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize);
   assert( !pFS->pCompress );
   return ( iPg && (iPg % nPagePerBlock)==0 );
@@ -944,7 +944,7 @@ static int fsIsLast(FileSystem *pFS, Pgno iPg){
 **
 ** This function is only called in non-compressed database mode.
 */
-static int fsIsFirst(FileSystem *pFS, Pgno iPg){
+static int fsIsFirst(FileSystem *pFS, LsmPgno iPg){
   const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize);
   assert( !pFS->pCompress );
   return ( (iPg % nPagePerBlock)==1
@@ -967,7 +967,7 @@ u8 *lsmFsPageData(Page *pPage, int *pnData){
 /*
 ** Return the page number of a page.
 */
-Pgno lsmFsPageNumber(Page *pPage){
+LsmPgno lsmFsPageNumber(Page *pPage){
   /* assert( (pPage->flags & PAGE_DIRTY)==0 ); */
   return pPage ? pPage->iPg : 0;
 }
@@ -1058,7 +1058,7 @@ void lsmFsPurgeCache(FileSystem *pFS){
 ** Either way, if argument piHash is not NULL set *piHash to the hash slot
 ** number that page iPg would be stored in before returning.
 */
-static Page *fsPageFindInHash(FileSystem *pFS, Pgno iPg, int *piHash){
+static Page *fsPageFindInHash(FileSystem *pFS, LsmPgno iPg, int *piHash){
   Page *p;                        /* Return value */
   int iHash = fsHashKey(pFS->nHash, iPg);
 
@@ -1189,8 +1189,8 @@ static int fsRedirectBlock(Redirect *p, int iBlk){
 ** object passed as the second argument, return the destination page to
 ** which it is redirected. Otherwise, return a copy of iPg.
 */
-Pgno lsmFsRedirectPage(FileSystem *pFS, Redirect *pRedir, Pgno iPg){
-  Pgno iReal = iPg;
+LsmPgno lsmFsRedirectPage(FileSystem *pFS, Redirect *pRedir, LsmPgno iPg){
+  LsmPgno iReal = iPg;
 
   if( pRedir ){
     const int nPagePerBlock = (
@@ -1203,7 +1203,7 @@ Pgno lsmFsRedirectPage(FileSystem *pFS, Redirect *pRedir, Pgno iPg){
       if( iFrom>iBlk ) break;
       if( iFrom==iBlk ){
         int iTo = pRedir->a[i].iTo;
-        iReal = iPg - (Pgno)(iFrom - iTo) * nPagePerBlock;
+        iReal = iPg - (LsmPgno)(iFrom - iTo) * nPagePerBlock;
         if( iTo==1 ){
           iReal += (fsFirstPageOnBlock(pFS, 1)-1);
         }
@@ -1217,7 +1217,7 @@ Pgno lsmFsRedirectPage(FileSystem *pFS, Redirect *pRedir, Pgno iPg){
 }
 
 /* Required by the circular fsBlockNext<->fsPageGet dependency. */
-static int fsPageGet(FileSystem *, Segment *, Pgno, int, Page **, int *);
+static int fsPageGet(FileSystem *, Segment *, LsmPgno, int, Page **, int *);
 
 /*
 ** Parameter iBlock is a database file block. This function reads the value 
@@ -1269,7 +1269,7 @@ static int fsBlockNext(
 /*
 ** Return the page number of the last page on the same block as page iPg.
 */
-Pgno fsLastPageOnPagesBlock(FileSystem *pFS, Pgno iPg){
+LsmPgno fsLastPageOnPagesBlock(FileSystem *pFS, LsmPgno iPg){
   return fsLastPageOnBlock(pFS, fsPageToBlock(pFS, iPg));
 }
 
@@ -1537,7 +1537,7 @@ static int fsReadPagedata(
 static int fsPageGet(
   FileSystem *pFS,                /* File-system handle */
   Segment *pSeg,                  /* Block redirection to use (or NULL) */
-  Pgno iPg,                       /* Page id */
+  LsmPgno iPg,                    /* Page id */
   int noContent,                  /* True to not load content from disk */
   Page **ppPg,                    /* OUT: New page handle */
   int *pnSpace                    /* OUT: Bytes of free space */
@@ -1549,7 +1549,7 @@ static int fsPageGet(
   /* In most cases iReal is the same as iPg. Except, if pSeg->pRedirect is 
   ** not NULL, and the block containing iPg has been redirected, then iReal
   ** is the page number after redirection.  */
-  Pgno iReal = lsmFsRedirectPage(pFS, (pSeg ? pSeg->pRedirect : 0), iPg);
+  LsmPgno iReal = lsmFsRedirectPage(pFS, (pSeg ? pSeg->pRedirect : 0), iPg);
 
   assert_lists_are_ok(pFS);
   assert( iPg>=fsFirstPageOnBlock(pFS, 1) );
@@ -1689,8 +1689,8 @@ int lsmFsReadSyncedId(lsm_db *db, int iMeta, i64 *piVal){
 static int fsRunEndsBetween(
   Segment *pRun, 
   Segment *pIgnore, 
-  Pgno iFirst, 
-  Pgno iLast
+  LsmPgno iFirst, 
+  LsmPgno iLast
 ){
   return (pRun!=pIgnore && (
         (pRun->iFirst>=iFirst && pRun->iFirst<=iLast)
@@ -1705,8 +1705,8 @@ static int fsRunEndsBetween(
 static int fsLevelEndsBetween(
   Level *pLevel, 
   Segment *pIgnore, 
-  Pgno iFirst, 
-  Pgno iLast
+  LsmPgno iFirst, 
+  LsmPgno iLast
 ){
   int i;
 
@@ -1733,13 +1733,13 @@ static int fsFreeBlock(
   int iBlk                        /* Block number of block to free */
 ){
   int rc = LSM_OK;                /* Return code */
-  Pgno iFirst;                    /* First page on block iBlk */
-  Pgno iLast;                     /* Last page on block iBlk */
+  LsmPgno iFirst;                 /* First page on block iBlk */
+  LsmPgno iLast;                  /* Last page on block iBlk */
   Level *pLevel;                  /* Used to iterate through levels */
 
   int iIn;                        /* Used to iterate through append points */
   int iOut = 0;                   /* Used to output append points */
-  Pgno *aApp = pSnapshot->aiAppend;
+  LsmPgno *aApp = pSnapshot->aiAppend;
 
   iFirst = fsFirstPageOnBlock(pFS, iBlk);
   iLast = fsLastPageOnBlock(pFS, iBlk);
@@ -1811,11 +1811,16 @@ int lsmFsSortedDelete(
 ** number from the array that falls on block iBlk. Or, if none of the pages
 ** in aPgno[] fall on block iBlk, return 0.
 */
-static Pgno firstOnBlock(FileSystem *pFS, int iBlk, Pgno *aPgno, int nPgno){
-  Pgno iRet = 0;
+static LsmPgno firstOnBlock(
+  FileSystem *pFS, 
+  int iBlk, 
+  LsmPgno *aPgno, 
+  int nPgno
+){
+  LsmPgno iRet = 0;
   int i;
   for(i=0; i<nPgno; i++){
-    Pgno iPg = aPgno[i];
+    LsmPgno iPg = aPgno[i];
     if( fsPageToBlock(pFS, iPg)==iBlk && (iRet==0 || iPg<iRet) ){
       iRet = iPg;
     }
@@ -1828,7 +1833,7 @@ static Pgno firstOnBlock(FileSystem *pFS, int iBlk, Pgno *aPgno, int nPgno){
 ** Return true if page iPg, which is a part of segment p, lies on
 ** a redirected block. 
 */
-static int fsPageRedirects(FileSystem *pFS, Segment *p, Pgno iPg){
+static int fsPageRedirects(FileSystem *pFS, Segment *p, LsmPgno iPg){
   return (iPg!=0 && iPg!=lsmFsRedirectPage(pFS, p->pRedirect, iPg));
 }
 
@@ -1854,7 +1859,7 @@ static int fsSegmentRedirects(FileSystem *pFS, Segment *p){
 void lsmFsGobble(
   lsm_db *pDb,
   Segment *pRun, 
-  Pgno *aPgno,
+  LsmPgno *aPgno,
   int nPgno
 ){
   int rc = LSM_OK;
@@ -1871,7 +1876,7 @@ void lsmFsGobble(
 
   while( rc==LSM_OK ){
     int iNext = 0;
-    Pgno iFirst = firstOnBlock(pFS, iBlk, aPgno, nPgno);
+    LsmPgno iFirst = firstOnBlock(pFS, iBlk, aPgno, nPgno);
     if( iFirst ){
       pRun->iFirst = iFirst;
       break;
@@ -1905,11 +1910,11 @@ void lsmFsGobble(
 static int fsNextPageOffset(
   FileSystem *pFS,                /* File system object */
   Segment *pSeg,                  /* Segment to move within */
-  Pgno iPg,                       /* Offset of current page */
+  LsmPgno iPg,                    /* Offset of current page */
   int nByte,                      /* Size of current page including headers */
-  Pgno *piNext                    /* OUT: Offset of next page. Or zero (EOF) */
+  LsmPgno *piNext                 /* OUT: Offset of next page. Or zero (EOF) */
 ){
-  Pgno iNext;
+  LsmPgno iNext;
   int rc;
 
   assert( pFS->pCompress );
@@ -1939,8 +1944,8 @@ static int fsNextPageOffset(
 static int fsGetPageBefore(
   FileSystem *pFS, 
   Segment *pSeg, 
-  Pgno iPg, 
-  Pgno *piPrev
+  LsmPgno iPg, 
+  LsmPgno *piPrev
 ){
   u8 aSz[3];
   int rc;
@@ -1990,7 +1995,7 @@ static int fsGetPageBefore(
 int lsmFsDbPageNext(Segment *pRun, Page *pPg, int eDir, Page **ppNext){
   int rc = LSM_OK;
   FileSystem *pFS = pPg->pFS;
-  Pgno iPg = pPg->iPg;
+  LsmPgno iPg = pPg->iPg;
 
   assert( 0==fsSegmentRedirects(pFS, pRun) );
   if( pFS->pCompress ){
@@ -2062,10 +2067,10 @@ int lsmFsDbPageNext(Segment *pRun, Page *pPg, int eDir, Page **ppNext){
 ** start the new segment immediately following any segment that is part
 ** of the right-hand-side of pLvl.
 */
-static Pgno findAppendPoint(FileSystem *pFS, Level *pLvl){
+static LsmPgno findAppendPoint(FileSystem *pFS, Level *pLvl){
   int i;
-  Pgno *aiAppend = pFS->pDb->pWorker->aiAppend;
-  Pgno iRet = 0;
+  LsmPgno *aiAppend = pFS->pDb->pWorker->aiAppend;
+  LsmPgno iRet = 0;
 
   for(i=LSM_APPLIST_SZ-1; iRet==0 && i>=0; i--){
     if( (iRet = aiAppend[i]) ){
@@ -2098,10 +2103,10 @@ int lsmFsSortedAppend(
 ){
   int rc = LSM_OK;
   Page *pPg = 0;
-  Pgno iApp = 0;
-  Pgno iNext = 0;
+  LsmPgno iApp = 0;
+  LsmPgno iNext = 0;
   Segment *p = &pLvl->lhs;
-  Pgno iPrev = p->iLastPg;
+  LsmPgno iPrev = p->iLastPg;
 
   *ppOut = 0;
   assert( p->pRedirect==0 );
@@ -2195,7 +2200,7 @@ int lsmFsSortedFinish(FileSystem *pFS, Segment *p){
     */
     if( fsLastPageOnPagesBlock(pFS, p->iLastPg)!=p->iLastPg ){
       int i;
-      Pgno *aiAppend = pFS->pDb->pWorker->aiAppend;
+      LsmPgno *aiAppend = pFS->pDb->pWorker->aiAppend;
       for(i=0; i<LSM_APPLIST_SZ; i++){
         if( aiAppend[i]==0 ){
           aiAppend[i] = p->iLastPg+1;
@@ -2226,7 +2231,7 @@ int lsmFsSortedFinish(FileSystem *pFS, Segment *p){
 **
 ** Return LSM_OK if successful, or an lsm error code if an error occurs.
 */
-int lsmFsDbPageGet(FileSystem *pFS, Segment *pSeg, Pgno iPg, Page **ppPg){
+int lsmFsDbPageGet(FileSystem *pFS, Segment *pSeg, LsmPgno iPg, Page **ppPg){
   return fsPageGet(pFS, pSeg, iPg, 0, ppPg, 0);
 }
 
@@ -2238,7 +2243,7 @@ int lsmFsDbPageGet(FileSystem *pFS, Segment *pSeg, Pgno iPg, Page **ppPg){
 */
 int lsmFsDbPageLast(FileSystem *pFS, Segment *pSeg, Page **ppPg){
   int rc;
-  Pgno iPg = pSeg->iLastPg;
+  LsmPgno iPg = pSeg->iLastPg;
   if( pFS->pCompress ){
     int nSpace;
     iPg++;
@@ -2366,14 +2371,14 @@ static void fsMovePage(
   FileSystem *pFS,                /* File system object */
   int iTo,                        /* Destination block */
   int iFrom,                      /* Source block */
-  Pgno *piPg                      /* IN/OUT: Page number */
+  LsmPgno *piPg                   /* IN/OUT: Page number */
 ){
-  Pgno iPg = *piPg;
+  LsmPgno iPg = *piPg;
   if( iFrom==fsPageToBlock(pFS, iPg) ){
     const int nPagePerBlock = (
         pFS->pCompress ? pFS ->nBlocksize : (pFS->nBlocksize / pFS->nPagesize)
     );
-    *piPg = iPg - (Pgno)(iFrom - iTo) * nPagePerBlock;
+    *piPg = iPg - (LsmPgno)(iFrom - iTo) * nPagePerBlock;
   }
 }
 
@@ -2457,21 +2462,21 @@ int lsmFsMoveBlock(FileSystem *pFS, Segment *pSeg, int iTo, int iFrom){
 **
 ** This function is only used in compressed database mode.
 */
-static Pgno fsAppendData(
+static LsmPgno fsAppendData(
   FileSystem *pFS,                /* File-system handle */
   Segment *pSeg,                  /* Segment to append to */
   const u8 *aData,                /* Buffer containing data to write */
   int nData,                      /* Size of buffer aData[] in bytes */
   int *pRc                        /* IN/OUT: Error code */
 ){
-  Pgno iRet = 0;
+  LsmPgno iRet = 0;
   int rc = *pRc;
   assert( pFS->pCompress );
   if( rc==LSM_OK ){
     int nRem = 0;
     int nWrite = 0;
-    Pgno iLastOnBlock;
-    Pgno iApp = pSeg->iLastPg+1;
+    LsmPgno iLastOnBlock;
+    LsmPgno iApp = pSeg->iLastPg+1;
 
     /* If this is the first data written into the segment, find an append-point
     ** or allocate a new block.  */
@@ -2519,7 +2524,7 @@ static Pgno fsAppendData(
 
         /* Set the "prev" pointer on the new block */
         if( rc==LSM_OK ){
-          Pgno iWrite;
+          LsmPgno iWrite;
           lsmPutU32(aPtr, fsPageToBlock(pFS, iApp));
           iWrite = fsFirstPageOnBlock(pFS, iBlk);
           rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iWrite-4, aPtr, sizeof(aPtr));
@@ -2588,11 +2593,11 @@ static int fsCompressIntoBuffer(FileSystem *pFS, Page *pPg){
 static int fsAppendPage(
   FileSystem *pFS, 
   Segment *pSeg,
-  Pgno *piNew,
+  LsmPgno *piNew,
   int *piPrev,
   int *piNext
 ){
-  Pgno iPrev = pSeg->iLastPg;
+  LsmPgno iPrev = pSeg->iLastPg;
   int rc;
   assert( iPrev!=0 );
 
@@ -2650,7 +2655,7 @@ void lsmFsFlushWaiting(FileSystem *pFS, int *pRc){
 /*
 ** If there exists a hash-table entry associated with page iPg, remove it.
 */
-static void fsRemoveHashEntry(FileSystem *pFS, Pgno iPg){
+static void fsRemoveHashEntry(FileSystem *pFS, LsmPgno iPg){
   Page *p;
   int iHash = fsHashKey(pFS->nHash, iPg);
 
@@ -2804,8 +2809,8 @@ int lsmFsSortedPadding(
 ){
   int rc = LSM_OK;
   if( pFS->pCompress && pSeg->iFirst ){
-    Pgno iLast2;
-    Pgno iLast = pSeg->iLastPg;     /* Current last page of segment */
+    LsmPgno iLast2;
+    LsmPgno iLast = pSeg->iLastPg;  /* Current last page of segment */
     int nPad;                       /* Bytes of padding required */
     u8 aSz[3];
 
@@ -2935,7 +2940,7 @@ int lsmFsSectorSize(FileSystem *pFS){
 /*
 ** Helper function for lsmInfoArrayStructure().
 */
-static Segment *startsWith(Segment *pRun, Pgno iFirst){
+static Segment *startsWith(Segment *pRun, LsmPgno iFirst){
   return (iFirst==pRun->iFirst) ? pRun : 0;
 }
 
@@ -2943,7 +2948,7 @@ static Segment *startsWith(Segment *pRun, Pgno iFirst){
 ** Return the segment that starts with page iFirst, if any. If no such segment
 ** can be found, return NULL.
 */
-static Segment *findSegment(Snapshot *pWorker, Pgno iFirst){
+static Segment *findSegment(Snapshot *pWorker, LsmPgno iFirst){
   Level *pLvl;                    /* Used to iterate through db levels */
   Segment *pSeg = 0;              /* Pointer to segment to return */
 
@@ -2970,7 +2975,7 @@ static Segment *findSegment(Snapshot *pWorker, Pgno iFirst){
 int lsmInfoArrayStructure(
   lsm_db *pDb, 
   int bBlock,                     /* True for block numbers only */
-  Pgno iFirst,
+  LsmPgno iFirst,
   char **pzOut
 ){
   int rc = LSM_OK;
@@ -3035,7 +3040,7 @@ int lsmInfoArrayStructure(
 int lsmFsSegmentContainsPg(
   FileSystem *pFS, 
   Segment *pSeg, 
-  Pgno iPg, 
+  LsmPgno iPg, 
   int *pbRes
 ){
   Redirect *pRedir = pSeg->pRedirect;
@@ -3064,7 +3069,7 @@ int lsmFsSegmentContainsPg(
 **
 ** If an error occurs, *pzOut is set to NULL and an LSM error code returned.
 */
-int lsmInfoArrayPages(lsm_db *pDb, Pgno iFirst, char **pzOut){
+int lsmInfoArrayPages(lsm_db *pDb, LsmPgno iFirst, char **pzOut){
   int rc = LSM_OK;
   Snapshot *pWorker;              /* Worker snapshot */
   Segment *pSeg = 0;              /* Array to report on */
@@ -3297,7 +3302,7 @@ int lsmFsIntegrityCheck(lsm_db *pDb){
 */
 int lsmFsDbPageIsLast(Segment *pSeg, Page *pPg){
   if( pPg->pFS->pCompress ){
-    Pgno iNext = 0;
+    LsmPgno iNext = 0;
     int rc;
     rc = fsNextPageOffset(pPg->pFS, pSeg, pPg->iPg, pPg->nCompress+6, &iNext);
     return (rc!=LSM_OK || iNext==0);
diff --git a/ext/lsm1/lsm_main.c b/ext/lsm1/lsm_main.c
index 8a324a3efe..539883c469 100644
--- a/ext/lsm1/lsm_main.c
+++ b/ext/lsm1/lsm_main.c
@@ -583,14 +583,14 @@ int lsm_info(lsm_db *pDb, int eParam, ...){
     }
 
     case LSM_INFO_ARRAY_STRUCTURE: {
-      Pgno pgno = va_arg(ap, Pgno);
+      LsmPgno pgno = va_arg(ap, LsmPgno);
       char **pzVal = va_arg(ap, char **);
       rc = lsmInfoArrayStructure(pDb, 0, pgno, pzVal);
       break;
     }
 
     case LSM_INFO_ARRAY_PAGES: {
-      Pgno pgno = va_arg(ap, Pgno);
+      LsmPgno pgno = va_arg(ap, LsmPgno);
       char **pzVal = va_arg(ap, char **);
       rc = lsmInfoArrayPages(pDb, pgno, pzVal);
       break;
@@ -598,7 +598,7 @@ int lsm_info(lsm_db *pDb, int eParam, ...){
 
     case LSM_INFO_PAGE_HEX_DUMP:
     case LSM_INFO_PAGE_ASCII_DUMP: {
-      Pgno pgno = va_arg(ap, Pgno);
+      LsmPgno pgno = va_arg(ap, LsmPgno);
       char **pzVal = va_arg(ap, char **);
       int bUnlock = 0;
       rc = infoGetWorker(pDb, 0, &bUnlock);
diff --git a/ext/lsm1/lsm_sorted.c b/ext/lsm1/lsm_sorted.c
index 9eff2d08cc..171d0ec05c 100644
--- a/ext/lsm1/lsm_sorted.c
+++ b/ext/lsm1/lsm_sorted.c
@@ -104,9 +104,9 @@
 #endif
 
 typedef struct SegmentPtr SegmentPtr;
-typedef struct Blob Blob;
+typedef struct LsmBlob LsmBlob;
 
-struct Blob {
+struct LsmBlob {
   lsm_env *pEnv;
   void *pData;
   int nData;
@@ -129,18 +129,18 @@ struct SegmentPtr {
   Page *pPg;                    /* Current page */
   u16 flags;                    /* Copy of page flags field */
   int nCell;                    /* Number of cells on pPg */
-  Pgno iPtr;                    /* Base cascade pointer */
+  LsmPgno iPtr;                 /* Base cascade pointer */
 
   /* Current cell. See segmentPtrLoadCell() */
   int iCell;                    /* Current record within page pPg */
   int eType;                    /* Type of current record */
-  Pgno iPgPtr;                  /* Cascade pointer offset */
+  LsmPgno iPgPtr;               /* Cascade pointer offset */
   void *pKey; int nKey;         /* Key associated with current record */
   void *pVal; int nVal;         /* Current record value (eType==WRITE only) */
 
   /* Blobs used to allocate buffers for pKey and pVal as required */
-  Blob blob1;
-  Blob blob2;
+  LsmBlob blob1;
+  LsmBlob blob2;
 };
 
 /*
@@ -171,10 +171,10 @@ struct BtreeCursor {
   void *pKey;
   int nKey;
   int eType;
-  Pgno iPtr;
+  LsmPgno iPtr;
 
   /* Storage for key, if not local */
-  Blob blob;
+  LsmBlob blob;
 };
 
 
@@ -203,8 +203,8 @@ struct MultiCursor {
   int flags;                      /* Mask of CURSOR_XXX flags */
 
   int eType;                      /* Cache of current key type */
-  Blob key;                       /* Cache of current key (or NULL) */
-  Blob val;                       /* Cache of current value */
+  LsmBlob key;                    /* Cache of current key (or NULL) */
+  LsmBlob val;                    /* Cache of current value */
 
   /* All the component cursors: */
   TreeCursor *apTreeCsr[2];       /* Up to two tree cursors */
@@ -221,7 +221,7 @@ struct MultiCursor {
   void *pSystemVal;               /* Pointer to buffer to free */
 
   /* Used by worker cursors only */
-  Pgno *pPrevMergePtr;
+  LsmPgno *pPrevMergePtr;
 };
 
 /*
@@ -295,11 +295,11 @@ struct MergeWorker {
   Hierarchy hier;                 /* B-tree hierarchy under construction */
   Page *pPage;                    /* Current output page */
   int nWork;                      /* Number of calls to mergeWorkerNextPage() */
-  Pgno *aGobble;                  /* Gobble point for each input segment */
+  LsmPgno *aGobble;               /* Gobble point for each input segment */
 
-  Pgno iIndirect;
+  LsmPgno iIndirect;
   struct SavedPgno {
-    Pgno iPgno;
+    LsmPgno iPgno;
     int bStore;
   } aSave[2];
 };
@@ -371,7 +371,7 @@ void lsmPutU64(u8 *aOut, u64 nVal){
   aOut[7] = (u8)((nVal    ) & 0xFF);
 }
 
-static int sortedBlobGrow(lsm_env *pEnv, Blob *pBlob, int nData){
+static int sortedBlobGrow(lsm_env *pEnv, LsmBlob *pBlob, int nData){
   assert( pBlob->pEnv==pEnv || (pBlob->pEnv==0 && pBlob->pData==0) );
   if( pBlob->nAlloc<nData ){
     pBlob->pData = lsmReallocOrFree(pEnv, pBlob->pData, nData);
@@ -382,7 +382,7 @@ static int sortedBlobGrow(lsm_env *pEnv, Blob *pBlob, int nData){
   return LSM_OK;
 }
 
-static int sortedBlobSet(lsm_env *pEnv, Blob *pBlob, void *pData, int nData){
+static int sortedBlobSet(lsm_env *pEnv, LsmBlob *pBlob, void *pData, int nData){
   if( sortedBlobGrow(pEnv, pBlob, nData) ) return LSM_NOMEM;
   memcpy(pBlob->pData, pData, nData);
   pBlob->nData = nData;
@@ -390,15 +390,15 @@ static int sortedBlobSet(lsm_env *pEnv, Blob *pBlob, void *pData, int nData){
 }
 
 #if 0
-static int sortedBlobCopy(Blob *pDest, Blob *pSrc){
+static int sortedBlobCopy(LsmBlob *pDest, LsmBlob *pSrc){
   return sortedBlobSet(pDest, pSrc->pData, pSrc->nData);
 }
 #endif
 
-static void sortedBlobFree(Blob *pBlob){
+static void sortedBlobFree(LsmBlob *pBlob){
   assert( pBlob->pEnv || pBlob->pData==0 );
   if( pBlob->pData ) lsmFree(pBlob->pEnv, pBlob->pData);
-  memset(pBlob, 0, sizeof(Blob));
+  memset(pBlob, 0, sizeof(LsmBlob));
 }
 
 static int sortedReadData(
@@ -407,7 +407,7 @@ static int sortedReadData(
   int iOff,
   int nByte,
   void **ppData,
-  Blob *pBlob
+  LsmBlob *pBlob
 ){
   int rc = LSM_OK;
   int iEnd;
@@ -481,8 +481,8 @@ static int pageGetNRec(u8 *aData, int nData){
   return (int)lsmGetU16(&aData[SEGMENT_NRECORD_OFFSET(nData)]);
 }
 
-static Pgno pageGetPtr(u8 *aData, int nData){
-  return (Pgno)lsmGetU64(&aData[SEGMENT_POINTER_OFFSET(nData)]);
+static LsmPgno pageGetPtr(u8 *aData, int nData){
+  return (LsmPgno)lsmGetU64(&aData[SEGMENT_POINTER_OFFSET(nData)]);
 }
 
 static int pageGetFlags(u8 *aData, int nData){
@@ -506,8 +506,8 @@ static int pageObjGetNRec(Page *pPg){
 ** Return the decoded (possibly relative) pointer value stored in cell 
 ** iCell from page aData/nData.
 */
-static Pgno pageGetRecordPtr(u8 *aData, int nData, int iCell){
-  Pgno iRet;                      /* Return value */
+static LsmPgno pageGetRecordPtr(u8 *aData, int nData, int iCell){
+  LsmPgno iRet;                   /* Return value */
   u8 *aCell;                      /* Pointer to cell iCell */
 
   assert( iCell<pageGetNRec(aData, nData) && iCell>=0 );
@@ -522,7 +522,7 @@ static u8 *pageGetKey(
   int iCell,                      /* Index of cell on page to read */
   int *piTopic,                   /* OUT: Topic associated with this key */
   int *pnKey,                     /* OUT: Size of key in bytes */
-  Blob *pBlob                     /* If required, use this for dynamic memory */
+  LsmBlob *pBlob                  /* If required, use this for dynamic memory */
 ){
   u8 *pKey;
   int nDummy;
@@ -554,7 +554,7 @@ static int pageGetKeyCopy(
   Page *pPg,                      /* Page to read from */
   int iCell,                      /* Index of cell on page to read */
   int *piTopic,                   /* OUT: Topic associated with this key */
-  Blob *pBlob                     /* If required, use this for dynamic memory */
+  LsmBlob *pBlob                  /* If required, use this for dynamic memory */
 ){
   int rc = LSM_OK;
   int nKey;
@@ -569,8 +569,8 @@ static int pageGetKeyCopy(
   return rc;
 }
 
-static Pgno pageGetBtreeRef(Page *pPg, int iKey){
-  Pgno iRef;
+static LsmPgno pageGetBtreeRef(Page *pPg, int iKey){
+  LsmPgno iRef;
   u8 *aData;
   int nData;
   u8 *aCell;
@@ -592,11 +592,11 @@ static int pageGetBtreeKey(
   Segment *pSeg,                  /* Segment page pPg belongs to */
   Page *pPg,
   int iKey, 
-  Pgno *piPtr, 
+  LsmPgno *piPtr, 
   int *piTopic, 
   void **ppKey,
   int *pnKey,
-  Blob *pBlob
+  LsmBlob *pBlob
 ){
   u8 *aData;
   int nData;
@@ -613,7 +613,7 @@ static int pageGetBtreeKey(
 
   if( eType==0 ){
     int rc;
-    Pgno iRef;                  /* Page number of referenced page */
+    LsmPgno iRef;               /* Page number of referenced page */
     Page *pRef;
     aCell += GETVARINT64(aCell, iRef);
     rc = lsmFsDbPageGet(lsmPageFS(pPg), pSeg, iRef, &pRef);
@@ -638,7 +638,7 @@ static int btreeCursorLoadKey(BtreeCursor *pCsr){
     pCsr->nKey = 0;
     pCsr->eType = 0;
   }else{
-    Pgno dummy;
+    LsmPgno dummy;
     int iPg = pCsr->iPg;
     int iCell = pCsr->aPg[iPg].iCell;
     while( iCell<0 && (--iPg)>=0 ){
@@ -683,7 +683,7 @@ static int btreeCursorNext(BtreeCursor *pCsr){
   assert( pPg->iCell<=nCell );
   pPg->iCell++;
   if( pPg->iCell==nCell ){
-    Pgno iLoad;
+    LsmPgno iLoad;
 
     /* Up to parent. */
     lsmFsPageRelease(pPg->pPage);
@@ -842,7 +842,7 @@ static int btreeCursorRestore(
   if( p->iPg ){
     lsm_env *pEnv = lsmFsEnv(pCsr->pFS);
     int iCell;                    /* Current cell number on leaf page */
-    Pgno iLeaf;                   /* Page number of current leaf page */
+    LsmPgno iLeaf;                /* Page number of current leaf page */
     int nDepth;                   /* Depth of b-tree structure */
     Segment *pSeg = pCsr->pSeg;
 
@@ -866,7 +866,7 @@ static int btreeCursorRestore(
 
     /* Populate any other aPg[] array entries */
     if( rc==LSM_OK && nDepth>1 ){
-      Blob blob = {0,0,0};
+      LsmBlob blob = {0,0,0};
       void *pSeek;
       int nSeek;
       int iTopicSeek;
@@ -883,7 +883,7 @@ static int btreeCursorRestore(
         pSeek = 0;
         nSeek = 0;
       }else{
-        Pgno dummy;
+        LsmPgno dummy;
         rc = pageGetBtreeKey(pSeg, pPg,
             0, &dummy, &iTopicSeek, &pSeek, &nSeek, &pCsr->blob
         );
@@ -912,7 +912,7 @@ static int btreeCursorRestore(
             int iTry = (iMin+iMax)/2;
             void *pKey; int nKey;         /* Key for cell iTry */
             int iTopic;                   /* Topic for key pKeyT/nKeyT */
-            Pgno iPtr;                    /* Pointer for cell iTry */
+            LsmPgno iPtr;                 /* Pointer for cell iTry */
             int res;                      /* (pSeek - pKeyT) */
 
             rc = pageGetBtreeKey(
@@ -955,7 +955,7 @@ static int btreeCursorRestore(
       aData = fsPageData(pBtreePg->pPage, &nData);
       pCsr->iPtr = btreeCursorPtr(aData, nData, pBtreePg->iCell+1);
       if( pBtreePg->iCell<0 ){
-        Pgno dummy;
+        LsmPgno dummy;
         int i;
         for(i=pCsr->iPg-1; i>=0; i--){
           if( pCsr->aPg[i].iCell>0 ) break;
@@ -1030,7 +1030,7 @@ static int segmentPtrReadData(
   int iOff,
   int nByte,
   void **ppData,
-  Blob *pBlob
+  LsmBlob *pBlob
 ){
   return sortedReadData(pPtr->pSeg, pPtr->pPg, iOff, nByte, ppData, pBlob);
 }
@@ -1123,7 +1123,7 @@ static void sortedSplitkey(lsm_db *pDb, Level *pLevel, int *pRc){
   }
   if( rc==LSM_OK ){
     int iTopic;
-    Blob blob = {0, 0, 0, 0};
+    LsmBlob blob = {0, 0, 0, 0};
     u8 *aData;
     int nData;
   
@@ -1131,7 +1131,7 @@ static void sortedSplitkey(lsm_db *pDb, Level *pLevel, int *pRc){
     if( pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG ){
       void *pKey;
       int nKey;
-      Pgno dummy;
+      LsmPgno dummy;
       rc = pageGetBtreeKey(pSeg,
           pPg, pMerge->splitkey.iCell, &dummy, &iTopic, &pKey, &nKey, &blob
       );
@@ -1342,7 +1342,7 @@ static int assertKeyLocation(
   void *pKey, int nKey
 ){
   lsm_env *pEnv = lsmFsEnv(pCsr->pDb->pFS);
-  Blob blob = {0, 0, 0};
+  LsmBlob blob = {0, 0, 0};
   int eDir;
   int iTopic = 0;                 /* TODO: Fix me */
 
@@ -1488,7 +1488,7 @@ static int ptrFwdPointer(
   Page *pPage,
   int iCell,
   Segment *pSeg,
-  Pgno *piPtr,
+  LsmPgno *piPtr,
   int *pbFound
 ){
   Page *pPg = pPage;
@@ -1573,14 +1573,14 @@ static int sortedRhsFirst(MultiCursor *pCsr, Level *pLvl, SegmentPtr *pPtr){
 static int segmentPtrFwdPointer(
   MultiCursor *pCsr,              /* Multi-cursor pPtr belongs to */
   SegmentPtr *pPtr,               /* Segment-pointer to extract FC ptr from */
-  Pgno *piPtr                     /* OUT: FC pointer value */
+  LsmPgno *piPtr                  /* OUT: FC pointer value */
 ){
   Level *pLvl = pPtr->pLevel;
   Level *pNext = pLvl->pNext;
   Page *pPg = pPtr->pPg;
   int rc;
   int bFound;
-  Pgno iOut = 0;
+  LsmPgno iOut = 0;
 
   if( pPtr->pSeg==&pLvl->lhs || pPtr->pSeg==&pLvl->aRhs[pLvl->nRight-1] ){
     if( pNext==0 
@@ -1641,7 +1641,7 @@ static int segmentPtrSeek(
   int rc = LSM_OK;
   int iMin;
   int iMax;
-  Pgno iPtrOut = 0;
+  LsmPgno iPtrOut = 0;
 
   /* If the current page contains an oversized entry, then there are no
   ** pointers to one or more of the subsequent pages in the sorted run.
@@ -1768,18 +1768,18 @@ static int seekInBtree(
   Segment *pSeg,                  /* Seek within this segment */
   int iTopic,
   void *pKey, int nKey,           /* Key to seek to */
-  Pgno *aPg,                      /* OUT: Page numbers */
+  LsmPgno *aPg,                   /* OUT: Page numbers */
   Page **ppPg                     /* OUT: Leaf (sorted-run) page reference */
 ){
   int i = 0;
   int rc;
   int iPg;
   Page *pPg = 0;
-  Blob blob = {0, 0, 0};
+  LsmBlob blob = {0, 0, 0};
 
   iPg = (int)pSeg->iRoot;
   do {
-    Pgno *piFirst = 0;
+    LsmPgno *piFirst = 0;
     if( aPg ){
       aPg[i++] = iPg;
       piFirst = &aPg[i];
@@ -1808,7 +1808,7 @@ static int seekInBtree(
         int iTry = (iMin+iMax)/2;
         void *pKeyT; int nKeyT;       /* Key for cell iTry */
         int iTopicT;                  /* Topic for key pKeyT/nKeyT */
-        Pgno iPtr;                    /* Pointer associated with cell iTry */
+        LsmPgno iPtr;                 /* Pointer associated with cell iTry */
         int res;                      /* (pKey - pKeyT) */
 
         rc = pageGetBtreeKey(
@@ -1899,7 +1899,7 @@ static int seekInLevel(
   int eSeek,                      /* Search bias - see above */
   int iTopic,                     /* Key topic to search for */
   void *pKey, int nKey,           /* Key to search for */
-  Pgno *piPgno,                   /* IN/OUT: fraction cascade pointer (or 0) */
+  LsmPgno *piPgno,                /* IN/OUT: fraction cascade pointer (or 0) */
   int *pbStop                     /* OUT: See above */
 ){
   Level *pLvl = aPtr[0].pLevel;   /* Level to seek within */
@@ -3055,7 +3055,7 @@ int lsmMCursorSeek(
   int bStop = 0;                  /* Set to true to halt search operation */
   int rc = LSM_OK;                /* Return code */
   int iPtr = 0;                   /* Used to iterate through pCsr->aPtr[] */
-  Pgno iPgno = 0;                 /* FC pointer value */
+  LsmPgno iPgno = 0;              /* FC pointer value */
 
   assert( pCsr->apTreeCsr[0]==0 || iTopic==0 );
   assert( pCsr->apTreeCsr[1]==0 || iTopic==0 );
@@ -3537,7 +3537,7 @@ static int mergeWorkerLoadHierarchy(MergeWorker *pMW){
 **         + Type byte (always SORTED_SEPARATOR or SORTED_SYSTEM_SEPARATOR),
 **         + Absolute pointer value (varint),
 **         + Number of bytes in key (varint),
-**         + Blob containing key data.
+**         + LsmBlob containing key data.
 **
 **   2. All pointer values are stored as absolute values (not offsets 
 **      relative to the footer pointer value).
@@ -3571,8 +3571,8 @@ static int mergeWorkerLoadHierarchy(MergeWorker *pMW){
 static int mergeWorkerBtreeWrite(
   MergeWorker *pMW,
   u8 eType,
-  Pgno iPtr,
-  Pgno iKeyPg,
+  LsmPgno iPtr,
+  LsmPgno iKeyPg,
   void *pKey,
   int nKey
 ){
@@ -3682,7 +3682,7 @@ static int mergeWorkerBtreeWrite(
 static int mergeWorkerBtreeIndirect(MergeWorker *pMW){
   int rc = LSM_OK;
   if( pMW->iIndirect ){
-    Pgno iKeyPg = pMW->aSave[1].iPgno;
+    LsmPgno iKeyPg = pMW->aSave[1].iPgno;
     rc = mergeWorkerBtreeWrite(pMW, 0, pMW->iIndirect, iKeyPg, 0, 0);
     pMW->iIndirect = 0;
   }
@@ -3703,7 +3703,7 @@ static int mergeWorkerPushHierarchy(
   int nKey                        /* Size of pKey buffer in bytes */
 ){
   int rc = LSM_OK;                /* Return Code */
-  Pgno iPtr;                      /* Pointer value to accompany pKey/nKey */
+  LsmPgno iPtr;                   /* Pointer value to accompany pKey/nKey */
 
   assert( pMW->aSave[0].bStore==0 );
   assert( pMW->aSave[1].bStore==0 );
@@ -3734,7 +3734,7 @@ static int mergeWorkerFinishHierarchy(
 ){
   int i;                          /* Used to loop through apHier[] */
   int rc = LSM_OK;                /* Return code */
-  Pgno iPtr;                      /* New right-hand-child pointer value */
+  LsmPgno iPtr;                   /* New right-hand-child pointer value */
 
   iPtr = pMW->aSave[0].iPgno;
   for(i=0; i<pMW->hier.nHier && rc==LSM_OK; i++){
@@ -3830,7 +3830,7 @@ static int mergeWorkerPersistAndRelease(MergeWorker *pMW){
 */
 static int mergeWorkerNextPage(
   MergeWorker *pMW,               /* Merge worker object to append page to */
-  Pgno iFPtr                      /* Pointer value for footer of new page */
+  LsmPgno iFPtr                   /* Pointer value for footer of new page */
 ){
   int rc = LSM_OK;                /* Return code */
   Page *pNext = 0;                /* New page appended to run */
@@ -4218,7 +4218,7 @@ static int mergeWorkerStep(MergeWorker *pMW){
   int rc = LSM_OK;              /* Return code */
   int eType;                    /* SORTED_SEPARATOR, WRITE or DELETE */
   void *pKey; int nKey;         /* Key */
-  Pgno iPtr;
+  LsmPgno iPtr;
   int iVal;
 
   pCsr = pMW->pCsr;
@@ -4371,7 +4371,7 @@ static int sortedNewToplevel(
   if( rc!=LSM_OK ){
     lsmMCursorClose(pCsr, 0);
   }else{
-    Pgno iLeftPtr = 0;
+    LsmPgno iLeftPtr = 0;
     Merge merge;                  /* Merge object used to create new level */
     MergeWorker mergeworker;      /* MergeWorker object for the same purpose */
 
@@ -4548,7 +4548,7 @@ static int mergeWorkerInit(
   memset(pMW, 0, sizeof(MergeWorker));
   pMW->pDb = pDb;
   pMW->pLevel = pLevel;
-  pMW->aGobble = lsmMallocZeroRc(pDb->pEnv, sizeof(Pgno) * pLevel->nRight, &rc);
+  pMW->aGobble = lsmMallocZeroRc(pDb->pEnv, sizeof(LsmPgno)*pLevel->nRight,&rc);
 
   /* Create a multi-cursor to read the data to write to the new
   ** segment. The new segment contains:
@@ -4630,7 +4630,7 @@ static int sortedBtreeGobble(
   int rc = LSM_OK;
   if( rtTopic(pCsr->eType)==0 ){
     Segment *pSeg = pCsr->aPtr[iGobble].pSeg;
-    Pgno *aPg;
+    LsmPgno *aPg;
     int nPg;
 
     /* Seek from the root of the b-tree to the segment leaf that may contain
@@ -4639,7 +4639,7 @@ static int sortedBtreeGobble(
     ** gobbled up to (but not including) the first of these page numbers.
     */
     assert( pSeg->iRoot>0 );
-    aPg = lsmMallocZeroRc(pDb->pEnv, sizeof(Pgno)*32, &rc);
+    aPg = lsmMallocZeroRc(pDb->pEnv, sizeof(LsmPgno)*32, &rc);
     if( rc==LSM_OK ){
       rc = seekInBtree(pCsr, pSeg, 
           rtTopic(pCsr->eType), pCsr->key.pData, pCsr->key.nData, aPg, 0
@@ -5466,9 +5466,9 @@ int lsmFlushTreeToDisk(lsm_db *pDb){
 */
 static char *segToString(lsm_env *pEnv, Segment *pSeg, int nMin){
   int nSize = pSeg->nSize;
-  Pgno iRoot = pSeg->iRoot;
-  Pgno iFirst = pSeg->iFirst;
-  Pgno iLast = pSeg->iLastPg;
+  LsmPgno iRoot = pSeg->iRoot;
+  LsmPgno iFirst = pSeg->iFirst;
+  LsmPgno iLast = pSeg->iLastPg;
   char *z;
 
   char *z1;
@@ -5527,7 +5527,7 @@ static int fileToString(
 }
 
 void sortedDumpPage(lsm_db *pDb, Segment *pRun, Page *pPg, int bVals){
-  Blob blob = {0, 0, 0};         /* Blob used for keys */
+  LsmBlob blob = {0, 0, 0};       /* LsmBlob used for keys */
   LsmString s;
   int i;
 
@@ -5563,7 +5563,7 @@ void sortedDumpPage(lsm_db *pDb, Segment *pRun, Page *pPg, int bVals){
     aCell += lsmVarintGet32(aCell, &iPgPtr);
 
     if( eType==0 ){
-      Pgno iRef;                  /* Page number of referenced page */
+      LsmPgno iRef;               /* Page number of referenced page */
       aCell += lsmVarintGet64(aCell, &iRef);
       lsmFsDbPageGet(pDb->pFS, pRun, iRef, &pRef);
       aKey = pageGetKey(pRun, pRef, 0, &iTopic, &nKey, &blob);
@@ -5607,7 +5607,7 @@ static void infoCellDump(
   int *piPgPtr,
   u8 **paKey, int *pnKey,
   u8 **paVal, int *pnVal,
-  Blob *pBlob
+  LsmBlob *pBlob
 ){
   u8 *aData; int nData;           /* Page data */
   u8 *aKey; int nKey = 0;         /* Key */
@@ -5625,7 +5625,7 @@ static void infoCellDump(
 
   if( eType==0 ){
     int dummy;
-    Pgno iRef;                  /* Page number of referenced page */
+    LsmPgno iRef;                 /* Page number of referenced page */
     aCell += lsmVarintGet64(aCell, &iRef);
     if( bIndirect ){
       lsmFsDbPageGet(pDb->pFS, pSeg, iRef, &pRef);
@@ -5671,7 +5671,7 @@ static int infoAppendBlob(LsmString *pStr, int bHex, u8 *z, int n){
 
 static int infoPageDump(
   lsm_db *pDb,                    /* Database handle */
-  Pgno iPg,                       /* Page number of page to dump */
+  LsmPgno iPg,                    /* Page number of page to dump */
   int flags,
   char **pzOut                    /* OUT: lsmMalloc'd string */
 ){
@@ -5712,7 +5712,7 @@ static int infoPageDump(
   }
 
   if( rc==LSM_OK ){
-    Blob blob = {0, 0, 0, 0};
+    LsmBlob blob = {0, 0, 0, 0};
     int nKeyWidth = 0;
     LsmString str;
     int nRec;
@@ -5747,7 +5747,7 @@ static int infoPageDump(
       u8 *aVal; int nVal = 0;       /* Value */
       int iPgPtr;
       int eType;
-      Pgno iAbsPtr;
+      LsmPgno iAbsPtr;
       char zFlags[8];
 
       infoCellDump(pDb, pSeg, bIndirect, pPg, iCell, &eType, &iPgPtr,
@@ -5813,7 +5813,7 @@ static int infoPageDump(
 
 int lsmInfoPageDump(
   lsm_db *pDb,                    /* Database handle */
-  Pgno iPg,                       /* Page number of page to dump */
+  LsmPgno iPg,                    /* Page number of page to dump */
   int bHex,                       /* True to output key/value in hex form */
   char **pzOut                    /* OUT: lsmMalloc'd string */
 ){
@@ -5989,8 +5989,8 @@ void lsmSortedExpandBtreePage(Page *pPg, int nOrig){
 #ifdef LSM_DEBUG_EXPENSIVE
 static void assertRunInOrder(lsm_db *pDb, Segment *pSeg){
   Page *pPg = 0;
-  Blob blob1 = {0, 0, 0, 0};
-  Blob blob2 = {0, 0, 0, 0};
+  LsmBlob blob1 = {0, 0, 0, 0};
+  LsmBlob blob2 = {0, 0, 0, 0};
 
   lsmFsDbPageGet(pDb->pFS, pSeg, pSeg->iFirst, &pPg);
   while( pPg ){
@@ -6052,7 +6052,7 @@ static int assertPointersOk(
   int rc = LSM_OK;                /* Error code */
   SegmentPtr ptr1;                /* Iterates through pOne */
   SegmentPtr ptr2;                /* Iterates through pTwo */
-  Pgno iPrev;
+  LsmPgno iPrev;
 
   assert( pOne && pTwo );
 
@@ -6075,7 +6075,7 @@ static int assertPointersOk(
   }
       
   while( rc==LSM_OK && ptr2.pPg ){
-    Pgno iThis;
+    LsmPgno iThis;
 
     /* Advance to the next page of segment pTwo that contains at least
     ** one cell. Break out of the loop if the iterator reaches EOF.  */
@@ -6137,7 +6137,7 @@ static int assertBtreeOk(
 ){
   int rc = LSM_OK;                /* Return code */
   if( pSeg->iRoot ){
-    Blob blob = {0, 0, 0};        /* Buffer used to cache overflow keys */
+    LsmBlob blob = {0, 0, 0};     /* Buffer used to cache overflow keys */
     FileSystem *pFS = pDb->pFS;   /* File system to read from */
     Page *pPg = 0;                /* Main run page */
     BtreeCursor *pCsr = 0;        /* Btree cursor */
diff --git a/manifest b/manifest
index cad9609c6b..f66d23cb12 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sscript\sto\samalgamate\sall\sLSM\sfiles\sinto\s"lsm1.c".
-D 2017-12-13T14:22:48.449
+C Change\ssome\sLSM\scode\sinternal\stypenames\sfrom\s"Blob"\sand\s"Pgno"\sto\s"LsmBlob"\nand\s"LsmPgno".\sThis\sallows\sthe\sLSM\scode\sto\sbe\scompiled\sas\sa\ssingle\sunit\swith\nsqlite3.c.
+D 2017-12-13T15:32:33.291
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
@@ -239,15 +239,15 @@ F ext/lsm1/lsm-test/lsmtest_tdb4.c 47e8bb5eba266472d690fb8264f1855ebdba0ae5a0e54
 F ext/lsm1/lsm-test/lsmtest_util.c 241622db5a332a09c8e6e7606b617d288a37b557f7d3bce0bb97809f67cc2806
 F ext/lsm1/lsm-test/lsmtest_win32.c 0e0a224674c4d3170631c41b026b56c7e1672b151f5261e1b4cc19068641da2d
 F ext/lsm1/lsm.h 0f6f64ff071471cb87bf98beb8386566f30ea001
-F ext/lsm1/lsmInt.h e9e5c5f08e35a104086102b3def94ee69cbc0d39002f6596f5c80a640439628e
-F ext/lsm1/lsm_ckpt.c ac6fb4581983291c2e0be6fbb68f12b26f0c08d606835c05417be1323d0fdd03
-F ext/lsm1/lsm_file.c 4b3fb56336fbc9d941e1b2042e809d986feebdc41e73dc7fc4fdc0dd1bd4274d
+F ext/lsm1/lsmInt.h 5983690e05e83653cc01ba9d8fbf8455e534ddf8349ed9adedbf46a7549760b0
+F ext/lsm1/lsm_ckpt.c 0eabfaf812ddb4ea43add38f05e430694cd054eb622c3e35af4c43118a2d5321
+F ext/lsm1/lsm_file.c 3c51841d5b3e7da162693cbac9a9f47eeedf6bcbbe2969a4d25e30c428c9fe36
 F ext/lsm1/lsm_log.c a8bf334532109bba05b09a504ee45fc393828b0d034ca61ab45e3940709d9a7c
-F ext/lsm1/lsm_main.c 15e73ccdafdd44ddeefc29e332079d88ba8f00c12c797b3c2b63d3171b5afce8
+F ext/lsm1/lsm_main.c 801295038b548ae2e5fae93f08c3f945154f40848a03ff26b16eab5d04ba573a
 F ext/lsm1/lsm_mem.c 4c51ea9fa285ee6e35301b33491642d071740a0a
 F ext/lsm1/lsm_mutex.c 378edf0a2b142b4f7640ee982df06d50b98788ea
 F ext/lsm1/lsm_shared.c 76adfc1ed9ffebaf92746dde4b370ccc48143ca8b05b563816eadd2aadf1c525
-F ext/lsm1/lsm_sorted.c 8f899fb64a4c736ff3c27d5126c7ce181129ddffde947fe5fb657a7a413f470b
+F ext/lsm1/lsm_sorted.c d07ff7c28758542b8b4da4b5a1fb67b22a4d33e50e7f684cffe1f6c45cf5182c
 F ext/lsm1/lsm_str.c 65e361b488c87b10bf3e5c0070b14ffc602cf84f094880bece77bbf6678bca82
 F ext/lsm1/lsm_tree.c 682679d7ef2b8b6f2fe77aeb532c8d29695bca671c220b0abac77069de5fb9fb
 F ext/lsm1/lsm_unix.c 57361bcf5b1a1a028f5d66571ee490e9064d2cfb145a2cc9e5ddade467bb551b
@@ -1680,7 +1680,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 9cede8a83ca4cd88d504050115e1e89e7b3d3cd3cb2ffb5b8961e311a23ff5e2
-R 0bb1b86efd302c3523a2137056206a8a
+P e32b69d73062e233b0ac853611d10b24546a346a603289ab0e339a3604ae2af4
+R 3d8a0d79c82564334f201ac883edd045
 U dan
-Z 0df44b61ec124ac51afd1eb16b579be7
+Z 7a59c9677684f4709aed2e3a45c7d151
diff --git a/manifest.uuid b/manifest.uuid
index f9c2f8f2b4..d947d934df 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e32b69d73062e233b0ac853611d10b24546a346a603289ab0e339a3604ae2af4
\ No newline at end of file
+30bf38d589adf0b2eb613e184ffb03ed7e625736cb04fd5c341328f72bc5ef4d
\ No newline at end of file

From 472e41ea16b2f338634ad31495d21dca62259eed Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 13 Dec 2017 18:01:52 +0000
Subject: [PATCH 142/488] Correctly invalidate a column cache line when it is
 overwritten with NULL. Fix for ticket [123c9ba32130a6c9d432].

FossilOrigin-Name: 65754dc3b830ef9482655d2e93c2cd4acaa7a8715a58d8309955d04378541f89
---
 manifest         | 16 ++++++++--------
 manifest.uuid    |  2 +-
 src/wherecode.c  |  1 +
 test/limit2.test | 17 +++++++++++++++++
 4 files changed, 27 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index f66d23cb12..ef06a4dad0 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Change\ssome\sLSM\scode\sinternal\stypenames\sfrom\s"Blob"\sand\s"Pgno"\sto\s"LsmBlob"\nand\s"LsmPgno".\sThis\sallows\sthe\sLSM\scode\sto\sbe\scompiled\sas\sa\ssingle\sunit\swith\nsqlite3.c.
-D 2017-12-13T15:32:33.291
+C Correctly\sinvalidate\sa\scolumn\scache\sline\swhen\sit\sis\soverwritten\swith\sNULL.\nFix\sfor\sticket\s[123c9ba32130a6c9d432].
+D 2017-12-13T18:01:52.248
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
@@ -557,7 +557,7 @@ F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f
 F src/where.c ee9dd4a438a07cd364c8449e834db4c4d6163a2576a69e937d7a4c37685612a2
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
-F src/wherecode.c 611fcabd05592ed2febd7d182f9621425b0466c5232d70e0981c842d429356d5
+F src/wherecode.c ff2f079097a3bdce6ebabfde1419fba448c9ce5feb7cb964e8bfa2a4e27274ef
 F src/whereexpr.c 427ea8e96ec24f2a7814c67b8024ad664a9c7656264c4566c34743cb23186e46
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd
@@ -1008,7 +1008,7 @@ F test/like.test 11cfd7d4ef8625389df9efc46735ff0b0b41d5e62047ef0f3bc24c380d28a7a
 F test/like2.test 3b2ee13149ba4a8a60b59756f4e5d345573852da
 F test/like3.test 3608a2042b6f922f900fbfd5d3ce4e7eca57f7c4
 F test/limit.test 0c99a27a87b14c646a9d583c7c89fd06c352663e
-F test/limit2.test e35f57bd3a62d7c5dcb5ac4306e675c75f974809
+F test/limit2.test 360982809e03211636d2b18ddbc97d5da06826941370607e4b00e113f827cb5a
 F test/loadext.test d077450695ddb5c1ea3ad7d48e5f5850fe732ad9
 F test/loadext2.test 0408380b57adca04004247179837a18e866a74f7
 F test/lock.test be4fe08118fb988fed741f429b7dd5d65e1c90db
@@ -1680,7 +1680,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e32b69d73062e233b0ac853611d10b24546a346a603289ab0e339a3604ae2af4
-R 3d8a0d79c82564334f201ac883edd045
-U dan
-Z 7a59c9677684f4709aed2e3a45c7d151
+P 30bf38d589adf0b2eb613e184ffb03ed7e625736cb04fd5c341328f72bc5ef4d
+R 0b90f4f4961013e5f05c73c6020a97dc
+U drh
+Z 53ad352984145a11e63d4c3ee646c195
diff --git a/manifest.uuid b/manifest.uuid
index d947d934df..15f6d50382 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-30bf38d589adf0b2eb613e184ffb03ed7e625736cb04fd5c341328f72bc5ef4d
\ No newline at end of file
+65754dc3b830ef9482655d2e93c2cd4acaa7a8715a58d8309955d04378541f89
\ No newline at end of file
diff --git a/src/wherecode.c b/src/wherecode.c
index 40c5f41182..3c166a1210 100644
--- a/src/wherecode.c
+++ b/src/wherecode.c
@@ -1690,6 +1690,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       }
     }else if( bStopAtNull ){
       sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq);
+      sqlite3ExprCacheRemove(pParse, regBase+nEq, 1);
       endEq = 0;
       nConstraint++;
     }
diff --git a/test/limit2.test b/test/limit2.test
index f6eaefb8aa..83c67506f5 100644
--- a/test/limit2.test
+++ b/test/limit2.test
@@ -150,4 +150,21 @@ do_execsql_test 502 {
   SELECT j FROM t502 WHERE i IN (1,2,3,4,5) ORDER BY j LIMIT 3;
 } {1 3 4}
 
+# Ticket https://www.sqlite.org/src/info/123c9ba32130a6c9 2017-12-13
+# Incorrect result when an idnex is used for an ordered join.
+#
+# This test case is in the limit2.test module because the problem was first
+# exposed by check-in https://www.sqlite.org/src/info/559733b09e which 
+# implemented the ORDER BY LIMIT optimization that limit2.test strives to
+# test.
+#
+do_execsql_test 600 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1(a, b);  INSERT INTO t1 VALUES(1,2);
+  DROP TABLE IF EXISTS t2;
+  CREATE TABLE t2(x, y);  INSERT INTO t2 VALUES(1,3);
+  CREATE INDEX t1ab ON t1(a,b);
+  SELECT y FROM t1, t2 WHERE a=x AND b<=y ORDER BY b DESC;
+} {3}
+
 finish_test

From a803a2cd9889cb3b4db1029110281370576bf354 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 13 Dec 2017 20:02:29 +0000
Subject: [PATCH 143/488] New result code SQLITE_READONLY_DIRECTORY is returned
 when an attempt is made to write on a database file that is in a read-only
 directory and hence the journal file could not be created.  This situation
 formerly returned SQLITE_CANTOPEN, which less helpful.

FossilOrigin-Name: 3ec73c38f878d73d278fce99ba10c708dcc475835774f1e17769ff7315be6d7c
---
 manifest        | 14 +++++++-------
 manifest.uuid   |  2 +-
 src/os_unix.c   | 12 +++++++++---
 src/sqlite.h.in |  1 +
 4 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index ef06a4dad0..c2d0952bf2 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Correctly\sinvalidate\sa\scolumn\scache\sline\swhen\sit\sis\soverwritten\swith\sNULL.\nFix\sfor\sticket\s[123c9ba32130a6c9d432].
-D 2017-12-13T18:01:52.248
+C New\sresult\scode\sSQLITE_READONLY_DIRECTORY\sis\sreturned\swhen\san\sattempt\sis\nmade\sto\swrite\son\sa\sdatabase\sfile\sthat\sis\sin\sa\sread-only\sdirectory\sand\shence\nthe\sjournal\sfile\scould\snot\sbe\screated.\s\sThis\ssituation\sformerly\sreturned\nSQLITE_CANTOPEN,\swhich\sless\shelpful.
+D 2017-12-13T20:02:29.496
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
@@ -456,7 +456,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104
 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
-F src/os_unix.c a40e2926d2cee18f5bd919356f560ff30f0189dd03cf298662e70ece89c23088
+F src/os_unix.c 1ec9390926a0626494466e572eb7754545acd3e423a4871a749fe35c09e0b6ab
 F src/os_win.c 0a4afa35cc8e812000df3ea2f64b476131b39e29e75d8007d0504726e4761de4
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903
@@ -474,7 +474,7 @@ F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
 F src/shell.c.in 6ffed0c589f5aff180789a8c8abf5b2d3e2eea7470c86b30e797887cb0c9d0e5
-F src/sqlite.h.in 4622dbb3bb94119795fd926c2a10cacfb86d59900fa9b037e8678acd9830d147
+F src/sqlite.h.in 364515dd186285f3c01f5cab42e7db7edc47c70e87b6a25de389a2e6b8c413fd
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
 F src/sqliteInt.h 55b8e7da85947eb61b13d4d2523ccdda7800a13e987c3fc4ca73d8518bbf02fa
@@ -1680,7 +1680,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 30bf38d589adf0b2eb613e184ffb03ed7e625736cb04fd5c341328f72bc5ef4d
-R 0b90f4f4961013e5f05c73c6020a97dc
+P 65754dc3b830ef9482655d2e93c2cd4acaa7a8715a58d8309955d04378541f89
+R 317d939eaca617f6f45ef08fff90ca48
 U drh
-Z 53ad352984145a11e63d4c3ee646c195
+Z f0826ac4a0e91b44749cba2450c52240
diff --git a/manifest.uuid b/manifest.uuid
index 15f6d50382..4128241972 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-65754dc3b830ef9482655d2e93c2cd4acaa7a8715a58d8309955d04378541f89
\ No newline at end of file
+3ec73c38f878d73d278fce99ba10c708dcc475835774f1e17769ff7315be6d7c
\ No newline at end of file
diff --git a/src/os_unix.c b/src/os_unix.c
index 3c2f15ed05..e82cec2333 100644
--- a/src/os_unix.c
+++ b/src/os_unix.c
@@ -5799,7 +5799,7 @@ static int unixOpen(
   ** a file-descriptor on the directory too. The first time unixSync()
   ** is called the directory file descriptor will be fsync()ed and close()d.
   */
-  int syncDir = (isCreate && (
+  int isNewJrnl = (isCreate && (
         eType==SQLITE_OPEN_MASTER_JOURNAL 
      || eType==SQLITE_OPEN_MAIN_JOURNAL 
      || eType==SQLITE_OPEN_WAL
@@ -5869,7 +5869,7 @@ static int unixOpen(
 
   }else if( !zName ){
     /* If zName is NULL, the upper layer is requesting a temp file. */
-    assert(isDelete && !syncDir);
+    assert(isDelete && !isNewJrnl);
     rc = unixGetTempname(pVfs->mxPathname, zTmpname);
     if( rc!=SQLITE_OK ){
       return rc;
@@ -5904,6 +5904,12 @@ static int unixOpen(
     fd = robust_open(zName, openFlags, openMode);
     OSTRACE(("OPENX   %-3d %s 0%o\n", fd, zName, openFlags));
     assert( !isExclusive || (openFlags & O_CREAT)!=0 );
+    if( fd<0 && isNewJrnl && osAccess(zName, F_OK) ){
+      /* Trying to create a journal file where we don't have write 
+      ** permission on the directory */
+      rc = unixLogError(SQLITE_READONLY_DIRECTORY, "open", zName);
+      goto open_finished;
+    }
     if( fd<0 && errno!=EISDIR && isReadWrite ){
       /* Failed to open the file for read/write access. Try read-only. */
       flags &= ~(SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE);
@@ -5974,7 +5980,7 @@ static int unixOpen(
   if( isReadonly )              ctrlFlags |= UNIXFILE_RDONLY;
   noLock = eType!=SQLITE_OPEN_MAIN_DB;
   if( noLock )                  ctrlFlags |= UNIXFILE_NOLOCK;
-  if( syncDir )                 ctrlFlags |= UNIXFILE_DIRSYNC;
+  if( isNewJrnl )               ctrlFlags |= UNIXFILE_DIRSYNC;
   if( flags & SQLITE_OPEN_URI ) ctrlFlags |= UNIXFILE_URI;
 
 #if SQLITE_ENABLE_LOCKING_STYLE
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 29b0458d76..7e26034bdd 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -517,6 +517,7 @@ int sqlite3_exec(
 #define SQLITE_READONLY_ROLLBACK       (SQLITE_READONLY | (3<<8))
 #define SQLITE_READONLY_DBMOVED        (SQLITE_READONLY | (4<<8))
 #define SQLITE_READONLY_CANTINIT       (SQLITE_READONLY | (5<<8))
+#define SQLITE_READONLY_DIRECTORY      (SQLITE_READONLY | (6<<8))
 #define SQLITE_ABORT_ROLLBACK          (SQLITE_ABORT | (2<<8))
 #define SQLITE_CONSTRAINT_CHECK        (SQLITE_CONSTRAINT | (1<<8))
 #define SQLITE_CONSTRAINT_COMMITHOOK   (SQLITE_CONSTRAINT | (2<<8))

From 3f67ddf8f22cae016c577dff9295b2fe523031c9 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 13 Dec 2017 20:04:53 +0000
Subject: [PATCH 144/488] Add support for the "--list" command. And for
 arguments to the "--extract" command.

FossilOrigin-Name: 32c4fa2552bb0fa7d7d143108457efae7a756d6cb14b1d59312e56efac3b2656
---
 manifest       |  12 ++---
 manifest.uuid  |   2 +-
 src/shell.c.in | 138 +++++++++++++++++++++++++++++++++++++++++++++----
 3 files changed, 135 insertions(+), 17 deletions(-)

diff --git a/manifest b/manifest
index 61046afc09..8be2f2242c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\stests\sand\sfixes\sfor\sthe\sshell\s".ar"\scommand\s-f\soption.
-D 2017-12-12T20:28:36.588
+C Add\ssupport\sfor\sthe\s"--list"\scommand.\sAnd\sfor\sarguments\sto\sthe\s"--extract"\ncommand.
+D 2017-12-13T20:04:53.034
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
@@ -474,7 +474,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
-F src/shell.c.in a09773c80a647f6ba4ef8dd9ce88840d52dbede5a9fa318333843deb8c8548b7
+F src/shell.c.in b53eddcb293a9d35c0673c1d3bf2cf120b55e6660b436f65d689776a56391562
 F src/sqlite.h.in 8fd97993d48b50b9bade38c52f12d175942c9497c960905610c7b03a3e4b5818
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1682,7 +1682,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 38dbeb1e777aa7ec742aa27002ad4dcee28af520dc43de96e5c56c39f16574ff
-R e6fcdae437270c2329ef751955f65f0e
+P 1a9867973c9d6675fa5254fdd74f36004707a98a91593a188033cf5a49cc7a0b
+R c505f9c0d154ac54b6761fcc5d07e027
 U dan
-Z 1b5cf819c180e1535b22a7c0cf2ece4d
+Z dcb23b384070f4f2014f2150f41e157a
diff --git a/manifest.uuid b/manifest.uuid
index 7640d02900..71ba76fdc3 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-1a9867973c9d6675fa5254fdd74f36004707a98a91593a188033cf5a49cc7a0b
\ No newline at end of file
+32c4fa2552bb0fa7d7d143108457efae7a756d6cb14b1d59312e56efac3b2656
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index ee9ccd6e76..0230e5f4d7 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -4092,6 +4092,26 @@ static void shellPrepare(
   }
 }
 
+static void shellPrepare2(
+  sqlite3 *db, 
+  int *pRc, 
+  const char *zSql, 
+  const char *zTail, 
+  sqlite3_stmt **ppStmt
+){
+  if( *pRc==SQLITE_OK && zTail ){
+    char *z = sqlite3_mprintf("%s %s", zSql, zTail);
+    if( z==0 ){
+      *pRc = SQLITE_NOMEM;
+    }else{
+      shellPrepare(db, pRc, z, ppStmt);
+      sqlite3_free(z);
+    }
+  }else{
+    shellPrepare(db, pRc, zSql, ppStmt);
+  }
+}
+
 static void shellFinalize(
   int *pRc, 
   sqlite3_stmt *pStmt
@@ -4316,12 +4336,96 @@ static int arUpdateCmd(ShellState *p, sqlite3 *db, ArCommand *pAr){
   return SQLITE_OK;
 }
 
+/*
+** This function assumes that all arguments within the ArCommand.azArg[]
+** array refer to archive members, as for the --extract or --list commands. 
+** It checks that each of them are present. If any specified file is not
+** present in the archive, an error is printed to stderr and an error
+** code returned. Otherwise, if all specified arguments are present in
+** the archive, SQLITE_OK is returned.
+**
+** This function strips any trailing '/' characters from each argument.
+** This is consistent with the way the [tar] command seems to work on
+** Linux.
+*/
+static int arCheckEntries(sqlite3 *db, ArCommand *pAr){
+  int rc = SQLITE_OK;
+  if( pAr->nArg ){
+    int i;
+    sqlite3_stmt *pTest = 0;
+
+    shellPrepare(db, &rc, "SELECT name FROM sqlar WHERE name=?", &pTest);
+    for(i=0; i<pAr->nArg && rc==SQLITE_OK; i++){
+      char *z = pAr->azArg[i];
+      int n = strlen(z);
+      int bOk = 0;
+      while( n>0 && z[n-1]=='/' ) n--;
+      z[n] = '\0';
+      sqlite3_bind_text(pTest, 1, z, -1, SQLITE_STATIC);
+      if( SQLITE_ROW==sqlite3_step(pTest) ){
+        bOk = 1;
+      }
+      shellReset(&rc, pTest);
+      if( rc==SQLITE_OK && bOk==0 ){
+        raw_printf(stderr, "not found in archive: %s\n", z);
+        rc = SQLITE_ERROR;
+      }
+    }
+    shellFinalize(&rc, pTest);
+  }
+
+  return rc;
+}
+
+/*
+** Format a WHERE clause that can be used against the "sqlar" table to
+** identify all archive members that match the command arguments held
+** in (*pAr). Leave this WHERE clause in (*pzWhere) before returning.
+** The caller is responsible for eventually calling sqlite3_free() on
+** any non-NULL (*pzWhere) value.
+*/
+static void arWhereClause(
+  int *pRc, 
+  ArCommand *pAr, 
+  char **pzWhere                  /* OUT: New WHERE clause (or NULL) */
+){
+  char *zWhere = 0;
+  if( *pRc==SQLITE_OK ){
+    int i;
+    const char *zSep = "WHERE ";
+    for(i=0; i<pAr->nArg; i++){
+      const char *z = pAr->azArg[i];
+      zWhere = sqlite3_mprintf(
+          "%z%s name = '%q' OR name BETWEEN '%q/' AND '%q0'", 
+          zWhere, zSep, z, z, z
+      );
+      if( zWhere==0 ){
+        *pRc = SQLITE_NOMEM;
+        break;
+      }
+      zSep = " OR ";
+    }
+  }
+  *pzWhere = zWhere;
+}
+
 /*
 ** Implementation of .ar "lisT" command. 
 */
 static int arListCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
-  raw_printf(stderr, "todo...\n");
-  return SQLITE_OK;
+  const char *zSql = "SELECT name FROM sqlar"; 
+  char *zWhere = 0;
+  sqlite3_stmt *pSql = 0;
+  int rc;
+
+  rc = arCheckEntries(db, pAr);
+  arWhereClause(&rc, pAr, &zWhere);
+
+  shellPrepare2(db, &rc, zSql, zWhere, &pSql);
+  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
+    raw_printf(p->out, "%s\n", sqlite3_column_text(pSql, 0));
+  }
+  return rc;
 }
 
 
@@ -4331,8 +4435,11 @@ static int arListCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
 static int arExtractCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
   const char *zSql1 = 
     "SELECT :1 || name, writefile(:1 || name, "
-    "CASE WHEN (data AND sz>=0 AND sz!=length(data)) THEN uncompress(data) "
-    "   ELSE data END, "
+    "CASE WHEN (data AND sz>=0 AND sz!=length(data)) THEN "
+    "    uncompress(data) "
+    "ELSE"
+    "    data "
+    "END, "
     "mode) FROM sqlar";
   const char *zSql2 = "SELECT :1 || name, mtime FROM sqlar"; 
 
@@ -4340,17 +4447,27 @@ static int arExtractCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
   sqlite3_stmt *pSql = 0;
   int rc = SQLITE_OK;
   char *zDir = 0;
+  char *zWhere = 0;
 
-  if( pAr->zDir ){
-    zDir = sqlite3_mprintf("%s/", pAr->zDir);
-  }else{
-    zDir = sqlite3_mprintf("");
+  /* If arguments are specified, check that they actually exist within
+  ** the archive before proceeding. And formulate a WHERE clause to
+  ** match them.  */
+  rc = arCheckEntries(db, pAr);
+  arWhereClause(&rc, pAr, &zWhere);
+
+  if( rc==SQLITE_OK ){
+    if( pAr->zDir ){
+      zDir = sqlite3_mprintf("%s/", pAr->zDir);
+    }else{
+      zDir = sqlite3_mprintf("");
+    }
+    if( zDir==0 ) rc = SQLITE_NOMEM;
   }
 
   memset(times, 0, sizeof(times));
   times[0].tv_sec = time(0);
 
-  shellPrepare(db, &rc, zSql1, &pSql);
+  shellPrepare2(db, &rc, zSql1, zWhere, &pSql);
   if( rc==SQLITE_OK ){
     sqlite3_bind_text(pSql, 1, zDir, -1, SQLITE_STATIC);
   }
@@ -4361,7 +4478,7 @@ static int arExtractCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
   }
   shellFinalize(&rc, pSql);
 
-  shellPrepare(db, &rc, zSql2, &pSql);
+  shellPrepare2(db, &rc, zSql2, zWhere, &pSql);
   if( rc==SQLITE_OK ){
     sqlite3_bind_text(pSql, 1, zDir, -1, SQLITE_STATIC);
   }
@@ -4377,6 +4494,7 @@ static int arExtractCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
   shellFinalize(&rc, pSql);
 
   sqlite3_free(zDir);
+  sqlite3_free(zWhere);
   return rc;
 }
 

From 06741a3c9b7bfb10c6d22ae74706b40af46ecd4a Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 13 Dec 2017 20:17:18 +0000
Subject: [PATCH 145/488] Add the shell tool ".ar --update" command.

FossilOrigin-Name: 825e3c037b03fc09d581aeda0193ff1d4062404414c7354cb649f99aa9022d25
---
 manifest       | 12 ++++++------
 manifest.uuid  |  2 +-
 src/shell.c.in | 48 +++++++++++++++++++++++++++++++++++-------------
 3 files changed, 42 insertions(+), 20 deletions(-)

diff --git a/manifest b/manifest
index 8be2f2242c..c7890a8a98 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\ssupport\sfor\sthe\s"--list"\scommand.\sAnd\sfor\sarguments\sto\sthe\s"--extract"\ncommand.
-D 2017-12-13T20:04:53.034
+C Add\sthe\sshell\stool\s".ar\s--update"\scommand.
+D 2017-12-13T20:17:18.759
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
@@ -474,7 +474,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
-F src/shell.c.in b53eddcb293a9d35c0673c1d3bf2cf120b55e6660b436f65d689776a56391562
+F src/shell.c.in 12313c0500b9b1958507d3a432c192f73f86da112f0f37467636530696d13152
 F src/sqlite.h.in 8fd97993d48b50b9bade38c52f12d175942c9497c960905610c7b03a3e4b5818
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1682,7 +1682,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 1a9867973c9d6675fa5254fdd74f36004707a98a91593a188033cf5a49cc7a0b
-R c505f9c0d154ac54b6761fcc5d07e027
+P 32c4fa2552bb0fa7d7d143108457efae7a756d6cb14b1d59312e56efac3b2656
+R 656e4fd0036ab859f27379daf7289767
 U dan
-Z dcb23b384070f4f2014f2150f41e157a
+Z 57a0e73a5198163aa720919c6750aab2
diff --git a/manifest.uuid b/manifest.uuid
index 71ba76fdc3..937d78b439 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-32c4fa2552bb0fa7d7d143108457efae7a756d6cb14b1d59312e56efac3b2656
\ No newline at end of file
+825e3c037b03fc09d581aeda0193ff1d4062404414c7354cb649f99aa9022d25
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 0230e5f4d7..e9a01ed779 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -4328,14 +4328,6 @@ static int arParseCommand(
   return SQLITE_OK;
 }
 
-/*
-** Implementation of .ar "Update" command. 
-*/
-static int arUpdateCmd(ShellState *p, sqlite3 *db, ArCommand *pAr){
-  raw_printf(stderr, "todo...\n");
-  return SQLITE_OK;
-}
-
 /*
 ** This function assumes that all arguments within the ArCommand.azArg[]
 ** array refer to archive members, as for the --extract or --list commands. 
@@ -4500,17 +4492,21 @@ static int arExtractCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
 
 
 /*
-** Implementation of .ar "Create" command. 
+** Implementation of .ar "create" and "update" commands.
 **
 ** Create the "sqlar" table in the database if it does not already exist.
 ** Then add each file in the azFile[] array to the archive. Directories
 ** are added recursively. If argument bVerbose is non-zero, a message is
 ** printed on stdout for each file archived.
+**
+** The create command is the same as update, except that it drops
+** any existing "sqlar" table before beginning.
 */
-static int arCreateCommand(
+static int arCreateUpdate(
   ShellState *p,                  /* Shell state pointer */
   sqlite3 *db,
-  ArCommand *pAr                  /* Command arguments and options */
+  ArCommand *pAr,                 /* Command arguments and options */
+  int bUpdate
 ){
   const char *zSql = "SELECT name, mode, mtime, data FROM fsdir(?, ?)";
   const char *zCreate = 
@@ -4535,8 +4531,10 @@ static int arCreateCommand(
   rc = sqlite3_exec(db, "SAVEPOINT ar;", 0, 0, 0);
   if( rc!=SQLITE_OK ) return rc;
 
-  rc = sqlite3_exec(db, zDrop, 0, 0, 0);
-  if( rc!=SQLITE_OK ) return rc;
+  if( bUpdate==0 ){
+    rc = sqlite3_exec(db, zDrop, 0, 0, 0);
+    if( rc!=SQLITE_OK ) return rc;
+  }
 
   rc = sqlite3_exec(db, zCreate, 0, 0, 0);
   shellPrepare(db, &rc, zInsert, &pInsert);
@@ -4610,6 +4608,30 @@ static int arCreateCommand(
   return rc;
 }
 
+/*
+** Implementation of .ar "Create" command. 
+**
+** Create the "sqlar" table in the database if it does not already exist.
+** Then add each file in the azFile[] array to the archive. Directories
+** are added recursively. If argument bVerbose is non-zero, a message is
+** printed on stdout for each file archived.
+*/
+static int arCreateCommand(
+  ShellState *p,                  /* Shell state pointer */
+  sqlite3 *db,
+  ArCommand *pAr                  /* Command arguments and options */
+){
+  return arCreateUpdate(p, db, pAr, 0);
+}
+
+/*
+** Implementation of .ar "Update" command. 
+*/
+static int arUpdateCmd(ShellState *p, sqlite3 *db, ArCommand *pAr){
+  return arCreateUpdate(p, db, pAr, 1);
+}
+
+
 /*
 ** Implementation of ".ar" dot command.
 */

From 258dfe48cf8edd4972c60545b7c08394af504fa7 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 13 Dec 2017 20:35:34 +0000
Subject: [PATCH 146/488] An improved way of generating the
 SQLITE_READONLY_DIRECTORY error. The error message submitted to sqlite3_log()
 is still correct this way.

FossilOrigin-Name: 1c0aa919ee429cd194820ec9c54084563f39e63fd399b23f859fc6703b429b15
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/os_unix.c |  9 +++------
 3 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/manifest b/manifest
index c2d0952bf2..4f30ae5e01 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C New\sresult\scode\sSQLITE_READONLY_DIRECTORY\sis\sreturned\swhen\san\sattempt\sis\nmade\sto\swrite\son\sa\sdatabase\sfile\sthat\sis\sin\sa\sread-only\sdirectory\sand\shence\nthe\sjournal\sfile\scould\snot\sbe\screated.\s\sThis\ssituation\sformerly\sreturned\nSQLITE_CANTOPEN,\swhich\sless\shelpful.
-D 2017-12-13T20:02:29.496
+C An\simproved\sway\sof\sgenerating\sthe\sSQLITE_READONLY_DIRECTORY\serror.\nThe\serror\smessage\ssubmitted\sto\ssqlite3_log()\sis\sstill\scorrect\sthis\sway.
+D 2017-12-13T20:35:34.439
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
@@ -456,7 +456,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104
 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
-F src/os_unix.c 1ec9390926a0626494466e572eb7754545acd3e423a4871a749fe35c09e0b6ab
+F src/os_unix.c 7fc2735390a7809d5d893ed735d994ff12521224b89738226fff6f1a0aa1c932
 F src/os_win.c 0a4afa35cc8e812000df3ea2f64b476131b39e29e75d8007d0504726e4761de4
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903
@@ -1680,7 +1680,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 65754dc3b830ef9482655d2e93c2cd4acaa7a8715a58d8309955d04378541f89
-R 317d939eaca617f6f45ef08fff90ca48
+P 3ec73c38f878d73d278fce99ba10c708dcc475835774f1e17769ff7315be6d7c
+R 22c3a7dd6bc608a3f08477eed3bb5065
 U drh
-Z f0826ac4a0e91b44749cba2450c52240
+Z 687b40dd4870cf20b77c228c17085d90
diff --git a/manifest.uuid b/manifest.uuid
index 4128241972..1669756c04 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-3ec73c38f878d73d278fce99ba10c708dcc475835774f1e17769ff7315be6d7c
\ No newline at end of file
+1c0aa919ee429cd194820ec9c54084563f39e63fd399b23f859fc6703b429b15
\ No newline at end of file
diff --git a/src/os_unix.c b/src/os_unix.c
index e82cec2333..2d377ef56a 100644
--- a/src/os_unix.c
+++ b/src/os_unix.c
@@ -5904,12 +5904,6 @@ static int unixOpen(
     fd = robust_open(zName, openFlags, openMode);
     OSTRACE(("OPENX   %-3d %s 0%o\n", fd, zName, openFlags));
     assert( !isExclusive || (openFlags & O_CREAT)!=0 );
-    if( fd<0 && isNewJrnl && osAccess(zName, F_OK) ){
-      /* Trying to create a journal file where we don't have write 
-      ** permission on the directory */
-      rc = unixLogError(SQLITE_READONLY_DIRECTORY, "open", zName);
-      goto open_finished;
-    }
     if( fd<0 && errno!=EISDIR && isReadWrite ){
       /* Failed to open the file for read/write access. Try read-only. */
       flags &= ~(SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE);
@@ -5921,6 +5915,9 @@ static int unixOpen(
     }
     if( fd<0 ){
       rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zName);
+      /* If unable to create a journal, change the error code to
+      ** indicate that the directory permissions are wrong. */
+      if( isNewJrnl && osAccess(zName, F_OK) ) rc = SQLITE_READONLY_DIRECTORY;
       goto open_finished;
     }
 

From 3384ccb8bb7055d58f875ad92ffd6ad0dfe75519 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Wed, 13 Dec 2017 22:42:55 +0000
Subject: [PATCH 147/488] Add support for the 'lsm1.c' target in the MSVC
 makefile.

FossilOrigin-Name: 6de21deac469ab25378656f6f58115a92f5892428c6f2f3545c9bafac37e4a41
---
 Makefile.msc  | 23 +++++++++++++++++++++++
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 3 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/Makefile.msc b/Makefile.msc
index b16f08adee..9eb443faf3 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -2075,6 +2075,24 @@ FTS5_SRC = \
    $(TOP)\ext\fts5\fts5_varint.c \
    $(TOP)\ext\fts5\fts5_vocab.c
 
+LSM1_SRC = \
+   $(TOP)\ext\lsm1\lsm.h \
+   $(TOP)\ext\lsm1\lsmInt.h \
+   $(TOP)\ext\lsm1\lsm_ckpt.c \
+   $(TOP)\ext\lsm1\lsm_file.c \
+   $(TOP)\ext\lsm1\lsm_log.c \
+   $(TOP)\ext\lsm1\lsm_main.c \
+   $(TOP)\ext\lsm1\lsm_mem.c \
+   $(TOP)\ext\lsm1\lsm_mutex.c \
+   $(TOP)\ext\lsm1\lsm_shared.c \
+   $(TOP)\ext\lsm1\lsm_sorted.c \
+   $(TOP)\ext\lsm1\lsm_str.c \
+   $(TOP)\ext\lsm1\lsm_tree.c \
+   $(TOP)\ext\lsm1\lsm_unix.c \
+   $(TOP)\ext\lsm1\lsm_varint.c \
+   $(TOP)\ext\lsm1\lsm_vtab.c \
+   $(TOP)\ext\lsm1\lsm_win32.c
+
 fts5parse.c:	$(TOP)\ext\fts5\fts5parse.y lemon.exe
 	copy $(TOP)\ext\fts5\fts5parse.y .
 	del /Q fts5parse.h 2>NUL
@@ -2086,6 +2104,10 @@ fts5.c:	$(FTS5_SRC)
 	$(TCLSH_CMD) $(TOP)\ext\fts5\tool\mkfts5c.tcl
 	copy $(TOP)\ext\fts5\fts5.h .
 
+lsm1.c:	$(LSM1_SRC)
+	$(TCLSH_CMD) $(TOP)\ext\lsm1\tool\mklsm1c.tcl
+	copy $(TOP)\ext\lsm1\lsm.h .
+
 fts5.lo:	fts5.c $(HDR) $(EXTHDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c fts5.c
 
@@ -2315,4 +2337,5 @@ clean:
 	del /Q sqlite-*-output.vsix 2>NUL
 	del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe dbhash.exe 2>NUL
 	del /Q fts5.* fts5parse.* 2>NUL
+	del /Q lsm.h lsm1.c 2>NUL
 # <</mark>>
diff --git a/manifest b/manifest
index 4f30ae5e01..4b4c5e5dc3 100644
--- a/manifest
+++ b/manifest
@@ -1,8 +1,8 @@
-C An\simproved\sway\sof\sgenerating\sthe\sSQLITE_READONLY_DIRECTORY\serror.\nThe\serror\smessage\ssubmitted\sto\ssqlite3_log()\sis\sstill\scorrect\sthis\sway.
-D 2017-12-13T20:35:34.439
+C Add\ssupport\sfor\sthe\s'lsm1.c'\starget\sin\sthe\sMSVC\smakefile.
+D 2017-12-13T22:42:55.057
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc e5d7606238f55816da99f719969598df5b091aa2e9a6935c9412fcae8f53fc44
+F Makefile.msc a2492b29176edc3c754aa7a2f7daa20cd3fa20a56e3ee64e376092836177c42a
 F README.md eeae1e552f93ef72ef7c5b8f6647b368a001c28820ad1df179d3dae602bef681
 F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -1680,7 +1680,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3ec73c38f878d73d278fce99ba10c708dcc475835774f1e17769ff7315be6d7c
-R 22c3a7dd6bc608a3f08477eed3bb5065
-U drh
-Z 687b40dd4870cf20b77c228c17085d90
+P 1c0aa919ee429cd194820ec9c54084563f39e63fd399b23f859fc6703b429b15
+R c15fac6ddfa8bdc9c58742ac799b0523
+U mistachkin
+Z f5d555af74f00cd8b8d0269819b3c3ed
diff --git a/manifest.uuid b/manifest.uuid
index 1669756c04..01b0fc2c62 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-1c0aa919ee429cd194820ec9c54084563f39e63fd399b23f859fc6703b429b15
\ No newline at end of file
+6de21deac469ab25378656f6f58115a92f5892428c6f2f3545c9bafac37e4a41
\ No newline at end of file

From 84a6c85c644981854e61ce9d9ebda5a9762ee27a Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 13 Dec 2017 23:47:55 +0000
Subject: [PATCH 148/488] In valueFromExpr() only generate a OOM fault if there
 have been now prior faults.

FossilOrigin-Name: 3765aaf712998af5ffb6bc680a0c1419f2b5deb47ecbc1835ba5879127c4dbe3
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/vdbemem.c |  5 ++++-
 3 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index 4b4c5e5dc3..1c8585e662 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\ssupport\sfor\sthe\s'lsm1.c'\starget\sin\sthe\sMSVC\smakefile.
-D 2017-12-13T22:42:55.057
+C In\svalueFromExpr()\sonly\sgenerate\sa\sOOM\sfault\sif\sthere\shave\sbeen\snow\sprior\nfaults.
+D 2017-12-13T23:47:55.478
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a2492b29176edc3c754aa7a2f7daa20cd3fa20a56e3ee64e376092836177c42a
@@ -547,7 +547,7 @@ F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9
 F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1
 F src/vdbeaux.c b02a1f842c0e916285643b8475b7189f10b76f9e7edb5e2353a913c7980f90b5
 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191
-F src/vdbemem.c 3e8c795e0874b2848658b7ca96930a7bcda3f80dc93be2a3422cd30967961e4c
+F src/vdbemem.c 8478f7fb1948bf8fdeec7c2cb59ea58155c31258b9cd43c56d485e03ed40bd07
 F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a
@@ -1680,7 +1680,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 1c0aa919ee429cd194820ec9c54084563f39e63fd399b23f859fc6703b429b15
-R c15fac6ddfa8bdc9c58742ac799b0523
-U mistachkin
-Z f5d555af74f00cd8b8d0269819b3c3ed
+P 6de21deac469ab25378656f6f58115a92f5892428c6f2f3545c9bafac37e4a41
+R 44abcc3575bcbddb8f261c0eaf812ba5
+U drh
+Z 3fcdf29f818607817f06d87e8d680060
diff --git a/manifest.uuid b/manifest.uuid
index 01b0fc2c62..f81e89e40c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6de21deac469ab25378656f6f58115a92f5892428c6f2f3545c9bafac37e4a41
\ No newline at end of file
+3765aaf712998af5ffb6bc680a0c1419f2b5deb47ecbc1835ba5879127c4dbe3
\ No newline at end of file
diff --git a/src/vdbemem.c b/src/vdbemem.c
index 17d57bbfa9..107d831f4c 100644
--- a/src/vdbemem.c
+++ b/src/vdbemem.c
@@ -1420,7 +1420,10 @@ static int valueFromExpr(
   return rc;
 
 no_mem:
-  sqlite3OomFault(db);
+#ifdef SQLITE_ENABLE_STAT3_OR_STAT4
+  if( pCtx==0 || pCtx->pParse->nErr==0 )
+#endif
+    sqlite3OomFault(db);
   sqlite3DbFree(db, zVal);
   assert( *ppVal==0 );
 #ifdef SQLITE_ENABLE_STAT3_OR_STAT4

From 0d0547fec67e1ef00b68fdfd648709f7f5477719 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 14 Dec 2017 15:40:42 +0000
Subject: [PATCH 149/488] Improve error and usage messages output by the shell
 ".ar" command.

FossilOrigin-Name: b9d2d5d97291bf3d1392232e3705cca89dc7b918db2b08067b2b013ea39320e0
---
 manifest       | 12 +++----
 manifest.uuid  |  2 +-
 src/shell.c.in | 92 +++++++++++++++++++++++++++++++++++++++-----------
 3 files changed, 80 insertions(+), 26 deletions(-)

diff --git a/manifest b/manifest
index d58279c9ae..dda85f8287 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\slatest\strunk\schanges\sinto\sthis\sbranch.
-D 2017-12-14T13:55:01.572
+C Improve\serror\sand\susage\smessages\soutput\sby\sthe\sshell\s".ar"\scommand.
+D 2017-12-14T15:40:42.931
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
@@ -475,7 +475,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
-F src/shell.c.in 12313c0500b9b1958507d3a432c192f73f86da112f0f37467636530696d13152
+F src/shell.c.in 4bdd2efe722005180365698f2a3de51e22ae1e9bb61c868006bc8f2c5e02eb98
 F src/sqlite.h.in 364515dd186285f3c01f5cab42e7db7edc47c70e87b6a25de389a2e6b8c413fd
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1683,7 +1683,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 825e3c037b03fc09d581aeda0193ff1d4062404414c7354cb649f99aa9022d25 3765aaf712998af5ffb6bc680a0c1419f2b5deb47ecbc1835ba5879127c4dbe3
-R bf083f80c8163c105f680457377723eb
+P 803156cba8b056a1cb8d1bb186a57454afe72341abe7de1dfe529234c3415cd2
+R 69895ee1194373ab4ba11578d798fe26
 U dan
-Z 36133c008e2dcaa09c3c6c120c31dee9
+Z 5ea750b0cf239f9442948a3eda934166
diff --git a/manifest.uuid b/manifest.uuid
index 4a8f53dbe3..ce1ae1dcb3 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-803156cba8b056a1cb8d1bb186a57454afe72341abe7de1dfe529234c3415cd2
\ No newline at end of file
+b9d2d5d97291bf3d1392232e3705cca89dc7b918db2b08067b2b013ea39320e0
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index e9a01ed779..2a0f9fde9e 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -4152,9 +4152,46 @@ struct ArCommand {
 /*
 ** Print a usage message for the .ar command to stderr and return SQLITE_ERROR.
 */
-static int arUsage(void){
-  /* todo */
-  raw_printf(stderr, "error in .ar command line\n");
+static int arUsage(FILE *f){
+  raw_printf(f,
+"\n"
+"Usage: .ar [OPTION...] [FILE...]\n"
+"The .ar command manages sqlar archives.\n"
+"\n"
+"Examples:\n"
+"  .ar -cf archive.sar foo bar    # Create archive.sar from files foo and bar\n"
+"  .ar -tf archive.sar            # List members of archive.sar\n"
+"  .ar -xvf archive.sar           # Verbosely extract files from archive.sar\n"
+"\n"
+"Each command line must feature exactly one command option:\n"
+"  -c, --create               Create a new archive\n"
+"  -u, --update               Update or add files to an existing archive\n"
+"  -t, --list                 List contents of archive\n"
+"  -x, --extract              Extract files from archive\n"
+"\n"
+"And zero or more optional options:\n"
+"  -v, --verbose              Print each filename as it is processed\n"
+"  -f FILE, --file FILE       Operate on archive FILE (default is current db)\n"
+"  -C DIR, --directory DIR    Change to directory DIR to read/extract files\n"
+"\n"
+"See also: http://sqlite.org/cli.html#sqlar_archive_support\n"
+"\n"
+);
+  return SQLITE_ERROR;
+}
+
+/*
+** Print an error message for the .ar command to stderr and return 
+** SQLITE_ERROR.
+*/
+static int arErrorMsg(const char *zFmt, ...){
+  va_list ap;
+  char *z;
+  va_start(ap, zFmt);
+  z = sqlite3_vmprintf(zFmt, ap);
+  va_end(ap);
+  raw_printf(stderr, "Error: %s (try \".ar --help\")\n", z);
+  sqlite3_free(z);
   return SQLITE_ERROR;
 }
 
@@ -4165,13 +4202,14 @@ static int arUsage(void){
 #define AR_CMD_EXTRACT      2
 #define AR_CMD_LIST         3
 #define AR_CMD_UPDATE       4
+#define AR_CMD_HELP         5
 
 /*
 ** Other (non-command) switches.
 */
-#define AR_SWITCH_VERBOSE   5
-#define AR_SWITCH_FILE      6
-#define AR_SWITCH_DIRECTORY 7
+#define AR_SWITCH_VERBOSE   6
+#define AR_SWITCH_FILE      7
+#define AR_SWITCH_DIRECTORY 8
 
 static int arProcessSwitch(ArCommand *pAr, int eSwitch, const char *zArg){
   switch( eSwitch ){
@@ -4179,7 +4217,10 @@ static int arProcessSwitch(ArCommand *pAr, int eSwitch, const char *zArg){
     case AR_CMD_EXTRACT:
     case AR_CMD_LIST:
     case AR_CMD_UPDATE:
-      if( pAr->eCmd ) return arUsage();
+    case AR_CMD_HELP:
+      if( pAr->eCmd ){
+        return arErrorMsg("multiple command options");
+      }
       pAr->eCmd = eSwitch;
       break;
 
@@ -4219,6 +4260,7 @@ static int arParseCommand(
     { 'x', "extract",   AR_CMD_EXTRACT, 0 },
     { 't', "list",      AR_CMD_LIST, 0 },
     { 'u', "update",    AR_CMD_UPDATE, 0 },
+    { 'h', "help",      AR_CMD_HELP, 0 },
     { 'v', "verbose",   AR_SWITCH_VERBOSE, 0 },
     { 'f', "file",      AR_SWITCH_FILE, 1 },
     { 'C', "directory", AR_SWITCH_DIRECTORY, 1 }
@@ -4227,7 +4269,7 @@ static int arParseCommand(
   struct ArSwitch *pEnd = &aSwitch[nSwitch];
 
   if( nArg<=1 ){
-    return arUsage();
+    return arUsage(stderr);
   }else{
     char *z = azArg[1];
     memset(pAr, 0, sizeof(ArCommand));
@@ -4242,9 +4284,13 @@ static int arParseCommand(
         for(pOpt=&aSwitch[0]; pOpt<pEnd; pOpt++){
           if( z[i]==pOpt->cShort ) break;
         }
-        if( pOpt==pEnd ) return arUsage();
+        if( pOpt==pEnd ){
+          return arErrorMsg("unrecognized option: %c", z[i]);
+        }
         if( pOpt->bArg ){
-          if( iArg>=nArg ) return arUsage();
+          if( iArg>=nArg ){
+            return arErrorMsg("option requires an argument: %c",z[i]);
+          }
           zArg = azArg[iArg++];
         }
         if( arProcessSwitch(pAr, pOpt->eSwitch, zArg) ) return SQLITE_ERROR;
@@ -4276,13 +4322,17 @@ static int arParseCommand(
             for(pOpt=&aSwitch[0]; pOpt<pEnd; pOpt++){
               if( z[i]==pOpt->cShort ) break;
             }
-            if( pOpt==pEnd ) return arUsage();
+            if( pOpt==pEnd ){
+              return arErrorMsg("unrecognized option: %c\n", z[i]);
+            }
             if( pOpt->bArg ){
               if( i<(n-1) ){
                 zArg = &z[i+1];
                 i = n;
               }else{
-                if( iArg>=(nArg-1) ) return arUsage();
+                if( iArg>=(nArg-1) ){
+                  return arErrorMsg("option requires an argument: %c\n",z[i]);
+                }
                 zArg = azArg[++iArg];
               }
             }
@@ -4303,8 +4353,7 @@ static int arParseCommand(
             const char *zLong = pOpt->zLong;
             if( (n-2)<=strlen(zLong) && 0==memcmp(&z[2], zLong, n-2) ){
               if( pMatch ){
-                /* ambiguous option */
-                return arUsage();
+                return arErrorMsg("ambiguous option: %s",z);
               }else{
                 pMatch = pOpt;
               }
@@ -4312,11 +4361,12 @@ static int arParseCommand(
           }
 
           if( pMatch==0 ){
-            /* no such option. */
-            return arUsage();
+            return arErrorMsg("unrecognized option: %s", z);
           }
           if( pMatch->bArg ){
-            if( iArg>=(nArg-1) ) return arUsage();
+            if( iArg>=(nArg-1) ){
+              return arErrorMsg("option requires an argument: %s", z);
+            }
             zArg = azArg[++iArg];
           }
           if( arProcessSwitch(pAr, pMatch->eSwitch, zArg) ) return SQLITE_ERROR;
@@ -4465,7 +4515,7 @@ static int arExtractCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
   }
   while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
     if( pAr->bVerbose ){
-      raw_printf(stdout, "%s\n", sqlite3_column_text(pSql, 0));
+      raw_printf(p->out, "%s\n", sqlite3_column_text(pSql, 0));
     }
   }
   shellFinalize(&rc, pSql);
@@ -4550,7 +4600,7 @@ static int arCreateUpdate(
       unsigned int mtime = sqlite3_column_int(pStmt, 2);
 
       if( pAr->bVerbose ){
-        raw_printf(stdout, "%s\n", zName);
+        raw_printf(p->out, "%s\n", zName);
       }
 
       sqlite3_bind_text(pInsert, 1, zName, -1, SQLITE_STATIC);
@@ -4680,6 +4730,10 @@ static int arDotCommand(
         rc = arListCommand(pState, db, &cmd);
         break;
 
+      case AR_CMD_HELP:
+        arUsage(pState->out);
+        break;
+
       default:
         assert( cmd.eCmd==AR_CMD_UPDATE );
         rc = arUpdateCmd(pState, db, &cmd);

From e483d349ab20b49cf704423f3d15e30d722f25e2 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 14 Dec 2017 16:28:27 +0000
Subject: [PATCH 150/488] Add the ability to write to an appended database. 
 This check-in compiles but is otherwise untested.

FossilOrigin-Name: e343c63cbd754f37c33c939cd0b6f1ecc6202e60c6e66cd65c23cc8d571a994e
---
 ext/misc/appendvfs.c | 187 ++++++++++++++++++++++++++++++++++---------
 manifest             |  12 +--
 manifest.uuid        |   2 +-
 3 files changed, 155 insertions(+), 46 deletions(-)

diff --git a/ext/misc/appendvfs.c b/ext/misc/appendvfs.c
index b650b80be3..d0096f1224 100644
--- a/ext/misc/appendvfs.c
+++ b/ext/misc/appendvfs.c
@@ -18,8 +18,31 @@
 ** best performance page 1 should be located at a disk page boundary, though
 ** that is not required.
 **
-** An appended database is considered immutable.  It is read-only and no
-** locks are ever taken.
+** When opening a database using this VFS, the connection might treat
+** the file as an ordinary SQLite database, or it might treat is as a
+** database appended onto some other file.  Here are the rules:
+**
+**  (1)  When opening a new empty file, that file is treated as an ordinary
+**       database.
+**
+**  (2)  When opening a file that begins with the standard SQLite prefix
+**       string "SQLite format 3", that file is treated as an ordinary
+**       database.
+**
+**  (3)  When opening a file that ends with the appendvfs trailer string
+**       "Start-Of-SQLite3-NNNNNNNN" that file is treated as an appended
+**       database.
+**
+**  (4)  If none of the above apply and the SQLITE_OPEN_CREATE flag is
+**       set, then a new database is appended to the already existing file.
+**
+**  (5)  Otherwise, SQLITE_CANTOPEN is returned.
+**
+** To avoid unnecessary complications with the PENDING_BYTE, the size of
+** the file containing the database is limited to 1GB.  This VFS will refuse
+** to read or write past the 1GB mark.  This restriction might be lifted in
+** future versions.  For now, if you need a large database, then keep the
+** database in a separate file.
 **
 ** If the file being opened is not an appended database, then this shim is
 ** a pass-through into the default underlying VFS.
@@ -41,6 +64,12 @@ SQLITE_EXTENSION_INIT1
 #define APND_MARK_PREFIX_SZ  17
 #define APND_MARK_SIZE       25
 
+/*
+** Maximum size of the combined prefix + database + append-mark.  This
+** must be less than 0x40000000 to avoid locking issues on Windows.
+*/
+#define APND_MAX_SIZE  (65536*15259)
+
 /*
 ** Forward declaration of objects used by this utility
 */
@@ -57,6 +86,7 @@ typedef struct ApndFile ApndFile;
 struct ApndFile {
   sqlite3_file base;              /* IO methods */
   sqlite3_int64 iPgOne;           /* File offset to page 1 */
+  sqlite3_int64 iMark;            /* Start of the append-mark */
 };
 
 /*
@@ -172,37 +202,75 @@ static int apndRead(
   return pFile->pMethods->xRead(pFile, zBuf, iAmt, iOfst+p->iPgOne);
 }
 
+/*
+** Add the append-mark onto the end of the file.
+*/
+static int apndWriteMark(ApndFile *p, sqlite3_file *pFile){
+  int i;
+  unsigned char a[APND_MARK_SIZE];
+  memcpy(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ);
+  for(i=0; i<8; i++){
+    a[APND_MARK_PREFIX_SZ+i] = (p->iPgOne >> (56 - i*8)) & 0xff;
+  }
+  return pFile->pMethods->xWrite(pFile, a, APND_MARK_PREFIX_SZ, p->iMark);
+}
+
 /*
 ** Write data to an apnd-file.
 */
 static int apndWrite(
   sqlite3_file *pFile,
-  const void *z,
+  const void *zBuf,
   int iAmt,
   sqlite_int64 iOfst
 ){
-  return SQLITE_READONLY;
+  int rc;
+  ApndFile *p = (ApndFile *)pFile;
+  pFile = ORIGFILE(pFile);
+  if( iOfst+iAmt>=APND_MAX_SIZE ) return SQLITE_FULL;
+  rc = pFile->pMethods->xWrite(pFile, zBuf, iAmt, iOfst+p->iPgOne);
+  if( rc==SQLITE_OK &&  iOfst + iAmt + p->iPgOne > p->iMark ){
+    sqlite3_int64 sz = 0;
+    rc = pFile->pMethods->xFileSize(pFile, &sz);
+    if( rc==SQLITE_OK ){
+      p->iMark = sz - APND_MARK_SIZE;
+      if( iOfst + iAmt + p->iPgOne > p->iMark ){
+        p->iMark = p->iPgOne + iOfst + iAmt;
+        rc = apndWriteMark(p, pFile);
+      }
+    }
+  }
+  return rc;
 }
 
 /*
 ** Truncate an apnd-file.
 */
 static int apndTruncate(sqlite3_file *pFile, sqlite_int64 size){
-  return SQLITE_READONLY;
+  int rc;
+  ApndFile *p = (ApndFile *)pFile;
+  pFile = ORIGFILE(pFile);
+  rc = pFile->pMethods->xTruncate(pFile, size+p->iPgOne+APND_MARK_SIZE);
+  if( rc==SQLITE_OK ){
+    p->iMark = p->iPgOne+size;
+    rc = apndWriteMark(p, pFile);
+  }
+  return rc;
 }
 
 /*
 ** Sync an apnd-file.
 */
 static int apndSync(sqlite3_file *pFile, int flags){
-  return SQLITE_READONLY;
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xSync(pFile, flags);
 }
 
 /*
 ** Return the current file-size of an apnd-file.
 */
 static int apndFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
-  ApndFile *p = (ApndFile*)pFile;
+  ApndFile *p = (ApndFile *)pFile;
   int rc;
   pFile = ORIGFILE(p);
   rc = pFile->pMethods->xFileSize(pFile, pSize);
@@ -216,22 +284,24 @@ static int apndFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
 ** Lock an apnd-file.
 */
 static int apndLock(sqlite3_file *pFile, int eLock){
-  return SQLITE_READONLY;
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xLock(pFile, eLock);
 }
 
 /*
 ** Unlock an apnd-file.
 */
 static int apndUnlock(sqlite3_file *pFile, int eLock){
-  return SQLITE_OK;
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xUnlock(pFile, eLock);
 }
 
 /*
 ** Check if another file-handle holds a RESERVED lock on an apnd-file.
 */
 static int apndCheckReservedLock(sqlite3_file *pFile, int *pResOut){
-  *pResOut = 0;
-  return SQLITE_OK;
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xCheckReservedLock(pFile, pResOut);
 }
 
 /*
@@ -273,22 +343,26 @@ static int apndShmMap(
   int bExtend,
   void volatile **pp
 ){
-  return SQLITE_READONLY;
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xShmMap(pFile,iPg,pgsz,bExtend,pp);
 }
 
 /* Perform locking on a shared-memory segment */
 static int apndShmLock(sqlite3_file *pFile, int offset, int n, int flags){
-  return SQLITE_READONLY;
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xShmLock(pFile,offset,n,flags);
 }
 
 /* Memory barrier operation on shared memory */
 static void apndShmBarrier(sqlite3_file *pFile){
-  return;
+  pFile = ORIGFILE(pFile);
+  pFile->pMethods->xShmBarrier(pFile);
 }
 
 /* Unmap a shared memory segment */
 static int apndShmUnmap(sqlite3_file *pFile, int deleteFlag){
-  return SQLITE_OK;
+  pFile = ORIGFILE(pFile);
+  return pFile->pMethods->xShmUnmap(pFile,deleteFlag);
 }
 
 /* Fetch a page of a memory-mapped file */
@@ -310,6 +384,40 @@ static int apndUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){
   return pFile->pMethods->xUnfetch(pFile, iOfst+p->iPgOne, pPage);
 }
 
+/*
+** Check to see if the file is an ordinary SQLite database file.
+*/
+static int apndIsOrdinaryDatabaseFile(sqlite3_int64 sz, sqlite3_file *pFile){
+  int rc;
+  char zHdr[16];
+  static const char aSqliteHdr[] = "SQLite format 3";
+  if( sz<512 ) return 0;
+  rc = pFile->pMethods->xRead(pFile, zHdr, sizeof(zHdr), 0);
+  if( rc ) return 0;
+  return memcmp(zHdr, aSqliteHdr, sizeof(zHdr))==0;
+}
+
+/*
+** Try to read the append-mark off the end of a file.  Return the
+** start of the appended database if the append-mark is present.  If
+** there is no append-mark, return -1;
+*/
+static sqlite3_int64 apndReadMark(sqlite3_int64 sz, sqlite3_file *pFile){
+  int rc, i;
+  sqlite3_int64 iMark;
+  unsigned char a[APND_MARK_SIZE];
+
+  if( sz<=APND_MARK_SIZE ) return -1;
+  rc = pFile->pMethods->xRead(pFile, a, APND_MARK_SIZE, sz-APND_MARK_SIZE);
+  if( rc ) return -1;
+  if( memcmp(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ)!=0 ) return -1;
+  iMark = ((sqlite3_int64)(a[APND_MARK_PREFIX_SZ]&0x7f))<<56;
+  for(i=1; i<8; i++){    
+    iMark += (sqlite3_int64)a[APND_MARK_PREFIX_SZ+i]<<(56-8*i);
+  }
+  return iMark;
+}
+
 /*
 ** Open an apnd file handle.
 */
@@ -321,38 +429,39 @@ static int apndOpen(
   int *pOutFlags
 ){
   ApndFile *p;
+  sqlite3_file *pSubFile;
+  sqlite3_vfs *pSubVfs;
   int rc;
   sqlite3_int64 sz;
-  pVfs = ORIGVFS(pVfs);
+  pSubVfs = ORIGVFS(pVfs);
   if( (flags & SQLITE_OPEN_MAIN_DB)==0 ){
-    return pVfs->xOpen(pVfs, zName, pFile, flags, pOutFlags);
+    return pSubVfs->xOpen(pSubVfs, zName, pFile, flags, pOutFlags);
   }
   p = (ApndFile*)pFile;
   memset(p, 0, sizeof(*p));
   p->base.pMethods = &apnd_io_methods;
-  pFile = ORIGFILE(pFile);
-  flags &= ~(SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE);
-  flags |= SQLITE_OPEN_READONLY;
-  rc = pVfs->xOpen(pVfs, zName, pFile, flags, pOutFlags);
+  pSubFile = ORIGFILE(pFile);
+  rc = pSubVfs->xOpen(pSubVfs, zName, pSubFile, flags, pOutFlags);
   if( rc ) return rc;
-  rc = pFile->pMethods->xFileSize(pFile, &sz);
-  if( rc==SQLITE_OK && sz>512 ){
-    unsigned char a[APND_MARK_SIZE];
-    rc = pFile->pMethods->xRead(pFile, a, APND_MARK_SIZE, sz-APND_MARK_SIZE);
-    if( rc==SQLITE_OK
-     && memcmp(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ)==0
-    ){
-      p->iPgOne =
-         ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ]<<56) +
-         ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+1]<<48) +
-         ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+2]<<40) +
-         ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+3]<<32) +
-         ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+4]<<24) +
-         ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+5]<<16) +
-         ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+6]<<8) +
-         ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+7]);
-    }
+  rc = pSubFile->pMethods->xFileSize(pSubFile, &sz);
+  if( rc ){
+    pSubFile->pMethods->xClose(pSubFile);
+    return rc;
   }
+  if( apndIsOrdinaryDatabaseFile(sz, pSubFile) ){
+    memmove(pFile, pSubFile, pSubVfs->szOsFile);
+    return SQLITE_OK;
+  }
+  p->iMark = 0;
+  p->iPgOne = apndReadMark(sz, pFile);
+  if( p->iPgOne>0 ){
+    return SQLITE_OK;
+  }
+  if( (flags & SQLITE_OPEN_CREATE)==0 ){
+    pSubFile->pMethods->xClose(pSubFile);
+    return SQLITE_CANTOPEN;
+  }
+  p->iPgOne = (sz+0xfff) & ~(sqlite3_int64)0xfff;
   return SQLITE_OK;
 }
 
@@ -442,7 +551,7 @@ int sqlite3_appendvfs_init(
   apnd_vfs.iVersion = pOrig->iVersion;
   apnd_vfs.pAppData = pOrig;
   apnd_vfs.szOsFile = sizeof(ApndFile);
-  rc = sqlite3_vfs_register(&apnd_vfs, 1);
+  rc = sqlite3_vfs_register(&apnd_vfs, 0);
 #ifdef APPENDVFS_TEST
   if( rc==SQLITE_OK ){
     rc = sqlite3_auto_extension((void(*)(void))apndvfsRegister);
diff --git a/manifest b/manifest
index 1b36095a3f..4ddbb0d1d5 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Bring\sin\sthe\slatest\strunk\schanges.
-D 2017-12-14T14:50:49.020
+C Add\sthe\sability\sto\swrite\sto\san\sappended\sdatabase.\s\sThis\scheck-in\scompiles\nbut\sis\sotherwise\suntested.
+D 2017-12-14T16:28:27.146
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a2492b29176edc3c754aa7a2f7daa20cd3fa20a56e3ee64e376092836177c42a
@@ -260,7 +260,7 @@ F ext/lsm1/tool/mklsm1c.tcl f31561bbee5349f0a554d1ad7236ac1991fc09176626f529f607
 F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601d90b2
 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87
 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb
-F ext/misc/appendvfs.c 487a5db4427d118413346780c4c90d0256315e8de3fb528ee54daef3bb73276f
+F ext/misc/appendvfs.c c8b18caae0d22762080a9c48a4e46933f48625bac7fde2af9ef8ec2a23755ac9
 F ext/misc/btreeinfo.c d7fd9a2fe2fa33ba28488e2fce703ebecc759219ea9e0bb3b254784866c0a676
 F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005
 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704
@@ -1681,7 +1681,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 1a1a73b821eb1a22ca335f582a0aea31c71ca9f5b09d54f26409691c90e38c4a 3765aaf712998af5ffb6bc680a0c1419f2b5deb47ecbc1835ba5879127c4dbe3
-R 29a07f33957881a5b837419ff7f21683
+P 75d8517703f7efa33437079108e2c4ef0de1a118bbe1f4a86afdc34da09d3008
+R 15eddfa7673da0a2579cc83183e243e0
 U drh
-Z d18bd4436a6dfca865f7b12ca3ce5865
+Z 4e88658420f57b7b81dd23d0961f71ee
diff --git a/manifest.uuid b/manifest.uuid
index 29cb2dde87..026378badf 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-75d8517703f7efa33437079108e2c4ef0de1a118bbe1f4a86afdc34da09d3008
\ No newline at end of file
+e343c63cbd754f37c33c939cd0b6f1ecc6202e60c6e66cd65c23cc8d571a994e
\ No newline at end of file

From 233ff96ec5beae1301731a3c0893f8cf358c86f0 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 14 Dec 2017 16:57:11 +0000
Subject: [PATCH 151/488] The "apndvfs" VFS shim now appears to be working for
 both reads and writes.

FossilOrigin-Name: 7f7b72d83633922e2b5dbf0d3455d0fea72cc6b8925ffcb78dfbad7b3c4b26e9
---
 ext/misc/appendvfs.c |  7 +++----
 manifest             | 12 ++++++------
 manifest.uuid        |  2 +-
 3 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/ext/misc/appendvfs.c b/ext/misc/appendvfs.c
index d0096f1224..bd1f37509e 100644
--- a/ext/misc/appendvfs.c
+++ b/ext/misc/appendvfs.c
@@ -212,7 +212,7 @@ static int apndWriteMark(ApndFile *p, sqlite3_file *pFile){
   for(i=0; i<8; i++){
     a[APND_MARK_PREFIX_SZ+i] = (p->iPgOne >> (56 - i*8)) & 0xff;
   }
-  return pFile->pMethods->xWrite(pFile, a, APND_MARK_PREFIX_SZ, p->iMark);
+  return pFile->pMethods->xWrite(pFile, a, APND_MARK_SIZE, p->iMark);
 }
 
 /*
@@ -331,8 +331,7 @@ static int apndSectorSize(sqlite3_file *pFile){
 */
 static int apndDeviceCharacteristics(sqlite3_file *pFile){
   pFile = ORIGFILE(pFile);
-  return SQLITE_IOCAP_IMMUTABLE |
-           pFile->pMethods->xDeviceCharacteristics(pFile);
+  return pFile->pMethods->xDeviceCharacteristics(pFile);
 }
 
 /* Create a shared memory file mapping */
@@ -550,7 +549,7 @@ int sqlite3_appendvfs_init(
   pOrig = sqlite3_vfs_find(0);
   apnd_vfs.iVersion = pOrig->iVersion;
   apnd_vfs.pAppData = pOrig;
-  apnd_vfs.szOsFile = sizeof(ApndFile);
+  apnd_vfs.szOsFile = pOrig->szOsFile + sizeof(ApndFile);
   rc = sqlite3_vfs_register(&apnd_vfs, 0);
 #ifdef APPENDVFS_TEST
   if( rc==SQLITE_OK ){
diff --git a/manifest b/manifest
index 4ddbb0d1d5..c5da940c02 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\sability\sto\swrite\sto\san\sappended\sdatabase.\s\sThis\scheck-in\scompiles\nbut\sis\sotherwise\suntested.
-D 2017-12-14T16:28:27.146
+C The\s"apndvfs"\sVFS\sshim\snow\sappears\sto\sbe\sworking\sfor\sboth\sreads\sand\swrites.
+D 2017-12-14T16:57:11.644
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a2492b29176edc3c754aa7a2f7daa20cd3fa20a56e3ee64e376092836177c42a
@@ -260,7 +260,7 @@ F ext/lsm1/tool/mklsm1c.tcl f31561bbee5349f0a554d1ad7236ac1991fc09176626f529f607
 F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601d90b2
 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87
 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb
-F ext/misc/appendvfs.c c8b18caae0d22762080a9c48a4e46933f48625bac7fde2af9ef8ec2a23755ac9
+F ext/misc/appendvfs.c ee71fbc918ab4c526a5935a3153c1cc4e0b601c081da8f9a549e0e9d01bdbca5
 F ext/misc/btreeinfo.c d7fd9a2fe2fa33ba28488e2fce703ebecc759219ea9e0bb3b254784866c0a676
 F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005
 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704
@@ -1681,7 +1681,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 75d8517703f7efa33437079108e2c4ef0de1a118bbe1f4a86afdc34da09d3008
-R 15eddfa7673da0a2579cc83183e243e0
+P e343c63cbd754f37c33c939cd0b6f1ecc6202e60c6e66cd65c23cc8d571a994e
+R 427fca803286f3061d4cdb1346b5bd46
 U drh
-Z 4e88658420f57b7b81dd23d0961f71ee
+Z 919006b84ebd33ef7833311db2cf1160
diff --git a/manifest.uuid b/manifest.uuid
index 026378badf..566e5b932a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e343c63cbd754f37c33c939cd0b6f1ecc6202e60c6e66cd65c23cc8d571a994e
\ No newline at end of file
+7f7b72d83633922e2b5dbf0d3455d0fea72cc6b8925ffcb78dfbad7b3c4b26e9
\ No newline at end of file

From ac15e2d7ccfb706d020d4a42fc4d809f6ca1cca8 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 14 Dec 2017 19:15:07 +0000
Subject: [PATCH 152/488] Have the writefile() function optionally set the
 modification-time of the files it writes or creates. And many small fixes to
 the new code on this branch.

FossilOrigin-Name: 7b51269caebe1492885fe9b965892f49a3f8bdb1d666b0203d594c30f9e83938
---
 ext/misc/fileio.c | 136 ++++++++++++++++++++++++++++++++++++----------
 manifest          |  16 +++---
 manifest.uuid     |   2 +-
 src/shell.c.in    |  91 ++++++++++++++++---------------
 test/shell8.test  |  15 +++++
 5 files changed, 177 insertions(+), 83 deletions(-)

diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c
index 2320173997..7dbac4043f 100644
--- a/ext/misc/fileio.c
+++ b/ext/misc/fileio.c
@@ -11,21 +11,67 @@
 ******************************************************************************
 **
 ** This SQLite extension implements SQL functions readfile() and
-** writefile().
+** writefile(), and eponymous virtual type "fsdir".
 **
-** Also, an eponymous virtual table type "fsdir". Used as follows:
+** WRITEFILE(FILE, DATA [, MODE [, MTIME]]):
 **
-**   SELECT * FROM fsdir($dirname);
+**   If neither of the optional arguments is present, then this UDF
+**   function writes blob DATA to file FILE. If successful, the number
+**   of bytes written is returned. If an error occurs, NULL is returned.
 **
-** Returns one row for each entry in the directory $dirname. No row is
-** returned for "." or "..". Row columns are as follows:
+**   If the first option argument - MODE - is present, then it must
+**   be passed an integer value that corresponds to a POSIX mode
+**   value (file type + permissions, as returned in the stat.st_mode
+**   field by the stat() system call). Three types of files may
+**   be written/created:
 **
-**   name:  Name of directory entry.
-**   mode:  Value of stat.st_mode for directory entry.
-**   mtime: Value of stat.st_mtime for directory entry.
-**   data:  For a regular file, a blob containing the file data. For a
-**          symlink, a text value containing the text of the link. For a
-**          directory, NULL.
+**     regular files:  (mode & 0170000)==0100000
+**     symbolic links: (mode & 0170000)==0120000
+**     directories:    (mode & 0170000)==0040000
+**
+**   For a directory, the DATA is ignored. For a symbolic link, it is
+**   interpreted as text and used as the target of the link. For a
+**   regular file, it is interpreted as a blob and written into the
+**   named file. Regardless of the type of file, its permissions are
+**   set to (mode & 0777) before returning.
+**
+**   If the optional MTIME argument is present, then it is interpreted
+**   as an integer - the number of seconds since the unix epoch. The
+**   modification-time of the target file is set to this value before
+**   returning.
+**
+**   If three or more arguments are passed to this function and an
+**   error is encountered, an exception is raised.
+**
+** READFILE(FILE):
+**
+**   Read and return the contents of file FILE (type blob) from disk.
+**
+** FSDIR:
+**
+**   Used as follows:
+**
+**     SELECT * FROM fsdir($path [, $dir]);
+**
+**   Parameter $path is an absolute or relative pathname. If the file that it
+**   refers to does not exist, it is an error. If the path refers to a regular
+**   file or symbolic link, it returns a single row. Or, if the path refers
+**   to a directory, it returns one row for the directory, and one row for each
+**   file within the hierarchy rooted at $path.
+**
+**   Each row has the following columns:
+**
+**     name:  Path to file or directory (text value).
+**     mode:  Value of stat.st_mode for directory entry (an integer).
+**     mtime: Value of stat.st_mtime for directory entry (an integer).
+**     data:  For a regular file, a blob containing the file data. For a
+**            symlink, a text value containing the text of the link. For a
+**            directory, NULL.
+**
+**   If a non-NULL value is specified for the optional $dir parameter and
+**   $path is a relative path, then $path is interpreted relative to $dir. 
+**   And the paths returned in the "name" column of the table are also 
+**   relative to directory $dir.
 */
 #include "sqlite3ext.h"
 SQLITE_EXTENSION_INIT1
@@ -45,6 +91,10 @@ SQLITE_EXTENSION_INIT1
 
 #define FSDIR_SCHEMA "(name,mode,mtime,data,path HIDDEN,dir HIDDEN)"
 
+/*
+** Set the result stored by context ctx to a blob containing the 
+** contents of file zName.
+*/
 static void readFileContents(sqlite3_context *ctx, const char *zName){
   FILE *in;
   long nIn;
@@ -81,6 +131,10 @@ static void readfileFunc(
   readFileContents(context, zName);
 }
 
+/*
+** Set the error message contained in context ctx to the results of
+** vprintf(zFmt, ...).
+*/
 static void ctxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){
   char *zMsg = 0;
   va_list ap;
@@ -138,11 +192,16 @@ static int makeDirectory(
   return rc;
 }
 
+/*
+** This function does the work for the writefile() UDF. Refer to 
+** header comments at the top of this file for details.
+*/
 static int writeFile(
-  sqlite3_context *pCtx, 
-  const char *zFile, 
-  mode_t mode, 
-  sqlite3_value *pData
+  sqlite3_context *pCtx,          /* Context to return bytes written in */
+  const char *zFile,              /* File to write */
+  sqlite3_value *pData,           /* Data to write */
+  mode_t mode,                    /* MODE parameter passed to writefile() */
+  sqlite3_int64 mtime             /* MTIME parameter (or -1 to not set time) */
 ){
   if( S_ISLNK(mode) ){
     const char *zTo = (const char*)sqlite3_value_text(pData);
@@ -178,22 +237,30 @@ static int writeFile(
         }
       }
       fclose(out);
-      if( rc==0 && chmod(zFile, mode & 0777) ){
+      if( rc==0 && mode && chmod(zFile, mode & 0777) ){
         rc = 1;
       }
       if( rc ) return 2;
       sqlite3_result_int64(pCtx, nWrite);
     }
   }
+
+  if( mtime>=0 ){
+    struct timespec times[2];
+    times[0].tv_nsec = times[1].tv_nsec = 0;
+    times[0].tv_sec = time(0);
+    times[1].tv_sec = mtime;
+    if( utimensat(AT_FDCWD, zFile, times, AT_SYMLINK_NOFOLLOW) ){
+      return 1;
+    }
+  }
+
   return 0;
 }
 
 /*
-** Implementation of the "writefile(W,X[,Y]])" SQL function.  
-**
-** The argument X is written into file W.  The number of bytes written is
-** returned. Or NULL is returned if something goes wrong, such as being unable
-** to open file X for writing.
+** Implementation of the "writefile(W,X[,Y[,Z]]])" SQL function.  
+** Refer to header comments at the top of this file for details.
 */
 static void writefileFunc(
   sqlite3_context *context,
@@ -203,8 +270,9 @@ static void writefileFunc(
   const char *zFile;
   mode_t mode = 0;
   int res;
+  sqlite3_int64 mtime = -1;
 
-  if( argc<2 || argc>3 ){
+  if( argc<2 || argc>4 ){
     sqlite3_result_error(context, 
         "wrong number of arguments to function writefile()", -1
     );
@@ -214,18 +282,20 @@ static void writefileFunc(
   zFile = (const char*)sqlite3_value_text(argv[0]);
   if( zFile==0 ) return;
   if( argc>=3 ){
-    sqlite3_result_int(context, 0);
     mode = sqlite3_value_int(argv[2]);
   }
+  if( argc==4 ){
+    mtime = sqlite3_value_int64(argv[3]);
+  }
 
-  res = writeFile(context, zFile, mode, argv[1]);
+  res = writeFile(context, zFile, argv[1], mode, mtime);
   if( res==1 && errno==ENOENT ){
     if( makeDirectory(zFile, mode)==SQLITE_OK ){
-      res = writeFile(context, zFile, mode, argv[1]);
+      res = writeFile(context, zFile, argv[1], mode, mtime);
     }
   }
 
-  if( res!=0 ){
+  if( argc>2 && res!=0 ){
     if( S_ISLNK(mode) ){
       ctxErrorMsg(context, "failed to create symlink: %s", zFile);
     }else if( S_ISDIR(mode) ){
@@ -313,6 +383,10 @@ static int fsdirOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
   return SQLITE_OK;
 }
 
+/*
+** Reset a cursor back to the state it was in when first returned
+** by fsdirOpen().
+*/
 static void fsdirResetCursor(fsdir_cursor *pCur){
   int i;
   for(i=0; i<=pCur->iLvl; i++){
@@ -341,6 +415,10 @@ static int fsdirClose(sqlite3_vtab_cursor *cur){
   return SQLITE_OK;
 }
 
+/*
+** Set the error message for the virtual table associated with cursor
+** pCur to the results of vprintf(zFmt, ...).
+*/
 static void fsdirSetErrmsg(fsdir_cursor *pCur, const char *zFmt, ...){
   va_list ap;
   va_start(ap, zFmt);
@@ -586,6 +664,9 @@ static int fsdirBestIndex(
   return SQLITE_OK;
 }
 
+/*
+** Register the "fsdir" virtual table.
+*/
 static int fsdirRegister(sqlite3 *db){
   static sqlite3_module fsdirModule = {
     0,                         /* iVersion */
@@ -611,9 +692,6 @@ static int fsdirRegister(sqlite3 *db){
   };
 
   int rc = sqlite3_create_module(db, "fsdir", &fsdirModule, 0);
-  if( rc==SQLITE_OK ){
-    rc = sqlite3_create_module(db, "fsentry", &fsdirModule, (void*)1);
-  }
   return rc;
 }
 #else         /* SQLITE_OMIT_VIRTUALTABLE */
diff --git a/manifest b/manifest
index dda85f8287..2ee47c023e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improve\serror\sand\susage\smessages\soutput\sby\sthe\sshell\s".ar"\scommand.
-D 2017-12-14T15:40:42.931
+C Have\sthe\swritefile()\sfunction\soptionally\sset\sthe\smodification-time\sof\sthe\nfiles\sit\swrites\sor\screates.\sAnd\smany\ssmall\sfixes\sto\sthe\snew\scode\son\sthis\nbranch.
+D 2017-12-14T19:15:07.381
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
@@ -270,7 +270,7 @@ F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83
 F ext/misc/csv.c 1a009b93650732e22334edc92459c4630b9fa703397cbb3c8ca279921a36ca11
 F ext/misc/dbdump.c 3509fa6b8932d04e932d6b6b827b6a82ca362781b8e8f3c77336f416793e215e
 F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2
-F ext/misc/fileio.c 29b7fc94752fff6245cf4a81455f98cf6778ec1102ca7e67bf693d41a7db4307
+F ext/misc/fileio.c 014152d4133e7b29eab8eb39d0c640659c23a6d23d882b4778f487ae7d1a457b
 F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25
 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c
 F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281984
@@ -475,7 +475,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
-F src/shell.c.in 4bdd2efe722005180365698f2a3de51e22ae1e9bb61c868006bc8f2c5e02eb98
+F src/shell.c.in 074b2129559a0aa712a367317f7e7daf4740925ec2c123b529800628eb10dc73
 F src/sqlite.h.in 364515dd186285f3c01f5cab42e7db7edc47c70e87b6a25de389a2e6b8c413fd
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1215,7 +1215,7 @@ F test/shell4.test 89ad573879a745974ff2df20ff97c5d6ffffbd5d
 F test/shell5.test 23939a4c51f0421330ea61dbd3c74f9c215f5f8d3d1a94846da6ffc777a35458
 F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3
 F test/shell7.test 115132f66d0463417f408562cc2cf534f6bbc6d83a6d50f0072a9eb171bae97f
-F test/shell8.test 0f7dfc5b33bde7143df8e37cbb4ae6ccc7e91f87232dc8e5e02be03117cdebb8
+F test/shell8.test 96f35965fe84d633fb2338696f5cbc1bcf6bdbdd79677244bc617a8452851dc7
 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3
 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5
 F test/shrink.test 1b4330b1fd9e818c04726d45cb28db73087535ce
@@ -1683,7 +1683,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 803156cba8b056a1cb8d1bb186a57454afe72341abe7de1dfe529234c3415cd2
-R 69895ee1194373ab4ba11578d798fe26
+P b9d2d5d97291bf3d1392232e3705cca89dc7b918db2b08067b2b013ea39320e0
+R 975f9981c88e18e3ddfa1fe4bb6e7fae
 U dan
-Z 5ea750b0cf239f9442948a3eda934166
+Z 1e84576de6ef44efce8abf1d868c1119
diff --git a/manifest.uuid b/manifest.uuid
index ce1ae1dcb3..31073ba115 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b9d2d5d97291bf3d1392232e3705cca89dc7b918db2b08067b2b013ea39320e0
\ No newline at end of file
+7b51269caebe1492885fe9b965892f49a3f8bdb1d666b0203d594c30f9e83938
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 2a0f9fde9e..8b16082add 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -4092,23 +4092,25 @@ static void shellPrepare(
   }
 }
 
-static void shellPrepare2(
+static void shellPreparePrintf(
   sqlite3 *db, 
   int *pRc, 
-  const char *zSql, 
-  const char *zTail, 
-  sqlite3_stmt **ppStmt
+  sqlite3_stmt **ppStmt,
+  const char *zFmt, 
+  ...
 ){
-  if( *pRc==SQLITE_OK && zTail ){
-    char *z = sqlite3_mprintf("%s %s", zSql, zTail);
+  *ppStmt = 0;
+  if( *pRc==SQLITE_OK ){
+    va_list ap;
+    char *z;
+    va_start(ap, zFmt);
+    z = sqlite3_vmprintf(zFmt, ap);
     if( z==0 ){
       *pRc = SQLITE_NOMEM;
     }else{
       shellPrepare(db, pRc, z, ppStmt);
       sqlite3_free(z);
     }
-  }else{
-    shellPrepare(db, pRc, zSql, ppStmt);
   }
 }
 
@@ -4429,23 +4431,27 @@ static int arCheckEntries(sqlite3 *db, ArCommand *pAr){
 static void arWhereClause(
   int *pRc, 
   ArCommand *pAr, 
-  char **pzWhere                  /* OUT: New WHERE clause (or NULL) */
+  char **pzWhere                  /* OUT: New WHERE clause */
 ){
   char *zWhere = 0;
   if( *pRc==SQLITE_OK ){
-    int i;
-    const char *zSep = "WHERE ";
-    for(i=0; i<pAr->nArg; i++){
-      const char *z = pAr->azArg[i];
-      zWhere = sqlite3_mprintf(
-          "%z%s name = '%q' OR name BETWEEN '%q/' AND '%q0'", 
-          zWhere, zSep, z, z, z
-      );
-      if( zWhere==0 ){
-        *pRc = SQLITE_NOMEM;
-        break;
+    if( pAr->nArg==0 ){
+      zWhere = sqlite3_mprintf("1");
+    }else{
+      int i;
+      const char *zSep = "";
+      for(i=0; i<pAr->nArg; i++){
+        const char *z = pAr->azArg[i];
+        zWhere = sqlite3_mprintf(
+            "%z%s name = '%q' OR name BETWEEN '%q/' AND '%q0'", 
+            zWhere, zSep, z, z, z
+            );
+        if( zWhere==0 ){
+          *pRc = SQLITE_NOMEM;
+          break;
+        }
+        zSep = " OR ";
       }
-      zSep = " OR ";
     }
   }
   *pzWhere = zWhere;
@@ -4455,7 +4461,7 @@ static void arWhereClause(
 ** Implementation of .ar "lisT" command. 
 */
 static int arListCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
-  const char *zSql = "SELECT name FROM sqlar"; 
+  const char *zSql = "SELECT name FROM sqlar WHERE %s"; 
   char *zWhere = 0;
   sqlite3_stmt *pSql = 0;
   int rc;
@@ -4463,7 +4469,7 @@ static int arListCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
   rc = arCheckEntries(db, pAr);
   arWhereClause(&rc, pAr, &zWhere);
 
-  shellPrepare2(db, &rc, zSql, zWhere, &pSql);
+  shellPreparePrintf(db, &rc, &pSql, zSql, zWhere);
   while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
     raw_printf(p->out, "%s\n", sqlite3_column_text(pSql, 0));
   }
@@ -4482,14 +4488,14 @@ static int arExtractCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
     "ELSE"
     "    data "
     "END, "
-    "mode) FROM sqlar";
-  const char *zSql2 = "SELECT :1 || name, mtime FROM sqlar"; 
+    "mode, mtime) FROM sqlar WHERE (%s) AND (data IS NULL OR :2 = 0)";
 
   struct timespec times[2];
   sqlite3_stmt *pSql = 0;
   int rc = SQLITE_OK;
   char *zDir = 0;
   char *zWhere = 0;
+  int i;
 
   /* If arguments are specified, check that they actually exist within
   ** the archive before proceeding. And formulate a WHERE clause to
@@ -4509,31 +4515,26 @@ static int arExtractCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
   memset(times, 0, sizeof(times));
   times[0].tv_sec = time(0);
 
-  shellPrepare2(db, &rc, zSql1, zWhere, &pSql);
+  shellPreparePrintf(db, &rc, &pSql, zSql1, zWhere);
   if( rc==SQLITE_OK ){
     sqlite3_bind_text(pSql, 1, zDir, -1, SQLITE_STATIC);
-  }
-  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
-    if( pAr->bVerbose ){
-      raw_printf(p->out, "%s\n", sqlite3_column_text(pSql, 0));
-    }
-  }
-  shellFinalize(&rc, pSql);
 
-  shellPrepare2(db, &rc, zSql2, zWhere, &pSql);
-  if( rc==SQLITE_OK ){
-    sqlite3_bind_text(pSql, 1, zDir, -1, SQLITE_STATIC);
-  }
-  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
-    const char *zPath = (const char*)sqlite3_column_text(pSql, 0);
-    times[1].tv_sec = (time_t)sqlite3_column_int64(pSql, 1);
-    if( utimensat(AT_FDCWD, zPath, times, AT_SYMLINK_NOFOLLOW) ){
-      raw_printf(stderr, "failed to set timestamp for %s\n", zPath);
-      rc = SQLITE_ERROR;
-      break;
+    /* Run the SELECT statement twice. The first time, writefile() is called
+    ** for all archive members that should be extracted. The second time,
+    ** only for the directories. This is because the timestamps for
+    ** extracted directories must be reset after they are populated (as
+    ** populating them changes the timestamp).  */
+    for(i=0; i<2; i++){
+      sqlite3_bind_int(pSql, 2, i);
+      while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
+        if( i==0 && pAr->bVerbose ){
+          raw_printf(p->out, "%s\n", sqlite3_column_text(pSql, 0));
+        }
+      }
+      shellReset(&rc, pSql);
     }
+    shellFinalize(&rc, pSql);
   }
-  shellFinalize(&rc, pSql);
 
   sqlite3_free(zDir);
   sqlite3_free(zWhere);
diff --git a/test/shell8.test b/test/shell8.test
index 07065b56bd..14980a84a5 100644
--- a/test/shell8.test
+++ b/test/shell8.test
@@ -139,6 +139,21 @@ foreach {tn tcl} {
     catchcmd ":memory:" $x3
     dir_to_list ar3
   } $expected
+
+  # This is a repeat of test 1.$tn.1, except that there is a 2 second 
+  # pause between creating the archive and extracting its contents.
+  # This is to test that timestamps are set correctly.
+  #
+  # Because it is slow, only do this for $tn==1.
+  if {$tn==1} {
+    do_test 1.$tn.1 {
+      catchcmd test_ar.db $c1
+      file delete -force ar1
+      after 2000
+      catchcmd test_ar.db $x1
+      dir_to_list ar1
+    } $expected
+  }
 }
 
 finish_test

From dd2a43a225a69a53bbdfe17eec319bbb85e674f9 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 14 Dec 2017 19:24:00 +0000
Subject: [PATCH 153/488] Fixes to the appendvfs.c extension.  Add the
 "sqltclsh" application that uses appendvfs.c to find its scripts.

FossilOrigin-Name: ee248b529c2396c5480fb99b0a1dc31032627ec8241eca4a8c0fff257bb4a088
---
 Makefile.in          |  9 +++++-
 ext/misc/appendvfs.c |  3 ++
 main.mk              |  9 +++++-
 manifest             | 18 ++++++------
 manifest.uuid        |  2 +-
 tool/sqltclsh.c.in   | 42 ++++++++++++++++++++++++++++
 tool/sqltclsh.tcl    | 66 ++++++++++++++++++++++++++++++++++++++++++++
 7 files changed, 138 insertions(+), 11 deletions(-)
 create mode 100644 tool/sqltclsh.c.in
 create mode 100644 tool/sqltclsh.tcl

diff --git a/Makefile.in b/Makefile.in
index 4883c1ac6b..5d377174f8 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -556,7 +556,8 @@ TESTPROGS = \
   sqlite3$(TEXE) \
   sqlite3_analyzer$(TEXE) \
   sqldiff$(TEXE) \
-  dbhash$(TEXE)
+  dbhash$(TEXE) \
+  sqltclsh$(TEXE)
 
 # Databases containing fuzzer test cases
 #
@@ -1190,6 +1191,12 @@ sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $
 sqlite3_analyzer$(TEXE): sqlite3_analyzer.c
 	$(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS)
 
+sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in	
+	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in >sqltclsh.c
+
+sqltclsh$(TEXE): sqltclsh.c
+	$(LTLINK) sqltclsh.c -o $@ $(LIBTCL) $(TLIBS)
+
 CHECKER_DEPS =\
   $(TOP)/tool/mkccode.tcl \
   sqlite3.c \
diff --git a/ext/misc/appendvfs.c b/ext/misc/appendvfs.c
index bd1f37509e..c6e9e0d58b 100644
--- a/ext/misc/appendvfs.c
+++ b/ext/misc/appendvfs.c
@@ -47,7 +47,9 @@
 ** If the file being opened is not an appended database, then this shim is
 ** a pass-through into the default underlying VFS.
 **/
+#if !defined(SQLITEINT_H)
 #include <sqlite3ext.h>
+#endif
 SQLITE_EXTENSION_INIT1
 #include <string.h>
 #include <assert.h>
@@ -458,6 +460,7 @@ static int apndOpen(
   }
   if( (flags & SQLITE_OPEN_CREATE)==0 ){
     pSubFile->pMethods->xClose(pSubFile);
+    pFile->pMethods = 0;
     return SQLITE_CANTOPEN;
   }
   p->iPgOne = (sz+0xfff) & ~(sqlite3_int64)0xfff;
diff --git a/main.mk b/main.mk
index e978b23a58..bc4e404020 100644
--- a/main.mk
+++ b/main.mk
@@ -484,7 +484,8 @@ TESTPROGS = \
   sqlite3_analyzer$(EXE) \
   sqlite3_checker$(EXE) \
   sqldiff$(EXE) \
-  dbhash$(EXE)
+  dbhash$(EXE) \
+  sqltclsh$(EXE)
 
 # Databases containing fuzzer test cases
 #
@@ -809,6 +810,12 @@ sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $
 sqlite3_analyzer$(EXE): sqlite3_analyzer.c
 	$(TCCX) $(TCL_FLAGS) sqlite3_analyzer.c -o $@ $(LIBTCL) $(THREADLIB) 
 
+sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl
+	tclsh $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in >sqltclsh.c
+
+sqltclsh$(EXE): sqltclsh.c
+	$(TCCX) $(TCL_FLAGS) sqltclsh.c -o $@ $(LIBTCL) $(THREADLIB) 
+
 CHECKER_DEPS =\
   $(TOP)/tool/mkccode.tcl \
   sqlite3.c \
diff --git a/manifest b/manifest
index c5da940c02..743670d293 100644
--- a/manifest
+++ b/manifest
@@ -1,6 +1,6 @@
-C The\s"apndvfs"\sVFS\sshim\snow\sappears\sto\sbe\sworking\sfor\sboth\sreads\sand\swrites.
-D 2017-12-14T16:57:11.644
-F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
+C Fixes\sto\sthe\sappendvfs.c\sextension.\s\sAdd\sthe\s"sqltclsh"\sapplication\sthat\nuses\sappendvfs.c\sto\sfind\sits\sscripts.
+D 2017-12-14T19:24:00.444
+F Makefile.in 053284f237e955fba2f386fd9f87020199e6dbcbce9f48bc02cf431458afbe07
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a2492b29176edc3c754aa7a2f7daa20cd3fa20a56e3ee64e376092836177c42a
 F README.md eeae1e552f93ef72ef7c5b8f6647b368a001c28820ad1df179d3dae602bef681
@@ -260,7 +260,7 @@ F ext/lsm1/tool/mklsm1c.tcl f31561bbee5349f0a554d1ad7236ac1991fc09176626f529f607
 F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601d90b2
 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87
 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb
-F ext/misc/appendvfs.c ee71fbc918ab4c526a5935a3153c1cc4e0b601c081da8f9a549e0e9d01bdbca5
+F ext/misc/appendvfs.c 8cc3ae6633e7d97ca9c3c9a48c647e9ba23f8d24f3d5f5f596272a81dc50c398
 F ext/misc/btreeinfo.c d7fd9a2fe2fa33ba28488e2fce703ebecc759219ea9e0bb3b254784866c0a676
 F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005
 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704
@@ -396,7 +396,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 6123b0b2db806ddb482c24786ad6603a289df720382a3bce8f532d76a94c84b1
+F main.mk d0278b8833203dc60ed81eed12412d90559fcd3470e0fd5d509d41ef74a3d64b
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -1650,6 +1650,8 @@ F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd
 F tool/split-sqlite3c.tcl 3efcd4240b738f6bb2b5af0aea7e1e0ef9bc1c61654f645076cec883030b710c
 F tool/sqldiff.c 30879bbc8de686df4624e86adce2d8981f500904c1cfb55b5d1eea2ffd9341eb
 F tool/sqlite3_analyzer.c.in 7eeaae8b0d7577662acaabbb11107af0659d1b41bc1dfdd4d91422de27127968
+F tool/sqltclsh.c.in e1f48150f755bfbe0194478cba50aa9f2f5183bb1efbdd6456532cce3cd2e18d
+F tool/sqltclsh.tcl 71bdb62c8f6b37c05b2d377e870c3dd13ee96cc4fee5b5ae5215ae9a0aca088d
 F tool/srcck1.c 371de5363b70154012955544f86fdee8f6e5326f
 F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43
 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d
@@ -1681,7 +1683,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e343c63cbd754f37c33c939cd0b6f1ecc6202e60c6e66cd65c23cc8d571a994e
-R 427fca803286f3061d4cdb1346b5bd46
+P 7f7b72d83633922e2b5dbf0d3455d0fea72cc6b8925ffcb78dfbad7b3c4b26e9
+R 4666f6f48678fbbcc815c221172efa48
 U drh
-Z 919006b84ebd33ef7833311db2cf1160
+Z 47302a3d5371e2c5e325f342d15c1218
diff --git a/manifest.uuid b/manifest.uuid
index 566e5b932a..dcab13a743 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7f7b72d83633922e2b5dbf0d3455d0fea72cc6b8925ffcb78dfbad7b3c4b26e9
\ No newline at end of file
+ee248b529c2396c5480fb99b0a1dc31032627ec8241eca4a8c0fff257bb4a088
\ No newline at end of file
diff --git a/tool/sqltclsh.c.in b/tool/sqltclsh.c.in
new file mode 100644
index 0000000000..bdfac7dd00
--- /dev/null
+++ b/tool/sqltclsh.c.in
@@ -0,0 +1,42 @@
+/*
+** This is the source code to a "tclsh" that has SQLite built-in.
+**
+** The startup script is located as follows:
+**
+**   (1)  Open the executable as an appended SQLite database and try to
+**        read the startup script out of that database.
+**
+**   (2)  If the first argument is a readable file, try to open that file
+**        as an SQLite database and read the startup script out of that
+**        database.
+**
+**   (3)  If the first argument is a readable file with a ".tcl" extension,
+**        then try to run that script directly.
+**
+** If none of the above steps work, then the program runs as an interactive
+** tclsh.
+*/
+#define TCLSH_INIT_PROC sqlite3_tclapp_init_proc
+#define SQLITE_ENABLE_DBSTAT_VTAB 1
+#undef SQLITE_THREADSAFE
+#define SQLITE_THREADSAFE 0
+#undef SQLITE_ENABLE_COLUMN_METADATA
+#define SQLITE_OMIT_DECLTYPE 1
+#define SQLITE_OMIT_DEPRECATED 1
+#define SQLITE_OMIT_PROGRESS_CALLBACK 1
+#define SQLITE_OMIT_SHARED_CACHE 1
+#define SQLITE_DEFAULT_MEMSTATUS 0
+#define SQLITE_MAX_EXPR_DEPTH 0
+INCLUDE sqlite3.c
+INCLUDE $ROOT/ext/misc/appendvfs.c
+INCLUDE $ROOT/src/tclsqlite.c
+
+const char *sqlite3_tclapp_init_proc(Tcl_Interp *interp){
+  (void)interp;
+  sqlite3_appendvfs_init(0,0,0);
+  return
+BEGIN_STRING
+INCLUDE $ROOT/tool/sqltclsh.tcl
+END_STRING
+;
+}
diff --git a/tool/sqltclsh.tcl b/tool/sqltclsh.tcl
new file mode 100644
index 0000000000..999f166ccb
--- /dev/null
+++ b/tool/sqltclsh.tcl
@@ -0,0 +1,66 @@
+# Try to open the executable as a database and read the "scripts.data"
+# field where "scripts.name" is 'main.tcl'
+#
+catch {
+  sqlite3 db $argv0 -vfs apndvfs -create 0
+  set mainscript [db one {SELECT data FROM scripts WHERE name='main.tcl'}]
+}
+if {[info exists mainscript]} {
+  eval $mainscript
+  return
+} else {
+  catch {db close}
+}
+
+# Try to open file named in the first argument as a database and
+# read the "scripts.data" field where "scripts.name" is 'main.tcl'
+#
+if {[llength $argv]>0 && [file readable [lindex $argv 0]]} {
+  catch {
+    sqlite3 db [lindex $argv 0] -vfs apndvfs -create 0
+    set mainscript [db one {SELECT data FROM scripts WHERE name='main.tcl'}]
+    set argv0 [lindex $argv 0]
+    set argv [lrange $argv 1 end]
+  }
+  if {[info exists mainscript]} {
+    eval $mainscript
+    return
+  } else {
+    catch {db close}
+  }
+  if {[string match *.tcl [lindex $argv 0]]} {
+    set fd [open [lindex $argv 0] rb]
+    set mainscript [read $fd]
+    close $fd
+    unset fd
+    set argv0 [lindex $argv 0]
+    set argv [lrange $argv 1 end]
+  }
+  if {[info exists mainscript]} {
+    eval $mainscript
+    return
+  }
+}
+
+# If all else fails, do an interactive loop
+#
+set line {}
+while {![eof stdin]} {
+  if {$line!=""} {
+    puts -nonewline "> "
+  } else {
+    puts -nonewline "% "
+  }
+  flush stdout
+  append line [gets stdin]
+  if {[info complete $line]} {
+    if {[catch {uplevel #0 $line} result]} {
+      puts stderr "Error: $result"
+    } elseif {$result!=""} {
+      puts $result
+    }
+    set line {}
+  } else {
+    append line \\n"
+  }
+}

From 087316ceecb47f18f0f0eb750e7fc3273f50dcca Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 15 Dec 2017 12:22:21 +0000
Subject: [PATCH 154/488] In the LEMON parser generator, provide reduce actions
 with access to the lookahead token.

FossilOrigin-Name: 42af190f4f86ad60de02800054010fafd484ac86ca41e2a13799b2e583eea98c
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 tool/lempar.c | 12 ++++++++++--
 3 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index 1c8585e662..40cc2db652 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\svalueFromExpr()\sonly\sgenerate\sa\sOOM\sfault\sif\sthere\shave\sbeen\snow\sprior\nfaults.
-D 2017-12-13T23:47:55.478
+C In\sthe\sLEMON\sparser\sgenerator,\sprovide\sreduce\sactions\swith\saccess\sto\sthe\nlookahead\stoken.
+D 2017-12-15T12:22:21.137
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a2492b29176edc3c754aa7a2f7daa20cd3fa20a56e3ee64e376092836177c42a
@@ -1602,7 +1602,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
 F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f
 F tool/lemon.c e6056373044d55296d21f81467dba7632bbb81dc49af072b3f0e76338771497e
-F tool/lempar.c 105d0d9cbe5a25d24d4769241ffbfc63ac7c09e6ccee0dc43dcc8a4c4ae4e426
+F tool/lempar.c 967ebf585cd09b11b89d255d213865109a9c4ff075680d22580a2826de288c89
 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9
 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca
@@ -1680,7 +1680,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6de21deac469ab25378656f6f58115a92f5892428c6f2f3545c9bafac37e4a41
-R 44abcc3575bcbddb8f261c0eaf812ba5
+P 3765aaf712998af5ffb6bc680a0c1419f2b5deb47ecbc1835ba5879127c4dbe3
+R 545f6d4adf455d1fa5987c428909cf33
 U drh
-Z 3fcdf29f818607817f06d87e8d680060
+Z 7981b7f16ad8b88f8d8fe8aa12f607a9
diff --git a/manifest.uuid b/manifest.uuid
index f81e89e40c..999df6a24e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-3765aaf712998af5ffb6bc680a0c1419f2b5deb47ecbc1835ba5879127c4dbe3
\ No newline at end of file
+42af190f4f86ad60de02800054010fafd484ac86ca41e2a13799b2e583eea98c
\ No newline at end of file
diff --git a/tool/lempar.c b/tool/lempar.c
index 37a5892195..da81ddd4bc 100644
--- a/tool/lempar.c
+++ b/tool/lempar.c
@@ -651,10 +651,18 @@ static void yy_accept(yyParser*);  /* Forward Declaration */
 /*
 ** Perform a reduce action and the shift that must immediately
 ** follow the reduce.
+**
+** The yyLookahead and yyLookaheadToken parameters provide reduce actions
+** access to the lookahead token (if any).  The yyLookahead will be YYNOCODE
+** if the lookahead token has already been consumed.  As this procedure is
+** only called from one place, optimizing compilers will in-line it, which
+** means that the extra parameters have no performance impact.
 */
 static void yy_reduce(
   yyParser *yypParser,         /* The parser */
-  unsigned int yyruleno        /* Number of the rule by which to reduce */
+  unsigned int yyruleno,       /* Number of the rule by which to reduce */
+  int yyLookahead,             /* Lookahead token, or YYNOCODE if none */
+  ParseTOKENTYPE yyLookaheadToken  /* Value of the lookahead token */
 ){
   int yygoto;                     /* The next state */
   int yyact;                      /* The next action */
@@ -853,7 +861,7 @@ void Parse(
 #endif
       yymajor = YYNOCODE;
     }else if( yyact <= YY_MAX_REDUCE ){
-      yy_reduce(yypParser,yyact-YY_MIN_REDUCE);
+      yy_reduce(yypParser,yyact-YY_MIN_REDUCE,yymajor,yyminor);
     }else{
       assert( yyact == YY_ERROR_ACTION );
       yyminorunion.yy0 = yyminor;

From 2c44e370f198037202dc1c44d8d3efe7aabceb70 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 15 Dec 2017 20:21:17 +0000
Subject: [PATCH 155/488] Enhance the "swarmvtab" extension. See header
 comments in ext/misc/unionvtab.c for details.

FossilOrigin-Name: 01c173651ab22b7b0c139eded6f2ad8504efd09088df8ae6a3471230ebf2306f
---
 ext/misc/unionvtab.c | 305 +++++++++++++++++++++++++++++++++++++++----
 manifest             |  19 +--
 manifest.uuid        |   2 +-
 test/swarmvtab.test  |   2 +-
 test/swarmvtab2.test |   2 +-
 test/swarmvtab3.test | 233 +++++++++++++++++++++++++++++++++
 6 files changed, 522 insertions(+), 41 deletions(-)
 create mode 100644 test/swarmvtab3.test

diff --git a/ext/misc/unionvtab.c b/ext/misc/unionvtab.c
index fc87915b38..14cce84510 100644
--- a/ext/misc/unionvtab.c
+++ b/ext/misc/unionvtab.c
@@ -56,6 +56,8 @@
 **
 ** SWARMVTAB
 **
+**  LEGACY SYNTAX:
+**
 **   A "swarmvtab" virtual table is created similarly to a unionvtab table:
 **
 **     CREATE VIRTUAL TABLE <name>
@@ -66,13 +68,78 @@
 **   the database file containing the source table.  The <callback> option
 **   is optional.  If included, it is the name of an application-defined
 **   SQL function that is invoked with the URI of the file, if the file
-**   does not already exist on disk.
+**   does not already exist on disk when required by swarmvtab.
+**
+**  NEW SYNTAX:
+**
+**   Using the new syntax, a swarmvtab table is created with:
+**
+**      CREATE VIRTUAL TABLE <name> USING swarmvtab(
+**        <sql-statement> [, <options>]
+**      );
+**
+**   where valid <options> are:
+**
+**      missing=<udf-function-name>
+**      openclose=<udf-function-name>
+**      maxopen=<integer>
+**      <sql-parameter>=<text-value>
+**
+**   The <sql-statement> must return the same 4 columns as for a swarmvtab
+**   table in legacy mode. However, it may also return a 5th column - the
+**   "context" column. The text value returned in this column is not used
+**   at all by the swarmvtab implementation, except that it is passed as
+**   an additional argument to the two UDF functions that may be invoked
+**   (see below).
+**
+**   The "missing" option, if present, specifies the name of an SQL UDF
+**   function to be invoked if a database file is not already present on
+**   disk when required by swarmvtab. If the <sql-statement> did not provide
+**   a context column, it is invoked as:
+**
+**     SELECT <missing-udf>(<database filename/uri>);
+**
+**   Or, if there was a context column:
+**
+**     SELECT <missing-udf>(<database filename/uri>, <context>);
+**
+**   The "openclose" option may also specify a UDF function. This function
+**   is invoked right before swarmvtab opens a database, and right after
+**   it closes one. The first argument - or first two arguments, if
+**   <sql-statement> supplied the context column - is the same as for
+**   the "missing" UDF. Following this, the UDF is passed integer value
+**   0 before a db is opened, and 1 right after it is closed. If both
+**   a missing and openclose UDF is supplied, the application should expect
+**   the following sequence of calls (for a single database):
+**
+**      SELECT <openclose-udf>(<db filename>, <context>, 0);
+**      if( db not already on disk ){
+**          SELECT <missing-udf>(<db filename>, <context>);
+**      }
+**      ... swarmvtab uses database ...
+**      SELECT <openclose-udf>(<db filename>, <context>, 1);
+**
+**   The "maxopen" option is used to configure the maximum number of
+**   database files swarmvtab will hold open simultaneously (default 9).
+**
+**   If an option name begins with a ":" character, then it is assumed
+**   to be an SQL parameter. In this case, the specified text value is
+**   bound to the same variable of the <sql-statement> before it is 
+**   executed. It is an error of the named SQL parameter does not exist.
+**   For example:
+**
+**     CREATE VIRTUAL TABLE swarm USING swarmvtab(
+**       'SELECT :path || localfile, tbl, min, max FROM swarmdir',
+**       :path='/home/user/databases/'
+**       missing='missing_func'
+**     );
 */
 
 #include "sqlite3ext.h"
 SQLITE_EXTENSION_INIT1
 #include <assert.h>
 #include <string.h>
+#include <stdlib.h>
 
 #ifndef SQLITE_OMIT_VIRTUALTABLE
 
@@ -128,6 +195,7 @@ struct UnionSrc {
 
   /* Fields used by swarmvtab only */
   char *zFile;                    /* Database file containing table zTab */
+  char *zContext;                 /* Context string, if any */
   int nUser;                      /* Current number of users */
   sqlite3 *db;                    /* Database handle */
   UnionSrc *pNextClosable;        /* Next in list of closable sources */
@@ -145,8 +213,11 @@ struct UnionTab {
   UnionSrc *aSrc;                 /* Array of source tables, sorted by rowid */
 
   /* Used by swarmvtab only */
+  int bHasContext;                /* Has context strings */
   char *zSourceStr;               /* Expected unionSourceToStr() value */
-  char *zNotFoundCallback;        /* UDF to invoke if file not found on open */
+  sqlite3_stmt *pNotFound;        /* UDF to invoke if file not found on open */
+  sqlite3_stmt *pOpenClose;       /* UDF to invoke on open and close */
+
   UnionSrc *pClosable;            /* First in list of closable sources */
   int nOpen;                      /* Current number of open sources */
   int nMaxOpen;                   /* Maximum number of open sources */
@@ -351,6 +422,39 @@ static void unionFinalize(int *pRc, sqlite3_stmt *pStmt, char **pzErr){
   }
 }
 
+/*
+** If an "openclose" UDF was supplied when this virtual table was created,
+** invoke it now. The first argument passed is the name of the database
+** file for source pSrc. The second is integer value bClose.
+**
+** If successful, return SQLITE_OK. Otherwise an SQLite error code. In this
+** case if argument pzErr is not NULL, also set (*pzErr) to an English
+** language error message. The caller is responsible for eventually freeing 
+** any error message using sqlite3_free().
+*/
+static int unionInvokeOpenClose(
+  UnionTab *pTab, 
+  UnionSrc *pSrc, 
+  int bClose,
+  char **pzErr
+){
+  int rc = SQLITE_OK;
+  if( pTab->pOpenClose ){
+    sqlite3_bind_text(pTab->pOpenClose, 1, pSrc->zFile, -1, SQLITE_STATIC);
+    if( pTab->bHasContext ){
+      sqlite3_bind_text(pTab->pOpenClose, 2, pSrc->zContext, -1, SQLITE_STATIC);
+    }
+    sqlite3_bind_int(pTab->pOpenClose, 2+pTab->bHasContext, bClose);
+    sqlite3_step(pTab->pOpenClose);
+    if( SQLITE_OK!=(rc = sqlite3_reset(pTab->pOpenClose)) ){
+      if( pzErr ){
+        *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db));
+      }
+    }
+  }
+  return rc;
+}
+
 /*
 ** This function is a no-op for unionvtab. For swarmvtab, it attempts to
 ** close open database files until at most nMax are open. An SQLite error
@@ -358,13 +462,16 @@ static void unionFinalize(int *pRc, sqlite3_stmt *pStmt, char **pzErr){
 */
 static void unionCloseSources(UnionTab *pTab, int nMax){
   while( pTab->pClosable && pTab->nOpen>nMax ){
+    UnionSrc *p;
     UnionSrc **pp;
     for(pp=&pTab->pClosable; (*pp)->pNextClosable; pp=&(*pp)->pNextClosable);
-    assert( (*pp)->db );
-    sqlite3_close((*pp)->db);
-    (*pp)->db = 0;
+    p = *pp;
+    assert( p->db );
+    sqlite3_close(p->db);
+    p->db = 0;
     *pp = 0;
     pTab->nOpen--;
+    unionInvokeOpenClose(pTab, p, 1, 0);
   }
 }
 
@@ -377,13 +484,18 @@ static int unionDisconnect(sqlite3_vtab *pVtab){
     int i;
     for(i=0; i<pTab->nSrc; i++){
       UnionSrc *pSrc = &pTab->aSrc[i];
+      if( pSrc->db ){
+        unionInvokeOpenClose(pTab, pSrc, 1, 0);
+      }
       sqlite3_free(pSrc->zDb);
       sqlite3_free(pSrc->zTab);
       sqlite3_free(pSrc->zFile);
+      sqlite3_free(pSrc->zContext);
       sqlite3_close(pSrc->db);
     }
+    sqlite3_finalize(pTab->pNotFound);
+    sqlite3_finalize(pTab->pOpenClose);
     sqlite3_free(pTab->zSourceStr);
-    sqlite3_free(pTab->zNotFoundCallback);
     sqlite3_free(pTab->aSrc);
     sqlite3_free(pTab);
   }
@@ -496,29 +608,31 @@ static int unionSourceCheck(UnionTab *pTab, char **pzErr){
   return rc;
 }
 
-
 /*
 ** Try to open the swarmvtab database.  If initially unable, invoke the
 ** not-found callback UDF and then try again.
 */
 static int unionOpenDatabaseInner(UnionTab *pTab, UnionSrc *pSrc, char **pzErr){
-  int rc = SQLITE_OK;
-  static const int openFlags = 
-       SQLITE_OPEN_READONLY | SQLITE_OPEN_URI;
+  static const int openFlags = SQLITE_OPEN_READONLY | SQLITE_OPEN_URI;
+  int rc;
+
+  rc = unionInvokeOpenClose(pTab, pSrc, 0, pzErr);
+  if( rc!=SQLITE_OK ) return rc;
+
   rc = sqlite3_open_v2(pSrc->zFile, &pSrc->db, openFlags, 0);
   if( rc==SQLITE_OK ) return rc;
-  if( pTab->zNotFoundCallback ){
-    char *zSql = sqlite3_mprintf("SELECT \"%w\"(%Q);",
-                    pTab->zNotFoundCallback, pSrc->zFile);
+  if( pTab->pNotFound ){
     sqlite3_close(pSrc->db);
     pSrc->db = 0;
-    if( zSql==0 ){
-      *pzErr = sqlite3_mprintf("out of memory");
-      return SQLITE_NOMEM;
+    sqlite3_bind_text(pTab->pNotFound, 1, pSrc->zFile, -1, SQLITE_STATIC);
+    if( pTab->bHasContext ){
+      sqlite3_bind_text(pTab->pNotFound, 2, pSrc->zContext, -1, SQLITE_STATIC);
+    }
+    sqlite3_step(pTab->pNotFound);
+    if( SQLITE_OK!=(rc = sqlite3_reset(pTab->pNotFound)) ){
+      *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db));
+      return rc;
     }
-    rc = sqlite3_exec(pTab->db, zSql, 0, 0, pzErr);
-    sqlite3_free(zSql);
-    if( rc ) return rc;
     rc = sqlite3_open_v2(pSrc->zFile, &pSrc->db, openFlags, 0);
   }
   if( rc!=SQLITE_OK ){
@@ -572,6 +686,7 @@ static int unionOpenDatabase(UnionTab *pTab, int iSrc, char **pzErr){
     }else{
       sqlite3_close(pSrc->db);
       pSrc->db = 0;
+      unionInvokeOpenClose(pTab, pSrc, 1, 0);
     }
   }
 
@@ -627,6 +742,132 @@ static int unionFinalizeCsrStmt(UnionCsr *pCsr){
   return rc;
 }
 
+/* 
+** Return true if the argument is a space, tab, CR or LF character.
+*/
+static int union_isspace(char c){
+  return (c==' ' || c=='\n' || c=='\r' || c=='\t');
+}
+
+/* 
+** Return true if the argument is an alphanumeric character in the 
+** ASCII range.
+*/
+static int union_isidchar(char c){
+  return ((c>='a' && c<='z') || (c>='A' && c<'Z') || (c>='0' && c<='9'));
+}
+
+/*
+** This function is called to handle all arguments following the first 
+** (the SQL statement) passed to a swarmvtab (not unionvtab) CREATE 
+** VIRTUAL TABLE statement. It may bind parameters to the SQL statement 
+** or configure members of the UnionTab object passed as the second
+** argument.
+**
+** Refer to header comments at the top of this file for a description
+** of the arguments parsed.
+**
+** This function is a no-op if *pRc is other than SQLITE_OK when it is
+** called. Otherwise, if an error occurs, *pRc is set to an SQLite error
+** code. In this case *pzErr may be set to point to a buffer containing
+** an English language error message. It is the responsibility of the 
+** caller to eventually free the buffer using sqlite3_free().
+*/
+static void unionConfigureVtab(
+  int *pRc,                       /* IN/OUT: Error code */
+  UnionTab *pTab,                 /* Table to configure */
+  sqlite3_stmt *pStmt,            /* SQL statement to find sources */
+  int nArg,                       /* Number of entries in azArg[] array */
+  const char * const *azArg,      /* Array of arguments to consider */
+  char **pzErr                    /* OUT: Error message */
+){
+  int rc = *pRc;
+  int i;
+  if( rc==SQLITE_OK ){
+    pTab->bHasContext = (sqlite3_column_count(pStmt)>4);
+  }
+  for(i=0; rc==SQLITE_OK && i<nArg; i++){
+    char *zArg = unionStrdup(&rc, azArg[i]);
+    if( zArg ){
+      int nOpt = 0;               /* Size of option name in bytes */
+      char *zOpt;                 /* Pointer to option name */
+      char *zVal;                 /* Pointer to value */
+
+      unionDequote(zArg);
+      zOpt = zArg;
+      while( union_isspace(*zOpt) ) zOpt++;
+      zVal = zOpt;
+      if( *zVal==':' ) zVal++;
+      while( union_isidchar(*zVal) ) zVal++;
+      nOpt = zVal-zOpt;
+
+      while( union_isspace(*zVal) ) zVal++;
+      if( *zVal=='=' ){
+        zOpt[nOpt] = '\0';
+        zVal++;
+        while( union_isspace(*zVal) ) zVal++;
+        zVal = unionStrdup(&rc, zVal);
+        if( zVal ){
+          unionDequote(zVal);
+          if( zOpt[0]==':' ){
+            /* A value to bind to the SQL statement */
+            int iParam = sqlite3_bind_parameter_index(pStmt, zOpt);
+            if( iParam==0 ){
+              *pzErr = sqlite3_mprintf(
+                  "swarmvtab: no such SQL parameter: %s", zOpt
+              );
+              rc = SQLITE_ERROR;
+            }else{
+              rc = sqlite3_bind_text(pStmt, iParam, zVal, -1, SQLITE_TRANSIENT);
+            }
+          }else if( nOpt==7 && 0==sqlite3_strnicmp(zOpt, "maxopen", 7) ){
+            pTab->nMaxOpen = atoi(zVal);
+            if( pTab->nMaxOpen<=0 ){
+              *pzErr = sqlite3_mprintf("swarmvtab: illegal maxopen value");
+              rc = SQLITE_ERROR;
+            }
+          }else if( nOpt==7 && 0==sqlite3_strnicmp(zOpt, "missing", 7) ){
+            if( pTab->pNotFound ){
+              *pzErr = sqlite3_mprintf(
+                  "swarmvtab: duplicate \"missing\" option");
+              rc = SQLITE_ERROR;
+            }else{
+              pTab->pNotFound = unionPreparePrintf(&rc, pzErr, pTab->db,
+                  "SELECT \"%w\"(?%s)", zVal, pTab->bHasContext ? ",?" : ""
+              );
+            }
+          }else if( nOpt==9 && 0==sqlite3_strnicmp(zOpt, "openclose", 9) ){
+            if( pTab->pOpenClose ){
+              *pzErr = sqlite3_mprintf(
+                  "swarmvtab: duplicate \"openclose\" option");
+              rc = SQLITE_ERROR;
+            }else{
+              pTab->pOpenClose = unionPreparePrintf(&rc, pzErr, pTab->db,
+                  "SELECT \"%w\"(?,?%s)", zVal, pTab->bHasContext ? ",?" : ""
+              );
+            }
+          }else{
+            *pzErr = sqlite3_mprintf("swarmvtab: unrecognized option: %s",zOpt);
+            rc = SQLITE_ERROR;
+          }
+          sqlite3_free(zVal);
+        }
+      }else{
+        if( i==0 && nArg==1 ){
+          pTab->pNotFound = unionPreparePrintf(&rc, pzErr, pTab->db,
+              "SELECT \"%w\"(?)", zArg
+          );
+        }else{
+          *pzErr = sqlite3_mprintf( "swarmvtab: parse error: %s", azArg[i]);
+          rc = SQLITE_ERROR;
+        }
+      }
+      sqlite3_free(zArg);
+    }
+  }
+  *pRc = rc;
+}
+
 /* 
 ** xConnect/xCreate method.
 **
@@ -654,7 +895,7 @@ static int unionConnect(
     /* unionvtab tables may only be created in the temp schema */
     *pzErr = sqlite3_mprintf("%s tables must be created in TEMP schema", zVtab);
     rc = SQLITE_ERROR;
-  }else if( argc!=4 && argc!=5 ){
+  }else if( argc<4 || (argc>4 && bSwarm==0) ){
     *pzErr = sqlite3_mprintf("wrong number of arguments for %s", zVtab);
     rc = SQLITE_ERROR;
   }else{
@@ -673,6 +914,17 @@ static int unionConnect(
 
     /* Allocate the UnionTab structure */
     pTab = unionMalloc(&rc, sizeof(UnionTab));
+    if( pTab ){
+      assert( rc==SQLITE_OK );
+      pTab->db = db;
+      pTab->bSwarm = bSwarm;
+      pTab->nMaxOpen = SWARMVTAB_MAX_OPEN;
+    }
+
+    /* Parse other CVT arguments, if any */
+    if( bSwarm ){
+      unionConfigureVtab(&rc, pTab, pStmt, argc-4, &argv[4], pzErr);
+    }
 
     /* Iterate through the rows returned by the SQL statement specified
     ** as an argument to the CREATE VIRTUAL TABLE statement. */
@@ -715,17 +967,15 @@ static int unionConnect(
         }else{
           pSrc->zDb = unionStrdup(&rc, zDb);
         }
+        if( pTab->bHasContext ){
+          const char *zContext = (const char*)sqlite3_column_text(pStmt, 4);
+          pSrc->zContext = unionStrdup(&rc, zContext);
+        }
       }
     }
     unionFinalize(&rc, pStmt, pzErr);
     pStmt = 0;
 
-    /* Capture the not-found callback UDF name */
-    if( rc==SQLITE_OK && argc>=5 ){
-      pTab->zNotFoundCallback = unionStrdup(&rc, argv[4]);
-      unionDequote(pTab->zNotFoundCallback);
-    }
-
     /* It is an error if the SELECT statement returned zero rows. If only
     ** because there is no way to determine the schema of the virtual 
     ** table in this case.  */
@@ -738,9 +988,6 @@ static int unionConnect(
     ** compatible schemas. For swarmvtab, attach the first database and
     ** check that the first table is a rowid table only.  */
     if( rc==SQLITE_OK ){
-      pTab->db = db;
-      pTab->bSwarm = bSwarm;
-      pTab->nMaxOpen = SWARMVTAB_MAX_OPEN;
       if( bSwarm ){
         rc = unionOpenDatabase(pTab, 0, pzErr);
       }else{
diff --git a/manifest b/manifest
index 40cc2db652..657245305a 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\sthe\sLEMON\sparser\sgenerator,\sprovide\sreduce\sactions\swith\saccess\sto\sthe\nlookahead\stoken.
-D 2017-12-15T12:22:21.137
+C Enhance\sthe\s"swarmvtab"\sextension.\sSee\sheader\scomments\sin\sext/misc/unionvtab.c\nfor\sdetails.
+D 2017-12-15T20:21:17.831
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a2492b29176edc3c754aa7a2f7daa20cd3fa20a56e3ee64e376092836177c42a
@@ -287,7 +287,7 @@ F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52
 F ext/misc/spellfix.c 41cf26c6b89fcaa8798ae10ae64d39c1f1d9d6995152e545bd491c13058b8fac
 F ext/misc/stmt.c 6f16443abb3551e3f5813bb13ba19a30e7032830015b0f92fe0c0453045c0a11
 F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512
-F ext/misc/unionvtab.c 1e0ebc5078e1a916db191bcd88f87e94ea7ba4aa563ee30ff706261cb4b39461
+F ext/misc/unionvtab.c de36c2c45583d68f99e45b392311967066b02e2651d05697da783698b245b387
 F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95
 F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
@@ -1260,8 +1260,9 @@ F test/subselect.test 0966aa8e720224dbd6a5e769a3ec2a723e332303
 F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a
 F test/subtype1.test 7fe09496352f97053af1437150751be2d0a0cae8
 F test/superlock.test ec94f0556b6488d97f71c79f9061ae08d9ab8f12
-F test/swarmvtab.test c2279311b44de032f86a8295a9b06818d864856f9428b4c99eee91a0d419cf25
-F test/swarmvtab2.test 9a3a68a1e58d00f4ed6c68d12d52f2df971b9e22a80a41f6f8c1409abba8e5b4
+F test/swarmvtab.test 9a3fd5ab3e9b3c976ad1b3d7646aab725114f2ac26b59395d0778b33bab6cdaf
+F test/swarmvtab2.test c948cb2fdfc5b01d85e8f6d6504854202dc1a0782ab2a0ed61538f27cbd0aa5c
+F test/swarmvtab3.test c4c8d09e56ae99b90187ac225458f13f373873ea296fc442c7ad7511f25e7314
 F test/swarmvtabfault.test 00aec54665909490f5c383f3cae3b5d18bd97c12490b429ff8752a3027acfa42
 F test/symlink.test c9ebe7330d228249e447038276bfc8a7b22f4849
 F test/sync.test 2f84bdbc2b2df1fcb0220575b4b9f8cea94b7529
@@ -1680,7 +1681,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3765aaf712998af5ffb6bc680a0c1419f2b5deb47ecbc1835ba5879127c4dbe3
-R 545f6d4adf455d1fa5987c428909cf33
-U drh
-Z 7981b7f16ad8b88f8d8fe8aa12f607a9
+P 42af190f4f86ad60de02800054010fafd484ac86ca41e2a13799b2e583eea98c
+R 23f432208fb9ea9e9693297333855e09
+U dan
+Z ee61a0a249e7c85137482dc433509d6a
diff --git a/manifest.uuid b/manifest.uuid
index 999df6a24e..72e6a84022 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-42af190f4f86ad60de02800054010fafd484ac86ca41e2a13799b2e583eea98c
\ No newline at end of file
+01c173651ab22b7b0c139eded6f2ad8504efd09088df8ae6a3471230ebf2306f
\ No newline at end of file
diff --git a/test/swarmvtab.test b/test/swarmvtab.test
index 4cdcf29ca8..9d2919bee1 100644
--- a/test/swarmvtab.test
+++ b/test/swarmvtab.test
@@ -213,7 +213,7 @@ do_catchsql_test 3.1 {
         ("test.db2", "t1", 11, 20)
     ', 'fetch_db_no_such_function'
   );
-} {1 {no such function: fetch_db_no_such_function}}
+} {1 {sql error: no such function: fetch_db_no_such_function}}
 
 do_catchsql_test 3.2 {
   CREATE VIRTUAL TABLE temp.xyz USING swarmvtab(
diff --git a/test/swarmvtab2.test b/test/swarmvtab2.test
index cf1bdd0af9..1cc7fbb378 100644
--- a/test/swarmvtab2.test
+++ b/test/swarmvtab2.test
@@ -14,7 +14,7 @@
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
-set testprefix swarmvtab
+set testprefix swarmvtab2
 do_not_use_codec
 
 ifcapable !vtab {
diff --git a/test/swarmvtab3.test b/test/swarmvtab3.test
new file mode 100644
index 0000000000..70d6c7dca0
--- /dev/null
+++ b/test/swarmvtab3.test
@@ -0,0 +1,233 @@
+# 2017-07-15
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is the "swarmvtab" extension
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix swarmvtab3
+do_not_use_codec
+
+ifcapable !vtab {
+  finish_test
+  return
+}
+
+load_static_extension db unionvtab
+
+set nFile $sqlite_open_file_count
+
+do_execsql_test 1.0 {
+  CREATE TEMP TABLE swarm(id, tbl, minval, maxval);
+}
+
+# Set up 100 databases with filenames "remote_test.dbN", where N is between
+# 0 and 99.
+do_test 1.1 {
+  for {set i 0} {$i < 100} {incr i} {
+    set file remote_test.db$i
+    forcedelete $file
+    forcedelete test.db$i
+    sqlite3 rrr $file
+    rrr eval {
+      CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
+      INSERT INTO t1 VALUES($i, $i);
+    }
+    rrr close
+    db eval {
+      INSERT INTO swarm VALUES($i, 't1', $i, $i);
+    }
+    set ::dbcache(test.db$i) 0
+  }
+} {}
+
+proc missing_db {filename} {
+  set remote "remote_$filename"
+  forcedelete $filename
+  file copy $remote $filename
+}
+db func missing_db missing_db
+
+proc openclose_db {filename bClose} {
+  if {$bClose} {
+    incr ::dbcache($filename) -1
+  } else {
+    incr ::dbcache($filename) 1
+  }
+  if {$::dbcache($filename)==0} {
+    forcedelete $filename
+  }
+}
+db func openclose_db openclose_db
+
+proc check_dbcache {} {
+  set n 0
+  for {set i 0} {$i<100} {incr i} {
+    set exists [file exists test.db$i]
+    if {$exists!=($::dbcache(test.db$i)!=0)} {
+      error "inconsistent ::dbcache and disk"
+    }
+    incr n $exists
+  }
+  return $n
+}
+
+foreach {tn nMaxOpen cvt} {
+  1 5 {
+    CREATE VIRTUAL TABLE temp.s USING swarmvtab(
+        'SELECT :prefix || id, tbl, minval, minval FROM swarm',
+        :prefix='test.db',
+        missing=missing_db,
+        openclose=openclose_db,
+        maxopen=5
+    )
+  }
+
+  2 3 {
+    CREATE VIRTUAL TABLE temp.s USING swarmvtab(
+        'SELECT :prefix || id, tbl, minval, minval FROM swarm',
+        :prefix='test.db',
+        missing =       'missing_db',
+        openclose=[openclose_db],
+        maxopen = 3
+    )
+  }
+
+  3 1 {
+    CREATE VIRTUAL TABLE temp.s USING swarmvtab(
+        'SELECT :prefix||''.''||:suffix||id, tbl, minval, minval FROM swarm',
+        :prefix=test, :suffix=db,
+        missing =       'missing_db',
+        openclose=[openclose_db],
+        maxopen = 1
+    )
+  }
+
+} {
+  execsql { DROP TABLE IF EXISTS s }
+  
+  do_execsql_test 1.$tn.1 $cvt
+
+  do_execsql_test 1.$tn.2 {
+    SELECT b FROM s WHERE a<10;
+  } {0 1 2 3 4 5 6 7 8 9}
+
+  do_test 1.$tn.3 { check_dbcache } $nMaxOpen
+
+  do_execsql_test 1.$tn.4 {
+    SELECT b FROM s WHERE (b%10)=0;
+  } {0 10 20 30 40 50 60 70 80 90}
+
+  do_test 1.$tn.5 { check_dbcache } $nMaxOpen
+}
+
+execsql { DROP TABLE IF EXISTS s }
+for {set i 0} {$i < 100} {incr i} {
+  forcedelete remote_test.db$i
+}
+
+#----------------------------------------------------------------------------
+#
+do_execsql_test 2.0 {
+  DROP TABLE IF EXISTS swarm;
+  CREATE TEMP TABLE swarm(file, tbl, minval, maxval, ctx);
+}
+
+catch { array unset ::dbcache }
+
+# Set up 100 databases with filenames "remote_test.dbN", where N is a
+# random integer between 0 and 1,000,000
+# 0 and 99.
+do_test 2.1 {
+  for {set i 0} {$i < 100} {incr i} {
+    while 1 {
+      set ctx [expr abs(int(rand() *1000000))]
+      if {[info exists ::dbcache($ctx)]==0} break
+    }
+
+    set file test_remote.db$ctx
+    forcedelete $file
+    forcedelete test.db$i
+    sqlite3 rrr $file
+    rrr eval {
+      CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
+      INSERT INTO t1 VALUES($i, $i);
+    }
+    rrr close
+    db eval {
+      INSERT INTO swarm VALUES('test.db' || $i, 't1', $i, $i, $file)
+    }
+    set ::dbcache(test.db$i) 0
+  }
+} {}
+
+proc missing_db {filename ctx} {
+  file copy $ctx $filename
+}
+db func missing_db missing_db
+
+proc openclose_db {filename ctx bClose} {
+  if {$bClose} {
+    incr ::dbcache($filename) -1
+  } else {
+    incr ::dbcache($filename) 1
+  }
+  if {$::dbcache($filename)==0} {
+    forcedelete $filename
+  }
+}
+db func openclose_db openclose_db
+
+proc check_dbcache {} {
+  set n 0
+  foreach k [array names ::dbcache] {
+    set exists [file exists $k]
+    if {$exists!=($::dbcache($k)!=0)} {
+      error "inconsistent ::dbcache and disk ($k)"
+    }
+    incr n $exists
+  }
+  return $n
+}
+
+foreach {tn nMaxOpen cvt} {
+  2 5 {
+    CREATE VIRTUAL TABLE temp.s USING swarmvtab(
+        'SELECT file, tbl, minval, minval, ctx FROM swarm',
+        missing=missing_db,
+        openclose=openclose_db,
+        maxopen=5
+    )
+  }
+} {
+  execsql { DROP TABLE IF EXISTS s }
+  
+  do_execsql_test 1.$tn.1 $cvt
+
+  do_execsql_test 1.$tn.2 {
+    SELECT b FROM s WHERE a<10;
+  } {0 1 2 3 4 5 6 7 8 9}
+
+  do_test 1.$tn.3 { check_dbcache } $nMaxOpen
+
+  do_execsql_test 1.$tn.4 {
+    SELECT b FROM s WHERE (b%10)=0;
+  } {0 10 20 30 40 50 60 70 80 90}
+
+  do_test 1.$tn.5 { check_dbcache } $nMaxOpen
+}
+
+db close
+forcedelete {*}[glob test_remote.db*]
+
+finish_test
+

From 468c649331dc631a5821adb24083bf5d96dfb9dc Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 16 Dec 2017 04:37:15 +0000
Subject: [PATCH 156/488] Add unnecessary initializations to some local
 variables in the rtree module to suppress false-positive compiler warnings
 coming out of MSVC.

FossilOrigin-Name: 64487d658cb3b6c8c67f1e198c70813c963de52599f3ea974bdc2aa432e74de9
---
 ext/rtree/rtree.c |  4 ++--
 manifest          | 14 +++++++-------
 manifest.uuid     |  2 +-
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c
index ff15a192a1..ddabacf408 100644
--- a/ext/rtree/rtree.c
+++ b/ext/rtree/rtree.c
@@ -2021,7 +2021,7 @@ static int ChooseLeaf(
 ){
   int rc;
   int ii;
-  RtreeNode *pNode;
+  RtreeNode *pNode = 0;
   rc = nodeAcquire(pRtree, 1, 0, &pNode);
 
   for(ii=0; rc==SQLITE_OK && ii<(pRtree->iDepth-iHeight); ii++){
@@ -2896,7 +2896,7 @@ static int rtreeDeleteRowid(Rtree *pRtree, sqlite3_int64 iDelete){
   */
   if( rc==SQLITE_OK && pRtree->iDepth>0 && NCELL(pRoot)==1 ){
     int rc2;
-    RtreeNode *pChild;
+    RtreeNode *pChild = 0;
     i64 iChild = nodeGetRowid(pRtree, pRoot, 0);
     rc = nodeAcquire(pRtree, iChild, pRoot, &pChild);
     if( rc==SQLITE_OK ){
diff --git a/manifest b/manifest
index 657245305a..c09e354d38 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Enhance\sthe\s"swarmvtab"\sextension.\sSee\sheader\scomments\sin\sext/misc/unionvtab.c\nfor\sdetails.
-D 2017-12-15T20:21:17.831
+C Add\sunnecessary\sinitializations\sto\ssome\slocal\svariables\sin\sthe\srtree\smodule\nto\ssuppress\sfalse-positive\scompiler\swarnings\scoming\sout\sof\sMSVC.
+D 2017-12-16T04:37:15.013
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a2492b29176edc3c754aa7a2f7daa20cd3fa20a56e3ee64e376092836177c42a
@@ -338,7 +338,7 @@ F ext/repair/test/checkfreelist01.test 3e8aa6aeb4007680c94a8d07b41c339aa635cc782
 F ext/repair/test/checkindex01.test 6945d0ffc0c1dc993b2ce88036b26e0f5d6fcc65da70fc9df27c2647bb358b0f
 F ext/repair/test/test.tcl 686d76d888dffd021f64260abf29a55c57b2cedfa7fc69150b42b1d6119aac3c
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
-F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f
+F ext/rtree/rtree.c 2111f685ae07988622c241f819b56fea60782f56e32f97e334473c59f6083481
 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e
 F ext/rtree/rtree1.test 82a353747fcab1083d114b2ac84723dfefdbf86c1a6e1df57bf588c7d4285436
 F ext/rtree/rtree2.test 5f25b01acd03470067a2d52783b2eb0a50bf836803d4342d20ca39e541220fe2
@@ -1681,7 +1681,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 42af190f4f86ad60de02800054010fafd484ac86ca41e2a13799b2e583eea98c
-R 23f432208fb9ea9e9693297333855e09
-U dan
-Z ee61a0a249e7c85137482dc433509d6a
+P 01c173651ab22b7b0c139eded6f2ad8504efd09088df8ae6a3471230ebf2306f
+R e261edbb5cf2d21265b23f8e0074b2f2
+U drh
+Z c0a264eb11df5fc17a94051fa60030ca
diff --git a/manifest.uuid b/manifest.uuid
index 72e6a84022..9c14ed954f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-01c173651ab22b7b0c139eded6f2ad8504efd09088df8ae6a3471230ebf2306f
\ No newline at end of file
+64487d658cb3b6c8c67f1e198c70813c963de52599f3ea974bdc2aa432e74de9
\ No newline at end of file

From d1b51d49551d03ac2d47f3164df4096e4b8c6efb Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 16 Dec 2017 19:11:26 +0000
Subject: [PATCH 157/488] Do not use the compress() and uncompress() functions
 in ext/misc/compress.c - they are not quite compatible with the spec. Instead
 use new functions in ext/misc/sqlar.c.

FossilOrigin-Name: 7652b3c2374084047b6c1da3e525e0cac34fe220597f81e793bc4fd9f33358da
---
 ext/misc/sqlar.c | 113 +++++++++++++++++++++++++++++++++++++++++++++++
 main.mk          |   3 +-
 manifest         |  15 ++++---
 manifest.uuid    |   2 +-
 src/shell.c.in   |  58 +++++++-----------------
 5 files changed, 139 insertions(+), 52 deletions(-)
 create mode 100644 ext/misc/sqlar.c

diff --git a/ext/misc/sqlar.c b/ext/misc/sqlar.c
new file mode 100644
index 0000000000..17d0875938
--- /dev/null
+++ b/ext/misc/sqlar.c
@@ -0,0 +1,113 @@
+/*
+** 2017-12-17
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** Utility functions sqlar_compress() and sqlar_uncompress(). Useful
+** for working with sqlar archives and used by the shell tool's built-in
+** sqlar support.
+*/
+#include "sqlite3ext.h"
+SQLITE_EXTENSION_INIT1
+#include <zlib.h>
+
+/*
+** Implementation of the "sqlar_compress(X)" SQL function.
+**
+** If the type of X is SQLITE_BLOB, and compressing that blob using
+** zlib utility function compress() yields a smaller blob, return the
+** compressed blob. Otherwise, return a copy of X.
+*/
+static void sqlarCompressFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  assert( argc==1 );
+  if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){
+    const Bytef *pData = sqlite3_value_blob(argv[0]);
+    uLong nData = sqlite3_value_bytes(argv[0]);
+    uLongf nOut = compressBound(nData);
+    Bytef *pOut;
+
+    pOut = (Bytef*)sqlite3_malloc(nOut);
+    if( pOut==0 ){
+      sqlite3_result_error_nomem(context);
+      return;
+    }else{
+      if( Z_OK!=compress(pOut, &nOut, pData, nData) ){
+        sqlite3_result_error(context, "error in compress()", -1);
+      }else if( nOut<nData ){
+        sqlite3_result_blob(context, pOut, nOut, SQLITE_TRANSIENT);
+      }else{
+        sqlite3_result_value(context, argv[0]);
+      }
+      sqlite3_free(pOut);
+    }
+  }else{
+    sqlite3_result_value(context, argv[0]);
+  }
+}
+
+/*
+** Implementation of the "sqlar_uncompress(X,SZ)" SQL function
+**
+** Parameter SZ is interpreted as an integer. If it is less than or
+** equal to zero, then this function returns a copy of X. Or, if
+** SZ is equal to the size of X when interpreted as a blob, also
+** return a copy of X. Otherwise, decompress blob X using zlib
+** utility function uncompress() and return the results (another
+** blob).
+*/
+static void sqlarUncompressFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  uLong nData;
+  uLongf sz;
+
+  assert( argc==2 );
+  sz = sqlite3_value_int(argv[1]);
+
+  if( sz<=0 || sz==(nData = sqlite3_value_bytes(argv[0])) ){
+    sqlite3_result_value(context, argv[0]);
+  }else{
+    const Bytef *pData= sqlite3_value_blob(argv[0]);
+    Bytef *pOut = sqlite3_malloc(sz);
+    if( Z_OK!=uncompress(pOut, &sz, pData, nData) ){
+      sqlite3_result_error(context, "error in uncompress()", -1);
+    }else{
+      sqlite3_result_blob(context, pOut, sz, SQLITE_TRANSIENT);
+    }
+    sqlite3_free(pOut);
+  }
+}
+
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_sqlar_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  int rc = SQLITE_OK;
+  SQLITE_EXTENSION_INIT2(pApi);
+  (void)pzErrMsg;  /* Unused parameter */
+  rc = sqlite3_create_function(db, "sqlar_compress", 1, SQLITE_UTF8, 0,
+                               sqlarCompressFunc, 0, 0);
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_create_function(db, "sqlar_uncompress", 2, SQLITE_UTF8, 0,
+                                 sqlarUncompressFunc, 0, 0);
+  }
+  return rc;
+}
diff --git a/main.mk b/main.mk
index e978b23a58..07cab6db1d 100644
--- a/main.mk
+++ b/main.mk
@@ -690,7 +690,8 @@ SHELL_SRC = \
 	$(TOP)/src/shell.c.in \
 	$(TOP)/ext/misc/shathree.c \
 	$(TOP)/ext/misc/fileio.c \
-	$(TOP)/ext/misc/completion.c
+	$(TOP)/ext/misc/completion.c \
+	$(TOP)/ext/misc/sqlar.c
 
 shell.c:	$(SHELL_SRC) $(TOP)/tool/mkshellc.tcl
 	tclsh $(TOP)/tool/mkshellc.tcl >shell.c
diff --git a/manifest b/manifest
index 2ee47c023e..bbdf7f6ae0 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Have\sthe\swritefile()\sfunction\soptionally\sset\sthe\smodification-time\sof\sthe\nfiles\sit\swrites\sor\screates.\sAnd\smany\ssmall\sfixes\sto\sthe\snew\scode\son\sthis\nbranch.
-D 2017-12-14T19:15:07.381
+C Do\snot\suse\sthe\scompress()\sand\suncompress()\sfunctions\sin\sext/misc/compress.c\s-\nthey\sare\snot\squite\scompatible\swith\sthe\sspec.\sInstead\suse\snew\sfunctions\sin\next/misc/sqlar.c.
+D 2017-12-16T19:11:26.054
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
@@ -287,6 +287,7 @@ F ext/misc/sha1.c 0b9e9b855354910d3ca467bf39099d570e73db56
 F ext/misc/shathree.c fa185d7aee0ad0aca5e091b4a2db7baff11796170e5793b5de99e511a13af448
 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52
 F ext/misc/spellfix.c 41cf26c6b89fcaa8798ae10ae64d39c1f1d9d6995152e545bd491c13058b8fac
+F ext/misc/sqlar.c d355cd8b6e7280d2f61d4737672922acb512a2ab1cee52399ffb88980476e31c
 F ext/misc/stmt.c 6f16443abb3551e3f5813bb13ba19a30e7032830015b0f92fe0c0453045c0a11
 F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512
 F ext/misc/unionvtab.c 1e0ebc5078e1a916db191bcd88f87e94ea7ba4aa563ee30ff706261cb4b39461
@@ -397,7 +398,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 6123b0b2db806ddb482c24786ad6603a289df720382a3bce8f532d76a94c84b1
+F main.mk f5f8e5e6bdd9ab3dabd94c6cf929813a601d5dbf312197fe463190c7d927c27c
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -475,7 +476,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
-F src/shell.c.in 074b2129559a0aa712a367317f7e7daf4740925ec2c123b529800628eb10dc73
+F src/shell.c.in ad8c13b9dee606d1ef11226f90d8e4e084d75986f96e6852e8e5b1c54aa4f0aa
 F src/sqlite.h.in 364515dd186285f3c01f5cab42e7db7edc47c70e87b6a25de389a2e6b8c413fd
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1683,7 +1684,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b9d2d5d97291bf3d1392232e3705cca89dc7b918db2b08067b2b013ea39320e0
-R 975f9981c88e18e3ddfa1fe4bb6e7fae
+P 7b51269caebe1492885fe9b965892f49a3f8bdb1d666b0203d594c30f9e83938
+R 6fdfedcd54ee170b8eb014864253aebf
 U dan
-Z 1e84576de6ef44efce8abf1d868c1119
+Z 7252a7eb50ba8ff9515237c2fa79b757
diff --git a/manifest.uuid b/manifest.uuid
index 31073ba115..4125d09f9b 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7b51269caebe1492885fe9b965892f49a3f8bdb1d666b0203d594c30f9e83938
\ No newline at end of file
+7652b3c2374084047b6c1da3e525e0cac34fe220597f81e793bc4fd9f33358da
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 8b16082add..adfbaa96d1 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -797,7 +797,7 @@ INCLUDE ../ext/misc/shathree.c
 INCLUDE ../ext/misc/fileio.c
 INCLUDE ../ext/misc/completion.c
 #ifdef SQLITE_HAVE_ZLIB
-INCLUDE ../ext/misc/compress.c
+INCLUDE ../ext/misc/sqlar.c
 #endif
 
 #if defined(SQLITE_ENABLE_SESSION)
@@ -2901,7 +2901,7 @@ static void open_db(ShellState *p, int keepAlive){
     sqlite3_shathree_init(p->db, 0, 0);
     sqlite3_completion_init(p->db, 0, 0);
 #ifdef SQLITE_HAVE_ZLIB
-    sqlite3_compress_init(p->db, 0, 0);
+    sqlite3_sqlar_init(p->db, 0, 0);
 #endif
     sqlite3_create_function(p->db, "shell_add_schema", 2, SQLITE_UTF8, 0,
                             shellAddSchemaName, 0, 0);
@@ -4482,13 +4482,10 @@ static int arListCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
 */
 static int arExtractCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
   const char *zSql1 = 
-    "SELECT :1 || name, writefile(:1 || name, "
-    "CASE WHEN (data AND sz>=0 AND sz!=length(data)) THEN "
-    "    uncompress(data) "
-    "ELSE"
-    "    data "
-    "END, "
-    "mode, mtime) FROM sqlar WHERE (%s) AND (data IS NULL OR :2 = 0)";
+    "SELECT "
+    "  :1 || name, "
+    "  writefile(:1 || name, sqlar_uncompress(data, sz), mode, mtime) "
+    "FROM sqlar WHERE (%s) AND (data IS NULL OR :2 = 0)";
 
   struct timespec times[2];
   sqlite3_stmt *pSql = 0;
@@ -4569,16 +4566,13 @@ static int arCreateUpdate(
       "data BLOB               -- compressed content\n"
   ")";
   const char *zDrop = "DROP TABLE IF EXISTS sqlar";
-  const char *zInsert = "REPLACE INTO sqlar VALUES(?, ?, ?, ?, ?)";
+  const char *zInsert = "REPLACE INTO sqlar VALUES(?,?,?,?,sqlar_compress(?))";
 
   sqlite3_stmt *pStmt = 0;        /* Directory traverser */
   sqlite3_stmt *pInsert = 0;      /* Compilation of zInsert */
   int i;                          /* For iterating through azFile[] */
   int rc;                         /* Return code */
 
-  Bytef *aCompress = 0;           /* Compression buffer */
-  int nCompress = 0;              /* Size of compression buffer */
-
   rc = sqlite3_exec(db, "SAVEPOINT ar;", 0, 0, 0);
   if( rc!=SQLITE_OK ) return rc;
 
@@ -4611,39 +4605,18 @@ static int arCreateUpdate(
       if( S_ISDIR(mode) ){
         sz = 0;
         sqlite3_bind_null(pInsert, 5);
-      }else if( S_ISLNK(mode) ){
-        sz = -1;
-        sqlite3_bind_value(pInsert, 5, sqlite3_column_value(pStmt, 3));
       }else{
-        uLongf nReq;              /* Required size of compression buffer */
-        const Bytef *pData = (const Bytef*)sqlite3_column_blob(pStmt, 3);
-        sz = sqlite3_column_bytes(pStmt, 3);
-        nReq = compressBound(sz);
-        if( aCompress==0 || nReq>nCompress ){
-          Bytef *aNew = sqlite3_realloc(aCompress, nReq);
-          if( aNew==0 ){
-            rc = SQLITE_NOMEM;
-          }else{
-            aCompress = aNew;
-            nCompress = nReq;
-          }
-        }
-
-        if( Z_OK!=compress(aCompress, &nReq, pData, sz) ){
-          rc = SQLITE_ERROR;
-        }
-        if( nReq<sz ){
-          sqlite3_bind_blob(pInsert, 5, aCompress, nReq, SQLITE_STATIC);
+        sqlite3_bind_value(pInsert, 5, sqlite3_column_value(pStmt, 3));
+        if( S_ISLNK(mode) ){
+          sz = -1;
         }else{
-          sqlite3_bind_blob(pInsert, 5, pData, sz, SQLITE_STATIC);
+          sz = sqlite3_column_bytes(pStmt, 3);
         }
       }
 
-      if( rc==SQLITE_OK ){
-        sqlite3_bind_int(pInsert, 4, sz);
-        sqlite3_step(pInsert);
-        rc = sqlite3_reset(pInsert);
-      }
+      sqlite3_bind_int(pInsert, 4, sz);
+      sqlite3_step(pInsert);
+      rc = sqlite3_reset(pInsert);
     }
     shellReset(&rc, pStmt);
   }
@@ -4655,7 +4628,6 @@ static int arCreateUpdate(
   }
   shellFinalize(&rc, pStmt);
   shellFinalize(&rc, pInsert);
-  sqlite3_free(aCompress);
   return rc;
 }
 
@@ -4713,7 +4685,7 @@ static int arDotCommand(
         return rc;
       }
       sqlite3_fileio_init(db, 0, 0);
-      sqlite3_compress_init(db, 0, 0);
+      sqlite3_sqlar_init(db, 0, 0);
     }else{
       db = pState->db;
     }

From 2fc865c1153d739208657ea652f74426bf20f678 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 16 Dec 2017 20:20:37 +0000
Subject: [PATCH 158/488] Add an experimental location(X) SQL function that
 attempt to return the location of the payload within the database for the
 record that contains column X.  location(X) returns NULL if X is not an
 ordinary table column or if SQLite cannot figure out the location because it
 is using a covering index.

FossilOrigin-Name: 51be9558164301c5dd4df23ab8b3e67de0b522f8d36f79f3d84d45d3dc2a83a4
---
 manifest        | 29 ++++++++++++++++-------------
 manifest.uuid   |  2 +-
 src/btree.c     | 13 +++++++++++++
 src/btree.h     |  1 +
 src/expr.c      | 15 ++++++++++++---
 src/func.c      |  2 ++
 src/sqliteInt.h |  1 +
 src/vdbe.c      | 20 ++++++++++++++++++++
 test/func6.test | 36 ++++++++++++++++++++++++++++++++++++
 9 files changed, 102 insertions(+), 17 deletions(-)
 create mode 100644 test/func6.test

diff --git a/manifest b/manifest
index c3b80112dd..8e46d8e601 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\ssqlite3_vtab_collation()\sfunction,\swhich\sallows\san\sxBestIndex\scallback\nto\sdetermine\sthe\scollation\ssequence\sthat\sSQLite\swill\suse\sfor\sa\scomparison.\sAnd\nthe\sSQLITE_DBCONFIG_FULL_EQP\sconfiguration\soption,\swhich\senhances\sthe\soutput\nof\s"EXPLAIN\sQUERY\sPLAN"\sso\sthat\sit\sincludes\sstatements\srun\sby\striggers.\sAnd\nthe\scode\sfor\sthe\ssqlite3_expert\sextension\sand\scommand\sline\sapplication.
-D 2017-12-16T19:36:52.802
+C Add\san\sexperimental\slocation(X)\sSQL\sfunction\sthat\sattempt\sto\sreturn\sthe\nlocation\sof\sthe\spayload\swithin\sthe\sdatabase\sfor\sthe\srecord\sthat\scontains\ncolumn\sX.\s\slocation(X)\sreturns\sNULL\sif\sX\sis\snot\san\sordinary\stable\scolumn\sor\nif\sSQLite\scannot\sfigure\sout\sthe\slocation\sbecause\sit\sis\susing\sa\scovering\sindex.
+D 2017-12-16T20:20:37.221
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -420,8 +420,8 @@ F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
-F src/btree.c b83a6b03f160528020bb965f0c3a40af5286cd4923c3870fd218177f03a120a7
-F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09
+F src/btree.c 6fba32ea06e9be55849dd486e88df6c467bf69cf306d38ca3c3a8df34e6499e9
+F src/btree.h e9d22d0475b37422cc2db53f4336cc814645dcca8634bc4aae25ed52d043ef53
 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc
 F src/build.c 87b68e3b45559ec404b12f095f0ba5f06f91a6dd2d21bd8443e41d8ac2e67196
 F src/callback.c fe677cb5f5abb02f7a772a62a98c2f516426081df68856e8f2d5f950929b966a
@@ -431,10 +431,10 @@ F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957
 F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
 F src/delete.c 74667ad914ac143731a444a1bacf29ceb18f6eded8a0dd17aafae80baa07f8bb
-F src/expr.c fe11b91bb65b869143bd42023427c4429778ae42c0a0db7762f68f75b347a958
+F src/expr.c 261d1eb0954a7813d512d8edafd6db26736a3dcb650785652a6234d5ddb394aa
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331
-F src/func.c 0fb9a2d678d3c8aba89b46468b309cd7e8fa9806a369a30aa89024660845bb13
+F src/func.c a23ea9b8869ca47e20a273985013b36a2294788fde5abe1a610be4bb90c8e6e8
 F src/global.c ac3094f1dc59fbeb919aef7cc0cc827a8459d1fb1adb7972ef75bd9e0c10b75b
 F src/hash.c a12580e143f10301ed5166ea4964ae2853d3905a511d4e0c44497245c7ce1f7a
 F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4
@@ -483,7 +483,7 @@ F src/shell.c.in 6ffed0c589f5aff180789a8c8abf5b2d3e2eea7470c86b30e797887cb0c9d0e
 F src/sqlite.h.in 95afbec6e623843d702e727765efc833586fecc688867f41f87be7db3ff1fa62
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
-F src/sqliteInt.h 4d6580ab911e9bc36447ef05bf4d2aab929dde161ac117c7de6220eef06d6175
+F src/sqliteInt.h 3aca12018eb1528f9bc7af4a9ea141759f2487ff42b9f9f688210d09742dfe1e
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -547,7 +547,7 @@ F src/update.c 961bd1265d4d1e5cd65c9a54fa5122fb7aefcb003fcf2de0c092fceb7e58972c
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c d01fa6f45bfad3b65fb2490513aa2e0676412c61b4b094340b513cf72c3704a4
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
-F src/vdbe.c 3393b508d9ad084ffce232a7c53e375ef5ac99b50b685c5131fcdfce97a9d534
+F src/vdbe.c beed55b540e01e9be3b87b0d3aa491b41d933bd391ca4d8470882f77e1090871
 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97
 F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9
 F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1
@@ -917,6 +917,7 @@ F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f
 F test/func3.test d202a7606d23f90988a664e88e268aed1087c11c
 F test/func4.test 6beacdfcb0e18c358e6c2dcacf1b65d1fa80955f
 F test/func5.test cdd224400bc3e48d891827cc913a57051a426fa4
+F test/func6.test 5d4b0526f5013db07e2f6de43df42d990500f636fa32258cfac0c87113a1c1ce
 F test/fuzz-oss1.test e58330d01cbbd8215ee636b17a03fe220b37dbfa
 F test/fuzz.test 96083052bf5765e4518c1ba686ce2bab785670d1
 F test/fuzz2.test 76dc35b32b6d6f965259508508abce75a6c4d7e1
@@ -1687,8 +1688,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 64487d658cb3b6c8c67f1e198c70813c963de52599f3ea974bdc2aa432e74de9 d5b597b52a1213cdf382d96f4df3535727be0852b25bafd12bbef54da946c5f2
-R 864c298149f75794a8b0c88eee2c8573
-T +closed d5b597b52a1213cdf382d96f4df3535727be0852b25bafd12bbef54da946c5f2
-U dan
-Z 532f71905753147684a848f63aad67ba
+P 4c782c950204c09c1d8f857c39c4cf476539ec4e7eee6fd86419d47cf0f8b9e0
+R c224d72a40caf69fa2477f10805805e4
+T *branch * location-function
+T *sym-location-function *
+T -sym-trunk *
+U drh
+Z 078a2a780a60eba456bb0addd6858536
diff --git a/manifest.uuid b/manifest.uuid
index 7b68239262..e971cdd459 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4c782c950204c09c1d8f857c39c4cf476539ec4e7eee6fd86419d47cf0f8b9e0
\ No newline at end of file
+51be9558164301c5dd4df23ab8b3e67de0b522f8d36f79f3d84d45d3dc2a83a4
\ No newline at end of file
diff --git a/src/btree.c b/src/btree.c
index 0fa00a2e12..d17cbde846 100644
--- a/src/btree.c
+++ b/src/btree.c
@@ -4432,6 +4432,19 @@ i64 sqlite3BtreeIntegerKey(BtCursor *pCur){
   return pCur->info.nKey;
 }
 
+/*
+** Return the offset into the database file for the start of the
+** payload to which the cursor is pointing.
+*/
+i64 sqlite3BtreeLocation(BtCursor *pCur){
+  assert( cursorHoldsMutex(pCur) );
+  assert( pCur->eState==CURSOR_VALID );
+  assert( pCur->curIntKey );
+  getCellInfo(pCur);
+  return (i64)pCur->pBt->pageSize*(i64)pCur->pPage->pgno +
+         (i64)(pCur->info.pPayload - pCur->pPage->aData);
+}
+
 /*
 ** Return the number of bytes of payload for the entry that pCur is
 ** currently pointing to.  For table btrees, this will be the amount
diff --git a/src/btree.h b/src/btree.h
index e2c271cdc5..43c0b9a8a4 100644
--- a/src/btree.h
+++ b/src/btree.h
@@ -291,6 +291,7 @@ int sqlite3BtreeNext(BtCursor*, int flags);
 int sqlite3BtreeEof(BtCursor*);
 int sqlite3BtreePrevious(BtCursor*, int flags);
 i64 sqlite3BtreeIntegerKey(BtCursor*);
+i64 sqlite3BtreeLocation(BtCursor*);
 int sqlite3BtreePayload(BtCursor*, u32 offset, u32 amt, void*);
 const void *sqlite3BtreePayloadFetch(BtCursor*, u32 *pAmt);
 u32 sqlite3BtreePayloadSize(BtCursor*);
diff --git a/src/expr.c b/src/expr.c
index 524e539349..b7c64d3c11 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -3871,9 +3871,18 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){
         if( !pColl ) pColl = db->pDfltColl; 
         sqlite3VdbeAddOp4(v, OP_CollSeq, 0, 0, 0, (char *)pColl, P4_COLLSEQ);
       }
-      sqlite3VdbeAddOp4(v, pParse->iSelfTab ? OP_PureFunc0 : OP_Function0,
-                        constMask, r1, target, (char*)pDef, P4_FUNCDEF);
-      sqlite3VdbeChangeP5(v, (u8)nFarg);
+      if( pDef->funcFlags & SQLITE_FUNC_LOCATION ){
+        Expr *pArg = pFarg->a[0].pExpr;
+        if( pArg->op==TK_COLUMN ){
+          sqlite3VdbeAddOp2(v, OP_Location, pArg->iTable, target);
+        }else{
+          sqlite3VdbeAddOp2(v, OP_Null, 0, target);
+        }
+      }else{
+        sqlite3VdbeAddOp4(v, pParse->iSelfTab ? OP_PureFunc0 : OP_Function0,
+                          constMask, r1, target, (char*)pDef, P4_FUNCDEF);
+        sqlite3VdbeChangeP5(v, (u8)nFarg);
+      }
       if( nFarg && constMask==0 ){
         sqlite3ReleaseTempRange(pParse, r1, nFarg);
       }
diff --git a/src/func.c b/src/func.c
index 7528fa8b4d..f81c4865d8 100644
--- a/src/func.c
+++ b/src/func.c
@@ -1799,6 +1799,8 @@ void sqlite3RegisterBuiltinFunctions(void){
 #ifdef SQLITE_DEBUG
     FUNCTION2(affinity,          1, 0, 0, noopFunc,  SQLITE_FUNC_AFFINITY),
 #endif
+    FUNCTION2(location,          1, 0, 0, noopFunc,  SQLITE_FUNC_LOCATION|
+                                                     SQLITE_FUNC_TYPEOF),
     FUNCTION(ltrim,              1, 1, 0, trimFunc         ),
     FUNCTION(ltrim,              2, 1, 0, trimFunc         ),
     FUNCTION(rtrim,              1, 2, 0, trimFunc         ),
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index efe823a654..f85682a393 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1630,6 +1630,7 @@ struct FuncDestructor {
 #define SQLITE_FUNC_SLOCHNG  0x2000 /* "Slow Change". Value constant during a
                                     ** single query - might change over time */
 #define SQLITE_FUNC_AFFINITY 0x4000 /* Built-in affinity() function */
+#define SQLITE_FUNC_LOCATION 0x8000 /* Built-in location() function */
 
 /*
 ** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are
diff --git a/src/vdbe.c b/src/vdbe.c
index cfe18a9d1e..74cf82c4c1 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -2349,6 +2349,26 @@ case OP_IfNullRow: {         /* jump */
   break;
 }
 
+/* Opcode: Location P1 P2 * * *
+** Synopsis: r[P2] = location(P1)
+**
+** Store in register r[P2] the location in the database file that is the
+** start of the payload for the record at which that cursor P1 is currently
+** pointing.
+*/
+case OP_Location: {          /* out2 */
+  VdbeCursor *pC;    /* The VDBE cursor */
+  assert( pOp->p1>=0 && pOp->p1<p->nCursor );
+  pC = p->apCsr[pOp->p1];
+  pOut = out2Prerelease(p, pOp);
+  if( pC==0 || pC->eCurType!=CURTYPE_BTREE ){
+    pOut->flags = MEM_Null;
+  }else{
+    pOut->u.i = sqlite3BtreeLocation(pC->uc.pCursor);
+  }
+  break;
+}
+
 /* Opcode: Column P1 P2 P3 P4 P5
 ** Synopsis: r[P3]=PX
 **
diff --git a/test/func6.test b/test/func6.test
new file mode 100644
index 0000000000..5350aeaa7b
--- /dev/null
+++ b/test/func6.test
@@ -0,0 +1,36 @@
+# 2017-12-16
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+# Test cases for the location() function.
+#
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+do_execsql_test func6-100 {
+  CREATE TABLE t1(a,b,c,d);
+  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100)
+   INSERT INTO t1(a,b,c,d) SELECT printf('abc%03x',x), x, 1000-x, NULL FROM c;
+}
+do_execsql_test func6-110 {
+  SELECT a, typeof(location(a)) FROM t1 ORDER BY rowid LIMIT 2;
+} {abc001 integer abc002 integer}
+do_execsql_test func6-120 {
+  SELECT a, typeof(location(+a)) FROM t1 ORDER BY rowid LIMIT 2;
+} {abc001 null abc002 null}
+do_execsql_test func6-130 {
+  CREATE INDEX t1a ON t1(a);
+  SELECT a, typeof(location(a)) FROM t1 ORDER BY a LIMIT 2;
+} {abc001 null abc002 null}
+do_execsql_test func6-140 {
+  SELECT a, typeof(location(a)) FROM t1 NOT INDEXED ORDER BY a LIMIT 2;
+} {abc001 integer abc002 integer}
+
+finish_test

From 43efc18669bc081f8937f8dc3f91a3068fd257a7 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 19 Dec 2017 17:42:13 +0000
Subject: [PATCH 159/488] Experimentally add the SQLite expert functionality to
 the shell tool.

FossilOrigin-Name: 51068dbaeaef13bb80af8126b8c4f3a454dee63de5127d706db50bf789533e60
---
 ext/expert/expert1.test |  18 +++--
 main.mk                 |   4 +-
 manifest                |  22 +++---
 manifest.uuid           |   2 +-
 src/shell.c.in          | 154 ++++++++++++++++++++++++++++++++++++++++
 test/tester.tcl         |   8 ++-
 6 files changed, 189 insertions(+), 19 deletions(-)

diff --git a/ext/expert/expert1.test b/ext/expert/expert1.test
index 7995327557..04396baf48 100644
--- a/ext/expert/expert1.test
+++ b/ext/expert/expert1.test
@@ -23,11 +23,8 @@ if {![info exists testdir]} {
 source $testdir/tester.tcl
 set testprefix expert1
 
-if {$tcl_platform(platform)=="windows"} {
-  set CMD "sqlite3_expert.exe"
-} else {
-  set CMD ".././sqlite3_expert"
-}
+set CLI [test_binary_name sqlite3]
+set CMD [test_binary_name sqlite3_expert]
 
 proc squish {txt} {
   regsub -all {[[:space:]]+} $txt { }
@@ -73,6 +70,17 @@ foreach {tn setup} {
       uplevel [list do_test $tn $tst [string trim [squish $res]]]
     }
   }
+  3 {
+    if {![file executable $CLI]} { continue }
+
+    proc do_rec_test {tn sql res} {
+      set res [squish [string trim $res]]
+      set tst [subst -nocommands { 
+        squish [string trim [exec $::CLI test.db ".expert" {$sql;}]]
+      }]
+      uplevel [list do_test $tn $tst $res]
+    }
+  }
 } {
 
   eval $setup
diff --git a/main.mk b/main.mk
index 13a8673e13..8692450441 100644
--- a/main.mk
+++ b/main.mk
@@ -692,7 +692,9 @@ SHELL_SRC = \
 	$(TOP)/src/shell.c.in \
 	$(TOP)/ext/misc/shathree.c \
 	$(TOP)/ext/misc/fileio.c \
-	$(TOP)/ext/misc/completion.c
+	$(TOP)/ext/misc/completion.c \
+	$(TOP)/ext/expert/sqlite3expert.c \
+	$(TOP)/ext/expert/sqlite3expert.h
 
 shell.c:	$(SHELL_SRC) $(TOP)/tool/mkshellc.tcl
 	tclsh $(TOP)/tool/mkshellc.tcl >shell.c
diff --git a/manifest b/manifest
index c3b80112dd..f9adc5dc9a 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\ssqlite3_vtab_collation()\sfunction,\swhich\sallows\san\sxBestIndex\scallback\nto\sdetermine\sthe\scollation\ssequence\sthat\sSQLite\swill\suse\sfor\sa\scomparison.\sAnd\nthe\sSQLITE_DBCONFIG_FULL_EQP\sconfiguration\soption,\swhich\senhances\sthe\soutput\nof\s"EXPLAIN\sQUERY\sPLAN"\sso\sthat\sit\sincludes\sstatements\srun\sby\striggers.\sAnd\nthe\scode\sfor\sthe\ssqlite3_expert\sextension\sand\scommand\sline\sapplication.
-D 2017-12-16T19:36:52.802
+C Experimentally\sadd\sthe\sSQLite\sexpert\sfunctionality\sto\sthe\sshell\stool.
+D 2017-12-19T17:42:13.030
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -42,7 +42,7 @@ F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3
 F ext/expert/expert.c 4791c5e064aea81b2b829fa95228b22283380ee370ea88a1e580103b75516ebf
-F ext/expert/expert1.test 1033e43071b69dc2f4e88fbf03fc7f18846c9865cac14f28c80f581437f09acb
+F ext/expert/expert1.test 4ffbecb378dc542d6fe030f936e2aff79a42c8cdf9fbcbff07db629fb7857b22
 F ext/expert/sqlite3expert.c 6ed4e84a06d1a29b2cf3009c0266573b88602d098055caa46c467154a64e0959
 F ext/expert/sqlite3expert.h af6354f8ee5c9e025024e63fec3bd640a802afcc3099a44d804752cf0791d811
 F ext/expert/test_expert.c 85f5c743a899063fa48296d21de2f32c26d09a21c8582b2a0bc482e8de183e7a
@@ -401,7 +401,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 6ef9e2b1c3f1e46c9e3c2b362531cbc277f7bea1a66fe610a3a7c4173b091ba4
+F main.mk 50bac9920024b5485f06398b3980f09e97ab28cd4b5b6dcd829d2a5e3ce22e7a
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -479,7 +479,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
-F src/shell.c.in 6ffed0c589f5aff180789a8c8abf5b2d3e2eea7470c86b30e797887cb0c9d0e5
+F src/shell.c.in 87a048fabcf0080a78bcdd01e57933369951f7fa7d628f04bad48e900c1899a1
 F src/sqlite.h.in 95afbec6e623843d702e727765efc833586fecc688867f41f87be7db3ff1fa62
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1287,7 +1287,7 @@ F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30
 F test/temptable2.test cd396beb41117a5302fff61767c35fa4270a0d5e
 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637
 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc
-F test/tester.tcl 9948bd856ce8a1c127f2f7900365387a42a917ce0dc87185bdd128fa5b11aff2
+F test/tester.tcl 3ed81b9e1d9718a8d9603596c8a877793d054294053c4277a3d3897eabab3866
 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5
 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58
 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7
@@ -1687,8 +1687,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 64487d658cb3b6c8c67f1e198c70813c963de52599f3ea974bdc2aa432e74de9 d5b597b52a1213cdf382d96f4df3535727be0852b25bafd12bbef54da946c5f2
-R 864c298149f75794a8b0c88eee2c8573
-T +closed d5b597b52a1213cdf382d96f4df3535727be0852b25bafd12bbef54da946c5f2
+P 4c782c950204c09c1d8f857c39c4cf476539ec4e7eee6fd86419d47cf0f8b9e0
+R 02a097d24d17c339f9cf394885a43a5e
+T *branch * expert-in-shell
+T *sym-expert-in-shell *
+T -sym-trunk *
 U dan
-Z 532f71905753147684a848f63aad67ba
+Z d356e0f9c90762c0d67a4b31b4b9510d
diff --git a/manifest.uuid b/manifest.uuid
index 7b68239262..c7ada37e5a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4c782c950204c09c1d8f857c39c4cf476539ec4e7eee6fd86419d47cf0f8b9e0
\ No newline at end of file
+51068dbaeaef13bb80af8126b8c4f3a454dee63de5127d706db50bf789533e60
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 062f76e475..cdd824835a 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -796,6 +796,8 @@ static void shellAddSchemaName(
 INCLUDE ../ext/misc/shathree.c
 INCLUDE ../ext/misc/fileio.c
 INCLUDE ../ext/misc/completion.c
+INCLUDE ../ext/expert/sqlite3expert.h
+INCLUDE ../ext/expert/sqlite3expert.c
 
 #if defined(SQLITE_ENABLE_SESSION)
 /*
@@ -822,6 +824,12 @@ struct SavedModeInfo {
   int colWidth[100];  /* Column widths prior to ".explain on" */
 };
 
+typedef struct ExpertInfo ExpertInfo;
+struct ExpertInfo {
+  sqlite3expert *pExpert;
+  int bVerbose;
+};
+
 /*
 ** State information about the database connection is contained in an
 ** instance of the following structure.
@@ -866,6 +874,7 @@ struct ShellState {
   int nSession;             /* Number of active sessions */
   OpenSession aSession[4];  /* Array of sessions.  [0] is in focus. */
 #endif
+  ExpertInfo expert;        /* Valid if previous command was ".expert OPT..." */
 };
 
 /*
@@ -2249,6 +2258,79 @@ static void exec_prepared_stmt(
   }
 }
 
+/*
+** This function is called to process SQL if the previous shell command
+** was ".expert". It passes the SQL in the second argument directly to
+** the sqlite3expert object.
+**
+** If successful, SQLITE_OK is returned. Otherwise, an SQLite error
+** code. In this case, (*pzErr) may be set to point to a buffer containing
+** an English language error message. It is the responsibility of the
+** caller to eventually free this buffer using sqlite3_free().
+*/
+static int expertHandleSQL(
+  ShellState *pState, 
+  const char *zSql, 
+  char **pzErr
+){
+  assert( pState->expert.pExpert );
+  assert( pzErr==0 || *pzErr==0 );
+  return sqlite3_expert_sql(pState->expert.pExpert, zSql, pzErr);
+}
+
+/*
+** This function is called either to silently clean up the object
+** created by the ".expert" command (if bCancel==1), or to generate a 
+** report from it and then clean it up (if bCancel==0).
+**
+** If successful, SQLITE_OK is returned. Otherwise, an SQLite error
+** code. In this case, (*pzErr) may be set to point to a buffer containing
+** an English language error message. It is the responsibility of the
+** caller to eventually free this buffer using sqlite3_free().
+*/
+static int expertFinish(
+  ShellState *pState,
+  int bCancel,
+  char **pzErr
+){
+  int rc = SQLITE_OK;
+  sqlite3expert *p = pState->expert.pExpert;
+  assert( p );
+  assert( bCancel || pzErr==0 || *pzErr==0 );
+  if( bCancel==0 ){
+    FILE *out = pState->out;
+    int bVerbose = pState->expert.bVerbose;
+
+    rc = sqlite3_expert_analyze(p, pzErr);
+    if( rc==SQLITE_OK ){
+      int nQuery = sqlite3_expert_count(p);
+      int i;
+
+      if( bVerbose ){
+        const char *zCand = sqlite3_expert_report(p,0,EXPERT_REPORT_CANDIDATES);
+        raw_printf(out, "-- Candidates -----------------------------\n");
+        raw_printf(out, "%s\n", zCand);
+      }
+      for(i=0; i<nQuery; i++){
+        const char *zSql = sqlite3_expert_report(p, i, EXPERT_REPORT_SQL);
+        const char *zIdx = sqlite3_expert_report(p, i, EXPERT_REPORT_INDEXES);
+        const char *zEQP = sqlite3_expert_report(p, i, EXPERT_REPORT_PLAN);
+        if( zIdx==0 ) zIdx = "(no new indexes)\n";
+        if( bVerbose ){
+          raw_printf(out, "-- Query %d --------------------------------\n",i+1);
+          raw_printf(out, "%s\n\n", zSql);
+        }
+        raw_printf(out, "%s\n", zIdx);
+        raw_printf(out, "%s\n", zEQP);
+      }
+    }
+  }
+  sqlite3_expert_destroy(p);
+  pState->expert.pExpert = 0;
+  return rc;
+}
+
+
 /*
 ** Execute a statement or set of statements.  Print
 ** any result rows/columns depending on the current mode
@@ -2275,6 +2357,11 @@ static int shell_exec(
     *pzErrMsg = NULL;
   }
 
+  if( pArg->expert.pExpert ){
+    rc = expertHandleSQL(pArg, zSql, pzErrMsg);
+    return expertFinish(pArg, (rc!=SQLITE_OK), pzErrMsg);
+  }
+
   while( zSql[0] && (SQLITE_OK == rc) ){
     static const char *zStmtSql;
     rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zLeftover);
@@ -4068,6 +4155,64 @@ static int lintDotCommand(
   return SQLITE_ERROR;
 }
 
+/*
+** Implementation of ".expert" dot command.
+*/
+static int expertDotCommand(
+  ShellState *pState,             /* Current shell tool state */
+  char **azArg,                   /* Array of arguments passed to dot command */
+  int nArg                        /* Number of entries in azArg[] */
+){
+  int rc = SQLITE_OK;
+  char *zErr = 0;
+  int i;
+  int iSample = 0;
+
+  assert( pState->expert.pExpert==0 );
+  memset(&pState->expert, 0, sizeof(ExpertInfo));
+
+  for(i=1; rc==SQLITE_OK && i<nArg; i++){
+    char *z = azArg[i];
+    int n;
+    if( z[0]=='-' && z[1]=='-' ) z++;
+    n = strlen(z);
+    if( n>=2 && 0==strncmp(z, "-verbose", n) ){
+      pState->expert.bVerbose = 1;
+    }
+    else if( n>=2 && 0==strncmp(z, "-sample", n) ){
+      if( i==(nArg-1) ){
+        raw_printf(stderr, "option requires an argument: %s\n", z);
+        rc = SQLITE_ERROR;
+      }else{
+        iSample = (int)integerValue(azArg[++i]);
+        if( iSample<0 || iSample>100 ){
+          raw_printf(stderr, "value out of range: %s\n", azArg[i]);
+          rc = SQLITE_ERROR;
+        }
+      }
+    }
+    else{
+      raw_printf(stderr, "unknown option: %s\n", z);
+      rc = SQLITE_ERROR;
+    }
+  }
+
+  if( rc==SQLITE_OK ){
+    pState->expert.pExpert = sqlite3_expert_new(pState->db, &zErr);
+    if( pState->expert.pExpert==0 ){
+      raw_printf(stderr, "sqlite3_expert_new: %s\n", zErr);
+      rc = SQLITE_ERROR;
+    }else{
+      sqlite3_expert_config(
+          pState->expert.pExpert, EXPERT_CONFIG_SAMPLE, iSample
+      );
+    }
+  }
+
+  return rc;
+}
+
+
 
 /*
 ** If an input line begins with "." then invoke this routine to
@@ -4082,6 +4227,10 @@ static int do_meta_command(char *zLine, ShellState *p){
   int rc = 0;
   char *azArg[50];
 
+  if( p->expert.pExpert ){
+    expertFinish(p, 1, 0);
+  }
+
   /* Parse the input line into tokens.
   */
   while( zLine[h] && nArg<ArraySize(azArg) ){
@@ -4436,6 +4585,11 @@ static int do_meta_command(char *zLine, ShellState *p){
     }
   }else
 
+  if( c=='e' && strncmp(azArg[0], "expert", n)==0 ){
+    open_db(p, 0);
+    expertDotCommand(p, azArg, nArg);
+  }else
+
   if( c=='f' && strncmp(azArg[0], "fullschema", n)==0 ){
     ShellState data;
     char *zErrMsg = 0;
diff --git a/test/tester.tcl b/test/tester.tcl
index 10a20a47d6..d0d6c92a60 100644
--- a/test/tester.tcl
+++ b/test/tester.tcl
@@ -2271,13 +2271,17 @@ proc test_restore_config_pagecache {} {
   sqlite3 db test.db
 }
 
-proc test_find_binary {nm} {
+proc test_binary_name {nm} {
   if {$::tcl_platform(platform)=="windows"} {
     set ret "$nm.exe"
   } else {
     set ret $nm
   }
-  set ret [file normalize [file join $::cmdlinearg(TESTFIXTURE_HOME) $ret]]
+  file normalize [file join $::cmdlinearg(TESTFIXTURE_HOME) $ret]
+}
+
+proc test_find_binary {nm} {
+  set ret [test_binary_name $nm]
   if {![file executable $ret]} {
     finish_test
     return ""

From e42e1bc5943f6e9e8358af755e6011a907d7527c Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 19 Dec 2017 18:56:28 +0000
Subject: [PATCH 160/488] Fix crash in sqlite3_vtab_collation() when called for
 an IS NOT NULL constraint.

FossilOrigin-Name: ad38d2c4f073705c02c7b38675e8ae86fe4a794d54eb796e7ed51a905824d5f5
---
 ext/expert/expert1.test |  9 +++++++++
 manifest                | 15 +++++++--------
 manifest.uuid           |  2 +-
 src/where.c             |  5 ++++-
 4 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/ext/expert/expert1.test b/ext/expert/expert1.test
index 7995327557..ff71207dcd 100644
--- a/ext/expert/expert1.test
+++ b/ext/expert/expert1.test
@@ -292,6 +292,15 @@ do_setup_rec_test $tn.15 {
   0|1|1|SEARCH TABLE t1 USING INTEGER PRIMARY KEY (rowid=?)
 }
 
+do_setup_rec_test $tn.16 {
+  CREATE TABLE t1(a, b);
+} {
+  SELECT * FROM t1 WHERE b IS NOT NULL;
+} {
+  (no new indexes)
+  0|0|0|SCAN TABLE t1
+}
+
 }
 
 proc do_candidates_test {tn sql res} {
diff --git a/manifest b/manifest
index c3b80112dd..19e4965990 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\ssqlite3_vtab_collation()\sfunction,\swhich\sallows\san\sxBestIndex\scallback\nto\sdetermine\sthe\scollation\ssequence\sthat\sSQLite\swill\suse\sfor\sa\scomparison.\sAnd\nthe\sSQLITE_DBCONFIG_FULL_EQP\sconfiguration\soption,\swhich\senhances\sthe\soutput\nof\s"EXPLAIN\sQUERY\sPLAN"\sso\sthat\sit\sincludes\sstatements\srun\sby\striggers.\sAnd\nthe\scode\sfor\sthe\ssqlite3_expert\sextension\sand\scommand\sline\sapplication.
-D 2017-12-16T19:36:52.802
+C Fix\scrash\sin\ssqlite3_vtab_collation()\swhen\scalled\sfor\san\sIS\sNOT\sNULL\nconstraint.
+D 2017-12-19T18:56:28.856
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -42,7 +42,7 @@ F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3
 F ext/expert/expert.c 4791c5e064aea81b2b829fa95228b22283380ee370ea88a1e580103b75516ebf
-F ext/expert/expert1.test 1033e43071b69dc2f4e88fbf03fc7f18846c9865cac14f28c80f581437f09acb
+F ext/expert/expert1.test 939265e01b8a40551b1338cc3c38b378d7c99479ececbfc5095b2e757f7b8944
 F ext/expert/sqlite3expert.c 6ed4e84a06d1a29b2cf3009c0266573b88602d098055caa46c467154a64e0959
 F ext/expert/sqlite3expert.h af6354f8ee5c9e025024e63fec3bd640a802afcc3099a44d804752cf0791d811
 F ext/expert/test_expert.c 85f5c743a899063fa48296d21de2f32c26d09a21c8582b2a0bc482e8de183e7a
@@ -561,7 +561,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 5a3f464edd64596f601683ed321d12e6fd93c5fb9afdfb3653d6ffd0fee9c48f
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f
-F src/where.c 1617e7e42daf0ace7799204e3e9b7c9e75069aa2a0c8ddd24d1b9569b2a0b7e4
+F src/where.c 0cdfda28708fe75699df2fcdb44d067dc94e7b1f83e3a622dca96a27aaaa2e42
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
 F src/wherecode.c ff2f079097a3bdce6ebabfde1419fba448c9ce5feb7cb964e8bfa2a4e27274ef
 F src/whereexpr.c 427ea8e96ec24f2a7814c67b8024ad664a9c7656264c4566c34743cb23186e46
@@ -1687,8 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 64487d658cb3b6c8c67f1e198c70813c963de52599f3ea974bdc2aa432e74de9 d5b597b52a1213cdf382d96f4df3535727be0852b25bafd12bbef54da946c5f2
-R 864c298149f75794a8b0c88eee2c8573
-T +closed d5b597b52a1213cdf382d96f4df3535727be0852b25bafd12bbef54da946c5f2
+P 4c782c950204c09c1d8f857c39c4cf476539ec4e7eee6fd86419d47cf0f8b9e0
+R 467752e7571913f5d5399376bb34bcd1
 U dan
-Z 532f71905753147684a848f63aad67ba
+Z b7b7f4c9939a0a5799e54a0fcc200864
diff --git a/manifest.uuid b/manifest.uuid
index 7b68239262..c2cbc0a906 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4c782c950204c09c1d8f857c39c4cf476539ec4e7eee6fd86419d47cf0f8b9e0
\ No newline at end of file
+ad38d2c4f073705c02c7b38675e8ae86fe4a794d54eb796e7ed51a905824d5f5
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index 0694031a51..f0ee78a1da 100644
--- a/src/where.c
+++ b/src/where.c
@@ -3160,9 +3160,12 @@ const char *sqlite3_vtab_collation(sqlite3 *db, int iCons){
   struct BestIndexCtx *p = (struct BestIndexCtx*)db->pBestIndexCtx;
   const char *zRet = 0;
   if( p && iCons>=0 && iCons<p->pIdxInfo->nConstraint ){
+    CollSeq *pC = 0;
     int iTerm = p->pIdxInfo->aConstraint[iCons].iTermOffset;
     Expr *pX = p->pWC->a[iTerm].pExpr;
-    CollSeq *pC = sqlite3BinaryCompareCollSeq(p->pParse,pX->pLeft,pX->pRight);
+    if( pX->pLeft ){
+      pC = sqlite3BinaryCompareCollSeq(p->pParse, pX->pLeft, pX->pRight);
+    }
     zRet = (pC ? pC->zName : "BINARY");
   }
   return zRet;

From 0872e268abf94766a92dcacc91681066a4e07d40 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 20 Dec 2017 23:46:29 +0000
Subject: [PATCH 161/488] Lowercase local variable names in the SHA3 extension
 in order to avoid collisions with macros in termios.h.

FossilOrigin-Name: 3ec7371161bd617e40328aa015b09acc2b37b0b5d269a87050a0c57163f92801
---
 ext/misc/shathree.c | 544 ++++++++++++++++++++++----------------------
 manifest            |  14 +-
 manifest.uuid       |   2 +-
 3 files changed, 280 insertions(+), 280 deletions(-)

diff --git a/ext/misc/shathree.c b/ext/misc/shathree.c
index 612e395ccb..e5c95407d6 100644
--- a/ext/misc/shathree.c
+++ b/ext/misc/shathree.c
@@ -78,9 +78,9 @@ struct SHA3Context {
 */
 static void KeccakF1600Step(SHA3Context *p){
   int i;
-  u64 B0, B1, B2, B3, B4;
-  u64 C0, C1, C2, C3, C4;
-  u64 D0, D1, D2, D3, D4;
+  u64 b0, b1, b2, b3, b4;
+  u64 c0, c1, c2, c3, c4;
+  u64 d0, d1, d2, d3, d4;
   static const u64 RC[] = {
     0x0000000000000001ULL,  0x0000000000008082ULL,
     0x800000000000808aULL,  0x8000000080008000ULL,
@@ -95,301 +95,301 @@ static void KeccakF1600Step(SHA3Context *p){
     0x8000000080008081ULL,  0x8000000000008080ULL,
     0x0000000080000001ULL,  0x8000000080008008ULL
   };
-# define A00 (p->u.s[0])
-# define A01 (p->u.s[1])
-# define A02 (p->u.s[2])
-# define A03 (p->u.s[3])
-# define A04 (p->u.s[4])
-# define A10 (p->u.s[5])
-# define A11 (p->u.s[6])
-# define A12 (p->u.s[7])
-# define A13 (p->u.s[8])
-# define A14 (p->u.s[9])
-# define A20 (p->u.s[10])
-# define A21 (p->u.s[11])
-# define A22 (p->u.s[12])
-# define A23 (p->u.s[13])
-# define A24 (p->u.s[14])
-# define A30 (p->u.s[15])
-# define A31 (p->u.s[16])
-# define A32 (p->u.s[17])
-# define A33 (p->u.s[18])
-# define A34 (p->u.s[19])
-# define A40 (p->u.s[20])
-# define A41 (p->u.s[21])
-# define A42 (p->u.s[22])
-# define A43 (p->u.s[23])
-# define A44 (p->u.s[24])
+# define a00 (p->u.s[0])
+# define a01 (p->u.s[1])
+# define a02 (p->u.s[2])
+# define a03 (p->u.s[3])
+# define a04 (p->u.s[4])
+# define a10 (p->u.s[5])
+# define a11 (p->u.s[6])
+# define a12 (p->u.s[7])
+# define a13 (p->u.s[8])
+# define a14 (p->u.s[9])
+# define a20 (p->u.s[10])
+# define a21 (p->u.s[11])
+# define a22 (p->u.s[12])
+# define a23 (p->u.s[13])
+# define a24 (p->u.s[14])
+# define a30 (p->u.s[15])
+# define a31 (p->u.s[16])
+# define a32 (p->u.s[17])
+# define a33 (p->u.s[18])
+# define a34 (p->u.s[19])
+# define a40 (p->u.s[20])
+# define a41 (p->u.s[21])
+# define a42 (p->u.s[22])
+# define a43 (p->u.s[23])
+# define a44 (p->u.s[24])
 # define ROL64(a,x) ((a<<x)|(a>>(64-x)))
 
   for(i=0; i<24; i+=4){
-    C0 = A00^A10^A20^A30^A40;
-    C1 = A01^A11^A21^A31^A41;
-    C2 = A02^A12^A22^A32^A42;
-    C3 = A03^A13^A23^A33^A43;
-    C4 = A04^A14^A24^A34^A44;
-    D0 = C4^ROL64(C1, 1);
-    D1 = C0^ROL64(C2, 1);
-    D2 = C1^ROL64(C3, 1);
-    D3 = C2^ROL64(C4, 1);
-    D4 = C3^ROL64(C0, 1);
+    c0 = a00^a10^a20^a30^a40;
+    c1 = a01^a11^a21^a31^a41;
+    c2 = a02^a12^a22^a32^a42;
+    c3 = a03^a13^a23^a33^a43;
+    c4 = a04^a14^a24^a34^a44;
+    d0 = c4^ROL64(c1, 1);
+    d1 = c0^ROL64(c2, 1);
+    d2 = c1^ROL64(c3, 1);
+    d3 = c2^ROL64(c4, 1);
+    d4 = c3^ROL64(c0, 1);
 
-    B0 = (A00^D0);
-    B1 = ROL64((A11^D1), 44);
-    B2 = ROL64((A22^D2), 43);
-    B3 = ROL64((A33^D3), 21);
-    B4 = ROL64((A44^D4), 14);
-    A00 =   B0 ^((~B1)&  B2 );
-    A00 ^= RC[i];
-    A11 =   B1 ^((~B2)&  B3 );
-    A22 =   B2 ^((~B3)&  B4 );
-    A33 =   B3 ^((~B4)&  B0 );
-    A44 =   B4 ^((~B0)&  B1 );
+    b0 = (a00^d0);
+    b1 = ROL64((a11^d1), 44);
+    b2 = ROL64((a22^d2), 43);
+    b3 = ROL64((a33^d3), 21);
+    b4 = ROL64((a44^d4), 14);
+    a00 =   b0 ^((~b1)&  b2 );
+    a00 ^= RC[i];
+    a11 =   b1 ^((~b2)&  b3 );
+    a22 =   b2 ^((~b3)&  b4 );
+    a33 =   b3 ^((~b4)&  b0 );
+    a44 =   b4 ^((~b0)&  b1 );
 
-    B2 = ROL64((A20^D0), 3);
-    B3 = ROL64((A31^D1), 45);
-    B4 = ROL64((A42^D2), 61);
-    B0 = ROL64((A03^D3), 28);
-    B1 = ROL64((A14^D4), 20);
-    A20 =   B0 ^((~B1)&  B2 );
-    A31 =   B1 ^((~B2)&  B3 );
-    A42 =   B2 ^((~B3)&  B4 );
-    A03 =   B3 ^((~B4)&  B0 );
-    A14 =   B4 ^((~B0)&  B1 );
+    b2 = ROL64((a20^d0), 3);
+    b3 = ROL64((a31^d1), 45);
+    b4 = ROL64((a42^d2), 61);
+    b0 = ROL64((a03^d3), 28);
+    b1 = ROL64((a14^d4), 20);
+    a20 =   b0 ^((~b1)&  b2 );
+    a31 =   b1 ^((~b2)&  b3 );
+    a42 =   b2 ^((~b3)&  b4 );
+    a03 =   b3 ^((~b4)&  b0 );
+    a14 =   b4 ^((~b0)&  b1 );
 
-    B4 = ROL64((A40^D0), 18);
-    B0 = ROL64((A01^D1), 1);
-    B1 = ROL64((A12^D2), 6);
-    B2 = ROL64((A23^D3), 25);
-    B3 = ROL64((A34^D4), 8);
-    A40 =   B0 ^((~B1)&  B2 );
-    A01 =   B1 ^((~B2)&  B3 );
-    A12 =   B2 ^((~B3)&  B4 );
-    A23 =   B3 ^((~B4)&  B0 );
-    A34 =   B4 ^((~B0)&  B1 );
+    b4 = ROL64((a40^d0), 18);
+    b0 = ROL64((a01^d1), 1);
+    b1 = ROL64((a12^d2), 6);
+    b2 = ROL64((a23^d3), 25);
+    b3 = ROL64((a34^d4), 8);
+    a40 =   b0 ^((~b1)&  b2 );
+    a01 =   b1 ^((~b2)&  b3 );
+    a12 =   b2 ^((~b3)&  b4 );
+    a23 =   b3 ^((~b4)&  b0 );
+    a34 =   b4 ^((~b0)&  b1 );
 
-    B1 = ROL64((A10^D0), 36);
-    B2 = ROL64((A21^D1), 10);
-    B3 = ROL64((A32^D2), 15);
-    B4 = ROL64((A43^D3), 56);
-    B0 = ROL64((A04^D4), 27);
-    A10 =   B0 ^((~B1)&  B2 );
-    A21 =   B1 ^((~B2)&  B3 );
-    A32 =   B2 ^((~B3)&  B4 );
-    A43 =   B3 ^((~B4)&  B0 );
-    A04 =   B4 ^((~B0)&  B1 );
+    b1 = ROL64((a10^d0), 36);
+    b2 = ROL64((a21^d1), 10);
+    b3 = ROL64((a32^d2), 15);
+    b4 = ROL64((a43^d3), 56);
+    b0 = ROL64((a04^d4), 27);
+    a10 =   b0 ^((~b1)&  b2 );
+    a21 =   b1 ^((~b2)&  b3 );
+    a32 =   b2 ^((~b3)&  b4 );
+    a43 =   b3 ^((~b4)&  b0 );
+    a04 =   b4 ^((~b0)&  b1 );
 
-    B3 = ROL64((A30^D0), 41);
-    B4 = ROL64((A41^D1), 2);
-    B0 = ROL64((A02^D2), 62);
-    B1 = ROL64((A13^D3), 55);
-    B2 = ROL64((A24^D4), 39);
-    A30 =   B0 ^((~B1)&  B2 );
-    A41 =   B1 ^((~B2)&  B3 );
-    A02 =   B2 ^((~B3)&  B4 );
-    A13 =   B3 ^((~B4)&  B0 );
-    A24 =   B4 ^((~B0)&  B1 );
+    b3 = ROL64((a30^d0), 41);
+    b4 = ROL64((a41^d1), 2);
+    b0 = ROL64((a02^d2), 62);
+    b1 = ROL64((a13^d3), 55);
+    b2 = ROL64((a24^d4), 39);
+    a30 =   b0 ^((~b1)&  b2 );
+    a41 =   b1 ^((~b2)&  b3 );
+    a02 =   b2 ^((~b3)&  b4 );
+    a13 =   b3 ^((~b4)&  b0 );
+    a24 =   b4 ^((~b0)&  b1 );
 
-    C0 = A00^A20^A40^A10^A30;
-    C1 = A11^A31^A01^A21^A41;
-    C2 = A22^A42^A12^A32^A02;
-    C3 = A33^A03^A23^A43^A13;
-    C4 = A44^A14^A34^A04^A24;
-    D0 = C4^ROL64(C1, 1);
-    D1 = C0^ROL64(C2, 1);
-    D2 = C1^ROL64(C3, 1);
-    D3 = C2^ROL64(C4, 1);
-    D4 = C3^ROL64(C0, 1);
+    c0 = a00^a20^a40^a10^a30;
+    c1 = a11^a31^a01^a21^a41;
+    c2 = a22^a42^a12^a32^a02;
+    c3 = a33^a03^a23^a43^a13;
+    c4 = a44^a14^a34^a04^a24;
+    d0 = c4^ROL64(c1, 1);
+    d1 = c0^ROL64(c2, 1);
+    d2 = c1^ROL64(c3, 1);
+    d3 = c2^ROL64(c4, 1);
+    d4 = c3^ROL64(c0, 1);
 
-    B0 = (A00^D0);
-    B1 = ROL64((A31^D1), 44);
-    B2 = ROL64((A12^D2), 43);
-    B3 = ROL64((A43^D3), 21);
-    B4 = ROL64((A24^D4), 14);
-    A00 =   B0 ^((~B1)&  B2 );
-    A00 ^= RC[i+1];
-    A31 =   B1 ^((~B2)&  B3 );
-    A12 =   B2 ^((~B3)&  B4 );
-    A43 =   B3 ^((~B4)&  B0 );
-    A24 =   B4 ^((~B0)&  B1 );
+    b0 = (a00^d0);
+    b1 = ROL64((a31^d1), 44);
+    b2 = ROL64((a12^d2), 43);
+    b3 = ROL64((a43^d3), 21);
+    b4 = ROL64((a24^d4), 14);
+    a00 =   b0 ^((~b1)&  b2 );
+    a00 ^= RC[i+1];
+    a31 =   b1 ^((~b2)&  b3 );
+    a12 =   b2 ^((~b3)&  b4 );
+    a43 =   b3 ^((~b4)&  b0 );
+    a24 =   b4 ^((~b0)&  b1 );
 
-    B2 = ROL64((A40^D0), 3);
-    B3 = ROL64((A21^D1), 45);
-    B4 = ROL64((A02^D2), 61);
-    B0 = ROL64((A33^D3), 28);
-    B1 = ROL64((A14^D4), 20);
-    A40 =   B0 ^((~B1)&  B2 );
-    A21 =   B1 ^((~B2)&  B3 );
-    A02 =   B2 ^((~B3)&  B4 );
-    A33 =   B3 ^((~B4)&  B0 );
-    A14 =   B4 ^((~B0)&  B1 );
+    b2 = ROL64((a40^d0), 3);
+    b3 = ROL64((a21^d1), 45);
+    b4 = ROL64((a02^d2), 61);
+    b0 = ROL64((a33^d3), 28);
+    b1 = ROL64((a14^d4), 20);
+    a40 =   b0 ^((~b1)&  b2 );
+    a21 =   b1 ^((~b2)&  b3 );
+    a02 =   b2 ^((~b3)&  b4 );
+    a33 =   b3 ^((~b4)&  b0 );
+    a14 =   b4 ^((~b0)&  b1 );
 
-    B4 = ROL64((A30^D0), 18);
-    B0 = ROL64((A11^D1), 1);
-    B1 = ROL64((A42^D2), 6);
-    B2 = ROL64((A23^D3), 25);
-    B3 = ROL64((A04^D4), 8);
-    A30 =   B0 ^((~B1)&  B2 );
-    A11 =   B1 ^((~B2)&  B3 );
-    A42 =   B2 ^((~B3)&  B4 );
-    A23 =   B3 ^((~B4)&  B0 );
-    A04 =   B4 ^((~B0)&  B1 );
+    b4 = ROL64((a30^d0), 18);
+    b0 = ROL64((a11^d1), 1);
+    b1 = ROL64((a42^d2), 6);
+    b2 = ROL64((a23^d3), 25);
+    b3 = ROL64((a04^d4), 8);
+    a30 =   b0 ^((~b1)&  b2 );
+    a11 =   b1 ^((~b2)&  b3 );
+    a42 =   b2 ^((~b3)&  b4 );
+    a23 =   b3 ^((~b4)&  b0 );
+    a04 =   b4 ^((~b0)&  b1 );
 
-    B1 = ROL64((A20^D0), 36);
-    B2 = ROL64((A01^D1), 10);
-    B3 = ROL64((A32^D2), 15);
-    B4 = ROL64((A13^D3), 56);
-    B0 = ROL64((A44^D4), 27);
-    A20 =   B0 ^((~B1)&  B2 );
-    A01 =   B1 ^((~B2)&  B3 );
-    A32 =   B2 ^((~B3)&  B4 );
-    A13 =   B3 ^((~B4)&  B0 );
-    A44 =   B4 ^((~B0)&  B1 );
+    b1 = ROL64((a20^d0), 36);
+    b2 = ROL64((a01^d1), 10);
+    b3 = ROL64((a32^d2), 15);
+    b4 = ROL64((a13^d3), 56);
+    b0 = ROL64((a44^d4), 27);
+    a20 =   b0 ^((~b1)&  b2 );
+    a01 =   b1 ^((~b2)&  b3 );
+    a32 =   b2 ^((~b3)&  b4 );
+    a13 =   b3 ^((~b4)&  b0 );
+    a44 =   b4 ^((~b0)&  b1 );
 
-    B3 = ROL64((A10^D0), 41);
-    B4 = ROL64((A41^D1), 2);
-    B0 = ROL64((A22^D2), 62);
-    B1 = ROL64((A03^D3), 55);
-    B2 = ROL64((A34^D4), 39);
-    A10 =   B0 ^((~B1)&  B2 );
-    A41 =   B1 ^((~B2)&  B3 );
-    A22 =   B2 ^((~B3)&  B4 );
-    A03 =   B3 ^((~B4)&  B0 );
-    A34 =   B4 ^((~B0)&  B1 );
+    b3 = ROL64((a10^d0), 41);
+    b4 = ROL64((a41^d1), 2);
+    b0 = ROL64((a22^d2), 62);
+    b1 = ROL64((a03^d3), 55);
+    b2 = ROL64((a34^d4), 39);
+    a10 =   b0 ^((~b1)&  b2 );
+    a41 =   b1 ^((~b2)&  b3 );
+    a22 =   b2 ^((~b3)&  b4 );
+    a03 =   b3 ^((~b4)&  b0 );
+    a34 =   b4 ^((~b0)&  b1 );
 
-    C0 = A00^A40^A30^A20^A10;
-    C1 = A31^A21^A11^A01^A41;
-    C2 = A12^A02^A42^A32^A22;
-    C3 = A43^A33^A23^A13^A03;
-    C4 = A24^A14^A04^A44^A34;
-    D0 = C4^ROL64(C1, 1);
-    D1 = C0^ROL64(C2, 1);
-    D2 = C1^ROL64(C3, 1);
-    D3 = C2^ROL64(C4, 1);
-    D4 = C3^ROL64(C0, 1);
+    c0 = a00^a40^a30^a20^a10;
+    c1 = a31^a21^a11^a01^a41;
+    c2 = a12^a02^a42^a32^a22;
+    c3 = a43^a33^a23^a13^a03;
+    c4 = a24^a14^a04^a44^a34;
+    d0 = c4^ROL64(c1, 1);
+    d1 = c0^ROL64(c2, 1);
+    d2 = c1^ROL64(c3, 1);
+    d3 = c2^ROL64(c4, 1);
+    d4 = c3^ROL64(c0, 1);
 
-    B0 = (A00^D0);
-    B1 = ROL64((A21^D1), 44);
-    B2 = ROL64((A42^D2), 43);
-    B3 = ROL64((A13^D3), 21);
-    B4 = ROL64((A34^D4), 14);
-    A00 =   B0 ^((~B1)&  B2 );
-    A00 ^= RC[i+2];
-    A21 =   B1 ^((~B2)&  B3 );
-    A42 =   B2 ^((~B3)&  B4 );
-    A13 =   B3 ^((~B4)&  B0 );
-    A34 =   B4 ^((~B0)&  B1 );
+    b0 = (a00^d0);
+    b1 = ROL64((a21^d1), 44);
+    b2 = ROL64((a42^d2), 43);
+    b3 = ROL64((a13^d3), 21);
+    b4 = ROL64((a34^d4), 14);
+    a00 =   b0 ^((~b1)&  b2 );
+    a00 ^= RC[i+2];
+    a21 =   b1 ^((~b2)&  b3 );
+    a42 =   b2 ^((~b3)&  b4 );
+    a13 =   b3 ^((~b4)&  b0 );
+    a34 =   b4 ^((~b0)&  b1 );
 
-    B2 = ROL64((A30^D0), 3);
-    B3 = ROL64((A01^D1), 45);
-    B4 = ROL64((A22^D2), 61);
-    B0 = ROL64((A43^D3), 28);
-    B1 = ROL64((A14^D4), 20);
-    A30 =   B0 ^((~B1)&  B2 );
-    A01 =   B1 ^((~B2)&  B3 );
-    A22 =   B2 ^((~B3)&  B4 );
-    A43 =   B3 ^((~B4)&  B0 );
-    A14 =   B4 ^((~B0)&  B1 );
+    b2 = ROL64((a30^d0), 3);
+    b3 = ROL64((a01^d1), 45);
+    b4 = ROL64((a22^d2), 61);
+    b0 = ROL64((a43^d3), 28);
+    b1 = ROL64((a14^d4), 20);
+    a30 =   b0 ^((~b1)&  b2 );
+    a01 =   b1 ^((~b2)&  b3 );
+    a22 =   b2 ^((~b3)&  b4 );
+    a43 =   b3 ^((~b4)&  b0 );
+    a14 =   b4 ^((~b0)&  b1 );
 
-    B4 = ROL64((A10^D0), 18);
-    B0 = ROL64((A31^D1), 1);
-    B1 = ROL64((A02^D2), 6);
-    B2 = ROL64((A23^D3), 25);
-    B3 = ROL64((A44^D4), 8);
-    A10 =   B0 ^((~B1)&  B2 );
-    A31 =   B1 ^((~B2)&  B3 );
-    A02 =   B2 ^((~B3)&  B4 );
-    A23 =   B3 ^((~B4)&  B0 );
-    A44 =   B4 ^((~B0)&  B1 );
+    b4 = ROL64((a10^d0), 18);
+    b0 = ROL64((a31^d1), 1);
+    b1 = ROL64((a02^d2), 6);
+    b2 = ROL64((a23^d3), 25);
+    b3 = ROL64((a44^d4), 8);
+    a10 =   b0 ^((~b1)&  b2 );
+    a31 =   b1 ^((~b2)&  b3 );
+    a02 =   b2 ^((~b3)&  b4 );
+    a23 =   b3 ^((~b4)&  b0 );
+    a44 =   b4 ^((~b0)&  b1 );
 
-    B1 = ROL64((A40^D0), 36);
-    B2 = ROL64((A11^D1), 10);
-    B3 = ROL64((A32^D2), 15);
-    B4 = ROL64((A03^D3), 56);
-    B0 = ROL64((A24^D4), 27);
-    A40 =   B0 ^((~B1)&  B2 );
-    A11 =   B1 ^((~B2)&  B3 );
-    A32 =   B2 ^((~B3)&  B4 );
-    A03 =   B3 ^((~B4)&  B0 );
-    A24 =   B4 ^((~B0)&  B1 );
+    b1 = ROL64((a40^d0), 36);
+    b2 = ROL64((a11^d1), 10);
+    b3 = ROL64((a32^d2), 15);
+    b4 = ROL64((a03^d3), 56);
+    b0 = ROL64((a24^d4), 27);
+    a40 =   b0 ^((~b1)&  b2 );
+    a11 =   b1 ^((~b2)&  b3 );
+    a32 =   b2 ^((~b3)&  b4 );
+    a03 =   b3 ^((~b4)&  b0 );
+    a24 =   b4 ^((~b0)&  b1 );
 
-    B3 = ROL64((A20^D0), 41);
-    B4 = ROL64((A41^D1), 2);
-    B0 = ROL64((A12^D2), 62);
-    B1 = ROL64((A33^D3), 55);
-    B2 = ROL64((A04^D4), 39);
-    A20 =   B0 ^((~B1)&  B2 );
-    A41 =   B1 ^((~B2)&  B3 );
-    A12 =   B2 ^((~B3)&  B4 );
-    A33 =   B3 ^((~B4)&  B0 );
-    A04 =   B4 ^((~B0)&  B1 );
+    b3 = ROL64((a20^d0), 41);
+    b4 = ROL64((a41^d1), 2);
+    b0 = ROL64((a12^d2), 62);
+    b1 = ROL64((a33^d3), 55);
+    b2 = ROL64((a04^d4), 39);
+    a20 =   b0 ^((~b1)&  b2 );
+    a41 =   b1 ^((~b2)&  b3 );
+    a12 =   b2 ^((~b3)&  b4 );
+    a33 =   b3 ^((~b4)&  b0 );
+    a04 =   b4 ^((~b0)&  b1 );
 
-    C0 = A00^A30^A10^A40^A20;
-    C1 = A21^A01^A31^A11^A41;
-    C2 = A42^A22^A02^A32^A12;
-    C3 = A13^A43^A23^A03^A33;
-    C4 = A34^A14^A44^A24^A04;
-    D0 = C4^ROL64(C1, 1);
-    D1 = C0^ROL64(C2, 1);
-    D2 = C1^ROL64(C3, 1);
-    D3 = C2^ROL64(C4, 1);
-    D4 = C3^ROL64(C0, 1);
+    c0 = a00^a30^a10^a40^a20;
+    c1 = a21^a01^a31^a11^a41;
+    c2 = a42^a22^a02^a32^a12;
+    c3 = a13^a43^a23^a03^a33;
+    c4 = a34^a14^a44^a24^a04;
+    d0 = c4^ROL64(c1, 1);
+    d1 = c0^ROL64(c2, 1);
+    d2 = c1^ROL64(c3, 1);
+    d3 = c2^ROL64(c4, 1);
+    d4 = c3^ROL64(c0, 1);
 
-    B0 = (A00^D0);
-    B1 = ROL64((A01^D1), 44);
-    B2 = ROL64((A02^D2), 43);
-    B3 = ROL64((A03^D3), 21);
-    B4 = ROL64((A04^D4), 14);
-    A00 =   B0 ^((~B1)&  B2 );
-    A00 ^= RC[i+3];
-    A01 =   B1 ^((~B2)&  B3 );
-    A02 =   B2 ^((~B3)&  B4 );
-    A03 =   B3 ^((~B4)&  B0 );
-    A04 =   B4 ^((~B0)&  B1 );
+    b0 = (a00^d0);
+    b1 = ROL64((a01^d1), 44);
+    b2 = ROL64((a02^d2), 43);
+    b3 = ROL64((a03^d3), 21);
+    b4 = ROL64((a04^d4), 14);
+    a00 =   b0 ^((~b1)&  b2 );
+    a00 ^= RC[i+3];
+    a01 =   b1 ^((~b2)&  b3 );
+    a02 =   b2 ^((~b3)&  b4 );
+    a03 =   b3 ^((~b4)&  b0 );
+    a04 =   b4 ^((~b0)&  b1 );
 
-    B2 = ROL64((A10^D0), 3);
-    B3 = ROL64((A11^D1), 45);
-    B4 = ROL64((A12^D2), 61);
-    B0 = ROL64((A13^D3), 28);
-    B1 = ROL64((A14^D4), 20);
-    A10 =   B0 ^((~B1)&  B2 );
-    A11 =   B1 ^((~B2)&  B3 );
-    A12 =   B2 ^((~B3)&  B4 );
-    A13 =   B3 ^((~B4)&  B0 );
-    A14 =   B4 ^((~B0)&  B1 );
+    b2 = ROL64((a10^d0), 3);
+    b3 = ROL64((a11^d1), 45);
+    b4 = ROL64((a12^d2), 61);
+    b0 = ROL64((a13^d3), 28);
+    b1 = ROL64((a14^d4), 20);
+    a10 =   b0 ^((~b1)&  b2 );
+    a11 =   b1 ^((~b2)&  b3 );
+    a12 =   b2 ^((~b3)&  b4 );
+    a13 =   b3 ^((~b4)&  b0 );
+    a14 =   b4 ^((~b0)&  b1 );
 
-    B4 = ROL64((A20^D0), 18);
-    B0 = ROL64((A21^D1), 1);
-    B1 = ROL64((A22^D2), 6);
-    B2 = ROL64((A23^D3), 25);
-    B3 = ROL64((A24^D4), 8);
-    A20 =   B0 ^((~B1)&  B2 );
-    A21 =   B1 ^((~B2)&  B3 );
-    A22 =   B2 ^((~B3)&  B4 );
-    A23 =   B3 ^((~B4)&  B0 );
-    A24 =   B4 ^((~B0)&  B1 );
+    b4 = ROL64((a20^d0), 18);
+    b0 = ROL64((a21^d1), 1);
+    b1 = ROL64((a22^d2), 6);
+    b2 = ROL64((a23^d3), 25);
+    b3 = ROL64((a24^d4), 8);
+    a20 =   b0 ^((~b1)&  b2 );
+    a21 =   b1 ^((~b2)&  b3 );
+    a22 =   b2 ^((~b3)&  b4 );
+    a23 =   b3 ^((~b4)&  b0 );
+    a24 =   b4 ^((~b0)&  b1 );
 
-    B1 = ROL64((A30^D0), 36);
-    B2 = ROL64((A31^D1), 10);
-    B3 = ROL64((A32^D2), 15);
-    B4 = ROL64((A33^D3), 56);
-    B0 = ROL64((A34^D4), 27);
-    A30 =   B0 ^((~B1)&  B2 );
-    A31 =   B1 ^((~B2)&  B3 );
-    A32 =   B2 ^((~B3)&  B4 );
-    A33 =   B3 ^((~B4)&  B0 );
-    A34 =   B4 ^((~B0)&  B1 );
+    b1 = ROL64((a30^d0), 36);
+    b2 = ROL64((a31^d1), 10);
+    b3 = ROL64((a32^d2), 15);
+    b4 = ROL64((a33^d3), 56);
+    b0 = ROL64((a34^d4), 27);
+    a30 =   b0 ^((~b1)&  b2 );
+    a31 =   b1 ^((~b2)&  b3 );
+    a32 =   b2 ^((~b3)&  b4 );
+    a33 =   b3 ^((~b4)&  b0 );
+    a34 =   b4 ^((~b0)&  b1 );
 
-    B3 = ROL64((A40^D0), 41);
-    B4 = ROL64((A41^D1), 2);
-    B0 = ROL64((A42^D2), 62);
-    B1 = ROL64((A43^D3), 55);
-    B2 = ROL64((A44^D4), 39);
-    A40 =   B0 ^((~B1)&  B2 );
-    A41 =   B1 ^((~B2)&  B3 );
-    A42 =   B2 ^((~B3)&  B4 );
-    A43 =   B3 ^((~B4)&  B0 );
-    A44 =   B4 ^((~B0)&  B1 );
+    b3 = ROL64((a40^d0), 41);
+    b4 = ROL64((a41^d1), 2);
+    b0 = ROL64((a42^d2), 62);
+    b1 = ROL64((a43^d3), 55);
+    b2 = ROL64((a44^d4), 39);
+    a40 =   b0 ^((~b1)&  b2 );
+    a41 =   b1 ^((~b2)&  b3 );
+    a42 =   b2 ^((~b3)&  b4 );
+    a43 =   b3 ^((~b4)&  b0 );
+    a44 =   b4 ^((~b0)&  b1 );
   }
 }
 
diff --git a/manifest b/manifest
index 19e4965990..c1dd4fcf09 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\scrash\sin\ssqlite3_vtab_collation()\swhen\scalled\sfor\san\sIS\sNOT\sNULL\nconstraint.
-D 2017-12-19T18:56:28.856
+C Lowercase\slocal\svariable\snames\sin\sthe\sSHA3\sextension\sin\sorder\sto\savoid\ncollisions\swith\smacros\sin\stermios.h.
+D 2017-12-20T23:46:29.934
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -288,7 +288,7 @@ F ext/misc/rot13.c 540a169cb0d74f15522a8930b0cccdcb37a4fd071d219a5a083a319fc6e8d
 F ext/misc/scrub.c 1c5bfb8b0cd18b602fcb55755e84abf0023ac2fb
 F ext/misc/series.c f3c0dba5c5c749ce1782b53076108f87cf0b71041eb6023f727a9c50681da564
 F ext/misc/sha1.c 0b9e9b855354910d3ca467bf39099d570e73db56
-F ext/misc/shathree.c fa185d7aee0ad0aca5e091b4a2db7baff11796170e5793b5de99e511a13af448
+F ext/misc/shathree.c 9e960ba50483214c6a7a4b1517f8d8cef799e9db381195178c3fd3ad207e10c0
 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52
 F ext/misc/spellfix.c 41cf26c6b89fcaa8798ae10ae64d39c1f1d9d6995152e545bd491c13058b8fac
 F ext/misc/stmt.c 6f16443abb3551e3f5813bb13ba19a30e7032830015b0f92fe0c0453045c0a11
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 4c782c950204c09c1d8f857c39c4cf476539ec4e7eee6fd86419d47cf0f8b9e0
-R 467752e7571913f5d5399376bb34bcd1
-U dan
-Z b7b7f4c9939a0a5799e54a0fcc200864
+P ad38d2c4f073705c02c7b38675e8ae86fe4a794d54eb796e7ed51a905824d5f5
+R 5daaca0715e80715d469ec030422b144
+U drh
+Z 1310a0a150441991e0cda5ffa4262cdc
diff --git a/manifest.uuid b/manifest.uuid
index c2cbc0a906..984372aeb6 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ad38d2c4f073705c02c7b38675e8ae86fe4a794d54eb796e7ed51a905824d5f5
\ No newline at end of file
+3ec7371161bd617e40328aa015b09acc2b37b0b5d269a87050a0c57163f92801
\ No newline at end of file

From 3c5d285762ee7293bd262a2e7f19c350ff275660 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 21 Dec 2017 14:46:11 +0000
Subject: [PATCH 162/488] Add the SQLITE_DBCONFIG_MAX preprocessor macro which
 will always be equal to the largest SQLITE_DECONFIG value.

FossilOrigin-Name: 95d4103348d14f9d4c3fee30bd1d882f717b047b2b92f8fef52bec4878281907
---
 manifest        | 13 ++++++-------
 manifest.uuid   |  2 +-
 src/sqlite.h.in |  3 ++-
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index df2d29e242..9e1163e1b8 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\sexperimental\s".expert"\scommand\sto\sthe\ssqlite3.exe\sshell.
-D 2017-12-21T02:17:02.540
+C Add\sthe\sSQLITE_DBCONFIG_MAX\spreprocessor\smacro\swhich\swill\salways\sbe\sequal\sto\nthe\slargest\sSQLITE_DECONFIG\svalue.
+D 2017-12-21T14:46:11.149
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -480,7 +480,7 @@ F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
 F src/shell.c.in 87a048fabcf0080a78bcdd01e57933369951f7fa7d628f04bad48e900c1899a1
-F src/sqlite.h.in 95afbec6e623843d702e727765efc833586fecc688867f41f87be7db3ff1fa62
+F src/sqlite.h.in e00a8c4910e64bd72c1808e9dff80a7a6323f1f036cdd3fdf927c8eae6bb33d2
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
 F src/sqliteInt.h 4d6580ab911e9bc36447ef05bf4d2aab929dde161ac117c7de6220eef06d6175
@@ -1687,8 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3ec7371161bd617e40328aa015b09acc2b37b0b5d269a87050a0c57163f92801 51068dbaeaef13bb80af8126b8c4f3a454dee63de5127d706db50bf789533e60
-R 4f8ab13364d6bba7088ea515e80d086d
-T +closed 51068dbaeaef13bb80af8126b8c4f3a454dee63de5127d706db50bf789533e60
+P 0821bae7afefed98102c81104b4a477e81816bb1f43353c80865411771e3c5a7
+R 0d03176c5debc93f715c8f8149153033
 U drh
-Z b5148bb6ba613d796ac4de4909e11d71
+Z b349f9c48e8b618dcebb916f768288b4
diff --git a/manifest.uuid b/manifest.uuid
index c17fb2dc41..f0979207cd 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0821bae7afefed98102c81104b4a477e81816bb1f43353c80865411771e3c5a7
\ No newline at end of file
+95d4103348d14f9d4c3fee30bd1d882f717b047b2b92f8fef52bec4878281907
\ No newline at end of file
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index f4d44b52ff..671d3dcebb 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -2091,6 +2091,7 @@ struct sqlite3_mem_methods {
 #define SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE      1006 /* int int* */
 #define SQLITE_DBCONFIG_ENABLE_QPSG           1007 /* int int* */
 #define SQLITE_DBCONFIG_FULL_EQP              1008 /* int int* */
+#define SQLITE_DBCONFIG_MAX                   1008 /* Largest DBCONFIG */
 
 /*
 ** CAPI3REF: Enable Or Disable Extended Result Codes
@@ -7041,7 +7042,7 @@ int sqlite3_test_control(int op, ...);
 #define SQLITE_TESTCTRL_ISINIT                  23
 #define SQLITE_TESTCTRL_SORTER_MMAP             24
 #define SQLITE_TESTCTRL_IMPOSTER                25
-#define SQLITE_TESTCTRL_LAST                    25
+#define SQLITE_TESTCTRL_LAST                    25  /* Largest TESTCTRL */
 
 /*
 ** CAPI3REF: SQLite Runtime Status

From 36e31c6915763de55f7e2968a20084fc7252fcd9 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 21 Dec 2017 18:23:26 +0000
Subject: [PATCH 163/488] Change the name of SQLITE_DBCONFIG_FULL_EQP to be
 SQLITE_DBCONFIG_TRIGGER_EQP (which we can do without breaking compatibility
 because the former name has not yet appeared in an official release) and
 streamline its implementation.

FossilOrigin-Name: fffc7685d19f78ec322a4e834ad727af20a17e2e1c35680e4b1c4162c4786f60
---
 ext/expert/sqlite3expert.c |  2 +-
 manifest                   | 20 ++++++++++----------
 manifest.uuid              |  2 +-
 src/main.c                 | 16 +---------------
 src/sqlite.h.in            |  4 ++--
 src/sqliteInt.h            |  5 +++--
 src/vdbeaux.c              |  6 +++---
 7 files changed, 21 insertions(+), 34 deletions(-)

diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index 8627d196f0..b8b19c66a6 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -1734,7 +1734,7 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){
   if( rc==SQLITE_OK ){
     rc = sqlite3_open(":memory:", &pNew->dbm);
     if( rc==SQLITE_OK ){
-      sqlite3_db_config(pNew->dbm, SQLITE_DBCONFIG_FULL_EQP, 1, (int*)0);
+      sqlite3_db_config(pNew->dbm, SQLITE_DBCONFIG_TRIGGER_EQP, 1, (int*)0);
     }
   }
   
diff --git a/manifest b/manifest
index 9e1163e1b8..361a345ff6 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\sSQLITE_DBCONFIG_MAX\spreprocessor\smacro\swhich\swill\salways\sbe\sequal\sto\nthe\slargest\sSQLITE_DECONFIG\svalue.
-D 2017-12-21T14:46:11.149
+C Change\sthe\sname\sof\sSQLITE_DBCONFIG_FULL_EQP\sto\sbe\sSQLITE_DBCONFIG_TRIGGER_EQP\n(which\swe\scan\sdo\swithout\sbreaking\scompatibility\sbecause\sthe\sformer\sname\shas\nnot\syet\sappeared\sin\san\sofficial\srelease)\sand\sstreamline\sits\simplementation.
+D 2017-12-21T18:23:26.999
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -43,7 +43,7 @@ F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3
 F ext/expert/expert.c 4791c5e064aea81b2b829fa95228b22283380ee370ea88a1e580103b75516ebf
 F ext/expert/expert1.test 0c71a3453ce3a0b4dbe952713aec0ae8d416dd846820dd027b08f305f5278b30
-F ext/expert/sqlite3expert.c 6ed4e84a06d1a29b2cf3009c0266573b88602d098055caa46c467154a64e0959
+F ext/expert/sqlite3expert.c be6452d15a85a59dfa503c3a890d72bd924ac0c39c0af6075b6437a38d7a64da
 F ext/expert/sqlite3expert.h af6354f8ee5c9e025024e63fec3bd640a802afcc3099a44d804752cf0791d811
 F ext/expert/test_expert.c 85f5c743a899063fa48296d21de2f32c26d09a21c8582b2a0bc482e8de183e7a
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
@@ -443,7 +443,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c cb67cc56ef2ddd13e6944b2c0dd08a920bcd9503230adef8b9928d338097c722
 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e
 F src/loadext.c 20865b183bb8a3723d59cf1efffc3c50217eb452c1021d077b908c94da26b0b2
-F src/main.c 903c4e3eb189d4ce94748dc7f442f9136177cb2eedb27185f83a525edb0c4530
+F src/main.c 7ce55fa3c0bf669944de309ebab1655ed06ec67869adb0372c7a1062e461c448
 F src/malloc.c a02c9e69bc76bee0f639416b947a946412890b606301454727feadcb313536d6
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -480,10 +480,10 @@ F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
 F src/shell.c.in 87a048fabcf0080a78bcdd01e57933369951f7fa7d628f04bad48e900c1899a1
-F src/sqlite.h.in e00a8c4910e64bd72c1808e9dff80a7a6323f1f036cdd3fdf927c8eae6bb33d2
+F src/sqlite.h.in 53611410ade98671c0a95b209273e52dfd0ec1268ebc9213622d221d83b50bfe
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
-F src/sqliteInt.h 4d6580ab911e9bc36447ef05bf4d2aab929dde161ac117c7de6220eef06d6175
+F src/sqliteInt.h 392de8f6d2e3d6c88cd1a998dc2fa90bbdb146cb440cc7a1abc62d478c69df12
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -551,7 +551,7 @@ F src/vdbe.c 3393b508d9ad084ffce232a7c53e375ef5ac99b50b685c5131fcdfce97a9d534
 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97
 F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9
 F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1
-F src/vdbeaux.c cf474d29372a30d0db84b0a9a74ab07117c9b831e1585a0fa335e2b445f4a95f
+F src/vdbeaux.c 7ae48b180e5dd5d282e6752d155f1ab7929196d8e6577b82742044188152ca85
 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191
 F src/vdbemem.c 8478f7fb1948bf8fdeec7c2cb59ea58155c31258b9cd43c56d485e03ed40bd07
 F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0821bae7afefed98102c81104b4a477e81816bb1f43353c80865411771e3c5a7
-R 0d03176c5debc93f715c8f8149153033
+P 95d4103348d14f9d4c3fee30bd1d882f717b047b2b92f8fef52bec4878281907
+R 8b3e23acfaff32d9421a31731659429a
 U drh
-Z b349f9c48e8b618dcebb916f768288b4
+Z aa5f57e4aaa4de538c8fad51c00ee6a2
diff --git a/manifest.uuid b/manifest.uuid
index f0979207cd..bd309e6fc0 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-95d4103348d14f9d4c3fee30bd1d882f717b047b2b92f8fef52bec4878281907
\ No newline at end of file
+fffc7685d19f78ec322a4e834ad727af20a17e2e1c35680e4b1c4162c4786f60
\ No newline at end of file
diff --git a/src/main.c b/src/main.c
index 52d3be9531..32ce1889f6 100644
--- a/src/main.c
+++ b/src/main.c
@@ -806,21 +806,6 @@ int sqlite3_db_config(sqlite3 *db, int op, ...){
       rc = setupLookaside(db, pBuf, sz, cnt);
       break;
     }
-    case SQLITE_DBCONFIG_FULL_EQP: {
-      int onoff = va_arg(ap, int);
-      int *pRes = va_arg(ap, int*);
-      if( onoff>0 ){
-        db->bFullEQP = 1;
-      }else if( onoff==0 ){
-        db->bFullEQP = 0;
-      }
-      sqlite3ExpirePreparedStatements(db);
-      if( pRes ){
-        *pRes = db->bFullEQP;
-      }
-      rc = SQLITE_OK;
-      break;
-    }
     default: {
       static const struct {
         int op;      /* The opcode */
@@ -832,6 +817,7 @@ int sqlite3_db_config(sqlite3 *db, int op, ...){
         { SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, SQLITE_LoadExtension  },
         { SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE,      SQLITE_NoCkptOnClose  },
         { SQLITE_DBCONFIG_ENABLE_QPSG,           SQLITE_EnableQPSG     },
+        { SQLITE_DBCONFIG_TRIGGER_EQP,           SQLITE_TriggerEQP     },
       };
       unsigned int i;
       rc = SQLITE_ERROR; /* IMP: R-42790-23372 */
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 671d3dcebb..67c90589cc 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -2070,7 +2070,7 @@ struct sqlite3_mem_methods {
 ** the QPSG active, SQLite will always use the same query plan in the field as
 ** was used during testing in the lab.
 ** </dd>
-** <dt>SQLITE_DBCONFIG_FULL_EQP</dt>
+** <dt>SQLITE_DBCONFIG_TRIGGER_EQP</dt>
 ** <dd> By default, the output of EXPLAIN QUERY PLAN commands does not 
 ** include output for any operations performed by trigger programs. This
 ** option is used to set or clear (the default) a flag that governs this
@@ -2090,7 +2090,7 @@ struct sqlite3_mem_methods {
 #define SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION 1005 /* int int* */
 #define SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE      1006 /* int int* */
 #define SQLITE_DBCONFIG_ENABLE_QPSG           1007 /* int int* */
-#define SQLITE_DBCONFIG_FULL_EQP              1008 /* int int* */
+#define SQLITE_DBCONFIG_TRIGGER_EQP           1008 /* int int* */
 #define SQLITE_DBCONFIG_MAX                   1008 /* Largest DBCONFIG */
 
 /*
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index efe823a654..8d4aea723b 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1356,7 +1356,6 @@ struct sqlite3 {
   u8 mTrace;                    /* zero or more SQLITE_TRACE flags */
   u8 skipBtreeMutex;            /* True if no shared-cache backends */
   u8 nSqlExec;                  /* Number of pending OP_SqlExec opcodes */
-  u8 bFullEQP;                  /* Include triggers in EQP output */
   int nextPagesize;             /* Pagesize after VACUUM if >0 */
   u32 magic;                    /* Magic number for detect library misuse */
   int nChange;                  /* Value returned by sqlite3_changes() */
@@ -1497,7 +1496,9 @@ struct sqlite3 {
 #define SQLITE_QueryOnly      0x00100000  /* Disable database changes */
 #define SQLITE_CellSizeCk     0x00200000  /* Check btree cell sizes on load */
 #define SQLITE_Fts3Tokenizer  0x00400000  /* Enable fts3_tokenizer(2) */
-#define SQLITE_EnableQPSG     0x00800000  /* Query Planner Stability Guarantee */
+#define SQLITE_EnableQPSG     0x00800000  /* Query Planner Stability Guarantee*/
+#define SQLITE_TriggerEQP     0x01000000  /* Show trigger EXPLAIN QUERY PLAN */
+
 /* Flags used only if debugging */
 #ifdef SQLITE_DEBUG
 #define SQLITE_SqlTrace       0x08000000  /* Debug print SQL as it executes */
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index 1c354e0be1..bc4bbda8e4 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -1639,7 +1639,7 @@ int sqlite3VdbeList(
   int i;                               /* Loop counter */
   int rc = SQLITE_OK;                  /* Return code */
   Mem *pMem = &p->aMem[1];             /* First Mem of result set */
-  int bFull = (p->explain==1 || db->bFullEQP);
+  int bListSubprogs = (p->explain==1 || (db->flags & SQLITE_TriggerEQP)!=0);
   Op *pOp = 0;
 
   assert( p->explain );
@@ -1668,7 +1668,7 @@ int sqlite3VdbeList(
   ** encountered, but p->pc will eventually catch up to nRow.
   */
   nRow = p->nOp;
-  if( bFull ){
+  if( bListSubprogs ){
     /* The first 8 memory cells are used for the result set.  So we will
     ** commandeer the 9th cell to use as storage for an array of pointers
     ** to trigger subprograms.  The VDBE is guaranteed to have at least 9
@@ -1713,7 +1713,7 @@ int sqlite3VdbeList(
     ** kept in p->aMem[9].z to hold the new program - assuming this subprogram
     ** has not already been seen.
     */
-    if( bFull && pOp->p4type==P4_SUBPROGRAM ){
+    if( bListSubprogs && pOp->p4type==P4_SUBPROGRAM ){
       int nByte = (nSub+1)*sizeof(SubProgram*);
       int j;
       for(j=0; j<nSub; j++){

From 2e1ea57f8bd08f8abdbb976b0772dbd995511718 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 21 Dec 2017 18:55:24 +0000
Subject: [PATCH 164/488] Add the ".expert" command to the shell tool's ".help"
 output.

FossilOrigin-Name: fc6193af5d216b4066fbc47e75a7d0538fd5fda40b94ee15e2ff4037ea89221b
---
 manifest       | 14 +++++++-------
 manifest.uuid  |  2 +-
 src/shell.c.in |  1 +
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 361a345ff6..b9641d26a6 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Change\sthe\sname\sof\sSQLITE_DBCONFIG_FULL_EQP\sto\sbe\sSQLITE_DBCONFIG_TRIGGER_EQP\n(which\swe\scan\sdo\swithout\sbreaking\scompatibility\sbecause\sthe\sformer\sname\shas\nnot\syet\sappeared\sin\san\sofficial\srelease)\sand\sstreamline\sits\simplementation.
-D 2017-12-21T18:23:26.999
+C Add\sthe\s".expert"\scommand\sto\sthe\sshell\stool's\s".help"\soutput.
+D 2017-12-21T18:55:24.380
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -479,7 +479,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
-F src/shell.c.in 87a048fabcf0080a78bcdd01e57933369951f7fa7d628f04bad48e900c1899a1
+F src/shell.c.in ef343d708f32e43bc5abff8a4a769e026252b77ff37583cbd7ad30a9cc169f26
 F src/sqlite.h.in 53611410ade98671c0a95b209273e52dfd0ec1268ebc9213622d221d83b50bfe
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 95d4103348d14f9d4c3fee30bd1d882f717b047b2b92f8fef52bec4878281907
-R 8b3e23acfaff32d9421a31731659429a
-U drh
-Z aa5f57e4aaa4de538c8fad51c00ee6a2
+P fffc7685d19f78ec322a4e834ad727af20a17e2e1c35680e4b1c4162c4786f60
+R d5bcde1015d19d1cdfe7f85d9291f783
+U dan
+Z f2c8e8756f16d691313b2a4f36b60cc0
diff --git a/manifest.uuid b/manifest.uuid
index bd309e6fc0..247556897e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-fffc7685d19f78ec322a4e834ad727af20a17e2e1c35680e4b1c4162c4786f60
\ No newline at end of file
+fc6193af5d216b4066fbc47e75a7d0538fd5fda40b94ee15e2ff4037ea89221b
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index cdd824835a..261a1971e7 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -2779,6 +2779,7 @@ static char zHelp[] =
   ".echo on|off           Turn command echo on or off\n"
   ".eqp on|off|full       Enable or disable automatic EXPLAIN QUERY PLAN\n"
   ".exit                  Exit this program\n"
+  ".expert                EXPERIMENTAL. Suggest indexes for specified queries\n"
 /* Because explain mode comes on automatically now, the ".explain" mode
 ** is removed from the help screen.  It is still supported for legacy, however */
 /*".explain ?on|off|auto? Turn EXPLAIN output mode on or off or to automatic\n"*/

From ada7045eabf23116f9c6b422fff7ca604198ce58 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 21 Dec 2017 21:02:27 +0000
Subject: [PATCH 165/488] Add the ".eqp trigger" option to the ".eqp" command
 in the command-line shell.  Implemented using the new
 SQLITE_DBCONFIG_TRIGGER_EQP control.

FossilOrigin-Name: 2c51644a12a638d89e4f7cc3fd561236ce424f2d4e1db31f1e8388f77add02b8
---
 manifest       | 14 +++++++-------
 manifest.uuid  |  2 +-
 src/shell.c.in | 27 +++++++++++++++++++++------
 3 files changed, 29 insertions(+), 14 deletions(-)

diff --git a/manifest b/manifest
index b9641d26a6..62b8ff3da8 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\s".expert"\scommand\sto\sthe\sshell\stool's\s".help"\soutput.
-D 2017-12-21T18:55:24.380
+C Add\sthe\s".eqp\strigger"\soption\sto\sthe\s".eqp"\scommand\sin\sthe\scommand-line\nshell.\s\sImplemented\susing\sthe\snew\sSQLITE_DBCONFIG_TRIGGER_EQP\scontrol.
+D 2017-12-21T21:02:27.410
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -479,7 +479,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
-F src/shell.c.in ef343d708f32e43bc5abff8a4a769e026252b77ff37583cbd7ad30a9cc169f26
+F src/shell.c.in 339169a3d1307b5566ebe9ce15832d03439206106724c78cc3d9125a7b851795
 F src/sqlite.h.in 53611410ade98671c0a95b209273e52dfd0ec1268ebc9213622d221d83b50bfe
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P fffc7685d19f78ec322a4e834ad727af20a17e2e1c35680e4b1c4162c4786f60
-R d5bcde1015d19d1cdfe7f85d9291f783
-U dan
-Z f2c8e8756f16d691313b2a4f36b60cc0
+P fc6193af5d216b4066fbc47e75a7d0538fd5fda40b94ee15e2ff4037ea89221b
+R 9e177759a0c3bcdc85d791f3df26085a
+U drh
+Z 1ea40405a7cdaae369b8eb1d1f8aee13
diff --git a/manifest.uuid b/manifest.uuid
index 247556897e..48249eb7dd 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-fc6193af5d216b4066fbc47e75a7d0538fd5fda40b94ee15e2ff4037ea89221b
\ No newline at end of file
+2c51644a12a638d89e4f7cc3fd561236ce424f2d4e1db31f1e8388f77add02b8
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 261a1971e7..13b1fcde39 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -877,6 +877,13 @@ struct ShellState {
   ExpertInfo expert;        /* Valid if previous command was ".expert OPT..." */
 };
 
+/* Allowed values for ShellState.autoEQP
+*/
+#define AUTOEQP_off      0
+#define AUTOEQP_on       1
+#define AUTOEQP_trigger  2
+#define AUTOEQP_full     3
+
 /*
 ** These are the allowed shellFlgs values
 */
@@ -2395,7 +2402,12 @@ static int shell_exec(
       if( pArg && pArg->autoEQP && sqlite3_strlike("EXPLAIN%",zStmtSql,0)!=0 ){
         sqlite3_stmt *pExplain;
         char *zEQP;
+        int triggerEQP = 0;
         disable_debug_trace_modes();
+        sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, -1, &triggerEQP);
+        if( pArg->autoEQP>=AUTOEQP_trigger ){
+          sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, 1, 0);
+        }
         zEQP = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", zStmtSql);
         rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0);
         if( rc==SQLITE_OK ){
@@ -2408,7 +2420,7 @@ static int shell_exec(
         }
         sqlite3_finalize(pExplain);
         sqlite3_free(zEQP);
-        if( pArg->autoEQP>=2 ){
+        if( pArg->autoEQP>=AUTOEQP_full ){
           /* Also do an EXPLAIN for ".eqp full" mode */
           zEQP = sqlite3_mprintf("EXPLAIN %s", zStmtSql);
           rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0);
@@ -2421,6 +2433,7 @@ static int shell_exec(
           sqlite3_finalize(pExplain);
           sqlite3_free(zEQP);
         }
+        sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, triggerEQP, 0);
         restore_debug_trace_modes();
       }
 
@@ -4547,12 +4560,14 @@ static int do_meta_command(char *zLine, ShellState *p){
   if( c=='e' && strncmp(azArg[0], "eqp", n)==0 ){
     if( nArg==2 ){
       if( strcmp(azArg[1],"full")==0 ){
-        p->autoEQP = 2;
+        p->autoEQP = AUTOEQP_full;
+      }else if( strcmp(azArg[1],"trigger")==0 ){
+        p->autoEQP = AUTOEQP_trigger;
       }else{
         p->autoEQP = booleanValue(azArg[1]);
       }
     }else{
-      raw_printf(stderr, "Usage: .eqp on|off|full\n");
+      raw_printf(stderr, "Usage: .eqp off|on|trigger|full\n");
       rc = 1;
     }
   }else
@@ -5905,7 +5920,7 @@ static int do_meta_command(char *zLine, ShellState *p){
   }else
 
   if( c=='s' && strncmp(azArg[0], "show", n)==0 ){
-    static const char *azBool[] = { "off", "on", "full", "unk" };
+    static const char *azBool[] = { "off", "on", "trigger", "full"};
     int i;
     if( nArg!=1 ){
       raw_printf(stderr, "Usage: .show\n");
@@ -7057,9 +7072,9 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
     }else if( strcmp(z,"-echo")==0 ){
       ShellSetFlag(&data, SHFLG_Echo);
     }else if( strcmp(z,"-eqp")==0 ){
-      data.autoEQP = 1;
+      data.autoEQP = AUTOEQP_on;
     }else if( strcmp(z,"-eqpfull")==0 ){
-      data.autoEQP = 2;
+      data.autoEQP = AUTOEQP_full;
     }else if( strcmp(z,"-stats")==0 ){
       data.statsOn = 1;
     }else if( strcmp(z,"-scanstats")==0 ){

From ef7231b8653d7e8e5d29d2f19805361099f491c6 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 21 Dec 2017 21:41:13 +0000
Subject: [PATCH 166/488] Fix SQLITE_DBCONFIG_TRIGGER_EQP so that it works even
 if SQLITE_DEBUG is not defined.

FossilOrigin-Name: afbbfff72002089fa73b5473cc98360df14288d489e93e667332d6e884ef60da
---
 manifest        | 12 ++++++------
 manifest.uuid   |  2 +-
 src/wherecode.c |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 62b8ff3da8..990fbed2ba 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\s".eqp\strigger"\soption\sto\sthe\s".eqp"\scommand\sin\sthe\scommand-line\nshell.\s\sImplemented\susing\sthe\snew\sSQLITE_DBCONFIG_TRIGGER_EQP\scontrol.
-D 2017-12-21T21:02:27.410
+C Fix\sSQLITE_DBCONFIG_TRIGGER_EQP\sso\sthat\sit\sworks\seven\sif\sSQLITE_DEBUG\sis\nnot\sdefined.
+D 2017-12-21T21:41:13.558
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -563,7 +563,7 @@ F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f
 F src/where.c 0cdfda28708fe75699df2fcdb44d067dc94e7b1f83e3a622dca96a27aaaa2e42
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
-F src/wherecode.c ff2f079097a3bdce6ebabfde1419fba448c9ce5feb7cb964e8bfa2a4e27274ef
+F src/wherecode.c af1e79154aaa88cd802d6f2e5b945f67eaca7c958d1525fbf8ee19d5bd7b9020
 F src/whereexpr.c 427ea8e96ec24f2a7814c67b8024ad664a9c7656264c4566c34743cb23186e46
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P fc6193af5d216b4066fbc47e75a7d0538fd5fda40b94ee15e2ff4037ea89221b
-R 9e177759a0c3bcdc85d791f3df26085a
+P 2c51644a12a638d89e4f7cc3fd561236ce424f2d4e1db31f1e8388f77add02b8
+R f003797eb1e0446ed94e2b0c9d22122e
 U drh
-Z 1ea40405a7cdaae369b8eb1d1f8aee13
+Z c4f59d8ddc7f0db54ec7405d6ea2d997
diff --git a/manifest.uuid b/manifest.uuid
index 48249eb7dd..194b1d49ef 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2c51644a12a638d89e4f7cc3fd561236ce424f2d4e1db31f1e8388f77add02b8
\ No newline at end of file
+afbbfff72002089fa73b5473cc98360df14288d489e93e667332d6e884ef60da
\ No newline at end of file
diff --git a/src/wherecode.c b/src/wherecode.c
index 3c166a1210..32dd2048bf 100644
--- a/src/wherecode.c
+++ b/src/wherecode.c
@@ -128,7 +128,7 @@ int sqlite3WhereExplainOneScan(
 ){
   int ret = 0;
 #if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS)
-  if( pParse->explain==2 )
+  if( sqlite3ParseToplevel(pParse)->explain==2 )
 #endif
   {
     struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom];

From efc88d020cc6872923616d4c22cc98488f117da2 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 22 Dec 2017 00:52:50 +0000
Subject: [PATCH 167/488] Modify the new sqlite3_vtab_collation() interface so
 that it takes a pointer to the sqlite3_index_info object passed into
 xBestIndex rather than an sqlite3 connection pointer, which the xBestIndex
 method might not have access to.

FossilOrigin-Name: 5c1fe6666019147a26480b5db1bf2f474a5d072c234c736f16ed5d2a9a040b3f
---
 ext/expert/sqlite3expert.c |  3 +-
 manifest                   | 18 +++++------
 manifest.uuid              |  2 +-
 src/sqlite.h.in            | 11 +++----
 src/sqliteInt.h            |  3 +-
 src/where.c                | 63 +++++++++++++++++++-------------------
 6 files changed, 48 insertions(+), 52 deletions(-)

diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index b8b19c66a6..e4060de681 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -434,7 +434,6 @@ static int expertDisconnect(sqlite3_vtab *pVtab){
 
 static int expertBestIndex(sqlite3_vtab *pVtab, sqlite3_index_info *pIdxInfo){
   ExpertVtab *p = (ExpertVtab*)pVtab;
-  sqlite3 *dbv = p->pExpert->dbv;
   int rc = SQLITE_OK;
   int n = 0;
   IdxScan *pScan;
@@ -461,7 +460,7 @@ static int expertBestIndex(sqlite3_vtab *pVtab, sqlite3_index_info *pIdxInfo){
        && (pCons->op & opmask) 
       ){
         IdxConstraint *pNew;
-        const char *zColl = sqlite3_vtab_collation(dbv, i);
+        const char *zColl = sqlite3_vtab_collation(pIdxInfo, i);
         pNew = idxNewConstraint(&rc, zColl);
         if( pNew ){
           pNew->iCol = pCons->iColumn;
diff --git a/manifest b/manifest
index 990fbed2ba..ba93b34166 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sSQLITE_DBCONFIG_TRIGGER_EQP\sso\sthat\sit\sworks\seven\sif\sSQLITE_DEBUG\sis\nnot\sdefined.
-D 2017-12-21T21:41:13.558
+C Modify\sthe\snew\ssqlite3_vtab_collation()\sinterface\sso\sthat\sit\stakes\sa\npointer\sto\sthe\ssqlite3_index_info\sobject\spassed\sinto\sxBestIndex\srather\sthan\nan\ssqlite3\sconnection\spointer,\swhich\sthe\sxBestIndex\smethod\smight\snot\shave\naccess\sto.
+D 2017-12-22T00:52:50.426
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -43,7 +43,7 @@ F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3
 F ext/expert/expert.c 4791c5e064aea81b2b829fa95228b22283380ee370ea88a1e580103b75516ebf
 F ext/expert/expert1.test 0c71a3453ce3a0b4dbe952713aec0ae8d416dd846820dd027b08f305f5278b30
-F ext/expert/sqlite3expert.c be6452d15a85a59dfa503c3a890d72bd924ac0c39c0af6075b6437a38d7a64da
+F ext/expert/sqlite3expert.c 252f3129f12a0e9df094a14711db98265c9c6d7afa033ec906d94e920f5c7ba7
 F ext/expert/sqlite3expert.h af6354f8ee5c9e025024e63fec3bd640a802afcc3099a44d804752cf0791d811
 F ext/expert/test_expert.c 85f5c743a899063fa48296d21de2f32c26d09a21c8582b2a0bc482e8de183e7a
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
@@ -480,10 +480,10 @@ F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
 F src/shell.c.in 339169a3d1307b5566ebe9ce15832d03439206106724c78cc3d9125a7b851795
-F src/sqlite.h.in 53611410ade98671c0a95b209273e52dfd0ec1268ebc9213622d221d83b50bfe
+F src/sqlite.h.in 2126192945019d4cdce335cb236b440a05ec75c93e4cd94c9c6d6e7fcc654cc4
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
-F src/sqliteInt.h 392de8f6d2e3d6c88cd1a998dc2fa90bbdb146cb440cc7a1abc62d478c69df12
+F src/sqliteInt.h 003b78433baae4e5c997f99f2f9cf98d90754f256baeacb32f8189569a48251f
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -561,7 +561,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 5a3f464edd64596f601683ed321d12e6fd93c5fb9afdfb3653d6ffd0fee9c48f
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f
-F src/where.c 0cdfda28708fe75699df2fcdb44d067dc94e7b1f83e3a622dca96a27aaaa2e42
+F src/where.c 5876c9100b622f7b9e5ee7f579b8b6a71ae5ba627724cea4546d9114c32b3cb5
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
 F src/wherecode.c af1e79154aaa88cd802d6f2e5b945f67eaca7c958d1525fbf8ee19d5bd7b9020
 F src/whereexpr.c 427ea8e96ec24f2a7814c67b8024ad664a9c7656264c4566c34743cb23186e46
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2c51644a12a638d89e4f7cc3fd561236ce424f2d4e1db31f1e8388f77add02b8
-R f003797eb1e0446ed94e2b0c9d22122e
+P afbbfff72002089fa73b5473cc98360df14288d489e93e667332d6e884ef60da
+R b5eb5cd81beccd0fb58b9d1b00a03e31
 U drh
-Z c4f59d8ddc7f0db54ec7405d6ea2d997
+Z 10e32a088c82fd051b685f8b12f5082b
diff --git a/manifest.uuid b/manifest.uuid
index 194b1d49ef..4ed62ec7c2 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-afbbfff72002089fa73b5473cc98360df14288d489e93e667332d6e884ef60da
\ No newline at end of file
+5c1fe6666019147a26480b5db1bf2f474a5d072c234c736f16ed5d2a9a040b3f
\ No newline at end of file
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 67c90589cc..f161eea6f2 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -8302,15 +8302,14 @@ int sqlite3_vtab_on_conflict(sqlite3 *);
 ** This function may only be called from within a call to the [xBestIndex]
 ** method of a [virtual table implementation]. 
 **
-** The first argument must be the database handle with which the virtual 
-** table is associated (the one passed to the [xConnect] or [xCreate] method 
-** to create the sqlite3_vtab object. The second argument must be an index
-** into the aConstraint[] array belonging to the sqlite3_index_info structure
-** passed to xBestIndex. This function returns a pointer to a buffer 
+** The first argument must be the sqlite3_index_info object that is the
+** first parameter to the xBestIndex() method. The second argument must be
+** an index into the aConstraint[] array belonging to the sqlite3_index_info
+** structure passed to xBestIndex. This function returns a pointer to a buffer 
 ** containing the name of the collation sequence for the corresponding
 ** constraint.
 */
-SQLITE_EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3*, int);
+SQLITE_EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3_index_info*,int);
 
 /*
 ** CAPI3REF: Conflict resolution modes
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 8d4aea723b..b0c4711b03 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1420,8 +1420,7 @@ struct sqlite3 {
   Hash aModule;                 /* populated by sqlite3_create_module() */
   VtabCtx *pVtabCtx;            /* Context for active vtab connect/create */
   VTable **aVTrans;             /* Virtual tables with open transactions */
-  VTable *pDisconnect;    /* Disconnect these in next sqlite3_prepare() */
-  void *pBestIndexCtx;          /* For sqlite3_vtab_collation() */
+  VTable *pDisconnect;          /* Disconnect these in next sqlite3_prepare() */
 #endif
   Hash aFunc;                   /* Hash table of connection functions */
   Hash aCollSeq;                /* All collating sequences */
diff --git a/src/where.c b/src/where.c
index f0ee78a1da..d3aec354ba 100644
--- a/src/where.c
+++ b/src/where.c
@@ -19,6 +19,21 @@
 #include "sqliteInt.h"
 #include "whereInt.h"
 
+/*
+** Extra information appended to the end of sqlite3_index_info but not
+** visible to the xBestIndex function, at least not directly.  The
+** sqlite3_vtab_collation() interface knows how to reach it, however.
+**
+** This object is not an API and can be changed from one release to the
+** next.  As long as allocateIndexInfo() and sqlite3_vtab_collation()
+** agree on the structure, all will be well.
+*/
+typedef struct HiddenIndexInfo HiddenIndexInfo;
+struct HiddenIndexInfo {
+  WhereClause *pWC;   /* The Where clause being analyzed */
+  Parse *pParse;      /* The parsing context */
+};
+
 /* Forward declaration of methods */
 static int whereLoopResize(sqlite3*, WhereLoop*, int);
 
@@ -841,11 +856,11 @@ end_auto_index_create:
 ** by passing the pointer returned by this function to sqlite3_free().
 */
 static sqlite3_index_info *allocateIndexInfo(
-  Parse *pParse,
-  WhereClause *pWC,
+  Parse *pParse,                  /* The parsing context */
+  WhereClause *pWC,               /* The WHERE clause being analyzed */
   Bitmask mUnusable,              /* Ignore terms with these prereqs */
-  struct SrcList_item *pSrc,
-  ExprList *pOrderBy,
+  struct SrcList_item *pSrc,      /* The FROM clause term that is the vtab */
+  ExprList *pOrderBy,             /* The ORDER BY clause */
   u16 *pmNoOmit                   /* Mask of terms not to omit */
 ){
   int i, j;
@@ -853,6 +868,7 @@ static sqlite3_index_info *allocateIndexInfo(
   struct sqlite3_index_constraint *pIdxCons;
   struct sqlite3_index_orderby *pIdxOrderBy;
   struct sqlite3_index_constraint_usage *pUsage;
+  struct HiddenIndexInfo *pHidden;
   WhereTerm *pTerm;
   int nOrderBy;
   sqlite3_index_info *pIdxInfo;
@@ -894,7 +910,7 @@ static sqlite3_index_info *allocateIndexInfo(
   */
   pIdxInfo = sqlite3DbMallocZero(pParse->db, sizeof(*pIdxInfo)
                            + (sizeof(*pIdxCons) + sizeof(*pUsage))*nTerm
-                           + sizeof(*pIdxOrderBy)*nOrderBy );
+                           + sizeof(*pIdxOrderBy)*nOrderBy + sizeof(*pHidden) );
   if( pIdxInfo==0 ){
     sqlite3ErrorMsg(pParse, "out of memory");
     return 0;
@@ -905,7 +921,8 @@ static sqlite3_index_info *allocateIndexInfo(
   ** changing them.  We have to do some funky casting in order to
   ** initialize those fields.
   */
-  pIdxCons = (struct sqlite3_index_constraint*)&pIdxInfo[1];
+  pHidden = (struct HiddenIndexInfo*)&pIdxInfo[1];
+  pIdxCons = (struct sqlite3_index_constraint*)&pHidden[1];
   pIdxOrderBy = (struct sqlite3_index_orderby*)&pIdxCons[nTerm];
   pUsage = (struct sqlite3_index_constraint_usage*)&pIdxOrderBy[nOrderBy];
   *(int*)&pIdxInfo->nConstraint = nTerm;
@@ -915,6 +932,8 @@ static sqlite3_index_info *allocateIndexInfo(
   *(struct sqlite3_index_constraint_usage**)&pIdxInfo->aConstraintUsage =
                                                                    pUsage;
 
+  pHidden->pWC = pWC;
+  pHidden->pParse = pParse;
   for(i=j=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){
     u16 op;
     if( pTerm->leftCursor != pSrc->iCursor ) continue;
@@ -3138,17 +3157,6 @@ static int whereLoopAddVirtualOne(
   return rc;
 }
 
-
-/*
-** Context object used to pass information from whereLoopAddVirtual()
-** to sqlite3_vtab_collation().
-*/
-struct BestIndexCtx {
-  WhereClause *pWC;
-  sqlite3_index_info *pIdxInfo;
-  Parse *pParse;
-};
-
 /*
 ** If this function is invoked from within an xBestIndex() callback, it
 ** returns a pointer to a buffer containing the name of the collation
@@ -3156,15 +3164,15 @@ struct BestIndexCtx {
 ** array. Or, if iCons is out of range or there is no active xBestIndex
 ** call, return NULL.
 */
-const char *sqlite3_vtab_collation(sqlite3 *db, int iCons){
-  struct BestIndexCtx *p = (struct BestIndexCtx*)db->pBestIndexCtx;
+const char *sqlite3_vtab_collation(sqlite3_index_info *pIdxInfo, int iCons){
+  HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
   const char *zRet = 0;
-  if( p && iCons>=0 && iCons<p->pIdxInfo->nConstraint ){
+  if( iCons>=0 && iCons<pIdxInfo->nConstraint ){
     CollSeq *pC = 0;
-    int iTerm = p->pIdxInfo->aConstraint[iCons].iTermOffset;
-    Expr *pX = p->pWC->a[iTerm].pExpr;
+    int iTerm = pIdxInfo->aConstraint[iCons].iTermOffset;
+    Expr *pX = pHidden->pWC->a[iTerm].pExpr;
     if( pX->pLeft ){
-      pC = sqlite3BinaryCompareCollSeq(p->pParse, pX->pLeft, pX->pRight);
+      pC = sqlite3BinaryCompareCollSeq(pHidden->pParse, pX->pLeft, pX->pRight);
     }
     zRet = (pC ? pC->zName : "BINARY");
   }
@@ -3212,8 +3220,6 @@ static int whereLoopAddVirtual(
   WhereLoop *pNew;
   Bitmask mBest;               /* Tables used by best possible plan */
   u16 mNoOmit;
-  struct BestIndexCtx bic;
-  void *pSaved;
 
   assert( (mPrereq & mUnusable)==0 );
   pWInfo = pBuilder->pWInfo;
@@ -3235,12 +3241,6 @@ static int whereLoopAddVirtual(
     return SQLITE_NOMEM_BKPT;
   }
 
-  bic.pWC = pWC;
-  bic.pIdxInfo = p;
-  bic.pParse = pParse;
-  pSaved = pParse->db->pBestIndexCtx;
-  pParse->db->pBestIndexCtx = (void*)&bic;
-
   /* First call xBestIndex() with all constraints usable. */
   WHERETRACE(0x40, ("  VirtualOne: all usable\n"));
   rc = whereLoopAddVirtualOne(pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn);
@@ -3317,7 +3317,6 @@ static int whereLoopAddVirtual(
 
   if( p->needToFreeIdxStr ) sqlite3_free(p->idxStr);
   sqlite3DbFreeNN(pParse->db, p);
-  pParse->db->pBestIndexCtx = pSaved;
   return rc;
 }
 #endif /* SQLITE_OMIT_VIRTUALTABLE */

From 5a699a01ca773e2f4cc90677f94efb3ff01b9a2a Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 22 Dec 2017 19:53:02 +0000
Subject: [PATCH 168/488] In the sqlite3TreeViewExprList() routine, show the
 "AS" alias name for each expression in the list, if it exists.

FossilOrigin-Name: 5efd854fe21470336ba4140294b6c90ef39af32e103b92c664438d7485c50f9a
---
 manifest       | 12 ++++++------
 manifest.uuid  |  2 +-
 src/treeview.c | 12 ++++++++++--
 3 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index ba93b34166..23b0f9e47e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Modify\sthe\snew\ssqlite3_vtab_collation()\sinterface\sso\sthat\sit\stakes\sa\npointer\sto\sthe\ssqlite3_index_info\sobject\spassed\sinto\sxBestIndex\srather\sthan\nan\ssqlite3\sconnection\spointer,\swhich\sthe\sxBestIndex\smethod\smight\snot\shave\naccess\sto.
-D 2017-12-22T00:52:50.426
+C In\sthe\ssqlite3TreeViewExprList()\sroutine,\sshow\sthe\s"AS"\salias\sname\sfor\s\neach\sexpression\sin\sthe\slist,\sif\sit\sexists.
+D 2017-12-22T19:53:02.654
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -541,7 +541,7 @@ F src/test_windirent.h 5d67483a55442e31e1bde0f4a230e6e932ad5906
 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
 F src/tokenize.c 1003d6d90c6783206c711f0a9397656fa5b055209f4d092caa43bb3bf5215db5
-F src/treeview.c 08a83195de8fad3f00542e3c8b3c1eb1222c999817c9e301ffb7f332882b96dd
+F src/treeview.c eae35972ff44f67064de2eaf35f04afe94e7aea3271a8b3bcebb3f954880fec3
 F src/trigger.c 775053eecf6b73062e243404b56f5064446254d5cce17d8704d5cdffd72a546a
 F src/update.c 961bd1265d4d1e5cd65c9a54fa5122fb7aefcb003fcf2de0c092fceb7e58972c
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P afbbfff72002089fa73b5473cc98360df14288d489e93e667332d6e884ef60da
-R b5eb5cd81beccd0fb58b9d1b00a03e31
+P 5c1fe6666019147a26480b5db1bf2f474a5d072c234c736f16ed5d2a9a040b3f
+R d3af759da34c77156099ceec10b188ae
 U drh
-Z 10e32a088c82fd051b685f8b12f5082b
+Z f7d659c7910069dbd59e1c3feefe9c0a
diff --git a/manifest.uuid b/manifest.uuid
index 4ed62ec7c2..d2b59b3c8a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5c1fe6666019147a26480b5db1bf2f474a5d072c234c736f16ed5d2a9a040b3f
\ No newline at end of file
+5efd854fe21470336ba4140294b6c90ef39af32e103b92c664438d7485c50f9a
\ No newline at end of file
diff --git a/src/treeview.c b/src/treeview.c
index 8d171a4733..6dd386ffe6 100644
--- a/src/treeview.c
+++ b/src/treeview.c
@@ -507,12 +507,20 @@ void sqlite3TreeViewBareExprList(
     sqlite3TreeViewLine(pView, "%s", zLabel);
     for(i=0; i<pList->nExpr; i++){
       int j = pList->a[i].u.x.iOrderByCol;
-      if( j ){
+      char *zName = pList->a[i].zName;
+      if( j || zName ){
         sqlite3TreeViewPush(pView, 0);
+      }
+      if( zName ){
+        sqlite3TreeViewLine(pView, "AS %s", zName);
+      }
+      if( j ){
         sqlite3TreeViewLine(pView, "iOrderByCol=%d", j);
       }
       sqlite3TreeViewExpr(pView, pList->a[i].pExpr, i<pList->nExpr-1);
-      if( j ) sqlite3TreeViewPop(pView);
+      if( j || zName ){
+        sqlite3TreeViewPop(pView);
+      }
     }
   }
 }

From 07859486c6194c4d9f67725101acd3b25f9f177a Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 23 Dec 2017 11:51:40 +0000
Subject: [PATCH 169/488] Add a SELECTTRACE() macro to indicate when column
 names are assigned to a SELECT statement.  This helps with debugging for
 tickets like [de3403bf5ae5f72e] and [3b4450072511e621].

FossilOrigin-Name: 8f194008c3aaa4ef287200e37bc5278ba9c377a7091ee3f95bad66513226b083
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/select.c  |  1 +
 3 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index 23b0f9e47e..d9e65a145e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\sthe\ssqlite3TreeViewExprList()\sroutine,\sshow\sthe\s"AS"\salias\sname\sfor\s\neach\sexpression\sin\sthe\slist,\sif\sit\sexists.
-D 2017-12-22T19:53:02.654
+C Add\sa\sSELECTTRACE()\smacro\sto\sindicate\swhen\scolumn\snames\sare\sassigned\sto\na\sSELECT\sstatement.\s\sThis\shelps\swith\sdebugging\sfor\stickets\slike\n[de3403bf5ae5f72e]\sand\s[3b4450072511e621].
+D 2017-12-23T11:51:40.933
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -478,7 +478,7 @@ F src/printf.c 9506b4b96e59c0467047155f09015750cb2878aeda3d39e5610c1192ddc3c41c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
+F src/select.c 6c84733262885f70675b7b4d5c3f5d21562cd378cf06d5b0cb3bdbdb4785b85f
 F src/shell.c.in 339169a3d1307b5566ebe9ce15832d03439206106724c78cc3d9125a7b851795
 F src/sqlite.h.in 2126192945019d4cdce335cb236b440a05ec75c93e4cd94c9c6d6e7fcc654cc4
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5c1fe6666019147a26480b5db1bf2f474a5d072c234c736f16ed5d2a9a040b3f
-R d3af759da34c77156099ceec10b188ae
+P 5efd854fe21470336ba4140294b6c90ef39af32e103b92c664438d7485c50f9a
+R 3deaa3da40d43e03f04500b413c136e7
 U drh
-Z f7d659c7910069dbd59e1c3feefe9c0a
+Z 1c7db6ece307b188f8e97c3dc484c292
diff --git a/manifest.uuid b/manifest.uuid
index d2b59b3c8a..8c8a394848 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5efd854fe21470336ba4140294b6c90ef39af32e103b92c664438d7485c50f9a
\ No newline at end of file
+8f194008c3aaa4ef287200e37bc5278ba9c377a7091ee3f95bad66513226b083
\ No newline at end of file
diff --git a/src/select.c b/src/select.c
index 97eaf21b4d..e37fffd147 100644
--- a/src/select.c
+++ b/src/select.c
@@ -1596,6 +1596,7 @@ static void generateColumnNames(
   if( pParse->colNamesSet || db->mallocFailed ) return;
   /* Column names are determined by the left-most term of a compound select */
   while( pSelect->pPrior ) pSelect = pSelect->pPrior;
+  SELECTTRACE(1,pParse,pSelect,("generating column names\n"));
   pTabList = pSelect->pSrc;
   pEList = pSelect->pEList;
   assert( v!=0 );

From 755b0fd358b355db11ce81e4a2b8b267567ca072 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 23 Dec 2017 12:33:40 +0000
Subject: [PATCH 170/488] Move the generation of output column names earlier in
 the case of a CREATE TABLE AS.  This is a fix for ticket [3b4450072511e62]
 and a continuation of check-in [ade7ddf1998190b2b63] that fixes cases of
 ticket [de3403bf5ae5f72ed6] that were missed previously.

FossilOrigin-Name: 6b2ff26c25bb9da344add79c93fb3e49fa034a89b38ef56e08e18d21de61f707
---
 manifest          | 16 ++++++++--------
 manifest.uuid     |  2 +-
 src/build.c       |  9 ++++-----
 src/select.c      | 11 +++++------
 test/colname.test | 21 +++++++++++++++++++++
 5 files changed, 39 insertions(+), 20 deletions(-)

diff --git a/manifest b/manifest
index d9e65a145e..3d7fcbcc18 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sa\sSELECTTRACE()\smacro\sto\sindicate\swhen\scolumn\snames\sare\sassigned\sto\na\sSELECT\sstatement.\s\sThis\shelps\swith\sdebugging\sfor\stickets\slike\n[de3403bf5ae5f72e]\sand\s[3b4450072511e621].
-D 2017-12-23T11:51:40.933
+C Move\sthe\sgeneration\sof\soutput\scolumn\snames\searlier\sin\sthe\scase\sof\sa\nCREATE\sTABLE\sAS.\s\sThis\sis\sa\sfix\sfor\sticket\s[3b4450072511e62]\sand\sa\ncontinuation\sof\scheck-in\s[ade7ddf1998190b2b63]\sthat\sfixes\scases\sof\nticket\s[de3403bf5ae5f72ed6]\sthat\swere\smissed\spreviously.
+D 2017-12-23T12:33:40.443
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -423,7 +423,7 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
 F src/btree.c b83a6b03f160528020bb965f0c3a40af5286cd4923c3870fd218177f03a120a7
 F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09
 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc
-F src/build.c 87b68e3b45559ec404b12f095f0ba5f06f91a6dd2d21bd8443e41d8ac2e67196
+F src/build.c ed567f088edbc305dad33a6b14e08f8216a3860f6bad1d180450d5a5414bf346
 F src/callback.c fe677cb5f5abb02f7a772a62a98c2f516426081df68856e8f2d5f950929b966a
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
@@ -478,7 +478,7 @@ F src/printf.c 9506b4b96e59c0467047155f09015750cb2878aeda3d39e5610c1192ddc3c41c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c 6c84733262885f70675b7b4d5c3f5d21562cd378cf06d5b0cb3bdbdb4785b85f
+F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
 F src/shell.c.in 339169a3d1307b5566ebe9ce15832d03439206106724c78cc3d9125a7b851795
 F src/sqlite.h.in 2126192945019d4cdce335cb236b440a05ec75c93e4cd94c9c6d6e7fcc654cc4
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
@@ -682,7 +682,7 @@ F test/collate9.test 3adcc799229545940df2f25308dd1ad65869145a
 F test/collateA.test b8218ab90d1fa5c59dcf156efabb1b2599c580d6
 F test/collateB.test 1e68906951b846570f29f20102ed91d29e634854ee47454d725f2151ecac0b95
 F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1
-F test/colname.test c47639d26cbeba6977457e5ef2c2c55c5b6c889478dd7eb0ed858ba894e7fa93
+F test/colname.test a7ecb8f1d6d8b30a6cf8fa84a2cd6f6e91cad8296376fabe485cf93cd5eb6229
 F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db
 F test/conflict2.test bb0b94cf7196c64a3cbd815c66d3ee98c2fecd9c
 F test/conflict3.test a83db76a6c3503b2fa057c7bfb08c318d8a422202d8bc5b86226e078e5b49ff9
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5efd854fe21470336ba4140294b6c90ef39af32e103b92c664438d7485c50f9a
-R 3deaa3da40d43e03f04500b413c136e7
+P 8f194008c3aaa4ef287200e37bc5278ba9c377a7091ee3f95bad66513226b083
+R 64e4839a73a8deb3e3e75445ed44ca78
 U drh
-Z 1c7db6ece307b188f8e97c3dc484c292
+Z d26f24526e4b1ec13c6ca34a79230a0c
diff --git a/manifest.uuid b/manifest.uuid
index 8c8a394848..521110fac7 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-8f194008c3aaa4ef287200e37bc5278ba9c377a7091ee3f95bad66513226b083
\ No newline at end of file
+6b2ff26c25bb9da344add79c93fb3e49fa034a89b38ef56e08e18d21de61f707
\ No newline at end of file
diff --git a/src/build.c b/src/build.c
index 9582f136c8..01d8972415 100644
--- a/src/build.c
+++ b/src/build.c
@@ -1965,11 +1965,6 @@ void sqlite3EndTable(
       pParse->nTab = 2;
       addrTop = sqlite3VdbeCurrentAddr(v) + 1;
       sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop);
-      sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield);
-      sqlite3Select(pParse, pSelect, &dest);
-      sqlite3VdbeEndCoroutine(v, regYield);
-      sqlite3VdbeJumpHere(v, addrTop - 1);
-      if( pParse->nErr ) return;
       pSelTab = sqlite3ResultSetOfSelect(pParse, pSelect);
       if( pSelTab==0 ) return;
       assert( p->aCol==0 );
@@ -1978,6 +1973,10 @@ void sqlite3EndTable(
       pSelTab->nCol = 0;
       pSelTab->aCol = 0;
       sqlite3DeleteTable(db, pSelTab);
+      sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield);
+      sqlite3Select(pParse, pSelect, &dest);
+      sqlite3VdbeEndCoroutine(v, regYield);
+      sqlite3VdbeJumpHere(v, addrTop - 1);
       addrInsLoop = sqlite3VdbeAddOp1(v, OP_Yield, dest.iSDParm);
       VdbeCoverage(v);
       sqlite3VdbeAddOp3(v, OP_MakeRecord, dest.iSdst, dest.nSdst, regRec);
diff --git a/src/select.c b/src/select.c
index e37fffd147..1a4b0a93af 100644
--- a/src/select.c
+++ b/src/select.c
@@ -1381,8 +1381,9 @@ static const char *columnTypeImpl(
 
   assert( pExpr!=0 );
   assert( pNC->pSrcList!=0 );
+  assert( pExpr->op!=TK_AGG_COLUMN );  /* This routine runes before aggregates
+                                       ** are processed */
   switch( pExpr->op ){
-    case TK_AGG_COLUMN:
     case TK_COLUMN: {
       /* The expression is a column. Locate the table the column is being
       ** extracted from in NameContext.pSrcList. This table may be real
@@ -1391,8 +1392,6 @@ static const char *columnTypeImpl(
       Table *pTab = 0;            /* Table structure column is extracted from */
       Select *pS = 0;             /* Select the column is extracted from */
       int iCol = pExpr->iColumn;  /* Index of column in pTab */
-      testcase( pExpr->op==TK_AGG_COLUMN );
-      testcase( pExpr->op==TK_COLUMN );
       while( pNC && !pTab ){
         SrcList *pTabList = pNC->pSrcList;
         for(j=0;j<pTabList->nSrc && pTabList->a[j].iCursor!=pExpr->iTable;j++);
@@ -1705,12 +1704,12 @@ int sqlite3ColumnsFromExprList(
         pColExpr = pColExpr->pRight;
         assert( pColExpr!=0 );
       }
-      if( (pColExpr->op==TK_COLUMN || pColExpr->op==TK_AGG_COLUMN)
-       && pColExpr->pTab!=0 
-      ){
+      assert( pColExpr->op!=TK_AGG_COLUMN );
+      if( pColExpr->op==TK_COLUMN ){
         /* For columns use the column name name */
         int iCol = pColExpr->iColumn;
         Table *pTab = pColExpr->pTab;
+        assert( pTab!=0 );
         if( iCol<0 ) iCol = pTab->iPKey;
         zName = iCol>=0 ? pTab->aCol[iCol].zName : "rowid";
       }else if( pColExpr->op==TK_ID ){
diff --git a/test/colname.test b/test/colname.test
index 2e4ae89008..5a40286773 100644
--- a/test/colname.test
+++ b/test/colname.test
@@ -378,6 +378,27 @@ do_test colname-9.210 {
   execsql2 {SELECT t1.a, v3.a AS n FROM t1 JOIN v3}
 } {a 1 n 3}
 
+# 2017-12-23:  Ticket https://www.sqlite.org/src/info/3b4450072511e621
+# Inconsistent column names in CREATE TABLE AS
+#
+# Verify that the names of columns in the created table of a CREATE TABLE AS
+# are the same as the names of result columns in the SELECT statement.
+#
+do_execsql_test colname-9.300 {
+  DROP TABLE IF EXISTS t1;
+  DROP TABLE IF EXISTS t2;
+  CREATE TABLE t1(aaa INT);
+  INSERT INTO t1(aaa) VALUES(123);
+}
+do_test colname-9.310 {
+  execsql2 {SELECT BBb FROM (SELECT aaa AS Bbb FROM t1)}
+} {Bbb 123}
+do_execsql_test colname-9.320 {
+  CREATE TABLE t2 AS SELECT BBb FROM (SELECT aaa AS Bbb FROM t1);
+  SELECT name FROM pragma_table_info('t2');
+} {Bbb}
+
+
 # Make sure the quotation marks get removed from the column names
 # when constructing a new table from an aggregate SELECT.
 # Email from Juergen Palm on 2017-07-11.

From ab42a2fe8ea01e7e1f21eddd4945e9ee3aff9cc5 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 23 Dec 2017 13:55:43 +0000
Subject: [PATCH 171/488] Improve the error message that comes back when
 sqlite3_load_extension() fails because the named file exists but is not a
 valid shared library.

FossilOrigin-Name: 05fee1a21ea398f1e4d6f1cf361657eff25ed6cd8f85ab398262dcfd30da57e9
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/loadext.c |  7 ++++++-
 3 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 3d7fcbcc18..1122cef153 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Move\sthe\sgeneration\sof\soutput\scolumn\snames\searlier\sin\sthe\scase\sof\sa\nCREATE\sTABLE\sAS.\s\sThis\sis\sa\sfix\sfor\sticket\s[3b4450072511e62]\sand\sa\ncontinuation\sof\scheck-in\s[ade7ddf1998190b2b63]\sthat\sfixes\scases\sof\nticket\s[de3403bf5ae5f72ed6]\sthat\swere\smissed\spreviously.
-D 2017-12-23T12:33:40.443
+C Improve\sthe\serror\smessage\sthat\scomes\sback\swhen\ssqlite3_load_extension()\nfails\sbecause\sthe\snamed\sfile\sexists\sbut\sis\snot\sa\svalid\sshared\slibrary.
+D 2017-12-23T13:55:43.819
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -442,7 +442,7 @@ F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c cb67cc56ef2ddd13e6944b2c0dd08a920bcd9503230adef8b9928d338097c722
 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e
-F src/loadext.c 20865b183bb8a3723d59cf1efffc3c50217eb452c1021d077b908c94da26b0b2
+F src/loadext.c c2eb35ab31035effc0429e367147a06c2a7aeb86a9114b0cbe9b882a9fb8f131
 F src/main.c 7ce55fa3c0bf669944de309ebab1655ed06ec67869adb0372c7a1062e461c448
 F src/malloc.c a02c9e69bc76bee0f639416b947a946412890b606301454727feadcb313536d6
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 8f194008c3aaa4ef287200e37bc5278ba9c377a7091ee3f95bad66513226b083
-R 64e4839a73a8deb3e3e75445ed44ca78
+P 6b2ff26c25bb9da344add79c93fb3e49fa034a89b38ef56e08e18d21de61f707
+R 7577ffeaa1cca163e13b3e78ba9bdfee
 U drh
-Z d26f24526e4b1ec13c6ca34a79230a0c
+Z 436076bd78ea4f3cfe09ab636dacef01
diff --git a/manifest.uuid b/manifest.uuid
index 521110fac7..304225ca85 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6b2ff26c25bb9da344add79c93fb3e49fa034a89b38ef56e08e18d21de61f707
\ No newline at end of file
+05fee1a21ea398f1e4d6f1cf361657eff25ed6cd8f85ab398262dcfd30da57e9
\ No newline at end of file
diff --git a/src/loadext.c b/src/loadext.c
index 6751425936..c28600aef8 100644
--- a/src/loadext.c
+++ b/src/loadext.c
@@ -496,8 +496,13 @@ static int sqlite3LoadExtension(
 #if SQLITE_OS_UNIX || SQLITE_OS_WIN
   for(ii=0; ii<ArraySize(azEndings) && handle==0; ii++){
     char *zAltFile = sqlite3_mprintf("%s.%s", zFile, azEndings[ii]);
+    int bOk = 0;
     if( zAltFile==0 ) return SQLITE_NOMEM_BKPT;
-    handle = sqlite3OsDlOpen(pVfs, zAltFile);
+    if( sqlite3OsAccess(pVfs,zAltFile,SQLITE_ACCESS_EXISTS,&bOk)==SQLITE_OK
+     && bOk
+    ){
+      handle = sqlite3OsDlOpen(pVfs, zAltFile);
+    }
     sqlite3_free(zAltFile);
   }
 #endif

From 88a921ce602c7d940a0ba4ed3eeeef31a4ce1455 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 23 Dec 2017 14:39:36 +0000
Subject: [PATCH 172/488] Simplification to the error handling logic in the
 extension loader.

FossilOrigin-Name: 07c773148d8db185fa54991df09298b64f4fef28879e6c9395759265e8183977
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/loadext.c |  9 +++------
 3 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/manifest b/manifest
index 1122cef153..f187cc0290 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improve\sthe\serror\smessage\sthat\scomes\sback\swhen\ssqlite3_load_extension()\nfails\sbecause\sthe\snamed\sfile\sexists\sbut\sis\snot\sa\svalid\sshared\slibrary.
-D 2017-12-23T13:55:43.819
+C Simplification\sto\sthe\serror\shandling\slogic\sin\sthe\sextension\sloader.
+D 2017-12-23T14:39:36.160
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -442,7 +442,7 @@ F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c cb67cc56ef2ddd13e6944b2c0dd08a920bcd9503230adef8b9928d338097c722
 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e
-F src/loadext.c c2eb35ab31035effc0429e367147a06c2a7aeb86a9114b0cbe9b882a9fb8f131
+F src/loadext.c 55bcc3c741059a1056859e8adaf133aa179e22be12215c0936b2f354ef71209b
 F src/main.c 7ce55fa3c0bf669944de309ebab1655ed06ec67869adb0372c7a1062e461c448
 F src/malloc.c a02c9e69bc76bee0f639416b947a946412890b606301454727feadcb313536d6
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6b2ff26c25bb9da344add79c93fb3e49fa034a89b38ef56e08e18d21de61f707
-R 7577ffeaa1cca163e13b3e78ba9bdfee
+P 05fee1a21ea398f1e4d6f1cf361657eff25ed6cd8f85ab398262dcfd30da57e9
+R 7fbfe3e61eca395fa5baf5f121a4d2ad
 U drh
-Z 436076bd78ea4f3cfe09ab636dacef01
+Z d82d9c01768cefc5beb206b92e1398bf
diff --git a/manifest.uuid b/manifest.uuid
index 304225ca85..8454418cd5 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-05fee1a21ea398f1e4d6f1cf361657eff25ed6cd8f85ab398262dcfd30da57e9
\ No newline at end of file
+07c773148d8db185fa54991df09298b64f4fef28879e6c9395759265e8183977
\ No newline at end of file
diff --git a/src/loadext.c b/src/loadext.c
index c28600aef8..abc6b3ff1d 100644
--- a/src/loadext.c
+++ b/src/loadext.c
@@ -496,13 +496,10 @@ static int sqlite3LoadExtension(
 #if SQLITE_OS_UNIX || SQLITE_OS_WIN
   for(ii=0; ii<ArraySize(azEndings) && handle==0; ii++){
     char *zAltFile = sqlite3_mprintf("%s.%s", zFile, azEndings[ii]);
-    int bOk = 0;
+    int bExists = 0;
     if( zAltFile==0 ) return SQLITE_NOMEM_BKPT;
-    if( sqlite3OsAccess(pVfs,zAltFile,SQLITE_ACCESS_EXISTS,&bOk)==SQLITE_OK
-     && bOk
-    ){
-      handle = sqlite3OsDlOpen(pVfs, zAltFile);
-    }
+    sqlite3OsAccess(pVfs, zAltFile, SQLITE_ACCESS_EXISTS, &bExists);
+    if( bExists )  handle = sqlite3OsDlOpen(pVfs, zAltFile);
     sqlite3_free(zAltFile);
   }
 #endif

From 1be266ba089bf463f1b0a5ea2839b7a1341d91ab Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sun, 24 Dec 2017 00:18:47 +0000
Subject: [PATCH 173/488] Remove the ExprSpan object. Instead, keep track of
 the test of subphrases in the parse using the "scanpt" non-terminal.

FossilOrigin-Name: 3eab7bdc44e0878b83dc86f27058a40c2ffafeacadc566f03693f6dc7e40a504
---
 manifest        |  21 +--
 manifest.uuid   |   2 +-
 src/build.c     |  23 ++-
 src/expr.c      |  11 +-
 src/parse.y     | 408 ++++++++++++++++++++----------------------------
 src/sqliteInt.h |  16 +-
 6 files changed, 207 insertions(+), 274 deletions(-)

diff --git a/manifest b/manifest
index f187cc0290..3cc840deb1 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Simplification\sto\sthe\serror\shandling\slogic\sin\sthe\sextension\sloader.
-D 2017-12-23T14:39:36.160
+C Remove\sthe\sExprSpan\sobject.\sInstead,\skeep\strack\sof\sthe\stest\sof\ssubphrases\sin\nthe\sparse\susing\sthe\s"scanpt"\snon-terminal.
+D 2017-12-24T00:18:47.740
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -423,7 +423,7 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
 F src/btree.c b83a6b03f160528020bb965f0c3a40af5286cd4923c3870fd218177f03a120a7
 F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09
 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc
-F src/build.c ed567f088edbc305dad33a6b14e08f8216a3860f6bad1d180450d5a5414bf346
+F src/build.c d3d2feeeee9c6b41a0bc7a00e4803724780f29f27ef16633b6e32aa0901c7994
 F src/callback.c fe677cb5f5abb02f7a772a62a98c2f516426081df68856e8f2d5f950929b966a
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
@@ -431,7 +431,7 @@ F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957
 F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
 F src/delete.c 74667ad914ac143731a444a1bacf29ceb18f6eded8a0dd17aafae80baa07f8bb
-F src/expr.c fe11b91bb65b869143bd42023427c4429778ae42c0a0db7762f68f75b347a958
+F src/expr.c 04188faa71b86bff3a1de46d92f41b9ec2b18fa1bbbdbf683cfff657fe803f3d
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331
 F src/func.c 0fb9a2d678d3c8aba89b46468b309cd7e8fa9806a369a30aa89024660845bb13
@@ -467,7 +467,7 @@ F src/os_win.c 0a4afa35cc8e812000df3ea2f64b476131b39e29e75d8007d0504726e4761de4
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
-F src/parse.y d79001da275bfe344b409006b85e81e486a0f6afc3762fdf0944f000f4aa0111
+F src/parse.y 1e29222ba074fea8d84438098e00812071d4cfd34c8620f34accab7cd523a276
 F src/pcache.c 7ae91a4557a43d77d449accbfdc68846e6516f8e2eda46e8bbe4536fb669b201
 F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170
 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
@@ -483,7 +483,7 @@ F src/shell.c.in 339169a3d1307b5566ebe9ce15832d03439206106724c78cc3d9125a7b85179
 F src/sqlite.h.in 2126192945019d4cdce335cb236b440a05ec75c93e4cd94c9c6d6e7fcc654cc4
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
-F src/sqliteInt.h 003b78433baae4e5c997f99f2f9cf98d90754f256baeacb32f8189569a48251f
+F src/sqliteInt.h 124be5e9bacd2a356ba17875bbaddf5f25aa1ba8a292233210ba8141ecc58316
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -1687,7 +1687,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 05fee1a21ea398f1e4d6f1cf361657eff25ed6cd8f85ab398262dcfd30da57e9
-R 7fbfe3e61eca395fa5baf5f121a4d2ad
+P 07c773148d8db185fa54991df09298b64f4fef28879e6c9395759265e8183977
+R 1cdf837f203b6ea58ba0352a2b260bed
+T *branch * span-refactor
+T *sym-span-refactor *
+T -sym-trunk *
 U drh
-Z d82d9c01768cefc5beb206b92e1398bf
+Z 03a6ffebedfd10412e429ce17606ec66
diff --git a/manifest.uuid b/manifest.uuid
index 8454418cd5..512e591247 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-07c773148d8db185fa54991df09298b64f4fef28879e6c9395759265e8183977
\ No newline at end of file
+3eab7bdc44e0878b83dc86f27058a40c2ffafeacadc566f03693f6dc7e40a504
\ No newline at end of file
diff --git a/src/build.c b/src/build.c
index 01d8972415..edf300628b 100644
--- a/src/build.c
+++ b/src/build.c
@@ -1221,34 +1221,41 @@ char sqlite3AffinityType(const char *zIn, u8 *pszEst){
 ** This routine is called by the parser while in the middle of
 ** parsing a CREATE TABLE statement.
 */
-void sqlite3AddDefaultValue(Parse *pParse, ExprSpan *pSpan){
+void sqlite3AddDefaultValue(
+  Parse *pParse,           /* Parsing context */
+  Expr *pExpr,             /* The parsed expression of the default value */
+  const char *zStart,      /* Start of the default value text */
+  const char *zEnd         /* First character past end of defaut value text */
+){
   Table *p;
   Column *pCol;
   sqlite3 *db = pParse->db;
   p = pParse->pNewTable;
   if( p!=0 ){
     pCol = &(p->aCol[p->nCol-1]);
-    if( !sqlite3ExprIsConstantOrFunction(pSpan->pExpr, db->init.busy) ){
+    if( !sqlite3ExprIsConstantOrFunction(pExpr, db->init.busy) ){
       sqlite3ErrorMsg(pParse, "default value of column [%s] is not constant",
           pCol->zName);
     }else{
       /* A copy of pExpr is used instead of the original, as pExpr contains
-      ** tokens that point to volatile memory. The 'span' of the expression
-      ** is required by pragma table_info.
+      ** tokens that point to volatile memory.	
       */
       Expr x;
+      int n;
       sqlite3ExprDelete(db, pCol->pDflt);
       memset(&x, 0, sizeof(x));
       x.op = TK_SPAN;
-      x.u.zToken = sqlite3DbStrNDup(db, (char*)pSpan->zStart,
-                                    (int)(pSpan->zEnd - pSpan->zStart));
-      x.pLeft = pSpan->pExpr;
+      while( sqlite3Isspace(zStart[0]) ) zStart++;
+      n = (int)(zEnd - zStart);
+      while( n>0 && sqlite3Isspace(zStart[n-1]) ) n--;
+      x.u.zToken = sqlite3DbStrNDup(db, zStart, n);
+      x.pLeft = pExpr;
       x.flags = EP_Skip;
       pCol->pDflt = sqlite3ExprDup(db, &x, EXPRDUP_REDUCE);
       sqlite3DbFree(db, x.u.zToken);
     }
   }
-  sqlite3ExprDelete(db, pSpan->pExpr);
+  sqlite3ExprDelete(db, pExpr);
 }
 
 /*
diff --git a/src/expr.c b/src/expr.c
index 524e539349..129c1299bf 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -1654,17 +1654,20 @@ void sqlite3ExprListSetName(
 void sqlite3ExprListSetSpan(
   Parse *pParse,          /* Parsing context */
   ExprList *pList,        /* List to which to add the span. */
-  ExprSpan *pSpan         /* The span to be added */
+  const char *zStart,     /* Start of the span */
+  const char *zEnd        /* End of the span */
 ){
   sqlite3 *db = pParse->db;
   assert( pList!=0 || db->mallocFailed!=0 );
   if( pList ){
     struct ExprList_item *pItem = &pList->a[pList->nExpr-1];
+    int n;
     assert( pList->nExpr>0 );
-    assert( db->mallocFailed || pItem->pExpr==pSpan->pExpr );
     sqlite3DbFree(db, pItem->zSpan);
-    pItem->zSpan = sqlite3DbStrNDup(db, (char*)pSpan->zStart,
-                                    (int)(pSpan->zEnd - pSpan->zStart));
+    while( sqlite3Isspace(zStart[0]) ) zStart++;
+    n = (int)(zEnd - zStart);
+    while( n>0 && sqlite3Isspace(zStart[n-1]) ) n--;
+    pItem->zSpan = sqlite3DbStrNDup(db, zStart, n);
   }
 }
 
diff --git a/src/parse.y b/src/parse.y
index 0867d1e0c8..d59426409c 100644
--- a/src/parse.y
+++ b/src/parse.y
@@ -273,26 +273,47 @@ typename(A) ::= typename(A) ids(Y). {A.n=Y.n+(int)(Y.z-A.z);}
 signed ::= plus_num.
 signed ::= minus_num.
 
+// The scanpt non-terminal takes a value which is a pointer to the
+// input text just past the last token that has been shifted into
+// the parser.  By surrounding some phrase in the grammar with two
+// scanpt non-terminals, we can capture the input text for that phrase.
+// For example:
+//
+//      something ::= .... scanpt(A) phrase scanpt(Z).
+//
+// The text that is parsed as "phrase" is a string starting at A
+// and containing (int)(Z-A) characters.  There might be some extra
+// whitespace on either end of the text, but that can be removed in
+// post-processing, if needed.
+//
+%type scanpt {const char*}
+scanpt(A) ::= . {
+  if( yyLookahead!=YYNOCODE ){
+    A = yyLookaheadToken.z;
+  }else{
+    A = pParse->sLastToken.z + pParse->sLastToken.n;
+  }
+}
+
 // "carglist" is a list of additional constraints that come after the
 // column name and column type in a CREATE TABLE statement.
 //
 carglist ::= carglist ccons.
 carglist ::= .
 ccons ::= CONSTRAINT nm(X).           {pParse->constraintName = X;}
-ccons ::= DEFAULT term(X).            {sqlite3AddDefaultValue(pParse,&X);}
-ccons ::= DEFAULT LP expr(X) RP.      {sqlite3AddDefaultValue(pParse,&X);}
-ccons ::= DEFAULT PLUS term(X).       {sqlite3AddDefaultValue(pParse,&X);}
-ccons ::= DEFAULT MINUS(A) term(X).      {
-  ExprSpan v;
-  v.pExpr = sqlite3PExpr(pParse, TK_UMINUS, X.pExpr, 0);
-  v.zStart = A.z;
-  v.zEnd = X.zEnd;
-  sqlite3AddDefaultValue(pParse,&v);
+ccons ::= DEFAULT scanpt(A) term(X) scanpt(Z).
+                            {sqlite3AddDefaultValue(pParse,X,A,Z);}
+ccons ::= DEFAULT LP(A) expr(X) RP(Z).
+                            {sqlite3AddDefaultValue(pParse,X,A.z+1,Z.z);}
+ccons ::= DEFAULT PLUS(A) term(X) scanpt(Z).
+                            {sqlite3AddDefaultValue(pParse,X,A.z,Z);}
+ccons ::= DEFAULT MINUS(A) term(X) scanpt(Z).      {
+  Expr *p = sqlite3PExpr(pParse, TK_UMINUS, X, 0);
+  sqlite3AddDefaultValue(pParse,p,A.z,Z);
 }
-ccons ::= DEFAULT id(X).              {
-  ExprSpan v;
-  spanExpr(&v, pParse, TK_STRING, X);
-  sqlite3AddDefaultValue(pParse,&v);
+ccons ::= DEFAULT scanpt id(X).       {
+  Expr *p = tokenExpr(pParse, TK_STRING, X);
+  sqlite3AddDefaultValue(pParse,p,X.z,X.z+X.n);
 }
 
 // In addition to the type name, we also care about the primary key and
@@ -304,7 +325,7 @@ ccons ::= PRIMARY KEY sortorder(Z) onconf(R) autoinc(I).
                                  {sqlite3AddPrimaryKey(pParse,0,R,I,Z);}
 ccons ::= UNIQUE onconf(R).      {sqlite3CreateIndex(pParse,0,0,0,0,R,0,0,0,0,
                                    SQLITE_IDXTYPE_UNIQUE);}
-ccons ::= CHECK LP expr(X) RP.   {sqlite3AddCheckConstraint(pParse,X.pExpr);}
+ccons ::= CHECK LP expr(X) RP.   {sqlite3AddCheckConstraint(pParse,X);}
 ccons ::= REFERENCES nm(T) eidlist_opt(TA) refargs(R).
                                  {sqlite3CreateForeignKey(pParse,0,&T,TA,R);}
 ccons ::= defer_subclause(D).    {sqlite3DeferForeignKey(pParse,D);}
@@ -355,7 +376,7 @@ tcons ::= UNIQUE LP sortlist(X) RP onconf(R).
                                  {sqlite3CreateIndex(pParse,0,0,0,X,R,0,0,0,0,
                                        SQLITE_IDXTYPE_UNIQUE);}
 tcons ::= CHECK LP expr(E) RP onconf.
-                                 {sqlite3AddCheckConstraint(pParse,E.pExpr);}
+                                 {sqlite3AddCheckConstraint(pParse,E);}
 tcons ::= FOREIGN KEY LP eidlist(FA) RP
           REFERENCES nm(T) eidlist_opt(TA) refargs(R) defer_subclause_opt(D). {
     sqlite3CreateForeignKey(pParse, FA, &T, TA, R);
@@ -549,16 +570,16 @@ distinct(A) ::= .           {A = 0;}
 %destructor sclp {sqlite3ExprListDelete(pParse->db, $$);}
 sclp(A) ::= selcollist(A) COMMA.
 sclp(A) ::= .                                {A = 0;}
-selcollist(A) ::= sclp(A) expr(X) as(Y).     {
-   A = sqlite3ExprListAppend(pParse, A, X.pExpr);
+selcollist(A) ::= sclp(A) scanpt(B) expr(X) scanpt(Z) as(Y).     {
+   A = sqlite3ExprListAppend(pParse, A, X);
    if( Y.n>0 ) sqlite3ExprListSetName(pParse, A, &Y, 1);
-   sqlite3ExprListSetSpan(pParse,A,&X);
+   sqlite3ExprListSetSpan(pParse,A,B,Z);
 }
 selcollist(A) ::= sclp(A) STAR. {
   Expr *p = sqlite3Expr(pParse->db, TK_ASTERISK, 0);
   A = sqlite3ExprListAppend(pParse, A, p);
 }
-selcollist(A) ::= sclp(A) nm(X) DOT STAR. {
+selcollist(A) ::= sclp(A) scanpt nm(X) DOT STAR. {
   Expr *pRight = sqlite3PExpr(pParse, TK_ASTERISK, 0, 0);
   Expr *pLeft = sqlite3ExprAlloc(pParse->db, TK_ID, &X, 1);
   Expr *pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight);
@@ -656,7 +677,7 @@ joinop(X) ::= JOIN_KW(A) nm(B) nm(C) JOIN.
 
 %type on_opt {Expr*}
 %destructor on_opt {sqlite3ExprDelete(pParse->db, $$);}
-on_opt(N) ::= ON expr(E).   {N = E.pExpr;}
+on_opt(N) ::= ON expr(E).   {N = E;}
 on_opt(N) ::= .             {N = 0;}
 
 // Note that this block abuses the Token type just a little. If there is
@@ -693,11 +714,11 @@ using_opt(U) ::= .                        {U = 0;}
 orderby_opt(A) ::= .                          {A = 0;}
 orderby_opt(A) ::= ORDER BY sortlist(X).      {A = X;}
 sortlist(A) ::= sortlist(A) COMMA expr(Y) sortorder(Z). {
-  A = sqlite3ExprListAppend(pParse,A,Y.pExpr);
+  A = sqlite3ExprListAppend(pParse,A,Y);
   sqlite3ExprListSetSortOrder(A,Z);
 }
 sortlist(A) ::= expr(Y) sortorder(Z). {
-  A = sqlite3ExprListAppend(pParse,0,Y.pExpr); /*A-overwrites-Y*/
+  A = sqlite3ExprListAppend(pParse,0,Y); /*A-overwrites-Y*/
   sqlite3ExprListSetSortOrder(A,Z);
 }
 
@@ -715,7 +736,7 @@ groupby_opt(A) ::= GROUP BY nexprlist(X). {A = X;}
 %type having_opt {Expr*}
 %destructor having_opt {sqlite3ExprDelete(pParse->db, $$);}
 having_opt(A) ::= .                {A = 0;}
-having_opt(A) ::= HAVING expr(X).  {A = X.pExpr;}
+having_opt(A) ::= HAVING expr(X).  {A = X;}
 
 %type limit_opt {Expr*}
 
@@ -729,11 +750,11 @@ having_opt(A) ::= HAVING expr(X).  {A = X.pExpr;}
 //%destructor limit_opt {sqlite3ExprDelete(pParse->db, $$);}
 limit_opt(A) ::= .       {A = 0;}
 limit_opt(A) ::= LIMIT expr(X).
-                         {A = sqlite3PExpr(pParse,TK_LIMIT,X.pExpr,0);}
+                         {A = sqlite3PExpr(pParse,TK_LIMIT,X,0);}
 limit_opt(A) ::= LIMIT expr(X) OFFSET expr(Y). 
-                         {A = sqlite3PExpr(pParse,TK_LIMIT,X.pExpr,Y.pExpr);}
+                         {A = sqlite3PExpr(pParse,TK_LIMIT,X,Y);}
 limit_opt(A) ::= LIMIT expr(X) COMMA expr(Y). 
-                         {A = sqlite3PExpr(pParse,TK_LIMIT,Y.pExpr,X.pExpr);}
+                         {A = sqlite3PExpr(pParse,TK_LIMIT,Y,X);}
 
 /////////////////////////// The DELETE statement /////////////////////////////
 //
@@ -757,7 +778,7 @@ cmd ::= with(C) DELETE FROM fullname(X) indexed_opt(I) where_opt(W). {
 %destructor where_opt {sqlite3ExprDelete(pParse->db, $$);}
 
 where_opt(A) ::= .                    {A = 0;}
-where_opt(A) ::= WHERE expr(X).       {A = X.pExpr;}
+where_opt(A) ::= WHERE expr(X).       {A = X;}
 
 ////////////////////////// The UPDATE command ////////////////////////////////
 //
@@ -784,18 +805,18 @@ cmd ::= with(C) UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y)
 %destructor setlist {sqlite3ExprListDelete(pParse->db, $$);}
 
 setlist(A) ::= setlist(A) COMMA nm(X) EQ expr(Y). {
-  A = sqlite3ExprListAppend(pParse, A, Y.pExpr);
+  A = sqlite3ExprListAppend(pParse, A, Y);
   sqlite3ExprListSetName(pParse, A, &X, 1);
 }
 setlist(A) ::= setlist(A) COMMA LP idlist(X) RP EQ expr(Y). {
-  A = sqlite3ExprListAppendVector(pParse, A, X, Y.pExpr);
+  A = sqlite3ExprListAppendVector(pParse, A, X, Y);
 }
 setlist(A) ::= nm(X) EQ expr(Y). {
-  A = sqlite3ExprListAppend(pParse, 0, Y.pExpr);
+  A = sqlite3ExprListAppend(pParse, 0, Y);
   sqlite3ExprListSetName(pParse, A, &X, 1);
 }
 setlist(A) ::= LP idlist(X) RP EQ expr(Y). {
-  A = sqlite3ExprListAppendVector(pParse, 0, X, Y.pExpr);
+  A = sqlite3ExprListAppendVector(pParse, 0, X, Y);
 }
 
 ////////////////////////// The INSERT command /////////////////////////////////
@@ -829,26 +850,18 @@ idlist(A) ::= nm(Y).
 /////////////////////////// Expression Processing /////////////////////////////
 //
 
-%type expr {ExprSpan}
-%destructor expr {sqlite3ExprDelete(pParse->db, $$.pExpr);}
-%type term {ExprSpan}
-%destructor term {sqlite3ExprDelete(pParse->db, $$.pExpr);}
+%type expr {Expr*}
+%destructor expr {sqlite3ExprDelete(pParse->db, $$);}
+%type term {Expr*}
+%destructor term {sqlite3ExprDelete(pParse->db, $$);}
 
 %include {
-  /* This is a utility routine used to set the ExprSpan.zStart and
-  ** ExprSpan.zEnd values of pOut so that the span covers the complete
-  ** range of text beginning with pStart and going to the end of pEnd.
-  */
-  static void spanSet(ExprSpan *pOut, Token *pStart, Token *pEnd){
-    pOut->zStart = pStart->z;
-    pOut->zEnd = &pEnd->z[pEnd->n];
-  }
 
   /* Construct a new Expr object from a single identifier.  Use the
   ** new Expr to populate pOut.  Set the span of pOut to be the identifier
   ** that created the expression.
   */
-  static void spanExpr(ExprSpan *pOut, Parse *pParse, int op, Token t){
+  static Expr *tokenExpr(Parse *pParse, int op, Token t){
     Expr *p = sqlite3DbMallocRawNN(pParse->db, sizeof(Expr)+t.n+1);
     if( p ){
       memset(p, 0, sizeof(Expr));
@@ -866,136 +879,98 @@ idlist(A) ::= nm(Y).
       p->nHeight = 1;
 #endif  
     }
-    pOut->pExpr = p;
-    pOut->zStart = t.z;
-    pOut->zEnd = &t.z[t.n];
+    return p;
   }
 }
 
 expr(A) ::= term(A).
-expr(A) ::= LP(B) expr(X) RP(E).
-            {spanSet(&A,&B,&E); /*A-overwrites-B*/  A.pExpr = X.pExpr;}
-expr(A) ::= id(X).          {spanExpr(&A,pParse,TK_ID,X); /*A-overwrites-X*/}
-expr(A) ::= JOIN_KW(X).     {spanExpr(&A,pParse,TK_ID,X); /*A-overwrites-X*/}
+expr(A) ::= LP expr(X) RP. {A = X;}
+expr(A) ::= id(X).          {A=tokenExpr(pParse,TK_ID,X); /*A-overwrites-X*/}
+expr(A) ::= JOIN_KW(X).     {A=tokenExpr(pParse,TK_ID,X); /*A-overwrites-X*/}
 expr(A) ::= nm(X) DOT nm(Y). {
   Expr *temp1 = sqlite3ExprAlloc(pParse->db, TK_ID, &X, 1);
   Expr *temp2 = sqlite3ExprAlloc(pParse->db, TK_ID, &Y, 1);
-  spanSet(&A,&X,&Y); /*A-overwrites-X*/
-  A.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp2);
+  A = sqlite3PExpr(pParse, TK_DOT, temp1, temp2);
 }
 expr(A) ::= nm(X) DOT nm(Y) DOT nm(Z). {
   Expr *temp1 = sqlite3ExprAlloc(pParse->db, TK_ID, &X, 1);
   Expr *temp2 = sqlite3ExprAlloc(pParse->db, TK_ID, &Y, 1);
   Expr *temp3 = sqlite3ExprAlloc(pParse->db, TK_ID, &Z, 1);
   Expr *temp4 = sqlite3PExpr(pParse, TK_DOT, temp2, temp3);
-  spanSet(&A,&X,&Z); /*A-overwrites-X*/
-  A.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp4);
+  A = sqlite3PExpr(pParse, TK_DOT, temp1, temp4);
 }
-term(A) ::= NULL|FLOAT|BLOB(X). {spanExpr(&A,pParse,@X,X); /*A-overwrites-X*/}
-term(A) ::= STRING(X).          {spanExpr(&A,pParse,@X,X); /*A-overwrites-X*/}
+term(A) ::= NULL|FLOAT|BLOB(X). {A=tokenExpr(pParse,@X,X); /*A-overwrites-X*/}
+term(A) ::= STRING(X).          {A=tokenExpr(pParse,@X,X); /*A-overwrites-X*/}
 term(A) ::= INTEGER(X). {
-  A.pExpr = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &X, 1);
-  A.zStart = X.z;
-  A.zEnd = X.z + X.n;
+  A = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &X, 1);
 }
 expr(A) ::= VARIABLE(X).     {
   if( !(X.z[0]=='#' && sqlite3Isdigit(X.z[1])) ){
     u32 n = X.n;
-    spanExpr(&A, pParse, TK_VARIABLE, X);
-    sqlite3ExprAssignVarNumber(pParse, A.pExpr, n);
+    A = tokenExpr(pParse, TK_VARIABLE, X);
+    sqlite3ExprAssignVarNumber(pParse, A, n);
   }else{
     /* When doing a nested parse, one can include terms in an expression
     ** that look like this:   #1 #2 ...  These terms refer to registers
     ** in the virtual machine.  #N is the N-th register. */
     Token t = X; /*A-overwrites-X*/
     assert( t.n>=2 );
-    spanSet(&A, &t, &t);
     if( pParse->nested==0 ){
       sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &t);
-      A.pExpr = 0;
+      A = 0;
     }else{
-      A.pExpr = sqlite3PExpr(pParse, TK_REGISTER, 0, 0);
-      if( A.pExpr ) sqlite3GetInt32(&t.z[1], &A.pExpr->iTable);
+      A = sqlite3PExpr(pParse, TK_REGISTER, 0, 0);
+      if( A ) sqlite3GetInt32(&t.z[1], &A->iTable);
     }
   }
 }
 expr(A) ::= expr(A) COLLATE ids(C). {
-  A.pExpr = sqlite3ExprAddCollateToken(pParse, A.pExpr, &C, 1);
-  A.zEnd = &C.z[C.n];
+  A = sqlite3ExprAddCollateToken(pParse, A, &C, 1);
 }
 %ifndef SQLITE_OMIT_CAST
-expr(A) ::= CAST(X) LP expr(E) AS typetoken(T) RP(Y). {
-  spanSet(&A,&X,&Y); /*A-overwrites-X*/
-  A.pExpr = sqlite3ExprAlloc(pParse->db, TK_CAST, &T, 1);
-  sqlite3ExprAttachSubtrees(pParse->db, A.pExpr, E.pExpr, 0);
+expr(A) ::= CAST LP expr(E) AS typetoken(T) RP. {
+  A = sqlite3ExprAlloc(pParse->db, TK_CAST, &T, 1);
+  sqlite3ExprAttachSubtrees(pParse->db, A, E, 0);
 }
 %endif  SQLITE_OMIT_CAST
-expr(A) ::= id(X) LP distinct(D) exprlist(Y) RP(E). {
+expr(A) ::= id(X) LP distinct(D) exprlist(Y) RP. {
   if( Y && Y->nExpr>pParse->db->aLimit[SQLITE_LIMIT_FUNCTION_ARG] ){
     sqlite3ErrorMsg(pParse, "too many arguments on function %T", &X);
   }
-  A.pExpr = sqlite3ExprFunction(pParse, Y, &X);
-  spanSet(&A,&X,&E);
-  if( D==SF_Distinct && A.pExpr ){
-    A.pExpr->flags |= EP_Distinct;
+  A = sqlite3ExprFunction(pParse, Y, &X);
+  if( D==SF_Distinct && A ){
+    A->flags |= EP_Distinct;
   }
 }
-expr(A) ::= id(X) LP STAR RP(E). {
-  A.pExpr = sqlite3ExprFunction(pParse, 0, &X);
-  spanSet(&A,&X,&E);
+expr(A) ::= id(X) LP STAR RP. {
+  A = sqlite3ExprFunction(pParse, 0, &X);
 }
 term(A) ::= CTIME_KW(OP). {
-  A.pExpr = sqlite3ExprFunction(pParse, 0, &OP);
-  spanSet(&A, &OP, &OP);
+  A = sqlite3ExprFunction(pParse, 0, &OP);
 }
 
-%include {
-  /* This routine constructs a binary expression node out of two ExprSpan
-  ** objects and uses the result to populate a new ExprSpan object.
-  */
-  static void spanBinaryExpr(
-    Parse *pParse,      /* The parsing context.  Errors accumulate here */
-    int op,             /* The binary operation */
-    ExprSpan *pLeft,    /* The left operand, and output */
-    ExprSpan *pRight    /* The right operand */
-  ){
-    pLeft->pExpr = sqlite3PExpr(pParse, op, pLeft->pExpr, pRight->pExpr);
-    pLeft->zEnd = pRight->zEnd;
-  }
-
-  /* If doNot is true, then add a TK_NOT Expr-node wrapper around the
-  ** outside of *ppExpr.
-  */
-  static void exprNot(Parse *pParse, int doNot, ExprSpan *pSpan){
-    if( doNot ){
-      pSpan->pExpr = sqlite3PExpr(pParse, TK_NOT, pSpan->pExpr, 0);
-    }
-  }
-}
-
-expr(A) ::= LP(L) nexprlist(X) COMMA expr(Y) RP(R). {
-  ExprList *pList = sqlite3ExprListAppend(pParse, X, Y.pExpr);
-  A.pExpr = sqlite3PExpr(pParse, TK_VECTOR, 0, 0);
-  if( A.pExpr ){
-    A.pExpr->x.pList = pList;
-    spanSet(&A, &L, &R);
+expr(A) ::= LP nexprlist(X) COMMA expr(Y) RP. {
+  ExprList *pList = sqlite3ExprListAppend(pParse, X, Y);
+  A = sqlite3PExpr(pParse, TK_VECTOR, 0, 0);
+  if( A ){
+    A->x.pList = pList;
   }else{
     sqlite3ExprListDelete(pParse->db, pList);
   }
 }
 
-expr(A) ::= expr(A) AND(OP) expr(Y).    {spanBinaryExpr(pParse,@OP,&A,&Y);}
-expr(A) ::= expr(A) OR(OP) expr(Y).     {spanBinaryExpr(pParse,@OP,&A,&Y);}
+expr(A) ::= expr(A) AND(OP) expr(Y).    {A=sqlite3PExpr(pParse,@OP,A,Y);}
+expr(A) ::= expr(A) OR(OP) expr(Y).     {A=sqlite3PExpr(pParse,@OP,A,Y);}
 expr(A) ::= expr(A) LT|GT|GE|LE(OP) expr(Y).
-                                        {spanBinaryExpr(pParse,@OP,&A,&Y);}
-expr(A) ::= expr(A) EQ|NE(OP) expr(Y).  {spanBinaryExpr(pParse,@OP,&A,&Y);}
+                                        {A=sqlite3PExpr(pParse,@OP,A,Y);}
+expr(A) ::= expr(A) EQ|NE(OP) expr(Y).  {A=sqlite3PExpr(pParse,@OP,A,Y);}
 expr(A) ::= expr(A) BITAND|BITOR|LSHIFT|RSHIFT(OP) expr(Y).
-                                        {spanBinaryExpr(pParse,@OP,&A,&Y);}
+                                        {A=sqlite3PExpr(pParse,@OP,A,Y);}
 expr(A) ::= expr(A) PLUS|MINUS(OP) expr(Y).
-                                        {spanBinaryExpr(pParse,@OP,&A,&Y);}
+                                        {A=sqlite3PExpr(pParse,@OP,A,Y);}
 expr(A) ::= expr(A) STAR|SLASH|REM(OP) expr(Y).
-                                        {spanBinaryExpr(pParse,@OP,&A,&Y);}
-expr(A) ::= expr(A) CONCAT(OP) expr(Y). {spanBinaryExpr(pParse,@OP,&A,&Y);}
+                                        {A=sqlite3PExpr(pParse,@OP,A,Y);}
+expr(A) ::= expr(A) CONCAT(OP) expr(Y). {A=sqlite3PExpr(pParse,@OP,A,Y);}
 %type likeop {Token}
 likeop(A) ::= LIKE_KW|MATCH(A).
 likeop(A) ::= NOT LIKE_KW|MATCH(X). {A=X; A.n|=0x80000000; /*A-overwrite-X*/}
@@ -1003,42 +978,26 @@ expr(A) ::= expr(A) likeop(OP) expr(Y).  [LIKE_KW]  {
   ExprList *pList;
   int bNot = OP.n & 0x80000000;
   OP.n &= 0x7fffffff;
-  pList = sqlite3ExprListAppend(pParse,0, Y.pExpr);
-  pList = sqlite3ExprListAppend(pParse,pList, A.pExpr);
-  A.pExpr = sqlite3ExprFunction(pParse, pList, &OP);
-  exprNot(pParse, bNot, &A);
-  A.zEnd = Y.zEnd;
-  if( A.pExpr ) A.pExpr->flags |= EP_InfixFunc;
+  pList = sqlite3ExprListAppend(pParse,0, Y);
+  pList = sqlite3ExprListAppend(pParse,pList, A);
+  A = sqlite3ExprFunction(pParse, pList, &OP);
+  if( bNot ) A = sqlite3PExpr(pParse, TK_NOT, A, 0);
+  if( A ) A->flags |= EP_InfixFunc;
 }
 expr(A) ::= expr(A) likeop(OP) expr(Y) ESCAPE expr(E).  [LIKE_KW]  {
   ExprList *pList;
   int bNot = OP.n & 0x80000000;
   OP.n &= 0x7fffffff;
-  pList = sqlite3ExprListAppend(pParse,0, Y.pExpr);
-  pList = sqlite3ExprListAppend(pParse,pList, A.pExpr);
-  pList = sqlite3ExprListAppend(pParse,pList, E.pExpr);
-  A.pExpr = sqlite3ExprFunction(pParse, pList, &OP);
-  exprNot(pParse, bNot, &A);
-  A.zEnd = E.zEnd;
-  if( A.pExpr ) A.pExpr->flags |= EP_InfixFunc;
+  pList = sqlite3ExprListAppend(pParse,0, Y);
+  pList = sqlite3ExprListAppend(pParse,pList, A);
+  pList = sqlite3ExprListAppend(pParse,pList, E);
+  A = sqlite3ExprFunction(pParse, pList, &OP);
+  if( bNot ) A = sqlite3PExpr(pParse, TK_NOT, A, 0);
+  if( A ) A->flags |= EP_InfixFunc;
 }
 
-%include {
-  /* Construct an expression node for a unary postfix operator
-  */
-  static void spanUnaryPostfix(
-    Parse *pParse,         /* Parsing context to record errors */
-    int op,                /* The operator */
-    ExprSpan *pOperand,    /* The operand, and output */
-    Token *pPostOp         /* The operand token for setting the span */
-  ){
-    pOperand->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0);
-    pOperand->zEnd = &pPostOp->z[pPostOp->n];
-  }                           
-}
-
-expr(A) ::= expr(A) ISNULL|NOTNULL(E).   {spanUnaryPostfix(pParse,@E,&A,&E);}
-expr(A) ::= expr(A) NOT NULL(E). {spanUnaryPostfix(pParse,TK_NOTNULL,&A,&E);}
+expr(A) ::= expr(A) ISNULL|NOTNULL(E).   {A = sqlite3PExpr(pParse,@E,A,0);}
+expr(A) ::= expr(A) NOT NULL.    {A = sqlite3PExpr(pParse,TK_NOTNULL,A,0);}
 
 %include {
   /* A routine to convert a binary TK_IS or TK_ISNOT expression into a
@@ -1060,61 +1019,42 @@ expr(A) ::= expr(A) NOT NULL(E). {spanUnaryPostfix(pParse,TK_NOTNULL,&A,&E);}
 // is any other expression, code as TK_IS or TK_ISNOT.
 // 
 expr(A) ::= expr(A) IS expr(Y).     {
-  spanBinaryExpr(pParse,TK_IS,&A,&Y);
-  binaryToUnaryIfNull(pParse, Y.pExpr, A.pExpr, TK_ISNULL);
+  A = sqlite3PExpr(pParse,TK_IS,A,Y);
+  binaryToUnaryIfNull(pParse, Y, A, TK_ISNULL);
 }
 expr(A) ::= expr(A) IS NOT expr(Y). {
-  spanBinaryExpr(pParse,TK_ISNOT,&A,&Y);
-  binaryToUnaryIfNull(pParse, Y.pExpr, A.pExpr, TK_NOTNULL);
+  A = sqlite3PExpr(pParse,TK_ISNOT,A,Y);
+  binaryToUnaryIfNull(pParse, Y, A, TK_NOTNULL);
 }
 
-%include {
-  /* Construct an expression node for a unary prefix operator
-  */
-  static void spanUnaryPrefix(
-    ExprSpan *pOut,        /* Write the new expression node here */
-    Parse *pParse,         /* Parsing context to record errors */
-    int op,                /* The operator */
-    ExprSpan *pOperand,    /* The operand */
-    Token *pPreOp         /* The operand token for setting the span */
-  ){
-    pOut->zStart = pPreOp->z;
-    pOut->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0);
-    pOut->zEnd = pOperand->zEnd;
-  }
-}
-
-
-
 expr(A) ::= NOT(B) expr(X).  
-              {spanUnaryPrefix(&A,pParse,@B,&X,&B);/*A-overwrites-B*/}
+              {A = sqlite3PExpr(pParse, @B, X, 0);/*A-overwrites-B*/}
 expr(A) ::= BITNOT(B) expr(X).
-              {spanUnaryPrefix(&A,pParse,@B,&X,&B);/*A-overwrites-B*/}
-expr(A) ::= MINUS(B) expr(X). [BITNOT]
-              {spanUnaryPrefix(&A,pParse,TK_UMINUS,&X,&B);/*A-overwrites-B*/}
-expr(A) ::= PLUS(B) expr(X). [BITNOT]
-              {spanUnaryPrefix(&A,pParse,TK_UPLUS,&X,&B);/*A-overwrites-B*/}
+              {A = sqlite3PExpr(pParse, @B, X, 0);/*A-overwrites-B*/}
+expr(A) ::= MINUS expr(X). [BITNOT]
+              {A = sqlite3PExpr(pParse, TK_UMINUS, X, 0);}
+expr(A) ::= PLUS expr(X). [BITNOT]
+              {A = sqlite3PExpr(pParse, TK_UPLUS, X, 0);}
 
 %type between_op {int}
 between_op(A) ::= BETWEEN.     {A = 0;}
 between_op(A) ::= NOT BETWEEN. {A = 1;}
 expr(A) ::= expr(A) between_op(N) expr(X) AND expr(Y). [BETWEEN] {
-  ExprList *pList = sqlite3ExprListAppend(pParse,0, X.pExpr);
-  pList = sqlite3ExprListAppend(pParse,pList, Y.pExpr);
-  A.pExpr = sqlite3PExpr(pParse, TK_BETWEEN, A.pExpr, 0);
-  if( A.pExpr ){
-    A.pExpr->x.pList = pList;
+  ExprList *pList = sqlite3ExprListAppend(pParse,0, X);
+  pList = sqlite3ExprListAppend(pParse,pList, Y);
+  A = sqlite3PExpr(pParse, TK_BETWEEN, A, 0);
+  if( A ){
+    A->x.pList = pList;
   }else{
     sqlite3ExprListDelete(pParse->db, pList);
   } 
-  exprNot(pParse, N, &A);
-  A.zEnd = Y.zEnd;
+  if( N ) A = sqlite3PExpr(pParse, TK_NOT, A, 0);
 }
 %ifndef SQLITE_OMIT_SUBQUERY
   %type in_op {int}
   in_op(A) ::= IN.      {A = 0;}
   in_op(A) ::= NOT IN.  {A = 1;}
-  expr(A) ::= expr(A) in_op(N) LP exprlist(Y) RP(E). [IN] {
+  expr(A) ::= expr(A) in_op(N) LP exprlist(Y) RP. [IN] {
     if( Y==0 ){
       /* Expressions of the form
       **
@@ -1124,8 +1064,8 @@ expr(A) ::= expr(A) between_op(N) expr(X) AND expr(Y). [BETWEEN] {
       ** simplify to constants 0 (false) and 1 (true), respectively,
       ** regardless of the value of expr1.
       */
-      sqlite3ExprDelete(pParse->db, A.pExpr);
-      A.pExpr = sqlite3ExprAlloc(pParse->db, TK_INTEGER,&sqlite3IntTokens[N],1);
+      sqlite3ExprDelete(pParse->db, A);
+      A = sqlite3ExprAlloc(pParse->db, TK_INTEGER,&sqlite3IntTokens[N],1);
     }else if( Y->nExpr==1 ){
       /* Expressions of the form:
       **
@@ -1152,54 +1092,48 @@ expr(A) ::= expr(A) between_op(N) expr(X) AND expr(Y). [BETWEEN] {
         pRHS->flags &= ~EP_Collate;
         pRHS->flags |= EP_Generic;
       }
-      A.pExpr = sqlite3PExpr(pParse, N ? TK_NE : TK_EQ, A.pExpr, pRHS);
+      A = sqlite3PExpr(pParse, N ? TK_NE : TK_EQ, A, pRHS);
     }else{
-      A.pExpr = sqlite3PExpr(pParse, TK_IN, A.pExpr, 0);
-      if( A.pExpr ){
-        A.pExpr->x.pList = Y;
-        sqlite3ExprSetHeightAndFlags(pParse, A.pExpr);
+      A = sqlite3PExpr(pParse, TK_IN, A, 0);
+      if( A ){
+        A->x.pList = Y;
+        sqlite3ExprSetHeightAndFlags(pParse, A);
       }else{
         sqlite3ExprListDelete(pParse->db, Y);
       }
-      exprNot(pParse, N, &A);
+      if( N ) A = sqlite3PExpr(pParse, TK_NOT, A, 0);
     }
-    A.zEnd = &E.z[E.n];
   }
-  expr(A) ::= LP(B) select(X) RP(E). {
-    spanSet(&A,&B,&E); /*A-overwrites-B*/
-    A.pExpr = sqlite3PExpr(pParse, TK_SELECT, 0, 0);
-    sqlite3PExprAddSelect(pParse, A.pExpr, X);
+  expr(A) ::= LP select(X) RP. {
+    A = sqlite3PExpr(pParse, TK_SELECT, 0, 0);
+    sqlite3PExprAddSelect(pParse, A, X);
   }
-  expr(A) ::= expr(A) in_op(N) LP select(Y) RP(E).  [IN] {
-    A.pExpr = sqlite3PExpr(pParse, TK_IN, A.pExpr, 0);
-    sqlite3PExprAddSelect(pParse, A.pExpr, Y);
-    exprNot(pParse, N, &A);
-    A.zEnd = &E.z[E.n];
+  expr(A) ::= expr(A) in_op(N) LP select(Y) RP.  [IN] {
+    A = sqlite3PExpr(pParse, TK_IN, A, 0);
+    sqlite3PExprAddSelect(pParse, A, Y);
+    if( N ) A = sqlite3PExpr(pParse, TK_NOT, A, 0);
   }
   expr(A) ::= expr(A) in_op(N) nm(Y) dbnm(Z) paren_exprlist(E). [IN] {
     SrcList *pSrc = sqlite3SrcListAppend(pParse->db, 0,&Y,&Z);
     Select *pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0);
     if( E )  sqlite3SrcListFuncArgs(pParse, pSelect ? pSrc : 0, E);
-    A.pExpr = sqlite3PExpr(pParse, TK_IN, A.pExpr, 0);
-    sqlite3PExprAddSelect(pParse, A.pExpr, pSelect);
-    exprNot(pParse, N, &A);
-    A.zEnd = Z.z ? &Z.z[Z.n] : &Y.z[Y.n];
+    A = sqlite3PExpr(pParse, TK_IN, A, 0);
+    sqlite3PExprAddSelect(pParse, A, pSelect);
+    if( N ) A = sqlite3PExpr(pParse, TK_NOT, A, 0);
   }
-  expr(A) ::= EXISTS(B) LP select(Y) RP(E). {
+  expr(A) ::= EXISTS LP select(Y) RP. {
     Expr *p;
-    spanSet(&A,&B,&E); /*A-overwrites-B*/
-    p = A.pExpr = sqlite3PExpr(pParse, TK_EXISTS, 0, 0);
+    p = A = sqlite3PExpr(pParse, TK_EXISTS, 0, 0);
     sqlite3PExprAddSelect(pParse, p, Y);
   }
 %endif SQLITE_OMIT_SUBQUERY
 
 /* CASE expressions */
-expr(A) ::= CASE(C) case_operand(X) case_exprlist(Y) case_else(Z) END(E). {
-  spanSet(&A,&C,&E);  /*A-overwrites-C*/
-  A.pExpr = sqlite3PExpr(pParse, TK_CASE, X, 0);
-  if( A.pExpr ){
-    A.pExpr->x.pList = Z ? sqlite3ExprListAppend(pParse,Y,Z) : Y;
-    sqlite3ExprSetHeightAndFlags(pParse, A.pExpr);
+expr(A) ::= CASE case_operand(X) case_exprlist(Y) case_else(Z) END. {
+  A = sqlite3PExpr(pParse, TK_CASE, X, 0);
+  if( A ){
+    A->x.pList = Z ? sqlite3ExprListAppend(pParse,Y,Z) : Y;
+    sqlite3ExprSetHeightAndFlags(pParse, A);
   }else{
     sqlite3ExprListDelete(pParse->db, Y);
     sqlite3ExprDelete(pParse->db, Z);
@@ -1208,20 +1142,20 @@ expr(A) ::= CASE(C) case_operand(X) case_exprlist(Y) case_else(Z) END(E). {
 %type case_exprlist {ExprList*}
 %destructor case_exprlist {sqlite3ExprListDelete(pParse->db, $$);}
 case_exprlist(A) ::= case_exprlist(A) WHEN expr(Y) THEN expr(Z). {
-  A = sqlite3ExprListAppend(pParse,A, Y.pExpr);
-  A = sqlite3ExprListAppend(pParse,A, Z.pExpr);
+  A = sqlite3ExprListAppend(pParse,A, Y);
+  A = sqlite3ExprListAppend(pParse,A, Z);
 }
 case_exprlist(A) ::= WHEN expr(Y) THEN expr(Z). {
-  A = sqlite3ExprListAppend(pParse,0, Y.pExpr);
-  A = sqlite3ExprListAppend(pParse,A, Z.pExpr);
+  A = sqlite3ExprListAppend(pParse,0, Y);
+  A = sqlite3ExprListAppend(pParse,A, Z);
 }
 %type case_else {Expr*}
 %destructor case_else {sqlite3ExprDelete(pParse->db, $$);}
-case_else(A) ::=  ELSE expr(X).         {A = X.pExpr;}
+case_else(A) ::=  ELSE expr(X).         {A = X;}
 case_else(A) ::=  .                     {A = 0;} 
 %type case_operand {Expr*}
 %destructor case_operand {sqlite3ExprDelete(pParse->db, $$);}
-case_operand(A) ::= expr(X).            {A = X.pExpr; /*A-overwrites-X*/} 
+case_operand(A) ::= expr(X).            {A = X; /*A-overwrites-X*/} 
 case_operand(A) ::= .                   {A = 0;} 
 
 %type exprlist {ExprList*}
@@ -1232,9 +1166,9 @@ case_operand(A) ::= .                   {A = 0;}
 exprlist(A) ::= nexprlist(A).
 exprlist(A) ::= .                            {A = 0;}
 nexprlist(A) ::= nexprlist(A) COMMA expr(Y).
-    {A = sqlite3ExprListAppend(pParse,A,Y.pExpr);}
+    {A = sqlite3ExprListAppend(pParse,A,Y);}
 nexprlist(A) ::= expr(Y).
-    {A = sqlite3ExprListAppend(pParse,0,Y.pExpr); /*A-overwrites-Y*/}
+    {A = sqlite3ExprListAppend(pParse,0,Y); /*A-overwrites-Y*/}
 
 %ifndef SQLITE_OMIT_SUBQUERY
 /* A paren_exprlist is an optional expression list contained inside
@@ -1388,7 +1322,7 @@ foreach_clause ::= FOR EACH ROW.
 %type when_clause {Expr*}
 %destructor when_clause {sqlite3ExprDelete(pParse->db, $$);}
 when_clause(A) ::= .             { A = 0; }
-when_clause(A) ::= WHEN expr(X). { A = X.pExpr; }
+when_clause(A) ::= WHEN expr(X). { A = X; }
 
 %type trigger_cmd_list {TriggerStep*}
 %destructor trigger_cmd_list {sqlite3DeleteTriggerStep(pParse->db, $$);}
@@ -1453,18 +1387,16 @@ trigger_cmd(A) ::= select(X).
    {A = sqlite3TriggerSelectStep(pParse->db, X); /*A-overwrites-X*/}
 
 // The special RAISE expression that may occur in trigger programs
-expr(A) ::= RAISE(X) LP IGNORE RP(Y).  {
-  spanSet(&A,&X,&Y);  /*A-overwrites-X*/
-  A.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0); 
-  if( A.pExpr ){
-    A.pExpr->affinity = OE_Ignore;
+expr(A) ::= RAISE LP IGNORE RP.  {
+  A = sqlite3PExpr(pParse, TK_RAISE, 0, 0); 
+  if( A ){
+    A->affinity = OE_Ignore;
   }
 }
-expr(A) ::= RAISE(X) LP raisetype(T) COMMA nm(Z) RP(Y).  {
-  spanSet(&A,&X,&Y);  /*A-overwrites-X*/
-  A.pExpr = sqlite3ExprAlloc(pParse->db, TK_RAISE, &Z, 1); 
-  if( A.pExpr ) {
-    A.pExpr->affinity = (char)T;
+expr(A) ::= RAISE LP raisetype(T) COMMA nm(Z) RP.  {
+  A = sqlite3ExprAlloc(pParse->db, TK_RAISE, &Z, 1); 
+  if( A ) {
+    A->affinity = (char)T;
   }
 }
 %endif  !SQLITE_OMIT_TRIGGER
@@ -1485,16 +1417,16 @@ cmd ::= DROP TRIGGER ifexists(NOERR) fullname(X). {
 //////////////////////// ATTACH DATABASE file AS name /////////////////////////
 %ifndef SQLITE_OMIT_ATTACH
 cmd ::= ATTACH database_kw_opt expr(F) AS expr(D) key_opt(K). {
-  sqlite3Attach(pParse, F.pExpr, D.pExpr, K);
+  sqlite3Attach(pParse, F, D, K);
 }
 cmd ::= DETACH database_kw_opt expr(D). {
-  sqlite3Detach(pParse, D.pExpr);
+  sqlite3Detach(pParse, D);
 }
 
 %type key_opt {Expr*}
 %destructor key_opt {sqlite3ExprDelete(pParse->db, $$);}
 key_opt(A) ::= .                     { A = 0; }
-key_opt(A) ::= KEY expr(X).          { A = X.pExpr; }
+key_opt(A) ::= KEY expr(X).          { A = X; }
 
 database_kw_opt ::= DATABASE.
 database_kw_opt ::= .
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index b0c4711b03..93099ca1d3 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1063,7 +1063,6 @@ typedef struct Db Db;
 typedef struct Schema Schema;
 typedef struct Expr Expr;
 typedef struct ExprList ExprList;
-typedef struct ExprSpan ExprSpan;
 typedef struct FKey FKey;
 typedef struct FuncDestructor FuncDestructor;
 typedef struct FuncDef FuncDef;
@@ -2504,17 +2503,6 @@ struct ExprList {
   } a[1];                  /* One slot for each expression in the list */
 };
 
-/*
-** An instance of this structure is used by the parser to record both
-** the parse tree for an expression and the span of input text for an
-** expression.
-*/
-struct ExprSpan {
-  Expr *pExpr;          /* The expression parse tree */
-  const char *zStart;   /* First character of input text */
-  const char *zEnd;     /* One character past the end of input text */
-};
-
 /*
 ** An instance of this structure can hold a simple list of identifiers,
 ** such as the list "a,b,c" in the following statements:
@@ -3663,7 +3651,7 @@ ExprList *sqlite3ExprListAppend(Parse*,ExprList*,Expr*);
 ExprList *sqlite3ExprListAppendVector(Parse*,ExprList*,IdList*,Expr*);
 void sqlite3ExprListSetSortOrder(ExprList*,int);
 void sqlite3ExprListSetName(Parse*,ExprList*,Token*,int);
-void sqlite3ExprListSetSpan(Parse*,ExprList*,ExprSpan*);
+void sqlite3ExprListSetSpan(Parse*,ExprList*,const char*,const char*);
 void sqlite3ExprListDelete(sqlite3*, ExprList*);
 u32 sqlite3ExprListFlags(const ExprList*);
 int sqlite3Init(sqlite3*, char**);
@@ -3693,7 +3681,7 @@ void sqlite3AddColumn(Parse*,Token*,Token*);
 void sqlite3AddNotNull(Parse*, int);
 void sqlite3AddPrimaryKey(Parse*, ExprList*, int, int, int);
 void sqlite3AddCheckConstraint(Parse*, Expr*);
-void sqlite3AddDefaultValue(Parse*,ExprSpan*);
+void sqlite3AddDefaultValue(Parse*,Expr*,const char*,const char*);
 void sqlite3AddCollateType(Parse*, Token*);
 void sqlite3EndTable(Parse*,Token*,Token*,u8,Select*);
 int sqlite3ParseUri(const char*,const char*,unsigned int*,

From ec64c6cfd2ecdc4d14032f8608833e2a0e11a1d2 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sun, 24 Dec 2017 14:14:21 +0000
Subject: [PATCH 174/488] Improved parser tracing output.

FossilOrigin-Name: c4951833c2b976223c2393d82fd2606068c71cd19612ca9df4e26debab980e32
---
 manifest      | 15 ++++++---------
 manifest.uuid |  2 +-
 tool/lempar.c | 36 +++++++++++++++++++++++++-----------
 3 files changed, 32 insertions(+), 21 deletions(-)

diff --git a/manifest b/manifest
index 3cc840deb1..8e6b14968f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\sthe\sExprSpan\sobject.\sInstead,\skeep\strack\sof\sthe\stest\sof\ssubphrases\sin\nthe\sparse\susing\sthe\s"scanpt"\snon-terminal.
-D 2017-12-24T00:18:47.740
+C Improved\sparser\stracing\soutput.
+D 2017-12-24T14:14:21.392
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -1609,7 +1609,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
 F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f
 F tool/lemon.c e6056373044d55296d21f81467dba7632bbb81dc49af072b3f0e76338771497e
-F tool/lempar.c 967ebf585cd09b11b89d255d213865109a9c4ff075680d22580a2826de288c89
+F tool/lempar.c a427c2375df118fa52e69174ffbbf9e26878096f0a109df6b77062d6032afe18
 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9
 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca
@@ -1687,10 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 07c773148d8db185fa54991df09298b64f4fef28879e6c9395759265e8183977
-R 1cdf837f203b6ea58ba0352a2b260bed
-T *branch * span-refactor
-T *sym-span-refactor *
-T -sym-trunk *
+P 3eab7bdc44e0878b83dc86f27058a40c2ffafeacadc566f03693f6dc7e40a504
+R 35c7a29b183ac93da1a9be4e6a82bb63
 U drh
-Z 03a6ffebedfd10412e429ce17606ec66
+Z 121e81b010860ccafac66f3ea34cc3c9
diff --git a/manifest.uuid b/manifest.uuid
index 512e591247..519f079775 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-3eab7bdc44e0878b83dc86f27058a40c2ffafeacadc566f03693f6dc7e40a504
\ No newline at end of file
+c4951833c2b976223c2393d82fd2606068c71cd19612ca9df4e26debab980e32
\ No newline at end of file
diff --git a/tool/lempar.c b/tool/lempar.c
index da81ddd4bc..89c1c80f70 100644
--- a/tool/lempar.c
+++ b/tool/lempar.c
@@ -578,20 +578,21 @@ static void yyStackOverflow(yyParser *yypParser){
 ** Print tracing information for a SHIFT action
 */
 #ifndef NDEBUG
-static void yyTraceShift(yyParser *yypParser, int yyNewState){
+static void yyTraceShift(yyParser *yypParser, int yyNewState, const char *zTag){
   if( yyTraceFILE ){
     if( yyNewState<YYNSTATE ){
-      fprintf(yyTraceFILE,"%sShift '%s', go to state %d\n",
-         yyTracePrompt,yyTokenName[yypParser->yytos->major],
+      fprintf(yyTraceFILE,"%s%s '%s', go to state %d\n",
+         yyTracePrompt, zTag, yyTokenName[yypParser->yytos->major],
          yyNewState);
     }else{
-      fprintf(yyTraceFILE,"%sShift '%s'\n",
-         yyTracePrompt,yyTokenName[yypParser->yytos->major]);
+      fprintf(yyTraceFILE,"%s%s '%s', pending reduce %d\n",
+         yyTracePrompt, zTag, yyTokenName[yypParser->yytos->major],
+         yyNewState - YY_MIN_REDUCE);
     }
   }
 }
 #else
-# define yyTraceShift(X,Y)
+# define yyTraceShift(X,Y,Z)
 #endif
 
 /*
@@ -633,7 +634,7 @@ static void yy_shift(
   yytos->stateno = (YYACTIONTYPE)yyNewState;
   yytos->major = (YYCODETYPE)yyMajor;
   yytos->minor.yy0 = yyMinor;
-  yyTraceShift(yypParser, yyNewState);
+  yyTraceShift(yypParser, yyNewState, "Shift");
 }
 
 /* The following table contains information about every rule that
@@ -673,8 +674,14 @@ static void yy_reduce(
 #ifndef NDEBUG
   if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){
     yysize = yyRuleInfo[yyruleno].nrhs;
-    fprintf(yyTraceFILE, "%sReduce [%s], go to state %d.\n", yyTracePrompt,
-      yyRuleName[yyruleno], yymsp[yysize].stateno);
+    if( yysize ){
+      fprintf(yyTraceFILE, "%sReduce %d [%s], go to state %d.\n",
+        yyTracePrompt,
+        yyruleno, yyRuleName[yyruleno], yymsp[yysize].stateno);
+    }else{
+      fprintf(yyTraceFILE, "%sReduce %d [%s].\n",
+        yyTracePrompt, yyruleno, yyRuleName[yyruleno]);
+    }
   }
 #endif /* NDEBUG */
 
@@ -737,7 +744,7 @@ static void yy_reduce(
     yypParser->yytos = yymsp;
     yymsp->stateno = (YYACTIONTYPE)yyact;
     yymsp->major = (YYCODETYPE)yygoto;
-    yyTraceShift(yypParser, yyact);
+    yyTraceShift(yypParser, yyact, "... then shift");
   }
 }
 
@@ -848,7 +855,14 @@ void Parse(
 
 #ifndef NDEBUG
   if( yyTraceFILE ){
-    fprintf(yyTraceFILE,"%sInput '%s'\n",yyTracePrompt,yyTokenName[yymajor]);
+    int stateno = yypParser->yytos->stateno;
+    if( stateno < YY_MIN_REDUCE ){
+      fprintf(yyTraceFILE,"%sInput '%s' in state %d\n",
+              yyTracePrompt,yyTokenName[yymajor],stateno);
+    }else{
+      fprintf(yyTraceFILE,"%sInput '%s' with pending reduce %d\n",
+              yyTracePrompt,yyTokenName[yymajor],stateno-YY_MIN_REDUCE);
+    }
   }
 #endif
 

From d3f5d6110b21b04acbe860f6a6d6242f6ed9fab4 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sun, 24 Dec 2017 17:01:54 +0000
Subject: [PATCH 175/488] Grammar changes:  the sclp non-terminal should always
 be followed by a scanpt.

FossilOrigin-Name: 74a0181fc016fb0b7934188cfcc2ec26c689f0d33cc42453875f4392b7240d3a
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/parse.y   |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 8e6b14968f..bc08e70fd1 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improved\sparser\stracing\soutput.
-D 2017-12-24T14:14:21.392
+C Grammar\schanges:\s\sthe\ssclp\snon-terminal\sshould\salways\sbe\sfollowed\sby\sa\sscanpt.
+D 2017-12-24T17:01:54.828
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -467,7 +467,7 @@ F src/os_win.c 0a4afa35cc8e812000df3ea2f64b476131b39e29e75d8007d0504726e4761de4
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
-F src/parse.y 1e29222ba074fea8d84438098e00812071d4cfd34c8620f34accab7cd523a276
+F src/parse.y 2b0653fd44534d941ff5289f372fedfc213e78c4698e28334d073cf4cfd146f2
 F src/pcache.c 7ae91a4557a43d77d449accbfdc68846e6516f8e2eda46e8bbe4536fb669b201
 F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170
 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3eab7bdc44e0878b83dc86f27058a40c2ffafeacadc566f03693f6dc7e40a504
-R 35c7a29b183ac93da1a9be4e6a82bb63
+P c4951833c2b976223c2393d82fd2606068c71cd19612ca9df4e26debab980e32
+R bc2b2a8774f331c7775262d09dd60c38
 U drh
-Z 121e81b010860ccafac66f3ea34cc3c9
+Z a70d3a97910b6c52f1408e750f661b18
diff --git a/manifest.uuid b/manifest.uuid
index 519f079775..b5e2fe7f2b 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c4951833c2b976223c2393d82fd2606068c71cd19612ca9df4e26debab980e32
\ No newline at end of file
+74a0181fc016fb0b7934188cfcc2ec26c689f0d33cc42453875f4392b7240d3a
\ No newline at end of file
diff --git a/src/parse.y b/src/parse.y
index d59426409c..fe2251db4b 100644
--- a/src/parse.y
+++ b/src/parse.y
@@ -575,7 +575,7 @@ selcollist(A) ::= sclp(A) scanpt(B) expr(X) scanpt(Z) as(Y).     {
    if( Y.n>0 ) sqlite3ExprListSetName(pParse, A, &Y, 1);
    sqlite3ExprListSetSpan(pParse,A,B,Z);
 }
-selcollist(A) ::= sclp(A) STAR. {
+selcollist(A) ::= sclp(A) scanpt STAR. {
   Expr *p = sqlite3Expr(pParse->db, TK_ASTERISK, 0);
   A = sqlite3ExprListAppend(pParse, A, p);
 }

From e58f74f680f97f382db35215b22203b3b10abb30 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sun, 24 Dec 2017 17:06:41 +0000
Subject: [PATCH 176/488] Improved parser tracing output.

FossilOrigin-Name: 25be5750545b7b0ed9e1a1baca31611b354519688f875aa1590b21bb6ff42f1b
---
 manifest      | 16 ++++++++++------
 manifest.uuid |  2 +-
 tool/lempar.c | 36 +++++++++++++++++++++++++-----------
 3 files changed, 36 insertions(+), 18 deletions(-)

diff --git a/manifest b/manifest
index f187cc0290..07c293333d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Simplification\sto\sthe\serror\shandling\slogic\sin\sthe\sextension\sloader.
-D 2017-12-23T14:39:36.160
+C Improved\sparser\stracing\soutput.
+D 2017-12-24T17:06:41.289
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -1609,7 +1609,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
 F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f
 F tool/lemon.c e6056373044d55296d21f81467dba7632bbb81dc49af072b3f0e76338771497e
-F tool/lempar.c 967ebf585cd09b11b89d255d213865109a9c4ff075680d22580a2826de288c89
+F tool/lempar.c a427c2375df118fa52e69174ffbbf9e26878096f0a109df6b77062d6032afe18
 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9
 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca
@@ -1687,7 +1687,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 05fee1a21ea398f1e4d6f1cf361657eff25ed6cd8f85ab398262dcfd30da57e9
-R 7fbfe3e61eca395fa5baf5f121a4d2ad
+P 07c773148d8db185fa54991df09298b64f4fef28879e6c9395759265e8183977
+Q +c4951833c2b976223c2393d82fd2606068c71cd19612ca9df4e26debab980e32
+R 3f4f31cde41f94adafa8015552d0561b
+T *branch * lemon-improvements
+T *sym-lemon-improvements *
+T -sym-trunk *
 U drh
-Z d82d9c01768cefc5beb206b92e1398bf
+Z 3f6d9f274761457d9d7dcc3ca6c0270c
diff --git a/manifest.uuid b/manifest.uuid
index 8454418cd5..bdf982cb88 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-07c773148d8db185fa54991df09298b64f4fef28879e6c9395759265e8183977
\ No newline at end of file
+25be5750545b7b0ed9e1a1baca31611b354519688f875aa1590b21bb6ff42f1b
\ No newline at end of file
diff --git a/tool/lempar.c b/tool/lempar.c
index da81ddd4bc..89c1c80f70 100644
--- a/tool/lempar.c
+++ b/tool/lempar.c
@@ -578,20 +578,21 @@ static void yyStackOverflow(yyParser *yypParser){
 ** Print tracing information for a SHIFT action
 */
 #ifndef NDEBUG
-static void yyTraceShift(yyParser *yypParser, int yyNewState){
+static void yyTraceShift(yyParser *yypParser, int yyNewState, const char *zTag){
   if( yyTraceFILE ){
     if( yyNewState<YYNSTATE ){
-      fprintf(yyTraceFILE,"%sShift '%s', go to state %d\n",
-         yyTracePrompt,yyTokenName[yypParser->yytos->major],
+      fprintf(yyTraceFILE,"%s%s '%s', go to state %d\n",
+         yyTracePrompt, zTag, yyTokenName[yypParser->yytos->major],
          yyNewState);
     }else{
-      fprintf(yyTraceFILE,"%sShift '%s'\n",
-         yyTracePrompt,yyTokenName[yypParser->yytos->major]);
+      fprintf(yyTraceFILE,"%s%s '%s', pending reduce %d\n",
+         yyTracePrompt, zTag, yyTokenName[yypParser->yytos->major],
+         yyNewState - YY_MIN_REDUCE);
     }
   }
 }
 #else
-# define yyTraceShift(X,Y)
+# define yyTraceShift(X,Y,Z)
 #endif
 
 /*
@@ -633,7 +634,7 @@ static void yy_shift(
   yytos->stateno = (YYACTIONTYPE)yyNewState;
   yytos->major = (YYCODETYPE)yyMajor;
   yytos->minor.yy0 = yyMinor;
-  yyTraceShift(yypParser, yyNewState);
+  yyTraceShift(yypParser, yyNewState, "Shift");
 }
 
 /* The following table contains information about every rule that
@@ -673,8 +674,14 @@ static void yy_reduce(
 #ifndef NDEBUG
   if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){
     yysize = yyRuleInfo[yyruleno].nrhs;
-    fprintf(yyTraceFILE, "%sReduce [%s], go to state %d.\n", yyTracePrompt,
-      yyRuleName[yyruleno], yymsp[yysize].stateno);
+    if( yysize ){
+      fprintf(yyTraceFILE, "%sReduce %d [%s], go to state %d.\n",
+        yyTracePrompt,
+        yyruleno, yyRuleName[yyruleno], yymsp[yysize].stateno);
+    }else{
+      fprintf(yyTraceFILE, "%sReduce %d [%s].\n",
+        yyTracePrompt, yyruleno, yyRuleName[yyruleno]);
+    }
   }
 #endif /* NDEBUG */
 
@@ -737,7 +744,7 @@ static void yy_reduce(
     yypParser->yytos = yymsp;
     yymsp->stateno = (YYACTIONTYPE)yyact;
     yymsp->major = (YYCODETYPE)yygoto;
-    yyTraceShift(yypParser, yyact);
+    yyTraceShift(yypParser, yyact, "... then shift");
   }
 }
 
@@ -848,7 +855,14 @@ void Parse(
 
 #ifndef NDEBUG
   if( yyTraceFILE ){
-    fprintf(yyTraceFILE,"%sInput '%s'\n",yyTracePrompt,yyTokenName[yymajor]);
+    int stateno = yypParser->yytos->stateno;
+    if( stateno < YY_MIN_REDUCE ){
+      fprintf(yyTraceFILE,"%sInput '%s' in state %d\n",
+              yyTracePrompt,yyTokenName[yymajor],stateno);
+    }else{
+      fprintf(yyTraceFILE,"%sInput '%s' with pending reduce %d\n",
+              yyTracePrompt,yyTokenName[yymajor],stateno-YY_MIN_REDUCE);
+    }
   }
 #endif
 

From 512795dfeae9dd816eefda3eee3222aedcd6d2b5 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sun, 24 Dec 2017 18:56:28 +0000
Subject: [PATCH 177/488] Fix a NULL pointer dereference after a syntax error
 that can occur as a result of check-in [6b2ff26c25bb9da3] yesterday.  This
 problem was discovered by the OSSFuzz.

FossilOrigin-Name: d49afb8f9804e96662d1e3cadc4c6643908706d848a53d5ed019919c98f2ccba
---
 manifest          | 14 +++++++-------
 manifest.uuid     |  2 +-
 src/build.c       |  1 +
 test/colname.test | 10 ++++++++++
 4 files changed, 19 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index f187cc0290..1c622c1102 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Simplification\sto\sthe\serror\shandling\slogic\sin\sthe\sextension\sloader.
-D 2017-12-23T14:39:36.160
+C Fix\sa\sNULL\spointer\sdereference\safter\sa\ssyntax\serror\sthat\scan\soccur\sas\sa\nresult\sof\scheck-in\s[6b2ff26c25bb9da3]\syesterday.\s\sThis\sproblem\swas\ndiscovered\sby\sthe\sOSSFuzz.
+D 2017-12-24T18:56:28.786
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -423,7 +423,7 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
 F src/btree.c b83a6b03f160528020bb965f0c3a40af5286cd4923c3870fd218177f03a120a7
 F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09
 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc
-F src/build.c ed567f088edbc305dad33a6b14e08f8216a3860f6bad1d180450d5a5414bf346
+F src/build.c ab5bdf955c85bcd56acbf310a48bbd50b4b92079efa40d997a7e4246f8e03741
 F src/callback.c fe677cb5f5abb02f7a772a62a98c2f516426081df68856e8f2d5f950929b966a
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
@@ -682,7 +682,7 @@ F test/collate9.test 3adcc799229545940df2f25308dd1ad65869145a
 F test/collateA.test b8218ab90d1fa5c59dcf156efabb1b2599c580d6
 F test/collateB.test 1e68906951b846570f29f20102ed91d29e634854ee47454d725f2151ecac0b95
 F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1
-F test/colname.test a7ecb8f1d6d8b30a6cf8fa84a2cd6f6e91cad8296376fabe485cf93cd5eb6229
+F test/colname.test 36da785927822ecd0de979459e27e9be63f458dd08d3edde41af3af37a337d58
 F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db
 F test/conflict2.test bb0b94cf7196c64a3cbd815c66d3ee98c2fecd9c
 F test/conflict3.test a83db76a6c3503b2fa057c7bfb08c318d8a422202d8bc5b86226e078e5b49ff9
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 05fee1a21ea398f1e4d6f1cf361657eff25ed6cd8f85ab398262dcfd30da57e9
-R 7fbfe3e61eca395fa5baf5f121a4d2ad
+P 07c773148d8db185fa54991df09298b64f4fef28879e6c9395759265e8183977
+R 2e4222d820aa06549d33319e9e33c627
 U drh
-Z d82d9c01768cefc5beb206b92e1398bf
+Z 67298e45eb2d4d992f0cdf5678deffdf
diff --git a/manifest.uuid b/manifest.uuid
index 8454418cd5..d58d2c0dd5 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-07c773148d8db185fa54991df09298b64f4fef28879e6c9395759265e8183977
\ No newline at end of file
+d49afb8f9804e96662d1e3cadc4c6643908706d848a53d5ed019919c98f2ccba
\ No newline at end of file
diff --git a/src/build.c b/src/build.c
index 01d8972415..390dae5d3b 100644
--- a/src/build.c
+++ b/src/build.c
@@ -1965,6 +1965,7 @@ void sqlite3EndTable(
       pParse->nTab = 2;
       addrTop = sqlite3VdbeCurrentAddr(v) + 1;
       sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop);
+      if( pParse->nErr ) return;
       pSelTab = sqlite3ResultSetOfSelect(pParse, pSelect);
       if( pSelTab==0 ) return;
       assert( p->aCol==0 );
diff --git a/test/colname.test b/test/colname.test
index 5a40286773..cacf91e4e3 100644
--- a/test/colname.test
+++ b/test/colname.test
@@ -398,6 +398,16 @@ do_execsql_test colname-9.320 {
   SELECT name FROM pragma_table_info('t2');
 } {Bbb}
 
+# Issue detected by clusterfuzz on 2017-12-24 (Christmas Eve)
+# caused by check-in https://sqlite.org/src/info/6b2ff26c25
+#
+# Prior to being fixed, the following CREATE TABLE was dereferencing
+# a NULL pointer and segfaulting.
+#
+do_catchsql_test colname-9.400 {
+  CREATE TABLE t4 AS SELECT #0;
+} {1 {near "#0": syntax error}}
+
 
 # Make sure the quotation marks get removed from the column names
 # when constructing a new table from an aggregate SELECT.

From 5c8241b8750ce919b4151761a1d721af58bfbf7b Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sun, 24 Dec 2017 23:38:10 +0000
Subject: [PATCH 178/488] In the LEMON-generated parser, rearrange the meanings
 of integer action codes so that reduce actions occur last.  This means that
 the most common case (reduce actions) can be recognized with a single
 comparison operation, thus speeding up the main parser loop, slightly.

FossilOrigin-Name: 7bfe7a360261ac7227840db49487c2f0fe338a2f1b868fcaada1e04a8d2b8f7a
---
 manifest      |  18 ++++-----
 manifest.uuid |   2 +-
 tool/lemon.c  | 100 ++++++++++++++++++++++++++++++++------------------
 tool/lempar.c |  16 ++++----
 4 files changed, 81 insertions(+), 55 deletions(-)

diff --git a/manifest b/manifest
index 07c293333d..d8e37c9644 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improved\sparser\stracing\soutput.
-D 2017-12-24T17:06:41.289
+C In\sthe\sLEMON-generated\sparser,\srearrange\sthe\smeanings\sof\sinteger\saction\scodes\nso\sthat\sreduce\sactions\soccur\slast.\s\sThis\smeans\sthat\sthe\smost\scommon\scase\n(reduce\sactions)\scan\sbe\srecognized\swith\sa\ssingle\scomparison\soperation,\sthus\nspeeding\sup\sthe\smain\sparser\sloop,\sslightly.
+D 2017-12-24T23:38:10.370
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -1608,8 +1608,8 @@ F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4
 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
 F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f
-F tool/lemon.c e6056373044d55296d21f81467dba7632bbb81dc49af072b3f0e76338771497e
-F tool/lempar.c a427c2375df118fa52e69174ffbbf9e26878096f0a109df6b77062d6032afe18
+F tool/lemon.c 7c6919d98e459c0f8a3673be64b03425553733dba01c12939b2fadc30e4e2804
+F tool/lempar.c c8dd4dcf0bca9d7c27c62f7df12882c30db749cd9bb83d6f71796b9fabb94f6c
 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9
 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca
@@ -1687,11 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 07c773148d8db185fa54991df09298b64f4fef28879e6c9395759265e8183977
-Q +c4951833c2b976223c2393d82fd2606068c71cd19612ca9df4e26debab980e32
-R 3f4f31cde41f94adafa8015552d0561b
-T *branch * lemon-improvements
-T *sym-lemon-improvements *
-T -sym-trunk *
+P 25be5750545b7b0ed9e1a1baca31611b354519688f875aa1590b21bb6ff42f1b
+R 02413a9b8686f5cb855fd9a72c28a347
 U drh
-Z 3f6d9f274761457d9d7dcc3ca6c0270c
+Z a4798d5b361261f3f5c39967bad11abf
diff --git a/manifest.uuid b/manifest.uuid
index bdf982cb88..d4db86c744 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-25be5750545b7b0ed9e1a1baca31611b354519688f875aa1590b21bb6ff42f1b
\ No newline at end of file
+7bfe7a360261ac7227840db49487c2f0fe338a2f1b868fcaada1e04a8d2b8f7a
\ No newline at end of file
diff --git a/tool/lemon.c b/tool/lemon.c
index acc5450c99..33ef43d1b1 100644
--- a/tool/lemon.c
+++ b/tool/lemon.c
@@ -384,6 +384,12 @@ struct lemon {
   int nrule;               /* Number of rules */
   int nsymbol;             /* Number of terminal and nonterminal symbols */
   int nterminal;           /* Number of terminal symbols */
+  int minShiftReduce;      /* Minimum shift-reduce action value */
+  int errAction;           /* Error action value */
+  int accAction;           /* Accept action value */
+  int noAction;            /* No-op action value */
+  int minReduce;           /* Minimum reduce action */
+  int maxAction;           /* Maximum action value of any kind */
   struct symbol **symbols; /* Sorted array of pointers to symbols */
   int errorcnt;            /* Number of errors */
   struct symbol *errsym;   /* The error symbol */
@@ -3020,6 +3026,27 @@ PRIVATE FILE *file_open(
   return fp;
 }
 
+/* Print the text of a rule
+*/
+void rule_print(FILE *out, struct rule *rp){
+  int i, j;
+  fprintf(out, "%s",rp->lhs->name);
+  /*    if( rp->lhsalias ) fprintf(out,"(%s)",rp->lhsalias); */
+  fprintf(out," ::=");
+  for(i=0; i<rp->nrhs; i++){
+    struct symbol *sp = rp->rhs[i];
+    if( sp->type==MULTITERMINAL ){
+      fprintf(out," %s", sp->subsym[0]->name);
+      for(j=1; j<sp->nsubsym; j++){
+        fprintf(out,"|%s", sp->subsym[j]->name);
+      }
+    }else{
+      fprintf(out," %s", sp->name);
+    }
+    /* if( rp->rhsalias[i] ) fprintf(out,"(%s)",rp->rhsalias[i]); */
+  }
+}
+
 /* Duplicate the input file without comments and without actions
 ** on rules */
 void Reprint(struct lemon *lemp)
@@ -3047,21 +3074,7 @@ void Reprint(struct lemon *lemp)
     printf("\n");
   }
   for(rp=lemp->rule; rp; rp=rp->next){
-    printf("%s",rp->lhs->name);
-    /*    if( rp->lhsalias ) printf("(%s)",rp->lhsalias); */
-    printf(" ::=");
-    for(i=0; i<rp->nrhs; i++){
-      sp = rp->rhs[i];
-      if( sp->type==MULTITERMINAL ){
-        printf(" %s", sp->subsym[0]->name);
-        for(j=1; j<sp->nsubsym; j++){
-          printf("|%s", sp->subsym[j]->name);
-        }
-      }else{
-        printf(" %s", sp->name);
-      }
-      /* if( rp->rhsalias[i] ) printf("(%s)",rp->rhsalias[i]); */
-    }
+    rule_print(stdout, rp);
     printf(".");
     if( rp->precsym ) printf(" [%s]",rp->precsym->name);
     /* if( rp->code ) printf("\n    %s",rp->code); */
@@ -3321,16 +3334,19 @@ PRIVATE int compute_action(struct lemon *lemp, struct action *ap)
   switch( ap->type ){
     case SHIFT:  act = ap->x.stp->statenum;                        break;
     case SHIFTREDUCE: {
-      act = ap->x.rp->iRule + lemp->nstate;
       /* Since a SHIFT is inherient after a prior REDUCE, convert any
       ** SHIFTREDUCE action with a nonterminal on the LHS into a simple
       ** REDUCE action: */
-      if( ap->sp->index>=lemp->nterminal ) act += lemp->nrule;
+      if( ap->sp->index>=lemp->nterminal ){
+        act = lemp->minReduce + ap->x.rp->iRule;
+      }else{
+        act = lemp->minShiftReduce + ap->x.rp->iRule;
+      }
       break;
     }
-    case REDUCE: act = ap->x.rp->iRule + lemp->nstate+lemp->nrule; break;
-    case ERROR:  act = lemp->nstate + lemp->nrule*2;               break;
-    case ACCEPT: act = lemp->nstate + lemp->nrule*2 + 1;           break;
+    case REDUCE: act = lemp->minReduce + ap->x.rp->iRule;          break;
+    case ERROR:  act = lemp->errAction;                            break;
+    case ACCEPT: act = lemp->accAction;                            break;
     default:     act = -1; break;
   }
   return act;
@@ -4038,6 +4054,13 @@ void ReportTable(
   int mnNtOfst, mxNtOfst;
   struct axset *ax;
 
+  lemp->minShiftReduce = lemp->nstate;
+  lemp->errAction = lemp->minShiftReduce + lemp->nrule;
+  lemp->accAction = lemp->errAction + 1;
+  lemp->noAction = lemp->accAction + 1;
+  lemp->minReduce = lemp->noAction + 1;
+  lemp->maxAction = lemp->minReduce + lemp->nrule;
+
   in = tplt_open(lemp);
   if( in==0 ) return;
   out = file_open(lemp,".c","wb");
@@ -4076,7 +4099,7 @@ void ReportTable(
     minimum_size_type(0, lemp->nsymbol+1, &szCodeType)); lineno++;
   fprintf(out,"#define YYNOCODE %d\n",lemp->nsymbol+1);  lineno++;
   fprintf(out,"#define YYACTIONTYPE %s\n",
-    minimum_size_type(0,lemp->nstate+lemp->nrule*2+5,&szActionType)); lineno++;
+    minimum_size_type(0,lemp->maxAction,&szActionType)); lineno++;
   if( lemp->wildcard ){
     fprintf(out,"#define YYWILDCARD %d\n",
        lemp->wildcard->index); lineno++;
@@ -4201,15 +4224,16 @@ void ReportTable(
   fprintf(out,"#define YYNSTATE             %d\n",lemp->nxstate);  lineno++;
   fprintf(out,"#define YYNRULE              %d\n",lemp->nrule);  lineno++;
   fprintf(out,"#define YY_MAX_SHIFT         %d\n",lemp->nxstate-1); lineno++;
-  fprintf(out,"#define YY_MIN_SHIFTREDUCE   %d\n",lemp->nstate); lineno++;
-  i = lemp->nstate + lemp->nrule;
+  i = lemp->minShiftReduce;
+  fprintf(out,"#define YY_MIN_SHIFTREDUCE   %d\n",i); lineno++;
+  i += lemp->nrule;
   fprintf(out,"#define YY_MAX_SHIFTREDUCE   %d\n", i-1); lineno++;
-  fprintf(out,"#define YY_MIN_REDUCE        %d\n", i); lineno++;
-  i = lemp->nstate + lemp->nrule*2;
+  fprintf(out,"#define YY_ERROR_ACTION      %d\n", lemp->errAction); lineno++;
+  fprintf(out,"#define YY_ACCEPT_ACTION     %d\n", lemp->accAction); lineno++;
+  fprintf(out,"#define YY_NO_ACTION         %d\n", lemp->noAction); lineno++;
+  fprintf(out,"#define YY_MIN_REDUCE        %d\n", lemp->minReduce); lineno++;
+  i = lemp->minReduce + lemp->nrule;
   fprintf(out,"#define YY_MAX_REDUCE        %d\n", i-1); lineno++;
-  fprintf(out,"#define YY_ERROR_ACTION      %d\n", i); lineno++;
-  fprintf(out,"#define YY_ACCEPT_ACTION     %d\n", i+1); lineno++;
-  fprintf(out,"#define YY_NO_ACTION         %d\n", i+2); lineno++;
   tplt_xfer(lemp->name,in,out,&lineno);
 
   /* Now output the action table and its associates:
@@ -4231,7 +4255,7 @@ void ReportTable(
   fprintf(out,"static const YYACTIONTYPE yy_action[] = {\n"); lineno++;
   for(i=j=0; i<n; i++){
     int action = acttab_yyaction(pActtab, i);
-    if( action<0 ) action = lemp->nstate + lemp->nrule + 2;
+    if( action<0 ) action = lemp->noAction;
     if( j==0 ) fprintf(out," /* %5d */ ", i);
     fprintf(out, " %4d,", action);
     if( j==9 || i==n-1 ){
@@ -4320,7 +4344,11 @@ void ReportTable(
   for(i=j=0; i<n; i++){
     stp = lemp->sorted[i];
     if( j==0 ) fprintf(out," /* %5d */ ", i);
-    fprintf(out, " %4d,", stp->iDfltReduce+lemp->nstate+lemp->nrule);
+    if( stp->iDfltReduce<0 ){
+      fprintf(out, " %4d,", lemp->errAction);
+    }else{
+      fprintf(out, " %4d,", stp->iDfltReduce + lemp->minReduce);
+    }
     if( j==9 || i==n-1 ){
       fprintf(out, "\n"); lineno++;
       j = 0;
@@ -4401,7 +4429,7 @@ void ReportTable(
       if( sp==0 || sp->type==TERMINAL ||
           sp->index<=0 || sp->destructor!=0 ) continue;
       if( once ){
-        fprintf(out, "      /* Default NON-TERMINAL Destructor */\n"); lineno++;
+        fprintf(out, "      /* Default NON-TERMINAL Destructor */\n");lineno++;
         once = 0;
       }
       fprintf(out,"    case %d: /* %s */\n", sp->index, sp->name); lineno++;
@@ -4444,8 +4472,10 @@ void ReportTable(
   ** Note: This code depends on the fact that rules are number
   ** sequentually beginning with 0.
   */
-  for(rp=lemp->rule; rp; rp=rp->next){
-    fprintf(out,"  { %d, %d },\n",rp->lhs->index,-rp->nrhs); lineno++;
+  for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){
+    fprintf(out,"  { %4d, %4d }, /* (%d) ",rp->lhs->index,-rp->nrhs,i);
+    rule_print(out, rp);
+    fprintf(out," */\n"); lineno++;
   }
   tplt_xfer(lemp->name,in,out,&lineno);
 
@@ -4711,7 +4741,7 @@ void ResortStates(struct lemon *lemp)
   for(i=0; i<lemp->nstate; i++){
     stp = lemp->sorted[i];
     stp->nTknAct = stp->nNtAct = 0;
-    stp->iDfltReduce = lemp->nrule;  /* Init dflt action to "syntax error" */
+    stp->iDfltReduce = -1; /* Init dflt action to "syntax error" */
     stp->iTknOfst = NO_OFFSET;
     stp->iNtOfst = NO_OFFSET;
     for(ap=stp->ap; ap; ap=ap->next){
@@ -4723,7 +4753,7 @@ void ResortStates(struct lemon *lemp)
           stp->nNtAct++;
         }else{
           assert( stp->autoReduce==0 || stp->pDfltReduce==ap->x.rp );
-          stp->iDfltReduce = iAction - lemp->nstate - lemp->nrule;
+          stp->iDfltReduce = iAction;
         }
       }
     }
diff --git a/tool/lempar.c b/tool/lempar.c
index 89c1c80f70..7d25175ca7 100644
--- a/tool/lempar.c
+++ b/tool/lempar.c
@@ -75,11 +75,11 @@
 **    YY_MAX_SHIFT       Maximum value for shift actions
 **    YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions
 **    YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions
-**    YY_MIN_REDUCE      Minimum value for reduce actions
-**    YY_MAX_REDUCE      Maximum value for reduce actions
 **    YY_ERROR_ACTION    The yy_action[] code for syntax error
 **    YY_ACCEPT_ACTION   The yy_action[] code for accept
 **    YY_NO_ACTION       The yy_action[] code for no-op
+**    YY_MIN_REDUCE      Minimum value for reduce actions
+**    YY_MAX_REDUCE      Maximum value for reduce actions
 */
 #ifndef INTERFACE
 # define INTERFACE 1
@@ -115,9 +115,6 @@
 **   N between YY_MIN_SHIFTREDUCE       Shift to an arbitrary state then
 **     and YY_MAX_SHIFTREDUCE           reduce by rule N-YY_MIN_SHIFTREDUCE.
 **
-**   N between YY_MIN_REDUCE            Reduce by rule N-YY_MIN_REDUCE
-**     and YY_MAX_REDUCE
-**
 **   N == YY_ERROR_ACTION               A syntax error has occurred.
 **
 **   N == YY_ACCEPT_ACTION              The parser accepts its input.
@@ -125,6 +122,9 @@
 **   N == YY_NO_ACTION                  No such action.  Denotes unused
 **                                      slots in the yy_action[] table.
 **
+**   N between YY_MIN_REDUCE            Reduce by rule N-YY_MIN_REDUCE
+**     and YY_MAX_REDUCE
+**
 ** The action table is constructed as a single large table named yy_action[].
 ** Given state S and lookahead X, the action is computed as either:
 **
@@ -868,14 +868,14 @@ void Parse(
 
   do{
     yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor);
-    if( yyact <= YY_MAX_SHIFTREDUCE ){
+    if( yyact >= YY_MIN_REDUCE ){
+      yy_reduce(yypParser,yyact-YY_MIN_REDUCE,yymajor,yyminor);
+    }else if( yyact <= YY_MAX_SHIFTREDUCE ){
       yy_shift(yypParser,yyact,yymajor,yyminor);
 #ifndef YYNOERRORRECOVERY
       yypParser->yyerrcnt--;
 #endif
       yymajor = YYNOCODE;
-    }else if( yyact <= YY_MAX_REDUCE ){
-      yy_reduce(yypParser,yyact-YY_MIN_REDUCE,yymajor,yyminor);
     }else{
       assert( yyact == YY_ERROR_ACTION );
       yyminorunion.yy0 = yyminor;

From ef53a9f0af73a66d8844fb2b58c669213677530e Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 25 Dec 2017 00:10:05 +0000
Subject: [PATCH 179/488] In the LEMON-generated parser, avoid unnecessary
 tests for the acceptance state.

FossilOrigin-Name: fdbb35c54f2b6cb65d04ac295f207ff3e69360e0558348c77eb5e62691807046
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 tool/lempar.c | 21 ++++++++++-----------
 3 files changed, 17 insertions(+), 18 deletions(-)

diff --git a/manifest b/manifest
index d8e37c9644..f8d8364cbb 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\sthe\sLEMON-generated\sparser,\srearrange\sthe\smeanings\sof\sinteger\saction\scodes\nso\sthat\sreduce\sactions\soccur\slast.\s\sThis\smeans\sthat\sthe\smost\scommon\scase\n(reduce\sactions)\scan\sbe\srecognized\swith\sa\ssingle\scomparison\soperation,\sthus\nspeeding\sup\sthe\smain\sparser\sloop,\sslightly.
-D 2017-12-24T23:38:10.370
+C In\sthe\sLEMON-generated\sparser,\savoid\sunnecessary\stests\sfor\sthe\sacceptance\nstate.
+D 2017-12-25T00:10:05.418
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -1609,7 +1609,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
 F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f
 F tool/lemon.c 7c6919d98e459c0f8a3673be64b03425553733dba01c12939b2fadc30e4e2804
-F tool/lempar.c c8dd4dcf0bca9d7c27c62f7df12882c30db749cd9bb83d6f71796b9fabb94f6c
+F tool/lempar.c 8062f219b4ce349853cb3ab3ebd3ab44466604235347457d703a9f4252e76dd5
 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9
 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 25be5750545b7b0ed9e1a1baca31611b354519688f875aa1590b21bb6ff42f1b
-R 02413a9b8686f5cb855fd9a72c28a347
+P 7bfe7a360261ac7227840db49487c2f0fe338a2f1b868fcaada1e04a8d2b8f7a
+R 904d9f58d13471f5c0ad3064ead9b812
 U drh
-Z a4798d5b361261f3f5c39967bad11abf
+Z c00f50853c78fa733a13b33ab92ee708
diff --git a/manifest.uuid b/manifest.uuid
index d4db86c744..e026b0ddc4 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7bfe7a360261ac7227840db49487c2f0fe338a2f1b868fcaada1e04a8d2b8f7a
\ No newline at end of file
+fdbb35c54f2b6cb65d04ac295f207ff3e69360e0558348c77eb5e62691807046
\ No newline at end of file
diff --git a/tool/lempar.c b/tool/lempar.c
index 7d25175ca7..6c6ca77dde 100644
--- a/tool/lempar.c
+++ b/tool/lempar.c
@@ -472,7 +472,7 @@ static unsigned int yy_find_shift_action(
   int i;
   int stateno = pParser->yytos->stateno;
  
-  if( stateno>=YY_MIN_REDUCE ) return stateno;
+  if( stateno>YY_MAX_SHIFT ) return stateno;
   assert( stateno <= YY_SHIFT_COUNT );
   do{
     i = yy_shift_ofst[stateno];
@@ -736,16 +736,11 @@ static void yy_reduce(
   /* It is not possible for a REDUCE to be followed by an error */
   assert( yyact!=YY_ERROR_ACTION );
 
-  if( yyact==YY_ACCEPT_ACTION ){
-    yypParser->yytos += yysize;
-    yy_accept(yypParser);
-  }else{
-    yymsp += yysize+1;
-    yypParser->yytos = yymsp;
-    yymsp->stateno = (YYACTIONTYPE)yyact;
-    yymsp->major = (YYCODETYPE)yygoto;
-    yyTraceShift(yypParser, yyact, "... then shift");
-  }
+  yymsp += yysize+1;
+  yypParser->yytos = yymsp;
+  yymsp->stateno = (YYACTIONTYPE)yyact;
+  yymsp->major = (YYCODETYPE)yygoto;
+  yyTraceShift(yypParser, yyact, "... then shift");
 }
 
 /*
@@ -876,6 +871,10 @@ void Parse(
       yypParser->yyerrcnt--;
 #endif
       yymajor = YYNOCODE;
+    }else if( yyact==YY_ACCEPT_ACTION ){
+      yypParser->yytos--;
+      yy_accept(yypParser);
+      return;
     }else{
       assert( yyact == YY_ERROR_ACTION );
       yyminorunion.yy0 = yyminor;

From 3a9d6c7156e33c63f03f12be8e2168c8d0aa7d2b Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 25 Dec 2017 04:15:38 +0000
Subject: [PATCH 180/488] Enhance LEMON so that it generates the action table
 in such a way that no range check is needed on the lookahead table to verify
 that the next input token is valid.  This makes the lookahead table slightly
 larger (about 120 bytes) but helps the parser to run faster.

FossilOrigin-Name: 7eb0198d0102e97e4b7ad9e359d95985e55e09c510ea4b360265ac8feb9ed814
---
 manifest      | 14 +++++------
 manifest.uuid |  2 +-
 tool/lemon.c  | 65 +++++++++++++++++++++++++++++++++++++--------------
 tool/lempar.c | 16 ++++---------
 4 files changed, 61 insertions(+), 36 deletions(-)

diff --git a/manifest b/manifest
index f8d8364cbb..d9f3940ae0 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\sthe\sLEMON-generated\sparser,\savoid\sunnecessary\stests\sfor\sthe\sacceptance\nstate.
-D 2017-12-25T00:10:05.418
+C Enhance\sLEMON\sso\sthat\sit\sgenerates\sthe\saction\stable\sin\ssuch\sa\sway\sthat\sno\nrange\scheck\sis\sneeded\son\sthe\slookahead\stable\sto\sverify\sthat\sthe\snext\sinput\ntoken\sis\svalid.\s\sThis\smakes\sthe\slookahead\stable\sslightly\slarger\s(about\s120\nbytes)\sbut\shelps\sthe\sparser\sto\srun\sfaster.
+D 2017-12-25T04:15:38.668
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -1608,8 +1608,8 @@ F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4
 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
 F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f
-F tool/lemon.c 7c6919d98e459c0f8a3673be64b03425553733dba01c12939b2fadc30e4e2804
-F tool/lempar.c 8062f219b4ce349853cb3ab3ebd3ab44466604235347457d703a9f4252e76dd5
+F tool/lemon.c c8d7ce4fe7a90f7fa6a5985452aa926fcf25376cf90095c9d06c432ab0bebdbc
+F tool/lempar.c 427ee280f3c3781e82bbee21f428bc8ae18ab245d4f66d65da46b598ded81648
 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9
 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 7bfe7a360261ac7227840db49487c2f0fe338a2f1b868fcaada1e04a8d2b8f7a
-R 904d9f58d13471f5c0ad3064ead9b812
+P fdbb35c54f2b6cb65d04ac295f207ff3e69360e0558348c77eb5e62691807046
+R f14cbf100834d359550a23c924f7ebaa
 U drh
-Z c00f50853c78fa733a13b33ab92ee708
+Z bb1d7d0db816330a6f1e2b8011c2819b
diff --git a/manifest.uuid b/manifest.uuid
index e026b0ddc4..b69ed5a1cc 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-fdbb35c54f2b6cb65d04ac295f207ff3e69360e0558348c77eb5e62691807046
\ No newline at end of file
+7eb0198d0102e97e4b7ad9e359d95985e55e09c510ea4b360265ac8feb9ed814
\ No newline at end of file
diff --git a/tool/lemon.c b/tool/lemon.c
index 33ef43d1b1..96dc756d1b 100644
--- a/tool/lemon.c
+++ b/tool/lemon.c
@@ -413,6 +413,7 @@ struct lemon {
   char *tokenprefix;       /* A prefix added to token names in the .h file */
   int nconflict;           /* Number of parsing conflicts */
   int nactiontab;          /* Number of entries in the yy_action[] table */
+  int nlookaheadtab;       /* Number of entries in yy_lookahead[] */
   int tablesize;           /* Total table size of all tables in bytes */
   int basisflag;           /* Print only basis configurations */
   int has_fallback;        /* True if any %fallback is seen in the grammar */
@@ -589,10 +590,12 @@ struct acttab {
   int mxLookahead;             /* Maximum aLookahead[].lookahead */
   int nLookahead;              /* Used slots in aLookahead[] */
   int nLookaheadAlloc;         /* Slots allocated in aLookahead[] */
+  int nterminal;               /* Number of terminal symbols */
+  int nsymbol;                 /* total number of symbols */
 };
 
 /* Return the number of entries in the yy_action table */
-#define acttab_size(X) ((X)->nAction)
+#define acttab_lookahead_size(X) ((X)->nAction)
 
 /* The value for the N-th entry in yy_action */
 #define acttab_yyaction(X,N)  ((X)->aAction[N].action)
@@ -608,13 +611,15 @@ void acttab_free(acttab *p){
 }
 
 /* Allocate a new acttab structure */
-acttab *acttab_alloc(void){
+acttab *acttab_alloc(int nsymbol, int nterminal){
   acttab *p = (acttab *) calloc( 1, sizeof(*p) );
   if( p==0 ){
     fprintf(stderr,"Unable to allocate memory for a new acttab.");
     exit(1);
   }
   memset(p, 0, sizeof(*p));
+  p->nsymbol = nsymbol;
+  p->nterminal = nterminal;
   return p;
 }
 
@@ -655,16 +660,24 @@ void acttab_action(acttab *p, int lookahead, int action){
 ** to an empty set in preparation for a new round of acttab_action() calls.
 **
 ** Return the offset into the action table of the new transaction.
+**
+** If the makeItSafe parameter is true, then the offset is chosen so that
+** it is impossible to overread the yy_lookaside[] table regardless of
+** the lookaside token.  This is done for the terminal symbols, as they
+** come from external inputs and can contain syntax errors.  When makeItSafe
+** is false, there is more flexibility in selecting offsets, resulting in
+** a smaller table.  For non-terminal symbols, which are never syntax errors,
+** makeItSafe can be false.
 */
-int acttab_insert(acttab *p){
-  int i, j, k, n;
+int acttab_insert(acttab *p, int makeItSafe){
+  int i, j, k, n, end;
   assert( p->nLookahead>0 );
 
   /* Make sure we have enough space to hold the expanded action table
   ** in the worst case.  The worst case occurs if the transaction set
   ** must be appended to the current action table
   */
-  n = p->mxLookahead + 1;
+  n = p->nsymbol + 1;
   if( p->nAction + n >= p->nActionAlloc ){
     int oldAlloc = p->nActionAlloc;
     p->nActionAlloc = p->nAction + n + p->nActionAlloc + 20;
@@ -686,7 +699,8 @@ int acttab_insert(acttab *p){
   **
   ** i is the index in p->aAction[] where p->mnLookahead is inserted.
   */
-  for(i=p->nAction-1; i>=0; i--){
+  end = makeItSafe ? p->mnLookahead : 0;
+  for(i=p->nAction-1; i>=end; i--){
     if( p->aAction[i].lookahead==p->mnLookahead ){
       /* All lookaheads and actions in the aLookahead[] transaction
       ** must match against the candidate aAction[i] entry. */
@@ -716,12 +730,13 @@ int acttab_insert(acttab *p){
   ** an empty offset in the aAction[] table in which we can add the
   ** aLookahead[] transaction.
   */
-  if( i<0 ){
+  if( i<end ){
     /* Look for holes in the aAction[] table that fit the current
     ** aLookahead[] transaction.  Leave i set to the offset of the hole.
     ** If no holes are found, i is left at p->nAction, which means the
     ** transaction will be appended. */
-    for(i=0; i<p->nActionAlloc - p->mxLookahead; i++){
+    i = makeItSafe ? p->mnLookahead : 0;
+    for(; i<p->nActionAlloc - p->mxLookahead; i++){
       if( p->aAction[i].lookahead<0 ){
         for(j=0; j<p->nLookahead; j++){
           k = p->aLookahead[j].lookahead - p->mnLookahead + i;
@@ -739,11 +754,19 @@ int acttab_insert(acttab *p){
     }
   }
   /* Insert transaction set at index i. */
+#if 0
+  printf("Acttab:");
+  for(j=0; j<p->nLookahead; j++){
+    printf(" %d", p->aLookahead[j].lookahead);
+  }
+  printf(" inserted at %d\n", i);
+#endif
   for(j=0; j<p->nLookahead; j++){
     k = p->aLookahead[j].lookahead - p->mnLookahead + i;
     p->aAction[k] = p->aLookahead[j];
     if( k>=p->nAction ) p->nAction = k+1;
   }
+  if( makeItSafe && i+p->nterminal>p->nAction ) p->nAction = i+p->nterminal;
   p->nLookahead = 0;
 
   /* Return the offset that is added to the lookahead in order to get the
@@ -751,6 +774,16 @@ int acttab_insert(acttab *p){
   return i - p->mnLookahead;
 }
 
+/*
+** Return the size of the action table without the trailing syntax error
+** entries.
+*/
+int acttab_action_size(acttab *p){
+  int n = p->nAction;
+  while( n>0 && p->aAction[n-1].lookahead<0 ){ n--; }
+  return n;
+}
+
 /********************** From the file "build.c" *****************************/
 /*
 ** Routines to construction the finite state machine for the LEMON
@@ -1724,6 +1757,7 @@ int main(int argc, char **argv)
     stats_line("states", lem.nxstate);
     stats_line("conflicts", lem.nconflict);
     stats_line("action table entries", lem.nactiontab);
+    stats_line("lookahead table entries", lem.nlookaheadtab);
     stats_line("total table size (bytes)", lem.tablesize);
   }
   if( lem.nconflict > 0 ){
@@ -4167,7 +4201,7 @@ void ReportTable(
   ** of placing the largest action sets first */
   for(i=0; i<lemp->nxstate*2; i++) ax[i].iOrder = i;
   qsort(ax, lemp->nxstate*2, sizeof(ax[0]), axset_compare);
-  pActtab = acttab_alloc();
+  pActtab = acttab_alloc(lemp->nsymbol, lemp->nterminal);
   for(i=0; i<lemp->nxstate*2 && ax[i].nAction>0; i++){
     stp = ax[i].stp;
     if( ax[i].isTkn ){
@@ -4178,7 +4212,7 @@ void ReportTable(
         if( action<0 ) continue;
         acttab_action(pActtab, ap->sp->index, action);
       }
-      stp->iTknOfst = acttab_insert(pActtab);
+      stp->iTknOfst = acttab_insert(pActtab, 1);
       if( stp->iTknOfst<mnTknOfst ) mnTknOfst = stp->iTknOfst;
       if( stp->iTknOfst>mxTknOfst ) mxTknOfst = stp->iTknOfst;
     }else{
@@ -4190,7 +4224,7 @@ void ReportTable(
         if( action<0 ) continue;
         acttab_action(pActtab, ap->sp->index, action);
       }
-      stp->iNtOfst = acttab_insert(pActtab);
+      stp->iNtOfst = acttab_insert(pActtab, 0);
       if( stp->iNtOfst<mnNtOfst ) mnNtOfst = stp->iNtOfst;
       if( stp->iNtOfst>mxNtOfst ) mxNtOfst = stp->iNtOfst;
     }
@@ -4249,7 +4283,7 @@ void ReportTable(
   */
 
   /* Output the yy_action table */
-  lemp->nactiontab = n = acttab_size(pActtab);
+  lemp->nactiontab = n = acttab_action_size(pActtab);
   lemp->tablesize += n*szActionType;
   fprintf(out,"#define YY_ACTTAB_COUNT (%d)\n", n); lineno++;
   fprintf(out,"static const YYACTIONTYPE yy_action[] = {\n"); lineno++;
@@ -4268,6 +4302,7 @@ void ReportTable(
   fprintf(out, "};\n"); lineno++;
 
   /* Output the yy_lookahead table */
+  lemp->nlookaheadtab = n = acttab_lookahead_size(pActtab);
   lemp->tablesize += n*szCodeType;
   fprintf(out,"static const YYCODETYPE yy_lookahead[] = {\n"); lineno++;
   for(i=j=0; i<n; i++){
@@ -4287,7 +4322,6 @@ void ReportTable(
   /* Output the yy_shift_ofst[] table */
   n = lemp->nxstate;
   while( n>0 && lemp->sorted[n-1]->iTknOfst==NO_OFFSET ) n--;
-  fprintf(out, "#define YY_SHIFT_USE_DFLT (%d)\n", lemp->nactiontab); lineno++;
   fprintf(out, "#define YY_SHIFT_COUNT    (%d)\n", n-1); lineno++;
   fprintf(out, "#define YY_SHIFT_MIN      (%d)\n", mnTknOfst); lineno++;
   fprintf(out, "#define YY_SHIFT_MAX      (%d)\n", mxTknOfst); lineno++;
@@ -4312,7 +4346,6 @@ void ReportTable(
   fprintf(out, "};\n"); lineno++;
 
   /* Output the yy_reduce_ofst[] table */
-  fprintf(out, "#define YY_REDUCE_USE_DFLT (%d)\n", mnNtOfst-1); lineno++;
   n = lemp->nxstate;
   while( n>0 && lemp->sorted[n-1]->iNtOfst==NO_OFFSET ) n--;
   fprintf(out, "#define YY_REDUCE_COUNT (%d)\n", n-1); lineno++;
@@ -4382,10 +4415,8 @@ void ReportTable(
   */
   for(i=0; i<lemp->nsymbol; i++){
     lemon_sprintf(line,"\"%s\",",lemp->symbols[i]->name);
-    fprintf(out,"  %-15s",line);
-    if( (i&3)==3 ){ fprintf(out,"\n"); lineno++; }
+    fprintf(out,"  /* %4d */ \"%s\",\n",i, lemp->symbols[i]->name); lineno++;
   }
-  if( (i&3)!=0 ){ fprintf(out,"\n"); lineno++; }
   tplt_xfer(lemp->name,in,out,&lineno);
 
   /* Generate a table containing a text string that describes every
diff --git a/tool/lempar.c b/tool/lempar.c
index 6c6ca77dde..7ceaaa6207 100644
--- a/tool/lempar.c
+++ b/tool/lempar.c
@@ -131,19 +131,13 @@
 **    (A)   N = yy_action[ yy_shift_ofst[S] + X ]
 **    (B)   N = yy_default[S]
 **
-** The (A) formula is preferred.  The B formula is used instead if:
-**    (1)  The yy_shift_ofst[S]+X value is out of range, or
-**    (2)  yy_lookahead[yy_shift_ofst[S]+X] is not equal to X, or
-**    (3)  yy_shift_ofst[S] equal YY_SHIFT_USE_DFLT.
-** (Implementation note: YY_SHIFT_USE_DFLT is chosen so that
-** YY_SHIFT_USE_DFLT+X will be out of range for all possible lookaheads X.
-** Hence only tests (1) and (2) need to be evaluated.)
+** The (A) formula is preferred.  The B formula is used instead if
+** yy_lookahead[yy_shift_ofst[S]+X] is not equal to X.
 **
 ** The formulas above are for computing the action when the lookahead is
 ** a terminal symbol.  If the lookahead is a non-terminal (as occurs after
 ** a reduce action) then the yy_reduce_ofst[] array is used in place of
-** the yy_shift_ofst[] array and YY_REDUCE_USE_DFLT is used in place of
-** YY_SHIFT_USE_DFLT.
+** the yy_shift_ofst[] array.
 **
 ** The following are the tables generated in this section:
 **
@@ -478,7 +472,8 @@ static unsigned int yy_find_shift_action(
     i = yy_shift_ofst[stateno];
     assert( iLookAhead!=YYNOCODE );
     i += iLookAhead;
-    if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){
+    assert( i>=0 && i<sizeof(yy_lookahead)/sizeof(yy_lookahead[0]) );
+    if( yy_lookahead[i]!=iLookAhead ){
 #ifdef YYFALLBACK
       YYCODETYPE iFallback;            /* Fallback token */
       if( iLookAhead<sizeof(yyFallback)/sizeof(yyFallback[0])
@@ -541,7 +536,6 @@ static int yy_find_reduce_action(
   assert( stateno<=YY_REDUCE_COUNT );
 #endif
   i = yy_reduce_ofst[stateno];
-  assert( i!=YY_REDUCE_USE_DFLT );
   assert( iLookAhead!=YYNOCODE );
   i += iLookAhead;
 #ifdef YYERRORSYMBOL

From 5060a67c4d825b0286966ec9df37e1105a734c2c Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 25 Dec 2017 13:43:54 +0000
Subject: [PATCH 181/488] Fix an assertion fault on a syntax error input caused
 by check-in [6b2ff26c25bb9da3].  Problem discovered by OSSFuzz.

FossilOrigin-Name: 90d6e4f10d3055a776d24854c442a2a68e726af8fc382cdb6241a834082e4c4c
---
 manifest          | 14 +++++++-------
 manifest.uuid     |  2 +-
 src/build.c       |  1 +
 test/colname.test | 11 ++++++++++-
 4 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index 1c622c1102..d9715194d3 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sNULL\spointer\sdereference\safter\sa\ssyntax\serror\sthat\scan\soccur\sas\sa\nresult\sof\scheck-in\s[6b2ff26c25bb9da3]\syesterday.\s\sThis\sproblem\swas\ndiscovered\sby\sthe\sOSSFuzz.
-D 2017-12-24T18:56:28.786
+C Fix\san\sassertion\sfault\son\sa\ssyntax\serror\sinput\scaused\sby\scheck-in\n[6b2ff26c25bb9da3].\s\sProblem\sdiscovered\sby\sOSSFuzz.
+D 2017-12-25T13:43:54.974
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -423,7 +423,7 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
 F src/btree.c b83a6b03f160528020bb965f0c3a40af5286cd4923c3870fd218177f03a120a7
 F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09
 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc
-F src/build.c ab5bdf955c85bcd56acbf310a48bbd50b4b92079efa40d997a7e4246f8e03741
+F src/build.c 6ab114a9a4717622df3745d26f2c55e9113f356441a8f76fa2b1d7e1f2472c20
 F src/callback.c fe677cb5f5abb02f7a772a62a98c2f516426081df68856e8f2d5f950929b966a
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
@@ -682,7 +682,7 @@ F test/collate9.test 3adcc799229545940df2f25308dd1ad65869145a
 F test/collateA.test b8218ab90d1fa5c59dcf156efabb1b2599c580d6
 F test/collateB.test 1e68906951b846570f29f20102ed91d29e634854ee47454d725f2151ecac0b95
 F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1
-F test/colname.test 36da785927822ecd0de979459e27e9be63f458dd08d3edde41af3af37a337d58
+F test/colname.test 101aa39392a1f6883278f588836a3ab99178f8103f78032433400475cc05109f
 F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db
 F test/conflict2.test bb0b94cf7196c64a3cbd815c66d3ee98c2fecd9c
 F test/conflict3.test a83db76a6c3503b2fa057c7bfb08c318d8a422202d8bc5b86226e078e5b49ff9
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 07c773148d8db185fa54991df09298b64f4fef28879e6c9395759265e8183977
-R 2e4222d820aa06549d33319e9e33c627
+P d49afb8f9804e96662d1e3cadc4c6643908706d848a53d5ed019919c98f2ccba
+R 22dc7c511444b2944fcb40dc7656150e
 U drh
-Z 67298e45eb2d4d992f0cdf5678deffdf
+Z df3951acae9acd7f479ba37833a7cfe6
diff --git a/manifest.uuid b/manifest.uuid
index d58d2c0dd5..55584bef0a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d49afb8f9804e96662d1e3cadc4c6643908706d848a53d5ed019919c98f2ccba
\ No newline at end of file
+90d6e4f10d3055a776d24854c442a2a68e726af8fc382cdb6241a834082e4c4c
\ No newline at end of file
diff --git a/src/build.c b/src/build.c
index 390dae5d3b..ae2a92c79b 100644
--- a/src/build.c
+++ b/src/build.c
@@ -1976,6 +1976,7 @@ void sqlite3EndTable(
       sqlite3DeleteTable(db, pSelTab);
       sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield);
       sqlite3Select(pParse, pSelect, &dest);
+      if( pParse->nErr ) return;
       sqlite3VdbeEndCoroutine(v, regYield);
       sqlite3VdbeJumpHere(v, addrTop - 1);
       addrInsLoop = sqlite3VdbeAddOp1(v, OP_Yield, dest.iSDParm);
diff --git a/test/colname.test b/test/colname.test
index cacf91e4e3..358ea77e07 100644
--- a/test/colname.test
+++ b/test/colname.test
@@ -398,7 +398,7 @@ do_execsql_test colname-9.320 {
   SELECT name FROM pragma_table_info('t2');
 } {Bbb}
 
-# Issue detected by clusterfuzz on 2017-12-24 (Christmas Eve)
+# Issue detected by OSSFuzz on 2017-12-24 (Christmas Eve)
 # caused by check-in https://sqlite.org/src/info/6b2ff26c25
 #
 # Prior to being fixed, the following CREATE TABLE was dereferencing
@@ -408,6 +408,15 @@ do_catchsql_test colname-9.400 {
   CREATE TABLE t4 AS SELECT #0;
 } {1 {near "#0": syntax error}}
 
+# Issue detected by OSSFuzz on 2017-12-25 (Christmas Day)
+# also caused by check-in https://sqlite.org/src/info/6b2ff26c25
+#
+# Prior to being fixed, the following CREATE TABLE caused an
+# assertion fault.
+#
+do_catchsql_test colname-9.410 {
+  CREATE TABLE t5 AS SELECT RAISE(abort,a);
+} {1 {RAISE() may only be used within a trigger-program}}
 
 # Make sure the quotation marks get removed from the column names
 # when constructing a new table from an aggregate SELECT.

From f6c37dbc8c9ae15689f91e1d9c1e43e7b8737120 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 26 Dec 2017 14:30:44 +0000
Subject: [PATCH 182/488] Add options to speedtest1.c and speed-check.sh for
 testing performance of floating-point computatations.

FossilOrigin-Name: ebfea8728fec955b1d74b1d0a3de498fd1a32e8b39572a8fdab606ed87b169b4
---
 manifest            | 17 +++++-----
 manifest.uuid       |  2 +-
 test/speedtest1.c   | 76 +++++++++++++++++++++++++++++++++++++++++++--
 tool/speed-check.sh |  6 ++++
 4 files changed, 90 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index d9715194d3..aa2114d805 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\san\sassertion\sfault\son\sa\ssyntax\serror\sinput\scaused\sby\scheck-in\n[6b2ff26c25bb9da3].\s\sProblem\sdiscovered\sby\sOSSFuzz.
-D 2017-12-25T13:43:54.974
+C Add\soptions\sto\sspeedtest1.c\sand\sspeed-check.sh\sfor\stesting\sperformance\sof\nfloating-point\scomputatations.
+D 2017-12-26T14:30:44.117
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -1248,7 +1248,7 @@ F test/speed3.test 694affeb9100526007436334cf7d08f3d74b85ef
 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715
 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa
 F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b
-F test/speedtest1.c e44c5fccddcfe916c3bf7fe2f87dcc4b4fd66a0d923eb83515f311212670f267
+F test/speedtest1.c a5faf4cbe5769eee4b721b3875cb3f12520a9b99d9026b1063b47c39603375b8
 F test/spellfix.test f9c1f431e2c096c8775fec032952320c0e4700db
 F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3
 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33
@@ -1647,7 +1647,7 @@ F tool/showstat4.c 0682ebea7abf4d3657f53c4a243f2e7eab48eab344ed36a94bb75dcd19a5c
 F tool/showwal.c ad9d768f96ca6199ad3a8c9562d679680bd032dd01204ea3e5ea6fb931d81847
 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe
 F tool/spaceanal.tcl 4bfd19aad7eb3ce0372ef0255f58035e0bba4ff5e9acfd763a10c6fb365c8dec
-F tool/speed-check.sh a97ae367e9172a706101901e7caef48f1a14fc8a49053b25e79f6a67296b3412
+F tool/speed-check.sh 9ae425da8819e54e780cf494fc6d8175dfb16e109ae3214a45a5c9bb2b74e2c4
 F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355
 F tool/speedtest16.c ecb6542862151c3e6509bbc00509b234562ae81e
 F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
@@ -1687,7 +1687,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d49afb8f9804e96662d1e3cadc4c6643908706d848a53d5ed019919c98f2ccba
-R 22dc7c511444b2944fcb40dc7656150e
-U drh
-Z df3951acae9acd7f479ba37833a7cfe6
+P 90d6e4f10d3055a776d24854c442a2a68e726af8fc382cdb6241a834082e4c4c
+Q +b3c6105181d22c8a53f5bf662e3d182bba962ee8afd1ad972ebd10094b20fc17
+R c2447707ab323a591ab8e05be0f0cae9
+U dan
+Z 1002781326ca68c114cfc46396055176
diff --git a/manifest.uuid b/manifest.uuid
index 55584bef0a..aaa3417a9c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-90d6e4f10d3055a776d24854c442a2a68e726af8fc382cdb6241a834082e4c4c
\ No newline at end of file
+ebfea8728fec955b1d74b1d0a3de498fd1a32e8b39572a8fdab606ed87b169b4
\ No newline at end of file
diff --git a/test/speedtest1.c b/test/speedtest1.c
index b92801a208..9342da79bf 100644
--- a/test/speedtest1.c
+++ b/test/speedtest1.c
@@ -32,7 +32,7 @@ static const char zHelp[] =
   "  --size N            Relative test size.  Default=100\n"
   "  --stats             Show statistics at the end\n"
   "  --temp N            N from 0 to 9.  0: no temp table. 9: all temp tables\n"
-  "  --testset T         Run test-set T (main, cte, rtree, orm, debug)\n"
+  "  --testset T         Run test-set T (main, cte, rtree, orm, fp, debug)\n"
   "  --trace             Turn on SQL tracing\n"
   "  --threads N         Use up to N threads for sorting\n"
   "  --utf16be           Set text encoding to UTF-16BE\n"
@@ -1120,7 +1120,77 @@ void testset_cte(void){
   );
   speedtest1_run();
   speedtest1_end_test();
+}
 
+/*
+** Compute a pseudo-random floating point ascii number.
+*/
+void speedtest1_random_ascii_fp(char *zFP){
+  int x = speedtest1_random();
+  int y = speedtest1_random();
+  int z;
+  z = y%10;
+  if( z<0 ) z = -z;
+  y /= 10;
+  sqlite3_snprintf(100,zFP,"%d.%de%d",y,z,x%200);
+}
+
+/*
+** A testset for floating-point numbers.
+*/
+void testset_fp(void){
+  int n;
+  int i;
+  char zFP1[100];
+  char zFP2[100];
+  
+  n = g.szTest*5000;
+  speedtest1_begin_test(100, "Fill a table with %d FP values", n*2);
+  speedtest1_exec("BEGIN");
+  speedtest1_exec("CREATE%s TABLE t1(a REAL %s, b REAL %s);",
+                  isTemp(1), g.zNN, g.zNN);
+  speedtest1_prepare("INSERT INTO t1 VALUES(?1,?2); -- %d times", n);
+  for(i=1; i<=n; i++){
+    speedtest1_random_ascii_fp(zFP1);
+    speedtest1_random_ascii_fp(zFP2);
+    sqlite3_bind_text(g.pStmt, 1, zFP1, -1, SQLITE_STATIC);
+    sqlite3_bind_text(g.pStmt, 2, zFP2, -1, SQLITE_STATIC);
+    speedtest1_run();
+  }
+  speedtest1_exec("COMMIT");
+  speedtest1_end_test();
+
+  n = g.szTest/25 + 2;
+  speedtest1_begin_test(110, "%d range queries", n);
+  speedtest1_prepare("SELECT sum(b) FROM t1 WHERE a BETWEEN ?1 AND ?2");
+  for(i=1; i<=n; i++){
+    speedtest1_random_ascii_fp(zFP1);
+    speedtest1_random_ascii_fp(zFP2);
+    sqlite3_bind_text(g.pStmt, 1, zFP1, -1, SQLITE_STATIC);
+    sqlite3_bind_text(g.pStmt, 2, zFP2, -1, SQLITE_STATIC);
+    speedtest1_run();
+  }
+  speedtest1_end_test();
+
+  speedtest1_begin_test(120, "CREATE INDEX three times");
+  speedtest1_exec("BEGIN;");
+  speedtest1_exec("CREATE INDEX t1a ON t1(a);");
+  speedtest1_exec("CREATE INDEX t1b ON t1(b);");
+  speedtest1_exec("CREATE INDEX t1ab ON t1(a,b);");
+  speedtest1_exec("COMMIT;");
+  speedtest1_end_test();
+
+  n = g.szTest/3 + 2;
+  speedtest1_begin_test(130, "%d indexed range queries", n);
+  speedtest1_prepare("SELECT sum(b) FROM t1 WHERE a BETWEEN ?1 AND ?2");
+  for(i=1; i<=n; i++){
+    speedtest1_random_ascii_fp(zFP1);
+    speedtest1_random_ascii_fp(zFP2);
+    sqlite3_bind_text(g.pStmt, 1, zFP1, -1, SQLITE_STATIC);
+    sqlite3_bind_text(g.pStmt, 2, zFP2, -1, SQLITE_STATIC);
+    speedtest1_run();
+  }
+  speedtest1_end_test();
 }
 
 #ifdef SQLITE_ENABLE_RTREE
@@ -1873,6 +1943,8 @@ int main(int argc, char **argv){
     testset_orm();
   }else if( strcmp(zTSet,"cte")==0 ){
     testset_cte();
+  }else if( strcmp(zTSet,"fp")==0 ){
+    testset_fp();
   }else if( strcmp(zTSet,"rtree")==0 ){
 #ifdef SQLITE_ENABLE_RTREE
     testset_rtree(6, 147);
@@ -1881,7 +1953,7 @@ int main(int argc, char **argv){
                 "the R-Tree tests\n");
 #endif
   }else{
-    fatal_error("unknown testset: \"%s\"\nChoices: main debug1 cte rtree\n",
+    fatal_error("unknown testset: \"%s\"\nChoices: main debug1 cte rtree fp\n",
                  zTSet);
   }
   speedtest1_final();
diff --git a/tool/speed-check.sh b/tool/speed-check.sh
index 2cda5c8078..6cc3018981 100644
--- a/tool/speed-check.sh
+++ b/tool/speed-check.sh
@@ -117,6 +117,12 @@ while test "$1" != ""; do
     --orm)
         SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset orm"
         ;;
+    --cte)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset cte"
+        ;;
+    --fp)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset fp"
+        ;;
     -*)
         CC_OPTS="$CC_OPTS $1"
         ;;

From c8f44d245c5034c8f96361026a49b748ac02a700 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 26 Dec 2017 14:32:25 +0000
Subject: [PATCH 183/488] Fix crashes that could occur if SQL NULL values were
 passed to the built-in FTS5 snippet function. Edit: breaks amalgamation
 builds.

FossilOrigin-Name: 6a790b67a0a5c698526db16ea262b13ecdd1b6ca74e80bdccfcad88ddbdc933a
---
 ext/fts5/fts5_aux.c       | 16 +++++++++++++---
 ext/fts5/test/fts5af.test | 10 ++++++++++
 manifest                  | 16 ++++++++--------
 manifest.uuid             |  2 +-
 4 files changed, 32 insertions(+), 12 deletions(-)

diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c
index 219ea6fff8..594b981dda 100644
--- a/ext/fts5/fts5_aux.c
+++ b/ext/fts5/fts5_aux.c
@@ -358,6 +358,16 @@ static int fts5SnippetScore(
   return rc;
 }
 
+/*
+** Return the value in pVal interpreted as utf-8 text. Except, if pVal 
+** contains a NULL value, return a pointer to a static string zero
+** bytes in length instead of a NULL pointer.
+*/
+static const char *fts5ValueToText(sqlite3_value *pVal){
+  const char *zRet = (const char*)sqlite3_value_text(pVal);
+  return zRet ? zRet : "";
+}
+
 /*
 ** Implementation of snippet() function.
 */
@@ -393,9 +403,9 @@ static void fts5SnippetFunction(
   nCol = pApi->xColumnCount(pFts);
   memset(&ctx, 0, sizeof(HighlightContext));
   iCol = sqlite3_value_int(apVal[0]);
-  ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]);
-  ctx.zClose = (const char*)sqlite3_value_text(apVal[2]);
-  zEllips = (const char*)sqlite3_value_text(apVal[3]);
+  ctx.zOpen = fts5ValueToText(apVal[1]);
+  ctx.zClose = fts5ValueToText(apVal[2]);
+  zEllips = fts5ValueToText(apVal[3]);
   nToken = sqlite3_value_int(apVal[4]);
 
   iBestCol = (iCol>=0 ? iCol : 0);
diff --git a/ext/fts5/test/fts5af.test b/ext/fts5/test/fts5af.test
index fa4ebd2955..86c8f753fa 100644
--- a/ext/fts5/test/fts5af.test
+++ b/ext/fts5/test/fts5af.test
@@ -175,6 +175,16 @@ do_execsql_test 5.1 {
   SELECT snippet(p1, 0, '[', ']', '...', 6) FROM p1('x');
 } {{[x] a a a a a...}}
 
+do_execsql_test 5.2 {
+  SELECT snippet(p1, 0, '[', ']', NULL, 6) FROM p1('x');
+} {{[x] a a a a a}}
+do_execsql_test 5.3 {
+  SELECT snippet(p1, 0, NULL, ']', '...', 6) FROM p1('x');
+} {{x] a a a a a...}}
+do_execsql_test 5.4 {
+  SELECT snippet(p1, 0, '[', NULL, '...', 6) FROM p1('x');
+} {{[x a a a a a...}}
+
 } ;# foreach_detail_mode 
 
 finish_test
diff --git a/manifest b/manifest
index aa2114d805..f3f08454e9 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\soptions\sto\sspeedtest1.c\sand\sspeed-check.sh\sfor\stesting\sperformance\sof\nfloating-point\scomputatations.
-D 2017-12-26T14:30:44.117
+C Fix\scrashes\sthat\scould\soccur\sif\sSQL\sNULL\svalues\swere\spassed\sto\sthe\sbuilt-in\nFTS5\ssnippet\sfunction.\sEdit:\sbreaks\samalgamation\sbuilds.
+D 2017-12-26T14:32:25.807
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -106,7 +106,7 @@ F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h 62f3e33ceeb9a428db139f9c012186b371da1cc7
 F ext/fts5/fts5Int.h eda28e3a0a5d87c412e8355fe35da875b04cb389908c8eb0d867ad662adbc491
-F ext/fts5/fts5_aux.c 67acf8d51723cf28ffc3828210ba662df4b8d267
+F ext/fts5/fts5_aux.c ca666a3bbe07c5a3bbe9fffaea19c935a1efaf337333e28bad7bdd1971ffd093
 F ext/fts5/fts5_buffer.c 1dd1ec0446b3acfc2d7d407eb894762a461613e2695273f48e449bfd13e973ff
 F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857
 F ext/fts5/fts5_expr.c 01048018d21524e2c302b063ff5c3cdcf546e03297215e577205d85b47499deb
@@ -129,7 +129,7 @@ F ext/fts5/test/fts5ab.test 9205c839332c908aaad2b01ab8670ece8b161e8f2ec8a9fabf18
 F ext/fts5/test/fts5ac.test a7aa7e1fefc6e1918aa4d3111d5c44a09177168e962c5fd2cca9620de8a7ed6d
 F ext/fts5/test/fts5ad.test e8cf959dfcd57c8e46d6f5f25665686f3b6627130a9a981371dafdf6482790de
 F ext/fts5/test/fts5ae.test 1142d16d9cc193894dc13cc8f9c7a8a21411ac61b5567a878514df6f9f0d7bb7
-F ext/fts5/test/fts5af.test aa635947bda31ac87fbe99483eef4d9a8571f58ad89c75dfb63312a35688eceb
+F ext/fts5/test/fts5af.test 724247405b13f8f06cc6ce464dc4f152dc5dd4e86b12c2099685d8f19747bf7b
 F ext/fts5/test/fts5ag.test 7816f25a0707578f08145ab539fc0ca025f8951e788b28a6a18a06b2099469dd
 F ext/fts5/test/fts5ah.test 27b5a33bfd0363ca8a4dc659e6e2a5df3dea1c3c5b04bc51ca6aeb1277bd9b21
 F ext/fts5/test/fts5ai.test d837c42249c0d8ad1a2912270e22cf2f303790a611f85c0be3a58e42a3696e3d
@@ -1687,8 +1687,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 90d6e4f10d3055a776d24854c442a2a68e726af8fc382cdb6241a834082e4c4c
-Q +b3c6105181d22c8a53f5bf662e3d182bba962ee8afd1ad972ebd10094b20fc17
-R c2447707ab323a591ab8e05be0f0cae9
+P ebfea8728fec955b1d74b1d0a3de498fd1a32e8b39572a8fdab606ed87b169b4
+Q +553a3ad32498ddda920216cd44a376a439a58fbb326d2d3800528867db1ffa9d
+R 742f62796704fae3ae9dc15618c72290
 U dan
-Z 1002781326ca68c114cfc46396055176
+Z ecb93df9cf8c6f4bd1c8f7a6ca97d753
diff --git a/manifest.uuid b/manifest.uuid
index aaa3417a9c..9c54144fee 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ebfea8728fec955b1d74b1d0a3de498fd1a32e8b39572a8fdab606ed87b169b4
\ No newline at end of file
+6a790b67a0a5c698526db16ea262b13ecdd1b6ca74e80bdccfcad88ddbdc933a
\ No newline at end of file

From 02a43f6defda835b5baaaee274923f1b80d31ceb Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 26 Dec 2017 14:46:20 +0000
Subject: [PATCH 184/488] Faster and smaller implementation of sqlite3AtoF()
 based on a suggestion from Cezary H. Noweta.

FossilOrigin-Name: fd2e0e7a770c2ce9355068aad1024c3d2861c104fd3be304a91c55ca742155fa
---
 manifest      | 15 +++++++--------
 manifest.uuid |  2 +-
 src/util.c    | 26 ++++++++++++++++++++------
 3 files changed, 28 insertions(+), 15 deletions(-)

diff --git a/manifest b/manifest
index f3f08454e9..3d3fdb11e9 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\scrashes\sthat\scould\soccur\sif\sSQL\sNULL\svalues\swere\spassed\sto\sthe\sbuilt-in\nFTS5\ssnippet\sfunction.\sEdit:\sbreaks\samalgamation\sbuilds.
-D 2017-12-26T14:32:25.807
+C Faster\sand\ssmaller\simplementation\sof\ssqlite3AtoF()\sbased\son\sa\ssuggestion\nfrom\sCezary\sH.\sNoweta.
+D 2017-12-26T14:46:20.800
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -545,7 +545,7 @@ F src/treeview.c eae35972ff44f67064de2eaf35f04afe94e7aea3271a8b3bcebb3f954880fec
 F src/trigger.c 775053eecf6b73062e243404b56f5064446254d5cce17d8704d5cdffd72a546a
 F src/update.c 961bd1265d4d1e5cd65c9a54fa5122fb7aefcb003fcf2de0c092fceb7e58972c
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
-F src/util.c d01fa6f45bfad3b65fb2490513aa2e0676412c61b4b094340b513cf72c3704a4
+F src/util.c 7315e97a8dc2c8e19ca64196c652cf0a65d13fd0a211b2cec082062372dc6261
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
 F src/vdbe.c 3393b508d9ad084ffce232a7c53e375ef5ac99b50b685c5131fcdfce97a9d534
 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97
@@ -1687,8 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ebfea8728fec955b1d74b1d0a3de498fd1a32e8b39572a8fdab606ed87b169b4
-Q +553a3ad32498ddda920216cd44a376a439a58fbb326d2d3800528867db1ffa9d
-R 742f62796704fae3ae9dc15618c72290
-U dan
-Z ecb93df9cf8c6f4bd1c8f7a6ca97d753
+P 6a790b67a0a5c698526db16ea262b13ecdd1b6ca74e80bdccfcad88ddbdc933a
+R c3240df89b3aa2504611bdd745085460
+U drh
+Z 3dc20c6e2c72c6078d443025934f152b
diff --git a/manifest.uuid b/manifest.uuid
index 9c54144fee..bdfe92339c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6a790b67a0a5c698526db16ea262b13ecdd1b6ca74e80bdccfcad88ddbdc933a
\ No newline at end of file
+fd2e0e7a770c2ce9355068aad1024c3d2861c104fd3be304a91c55ca742155fa
\ No newline at end of file
diff --git a/src/util.c b/src/util.c
index a4dbe8fdaf..75de4b3b30 100644
--- a/src/util.c
+++ b/src/util.c
@@ -320,6 +320,24 @@ int sqlite3_strnicmp(const char *zLeft, const char *zRight, int N){
   return N<0 ? 0 : UpperToLower[*a] - UpperToLower[*b];
 }
 
+/*
+** Compute 10 to the E-th power.  Examples:  E==1 results in 10.
+** E==2 results in 100.  E==50 results in 1.0e50.
+**
+** This routine only works for values of E between 1 and 341.
+*/
+static LONGDOUBLE_TYPE sqlite3Pow10(int E){
+  LONGDOUBLE_TYPE x = 10.0;
+  LONGDOUBLE_TYPE r = 1.0;
+  while(1){
+    if( E & 1 ) r *= x;
+    E >>= 1;
+    if( E==0 ) break;
+    x *= x;
+  }
+  return r; 
+}
+
 /*
 ** The string z[] is an text representation of a real number.
 ** Convert this string to a double and write it into *pResult.
@@ -475,11 +493,10 @@ do_atof_calc:
     if( e==0 ){                                         /*OPTIMIZATION-IF-TRUE*/
       result = (double)s;
     }else{
-      LONGDOUBLE_TYPE scale = 1.0;
       /* attempt to handle extremely small/large numbers better */
       if( e>307 ){                                      /*OPTIMIZATION-IF-TRUE*/
         if( e<342 ){                                    /*OPTIMIZATION-IF-TRUE*/
-          while( e%308 ) { scale *= 1.0e+1; e -= 1; }
+          LONGDOUBLE_TYPE scale = sqlite3Pow10(e-308);
           if( esign<0 ){
             result = s / scale;
             result /= 1.0e+308;
@@ -499,10 +516,7 @@ do_atof_calc:
           }
         }
       }else{
-        /* 1.0e+22 is the largest power of 10 than can be 
-        ** represented exactly. */
-        while( e%22 ) { scale *= 1.0e+1; e -= 1; }
-        while( e>0 ) { scale *= 1.0e+22; e -= 22; }
+        LONGDOUBLE_TYPE scale = sqlite3Pow10(e);
         if( esign<0 ){
           result = s / scale;
         }else{

From 0d9de99c5cbc065fa0243f1a23ab79081e05fea4 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 26 Dec 2017 18:04:23 +0000
Subject: [PATCH 185/488] Add support for measuring and reporting coverage of
 the parser state machine using the SQLITE_TESTCTRL_PARSER_COVERAGE
 test-control.

FossilOrigin-Name: 1253a872dbf48656d4efd588ab61223a5ac550d9b2b932249d6ba585276ba573
---
 manifest        | 22 +++++++++++-----------
 manifest.uuid   |  2 +-
 src/main.c      | 16 ++++++++++++++++
 src/shell.c.in  | 11 +++++++++++
 src/sqlite.h.in |  3 ++-
 src/sqliteInt.h |  3 +++
 tool/lemon.c    |  1 +
 tool/lempar.c   | 37 +++++++++++++++++++++++++++++++++++--
 8 files changed, 80 insertions(+), 15 deletions(-)

diff --git a/manifest b/manifest
index d9f3940ae0..7f1e56cfbd 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Enhance\sLEMON\sso\sthat\sit\sgenerates\sthe\saction\stable\sin\ssuch\sa\sway\sthat\sno\nrange\scheck\sis\sneeded\son\sthe\slookahead\stable\sto\sverify\sthat\sthe\snext\sinput\ntoken\sis\svalid.\s\sThis\smakes\sthe\slookahead\stable\sslightly\slarger\s(about\s120\nbytes)\sbut\shelps\sthe\sparser\sto\srun\sfaster.
-D 2017-12-25T04:15:38.668
+C Add\ssupport\sfor\smeasuring\sand\sreporting\scoverage\sof\sthe\sparser\sstate\smachine\nusing\sthe\sSQLITE_TESTCTRL_PARSER_COVERAGE\stest-control.
+D 2017-12-26T18:04:23.951
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -443,7 +443,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c cb67cc56ef2ddd13e6944b2c0dd08a920bcd9503230adef8b9928d338097c722
 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e
 F src/loadext.c 55bcc3c741059a1056859e8adaf133aa179e22be12215c0936b2f354ef71209b
-F src/main.c 7ce55fa3c0bf669944de309ebab1655ed06ec67869adb0372c7a1062e461c448
+F src/main.c 690c4134f944cbd5b71d59dd6e61ce4131f6a50ab774f38108e57d07d79cf876
 F src/malloc.c a02c9e69bc76bee0f639416b947a946412890b606301454727feadcb313536d6
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -479,11 +479,11 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in 339169a3d1307b5566ebe9ce15832d03439206106724c78cc3d9125a7b851795
-F src/sqlite.h.in 2126192945019d4cdce335cb236b440a05ec75c93e4cd94c9c6d6e7fcc654cc4
+F src/shell.c.in f3ec8f90dd698ea98781a90642c91eacbc24f4e55bb551c7b2762000d3ef55dc
+F src/sqlite.h.in b4dc75265ed04b98e2184011a7dd0054ce2137ff84867a6be8b4f3bdfbc03d30
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
-F src/sqliteInt.h 003b78433baae4e5c997f99f2f9cf98d90754f256baeacb32f8189569a48251f
+F src/sqliteInt.h 26bf7cc7aaa6a6eb3d3c0581f9ef5523da01babb8de1d796793d5165fea95958
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -1608,8 +1608,8 @@ F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4
 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
 F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f
-F tool/lemon.c c8d7ce4fe7a90f7fa6a5985452aa926fcf25376cf90095c9d06c432ab0bebdbc
-F tool/lempar.c 427ee280f3c3781e82bbee21f428bc8ae18ab245d4f66d65da46b598ded81648
+F tool/lemon.c d92031adf63377ff5055441c9d26a41a03378e8cac8dbcc1cd6ef7190f51aa6a
+F tool/lempar.c 2a688bf343f21c25908cf082d0d707f45bc4416eea8550787809da4025c3d7c6
 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9
 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P fdbb35c54f2b6cb65d04ac295f207ff3e69360e0558348c77eb5e62691807046
-R f14cbf100834d359550a23c924f7ebaa
+P 7eb0198d0102e97e4b7ad9e359d95985e55e09c510ea4b360265ac8feb9ed814
+R e80238b7e2c8eabf5efe2bfae85580fc
 U drh
-Z bb1d7d0db816330a6f1e2b8011c2819b
+Z d5a5596d3c74652b6ba118722fb4ea75
diff --git a/manifest.uuid b/manifest.uuid
index b69ed5a1cc..dce691ea27 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7eb0198d0102e97e4b7ad9e359d95985e55e09c510ea4b360265ac8feb9ed814
\ No newline at end of file
+1253a872dbf48656d4efd588ab61223a5ac550d9b2b932249d6ba585276ba573
\ No newline at end of file
diff --git a/src/main.c b/src/main.c
index 32ce1889f6..3c8035c120 100644
--- a/src/main.c
+++ b/src/main.c
@@ -3911,6 +3911,22 @@ int sqlite3_test_control(int op, ...){
       sqlite3_mutex_leave(db->mutex);
       break;
     }
+
+#if defined(YYCOVERAGE)
+    /*  sqlite3_test_control(SQLITE_TESTCTRL_PARSER_COVERAGE, FILE *out)
+    **
+    ** This test control (only available when SQLite is compiled with
+    ** -DYYCOVERAGE) writes a report onto "out" that shows all
+    ** state/lookahead combinations in the parser state machine
+    ** which are never exercised.  If any state is missed, make the
+    ** return code SQLITE_ERROR.
+    */
+    case SQLITE_TESTCTRL_PARSER_COVERAGE: {
+      FILE *out = va_arg(ap, FILE*);
+      if( sqlite3ParserCoverage(out) ) rc = SQLITE_ERROR;
+      break;
+    }
+#endif /* defined(YYCOVERAGE) */
   }
   va_end(ap);
 #endif /* SQLITE_UNTESTABLE */
diff --git a/src/shell.c.in b/src/shell.c.in
index 13b1fcde39..155ca708a5 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -6108,6 +6108,9 @@ static int do_meta_command(char *zLine, ShellState *p){
       { "localtime_fault",    SQLITE_TESTCTRL_LOCALTIME_FAULT,"BOOLEAN"           },
       { "never_corrupt",      SQLITE_TESTCTRL_NEVER_CORRUPT, "BOOLEAN"            },
       { "optimizations",      SQLITE_TESTCTRL_OPTIMIZATIONS, "DISABLE-MASK"       },
+#ifdef YYCOVERAGE
+      { "parser_coverage",    SQLITE_TESTCTRL_PARSER_COVERAGE, ""                 },
+#endif
       { "pending_byte",       SQLITE_TESTCTRL_PENDING_BYTE,  "OFFSET  "           },
       { "prng_reset",         SQLITE_TESTCTRL_PRNG_RESET,    ""                   },
       { "prng_restore",       SQLITE_TESTCTRL_PRNG_RESTORE,  ""                   },
@@ -6233,6 +6236,14 @@ static int do_meta_command(char *zLine, ShellState *p){
             isOk = 3;
           }
           break;
+
+#ifdef YYCOVERAGE
+        case SQLITE_TESTCTRL_PARSER_COVERAGE:
+          if( nArg==2 ){
+            sqlite3_test_control(testctrl, p->out);
+            isOk = 3;
+          }
+#endif
       }
     }
     if( isOk==0 && iCtrl>=0 ){
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index f161eea6f2..7e5fccdae6 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -7042,7 +7042,8 @@ int sqlite3_test_control(int op, ...);
 #define SQLITE_TESTCTRL_ISINIT                  23
 #define SQLITE_TESTCTRL_SORTER_MMAP             24
 #define SQLITE_TESTCTRL_IMPOSTER                25
-#define SQLITE_TESTCTRL_LAST                    25  /* Largest TESTCTRL */
+#define SQLITE_TESTCTRL_PARSER_COVERAGE         26
+#define SQLITE_TESTCTRL_LAST                    26  /* Largest TESTCTRL */
 
 /*
 ** CAPI3REF: SQLite Runtime Status
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index b0c4711b03..074d363d48 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -4348,6 +4348,9 @@ void sqlite3Put4byte(u8*, u32);
 #ifdef SQLITE_DEBUG
   void sqlite3ParserTrace(FILE*, char *);
 #endif
+#if defined(YYCOVERAGE)
+  int sqlite3ParserCoverage(FILE*);
+#endif
 
 /*
 ** If the SQLITE_ENABLE IOTRACE exists then the global variable
diff --git a/tool/lemon.c b/tool/lemon.c
index 96dc756d1b..111082c72a 100644
--- a/tool/lemon.c
+++ b/tool/lemon.c
@@ -4257,6 +4257,7 @@ void ReportTable(
   ** been computed */
   fprintf(out,"#define YYNSTATE             %d\n",lemp->nxstate);  lineno++;
   fprintf(out,"#define YYNRULE              %d\n",lemp->nrule);  lineno++;
+  fprintf(out,"#define YYNTOKEN             %d\n",lemp->nterminal); lineno++;
   fprintf(out,"#define YY_MAX_SHIFT         %d\n",lemp->nxstate-1); lineno++;
   i = lemp->minShiftReduce;
   fprintf(out,"#define YY_MIN_SHIFTREDUCE   %d\n",i); lineno++;
diff --git a/tool/lempar.c b/tool/lempar.c
index 7ceaaa6207..29dc15d03d 100644
--- a/tool/lempar.c
+++ b/tool/lempar.c
@@ -72,6 +72,7 @@
 **                       defined, then do no error processing.
 **    YYNSTATE           the combined number of states.
 **    YYNRULE            the number of rules in the grammar
+**    YYNTOKEN           Number of terminal symbols
 **    YY_MAX_SHIFT       Maximum value for shift actions
 **    YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions
 **    YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions
@@ -253,13 +254,13 @@ void ParseTrace(FILE *TraceFILE, char *zTracePrompt){
 }
 #endif /* NDEBUG */
 
-#ifndef NDEBUG
+#if defined(YYCOVERAGE) || !defined(NDEBUG)
 /* For tracing shifts, the names of all terminals and nonterminals
 ** are required.  The following table supplies these names */
 static const char *const yyTokenName[] = { 
 %%
 };
-#endif /* NDEBUG */
+#endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */
 
 #ifndef NDEBUG
 /* For tracing reduce actions, the names of all rules are required.
@@ -455,6 +456,35 @@ int ParseStackPeak(void *p){
 }
 #endif
 
+/* This array of booleans keeps track of the parser statement
+** coverage.  The element yycoverage[X][Y] is set when the parser
+** is in state X and has a lookahead token Y.  In a well-tested
+** systems, every element of this matrix should end up being set.
+*/
+#if defined(YYCOVERAGE)
+static unsigned char yycoverage[YYNSTATE][YYNTOKEN];
+#endif
+
+/*
+** Write into out a description of every state/lookahead combination that
+** has not previously been seen by the parser.  Return the number of 
+** missed state/lookahead combinations.
+*/
+#if defined(YYCOVERAGE)
+int ParseCoverage(FILE *out){
+  int i, j;
+  int nMissed = 0;
+  for(i=0; i<YYNSTATE; i++){
+    for(j=0; j<YYNTOKEN; j++){
+      if( yycoverage[i][j] ) continue;
+      nMissed++;
+      if( out ) fprintf(out,"State %d lookahead %s\n", i, yyTokenName[j]);
+    }
+  }
+  return nMissed;
+}
+#endif
+
 /*
 ** Find the appropriate action for a parser given the terminal
 ** look-ahead token iLookAhead.
@@ -468,6 +498,9 @@ static unsigned int yy_find_shift_action(
  
   if( stateno>YY_MAX_SHIFT ) return stateno;
   assert( stateno <= YY_SHIFT_COUNT );
+#if defined(YYCOVERAGE)
+  yycoverage[stateno][iLookAhead] = 1;
+#endif
   do{
     i = yy_shift_ofst[stateno];
     assert( iLookAhead!=YYNOCODE );

From 22716cbb0baa9b07231b033247a7cae0f1ae0c3a Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 26 Dec 2017 18:32:06 +0000
Subject: [PATCH 186/488] Change the lemon-parser coverage report format to
 report all state/lookahead pairs and indicate on each whether it is hit or
 missed.

FossilOrigin-Name: 86e30fc284c740b55f75884b67988fe837b28878f586f6ec8850ecf80164e700
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 tool/lempar.c |  8 +++++---
 3 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index 9adf601335..ffec829b50 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\sbug\sfixes\sfrom\strunk.
-D 2017-12-26T18:14:53.001
+C Change\sthe\slemon-parser\scoverage\sreport\sformat\sto\sreport\sall\sstate/lookahead\npairs\sand\sindicate\son\seach\swhether\sit\sis\shit\sor\smissed.
+D 2017-12-26T18:32:06.041
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -1609,7 +1609,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
 F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f
 F tool/lemon.c d92031adf63377ff5055441c9d26a41a03378e8cac8dbcc1cd6ef7190f51aa6a
-F tool/lempar.c 2a688bf343f21c25908cf082d0d707f45bc4416eea8550787809da4025c3d7c6
+F tool/lempar.c 64e760f7cc21ea7e73513b062a86a3c478dfd1ed41fe9837a6f49958cdb61e03
 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9
 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 1253a872dbf48656d4efd588ab61223a5ac550d9b2b932249d6ba585276ba573 fd2e0e7a770c2ce9355068aad1024c3d2861c104fd3be304a91c55ca742155fa
-R 2e2b89187d0becf4c2715a566c5e3287
+P d76e12066fa6950f877cbe33b1892e2b0afa948978815d5b9a90cde1fff8ec98
+R 930f1a264ff138489be35924f879ef78
 U drh
-Z ade3bde65575a1ed1ca48cdbaee1adc1
+Z 2620b1a658dd06210670b65491310e1c
diff --git a/manifest.uuid b/manifest.uuid
index e738275177..97230d7170 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d76e12066fa6950f877cbe33b1892e2b0afa948978815d5b9a90cde1fff8ec98
\ No newline at end of file
+86e30fc284c740b55f75884b67988fe837b28878f586f6ec8850ecf80164e700
\ No newline at end of file
diff --git a/tool/lempar.c b/tool/lempar.c
index 29dc15d03d..5404fa5f5f 100644
--- a/tool/lempar.c
+++ b/tool/lempar.c
@@ -476,9 +476,11 @@ int ParseCoverage(FILE *out){
   int nMissed = 0;
   for(i=0; i<YYNSTATE; i++){
     for(j=0; j<YYNTOKEN; j++){
-      if( yycoverage[i][j] ) continue;
-      nMissed++;
-      if( out ) fprintf(out,"State %d lookahead %s\n", i, yyTokenName[j]);
+      if( !yycoverage[i][j] ) nMissed++;
+      if( out ){
+        fprintf(out,"State %d lookahead %s %s\n",
+           i, yyTokenName[j], yycoverage[i][j] ? "ok" : "missed");
+      }
     }
   }
   return nMissed;

From 9ebfaad25db43869ccfae50c02e0fd3310a95e9b Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 26 Dec 2017 20:39:58 +0000
Subject: [PATCH 187/488] Add new file ext/misc/zipfile.c, containing a virtual
 table for read-only access to simple zip archives.

FossilOrigin-Name: 8e366b99b13d765d8bf000a7ec5919e582702e51dc07c27a746b6002898a2302
---
 ext/misc/zipfile.c | 696 +++++++++++++++++++++++++++++++++++++++++++++
 main.mk            |   3 +-
 manifest           |  17 +-
 manifest.uuid      |   2 +-
 src/shell.c.in     |   2 +
 5 files changed, 710 insertions(+), 10 deletions(-)
 create mode 100644 ext/misc/zipfile.c

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
new file mode 100644
index 0000000000..bcee873044
--- /dev/null
+++ b/ext/misc/zipfile.c
@@ -0,0 +1,696 @@
+/*
+** 2017-12-26
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+*/
+#include "sqlite3ext.h"
+SQLITE_EXTENSION_INIT1
+#include <stdio.h>
+#include <string.h>
+#include <assert.h>
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <dirent.h>
+#include <time.h>
+#include <utime.h>
+#include <errno.h>
+
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+
+#ifndef SQLITE_AMALGAMATION
+typedef sqlite3_int64 i64;
+typedef unsigned char u8;
+typedef unsigned short u16;
+typedef unsigned long u32;
+#define MIN(a,b) ((a)<(b) ? (a) : (b))
+#endif
+
+#define ZIPFILE_SCHEMA "CREATE TABLE y("                           \
+  "name,      /* Name of file in zip archive */"                   \
+  "mode,      /* POSIX mode for file */"                           \
+  "mtime,     /* Last modification time in seconds since epoch */" \
+  "sz,        /* Size of object */"                                \
+  "data,      /* Data stored in zip file (possibly compressed) */" \
+  "method,    /* Compression method (integer) */"                  \
+  "f HIDDEN   /* Name of zip file */"                              \
+");"
+
+#define ZIPFILE_F_COLUMN_IDX 6    /* Index of column "f" in the above */
+
+#define ZIPFILE_BUFFER_SIZE (64*1024)
+
+/*
+** Set the error message contained in context ctx to the results of
+** vprintf(zFmt, ...).
+*/
+static void zipfileCtxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){
+  char *zMsg = 0;
+  va_list ap;
+  va_start(ap, zFmt);
+  zMsg = sqlite3_vmprintf(zFmt, ap);
+  sqlite3_result_error(ctx, zMsg, -1);
+  sqlite3_free(zMsg);
+  va_end(ap);
+}
+
+
+/*
+*** 4.3.16  End of central directory record:
+***
+***   end of central dir signature    4 bytes  (0x06054b50)
+***   number of this disk             2 bytes
+***   number of the disk with the
+***   start of the central directory  2 bytes
+***   total number of entries in the
+***   central directory on this disk  2 bytes
+***   total number of entries in
+***   the central directory           2 bytes
+***   size of the central directory   4 bytes
+***   offset of start of central
+***   directory with respect to
+***   the starting disk number        4 bytes
+***   .ZIP file comment length        2 bytes
+***   .ZIP file comment       (variable size)
+*/
+typedef struct ZipfileEOCD ZipfileEOCD;
+struct ZipfileEOCD {
+  u16 iDisk;
+  u16 iFirstDisk;
+  u16 nEntry;
+  u16 nEntryTotal;
+  u32 nSize;
+  u32 iOffset;
+};
+
+/*
+*** 4.3.12  Central directory structure:
+***
+*** ...
+***
+***   central file header signature   4 bytes  (0x02014b50)
+***   version made by                 2 bytes
+***   version needed to extract       2 bytes
+***   general purpose bit flag        2 bytes
+***   compression method              2 bytes
+***   last mod file time              2 bytes
+***   last mod file date              2 bytes
+***   crc-32                          4 bytes
+***   compressed size                 4 bytes
+***   uncompressed size               4 bytes
+***   file name length                2 bytes
+***   extra field length              2 bytes
+***   file comment length             2 bytes
+***   disk number start               2 bytes
+***   internal file attributes        2 bytes
+***   external file attributes        4 bytes
+***   relative offset of local header 4 bytes
+*/
+typedef struct ZipfileCDS ZipfileCDS;
+struct ZipfileCDS {
+  u16 iVersionMadeBy;
+  u16 iVersionExtract;
+  u16 flags;
+  u16 iCompression;
+  u16 mTime;
+  u16 mDate;
+  u32 crc32;
+  u32 szCompressed;
+  u32 szUncompressed;
+  u16 nFile;
+  u16 nExtra;
+  u16 nComment;
+  u16 iDiskStart;
+  u16 iInternalAttr;
+  u32 iExternalAttr;
+  u32 iOffset;
+  char *zFile;                    /* Filename (sqlite3_malloc()) */
+};
+
+/*
+*** 4.3.7  Local file header:
+***
+***   local file header signature     4 bytes  (0x04034b50)
+***   version needed to extract       2 bytes
+***   general purpose bit flag        2 bytes
+***   compression method              2 bytes
+***   last mod file time              2 bytes
+***   last mod file date              2 bytes
+***   crc-32                          4 bytes
+***   compressed size                 4 bytes
+***   uncompressed size               4 bytes
+***   file name length                2 bytes
+***   extra field length              2 bytes
+***   
+*/
+typedef struct ZipfileLFH ZipfileLFH;
+struct ZipfileLFH {
+  u16 iVersionExtract;
+  u16 flags;
+  u16 iCompression;
+  u16 mTime;
+  u16 mDate;
+  u32 crc32;
+  u32 szCompressed;
+  u32 szUncompressed;
+  u16 nFile;
+  u16 nExtra;
+};
+
+/* 
+** Cursor type for recursively iterating through a directory structure.
+*/
+typedef struct ZipfileCsr ZipfileCsr;
+
+struct ZipfileCsr {
+  sqlite3_vtab_cursor base;  /* Base class - must be first */
+  i64 iRowid;                /* Rowid for current row */
+  FILE *pFile;               /* Zip file */
+  i64 nByte;                 /* Size of zip file on disk */
+  int bEof;                  /* True when at EOF */
+  i64 iNextOff;              /* Offset of next record in central directory */
+  ZipfileEOCD eocd;          /* Parse of central directory record */
+  ZipfileCDS cds;            /* Central Directory Structure */
+  ZipfileLFH lfh;            /* Local File Header for current entry */
+  i64 iDataOff;              /* Offset in zipfile to data */
+  u32 mTime;                 /* Extended mtime value */
+  int flags;
+  u8 *aBuffer;               /* Buffer used for various tasks */
+};
+
+#define ZIPFILE_MTIME_VALID 0x0001
+
+typedef struct ZipfileTab ZipfileTab;
+struct ZipfileTab {
+  sqlite3_vtab base;         /* Base class - must be first */
+};
+
+/*
+** Construct a new ZipfileTab virtual table object.
+*/
+static int zipfileConnect(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  ZipfileTab *pNew = 0;
+  int rc;
+
+  rc = sqlite3_declare_vtab(db, ZIPFILE_SCHEMA);
+  if( rc==SQLITE_OK ){
+    pNew = (ZipfileTab*)sqlite3_malloc( sizeof(*pNew) );
+    if( pNew==0 ) return SQLITE_NOMEM;
+    memset(pNew, 0, sizeof(*pNew));
+  }
+  *ppVtab = (sqlite3_vtab*)pNew;
+  return rc;
+}
+
+/*
+** This method is the destructor for zipfile vtab objects.
+*/
+static int zipfileDisconnect(sqlite3_vtab *pVtab){
+  sqlite3_free(pVtab);
+  return SQLITE_OK;
+}
+
+/*
+** Constructor for a new ZipfileCsr object.
+*/
+static int zipfileOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){
+  ZipfileCsr *pCsr;
+  pCsr = sqlite3_malloc( sizeof(*pCsr) + ZIPFILE_BUFFER_SIZE);
+  if( pCsr==0 ) return SQLITE_NOMEM;
+  memset(pCsr, 0, sizeof(*pCsr));
+  pCsr->aBuffer = (u8*)&pCsr[1];
+  *ppCsr = &pCsr->base;
+  return SQLITE_OK;
+}
+
+/*
+** Reset a cursor back to the state it was in when first returned
+** by zipfileOpen().
+*/
+static void zipfileResetCursor(ZipfileCsr *pCsr){
+  pCsr->iRowid = 0;
+  pCsr->bEof = 0;
+  if( pCsr->pFile ){
+    fclose(pCsr->pFile);
+    pCsr->pFile = 0;
+  }
+}
+
+/*
+** Destructor for an ZipfileCsr.
+*/
+static int zipfileClose(sqlite3_vtab_cursor *cur){
+  ZipfileCsr *pCsr = (ZipfileCsr*)cur;
+  zipfileResetCursor(pCsr);
+  sqlite3_free(pCsr);
+  return SQLITE_OK;
+}
+
+/*
+** Set the error message for the virtual table associated with cursor
+** pCsr to the results of vprintf(zFmt, ...).
+*/
+static void zipfileSetErrmsg(ZipfileCsr *pCsr, const char *zFmt, ...){
+  va_list ap;
+  va_start(ap, zFmt);
+  pCsr->base.pVtab->zErrMsg = sqlite3_vmprintf(zFmt, ap);
+  va_end(ap);
+}
+
+static int zipfileReadData(ZipfileCsr *pCsr, u8 *aRead, int nRead, i64 iOff){
+  size_t n;
+  fseek(pCsr->pFile, iOff, SEEK_SET);
+  n = fread(aRead, 1, nRead, pCsr->pFile);
+  if( n!=nRead ){
+    zipfileSetErrmsg(pCsr, "error in fread()");
+    return SQLITE_ERROR;
+  }
+  return SQLITE_OK;
+}
+
+static u16 zipfileGetU16(const u8 *aBuf){
+  return (aBuf[1] << 8) + aBuf[0];
+}
+static u32 zipfileGetU32(const u8 *aBuf){
+  return ((u32)(aBuf[3]) << 24)
+       + ((u32)(aBuf[2]) << 16)
+       + ((u32)(aBuf[1]) <<  8)
+       + ((u32)(aBuf[0]) <<  0);
+}
+
+#define zipfileRead32(aBuf) ( aBuf+=4, zipfileGetU32(aBuf-4) )
+#define zipfileRead16(aBuf) ( aBuf+=2, zipfileGetU16(aBuf-2) )
+
+static int zipfileReadCDS(ZipfileCsr *pCsr){
+  static const int szFix = 46;    /* Size of fixed-size part of CDS */
+  u8 *aRead = pCsr->aBuffer;
+  int rc;
+
+  rc = zipfileReadData(pCsr, aRead, szFix, pCsr->iNextOff);
+  if( rc==SQLITE_OK ){
+    u32 sig = zipfileRead32(aRead);
+    if( sig!=0x02014b50 ){
+      zipfileSetErrmsg(pCsr,"failed to read CDS at offset %lld",pCsr->iNextOff);
+      rc = SQLITE_ERROR;
+    }else{
+      int nRead;
+      pCsr->cds.iVersionMadeBy = zipfileRead16(aRead);
+      pCsr->cds.iVersionExtract = zipfileRead16(aRead);
+      pCsr->cds.flags = zipfileRead16(aRead);
+      pCsr->cds.iCompression = zipfileRead16(aRead);
+      pCsr->cds.mTime = zipfileRead16(aRead);
+      pCsr->cds.mDate = zipfileRead16(aRead);
+      pCsr->cds.crc32 = zipfileRead32(aRead);
+      pCsr->cds.szCompressed = zipfileRead32(aRead);
+      pCsr->cds.szUncompressed = zipfileRead32(aRead);
+      pCsr->cds.nFile = zipfileRead16(aRead);
+      pCsr->cds.nExtra = zipfileRead16(aRead);
+      pCsr->cds.nComment = zipfileRead16(aRead);
+      pCsr->cds.iDiskStart = zipfileRead16(aRead);
+      pCsr->cds.iInternalAttr = zipfileRead16(aRead);
+      pCsr->cds.iExternalAttr = zipfileRead32(aRead);
+      pCsr->cds.iOffset = zipfileRead32(aRead);
+
+      assert( aRead==&pCsr->aBuffer[szFix] );
+
+      nRead = pCsr->cds.nFile + pCsr->cds.nExtra;
+      aRead = pCsr->aBuffer;
+      rc = zipfileReadData(pCsr, aRead, nRead, pCsr->iNextOff+szFix);
+
+      if( rc==SQLITE_OK ){
+        pCsr->cds.zFile = sqlite3_mprintf("%.*s", (int)pCsr->cds.nFile, aRead);
+        pCsr->iNextOff += szFix;
+        pCsr->iNextOff += pCsr->cds.nFile;
+        pCsr->iNextOff += pCsr->cds.nExtra;
+        pCsr->iNextOff += pCsr->cds.nComment;
+      }
+
+      /* Scan the "extra" fields */
+      if( rc==SQLITE_OK ){
+        u8 *p = &aRead[pCsr->cds.nFile];
+        u8 *pEnd = &p[pCsr->cds.nExtra];
+
+        while( p<pEnd ){
+          u16 id = zipfileRead16(p);
+          u16 nByte = zipfileRead16(p);
+
+          switch( id ){
+            case 0x5455: {        /* Extended timestamp */
+              u8 b = p[0];
+              if( b & 0x01 ){     /* 0x01 -> modtime is present */
+                pCsr->mTime = zipfileGetU32(&p[1]);
+                pCsr->flags |= ZIPFILE_MTIME_VALID;
+              }
+              break;
+            }
+
+            case 0x7875:          /* Info-ZIP Unix (new) */
+              break;
+          }
+
+          p += nByte;
+        }
+      }
+    }
+  }
+
+  return rc;
+}
+
+static int zipfileReadLFH(ZipfileCsr *pCsr){
+  static const int szFix = 30;    /* Size of fixed-size part of LFH */
+  u8 *aRead = pCsr->aBuffer;
+  int rc;
+
+  rc = zipfileReadData(pCsr, aRead, szFix, pCsr->cds.iOffset);
+  if( rc==SQLITE_OK ){
+    u32 sig = zipfileRead32(aRead);
+    if( sig!=0x04034b50 ){
+      zipfileSetErrmsg(pCsr, "failed to read LFH at offset %d", 
+          (int)pCsr->cds.iOffset
+      );
+      rc = SQLITE_ERROR;
+    }else{
+      pCsr->lfh.iVersionExtract = zipfileRead16(aRead);
+      pCsr->lfh.flags = zipfileRead16(aRead);
+      pCsr->lfh.iCompression = zipfileRead16(aRead);
+      pCsr->lfh.mTime = zipfileRead16(aRead);
+      pCsr->lfh.mDate = zipfileRead16(aRead);
+      pCsr->lfh.crc32 = zipfileRead32(aRead);
+      pCsr->lfh.szCompressed = zipfileRead32(aRead);
+      pCsr->lfh.szUncompressed = zipfileRead32(aRead);
+      pCsr->lfh.nFile = zipfileRead16(aRead);
+      pCsr->lfh.nExtra = zipfileRead16(aRead);
+      assert( aRead==&pCsr->aBuffer[szFix] );
+      pCsr->iDataOff = pCsr->cds.iOffset+szFix+pCsr->lfh.nFile+pCsr->lfh.nExtra;
+    }
+  }
+
+  return rc;
+}
+
+
+/*
+** Advance an ZipfileCsr to its next row of output.
+*/
+static int zipfileNext(sqlite3_vtab_cursor *cur){
+  ZipfileCsr *pCsr = (ZipfileCsr*)cur;
+  i64 iEof = pCsr->eocd.iOffset + pCsr->eocd.nSize;
+  int rc = SQLITE_OK;
+
+  if( pCsr->iNextOff>=iEof ){
+    pCsr->bEof = 1;
+  }else{
+    pCsr->iRowid++;
+    pCsr->flags = 0;
+    rc = zipfileReadCDS(pCsr);
+    if( rc==SQLITE_OK ){
+      rc = zipfileReadLFH(pCsr);
+    }
+  }
+  return rc;
+}
+
+/*
+** "Standard" MS-DOS time format:
+**
+**   File modification time:
+**     Bits 00-04: seconds divided by 2
+**     Bits 05-10: minute
+**     Bits 11-15: hour
+**   File modification date:
+**     Bits 00-04: day
+**     Bits 05-08: month (1-12)
+**     Bits 09-15: years from 1980 
+*/
+static time_t zipfileMtime(ZipfileCsr *pCsr){
+  struct tm t;
+  memset(&t, 0, sizeof(t));
+  t.tm_sec = (pCsr->cds.mTime & 0x1F)*2;
+  t.tm_min = (pCsr->cds.mTime >> 5) & 0x2F;
+  t.tm_hour = (pCsr->cds.mTime >> 11) & 0x1F;
+
+  t.tm_mday = (pCsr->cds.mDate & 0x1F);
+  t.tm_mon = ((pCsr->cds.mDate >> 5) & 0x0F) - 1;
+  t.tm_year = 80 + ((pCsr->cds.mDate >> 9) & 0x7F);
+
+  return mktime(&t);
+}
+
+/*
+** Return values of columns for the row at which the series_cursor
+** is currently pointing.
+*/
+static int zipfileColumn(
+  sqlite3_vtab_cursor *cur,   /* The cursor */
+  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
+  int i                       /* Which column to return */
+){
+  ZipfileCsr *pCsr = (ZipfileCsr*)cur;
+  int rc = SQLITE_OK;
+  switch( i ){
+    case 0:   /* name */
+      sqlite3_result_text(ctx, pCsr->cds.zFile, -1, SQLITE_TRANSIENT);
+      break;
+    case 1:   /* mode */
+      /* TODO: Whether or not the following is correct surely depends on
+      ** the platform on which the archive was created.  */
+      sqlite3_result_int(ctx, pCsr->cds.iExternalAttr >> 16);
+      break;
+    case 2: { /* mtime */
+      if( pCsr->flags & ZIPFILE_MTIME_VALID ){
+        sqlite3_result_int64(ctx, pCsr->mTime);
+      }else{
+        sqlite3_result_int64(ctx, zipfileMtime(pCsr));
+      }
+      break;
+    }
+    case 3: { /* sz */
+      sqlite3_result_int64(ctx, pCsr->cds.szUncompressed);
+      break;
+    }
+    case 4: { /* data */
+      int sz = pCsr->cds.szCompressed;
+      if( sz>0 ){
+        u8 *aBuf = sqlite3_malloc(sz);
+        if( aBuf==0 ){
+          rc = SQLITE_NOMEM;
+        }else{
+          rc = zipfileReadData(pCsr, aBuf, sz, pCsr->iDataOff);
+        }
+        if( rc==SQLITE_OK ){
+          sqlite3_result_blob(ctx, aBuf, sz, SQLITE_TRANSIENT);
+          sqlite3_free(aBuf);
+        }
+      }
+      break;
+    }
+    case 5:   /* method */
+      sqlite3_result_int(ctx, pCsr->cds.iCompression);
+      break;
+  }
+
+  return SQLITE_OK;
+}
+
+/*
+** Return the rowid for the current row. In this implementation, the
+** first row returned is assigned rowid value 1, and each subsequent
+** row a value 1 more than that of the previous.
+*/
+static int zipfileRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
+  ZipfileCsr *pCsr = (ZipfileCsr*)cur;
+  *pRowid = pCsr->iRowid;
+  return SQLITE_OK;
+}
+
+/*
+** Return TRUE if the cursor has been moved off of the last
+** row of output.
+*/
+static int zipfileEof(sqlite3_vtab_cursor *cur){
+  ZipfileCsr *pCsr = (ZipfileCsr*)cur;
+  return pCsr->bEof;
+}
+
+/*
+** The zip file has been successfully opened (so pCsr->pFile is valid). 
+** This function attempts to locate and read the End of central
+** directory record from the file.
+**
+*/
+static int zipfileReadEOCD(ZipfileCsr *pCsr, ZipfileEOCD *pEOCD){
+  u8 *aRead = pCsr->aBuffer;
+  int nRead = (int)(MIN(pCsr->nByte, ZIPFILE_BUFFER_SIZE));
+  i64 iOff = pCsr->nByte - nRead;
+
+  int rc = zipfileReadData(pCsr, aRead, nRead, iOff);
+  if( rc==SQLITE_OK ){
+    int i;
+
+    /* Scan backwards looking for the signature bytes */
+    for(i=nRead-20; i>=0; i--){
+      if( aRead[i]==0x50 && aRead[i+1]==0x4b 
+       && aRead[i+2]==0x05 && aRead[i+3]==0x06 
+      ){
+        break;
+      }
+    }
+    if( i<0 ){
+      zipfileSetErrmsg(pCsr, "cannot find end of central directory record");
+      return SQLITE_ERROR;
+    }
+
+    aRead += i+4;
+    pEOCD->iDisk = zipfileRead16(aRead);
+    pEOCD->iFirstDisk = zipfileRead16(aRead);
+    pEOCD->nEntry = zipfileRead16(aRead);
+    pEOCD->nEntryTotal = zipfileRead16(aRead);
+    pEOCD->nSize = zipfileRead32(aRead);
+    pEOCD->iOffset = zipfileRead32(aRead);
+
+#if 0
+    printf("iDisk=%d  iFirstDisk=%d  nEntry=%d  "
+           "nEntryTotal=%d  nSize=%d  iOffset=%d", 
+           (int)pEOCD->iDisk, (int)pEOCD->iFirstDisk, (int)pEOCD->nEntry,
+           (int)pEOCD->nEntryTotal, (int)pEOCD->nSize, (int)pEOCD->iOffset
+    );
+#endif
+  }
+
+  return SQLITE_OK;
+}
+
+/*
+** xFilter callback.
+*/
+static int zipfileFilter(
+  sqlite3_vtab_cursor *cur, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  ZipfileCsr *pCsr = (ZipfileCsr*)cur;
+  const char *zFile;              /* Zip file to scan */
+  int rc = SQLITE_OK;             /* Return Code */
+
+  zipfileResetCursor(pCsr);
+
+  assert( idxNum==argc && (idxNum==0 || idxNum==1) );
+  if( idxNum==0 ){
+    /* Error. User did not supply a file name. */
+    zipfileSetErrmsg(pCsr, "table function zipfile() requires an argument");
+    return SQLITE_ERROR;
+  }
+
+  zFile = sqlite3_value_text(argv[0]);
+  pCsr->pFile = fopen(zFile, "rb");
+  if( pCsr->pFile==0 ){
+    zipfileSetErrmsg(pCsr, "cannot open file: %s", zFile);
+    rc = SQLITE_ERROR;
+  }else{
+    fseek(pCsr->pFile, 0, SEEK_END);
+    pCsr->nByte = (i64)ftell(pCsr->pFile);
+    rc = zipfileReadEOCD(pCsr, &pCsr->eocd);
+    if( rc==SQLITE_OK ){
+      pCsr->iNextOff = pCsr->eocd.iOffset;
+      rc = zipfileNext(cur);
+    }
+  }
+
+  return rc;
+}
+
+/*
+** xBestIndex callback.
+*/
+static int zipfileBestIndex(
+  sqlite3_vtab *tab,
+  sqlite3_index_info *pIdxInfo
+){
+  int i;
+
+  for(i=0; i<pIdxInfo->nConstraint; i++){
+    const struct sqlite3_index_constraint *pCons = &pIdxInfo->aConstraint[i];
+    if( pCons->usable==0 ) continue;
+    if( pCons->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
+    if( pCons->iColumn!=ZIPFILE_F_COLUMN_IDX ) continue;
+    break;
+  }
+
+  if( i<pIdxInfo->nConstraint ){
+    pIdxInfo->aConstraintUsage[i].argvIndex = 1;
+    pIdxInfo->aConstraintUsage[i].omit = 1;
+    pIdxInfo->estimatedCost = 1000.0;
+    pIdxInfo->idxNum = 1;
+  }else{
+    pIdxInfo->estimatedCost = (double)(((sqlite3_int64)1) << 50);
+    pIdxInfo->idxNum = 0;
+  }
+
+  return SQLITE_OK;
+}
+
+/*
+** Register the "zipfile" virtual table.
+*/
+static int zipfileRegister(sqlite3 *db){
+  static sqlite3_module zipfileModule = {
+    0,                         /* iVersion */
+    0,                         /* xCreate */
+    zipfileConnect,            /* xConnect */
+    zipfileBestIndex,          /* xBestIndex */
+    zipfileDisconnect,         /* xDisconnect */
+    0,                         /* xDestroy */
+    zipfileOpen,               /* xOpen - open a cursor */
+    zipfileClose,              /* xClose - close a cursor */
+    zipfileFilter,             /* xFilter - configure scan constraints */
+    zipfileNext,               /* xNext - advance a cursor */
+    zipfileEof,                /* xEof - check for end of scan */
+    zipfileColumn,             /* xColumn - read data */
+    zipfileRowid,              /* xRowid - read data */
+    0,                         /* xUpdate */
+    0,                         /* xBegin */
+    0,                         /* xSync */
+    0,                         /* xCommit */
+    0,                         /* xRollback */
+    0,                         /* xFindMethod */
+    0,                         /* xRename */
+  };
+
+  int rc = sqlite3_create_module(db, "zipfile"  , &zipfileModule, 0);
+  return rc;
+}
+#else         /* SQLITE_OMIT_VIRTUALTABLE */
+# define zipfileRegister(x) SQLITE_OK
+#endif
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_zipfile_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  int rc = SQLITE_OK;
+  SQLITE_EXTENSION_INIT2(pApi);
+  (void)pzErrMsg;  /* Unused parameter */
+  return zipfileRegister(db);
+}
+
diff --git a/main.mk b/main.mk
index ea0ff8bc2e..ff379da861 100644
--- a/main.mk
+++ b/main.mk
@@ -695,7 +695,8 @@ SHELL_SRC = \
 	$(TOP)/ext/misc/completion.c \
 	$(TOP)/ext/misc/sqlar.c \
 	$(TOP)/ext/expert/sqlite3expert.c \
-	$(TOP)/ext/expert/sqlite3expert.h
+	$(TOP)/ext/expert/sqlite3expert.h \
+	$(TOP)/ext/misc/zipfile.c
 
 shell.c:	$(SHELL_SRC) $(TOP)/tool/mkshellc.tcl
 	tclsh $(TOP)/tool/mkshellc.tcl >shell.c
diff --git a/manifest b/manifest
index 0293660ba4..a788cbbd26 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\senhancements\sfrom\strunk.
-D 2017-12-23T18:34:49.545
+C Add\snew\sfile\sext/misc/zipfile.c,\scontaining\sa\svirtual\stable\sfor\sread-only\naccess\sto\ssimple\szip\sarchives.
+D 2017-12-26T20:39:58.777
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
@@ -302,6 +302,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
+F ext/misc/zipfile.c 9736694a5eb029397e769f06517250be8b8e3836f4869246bfb60942a4047227
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -404,7 +405,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk eef9a9918485b5df70d7a69ed3d0e1dd182bf714740122a144905a59d42da5c6
+F main.mk fc13303745f7a06e2eac69406ee0e7ba481f6df37a528ceb6ebacc03e310a020
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -482,7 +483,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in e739db2809b9ad38ac389a89bead6986542a679ca33b153aef5850856998b525
+F src/shell.c.in 1c927f9407fa4e58ed114577971525209ea12a293d25fe689d1973d9fef17f74
 F src/sqlite.h.in 2126192945019d4cdce335cb236b440a05ec75c93e4cd94c9c6d6e7fcc654cc4
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1691,7 +1692,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 7652b3c2374084047b6c1da3e525e0cac34fe220597f81e793bc4fd9f33358da 07c773148d8db185fa54991df09298b64f4fef28879e6c9395759265e8183977
-R 0cdaf396f246262d8fd00807c02fe094
-U drh
-Z d44ebee6c3c37877974b2b88daec7bfe
+P 150f07fec1e6d1fc0601820d717d8712fc513fe0d4bed67c8679eb51bca30d53
+R ab04bfd243997fb0b46867b0347c7799
+U dan
+Z 06eecfc1ea586319973b447240342e1d
diff --git a/manifest.uuid b/manifest.uuid
index 3fe37939e4..85c9e9cdf5 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-150f07fec1e6d1fc0601820d717d8712fc513fe0d4bed67c8679eb51bca30d53
\ No newline at end of file
+8e366b99b13d765d8bf000a7ec5919e582702e51dc07c27a746b6002898a2302
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 444df58899..b3ef375c0b 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -797,6 +797,7 @@ INCLUDE ../ext/misc/shathree.c
 INCLUDE ../ext/misc/fileio.c
 INCLUDE ../ext/misc/completion.c
 #ifdef SQLITE_HAVE_ZLIB
+INCLUDE ../ext/misc/zipfile.c
 INCLUDE ../ext/misc/sqlar.c
 #endif
 INCLUDE ../ext/expert/sqlite3expert.h
@@ -3002,6 +3003,7 @@ static void open_db(ShellState *p, int keepAlive){
     sqlite3_shathree_init(p->db, 0, 0);
     sqlite3_completion_init(p->db, 0, 0);
 #ifdef SQLITE_HAVE_ZLIB
+    sqlite3_zipfile_init(p->db, 0, 0);
     sqlite3_sqlar_init(p->db, 0, 0);
 #endif
     sqlite3_create_function(p->db, "shell_add_schema", 2, SQLITE_UTF8, 0,

From 4396c61959eda774c250f50a2dc36f840658abda Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 27 Dec 2017 15:21:16 +0000
Subject: [PATCH 188/488] In LEMON, fix an off-by-one error that can make the
 lookahead table one byte too smal.

FossilOrigin-Name: 93792bc58a2eccc7e07b14307388350bb376db32c5055b79a44e4fa8ff91d58e
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 tool/lemon.c  |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index ffec829b50..784faa63b2 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Change\sthe\slemon-parser\scoverage\sreport\sformat\sto\sreport\sall\sstate/lookahead\npairs\sand\sindicate\son\seach\swhether\sit\sis\shit\sor\smissed.
-D 2017-12-26T18:32:06.041
+C In\sLEMON,\sfix\san\soff-by-one\serror\sthat\scan\smake\sthe\slookahead\stable\sone\nbyte\stoo\ssmal.
+D 2017-12-27T15:21:16.478
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -1608,7 +1608,7 @@ F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4
 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
 F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f
-F tool/lemon.c d92031adf63377ff5055441c9d26a41a03378e8cac8dbcc1cd6ef7190f51aa6a
+F tool/lemon.c 7f7735326ca9c3b48327b241063cee52d35d44e20ebe1b3624a81658052a4d39
 F tool/lempar.c 64e760f7cc21ea7e73513b062a86a3c478dfd1ed41fe9837a6f49958cdb61e03
 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9
 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d76e12066fa6950f877cbe33b1892e2b0afa948978815d5b9a90cde1fff8ec98
-R 930f1a264ff138489be35924f879ef78
+P 86e30fc284c740b55f75884b67988fe837b28878f586f6ec8850ecf80164e700
+R 4a4c12385039bdde527999ce093c45db
 U drh
-Z 2620b1a658dd06210670b65491310e1c
+Z dbb131aa448f925318d1af4566b5714c
diff --git a/manifest.uuid b/manifest.uuid
index 97230d7170..d4b1dea234 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-86e30fc284c740b55f75884b67988fe837b28878f586f6ec8850ecf80164e700
\ No newline at end of file
+93792bc58a2eccc7e07b14307388350bb376db32c5055b79a44e4fa8ff91d58e
\ No newline at end of file
diff --git a/tool/lemon.c b/tool/lemon.c
index 111082c72a..96bbed7473 100644
--- a/tool/lemon.c
+++ b/tool/lemon.c
@@ -766,7 +766,7 @@ int acttab_insert(acttab *p, int makeItSafe){
     p->aAction[k] = p->aLookahead[j];
     if( k>=p->nAction ) p->nAction = k+1;
   }
-  if( makeItSafe && i+p->nterminal>p->nAction ) p->nAction = i+p->nterminal;
+  if( makeItSafe && i+p->nterminal>=p->nAction ) p->nAction = i+p->nterminal+1;
   p->nLookahead = 0;
 
   /* Return the offset that is added to the lookahead in order to get the

From 7e7b753158f6c7fa0442f6609179df2512da2633 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 27 Dec 2017 16:13:22 +0000
Subject: [PATCH 189/488] In the lemon-generated parser, do not report the
 End-of-input character and the wildcard character as missed coverage.

FossilOrigin-Name: 3fe964873da16c0e0b1c4f1945f965d4137df7a307acd6a3eb6585ffbaa2afd1
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 tool/lempar.c |  3 ++-
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 784faa63b2..74d89bf960 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\sLEMON,\sfix\san\soff-by-one\serror\sthat\scan\smake\sthe\slookahead\stable\sone\nbyte\stoo\ssmal.
-D 2017-12-27T15:21:16.478
+C In\sthe\slemon-generated\sparser,\sdo\snot\sreport\sthe\sEnd-of-input\scharacter\sand\nthe\swildcard\scharacter\sas\smissed\scoverage.
+D 2017-12-27T16:13:22.261
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -1609,7 +1609,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
 F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f
 F tool/lemon.c 7f7735326ca9c3b48327b241063cee52d35d44e20ebe1b3624a81658052a4d39
-F tool/lempar.c 64e760f7cc21ea7e73513b062a86a3c478dfd1ed41fe9837a6f49958cdb61e03
+F tool/lempar.c 48ca9d9f280762da24d667f423c5421e09a6839d08a7d6a516d009db22974deb
 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9
 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 86e30fc284c740b55f75884b67988fe837b28878f586f6ec8850ecf80164e700
-R 4a4c12385039bdde527999ce093c45db
+P 93792bc58a2eccc7e07b14307388350bb376db32c5055b79a44e4fa8ff91d58e
+R 930e0d52d43acba766af53475563b556
 U drh
-Z dbb131aa448f925318d1af4566b5714c
+Z 594a8d4b6a5a5a1f42055bf091b10e50
diff --git a/manifest.uuid b/manifest.uuid
index d4b1dea234..5d9f174839 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-93792bc58a2eccc7e07b14307388350bb376db32c5055b79a44e4fa8ff91d58e
\ No newline at end of file
+3fe964873da16c0e0b1c4f1945f965d4137df7a307acd6a3eb6585ffbaa2afd1
\ No newline at end of file
diff --git a/tool/lempar.c b/tool/lempar.c
index 5404fa5f5f..e5dd50cfb7 100644
--- a/tool/lempar.c
+++ b/tool/lempar.c
@@ -475,7 +475,8 @@ int ParseCoverage(FILE *out){
   int i, j;
   int nMissed = 0;
   for(i=0; i<YYNSTATE; i++){
-    for(j=0; j<YYNTOKEN; j++){
+    for(j=1; j<YYNTOKEN; j++){
+      if( j==YYWILDCARD ) continue;
       if( !yycoverage[i][j] ) nMissed++;
       if( out ){
         fprintf(out,"State %d lookahead %s %s\n",

From 7038a991b049d35e7207d05fe587dc1f4cae8da1 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 27 Dec 2017 17:14:50 +0000
Subject: [PATCH 190/488] Change the coverage measurement logic in the
 lemon-generated parser so that it only checks for coverage of state/lookahead
 pairs that are valid syntax. It turns out that some states are unreachable if
 the lookahead is not valid syntax, because the states are only reachable
 through a shift following a reduce, and the reduce does not happen if the
 lookahead is a syntax error.

FossilOrigin-Name: 9dce46508772bd0f9e940c4d44933154044bb58c1b3511dd0143287bf795dd6b
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 tool/lempar.c | 26 ++++++++++++++++----------
 3 files changed, 23 insertions(+), 17 deletions(-)

diff --git a/manifest b/manifest
index 74d89bf960..cb95612f6f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\sthe\slemon-generated\sparser,\sdo\snot\sreport\sthe\sEnd-of-input\scharacter\sand\nthe\swildcard\scharacter\sas\smissed\scoverage.
-D 2017-12-27T16:13:22.261
+C Change\sthe\scoverage\smeasurement\slogic\sin\sthe\slemon-generated\sparser\sso\sthat\nit\sonly\schecks\sfor\scoverage\sof\sstate/lookahead\spairs\sthat\sare\svalid\ssyntax.\nIt\sturns\sout\sthat\ssome\sstates\sare\sunreachable\sif\sthe\slookahead\sis\snot\svalid\nsyntax,\sbecause\sthe\sstates\sare\sonly\sreachable\sthrough\sa\sshift\sfollowing\sa\nreduce,\sand\sthe\sreduce\sdoes\snot\shappen\sif\sthe\slookahead\sis\sa\ssyntax\serror.
+D 2017-12-27T17:14:50.938
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -1609,7 +1609,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
 F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f
 F tool/lemon.c 7f7735326ca9c3b48327b241063cee52d35d44e20ebe1b3624a81658052a4d39
-F tool/lempar.c 48ca9d9f280762da24d667f423c5421e09a6839d08a7d6a516d009db22974deb
+F tool/lempar.c 1b1279a362b7045b4e3214301665c12e32689e2c9caf33b35587669bac6b22fa
 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9
 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 93792bc58a2eccc7e07b14307388350bb376db32c5055b79a44e4fa8ff91d58e
-R 930e0d52d43acba766af53475563b556
+P 3fe964873da16c0e0b1c4f1945f965d4137df7a307acd6a3eb6585ffbaa2afd1
+R 43f297c7f87ea228ee76b1c746f9c4a6
 U drh
-Z 594a8d4b6a5a5a1f42055bf091b10e50
+Z 38efd78b8cbae26fc786eb4139191684
diff --git a/manifest.uuid b/manifest.uuid
index 5d9f174839..8cb561dc1a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-3fe964873da16c0e0b1c4f1945f965d4137df7a307acd6a3eb6585ffbaa2afd1
\ No newline at end of file
+9dce46508772bd0f9e940c4d44933154044bb58c1b3511dd0143287bf795dd6b
\ No newline at end of file
diff --git a/tool/lempar.c b/tool/lempar.c
index e5dd50cfb7..97fd421334 100644
--- a/tool/lempar.c
+++ b/tool/lempar.c
@@ -467,20 +467,25 @@ static unsigned char yycoverage[YYNSTATE][YYNTOKEN];
 
 /*
 ** Write into out a description of every state/lookahead combination that
-** has not previously been seen by the parser.  Return the number of 
-** missed state/lookahead combinations.
+**
+**   (1)  has not been used by the parser, and
+**   (2)  is not a syntax error.
+**
+** Return the number of missed state/lookahead combinations.
 */
 #if defined(YYCOVERAGE)
 int ParseCoverage(FILE *out){
-  int i, j;
+  int stateno, iLookAhead, i;
   int nMissed = 0;
-  for(i=0; i<YYNSTATE; i++){
-    for(j=1; j<YYNTOKEN; j++){
-      if( j==YYWILDCARD ) continue;
-      if( !yycoverage[i][j] ) nMissed++;
+  for(stateno=0; stateno<YYNSTATE; stateno++){
+    i = yy_shift_ofst[stateno];
+    for(iLookAhead=0; iLookAhead<YYNTOKEN; iLookAhead++){
+      if( yy_lookahead[i]!=iLookAhead ) continue;
+      if( yycoverage[stateno][iLookAhead]==0 ) nMissed++;
       if( out ){
-        fprintf(out,"State %d lookahead %s %s\n",
-           i, yyTokenName[j], yycoverage[i][j] ? "ok" : "missed");
+        fprintf(out,"State %d lookahead %s %s\n", stateno,
+                yyTokenName[iLookAhead],
+                yycoverage[stateno][iLookAhead] ? "ok" : "missed");
       }
     }
   }
@@ -506,9 +511,10 @@ static unsigned int yy_find_shift_action(
 #endif
   do{
     i = yy_shift_ofst[stateno];
+    assert( i>=0 && i+YYNTOKEN<=sizeof(yy_lookahead)/sizeof(yy_lookahead[0]) );
     assert( iLookAhead!=YYNOCODE );
+    assert( iLookAhead < YYNTOKEN );
     i += iLookAhead;
-    assert( i>=0 && i<sizeof(yy_lookahead)/sizeof(yy_lookahead[0]) );
     if( yy_lookahead[i]!=iLookAhead ){
 #ifdef YYFALLBACK
       YYCODETYPE iFallback;            /* Fallback token */

From cf8e0e9cb7bb334322d4e9688728e2193b6f84ce Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 27 Dec 2017 17:36:58 +0000
Subject: [PATCH 191/488] The previous check-in had an error in the coverage
 reporting logic.

FossilOrigin-Name: ec9b19eb652e81cd3c8a5595eb39aac3aca4abe72136c4b65c9cc257594a9c92
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 tool/lempar.c |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index cb95612f6f..2ca894e566 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Change\sthe\scoverage\smeasurement\slogic\sin\sthe\slemon-generated\sparser\sso\sthat\nit\sonly\schecks\sfor\scoverage\sof\sstate/lookahead\spairs\sthat\sare\svalid\ssyntax.\nIt\sturns\sout\sthat\ssome\sstates\sare\sunreachable\sif\sthe\slookahead\sis\snot\svalid\nsyntax,\sbecause\sthe\sstates\sare\sonly\sreachable\sthrough\sa\sshift\sfollowing\sa\nreduce,\sand\sthe\sreduce\sdoes\snot\shappen\sif\sthe\slookahead\sis\sa\ssyntax\serror.
-D 2017-12-27T17:14:50.938
+C The\sprevious\scheck-in\shad\san\serror\sin\sthe\scoverage\sreporting\slogic.
+D 2017-12-27T17:36:58.482
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -1609,7 +1609,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
 F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f
 F tool/lemon.c 7f7735326ca9c3b48327b241063cee52d35d44e20ebe1b3624a81658052a4d39
-F tool/lempar.c 1b1279a362b7045b4e3214301665c12e32689e2c9caf33b35587669bac6b22fa
+F tool/lempar.c dddd4f592b8bad36aec4500d456c5db5fe42fefc4ee384913880439d8917f87a
 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9
 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3fe964873da16c0e0b1c4f1945f965d4137df7a307acd6a3eb6585ffbaa2afd1
-R 43f297c7f87ea228ee76b1c746f9c4a6
+P 9dce46508772bd0f9e940c4d44933154044bb58c1b3511dd0143287bf795dd6b
+R 0e64cfa85d322f92630985625ab9cde3
 U drh
-Z 38efd78b8cbae26fc786eb4139191684
+Z afc4d6b44fe600f4cffd568a58de353c
diff --git a/manifest.uuid b/manifest.uuid
index 8cb561dc1a..3498a34465 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-9dce46508772bd0f9e940c4d44933154044bb58c1b3511dd0143287bf795dd6b
\ No newline at end of file
+ec9b19eb652e81cd3c8a5595eb39aac3aca4abe72136c4b65c9cc257594a9c92
\ No newline at end of file
diff --git a/tool/lempar.c b/tool/lempar.c
index 97fd421334..9164eb0c1e 100644
--- a/tool/lempar.c
+++ b/tool/lempar.c
@@ -480,7 +480,7 @@ int ParseCoverage(FILE *out){
   for(stateno=0; stateno<YYNSTATE; stateno++){
     i = yy_shift_ofst[stateno];
     for(iLookAhead=0; iLookAhead<YYNTOKEN; iLookAhead++){
-      if( yy_lookahead[i]!=iLookAhead ) continue;
+      if( yy_lookahead[i+iLookAhead]!=iLookAhead ) continue;
       if( yycoverage[stateno][iLookAhead]==0 ) nMissed++;
       if( out ){
         fprintf(out,"State %d lookahead %s %s\n", stateno,

From 5a78b81b1b5b8ab70ab61feeac7407379e3efeee Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 27 Dec 2017 18:54:11 +0000
Subject: [PATCH 192/488] Have the shell tool ".ar --list" and ".ar --extract"
 commands support zip files. Currently the "-zip" switch is required.

FossilOrigin-Name: a532a0f6fd59e81086d46f09151ba7fb26725198231d902c71d0f95cb01dbe91
---
 ext/misc/zipfile.c | 52 ++++++++++++++++++++++++++++++++++
 manifest           | 14 +++++-----
 manifest.uuid      |  2 +-
 src/shell.c.in     | 70 ++++++++++++++++++++++++++++++++++++++++------
 4 files changed, 121 insertions(+), 17 deletions(-)

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index bcee873044..cf9666a7ae 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -680,6 +680,54 @@ static int zipfileRegister(sqlite3 *db){
 # define zipfileRegister(x) SQLITE_OK
 #endif
 
+#include <zlib.h>
+
+/*
+** zipfile_uncompress(DATA, SZ, METHOD)
+*/
+static void zipfileUncompressFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  int iMethod;
+
+  iMethod = sqlite3_value_int(argv[2]);
+  if( iMethod==0 ){
+    sqlite3_result_value(context, argv[0]);
+  }else if( iMethod==8 ){
+    Byte *res;
+    int sz = sqlite3_value_int(argv[1]);
+    z_stream str;
+    memset(&str, 0, sizeof(str));
+    str.next_in = (Byte*)sqlite3_value_blob(argv[0]);
+    str.avail_in = sqlite3_value_bytes(argv[0]);
+    res = str.next_out = (Byte*)sqlite3_malloc(sz);
+    if( res==0 ){
+      sqlite3_result_error_nomem(context);
+    }else{
+      int err;
+      str.avail_out = sz;
+
+      err = inflateInit2(&str, -15);
+      if( err!=Z_OK ){
+        zipfileCtxErrorMsg(context, "inflateInit2() failed (%d)", err);
+      }else{
+        err = inflate(&str, Z_NO_FLUSH);
+        if( err!=Z_STREAM_END ){
+          zipfileCtxErrorMsg(context, "inflate() failed (%d)", err);
+        }else{
+          sqlite3_result_blob(context, res, sz, SQLITE_TRANSIENT);
+        }
+      }
+      sqlite3_free(res);
+      inflateEnd(&str);
+    }
+  }else{
+    zipfileCtxErrorMsg(context, "unrecognized compression method: %d", iMethod);
+  }
+}
+
 #ifdef _WIN32
 __declspec(dllexport)
 #endif
@@ -691,6 +739,10 @@ int sqlite3_zipfile_init(
   int rc = SQLITE_OK;
   SQLITE_EXTENSION_INIT2(pApi);
   (void)pzErrMsg;  /* Unused parameter */
+  rc = sqlite3_create_function(db, "zipfile_uncompress", 3,
+      SQLITE_UTF8, 0, zipfileUncompressFunc, 0, 0
+  );
+  if( rc!=SQLITE_OK ) return rc;
   return zipfileRegister(db);
 }
 
diff --git a/manifest b/manifest
index a788cbbd26..0cc3ca2bdd 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\snew\sfile\sext/misc/zipfile.c,\scontaining\sa\svirtual\stable\sfor\sread-only\naccess\sto\ssimple\szip\sarchives.
-D 2017-12-26T20:39:58.777
+C Have\sthe\sshell\stool\s".ar\s--list"\sand\s".ar\s--extract"\scommands\ssupport\szip\nfiles.\sCurrently\sthe\s"-zip"\sswitch\sis\srequired.
+D 2017-12-27T18:54:11.166
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
@@ -302,7 +302,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c 9736694a5eb029397e769f06517250be8b8e3836f4869246bfb60942a4047227
+F ext/misc/zipfile.c 96148b78b56664fe82f774e50dcdf6c83d693a1449b88011eba00cd6c697fedf
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -483,7 +483,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in 1c927f9407fa4e58ed114577971525209ea12a293d25fe689d1973d9fef17f74
+F src/shell.c.in d1be3030ee7afbbfb67972e4614b6d08dcae2e76460114d6b200fd28d0f008fb
 F src/sqlite.h.in 2126192945019d4cdce335cb236b440a05ec75c93e4cd94c9c6d6e7fcc654cc4
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1692,7 +1692,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 150f07fec1e6d1fc0601820d717d8712fc513fe0d4bed67c8679eb51bca30d53
-R ab04bfd243997fb0b46867b0347c7799
+P 8e366b99b13d765d8bf000a7ec5919e582702e51dc07c27a746b6002898a2302
+R 0d0cb33644ad535bd9f83812fd2ec78d
 U dan
-Z 06eecfc1ea586319973b447240342e1d
+Z 54715bdb695d1dc399a9b1cd512f6d24
diff --git a/manifest.uuid b/manifest.uuid
index 85c9e9cdf5..6167e93eb4 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-8e366b99b13d765d8bf000a7ec5919e582702e51dc07c27a746b6002898a2302
\ No newline at end of file
+a532a0f6fd59e81086d46f09151ba7fb26725198231d902c71d0f95cb01dbe91
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index b3ef375c0b..15a018bb9c 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -4238,7 +4238,13 @@ static void shellReset(
   sqlite3_stmt *pStmt
 ){
   int rc = sqlite3_reset(pStmt);
-  if( *pRc==SQLITE_OK ) *pRc = rc;
+  if( *pRc==SQLITE_OK ){
+    if( rc!=SQLITE_OK ){
+      sqlite3 *db = sqlite3_db_handle(pStmt);
+      raw_printf(stderr, "SQL error: %s\n", sqlite3_errmsg(db));
+    }
+    *pRc = rc;
+  }
 }
 
 /* 
@@ -4250,6 +4256,7 @@ struct ArCommand {
   const char *zFile;              /* --file argument, or NULL */
   const char *zDir;               /* --directory argument, or NULL */
   int bVerbose;                   /* True if --verbose */
+  int bZip;                       /* True if --zip */
   int nArg;                       /* Number of command arguments */
   char **azArg;                   /* Array of command arguments */
 };
@@ -4315,6 +4322,7 @@ static int arErrorMsg(const char *zFmt, ...){
 #define AR_SWITCH_VERBOSE   6
 #define AR_SWITCH_FILE      7
 #define AR_SWITCH_DIRECTORY 8
+#define AR_SWITCH_ZIP       9
 
 static int arProcessSwitch(ArCommand *pAr, int eSwitch, const char *zArg){
   switch( eSwitch ){
@@ -4332,6 +4340,9 @@ static int arProcessSwitch(ArCommand *pAr, int eSwitch, const char *zArg){
     case AR_SWITCH_VERBOSE:
       pAr->bVerbose = 1;
       break;
+    case AR_SWITCH_ZIP:
+      pAr->bZip = 1;
+      break;
 
     case AR_SWITCH_FILE:
       pAr->zFile = zArg;
@@ -4368,7 +4379,8 @@ static int arParseCommand(
     { 'h', "help",      AR_CMD_HELP, 0 },
     { 'v', "verbose",   AR_SWITCH_VERBOSE, 0 },
     { 'f', "file",      AR_SWITCH_FILE, 1 },
-    { 'C', "directory", AR_SWITCH_DIRECTORY, 1 }
+    { 'C', "directory", AR_SWITCH_DIRECTORY, 1 },
+    { 'z', "zip",       AR_SWITCH_ZIP, 0 }
   };
   int nSwitch = sizeof(aSwitch) / sizeof(struct ArSwitch);
   struct ArSwitch *pEnd = &aSwitch[nSwitch];
@@ -4501,7 +4513,12 @@ static int arCheckEntries(sqlite3 *db, ArCommand *pAr){
     int i;
     sqlite3_stmt *pTest = 0;
 
-    shellPrepare(db, &rc, "SELECT name FROM sqlar WHERE name=?", &pTest);
+    shellPreparePrintf(db, &rc, &pTest, "SELECT name FROM %s WHERE name=?1", 
+        pAr->bZip ? "zipfile(?2)" : "sqlar"
+    );
+    if( rc==SQLITE_OK && pAr->bZip ){
+      sqlite3_bind_text(pTest, 2, pAr->zFile, -1, SQLITE_TRANSIENT);
+    }
     for(i=0; i<pAr->nArg && rc==SQLITE_OK; i++){
       char *z = pAr->azArg[i];
       int n = strlen(z);
@@ -4564,7 +4581,9 @@ static void arWhereClause(
 ** Implementation of .ar "lisT" command. 
 */
 static int arListCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
-  const char *zSql = "SELECT name FROM sqlar WHERE %s"; 
+  const char *zSql = "SELECT name FROM %s WHERE %s"; 
+  const char *zTbl = (pAr->bZip ? "zipfile(?)" : "sqlar");
+
   char *zWhere = 0;
   sqlite3_stmt *pSql = 0;
   int rc;
@@ -4572,10 +4591,15 @@ static int arListCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
   rc = arCheckEntries(db, pAr);
   arWhereClause(&rc, pAr, &zWhere);
 
-  shellPreparePrintf(db, &rc, &pSql, zSql, zWhere);
+  shellPreparePrintf(db, &rc, &pSql, zSql, zTbl, zWhere);
+  if( rc==SQLITE_OK && pAr->bZip ){
+    sqlite3_bind_text(pSql, 1, pAr->zFile, -1, SQLITE_TRANSIENT);
+  }
   while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
     raw_printf(p->out, "%s\n", sqlite3_column_text(pSql, 0));
   }
+
+  shellFinalize(&rc, pSql);
   return rc;
 }
 
@@ -4587,8 +4611,18 @@ static int arExtractCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
   const char *zSql1 = 
     "SELECT "
     "  :1 || name, "
-    "  writefile(:1 || name, sqlar_uncompress(data, sz), mode, mtime) "
-    "FROM sqlar WHERE (%s) AND (data IS NULL OR :2 = 0)";
+    "  writefile(?1 || name, %s, mode, mtime) "
+    "FROM %s WHERE (%s) AND (data IS NULL OR ?2 = 0)";
+
+  const char *azExtraArg[] = { 
+    "sqlar_uncompress(data, sz)",
+    "zipfile_uncompress(data, sz, method)"
+  };
+  const char *azSource[] = {
+    "sqlar", "zipfile(?3)"
+  };
+
+
 
   struct timespec times[2];
   sqlite3_stmt *pSql = 0;
@@ -4615,9 +4649,15 @@ static int arExtractCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
   memset(times, 0, sizeof(times));
   times[0].tv_sec = time(0);
 
-  shellPreparePrintf(db, &rc, &pSql, zSql1, zWhere);
+  shellPreparePrintf(db, &rc, &pSql, zSql1, 
+      azExtraArg[pAr->bZip], azSource[pAr->bZip], zWhere
+  );
+
   if( rc==SQLITE_OK ){
     sqlite3_bind_text(pSql, 1, zDir, -1, SQLITE_STATIC);
+    if( pAr->bZip ){
+      sqlite3_bind_text(pSql, 3, pAr->zFile, -1, SQLITE_STATIC);
+    }
 
     /* Run the SELECT statement twice. The first time, writefile() is called
     ** for all archive members that should be extracted. The second time,
@@ -4676,6 +4716,8 @@ static int arCreateUpdate(
   int i;                          /* For iterating through azFile[] */
   int rc;                         /* Return code */
 
+  assert( pAr->bZip==0 );
+
   rc = sqlite3_exec(db, "SAVEPOINT ar;", 0, 0, 0);
   if( rc!=SQLITE_OK ) return rc;
 
@@ -4772,7 +4814,17 @@ static int arDotCommand(
   if( rc==SQLITE_OK ){
     sqlite3 *db = 0;              /* Database handle to use as archive */
 
-    if( cmd.zFile ){
+    if( cmd.bZip ){
+      if( cmd.zFile==0 ){
+        raw_printf(stderr, "zip format requires a --file switch\n");
+        return SQLITE_ERROR;
+      }else
+      if( cmd.eCmd==AR_CMD_CREATE || cmd.eCmd==AR_CMD_UPDATE ){
+        raw_printf(stderr, "zip archives are read-only\n");
+        return SQLITE_ERROR;
+      }
+      db = pState->db;
+    }else if( cmd.zFile ){
       int flags;
       if( cmd.eCmd==AR_CMD_CREATE || cmd.eCmd==AR_CMD_UPDATE ){
         flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE;

From 9b2e0435d2f1d993af09645d6b03320c8e638d74 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 27 Dec 2017 19:43:22 +0000
Subject: [PATCH 193/488] Add and use the internal sqlite3DbSpanDup()
 interface.

FossilOrigin-Name: a8e1545cb7aacb6a26a8c92a3ad4a3d584d150c3a00d2828c8adbb1ee19fcb6d
---
 manifest        | 20 ++++++++++----------
 manifest.uuid   |  2 +-
 src/build.c     |  6 +-----
 src/expr.c      |  6 +-----
 src/malloc.c    | 13 +++++++++++++
 src/sqliteInt.h |  1 +
 test/capi2.test |  2 +-
 7 files changed, 28 insertions(+), 22 deletions(-)

diff --git a/manifest b/manifest
index c8a3a065d3..0b31af5927 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\srecent\senhancements\sfrom\strunk.
-D 2017-12-27T19:27:26.668
+C Add\sand\suse\sthe\sinternal\ssqlite3DbSpanDup()\sinterface.
+D 2017-12-27T19:43:22.931
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -423,7 +423,7 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
 F src/btree.c b83a6b03f160528020bb965f0c3a40af5286cd4923c3870fd218177f03a120a7
 F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09
 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc
-F src/build.c 972224edcc3385f1d6ad265f84a8120f777a2c4f7405ad848def0aee082a9573
+F src/build.c a03eb5a1cfff74784c24a4478ba5455711571936f1ac9d43f94fa7df57509977
 F src/callback.c fe677cb5f5abb02f7a772a62a98c2f516426081df68856e8f2d5f950929b966a
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
@@ -431,7 +431,7 @@ F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957
 F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
 F src/delete.c 74667ad914ac143731a444a1bacf29ceb18f6eded8a0dd17aafae80baa07f8bb
-F src/expr.c 04188faa71b86bff3a1de46d92f41b9ec2b18fa1bbbdbf683cfff657fe803f3d
+F src/expr.c f03666d7f50caf4139eb777085833808028e8af6b6dfafa722b3e843a369a126
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331
 F src/func.c 0fb9a2d678d3c8aba89b46468b309cd7e8fa9806a369a30aa89024660845bb13
@@ -444,7 +444,7 @@ F src/insert.c cb67cc56ef2ddd13e6944b2c0dd08a920bcd9503230adef8b9928d338097c722
 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e
 F src/loadext.c 55bcc3c741059a1056859e8adaf133aa179e22be12215c0936b2f354ef71209b
 F src/main.c 690c4134f944cbd5b71d59dd6e61ce4131f6a50ab774f38108e57d07d79cf876
-F src/malloc.c a02c9e69bc76bee0f639416b947a946412890b606301454727feadcb313536d6
+F src/malloc.c 6f684fd039f53bf9195193eb0cde731a2954970fabc6ef054ba379b6052dc296
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
 F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3
@@ -483,7 +483,7 @@ F src/shell.c.in f3ec8f90dd698ea98781a90642c91eacbc24f4e55bb551c7b2762000d3ef55d
 F src/sqlite.h.in b4dc75265ed04b98e2184011a7dd0054ce2137ff84867a6be8b4f3bdfbc03d30
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
-F src/sqliteInt.h d7054a71c87fe6b21c42147d4857f5baa8f19e959003b6011e473a1f5f93a542
+F src/sqliteInt.h 707e078db16dafad27401bb023da2dd12bafb0a2a421900ef4a1ff1e38c2b01f
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -658,7 +658,7 @@ F test/busy.test 510dc6daaad18bcbbc085bcc6217d6dc418def5e73f72ce1475eea0cb783472
 F test/cache.test 13bc046b26210471ca6f2889aceb1ea52dc717de
 F test/cacheflush.test af25bb1509df04c1da10e38d8f322d66eceedf61
 F test/cachespill.test 895997f84a25b323b166aecb69baab2d6380ea98f9e0bcc688c4493c535cfab9
-F test/capi2.test 011c16da245fdc0106a2785035de6b242c05e738
+F test/capi2.test d3267a146df2251e7ad09a672d59e33e836d74c6fbeb843c18e87835ef646c12
 F test/capi3.test 986e57cea8ab423b3fc8c2e3b69330394252d3d2a4496122ff3749e258305695
 F test/capi3b.test efb2b9cfd127efa84433cd7a2d72ce0454ae0dc4
 F test/capi3c.test 7ebed1d8fa2f3190149d556fe8cff5a006be62af437c5c4640db614470126098
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 74a0181fc016fb0b7934188cfcc2ec26c689f0d33cc42453875f4392b7240d3a 1b22b42e59793af19c69a2e5f6822883cc2687d4a0d9b9280bbff885276c6baa
-R a3246aa6793f9a1aed1e1936973e7f16
+P 76373091354f27577e5235e96cfaa4b31429a38d1e0e1d6e094c4cc323b92a0a
+R 913c4466610a0123883763647119d1c7
 U drh
-Z 2e8cea4c20132476e571ae504e9351f9
+Z 28dbbdc669e3541be1657ab76b942c72
diff --git a/manifest.uuid b/manifest.uuid
index c3e1771bba..9cbf31ff1c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-76373091354f27577e5235e96cfaa4b31429a38d1e0e1d6e094c4cc323b92a0a
\ No newline at end of file
+a8e1545cb7aacb6a26a8c92a3ad4a3d584d150c3a00d2828c8adbb1ee19fcb6d
\ No newline at end of file
diff --git a/src/build.c b/src/build.c
index 9447d02d48..26eb0579e6 100644
--- a/src/build.c
+++ b/src/build.c
@@ -1241,14 +1241,10 @@ void sqlite3AddDefaultValue(
       ** tokens that point to volatile memory.	
       */
       Expr x;
-      int n;
       sqlite3ExprDelete(db, pCol->pDflt);
       memset(&x, 0, sizeof(x));
       x.op = TK_SPAN;
-      while( sqlite3Isspace(zStart[0]) ) zStart++;
-      n = (int)(zEnd - zStart);
-      while( n>0 && sqlite3Isspace(zStart[n-1]) ) n--;
-      x.u.zToken = sqlite3DbStrNDup(db, zStart, n);
+      x.u.zToken = sqlite3DbSpanDup(db, zStart, zEnd);
       x.pLeft = pExpr;
       x.flags = EP_Skip;
       pCol->pDflt = sqlite3ExprDup(db, &x, EXPRDUP_REDUCE);
diff --git a/src/expr.c b/src/expr.c
index 129c1299bf..1592f10408 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -1661,13 +1661,9 @@ void sqlite3ExprListSetSpan(
   assert( pList!=0 || db->mallocFailed!=0 );
   if( pList ){
     struct ExprList_item *pItem = &pList->a[pList->nExpr-1];
-    int n;
     assert( pList->nExpr>0 );
     sqlite3DbFree(db, pItem->zSpan);
-    while( sqlite3Isspace(zStart[0]) ) zStart++;
-    n = (int)(zEnd - zStart);
-    while( n>0 && sqlite3Isspace(zStart[n-1]) ) n--;
-    pItem->zSpan = sqlite3DbStrNDup(db, zStart, n);
+    pItem->zSpan = sqlite3DbSpanDup(db, zStart, zEnd);
   }
 }
 
diff --git a/src/malloc.c b/src/malloc.c
index b750f6e72f..ec2d93ac89 100644
--- a/src/malloc.c
+++ b/src/malloc.c
@@ -627,6 +627,19 @@ char *sqlite3DbStrNDup(sqlite3 *db, const char *z, u64 n){
   return zNew;
 }
 
+/*
+** The text between zStart and zEnd represents a phrase within a larger
+** SQL statement.  Make a copy of this phrase in space obtained form
+** sqlite3DbMalloc().  Omit leading and trailing whitespace.
+*/
+char *sqlite3DbSpanDup(sqlite3 *db, const char *zStart, const char *zEnd){
+  int n;
+  while( sqlite3Isspace(zStart[0]) ) zStart++;
+  n = (int)(zEnd - zStart);
+  while( n>0 && sqlite3Isspace(zStart[n-1]) ) n--;
+  return sqlite3DbStrNDup(db, zStart, n);
+}
+
 /*
 ** Free any prior content in *pz and replace it with a copy of zNew.
 */
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index af351fc114..a51bae25c1 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -3513,6 +3513,7 @@ void *sqlite3DbMallocRaw(sqlite3*, u64);
 void *sqlite3DbMallocRawNN(sqlite3*, u64);
 char *sqlite3DbStrDup(sqlite3*,const char*);
 char *sqlite3DbStrNDup(sqlite3*,const char*, u64);
+char *sqlite3DbSpanDup(sqlite3*,const char*,const char*);
 void *sqlite3Realloc(void*, u64);
 void *sqlite3DbReallocOrFree(sqlite3 *, void *, u64);
 void *sqlite3DbRealloc(sqlite3 *, void *, u64);
diff --git a/test/capi2.test b/test/capi2.test
index 39f50dd079..5e51871814 100644
--- a/test/capi2.test
+++ b/test/capi2.test
@@ -184,7 +184,7 @@ do_test capi2-3.5 {
 } {1 {(1) no such column: bogus} {;;x;}}
 do_test capi2-3.6 {
   set rc [catch {
-      sqlite3_prepare $DB {select 5/0} -1 TAIL
+      sqlite3_prepare $DB {select 5/0;} -1 TAIL
   } VM]
   lappend rc $TAIL
 } {0 {}}

From f259df5fc8f6cff45a322e3cd3eca4db54fa6e1d Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 27 Dec 2017 20:38:35 +0000
Subject: [PATCH 194/488] Show the text of individual statements within a
 trigger, as they execute, as comments in the output from sqlite3_trace() and
 sqlite3_trace_v2().

FossilOrigin-Name: fe3d2b97d8945e6a9636472f77bcdc92cc5dda8b3e6e3a4fcbdd0a212ea5b9aa
---
 manifest        | 20 ++++++++++----------
 manifest.uuid   |  2 +-
 src/parse.y     | 24 +++++++++++-------------
 src/sqliteInt.h | 12 ++++++++----
 src/trigger.c   | 49 +++++++++++++++++++++++++++++++++++++++++--------
 src/vdbe.c      | 14 ++++++++++++--
 test/trace.test |  2 +-
 7 files changed, 84 insertions(+), 39 deletions(-)

diff --git a/manifest b/manifest
index 0b31af5927..e45bddc5bc 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sand\suse\sthe\sinternal\ssqlite3DbSpanDup()\sinterface.
-D 2017-12-27T19:43:22.931
+C Show\sthe\stext\sof\sindividual\sstatements\swithin\sa\strigger,\sas\sthey\nexecute,\sas\scomments\sin\sthe\soutput\sfrom\ssqlite3_trace()\sand\ssqlite3_trace_v2().
+D 2017-12-27T20:38:35.013
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -467,7 +467,7 @@ F src/os_win.c 0a4afa35cc8e812000df3ea2f64b476131b39e29e75d8007d0504726e4761de4
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
-F src/parse.y 2b0653fd44534d941ff5289f372fedfc213e78c4698e28334d073cf4cfd146f2
+F src/parse.y 44cbbc3e132ea128258eff7be7f6d5c5dfa25522f89ec8b5501276966511bd50
 F src/pcache.c 7ae91a4557a43d77d449accbfdc68846e6516f8e2eda46e8bbe4536fb669b201
 F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170
 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
@@ -483,7 +483,7 @@ F src/shell.c.in f3ec8f90dd698ea98781a90642c91eacbc24f4e55bb551c7b2762000d3ef55d
 F src/sqlite.h.in b4dc75265ed04b98e2184011a7dd0054ce2137ff84867a6be8b4f3bdfbc03d30
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
-F src/sqliteInt.h 707e078db16dafad27401bb023da2dd12bafb0a2a421900ef4a1ff1e38c2b01f
+F src/sqliteInt.h 394655443ab2ee9bde2c2f629bd1e613d5ee211dd6c16995f68e6014bbe60b5d
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -542,12 +542,12 @@ F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
 F src/tokenize.c 1003d6d90c6783206c711f0a9397656fa5b055209f4d092caa43bb3bf5215db5
 F src/treeview.c eae35972ff44f67064de2eaf35f04afe94e7aea3271a8b3bcebb3f954880fec3
-F src/trigger.c 775053eecf6b73062e243404b56f5064446254d5cce17d8704d5cdffd72a546a
+F src/trigger.c 5fa038ae62adae53b62776bacc34dc56191385eedac79d49264389620ddc2526
 F src/update.c 961bd1265d4d1e5cd65c9a54fa5122fb7aefcb003fcf2de0c092fceb7e58972c
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c 7315e97a8dc2c8e19ca64196c652cf0a65d13fd0a211b2cec082062372dc6261
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
-F src/vdbe.c 3393b508d9ad084ffce232a7c53e375ef5ac99b50b685c5131fcdfce97a9d534
+F src/vdbe.c e0a1ca7539d7071d5fa582bdf27195a51197b5ae1ab0f143888b1561df592fd0
 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97
 F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9
 F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1
@@ -1443,7 +1443,7 @@ F test/tkt3997.test a335fa41ca3985660a139df7b734a26ef53284bd
 F test/tkt4018.test 18dbc6617f7a4b90e938d1bd6d26ad18daafaf08
 F test/tokenize.test ce430a7aed48fc98301611429595883fdfcab5d7
 F test/tpch01.test 04adbf8d8300fa60a222f28d901abd76e7be6dd4
-F test/trace.test 6f676313e3ebd2a50585036d2f212a3319dd5836
+F test/trace.test a659a9862957f4789e37a92b3bf6d2caf5c86b02cdeefc41e850ae53acf6992a
 F test/trace2.test f5cb67ad3bc09e0c58e8cca78dfd0b5639259983
 F test/trace3.test 56ab944fddacf628b118cc298503fc45c2e50ab0
 F test/trans.test 6e1b4c6a42dba31bd65f8fa5e61a2708e08ddde6
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 76373091354f27577e5235e96cfaa4b31429a38d1e0e1d6e094c4cc323b92a0a
-R 913c4466610a0123883763647119d1c7
+P a8e1545cb7aacb6a26a8c92a3ad4a3d584d150c3a00d2828c8adbb1ee19fcb6d
+R 0ae5195579783355d0e4696ba0c5bdad
 U drh
-Z 28dbbdc669e3541be1657ab76b942c72
+Z 4ebaaf3c94dd44dc4a2d0b266bee40f3
diff --git a/manifest.uuid b/manifest.uuid
index 9cbf31ff1c..49211259e2 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a8e1545cb7aacb6a26a8c92a3ad4a3d584d150c3a00d2828c8adbb1ee19fcb6d
\ No newline at end of file
+fe3d2b97d8945e6a9636472f77bcdc92cc5dda8b3e6e3a4fcbdd0a212ea5b9aa
\ No newline at end of file
diff --git a/src/parse.y b/src/parse.y
index fe2251db4b..e780f8c3ab 100644
--- a/src/parse.y
+++ b/src/parse.y
@@ -288,11 +288,8 @@ signed ::= minus_num.
 //
 %type scanpt {const char*}
 scanpt(A) ::= . {
-  if( yyLookahead!=YYNOCODE ){
-    A = yyLookaheadToken.z;
-  }else{
-    A = pParse->sLastToken.z + pParse->sLastToken.n;
-  }
+  assert( yyLookahead!=YYNOCODE );
+  A = yyLookaheadToken.z;
 }
 
 // "carglist" is a list of additional constraints that come after the
@@ -1371,20 +1368,21 @@ tridxby ::= NOT INDEXED. {
 %destructor trigger_cmd {sqlite3DeleteTriggerStep(pParse->db, $$);}
 // UPDATE 
 trigger_cmd(A) ::=
-   UPDATE orconf(R) trnm(X) tridxby SET setlist(Y) where_opt(Z).  
-   {A = sqlite3TriggerUpdateStep(pParse->db, &X, Y, Z, R);}
+   UPDATE(B) orconf(R) trnm(X) tridxby SET setlist(Y) where_opt(Z) scanpt(E).  
+   {A = sqlite3TriggerUpdateStep(pParse->db, &X, Y, Z, R, B.z, E);}
 
 // INSERT
-trigger_cmd(A) ::= insert_cmd(R) INTO trnm(X) idlist_opt(F) select(S).
-   {A = sqlite3TriggerInsertStep(pParse->db, &X, F, S, R);/*A-overwrites-R*/}
+trigger_cmd(A) ::= scanpt(B) insert_cmd(R) INTO
+                      trnm(X) idlist_opt(F) select(S) scanpt(Z).
+   {A = sqlite3TriggerInsertStep(pParse->db,&X,F,S,R,B,Z);/*A-overwrites-R*/}
 
 // DELETE
-trigger_cmd(A) ::= DELETE FROM trnm(X) tridxby where_opt(Y).
-   {A = sqlite3TriggerDeleteStep(pParse->db, &X, Y);}
+trigger_cmd(A) ::= DELETE(B) FROM trnm(X) tridxby where_opt(Y) scanpt(E).
+   {A = sqlite3TriggerDeleteStep(pParse->db, &X, Y, B.z, E);}
 
 // SELECT
-trigger_cmd(A) ::= select(X).
-   {A = sqlite3TriggerSelectStep(pParse->db, X); /*A-overwrites-X*/}
+trigger_cmd(A) ::= scanpt(B) select(X) scanpt(E).
+   {A = sqlite3TriggerSelectStep(pParse->db, X, B, E); /*A-overwrites-X*/}
 
 // The special RAISE expression that may occur in trigger programs
 expr(A) ::= RAISE LP IGNORE RP.  {
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index a51bae25c1..b11b5a8ada 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -3203,6 +3203,7 @@ struct TriggerStep {
   Expr *pWhere;        /* The WHERE clause for DELETE or UPDATE steps */
   ExprList *pExprList; /* SET clause for UPDATE. */
   IdList *pIdList;     /* Column names for INSERT */
+  char *zSpan;         /* Original SQL text of this command */
   TriggerStep *pNext;  /* Next in the link-list */
   TriggerStep *pLast;  /* Last element in link-list. Valid for 1st elem only */
 };
@@ -3903,11 +3904,14 @@ void sqlite3MaterializeView(Parse*, Table*, Expr*, ExprList*,Expr*,int);
   void sqlite3CodeRowTriggerDirect(Parse *, Trigger *, Table *, int, int, int);
   void sqliteViewTriggers(Parse*, Table*, Expr*, int, ExprList*);
   void sqlite3DeleteTriggerStep(sqlite3*, TriggerStep*);
-  TriggerStep *sqlite3TriggerSelectStep(sqlite3*,Select*);
+  TriggerStep *sqlite3TriggerSelectStep(sqlite3*,Select*,
+                                        const char*,const char*);
   TriggerStep *sqlite3TriggerInsertStep(sqlite3*,Token*, IdList*,
-                                        Select*,u8);
-  TriggerStep *sqlite3TriggerUpdateStep(sqlite3*,Token*,ExprList*, Expr*, u8);
-  TriggerStep *sqlite3TriggerDeleteStep(sqlite3*,Token*, Expr*);
+                                        Select*,u8,const char*,const char*);
+  TriggerStep *sqlite3TriggerUpdateStep(sqlite3*,Token*,ExprList*, Expr*, u8,
+                                        const char*,const char*);
+  TriggerStep *sqlite3TriggerDeleteStep(sqlite3*,Token*, Expr*,
+                                        const char*,const char*);
   void sqlite3DeleteTrigger(sqlite3*, Trigger*);
   void sqlite3UnlinkAndDeleteTrigger(sqlite3*,int,const char*);
   u32 sqlite3TriggerColmask(Parse*,Trigger*,ExprList*,int,int,Table*,int);
diff --git a/src/trigger.c b/src/trigger.c
index d8aac2cc22..cf248a57bc 100644
--- a/src/trigger.c
+++ b/src/trigger.c
@@ -25,6 +25,7 @@ void sqlite3DeleteTriggerStep(sqlite3 *db, TriggerStep *pTriggerStep){
     sqlite3ExprListDelete(db, pTmp->pExprList);
     sqlite3SelectDelete(db, pTmp->pSelect);
     sqlite3IdListDelete(db, pTmp->pIdList);
+    sqlite3DbFree(db, pTmp->zSpan);
 
     sqlite3DbFree(db, pTmp);
   }
@@ -339,6 +340,17 @@ triggerfinish_cleanup:
   sqlite3DeleteTriggerStep(db, pStepList);
 }
 
+/*
+** Duplicate a range of text from an SQL statement, then convert all
+** whitespace characters into ordinary space characters.
+*/
+static char *triggerSpanDup(sqlite3 *db, const char *zStart, const char *zEnd){
+  char *z = sqlite3DbSpanDup(db, zStart, zEnd);
+  int i;
+  if( z ) for(i=0; z[i]; i++) if( sqlite3Isspace(z[i]) ) z[i] = ' ';
+  return z;
+}    
+
 /*
 ** Turn a SELECT statement (that the pSelect parameter points to) into
 ** a trigger step.  Return a pointer to a TriggerStep structure.
@@ -346,7 +358,12 @@ triggerfinish_cleanup:
 ** The parser calls this routine when it finds a SELECT statement in
 ** body of a TRIGGER.  
 */
-TriggerStep *sqlite3TriggerSelectStep(sqlite3 *db, Select *pSelect){
+TriggerStep *sqlite3TriggerSelectStep(
+  sqlite3 *db,                /* Database connection */
+  Select *pSelect,            /* The SELECT statement */
+  const char *zStart,         /* Start of SQL text */
+  const char *zEnd            /* End of SQL text */
+){
   TriggerStep *pTriggerStep = sqlite3DbMallocZero(db, sizeof(TriggerStep));
   if( pTriggerStep==0 ) {
     sqlite3SelectDelete(db, pSelect);
@@ -355,6 +372,7 @@ TriggerStep *sqlite3TriggerSelectStep(sqlite3 *db, Select *pSelect){
   pTriggerStep->op = TK_SELECT;
   pTriggerStep->pSelect = pSelect;
   pTriggerStep->orconf = OE_Default;
+  pTriggerStep->zSpan = triggerSpanDup(db, zStart, zEnd);
   return pTriggerStep;
 }
 
@@ -367,7 +385,9 @@ TriggerStep *sqlite3TriggerSelectStep(sqlite3 *db, Select *pSelect){
 static TriggerStep *triggerStepAllocate(
   sqlite3 *db,                /* Database connection */
   u8 op,                      /* Trigger opcode */
-  Token *pName                /* The target name */
+  Token *pName,               /* The target name */
+  const char *zStart,         /* Start of SQL text */
+  const char *zEnd            /* End of SQL text */
 ){
   TriggerStep *pTriggerStep;
 
@@ -378,6 +398,7 @@ static TriggerStep *triggerStepAllocate(
     sqlite3Dequote(z);
     pTriggerStep->zTarget = z;
     pTriggerStep->op = op;
+    pTriggerStep->zSpan = triggerSpanDup(db, zStart, zEnd);
   }
   return pTriggerStep;
 }
@@ -394,13 +415,15 @@ TriggerStep *sqlite3TriggerInsertStep(
   Token *pTableName,  /* Name of the table into which we insert */
   IdList *pColumn,    /* List of columns in pTableName to insert into */
   Select *pSelect,    /* A SELECT statement that supplies values */
-  u8 orconf           /* The conflict algorithm (OE_Abort, OE_Replace, etc.) */
+  u8 orconf,          /* The conflict algorithm (OE_Abort, OE_Replace, etc.) */
+  const char *zStart, /* Start of SQL text */
+  const char *zEnd    /* End of SQL text */
 ){
   TriggerStep *pTriggerStep;
 
   assert(pSelect != 0 || db->mallocFailed);
 
-  pTriggerStep = triggerStepAllocate(db, TK_INSERT, pTableName);
+  pTriggerStep = triggerStepAllocate(db, TK_INSERT, pTableName, zStart, zEnd);
   if( pTriggerStep ){
     pTriggerStep->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE);
     pTriggerStep->pIdList = pColumn;
@@ -423,11 +446,13 @@ TriggerStep *sqlite3TriggerUpdateStep(
   Token *pTableName,   /* Name of the table to be updated */
   ExprList *pEList,    /* The SET clause: list of column and new values */
   Expr *pWhere,        /* The WHERE clause */
-  u8 orconf            /* The conflict algorithm. (OE_Abort, OE_Ignore, etc) */
+  u8 orconf,           /* The conflict algorithm. (OE_Abort, OE_Ignore, etc) */
+  const char *zStart,  /* Start of SQL text */
+  const char *zEnd     /* End of SQL text */
 ){
   TriggerStep *pTriggerStep;
 
-  pTriggerStep = triggerStepAllocate(db, TK_UPDATE, pTableName);
+  pTriggerStep = triggerStepAllocate(db, TK_UPDATE, pTableName, zStart, zEnd);
   if( pTriggerStep ){
     pTriggerStep->pExprList = sqlite3ExprListDup(db, pEList, EXPRDUP_REDUCE);
     pTriggerStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE);
@@ -446,11 +471,13 @@ TriggerStep *sqlite3TriggerUpdateStep(
 TriggerStep *sqlite3TriggerDeleteStep(
   sqlite3 *db,            /* Database connection */
   Token *pTableName,      /* The table from which rows are deleted */
-  Expr *pWhere            /* The WHERE clause */
+  Expr *pWhere,           /* The WHERE clause */
+  const char *zStart,     /* Start of SQL text */
+  const char *zEnd        /* End of SQL text */
 ){
   TriggerStep *pTriggerStep;
 
-  pTriggerStep = triggerStepAllocate(db, TK_DELETE, pTableName);
+  pTriggerStep = triggerStepAllocate(db, TK_DELETE, pTableName, zStart, zEnd);
   if( pTriggerStep ){
     pTriggerStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE);
     pTriggerStep->orconf = OE_Default;
@@ -705,6 +732,12 @@ static int codeTriggerProgram(
     pParse->eOrconf = (orconf==OE_Default)?pStep->orconf:(u8)orconf;
     assert( pParse->okConstFactor==0 );
 
+#ifndef SQLITE_OMIT_TRACE
+    sqlite3VdbeAddOp4(v, OP_Trace, 0x7fffffff, 1, 0,
+                      sqlite3MPrintf(db, "-- %s", pStep->zSpan),
+                      P4_DYNAMIC);
+#endif
+
     switch( pStep->op ){
       case TK_UPDATE: {
         sqlite3Update(pParse, 
diff --git a/src/vdbe.c b/src/vdbe.c
index cfe18a9d1e..1005560562 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -7016,7 +7016,13 @@ case OP_Function: {
   break;
 }
 
-
+/* Opcode: Trace P1 P2 * P4 *
+**
+** Write P4 on the statement trace output if statement tracing is
+** enabled.
+**
+** Operand P1 must be 0x7fffffff and P2 must positive.
+*/
 /* Opcode: Init P1 P2 P3 P4 *
 ** Synopsis: Start at P2
 **
@@ -7035,6 +7041,7 @@ case OP_Function: {
 ** If P3 is not zero, then it is an address to jump to if an SQLITE_CORRUPT
 ** error is encountered.
 */
+case OP_Trace:
 case OP_Init: {          /* jump */
   char *zTrace;
   int i;
@@ -7049,7 +7056,9 @@ case OP_Init: {          /* jump */
   ** sqlite3_expanded_sql(P) otherwise.
   */
   assert( pOp->p4.z==0 || strncmp(pOp->p4.z, "-" "- ", 3)==0 );
-  assert( pOp==p->aOp );  /* Always instruction 0 */
+
+  /* OP_Init is always instruction 0 */
+  assert( pOp==p->aOp || pOp->opcode==OP_Trace );
 
 #ifndef SQLITE_OMIT_TRACE
   if( (db->mTrace & (SQLITE_TRACE_STMT|SQLITE_TRACE_LEGACY))!=0
@@ -7092,6 +7101,7 @@ case OP_Init: {          /* jump */
 #endif /* SQLITE_OMIT_TRACE */
   assert( pOp->p2>0 );
   if( pOp->p1>=sqlite3GlobalConfig.iOnceResetThreshold ){
+    if( pOp->opcode==OP_Trace ) break;
     for(i=1; i<p->nOp; i++){
       if( p->aOp[i].opcode==OP_Once ) p->aOp[i].p1 = 0;
     }
diff --git a/test/trace.test b/test/trace.test
index fd51d7ab64..37914857fc 100644
--- a/test/trace.test
+++ b/test/trace.test
@@ -197,7 +197,7 @@ ifcapable trigger {
       UPDATE t1 SET a=a+1;
     }
     set TRACE_OUT
-  } {{UPDATE t1 SET a=a+1;} {-- TRIGGER r1t1} {-- TRIGGER r1t2} {-- TRIGGER r1t1} {-- TRIGGER r1t2} {-- TRIGGER r1t1} {-- TRIGGER r1t2}}
+  } {{UPDATE t1 SET a=a+1;} {-- TRIGGER r1t1} {-- UPDATE t2 SET a=new.a WHERE rowid=new.rowid} {-- TRIGGER r1t2} {-- SELECT 'hello'} {-- TRIGGER r1t1} {-- UPDATE t2 SET a=new.a WHERE rowid=new.rowid} {-- TRIGGER r1t2} {-- SELECT 'hello'} {-- TRIGGER r1t1} {-- UPDATE t2 SET a=new.a WHERE rowid=new.rowid} {-- TRIGGER r1t2} {-- SELECT 'hello'}}
 }
 
 # With 3.6.21, we add the ability to expand host parameters in the trace

From b5090e489784941c91e559cc23e5cbe497dab52c Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 27 Dec 2017 21:13:21 +0000
Subject: [PATCH 195/488] Improve the shell tool ".ar --list --verbose"
 command.

FossilOrigin-Name: b64681a644c419bb98d00980a6cb56ef5a0aff5ef5321955631f0b4c88aac283
---
 ext/misc/zipfile.c | 17 +++++++++++-----
 manifest           | 14 +++++++-------
 manifest.uuid      |  2 +-
 src/shell.c.in     | 48 +++++++++++++++++++++++++++++++++++++++++++---
 4 files changed, 65 insertions(+), 16 deletions(-)

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index cf9666a7ae..188d4fc2ea 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -50,6 +50,9 @@ typedef unsigned long u32;
 
 #define ZIPFILE_BUFFER_SIZE (64*1024)
 
+
+#define ZIPFILE_EXTRA_TIMESTAMP 0x5455
+
 /*
 ** Set the error message contained in context ctx to the results of
 ** vprintf(zFmt, ...).
@@ -341,7 +344,14 @@ static int zipfileReadCDS(ZipfileCsr *pCsr){
         pCsr->iNextOff += pCsr->cds.nComment;
       }
 
-      /* Scan the "extra" fields */
+      /* Scan the cds.nExtra bytes of "extra" fields for any that can
+      ** be interpreted. The general format of an extra field is:
+      **
+      **   Header ID    2 bytes
+      **   Data Size    2 bytes
+      **   Data         N bytes
+      **
+      */
       if( rc==SQLITE_OK ){
         u8 *p = &aRead[pCsr->cds.nFile];
         u8 *pEnd = &p[pCsr->cds.nExtra];
@@ -351,7 +361,7 @@ static int zipfileReadCDS(ZipfileCsr *pCsr){
           u16 nByte = zipfileRead16(p);
 
           switch( id ){
-            case 0x5455: {        /* Extended timestamp */
+            case ZIPFILE_EXTRA_TIMESTAMP: {
               u8 b = p[0];
               if( b & 0x01 ){     /* 0x01 -> modtime is present */
                 pCsr->mTime = zipfileGetU32(&p[1]);
@@ -359,9 +369,6 @@ static int zipfileReadCDS(ZipfileCsr *pCsr){
               }
               break;
             }
-
-            case 0x7875:          /* Info-ZIP Unix (new) */
-              break;
           }
 
           p += nByte;
diff --git a/manifest b/manifest
index 0cc3ca2bdd..d6c9a543e5 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Have\sthe\sshell\stool\s".ar\s--list"\sand\s".ar\s--extract"\scommands\ssupport\szip\nfiles.\sCurrently\sthe\s"-zip"\sswitch\sis\srequired.
-D 2017-12-27T18:54:11.166
+C Improve\sthe\sshell\stool\s".ar\s--list\s--verbose"\scommand.
+D 2017-12-27T21:13:21.423
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
@@ -302,7 +302,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c 96148b78b56664fe82f774e50dcdf6c83d693a1449b88011eba00cd6c697fedf
+F ext/misc/zipfile.c ced1aa768904cf8182c3da93d42df4e3cf30fe7a2e66c9216f6bc64a8df4fd5b
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -483,7 +483,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in d1be3030ee7afbbfb67972e4614b6d08dcae2e76460114d6b200fd28d0f008fb
+F src/shell.c.in 9177b6cc706b1dd1ed81b05344641597d7ed8bba97a8fc31192309189846fab7
 F src/sqlite.h.in 2126192945019d4cdce335cb236b440a05ec75c93e4cd94c9c6d6e7fcc654cc4
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1692,7 +1692,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 8e366b99b13d765d8bf000a7ec5919e582702e51dc07c27a746b6002898a2302
-R 0d0cb33644ad535bd9f83812fd2ec78d
+P a532a0f6fd59e81086d46f09151ba7fb26725198231d902c71d0f95cb01dbe91
+R 7075030d907b0ffe212621eb709bb967
 U dan
-Z 54715bdb695d1dc399a9b1cd512f6d24
+Z 1c430e112380037cc2bfad44bf8be92e
diff --git a/manifest.uuid b/manifest.uuid
index 6167e93eb4..aaba04f0ec 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a532a0f6fd59e81086d46f09151ba7fb26725198231d902c71d0f95cb01dbe91
\ No newline at end of file
+b64681a644c419bb98d00980a6cb56ef5a0aff5ef5321955631f0b4c88aac283
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 15a018bb9c..f7edb53f06 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -4577,12 +4577,43 @@ static void arWhereClause(
   *pzWhere = zWhere;
 }
 
+/*
+** Argument zMode must point to a buffer at least 11 bytes in size. This
+** function populates this buffer with the string interpretation of
+** the unix file mode passed as the second argument (e.g. "drwxr-xr-x").
+*/
+static void shellModeToString(char *zMode, int mode){
+  int i;
+
+  /* Magic numbers copied from [man 2 stat] */
+  if( mode & 0040000 ){
+    zMode[0] = 'd';
+  }else if( (mode & 0120000)==0120000 ){
+    zMode[0] = 'l';
+  }else{
+    zMode[0] = '-';
+  }
+
+  for(i=0; i<3; i++){
+    int m = (mode >> ((2-i)*3));
+    char *a = &zMode[1 + i*3];
+    a[0] = (m & 0x4) ? 'r' : '-';
+    a[1] = (m & 0x2) ? 'w' : '-';
+    a[2] = (m & 0x1) ? 'x' : '-';
+  }
+  zMode[10] = '\0';
+}
+
 /*
 ** Implementation of .ar "lisT" command. 
 */
 static int arListCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
-  const char *zSql = "SELECT name FROM %s WHERE %s"; 
+  const char *zSql = "SELECT %s FROM %s WHERE %s"; 
   const char *zTbl = (pAr->bZip ? "zipfile(?)" : "sqlar");
+  const char *azCols[] = {
+    "name",
+    "mode, sz, datetime(mtime, 'unixepoch'), name"
+  };
 
   char *zWhere = 0;
   sqlite3_stmt *pSql = 0;
@@ -4591,12 +4622,23 @@ static int arListCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
   rc = arCheckEntries(db, pAr);
   arWhereClause(&rc, pAr, &zWhere);
 
-  shellPreparePrintf(db, &rc, &pSql, zSql, zTbl, zWhere);
+  shellPreparePrintf(db, &rc, &pSql, zSql, azCols[pAr->bVerbose], zTbl, zWhere);
   if( rc==SQLITE_OK && pAr->bZip ){
     sqlite3_bind_text(pSql, 1, pAr->zFile, -1, SQLITE_TRANSIENT);
   }
   while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
-    raw_printf(p->out, "%s\n", sqlite3_column_text(pSql, 0));
+    if( pAr->bVerbose ){
+      char zMode[11];
+      shellModeToString(zMode, sqlite3_column_int(pSql, 0));
+
+      raw_printf(p->out, "%s % 10d  %s  %s\n", zMode,
+          sqlite3_column_int(pSql, 1), 
+          sqlite3_column_text(pSql, 2),
+          sqlite3_column_text(pSql, 3)
+      );
+    }else{
+      raw_printf(p->out, "%s\n", sqlite3_column_text(pSql, 0));
+    }
   }
 
   shellFinalize(&rc, pSql);

From e307e11d0a38163d332554f54632d7b8eeb78d5d Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 27 Dec 2017 21:30:34 +0000
Subject: [PATCH 196/488] Omit all sqlite3_trace() output from the triggers
 associated with foreign key constraints.

FossilOrigin-Name: fda08e3d10cc850664a356efdafcfc68187053849e1b00991b0b35d892a6776b
---
 manifest        | 14 +++++++-------
 manifest.uuid   |  2 +-
 src/trigger.c   | 16 ++++++++++------
 test/fkey1.test | 14 ++++++++++++++
 4 files changed, 32 insertions(+), 14 deletions(-)

diff --git a/manifest b/manifest
index e45bddc5bc..ea3a94013a 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Show\sthe\stext\sof\sindividual\sstatements\swithin\sa\strigger,\sas\sthey\nexecute,\sas\scomments\sin\sthe\soutput\sfrom\ssqlite3_trace()\sand\ssqlite3_trace_v2().
-D 2017-12-27T20:38:35.013
+C Omit\sall\ssqlite3_trace()\soutput\sfrom\sthe\striggers\sassociated\swith\nforeign\skey\sconstraints.
+D 2017-12-27T21:30:34.358
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -542,7 +542,7 @@ F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
 F src/tokenize.c 1003d6d90c6783206c711f0a9397656fa5b055209f4d092caa43bb3bf5215db5
 F src/treeview.c eae35972ff44f67064de2eaf35f04afe94e7aea3271a8b3bcebb3f954880fec3
-F src/trigger.c 5fa038ae62adae53b62776bacc34dc56191385eedac79d49264389620ddc2526
+F src/trigger.c a34539c69433276d37b0da9a89c117726ff2d292c0902895af1f393a983cd3a1
 F src/update.c 961bd1265d4d1e5cd65c9a54fa5122fb7aefcb003fcf2de0c092fceb7e58972c
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c 7315e97a8dc2c8e19ca64196c652cf0a65d13fd0a211b2cec082062372dc6261
@@ -788,7 +788,7 @@ F test/extraquick.test cb254400bd42bfb777ff675356aabf3287978f79
 F test/fallocate.test 87b5e43c872b7e69cd80b7b8813eb102b571a75d45dda24e38b65537bcc85733
 F test/filectrl.test 6e871c2d35dead1d9a88e176e8d2ca094fec6bb3
 F test/filefmt.test f393e80c4b8d493b7a7f8f3809a8425bbf4292af1f5140f01cb1427798a2bbd4
-F test/fkey1.test ba64806ff9a04eecab2679caad377ae99a5e94e4
+F test/fkey1.test 9d7e3a0d409e7f64ab077af3b4fc5e5ce1a4e8d8f1272b65b9d93480aeb1fa2b
 F test/fkey2.test 155809016fad6b2a1491facf2ac53a551bc57c2c
 F test/fkey3.test 76d475c80b84ee7a5d062e56ccb6ea68882e2b49
 F test/fkey4.test 86446017011273aad8f9a99c1a65019e7bd9ca9d
@@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a8e1545cb7aacb6a26a8c92a3ad4a3d584d150c3a00d2828c8adbb1ee19fcb6d
-R 0ae5195579783355d0e4696ba0c5bdad
+P fe3d2b97d8945e6a9636472f77bcdc92cc5dda8b3e6e3a4fcbdd0a212ea5b9aa
+R 22de62f1b2836f49ea0af57f6725c723
 U drh
-Z 4ebaaf3c94dd44dc4a2d0b266bee40f3
+Z 3aa637606646f6f00ff3a9a21b2a3987
diff --git a/manifest.uuid b/manifest.uuid
index 49211259e2..2b2a68d7fa 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-fe3d2b97d8945e6a9636472f77bcdc92cc5dda8b3e6e3a4fcbdd0a212ea5b9aa
\ No newline at end of file
+fda08e3d10cc850664a356efdafcfc68187053849e1b00991b0b35d892a6776b
\ No newline at end of file
diff --git a/src/trigger.c b/src/trigger.c
index cf248a57bc..9f7bff5052 100644
--- a/src/trigger.c
+++ b/src/trigger.c
@@ -733,9 +733,11 @@ static int codeTriggerProgram(
     assert( pParse->okConstFactor==0 );
 
 #ifndef SQLITE_OMIT_TRACE
-    sqlite3VdbeAddOp4(v, OP_Trace, 0x7fffffff, 1, 0,
-                      sqlite3MPrintf(db, "-- %s", pStep->zSpan),
-                      P4_DYNAMIC);
+    if( pStep->zSpan ){
+      sqlite3VdbeAddOp4(v, OP_Trace, 0x7fffffff, 1, 0,
+                        sqlite3MPrintf(db, "-- %s", pStep->zSpan),
+                        P4_DYNAMIC);
+    }
 #endif
 
     switch( pStep->op ){
@@ -878,9 +880,11 @@ static TriggerPrg *codeRowTrigger(
       pTab->zName
     ));
 #ifndef SQLITE_OMIT_TRACE
-    sqlite3VdbeChangeP4(v, -1, 
-      sqlite3MPrintf(db, "-- TRIGGER %s", pTrigger->zName), P4_DYNAMIC
-    );
+    if( pTrigger->zName ){
+      sqlite3VdbeChangeP4(v, -1, 
+        sqlite3MPrintf(db, "-- TRIGGER %s", pTrigger->zName), P4_DYNAMIC
+      );
+    }
 #endif
 
     /* If one was specified, code the WHEN clause. If it evaluates to false
diff --git a/test/fkey1.test b/test/fkey1.test
index d9b038a022..2530327fce 100644
--- a/test/fkey1.test
+++ b/test/fkey1.test
@@ -171,6 +171,20 @@ do_catchsql_test fkey1-5.2 {
   INSERT OR REPLACE INTO t11 VALUES (2, 3);
 } {1 {FOREIGN KEY constraint failed}}
 
+# Make sure sqlite3_trace() output works with triggers used to implement
+# FK constraints
+#
+proc sqltrace {txt} {
+  global traceoutput
+  lappend traceoutput $txt
+}
+do_test fkey1-5.2.1 {
+  unset -nocomplain traceoutput
+  db trace sqltrace
+  catch {db eval {INSERT OR REPLACE INTO t11 VALUES(2,3);}}
+  set traceoutput
+} {{INSERT OR REPLACE INTO t11 VALUES(2,3);} {INSERT OR REPLACE INTO t11 VALUES(2,3);} {INSERT OR REPLACE INTO t11 VALUES(2,3);}}
+
 # A similar test to the above.
 do_execsql_test fkey1-5.3 {
   CREATE TABLE Foo (

From ee052a1c941920dab3fab38785fbf03237677b7a Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 29 Dec 2017 12:50:43 +0000
Subject: [PATCH 197/488] Add test cases for the undocumented behavior of
 duplicate columns on an INSERT or UPDATE.

FossilOrigin-Name: f4349c0c26611de8a7d5beb99431a575cf531cdeb0ca2413efabcf0a61e6f424
---
 manifest        | 13 ++++++-------
 manifest.uuid   |  2 +-
 test/misc1.test | 20 ++++++++++++++++++++
 3 files changed, 27 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index d4345bd99c..3e7d60e06a 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C The\soutput\sof\ssqlite3_trace()\snow\sshows\seach\scommand\sof\sa\strigger\sas\sit\sis\nevaluated.\s\sThis\sfeature\sinvolved\smajor\schanges\sto\sthe\sparser,\ssuch\sas\nremoving\sthe\sExprSpan\sobject\sand\sreplacing\sit\swith\sa\snew\smechanism\sfor\ncapturing\sthe\soriginal\sSQL\stext\sof\sphrases\sin\sthe\sinput\sSQL.
-D 2017-12-27T22:09:53.804
+C Add\stest\scases\sfor\sthe\sundocumented\sbehavior\sof\sduplicate\scolumns\son\san\nINSERT\sor\sUPDATE.
+D 2017-12-29T12:50:43.938
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -1061,7 +1061,7 @@ F test/minmax.test 6751e87b409fe11b02e70a306d846fa544e25a41
 F test/minmax2.test dae92964ac87c1d2ef978c582e81a95e11c00f1cbef68980bfb2abaf10315063
 F test/minmax3.test cc1e8b010136db0d01a6f2a29ba5a9f321034354
 F test/minmax4.test 936941484ebdceb8adec7c86b6cd9b6e5e897c1f
-F test/misc1.test 76737c259537586355f45e2a1e121b6e91b5476c4604ad5c53d1abfcb3acf786
+F test/misc1.test 704ea2cc7e7b9deb622b37953f0e77d0879826e8c3bfc1d7a691528035405061
 F test/misc2.test 00d7de54eda90e237fc9a38b9e5ccc769ebf6d4d
 F test/misc3.test cf3dda47d5dda3e53fc5804a100d3c82be736c9d
 F test/misc4.test 0d8be3466adf123a7791a66ba2bc8e8d229e87f3
@@ -1687,8 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 1b22b42e59793af19c69a2e5f6822883cc2687d4a0d9b9280bbff885276c6baa fda08e3d10cc850664a356efdafcfc68187053849e1b00991b0b35d892a6776b
-R 22de62f1b2836f49ea0af57f6725c723
-T +closed fda08e3d10cc850664a356efdafcfc68187053849e1b00991b0b35d892a6776b
+P 0fdf97efe5df745510c6b4b377a8ee5683a3a237630bfbd0f56e57c7c6e5b246
+R 9fe2cb7c2c9b946d64749c1df6cc4954
 U drh
-Z e235709d3662d8065d80f758148733ed
+Z 157cf27648c17a2b2af97b1730aa1c08
diff --git a/manifest.uuid b/manifest.uuid
index cf17777f87..3c028df06d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0fdf97efe5df745510c6b4b377a8ee5683a3a237630bfbd0f56e57c7c6e5b246
\ No newline at end of file
+f4349c0c26611de8a7d5beb99431a575cf531cdeb0ca2413efabcf0a61e6f424
\ No newline at end of file
diff --git a/test/misc1.test b/test/misc1.test
index 2acfa5c2dd..05b1b1980f 100644
--- a/test/misc1.test
+++ b/test/misc1.test
@@ -722,4 +722,24 @@ do_execsql_test misc1-26.0 {
   SELECT randomblob(min(max(coalesce(EXISTS (SELECT 1 FROM ( SELECT (SELECT 2147483647) NOT IN (SELECT 2147483649 UNION ALL SELECT DISTINCT -1) IN (SELECT 2147483649), 'fault', (SELECT ALL -1 INTERSECT SELECT 'experiments') IN (SELECT ALL 56.1 ORDER BY 'experiments' DESC) FROM (SELECT DISTINCT 2147483648, 'hardware' UNION ALL SELECT -2147483648, 'experiments' ORDER BY 2147483648 LIMIT 1 OFFSET 123456789.1234567899) GROUP BY (SELECT ALL 0 INTERSECT SELECT 'in') IN (SELECT DISTINCT 'experiments' ORDER BY zeroblob(1000) LIMIT 56.1 OFFSET -456) HAVING EXISTS (SELECT 'fault' EXCEPT    SELECT DISTINCT 56.1) UNION SELECT 'The', 'The', 2147483649 UNION ALL SELECT DISTINCT 'hardware', 'first', 'experiments' ORDER BY 'hardware' LIMIT 123456789.1234567899 OFFSET -2147483647)) NOT IN (SELECT (SELECT DISTINCT (SELECT 'The') FROM abc ORDER BY EXISTS (SELECT -1 INTERSECT SELECT ALL NULL) ASC) IN (SELECT DISTINCT EXISTS (SELECT ALL 123456789.1234567899 ORDER BY 1 ASC, NULL DESC) FROM sqlite_master INTERSECT SELECT 456)), (SELECT ALL 'injection' UNION ALL SELECT ALL (SELECT DISTINCT 'first' UNION     SELECT DISTINCT 'The') FROM (SELECT 456, 'in', 2147483649))),1), 500)), 'first', EXISTS (SELECT DISTINCT 456 FROM abc ORDER BY 'experiments' DESC) FROM abc;
 } {}
 
+# 2017-12-29
+#
+# The following behaviors (duplicate column names on an INSERT or UPDATE)
+# are undocumented.  These tests are added to ensure that historical behavior
+# does not change accidentally.
+#
+# For duplication columns on an INSERT, the first value is used.
+# For duplication columns on an UPDATE, the last value is used.
+#
+do_execsql_test misc1-27.0 {
+  CREATE TABLE dup1(a,b,c);
+  INSERT INTO dup1(a,b,c,a,b,c) VALUES(1,2,3,4,5,6);
+  SELECT a,b,c FROM dup1;
+} {1 2 3}
+do_execsql_test misc1-27.1 {
+  UPDATE dup1 SET a=7, b=8, c=9, a=10, b=11, c=12;
+  SELECT a,b,c FROM dup1;
+} {10 11 12}
+
+
 finish_test

From fe6d20e9f4d17ad82fd4c4cc69181ad5f52d9d93 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 29 Dec 2017 14:33:54 +0000
Subject: [PATCH 198/488] Enhance location(X) so that it works with indexes and
 WITHOUT ROWID tables. The function might return an offset to the main table
 or to an index, depending on whether the column X would be loaded from the
 main table or from the index.

FossilOrigin-Name: dd94d6a880dfec4bddd247239b815b84964f804d24841e25f33f1d46a4b5274d
---
 manifest        | 20 ++++++++++----------
 manifest.uuid   |  2 +-
 src/btree.c     |  3 +--
 src/expr.c      |  2 +-
 src/vdbe.c      | 19 ++++++++++++-------
 src/where.c     |  2 +-
 test/func6.test |  2 +-
 7 files changed, 27 insertions(+), 23 deletions(-)

diff --git a/manifest b/manifest
index 89d90d3cf5..12c0ae8735 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\srecent\senhancements\sfrom\strunk.
-D 2017-12-29T13:35:09.867
+C Enhance\slocation(X)\sso\sthat\sit\sworks\swith\sindexes\sand\sWITHOUT\sROWID\stables.\nThe\sfunction\smight\sreturn\san\soffset\sto\sthe\smain\stable\sor\sto\san\sindex,\ndepending\son\swhether\sthe\scolumn\sX\swould\sbe\sloaded\sfrom\sthe\smain\stable\sor\nfrom\sthe\sindex.
+D 2017-12-29T14:33:54.266
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
@@ -420,7 +420,7 @@ F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
-F src/btree.c 6fba32ea06e9be55849dd486e88df6c467bf69cf306d38ca3c3a8df34e6499e9
+F src/btree.c 516fea6cad0d904ba1f7d599cb3d8ec9f14c2c6d1b71cf883dcef81d77bc6a5a
 F src/btree.h e9d22d0475b37422cc2db53f4336cc814645dcca8634bc4aae25ed52d043ef53
 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc
 F src/build.c a03eb5a1cfff74784c24a4478ba5455711571936f1ac9d43f94fa7df57509977
@@ -431,7 +431,7 @@ F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957
 F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
 F src/delete.c 74667ad914ac143731a444a1bacf29ceb18f6eded8a0dd17aafae80baa07f8bb
-F src/expr.c db1f29a1ee0c8a3abec55134ae7e149fa5684d77b569c6a62121b9e13becc43a
+F src/expr.c 119aa8d492bfc05ecf73ff4160b48c65d696972b2c954149271a1363e795874b
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331
 F src/func.c a23ea9b8869ca47e20a273985013b36a2294788fde5abe1a610be4bb90c8e6e8
@@ -547,7 +547,7 @@ F src/update.c 961bd1265d4d1e5cd65c9a54fa5122fb7aefcb003fcf2de0c092fceb7e58972c
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c 7315e97a8dc2c8e19ca64196c652cf0a65d13fd0a211b2cec082062372dc6261
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
-F src/vdbe.c 742812438ab9c897db18025c6040af23feb4679fc63c53bdf25a9b190c7f91ec
+F src/vdbe.c 42e79ef2da625fd8c4364bf952b0515760958775beb38cc9f9e84bcec4115964
 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97
 F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9
 F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1
@@ -561,7 +561,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 5a3f464edd64596f601683ed321d12e6fd93c5fb9afdfb3653d6ffd0fee9c48f
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f
-F src/where.c 5876c9100b622f7b9e5ee7f579b8b6a71ae5ba627724cea4546d9114c32b3cb5
+F src/where.c 9c1fca74de02cf5d8ac6437fc385b683e194c403a2ae675eb4f15a5c19086726
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
 F src/wherecode.c af1e79154aaa88cd802d6f2e5b945f67eaca7c958d1525fbf8ee19d5bd7b9020
 F src/whereexpr.c 427ea8e96ec24f2a7814c67b8024ad664a9c7656264c4566c34743cb23186e46
@@ -917,7 +917,7 @@ F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f
 F test/func3.test d202a7606d23f90988a664e88e268aed1087c11c
 F test/func4.test 6beacdfcb0e18c358e6c2dcacf1b65d1fa80955f
 F test/func5.test cdd224400bc3e48d891827cc913a57051a426fa4
-F test/func6.test 5d4b0526f5013db07e2f6de43df42d990500f636fa32258cfac0c87113a1c1ce
+F test/func6.test afd2f720b333f0fbbb220887b3f1b3637ef9bfde47015f664e9bb4ba14a01535
 F test/fuzz-oss1.test e58330d01cbbd8215ee636b17a03fe220b37dbfa
 F test/fuzz.test 96083052bf5765e4518c1ba686ce2bab785670d1
 F test/fuzz2.test 76dc35b32b6d6f965259508508abce75a6c4d7e1
@@ -1688,7 +1688,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 51be9558164301c5dd4df23ab8b3e67de0b522f8d36f79f3d84d45d3dc2a83a4 f4349c0c26611de8a7d5beb99431a575cf531cdeb0ca2413efabcf0a61e6f424
-R 21534cc483241840a2ab1c4e82c69566
+P 6251e438f2a76170fd1e95aa512a46086ed88ab93b9b97a1dba97c4558689305
+R f543ccd2667c3e9285e509d3e9a274bf
 U drh
-Z 27f7d54e41b653412b0ccbce7bf65d00
+Z e0ddcaae57472f72e53764b52bf6363b
diff --git a/manifest.uuid b/manifest.uuid
index c839229c61..b83335d6cc 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6251e438f2a76170fd1e95aa512a46086ed88ab93b9b97a1dba97c4558689305
\ No newline at end of file
+dd94d6a880dfec4bddd247239b815b84964f804d24841e25f33f1d46a4b5274d
\ No newline at end of file
diff --git a/src/btree.c b/src/btree.c
index d17cbde846..a8d91cc0c9 100644
--- a/src/btree.c
+++ b/src/btree.c
@@ -4439,9 +4439,8 @@ i64 sqlite3BtreeIntegerKey(BtCursor *pCur){
 i64 sqlite3BtreeLocation(BtCursor *pCur){
   assert( cursorHoldsMutex(pCur) );
   assert( pCur->eState==CURSOR_VALID );
-  assert( pCur->curIntKey );
   getCellInfo(pCur);
-  return (i64)pCur->pBt->pageSize*(i64)pCur->pPage->pgno +
+  return (i64)pCur->pBt->pageSize*((i64)pCur->pPage->pgno - 1) +
          (i64)(pCur->info.pPayload - pCur->pPage->aData);
 }
 
diff --git a/src/expr.c b/src/expr.c
index dbfc1b7ed3..dd7d548edc 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -3873,7 +3873,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){
       if( pDef->funcFlags & SQLITE_FUNC_LOCATION ){
         Expr *pArg = pFarg->a[0].pExpr;
         if( pArg->op==TK_COLUMN ){
-          sqlite3VdbeAddOp2(v, OP_Location, pArg->iTable, target);
+          sqlite3VdbeAddOp3(v, OP_Location, pArg->iTable, pArg->iColumn,target);
         }else{
           sqlite3VdbeAddOp2(v, OP_Null, 0, target);
         }
diff --git a/src/vdbe.c b/src/vdbe.c
index 975a098198..68762ae5e1 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -2349,22 +2349,27 @@ case OP_IfNullRow: {         /* jump */
   break;
 }
 
-/* Opcode: Location P1 P2 * * *
-** Synopsis: r[P2] = location(P1)
+/* Opcode: Location P1 P2 P3 * *
+** Synopsis: r[P3] = location(P1)
 **
-** Store in register r[P2] the location in the database file that is the
+** Store in register r[P3] the location in the database file that is the
 ** start of the payload for the record at which that cursor P1 is currently
 ** pointing.
+**
+** P2 is the column number for the argument to the location() function.
+** This opcode does not use P2 itself, but the P2 value is used by the
+** code generator.  The P1, P2, and P3 operands to this opcode are the
+** as as for OP_Column.
 */
-case OP_Location: {          /* out2 */
+case OP_Location: {          /* out3 */
   VdbeCursor *pC;    /* The VDBE cursor */
   assert( pOp->p1>=0 && pOp->p1<p->nCursor );
   pC = p->apCsr[pOp->p1];
-  pOut = out2Prerelease(p, pOp);
+  pOut = &p->aMem[pOp->p3];
   if( pC==0 || pC->eCurType!=CURTYPE_BTREE ){
-    pOut->flags = MEM_Null;
+    sqlite3VdbeMemSetNull(pOut);
   }else{
-    pOut->u.i = sqlite3BtreeLocation(pC->uc.pCursor);
+    sqlite3VdbeMemSetInt64(pOut, sqlite3BtreeLocation(pC->uc.pCursor));
   }
   break;
 }
diff --git a/src/where.c b/src/where.c
index d3aec354ba..146c8124c8 100644
--- a/src/where.c
+++ b/src/where.c
@@ -5146,7 +5146,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
       pOp = sqlite3VdbeGetOp(v, k);
       for(; k<last; k++, pOp++){
         if( pOp->p1!=pLevel->iTabCur ) continue;
-        if( pOp->opcode==OP_Column ){
+        if( pOp->opcode==OP_Column || pOp->opcode==OP_Location ){
           int x = pOp->p2;
           assert( pIdx->pTable==pTab );
           if( !HasRowid(pTab) ){
diff --git a/test/func6.test b/test/func6.test
index 5350aeaa7b..3630aa33db 100644
--- a/test/func6.test
+++ b/test/func6.test
@@ -28,7 +28,7 @@ do_execsql_test func6-120 {
 do_execsql_test func6-130 {
   CREATE INDEX t1a ON t1(a);
   SELECT a, typeof(location(a)) FROM t1 ORDER BY a LIMIT 2;
-} {abc001 null abc002 null}
+} {abc001 integer abc002 integer}
 do_execsql_test func6-140 {
   SELECT a, typeof(location(a)) FROM t1 NOT INDEXED ORDER BY a LIMIT 2;
 } {abc001 integer abc002 integer}

From 092457b18ce71b6b0f30ea8a67817310feffaebc Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 29 Dec 2017 15:04:49 +0000
Subject: [PATCH 199/488] Change the function name to
 sqlite_unsupported_offset(X).  Only enable the function if compiled with
 -DSQLITE_ENABLE_OFFSET_SQL_FUNC.  The makefiles add that definition to shell
 builds.

FossilOrigin-Name: 7a7f826e324b1a2c332e2f1d0740fd0babffcaca6275a798572f02ad367b99ab
---
 Makefile.in       |  1 +
 Makefile.msc      |  1 +
 main.mk           |  1 +
 manifest          | 34 +++++++++++++++++-----------------
 manifest.uuid     |  2 +-
 src/btree.c       |  4 +++-
 src/btree.h       |  4 +++-
 src/expr.c        |  9 ++++++---
 src/func.c        |  5 ++++-
 src/sqliteInt.h   |  2 +-
 src/test_config.c |  6 ++++++
 src/vdbe.c        | 17 +++++++++++------
 src/where.c       |  6 +++++-
 test/func6.test   | 18 +++++++++++++-----
 14 files changed, 73 insertions(+), 37 deletions(-)

diff --git a/Makefile.in b/Makefile.in
index 009ab2e061..3c9e7b3a96 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -582,6 +582,7 @@ SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
 SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB
 SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB
 SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
+SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC
 FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
 FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
diff --git a/Makefile.msc b/Makefile.msc
index 652bd81fd0..94fa32dc3a 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -1513,6 +1513,7 @@ FUZZDATA = \
 !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_DBSTAT_VTAB
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_EANBLE_OFFSET_SQL_FUNC
 !ENDIF
 
 # <<mark>>
diff --git a/main.mk b/main.mk
index 8692450441..b46ffa1a52 100644
--- a/main.mk
+++ b/main.mk
@@ -509,6 +509,7 @@ SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
 SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB
 SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB
 SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
+SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC
 FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5
 FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
diff --git a/manifest b/manifest
index 12c0ae8735..1328eaf9ce 100644
--- a/manifest
+++ b/manifest
@@ -1,8 +1,8 @@
-C Enhance\slocation(X)\sso\sthat\sit\sworks\swith\sindexes\sand\sWITHOUT\sROWID\stables.\nThe\sfunction\smight\sreturn\san\soffset\sto\sthe\smain\stable\sor\sto\san\sindex,\ndepending\son\swhether\sthe\scolumn\sX\swould\sbe\sloaded\sfrom\sthe\smain\stable\sor\nfrom\sthe\sindex.
-D 2017-12-29T14:33:54.266
-F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
+C Change\sthe\sfunction\sname\sto\ssqlite_unsupported_offset(X).\s\sOnly\senable\sthe\nfunction\sif\scompiled\swith\s-DSQLITE_ENABLE_OFFSET_SQL_FUNC.\s\sThe\smakefiles\sadd\nthat\sdefinition\sto\sshell\sbuilds.
+D 2017-12-29T15:04:49.734
+F Makefile.in 804c347948b0ad5a962aea3e59413e897227e0173d0e76910f11f473e6c7ae9b
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
+F Makefile.msc f11644a365122b16628da7839a8ed2e3a781b578a94cb1ebd25ebad260bdc9c2
 F README.md eeae1e552f93ef72ef7c5b8f6647b368a001c28820ad1df179d3dae602bef681
 F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -401,7 +401,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 50bac9920024b5485f06398b3980f09e97ab28cd4b5b6dcd829d2a5e3ce22e7a
+F main.mk b5f569262e4f8111bb8cf6110784b8210fe14ea45f6c7ea50fb4230ebe7a4880
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -420,8 +420,8 @@ F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
-F src/btree.c 516fea6cad0d904ba1f7d599cb3d8ec9f14c2c6d1b71cf883dcef81d77bc6a5a
-F src/btree.h e9d22d0475b37422cc2db53f4336cc814645dcca8634bc4aae25ed52d043ef53
+F src/btree.c 0a1f63b50ab1ac5d4b1637c30cb1ae123fbc162ec8cb6336ddb9491a0bc1e363
+F src/btree.h 0866c0a08255142ea0e754aabd211c843cab32045c978a592a43152405ed0c84
 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc
 F src/build.c a03eb5a1cfff74784c24a4478ba5455711571936f1ac9d43f94fa7df57509977
 F src/callback.c fe677cb5f5abb02f7a772a62a98c2f516426081df68856e8f2d5f950929b966a
@@ -431,10 +431,10 @@ F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957
 F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
 F src/delete.c 74667ad914ac143731a444a1bacf29ceb18f6eded8a0dd17aafae80baa07f8bb
-F src/expr.c 119aa8d492bfc05ecf73ff4160b48c65d696972b2c954149271a1363e795874b
+F src/expr.c ad6e7a9c34a4bab9d10cc857d647ae7ce370a633b5c0bfa71f1c29b81ae364b8
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331
-F src/func.c a23ea9b8869ca47e20a273985013b36a2294788fde5abe1a610be4bb90c8e6e8
+F src/func.c 697a3ee3b8c8481db497226e9e71935727d6a9fb3d3cc049e82a1c717a0a8342
 F src/global.c ac3094f1dc59fbeb919aef7cc0cc827a8459d1fb1adb7972ef75bd9e0c10b75b
 F src/hash.c a12580e143f10301ed5166ea4964ae2853d3905a511d4e0c44497245c7ce1f7a
 F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4
@@ -483,7 +483,7 @@ F src/shell.c.in f3ec8f90dd698ea98781a90642c91eacbc24f4e55bb551c7b2762000d3ef55d
 F src/sqlite.h.in b4dc75265ed04b98e2184011a7dd0054ce2137ff84867a6be8b4f3bdfbc03d30
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
-F src/sqliteInt.h 623712d1ae728cc0ca78c9d8985b9665d89afff3082c1b664cbba08b0b01476d
+F src/sqliteInt.h fd8702c65994d7de3e2d8f7d85d958731da1ed29476571fdfa2290fd8ec0bf80
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -503,7 +503,7 @@ F src/test_backup.c bf5da90c9926df0a4b941f2d92825a01bbe090a0
 F src/test_bestindex.c 78809f11026f18a93fcfd798d9479cba37e1201c830260bf1edc674b2fa9b857
 F src/test_blob.c ae4a0620b478548afb67963095a7417cd06a4ec0a56adb453542203bfdcb31ce
 F src/test_btree.c 8b2dc8b8848cf3a4db93f11578f075e82252a274
-F src/test_config.c 3904a8682aac58b77d20ca236face2b11e50781be4116004ba1ba79f69896ec9
+F src/test_config.c cc8a1d44648d9392a14f4ecfc841d027daaf61f952b9f70792edf11373aaa3dd
 F src/test_delete.c e2fe07646dff6300b48d49b2fee2fe192ed389e834dd635e3b3bac0ce0bf9f8f
 F src/test_demovfs.c a0c3bdd45ed044115c2c9f7779e56eafff18741e
 F src/test_devsym.c 1960abbb234b97e9b920f07e99503fc04b443f62bbc3c6ff2c2cea2133e3b8a2
@@ -547,7 +547,7 @@ F src/update.c 961bd1265d4d1e5cd65c9a54fa5122fb7aefcb003fcf2de0c092fceb7e58972c
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c 7315e97a8dc2c8e19ca64196c652cf0a65d13fd0a211b2cec082062372dc6261
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
-F src/vdbe.c 42e79ef2da625fd8c4364bf952b0515760958775beb38cc9f9e84bcec4115964
+F src/vdbe.c cb3b5035f337c9fed4a0170ca20325dc4500b832c57d00561310cbb78149a73c
 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97
 F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9
 F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1
@@ -561,7 +561,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 5a3f464edd64596f601683ed321d12e6fd93c5fb9afdfb3653d6ffd0fee9c48f
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f
-F src/where.c 9c1fca74de02cf5d8ac6437fc385b683e194c403a2ae675eb4f15a5c19086726
+F src/where.c 36b92103f726609cc3dbe07c619426bd6886bede455de56ccff54c8e567f5582
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
 F src/wherecode.c af1e79154aaa88cd802d6f2e5b945f67eaca7c958d1525fbf8ee19d5bd7b9020
 F src/whereexpr.c 427ea8e96ec24f2a7814c67b8024ad664a9c7656264c4566c34743cb23186e46
@@ -917,7 +917,7 @@ F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f
 F test/func3.test d202a7606d23f90988a664e88e268aed1087c11c
 F test/func4.test 6beacdfcb0e18c358e6c2dcacf1b65d1fa80955f
 F test/func5.test cdd224400bc3e48d891827cc913a57051a426fa4
-F test/func6.test afd2f720b333f0fbbb220887b3f1b3637ef9bfde47015f664e9bb4ba14a01535
+F test/func6.test 8a6ecb974a9d0bdeef6a06aff8e305b1667c3b922103037735b39c716360ebc2
 F test/fuzz-oss1.test e58330d01cbbd8215ee636b17a03fe220b37dbfa
 F test/fuzz.test 96083052bf5765e4518c1ba686ce2bab785670d1
 F test/fuzz2.test 76dc35b32b6d6f965259508508abce75a6c4d7e1
@@ -1688,7 +1688,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6251e438f2a76170fd1e95aa512a46086ed88ab93b9b97a1dba97c4558689305
-R f543ccd2667c3e9285e509d3e9a274bf
+P dd94d6a880dfec4bddd247239b815b84964f804d24841e25f33f1d46a4b5274d
+R 78409e335314d524c66761480a13a3e4
 U drh
-Z e0ddcaae57472f72e53764b52bf6363b
+Z 5cd44f60a8e786e38b3cacae6fe91794
diff --git a/manifest.uuid b/manifest.uuid
index b83335d6cc..9bbedd7b51 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-dd94d6a880dfec4bddd247239b815b84964f804d24841e25f33f1d46a4b5274d
\ No newline at end of file
+7a7f826e324b1a2c332e2f1d0740fd0babffcaca6275a798572f02ad367b99ab
\ No newline at end of file
diff --git a/src/btree.c b/src/btree.c
index a8d91cc0c9..8cd5ee6739 100644
--- a/src/btree.c
+++ b/src/btree.c
@@ -4432,17 +4432,19 @@ i64 sqlite3BtreeIntegerKey(BtCursor *pCur){
   return pCur->info.nKey;
 }
 
+#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
 /*
 ** Return the offset into the database file for the start of the
 ** payload to which the cursor is pointing.
 */
-i64 sqlite3BtreeLocation(BtCursor *pCur){
+i64 sqlite3BtreeOffset(BtCursor *pCur){
   assert( cursorHoldsMutex(pCur) );
   assert( pCur->eState==CURSOR_VALID );
   getCellInfo(pCur);
   return (i64)pCur->pBt->pageSize*((i64)pCur->pPage->pgno - 1) +
          (i64)(pCur->info.pPayload - pCur->pPage->aData);
 }
+#endif /* SQLITE_ENABLE_OFFSET_SQL_FUNC */
 
 /*
 ** Return the number of bytes of payload for the entry that pCur is
diff --git a/src/btree.h b/src/btree.h
index 43c0b9a8a4..e8e114bd2a 100644
--- a/src/btree.h
+++ b/src/btree.h
@@ -291,7 +291,9 @@ int sqlite3BtreeNext(BtCursor*, int flags);
 int sqlite3BtreeEof(BtCursor*);
 int sqlite3BtreePrevious(BtCursor*, int flags);
 i64 sqlite3BtreeIntegerKey(BtCursor*);
-i64 sqlite3BtreeLocation(BtCursor*);
+#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
+i64 sqlite3BtreeOffset(BtCursor*);
+#endif
 int sqlite3BtreePayload(BtCursor*, u32 offset, u32 amt, void*);
 const void *sqlite3BtreePayloadFetch(BtCursor*, u32 *pAmt);
 u32 sqlite3BtreePayloadSize(BtCursor*);
diff --git a/src/expr.c b/src/expr.c
index dd7d548edc..32cc4423fa 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -3870,14 +3870,17 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){
         if( !pColl ) pColl = db->pDfltColl; 
         sqlite3VdbeAddOp4(v, OP_CollSeq, 0, 0, 0, (char *)pColl, P4_COLLSEQ);
       }
-      if( pDef->funcFlags & SQLITE_FUNC_LOCATION ){
+#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
+      if( pDef->funcFlags & SQLITE_FUNC_OFFSET ){
         Expr *pArg = pFarg->a[0].pExpr;
         if( pArg->op==TK_COLUMN ){
-          sqlite3VdbeAddOp3(v, OP_Location, pArg->iTable, pArg->iColumn,target);
+          sqlite3VdbeAddOp3(v, OP_Offset, pArg->iTable, pArg->iColumn, target);
         }else{
           sqlite3VdbeAddOp2(v, OP_Null, 0, target);
         }
-      }else{
+      }else
+#endif
+      {
         sqlite3VdbeAddOp4(v, pParse->iSelfTab ? OP_PureFunc0 : OP_Function0,
                           constMask, r1, target, (char*)pDef, P4_FUNCDEF);
         sqlite3VdbeChangeP5(v, (u8)nFarg);
diff --git a/src/func.c b/src/func.c
index f81c4865d8..4758b0191d 100644
--- a/src/func.c
+++ b/src/func.c
@@ -1799,8 +1799,11 @@ void sqlite3RegisterBuiltinFunctions(void){
 #ifdef SQLITE_DEBUG
     FUNCTION2(affinity,          1, 0, 0, noopFunc,  SQLITE_FUNC_AFFINITY),
 #endif
-    FUNCTION2(location,          1, 0, 0, noopFunc,  SQLITE_FUNC_LOCATION|
+#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
+    FUNCTION2(sqlite_unsupported_offset,
+                                 1, 0, 0, noopFunc,  SQLITE_FUNC_OFFSET|
                                                      SQLITE_FUNC_TYPEOF),
+#endif
     FUNCTION(ltrim,              1, 1, 0, trimFunc         ),
     FUNCTION(ltrim,              2, 1, 0, trimFunc         ),
     FUNCTION(rtrim,              1, 2, 0, trimFunc         ),
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 24a36e2c50..8e79133200 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1629,7 +1629,7 @@ struct FuncDestructor {
 #define SQLITE_FUNC_SLOCHNG  0x2000 /* "Slow Change". Value constant during a
                                     ** single query - might change over time */
 #define SQLITE_FUNC_AFFINITY 0x4000 /* Built-in affinity() function */
-#define SQLITE_FUNC_LOCATION 0x8000 /* Built-in location() function */
+#define SQLITE_FUNC_OFFSET   0x8000 /* Built-in sqlite_offset() function */
 
 /*
 ** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are
diff --git a/src/test_config.c b/src/test_config.c
index 24a7287da2..ad63016baf 100644
--- a/src/test_config.c
+++ b/src/test_config.c
@@ -160,6 +160,12 @@ static void set_options(Tcl_Interp *interp){
   Tcl_SetVar2(interp, "sqlite_options", "mem5", "0", TCL_GLOBAL_ONLY);
 #endif
 
+#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
+  Tcl_SetVar2(interp, "sqlite_options", "offset_sql_func","1",TCL_GLOBAL_ONLY);
+#else
+  Tcl_SetVar2(interp, "sqlite_options", "offset_sql_func","0",TCL_GLOBAL_ONLY);
+#endif
+
 #ifdef SQLITE_ENABLE_PREUPDATE_HOOK
   Tcl_SetVar2(interp, "sqlite_options", "preupdate", "1", TCL_GLOBAL_ONLY);
 #else
diff --git a/src/vdbe.c b/src/vdbe.c
index 68762ae5e1..26336d940b 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -2349,19 +2349,23 @@ case OP_IfNullRow: {         /* jump */
   break;
 }
 
-/* Opcode: Location P1 P2 P3 * *
-** Synopsis: r[P3] = location(P1)
+#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
+/* Opcode: Offset P1 P2 P3 * *
+** Synopsis: r[P3] = sqlite_offset(P1)
 **
-** Store in register r[P3] the location in the database file that is the
+** Store in register r[P3] the byte offset into the database file that is the
 ** start of the payload for the record at which that cursor P1 is currently
 ** pointing.
 **
-** P2 is the column number for the argument to the location() function.
+** P2 is the column number for the argument to the sqlite_offset() function.
 ** This opcode does not use P2 itself, but the P2 value is used by the
 ** code generator.  The P1, P2, and P3 operands to this opcode are the
 ** as as for OP_Column.
+**
+** This opcode is only available if SQLite is compiled with the
+** -DSQLITE_ENABLE_OFFSET_SQL_FUNC option.
 */
-case OP_Location: {          /* out3 */
+case OP_Offset: {          /* out3 */
   VdbeCursor *pC;    /* The VDBE cursor */
   assert( pOp->p1>=0 && pOp->p1<p->nCursor );
   pC = p->apCsr[pOp->p1];
@@ -2369,10 +2373,11 @@ case OP_Location: {          /* out3 */
   if( pC==0 || pC->eCurType!=CURTYPE_BTREE ){
     sqlite3VdbeMemSetNull(pOut);
   }else{
-    sqlite3VdbeMemSetInt64(pOut, sqlite3BtreeLocation(pC->uc.pCursor));
+    sqlite3VdbeMemSetInt64(pOut, sqlite3BtreeOffset(pC->uc.pCursor));
   }
   break;
 }
+#endif /* SQLITE_ENABLE_OFFSET_SQL_FUNC */
 
 /* Opcode: Column P1 P2 P3 P4 P5
 ** Synopsis: r[P3]=PX
diff --git a/src/where.c b/src/where.c
index 146c8124c8..ec53527943 100644
--- a/src/where.c
+++ b/src/where.c
@@ -5146,7 +5146,11 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
       pOp = sqlite3VdbeGetOp(v, k);
       for(; k<last; k++, pOp++){
         if( pOp->p1!=pLevel->iTabCur ) continue;
-        if( pOp->opcode==OP_Column || pOp->opcode==OP_Location ){
+        if( pOp->opcode==OP_Column
+#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
+         || pOp->opcode==OP_Offset
+#endif
+        ){
           int x = pOp->p2;
           assert( pIdx->pTable==pTab );
           if( !HasRowid(pTab) ){
diff --git a/test/func6.test b/test/func6.test
index 3630aa33db..ff05ee78c5 100644
--- a/test/func6.test
+++ b/test/func6.test
@@ -9,10 +9,14 @@
 #
 #*************************************************************************
 #
-# Test cases for the location() function.
+# Test cases for the sqlite_unsupported_offset() function.
 #
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
+ifcapable !offset_sql_func {
+  finish_test
+  return
+}
 
 do_execsql_test func6-100 {
   CREATE TABLE t1(a,b,c,d);
@@ -20,17 +24,21 @@ do_execsql_test func6-100 {
    INSERT INTO t1(a,b,c,d) SELECT printf('abc%03x',x), x, 1000-x, NULL FROM c;
 }
 do_execsql_test func6-110 {
-  SELECT a, typeof(location(a)) FROM t1 ORDER BY rowid LIMIT 2;
+  SELECT a, typeof(sqlite_unsupported_offset(a)) FROM t1
+   ORDER BY rowid LIMIT 2;
 } {abc001 integer abc002 integer}
 do_execsql_test func6-120 {
-  SELECT a, typeof(location(+a)) FROM t1 ORDER BY rowid LIMIT 2;
+  SELECT a, typeof(sqlite_unsupported_offset(+a)) FROM t1
+   ORDER BY rowid LIMIT 2;
 } {abc001 null abc002 null}
 do_execsql_test func6-130 {
   CREATE INDEX t1a ON t1(a);
-  SELECT a, typeof(location(a)) FROM t1 ORDER BY a LIMIT 2;
+  SELECT a, typeof(sqlite_unsupported_offset(a)) FROM t1
+   ORDER BY a LIMIT 2;
 } {abc001 integer abc002 integer}
 do_execsql_test func6-140 {
-  SELECT a, typeof(location(a)) FROM t1 NOT INDEXED ORDER BY a LIMIT 2;
+  SELECT a, typeof(sqlite_unsupported_offset(a)) FROM t1 NOT INDEXED
+   ORDER BY a LIMIT 2;
 } {abc001 integer abc002 integer}
 
 finish_test

From 23b440f8832a0d239d32008cb1f611508d88b89d Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 29 Dec 2017 15:19:03 +0000
Subject: [PATCH 200/488] Fix a typo in the MSVC makefile.

FossilOrigin-Name: 89e5720a8d8d46d4e2f16f15678e91336117e64abc977174e2b345a30308d0d2
---
 Makefile.msc  |  2 +-
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/Makefile.msc b/Makefile.msc
index 94fa32dc3a..27eb23a240 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -1513,7 +1513,7 @@ FUZZDATA = \
 !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_DBSTAT_VTAB
-SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_EANBLE_OFFSET_SQL_FUNC
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC
 !ENDIF
 
 # <<mark>>
diff --git a/manifest b/manifest
index 1328eaf9ce..3d0577aaea 100644
--- a/manifest
+++ b/manifest
@@ -1,8 +1,8 @@
-C Change\sthe\sfunction\sname\sto\ssqlite_unsupported_offset(X).\s\sOnly\senable\sthe\nfunction\sif\scompiled\swith\s-DSQLITE_ENABLE_OFFSET_SQL_FUNC.\s\sThe\smakefiles\sadd\nthat\sdefinition\sto\sshell\sbuilds.
-D 2017-12-29T15:04:49.734
+C Fix\sa\stypo\sin\sthe\sMSVC\smakefile.
+D 2017-12-29T15:19:03.549
 F Makefile.in 804c347948b0ad5a962aea3e59413e897227e0173d0e76910f11f473e6c7ae9b
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc f11644a365122b16628da7839a8ed2e3a781b578a94cb1ebd25ebad260bdc9c2
+F Makefile.msc 57dcce0595d4a0c4b94b6f6ba732b8e33540358adf5fb8e1aa8389d34a9677e8
 F README.md eeae1e552f93ef72ef7c5b8f6647b368a001c28820ad1df179d3dae602bef681
 F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -1688,7 +1688,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P dd94d6a880dfec4bddd247239b815b84964f804d24841e25f33f1d46a4b5274d
-R 78409e335314d524c66761480a13a3e4
+P 7a7f826e324b1a2c332e2f1d0740fd0babffcaca6275a798572f02ad367b99ab
+R 0e2a4a2bb72ba4dcfe2f3a815db6ddb3
 U drh
-Z 5cd44f60a8e786e38b3cacae6fe91794
+Z 9e1beeafb001196cab204d1af69f0fe2
diff --git a/manifest.uuid b/manifest.uuid
index 9bbedd7b51..a57419f54c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7a7f826e324b1a2c332e2f1d0740fd0babffcaca6275a798572f02ad367b99ab
\ No newline at end of file
+89e5720a8d8d46d4e2f16f15678e91336117e64abc977174e2b345a30308d0d2
\ No newline at end of file

From 9af41ff4524bce2756d807c43c09a15944f52ed1 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 29 Dec 2017 16:37:33 +0000
Subject: [PATCH 201/488] Stricter test cases.

FossilOrigin-Name: 9406c0a685fd5ff3516a66402b0514a1440652822a5eecf0b7af85929f3079e8
---
 manifest        | 12 +++++------
 manifest.uuid   |  2 +-
 test/func6.test | 56 ++++++++++++++++++++++++++++++++++++++++++-------
 3 files changed, 56 insertions(+), 14 deletions(-)

diff --git a/manifest b/manifest
index 3d0577aaea..6b6ae42bc2 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\stypo\sin\sthe\sMSVC\smakefile.
-D 2017-12-29T15:19:03.549
+C Stricter\stest\scases.
+D 2017-12-29T16:37:33.070
 F Makefile.in 804c347948b0ad5a962aea3e59413e897227e0173d0e76910f11f473e6c7ae9b
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 57dcce0595d4a0c4b94b6f6ba732b8e33540358adf5fb8e1aa8389d34a9677e8
@@ -917,7 +917,7 @@ F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f
 F test/func3.test d202a7606d23f90988a664e88e268aed1087c11c
 F test/func4.test 6beacdfcb0e18c358e6c2dcacf1b65d1fa80955f
 F test/func5.test cdd224400bc3e48d891827cc913a57051a426fa4
-F test/func6.test 8a6ecb974a9d0bdeef6a06aff8e305b1667c3b922103037735b39c716360ebc2
+F test/func6.test 1a2bc511fedb779e2bd3154361385216c6d35080f36bfe896b8cdd1b1954294a
 F test/fuzz-oss1.test e58330d01cbbd8215ee636b17a03fe220b37dbfa
 F test/fuzz.test 96083052bf5765e4518c1ba686ce2bab785670d1
 F test/fuzz2.test 76dc35b32b6d6f965259508508abce75a6c4d7e1
@@ -1688,7 +1688,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 7a7f826e324b1a2c332e2f1d0740fd0babffcaca6275a798572f02ad367b99ab
-R 0e2a4a2bb72ba4dcfe2f3a815db6ddb3
+P 89e5720a8d8d46d4e2f16f15678e91336117e64abc977174e2b345a30308d0d2
+R 9cb794037b97521b258cb5fdbbf60d53
 U drh
-Z 9e1beeafb001196cab204d1af69f0fe2
+Z 8d1ee588df9bbfd069e3afd64819112f
diff --git a/manifest.uuid b/manifest.uuid
index a57419f54c..b37d8e1f54 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-89e5720a8d8d46d4e2f16f15678e91336117e64abc977174e2b345a30308d0d2
\ No newline at end of file
+9406c0a685fd5ff3516a66402b0514a1440652822a5eecf0b7af85929f3079e8
\ No newline at end of file
diff --git a/test/func6.test b/test/func6.test
index ff05ee78c5..0d3de8ef49 100644
--- a/test/func6.test
+++ b/test/func6.test
@@ -11,6 +11,10 @@
 #
 # Test cases for the sqlite_unsupported_offset() function.
 #
+# Some of the tests in this file depend on the exact placement of content
+# within b-tree pages.  Such placement is at the implementations discretion,
+# and so it is possible for results to change from one release to the next.
+#
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 ifcapable !offset_sql_func {
@@ -19,26 +23,64 @@ ifcapable !offset_sql_func {
 }
 
 do_execsql_test func6-100 {
+  PRAGMA page_size=4096;
+  PRAGMA auto_vacuum=NONE;
   CREATE TABLE t1(a,b,c,d);
   WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100)
    INSERT INTO t1(a,b,c,d) SELECT printf('abc%03x',x), x, 1000-x, NULL FROM c;
+  CREATE INDEX t1a ON t1(a);
+  CREATE INDEX t1bc ON t1(b,c);
+  CREATE TABLE t2(x TEXT PRIMARY KEY, y) WITHOUT ROWID;
+  INSERT INTO t2(x,y) SELECT a, b FROM t1;
 }
 do_execsql_test func6-110 {
-  SELECT a, typeof(sqlite_unsupported_offset(a)) FROM t1
+  SELECT a, sqlite_unsupported_offset(d)/4096 + 1,
+            sqlite_unsupported_offset(d)%4096 FROM t1
    ORDER BY rowid LIMIT 2;
-} {abc001 integer abc002 integer}
+} {abc001 2 4084 abc002 2 4069}
 do_execsql_test func6-120 {
   SELECT a, typeof(sqlite_unsupported_offset(+a)) FROM t1
    ORDER BY rowid LIMIT 2;
 } {abc001 null abc002 null}
 do_execsql_test func6-130 {
-  CREATE INDEX t1a ON t1(a);
-  SELECT a, typeof(sqlite_unsupported_offset(a)) FROM t1
+  SELECT a, sqlite_unsupported_offset(a)/4096+1, 
+         sqlite_unsupported_offset(a)%4096
+   FROM t1
    ORDER BY a LIMIT 2;
-} {abc001 integer abc002 integer}
+} {abc001 3 4087 abc002 3 4076}
 do_execsql_test func6-140 {
-  SELECT a, typeof(sqlite_unsupported_offset(a)) FROM t1 NOT INDEXED
+  SELECT a, sqlite_unsupported_offset(d)/4096+1, 
+         sqlite_unsupported_offset(d)%4096
+   FROM t1
    ORDER BY a LIMIT 2;
-} {abc001 integer abc002 integer}
+} {abc001 2 4084 abc002 2 4069}
+do_execsql_test func6-150 {
+  SELECT a,
+         sqlite_unsupported_offset(a)/4096+1, 
+         sqlite_unsupported_offset(a)%4096,
+         sqlite_unsupported_offset(d)/4096+1, 
+         sqlite_unsupported_offset(d)%4096
+   FROM t1
+   ORDER BY a LIMIT 2;
+} {abc001 3 4087 2 4084 abc002 3 4076 2 4069}
+do_execsql_test func6-160 {
+  SELECT b,
+         sqlite_unsupported_offset(b)/4096+1, 
+         sqlite_unsupported_offset(b)%4096,
+         sqlite_unsupported_offset(c)/4096+1, 
+         sqlite_unsupported_offset(c)%4096,
+         sqlite_unsupported_offset(d)/4096+1, 
+         sqlite_unsupported_offset(d)%4096
+   FROM t1
+   ORDER BY b LIMIT 2;
+} {1 4 4090 4 4090 2 4084 2 4 4081 4 4081 2 4069}
+
+
+do_execsql_test func6-200 {
+  SELECT y, sqlite_unsupported_offset(y)/4096+1,
+         sqlite_unsupported_offset(y)%4096
+   FROM t2
+   ORDER BY x LIMIT 2;
+} {1 5 4087 2 5 4076}
 
 finish_test

From 373dc3bb3f00c19e84fab3a997194c7316f4f0e5 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 29 Dec 2017 20:19:03 +0000
Subject: [PATCH 202/488] Update ext/misc/zipfile.c to support creating and
 adding entries to existing zip archives.

FossilOrigin-Name: 2dec2dec592c0726ebe87b841b9c8d493dea7074a99f278eb1bf0b744d658a9d
---
 ext/misc/zipfile.c | 670 +++++++++++++++++++++++++++++++++++++++++----
 main.mk            |   1 +
 manifest           |  17 +-
 manifest.uuid      |   2 +-
 src/test1.c        |   2 +
 test/zipfile.test  |  60 ++++
 6 files changed, 689 insertions(+), 63 deletions(-)
 create mode 100644 test/zipfile.test

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index 188d4fc2ea..5824965032 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -26,6 +26,8 @@ SQLITE_EXTENSION_INIT1
 #include <utime.h>
 #include <errno.h>
 
+#include <zlib.h>
+
 #ifndef SQLITE_OMIT_VIRTUALTABLE
 
 #ifndef SQLITE_AMALGAMATION
@@ -47,11 +49,40 @@ typedef unsigned long u32;
 ");"
 
 #define ZIPFILE_F_COLUMN_IDX 6    /* Index of column "f" in the above */
-
 #define ZIPFILE_BUFFER_SIZE (64*1024)
 
 
-#define ZIPFILE_EXTRA_TIMESTAMP 0x5455
+/*
+** Magic numbers used to read and write zip files.
+**
+** ZIPFILE_NEWENTRY_MADEBY:
+**   Use this value for the "version-made-by" field in new zip file
+**   entries. The upper byte indicates "unix", and the lower byte 
+**   indicates that the zip file matches pkzip specification 3.0. 
+**   This is what info-zip seems to do.
+**
+** ZIPFILE_NEWENTRY_REQUIRED:
+**   Value for "version-required-to-extract" field of new entries.
+**   Version 2.0 is required to support folders and deflate compression.
+**
+** ZIPFILE_NEWENTRY_FLAGS:
+**   Value for "general-purpose-bit-flags" field of new entries. Bit
+**   11 means "utf-8 filename and comment".
+**
+** ZIPFILE_SIGNATURE_CDS:
+**   First 4 bytes of a valid CDS record.
+**
+** ZIPFILE_SIGNATURE_LFH:
+**   First 4 bytes of a valid LFH record.
+*/
+#define ZIPFILE_EXTRA_TIMESTAMP   0x5455
+#define ZIPFILE_NEWENTRY_MADEBY   ((3<<8) + 30)
+#define ZIPFILE_NEWENTRY_REQUIRED 20
+#define ZIPFILE_NEWENTRY_FLAGS    0x800
+#define ZIPFILE_SIGNATURE_CDS     0x02014b50
+#define ZIPFILE_SIGNATURE_LFH     0x04034b50
+#define ZIPFILE_SIGNATURE_EOCD    0x06054b50
+#define ZIPFILE_LFH_FIXED_SZ      30
 
 /*
 ** Set the error message contained in context ctx to the results of
@@ -188,18 +219,58 @@ struct ZipfileCsr {
   i64 iDataOff;              /* Offset in zipfile to data */
   u32 mTime;                 /* Extended mtime value */
   int flags;
-  u8 *aBuffer;               /* Buffer used for various tasks */
 };
 
+/*
+** Values for ZipfileCsr.flags.
+*/
 #define ZIPFILE_MTIME_VALID 0x0001
 
+typedef struct ZipfileEntry ZipfileEntry;
+struct ZipfileEntry {
+  char *zPath;               /* Path of zipfile entry */
+  u8 *aCdsEntry;             /* Buffer containing entire CDS entry */
+  int nCdsEntry;             /* Size of buffer aCdsEntry[] in bytes */
+  ZipfileEntry *pNext;
+};
+
 typedef struct ZipfileTab ZipfileTab;
 struct ZipfileTab {
   sqlite3_vtab base;         /* Base class - must be first */
+  char *zFile;               /* Zip file this table accesses (may be NULL) */
+  u8 *aBuffer;               /* Temporary buffer used for various tasks */
+
+  /* The following are used by write transactions only */
+  ZipfileEntry *pEntry;      /* Linked list of all files (if pWriteFd!=0) */
+  FILE *pWriteFd;            /* File handle open on zip archive */
+  i64 szCurrent;             /* Current size of zip archive */
+  i64 szOrig;                /* Size of archive at start of transaction */
 };
 
+static void zipfileDequote(char *zIn){
+  char q = zIn[0];
+  if( q=='"' || q=='\'' || q=='`' || q=='[' ){
+    char c;
+    int iIn = 1;
+    int iOut = 0;
+    if( q=='[' ) q = ']';
+    while( (c = zIn[iIn++]) ){
+      if( c==q ){
+        if( zIn[iIn++]!=q ) break;
+      }
+      zIn[iOut++] = c;
+    }
+    zIn[iOut] = '\0';
+  }
+}
+
 /*
 ** Construct a new ZipfileTab virtual table object.
+** 
+**   argv[0]   -> module name  ("zipfile")
+**   argv[1]   -> database name
+**   argv[2]   -> table name
+**   argv[...] -> "column name" and other module argument fields.
 */
 static int zipfileConnect(
   sqlite3 *db,
@@ -208,14 +279,28 @@ static int zipfileConnect(
   sqlite3_vtab **ppVtab,
   char **pzErr
 ){
+  int nByte = sizeof(ZipfileTab) + ZIPFILE_BUFFER_SIZE;
+  int nFile = 0;
+  const char *zFile = 0;
   ZipfileTab *pNew = 0;
   int rc;
 
+  if( argc>3 ){
+    zFile = argv[3];
+    nFile = strlen(zFile)+1;
+  }
+
   rc = sqlite3_declare_vtab(db, ZIPFILE_SCHEMA);
   if( rc==SQLITE_OK ){
-    pNew = (ZipfileTab*)sqlite3_malloc( sizeof(*pNew) );
+    pNew = (ZipfileTab*)sqlite3_malloc(nByte+nFile);
     if( pNew==0 ) return SQLITE_NOMEM;
-    memset(pNew, 0, sizeof(*pNew));
+    memset(pNew, 0, nByte+nFile);
+    pNew->aBuffer = (u8*)&pNew[1];
+    if( zFile ){
+      pNew->zFile = (char*)&pNew->aBuffer[ZIPFILE_BUFFER_SIZE];
+      memcpy(pNew->zFile, zFile, nFile);
+      zipfileDequote(pNew->zFile);
+    }
   }
   *ppVtab = (sqlite3_vtab*)pNew;
   return rc;
@@ -234,11 +319,12 @@ static int zipfileDisconnect(sqlite3_vtab *pVtab){
 */
 static int zipfileOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){
   ZipfileCsr *pCsr;
-  pCsr = sqlite3_malloc( sizeof(*pCsr) + ZIPFILE_BUFFER_SIZE);
-  if( pCsr==0 ) return SQLITE_NOMEM;
+  pCsr = sqlite3_malloc(sizeof(*pCsr));
+  *ppCsr = (sqlite3_vtab_cursor*)pCsr;
+  if( pCsr==0 ){
+    return SQLITE_NOMEM;
+  }
   memset(pCsr, 0, sizeof(*pCsr));
-  pCsr->aBuffer = (u8*)&pCsr[1];
-  *ppCsr = &pCsr->base;
   return SQLITE_OK;
 }
 
@@ -247,6 +333,8 @@ static int zipfileOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){
 ** by zipfileOpen().
 */
 static void zipfileResetCursor(ZipfileCsr *pCsr){
+  sqlite3_free(pCsr->cds.zFile);
+  pCsr->cds.zFile = 0;
   pCsr->iRowid = 0;
   pCsr->bEof = 0;
   if( pCsr->pFile ){
@@ -276,17 +364,39 @@ static void zipfileSetErrmsg(ZipfileCsr *pCsr, const char *zFmt, ...){
   va_end(ap);
 }
 
-static int zipfileReadData(ZipfileCsr *pCsr, u8 *aRead, int nRead, i64 iOff){
+static int zipfileReadData(
+  FILE *pFile,                    /* Read from this file */
+  u8 *aRead,                      /* Read into this buffer */
+  int nRead,                      /* Number of bytes to read */
+  i64 iOff,                       /* Offset to read from */
+  char **pzErrmsg                 /* OUT: Error message (from sqlite3_malloc) */
+){
   size_t n;
-  fseek(pCsr->pFile, iOff, SEEK_SET);
-  n = fread(aRead, 1, nRead, pCsr->pFile);
+  fseek(pFile, iOff, SEEK_SET);
+  n = fread(aRead, 1, nRead, pFile);
   if( n!=nRead ){
-    zipfileSetErrmsg(pCsr, "error in fread()");
+    *pzErrmsg = sqlite3_mprintf("error in fread()");
     return SQLITE_ERROR;
   }
   return SQLITE_OK;
 }
 
+static int zipfileAppendData(
+  ZipfileTab *pTab,
+  const u8 *aWrite,
+  int nWrite
+){
+  size_t n;
+  fseek(pTab->pWriteFd, pTab->szCurrent, SEEK_SET);
+  n = fwrite(aWrite, 1, nWrite, pTab->pWriteFd);
+  if( n!=nWrite ){
+    pTab->base.zErrMsg = sqlite3_mprintf("error in fwrite()");
+    return SQLITE_ERROR;
+  }
+  pTab->szCurrent += nWrite;
+  return SQLITE_OK;
+}
+
 static u16 zipfileGetU16(const u8 *aBuf){
   return (aBuf[1] << 8) + aBuf[0];
 }
@@ -297,18 +407,47 @@ static u32 zipfileGetU32(const u8 *aBuf){
        + ((u32)(aBuf[0]) <<  0);
 }
 
+static void zipfilePutU16(u8 *aBuf, u16 val){
+  aBuf[0] = val & 0xFF;
+  aBuf[1] = (val>>8) & 0xFF;
+}
+static void zipfilePutU32(u8 *aBuf, u32 val){
+  aBuf[0] = val & 0xFF;
+  aBuf[1] = (val>>8) & 0xFF;
+  aBuf[2] = (val>>16) & 0xFF;
+  aBuf[3] = (val>>24) & 0xFF;
+}
+
 #define zipfileRead32(aBuf) ( aBuf+=4, zipfileGetU32(aBuf-4) )
 #define zipfileRead16(aBuf) ( aBuf+=2, zipfileGetU16(aBuf-2) )
 
+#define zipfileWrite32(aBuf,val) { zipfilePutU32(aBuf,val); aBuf+=4; }
+#define zipfileWrite16(aBuf,val) { zipfilePutU16(aBuf,val); aBuf+=2; }
+
+static u8* zipfileCsrBuffer(ZipfileCsr *pCsr){
+  return ((ZipfileTab*)(pCsr->base.pVtab))->aBuffer;
+}
+
+/*
+** Magic numbers used to read CDS records.
+*/
+#define ZIPFILE_CDS_FIXED_SZ  46
+#define ZIPFILE_CDS_NFILE_OFF 28
+
 static int zipfileReadCDS(ZipfileCsr *pCsr){
-  static const int szFix = 46;    /* Size of fixed-size part of CDS */
-  u8 *aRead = pCsr->aBuffer;
+  char **pzErr = &pCsr->base.pVtab->zErrMsg;
+  u8 *aRead = zipfileCsrBuffer(pCsr);
   int rc;
 
-  rc = zipfileReadData(pCsr, aRead, szFix, pCsr->iNextOff);
+  sqlite3_free(pCsr->cds.zFile);
+  pCsr->cds.zFile = 0;
+
+  rc = zipfileReadData(
+      pCsr->pFile, aRead, ZIPFILE_CDS_FIXED_SZ, pCsr->iNextOff, pzErr
+  );
   if( rc==SQLITE_OK ){
     u32 sig = zipfileRead32(aRead);
-    if( sig!=0x02014b50 ){
+    if( sig!=ZIPFILE_SIGNATURE_CDS ){
       zipfileSetErrmsg(pCsr,"failed to read CDS at offset %lld",pCsr->iNextOff);
       rc = SQLITE_ERROR;
     }else{
@@ -322,6 +461,7 @@ static int zipfileReadCDS(ZipfileCsr *pCsr){
       pCsr->cds.crc32 = zipfileRead32(aRead);
       pCsr->cds.szCompressed = zipfileRead32(aRead);
       pCsr->cds.szUncompressed = zipfileRead32(aRead);
+      assert( aRead==zipfileCsrBuffer(pCsr)+ZIPFILE_CDS_NFILE_OFF );
       pCsr->cds.nFile = zipfileRead16(aRead);
       pCsr->cds.nExtra = zipfileRead16(aRead);
       pCsr->cds.nComment = zipfileRead16(aRead);
@@ -330,15 +470,15 @@ static int zipfileReadCDS(ZipfileCsr *pCsr){
       pCsr->cds.iExternalAttr = zipfileRead32(aRead);
       pCsr->cds.iOffset = zipfileRead32(aRead);
 
-      assert( aRead==&pCsr->aBuffer[szFix] );
+      assert( aRead==zipfileCsrBuffer(pCsr)+ZIPFILE_CDS_FIXED_SZ );
 
       nRead = pCsr->cds.nFile + pCsr->cds.nExtra;
-      aRead = pCsr->aBuffer;
-      rc = zipfileReadData(pCsr, aRead, nRead, pCsr->iNextOff+szFix);
+      aRead = zipfileCsrBuffer(pCsr);
+      pCsr->iNextOff += ZIPFILE_CDS_FIXED_SZ;
+      rc = zipfileReadData(pCsr->pFile, aRead, nRead, pCsr->iNextOff, pzErr);
 
       if( rc==SQLITE_OK ){
         pCsr->cds.zFile = sqlite3_mprintf("%.*s", (int)pCsr->cds.nFile, aRead);
-        pCsr->iNextOff += szFix;
         pCsr->iNextOff += pCsr->cds.nFile;
         pCsr->iNextOff += pCsr->cds.nExtra;
         pCsr->iNextOff += pCsr->cds.nComment;
@@ -381,14 +521,15 @@ static int zipfileReadCDS(ZipfileCsr *pCsr){
 }
 
 static int zipfileReadLFH(ZipfileCsr *pCsr){
-  static const int szFix = 30;    /* Size of fixed-size part of LFH */
-  u8 *aRead = pCsr->aBuffer;
+  char **pzErr = &pCsr->base.pVtab->zErrMsg;
+  static const int szFix = ZIPFILE_LFH_FIXED_SZ;
+  u8 *aRead = zipfileCsrBuffer(pCsr);
   int rc;
 
-  rc = zipfileReadData(pCsr, aRead, szFix, pCsr->cds.iOffset);
+  rc = zipfileReadData(pCsr->pFile, aRead, szFix, pCsr->cds.iOffset, pzErr);
   if( rc==SQLITE_OK ){
     u32 sig = zipfileRead32(aRead);
-    if( sig!=0x04034b50 ){
+    if( sig!=ZIPFILE_SIGNATURE_LFH ){
       zipfileSetErrmsg(pCsr, "failed to read LFH at offset %d", 
           (int)pCsr->cds.iOffset
       );
@@ -404,7 +545,7 @@ static int zipfileReadLFH(ZipfileCsr *pCsr){
       pCsr->lfh.szUncompressed = zipfileRead32(aRead);
       pCsr->lfh.nFile = zipfileRead16(aRead);
       pCsr->lfh.nExtra = zipfileRead16(aRead);
-      assert( aRead==&pCsr->aBuffer[szFix] );
+      assert( aRead==zipfileCsrBuffer(pCsr)+szFix );
       pCsr->iDataOff = pCsr->cds.iOffset+szFix+pCsr->lfh.nFile+pCsr->lfh.nExtra;
     }
   }
@@ -460,6 +601,22 @@ static time_t zipfileMtime(ZipfileCsr *pCsr){
   return mktime(&t);
 }
 
+static void zipfileMtimeToDos(ZipfileCDS *pCds, u32 mTime){
+  time_t t = (time_t)mTime;
+  struct tm res;
+  localtime_r(&t, &res);
+
+  pCds->mTime = 
+    (res.tm_sec / 2) + 
+    (res.tm_min << 5) +
+    (res.tm_hour << 11);
+
+  pCds->mDate = 
+    (res.tm_mday-1) +
+    ((res.tm_mon+1) << 5) +
+    ((res.tm_year-80) << 9);
+}
+
 /*
 ** Return values of columns for the row at which the series_cursor
 ** is currently pointing.
@@ -499,7 +656,9 @@ static int zipfileColumn(
         if( aBuf==0 ){
           rc = SQLITE_NOMEM;
         }else{
-          rc = zipfileReadData(pCsr, aBuf, sz, pCsr->iDataOff);
+          rc = zipfileReadData(pCsr->pFile, aBuf, sz, pCsr->iDataOff,
+              &pCsr->base.pVtab->zErrMsg
+          );
         }
         if( rc==SQLITE_OK ){
           sqlite3_result_blob(ctx, aBuf, sz, SQLITE_TRANSIENT);
@@ -537,17 +696,26 @@ static int zipfileEof(sqlite3_vtab_cursor *cur){
 }
 
 /*
-** The zip file has been successfully opened (so pCsr->pFile is valid). 
-** This function attempts to locate and read the End of central
-** directory record from the file.
-**
 */
-static int zipfileReadEOCD(ZipfileCsr *pCsr, ZipfileEOCD *pEOCD){
-  u8 *aRead = pCsr->aBuffer;
-  int nRead = (int)(MIN(pCsr->nByte, ZIPFILE_BUFFER_SIZE));
-  i64 iOff = pCsr->nByte - nRead;
+static int zipfileReadEOCD(
+  ZipfileTab *pTab,               /* Return errors here */
+  FILE *pFile,                    /* Read from this file */
+  ZipfileEOCD *pEOCD              /* Object to populate */
+){
+  u8 *aRead = pTab->aBuffer;      /* Temporary buffer */
+  i64 szFile;                     /* Total size of file in bytes */
+  int nRead;                      /* Bytes to read from file */
+  i64 iOff;                       /* Offset to read from */
 
-  int rc = zipfileReadData(pCsr, aRead, nRead, iOff);
+  fseek(pFile, 0, SEEK_END);
+  szFile = (i64)ftell(pFile);
+  if( szFile==0 ){
+    return SQLITE_EMPTY;
+  }
+  nRead = (int)(MIN(szFile, ZIPFILE_BUFFER_SIZE));
+  iOff = szFile - nRead;
+
+  int rc = zipfileReadData(pFile, aRead, nRead, iOff, &pTab->base.zErrMsg);
   if( rc==SQLITE_OK ){
     int i;
 
@@ -560,7 +728,9 @@ static int zipfileReadEOCD(ZipfileCsr *pCsr, ZipfileEOCD *pEOCD){
       }
     }
     if( i<0 ){
-      zipfileSetErrmsg(pCsr, "cannot find end of central directory record");
+      pTab->base.zErrMsg = sqlite3_mprintf(
+          "cannot find end of central directory record"
+      );
       return SQLITE_ERROR;
     }
 
@@ -592,6 +762,7 @@ static int zipfileFilter(
   int idxNum, const char *idxStr,
   int argc, sqlite3_value **argv
 ){
+  ZipfileTab *pTab = (ZipfileTab*)cur->pVtab;
   ZipfileCsr *pCsr = (ZipfileCsr*)cur;
   const char *zFile;              /* Zip file to scan */
   int rc = SQLITE_OK;             /* Return Code */
@@ -600,23 +771,29 @@ static int zipfileFilter(
 
   assert( idxNum==argc && (idxNum==0 || idxNum==1) );
   if( idxNum==0 ){
-    /* Error. User did not supply a file name. */
-    zipfileSetErrmsg(pCsr, "table function zipfile() requires an argument");
-    return SQLITE_ERROR;
+    ZipfileTab *pTab = (ZipfileTab*)cur->pVtab;
+    zFile = pTab->zFile;
+    if( zFile==0 ){
+      /* Error. This is an eponymous virtual table and the user has not 
+      ** supplied a file name. */
+      zipfileSetErrmsg(pCsr, "table function zipfile() requires an argument");
+      return SQLITE_ERROR;
+    }
+  }else{
+    zFile = (const char*)sqlite3_value_text(argv[0]);
   }
-
-  zFile = sqlite3_value_text(argv[0]);
   pCsr->pFile = fopen(zFile, "rb");
   if( pCsr->pFile==0 ){
     zipfileSetErrmsg(pCsr, "cannot open file: %s", zFile);
     rc = SQLITE_ERROR;
   }else{
-    fseek(pCsr->pFile, 0, SEEK_END);
-    pCsr->nByte = (i64)ftell(pCsr->pFile);
-    rc = zipfileReadEOCD(pCsr, &pCsr->eocd);
+    rc = zipfileReadEOCD(pTab, pCsr->pFile, &pCsr->eocd);
     if( rc==SQLITE_OK ){
       pCsr->iNextOff = pCsr->eocd.iOffset;
       rc = zipfileNext(cur);
+    }else if( rc==SQLITE_EMPTY ){
+      rc = SQLITE_OK;
+      pCsr->bEof = 1;
     }
   }
 
@@ -653,17 +830,404 @@ static int zipfileBestIndex(
   return SQLITE_OK;
 }
 
+static int zipfileLoadDirectory(ZipfileTab *pTab){
+  ZipfileEOCD eocd;
+  int rc;
+
+  rc = zipfileReadEOCD(pTab, pTab->pWriteFd, &eocd);
+  if( rc==SQLITE_OK ){
+    int i;
+    int iOff = 0;
+    u8 *aBuf = sqlite3_malloc(eocd.nSize);
+    if( aBuf==0 ){
+      rc = SQLITE_NOMEM;
+    }else{
+      rc = zipfileReadData(
+          pTab->pWriteFd, aBuf, eocd.nSize, eocd.iOffset, &pTab->base.zErrMsg
+      );
+    }
+
+    for(i=0; rc==SQLITE_OK && i<eocd.nEntry; i++){
+      u16 nFile;
+      u16 nExtra;
+      u16 nComment;
+      ZipfileEntry *pNew;
+      u8 *aRec = &aBuf[iOff];
+
+      nFile = zipfileGetU16(&aRec[ZIPFILE_CDS_NFILE_OFF]);
+      nExtra = zipfileGetU16(&aRec[ZIPFILE_CDS_NFILE_OFF+2]);
+      nComment = zipfileGetU16(&aRec[ZIPFILE_CDS_NFILE_OFF+4]);
+
+      pNew = sqlite3_malloc(
+          sizeof(ZipfileEntry) 
+        + nFile+1 
+        + ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment
+      );
+      if( pNew==0 ){
+        rc = SQLITE_NOMEM;
+      }else{
+        pNew->zPath = (char*)&pNew[1];
+        memcpy(pNew->zPath, &aBuf[ZIPFILE_CDS_FIXED_SZ], nFile);
+        pNew->zPath[nFile] = '\0';
+        pNew->aCdsEntry = (u8*)&pNew->zPath[nFile+1];
+        pNew->nCdsEntry = ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment;
+        memcpy(pNew->aCdsEntry, aRec, pNew->nCdsEntry);
+
+        pNew->pNext = pTab->pEntry;
+        pTab->pEntry = pNew;
+      }
+
+      iOff += ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment;
+    }
+
+    sqlite3_free(aBuf);
+  }else if( rc==SQLITE_EMPTY ){
+    rc = SQLITE_OK;
+  }
+
+  return rc;
+}
+
+static ZipfileEntry *zipfileNewEntry(
+  ZipfileCDS *pCds,               /* Values for fixed size part of CDS */
+  const char *zPath,              /* Path for new entry */
+  int nPath,                      /* strlen(zPath) */
+  u32 mTime                       /* Modification time (or 0) */
+){
+  u8 *aWrite;
+  ZipfileEntry *pNew;
+  pCds->nFile = nPath;
+  pCds->nExtra = mTime ? 9 : 0;
+  pNew = (ZipfileEntry*)sqlite3_malloc(
+    sizeof(ZipfileEntry) + 
+    nPath+1 + 
+    ZIPFILE_CDS_FIXED_SZ + nPath + pCds->nExtra
+  );
+
+  if( pNew ){
+    pNew->zPath = (char*)&pNew[1];
+    pNew->aCdsEntry = (u8*)&pNew->zPath[nPath+1];
+    pNew->nCdsEntry = ZIPFILE_CDS_FIXED_SZ + nPath + pCds->nExtra;
+    memcpy(pNew->zPath, zPath, nPath+1);
+
+    aWrite = pNew->aCdsEntry;
+    zipfileWrite32(aWrite, ZIPFILE_SIGNATURE_CDS);
+    zipfileWrite16(aWrite, pCds->iVersionMadeBy);
+    zipfileWrite16(aWrite, pCds->iVersionExtract);
+    zipfileWrite16(aWrite, pCds->flags);
+    zipfileWrite16(aWrite, pCds->iCompression);
+    zipfileWrite16(aWrite, pCds->mTime);
+    zipfileWrite16(aWrite, pCds->mDate);
+    zipfileWrite32(aWrite, pCds->crc32);
+    zipfileWrite32(aWrite, pCds->szCompressed);
+    zipfileWrite32(aWrite, pCds->szUncompressed);
+    zipfileWrite16(aWrite, pCds->nFile);
+    zipfileWrite16(aWrite, pCds->nExtra);
+    zipfileWrite16(aWrite, pCds->nComment);      assert( pCds->nComment==0 );
+    zipfileWrite16(aWrite, pCds->iDiskStart);
+    zipfileWrite16(aWrite, pCds->iInternalAttr);
+    zipfileWrite32(aWrite, pCds->iExternalAttr);
+    zipfileWrite32(aWrite, pCds->iOffset);
+    assert( aWrite==&pNew->aCdsEntry[ZIPFILE_CDS_FIXED_SZ] );
+    memcpy(aWrite, zPath, nPath);
+    if( pCds->nExtra ){
+      aWrite += nPath;
+      zipfileWrite16(aWrite, ZIPFILE_EXTRA_TIMESTAMP);
+      zipfileWrite16(aWrite, 5);
+      *aWrite++ = 0x01;
+      zipfileWrite32(aWrite, mTime);
+    }
+  }
+
+  return pNew;
+}
+
+static int zipfileAppendEntry(
+  ZipfileTab *pTab,
+  ZipfileCDS *pCds,
+  const char *zPath,              /* Path for new entry */
+  int nPath,                      /* strlen(zPath) */
+  const u8 *pData,
+  int nData,
+  u32 mTime
+){
+  u8 *aBuf = pTab->aBuffer;
+  int rc;
+
+  zipfileWrite32(aBuf, ZIPFILE_SIGNATURE_LFH);
+  zipfileWrite16(aBuf, pCds->iVersionExtract);
+  zipfileWrite16(aBuf, pCds->flags);
+  zipfileWrite16(aBuf, pCds->iCompression);
+  zipfileWrite16(aBuf, pCds->mTime);
+  zipfileWrite16(aBuf, pCds->mDate);
+  zipfileWrite32(aBuf, pCds->crc32);
+  zipfileWrite32(aBuf, pCds->szCompressed);
+  zipfileWrite32(aBuf, pCds->szUncompressed);
+  zipfileWrite16(aBuf, nPath);
+  zipfileWrite16(aBuf, pCds->nExtra);
+  assert( aBuf==&pTab->aBuffer[ZIPFILE_LFH_FIXED_SZ] );
+  rc = zipfileAppendData(pTab, pTab->aBuffer, aBuf - pTab->aBuffer);
+  if( rc==SQLITE_OK ){
+    rc = zipfileAppendData(pTab, (const u8*)zPath, nPath);
+  }
+
+  if( rc==SQLITE_OK && pCds->nExtra ){
+    aBuf = pTab->aBuffer;
+    zipfileWrite16(aBuf, ZIPFILE_EXTRA_TIMESTAMP);
+    zipfileWrite16(aBuf, 5);
+    *aBuf++ = 0x01;
+    zipfileWrite32(aBuf, mTime);
+    rc = zipfileAppendData(pTab, pTab->aBuffer, 9);
+  }
+
+  if( rc==SQLITE_OK ){
+    rc = zipfileAppendData(pTab, pData, nData);
+  }
+
+  return rc;
+}
+
+static int zipfileGetMode(ZipfileTab *pTab, sqlite3_value *pVal, int *pMode){
+  const char *z = (const char*)sqlite3_value_text(pVal);
+  int mode = 0;
+  if( z==0 || (z[0]>=0 && z[0]<=9) ){
+    mode = sqlite3_value_int(pVal);
+  }else{
+    const char zTemplate[10] = "-rwxrwxrwx";
+    int i;
+    if( strlen(z)!=10 ) goto parse_error;
+    switch( z[0] ){
+      case '-': mode |= S_IFREG; break;
+      case 'd': mode |= S_IFDIR; break;
+      case 'l': mode |= S_IFLNK; break;
+      default: goto parse_error;
+    }
+    for(i=1; i<10; i++){
+      if( z[i]==zTemplate[i] ) mode |= 1 << (9-i);
+      else if( z[i]!='-' ) goto parse_error;
+    }
+  }
+  *pMode = mode;
+  return SQLITE_OK;
+
+ parse_error:
+  pTab->base.zErrMsg = sqlite3_mprintf("zipfile: parse error in mode: %s", z);
+  return SQLITE_ERROR;
+}
+
+/*
+** xUpdate method.
+*/
+static int zipfileUpdate(
+  sqlite3_vtab *pVtab, 
+  int nVal, 
+  sqlite3_value **apVal, 
+  sqlite_int64 *pRowid
+){
+  ZipfileTab *pTab = (ZipfileTab*)pVtab;
+  int rc = SQLITE_OK;             /* Return Code */
+  ZipfileEntry *pNew = 0;         /* New in-memory CDS entry */
+
+  int mode;                       /* Mode for new entry */
+  i64 mTime;                      /* Modification time for new entry */
+  i64 sz;                         /* Uncompressed size */
+  const char *zPath;              /* Path for new entry */
+  int nPath;                      /* strlen(zPath) */
+  const u8 *pData;                /* Pointer to buffer containing content */
+  int nData;                      /* Size of pData buffer in bytes */
+  int iMethod = 0;                /* Compression method for new entry */
+  u8 *pFree = 0;                  /* Free this */
+  ZipfileCDS cds;                 /* New Central Directory Structure entry */
+
+  if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
+    pTab->base.zErrMsg = sqlite3_mprintf(
+        "zipfile: UPDATE/DELETE not yet supported"
+    );
+    return SQLITE_ERROR;
+  }
+
+  /* This table is only writable if a default archive path was specified 
+  ** as part of the CREATE VIRTUAL TABLE statement. */
+  if( pTab->zFile==0 ){
+    pTab->base.zErrMsg = sqlite3_mprintf(
+        "zipfile: writing requires a default archive"
+    );
+    return SQLITE_ERROR;
+  }
+
+  /* Open a write fd on the file. Also load the entire central directory
+  ** structure into memory. During the transaction any new file data is 
+  ** appended to the archive file, but the central directory is accumulated
+  ** in main-memory until the transaction is committed.  */
+  if( pTab->pWriteFd==0 ){
+    pTab->pWriteFd = fopen(pTab->zFile, "ab+");
+    if( pTab->pWriteFd==0 ){
+      pTab->base.zErrMsg = sqlite3_mprintf(
+          "zipfile: failed to open file %s for writing", pTab->zFile
+      );
+      return SQLITE_ERROR;
+    }
+    fseek(pTab->pWriteFd, 0, SEEK_END);
+    pTab->szCurrent = pTab->szOrig = (i64)ftell(pTab->pWriteFd);
+    rc = zipfileLoadDirectory(pTab);
+    if( rc!=SQLITE_OK ) return rc;
+  }
+
+  zPath = (const char*)sqlite3_value_text(apVal[2]);
+  nPath = strlen(zPath);
+  rc = zipfileGetMode(pTab, apVal[3], &mode);
+  if( rc!=SQLITE_OK ) return rc;
+  mTime = sqlite3_value_int64(apVal[4]);
+  sz = sqlite3_value_int(apVal[5]);
+  pData = sqlite3_value_blob(apVal[6]);
+  nData = sqlite3_value_bytes(apVal[6]);
+
+  /* If a NULL value is inserted into the 'method' column, do automatic
+  ** compression. */
+  if( nData>0 && sqlite3_value_type(apVal[7])==SQLITE_NULL ){
+    pFree = (u8*)sqlite3_malloc(nData);
+    if( pFree==0 ){
+      rc = SQLITE_NOMEM;
+    }else{
+      int res;
+      z_stream str;
+      memset(&str, 0, sizeof(str));
+      str.next_in = (z_const Bytef*)pData;
+      str.avail_in = nData;
+      str.next_out = pFree;
+      str.avail_out = nData;
+      deflateInit2(&str, 9, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY);
+      res = deflate(&str, Z_FINISH);
+      if( res==Z_STREAM_END ){
+        pData = pFree;
+        nData = str.total_out;
+        iMethod = 8;
+      }else if( res!=Z_OK ){
+        pTab->base.zErrMsg = sqlite3_mprintf("zipfile: deflate() error");
+        rc = SQLITE_ERROR;
+      }
+      deflateEnd(&str);
+    }
+  }else{
+    iMethod = sqlite3_value_int(apVal[7]);
+    if( iMethod<0 || iMethod>65535 ){
+      pTab->base.zErrMsg = sqlite3_mprintf(
+          "zipfile: invalid compression method: %d", iMethod
+      );
+      rc = SQLITE_ERROR;
+    }
+  }
+
+  if( rc==SQLITE_OK ){
+    /* Create the new CDS record. */
+    memset(&cds, 0, sizeof(cds));
+    cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY;
+    cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED;
+    cds.flags = ZIPFILE_NEWENTRY_FLAGS;
+    cds.iCompression = iMethod;
+    zipfileMtimeToDos(&cds, (u32)mTime);
+    cds.crc32 = crc32(0, pData, nData);
+    cds.szCompressed = nData;
+    cds.szUncompressed = sz;
+    cds.iExternalAttr = (mode<<16);
+    cds.iOffset = pTab->szCurrent;
+    pNew = zipfileNewEntry(&cds, zPath, nPath, (u32)mTime);
+    if( pNew==0 ){
+      rc = SQLITE_NOMEM;
+    }else{
+      pNew->pNext = pTab->pEntry;
+      pTab->pEntry = pNew;
+    }
+  }
+
+  /* Append the new header+file to the archive */
+  if( rc==SQLITE_OK ){
+    rc = zipfileAppendEntry(pTab, &cds, zPath, nPath, pData, nData, (u32)mTime);
+  }
+
+  sqlite3_free(pFree);
+  return rc;
+}
+
+static int zipfileAppendEOCD(ZipfileTab *pTab, ZipfileEOCD *p){
+  u8 *aBuf = pTab->aBuffer;
+
+  zipfileWrite32(aBuf, ZIPFILE_SIGNATURE_EOCD);
+  zipfileWrite16(aBuf, p->iDisk);
+  zipfileWrite16(aBuf, p->iFirstDisk);
+  zipfileWrite16(aBuf, p->nEntry);
+  zipfileWrite16(aBuf, p->nEntryTotal);
+  zipfileWrite32(aBuf, p->nSize);
+  zipfileWrite32(aBuf, p->iOffset);
+  zipfileWrite16(aBuf, 0);        /* Size of trailing comment in bytes*/
+
+  assert( (aBuf-pTab->aBuffer)==22 );
+  return zipfileAppendData(pTab, pTab->aBuffer, aBuf - pTab->aBuffer);
+}
+
+static void zipfileCleanupTransaction(ZipfileTab *pTab){
+  ZipfileEntry *pEntry;
+  ZipfileEntry *pNext;
+
+  for(pEntry=pTab->pEntry; pEntry; pEntry=pNext){
+    pNext = pEntry->pNext;
+    sqlite3_free(pEntry);
+  }
+  pTab->pEntry = 0;
+  fclose(pTab->pWriteFd);
+  pTab->pWriteFd = 0;
+  pTab->szCurrent = 0;
+  pTab->szOrig = 0;
+}
+
+static int zipfileBegin(sqlite3_vtab *pVtab){
+  return SQLITE_OK;
+}
+
+static int zipfileCommit(sqlite3_vtab *pVtab){
+  ZipfileTab *pTab = (ZipfileTab*)pVtab;
+  int rc = SQLITE_OK;
+  if( pTab->pWriteFd ){
+    i64 iOffset = pTab->szCurrent;
+    ZipfileEntry *pEntry;
+    ZipfileEOCD eocd;
+    int nEntry = 0;
+
+    /* Write out all entries */
+    for(pEntry=pTab->pEntry; rc==SQLITE_OK && pEntry; pEntry=pEntry->pNext){
+      rc = zipfileAppendData(pTab, pEntry->aCdsEntry, pEntry->nCdsEntry);
+      nEntry++;
+    }
+
+    /* Write out the EOCD record */
+    eocd.iDisk = 0;
+    eocd.iFirstDisk = 0;
+    eocd.nEntry = nEntry;
+    eocd.nEntryTotal = nEntry;
+    eocd.nSize = pTab->szCurrent - iOffset;;
+    eocd.iOffset = iOffset;
+    rc = zipfileAppendEOCD(pTab, &eocd);
+
+    zipfileCleanupTransaction(pTab);
+  }
+  return rc;
+}
+
+static int zipfileRollback(sqlite3_vtab *pVtab){
+  return zipfileCommit(pVtab);
+}
+
 /*
 ** Register the "zipfile" virtual table.
 */
 static int zipfileRegister(sqlite3 *db){
   static sqlite3_module zipfileModule = {
-    0,                         /* iVersion */
-    0,                         /* xCreate */
+    1,                         /* iVersion */
+    zipfileConnect,            /* xCreate */
     zipfileConnect,            /* xConnect */
     zipfileBestIndex,          /* xBestIndex */
     zipfileDisconnect,         /* xDisconnect */
-    0,                         /* xDestroy */
+    zipfileDisconnect,         /* xDestroy */
     zipfileOpen,               /* xOpen - open a cursor */
     zipfileClose,              /* xClose - close a cursor */
     zipfileFilter,             /* xFilter - configure scan constraints */
@@ -671,11 +1235,11 @@ static int zipfileRegister(sqlite3 *db){
     zipfileEof,                /* xEof - check for end of scan */
     zipfileColumn,             /* xColumn - read data */
     zipfileRowid,              /* xRowid - read data */
-    0,                         /* xUpdate */
-    0,                         /* xBegin */
+    zipfileUpdate,             /* xUpdate */
+    zipfileBegin,              /* xBegin */
     0,                         /* xSync */
-    0,                         /* xCommit */
-    0,                         /* xRollback */
+    zipfileCommit,             /* xCommit */
+    zipfileRollback,           /* xRollback */
     0,                         /* xFindMethod */
     0,                         /* xRename */
   };
@@ -687,8 +1251,6 @@ static int zipfileRegister(sqlite3 *db){
 # define zipfileRegister(x) SQLITE_OK
 #endif
 
-#include <zlib.h>
-
 /*
 ** zipfile_uncompress(DATA, SZ, METHOD)
 */
diff --git a/main.mk b/main.mk
index ff379da861..9cdcc051e3 100644
--- a/main.mk
+++ b/main.mk
@@ -370,6 +370,7 @@ TESTSRC += \
   $(TOP)/ext/misc/unionvtab.c \
   $(TOP)/ext/misc/wholenumber.c \
   $(TOP)/ext/misc/vfslog.c \
+  $(TOP)/ext/misc/zipfile.c \
   $(TOP)/ext/fts5/fts5_tcl.c \
   $(TOP)/ext/fts5/fts5_test_mi.c \
   $(TOP)/ext/fts5/fts5_test_tok.c 
diff --git a/manifest b/manifest
index d6c9a543e5..c8433dc8ca 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improve\sthe\sshell\stool\s".ar\s--list\s--verbose"\scommand.
-D 2017-12-27T21:13:21.423
+C Update\sext/misc/zipfile.c\sto\ssupport\screating\sand\sadding\sentries\sto\sexisting\nzip\sarchives.
+D 2017-12-29T20:19:03.196
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
@@ -302,7 +302,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c ced1aa768904cf8182c3da93d42df4e3cf30fe7a2e66c9216f6bc64a8df4fd5b
+F ext/misc/zipfile.c d9e0b0a4fdb4cb82ad22fd19d75563efdb1d6f812d62438bd30819781c282357
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -405,7 +405,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk fc13303745f7a06e2eac69406ee0e7ba481f6df37a528ceb6ebacc03e310a020
+F main.mk 9109d5786263a20fcf321a77c463d39c9ac84c871ea3774b0d213d8311dda105
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -492,7 +492,7 @@ F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
 F src/tclsqlite.c 1833388c01e3b77f4c712185ee7250b9423ee0981ce6ae7e401e47db0319a696
-F src/test1.c 8ef15f7a357f85dfc41c6c748ce9c947b4f676e01bb5ae6a45bee4923dff8b51
+F src/test1.c 1879ff5095def6091f83f54c6233bc19e2b7223068fb02da41f2396d55729764
 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5
 F src/test3.c b8434949dfb8aff8dfa082c8b592109e77844c2135ed3c492113839b6956255b
 F src/test4.c 18ec393bb4d0ad1de729f0b94da7267270f3d8e6
@@ -1596,6 +1596,7 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf
 F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc
 F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa
 F test/zerodamage.test e59a56443d6298ecf7435f618f0b27654f0c849e
+F test/zipfile.test c5167d0d13ed7165b1982ac46b8aa94d65bdf41d94dd82f1d17ad2250650356c
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d
 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91
@@ -1692,7 +1693,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a532a0f6fd59e81086d46f09151ba7fb26725198231d902c71d0f95cb01dbe91
-R 7075030d907b0ffe212621eb709bb967
+P b64681a644c419bb98d00980a6cb56ef5a0aff5ef5321955631f0b4c88aac283
+R 407639481a191f472cd3497cc84b9ac5
 U dan
-Z 1c430e112380037cc2bfad44bf8be92e
+Z a9a0bf4010ca982b681b075b8e4b53ca
diff --git a/manifest.uuid b/manifest.uuid
index aaba04f0ec..d0d83a4e81 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b64681a644c419bb98d00980a6cb56ef5a0aff5ef5321955631f0b4c88aac283
\ No newline at end of file
+2dec2dec592c0726ebe87b841b9c8d493dea7074a99f278eb1bf0b744d658a9d
\ No newline at end of file
diff --git a/src/test1.c b/src/test1.c
index 446317d803..f9b3b69711 100644
--- a/src/test1.c
+++ b/src/test1.c
@@ -6960,6 +6960,7 @@ static int SQLITE_TCLAPI tclLoadStaticExtensionCmd(
   extern int sqlite3_totype_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_wholenumber_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_unionvtab_init(sqlite3*,char**,const sqlite3_api_routines*);
+  extern int sqlite3_zipfile_init(sqlite3*,char**,const sqlite3_api_routines*);
   static const struct {
     const char *zExtName;
     int (*pInit)(sqlite3*,char**,const sqlite3_api_routines*);
@@ -6981,6 +6982,7 @@ static int SQLITE_TCLAPI tclLoadStaticExtensionCmd(
     { "totype",                sqlite3_totype_init               },
     { "unionvtab",             sqlite3_unionvtab_init            },
     { "wholenumber",           sqlite3_wholenumber_init          },
+    { "zipfile",               sqlite3_zipfile_init              },
   };
   sqlite3 *db;
   const char *zName;
diff --git a/test/zipfile.test b/test/zipfile.test
new file mode 100644
index 0000000000..6c0ed4ba62
--- /dev/null
+++ b/test/zipfile.test
@@ -0,0 +1,60 @@
+# 2017 December 9
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix zipfile
+
+load_static_extension db zipfile
+
+forcedelete test.zip
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE temp.zz USING zipfile('test.zip');
+  PRAGMA table_info(zz);
+} {
+  0 name {} 0 {} 0 
+  1 mode {} 0 {} 0 
+  2 mtime {} 0 {} 0 
+  3 sz {} 0 {} 0 
+  4 data {} 0 {} 0
+  5 method {} 0 {} 0
+}
+
+do_execsql_test 1.1 {
+  INSERT INTO zz VALUES('f.txt', '-rw-r--r--', 1000000000, 5, 'abcde', 0);
+  INSERT INTO zz VALUES('g.txt', '-rw-r--r--', 1000000002, 5, '12345', 0);
+}
+
+do_execsql_test 1.2 {
+  SELECT name, mtime, data FROM zipfile('test.zip');
+} {
+  g.txt 1000000002 12345
+  f.txt 1000000000 abcde 
+}
+
+do_execsql_test 1.3 {
+  INSERT INTO zz VALUES('h.txt', 
+    '-rw-r--r--', 1000000004, 20, 'aaaaaaaaaabbbbbbbbbb', NULL
+  );
+}
+
+do_execsql_test 1.4 {
+  SELECT name, mtime, zipfile_uncompress(data, sz, method), method
+  FROM zipfile('test.zip');
+} {
+  h.txt 1000000004 aaaaaaaaaabbbbbbbbbb 8
+  f.txt 1000000000 abcde 0
+  g.txt 1000000002 12345 0
+}
+
+finish_test
+

From db0cb303ad93ed19e3f0cc3e20afb9b4fc7da22b Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 30 Dec 2017 14:26:29 +0000
Subject: [PATCH 203/488] Rearrange things a bit so that writing to a zipfile
 does not invert the order of objects it contains.

FossilOrigin-Name: f69e8194bfa7de436c96028730ebd57f186d2e6207792e172e1aa38c7f4211c9
---
 ext/misc/zipfile.c | 44 +++++++++++++++++++++++++++++++-------------
 manifest           | 14 +++++++-------
 manifest.uuid      |  2 +-
 test/zipfile.test  |  4 ++--
 4 files changed, 41 insertions(+), 23 deletions(-)

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index 5824965032..7c2a11b1ff 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -218,7 +218,7 @@ struct ZipfileCsr {
   ZipfileLFH lfh;            /* Local File Header for current entry */
   i64 iDataOff;              /* Offset in zipfile to data */
   u32 mTime;                 /* Extended mtime value */
-  int flags;
+  int flags;                 /* Flags byte (see below for bits) */
 };
 
 /*
@@ -229,9 +229,10 @@ struct ZipfileCsr {
 typedef struct ZipfileEntry ZipfileEntry;
 struct ZipfileEntry {
   char *zPath;               /* Path of zipfile entry */
+  i64 iRowid;                /* Rowid for this value if queried */
   u8 *aCdsEntry;             /* Buffer containing entire CDS entry */
   int nCdsEntry;             /* Size of buffer aCdsEntry[] in bytes */
-  ZipfileEntry *pNext;
+  ZipfileEntry *pNext;       /* Next element in in-memory CDS */
 };
 
 typedef struct ZipfileTab ZipfileTab;
@@ -241,7 +242,8 @@ struct ZipfileTab {
   u8 *aBuffer;               /* Temporary buffer used for various tasks */
 
   /* The following are used by write transactions only */
-  ZipfileEntry *pEntry;      /* Linked list of all files (if pWriteFd!=0) */
+  ZipfileEntry *pFirstEntry; /* Linked list of all files (if pWriteFd!=0) */
+  ZipfileEntry *pLastEntry;  /* Last element in pFirstEntry list */
   FILE *pWriteFd;            /* File handle open on zip archive */
   i64 szCurrent;             /* Current size of zip archive */
   i64 szOrig;                /* Size of archive at start of transaction */
@@ -830,6 +832,22 @@ static int zipfileBestIndex(
   return SQLITE_OK;
 }
 
+/*
+** Add object pNew to the end of the linked list that begins at
+** ZipfileTab.pFirstEntry and ends with pLastEntry.
+*/
+static void zipfileAddEntry(ZipfileTab *pTab, ZipfileEntry *pNew){
+  assert( (pTab->pFirstEntry==0)==(pTab->pLastEntry==0) );
+  assert( pNew->pNext==0 );
+  if( pTab->pFirstEntry==0 ){
+    pTab->pFirstEntry = pTab->pLastEntry = pNew;
+  }else{
+    assert( pTab->pLastEntry->pNext==0 );
+    pTab->pLastEntry->pNext = pNew;
+    pTab->pLastEntry = pNew;
+  }
+}
+
 static int zipfileLoadDirectory(ZipfileTab *pTab){
   ZipfileEOCD eocd;
   int rc;
@@ -871,10 +889,9 @@ static int zipfileLoadDirectory(ZipfileTab *pTab){
         pNew->zPath[nFile] = '\0';
         pNew->aCdsEntry = (u8*)&pNew->zPath[nFile+1];
         pNew->nCdsEntry = ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment;
+        pNew->pNext = 0;
         memcpy(pNew->aCdsEntry, aRec, pNew->nCdsEntry);
-
-        pNew->pNext = pTab->pEntry;
-        pTab->pEntry = pNew;
+        zipfileAddEntry(pTab, pNew);
       }
 
       iOff += ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment;
@@ -908,6 +925,7 @@ static ZipfileEntry *zipfileNewEntry(
     pNew->zPath = (char*)&pNew[1];
     pNew->aCdsEntry = (u8*)&pNew->zPath[nPath+1];
     pNew->nCdsEntry = ZIPFILE_CDS_FIXED_SZ + nPath + pCds->nExtra;
+    pNew->pNext = 0;
     memcpy(pNew->zPath, zPath, nPath+1);
 
     aWrite = pNew->aCdsEntry;
@@ -1135,8 +1153,7 @@ static int zipfileUpdate(
     if( pNew==0 ){
       rc = SQLITE_NOMEM;
     }else{
-      pNew->pNext = pTab->pEntry;
-      pTab->pEntry = pNew;
+      zipfileAddEntry(pTab, pNew);
     }
   }
 
@@ -1169,11 +1186,12 @@ static void zipfileCleanupTransaction(ZipfileTab *pTab){
   ZipfileEntry *pEntry;
   ZipfileEntry *pNext;
 
-  for(pEntry=pTab->pEntry; pEntry; pEntry=pNext){
+  for(pEntry=pTab->pFirstEntry; pEntry; pEntry=pNext){
     pNext = pEntry->pNext;
     sqlite3_free(pEntry);
   }
-  pTab->pEntry = 0;
+  pTab->pFirstEntry = 0;
+  pTab->pLastEntry = 0;
   fclose(pTab->pWriteFd);
   pTab->pWriteFd = 0;
   pTab->szCurrent = 0;
@@ -1189,13 +1207,13 @@ static int zipfileCommit(sqlite3_vtab *pVtab){
   int rc = SQLITE_OK;
   if( pTab->pWriteFd ){
     i64 iOffset = pTab->szCurrent;
-    ZipfileEntry *pEntry;
+    ZipfileEntry *p;
     ZipfileEOCD eocd;
     int nEntry = 0;
 
     /* Write out all entries */
-    for(pEntry=pTab->pEntry; rc==SQLITE_OK && pEntry; pEntry=pEntry->pNext){
-      rc = zipfileAppendData(pTab, pEntry->aCdsEntry, pEntry->nCdsEntry);
+    for(p=pTab->pFirstEntry; rc==SQLITE_OK && p; p=p->pNext){
+      rc = zipfileAppendData(pTab, p->aCdsEntry, p->nCdsEntry);
       nEntry++;
     }
 
diff --git a/manifest b/manifest
index c8433dc8ca..bc7687ee6e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\sext/misc/zipfile.c\sto\ssupport\screating\sand\sadding\sentries\sto\sexisting\nzip\sarchives.
-D 2017-12-29T20:19:03.196
+C Rearrange\sthings\sa\sbit\sso\sthat\swriting\sto\sa\szipfile\sdoes\snot\sinvert\sthe\sorder\nof\sobjects\sit\scontains.
+D 2017-12-30T14:26:29.195
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
@@ -302,7 +302,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c d9e0b0a4fdb4cb82ad22fd19d75563efdb1d6f812d62438bd30819781c282357
+F ext/misc/zipfile.c cf093f797331e51fa6c0358698bc89d08a88d06cdf11484ee4ea9d079145289b
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -1596,7 +1596,7 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf
 F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc
 F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa
 F test/zerodamage.test e59a56443d6298ecf7435f618f0b27654f0c849e
-F test/zipfile.test c5167d0d13ed7165b1982ac46b8aa94d65bdf41d94dd82f1d17ad2250650356c
+F test/zipfile.test 63059e59024cacd01ba4c32a445cf75aac21393ff4460bddd1dc1ef4b78dd8bc
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d
 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91
@@ -1693,7 +1693,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b64681a644c419bb98d00980a6cb56ef5a0aff5ef5321955631f0b4c88aac283
-R 407639481a191f472cd3497cc84b9ac5
+P 2dec2dec592c0726ebe87b841b9c8d493dea7074a99f278eb1bf0b744d658a9d
+R 64b99b9622049272ef5b18e73b616095
 U dan
-Z a9a0bf4010ca982b681b075b8e4b53ca
+Z 6ef5282c9c40bbaf3432c6295db23abc
diff --git a/manifest.uuid b/manifest.uuid
index d0d83a4e81..f242346de1 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2dec2dec592c0726ebe87b841b9c8d493dea7074a99f278eb1bf0b744d658a9d
\ No newline at end of file
+f69e8194bfa7de436c96028730ebd57f186d2e6207792e172e1aa38c7f4211c9
\ No newline at end of file
diff --git a/test/zipfile.test b/test/zipfile.test
index 6c0ed4ba62..a0860f40bb 100644
--- a/test/zipfile.test
+++ b/test/zipfile.test
@@ -37,8 +37,8 @@ do_execsql_test 1.1 {
 do_execsql_test 1.2 {
   SELECT name, mtime, data FROM zipfile('test.zip');
 } {
-  g.txt 1000000002 12345
   f.txt 1000000000 abcde 
+  g.txt 1000000002 12345
 }
 
 do_execsql_test 1.3 {
@@ -51,9 +51,9 @@ do_execsql_test 1.4 {
   SELECT name, mtime, zipfile_uncompress(data, sz, method), method
   FROM zipfile('test.zip');
 } {
-  h.txt 1000000004 aaaaaaaaaabbbbbbbbbb 8
   f.txt 1000000000 abcde 0
   g.txt 1000000002 12345 0
+  h.txt 1000000004 aaaaaaaaaabbbbbbbbbb 8
 }
 
 finish_test

From 0cde0c62b1e53c01d72a9a9227010e6afc4032dc Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 30 Dec 2017 18:32:27 +0000
Subject: [PATCH 204/488] Have zipfile support DELETE commands.

FossilOrigin-Name: 01d4e866fb7b01aeada537d41c4a47747c7810e2028f51077ee5b8b78c348954
---
 ext/misc/zipfile.c | 279 ++++++++++++++++++++++++++++-----------------
 manifest           |  14 +--
 manifest.uuid      |   2 +-
 test/zipfile.test  |  21 +++-
 4 files changed, 202 insertions(+), 114 deletions(-)

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index 7c2a11b1ff..9d0865377d 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -201,40 +201,45 @@ struct ZipfileLFH {
   u16 nExtra;
 };
 
-/* 
-** Cursor type for recursively iterating through a directory structure.
-*/
-typedef struct ZipfileCsr ZipfileCsr;
-
-struct ZipfileCsr {
-  sqlite3_vtab_cursor base;  /* Base class - must be first */
-  i64 iRowid;                /* Rowid for current row */
-  FILE *pFile;               /* Zip file */
-  i64 nByte;                 /* Size of zip file on disk */
-  int bEof;                  /* True when at EOF */
-  i64 iNextOff;              /* Offset of next record in central directory */
-  ZipfileEOCD eocd;          /* Parse of central directory record */
-  ZipfileCDS cds;            /* Central Directory Structure */
-  ZipfileLFH lfh;            /* Local File Header for current entry */
-  i64 iDataOff;              /* Offset in zipfile to data */
-  u32 mTime;                 /* Extended mtime value */
-  int flags;                 /* Flags byte (see below for bits) */
-};
-
-/*
-** Values for ZipfileCsr.flags.
-*/
-#define ZIPFILE_MTIME_VALID 0x0001
-
 typedef struct ZipfileEntry ZipfileEntry;
 struct ZipfileEntry {
   char *zPath;               /* Path of zipfile entry */
   i64 iRowid;                /* Rowid for this value if queried */
   u8 *aCdsEntry;             /* Buffer containing entire CDS entry */
   int nCdsEntry;             /* Size of buffer aCdsEntry[] in bytes */
+  int bDeleted;              /* True if entry has been deleted */
   ZipfileEntry *pNext;       /* Next element in in-memory CDS */
 };
 
+/* 
+** Cursor type for recursively iterating through a directory structure.
+*/
+typedef struct ZipfileCsr ZipfileCsr;
+struct ZipfileCsr {
+  sqlite3_vtab_cursor base;  /* Base class - must be first */
+  int bEof;                  /* True when at EOF */
+
+  /* Used outside of write transactions */
+  FILE *pFile;               /* Zip file */
+  i64 iNextOff;              /* Offset of next record in central directory */
+  ZipfileEOCD eocd;          /* Parse of central directory record */
+
+  /* Used inside write transactions */
+  ZipfileEntry *pCurrent;
+
+  ZipfileCDS cds;            /* Central Directory Structure */
+  ZipfileLFH lfh;            /* Local File Header for current entry */
+  i64 iDataOff;              /* Offset in zipfile to data */
+  u32 mTime;                 /* Extended mtime value */
+  int flags;                 /* Flags byte (see below for bits) */
+  ZipfileCsr *pCsrNext;      /* Next cursor on same virtual table */
+};
+
+/*
+** Values for ZipfileCsr.flags.
+*/
+#define ZIPFILE_MTIME_VALID 0x0001
+
 typedef struct ZipfileTab ZipfileTab;
 struct ZipfileTab {
   sqlite3_vtab base;         /* Base class - must be first */
@@ -242,6 +247,7 @@ struct ZipfileTab {
   u8 *aBuffer;               /* Temporary buffer used for various tasks */
 
   /* The following are used by write transactions only */
+  ZipfileCsr *pCsrList;      /* List of cursors */
   ZipfileEntry *pFirstEntry; /* Linked list of all files (if pWriteFd!=0) */
   ZipfileEntry *pLastEntry;  /* Last element in pFirstEntry list */
   FILE *pWriteFd;            /* File handle open on zip archive */
@@ -335,9 +341,16 @@ static int zipfileOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){
 ** by zipfileOpen().
 */
 static void zipfileResetCursor(ZipfileCsr *pCsr){
+  ZipfileTab *pTab = (ZipfileTab*)(pCsr->base.pVtab);
+  ZipfileCsr **pp;
+
+  /* Remove this cursor from the ZipfileTab.pCsrList list. */
+  for(pp=&pTab->pCsrList; *pp; pp=&((*pp)->pCsrNext)){
+    if( *pp==pCsr ) *pp = pCsr->pCsrNext;
+  }
+
   sqlite3_free(pCsr->cds.zFile);
   pCsr->cds.zFile = 0;
-  pCsr->iRowid = 0;
   pCsr->bEof = 0;
   if( pCsr->pFile ){
     fclose(pCsr->pFile);
@@ -438,18 +451,25 @@ static u8* zipfileCsrBuffer(ZipfileCsr *pCsr){
 
 static int zipfileReadCDS(ZipfileCsr *pCsr){
   char **pzErr = &pCsr->base.pVtab->zErrMsg;
-  u8 *aRead = zipfileCsrBuffer(pCsr);
-  int rc;
+  u8 *aRead;
+  int rc = SQLITE_OK;
 
   sqlite3_free(pCsr->cds.zFile);
   pCsr->cds.zFile = 0;
 
-  rc = zipfileReadData(
-      pCsr->pFile, aRead, ZIPFILE_CDS_FIXED_SZ, pCsr->iNextOff, pzErr
-  );
+  if( pCsr->pCurrent==0 ){
+    aRead = zipfileCsrBuffer(pCsr);
+    rc = zipfileReadData(
+        pCsr->pFile, aRead, ZIPFILE_CDS_FIXED_SZ, pCsr->iNextOff, pzErr
+    );
+  }else{
+    aRead = pCsr->pCurrent->aCdsEntry;
+  }
+
   if( rc==SQLITE_OK ){
     u32 sig = zipfileRead32(aRead);
     if( sig!=ZIPFILE_SIGNATURE_CDS ){
+      assert( pCsr->pCurrent==0 );
       zipfileSetErrmsg(pCsr,"failed to read CDS at offset %lld",pCsr->iNextOff);
       rc = SQLITE_ERROR;
     }else{
@@ -463,7 +483,9 @@ static int zipfileReadCDS(ZipfileCsr *pCsr){
       pCsr->cds.crc32 = zipfileRead32(aRead);
       pCsr->cds.szCompressed = zipfileRead32(aRead);
       pCsr->cds.szUncompressed = zipfileRead32(aRead);
-      assert( aRead==zipfileCsrBuffer(pCsr)+ZIPFILE_CDS_NFILE_OFF );
+      assert( pCsr->pCurrent 
+           || aRead==zipfileCsrBuffer(pCsr)+ZIPFILE_CDS_NFILE_OFF 
+      );
       pCsr->cds.nFile = zipfileRead16(aRead);
       pCsr->cds.nExtra = zipfileRead16(aRead);
       pCsr->cds.nComment = zipfileRead16(aRead);
@@ -471,13 +493,16 @@ static int zipfileReadCDS(ZipfileCsr *pCsr){
       pCsr->cds.iInternalAttr = zipfileRead16(aRead);
       pCsr->cds.iExternalAttr = zipfileRead32(aRead);
       pCsr->cds.iOffset = zipfileRead32(aRead);
+      assert( pCsr->pCurrent 
+           || aRead==zipfileCsrBuffer(pCsr)+ZIPFILE_CDS_FIXED_SZ
+      );
 
-      assert( aRead==zipfileCsrBuffer(pCsr)+ZIPFILE_CDS_FIXED_SZ );
-
-      nRead = pCsr->cds.nFile + pCsr->cds.nExtra;
-      aRead = zipfileCsrBuffer(pCsr);
-      pCsr->iNextOff += ZIPFILE_CDS_FIXED_SZ;
-      rc = zipfileReadData(pCsr->pFile, aRead, nRead, pCsr->iNextOff, pzErr);
+      if( pCsr->pCurrent==0 ){
+        nRead = pCsr->cds.nFile + pCsr->cds.nExtra;
+        aRead = zipfileCsrBuffer(pCsr);
+        pCsr->iNextOff += ZIPFILE_CDS_FIXED_SZ;
+        rc = zipfileReadData(pCsr->pFile, aRead, nRead, pCsr->iNextOff, pzErr);
+      }
 
       if( rc==SQLITE_OK ){
         pCsr->cds.zFile = sqlite3_mprintf("%.*s", (int)pCsr->cds.nFile, aRead);
@@ -522,13 +547,19 @@ static int zipfileReadCDS(ZipfileCsr *pCsr){
   return rc;
 }
 
+static FILE *zipfileGetFd(ZipfileCsr *pCsr){
+  if( pCsr->pFile ) return pCsr->pFile;
+  return ((ZipfileTab*)(pCsr->base.pVtab))->pWriteFd;
+}
+
 static int zipfileReadLFH(ZipfileCsr *pCsr){
+  FILE *pFile = zipfileGetFd(pCsr);
   char **pzErr = &pCsr->base.pVtab->zErrMsg;
   static const int szFix = ZIPFILE_LFH_FIXED_SZ;
   u8 *aRead = zipfileCsrBuffer(pCsr);
   int rc;
 
-  rc = zipfileReadData(pCsr->pFile, aRead, szFix, pCsr->cds.iOffset, pzErr);
+  rc = zipfileReadData(pFile, aRead, szFix, pCsr->cds.iOffset, pzErr);
   if( rc==SQLITE_OK ){
     u32 sig = zipfileRead32(aRead);
     if( sig!=ZIPFILE_SIGNATURE_LFH ){
@@ -561,19 +592,31 @@ static int zipfileReadLFH(ZipfileCsr *pCsr){
 */
 static int zipfileNext(sqlite3_vtab_cursor *cur){
   ZipfileCsr *pCsr = (ZipfileCsr*)cur;
-  i64 iEof = pCsr->eocd.iOffset + pCsr->eocd.nSize;
   int rc = SQLITE_OK;
+  pCsr->flags = 0;
 
-  if( pCsr->iNextOff>=iEof ){
-    pCsr->bEof = 1;
+  if( pCsr->pCurrent==0 ){
+    i64 iEof = pCsr->eocd.iOffset + pCsr->eocd.nSize;
+    if( pCsr->iNextOff>=iEof ){
+      pCsr->bEof = 1;
+    }
   }else{
-    pCsr->iRowid++;
-    pCsr->flags = 0;
+    assert( pCsr->pFile==0 );
+    do {
+      pCsr->pCurrent = pCsr->pCurrent->pNext;
+    }while( pCsr->pCurrent && pCsr->pCurrent->bDeleted );
+    if( pCsr->pCurrent==0 ){
+      pCsr->bEof = 1;
+    }
+  }
+
+  if( pCsr->bEof==0 ){
     rc = zipfileReadCDS(pCsr);
     if( rc==SQLITE_OK ){
       rc = zipfileReadLFH(pCsr);
     }
   }
+
   return rc;
 }
 
@@ -658,7 +701,8 @@ static int zipfileColumn(
         if( aBuf==0 ){
           rc = SQLITE_NOMEM;
         }else{
-          rc = zipfileReadData(pCsr->pFile, aBuf, sz, pCsr->iDataOff,
+          FILE *pFile = zipfileGetFd(pCsr);
+          rc = zipfileReadData(pFile, aBuf, sz, pCsr->iDataOff,
               &pCsr->base.pVtab->zErrMsg
           );
         }
@@ -678,13 +722,15 @@ static int zipfileColumn(
 }
 
 /*
-** Return the rowid for the current row. In this implementation, the
-** first row returned is assigned rowid value 1, and each subsequent
-** row a value 1 more than that of the previous.
+** Return the rowid for the current row.
 */
 static int zipfileRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
   ZipfileCsr *pCsr = (ZipfileCsr*)cur;
-  *pRowid = pCsr->iRowid;
+  if( pCsr->pCurrent ){
+    *pRowid = pCsr->pCurrent->iRowid;
+  }else{
+    *pRowid = pCsr->cds.iOffset;
+  }
   return SQLITE_OK;
 }
 
@@ -771,32 +817,39 @@ static int zipfileFilter(
 
   zipfileResetCursor(pCsr);
 
-  assert( idxNum==argc && (idxNum==0 || idxNum==1) );
-  if( idxNum==0 ){
-    ZipfileTab *pTab = (ZipfileTab*)cur->pVtab;
+  if( pTab->zFile ){
     zFile = pTab->zFile;
-    if( zFile==0 ){
-      /* Error. This is an eponymous virtual table and the user has not 
-      ** supplied a file name. */
-      zipfileSetErrmsg(pCsr, "table function zipfile() requires an argument");
-      return SQLITE_ERROR;
-    }
+  }else if( idxNum==0 ){
+    /* Error. This is an eponymous virtual table and the user has not 
+    ** supplied a file name. */
+    zipfileSetErrmsg(pCsr, "table function zipfile() requires an argument");
+    return SQLITE_ERROR;
   }else{
     zFile = (const char*)sqlite3_value_text(argv[0]);
   }
-  pCsr->pFile = fopen(zFile, "rb");
-  if( pCsr->pFile==0 ){
-    zipfileSetErrmsg(pCsr, "cannot open file: %s", zFile);
-    rc = SQLITE_ERROR;
-  }else{
-    rc = zipfileReadEOCD(pTab, pCsr->pFile, &pCsr->eocd);
-    if( rc==SQLITE_OK ){
-      pCsr->iNextOff = pCsr->eocd.iOffset;
-      rc = zipfileNext(cur);
-    }else if( rc==SQLITE_EMPTY ){
-      rc = SQLITE_OK;
-      pCsr->bEof = 1;
+
+  if( pTab->pWriteFd==0 ){
+    pCsr->pFile = fopen(zFile, "rb");
+    if( pCsr->pFile==0 ){
+      zipfileSetErrmsg(pCsr, "cannot open file: %s", zFile);
+      rc = SQLITE_ERROR;
+    }else{
+      rc = zipfileReadEOCD(pTab, pCsr->pFile, &pCsr->eocd);
+      if( rc==SQLITE_OK ){
+        pCsr->iNextOff = pCsr->eocd.iOffset;
+        rc = zipfileNext(cur);
+      }else if( rc==SQLITE_EMPTY ){
+        rc = SQLITE_OK;
+        pCsr->bEof = 1;
+      }
     }
+  }else{
+    ZipfileEntry e;
+    memset(&e, 0, sizeof(e));
+    e.pNext = pTab->pFirstEntry;
+    pCsr->pCurrent = &e;
+    rc = zipfileNext(cur);
+    assert( pCsr->pCurrent!=&e );
   }
 
   return rc;
@@ -840,9 +893,11 @@ static void zipfileAddEntry(ZipfileTab *pTab, ZipfileEntry *pNew){
   assert( (pTab->pFirstEntry==0)==(pTab->pLastEntry==0) );
   assert( pNew->pNext==0 );
   if( pTab->pFirstEntry==0 ){
+    pNew->iRowid = 1;
     pTab->pFirstEntry = pTab->pLastEntry = pNew;
   }else{
     assert( pTab->pLastEntry->pNext==0 );
+    pNew->iRowid = pTab->pLastEntry->iRowid+1;
     pTab->pLastEntry->pNext = pNew;
     pTab->pLastEntry = pNew;
   }
@@ -884,12 +939,12 @@ static int zipfileLoadDirectory(ZipfileTab *pTab){
       if( pNew==0 ){
         rc = SQLITE_NOMEM;
       }else{
+        memset(pNew, 0, sizeof(ZipfileEntry));
         pNew->zPath = (char*)&pNew[1];
         memcpy(pNew->zPath, &aBuf[ZIPFILE_CDS_FIXED_SZ], nFile);
         pNew->zPath[nFile] = '\0';
         pNew->aCdsEntry = (u8*)&pNew->zPath[nFile+1];
         pNew->nCdsEntry = ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment;
-        pNew->pNext = 0;
         memcpy(pNew->aCdsEntry, aRec, pNew->nCdsEntry);
         zipfileAddEntry(pTab, pNew);
       }
@@ -922,10 +977,10 @@ static ZipfileEntry *zipfileNewEntry(
   );
 
   if( pNew ){
+    memset(pNew, 0, sizeof(ZipfileEntry));
     pNew->zPath = (char*)&pNew[1];
     pNew->aCdsEntry = (u8*)&pNew->zPath[nPath+1];
     pNew->nCdsEntry = ZIPFILE_CDS_FIXED_SZ + nPath + pCds->nExtra;
-    pNew->pNext = 0;
     memcpy(pNew->zPath, zPath, nPath+1);
 
     aWrite = pNew->aCdsEntry;
@@ -1057,38 +1112,19 @@ static int zipfileUpdate(
   u8 *pFree = 0;                  /* Free this */
   ZipfileCDS cds;                 /* New Central Directory Structure entry */
 
+  assert( pTab->zFile );
+  assert( pTab->pWriteFd );
+
   if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
-    pTab->base.zErrMsg = sqlite3_mprintf(
-        "zipfile: UPDATE/DELETE not yet supported"
-    );
-    return SQLITE_ERROR;
-  }
-
-  /* This table is only writable if a default archive path was specified 
-  ** as part of the CREATE VIRTUAL TABLE statement. */
-  if( pTab->zFile==0 ){
-    pTab->base.zErrMsg = sqlite3_mprintf(
-        "zipfile: writing requires a default archive"
-    );
-    return SQLITE_ERROR;
-  }
-
-  /* Open a write fd on the file. Also load the entire central directory
-  ** structure into memory. During the transaction any new file data is 
-  ** appended to the archive file, but the central directory is accumulated
-  ** in main-memory until the transaction is committed.  */
-  if( pTab->pWriteFd==0 ){
-    pTab->pWriteFd = fopen(pTab->zFile, "ab+");
-    if( pTab->pWriteFd==0 ){
-      pTab->base.zErrMsg = sqlite3_mprintf(
-          "zipfile: failed to open file %s for writing", pTab->zFile
-      );
-      return SQLITE_ERROR;
+    i64 iDelete = sqlite3_value_int64(apVal[0]);
+    ZipfileEntry *p;
+    for(p=pTab->pFirstEntry; p; p=p->pNext){
+      if( p->iRowid==iDelete ){
+        p->bDeleted = 1;
+        break;
+      }
     }
-    fseek(pTab->pWriteFd, 0, SEEK_END);
-    pTab->szCurrent = pTab->szOrig = (i64)ftell(pTab->pWriteFd);
-    rc = zipfileLoadDirectory(pTab);
-    if( rc!=SQLITE_OK ) return rc;
+    if( nVal==1 ) return SQLITE_OK;
   }
 
   zPath = (const char*)sqlite3_value_text(apVal[2]);
@@ -1199,7 +1235,41 @@ static void zipfileCleanupTransaction(ZipfileTab *pTab){
 }
 
 static int zipfileBegin(sqlite3_vtab *pVtab){
-  return SQLITE_OK;
+  ZipfileTab *pTab = (ZipfileTab*)pVtab;
+  int rc = SQLITE_OK;
+
+  assert( pTab->pWriteFd==0 );
+
+  /* This table is only writable if a default archive path was specified 
+  ** as part of the CREATE VIRTUAL TABLE statement. */
+  if( pTab->zFile==0 ){
+    pTab->base.zErrMsg = sqlite3_mprintf(
+        "zipfile: writing requires a default archive"
+    );
+    return SQLITE_ERROR;
+  }
+
+  /* Open a write fd on the file. Also load the entire central directory
+  ** structure into memory. During the transaction any new file data is 
+  ** appended to the archive file, but the central directory is accumulated
+  ** in main-memory until the transaction is committed.  */
+  pTab->pWriteFd = fopen(pTab->zFile, "ab+");
+  if( pTab->pWriteFd==0 ){
+    pTab->base.zErrMsg = sqlite3_mprintf(
+        "zipfile: failed to open file %s for writing", pTab->zFile
+    );
+    rc = SQLITE_ERROR;
+  }else{
+    fseek(pTab->pWriteFd, 0, SEEK_END);
+    pTab->szCurrent = pTab->szOrig = (i64)ftell(pTab->pWriteFd);
+    rc = zipfileLoadDirectory(pTab);
+  }
+
+  if( rc!=SQLITE_OK ){
+    zipfileCleanupTransaction(pTab);
+  }
+
+  return rc;
 }
 
 static int zipfileCommit(sqlite3_vtab *pVtab){
@@ -1211,8 +1281,9 @@ static int zipfileCommit(sqlite3_vtab *pVtab){
     ZipfileEOCD eocd;
     int nEntry = 0;
 
-    /* Write out all entries */
+    /* Write out all undeleted entries */
     for(p=pTab->pFirstEntry; rc==SQLITE_OK && p; p=p->pNext){
+      if( p->bDeleted ) continue;
       rc = zipfileAppendData(pTab, p->aCdsEntry, p->nCdsEntry);
       nEntry++;
     }
diff --git a/manifest b/manifest
index bc7687ee6e..6a47f6372c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Rearrange\sthings\sa\sbit\sso\sthat\swriting\sto\sa\szipfile\sdoes\snot\sinvert\sthe\sorder\nof\sobjects\sit\scontains.
-D 2017-12-30T14:26:29.195
+C Have\szipfile\ssupport\sDELETE\scommands.
+D 2017-12-30T18:32:27.545
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5
@@ -302,7 +302,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c cf093f797331e51fa6c0358698bc89d08a88d06cdf11484ee4ea9d079145289b
+F ext/misc/zipfile.c 2df8f94003903fe3cc104b807418c54e68040964d4319c522ac2f485152f5abd
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -1596,7 +1596,7 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf
 F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc
 F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa
 F test/zerodamage.test e59a56443d6298ecf7435f618f0b27654f0c849e
-F test/zipfile.test 63059e59024cacd01ba4c32a445cf75aac21393ff4460bddd1dc1ef4b78dd8bc
+F test/zipfile.test d4f342ca918fd4d7981a12c1523f5041074cc592f25fecce4ee11446cc984f56
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d
 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91
@@ -1693,7 +1693,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2dec2dec592c0726ebe87b841b9c8d493dea7074a99f278eb1bf0b744d658a9d
-R 64b99b9622049272ef5b18e73b616095
+P f69e8194bfa7de436c96028730ebd57f186d2e6207792e172e1aa38c7f4211c9
+R 145246df93a24ee9820cce32c6b5afb4
 U dan
-Z 6ef5282c9c40bbaf3432c6295db23abc
+Z 640cac8c8ac89e53412f930b0b057c7a
diff --git a/manifest.uuid b/manifest.uuid
index f242346de1..bc6078e242 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f69e8194bfa7de436c96028730ebd57f186d2e6207792e172e1aa38c7f4211c9
\ No newline at end of file
+01d4e866fb7b01aeada537d41c4a47747c7810e2028f51077ee5b8b78c348954
\ No newline at end of file
diff --git a/test/zipfile.test b/test/zipfile.test
index a0860f40bb..3f7b34d55d 100644
--- a/test/zipfile.test
+++ b/test/zipfile.test
@@ -29,13 +29,15 @@ do_execsql_test 1.0 {
   5 method {} 0 {} 0
 }
 
-do_execsql_test 1.1 {
+do_execsql_test 1.1.1 {
   INSERT INTO zz VALUES('f.txt', '-rw-r--r--', 1000000000, 5, 'abcde', 0);
+}
+do_execsql_test 1.1.2 {
   INSERT INTO zz VALUES('g.txt', '-rw-r--r--', 1000000002, 5, '12345', 0);
 }
 
 do_execsql_test 1.2 {
-  SELECT name, mtime, data FROM zipfile('test.zip');
+  SELECT name, mtime, data FROM zipfile('test.zip')
 } {
   f.txt 1000000000 abcde 
   g.txt 1000000002 12345
@@ -56,5 +58,20 @@ do_execsql_test 1.4 {
   h.txt 1000000004 aaaaaaaaaabbbbbbbbbb 8
 }
 
+do_execsql_test 1.5.1 {
+  BEGIN;
+    INSERT INTO zz VALUES('i.txt', '-rw-r--r--', 1000000006, 5, 'zxcvb', 0);
+    SELECT name FROM zz;
+  COMMIT;
+} {f.txt g.txt h.txt i.txt}
+do_execsql_test 1.5.2 {
+  SELECT name FROM zz;
+} {f.txt g.txt h.txt i.txt}
+
+do_execsql_test 1.6.0 {
+  DELETE FROM zz WHERE name='g.txt';
+  SELECT name FROM zz;
+} {f.txt h.txt i.txt}
+
 finish_test
 

From 470fc9ddaabf6475d38eb1ca2071279271404d19 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 1 Jan 2018 19:33:12 +0000
Subject: [PATCH 205/488] Enable the introspection pragmas for command-line
 shell builds.

FossilOrigin-Name: 0b04223f256f5c1e06fa6e899727815ebf650f2dfcf8f36d6cbfb6cb62d4ead8
---
 Makefile.in   |  1 +
 Makefile.msc  |  2 +-
 main.mk       |  1 +
 manifest      | 17 ++++++++---------
 manifest.uuid |  2 +-
 5 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/Makefile.in b/Makefile.in
index 3c9e7b3a96..e3dfb01baa 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -583,6 +583,7 @@ SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB
 SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB
 SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
 SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC
+SHELL_OPT += -DSQLITE_INTROSPECTION_PRAGMAS
 FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
 FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
diff --git a/Makefile.msc b/Makefile.msc
index 27eb23a240..2f038ab1c1 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -1513,7 +1513,7 @@ FUZZDATA = \
 !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_DBSTAT_VTAB
-SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC -DSQLITE_INTROSPECTION_PRAGMAS
 !ENDIF
 
 # <<mark>>
diff --git a/main.mk b/main.mk
index b46ffa1a52..85588cd01a 100644
--- a/main.mk
+++ b/main.mk
@@ -510,6 +510,7 @@ SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB
 SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB
 SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
 SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC
+SHELL_OPT += -DSQLITE_INTROSPECTION_PRAGMAS
 FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5
 FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
diff --git a/manifest b/manifest
index 682ec0eee7..aede5dedf5 100644
--- a/manifest
+++ b/manifest
@@ -1,8 +1,8 @@
-C Add\ssupport\sfor\sthe\ssqlite_unsupported_offset()\sSQL\sfunction\sif\sand\sonly\nif\scompiled\susing\s-DSQLITE_ENABLE_OFFSET_SQL_FUNC.\s\sUse\sthat\sdefinition\nwhen\scompiling\sthe\scommand-line\sshell.
-D 2017-12-29T17:21:21.319
-F Makefile.in 804c347948b0ad5a962aea3e59413e897227e0173d0e76910f11f473e6c7ae9b
+C Enable\sthe\sintrospection\spragmas\sfor\scommand-line\sshell\sbuilds.
+D 2018-01-01T19:33:12.194
+F Makefile.in 1b11037c5ed3399a79433cc82c59b5e36a7b3a3e4e195bb27640d0d2145e03e1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 57dcce0595d4a0c4b94b6f6ba732b8e33540358adf5fb8e1aa8389d34a9677e8
+F Makefile.msc f68b4f9b83cfeb057b6265e0288ad653f319e2ceacca731e0f22e19617829a89
 F README.md eeae1e552f93ef72ef7c5b8f6647b368a001c28820ad1df179d3dae602bef681
 F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -401,7 +401,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk b5f569262e4f8111bb8cf6110784b8210fe14ea45f6c7ea50fb4230ebe7a4880
+F main.mk 59744c818d349d170ff56cdbdfb5af0e0a2029db18ce2824fcd1b0a3fa317d84
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -1688,8 +1688,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P f4349c0c26611de8a7d5beb99431a575cf531cdeb0ca2413efabcf0a61e6f424 9406c0a685fd5ff3516a66402b0514a1440652822a5eecf0b7af85929f3079e8
-R 35346abc344e9d51558b9d8d5c8f2d13
-T +closed 9406c0a685fd5ff3516a66402b0514a1440652822a5eecf0b7af85929f3079e8
+P 4f1f1f521ae6da96b899e10bfeff6bc1ab7a45de0705076febaae20b441f48c6
+R c9a22e6c8d8b0c74afd8179337182631
 U drh
-Z b5b643a0e7da393bb8589918154fc3c1
+Z ad901b13942be312e9eb272912737269
diff --git a/manifest.uuid b/manifest.uuid
index 1c26686ec7..86076c7330 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4f1f1f521ae6da96b899e10bfeff6bc1ab7a45de0705076febaae20b441f48c6
\ No newline at end of file
+0b04223f256f5c1e06fa6e899727815ebf650f2dfcf8f36d6cbfb6cb62d4ead8
\ No newline at end of file

From c22b716e316af27b96e3717bd350c828841319b1 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 1 Jan 2018 20:11:23 +0000
Subject: [PATCH 206/488] Minor simplification to the ".schema" logic in the
 command-line shell.

FossilOrigin-Name: add45c47288248ca3745e0fceb5e0d9062a852d173d93ff2ff1a0f8fdf3237dc
---
 manifest       | 12 ++++++------
 manifest.uuid  |  2 +-
 src/shell.c.in | 24 +++++++-----------------
 3 files changed, 14 insertions(+), 24 deletions(-)

diff --git a/manifest b/manifest
index aede5dedf5..ac3df26d19 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Enable\sthe\sintrospection\spragmas\sfor\scommand-line\sshell\sbuilds.
-D 2018-01-01T19:33:12.194
+C Minor\ssimplification\sto\sthe\s".schema"\slogic\sin\sthe\scommand-line\sshell.
+D 2018-01-01T20:11:23.297
 F Makefile.in 1b11037c5ed3399a79433cc82c59b5e36a7b3a3e4e195bb27640d0d2145e03e1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc f68b4f9b83cfeb057b6265e0288ad653f319e2ceacca731e0f22e19617829a89
@@ -479,7 +479,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in f3ec8f90dd698ea98781a90642c91eacbc24f4e55bb551c7b2762000d3ef55dc
+F src/shell.c.in a3f30523bbf436e886831c9f867419edb957dec60a2b5d70d2a071298206e37d
 F src/sqlite.h.in b4dc75265ed04b98e2184011a7dd0054ce2137ff84867a6be8b4f3bdfbc03d30
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1688,7 +1688,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 4f1f1f521ae6da96b899e10bfeff6bc1ab7a45de0705076febaae20b441f48c6
-R c9a22e6c8d8b0c74afd8179337182631
+P 0b04223f256f5c1e06fa6e899727815ebf650f2dfcf8f36d6cbfb6cb62d4ead8
+R 2690a690f31be0b2523230103d44d965
 U drh
-Z ad901b13942be312e9eb272912737269
+Z 17d3ed202c106ebc7057d8eede16f971
diff --git a/manifest.uuid b/manifest.uuid
index 86076c7330..953c3c5663 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0b04223f256f5c1e06fa6e899727815ebf650f2dfcf8f36d6cbfb6cb62d4ead8
\ No newline at end of file
+add45c47288248ca3745e0fceb5e0d9062a852d173d93ff2ff1a0f8fdf3237dc
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 155ca708a5..7b994336ea 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -5334,33 +5334,23 @@ static int do_meta_command(char *zLine, ShellState *p){
     if( nArg==2 && azArg[1][0]!='-' ){
       int i;
       for(i=0; azArg[1][i]; i++) azArg[1][i] = ToLower(azArg[1][i]);
-      if( strcmp(azArg[1],"sqlite_master")==0 ){
+      if( strcmp(azArg[1],"sqlite_master")==0
+       || strcmp(azArg[1],"sqlite_temp_master")==0
+      ){
         char *new_argv[2], *new_colv[2];
-        new_argv[0] = "CREATE TABLE sqlite_master (\n"
+        new_argv[0] = sqlite3_mprintf(
+                      "CREATE TABLE %s (\n"
                       "  type text,\n"
                       "  name text,\n"
                       "  tbl_name text,\n"
                       "  rootpage integer,\n"
                       "  sql text\n"
-                      ")";
-        new_argv[1] = 0;
-        new_colv[0] = "sql";
-        new_colv[1] = 0;
-        callback(&data, 1, new_argv, new_colv);
-        rc = SQLITE_OK;
-      }else if( strcmp(azArg[1],"sqlite_temp_master")==0 ){
-        char *new_argv[2], *new_colv[2];
-        new_argv[0] = "CREATE TEMP TABLE sqlite_temp_master (\n"
-                      "  type text,\n"
-                      "  name text,\n"
-                      "  tbl_name text,\n"
-                      "  rootpage integer,\n"
-                      "  sql text\n"
-                      ")";
+                      ")", azArg[1]);
         new_argv[1] = 0;
         new_colv[0] = "sql";
         new_colv[1] = 0;
         callback(&data, 1, new_argv, new_colv);
+        sqlite3_free(new_argv[0]);
         rc = SQLITE_OK;
       }else{
         zDiv = "(";

From ceba792a2fb5c242432ab9e4d32277762e1476bb Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 1 Jan 2018 21:28:25 +0000
Subject: [PATCH 207/488] In the output of ".schema", show the column names of
 virtual tables and views in a separate comment.

FossilOrigin-Name: 2234a87fa905312b23f46d52e06cff7cacbf23b187e16c4398a42e6bdae0ee9f
---
 manifest          |  16 +++---
 manifest.uuid     |   2 +-
 src/shell.c.in    | 127 +++++++++++++++++++++++++++++++---------------
 test/pragma5.test |   2 +-
 test/shell1.test  |   6 ++-
 5 files changed, 99 insertions(+), 54 deletions(-)

diff --git a/manifest b/manifest
index ac3df26d19..b8745cef92 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Minor\ssimplification\sto\sthe\s".schema"\slogic\sin\sthe\scommand-line\sshell.
-D 2018-01-01T20:11:23.297
+C In\sthe\soutput\sof\s".schema",\sshow\sthe\scolumn\snames\sof\svirtual\stables\sand\nviews\sin\sa\sseparate\scomment.
+D 2018-01-01T21:28:25.565
 F Makefile.in 1b11037c5ed3399a79433cc82c59b5e36a7b3a3e4e195bb27640d0d2145e03e1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc f68b4f9b83cfeb057b6265e0288ad653f319e2ceacca731e0f22e19617829a89
@@ -479,7 +479,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in a3f30523bbf436e886831c9f867419edb957dec60a2b5d70d2a071298206e37d
+F src/shell.c.in f151a1181ed4da01c48ecbbeeb4e102d298dd239102c252f92cc331c85a1436b
 F src/sqlite.h.in b4dc75265ed04b98e2184011a7dd0054ce2137ff84867a6be8b4f3bdfbc03d30
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1127,7 +1127,7 @@ F test/pragma.test 7c8cfc328a1717a95663cf8edb06c52ddfeaf97bb0aee69ae7457132e8d39
 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f
 F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed
 F test/pragma4.test 3046501bee2f652dc2a4f9c87781e2741361d6864439c8381aba6c3b774b335c
-F test/pragma5.test fd517f42ee847e126afbbbd9fd0fb9e5a4a61a962496a350adb8a22583fbdc37
+F test/pragma5.test 824ce6ced5d6b7ec71abe37fc6005ff836fe39d638273dc5192b39864b9ee983
 F test/pragmafault.test 275edaf3161771d37de60e5c2b412627ac94cef11739236bec12ed1258b240f8
 F test/printf.test b3ff34e73d59124140eaf89f7672e21bc2ca5fcc
 F test/printf2.test 9e6db85f81c63f2367c34a9d7db384088bd374ad
@@ -1213,7 +1213,7 @@ F test/sharedA.test 0cdf1a76dfa00e6beee66af5b534b1e8df2720f5
 F test/sharedB.test 16cc7178e20965d75278f410943109b77b2e645e
 F test/shared_err.test 32634e404a3317eeb94abc7a099c556a346fdb8fb3858dbe222a4cbb8926a939
 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304
-F test/shell1.test f78ea0e2637ae9f497cb41206f6346f983052ffc7a359e664aaf6847fc704ea7
+F test/shell1.test 3f23fe267fc591dc6a004a163b719a7a6509e4b46915bab8781e32e854c0a0a4
 F test/shell2.test e242a9912f44f4c23c3d1d802a83e934e84c853b
 F test/shell3.test 9b95ba643eaa228376f06a898fb410ee9b6e57c1
 F test/shell4.test 89ad573879a745974ff2df20ff97c5d6ffffbd5d
@@ -1688,7 +1688,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0b04223f256f5c1e06fa6e899727815ebf650f2dfcf8f36d6cbfb6cb62d4ead8
-R 2690a690f31be0b2523230103d44d965
+P add45c47288248ca3745e0fceb5e0d9062a852d173d93ff2ff1a0f8fdf3237dc
+R dcc0e8850f6f2345f6beb728eed33eca
 U drh
-Z 17d3ed202c106ebc7057d8eede16f971
+Z b932e40f742685fed6e88a07d4738316
diff --git a/manifest.uuid b/manifest.uuid
index 953c3c5663..6be3b12497 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-add45c47288248ca3745e0fceb5e0d9062a852d173d93ff2ff1a0f8fdf3237dc
\ No newline at end of file
+2234a87fa905312b23f46d52e06cff7cacbf23b187e16c4398a42e6bdae0ee9f
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 7b994336ea..a1e4362f58 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -729,6 +729,33 @@ static char quoteChar(const char *zName){
   return 0;
 }
 
+/*
+** Construct a fake CREATE TABLE statement (or at least the part that comes
+** after the "CREATE TABLE" keywords) that describes the columns of
+** the view, virtual table, or table valued function zName in zSchema.
+*/
+static char *shellFakeCrTab(
+  sqlite3 *db,            /* The database connection containing the vtab */
+  const char *zSchema,    /* Schema of the database holding the vtab */
+  const char *zName       /* The name of the virtual table */
+){
+  sqlite3_stmt *pStmt = 0;
+  char *zSql;
+  char *z = 0;
+
+  zSql = sqlite3_mprintf("SELECT group_concat(name,',')"
+                         " FROM pragma_table_info(%Q,%Q);",
+                         zName, zSchema);
+  sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+  sqlite3_free(zSql);
+  if( sqlite3_step(pStmt)==SQLITE_ROW ){
+    z = sqlite3_mprintf("/* %s.%s(%s) */",
+           zSchema ? zSchema : "main", zName, sqlite3_column_text(pStmt, 0));
+  }
+  sqlite3_finalize(pStmt);
+  return z;
+}
+
 /*
 ** SQL function:  shell_add_schema(S,X)
 **
@@ -764,20 +791,29 @@ static void shellAddSchemaName(
   int i = 0;
   const char *zIn = (const char*)sqlite3_value_text(apVal[0]);
   const char *zSchema = (const char*)sqlite3_value_text(apVal[1]);
-  assert( nVal==2 );
+  sqlite3 *db = sqlite3_context_db_handle(pCtx);
   if( zIn!=0 && strncmp(zIn, "CREATE ", 7)==0 ){
     for(i=0; i<(int)(sizeof(aPrefix)/sizeof(aPrefix[0])); i++){
       int n = strlen30(aPrefix[i]);
       if( strncmp(zIn+7, aPrefix[i], n)==0 && zIn[n+7]==' ' ){
-        char cQuote = quoteChar(zSchema);
-        char *z;
-        if( cQuote ){
-         z = sqlite3_mprintf("%.*s \"%w\".%s", n+7, zIn, zSchema, zIn+n+8);
-        }else{
-          z = sqlite3_mprintf("%.*s %s.%s", n+7, zIn, zSchema, zIn+n+8);
+        char *z = 0;
+        if( zSchema ){
+          char cQuote = quoteChar(zSchema);
+          if( cQuote && sqlite3_stricmp(zSchema,"temp")!=0 ){
+            z = sqlite3_mprintf("%.*s \"%w\".%s", n+7, zIn, zSchema, zIn+n+8);
+          }else{
+            z = sqlite3_mprintf("%.*s %s.%s", n+7, zIn, zSchema, zIn+n+8);
+          }
+        }
+        if( aPrefix[i][0]=='V' ){
+          const char *zName = (const char*)sqlite3_value_text(apVal[2]);
+          if( z==0 ) z = sqlite3_mprintf("%s", zIn);
+          z = sqlite3_mprintf("%z\n%z", z,  shellFakeCrTab(db, zSchema, zName));
+        }
+        if( z ){
+          sqlite3_result_text(pCtx, z, -1, sqlite3_free);
+          return;
         }
-        sqlite3_result_text(pCtx, z, -1, sqlite3_free);
-        return;
       }
     }
   }
@@ -2998,7 +3034,7 @@ static void open_db(ShellState *p, int keepAlive){
     sqlite3_fileio_init(p->db, 0, 0);
     sqlite3_shathree_init(p->db, 0, 0);
     sqlite3_completion_init(p->db, 0, 0);
-    sqlite3_create_function(p->db, "shell_add_schema", 2, SQLITE_UTF8, 0,
+    sqlite3_create_function(p->db, "shell_add_schema", 3, SQLITE_UTF8, 0,
                             shellAddSchemaName, 0, 0);
   }
 }
@@ -5319,23 +5355,32 @@ static int do_meta_command(char *zLine, ShellState *p){
     ShellState data;
     char *zErrMsg = 0;
     const char *zDiv = 0;
+    const char *zName = 0;
     int iSchema = 0;
+    int bDebug = 0;
+    int ii;
 
     open_db(p, 0);
     memcpy(&data, p, sizeof(data));
     data.showHeader = 0;
     data.cMode = data.mode = MODE_Semi;
     initText(&sSelect);
-    if( nArg>=2 && optionMatch(azArg[1], "indent") ){
-      data.cMode = data.mode = MODE_Pretty;
-      nArg--;
-      if( nArg==2 ) azArg[1] = azArg[2];
+    for(ii=1; ii<nArg; ii++){
+      if( optionMatch(azArg[ii],"indent") ){
+        data.cMode = data.mode = MODE_Pretty;
+      }else if( optionMatch(azArg[ii],"debug") ){
+        bDebug = 1;
+      }else if( zName==0 ){
+        zName = azArg[ii];
+      }else{
+        raw_printf(stderr, "Usage: .schema ?--indent? ?LIKE-PATTERN?\n");
+        rc = 1;
+        goto meta_command_exit;
+      }
     }
-    if( nArg==2 && azArg[1][0]!='-' ){
-      int i;
-      for(i=0; azArg[1][i]; i++) azArg[1][i] = ToLower(azArg[1][i]);
-      if( strcmp(azArg[1],"sqlite_master")==0
-       || strcmp(azArg[1],"sqlite_temp_master")==0
+    if( zName!=0 ){
+      if( sqlite3_stricmp(zName,"sqlite_master")==0
+       || sqlite3_stricmp(zName,"sqlite_temp_master")==0
       ){
         char *new_argv[2], *new_colv[2];
         new_argv[0] = sqlite3_mprintf(
@@ -5345,7 +5390,7 @@ static int do_meta_command(char *zLine, ShellState *p){
                       "  tbl_name text,\n"
                       "  rootpage integer,\n"
                       "  sql text\n"
-                      ")", azArg[1]);
+                      ")", zName);
         new_argv[1] = 0;
         new_colv[0] = "sql";
         new_colv[1] = 0;
@@ -5355,12 +5400,8 @@ static int do_meta_command(char *zLine, ShellState *p){
       }else{
         zDiv = "(";
       }
-    }else if( nArg==1 ){
+    }else if( zName==0 ){
       zDiv = "(";
-    }else{
-      raw_printf(stderr, "Usage: .schema ?--indent? ?LIKE-PATTERN?\n");
-      rc = 1;
-      goto meta_command_exit;
     }
     if( zDiv ){
       sqlite3_stmt *pStmt = 0;
@@ -5380,39 +5421,41 @@ static int do_meta_command(char *zLine, ShellState *p){
         sqlite3_snprintf(sizeof(zScNum), zScNum, "%d", ++iSchema);
         appendText(&sSelect, zDiv, 0);
         zDiv = " UNION ALL ";
-        if( strcmp(zDb, "main")!=0 ){
-          appendText(&sSelect, "SELECT shell_add_schema(sql,", 0);
+        appendText(&sSelect, "SELECT shell_add_schema(sql,", 0);
+        if( sqlite3_stricmp(zDb, "main")!=0 ){
           appendText(&sSelect, zDb, '"');
-          appendText(&sSelect, ") AS sql, type, tbl_name, name, rowid,", 0);
-          appendText(&sSelect, zScNum, 0);
-          appendText(&sSelect, " AS snum, ", 0);
-          appendText(&sSelect, zDb, '\'');
-          appendText(&sSelect, " AS sname FROM ", 0);
-          appendText(&sSelect, zDb, '"');
-          appendText(&sSelect, ".sqlite_master", 0);
         }else{
-          appendText(&sSelect, "SELECT sql, type, tbl_name, name, rowid, ", 0);
-          appendText(&sSelect, zScNum, 0);
-          appendText(&sSelect, " AS snum, 'main' AS sname FROM sqlite_master",0);
+          appendText(&sSelect, "NULL", 0);
         }
+        appendText(&sSelect, ",name) AS sql, type, tbl_name, name, rowid,", 0);
+        appendText(&sSelect, zScNum, 0);
+        appendText(&sSelect, " AS snum, ", 0);
+        appendText(&sSelect, zDb, '\'');
+        appendText(&sSelect, " AS sname FROM ", 0);
+        appendText(&sSelect, zDb, '"');
+        appendText(&sSelect, ".sqlite_master", 0);
       }
       sqlite3_finalize(pStmt);
       appendText(&sSelect, ") WHERE ", 0);
-      if( nArg>1 ){
-        char *zQarg = sqlite3_mprintf("%Q", azArg[1]);
-        if( strchr(azArg[1], '.') ){
+      if( zName ){
+        char *zQarg = sqlite3_mprintf("%Q", zName);
+        if( strchr(zName, '.') ){
           appendText(&sSelect, "lower(printf('%s.%s',sname,tbl_name))", 0);
         }else{
           appendText(&sSelect, "lower(tbl_name)", 0);
         }
-        appendText(&sSelect, strchr(azArg[1], '*') ? " GLOB " : " LIKE ", 0);
+        appendText(&sSelect, strchr(zName, '*') ? " GLOB " : " LIKE ", 0);
         appendText(&sSelect, zQarg, 0);
         appendText(&sSelect, " AND ", 0);
         sqlite3_free(zQarg);
       }
       appendText(&sSelect, "type!='meta' AND sql IS NOT NULL"
                            " ORDER BY snum, rowid", 0);
-      rc = sqlite3_exec(p->db, sSelect.z, callback, &data, &zErrMsg);
+      if( bDebug ){
+        utf8_printf(p->out, "SQL: %s;\n", sSelect.z);
+      }else{
+        rc = sqlite3_exec(p->db, sSelect.z, callback, &data, &zErrMsg);
+      }
       freeText(&sSelect);
     }
     if( zErrMsg ){
diff --git a/test/pragma5.test b/test/pragma5.test
index 625ab92012..d2c58000cf 100644
--- a/test/pragma5.test
+++ b/test/pragma5.test
@@ -34,7 +34,7 @@ do_execsql_test 1.0 {
   1 builtin {} 0 {} 0
 }
 do_execsql_test 1.1 {
-  SELECT * FROM pragma_function_list WHERE name='upper'
+  SELECT * FROM pragma_function_list WHERE name='upper' AND builtin
 } {upper 1}
 do_execsql_test 1.2 {
   SELECT * FROM pragma_function_list WHERE name LIKE 'exter%';
diff --git a/test/shell1.test b/test/shell1.test
index ddd72c7180..c2f39435b8 100644
--- a/test/shell1.test
+++ b/test/shell1.test
@@ -581,8 +581,10 @@ do_test shell1-3.21.4 {
   }
   catchcmd "test.db" ".schema"
 } {0 {CREATE TABLE t1(x);
-CREATE VIEW v2 AS SELECT x+1 AS y FROM t1;
-CREATE VIEW v1 AS SELECT y+1 FROM v2;}}
+CREATE VIEW v2 AS SELECT x+1 AS y FROM t1
+/* main.v2(y) */;
+CREATE VIEW v1 AS SELECT y+1 FROM v2
+/* main.v1(y+1) */;}}
 db eval {DROP VIEW v1; DROP VIEW v2; DROP TABLE t1;}
 }
 

From 1d315cf7b86052178e4537e57e249499fa0e6187 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 1 Jan 2018 21:49:43 +0000
Subject: [PATCH 208/488] Improved name quoting and escaping in the auxiliary
 column info section of the ".schema" output for views and virtual tables.

FossilOrigin-Name: d64b14e37d9624bf5d86059ddd091170d8e6d341a8043f84548b9b3dbb96a908
---
 manifest         | 14 +++++++-------
 manifest.uuid    |  2 +-
 src/shell.c.in   | 34 +++++++++++++++++++++++++---------
 test/shell1.test |  4 ++--
 4 files changed, 35 insertions(+), 19 deletions(-)

diff --git a/manifest b/manifest
index b8745cef92..71d3959150 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\sthe\soutput\sof\s".schema",\sshow\sthe\scolumn\snames\sof\svirtual\stables\sand\nviews\sin\sa\sseparate\scomment.
-D 2018-01-01T21:28:25.565
+C Improved\sname\squoting\sand\sescaping\sin\sthe\sauxiliary\scolumn\sinfo\ssection\nof\sthe\s".schema"\soutput\sfor\sviews\sand\svirtual\stables.
+D 2018-01-01T21:49:43.315
 F Makefile.in 1b11037c5ed3399a79433cc82c59b5e36a7b3a3e4e195bb27640d0d2145e03e1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc f68b4f9b83cfeb057b6265e0288ad653f319e2ceacca731e0f22e19617829a89
@@ -479,7 +479,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in f151a1181ed4da01c48ecbbeeb4e102d298dd239102c252f92cc331c85a1436b
+F src/shell.c.in e0d3ef676265771a001446906765dd89617df2cece3249b08b1dcd24ba296f63
 F src/sqlite.h.in b4dc75265ed04b98e2184011a7dd0054ce2137ff84867a6be8b4f3bdfbc03d30
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1213,7 +1213,7 @@ F test/sharedA.test 0cdf1a76dfa00e6beee66af5b534b1e8df2720f5
 F test/sharedB.test 16cc7178e20965d75278f410943109b77b2e645e
 F test/shared_err.test 32634e404a3317eeb94abc7a099c556a346fdb8fb3858dbe222a4cbb8926a939
 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304
-F test/shell1.test 3f23fe267fc591dc6a004a163b719a7a6509e4b46915bab8781e32e854c0a0a4
+F test/shell1.test 6d69e08039aea13f2c42749f162fe05eab7b5c93729f31d49d7d27cf36226e5a
 F test/shell2.test e242a9912f44f4c23c3d1d802a83e934e84c853b
 F test/shell3.test 9b95ba643eaa228376f06a898fb410ee9b6e57c1
 F test/shell4.test 89ad573879a745974ff2df20ff97c5d6ffffbd5d
@@ -1688,7 +1688,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P add45c47288248ca3745e0fceb5e0d9062a852d173d93ff2ff1a0f8fdf3237dc
-R dcc0e8850f6f2345f6beb728eed33eca
+P 2234a87fa905312b23f46d52e06cff7cacbf23b187e16c4398a42e6bdae0ee9f
+R ca3aef501320cce9e28be13c23be1622
 U drh
-Z b932e40f742685fed6e88a07d4738316
+Z 2aaca99c49ba21a51639392b75414605
diff --git a/manifest.uuid b/manifest.uuid
index 6be3b12497..687c35b1d6 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2234a87fa905312b23f46d52e06cff7cacbf23b187e16c4398a42e6bdae0ee9f
\ No newline at end of file
+d64b14e37d9624bf5d86059ddd091170d8e6d341a8043f84548b9b3dbb96a908
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index a1e4362f58..4861d4b506 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -741,19 +741,34 @@ static char *shellFakeCrTab(
 ){
   sqlite3_stmt *pStmt = 0;
   char *zSql;
-  char *z = 0;
+  ShellText s;
+  char cQuote;
+  char *zDiv = "(";
 
-  zSql = sqlite3_mprintf("SELECT group_concat(name,',')"
-                         " FROM pragma_table_info(%Q,%Q);",
-                         zName, zSchema);
+  zSql = sqlite3_mprintf("PRAGMA \"%w\".table_info=%Q;",
+                         zSchema ? zSchema : "main", zName);
   sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
   sqlite3_free(zSql);
-  if( sqlite3_step(pStmt)==SQLITE_ROW ){
-    z = sqlite3_mprintf("/* %s.%s(%s) */",
-           zSchema ? zSchema : "main", zName, sqlite3_column_text(pStmt, 0));
+  initText(&s);
+  if( zSchema ){
+    cQuote = quoteChar(zSchema);
+    if( cQuote && sqlite3_stricmp(zSchema,"temp")==0 ) cQuote = 0;
+    appendText(&s, zSchema, cQuote);
+    appendText(&s, ".", 0);
   }
+  cQuote = quoteChar(zName);
+  appendText(&s, zName, cQuote);
+  while( sqlite3_step(pStmt)==SQLITE_ROW ){
+    const char *zCol = (const char*)sqlite3_column_text(pStmt, 1);
+    appendText(&s, zDiv, 0);
+    zDiv = ",";
+    cQuote = quoteChar(zCol);
+    appendText(&s, zCol, cQuote);
+  }
+  appendText(&s, ")", 0);
+
   sqlite3_finalize(pStmt);
-  return z;
+  return s.z;
 }
 
 /*
@@ -808,7 +823,8 @@ static void shellAddSchemaName(
         if( aPrefix[i][0]=='V' ){
           const char *zName = (const char*)sqlite3_value_text(apVal[2]);
           if( z==0 ) z = sqlite3_mprintf("%s", zIn);
-          z = sqlite3_mprintf("%z\n%z", z,  shellFakeCrTab(db, zSchema, zName));
+          z = sqlite3_mprintf("%z\n/* %z */", z,
+                              shellFakeCrTab(db, zSchema, zName));
         }
         if( z ){
           sqlite3_result_text(pCtx, z, -1, sqlite3_free);
diff --git a/test/shell1.test b/test/shell1.test
index c2f39435b8..0d03c64f78 100644
--- a/test/shell1.test
+++ b/test/shell1.test
@@ -582,9 +582,9 @@ do_test shell1-3.21.4 {
   catchcmd "test.db" ".schema"
 } {0 {CREATE TABLE t1(x);
 CREATE VIEW v2 AS SELECT x+1 AS y FROM t1
-/* main.v2(y) */;
+/* v2(y) */;
 CREATE VIEW v1 AS SELECT y+1 FROM v2
-/* main.v1(y+1) */;}}
+/* v1("y+1") */;}}
 db eval {DROP VIEW v1; DROP VIEW v2; DROP TABLE t1;}
 }
 

From 667a2a25bad684983c4e647fdfa4acfcfab7f67d Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 2 Jan 2018 00:04:37 +0000
Subject: [PATCH 209/488] The ".schema" command in the command-line shell now
 shows the structure of table-valued functions and eponymous virtual tables if
 they are named on the ".schema" command line.  Example:  ".schema sql%" shows
 the structure of the "sqlite_dbstat" and "sqlite_stmt" virtual tables.

FossilOrigin-Name: f80f6651df0b2843c6c9619a8f3e05c56cd50363402800a2e166e6eb664f7768
---
 manifest       | 12 ++++-----
 manifest.uuid  |  2 +-
 src/shell.c.in | 70 +++++++++++++++++++++++++++++++++++---------------
 3 files changed, 57 insertions(+), 27 deletions(-)

diff --git a/manifest b/manifest
index 71d3959150..cf7032a36b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improved\sname\squoting\sand\sescaping\sin\sthe\sauxiliary\scolumn\sinfo\ssection\nof\sthe\s".schema"\soutput\sfor\sviews\sand\svirtual\stables.
-D 2018-01-01T21:49:43.315
+C The\s".schema"\scommand\sin\sthe\scommand-line\sshell\snow\sshows\sthe\sstructure\sof\ntable-valued\sfunctions\sand\seponymous\svirtual\stables\sif\sthey\sare\snamed\son\sthe\n".schema"\scommand\sline.\s\sExample:\s\s".schema\ssql%"\sshows\sthe\sstructure\sof\nthe\s"sqlite_dbstat"\sand\s"sqlite_stmt"\svirtual\stables.
+D 2018-01-02T00:04:37.286
 F Makefile.in 1b11037c5ed3399a79433cc82c59b5e36a7b3a3e4e195bb27640d0d2145e03e1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc f68b4f9b83cfeb057b6265e0288ad653f319e2ceacca731e0f22e19617829a89
@@ -479,7 +479,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in e0d3ef676265771a001446906765dd89617df2cece3249b08b1dcd24ba296f63
+F src/shell.c.in 6b1ec35f31058505650a79807c1facaec399a9a910fb317a40b9e267cb4bb312
 F src/sqlite.h.in b4dc75265ed04b98e2184011a7dd0054ce2137ff84867a6be8b4f3bdfbc03d30
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1688,7 +1688,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2234a87fa905312b23f46d52e06cff7cacbf23b187e16c4398a42e6bdae0ee9f
-R ca3aef501320cce9e28be13c23be1622
+P d64b14e37d9624bf5d86059ddd091170d8e6d341a8043f84548b9b3dbb96a908
+R 58693511e34cddcc08538ba153bd6535
 U drh
-Z 2aaca99c49ba21a51639392b75414605
+Z 527d9b7c91bfab98cfe62f567d2e9f84
diff --git a/manifest.uuid b/manifest.uuid
index 687c35b1d6..aed8d4df66 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d64b14e37d9624bf5d86059ddd091170d8e6d341a8043f84548b9b3dbb96a908
\ No newline at end of file
+f80f6651df0b2843c6c9619a8f3e05c56cd50363402800a2e166e6eb664f7768
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 4861d4b506..e0e998632d 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -730,11 +730,10 @@ static char quoteChar(const char *zName){
 }
 
 /*
-** Construct a fake CREATE TABLE statement (or at least the part that comes
-** after the "CREATE TABLE" keywords) that describes the columns of
-** the view, virtual table, or table valued function zName in zSchema.
+** Construct a fake object name and column list to describe the structure
+** of the view, virtual table, or table valued function zSchema.zName.
 */
-static char *shellFakeCrTab(
+static char *shellFakeSchema(
   sqlite3 *db,            /* The database connection containing the vtab */
   const char *zSchema,    /* Schema of the database holding the vtab */
   const char *zName       /* The name of the virtual table */
@@ -744,6 +743,7 @@ static char *shellFakeCrTab(
   ShellText s;
   char cQuote;
   char *zDiv = "(";
+  int nRow = 0;
 
   zSql = sqlite3_mprintf("PRAGMA \"%w\".table_info=%Q;",
                          zSchema ? zSchema : "main", zName);
@@ -760,17 +760,40 @@ static char *shellFakeCrTab(
   appendText(&s, zName, cQuote);
   while( sqlite3_step(pStmt)==SQLITE_ROW ){
     const char *zCol = (const char*)sqlite3_column_text(pStmt, 1);
+    nRow++;
     appendText(&s, zDiv, 0);
     zDiv = ",";
     cQuote = quoteChar(zCol);
     appendText(&s, zCol, cQuote);
   }
   appendText(&s, ")", 0);
-
   sqlite3_finalize(pStmt);
+  if( nRow==0 ){
+    freeText(&s);
+    s.z = 0;
+  }
   return s.z;
 }
 
+/*
+** SQL function:  shell_module_schema(X)
+**
+** Return a fake schema for the table-valued function or eponymous virtual
+** table X.
+*/
+static void shellModuleSchema(
+  sqlite3_context *pCtx,
+  int nVal,
+  sqlite3_value **apVal
+){
+  const char *zName = (const char*)sqlite3_value_text(apVal[0]);
+  char *zFake = shellFakeSchema(sqlite3_context_db_handle(pCtx), 0, zName);
+  if( zFake ){
+    sqlite3_result_text(pCtx, sqlite3_mprintf("/* %z */", zFake),
+                        -1, sqlite3_free);
+  }
+}
+
 /*
 ** SQL function:  shell_add_schema(S,X)
 **
@@ -806,12 +829,14 @@ static void shellAddSchemaName(
   int i = 0;
   const char *zIn = (const char*)sqlite3_value_text(apVal[0]);
   const char *zSchema = (const char*)sqlite3_value_text(apVal[1]);
+  const char *zName = (const char*)sqlite3_value_text(apVal[2]);
   sqlite3 *db = sqlite3_context_db_handle(pCtx);
   if( zIn!=0 && strncmp(zIn, "CREATE ", 7)==0 ){
     for(i=0; i<(int)(sizeof(aPrefix)/sizeof(aPrefix[0])); i++){
       int n = strlen30(aPrefix[i]);
       if( strncmp(zIn+7, aPrefix[i], n)==0 && zIn[n+7]==' ' ){
         char *z = 0;
+        char *zFake = 0;
         if( zSchema ){
           char cQuote = quoteChar(zSchema);
           if( cQuote && sqlite3_stricmp(zSchema,"temp")!=0 ){
@@ -820,11 +845,15 @@ static void shellAddSchemaName(
             z = sqlite3_mprintf("%.*s %s.%s", n+7, zIn, zSchema, zIn+n+8);
           }
         }
-        if( aPrefix[i][0]=='V' ){
-          const char *zName = (const char*)sqlite3_value_text(apVal[2]);
-          if( z==0 ) z = sqlite3_mprintf("%s", zIn);
-          z = sqlite3_mprintf("%z\n/* %z */", z,
-                              shellFakeCrTab(db, zSchema, zName));
+        if( zName
+         && aPrefix[i][0]=='V'
+         && (zFake = shellFakeSchema(db, zSchema, zName))!=0
+        ){
+          if( z==0 ){
+            z = sqlite3_mprintf("%s\n/* %z */", zIn, zFake);
+          }else{
+            z = sqlite3_mprintf("%z\n/* %z */", z, zFake);
+          }
         }
         if( z ){
           sqlite3_result_text(pCtx, z, -1, sqlite3_free);
@@ -3052,6 +3081,8 @@ static void open_db(ShellState *p, int keepAlive){
     sqlite3_completion_init(p->db, 0, 0);
     sqlite3_create_function(p->db, "shell_add_schema", 3, SQLITE_UTF8, 0,
                             shellAddSchemaName, 0, 0);
+    sqlite3_create_function(p->db, "shell_module_schema", 1, SQLITE_UTF8, 0,
+                            shellModuleSchema, 0, 0);
   }
 }
 
@@ -5370,7 +5401,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     ShellText sSelect;
     ShellState data;
     char *zErrMsg = 0;
-    const char *zDiv = 0;
+    const char *zDiv = "(";
     const char *zName = 0;
     int iSchema = 0;
     int bDebug = 0;
@@ -5395,9 +5426,8 @@ static int do_meta_command(char *zLine, ShellState *p){
       }
     }
     if( zName!=0 ){
-      if( sqlite3_stricmp(zName,"sqlite_master")==0
-       || sqlite3_stricmp(zName,"sqlite_temp_master")==0
-      ){
+      int isMaster = sqlite3_strlike(zName, "sqlite_master", 0)==0;
+      if( isMaster || sqlite3_strlike(zName,"sqlite_temp_master",0)==0 ){
         char *new_argv[2], *new_colv[2];
         new_argv[0] = sqlite3_mprintf(
                       "CREATE TABLE %s (\n"
@@ -5406,18 +5436,13 @@ static int do_meta_command(char *zLine, ShellState *p){
                       "  tbl_name text,\n"
                       "  rootpage integer,\n"
                       "  sql text\n"
-                      ")", zName);
+                      ")", isMaster ? "sqlite_master" : "sqlite_temp_master");
         new_argv[1] = 0;
         new_colv[0] = "sql";
         new_colv[1] = 0;
         callback(&data, 1, new_argv, new_colv);
         sqlite3_free(new_argv[0]);
-        rc = SQLITE_OK;
-      }else{
-        zDiv = "(";
       }
-    }else if( zName==0 ){
-      zDiv = "(";
     }
     if( zDiv ){
       sqlite3_stmt *pStmt = 0;
@@ -5452,6 +5477,11 @@ static int do_meta_command(char *zLine, ShellState *p){
         appendText(&sSelect, ".sqlite_master", 0);
       }
       sqlite3_finalize(pStmt);
+      if( zName ){
+        appendText(&sSelect,
+           " UNION ALL SELECT shell_module_schema(name),"
+           " 'table', name, name, name, 9e+99, 'main' FROM pragma_module_list", 0);
+      }
       appendText(&sSelect, ") WHERE ", 0);
       if( zName ){
         char *zQarg = sqlite3_mprintf("%Q", zName);

From cde7b77932eade5cd5ed44fec353c78997a91ada Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 2 Jan 2018 12:50:40 +0000
Subject: [PATCH 210/488] Do not attempt to use pragma_module_list in the
 command-line shell unless it has been compiled with
 -DSQLITE_INTROSPECTION_PRAGMAS.

FossilOrigin-Name: 39ca5fcd31c902cbade4da05125b0dbf2bc5e2b8901af1645f113c9d3dbc8209
---
 manifest       | 12 ++++++------
 manifest.uuid  |  2 +-
 src/shell.c.in |  2 ++
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index cf7032a36b..df6e2b86dd 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C The\s".schema"\scommand\sin\sthe\scommand-line\sshell\snow\sshows\sthe\sstructure\sof\ntable-valued\sfunctions\sand\seponymous\svirtual\stables\sif\sthey\sare\snamed\son\sthe\n".schema"\scommand\sline.\s\sExample:\s\s".schema\ssql%"\sshows\sthe\sstructure\sof\nthe\s"sqlite_dbstat"\sand\s"sqlite_stmt"\svirtual\stables.
-D 2018-01-02T00:04:37.286
+C Do\snot\sattempt\sto\suse\spragma_module_list\sin\sthe\scommand-line\sshell\sunless\s\nit\shas\sbeen\scompiled\swith\s-DSQLITE_INTROSPECTION_PRAGMAS.
+D 2018-01-02T12:50:40.391
 F Makefile.in 1b11037c5ed3399a79433cc82c59b5e36a7b3a3e4e195bb27640d0d2145e03e1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc f68b4f9b83cfeb057b6265e0288ad653f319e2ceacca731e0f22e19617829a89
@@ -479,7 +479,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in 6b1ec35f31058505650a79807c1facaec399a9a910fb317a40b9e267cb4bb312
+F src/shell.c.in a418ddceef7a2789f18bdc2bcdd481b2562fe4a7754b8009c8dd33d5a67da332
 F src/sqlite.h.in b4dc75265ed04b98e2184011a7dd0054ce2137ff84867a6be8b4f3bdfbc03d30
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1688,7 +1688,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d64b14e37d9624bf5d86059ddd091170d8e6d341a8043f84548b9b3dbb96a908
-R 58693511e34cddcc08538ba153bd6535
+P f80f6651df0b2843c6c9619a8f3e05c56cd50363402800a2e166e6eb664f7768
+R 6990ef46fea99f4195aa8641bdb529e8
 U drh
-Z 527d9b7c91bfab98cfe62f567d2e9f84
+Z 071c90bd35c02d763da61d6f60dc05ae
diff --git a/manifest.uuid b/manifest.uuid
index aed8d4df66..4f0c82c41f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f80f6651df0b2843c6c9619a8f3e05c56cd50363402800a2e166e6eb664f7768
\ No newline at end of file
+39ca5fcd31c902cbade4da05125b0dbf2bc5e2b8901af1645f113c9d3dbc8209
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index e0e998632d..4c9e60769a 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -5477,11 +5477,13 @@ static int do_meta_command(char *zLine, ShellState *p){
         appendText(&sSelect, ".sqlite_master", 0);
       }
       sqlite3_finalize(pStmt);
+#ifdef SQLITE_INTROSPECTION_PRAGMAS
       if( zName ){
         appendText(&sSelect,
            " UNION ALL SELECT shell_module_schema(name),"
            " 'table', name, name, name, 9e+99, 'main' FROM pragma_module_list", 0);
       }
+#endif
       appendText(&sSelect, ") WHERE ", 0);
       if( zName ){
         char *zQarg = sqlite3_mprintf("%Q", zName);

From 9199ac1048e4739c3605f411868c7a5cf3959d4b Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 2 Jan 2018 13:48:48 +0000
Subject: [PATCH 211/488] Updates to the sqlite3_file_control() documentation. 
 No changes to code.

FossilOrigin-Name: 240e32ab1f2a18e3c9b92f577b1cc8f8ecb4c68c44eac863d859491e042cb72a
---
 manifest        | 12 ++++++------
 manifest.uuid   |  2 +-
 src/sqlite.h.in |  6 +++---
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index df6e2b86dd..8af4fc7c93 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Do\snot\sattempt\sto\suse\spragma_module_list\sin\sthe\scommand-line\sshell\sunless\s\nit\shas\sbeen\scompiled\swith\s-DSQLITE_INTROSPECTION_PRAGMAS.
-D 2018-01-02T12:50:40.391
+C Updates\sto\sthe\ssqlite3_file_control()\sdocumentation.\s\sNo\schanges\sto\scode.
+D 2018-01-02T13:48:48.784
 F Makefile.in 1b11037c5ed3399a79433cc82c59b5e36a7b3a3e4e195bb27640d0d2145e03e1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc f68b4f9b83cfeb057b6265e0288ad653f319e2ceacca731e0f22e19617829a89
@@ -480,7 +480,7 @@ F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
 F src/shell.c.in a418ddceef7a2789f18bdc2bcdd481b2562fe4a7754b8009c8dd33d5a67da332
-F src/sqlite.h.in b4dc75265ed04b98e2184011a7dd0054ce2137ff84867a6be8b4f3bdfbc03d30
+F src/sqlite.h.in c597ba5d11666bb2d0179a173cd25e44f0b0333f9e18ce60b07eac8bdc3de67f
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
 F src/sqliteInt.h fd8702c65994d7de3e2d8f7d85d958731da1ed29476571fdfa2290fd8ec0bf80
@@ -1688,7 +1688,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P f80f6651df0b2843c6c9619a8f3e05c56cd50363402800a2e166e6eb664f7768
-R 6990ef46fea99f4195aa8641bdb529e8
+P 39ca5fcd31c902cbade4da05125b0dbf2bc5e2b8901af1645f113c9d3dbc8209
+R 1ce62a1c4d9b56a67e8d1f1aed222781
 U drh
-Z 071c90bd35c02d763da61d6f60dc05ae
+Z d9ace2c20dd904e2b4c8de8f33d7aa5e
diff --git a/manifest.uuid b/manifest.uuid
index 4f0c82c41f..3dd953071f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-39ca5fcd31c902cbade4da05125b0dbf2bc5e2b8901af1645f113c9d3dbc8209
\ No newline at end of file
+240e32ab1f2a18e3c9b92f577b1cc8f8ecb4c68c44eac863d859491e042cb72a
\ No newline at end of file
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 7e5fccdae6..a6d328360a 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -6971,9 +6971,9 @@ sqlite3_mutex *sqlite3_db_mutex(sqlite3*);
 ** the xFileControl method.  ^The return value of the xFileControl
 ** method becomes the return value of this routine.
 **
-** ^The SQLITE_FCNTL_FILE_POINTER value for the op parameter causes
+** ^The [SQLITE_FCNTL_FILE_POINTER] value for the op parameter causes
 ** a pointer to the underlying [sqlite3_file] object to be written into
-** the space pointed to by the 4th parameter.  ^The SQLITE_FCNTL_FILE_POINTER
+** the space pointed to by the 4th parameter.  ^The [SQLITE_FCNTL_FILE_POINTER]
 ** case is a short-circuit path which does not actually invoke the
 ** underlying sqlite3_io_methods.xFileControl method.
 **
@@ -6985,7 +6985,7 @@ sqlite3_mutex *sqlite3_db_mutex(sqlite3*);
 ** an incorrect zDbName and an SQLITE_ERROR return from the underlying
 ** xFileControl method.
 **
-** See also: [SQLITE_FCNTL_LOCKSTATE]
+** See also: [file control opcodes]
 */
 int sqlite3_file_control(sqlite3*, const char *zDbName, int op, void*);
 

From 595a0e2a4a5d6781abaebd4f0da2720568e1c5bb Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 2 Jan 2018 16:02:50 +0000
Subject: [PATCH 212/488] Enhance the memvfs extension so that it can be
 read/write.

FossilOrigin-Name: 04c9197d589666299aef86ee6a56df63448c050274c9fba4af94f932752be237
---
 ext/misc/memvfs.c | 161 +++++++++++++++++++++++++++++++++++-----------
 manifest          |  12 ++--
 manifest.uuid     |   2 +-
 3 files changed, 129 insertions(+), 46 deletions(-)

diff --git a/ext/misc/memvfs.c b/ext/misc/memvfs.c
index 62a8a033d7..27a61c35e4 100644
--- a/ext/misc/memvfs.c
+++ b/ext/misc/memvfs.c
@@ -10,23 +10,33 @@
 **
 ******************************************************************************
 **
-** This is an in-memory read-only VFS implementation.  The application
-** supplies a block of memory which is the database file, and this VFS
-** uses that block of memory.
+** This is an in-memory VFS implementation.  The application supplies
+** a chunk of memory to hold the database file.
 **
-** Because there is no place to store journals and no good way to lock
-** the "file", this VFS is read-only.
+** Because there is place to store a rollback or wal journal, the database
+** must use one of journal_mode=MEMORY or journal_mode=NONE.
 **
 ** USAGE:
 **
-**    sqlite3_open_v2("file:/whatever?ptr=0xf05538&sz=14336", &db,
-**                    SQLITE_OPEN_READONLY | SQLITE_OPEN_URI,
+**    sqlite3_open_v2("file:/whatever?ptr=0xf05538&sz=14336&max=65536", &db,
+**                    SQLITE_OPEN_READWRITE | SQLITE_OPEN_URI,
 **                    "memvfs");
 **
-** The ptr= and sz= query parameters are required or the open will fail.
-** The ptr= parameter gives the memory address of the buffer holding the
-** read-only database and sz= gives the size of the database.  The parameter
-** values may be in hexadecimal or decimal.  The filename is ignored.
+** These are the query parameters:
+**
+**    ptr=          The address of the memory buffer that holds the database.
+**
+**    sz=           The current size the database file
+**
+**    maxsz=        The maximum size of the database.  In other words, the
+**                  amount of space allocated for the ptr= buffer.
+**
+**    freeonclose=  If true, then sqlite3_free() is called on the ptr=
+**                  value when the connection closes.
+**
+** The ptr= and sz= query parameters are required.  If maxsz= is omitted,
+** then it defaults to the sz= value.  Parameter values can be in either
+** decimal or hexadecimal.  The filename in the URI is ignored.
 */
 #include <sqlite3ext.h>
 SQLITE_EXTENSION_INIT1
@@ -49,7 +59,9 @@ typedef struct MemFile MemFile;
 struct MemFile {
   sqlite3_file base;              /* IO methods */
   sqlite3_int64 sz;               /* Size of the file */
+  sqlite3_int64 szMax;            /* Space allocated to aData */
   unsigned char *aData;           /* content of the file */
+  int bFreeOnClose;               /* Invoke sqlite3_free() on aData at close */
 };
 
 /*
@@ -144,6 +156,8 @@ static const sqlite3_io_methods mem_io_methods = {
 ** to free.
 */
 static int memClose(sqlite3_file *pFile){
+  MemFile *p = (MemFile *)pFile;
+  if( p->bFreeOnClose ) sqlite3_free(p->aData);
   return SQLITE_OK;
 }
 
@@ -170,21 +184,34 @@ static int memWrite(
   int iAmt,
   sqlite_int64 iOfst
 ){
-  return SQLITE_READONLY;
+  MemFile *p = (MemFile *)pFile;
+  if( iOfst+iAmt>p->sz ){
+    if( iOfst+iAmt>p->szMax ) return SQLITE_FULL;
+    if( iOfst>p->sz ) memset(p->aData+p->sz, 0, iOfst-p->sz);
+    p->sz = iOfst+iAmt;
+  }
+  memcpy(p->aData+iOfst, z, iAmt);
+  return SQLITE_OK;
 }
 
 /*
 ** Truncate an mem-file.
 */
 static int memTruncate(sqlite3_file *pFile, sqlite_int64 size){
-  return SQLITE_READONLY;
+  MemFile *p = (MemFile *)pFile;
+  if( size>p->sz ){
+    if( size>p->szMax ) return SQLITE_FULL;
+    memset(p->aData+p->sz, 0, size-p->sz);
+  }
+  p->sz = size; 
+  return SQLITE_OK;
 }
 
 /*
 ** Sync an mem-file.
 */
 static int memSync(sqlite3_file *pFile, int flags){
-  return SQLITE_READONLY;
+  return SQLITE_OK;
 }
 
 /*
@@ -200,7 +227,7 @@ static int memFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
 ** Lock an mem-file.
 */
 static int memLock(sqlite3_file *pFile, int eLock){
-  return SQLITE_READONLY;
+  return SQLITE_OK;
 }
 
 /*
@@ -242,7 +269,10 @@ static int memSectorSize(sqlite3_file *pFile){
 ** Return the device characteristic flags supported by an mem-file.
 */
 static int memDeviceCharacteristics(sqlite3_file *pFile){
-  return SQLITE_IOCAP_IMMUTABLE;
+  return SQLITE_IOCAP_ATOMIC | 
+         SQLITE_IOCAP_POWERSAFE_OVERWRITE |
+         SQLITE_IOCAP_SAFE_APPEND |
+         SQLITE_IOCAP_SEQUENTIAL;
 }
 
 /* Create a shared memory file mapping */
@@ -253,12 +283,12 @@ static int memShmMap(
   int bExtend,
   void volatile **pp
 ){
-  return SQLITE_READONLY;
+  return SQLITE_IOERR_SHMMAP;
 }
 
 /* Perform locking on a shared-memory segment */
 static int memShmLock(sqlite3_file *pFile, int offset, int n, int flags){
-  return SQLITE_READONLY;
+  return SQLITE_IOERR_SHMLOCK;
 }
 
 /* Memory barrier operation on shared memory */
@@ -305,6 +335,9 @@ static int memOpen(
   if( p->aData==0 ) return SQLITE_CANTOPEN;
   p->sz = sqlite3_uri_int64(zName,"sz",0);
   if( p->sz<0 ) return SQLITE_CANTOPEN;
+  p->szMax = sqlite3_uri_int64(zName,"max",p->sz);
+  if( p->szMax<p->sz ) return SQLITE_CANTOPEN;
+  p->bFreeOnClose = sqlite3_uri_boolean(zName,"freeonclose",0);
   pFile->pMethods = &mem_io_methods;
   return SQLITE_OK;
 }
@@ -315,7 +348,7 @@ static int memOpen(
 ** returning.
 */
 static int memDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
-  return SQLITE_READONLY;
+  return SQLITE_IOERR_DELETE;
 }
 
 /*
@@ -328,14 +361,7 @@ static int memAccess(
   int flags, 
   int *pResOut
 ){
-  /* The spec says there are three possible values for flags.  But only
-  ** two of them are actually used */
-  assert( flags==SQLITE_ACCESS_EXISTS || flags==SQLITE_ACCESS_READWRITE );
-  if( flags==SQLITE_ACCESS_READWRITE ){
-    *pResOut = 0;
-  }else{
-    *pResOut = 1;
-  }
+  *pResOut = 0;
   return SQLITE_OK;
 }
 
@@ -416,31 +442,43 @@ static int memCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){
 
 #ifdef MEMVFS_TEST
 /*
-**       memload(FILENAME)
+**       memvfs_from_file(FILENAME, MAXSIZE)
 **
 ** This an SQL function used to help in testing the memvfs VFS.  The
 ** function reads the content of a file into memory and then returns
-** a string that gives the locate and size of the in-memory buffer.
+** a URI that can be handed to ATTACH to attach the memory buffer as
+** a database.  Example:
+**
+**       ATTACH memvfs_from_file('test.db',1048576) AS inmem;
+**
+** The optional MAXSIZE argument gives the size of the memory allocation
+** used to hold the database.  If omitted, it defaults to the size of the
+** file on disk.
 */
 #include <stdio.h>
-static void memvfsMemloadFunc(
+static void memvfsFromFileFunc(
   sqlite3_context *context,
   int argc,
   sqlite3_value **argv
 ){
   unsigned char *p;
   sqlite3_int64 sz;
+  sqlite3_int64 szMax;
   FILE *in;
   const char *zFilename = (const char*)sqlite3_value_text(argv[0]);
-  char zReturn[100];
+  char *zUri;
 
   if( zFilename==0 ) return;
   in = fopen(zFilename, "rb");
   if( in==0 ) return;
   fseek(in, 0, SEEK_END);
-  sz = ftell(in);
+  szMax = sz = ftell(in);
   rewind(in);
-  p = sqlite3_malloc( sz );
+  if( argc>=2 ){
+    szMax = sqlite3_value_int64(argv[1]);
+    if( szMax<sz ) szMax = sz;
+  }
+  p = sqlite3_malloc64( szMax );
   if( p==0 ){
     fclose(in);
     sqlite3_result_error_nomem(context);
@@ -448,18 +486,60 @@ static void memvfsMemloadFunc(
   }
   fread(p, sz, 1, in);
   fclose(in);
-  sqlite3_snprintf(sizeof(zReturn),zReturn,"ptr=%lld&sz=%lld",
-                   (sqlite3_int64)p, sz);
-  sqlite3_result_text(context, zReturn, -1, SQLITE_TRANSIENT);
+  zUri = sqlite3_mprintf(
+           "file:/mem?vfs=memvfs&ptr=%lld&sz=%lld&max=%lld&freeonclose=1",
+                         (sqlite3_int64)p, sz, szMax);
+  sqlite3_result_text(context, zUri, -1, sqlite3_free);
 }
+#endif /* MEMVFS_TEST */
+
+#ifdef MEMVFS_TEST
+/*
+**       memvfs_to_file(SCHEMA, FILENAME)
+**
+** The schema identified by SCHEMA must be a memvfs database.  Write
+** the content of this database into FILENAME.
+*/
+static void memvfsToFileFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  MemFile *p = 0;
+  FILE *out;
+  int rc;
+  sqlite3 *db = sqlite3_context_db_handle(context);
+  sqlite3_vfs *pVfs = 0;
+  const char *zSchema = (const char*)sqlite3_value_text(argv[0]);
+  const char *zFilename = (const char*)sqlite3_value_text(argv[1]);
+
+  if( zFilename==0 ) return;
+  out = fopen(zFilename, "wb");
+  if( out==0 ) return;
+  rc = sqlite3_file_control(db, zSchema, SQLITE_FCNTL_VFS_POINTER, &pVfs);
+  if( rc || pVfs==0 ) return;
+  if( strcmp(pVfs->zName,"memvfs")!=0 ) return;
+  rc = sqlite3_file_control(db, zSchema, SQLITE_FCNTL_FILE_POINTER, &p);
+  if( rc ) return;
+  fwrite(p->aData, 1, (size_t)p->sz, out);
+  fclose(out);
+}
+#endif /* MEMVFS_TEST */
+
+#ifdef MEMVFS_TEST
 /* Called for each new database connection */
 static int memvfsRegister(
   sqlite3 *db,
-  const char **pzErrMsg,
+  char **pzErrMsg,
   const struct sqlite3_api_routines *pThunk
 ){
-  return sqlite3_create_function(db, "memload", 1, SQLITE_UTF8, 0,
-                                 memvfsMemloadFunc, 0, 0);
+  sqlite3_create_function(db, "memvfs_from_file", 1, SQLITE_UTF8, 0,
+                          memvfsFromFileFunc, 0, 0);
+  sqlite3_create_function(db, "memvfs_from_file", 2, SQLITE_UTF8, 0,
+                          memvfsFromFileFunc, 0, 0);
+  sqlite3_create_function(db, "memvfs_to_file", 2, SQLITE_UTF8, 0,
+                          memvfsToFileFunc, 0, 0);
+  return SQLITE_OK;
 }
 #endif /* MEMVFS_TEST */
 
@@ -485,6 +565,9 @@ int sqlite3_memvfs_init(
   if( rc==SQLITE_OK ){
     rc = sqlite3_auto_extension((void(*)(void))memvfsRegister);
   }
+  if( rc==SQLITE_OK ){
+    rc = memvfsRegister(db, pzErrMsg, pApi);
+  }
 #endif
   if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY;
   return rc;
diff --git a/manifest b/manifest
index 8af4fc7c93..b3405dc4c3 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Updates\sto\sthe\ssqlite3_file_control()\sdocumentation.\s\sNo\schanges\sto\scode.
-D 2018-01-02T13:48:48.784
+C Enhance\sthe\smemvfs\sextension\sso\sthat\sit\scan\sbe\sread/write.
+D 2018-01-02T16:02:50.552
 F Makefile.in 1b11037c5ed3399a79433cc82c59b5e36a7b3a3e4e195bb27640d0d2145e03e1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc f68b4f9b83cfeb057b6265e0288ad653f319e2ceacca731e0f22e19617829a89
@@ -278,7 +278,7 @@ F ext/misc/fileio.c b1aa06c0f1dac277695d4529e5e976c65ab5678dcbb53a0304deaa8adc44
 F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25
 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c
 F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281984
-F ext/misc/memvfs.c e5225bc22e79dde6b28380f3a068ddf600683a33
+F ext/misc/memvfs.c ab36f49e02ebcdf85a1e08dc4d8599ea8f343e073ac9e0bca18a98b7e1ec9567
 F ext/misc/mmapwarm.c 70b618f2d0bde43fae288ad0b7498a629f2b6f61b50a27e06fae3cd23c83af29
 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342
 F ext/misc/percentile.c 92699c8cd7d517ff610e6037e56506f8904dae2e
@@ -1688,7 +1688,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 39ca5fcd31c902cbade4da05125b0dbf2bc5e2b8901af1645f113c9d3dbc8209
-R 1ce62a1c4d9b56a67e8d1f1aed222781
+P 240e32ab1f2a18e3c9b92f577b1cc8f8ecb4c68c44eac863d859491e042cb72a
+R e6325f7b9d9fed8be88a348d63977f27
 U drh
-Z d9ace2c20dd904e2b4c8de8f33d7aa5e
+Z de6d00886014fb58a1b54f4eb1670d54
diff --git a/manifest.uuid b/manifest.uuid
index 3dd953071f..c9ede0a08a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-240e32ab1f2a18e3c9b92f577b1cc8f8ecb4c68c44eac863d859491e042cb72a
\ No newline at end of file
+04c9197d589666299aef86ee6a56df63448c050274c9fba4af94f932752be237
\ No newline at end of file

From 4d795ef7e4b6e673b2ab84cc2155afc72221d386 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 2 Jan 2018 18:11:11 +0000
Subject: [PATCH 213/488] In the constraint resolution logic, be careful not to
 cache column values in registers whose initialization might be bypassed by an
 OP_NoConflict opcode. Fix for ticket [dc3f932f5a147771] reported by OSSFuzz.

FossilOrigin-Name: 2846458af5d029a8e4fdcc8f50873a44e57897bbfe6aee8a23a01ffc34c5579f
---
 manifest             | 14 +++++++-------
 manifest.uuid        |  2 +-
 src/insert.c         |  2 ++
 test/indexexpr1.test | 11 +++++++++++
 4 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index b3405dc4c3..e31c0bce91 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Enhance\sthe\smemvfs\sextension\sso\sthat\sit\scan\sbe\sread/write.
-D 2018-01-02T16:02:50.552
+C In\sthe\sconstraint\sresolution\slogic,\sbe\scareful\snot\sto\scache\scolumn\svalues\s\nin\sregisters\swhose\sinitialization\smight\sbe\sbypassed\sby\san\sOP_NoConflict\sopcode.\nFix\sfor\sticket\s[dc3f932f5a147771]\sreported\sby\sOSSFuzz.
+D 2018-01-02T18:11:11.985
 F Makefile.in 1b11037c5ed3399a79433cc82c59b5e36a7b3a3e4e195bb27640d0d2145e03e1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc f68b4f9b83cfeb057b6265e0288ad653f319e2ceacca731e0f22e19617829a89
@@ -440,7 +440,7 @@ F src/hash.c a12580e143f10301ed5166ea4964ae2853d3905a511d4e0c44497245c7ce1f7a
 F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4
 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
-F src/insert.c cb67cc56ef2ddd13e6944b2c0dd08a920bcd9503230adef8b9928d338097c722
+F src/insert.c 14686083cedc198540b15a79586cdd4be2acf6d5fa97627e355f817ab07e9fee
 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e
 F src/loadext.c 55bcc3c741059a1056859e8adaf133aa179e22be12215c0936b2f354ef71209b
 F src/main.c 690c4134f944cbd5b71d59dd6e61ce4131f6a50ab774f38108e57d07d79cf876
@@ -969,7 +969,7 @@ F test/index7.test 7feababe16f2091b229c22aff2bcc1d4d6b9d2bb
 F test/index8.test bc2e3db70e8e62459aaa1bd7e4a9b39664f8f9d7
 F test/index9.test 0aa3e509dddf81f93380396e40e9bb386904c1054924ba8fa9bcdfe85a8e7721
 F test/indexedby.test faa585e315e868f09bce0eb39c41d6134649b13d2801638294d3ae616edf1609
-F test/indexexpr1.test 84100e880154a4b645db9f4fc7642756d9a2b6011b68f73c8efda4d244816de9
+F test/indexexpr1.test 60e2d6f1d1337fd213208270295c650d268503ff215de728f540ea31eb237f70
 F test/indexexpr2.test 13247bac49143196556eb3f65e97ef301bd3e993f4511558b5db322ddc370ea6
 F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d
 F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7
@@ -1688,7 +1688,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 240e32ab1f2a18e3c9b92f577b1cc8f8ecb4c68c44eac863d859491e042cb72a
-R e6325f7b9d9fed8be88a348d63977f27
+P 04c9197d589666299aef86ee6a56df63448c050274c9fba4af94f932752be237
+R 264e263a982eceab94b4544263056f03
 U drh
-Z de6d00886014fb58a1b54f4eb1670d54
+Z 317cb96d5817cef99d6f004b6578f45e
diff --git a/manifest.uuid b/manifest.uuid
index c9ede0a08a..710f42fb8d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-04c9197d589666299aef86ee6a56df63448c050274c9fba4af94f932752be237
\ No newline at end of file
+2846458af5d029a8e4fdcc8f50873a44e57897bbfe6aee8a23a01ffc34c5579f
\ No newline at end of file
diff --git a/src/insert.c b/src/insert.c
index f0af0fbd10..e1514692cc 100644
--- a/src/insert.c
+++ b/src/insert.c
@@ -1571,6 +1571,7 @@ void sqlite3GenerateConstraintChecks(
     }
 
     /* Check to see if the new index entry will be unique */
+    sqlite3ExprCachePush(pParse);
     sqlite3VdbeAddOp4Int(v, OP_NoConflict, iThisCur, addrUniqueOk,
                          regIdx, pIdx->nKeyCol); VdbeCoverage(v);
 
@@ -1659,6 +1660,7 @@ void sqlite3GenerateConstraintChecks(
       }
     }
     sqlite3VdbeResolveLabel(v, addrUniqueOk);
+    sqlite3ExprCachePop(pParse);
     if( regR!=regIdx ) sqlite3ReleaseTempRange(pParse, regR, nPkField);
   }
   if( ipkTop ){
diff --git a/test/indexexpr1.test b/test/indexexpr1.test
index 0e24c8066f..1caa3086be 100644
--- a/test/indexexpr1.test
+++ b/test/indexexpr1.test
@@ -401,5 +401,16 @@ do_execsql_test indexexpr1-1430 {
   SELECT abs(15+3) IN (SELECT 17 UNION ALL SELECT 18) FROM t1;
 } {1 1}
 
+# 2018-01-02 ticket https://sqlite.org/src/info/dc3f932f5a147771
+# A REPLACE into a table that uses an index on an expression causes
+# an assertion fault.  Problem discovered by OSSFuzz.
+#
+do_execsql_test indexexpr1-1500 {
+  CREATE TABLE t1500(a INT PRIMARY KEY, b INT UNIQUE);
+  CREATE INDEX t1500ab ON t1500(a*b);
+  INSERT INTO t1500(a,b) VALUES(1,2);
+  REPLACE INTO t1500(a,b) VALUES(1,3);  -- formerly caused assertion fault
+  SELECT * FROM t1500;
+} {1 3}
 
 finish_test

From d9620b8555f1bd62847400b8c9d8d3c2adf60ccb Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 2 Jan 2018 21:29:42 +0000
Subject: [PATCH 214/488] Fix code indentation.  No logic changes.

FossilOrigin-Name: e115f2583499df0c7ee991d372bed4b89aa717b10a4e4b10977864390cb4fc11
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/pager.c   |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index e31c0bce91..774549626b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\sthe\sconstraint\sresolution\slogic,\sbe\scareful\snot\sto\scache\scolumn\svalues\s\nin\sregisters\swhose\sinitialization\smight\sbe\sbypassed\sby\san\sOP_NoConflict\sopcode.\nFix\sfor\sticket\s[dc3f932f5a147771]\sreported\sby\sOSSFuzz.
-D 2018-01-02T18:11:11.985
+C Fix\scode\sindentation.\s\sNo\slogic\schanges.
+D 2018-01-02T21:29:42.540
 F Makefile.in 1b11037c5ed3399a79433cc82c59b5e36a7b3a3e4e195bb27640d0d2145e03e1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc f68b4f9b83cfeb057b6265e0288ad653f319e2ceacca731e0f22e19617829a89
@@ -465,7 +465,7 @@ F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
 F src/os_unix.c 7fc2735390a7809d5d893ed735d994ff12521224b89738226fff6f1a0aa1c932
 F src/os_win.c 0a4afa35cc8e812000df3ea2f64b476131b39e29e75d8007d0504726e4761de4
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
-F src/pager.c 07cf850241667874fcce9d7d924c814305e499b26c804322e2261247b5921903
+F src/pager.c 9b9cb4e06c03d43d62480a7a685a012d645fcf3a39e7767ccb505fb41ee083ec
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
 F src/parse.y 44cbbc3e132ea128258eff7be7f6d5c5dfa25522f89ec8b5501276966511bd50
 F src/pcache.c 7ae91a4557a43d77d449accbfdc68846e6516f8e2eda46e8bbe4536fb669b201
@@ -1688,7 +1688,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 04c9197d589666299aef86ee6a56df63448c050274c9fba4af94f932752be237
-R 264e263a982eceab94b4544263056f03
+P 2846458af5d029a8e4fdcc8f50873a44e57897bbfe6aee8a23a01ffc34c5579f
+R 9613ba300b7bd9cce413f2672960fb05
 U drh
-Z 317cb96d5817cef99d6f004b6578f45e
+Z 9fca118926b581510cc96e0ce3702687
diff --git a/manifest.uuid b/manifest.uuid
index 710f42fb8d..4bf24d86c2 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2846458af5d029a8e4fdcc8f50873a44e57897bbfe6aee8a23a01ffc34c5579f
\ No newline at end of file
+e115f2583499df0c7ee991d372bed4b89aa717b10a4e4b10977864390cb4fc11
\ No newline at end of file
diff --git a/src/pager.c b/src/pager.c
index dbb7636ca7..295cbe04c5 100644
--- a/src/pager.c
+++ b/src/pager.c
@@ -5579,7 +5579,7 @@ static int getPageMMap(
       }
       if( pPg==0 ){
         rc = pagerAcquireMapPage(pPager, pgno, pData, &pPg);
-     }else{
+      }else{
         sqlite3OsUnfetch(pPager->fd, (i64)(pgno-1)*pPager->pageSize, pData);
       }
       if( pPg ){

From 9eb8dbad4d09d3f65444f667f92d7ad38739e178 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 3 Jan 2018 01:47:30 +0000
Subject: [PATCH 215/488] Add another test case (found by OSSFuzz) for the
 problem fixed in check-in [2846458a] and described by ticket
 [dc3f932f5a147771]. No changes to code.

FossilOrigin-Name: 4165fae920a1176c68bfaa1f2cfe17df7f8dd8793519d458b71e3e63670b5495
---
 manifest             | 12 ++++++------
 manifest.uuid        |  2 +-
 test/indexexpr1.test | 12 ++++++++++++
 3 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index 774549626b..ab818a843a 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\scode\sindentation.\s\sNo\slogic\schanges.
-D 2018-01-02T21:29:42.540
+C Add\sanother\stest\scase\s(found\sby\sOSSFuzz)\sfor\sthe\sproblem\sfixed\sin\s\ncheck-in\s[2846458a]\sand\sdescribed\sby\sticket\s[dc3f932f5a147771].\nNo\schanges\sto\scode.
+D 2018-01-03T01:47:30.287
 F Makefile.in 1b11037c5ed3399a79433cc82c59b5e36a7b3a3e4e195bb27640d0d2145e03e1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc f68b4f9b83cfeb057b6265e0288ad653f319e2ceacca731e0f22e19617829a89
@@ -969,7 +969,7 @@ F test/index7.test 7feababe16f2091b229c22aff2bcc1d4d6b9d2bb
 F test/index8.test bc2e3db70e8e62459aaa1bd7e4a9b39664f8f9d7
 F test/index9.test 0aa3e509dddf81f93380396e40e9bb386904c1054924ba8fa9bcdfe85a8e7721
 F test/indexedby.test faa585e315e868f09bce0eb39c41d6134649b13d2801638294d3ae616edf1609
-F test/indexexpr1.test 60e2d6f1d1337fd213208270295c650d268503ff215de728f540ea31eb237f70
+F test/indexexpr1.test ace1ad489adc25325ad298434f13b1a515b36bf5dca9fe2a4b66cdf17aea3fa0
 F test/indexexpr2.test 13247bac49143196556eb3f65e97ef301bd3e993f4511558b5db322ddc370ea6
 F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d
 F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7
@@ -1688,7 +1688,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2846458af5d029a8e4fdcc8f50873a44e57897bbfe6aee8a23a01ffc34c5579f
-R 9613ba300b7bd9cce413f2672960fb05
+P e115f2583499df0c7ee991d372bed4b89aa717b10a4e4b10977864390cb4fc11
+R 83a0b36c500269c331a5b5ebe1ebb55e
 U drh
-Z 9fca118926b581510cc96e0ce3702687
+Z 192675b51835fbb18bda08bad08dd763
diff --git a/manifest.uuid b/manifest.uuid
index 4bf24d86c2..01fb7976a7 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e115f2583499df0c7ee991d372bed4b89aa717b10a4e4b10977864390cb4fc11
\ No newline at end of file
+4165fae920a1176c68bfaa1f2cfe17df7f8dd8793519d458b71e3e63670b5495
\ No newline at end of file
diff --git a/test/indexexpr1.test b/test/indexexpr1.test
index 1caa3086be..28c23b9089 100644
--- a/test/indexexpr1.test
+++ b/test/indexexpr1.test
@@ -413,4 +413,16 @@ do_execsql_test indexexpr1-1500 {
   SELECT * FROM t1500;
 } {1 3}
 
+# 2018-01-03 OSSFuzz discovers another test case for the same problem
+# above.
+#
+do_execsql_test indexexpr-1510 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1(a PRIMARY KEY,b UNIQUE);
+  REPLACE INTO t1 VALUES(2, 1);
+  REPLACE INTO t1 SELECT 6,1;
+  CREATE INDEX t1aa ON t1(a-a);
+  REPLACE INTO t1 SELECT a, randomblob(a) FROM t1
+} {}
+
 finish_test

From 64c1990c0044718bd3c11d238eb5be3eef48a4d8 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 4 Jan 2018 16:40:44 +0000
Subject: [PATCH 216/488] Fix a broken documentation hyperlink.  No code
 changes.

FossilOrigin-Name: d91e3f3d343d281af374dd23eea333e61228539023ad5c5aea622085e5863bc7
---
 manifest        | 12 ++++++------
 manifest.uuid   |  2 +-
 src/sqlite.h.in |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index ab818a843a..d48a41447d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sanother\stest\scase\s(found\sby\sOSSFuzz)\sfor\sthe\sproblem\sfixed\sin\s\ncheck-in\s[2846458a]\sand\sdescribed\sby\sticket\s[dc3f932f5a147771].\nNo\schanges\sto\scode.
-D 2018-01-03T01:47:30.287
+C Fix\sa\sbroken\sdocumentation\shyperlink.\s\sNo\scode\schanges.
+D 2018-01-04T16:40:44.063
 F Makefile.in 1b11037c5ed3399a79433cc82c59b5e36a7b3a3e4e195bb27640d0d2145e03e1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc f68b4f9b83cfeb057b6265e0288ad653f319e2ceacca731e0f22e19617829a89
@@ -480,7 +480,7 @@ F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
 F src/shell.c.in a418ddceef7a2789f18bdc2bcdd481b2562fe4a7754b8009c8dd33d5a67da332
-F src/sqlite.h.in c597ba5d11666bb2d0179a173cd25e44f0b0333f9e18ce60b07eac8bdc3de67f
+F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
 F src/sqliteInt.h fd8702c65994d7de3e2d8f7d85d958731da1ed29476571fdfa2290fd8ec0bf80
@@ -1688,7 +1688,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e115f2583499df0c7ee991d372bed4b89aa717b10a4e4b10977864390cb4fc11
-R 83a0b36c500269c331a5b5ebe1ebb55e
+P 4165fae920a1176c68bfaa1f2cfe17df7f8dd8793519d458b71e3e63670b5495
+R d66f50893f4ffe9e2cfb6738dd929b34
 U drh
-Z 192675b51835fbb18bda08bad08dd763
+Z 6dbec8e8e1444dbd0ed402b28d5f012a
diff --git a/manifest.uuid b/manifest.uuid
index 01fb7976a7..66b07243fd 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4165fae920a1176c68bfaa1f2cfe17df7f8dd8793519d458b71e3e63670b5495
\ No newline at end of file
+d91e3f3d343d281af374dd23eea333e61228539023ad5c5aea622085e5863bc7
\ No newline at end of file
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index a6d328360a..f63b029312 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -8301,7 +8301,7 @@ int sqlite3_vtab_on_conflict(sqlite3 *);
 ** CAPI3REF: Determine The Collation For a Virtual Table Constraint
 **
 ** This function may only be called from within a call to the [xBestIndex]
-** method of a [virtual table implementation]. 
+** method of a [virtual table]. 
 **
 ** The first argument must be the sqlite3_index_info object that is the
 ** first parameter to the xBestIndex() method. The second argument must be

From 35100fb194cfbaf75b31b504e7bf7d4480900c26 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 4 Jan 2018 19:20:37 +0000
Subject: [PATCH 217/488] Remove the snarky "_supported_" qualifier from the
 name of the sqlite_offset() SQL function.

FossilOrigin-Name: a6eee0fcd89d3958f8720ebdb5f0a8558b4795d747128091dae283eb81c4f74f
---
 manifest        | 14 +++++++-------
 manifest.uuid   |  2 +-
 src/func.c      |  3 +--
 test/func6.test | 40 ++++++++++++++++++++--------------------
 4 files changed, 29 insertions(+), 30 deletions(-)

diff --git a/manifest b/manifest
index d48a41447d..3552e5e8ce 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sbroken\sdocumentation\shyperlink.\s\sNo\scode\schanges.
-D 2018-01-04T16:40:44.063
+C Remove\sthe\ssnarky\s"_supported_"\squalifier\sfrom\sthe\sname\sof\sthe\nsqlite_offset()\sSQL\sfunction.
+D 2018-01-04T19:20:37.203
 F Makefile.in 1b11037c5ed3399a79433cc82c59b5e36a7b3a3e4e195bb27640d0d2145e03e1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc f68b4f9b83cfeb057b6265e0288ad653f319e2ceacca731e0f22e19617829a89
@@ -434,7 +434,7 @@ F src/delete.c 74667ad914ac143731a444a1bacf29ceb18f6eded8a0dd17aafae80baa07f8bb
 F src/expr.c ad6e7a9c34a4bab9d10cc857d647ae7ce370a633b5c0bfa71f1c29b81ae364b8
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331
-F src/func.c 697a3ee3b8c8481db497226e9e71935727d6a9fb3d3cc049e82a1c717a0a8342
+F src/func.c bd528d5ed68ce5cbf78a762e3b735fa75009f7197ff07fab07fd771f35ebaa1b
 F src/global.c ac3094f1dc59fbeb919aef7cc0cc827a8459d1fb1adb7972ef75bd9e0c10b75b
 F src/hash.c a12580e143f10301ed5166ea4964ae2853d3905a511d4e0c44497245c7ce1f7a
 F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4
@@ -917,7 +917,7 @@ F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f
 F test/func3.test d202a7606d23f90988a664e88e268aed1087c11c
 F test/func4.test 6beacdfcb0e18c358e6c2dcacf1b65d1fa80955f
 F test/func5.test cdd224400bc3e48d891827cc913a57051a426fa4
-F test/func6.test 1a2bc511fedb779e2bd3154361385216c6d35080f36bfe896b8cdd1b1954294a
+F test/func6.test a4281c8fcd42b56f7a60f28e8e4d444e8b2256f9e82658b7ab87699f8318f564
 F test/fuzz-oss1.test e58330d01cbbd8215ee636b17a03fe220b37dbfa
 F test/fuzz.test 96083052bf5765e4518c1ba686ce2bab785670d1
 F test/fuzz2.test 76dc35b32b6d6f965259508508abce75a6c4d7e1
@@ -1688,7 +1688,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 4165fae920a1176c68bfaa1f2cfe17df7f8dd8793519d458b71e3e63670b5495
-R d66f50893f4ffe9e2cfb6738dd929b34
+P d91e3f3d343d281af374dd23eea333e61228539023ad5c5aea622085e5863bc7
+R 314fe8cfc8ec7f02f3acb6e4d50707fb
 U drh
-Z 6dbec8e8e1444dbd0ed402b28d5f012a
+Z 7ea7a34ef1d7c0cfd34c8e9e5f226383
diff --git a/manifest.uuid b/manifest.uuid
index 66b07243fd..561f8b3f92 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d91e3f3d343d281af374dd23eea333e61228539023ad5c5aea622085e5863bc7
\ No newline at end of file
+a6eee0fcd89d3958f8720ebdb5f0a8558b4795d747128091dae283eb81c4f74f
\ No newline at end of file
diff --git a/src/func.c b/src/func.c
index 4758b0191d..1076b97161 100644
--- a/src/func.c
+++ b/src/func.c
@@ -1800,8 +1800,7 @@ void sqlite3RegisterBuiltinFunctions(void){
     FUNCTION2(affinity,          1, 0, 0, noopFunc,  SQLITE_FUNC_AFFINITY),
 #endif
 #ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
-    FUNCTION2(sqlite_unsupported_offset,
-                                 1, 0, 0, noopFunc,  SQLITE_FUNC_OFFSET|
+    FUNCTION2(sqlite_offset,     1, 0, 0, noopFunc,  SQLITE_FUNC_OFFSET|
                                                      SQLITE_FUNC_TYPEOF),
 #endif
     FUNCTION(ltrim,              1, 1, 0, trimFunc         ),
diff --git a/test/func6.test b/test/func6.test
index 0d3de8ef49..02e1998cd0 100644
--- a/test/func6.test
+++ b/test/func6.test
@@ -9,7 +9,7 @@
 #
 #*************************************************************************
 #
-# Test cases for the sqlite_unsupported_offset() function.
+# Test cases for the sqlite_offset() function.
 #
 # Some of the tests in this file depend on the exact placement of content
 # within b-tree pages.  Such placement is at the implementations discretion,
@@ -34,51 +34,51 @@ do_execsql_test func6-100 {
   INSERT INTO t2(x,y) SELECT a, b FROM t1;
 }
 do_execsql_test func6-110 {
-  SELECT a, sqlite_unsupported_offset(d)/4096 + 1,
-            sqlite_unsupported_offset(d)%4096 FROM t1
+  SELECT a, sqlite_offset(d)/4096 + 1,
+            sqlite_offset(d)%4096 FROM t1
    ORDER BY rowid LIMIT 2;
 } {abc001 2 4084 abc002 2 4069}
 do_execsql_test func6-120 {
-  SELECT a, typeof(sqlite_unsupported_offset(+a)) FROM t1
+  SELECT a, typeof(sqlite_offset(+a)) FROM t1
    ORDER BY rowid LIMIT 2;
 } {abc001 null abc002 null}
 do_execsql_test func6-130 {
-  SELECT a, sqlite_unsupported_offset(a)/4096+1, 
-         sqlite_unsupported_offset(a)%4096
+  SELECT a, sqlite_offset(a)/4096+1, 
+         sqlite_offset(a)%4096
    FROM t1
    ORDER BY a LIMIT 2;
 } {abc001 3 4087 abc002 3 4076}
 do_execsql_test func6-140 {
-  SELECT a, sqlite_unsupported_offset(d)/4096+1, 
-         sqlite_unsupported_offset(d)%4096
+  SELECT a, sqlite_offset(d)/4096+1, 
+         sqlite_offset(d)%4096
    FROM t1
    ORDER BY a LIMIT 2;
 } {abc001 2 4084 abc002 2 4069}
 do_execsql_test func6-150 {
   SELECT a,
-         sqlite_unsupported_offset(a)/4096+1, 
-         sqlite_unsupported_offset(a)%4096,
-         sqlite_unsupported_offset(d)/4096+1, 
-         sqlite_unsupported_offset(d)%4096
+         sqlite_offset(a)/4096+1, 
+         sqlite_offset(a)%4096,
+         sqlite_offset(d)/4096+1, 
+         sqlite_offset(d)%4096
    FROM t1
    ORDER BY a LIMIT 2;
 } {abc001 3 4087 2 4084 abc002 3 4076 2 4069}
 do_execsql_test func6-160 {
   SELECT b,
-         sqlite_unsupported_offset(b)/4096+1, 
-         sqlite_unsupported_offset(b)%4096,
-         sqlite_unsupported_offset(c)/4096+1, 
-         sqlite_unsupported_offset(c)%4096,
-         sqlite_unsupported_offset(d)/4096+1, 
-         sqlite_unsupported_offset(d)%4096
+         sqlite_offset(b)/4096+1, 
+         sqlite_offset(b)%4096,
+         sqlite_offset(c)/4096+1, 
+         sqlite_offset(c)%4096,
+         sqlite_offset(d)/4096+1, 
+         sqlite_offset(d)%4096
    FROM t1
    ORDER BY b LIMIT 2;
 } {1 4 4090 4 4090 2 4084 2 4 4081 4 4081 2 4069}
 
 
 do_execsql_test func6-200 {
-  SELECT y, sqlite_unsupported_offset(y)/4096+1,
-         sqlite_unsupported_offset(y)%4096
+  SELECT y, sqlite_offset(y)/4096+1,
+         sqlite_offset(y)%4096
    FROM t2
    ORDER BY x LIMIT 2;
 } {1 5 4087 2 5 4076}

From dfdfd8c7f24dc5499019e5ed70f3e26a6118f837 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Thu, 4 Jan 2018 22:46:08 +0000
Subject: [PATCH 218/488] Get the shell enhancements compiling with MSVC.

FossilOrigin-Name: 335387f9e0d4569097d34cd99cd332b38a282e9b7ae25f088eb47df5c25837ef
---
 Makefile.msc         |  9 +++++--
 ext/misc/fileio.c    | 49 +++++++++++++++++++++++++++++++++----
 ext/misc/zipfile.c   | 58 ++++++++++++++++++++++++++------------------
 manifest             | 24 +++++++++---------
 manifest.uuid        |  2 +-
 src/shell.c.in       | 10 ++++++--
 src/test_windirent.h | 48 +++++++++++++++++++++++++++++++++---
 test/shell8.test     |  6 ++++-
 8 files changed, 157 insertions(+), 49 deletions(-)

diff --git a/Makefile.msc b/Makefile.msc
index 5a479efb16..4fb128c567 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -1506,7 +1506,8 @@ TESTEXT = \
   $(TOP)\ext\misc\spellfix.c \
   $(TOP)\ext\misc\totype.c \
   $(TOP)\ext\misc\unionvtab.c \
-  $(TOP)\ext\misc\wholenumber.c
+  $(TOP)\ext\misc\wholenumber.c \
+  $(TOP)\ext\misc\zipfile.c
 
 # Source code to the library files needed by the test fixture
 # (non-amalgamation)
@@ -2069,7 +2070,11 @@ SHELL_SRC = \
 	$(TOP)\src\shell.c.in \
 	$(TOP)\ext\misc\shathree.c \
 	$(TOP)\ext\misc\fileio.c \
-	$(TOP)\ext\misc\completion.c
+	$(TOP)\ext\misc\completion.c \
+	$(TOP)\ext\misc\sqlar.c \
+	$(TOP)\ext\expert\sqlite3expert.c \
+	$(TOP)\ext\expert\sqlite3expert.h \
+	$(TOP)\ext\misc\zipfile.c
 
 shell.c:	$(SHELL_SRC) $(TOP)\tool\mkshellc.tcl
 	$(TCLSH_CMD) $(TOP)\tool\mkshellc.tcl > shell.c
diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c
index 7dbac4043f..2464d18f80 100644
--- a/ext/misc/fileio.c
+++ b/ext/misc/fileio.c
@@ -82,10 +82,21 @@ SQLITE_EXTENSION_INIT1
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <fcntl.h>
-#include <unistd.h>
-#include <dirent.h>
+#if !defined(_WIN32) && !defined(WIN32)
+#  include <unistd.h>
+#  include <dirent.h>
+#  include <utime.h>
+#else
+#  include "windows.h"
+#  include <io.h>
+#  include <direct.h>
+#  include "test_windirent.h"
+#  define dirent DIRENT
+#  define timespec TIMESPEC
+#  define mkdir(path,mode) _mkdir(path)
+#  define lstat(path,buf) _stat(path,buf)
+#endif
 #include <time.h>
-#include <utime.h>
 #include <errno.h>
 
 
@@ -203,10 +214,13 @@ static int writeFile(
   mode_t mode,                    /* MODE parameter passed to writefile() */
   sqlite3_int64 mtime             /* MTIME parameter (or -1 to not set time) */
 ){
+#if !defined(_WIN32) && !defined(WIN32)
   if( S_ISLNK(mode) ){
     const char *zTo = (const char*)sqlite3_value_text(pData);
     if( symlink(zTo, zFile)<0 ) return 1;
-  }else{
+  }else
+#endif
+  {
     if( S_ISDIR(mode) ){
       if( mkdir(zFile, mode) ){
         /* The mkdir() call to create the directory failed. This might not
@@ -246,6 +260,7 @@ static int writeFile(
   }
 
   if( mtime>=0 ){
+#if !defined(_WIN32) && !defined(WIN32)
     struct timespec times[2];
     times[0].tv_nsec = times[1].tv_nsec = 0;
     times[0].tv_sec = time(0);
@@ -253,6 +268,28 @@ static int writeFile(
     if( utimensat(AT_FDCWD, zFile, times, AT_SYMLINK_NOFOLLOW) ){
       return 1;
     }
+#else
+    FILETIME lastAccess;
+    FILETIME lastWrite;
+    SYSTEMTIME currentTime;
+    LONGLONG intervals;
+    HANDLE hFile;
+    GetSystemTime(&currentTime);
+    SystemTimeToFileTime(&currentTime, &lastAccess);
+    intervals = Int32x32To64(mtime, 10000000) + 116444736000000000;
+    lastWrite.dwLowDateTime = (DWORD)intervals;
+    lastWrite.dwHighDateTime = intervals >> 32;
+    hFile = CreateFile(
+      zFile, FILE_WRITE_ATTRIBUTES, 0, NULL, OPEN_EXISTING, 0, NULL
+    );
+    if( hFile!=INVALID_HANDLE_VALUE ){
+      BOOL bResult = SetFileTime(hFile, NULL, &lastAccess, &lastWrite);
+      CloseHandle(hFile);
+      return !bResult;
+    }else{
+      return 1;
+    }
+#endif
   }
 
   return 0;
@@ -282,7 +319,7 @@ static void writefileFunc(
   zFile = (const char*)sqlite3_value_text(argv[0]);
   if( zFile==0 ) return;
   if( argc>=3 ){
-    mode = sqlite3_value_int(argv[2]);
+    mode = (mode_t)sqlite3_value_int(argv[2]);
   }
   if( argc==4 ){
     mtime = sqlite3_value_int64(argv[3]);
@@ -518,6 +555,7 @@ static int fsdirColumn(
       mode_t m = pCur->sStat.st_mode;
       if( S_ISDIR(m) ){
         sqlite3_result_null(ctx);
+#if !defined(_WIN32) && !defined(WIN32)
       }else if( S_ISLNK(m) ){
         char aStatic[64];
         char *aBuf = aStatic;
@@ -538,6 +576,7 @@ static int fsdirColumn(
 
         sqlite3_result_text(ctx, aBuf, n, SQLITE_TRANSIENT);
         if( aBuf!=aStatic ) sqlite3_free(aBuf);
+#endif
       }else{
         readFileContents(ctx, pCur->zPath);
       }
diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index 9d0865377d..c3dc5f2d3e 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -20,10 +20,14 @@ SQLITE_EXTENSION_INIT1
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <fcntl.h>
-#include <unistd.h>
-#include <dirent.h>
+#if !defined(_WIN32) && !defined(WIN32)
+#  include <unistd.h>
+#  include <dirent.h>
+#  include <utime.h>
+#else
+#  include <io.h>
+#endif
 #include <time.h>
-#include <utime.h>
 #include <errno.h>
 
 #include <zlib.h>
@@ -387,9 +391,9 @@ static int zipfileReadData(
   char **pzErrmsg                 /* OUT: Error message (from sqlite3_malloc) */
 ){
   size_t n;
-  fseek(pFile, iOff, SEEK_SET);
+  fseek(pFile, (long)iOff, SEEK_SET);
   n = fread(aRead, 1, nRead, pFile);
-  if( n!=nRead ){
+  if( (int)n!=nRead ){
     *pzErrmsg = sqlite3_mprintf("error in fread()");
     return SQLITE_ERROR;
   }
@@ -402,9 +406,9 @@ static int zipfileAppendData(
   int nWrite
 ){
   size_t n;
-  fseek(pTab->pWriteFd, pTab->szCurrent, SEEK_SET);
+  fseek(pTab->pWriteFd, (long)pTab->szCurrent, SEEK_SET);
   n = fwrite(aWrite, 1, nWrite, pTab->pWriteFd);
-  if( n!=nWrite ){
+  if( (int)n!=nWrite ){
     pTab->base.zErrMsg = sqlite3_mprintf("error in fwrite()");
     return SQLITE_ERROR;
   }
@@ -649,17 +653,22 @@ static time_t zipfileMtime(ZipfileCsr *pCsr){
 static void zipfileMtimeToDos(ZipfileCDS *pCds, u32 mTime){
   time_t t = (time_t)mTime;
   struct tm res;
-  localtime_r(&t, &res);
 
-  pCds->mTime = 
+#if !defined(_WIN32) && !defined(WIN32)
+  localtime_r(&t, &res);
+#else
+  memcpy(&res, localtime(&t), sizeof(struct tm));
+#endif
+
+  pCds->mTime = (u16)(
     (res.tm_sec / 2) + 
     (res.tm_min << 5) +
-    (res.tm_hour << 11);
+    (res.tm_hour << 11));
 
-  pCds->mDate = 
+  pCds->mDate = (u16)(
     (res.tm_mday-1) +
     ((res.tm_mon+1) << 5) +
-    ((res.tm_year-80) << 9);
+    ((res.tm_year-80) << 9));
 }
 
 /*
@@ -754,6 +763,7 @@ static int zipfileReadEOCD(
   i64 szFile;                     /* Total size of file in bytes */
   int nRead;                      /* Bytes to read from file */
   i64 iOff;                       /* Offset to read from */
+  int rc;
 
   fseek(pFile, 0, SEEK_END);
   szFile = (i64)ftell(pFile);
@@ -763,7 +773,7 @@ static int zipfileReadEOCD(
   nRead = (int)(MIN(szFile, ZIPFILE_BUFFER_SIZE));
   iOff = szFile - nRead;
 
-  int rc = zipfileReadData(pFile, aRead, nRead, iOff, &pTab->base.zErrMsg);
+  rc = zipfileReadData(pFile, aRead, nRead, iOff, &pTab->base.zErrMsg);
   if( rc==SQLITE_OK ){
     int i;
 
@@ -968,7 +978,7 @@ static ZipfileEntry *zipfileNewEntry(
 ){
   u8 *aWrite;
   ZipfileEntry *pNew;
-  pCds->nFile = nPath;
+  pCds->nFile = (u16)nPath;
   pCds->nExtra = mTime ? 9 : 0;
   pNew = (ZipfileEntry*)sqlite3_malloc(
     sizeof(ZipfileEntry) + 
@@ -1036,7 +1046,7 @@ static int zipfileAppendEntry(
   zipfileWrite32(aBuf, pCds->crc32);
   zipfileWrite32(aBuf, pCds->szCompressed);
   zipfileWrite32(aBuf, pCds->szUncompressed);
-  zipfileWrite16(aBuf, nPath);
+  zipfileWrite16(aBuf, (u16)nPath);
   zipfileWrite16(aBuf, pCds->nExtra);
   assert( aBuf==&pTab->aBuffer[ZIPFILE_LFH_FIXED_SZ] );
   rc = zipfileAppendData(pTab, pTab->aBuffer, aBuf - pTab->aBuffer);
@@ -1066,13 +1076,15 @@ static int zipfileGetMode(ZipfileTab *pTab, sqlite3_value *pVal, int *pMode){
   if( z==0 || (z[0]>=0 && z[0]<=9) ){
     mode = sqlite3_value_int(pVal);
   }else{
-    const char zTemplate[10] = "-rwxrwxrwx";
+    const char zTemplate[11] = "-rwxrwxrwx";
     int i;
     if( strlen(z)!=10 ) goto parse_error;
     switch( z[0] ){
       case '-': mode |= S_IFREG; break;
       case 'd': mode |= S_IFDIR; break;
+#if !defined(_WIN32) && !defined(WIN32)
       case 'l': mode |= S_IFLNK; break;
+#endif
       default: goto parse_error;
     }
     for(i=1; i<10; i++){
@@ -1178,13 +1190,13 @@ static int zipfileUpdate(
     cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY;
     cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED;
     cds.flags = ZIPFILE_NEWENTRY_FLAGS;
-    cds.iCompression = iMethod;
+    cds.iCompression = (u16)iMethod;
     zipfileMtimeToDos(&cds, (u32)mTime);
     cds.crc32 = crc32(0, pData, nData);
     cds.szCompressed = nData;
-    cds.szUncompressed = sz;
+    cds.szUncompressed = (u32)sz;
     cds.iExternalAttr = (mode<<16);
-    cds.iOffset = pTab->szCurrent;
+    cds.iOffset = (u32)pTab->szCurrent;
     pNew = zipfileNewEntry(&cds, zPath, nPath, (u32)mTime);
     if( pNew==0 ){
       rc = SQLITE_NOMEM;
@@ -1291,10 +1303,10 @@ static int zipfileCommit(sqlite3_vtab *pVtab){
     /* Write out the EOCD record */
     eocd.iDisk = 0;
     eocd.iFirstDisk = 0;
-    eocd.nEntry = nEntry;
-    eocd.nEntryTotal = nEntry;
-    eocd.nSize = pTab->szCurrent - iOffset;;
-    eocd.iOffset = iOffset;
+    eocd.nEntry = (u16)nEntry;
+    eocd.nEntryTotal = (u16)nEntry;
+    eocd.nSize = (u32)(pTab->szCurrent - iOffset);
+    eocd.iOffset = (u32)iOffset;
     rc = zipfileAppendEOCD(pTab, &eocd);
 
     zipfileCleanupTransaction(pTab);
diff --git a/manifest b/manifest
index eb4d9552dd..32e1f5b38a 100644
--- a/manifest
+++ b/manifest
@@ -1,10 +1,10 @@
-C Merge\sin\sall\srecent\strunk\senhancements.
-D 2018-01-04T19:54:55.108
+C Get\sthe\sshell\senhancements\scompiling\swith\sMSVC.
+D 2018-01-04T22:46:08.740
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 1b11037c5ed3399a79433cc82c59b5e36a7b3a3e4e195bb27640d0d2145e03e1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 8723bebdec08013054d1ade8f65a13cad34bf8dd015f09649754be2b5f6edc59
+F Makefile.msc feaf722defab458cbf9583249e441239bae08fcf7fb907496c7d3fed16862f21
 F README.md eeae1e552f93ef72ef7c5b8f6647b368a001c28820ad1df179d3dae602bef681
 F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -276,7 +276,7 @@ F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83
 F ext/misc/csv.c 1a009b93650732e22334edc92459c4630b9fa703397cbb3c8ca279921a36ca11
 F ext/misc/dbdump.c 3509fa6b8932d04e932d6b6b827b6a82ca362781b8e8f3c77336f416793e215e
 F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2
-F ext/misc/fileio.c 014152d4133e7b29eab8eb39d0c640659c23a6d23d882b4778f487ae7d1a457b
+F ext/misc/fileio.c 16cf8d9b9372269a61644717929578a71c48eb018a76709f3bf1196bdc957d46
 F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25
 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c
 F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281984
@@ -302,7 +302,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c 2df8f94003903fe3cc104b807418c54e68040964d4319c522ac2f485152f5abd
+F ext/misc/zipfile.c d88033b4748db9929a0096f627d3a75e9fe0e11d7a92724a6c1c575d5448cea4
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -483,7 +483,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in 3e2db269982c4a6f7e8e32ef5620eda718a21a71bb2b5cd73c3ea9b87c6d21bc
+F src/shell.c.in e17f15b3394206e5f0002267426d03fc4932da40af60a48e409238a8ee6c40ec
 F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -541,7 +541,7 @@ F src/test_thread.c 911d15fb14e19c0c542bdc8aabf981c2f10a4858
 F src/test_vfs.c f0186261a24de2671d080bcd8050732f0cb64f6e
 F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698
 F src/test_windirent.c 17f91f5f2aa1bb7328abb49414c363b5d2a9d3ff
-F src/test_windirent.h 5d67483a55442e31e1bde0f4a230e6e932ad5906
+F src/test_windirent.h e3de7323538e5233dfffd33538fc7e9d56c85e266d36540adb1cedb566e14eea
 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
 F src/tokenize.c 1003d6d90c6783206c711f0a9397656fa5b055209f4d092caa43bb3bf5215db5
@@ -1224,7 +1224,7 @@ F test/shell4.test 89ad573879a745974ff2df20ff97c5d6ffffbd5d
 F test/shell5.test 23939a4c51f0421330ea61dbd3c74f9c215f5f8d3d1a94846da6ffc777a35458
 F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3
 F test/shell7.test 115132f66d0463417f408562cc2cf534f6bbc6d83a6d50f0072a9eb171bae97f
-F test/shell8.test 96f35965fe84d633fb2338696f5cbc1bcf6bdbdd79677244bc617a8452851dc7
+F test/shell8.test 7585847402452d594f0e5f93430d34ed63b2f34ca7e956f63db157f9327c6896
 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3
 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5
 F test/shrink.test 1b4330b1fd9e818c04726d45cb28db73087535ce
@@ -1694,7 +1694,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 01d4e866fb7b01aeada537d41c4a47747c7810e2028f51077ee5b8b78c348954 a6eee0fcd89d3958f8720ebdb5f0a8558b4795d747128091dae283eb81c4f74f
-R d4f5940088399233b1f4f56f19294263
-U drh
-Z c1193b259bc575590a62665cc9281689
+P 406f79183736b6ad360169b837172afef2c82a4312f5787db08c54167a44b15e
+R 73aa9301733dc3abdd11287ef3222fd8
+U mistachkin
+Z 751be0dac1987ad178cccedeeaed1d64
diff --git a/manifest.uuid b/manifest.uuid
index 0fefe4597b..f23cbae652 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-406f79183736b6ad360169b837172afef2c82a4312f5787db08c54167a44b15e
\ No newline at end of file
+335387f9e0d4569097d34cd99cd332b38a282e9b7ae25f088eb47df5c25837ef
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 6f1c92e006..21ca7edbb8 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -73,8 +73,9 @@
 #  include <pwd.h>
 # endif
 # include <unistd.h>
-# include <sys/types.h>
 #endif
+#include <sys/types.h>
+#include <sys/stat.h>
 
 #if HAVE_READLINE
 # include <readline/readline.h>
@@ -874,6 +875,11 @@ static void shellAddSchemaName(
 #define SQLITE_EXTENSION_INIT1
 #define SQLITE_EXTENSION_INIT2(X) (void)(X)
 
+#if defined(_WIN32) || defined(WIN32)
+INCLUDE test_windirent.c
+#define dirent DIRENT
+#define timespec TIMESPEC
+#endif
 INCLUDE ../ext/misc/shathree.c
 INCLUDE ../ext/misc/fileio.c
 INCLUDE ../ext/misc/completion.c
@@ -4551,7 +4557,7 @@ static int arParseCommand(
           struct ArSwitch *pOpt;            /* Iterator */
           for(pOpt=&aSwitch[0]; pOpt<pEnd; pOpt++){
             const char *zLong = pOpt->zLong;
-            if( (n-2)<=strlen(zLong) && 0==memcmp(&z[2], zLong, n-2) ){
+            if( (n-2)<=(int)strlen(zLong) && 0==memcmp(&z[2], zLong, n-2) ){
               if( pMatch ){
                 return arErrorMsg("ambiguous option: %s",z);
               }else{
diff --git a/src/test_windirent.h b/src/test_windirent.h
index 578e2a7c22..2303c0c8fb 100644
--- a/src/test_windirent.h
+++ b/src/test_windirent.h
@@ -37,6 +37,33 @@
 #include <errno.h>
 #include <io.h>
 #include <limits.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+
+/*
+** We may need several defines that should have been in "sys/stat.h".
+*/
+
+#ifndef S_ISREG
+#define S_ISREG(mode) (((mode) & S_IFMT) == S_IFREG)
+#endif
+
+#ifndef S_ISDIR
+#define S_ISDIR(mode) (((mode) & S_IFMT) == S_IFDIR)
+#endif
+
+#ifndef S_ISLNK
+#define S_ISLNK(mode) (0)
+#endif
+
+/*
+** We may need to provide the "mode_t" type.
+*/
+
+#ifndef MODE_T_DEFINED
+  #define MODE_T_DEFINED
+  typedef unsigned short mode_t;
+#endif
 
 /*
 ** We may need to provide the "ino_t" type.
@@ -75,22 +102,37 @@
 ** We need to provide the necessary structures and related types.
 */
 
+#ifndef DIRENT_DEFINED
+#define DIRENT_DEFINED
 typedef struct DIRENT DIRENT;
-typedef struct DIR DIR;
 typedef DIRENT *LPDIRENT;
-typedef DIR *LPDIR;
-
 struct DIRENT {
   ino_t d_ino;               /* Sequence number, do not use. */
   unsigned d_attributes;     /* Win32 file attributes. */
   char d_name[NAME_MAX + 1]; /* Name within the directory. */
 };
+#endif
 
+#ifndef DIR_DEFINED
+#define DIR_DEFINED
+typedef struct DIR DIR;
+typedef DIR *LPDIR;
 struct DIR {
   intptr_t d_handle; /* Value returned by "_findfirst". */
   DIRENT d_first;    /* DIRENT constructed based on "_findfirst". */
   DIRENT d_next;     /* DIRENT constructed based on "_findnext". */
 };
+#endif
+
+#ifndef TIMESPEC_DEFINED
+#define TIMESPEC_DEFINED
+typedef struct TIMESPEC TIMESPEC;
+typedef TIMESPEC *LPTIMESPEC;
+struct TIMESPEC {
+  time_t tv_sec; /* Number of whole seconds. */
+  long tv_nsec;  /* Number of whole nanoseconds. */
+};
+#endif
 
 /*
 ** Provide a macro, for use by the implementation, to determine if a
diff --git a/test/shell8.test b/test/shell8.test
index 14980a84a5..50e269ef3e 100644
--- a/test/shell8.test
+++ b/test/shell8.test
@@ -40,7 +40,11 @@ proc dir_to_list {dirname {n -1}} {
   set res [list]
   foreach f [glob -nocomplain $dirname/*] {
     set mtime [file mtime $f]
-    set perm [file attributes $f -perm]
+    if {$::tcl_platform(platform)!="windows"} {
+      set perm [file attributes $f -perm]
+    } else {
+      set perm 0
+    }
     set relpath [file join {*}[lrange [file split $f] $n end]]
     lappend res 
     if {[file isdirectory $f]} {

From e55988074fb9d5bb1936c66909251776482e2d5a Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Thu, 4 Jan 2018 22:50:52 +0000
Subject: [PATCH 219/488] Remove 'timespec' related code from the shell that
 has no effect and a (now) superfluous 'timespec' typedef from the Win32
 dirent header file.

FossilOrigin-Name: 57dac995dd028e4b7c8ce6806f5769831715c1bb9b886318fceb3fa50563d537
---
 manifest             | 14 +++++++-------
 manifest.uuid        |  2 +-
 src/shell.c.in       |  4 ----
 src/test_windirent.h | 10 ----------
 4 files changed, 8 insertions(+), 22 deletions(-)

diff --git a/manifest b/manifest
index 32e1f5b38a..dd247b2868 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Get\sthe\sshell\senhancements\scompiling\swith\sMSVC.
-D 2018-01-04T22:46:08.740
+C Remove\s'timespec'\srelated\scode\sfrom\sthe\sshell\sthat\shas\sno\seffect\sand\sa\s(now)\ssuperfluous\s'timespec'\stypedef\sfrom\sthe\sWin32\sdirent\sheader\sfile.
+D 2018-01-04T22:50:52.352
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 1b11037c5ed3399a79433cc82c59b5e36a7b3a3e4e195bb27640d0d2145e03e1
@@ -483,7 +483,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in e17f15b3394206e5f0002267426d03fc4932da40af60a48e409238a8ee6c40ec
+F src/shell.c.in 84af7b92b6c937a0b4f323719f2069e669bd685a7dcc81a42c3d9b21d040ba23
 F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -541,7 +541,7 @@ F src/test_thread.c 911d15fb14e19c0c542bdc8aabf981c2f10a4858
 F src/test_vfs.c f0186261a24de2671d080bcd8050732f0cb64f6e
 F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698
 F src/test_windirent.c 17f91f5f2aa1bb7328abb49414c363b5d2a9d3ff
-F src/test_windirent.h e3de7323538e5233dfffd33538fc7e9d56c85e266d36540adb1cedb566e14eea
+F src/test_windirent.h 5afeb2b1e2920d5149573a4c30a69cba91f2e8a80e00941b738036fca39aca61
 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
 F src/tokenize.c 1003d6d90c6783206c711f0a9397656fa5b055209f4d092caa43bb3bf5215db5
@@ -1694,7 +1694,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 406f79183736b6ad360169b837172afef2c82a4312f5787db08c54167a44b15e
-R 73aa9301733dc3abdd11287ef3222fd8
+P 335387f9e0d4569097d34cd99cd332b38a282e9b7ae25f088eb47df5c25837ef
+R 1281cdb7398acc29fce34993c7e9c8c8
 U mistachkin
-Z 751be0dac1987ad178cccedeeaed1d64
+Z d412c9c0d87063cd46a5fbdafa7a7569
diff --git a/manifest.uuid b/manifest.uuid
index f23cbae652..860fd9f513 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-335387f9e0d4569097d34cd99cd332b38a282e9b7ae25f088eb47df5c25837ef
\ No newline at end of file
+57dac995dd028e4b7c8ce6806f5769831715c1bb9b886318fceb3fa50563d537
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 21ca7edbb8..9cf62c7c22 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -4755,7 +4755,6 @@ static int arExtractCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
 
 
 
-  struct timespec times[2];
   sqlite3_stmt *pSql = 0;
   int rc = SQLITE_OK;
   char *zDir = 0;
@@ -4777,9 +4776,6 @@ static int arExtractCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
     if( zDir==0 ) rc = SQLITE_NOMEM;
   }
 
-  memset(times, 0, sizeof(times));
-  times[0].tv_sec = time(0);
-
   shellPreparePrintf(db, &rc, &pSql, zSql1, 
       azExtraArg[pAr->bZip], azSource[pAr->bZip], zWhere
   );
diff --git a/src/test_windirent.h b/src/test_windirent.h
index 2303c0c8fb..d4474f80e0 100644
--- a/src/test_windirent.h
+++ b/src/test_windirent.h
@@ -124,16 +124,6 @@ struct DIR {
 };
 #endif
 
-#ifndef TIMESPEC_DEFINED
-#define TIMESPEC_DEFINED
-typedef struct TIMESPEC TIMESPEC;
-typedef TIMESPEC *LPTIMESPEC;
-struct TIMESPEC {
-  time_t tv_sec; /* Number of whole seconds. */
-  long tv_nsec;  /* Number of whole nanoseconds. */
-};
-#endif
-
 /*
 ** Provide a macro, for use by the implementation, to determine if a
 ** particular directory entry should be skipped over when searching for

From 104958cab09d00df8ad2a3d6a770e08dfb064811 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Thu, 4 Jan 2018 23:49:08 +0000
Subject: [PATCH 220/488] In the 'fileio' extension code for Win32, use the
 FILE_FLAG_BACKUP_SEMANTICS flag when setting the file times, just in case the
 file is actually a directory.

FossilOrigin-Name: 4f3444060057127bce81787ed83cb5225cdbdd577596bc3fb06a40be2208f238
---
 ext/misc/fileio.c |  3 ++-
 manifest          | 12 ++++++------
 manifest.uuid     |  2 +-
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c
index 2464d18f80..835b53f4fa 100644
--- a/ext/misc/fileio.c
+++ b/ext/misc/fileio.c
@@ -280,7 +280,8 @@ static int writeFile(
     lastWrite.dwLowDateTime = (DWORD)intervals;
     lastWrite.dwHighDateTime = intervals >> 32;
     hFile = CreateFile(
-      zFile, FILE_WRITE_ATTRIBUTES, 0, NULL, OPEN_EXISTING, 0, NULL
+      zFile, FILE_WRITE_ATTRIBUTES, 0, NULL, OPEN_EXISTING,
+      FILE_FLAG_BACKUP_SEMANTICS, NULL
     );
     if( hFile!=INVALID_HANDLE_VALUE ){
       BOOL bResult = SetFileTime(hFile, NULL, &lastAccess, &lastWrite);
diff --git a/manifest b/manifest
index dd247b2868..7dd19634db 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\s'timespec'\srelated\scode\sfrom\sthe\sshell\sthat\shas\sno\seffect\sand\sa\s(now)\ssuperfluous\s'timespec'\stypedef\sfrom\sthe\sWin32\sdirent\sheader\sfile.
-D 2018-01-04T22:50:52.352
+C In\sthe\s'fileio'\sextension\scode\sfor\sWin32,\suse\sthe\sFILE_FLAG_BACKUP_SEMANTICS\sflag\swhen\ssetting\sthe\sfile\stimes,\sjust\sin\scase\sthe\sfile\sis\sactually\sa\sdirectory.
+D 2018-01-04T23:49:08.958
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 1b11037c5ed3399a79433cc82c59b5e36a7b3a3e4e195bb27640d0d2145e03e1
@@ -276,7 +276,7 @@ F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83
 F ext/misc/csv.c 1a009b93650732e22334edc92459c4630b9fa703397cbb3c8ca279921a36ca11
 F ext/misc/dbdump.c 3509fa6b8932d04e932d6b6b827b6a82ca362781b8e8f3c77336f416793e215e
 F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2
-F ext/misc/fileio.c 16cf8d9b9372269a61644717929578a71c48eb018a76709f3bf1196bdc957d46
+F ext/misc/fileio.c 868aa0572b08484323b172918882d5f8444e287b1e29e1d1b161633cf03aed10
 F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25
 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c
 F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281984
@@ -1694,7 +1694,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 335387f9e0d4569097d34cd99cd332b38a282e9b7ae25f088eb47df5c25837ef
-R 1281cdb7398acc29fce34993c7e9c8c8
+P 57dac995dd028e4b7c8ce6806f5769831715c1bb9b886318fceb3fa50563d537
+R 02d2a190c58a7a01c71feee97e36cbb6
 U mistachkin
-Z d412c9c0d87063cd46a5fbdafa7a7569
+Z 088dbce32336167e7acacd4de5ece530
diff --git a/manifest.uuid b/manifest.uuid
index 860fd9f513..d3c6c198e3 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-57dac995dd028e4b7c8ce6806f5769831715c1bb9b886318fceb3fa50563d537
\ No newline at end of file
+4f3444060057127bce81787ed83cb5225cdbdd577596bc3fb06a40be2208f238
\ No newline at end of file

From 3acaf4c07aa9f175334232d6e30091c853c1598d Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Fri, 5 Jan 2018 00:53:03 +0000
Subject: [PATCH 221/488] Add missing #ifdef.

FossilOrigin-Name: a0e18aea0950f5ebdf4112f826ff64d24e8660b341031132dcb65bb15579ef1c
---
 manifest       | 12 ++++++------
 manifest.uuid  |  2 +-
 src/shell.c.in |  2 ++
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index 7dd19634db..dfb9ae6a88 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\sthe\s'fileio'\sextension\scode\sfor\sWin32,\suse\sthe\sFILE_FLAG_BACKUP_SEMANTICS\sflag\swhen\ssetting\sthe\sfile\stimes,\sjust\sin\scase\sthe\sfile\sis\sactually\sa\sdirectory.
-D 2018-01-04T23:49:08.958
+C Add\smissing\s#ifdef.
+D 2018-01-05T00:53:03.495
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 1b11037c5ed3399a79433cc82c59b5e36a7b3a3e4e195bb27640d0d2145e03e1
@@ -483,7 +483,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in 84af7b92b6c937a0b4f323719f2069e669bd685a7dcc81a42c3d9b21d040ba23
+F src/shell.c.in f57fa72ec6aa5e330966adaa51a476f08a877a82e482cd26a06a7d7c1ecb666b
 F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1694,7 +1694,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 57dac995dd028e4b7c8ce6806f5769831715c1bb9b886318fceb3fa50563d537
-R 02d2a190c58a7a01c71feee97e36cbb6
+P 4f3444060057127bce81787ed83cb5225cdbdd577596bc3fb06a40be2208f238
+R c50a08a594d751082d54ba21ce6e9836
 U mistachkin
-Z 088dbce32336167e7acacd4de5ece530
+Z bad7a2c44b50561368222d84fba1c0b2
diff --git a/manifest.uuid b/manifest.uuid
index d3c6c198e3..0737aa4078 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4f3444060057127bce81787ed83cb5225cdbdd577596bc3fb06a40be2208f238
\ No newline at end of file
+a0e18aea0950f5ebdf4112f826ff64d24e8660b341031132dcb65bb15579ef1c
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 9cf62c7c22..1844b39d61 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -4967,7 +4967,9 @@ static int arDotCommand(
         return rc;
       }
       sqlite3_fileio_init(db, 0, 0);
+#ifdef SQLITE_HAVE_ZLIB
       sqlite3_sqlar_init(db, 0, 0);
+#endif
     }else{
       db = pState->db;
     }

From 1152250b448495f777c2e2a34ba1672071777f78 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Fri, 5 Jan 2018 01:00:17 +0000
Subject: [PATCH 222/488] Enhance the 'zlib' build target for MSVC.

FossilOrigin-Name: 0bc3b76ec9b83f3034e282ea0369a53673e2cb64dde42e8cf5e800f6d642d527
---
 Makefile.msc  |  2 +-
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/Makefile.msc b/Makefile.msc
index 4fb128c567..c476f54f9e 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -2080,7 +2080,7 @@ shell.c:	$(SHELL_SRC) $(TOP)\tool\mkshellc.tcl
 	$(TCLSH_CMD) $(TOP)\tool\mkshellc.tcl > shell.c
 
 zlib:
-	pushd $(ZLIBDIR) && $(MAKE) /f win32\Makefile.msc $(ZLIBLIB) && popd
+	pushd $(ZLIBDIR) && $(MAKE) /f win32\Makefile.msc clean $(ZLIBLIB) && popd
 
 # Rules to build the extension objects.
 #
diff --git a/manifest b/manifest
index dfb9ae6a88..c97886d88f 100644
--- a/manifest
+++ b/manifest
@@ -1,10 +1,10 @@
-C Add\smissing\s#ifdef.
-D 2018-01-05T00:53:03.495
+C Enhance\sthe\s'zlib'\sbuild\starget\sfor\sMSVC.
+D 2018-01-05T01:00:17.962
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 1b11037c5ed3399a79433cc82c59b5e36a7b3a3e4e195bb27640d0d2145e03e1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc feaf722defab458cbf9583249e441239bae08fcf7fb907496c7d3fed16862f21
+F Makefile.msc e8b973cd249b0db11c221c1bd7e9f738e8dac369275160342e1011f3932074c9
 F README.md eeae1e552f93ef72ef7c5b8f6647b368a001c28820ad1df179d3dae602bef681
 F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -1694,7 +1694,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 4f3444060057127bce81787ed83cb5225cdbdd577596bc3fb06a40be2208f238
-R c50a08a594d751082d54ba21ce6e9836
+P a0e18aea0950f5ebdf4112f826ff64d24e8660b341031132dcb65bb15579ef1c
+R ae30dd8990fa7c0deb74a2452807863c
 U mistachkin
-Z bad7a2c44b50561368222d84fba1c0b2
+Z 062c561092c9190b73f85a153a52c8b7
diff --git a/manifest.uuid b/manifest.uuid
index 0737aa4078..eacddc6a99 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a0e18aea0950f5ebdf4112f826ff64d24e8660b341031132dcb65bb15579ef1c
\ No newline at end of file
+0bc3b76ec9b83f3034e282ea0369a53673e2cb64dde42e8cf5e800f6d642d527
\ No newline at end of file

From 47be3b23e9c935930cc0dcb76c853512eabadc98 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Fri, 5 Jan 2018 01:22:37 +0000
Subject: [PATCH 223/488] Revise detection of 'tclsh.exe' in the Makefile for
 MSVC.

FossilOrigin-Name: 45fabd868dc690894f5a911d373a3d6410ba2d95d177307a42009afc8ae296cc
---
 Makefile.msc  |  6 +++---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/Makefile.msc b/Makefile.msc
index c476f54f9e..8f0fd1b82d 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -93,12 +93,12 @@ SPLIT_AMALGAMATION = 0
 
 # <<mark>>
 # Set this non-0 to have this makefile assume the Tcl shell executable
-# (tclsh*.exe) is available in the PATH.  By default, this is enabled
+# (tclsh*.exe) is available in the PATH.  By default, this is disabled
 # for compatibility with older build environments.  This setting only
 # applies if TCLSH_CMD is not set manually.
 #
 !IFNDEF USE_TCLSH_IN_PATH
-USE_TCLSH_IN_PATH = 1
+USE_TCLSH_IN_PATH = 0
 !ENDIF
 
 # Set this non-0 to use zlib, possibly compiling it from source code.
@@ -909,7 +909,7 @@ LIBICU = icuuc.lib icuin.lib
 # specific Tcl shell to use.
 #
 !IFNDEF TCLSH_CMD
-!IF $(USE_TCLSH_IN_PATH)!=0
+!IF $(USE_TCLSH_IN_PATH)!=0 || !EXIST("$(TCLDIR)\bin\tclsh.exe")
 TCLSH_CMD = tclsh
 !ELSE
 TCLSH_CMD = $(TCLDIR)\bin\tclsh.exe
diff --git a/manifest b/manifest
index c97886d88f..13547fdc1b 100644
--- a/manifest
+++ b/manifest
@@ -1,10 +1,10 @@
-C Enhance\sthe\s'zlib'\sbuild\starget\sfor\sMSVC.
-D 2018-01-05T01:00:17.962
+C Revise\sdetection\sof\s'tclsh.exe'\sin\sthe\sMakefile\sfor\sMSVC.
+D 2018-01-05T01:22:37.774
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 1b11037c5ed3399a79433cc82c59b5e36a7b3a3e4e195bb27640d0d2145e03e1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc e8b973cd249b0db11c221c1bd7e9f738e8dac369275160342e1011f3932074c9
+F Makefile.msc c4eaeff29b73d76437cf9c13d7a9654fc0e9e2578e20c12d0067c3d510e4e8e4
 F README.md eeae1e552f93ef72ef7c5b8f6647b368a001c28820ad1df179d3dae602bef681
 F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -1694,7 +1694,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a0e18aea0950f5ebdf4112f826ff64d24e8660b341031132dcb65bb15579ef1c
-R ae30dd8990fa7c0deb74a2452807863c
+P 0bc3b76ec9b83f3034e282ea0369a53673e2cb64dde42e8cf5e800f6d642d527
+R ea85632748cb8c061fa075c4e05b7d5d
 U mistachkin
-Z 062c561092c9190b73f85a153a52c8b7
+Z a7a75d6fbb0bbc541be826de9dbf52e8
diff --git a/manifest.uuid b/manifest.uuid
index eacddc6a99..d4f3a5c95c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0bc3b76ec9b83f3034e282ea0369a53673e2cb64dde42e8cf5e800f6d642d527
\ No newline at end of file
+45fabd868dc690894f5a911d373a3d6410ba2d95d177307a42009afc8ae296cc
\ No newline at end of file

From 8503d6451b9f09c0a862f14f909e5a0d453ab8d7 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 5 Jan 2018 07:57:54 +0000
Subject: [PATCH 224/488] Fix an LSM crash that could occur if
 LSM_CONFIG_AUTOFLUSH was set to 0.

FossilOrigin-Name: 05346f83d587e6049da0e8ec5f62c749daa9e80359cf48f8c37e333e6a1e7d2a
---
 ext/lsm1/lsm-test/lsmtest.h      |  1 +
 ext/lsm1/lsm-test/lsmtest_tdb.c  |  1 +
 ext/lsm1/lsm-test/lsmtest_tdb3.c | 13 +++++++++++++
 ext/lsm1/lsm_main.c              |  2 +-
 manifest                         | 20 ++++++++++----------
 manifest.uuid                    |  2 +-
 6 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/ext/lsm1/lsm-test/lsmtest.h b/ext/lsm1/lsm-test/lsmtest.h
index 249bc999e0..ca60424add 100644
--- a/ext/lsm1/lsm-test/lsmtest.h
+++ b/ext/lsm1/lsm-test/lsmtest.h
@@ -121,6 +121,7 @@ int test_mdb_scan(TestDb *, void *, int, void *, int, void *, int,
 */
 int test_lsm_open(const char*, const char *zFile, int bClear, TestDb **ppDb);
 int test_lsm_lomem_open(const char*, const char*, int bClear, TestDb **ppDb);
+int test_lsm_lomem2_open(const char*, const char*, int bClear, TestDb **ppDb);
 int test_lsm_zip_open(const char*, const char*, int bClear, TestDb **ppDb);
 int test_lsm_small_open(const char*, const char*, int bClear, TestDb **ppDb);
 int test_lsm_mt2(const char*, const char *zFile, int bClear, TestDb **ppDb);
diff --git a/ext/lsm1/lsm-test/lsmtest_tdb.c b/ext/lsm1/lsm-test/lsmtest_tdb.c
index 8377bc2ed2..9c4f9df8a4 100644
--- a/ext/lsm1/lsm-test/lsmtest_tdb.c
+++ b/ext/lsm1/lsm-test/lsmtest_tdb.c
@@ -721,6 +721,7 @@ static struct Lib {
   { "sqlite3",      "testdb.sqlite",    sql_open },
   { "lsm_small",    "testdb.lsm_small", test_lsm_small_open },
   { "lsm_lomem",    "testdb.lsm_lomem", test_lsm_lomem_open },
+  { "lsm_lomem2",   "testdb.lsm_lomem2", test_lsm_lomem2_open },
 #ifdef HAVE_ZLIB
   { "lsm_zip",      "testdb.lsm_zip",   test_lsm_zip_open },
 #endif
diff --git a/ext/lsm1/lsm-test/lsmtest_tdb3.c b/ext/lsm1/lsm-test/lsmtest_tdb3.c
index 1862023bad..c21e243b5c 100644
--- a/ext/lsm1/lsm-test/lsmtest_tdb3.c
+++ b/ext/lsm1/lsm-test/lsmtest_tdb3.c
@@ -1033,6 +1033,19 @@ int test_lsm_lomem_open(
   return testLsmOpen(zCfg, zFilename, bClear, ppDb);
 }
 
+int test_lsm_lomem2_open(
+  const char *zSpec, 
+  const char *zFilename, 
+  int bClear, 
+  TestDb **ppDb
+){
+    /* "max_freelist=4 autocheckpoint=32" */
+  const char *zCfg = 
+    "page_size=512 block_size=64 autoflush=0 mmap=0 "
+  ;
+  return testLsmOpen(zCfg, zFilename, bClear, ppDb);
+}
+
 int test_lsm_zip_open(
   const char *zSpec, 
   const char *zFilename, 
diff --git a/ext/lsm1/lsm_main.c b/ext/lsm1/lsm_main.c
index 539883c469..a9c48e004e 100644
--- a/ext/lsm1/lsm_main.c
+++ b/ext/lsm1/lsm_main.c
@@ -683,7 +683,7 @@ static int doWriteOp(
     int nDiff;
 
     if( nQuant>pDb->nTreeLimit ){
-      nQuant = pDb->nTreeLimit;
+      nQuant = LSM_MAX(pDb->nTreeLimit, pgsz);
     }
 
     nBefore = lsmTreeSize(pDb);
diff --git a/manifest b/manifest
index 3552e5e8ce..f1eb6f56c5 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\sthe\ssnarky\s"_supported_"\squalifier\sfrom\sthe\sname\sof\sthe\nsqlite_offset()\sSQL\sfunction.
-D 2018-01-04T19:20:37.203
+C Fix\san\sLSM\scrash\sthat\scould\soccur\sif\sLSM_CONFIG_AUTOFLUSH\swas\sset\sto\s0.
+D 2018-01-05T07:57:54.787
 F Makefile.in 1b11037c5ed3399a79433cc82c59b5e36a7b3a3e4e195bb27640d0d2145e03e1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc f68b4f9b83cfeb057b6265e0288ad653f319e2ceacca731e0f22e19617829a89
@@ -221,7 +221,7 @@ F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37
 F ext/lsm1/Makefile 98b0a24b45e248283d6bea4b6cb3e58d7b394edd8e96a0ac28c5fa5104813bad
 F ext/lsm1/Makefile.msc f8c878b467232226de288da320e1ac71c131f5ec91e08b21f502303347260013
 F ext/lsm1/lsm-test/README 87ea529d2abe615e856d4714bfe8bb185e6c2771b8612aa6298588b7b43e6f86
-F ext/lsm1/lsm-test/lsmtest.h 5847594d4b43ec3412e1fd97104f7eb5fd770be55e691e6cb2e80929f86bebe3
+F ext/lsm1/lsm-test/lsmtest.h cf58528ffe0cfe535e91b44584e2ec5fb1caacdabecef0d8dcf83bf83168bf28
 F ext/lsm1/lsm-test/lsmtest1.c 33158978327f800e82b6a47c09b86ace809f56a9ff10b0162273ec1186cc3153
 F ext/lsm1/lsm-test/lsmtest2.c 188b09aec776516aeedcfd13b9c6faf85ba16b3671a0897a2c740ee00a5dc4f8
 F ext/lsm1/lsm-test/lsmtest3.c 9ab87528a36dbf4a61d7c8ad954f5ee368c0878c127b84b942b2e2abe522de26
@@ -237,10 +237,10 @@ F ext/lsm1/lsm-test/lsmtest_func.c 159aa401bc8032bfa3d8cf2977bd687abebab88025589
 F ext/lsm1/lsm-test/lsmtest_io.c cf11b27b129c6bd5818fa1d440176502dc27229f0db892b4479118d61993ea20
 F ext/lsm1/lsm-test/lsmtest_main.c a9bc647738c0dcaebf205d6d194b3ce4a6ef3925801cd2d919f0a4ea33a15aeb
 F ext/lsm1/lsm-test/lsmtest_mem.c 4e63c764345ab1df59d4f13a77980c6f3643798210b10d6cdbd785b4b888fda5
-F ext/lsm1/lsm-test/lsmtest_tdb.c 555fb101d2fe638abdd133e9294536c857fb38e0f227e049c00a67f51eaece06
+F ext/lsm1/lsm-test/lsmtest_tdb.c 618a8619183fda4f5540fcde15f9068293c5e3180e1a246e34409b0c148758b3
 F ext/lsm1/lsm-test/lsmtest_tdb.h 8733eee249b12956a9df8322994b43d19bd8c02ad2e8b0bb5164db4d6ccc1735
 F ext/lsm1/lsm-test/lsmtest_tdb2.cc 99ea7f2dd9c7536c8fb9bdd329e4cfeb76899f3ddf6f48bdd3926e016922b715
-F ext/lsm1/lsm-test/lsmtest_tdb3.c e44bf94e8bd724cd6ac161fd2cd44ffe43193932ad3a6bee1b07d80bb74012bb
+F ext/lsm1/lsm-test/lsmtest_tdb3.c b4e46b1d2fec553fe4efb44e341b43abd20556fde610db0cfffdc2300b72defe
 F ext/lsm1/lsm-test/lsmtest_tdb4.c 47e8bb5eba266472d690fb8264f1855ebdba0ae5a0e541e35fcda61ebf1d277f
 F ext/lsm1/lsm-test/lsmtest_util.c 241622db5a332a09c8e6e7606b617d288a37b557f7d3bce0bb97809f67cc2806
 F ext/lsm1/lsm-test/lsmtest_win32.c 0e0a224674c4d3170631c41b026b56c7e1672b151f5261e1b4cc19068641da2d
@@ -249,7 +249,7 @@ F ext/lsm1/lsmInt.h 5983690e05e83653cc01ba9d8fbf8455e534ddf8349ed9adedbf46a75497
 F ext/lsm1/lsm_ckpt.c 0eabfaf812ddb4ea43add38f05e430694cd054eb622c3e35af4c43118a2d5321
 F ext/lsm1/lsm_file.c 3c51841d5b3e7da162693cbac9a9f47eeedf6bcbbe2969a4d25e30c428c9fe36
 F ext/lsm1/lsm_log.c a8bf334532109bba05b09a504ee45fc393828b0d034ca61ab45e3940709d9a7c
-F ext/lsm1/lsm_main.c 801295038b548ae2e5fae93f08c3f945154f40848a03ff26b16eab5d04ba573a
+F ext/lsm1/lsm_main.c b5703f8042e71d3a2d65e671f6832e077e79e89e9975818f67f969922618db63
 F ext/lsm1/lsm_mem.c 4c51ea9fa285ee6e35301b33491642d071740a0a
 F ext/lsm1/lsm_mutex.c 378edf0a2b142b4f7640ee982df06d50b98788ea
 F ext/lsm1/lsm_shared.c 76adfc1ed9ffebaf92746dde4b370ccc48143ca8b05b563816eadd2aadf1c525
@@ -1688,7 +1688,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d91e3f3d343d281af374dd23eea333e61228539023ad5c5aea622085e5863bc7
-R 314fe8cfc8ec7f02f3acb6e4d50707fb
-U drh
-Z 7ea7a34ef1d7c0cfd34c8e9e5f226383
+P a6eee0fcd89d3958f8720ebdb5f0a8558b4795d747128091dae283eb81c4f74f
+R 2eb065da8ca94639b242562f929b4d2b
+U dan
+Z 4b1a53b69d218d7d45a85af2c33ced1c
diff --git a/manifest.uuid b/manifest.uuid
index 561f8b3f92..d679b61f5b 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a6eee0fcd89d3958f8720ebdb5f0a8558b4795d747128091dae283eb81c4f74f
\ No newline at end of file
+05346f83d587e6049da0e8ec5f62c749daa9e80359cf48f8c37e333e6a1e7d2a
\ No newline at end of file

From 303b02cc0b3a3a85409a8b34e81db5eace1de9c8 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 5 Jan 2018 11:34:18 +0000
Subject: [PATCH 225/488] Fix an LSM problem causing the wrong amount of
 "auto-work" to be performed under fairly obscure circumstances.

FossilOrigin-Name: a4876672edea4e96103efd2463ce9a34a0b994a8744c941660940578aafbd454
---
 ext/lsm1/lsm_sorted.c |  5 ++---
 manifest              | 12 ++++++------
 manifest.uuid         |  2 +-
 3 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/ext/lsm1/lsm_sorted.c b/ext/lsm1/lsm_sorted.c
index 171d0ec05c..de962b8808 100644
--- a/ext/lsm1/lsm_sorted.c
+++ b/ext/lsm1/lsm_sorted.c
@@ -5252,16 +5252,15 @@ static int doLsmSingleWork(
   /* If the in-memory part of the free-list is too large, write a new 
   ** top-level containing just the in-memory free-list entries to disk. */
   if( rc==LSM_OK && pDb->pWorker->freelist.nEntry > pDb->nMaxFreelist ){
-    int nPg = 0;
     while( rc==LSM_OK && lsmDatabaseFull(pDb) ){
+      int nPg = 0;
       rc = sortedWork(pDb, 16, nMerge, 1, &nPg);
       nRem -= nPg;
     }
     if( rc==LSM_OK ){
       rc = sortedNewFreelistOnly(pDb);
     }
-    nRem -= nPg;
-    if( nPg ) bDirty = 1;
+    bDirty = 1;
   }
 
   if( rc==LSM_OK ){
diff --git a/manifest b/manifest
index f1eb6f56c5..ba1764d8aa 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\san\sLSM\scrash\sthat\scould\soccur\sif\sLSM_CONFIG_AUTOFLUSH\swas\sset\sto\s0.
-D 2018-01-05T07:57:54.787
+C Fix\san\sLSM\sproblem\scausing\sthe\swrong\samount\sof\s"auto-work"\sto\sbe\sperformed\nunder\sfairly\sobscure\scircumstances.
+D 2018-01-05T11:34:18.736
 F Makefile.in 1b11037c5ed3399a79433cc82c59b5e36a7b3a3e4e195bb27640d0d2145e03e1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc f68b4f9b83cfeb057b6265e0288ad653f319e2ceacca731e0f22e19617829a89
@@ -253,7 +253,7 @@ F ext/lsm1/lsm_main.c b5703f8042e71d3a2d65e671f6832e077e79e89e9975818f67f9699226
 F ext/lsm1/lsm_mem.c 4c51ea9fa285ee6e35301b33491642d071740a0a
 F ext/lsm1/lsm_mutex.c 378edf0a2b142b4f7640ee982df06d50b98788ea
 F ext/lsm1/lsm_shared.c 76adfc1ed9ffebaf92746dde4b370ccc48143ca8b05b563816eadd2aadf1c525
-F ext/lsm1/lsm_sorted.c d07ff7c28758542b8b4da4b5a1fb67b22a4d33e50e7f684cffe1f6c45cf5182c
+F ext/lsm1/lsm_sorted.c df7b393d9e4b85e6ad07181c6434d58a69a9145ee925d89c3ee1d2a210adaf9a
 F ext/lsm1/lsm_str.c 65e361b488c87b10bf3e5c0070b14ffc602cf84f094880bece77bbf6678bca82
 F ext/lsm1/lsm_tree.c 682679d7ef2b8b6f2fe77aeb532c8d29695bca671c220b0abac77069de5fb9fb
 F ext/lsm1/lsm_unix.c 57361bcf5b1a1a028f5d66571ee490e9064d2cfb145a2cc9e5ddade467bb551b
@@ -1688,7 +1688,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a6eee0fcd89d3958f8720ebdb5f0a8558b4795d747128091dae283eb81c4f74f
-R 2eb065da8ca94639b242562f929b4d2b
+P 05346f83d587e6049da0e8ec5f62c749daa9e80359cf48f8c37e333e6a1e7d2a
+R 3aef1a930039ecc41e90c5782e1f14e5
 U dan
-Z 4b1a53b69d218d7d45a85af2c33ced1c
+Z 26afacfab65aa4f19216c821cd8450d4
diff --git a/manifest.uuid b/manifest.uuid
index d679b61f5b..78301508b4 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-05346f83d587e6049da0e8ec5f62c749daa9e80359cf48f8c37e333e6a1e7d2a
\ No newline at end of file
+a4876672edea4e96103efd2463ce9a34a0b994a8744c941660940578aafbd454
\ No newline at end of file

From af2770f842d025b8b11c704688a937d52faa6ee2 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 5 Jan 2018 14:55:43 +0000
Subject: [PATCH 226/488] Fix compiler warnings.

FossilOrigin-Name: 364ac333b030f0e3372937df723f1098183da87913ba0e8ae162864ee24a50d3
---
 ext/misc/fileio.c |  5 ++---
 manifest          | 16 ++++++++--------
 manifest.uuid     |  2 +-
 src/shell.c.in    | 33 +++++++++++++++------------------
 4 files changed, 26 insertions(+), 30 deletions(-)

diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c
index 835b53f4fa..f6df42f98e 100644
--- a/ext/misc/fileio.c
+++ b/ext/misc/fileio.c
@@ -176,12 +176,11 @@ static int makeDirectory(
   if( zCopy==0 ){
     rc = SQLITE_NOMEM;
   }else{
-    int nCopy = strlen(zCopy);
+    int nCopy = (int)strlen(zCopy);
     int i = 1;
 
     while( rc==SQLITE_OK ){
       struct stat sStat;
-      int rc;
 
       for(; zCopy[i]!='/' && i<nCopy; i++);
       if( i==nCopy ) break;
@@ -634,7 +633,7 @@ static int fsdirFilter(
     pCur->zBase = (const char*)sqlite3_value_text(argv[1]);
   }
   if( pCur->zBase ){
-    pCur->nBase = strlen(pCur->zBase)+1;
+    pCur->nBase = (int)strlen(pCur->zBase)+1;
     pCur->zPath = sqlite3_mprintf("%s/%s", pCur->zBase, zDir);
   }else{
     pCur->zPath = sqlite3_mprintf("%s", zDir);
diff --git a/manifest b/manifest
index 13547fdc1b..2557324a67 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Revise\sdetection\sof\s'tclsh.exe'\sin\sthe\sMakefile\sfor\sMSVC.
-D 2018-01-05T01:22:37.774
+C Fix\scompiler\swarnings.
+D 2018-01-05T14:55:43.233
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 1b11037c5ed3399a79433cc82c59b5e36a7b3a3e4e195bb27640d0d2145e03e1
@@ -276,7 +276,7 @@ F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83
 F ext/misc/csv.c 1a009b93650732e22334edc92459c4630b9fa703397cbb3c8ca279921a36ca11
 F ext/misc/dbdump.c 3509fa6b8932d04e932d6b6b827b6a82ca362781b8e8f3c77336f416793e215e
 F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2
-F ext/misc/fileio.c 868aa0572b08484323b172918882d5f8444e287b1e29e1d1b161633cf03aed10
+F ext/misc/fileio.c 864984342ede6775976322a3a2731be5e3a70a2c71557e489e1730bd8ca57dd2
 F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25
 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c
 F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281984
@@ -483,7 +483,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in f57fa72ec6aa5e330966adaa51a476f08a877a82e482cd26a06a7d7c1ecb666b
+F src/shell.c.in 17fc28661aae277767db63fa90e644b8c9bf1242fbd167dcc2e7af0f0e620bb7
 F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1694,7 +1694,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0bc3b76ec9b83f3034e282ea0369a53673e2cb64dde42e8cf5e800f6d642d527
-R ea85632748cb8c061fa075c4e05b7d5d
-U mistachkin
-Z a7a75d6fbb0bbc541be826de9dbf52e8
+P 45fabd868dc690894f5a911d373a3d6410ba2d95d177307a42009afc8ae296cc
+R 67924405c342403978f2d6d8e94eaa8e
+U drh
+Z e8ddc1cfcd8bdce50b891a625082402b
diff --git a/manifest.uuid b/manifest.uuid
index d4f3a5c95c..9b6d19bf86 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-45fabd868dc690894f5a911d373a3d6410ba2d95d177307a42009afc8ae296cc
\ No newline at end of file
+364ac333b030f0e3372937df723f1098183da87913ba0e8ae162864ee24a50d3
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 1844b39d61..eb5f0b42ed 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -1955,7 +1955,7 @@ static void displayLinuxIoStats(FILE *out){
     };
     int i;
     for(i=0; i<ArraySize(aTrans); i++){
-      int n = (int)strlen(aTrans[i].zPattern);
+      int n = strlen30(aTrans[i].zPattern);
       if( strncmp(aTrans[i].zPattern, z, n)==0 ){
         utf8_printf(out, "%-36s %s", aTrans[i].zDesc, &z[n]);
         break;
@@ -3134,7 +3134,7 @@ static char **readline_completion(const char *zText, int iStart, int iEnd){
 ** Linenoise completion callback
 */
 static void linenoise_completion(const char *zLine, linenoiseCompletions *lc){
-  int nLine = (int)strlen(zLine);
+  int nLine = strlen30(zLine);
   int i, iStart;
   sqlite3_stmt *pStmt = 0;
   char *zSql;
@@ -3360,7 +3360,7 @@ static int sql_trace_callback(
   UNUSED_PARAMETER(pP);
   if( f ){
     const char *z = (const char*)pX;
-    int i = (int)strlen(z);
+    int i = strlen30(z);
     while( i>0 && z[i-1]==';' ){ i--; }
     utf8_printf(f, "%.*s;\n", i, z);
   }
@@ -3549,7 +3549,7 @@ static void tryToCloneData(
   char *zInsert = 0;
   int rc;
   int i, j, n;
-  int nTable = (int)strlen(zTable);
+  int nTable = strlen30(zTable);
   int k = 0;
   int cnt = 0;
   const int spinRate = 10000;
@@ -3570,7 +3570,7 @@ static void tryToCloneData(
   }
   sqlite3_snprintf(200+nTable,zInsert,
                    "INSERT OR IGNORE INTO \"%s\" VALUES(?", zTable);
-  i = (int)strlen(zInsert);
+  i = strlen30(zInsert);
   for(j=1; j<n; j++){
     memcpy(zInsert+i, ",?", 2);
     i += 2;
@@ -4152,7 +4152,7 @@ static int lintFkeyIndexes(
   const char *zGlobIPK = "SEARCH TABLE * USING INTEGER PRIMARY KEY (rowid=?)";
 
   for(i=2; i<nArg; i++){
-    int n = (int)strlen(azArg[i]);
+    int n = strlen30(azArg[i]);
     if( n>1 && sqlite3_strnicmp("-verbose", azArg[i], n)==0 ){
       bVerbose = 1;
     }
@@ -4255,7 +4255,7 @@ static int lintDotCommand(
   int nArg                        /* Number of entries in azArg[] */
 ){
   int n;
-  n = (nArg>=2 ? (int)strlen(azArg[1]) : 0);
+  n = (nArg>=2 ? strlen30(azArg[1]) : 0);
   if( n<1 || sqlite3_strnicmp(azArg[1], "fkey-indexes", n) ) goto usage;
   return lintFkeyIndexes(pState, azArg, nArg);
 
@@ -4517,7 +4517,7 @@ static int arParseCommand(
           pAr->nArg = nArg-iArg;
           break;
         }
-        n = strlen(z);
+        n = strlen30(z);
 
         if( z[1]!='-' ){
           int i;
@@ -4557,7 +4557,7 @@ static int arParseCommand(
           struct ArSwitch *pOpt;            /* Iterator */
           for(pOpt=&aSwitch[0]; pOpt<pEnd; pOpt++){
             const char *zLong = pOpt->zLong;
-            if( (n-2)<=(int)strlen(zLong) && 0==memcmp(&z[2], zLong, n-2) ){
+            if( (n-2)<=strlen30(zLong) && 0==memcmp(&z[2], zLong, n-2) ){
               if( pMatch ){
                 return arErrorMsg("ambiguous option: %s",z);
               }else{
@@ -4610,7 +4610,7 @@ static int arCheckEntries(sqlite3 *db, ArCommand *pAr){
     }
     for(i=0; i<pAr->nArg && rc==SQLITE_OK; i++){
       char *z = pAr->azArg[i];
-      int n = strlen(z);
+      int n = strlen30(z);
       int bOk = 0;
       while( n>0 && z[n-1]=='/' ) n--;
       z[n] = '\0';
@@ -5025,7 +5025,7 @@ static int expertDotCommand(
     char *z = azArg[i];
     int n;
     if( z[0]=='-' && z[1]=='-' ) z++;
-    n = strlen(z);
+    n = strlen30(z);
     if( n>=2 && 0==strncmp(z, "-verbose", n) ){
       pState->expert.bVerbose = 1;
     }
@@ -5908,7 +5908,7 @@ static int do_meta_command(char *zLine, ShellState *p){
 
   if( c=='m' && strncmp(azArg[0], "mode", n)==0 ){
     const char *zMode = nArg>=2 ? azArg[1] : "";
-    int n2 = (int)strlen(zMode);
+    int n2 = strlen30(zMode);
     int c2 = zMode[0];
     if( c2=='l' && n2>2 && strncmp(azArg[1],"lines",n2)==0 ){
       p->mode = MODE_Line;
@@ -7147,8 +7147,7 @@ static int do_meta_command(char *zLine, ShellState *p){
         rc = 1;
         goto meta_command_exit;
       }
-      rc = sqlite3_user_authenticate(p->db, azArg[2], azArg[3],
-                                    (int)strlen(azArg[3]));
+      rc = sqlite3_user_authenticate(p->db, azArg[2], azArg[3], strlen30(azArg[3]));
       if( rc ){
         utf8_printf(stderr, "Authentication failed for user %s\n", azArg[2]);
         rc = 1;
@@ -7159,8 +7158,7 @@ static int do_meta_command(char *zLine, ShellState *p){
         rc = 1;
         goto meta_command_exit;
       }
-      rc = sqlite3_user_add(p->db, azArg[2],
-                            azArg[3], (int)strlen(azArg[3]),
+      rc = sqlite3_user_add(p->db, azArg[2], azArg[3], strlen30(azArg[3]),
                             booleanValue(azArg[4]));
       if( rc ){
         raw_printf(stderr, "User-Add failed: %d\n", rc);
@@ -7172,8 +7170,7 @@ static int do_meta_command(char *zLine, ShellState *p){
         rc = 1;
         goto meta_command_exit;
       }
-      rc = sqlite3_user_change(p->db, azArg[2],
-                              azArg[3], (int)strlen(azArg[3]),
+      rc = sqlite3_user_change(p->db, azArg[2], azArg[3], strlen30(azArg[3]),
                               booleanValue(azArg[4]));
       if( rc ){
         raw_printf(stderr, "User-Edit failed: %d\n", rc);

From 91f7211caa24dc0a6f83b7b81a386178111c4b8d Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Fri, 5 Jan 2018 16:03:52 +0000
Subject: [PATCH 227/488] Add some missing #ifdefs for building without zlib.

FossilOrigin-Name: c63fb1700c0f67d90857b1f3859c203880d0939f356a64d8ae7cafc814ea72bf
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/test1.c   |  4 ++++
 3 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 2557324a67..53a4f9a14e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\scompiler\swarnings.
-D 2018-01-05T14:55:43.233
+C Add\ssome\smissing\s#ifdefs\sfor\sbuilding\swithout\szlib.
+D 2018-01-05T16:03:52.073
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 1b11037c5ed3399a79433cc82c59b5e36a7b3a3e4e195bb27640d0d2145e03e1
@@ -492,7 +492,7 @@ F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
 F src/tclsqlite.c 1833388c01e3b77f4c712185ee7250b9423ee0981ce6ae7e401e47db0319a696
-F src/test1.c 1879ff5095def6091f83f54c6233bc19e2b7223068fb02da41f2396d55729764
+F src/test1.c b52f9e7fe62016d357c3266fcfa0793cc1883d3cb2b11dfa39fcba2e70b0305c
 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5
 F src/test3.c b8434949dfb8aff8dfa082c8b592109e77844c2135ed3c492113839b6956255b
 F src/test4.c 18ec393bb4d0ad1de729f0b94da7267270f3d8e6
@@ -1694,7 +1694,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 45fabd868dc690894f5a911d373a3d6410ba2d95d177307a42009afc8ae296cc
-R 67924405c342403978f2d6d8e94eaa8e
-U drh
-Z e8ddc1cfcd8bdce50b891a625082402b
+P 364ac333b030f0e3372937df723f1098183da87913ba0e8ae162864ee24a50d3
+R 0ea7c4065b03d537986e987f96237bd2
+U mistachkin
+Z 7bfc76075fe03643e9086a52bceca2fa
diff --git a/manifest.uuid b/manifest.uuid
index 9b6d19bf86..cff8285416 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-364ac333b030f0e3372937df723f1098183da87913ba0e8ae162864ee24a50d3
\ No newline at end of file
+c63fb1700c0f67d90857b1f3859c203880d0939f356a64d8ae7cafc814ea72bf
\ No newline at end of file
diff --git a/src/test1.c b/src/test1.c
index f9b3b69711..55d92eb1fd 100644
--- a/src/test1.c
+++ b/src/test1.c
@@ -6960,7 +6960,9 @@ static int SQLITE_TCLAPI tclLoadStaticExtensionCmd(
   extern int sqlite3_totype_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_wholenumber_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_unionvtab_init(sqlite3*,char**,const sqlite3_api_routines*);
+#ifdef SQLITE_HAVE_ZLIB
   extern int sqlite3_zipfile_init(sqlite3*,char**,const sqlite3_api_routines*);
+#endif
   static const struct {
     const char *zExtName;
     int (*pInit)(sqlite3*,char**,const sqlite3_api_routines*);
@@ -6982,7 +6984,9 @@ static int SQLITE_TCLAPI tclLoadStaticExtensionCmd(
     { "totype",                sqlite3_totype_init               },
     { "unionvtab",             sqlite3_unionvtab_init            },
     { "wholenumber",           sqlite3_wholenumber_init          },
+#ifdef SQLITE_HAVE_ZLIB
     { "zipfile",               sqlite3_zipfile_init              },
+#endif
   };
   sqlite3 *db;
   const char *zName;

From 590522c18e06f6e18910c23e20535b43ac306a64 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Fri, 5 Jan 2018 16:05:51 +0000
Subject: [PATCH 228/488] In the Makefile for MSVC, only attempt to link
 against Tcl when it is needed.  Also, only compile 'zipfile.c' when use of
 zlib is enabled.

FossilOrigin-Name: a33ad33cf031edee273afa4735c8564870465be22962a9c277e4ac43a307ff2c
---
 Makefile.msc  | 28 +++++++++++++++++++---------
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 3 files changed, 26 insertions(+), 16 deletions(-)

diff --git a/Makefile.msc b/Makefile.msc
index 8f0fd1b82d..e2a6bbe86d 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -1148,8 +1148,8 @@ LDFLAGS = $(LDOPTS)
 # Start with the Tcl related linker options.
 #
 !IF $(NO_TCL)==0
-LTLIBPATHS = /LIBPATH:$(TCLLIBDIR)
-LTLIBS = $(LTLIBS) $(LIBTCL)
+TCLLIBPATHS = $(TCLLIBPATHS) /LIBPATH:$(TCLLIBDIR)
+TCLLIBS = $(TCLLIBS) $(LIBTCL)
 !ENDIF
 
 # If zlib support is enabled, add the linker options for it.
@@ -1506,8 +1506,13 @@ TESTEXT = \
   $(TOP)\ext\misc\spellfix.c \
   $(TOP)\ext\misc\totype.c \
   $(TOP)\ext\misc\unionvtab.c \
-  $(TOP)\ext\misc\wholenumber.c \
-  $(TOP)\ext\misc\zipfile.c
+  $(TOP)\ext\misc\wholenumber.c
+
+# If use of zlib is enabled, add the "zipfile.c" source file.
+#
+!IF $(USE_ZLIB)!=0
+TESTEXT = $(TESTEXT) $(TOP)\ext\misc\zipfile.c
+!ENDIF
 
 # Source code to the library files needed by the test fixture
 # (non-amalgamation)
@@ -2025,7 +2030,7 @@ tclsqlite-shell.lo:	$(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP)
 	$(LTCOMPILE) $(NO_WARN) -DTCLSH -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c
 
 tclsqlite3.exe:	tclsqlite-shell.lo $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS)
-	$(LTLINK) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite-shell.lo $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
+	$(LTLINK) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) /OUT:$@ tclsqlite-shell.lo $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
 
 # Rules to build opcodes.c and opcodes.h
 #
@@ -2073,8 +2078,13 @@ SHELL_SRC = \
 	$(TOP)\ext\misc\completion.c \
 	$(TOP)\ext\misc\sqlar.c \
 	$(TOP)\ext\expert\sqlite3expert.c \
-	$(TOP)\ext\expert\sqlite3expert.h \
-	$(TOP)\ext\misc\zipfile.c
+	$(TOP)\ext\expert\sqlite3expert.h
+
+# If use of zlib is enabled, add the "zipfile.c" source file.
+#
+!IF $(USE_ZLIB)!=0
+SHELL_SRC = $(SHELL_SRC) $(TOP)\ext\misc\zipfile.c
+!ENDIF
 
 shell.c:	$(SHELL_SRC) $(TOP)\tool\mkshellc.tcl
 	$(TCLSH_CMD) $(TOP)\tool\mkshellc.tcl > shell.c
@@ -2262,7 +2272,7 @@ testfixture.exe:	$(TESTFIXTURE_SRC) $(SQLITE3H) $(LIBRESOBJS) $(HDR) $(SQLITE_TC
 	$(LTLINK) -DSQLITE_NO_SYNC=1 $(TESTFIXTURE_FLAGS) \
 		-DBUILD_sqlite -I$(TCLINCDIR) \
 		$(TESTFIXTURE_SRC) \
-		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
+		/link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
 
 extensiontest:	testfixture.exe testloadext.dll
 	@set PATH=$(LIBTCLPATH);$(PATH)
@@ -2312,7 +2322,7 @@ sqlite3_analyzer.c:	$(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\s
 
 sqlite3_analyzer.exe:	sqlite3_analyzer.c $(LIBRESOBJS)
 	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \
-		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
+		/link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
 
 sqlite3_expert.exe: $(SQLITE3C) $(TOP)\ext\expert\sqlite3expert.h $(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c
 	$(LTLINK) $(NO_WARN)	$(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c $(SQLITE3C) $(TLIBS)
diff --git a/manifest b/manifest
index 53a4f9a14e..eb04d9429e 100644
--- a/manifest
+++ b/manifest
@@ -1,10 +1,10 @@
-C Add\ssome\smissing\s#ifdefs\sfor\sbuilding\swithout\szlib.
-D 2018-01-05T16:03:52.073
+C In\sthe\sMakefile\sfor\sMSVC,\sonly\sattempt\sto\slink\sagainst\sTcl\swhen\sit\sis\sneeded.\s\sAlso,\sonly\scompile\s'zipfile.c'\swhen\suse\sof\szlib\sis\senabled.
+D 2018-01-05T16:05:51.162
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 1b11037c5ed3399a79433cc82c59b5e36a7b3a3e4e195bb27640d0d2145e03e1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc c4eaeff29b73d76437cf9c13d7a9654fc0e9e2578e20c12d0067c3d510e4e8e4
+F Makefile.msc 72f437d37191eb122ae289f5583af5eb9cae288fbd7914111f029066c84fbc84
 F README.md eeae1e552f93ef72ef7c5b8f6647b368a001c28820ad1df179d3dae602bef681
 F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -1694,7 +1694,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 364ac333b030f0e3372937df723f1098183da87913ba0e8ae162864ee24a50d3
-R 0ea7c4065b03d537986e987f96237bd2
+P c63fb1700c0f67d90857b1f3859c203880d0939f356a64d8ae7cafc814ea72bf
+R b5c6776760b2ddd7e6d28c4945f67802
 U mistachkin
-Z 7bfc76075fe03643e9086a52bceca2fa
+Z 02b062c480e829873866b47ba5cd168a
diff --git a/manifest.uuid b/manifest.uuid
index cff8285416..8f4fdf47f5 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c63fb1700c0f67d90857b1f3859c203880d0939f356a64d8ae7cafc814ea72bf
\ No newline at end of file
+a33ad33cf031edee273afa4735c8564870465be22962a9c277e4ac43a307ff2c
\ No newline at end of file

From c93203363096c5946ae20f8339cc1539a8d83a5b Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 5 Jan 2018 16:23:43 +0000
Subject: [PATCH 229/488] Fix missing dependencies for shell.c in all
 makefiles.

FossilOrigin-Name: 45495d3e256fef4d0669754726878ed17248fc781397ebb0421149ee9492f977
---
 Makefile.in   |  7 ++++++-
 Makefile.msc  |  4 +++-
 main.mk       |  3 ++-
 manifest      | 18 +++++++++---------
 manifest.uuid |  2 +-
 5 files changed, 21 insertions(+), 13 deletions(-)

diff --git a/Makefile.in b/Makefile.in
index e3dfb01baa..f6933f6c7e 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -994,7 +994,12 @@ SHELL_SRC = \
 	$(TOP)/src/shell.c.in \
 	$(TOP)/ext/misc/shathree.c \
 	$(TOP)/ext/misc/fileio.c \
-	$(TOP)/ext/misc/completion.c
+	$(TOP)/ext/misc/completion.c \
+	$(TOP)/ext/misc/sqlar.c \
+	$(TOP)/ext/expert/sqlite3expert.c \
+	$(TOP)/ext/expert/sqlite3expert.h \
+	$(TOP)/ext/misc/zipfile.c \
+        $(TOP)/src/test_windirent.c
 
 shell.c:	$(SHELL_SRC) $(TOP)/tool/mkshellc.tcl
 	$(TCLSH_CMD) $(TOP)/tool/mkshellc.tcl >shell.c
diff --git a/Makefile.msc b/Makefile.msc
index e2a6bbe86d..c9b78b68f8 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -2078,7 +2078,9 @@ SHELL_SRC = \
 	$(TOP)\ext\misc\completion.c \
 	$(TOP)\ext\misc\sqlar.c \
 	$(TOP)\ext\expert\sqlite3expert.c \
-	$(TOP)\ext\expert\sqlite3expert.h
+	$(TOP)\ext\expert\sqlite3expert.h \
+	$(TOP)\ext\misc\zipfile.c \
+        $(TOP)\src\test_windirent.c
 
 # If use of zlib is enabled, add the "zipfile.c" source file.
 #
diff --git a/main.mk b/main.mk
index a6a7dbf1c2..5b6775a841 100644
--- a/main.mk
+++ b/main.mk
@@ -699,7 +699,8 @@ SHELL_SRC = \
 	$(TOP)/ext/misc/sqlar.c \
 	$(TOP)/ext/expert/sqlite3expert.c \
 	$(TOP)/ext/expert/sqlite3expert.h \
-	$(TOP)/ext/misc/zipfile.c
+	$(TOP)/ext/misc/zipfile.c \
+        $(TOP)/src/test_windirent.c
 
 shell.c:	$(SHELL_SRC) $(TOP)/tool/mkshellc.tcl
 	tclsh $(TOP)/tool/mkshellc.tcl >shell.c
diff --git a/manifest b/manifest
index eb04d9429e..9f339b15a4 100644
--- a/manifest
+++ b/manifest
@@ -1,10 +1,10 @@
-C In\sthe\sMakefile\sfor\sMSVC,\sonly\sattempt\sto\slink\sagainst\sTcl\swhen\sit\sis\sneeded.\s\sAlso,\sonly\scompile\s'zipfile.c'\swhen\suse\sof\szlib\sis\senabled.
-D 2018-01-05T16:05:51.162
+C Fix\smissing\sdependencies\sfor\sshell.c\sin\sall\smakefiles.
+D 2018-01-05T16:23:43.149
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
-F Makefile.in 1b11037c5ed3399a79433cc82c59b5e36a7b3a3e4e195bb27640d0d2145e03e1
+F Makefile.in f5c6285ac43b2e567d6cf463dff3744da960a19f2cf141744b4472842a97681e
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 72f437d37191eb122ae289f5583af5eb9cae288fbd7914111f029066c84fbc84
+F Makefile.msc 69711303f4062857b1f07ecfe74d7952ce56e7401ed0248eea0569fd63b7f664
 F README.md eeae1e552f93ef72ef7c5b8f6647b368a001c28820ad1df179d3dae602bef681
 F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -405,7 +405,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 739f832da37d29e252dd9f609c3864f3ecfa42136132eab9b11acb5162b7d02d
+F main.mk 0c0ed98340ad4b7ebdfe587144a13cb12dfe3f09c16a6af5b4d4a7f3a881f2f8
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -1694,7 +1694,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c63fb1700c0f67d90857b1f3859c203880d0939f356a64d8ae7cafc814ea72bf
-R b5c6776760b2ddd7e6d28c4945f67802
-U mistachkin
-Z 02b062c480e829873866b47ba5cd168a
+P a33ad33cf031edee273afa4735c8564870465be22962a9c277e4ac43a307ff2c
+R c9a59c27332856c5c3df1bda98801aaa
+U drh
+Z 3de3b81c4ea3d2cc9df2aeeb1cbc4214
diff --git a/manifest.uuid b/manifest.uuid
index 8f4fdf47f5..bb30f4d234 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a33ad33cf031edee273afa4735c8564870465be22962a9c277e4ac43a307ff2c
\ No newline at end of file
+45495d3e256fef4d0669754726878ed17248fc781397ebb0421149ee9492f977
\ No newline at end of file

From 9a5efdec7079a7f8a548631d5958b9f6f4a04b54 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Fri, 5 Jan 2018 17:10:49 +0000
Subject: [PATCH 230/488] Fix compiler warnings.

FossilOrigin-Name: 19aabccfe34c956599f33cd3ddc846816adc06784c2496838f1ef53059e038e5
---
 ext/fts5/fts5_tcl.c  |  2 +-
 ext/misc/fileio.c    |  1 +
 ext/misc/unionvtab.c |  2 +-
 ext/misc/zipfile.c   |  8 ++++----
 manifest             | 20 ++++++++++----------
 manifest.uuid        |  2 +-
 6 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c
index e8d4c32a46..8f79397fbf 100644
--- a/ext/fts5/fts5_tcl.c
+++ b/ext/fts5/fts5_tcl.c
@@ -433,7 +433,7 @@ static int SQLITE_TCLAPI xF5tApi(
       int iVal;
       int bClear;
       if( Tcl_GetBooleanFromObj(interp, objv[2], &bClear) ) return TCL_ERROR;
-      iVal = ((char*)p->pApi->xGetAuxdata(p->pFts, bClear) - (char*)0);
+      iVal = (int)((char*)p->pApi->xGetAuxdata(p->pFts, bClear) - (char*)0);
       Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal));
       break;
     }
diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c
index f6df42f98e..796bee399c 100644
--- a/ext/misc/fileio.c
+++ b/ext/misc/fileio.c
@@ -93,6 +93,7 @@ SQLITE_EXTENSION_INIT1
 #  include "test_windirent.h"
 #  define dirent DIRENT
 #  define timespec TIMESPEC
+#  define stat _stat
 #  define mkdir(path,mode) _mkdir(path)
 #  define lstat(path,buf) _stat(path,buf)
 #endif
diff --git a/ext/misc/unionvtab.c b/ext/misc/unionvtab.c
index 14cce84510..92d0b833c4 100644
--- a/ext/misc/unionvtab.c
+++ b/ext/misc/unionvtab.c
@@ -799,7 +799,7 @@ static void unionConfigureVtab(
       zVal = zOpt;
       if( *zVal==':' ) zVal++;
       while( union_isidchar(*zVal) ) zVal++;
-      nOpt = zVal-zOpt;
+      nOpt = (int)(zVal-zOpt);
 
       while( union_isspace(*zVal) ) zVal++;
       if( *zVal=='=' ){
diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index c3dc5f2d3e..d2715c4400 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -299,7 +299,7 @@ static int zipfileConnect(
 
   if( argc>3 ){
     zFile = argv[3];
-    nFile = strlen(zFile)+1;
+    nFile = (int)strlen(zFile)+1;
   }
 
   rc = sqlite3_declare_vtab(db, ZIPFILE_SCHEMA);
@@ -1049,7 +1049,7 @@ static int zipfileAppendEntry(
   zipfileWrite16(aBuf, (u16)nPath);
   zipfileWrite16(aBuf, pCds->nExtra);
   assert( aBuf==&pTab->aBuffer[ZIPFILE_LFH_FIXED_SZ] );
-  rc = zipfileAppendData(pTab, pTab->aBuffer, aBuf - pTab->aBuffer);
+  rc = zipfileAppendData(pTab, pTab->aBuffer, (int)(aBuf - pTab->aBuffer));
   if( rc==SQLITE_OK ){
     rc = zipfileAppendData(pTab, (const u8*)zPath, nPath);
   }
@@ -1140,7 +1140,7 @@ static int zipfileUpdate(
   }
 
   zPath = (const char*)sqlite3_value_text(apVal[2]);
-  nPath = strlen(zPath);
+  nPath = (int)strlen(zPath);
   rc = zipfileGetMode(pTab, apVal[3], &mode);
   if( rc!=SQLITE_OK ) return rc;
   mTime = sqlite3_value_int64(apVal[4]);
@@ -1227,7 +1227,7 @@ static int zipfileAppendEOCD(ZipfileTab *pTab, ZipfileEOCD *p){
   zipfileWrite16(aBuf, 0);        /* Size of trailing comment in bytes*/
 
   assert( (aBuf-pTab->aBuffer)==22 );
-  return zipfileAppendData(pTab, pTab->aBuffer, aBuf - pTab->aBuffer);
+  return zipfileAppendData(pTab, pTab->aBuffer, (int)(aBuf - pTab->aBuffer));
 }
 
 static void zipfileCleanupTransaction(ZipfileTab *pTab){
diff --git a/manifest b/manifest
index 9f339b15a4..fd16cab107 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\smissing\sdependencies\sfor\sshell.c\sin\sall\smakefiles.
-D 2018-01-05T16:23:43.149
+C Fix\scompiler\swarnings.
+D 2018-01-05T17:10:49.665
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in f5c6285ac43b2e567d6cf463dff3744da960a19f2cf141744b4472842a97681e
@@ -116,7 +116,7 @@ F ext/fts5/fts5_hash.c 32be400cf761868c9db33efe81a06eb19a17c5402ad477ee9efb51301
 F ext/fts5/fts5_index.c 5fe14375a29e8a7aa8f3e863babe180a19269206c254c8f47b216821d4ac1e15
 F ext/fts5/fts5_main.c 24868f88ab2a865defbba7a92eebeb726cc991eb092b71b5f5508f180c72605b
 F ext/fts5/fts5_storage.c fb5ef3c27073f67ade2e1bea08405f9e43f68f5f3676ed0ab7013bce5ba10be6
-F ext/fts5/fts5_tcl.c a7df39442ae674dde877cf06fe02ebb7658e69c179a4d223241c90df4f14b54e
+F ext/fts5/fts5_tcl.c b470467be4c5cab2d8b026992c05d86cd2293e7d8c4a10ba56d5f4f707981097
 F ext/fts5/fts5_test_mi.c 65864ba1e5c34a61d409c4c587e0bbe0466eb4f8f478d85dc42a92caad1338e6
 F ext/fts5/fts5_test_tok.c ffd657dd67e7fcdb31bf63fb60b6d867299a581d0f46e97086abacd66c2a9b26
 F ext/fts5/fts5_tokenize.c 2ce7b44183538ec46b7907726262ee43ffdd39a8
@@ -276,7 +276,7 @@ F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83
 F ext/misc/csv.c 1a009b93650732e22334edc92459c4630b9fa703397cbb3c8ca279921a36ca11
 F ext/misc/dbdump.c 3509fa6b8932d04e932d6b6b827b6a82ca362781b8e8f3c77336f416793e215e
 F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2
-F ext/misc/fileio.c 864984342ede6775976322a3a2731be5e3a70a2c71557e489e1730bd8ca57dd2
+F ext/misc/fileio.c 1e3694706406fcca3a19d09dde9d01624f1d45f4b9e328f9adf61ffe568467ee
 F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25
 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c
 F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281984
@@ -296,13 +296,13 @@ F ext/misc/spellfix.c 41cf26c6b89fcaa8798ae10ae64d39c1f1d9d6995152e545bd491c1305
 F ext/misc/sqlar.c d355cd8b6e7280d2f61d4737672922acb512a2ab1cee52399ffb88980476e31c
 F ext/misc/stmt.c 6f16443abb3551e3f5813bb13ba19a30e7032830015b0f92fe0c0453045c0a11
 F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512
-F ext/misc/unionvtab.c de36c2c45583d68f99e45b392311967066b02e2651d05697da783698b245b387
+F ext/misc/unionvtab.c 2aa94902ea646e1aaf6c05eac944a14276cddd67735b2ad856030ffffbb6626c
 F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95
 F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c d88033b4748db9929a0096f627d3a75e9fe0e11d7a92724a6c1c575d5448cea4
+F ext/misc/zipfile.c 8075df9296beeebc344567927d114c6d3201110a29110013388d233fa7d4fb2c
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -1694,7 +1694,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a33ad33cf031edee273afa4735c8564870465be22962a9c277e4ac43a307ff2c
-R c9a59c27332856c5c3df1bda98801aaa
-U drh
-Z 3de3b81c4ea3d2cc9df2aeeb1cbc4214
+P 45495d3e256fef4d0669754726878ed17248fc781397ebb0421149ee9492f977
+R fdf4e94235537d819e1c48759efffe29
+U mistachkin
+Z 11d1a9bb2052215696c761adb1621865
diff --git a/manifest.uuid b/manifest.uuid
index bb30f4d234..23f6ac3e69 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-45495d3e256fef4d0669754726878ed17248fc781397ebb0421149ee9492f977
\ No newline at end of file
+19aabccfe34c956599f33cd3ddc846816adc06784c2496838f1ef53059e038e5
\ No newline at end of file

From 5d7503a7894b0d263fe0103d15e2e9a023a75ff9 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Fri, 5 Jan 2018 17:12:13 +0000
Subject: [PATCH 231/488] More adjustments to MSVC Makefile dependencies for
 zlib.

FossilOrigin-Name: bb650e5d14642b925c5ea65ba7991b35c4e941405d2557d542b99c66ca4bc130
---
 Makefile.msc  | 10 +++++-----
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 3 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/Makefile.msc b/Makefile.msc
index c9b78b68f8..f1d43c22c4 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -636,10 +636,11 @@ SHELL_CORE_DEP =
 !ENDIF
 
 # <<mark>>
-# If zlib support is enabled, add the shell dependency for it.
+# If zlib support is enabled, add the dependencies for it.
 #
 !IF $(USE_ZLIB)!=0 && $(BUILD_ZLIB)!=0
 SHELL_CORE_DEP = zlib $(SHELL_CORE_DEP)
+TESTFIXTURE_DEP = zlib $(TESTFIXTURE_DEP)
 !ENDIF
 # <</mark>>
 
@@ -2076,15 +2077,14 @@ SHELL_SRC = \
 	$(TOP)\ext\misc\shathree.c \
 	$(TOP)\ext\misc\fileio.c \
 	$(TOP)\ext\misc\completion.c \
-	$(TOP)\ext\misc\sqlar.c \
 	$(TOP)\ext\expert\sqlite3expert.c \
 	$(TOP)\ext\expert\sqlite3expert.h \
-	$(TOP)\ext\misc\zipfile.c \
-        $(TOP)\src\test_windirent.c
+	$(TOP)\src\test_windirent.c
 
 # If use of zlib is enabled, add the "zipfile.c" source file.
 #
 !IF $(USE_ZLIB)!=0
+SHELL_SRC = $(SHELL_SRC) $(TOP)\ext\misc\sqlar.c
 SHELL_SRC = $(SHELL_SRC) $(TOP)\ext\misc\zipfile.c
 !ENDIF
 
@@ -2270,7 +2270,7 @@ sqlite_tcl.h:
 		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact "Tcl_HashEntry *(*createProc)" "Tcl_HashEntry *(SQLITE_TCLAPI *createProc)" >> $(SQLITETCLH)
 !ENDIF
 
-testfixture.exe:	$(TESTFIXTURE_SRC) $(SQLITE3H) $(LIBRESOBJS) $(HDR) $(SQLITE_TCL_DEP)
+testfixture.exe:	$(TESTFIXTURE_SRC) $(TESTFIXTURE_DEP) $(SQLITE3H) $(LIBRESOBJS) $(HDR) $(SQLITE_TCL_DEP)
 	$(LTLINK) -DSQLITE_NO_SYNC=1 $(TESTFIXTURE_FLAGS) \
 		-DBUILD_sqlite -I$(TCLINCDIR) \
 		$(TESTFIXTURE_SRC) \
diff --git a/manifest b/manifest
index fd16cab107..bc639547e1 100644
--- a/manifest
+++ b/manifest
@@ -1,10 +1,10 @@
-C Fix\scompiler\swarnings.
-D 2018-01-05T17:10:49.665
+C More\sadjustments\sto\sMSVC\sMakefile\sdependencies\sfor\szlib.
+D 2018-01-05T17:12:13.607
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in f5c6285ac43b2e567d6cf463dff3744da960a19f2cf141744b4472842a97681e
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 69711303f4062857b1f07ecfe74d7952ce56e7401ed0248eea0569fd63b7f664
+F Makefile.msc de37dcd0e875a72b3e4f0d5d962df007e7fffbdebcf588547aa9a414dc690dbc
 F README.md eeae1e552f93ef72ef7c5b8f6647b368a001c28820ad1df179d3dae602bef681
 F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -1694,7 +1694,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 45495d3e256fef4d0669754726878ed17248fc781397ebb0421149ee9492f977
-R fdf4e94235537d819e1c48759efffe29
+P 19aabccfe34c956599f33cd3ddc846816adc06784c2496838f1ef53059e038e5
+R 4f7a61feaaf912b14f8d84b71ecd0523
 U mistachkin
-Z 11d1a9bb2052215696c761adb1621865
+Z 232e8c03263ea4c4f59899620c8c44c4
diff --git a/manifest.uuid b/manifest.uuid
index 23f6ac3e69..4cf798be0b 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-19aabccfe34c956599f33cd3ddc846816adc06784c2496838f1ef53059e038e5
\ No newline at end of file
+bb650e5d14642b925c5ea65ba7991b35c4e941405d2557d542b99c66ca4bc130
\ No newline at end of file

From af23899318236a918af9cbfed85dcc57769077ec Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Fri, 5 Jan 2018 17:40:38 +0000
Subject: [PATCH 232/488] In the Makefile for MSVC, the default target should
 not include binaries that link against the Tcl library.

FossilOrigin-Name: 5c6d0a1d58533feb4ea6926f4d3611664a275ad216492b424056276da38e84a2
---
 Makefile.msc  | 10 +++++++++-
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 3 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/Makefile.msc b/Makefile.msc
index f1d43c22c4..f485e37ca1 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -1636,7 +1636,15 @@ ALL_TCL_TARGETS =
 # This is the default Makefile target.  The objects listed here
 # are what get build when you type just "make" with no arguments.
 #
-all:	dll libsqlite3.lib shell $(ALL_TCL_TARGETS)
+core:	dll libsqlite3.lib shell
+
+# Targets that require the Tcl library.
+#
+tcl:	$(ALL_TCL_TARGETS)
+
+# This Makefile target builds all of the standard binaries.
+#
+all:	core tcl
 
 # Dynamic link library section.
 #
diff --git a/manifest b/manifest
index bc639547e1..0c20fb2e6a 100644
--- a/manifest
+++ b/manifest
@@ -1,10 +1,10 @@
-C More\sadjustments\sto\sMSVC\sMakefile\sdependencies\sfor\szlib.
-D 2018-01-05T17:12:13.607
+C In\sthe\sMakefile\sfor\sMSVC,\sthe\sdefault\starget\sshould\snot\sinclude\sbinaries\sthat\slink\sagainst\sthe\sTcl\slibrary.
+D 2018-01-05T17:40:38.949
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in f5c6285ac43b2e567d6cf463dff3744da960a19f2cf141744b4472842a97681e
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc de37dcd0e875a72b3e4f0d5d962df007e7fffbdebcf588547aa9a414dc690dbc
+F Makefile.msc 2335db317697a3ea33ec3a258be9f46c973fcfd4e7cffe6798c6f3d55bc83d5b
 F README.md eeae1e552f93ef72ef7c5b8f6647b368a001c28820ad1df179d3dae602bef681
 F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -1694,7 +1694,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 19aabccfe34c956599f33cd3ddc846816adc06784c2496838f1ef53059e038e5
-R 4f7a61feaaf912b14f8d84b71ecd0523
+P bb650e5d14642b925c5ea65ba7991b35c4e941405d2557d542b99c66ca4bc130
+R cf861d2a1bc231834a93f636736946e1
 U mistachkin
-Z 232e8c03263ea4c4f59899620c8c44c4
+Z d9fbefa44671c9c82483590ecbf133ab
diff --git a/manifest.uuid b/manifest.uuid
index 4cf798be0b..a7d2b46577 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-bb650e5d14642b925c5ea65ba7991b35c4e941405d2557d542b99c66ca4bc130
\ No newline at end of file
+5c6d0a1d58533feb4ea6926f4d3611664a275ad216492b424056276da38e84a2
\ No newline at end of file

From 7ee8836e08cca0ecd8af0749e5072e5bc36352cc Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 5 Jan 2018 18:51:25 +0000
Subject: [PATCH 233/488] Fix a bug introduced by efforts to reduce compiler
 warnings in check-in [364ac333b030f0]

FossilOrigin-Name: 1d6cee9ad448b10e69f351ef9dbec09110c5b189cba8734e637f41abe8f35bf8
---
 ext/misc/fileio.c |  5 +++--
 manifest          | 14 +++++++-------
 manifest.uuid     |  2 +-
 3 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c
index 796bee399c..030b417e40 100644
--- a/ext/misc/fileio.c
+++ b/ext/misc/fileio.c
@@ -182,13 +182,14 @@ static int makeDirectory(
 
     while( rc==SQLITE_OK ){
       struct stat sStat;
+      int rc2;
 
       for(; zCopy[i]!='/' && i<nCopy; i++);
       if( i==nCopy ) break;
       zCopy[i] = '\0';
 
-      rc = stat(zCopy, &sStat);
-      if( rc!=0 ){
+      rc2 = stat(zCopy, &sStat);
+      if( rc2!=0 ){
         if( mkdir(zCopy, mode & 0777) ) rc = SQLITE_ERROR;
       }else{
         if( !S_ISDIR(sStat.st_mode) ) rc = SQLITE_ERROR;
diff --git a/manifest b/manifest
index 0c20fb2e6a..816bff3ecf 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\sthe\sMakefile\sfor\sMSVC,\sthe\sdefault\starget\sshould\snot\sinclude\sbinaries\sthat\slink\sagainst\sthe\sTcl\slibrary.
-D 2018-01-05T17:40:38.949
+C Fix\sa\sbug\sintroduced\sby\sefforts\sto\sreduce\scompiler\swarnings\sin\ncheck-in\s[364ac333b030f0]
+D 2018-01-05T18:51:25.650
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in f5c6285ac43b2e567d6cf463dff3744da960a19f2cf141744b4472842a97681e
@@ -276,7 +276,7 @@ F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83
 F ext/misc/csv.c 1a009b93650732e22334edc92459c4630b9fa703397cbb3c8ca279921a36ca11
 F ext/misc/dbdump.c 3509fa6b8932d04e932d6b6b827b6a82ca362781b8e8f3c77336f416793e215e
 F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2
-F ext/misc/fileio.c 1e3694706406fcca3a19d09dde9d01624f1d45f4b9e328f9adf61ffe568467ee
+F ext/misc/fileio.c f16e2ef87928b768883b259b6c53a67076e98e8daa0bbb624910406d59630b26
 F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25
 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c
 F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281984
@@ -1694,7 +1694,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P bb650e5d14642b925c5ea65ba7991b35c4e941405d2557d542b99c66ca4bc130
-R cf861d2a1bc231834a93f636736946e1
-U mistachkin
-Z d9fbefa44671c9c82483590ecbf133ab
+P 5c6d0a1d58533feb4ea6926f4d3611664a275ad216492b424056276da38e84a2
+R ed8496ab8972efeecd73ab10aa079f0a
+U drh
+Z ab3dd049af31828f3cc751c8ec71ac20
diff --git a/manifest.uuid b/manifest.uuid
index a7d2b46577..7b97eab290 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5c6d0a1d58533feb4ea6926f4d3611664a275ad216492b424056276da38e84a2
\ No newline at end of file
+1d6cee9ad448b10e69f351ef9dbec09110c5b189cba8734e637f41abe8f35bf8
\ No newline at end of file

From 370732a83a93df8c3b63f29109347c73e4d1fbd0 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Fri, 5 Jan 2018 19:25:52 +0000
Subject: [PATCH 234/488] For the MSVC Makefile, the 'sqlite3_checker.exe'
 target requires the Tcl library.

FossilOrigin-Name: 38109a47ffa977f1f962af2e183285d4e5db6fac2344868c5f1de64779dd0839
---
 Makefile.msc  |  2 +-
 manifest      | 15 +++++++--------
 manifest.uuid |  2 +-
 3 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/Makefile.msc b/Makefile.msc
index f485e37ca1..74518e11a9 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -2352,7 +2352,7 @@ sqlite3_checker.c:	$(CHECKER_DEPS)
 
 sqlite3_checker.exe:	sqlite3_checker.c $(LIBRESOBJS)
 	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_checker.c \
-		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
+		/link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
 
 dbdump.exe:	$(TOP)\ext\misc\dbdump.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DDBDUMP_STANDALONE $(TOP)\ext\misc\dbdump.c $(SQLITE3C) \
diff --git a/manifest b/manifest
index 7a5a46f247..992a88310c 100644
--- a/manifest
+++ b/manifest
@@ -1,10 +1,10 @@
-C Add\sthe\s".ar"\scommand\sto\sthe\scommand-line\sshell.
-D 2018-01-05T19:01:05.842
+C For\sthe\sMSVC\sMakefile,\sthe\s'sqlite3_checker.exe'\starget\srequires\sthe\sTcl\slibrary.
+D 2018-01-05T19:25:52.005
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in f5c6285ac43b2e567d6cf463dff3744da960a19f2cf141744b4472842a97681e
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 2335db317697a3ea33ec3a258be9f46c973fcfd4e7cffe6798c6f3d55bc83d5b
+F Makefile.msc d429170f3c6d35390280ce0dff79fa0ef3ec5bb11ff981d0195a77a97cd8f67f
 F README.md eeae1e552f93ef72ef7c5b8f6647b368a001c28820ad1df179d3dae602bef681
 F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -1694,8 +1694,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a4876672edea4e96103efd2463ce9a34a0b994a8744c941660940578aafbd454 1d6cee9ad448b10e69f351ef9dbec09110c5b189cba8734e637f41abe8f35bf8
-R af65ab601703ff43220f8cbd5a0328ec
-T +closed 1d6cee9ad448b10e69f351ef9dbec09110c5b189cba8734e637f41abe8f35bf8
-U drh
-Z 35e4e889b7dd1f4bdfa8de9e35b21bb1
+P 148b8aee78e40cab9a758a920589bd3ca8fc1c45cc93598bc50d96b85cd17e6c
+R 3bd57aef0f7576f9a9295eae8b71396f
+U mistachkin
+Z 21f077ad1d1dc6de7af623b34ca85770
diff --git a/manifest.uuid b/manifest.uuid
index ef03cb90bf..77e9c2d595 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-148b8aee78e40cab9a758a920589bd3ca8fc1c45cc93598bc50d96b85cd17e6c
\ No newline at end of file
+38109a47ffa977f1f962af2e183285d4e5db6fac2344868c5f1de64779dd0839
\ No newline at end of file

From 992fdfc5d88f8ba7fca981296b259f2d9c2a3ae2 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Fri, 5 Jan 2018 19:27:19 +0000
Subject: [PATCH 235/488] The 'releasetest' tool should pass the TCLDIR macro
 for MSVC to nmake.

FossilOrigin-Name: eaa9c0dda73808f1458f9968be59947636fbd4781c9fc535eb50bcaf790093c1
---
 manifest             | 12 ++++++------
 manifest.uuid        |  2 +-
 test/releasetest.tcl |  3 +++
 3 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index 992a88310c..56e9862e98 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C For\sthe\sMSVC\sMakefile,\sthe\s'sqlite3_checker.exe'\starget\srequires\sthe\sTcl\slibrary.
-D 2018-01-05T19:25:52.005
+C The\s'releasetest'\stool\sshould\spass\sthe\sTCLDIR\smacro\sfor\sMSVC\sto\snmake.
+D 2018-01-05T19:27:19.800
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in f5c6285ac43b2e567d6cf463dff3744da960a19f2cf141744b4472842a97681e
@@ -1151,7 +1151,7 @@ F test/rdonly.test 64e2696c322e3538df0b1ed624e21f9a23ed9ff8
 F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8
 F test/regexp2.test 40e894223b3d6672655481493f1be12012f2b33c
 F test/reindex.test 44edd3966b474468b823d481eafef0c305022254
-F test/releasetest.tcl 0b0b3d926e36822ff63b405d683544ce1014303b029f2678bbcf40c162b5f246 x
+F test/releasetest.tcl 6aaa853f7a7bbdc458d4cb42c0425228729b0f3e5769e9b41088c08eee999a49 x
 F test/resolver01.test f4022acafda7f4d40eca94dbf16bc5fc4ac30ceb
 F test/rollback.test f580934279800d480a19176c6b44909df31ce7ad45267ea475a541daa522f3d3
 F test/rollback2.test 8435d6ff0f13f51d2a4181c232e706005fa90fc5
@@ -1694,7 +1694,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 148b8aee78e40cab9a758a920589bd3ca8fc1c45cc93598bc50d96b85cd17e6c
-R 3bd57aef0f7576f9a9295eae8b71396f
+P 38109a47ffa977f1f962af2e183285d4e5db6fac2344868c5f1de64779dd0839
+R 9eed99f952d882095a7640a2b2c94c61
 U mistachkin
-Z 21f077ad1d1dc6de7af623b34ca85770
+Z 7e88bc6a5eaa12b61e53263c5085cc0a
diff --git a/manifest.uuid b/manifest.uuid
index 77e9c2d595..f9e69f73df 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-38109a47ffa977f1f962af2e183285d4e5db6fac2344868c5f1de64779dd0839
\ No newline at end of file
+eaa9c0dda73808f1458f9968be59947636fbd4781c9fc535eb50bcaf790093c1
\ No newline at end of file
diff --git a/test/releasetest.tcl b/test/releasetest.tcl
index 5055e81129..599ebd791d 100755
--- a/test/releasetest.tcl
+++ b/test/releasetest.tcl
@@ -734,6 +734,9 @@ proc makeCommand { targets makeOpts cflags opts } {
     set nmakeDir [file nativename $::SRCDIR]
     set nmakeFile [file nativename [file join $nmakeDir Makefile.msc]]
     lappend result nmake /f $nmakeFile TOP=$nmakeDir
+    set tclDir [file nativename [file normalize \
+        [file dirname [file dirname [info nameofexecutable]]]]]
+    lappend result "TCLDIR=$tclDir"
     if {[regexp {USE_STDCALL=1} $cflags]} {
       lappend result USE_STDCALL=1
     }

From b3ec2e1f4a6d75c7e1b959c8fdab2efc94f0a7c9 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Fri, 5 Jan 2018 19:28:16 +0000
Subject: [PATCH 236/488] Sync up the autoconf Makefile for MSVC.

FossilOrigin-Name: da8712bdb8f69dee9efb42b74d0b2ec10043f281c5f93e2724078bdf1370ecb1
---
 autoconf/Makefile.msc | 11 ++++++++++-
 manifest              | 12 ++++++------
 manifest.uuid         |  2 +-
 3 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/autoconf/Makefile.msc b/autoconf/Makefile.msc
index 5b5133c8b9..5f7c693d42 100644
--- a/autoconf/Makefile.msc
+++ b/autoconf/Makefile.msc
@@ -930,13 +930,22 @@ LIBRESOBJS =
 !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_DBSTAT_VTAB
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC -DSQLITE_INTROSPECTION_PRAGMAS
 !ENDIF
 
 
 # This is the default Makefile target.  The objects listed here
 # are what get build when you type just "make" with no arguments.
 #
-all:	dll shell
+core:	dll shell
+
+# Targets that require the Tcl library.
+#
+tcl:	$(ALL_TCL_TARGETS)
+
+# This Makefile target builds all of the standard binaries.
+#
+all:	core tcl
 
 # Dynamic link library section.
 #
diff --git a/manifest b/manifest
index 56e9862e98..053d4e4270 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C The\s'releasetest'\stool\sshould\spass\sthe\sTCLDIR\smacro\sfor\sMSVC\sto\snmake.
-D 2018-01-05T19:27:19.800
+C Sync\sup\sthe\sautoconf\sMakefile\sfor\sMSVC.
+D 2018-01-05T19:28:16.538
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in f5c6285ac43b2e567d6cf463dff3744da960a19f2cf141744b4472842a97681e
@@ -13,7 +13,7 @@ F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90
 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2
 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903
 F autoconf/Makefile.am 66c0befa511f0d95ba229e180067cf0357a9ebf8b3201b06d683c5ba6220fb39
-F autoconf/Makefile.msc 2b4b5e5ff7e7a9806ebdd4b441f8fb54695a3628701a97ae53860e67e872acc3
+F autoconf/Makefile.msc 2c50a59319af7da4eaca8c13e3240881b1bc245fd175845a055faab7d03d6e67
 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7
 F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1
 F autoconf/configure.ac 8dd08ca564279fff091c9bfdd2599d8f992c9f1f70c5396de2126ad2bd1b3bed
@@ -1694,7 +1694,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 38109a47ffa977f1f962af2e183285d4e5db6fac2344868c5f1de64779dd0839
-R 9eed99f952d882095a7640a2b2c94c61
+P eaa9c0dda73808f1458f9968be59947636fbd4781c9fc535eb50bcaf790093c1
+R aebea1e00e37b18810f9236f36666933
 U mistachkin
-Z 7e88bc6a5eaa12b61e53263c5085cc0a
+Z b1ea0ac834ae5e03c81c10b82e8229f6
diff --git a/manifest.uuid b/manifest.uuid
index f9e69f73df..b571106f7f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-eaa9c0dda73808f1458f9968be59947636fbd4781c9fc535eb50bcaf790093c1
\ No newline at end of file
+da8712bdb8f69dee9efb42b74d0b2ec10043f281c5f93e2724078bdf1370ecb1
\ No newline at end of file

From 411dfd4e6dc25a3b5e763b03ac1e7aeca46ee44d Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 5 Jan 2018 19:49:54 +0000
Subject: [PATCH 237/488] Fix the autoconf makefile so that it builds
 testfixture correctly.

FossilOrigin-Name: fbfe04c0b4f261789cbda3d2e98d12508181c283eab0bb757b081f0383e891bd
---
 Makefile.in   |  3 ++-
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 3 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/Makefile.in b/Makefile.in
index f6933f6c7e..da01248f4f 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -447,7 +447,8 @@ TESTSRC += \
   $(TOP)/ext/misc/spellfix.c \
   $(TOP)/ext/misc/totype.c \
   $(TOP)/ext/misc/unionvtab.c \
-  $(TOP)/ext/misc/wholenumber.c
+  $(TOP)/ext/misc/wholenumber.c \
+  $(TOP)/ext/misc/zipfile.c
 
 # Source code to the library files needed by the test fixture
 #
diff --git a/manifest b/manifest
index 053d4e4270..bc23c568bb 100644
--- a/manifest
+++ b/manifest
@@ -1,8 +1,8 @@
-C Sync\sup\sthe\sautoconf\sMakefile\sfor\sMSVC.
-D 2018-01-05T19:28:16.538
+C Fix\sthe\sautoconf\smakefile\sso\sthat\sit\sbuilds\stestfixture\scorrectly.
+D 2018-01-05T19:49:54.732
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
-F Makefile.in f5c6285ac43b2e567d6cf463dff3744da960a19f2cf141744b4472842a97681e
+F Makefile.in c4a9842dcd07572a106da739fa4f76d5ae228e3c4c40c859c84ceaa71c36348e
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc d429170f3c6d35390280ce0dff79fa0ef3ec5bb11ff981d0195a77a97cd8f67f
 F README.md eeae1e552f93ef72ef7c5b8f6647b368a001c28820ad1df179d3dae602bef681
@@ -1694,7 +1694,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P eaa9c0dda73808f1458f9968be59947636fbd4781c9fc535eb50bcaf790093c1
-R aebea1e00e37b18810f9236f36666933
-U mistachkin
-Z b1ea0ac834ae5e03c81c10b82e8229f6
+P da8712bdb8f69dee9efb42b74d0b2ec10043f281c5f93e2724078bdf1370ecb1
+R 3f6a7684abe5e233e60eee143543b86b
+U drh
+Z 40c46b9c1ab67658db9ed4b73dac85b2
diff --git a/manifest.uuid b/manifest.uuid
index b571106f7f..23f8f09cd2 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-da8712bdb8f69dee9efb42b74d0b2ec10043f281c5f93e2724078bdf1370ecb1
\ No newline at end of file
+fbfe04c0b4f261789cbda3d2e98d12508181c283eab0bb757b081f0383e891bd
\ No newline at end of file

From acae8c3e5d663907ab517f0575c28ef8cbac7dda Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Fri, 5 Jan 2018 20:08:46 +0000
Subject: [PATCH 238/488] Fix shell compilation with MinGW by including the
 'dirent.h' header.

FossilOrigin-Name: dfe510b51046a53c7f5c54fe73ec9de3364923eabd4b909c11984552e6bea406
---
 manifest       | 14 +++++++-------
 manifest.uuid  |  2 +-
 src/shell.c.in |  9 +++++++--
 3 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index bc23c568bb..98c8245baf 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sthe\sautoconf\smakefile\sso\sthat\sit\sbuilds\stestfixture\scorrectly.
-D 2018-01-05T19:49:54.732
+C Fix\sshell\scompilation\swith\sMinGW\sby\sincluding\sthe\s'dirent.h'\sheader.
+D 2018-01-05T20:08:46.463
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in c4a9842dcd07572a106da739fa4f76d5ae228e3c4c40c859c84ceaa71c36348e
@@ -483,7 +483,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in 17fc28661aae277767db63fa90e644b8c9bf1242fbd167dcc2e7af0f0e620bb7
+F src/shell.c.in 54b689b5c4301c77c27cb74030387a8fa311ca8e419efde97c32368447496adf
 F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1694,7 +1694,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P da8712bdb8f69dee9efb42b74d0b2ec10043f281c5f93e2724078bdf1370ecb1
-R 3f6a7684abe5e233e60eee143543b86b
-U drh
-Z 40c46b9c1ab67658db9ed4b73dac85b2
+P fbfe04c0b4f261789cbda3d2e98d12508181c283eab0bb757b081f0383e891bd
+R 67b7409129088d9700def94a0c3a7e6a
+U mistachkin
+Z 6191ddbb449a249c839487fd17b108da
diff --git a/manifest.uuid b/manifest.uuid
index 23f8f09cd2..26377c9417 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-fbfe04c0b4f261789cbda3d2e98d12508181c283eab0bb757b081f0383e891bd
\ No newline at end of file
+dfe510b51046a53c7f5c54fe73ec9de3364923eabd4b909c11984552e6bea406
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index eb5f0b42ed..691e7750ad 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -72,7 +72,13 @@
 # if !defined(__RTP__) && !defined(_WRS_KERNEL)
 #  include <pwd.h>
 # endif
+#endif
+#if (!defined(_WIN32) && !defined(WIN32)) || defined(__MINGW_H)
 # include <unistd.h>
+# include <dirent.h>
+# if defined(__MINGW_H)
+#  define DIRENT dirent
+# endif
 #endif
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -875,10 +881,9 @@ static void shellAddSchemaName(
 #define SQLITE_EXTENSION_INIT1
 #define SQLITE_EXTENSION_INIT2(X) (void)(X)
 
-#if defined(_WIN32) || defined(WIN32)
+#if defined(_WIN32) && defined(_MSC_VER)
 INCLUDE test_windirent.c
 #define dirent DIRENT
-#define timespec TIMESPEC
 #endif
 INCLUDE ../ext/misc/shathree.c
 INCLUDE ../ext/misc/fileio.c

From f2e8aa637e91814b4e823af4e474001fbddb1449 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Fri, 5 Jan 2018 20:13:33 +0000
Subject: [PATCH 239/488] Skip running the 'zipfile' test if the necessary
 static package cannot be loaded.

FossilOrigin-Name: 5bc816ec4010c20407396a1051e9f9b0a360ca27c0a99dafa867225cf974e306
---
 manifest          | 12 ++++++------
 manifest.uuid     |  2 +-
 test/zipfile.test |  5 ++++-
 3 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 98c8245baf..b645c837cd 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sshell\scompilation\swith\sMinGW\sby\sincluding\sthe\s'dirent.h'\sheader.
-D 2018-01-05T20:08:46.463
+C Skip\srunning\sthe\s'zipfile'\stest\sif\sthe\snecessary\sstatic\spackage\scannot\sbe\sloaded.
+D 2018-01-05T20:13:33.577
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in c4a9842dcd07572a106da739fa4f76d5ae228e3c4c40c859c84ceaa71c36348e
@@ -1597,7 +1597,7 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf
 F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc
 F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa
 F test/zerodamage.test e59a56443d6298ecf7435f618f0b27654f0c849e
-F test/zipfile.test d4f342ca918fd4d7981a12c1523f5041074cc592f25fecce4ee11446cc984f56
+F test/zipfile.test a1dd0429294cb9487900fc2b29aa9921329f20a7314aa0921b668246172ac090
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d
 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91
@@ -1694,7 +1694,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P fbfe04c0b4f261789cbda3d2e98d12508181c283eab0bb757b081f0383e891bd
-R 67b7409129088d9700def94a0c3a7e6a
+P dfe510b51046a53c7f5c54fe73ec9de3364923eabd4b909c11984552e6bea406
+R 77b0d484d37ac0b70535216ad42f574c
 U mistachkin
-Z 6191ddbb449a249c839487fd17b108da
+Z 2a7cfd5e6817597b58865e4e49810899
diff --git a/manifest.uuid b/manifest.uuid
index 26377c9417..325fc3f2c3 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-dfe510b51046a53c7f5c54fe73ec9de3364923eabd4b909c11984552e6bea406
\ No newline at end of file
+5bc816ec4010c20407396a1051e9f9b0a360ca27c0a99dafa867225cf974e306
\ No newline at end of file
diff --git a/test/zipfile.test b/test/zipfile.test
index 3f7b34d55d..627945651b 100644
--- a/test/zipfile.test
+++ b/test/zipfile.test
@@ -14,7 +14,10 @@ set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 set testprefix zipfile
 
-load_static_extension db zipfile
+if {[catch {load_static_extension db zipfile} error]} {
+  puts "Skipping zipfile tests, hit load error: $error"
+  finish_test; return
+}
 
 forcedelete test.zip
 do_execsql_test 1.0 {

From 2f74b3c3abbad9773b0cbca215b1fbf23d6ce068 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Fri, 5 Jan 2018 20:26:06 +0000
Subject: [PATCH 240/488] MinGW does not define the S_ISLNK macro; therefore,
 define it in the shell when needed.

FossilOrigin-Name: 73023febbe57495a22db05904bd6ddff439d6c25970cbe87e0ba4a923f3e8a29
---
 manifest       | 12 ++++++------
 manifest.uuid  |  2 +-
 src/shell.c.in |  3 +++
 3 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index b645c837cd..bbcb9ab663 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Skip\srunning\sthe\s'zipfile'\stest\sif\sthe\snecessary\sstatic\spackage\scannot\sbe\sloaded.
-D 2018-01-05T20:13:33.577
+C MinGW\sdoes\snot\sdefine\sthe\sS_ISLNK\smacro;\stherefore,\sdefine\sit\sin\sthe\sshell\swhen\sneeded.
+D 2018-01-05T20:26:06.480
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in c4a9842dcd07572a106da739fa4f76d5ae228e3c4c40c859c84ceaa71c36348e
@@ -483,7 +483,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in 54b689b5c4301c77c27cb74030387a8fa311ca8e419efde97c32368447496adf
+F src/shell.c.in df0e5728a6b41c676f0bb55716ca317081eba5e741ba45e513c842c4533bda58
 F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1694,7 +1694,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P dfe510b51046a53c7f5c54fe73ec9de3364923eabd4b909c11984552e6bea406
-R 77b0d484d37ac0b70535216ad42f574c
+P 5bc816ec4010c20407396a1051e9f9b0a360ca27c0a99dafa867225cf974e306
+R 1b1f2c1c694e2046fcfe9404f479b19a
 U mistachkin
-Z 2a7cfd5e6817597b58865e4e49810899
+Z 8141067c4aab566b846a47904d7a2b0b
diff --git a/manifest.uuid b/manifest.uuid
index 325fc3f2c3..c2654be9be 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5bc816ec4010c20407396a1051e9f9b0a360ca27c0a99dafa867225cf974e306
\ No newline at end of file
+73023febbe57495a22db05904bd6ddff439d6c25970cbe87e0ba4a923f3e8a29
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 691e7750ad..832450fea7 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -78,6 +78,9 @@
 # include <dirent.h>
 # if defined(__MINGW_H)
 #  define DIRENT dirent
+#  ifndef S_ISLNK
+#   define S_ISLNK(mode) (0)
+#  endif
 # endif
 #endif
 #include <sys/types.h>

From f5c75626ffeaa939a294259c4028a2bc27db6c50 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 5 Jan 2018 20:30:54 +0000
Subject: [PATCH 241/488] Fix the shell.c.in amalgamator script so that it
 avoids generating redundant typedef statements.

FossilOrigin-Name: 6c53c740cb4f62305689ea0a73beca83e71ed86707094e850bed975965c8c9f2
---
 manifest          | 14 +++++++-------
 manifest.uuid     |  2 +-
 tool/mkshellc.tcl | 14 ++++++++++++--
 3 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index bbcb9ab663..9634780370 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C MinGW\sdoes\snot\sdefine\sthe\sS_ISLNK\smacro;\stherefore,\sdefine\sit\sin\sthe\sshell\swhen\sneeded.
-D 2018-01-05T20:26:06.480
+C Fix\sthe\sshell.c.in\samalgamator\sscript\sso\sthat\sit\savoids\sgenerating\nredundant\stypedef\sstatements.
+D 2018-01-05T20:30:54.903
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in c4a9842dcd07572a106da739fa4f76d5ae228e3c4c40c859c84ceaa71c36348e
@@ -1630,7 +1630,7 @@ F tool/mkopcodec.tcl d1b6362bd3aa80d5520d4d6f3765badf01f6c43c
 F tool/mkopcodeh.tcl 4ee2a30ccbd900dc4d5cdb61bdab87cd2166cd2affcc78c9cc0b8d22a65b2eee
 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e
 F tool/mkpragmatab.tcl 2144bc8550a6471a029db262a132d2df4b9e0db61b90398bf64f5b7b3f8d92cd
-F tool/mkshellc.tcl 574307265b49d813301fba91ccd74e6a26d33f65f74b6891c320a0ffbee07895
+F tool/mkshellc.tcl 8cb90170e4aed5bd26f1664a83727c5d451f86948d3b759be26c1d7879fa346f
 F tool/mksourceid.c d458f9004c837bee87a6382228ac20d3eae3c49ea3b0a5aace936f8b60748d3b
 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97
 F tool/mksqlite3c-noext.tcl fef88397668ae83166735c41af99d79f56afaabb
@@ -1694,7 +1694,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5bc816ec4010c20407396a1051e9f9b0a360ca27c0a99dafa867225cf974e306
-R 1b1f2c1c694e2046fcfe9404f479b19a
-U mistachkin
-Z 8141067c4aab566b846a47904d7a2b0b
+P 73023febbe57495a22db05904bd6ddff439d6c25970cbe87e0ba4a923f3e8a29
+R 9251688fbd83640e3673feb22bd548ce
+U drh
+Z 4f1367bc1221f28464fd760bb035ab98
diff --git a/manifest.uuid b/manifest.uuid
index c2654be9be..3810fe5730 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-73023febbe57495a22db05904bd6ddff439d6c25970cbe87e0ba4a923f3e8a29
\ No newline at end of file
+6c53c740cb4f62305689ea0a73beca83e71ed86707094e850bed975965c8c9f2
\ No newline at end of file
diff --git a/tool/mkshellc.tcl b/tool/mkshellc.tcl
index 807a169826..1a98511ef9 100644
--- a/tool/mkshellc.tcl
+++ b/tool/mkshellc.tcl
@@ -30,15 +30,25 @@ puts $out {/* DO NOT EDIT!
 ** by "src/shell.c.in", then rerun the tool/mkshellc.tcl script.
 */}
 set in [open $topdir/src/shell.c.in rb]
+proc omit_redundant_typedefs {line} {
+  global typedef_seen
+  if {[regexp {^typedef .*;} $line]} {
+    if {[info exists typedef_seen($line)]} {
+      return "/* $line */"
+    }
+    set typedef_seen($line) 1
+  }
+  return $line
+}
 while {1} {
-  set lx [gets $in]
+  set lx [omit_redundant_typedefs [gets $in]]
   if {[eof $in]} break;
   if {[regexp {^INCLUDE } $lx]} {
     set cfile [lindex $lx 1]
     puts $out "/************************* Begin $cfile ******************/"
     set in2 [open $topdir/src/$cfile rb]
     while {![eof $in2]} {
-      set lx [gets $in2]
+      set lx [omit_redundant_typedefs [gets $in2]]
       if {[regexp {^#include "sqlite} $lx]} continue
       set lx [string map [list __declspec(dllexport) {}] $lx]
       puts $out $lx

From 1e506b5527a1fd842610b973cf21969cab4ac269 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 5 Jan 2018 21:01:37 +0000
Subject: [PATCH 242/488] Make sure i64 and u64 typedefs are available
 throughout shell.c.

FossilOrigin-Name: 656fb5aab32f40412389e43de209867cedee1589533949cc8c01ba24218ea434
---
 manifest       | 12 ++++++------
 manifest.uuid  |  2 +-
 src/shell.c.in |  2 ++
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index 9634780370..d848453f6a 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sthe\sshell.c.in\samalgamator\sscript\sso\sthat\sit\savoids\sgenerating\nredundant\stypedef\sstatements.
-D 2018-01-05T20:30:54.903
+C Make\ssure\si64\sand\su64\stypedefs\sare\savailable\sthroughout\sshell.c.
+D 2018-01-05T21:01:37.129
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in c4a9842dcd07572a106da739fa4f76d5ae228e3c4c40c859c84ceaa71c36348e
@@ -483,7 +483,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in df0e5728a6b41c676f0bb55716ca317081eba5e741ba45e513c842c4533bda58
+F src/shell.c.in 4cfa9394737306c5a5f3d7f72e74868df5f3010676d6a1a2a52db37b61b0c13a
 F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1694,7 +1694,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 73023febbe57495a22db05904bd6ddff439d6c25970cbe87e0ba4a923f3e8a29
-R 9251688fbd83640e3673feb22bd548ce
+P 6c53c740cb4f62305689ea0a73beca83e71ed86707094e850bed975965c8c9f2
+R 7d026919c293c5c42c6d20698632fda7
 U drh
-Z 4f1367bc1221f28464fd760bb035ab98
+Z 6c996af85ed6472bcfa0c4295fc91bf0
diff --git a/manifest.uuid b/manifest.uuid
index 3810fe5730..b50e96dbc6 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6c53c740cb4f62305689ea0a73beca83e71ed86707094e850bed975965c8c9f2
\ No newline at end of file
+656fb5aab32f40412389e43de209867cedee1589533949cc8c01ba24218ea434
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 832450fea7..bb4428fbc5 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -61,6 +61,8 @@
 #include <stdio.h>
 #include <assert.h>
 #include "sqlite3.h"
+typedef sqlite3_int64 i64;
+typedef sqlite3_uint64 u64;
 #if SQLITE_USER_AUTHENTICATION
 # include "sqlite3userauth.h"
 #endif

From 9dea173154d490489c7794f945a153d1a9de058b Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 6 Jan 2018 04:34:05 +0000
Subject: [PATCH 243/488] Add sqltclsh.exe to the windows makefile.

FossilOrigin-Name: a6d5c7c2aabd70b4fb2bd8f6278f70272a5bd166adf2f7225ea582e10a46fafa
---
 Makefile.msc  | 11 ++++++++++-
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 3 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/Makefile.msc b/Makefile.msc
index 652bd81fd0..5bea16eab0 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -1495,7 +1495,8 @@ TESTPROGS = \
   sqlite3_analyzer.exe \
   sqlite3_checker.exe \
   sqldiff.exe \
-  dbhash.exe
+  dbhash.exe \
+  sqltclsh.exe
 
 # Databases containing fuzzer test cases
 #
@@ -2222,6 +2223,14 @@ sqlite3_analyzer.exe:	sqlite3_analyzer.c $(LIBRESOBJS)
 	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \
 		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
 
+sqltclsh.c: sqlite3.c $(TOP)\src\tclsqlite.c $(TOP)\tool\sqltclsh.tcl $(TOP)\ext\misc\appendvfs.c $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqltclsh.c.in	
+	$(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqltclsh.c.in >sqltclsh.c
+
+sqltclsh.exe: sqltclsh.c
+	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqltclsh.c \
+               /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LTLIBS) $(TLIBS)
+
+
 sqlite3_expert.exe: $(SQLITE3C) $(TOP)\ext\expert\sqlite3expert.h $(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c
 	$(LTLINK) $(NO_WARN)	$(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c $(SQLITE3C) $(TLIBS)
 
diff --git a/manifest b/manifest
index c5aed3eac5..3f55a3838d 100644
--- a/manifest
+++ b/manifest
@@ -1,8 +1,8 @@
-C Merge\srecent\senhancements\sfrom\strunk.
-D 2017-12-23T18:40:39.094
+C Add\ssqltclsh.exe\sto\sthe\swindows\smakefile.
+D 2018-01-06T04:34:05.887
 F Makefile.in f2dc8c140e1d728157834da295eaaa8a0cb29620595c2a9f0efc7258797e6f24
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69
+F Makefile.msc 80cca5fd5760cb3e26aaedaf6ef0df155cab3761a98efd0aa819973713839ca9
 F README.md eeae1e552f93ef72ef7c5b8f6647b368a001c28820ad1df179d3dae602bef681
 F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -1690,7 +1690,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ee248b529c2396c5480fb99b0a1dc31032627ec8241eca4a8c0fff257bb4a088 07c773148d8db185fa54991df09298b64f4fef28879e6c9395759265e8183977
-R 38880472a25b77bfdeb87344d5313674
+P edceaccd66a65d6b36e53ce33d760a7bd9c2261a592d12189f5f55417b5d5d74
+R 724c729f15e9ebb158a68ffbc1395b1f
 U drh
-Z 5274401ef77d35314ed9b1a10cd6e401
+Z adb127fe5f715fb70b7df6486f1fd9f2
diff --git a/manifest.uuid b/manifest.uuid
index 9da9bac5e4..3d96106fb5 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-edceaccd66a65d6b36e53ce33d760a7bd9c2261a592d12189f5f55417b5d5d74
\ No newline at end of file
+a6d5c7c2aabd70b4fb2bd8f6278f70272a5bd166adf2f7225ea582e10a46fafa
\ No newline at end of file

From 6b9986e84a3e0b3edce6c7f5a7693c1d9d725616 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 6 Jan 2018 13:33:21 +0000
Subject: [PATCH 244/488] Fix the appendvfs so that it leaves
 sqlite3_file->pMethods as NULL if it fails to open.

FossilOrigin-Name: 46b341e3ad11b807ae50f001b970299de7ea0d523dbb639ee10f1d5aca5d958b
---
 ext/misc/appendvfs.c | 13 +++++++------
 manifest             | 14 +++++++-------
 manifest.uuid        |  2 +-
 tool/sqltclsh.tcl    |  3 +++
 4 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/ext/misc/appendvfs.c b/ext/misc/appendvfs.c
index c6e9e0d58b..24d5e5f15b 100644
--- a/ext/misc/appendvfs.c
+++ b/ext/misc/appendvfs.c
@@ -440,14 +440,14 @@ static int apndOpen(
   }
   p = (ApndFile*)pFile;
   memset(p, 0, sizeof(*p));
-  p->base.pMethods = &apnd_io_methods;
   pSubFile = ORIGFILE(pFile);
+  p->base.pMethods = &apnd_io_methods;
   rc = pSubVfs->xOpen(pSubVfs, zName, pSubFile, flags, pOutFlags);
-  if( rc ) return rc;
+  if( rc ) goto apnd_open_done;
   rc = pSubFile->pMethods->xFileSize(pSubFile, &sz);
   if( rc ){
     pSubFile->pMethods->xClose(pSubFile);
-    return rc;
+    goto apnd_open_done;
   }
   if( apndIsOrdinaryDatabaseFile(sz, pSubFile) ){
     memmove(pFile, pSubFile, pSubVfs->szOsFile);
@@ -460,11 +460,12 @@ static int apndOpen(
   }
   if( (flags & SQLITE_OPEN_CREATE)==0 ){
     pSubFile->pMethods->xClose(pSubFile);
-    pFile->pMethods = 0;
-    return SQLITE_CANTOPEN;
+    rc = SQLITE_CANTOPEN;
   }
   p->iPgOne = (sz+0xfff) & ~(sqlite3_int64)0xfff;
-  return SQLITE_OK;
+apnd_open_done:
+  if( rc ) pFile->pMethods = 0;
+  return rc;
 }
 
 /*
diff --git a/manifest b/manifest
index 3f55a3838d..f93f6fa978 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\ssqltclsh.exe\sto\sthe\swindows\smakefile.
-D 2018-01-06T04:34:05.887
+C Fix\sthe\sappendvfs\sso\sthat\sit\sleaves\ssqlite3_file->pMethods\sas\sNULL\sif\sit\nfails\sto\sopen.
+D 2018-01-06T13:33:21.118
 F Makefile.in f2dc8c140e1d728157834da295eaaa8a0cb29620595c2a9f0efc7258797e6f24
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 80cca5fd5760cb3e26aaedaf6ef0df155cab3761a98efd0aa819973713839ca9
@@ -266,7 +266,7 @@ F ext/lsm1/tool/mklsm1c.tcl f31561bbee5349f0a554d1ad7236ac1991fc09176626f529f607
 F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601d90b2
 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87
 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb
-F ext/misc/appendvfs.c 8cc3ae6633e7d97ca9c3c9a48c647e9ba23f8d24f3d5f5f596272a81dc50c398
+F ext/misc/appendvfs.c 24b72d0556e81939c19316ca48c628938b4bd0ec81dac076406260c3ccd51def
 F ext/misc/btreeinfo.c d7fd9a2fe2fa33ba28488e2fce703ebecc759219ea9e0bb3b254784866c0a676
 F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005
 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704
@@ -1658,7 +1658,7 @@ F tool/split-sqlite3c.tcl 3efcd4240b738f6bb2b5af0aea7e1e0ef9bc1c61654f645076cec8
 F tool/sqldiff.c 30879bbc8de686df4624e86adce2d8981f500904c1cfb55b5d1eea2ffd9341eb
 F tool/sqlite3_analyzer.c.in 7eeaae8b0d7577662acaabbb11107af0659d1b41bc1dfdd4d91422de27127968
 F tool/sqltclsh.c.in e1f48150f755bfbe0194478cba50aa9f2f5183bb1efbdd6456532cce3cd2e18d
-F tool/sqltclsh.tcl 71bdb62c8f6b37c05b2d377e870c3dd13ee96cc4fee5b5ae5215ae9a0aca088d
+F tool/sqltclsh.tcl 18adb7d4a24374a3c79a2acc011b5f54a22186beec13056b0fa3ca011711c4f3
 F tool/srcck1.c 371de5363b70154012955544f86fdee8f6e5326f
 F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43
 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d
@@ -1690,7 +1690,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P edceaccd66a65d6b36e53ce33d760a7bd9c2261a592d12189f5f55417b5d5d74
-R 724c729f15e9ebb158a68ffbc1395b1f
+P a6d5c7c2aabd70b4fb2bd8f6278f70272a5bd166adf2f7225ea582e10a46fafa
+R 11d42f816c24c039c3f505e6266a4bae
 U drh
-Z adb127fe5f715fb70b7df6486f1fd9f2
+Z 83fb91d11f3679f988cee9a994d344b9
diff --git a/manifest.uuid b/manifest.uuid
index 3d96106fb5..73d812be11 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a6d5c7c2aabd70b4fb2bd8f6278f70272a5bd166adf2f7225ea582e10a46fafa
\ No newline at end of file
+46b341e3ad11b807ae50f001b970299de7ea0d523dbb639ee10f1d5aca5d958b
\ No newline at end of file
diff --git a/tool/sqltclsh.tcl b/tool/sqltclsh.tcl
index 999f166ccb..61d00f1df2 100644
--- a/tool/sqltclsh.tcl
+++ b/tool/sqltclsh.tcl
@@ -2,6 +2,9 @@
 # field where "scripts.name" is 'main.tcl'
 #
 catch {
+  if {![file exists $argv0] && [file exists $argv0.exe]} {
+    append argv0 .exe
+  }
   sqlite3 db $argv0 -vfs apndvfs -create 0
   set mainscript [db one {SELECT data FROM scripts WHERE name='main.tcl'}]
 }

From 0476b95c4333e0f0ee6854326518f2ecb00d0060 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Sat, 6 Jan 2018 14:43:33 +0000
Subject: [PATCH 245/488] Minor tweaks to the Makefile for MSVC.

FossilOrigin-Name: 2f6cc5d8a0d9d961d1daf46e8725f7373f740afa788cf99ede9689f49d3a54ec
---
 Makefile.msc  |  4 ++--
 manifest      | 15 +++++++--------
 manifest.uuid |  2 +-
 3 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/Makefile.msc b/Makefile.msc
index feab1b0a40..88a0a23a4d 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -2340,8 +2340,7 @@ sqltclsh.c: sqlite3.c $(TOP)\src\tclsqlite.c $(TOP)\tool\sqltclsh.tcl $(TOP)\ext
 
 sqltclsh.exe: sqltclsh.c
 	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqltclsh.c \
-               /link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
-
+		/link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
 
 sqlite3_expert.exe: $(SQLITE3C) $(TOP)\ext\expert\sqlite3expert.h $(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c
 	$(LTLINK) $(NO_WARN)	$(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c $(SQLITE3C) $(TLIBS)
@@ -2462,6 +2461,7 @@ clean:
 	del /Q sqlite3_analyzer.exe sqlite3_analyzer.c 2>NUL
 	del /Q sqlite-*-output.vsix 2>NUL
 	del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe dbhash.exe 2>NUL
+	del /Q sqltclsh.exe 2>NUL
 	del /Q fts5.* fts5parse.* 2>NUL
 	del /Q lsm.h lsm1.c 2>NUL
 # <</mark>>
diff --git a/manifest b/manifest
index 5b51a62935..e01a917ad1 100644
--- a/manifest
+++ b/manifest
@@ -1,10 +1,10 @@
-C Add\sthe\sappendvfs\sextension\sand\scode\sfor\sthe\ssqltclsh.exe\sexecutable.
-D 2018-01-06T13:42:27.074
+C Minor\stweaks\sto\sthe\sMakefile\sfor\sMSVC.
+D 2018-01-06T14:43:33.919
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 9536f61ce33172d4868707ecc10844a0abef9e2e775ad2434245a60406fd7e38
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc b85c373ee4d962cb52beddda3bc78d289a768f8ca8bd3d68eea8eadf6dd04d43
+F Makefile.msc 50cadb65af908a048a5624330782a91bc831a79da19ba17dcf7d273739191413
 F README.md eeae1e552f93ef72ef7c5b8f6647b368a001c28820ad1df179d3dae602bef681
 F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -1697,8 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 656fb5aab32f40412389e43de209867cedee1589533949cc8c01ba24218ea434 46b341e3ad11b807ae50f001b970299de7ea0d523dbb639ee10f1d5aca5d958b
-R 554a8f967a9f4add06a7ff77d66966de
-T +closed 46b341e3ad11b807ae50f001b970299de7ea0d523dbb639ee10f1d5aca5d958b
-U drh
-Z 85cb71c6d2228eace4706c2b9c3e1fa8
+P b0a43e325c53ec0df50fd41acffbbee65eac47b9318868e6959cd39da66fa8d6
+R c768913b6d54732f0058c9759aabde34
+U mistachkin
+Z a162015bb17d305104612ae9f13f8b23
diff --git a/manifest.uuid b/manifest.uuid
index 06963ea079..588573c2f4 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b0a43e325c53ec0df50fd41acffbbee65eac47b9318868e6959cd39da66fa8d6
\ No newline at end of file
+2f6cc5d8a0d9d961d1daf46e8725f7373f740afa788cf99ede9689f49d3a54ec
\ No newline at end of file

From 5e6f782f118c42b766163434a105071cb4904c0f Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Sat, 6 Jan 2018 14:44:29 +0000
Subject: [PATCH 246/488] Remove a trailing tab from the Makefile for MSVC.

FossilOrigin-Name: 00cc26e34d2b81f140b031aa2f9ae0e2a4835cdd261d68f94b3e18a1388ca73d
---
 Makefile.msc  |  2 +-
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/Makefile.msc b/Makefile.msc
index 88a0a23a4d..d103cdc6e3 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -2335,7 +2335,7 @@ sqlite3_analyzer.exe:	sqlite3_analyzer.c $(LIBRESOBJS)
 	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \
 		/link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
 
-sqltclsh.c: sqlite3.c $(TOP)\src\tclsqlite.c $(TOP)\tool\sqltclsh.tcl $(TOP)\ext\misc\appendvfs.c $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqltclsh.c.in	
+sqltclsh.c: sqlite3.c $(TOP)\src\tclsqlite.c $(TOP)\tool\sqltclsh.tcl $(TOP)\ext\misc\appendvfs.c $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqltclsh.c.in
 	$(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqltclsh.c.in >sqltclsh.c
 
 sqltclsh.exe: sqltclsh.c
diff --git a/manifest b/manifest
index e01a917ad1..7690a6a790 100644
--- a/manifest
+++ b/manifest
@@ -1,10 +1,10 @@
-C Minor\stweaks\sto\sthe\sMakefile\sfor\sMSVC.
-D 2018-01-06T14:43:33.919
+C Remove\sa\strailing\stab\sfrom\sthe\sMakefile\sfor\sMSVC.
+D 2018-01-06T14:44:29.580
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 9536f61ce33172d4868707ecc10844a0abef9e2e775ad2434245a60406fd7e38
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 50cadb65af908a048a5624330782a91bc831a79da19ba17dcf7d273739191413
+F Makefile.msc 688fb6375f78dc41f3a4a67201e4d5ef62ef556faa776c7cbe2c50a887d1fa5d
 F README.md eeae1e552f93ef72ef7c5b8f6647b368a001c28820ad1df179d3dae602bef681
 F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b0a43e325c53ec0df50fd41acffbbee65eac47b9318868e6959cd39da66fa8d6
-R c768913b6d54732f0058c9759aabde34
+P 2f6cc5d8a0d9d961d1daf46e8725f7373f740afa788cf99ede9689f49d3a54ec
+R 5465f05e31d38450dc68aa4d6f2bc0a7
 U mistachkin
-Z a162015bb17d305104612ae9f13f8b23
+Z 16a3cdc1a076fc6d00702b3c46501e75
diff --git a/manifest.uuid b/manifest.uuid
index 588573c2f4..165554be57 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2f6cc5d8a0d9d961d1daf46e8725f7373f740afa788cf99ede9689f49d3a54ec
\ No newline at end of file
+00cc26e34d2b81f140b031aa2f9ae0e2a4835cdd261d68f94b3e18a1388ca73d
\ No newline at end of file

From 11be81d41365f3369434c81c8d62ced4c25f5b70 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 6 Jan 2018 15:46:20 +0000
Subject: [PATCH 247/488] Improved output from ".schema --indent" when a column
 definition is followed by a comment.

FossilOrigin-Name: 87da7efff07327278b1437f862ed8683c2d5d6ada7ea7461601a58f9762646b4
---
 manifest       | 14 +++++++-------
 manifest.uuid  |  2 +-
 src/shell.c.in | 24 ++++++++++++++++++++++--
 3 files changed, 30 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index 7690a6a790..b76e9b121f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\sa\strailing\stab\sfrom\sthe\sMakefile\sfor\sMSVC.
-D 2018-01-06T14:44:29.580
+C Improved\soutput\sfrom\s".schema\s--indent"\swhen\sa\scolumn\sdefinition\sis\sfollowed\nby\sa\scomment.
+D 2018-01-06T15:46:20.603
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 9536f61ce33172d4868707ecc10844a0abef9e2e775ad2434245a60406fd7e38
@@ -484,7 +484,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in 4cfa9394737306c5a5f3d7f72e74868df5f3010676d6a1a2a52db37b61b0c13a
+F src/shell.c.in 8874828ece41dc20767b1dcee40b0784dfb934cb8affe020b616542f57b42ea9
 F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2f6cc5d8a0d9d961d1daf46e8725f7373f740afa788cf99ede9689f49d3a54ec
-R 5465f05e31d38450dc68aa4d6f2bc0a7
-U mistachkin
-Z 16a3cdc1a076fc6d00702b3c46501e75
+P 00cc26e34d2b81f140b031aa2f9ae0e2a4835cdd261d68f94b3e18a1388ca73d
+R eb5f4dd564d1c772c17fe91ddfa9e74c
+U drh
+Z 0810beaa8d664bc298dc9e0f39c4629f
diff --git a/manifest.uuid b/manifest.uuid
index 165554be57..d009b990da 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-00cc26e34d2b81f140b031aa2f9ae0e2a4835cdd261d68f94b3e18a1388ca73d
\ No newline at end of file
+87da7efff07327278b1437f862ed8683c2d5d6ada7ea7461601a58f9762646b4
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index bb4428fbc5..c113bdb934 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -1411,6 +1411,22 @@ static void printSchemaLineN(FILE *out, char *z, int n, const char *zTail){
   z[n] = c;
 }
 
+/*
+** Return true if string z[] has nothing but whitespace and comments to the
+** end of the first line.
+*/
+static int wsToEol(const char *z){
+  int i;
+  for(i=0; z[i]; i++){
+    if( z[i]=='\n' ) return 1;
+    if( IsSpace(z[i]) ) continue;
+    if( z[i]=='-' && z[i+1]=='-' ) return 1;
+    return 0;
+  }
+  return 1;
+}
+    
+
 /*
 ** This is the callback routine that the shell
 ** invokes for each row of a query result.
@@ -1550,13 +1566,15 @@ static int shell_callback(
       while( j>0 && IsSpace(z[j-1]) ){ j--; }
       z[j] = 0;
       if( strlen30(z)>=79 ){
-        for(i=j=0; (c = z[i])!=0; i++){
+        for(i=j=0; (c = z[i])!=0; i++){  /* Copy changes from z[i] back to z[j] */
           if( c==cEnd ){
             cEnd = 0;
           }else if( c=='"' || c=='\'' || c=='`' ){
             cEnd = c;
           }else if( c=='[' ){
             cEnd = ']';
+          }else if( c=='-' && z[i+1]=='-' ){
+            cEnd = '\n';
           }else if( c=='(' ){
             nParen++;
           }else if( c==')' ){
@@ -1567,7 +1585,9 @@ static int shell_callback(
             }
           }
           z[j++] = c;
-          if( nParen==1 && (c=='(' || c==',' || c=='\n') ){
+          if( nParen==1 && cEnd==0
+           && (c=='(' || c=='\n' || (c==',' && !wsToEol(z+i+1)))
+          ){
             if( c=='\n' ) j--;
             printSchemaLineN(p->out, z, j, "\n  ");
             j = 0;

From afba1802a059c5e2a1ca549c0ca6abb31586ab43 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 6 Jan 2018 15:49:57 +0000
Subject: [PATCH 248/488] Better whitespace in the "CREATE TABLE sqlar()"
 statement for the ".ar" command.

FossilOrigin-Name: 252ee55a7fc0b068b707af27bd912e684c28320996e78f0675217046b8c2fb49
---
 manifest       | 12 ++++++------
 manifest.uuid  |  2 +-
 src/shell.c.in | 14 +++++++-------
 3 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/manifest b/manifest
index b76e9b121f..d08eceabe8 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improved\soutput\sfrom\s".schema\s--indent"\swhen\sa\scolumn\sdefinition\sis\sfollowed\nby\sa\scomment.
-D 2018-01-06T15:46:20.603
+C Better\swhitespace\sin\sthe\s"CREATE\sTABLE\ssqlar()"\sstatement\sfor\sthe\s".ar"\ncommand.
+D 2018-01-06T15:49:57.663
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 9536f61ce33172d4868707ecc10844a0abef9e2e775ad2434245a60406fd7e38
@@ -484,7 +484,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in 8874828ece41dc20767b1dcee40b0784dfb934cb8affe020b616542f57b42ea9
+F src/shell.c.in 1f1d02e83a4805db87d9405893e5375c9f7e6906166dead2a783d3f175cebcb0
 F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 00cc26e34d2b81f140b031aa2f9ae0e2a4835cdd261d68f94b3e18a1388ca73d
-R eb5f4dd564d1c772c17fe91ddfa9e74c
+P 87da7efff07327278b1437f862ed8683c2d5d6ada7ea7461601a58f9762646b4
+R 1cf086413b922edba322b9602699139d
 U drh
-Z 0810beaa8d664bc298dc9e0f39c4629f
+Z a5d68d44f94deab667b4cbaa7e1747ee
diff --git a/manifest.uuid b/manifest.uuid
index d009b990da..24f4d757a4 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-87da7efff07327278b1437f862ed8683c2d5d6ada7ea7461601a58f9762646b4
\ No newline at end of file
+252ee55a7fc0b068b707af27bd912e684c28320996e78f0675217046b8c2fb49
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index c113bdb934..78c9124736 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -4858,13 +4858,13 @@ static int arCreateUpdate(
 ){
   const char *zSql = "SELECT name, mode, mtime, data FROM fsdir(?, ?)";
   const char *zCreate = 
-      "CREATE TABLE IF NOT EXISTS sqlar("
-      "name TEXT PRIMARY KEY,  -- name of the file\n"
-      "mode INT,               -- access permissions\n"
-      "mtime INT,              -- last modification time\n"
-      "sz INT,                 -- original file size\n"
-      "data BLOB               -- compressed content\n"
-  ")";
+      "CREATE TABLE IF NOT EXISTS sqlar(\n"
+      "  name TEXT PRIMARY KEY,  -- name of the file\n"
+      "  mode INT,               -- access permissions\n"
+      "  mtime INT,              -- last modification time\n"
+      "  sz INT,                 -- original file size\n"
+      "  data BLOB               -- compressed content\n"
+      ")";
   const char *zDrop = "DROP TABLE IF EXISTS sqlar";
   const char *zInsert = "REPLACE INTO sqlar VALUES(?,?,?,?,sqlar_compress(?))";
 

From e37c0e1ce786fd87e10ba40456ceff7f2f0e4d51 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 6 Jan 2018 19:19:50 +0000
Subject: [PATCH 249/488] In the shell, include the ".archive" command only if
 compiling with SQLITE_HAVE_ZLIB.  Add ".archive" to the ".help" output.

FossilOrigin-Name: 366469f5603367fabcadfc9ffe8cd1e23c649fea49a560178ca0858a16a7e4d1
---
 manifest       | 12 ++++++------
 manifest.uuid  |  2 +-
 src/shell.c.in | 17 +++++++++++++----
 3 files changed, 20 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index d08eceabe8..59f92e796c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Better\swhitespace\sin\sthe\s"CREATE\sTABLE\ssqlar()"\sstatement\sfor\sthe\s".ar"\ncommand.
-D 2018-01-06T15:49:57.663
+C In\sthe\sshell,\sinclude\sthe\s".archive"\scommand\sonly\sif\scompiling\swith\nSQLITE_HAVE_ZLIB.\s\sAdd\s".archive"\sto\sthe\s".help"\soutput.
+D 2018-01-06T19:19:50.442
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 9536f61ce33172d4868707ecc10844a0abef9e2e775ad2434245a60406fd7e38
@@ -484,7 +484,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in 1f1d02e83a4805db87d9405893e5375c9f7e6906166dead2a783d3f175cebcb0
+F src/shell.c.in 9f2ab2d0b4b07310950ec84492c067f9c65a2c934b2704f07bf3f7abd81b1326
 F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 87da7efff07327278b1437f862ed8683c2d5d6ada7ea7461601a58f9762646b4
-R 1cf086413b922edba322b9602699139d
+P 252ee55a7fc0b068b707af27bd912e684c28320996e78f0675217046b8c2fb49
+R 1106ed2dd57a38999d04b0134db88743
 U drh
-Z a5d68d44f94deab667b4cbaa7e1747ee
+Z 0971e8c5cac3e7845d10877eae148e13
diff --git a/manifest.uuid b/manifest.uuid
index 24f4d757a4..289c74b7d1 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-252ee55a7fc0b068b707af27bd912e684c28320996e78f0675217046b8c2fb49
\ No newline at end of file
+366469f5603367fabcadfc9ffe8cd1e23c649fea49a560178ca0858a16a7e4d1
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 78c9124736..bff3684321 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -2895,6 +2895,9 @@ static int run_schema_dump_query(
 ** Text of a help message
 */
 static char zHelp[] =
+#if defined(SQLITE_HAVE_ZLIB) && !defined(SQLITE_OMIT_VIRTUALTABLE)
+  ".archive ...           Manage SQL archives: \".archive --help\" for details\n"
+#endif
 #ifndef SQLITE_OMIT_AUTHORIZATION
   ".auth ON|OFF           Show authorizer callbacks\n"
 #endif
@@ -4296,6 +4299,10 @@ static int lintDotCommand(
   return SQLITE_ERROR;
 }
 
+#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB)
+/*********************************************************************************
+** The ".archive" or ".ar" command.
+*/
 static void shellPrepare(
   sqlite3 *db, 
   int *pRc, 
@@ -4365,8 +4372,7 @@ static void shellReset(
     *pRc = rc;
   }
 }
-
-/* 
+/*
 ** Structure representing a single ".ar" command.
 */
 typedef struct ArCommand ArCommand;
@@ -5034,6 +5040,9 @@ static int arDotCommand(
 
   return rc;
 }
+/* End of the ".archive" or ".ar" command logic
+**********************************************************************************/
+#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) */
 
 /*
 ** Implementation of ".expert" dot command.
@@ -5156,8 +5165,8 @@ static int do_meta_command(char *zLine, ShellState *p){
   }else
 #endif
 
-#ifndef SQLITE_OMIT_VIRTUALTABLE
-  if( c=='a' && strncmp(azArg[0], "ar", n)==0 ){
+#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB)
+  if( c=='a' && strncmp(azArg[0], "archive", n)==0 ){
     open_db(p, 0);
     rc = arDotCommand(p, azArg, nArg);
   }else

From 1fa6d9f96f8cda45b2a3123241cd7e4971914830 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 6 Jan 2018 21:46:01 +0000
Subject: [PATCH 250/488] The shell detects and opens ZIP archives using the
 zipfile extension.

FossilOrigin-Name: 05c99eb8cefbb3366b6d4ae91e10aa0c82bdf5ea361f4b3375413783af9167ac
---
 manifest       | 12 ++++----
 manifest.uuid  |  2 +-
 src/shell.c.in | 83 +++++++++++++++++++++++++++++++++++++++++++++++---
 3 files changed, 85 insertions(+), 12 deletions(-)

diff --git a/manifest b/manifest
index 59f92e796c..83bebcd068 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\sthe\sshell,\sinclude\sthe\s".archive"\scommand\sonly\sif\scompiling\swith\nSQLITE_HAVE_ZLIB.\s\sAdd\s".archive"\sto\sthe\s".help"\soutput.
-D 2018-01-06T19:19:50.442
+C The\sshell\sdetects\sand\sopens\sZIP\sarchives\susing\sthe\szipfile\sextension.
+D 2018-01-06T21:46:01.434
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 9536f61ce33172d4868707ecc10844a0abef9e2e775ad2434245a60406fd7e38
@@ -484,7 +484,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in 9f2ab2d0b4b07310950ec84492c067f9c65a2c934b2704f07bf3f7abd81b1326
+F src/shell.c.in c2231d96fc059e2a6c86d67571db0dc7e029de25553a42c3334a6ef4c8e92484
 F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 252ee55a7fc0b068b707af27bd912e684c28320996e78f0675217046b8c2fb49
-R 1106ed2dd57a38999d04b0134db88743
+P 366469f5603367fabcadfc9ffe8cd1e23c649fea49a560178ca0858a16a7e4d1
+R 0f23134ec1d9b30706dc56d4edcd4f2b
 U drh
-Z 0971e8c5cac3e7845d10877eae148e13
+Z bf2fee513f6949a58534392bd37a9abf
diff --git a/manifest.uuid b/manifest.uuid
index 289c74b7d1..95d424063b 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-366469f5603367fabcadfc9ffe8cd1e23c649fea49a560178ca0858a16a7e4d1
\ No newline at end of file
+05c99eb8cefbb3366b6d4ae91e10aa0c82bdf5ea361f4b3375413783af9167ac
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index bff3684321..71f9f19cb2 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -63,6 +63,7 @@
 #include "sqlite3.h"
 typedef sqlite3_int64 i64;
 typedef sqlite3_uint64 u64;
+typedef unsigned char u8;
 #if SQLITE_USER_AUTHENTICATION
 # include "sqlite3userauth.h"
 #endif
@@ -938,10 +939,11 @@ struct ExpertInfo {
 typedef struct ShellState ShellState;
 struct ShellState {
   sqlite3 *db;           /* The database */
-  int autoExplain;       /* Automatically turn on .explain mode */
-  int autoEQP;           /* Run EXPLAIN QUERY PLAN prior to seach SQL stmt */
-  int statsOn;           /* True to display memory stats before each finalize */
-  int scanstatsOn;       /* True to display scan stats before each finalize */
+  u8 autoExplain;        /* Automatically turn on .explain mode */
+  u8 autoEQP;            /* Run EXPLAIN QUERY PLAN prior to seach SQL stmt */
+  u8 statsOn;            /* True to display memory stats before each finalize */
+  u8 scanstatsOn;        /* True to display scan stats before each finalize */
+  u8 openMode;           /* SHELL_OPEN_NORMAL, _APPENDVFS, or _ZIPFILE */
   int outCount;          /* Revert to stdout when reaching zero */
   int cnt;               /* Number of records displayed so far */
   FILE *out;             /* Write results here */
@@ -978,6 +980,7 @@ struct ShellState {
   ExpertInfo expert;        /* Valid if previous command was ".expert OPT..." */
 };
 
+
 /* Allowed values for ShellState.autoEQP
 */
 #define AUTOEQP_off      0
@@ -985,6 +988,13 @@ struct ShellState {
 #define AUTOEQP_trigger  2
 #define AUTOEQP_full     3
 
+/* Allowed values for ShellState.openMode
+*/
+#define SHELL_OPEN_UNSPEC     0      /* No open-mode specified */
+#define SHELL_OPEN_NORMAL     1      /* Normal database file */
+#define SHELL_OPEN_APPENDVFS  2      /* Use appendvfs */
+#define SHELL_OPEN_ZIPFILE    3      /* Use the zipfile virtual table */
+
 /*
 ** These are the allowed shellFlgs values
 */
@@ -3101,6 +3111,32 @@ static int session_filter(void *pCtx, const char *zTab){
 }
 #endif
 
+/*
+** Try to deduce the type of file for zName based on its content.  Return
+** one of the SHELL_OPEN_* constants.
+*/
+static int deduceDatabaseType(const char *zName){
+  FILE *f = fopen(zName, "rb");
+  size_t n;
+  int rc = SHELL_OPEN_UNSPEC;
+  char zBuf[100];
+  if( f==0 ) return SHELL_OPEN_NORMAL;
+  fseek(f, -25, SEEK_END);
+  n = fread(zBuf, 25, 1, f);
+  if( n==1 && memcmp(zBuf, "Start-Of-SQLite3-", 17)==0 ){
+    rc = SHELL_OPEN_APPENDVFS;
+  }else{
+    fseek(f, -22, SEEK_END);
+    n = fread(zBuf, 22, 1, f);
+    if( n==1 && zBuf[0]==0x50 && zBuf[1]==0x4b && zBuf[2]==0x05
+       && zBuf[3]==0x06 ){
+      rc = SHELL_OPEN_ZIPFILE;
+    }
+  }
+  fclose(f);
+  return rc;  
+}
+
 /*
 ** Make sure the database is open.  If it is not, then open it.  If
 ** the database fails to open, print an error message and exit.
@@ -3108,7 +3144,25 @@ static int session_filter(void *pCtx, const char *zTab){
 static void open_db(ShellState *p, int keepAlive){
   if( p->db==0 ){
     sqlite3_initialize();
-    sqlite3_open(p->zDbFilename, &p->db);
+    if( p->openMode==SHELL_OPEN_UNSPEC && access(p->zDbFilename,0)==0 ){
+      p->openMode = deduceDatabaseType(p->zDbFilename);
+    }
+    switch( p->openMode ){
+      case SHELL_OPEN_APPENDVFS: {
+        sqlite3_open_v2(p->zDbFilename, &p->db, 
+           SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, "apndvfs");
+        break;
+      }
+      case SHELL_OPEN_ZIPFILE: {
+        sqlite3_open(":memory:", &p->db);
+        break;
+      }
+      case SHELL_OPEN_UNSPEC:
+      case SHELL_OPEN_NORMAL: {
+        sqlite3_open(p->zDbFilename, &p->db);
+        break;
+      }
+    }
     globalDb = p->db;
     if( p->db==0 || SQLITE_OK!=sqlite3_errcode(p->db) ){
       utf8_printf(stderr,"Error: unable to open database \"%s\": %s\n",
@@ -3130,6 +3184,12 @@ static void open_db(ShellState *p, int keepAlive){
                             shellAddSchemaName, 0, 0);
     sqlite3_create_function(p->db, "shell_module_schema", 1, SQLITE_UTF8, 0,
                             shellModuleSchema, 0, 0);
+    if( p->openMode==SHELL_OPEN_ZIPFILE ){
+      char *zSql = sqlite3_mprintf(
+         "CREATE VIRTUAL TABLE zip USING zipfile(%Q);", p->zDbFilename);
+      sqlite3_exec(p->db, zSql, 0, 0, 0);
+      sqlite3_free(zSql);
+    }
   }
 }
 
@@ -6012,11 +6072,18 @@ static int do_meta_command(char *zLine, ShellState *p){
     p->zDbFilename = 0;
     sqlite3_free(p->zFreeOnClose);
     p->zFreeOnClose = 0;
+    p->openMode = SHELL_OPEN_UNSPEC;
     /* Check for command-line arguments */
     for(iName=1; iName<nArg && azArg[iName][0]=='-'; iName++){
       const char *z = azArg[iName];
       if( optionMatch(z,"new") ){
         newFlag = 1;
+#ifdef SQLITE_HAVE_ZIP
+      }else if( optionMatch(z, "zip") ){
+        p->openMode = SHELL_OPEN_ZIPFILE;
+#endif
+      }else if( optionMatch(z, "append") ){
+        p->openMode = SHELL_OPEN_APPENDVFS;
       }else if( z[0]=='-' ){
         utf8_printf(stderr, "unknown option: %s\n", z);
         rc = 1;
@@ -7937,6 +8004,12 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
     }else if( strcmp(z,"-csv")==0 ){
       data.mode = MODE_Csv;
       memcpy(data.colSeparator,",",2);
+#ifdef SQLITE_HAVE_ZIP
+    }else if( strcmp(z,"-zip")==0 ){
+      data.openMode = SHELL_OPEN_ZIPFILE;
+#endif
+    }else if( strcmp(z,"-append")==0 ){
+      data.openMode = SHELL_OPEN_APPENDVFS;
     }else if( strcmp(z,"-ascii")==0 ){
       data.mode = MODE_Ascii;
       sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator,

From acd6fdee98ba573ab1e3be9a7f6dd4eeb4a213c8 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sun, 7 Jan 2018 19:52:28 +0000
Subject: [PATCH 251/488] Improved comments on the sqlar.c and compress.c
 extensions that describe the differences between the "zlib format" used by
 SQLAR, the raw deflate format used by ZIP, and the custom format used by
 compress.c.  No changes to code.

FossilOrigin-Name: c13415c5caf06eaa73086c500907451dfcdfd1354ac4a9ab63edb7255edbfa1b
---
 ext/misc/compress.c | 15 +++++++++++++++
 ext/misc/sqlar.c    |  8 ++++++++
 manifest            | 14 +++++++-------
 manifest.uuid       |  2 +-
 4 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/ext/misc/compress.c b/ext/misc/compress.c
index bf38d4c93c..6e7d8b6148 100644
--- a/ext/misc/compress.c
+++ b/ext/misc/compress.c
@@ -27,6 +27,21 @@ SQLITE_EXTENSION_INIT1
 ** seven bits per integer stored in the lower seven bits of each byte.
 ** More significant bits occur first.  The most significant bit (0x80)
 ** is a flag to indicate the end of the integer.
+**
+** This function, SQLAR, and ZIP all use the same "deflate" compression
+** algorithm, but each is subtly different:
+**
+**   *  ZIP uses raw deflate.
+**
+**   *  SQLAR uses the "zlib format" which is raw deflate with a two-byte
+**      algorithm-identification header and a four-byte checksum at the end.
+**
+**   *  This utility uses the "zlib format" like SQLAR, but adds the variable-
+**      length integer uncompressed size value at the beginning.
+**
+** This function might be extended in the future to support compression
+** formats other than deflate, by providing a different algorithm-id
+** mark following the variable-length integer size parameter.
 */
 static void compressFunc(
   sqlite3_context *context,
diff --git a/ext/misc/sqlar.c b/ext/misc/sqlar.c
index 17d0875938..e812d70c99 100644
--- a/ext/misc/sqlar.c
+++ b/ext/misc/sqlar.c
@@ -24,6 +24,14 @@ SQLITE_EXTENSION_INIT1
 ** If the type of X is SQLITE_BLOB, and compressing that blob using
 ** zlib utility function compress() yields a smaller blob, return the
 ** compressed blob. Otherwise, return a copy of X.
+**
+** SQLar uses the "zlib format" for compressed content.  The zlib format
+** contains a two-byte identification header and a four-byte checksum at
+** the end.  This is different from ZIP which uses the raw deflate format.
+**
+** Future enhancements to SQLar might add support for new compression formats.
+** If so, those new formats will be identified by alternative headers in the
+** compressed data.
 */
 static void sqlarCompressFunc(
   sqlite3_context *context,
diff --git a/manifest b/manifest
index 83bebcd068..88ec153d89 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C The\sshell\sdetects\sand\sopens\sZIP\sarchives\susing\sthe\szipfile\sextension.
-D 2018-01-06T21:46:01.434
+C Improved\scomments\son\sthe\ssqlar.c\sand\scompress.c\sextensions\sthat\sdescribe\nthe\sdifferences\sbetween\sthe\s"zlib\sformat"\sused\sby\sSQLAR,\sthe\sraw\sdeflate\nformat\sused\sby\sZIP,\sand\sthe\scustom\sformat\sused\sby\scompress.c.\s\sNo\schanges\nto\scode.
+D 2018-01-07T19:52:28.395
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 9536f61ce33172d4868707ecc10844a0abef9e2e775ad2434245a60406fd7e38
@@ -273,7 +273,7 @@ F ext/misc/btreeinfo.c d7fd9a2fe2fa33ba28488e2fce703ebecc759219ea9e0bb3b25478486
 F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005
 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704
 F ext/misc/completion.c 52c3f01523e3e387eb321b4739a89d1fe47cbe6025aa1f2d8d3685e9e365df0f
-F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83
+F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f035b189
 F ext/misc/csv.c 1a009b93650732e22334edc92459c4630b9fa703397cbb3c8ca279921a36ca11
 F ext/misc/dbdump.c 3509fa6b8932d04e932d6b6b827b6a82ca362781b8e8f3c77336f416793e215e
 F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2
@@ -294,7 +294,7 @@ F ext/misc/sha1.c 0b9e9b855354910d3ca467bf39099d570e73db56
 F ext/misc/shathree.c 9e960ba50483214c6a7a4b1517f8d8cef799e9db381195178c3fd3ad207e10c0
 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52
 F ext/misc/spellfix.c 41cf26c6b89fcaa8798ae10ae64d39c1f1d9d6995152e545bd491c13058b8fac
-F ext/misc/sqlar.c d355cd8b6e7280d2f61d4737672922acb512a2ab1cee52399ffb88980476e31c
+F ext/misc/sqlar.c 57d5bc45cd5492208e451f697404be88f8612527d64c9d42f96b325b64983d74
 F ext/misc/stmt.c 6f16443abb3551e3f5813bb13ba19a30e7032830015b0f92fe0c0453045c0a11
 F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512
 F ext/misc/unionvtab.c 2aa94902ea646e1aaf6c05eac944a14276cddd67735b2ad856030ffffbb6626c
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 366469f5603367fabcadfc9ffe8cd1e23c649fea49a560178ca0858a16a7e4d1
-R 0f23134ec1d9b30706dc56d4edcd4f2b
+P 05c99eb8cefbb3366b6d4ae91e10aa0c82bdf5ea361f4b3375413783af9167ac
+R 1813bb08dedce0701d741a99a479782e
 U drh
-Z bf2fee513f6949a58534392bd37a9abf
+Z e662d2fceaa8f576c7d1d1fef9cf3ce7
diff --git a/manifest.uuid b/manifest.uuid
index 95d424063b..348cdf03c2 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-05c99eb8cefbb3366b6d4ae91e10aa0c82bdf5ea361f4b3375413783af9167ac
\ No newline at end of file
+c13415c5caf06eaa73086c500907451dfcdfd1354ac4a9ab63edb7255edbfa1b
\ No newline at end of file

From 8682e121906363c9573f66f4dd893969e095cc79 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sun, 7 Jan 2018 20:38:10 +0000
Subject: [PATCH 252/488] Modify the sqltclsh startup script to look for a
 properly formatted SQLAR at the end of the executable.  Fix the CLI so that
 it automatically links against appendvfs and so that the --append
 command-line option works.

FossilOrigin-Name: 67c4a8c6881e33b830aa27c80e7e3d697a4222939edd77cd5ca77ece16471ea4
---
 Makefile.in          |  1 +
 Makefile.msc         |  1 +
 ext/misc/appendvfs.c |  4 +---
 main.mk              |  1 +
 manifest             | 24 ++++++++++++------------
 manifest.uuid        |  2 +-
 src/shell.c.in       |  8 ++++++++
 tool/sqltclsh.c.in   |  5 +++++
 tool/sqltclsh.tcl    |  4 +++-
 9 files changed, 33 insertions(+), 17 deletions(-)

diff --git a/Makefile.in b/Makefile.in
index 64cb62a135..af1bd61548 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -994,6 +994,7 @@ keywordhash.h:	$(TOP)/tool/mkkeywordhash.c
 # Source files that go into making shell.c
 SHELL_SRC = \
 	$(TOP)/src/shell.c.in \
+        $(TOP)/ext/misc/appendvfs.c \
 	$(TOP)/ext/misc/shathree.c \
 	$(TOP)/ext/misc/fileio.c \
 	$(TOP)/ext/misc/completion.c \
diff --git a/Makefile.msc b/Makefile.msc
index d103cdc6e3..4d6c5c0cc2 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -2083,6 +2083,7 @@ keywordhash.h:	$(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe
 # Source files that go into making shell.c
 SHELL_SRC = \
 	$(TOP)\src\shell.c.in \
+        $(TOP)\ext\misc\appendvfs.c \
 	$(TOP)\ext\misc\shathree.c \
 	$(TOP)\ext\misc\fileio.c \
 	$(TOP)\ext\misc\completion.c \
diff --git a/ext/misc/appendvfs.c b/ext/misc/appendvfs.c
index 24d5e5f15b..1454243057 100644
--- a/ext/misc/appendvfs.c
+++ b/ext/misc/appendvfs.c
@@ -47,9 +47,7 @@
 ** If the file being opened is not an appended database, then this shim is
 ** a pass-through into the default underlying VFS.
 **/
-#if !defined(SQLITEINT_H)
-#include <sqlite3ext.h>
-#endif
+#include "sqlite3ext.h"
 SQLITE_EXTENSION_INIT1
 #include <string.h>
 #include <assert.h>
diff --git a/main.mk b/main.mk
index a16aad3860..fc68d9d307 100644
--- a/main.mk
+++ b/main.mk
@@ -694,6 +694,7 @@ keywordhash.h:	$(TOP)/tool/mkkeywordhash.c
 # Source files that go into making shell.c
 SHELL_SRC = \
 	$(TOP)/src/shell.c.in \
+        $(TOP)/ext/misc/appendvfs.c \
 	$(TOP)/ext/misc/shathree.c \
 	$(TOP)/ext/misc/fileio.c \
 	$(TOP)/ext/misc/completion.c \
diff --git a/manifest b/manifest
index 88ec153d89..f28f322507 100644
--- a/manifest
+++ b/manifest
@@ -1,10 +1,10 @@
-C Improved\scomments\son\sthe\ssqlar.c\sand\scompress.c\sextensions\sthat\sdescribe\nthe\sdifferences\sbetween\sthe\s"zlib\sformat"\sused\sby\sSQLAR,\sthe\sraw\sdeflate\nformat\sused\sby\sZIP,\sand\sthe\scustom\sformat\sused\sby\scompress.c.\s\sNo\schanges\nto\scode.
-D 2018-01-07T19:52:28.395
+C Modify\sthe\ssqltclsh\sstartup\sscript\sto\slook\sfor\sa\sproperly\sformatted\nSQLAR\sat\sthe\send\sof\sthe\sexecutable.\s\sFix\sthe\sCLI\sso\sthat\sit\sautomatically\nlinks\sagainst\sappendvfs\sand\sso\sthat\sthe\s--append\scommand-line\soption\sworks.
+D 2018-01-07T20:38:10.880
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
-F Makefile.in 9536f61ce33172d4868707ecc10844a0abef9e2e775ad2434245a60406fd7e38
+F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 688fb6375f78dc41f3a4a67201e4d5ef62ef556faa776c7cbe2c50a887d1fa5d
+F Makefile.msc b33f630258ce72fb3150c1a6819a484eba3950dc9397571d48672087e12abf4a
 F README.md eeae1e552f93ef72ef7c5b8f6647b368a001c28820ad1df179d3dae602bef681
 F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -268,7 +268,7 @@ F ext/lsm1/tool/mklsm1c.tcl f31561bbee5349f0a554d1ad7236ac1991fc09176626f529f607
 F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601d90b2
 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87
 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb
-F ext/misc/appendvfs.c 24b72d0556e81939c19316ca48c628938b4bd0ec81dac076406260c3ccd51def
+F ext/misc/appendvfs.c 4c65f0b79686ae5a483134233d7fd912f0f2d4fd76023404f96f2290fff13b19
 F ext/misc/btreeinfo.c d7fd9a2fe2fa33ba28488e2fce703ebecc759219ea9e0bb3b254784866c0a676
 F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005
 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704
@@ -406,7 +406,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk a1ecafe75de24b10286572033e195d0da53a7a05773b4f23ff509b54f7056f0f
+F main.mk 7965d01485f7bec7945407c5238985ea8c7cb2cb686d2bfdbe3d5f79d6fd4eb2
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -484,7 +484,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in c2231d96fc059e2a6c86d67571db0dc7e029de25553a42c3334a6ef4c8e92484
+F src/shell.c.in ffcef0e1b2497b32c13f00ab9a1fececeeeb678dd04f5b17c0b5fa0cc99cd105
 F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1664,8 +1664,8 @@ F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd
 F tool/split-sqlite3c.tcl 3efcd4240b738f6bb2b5af0aea7e1e0ef9bc1c61654f645076cec883030b710c
 F tool/sqldiff.c 30879bbc8de686df4624e86adce2d8981f500904c1cfb55b5d1eea2ffd9341eb
 F tool/sqlite3_analyzer.c.in 7eeaae8b0d7577662acaabbb11107af0659d1b41bc1dfdd4d91422de27127968
-F tool/sqltclsh.c.in e1f48150f755bfbe0194478cba50aa9f2f5183bb1efbdd6456532cce3cd2e18d
-F tool/sqltclsh.tcl 18adb7d4a24374a3c79a2acc011b5f54a22186beec13056b0fa3ca011711c4f3
+F tool/sqltclsh.c.in 8b2529b6c3cdd8ad6aaff21e80eb58370c428cb207b4607d0da4def064dcec56
+F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848
 F tool/srcck1.c 371de5363b70154012955544f86fdee8f6e5326f
 F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43
 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 05c99eb8cefbb3366b6d4ae91e10aa0c82bdf5ea361f4b3375413783af9167ac
-R 1813bb08dedce0701d741a99a479782e
+P c13415c5caf06eaa73086c500907451dfcdfd1354ac4a9ab63edb7255edbfa1b
+R 08ba8efa22e9576ef9e933cfe51a8274
 U drh
-Z e662d2fceaa8f576c7d1d1fef9cf3ce7
+Z 0db18b9089efefc8f8c5c54f23707a0d
diff --git a/manifest.uuid b/manifest.uuid
index 348cdf03c2..ec938d37c0 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c13415c5caf06eaa73086c500907451dfcdfd1354ac4a9ab63edb7255edbfa1b
\ No newline at end of file
+67c4a8c6881e33b830aa27c80e7e3d697a4222939edd77cd5ca77ece16471ea4
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 71f9f19cb2..21ad0aea7e 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -894,6 +894,7 @@ INCLUDE test_windirent.c
 INCLUDE ../ext/misc/shathree.c
 INCLUDE ../ext/misc/fileio.c
 INCLUDE ../ext/misc/completion.c
+INCLUDE ../ext/misc/appendvfs.c
 #ifdef SQLITE_HAVE_ZLIB
 INCLUDE ../ext/misc/zipfile.c
 INCLUDE ../ext/misc/sqlar.c
@@ -7952,6 +7953,12 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
         utf8_printf(stderr, "no such VFS: \"%s\"\n", argv[i]);
         exit(1);
       }
+#ifdef SQLITE_HAVE_ZIP
+    }else if( strcmp(z,"-zip")==0 ){
+      data.openMode = SHELL_OPEN_ZIPFILE;
+#endif
+    }else if( strcmp(z,"-append")==0 ){
+      data.openMode = SHELL_OPEN_APPENDVFS;
     }
   }
   if( data.zDbFilename==0 ){
@@ -7964,6 +7971,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
 #endif
   }
   data.out = stdout;
+  sqlite3_appendvfs_init(0,0,0);
 
   /* Go ahead and open the database file if it already exists.  If the
   ** file does not exist, delay opening it.  This prevents empty database
diff --git a/tool/sqltclsh.c.in b/tool/sqltclsh.c.in
index bdfac7dd00..2be05e136e 100644
--- a/tool/sqltclsh.c.in
+++ b/tool/sqltclsh.c.in
@@ -29,11 +29,16 @@
 #define SQLITE_MAX_EXPR_DEPTH 0
 INCLUDE sqlite3.c
 INCLUDE $ROOT/ext/misc/appendvfs.c
+INCLUDE $ROOT/ext/misc/zipfile.c
+INCLUDE $ROOT/ext/misc/sqlar.c
 INCLUDE $ROOT/src/tclsqlite.c
 
 const char *sqlite3_tclapp_init_proc(Tcl_Interp *interp){
   (void)interp;
   sqlite3_appendvfs_init(0,0,0);
+  sqlite3_auto_extension((void(*)(void))sqlite3_sqlar_init);
+  sqlite3_auto_extension((void(*)(void))sqlite3_zipfile_init);
+
   return
 BEGIN_STRING
 INCLUDE $ROOT/tool/sqltclsh.tcl
diff --git a/tool/sqltclsh.tcl b/tool/sqltclsh.tcl
index 61d00f1df2..6a4b1fe1f0 100644
--- a/tool/sqltclsh.tcl
+++ b/tool/sqltclsh.tcl
@@ -6,7 +6,9 @@ catch {
     append argv0 .exe
   }
   sqlite3 db $argv0 -vfs apndvfs -create 0
-  set mainscript [db one {SELECT data FROM scripts WHERE name='main.tcl'}]
+  set mainscript [db one {
+      SELECT sqlar_uncompress(data,sz) FROM sqlar WHERE name='main.tcl'
+  }]
 }
 if {[info exists mainscript]} {
   eval $mainscript

From 03491a1a1a29aa560e96b8f524a21e802a44db17 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sun, 7 Jan 2018 21:58:17 +0000
Subject: [PATCH 253/488] Remove the dependency on test_windirent.h from the
 generated shell.c file.

FossilOrigin-Name: 0a50c9e3bb0dbdaaec819ac6453276ba287b475ea322918ddda1ab3a1ec4b58b
---
 manifest             | 18 +++++++++---------
 manifest.uuid        |  2 +-
 src/shell.c.in       |  1 +
 src/test_windirent.c |  1 -
 src/test_windirent.h |  3 ++-
 tool/mkshellc.tcl    |  3 +++
 6 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/manifest b/manifest
index f28f322507..68cc1c4a7c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Modify\sthe\ssqltclsh\sstartup\sscript\sto\slook\sfor\sa\sproperly\sformatted\nSQLAR\sat\sthe\send\sof\sthe\sexecutable.\s\sFix\sthe\sCLI\sso\sthat\sit\sautomatically\nlinks\sagainst\sappendvfs\sand\sso\sthat\sthe\s--append\scommand-line\soption\sworks.
-D 2018-01-07T20:38:10.880
+C Remove\sthe\sdependency\son\stest_windirent.h\sfrom\sthe\sgenerated\sshell.c\sfile.
+D 2018-01-07T21:58:17.629
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -484,7 +484,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in ffcef0e1b2497b32c13f00ab9a1fececeeeb678dd04f5b17c0b5fa0cc99cd105
+F src/shell.c.in 0e20ac49a6c4458620084bf7715ad39cd9557d9116e447548d2282000e8ca361
 F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -541,8 +541,8 @@ F src/test_tclvar.c 33ff42149494a39c5fbb0df3d25d6fafb2f668888e41c0688d07273dcb26
 F src/test_thread.c 911d15fb14e19c0c542bdc8aabf981c2f10a4858
 F src/test_vfs.c f0186261a24de2671d080bcd8050732f0cb64f6e
 F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698
-F src/test_windirent.c 17f91f5f2aa1bb7328abb49414c363b5d2a9d3ff
-F src/test_windirent.h 5afeb2b1e2920d5149573a4c30a69cba91f2e8a80e00941b738036fca39aca61
+F src/test_windirent.c a895e2c068a06644eef91a7f0a32182445a893b9a0f33d0cdb4283dca2486ac1
+F src/test_windirent.h 8782864172ba5ae52c5c313c70faeadb324ff74de9c3dcc6b56a557dccaa1de6
 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
 F src/tokenize.c 1003d6d90c6783206c711f0a9397656fa5b055209f4d092caa43bb3bf5215db5
@@ -1631,7 +1631,7 @@ F tool/mkopcodec.tcl d1b6362bd3aa80d5520d4d6f3765badf01f6c43c
 F tool/mkopcodeh.tcl 4ee2a30ccbd900dc4d5cdb61bdab87cd2166cd2affcc78c9cc0b8d22a65b2eee
 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e
 F tool/mkpragmatab.tcl 2144bc8550a6471a029db262a132d2df4b9e0db61b90398bf64f5b7b3f8d92cd
-F tool/mkshellc.tcl 8cb90170e4aed5bd26f1664a83727c5d451f86948d3b759be26c1d7879fa346f
+F tool/mkshellc.tcl 1f45770aea226ac093a9c72f718efbb88a2a2833409ec2e1c4cecae4202626f5
 F tool/mksourceid.c d458f9004c837bee87a6382228ac20d3eae3c49ea3b0a5aace936f8b60748d3b
 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97
 F tool/mksqlite3c-noext.tcl fef88397668ae83166735c41af99d79f56afaabb
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c13415c5caf06eaa73086c500907451dfcdfd1354ac4a9ab63edb7255edbfa1b
-R 08ba8efa22e9576ef9e933cfe51a8274
+P 67c4a8c6881e33b830aa27c80e7e3d697a4222939edd77cd5ca77ece16471ea4
+R 197244392d7f5491887179479c1074da
 U drh
-Z 0db18b9089efefc8f8c5c54f23707a0d
+Z 398e4f42db7ae412d7f484f52a78c5ae
diff --git a/manifest.uuid b/manifest.uuid
index ec938d37c0..cd48fce8f8 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-67c4a8c6881e33b830aa27c80e7e3d697a4222939edd77cd5ca77ece16471ea4
\ No newline at end of file
+0a50c9e3bb0dbdaaec819ac6453276ba287b475ea322918ddda1ab3a1ec4b58b
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 21ad0aea7e..fe4eb624a1 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -888,6 +888,7 @@ static void shellAddSchemaName(
 #define SQLITE_EXTENSION_INIT2(X) (void)(X)
 
 #if defined(_WIN32) && defined(_MSC_VER)
+INCLUDE test_windirent.h
 INCLUDE test_windirent.c
 #define dirent DIRENT
 #endif
diff --git a/src/test_windirent.c b/src/test_windirent.c
index ca78d345d9..62165c4bea 100644
--- a/src/test_windirent.c
+++ b/src/test_windirent.c
@@ -14,7 +14,6 @@
 */
 
 #if defined(_WIN32) && defined(_MSC_VER)
-
 #include "test_windirent.h"
 
 /*
diff --git a/src/test_windirent.h b/src/test_windirent.h
index d4474f80e0..d71b49f684 100644
--- a/src/test_windirent.h
+++ b/src/test_windirent.h
@@ -13,7 +13,8 @@
 ** POSIX functions on Win32 using the MSVCRT.
 */
 
-#if defined(_WIN32) && defined(_MSC_VER)
+#if defined(_WIN32) && defined(_MSC_VER) && !defined(SQLITE_WINDIRENT_H)
+#define SQLITE_WINDIRENT_H
 
 /*
 ** We need several data types from the Windows SDK header.
diff --git a/tool/mkshellc.tcl b/tool/mkshellc.tcl
index 1a98511ef9..534ac6156a 100644
--- a/tool/mkshellc.tcl
+++ b/tool/mkshellc.tcl
@@ -50,6 +50,9 @@ while {1} {
     while {![eof $in2]} {
       set lx [omit_redundant_typedefs [gets $in2]]
       if {[regexp {^#include "sqlite} $lx]} continue
+      if {[regexp {^# *include "test_windirent.h"} $lx]} {
+        set lx "/* $lx */"
+      }
       set lx [string map [list __declspec(dllexport) {}] $lx]
       puts $out $lx
     }

From a5da4ef4ad24b2353e6dde41eb2b357bd9129ed8 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sun, 7 Jan 2018 23:28:10 +0000
Subject: [PATCH 254/488] Avoid the use of utimensat() on older unix platforms.

FossilOrigin-Name: 90cb01d8d6ac12d0b88f2952a75aeefa81ba66f5e4a5377fdd8b9f86aec8e927
---
 ext/misc/fileio.c | 29 ++++++++++++++++++++---------
 manifest          | 12 ++++++------
 manifest.uuid     |  2 +-
 3 files changed, 27 insertions(+), 16 deletions(-)

diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c
index 030b417e40..bb87f9524d 100644
--- a/ext/misc/fileio.c
+++ b/ext/misc/fileio.c
@@ -261,15 +261,8 @@ static int writeFile(
   }
 
   if( mtime>=0 ){
-#if !defined(_WIN32) && !defined(WIN32)
-    struct timespec times[2];
-    times[0].tv_nsec = times[1].tv_nsec = 0;
-    times[0].tv_sec = time(0);
-    times[1].tv_sec = mtime;
-    if( utimensat(AT_FDCWD, zFile, times, AT_SYMLINK_NOFOLLOW) ){
-      return 1;
-    }
-#else
+#if defined(_WIN32)
+    /* Windows */
     FILETIME lastAccess;
     FILETIME lastWrite;
     SYSTEMTIME currentTime;
@@ -291,6 +284,24 @@ static int writeFile(
     }else{
       return 1;
     }
+#elif defined(AT_FDCWD)
+    /* Recent unix */
+    struct timespec times[2];
+    times[0].tv_nsec = times[1].tv_nsec = 0;
+    times[0].tv_sec = time(0);
+    times[1].tv_sec = mtime;
+    if( utimensat(AT_FDCWD, zFile, times, AT_SYMLINK_NOFOLLOW) ){
+      return 1;
+    }
+#else
+    /* Legacy unix */
+    struct timeval times[2];
+    times[0].tv_usec = times[1].tv_usec = 0;
+    times[0].tv_sec = time(0);
+    times[1].tv_sec = mtime;
+    if( utimes(zFile, times) ){
+      return 1;
+    }
 #endif
   }
 
diff --git a/manifest b/manifest
index 68cc1c4a7c..15727841f8 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\sthe\sdependency\son\stest_windirent.h\sfrom\sthe\sgenerated\sshell.c\sfile.
-D 2018-01-07T21:58:17.629
+C Avoid\sthe\suse\sof\sutimensat()\son\solder\sunix\splatforms.
+D 2018-01-07T23:28:10.983
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -277,7 +277,7 @@ F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f0
 F ext/misc/csv.c 1a009b93650732e22334edc92459c4630b9fa703397cbb3c8ca279921a36ca11
 F ext/misc/dbdump.c 3509fa6b8932d04e932d6b6b827b6a82ca362781b8e8f3c77336f416793e215e
 F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2
-F ext/misc/fileio.c f16e2ef87928b768883b259b6c53a67076e98e8daa0bbb624910406d59630b26
+F ext/misc/fileio.c 672e89e125846848e07e025cbe601dc370a93433d571d9422541a9cefbd12b76
 F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25
 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c
 F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281984
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 67c4a8c6881e33b830aa27c80e7e3d697a4222939edd77cd5ca77ece16471ea4
-R 197244392d7f5491887179479c1074da
+P 0a50c9e3bb0dbdaaec819ac6453276ba287b475ea322918ddda1ab3a1ec4b58b
+R 1b4834f0a49c166cef0f6faa748094c7
 U drh
-Z 398e4f42db7ae412d7f484f52a78c5ae
+Z 0323f752d28821f22e44c20f1850c6dd
diff --git a/manifest.uuid b/manifest.uuid
index cd48fce8f8..253d7c078e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0a50c9e3bb0dbdaaec819ac6453276ba287b475ea322918ddda1ab3a1ec4b58b
\ No newline at end of file
+90cb01d8d6ac12d0b88f2952a75aeefa81ba66f5e4a5377fdd8b9f86aec8e927
\ No newline at end of file

From 98dfb2fc0843ca480d6753c91ceb3dd6dd7b66af Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Mon, 8 Jan 2018 14:24:08 +0000
Subject: [PATCH 255/488] Fix compilation of 'sqltclsh.exe' for MSVC.

FossilOrigin-Name: abb112d9e1a51c1c2f15d1e5f3dc161c9a947841f69a56715f5f1c62603096b2
---
 Makefile.msc       |  2 +-
 manifest           | 16 ++++++++--------
 manifest.uuid      |  2 +-
 tool/sqltclsh.c.in |  4 ++++
 4 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/Makefile.msc b/Makefile.msc
index 4d6c5c0cc2..6d4538706c 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -2339,7 +2339,7 @@ sqlite3_analyzer.exe:	sqlite3_analyzer.c $(LIBRESOBJS)
 sqltclsh.c: sqlite3.c $(TOP)\src\tclsqlite.c $(TOP)\tool\sqltclsh.tcl $(TOP)\ext\misc\appendvfs.c $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqltclsh.c.in
 	$(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqltclsh.c.in >sqltclsh.c
 
-sqltclsh.exe: sqltclsh.c
+sqltclsh.exe: sqltclsh.c  $(SHELL_CORE_DEP) $(LIBRESOBJS)
 	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqltclsh.c \
 		/link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
 
diff --git a/manifest b/manifest
index 15727841f8..43e25b3a81 100644
--- a/manifest
+++ b/manifest
@@ -1,10 +1,10 @@
-C Avoid\sthe\suse\sof\sutimensat()\son\solder\sunix\splatforms.
-D 2018-01-07T23:28:10.983
+C Fix\scompilation\sof\s'sqltclsh.exe'\sfor\sMSVC.
+D 2018-01-08T14:24:08.629
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc b33f630258ce72fb3150c1a6819a484eba3950dc9397571d48672087e12abf4a
+F Makefile.msc 085d3b65cebdebec89dc8c91901c06f18e357eb320a0434bfa67a53e917f10de
 F README.md eeae1e552f93ef72ef7c5b8f6647b368a001c28820ad1df179d3dae602bef681
 F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -1664,7 +1664,7 @@ F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd
 F tool/split-sqlite3c.tcl 3efcd4240b738f6bb2b5af0aea7e1e0ef9bc1c61654f645076cec883030b710c
 F tool/sqldiff.c 30879bbc8de686df4624e86adce2d8981f500904c1cfb55b5d1eea2ffd9341eb
 F tool/sqlite3_analyzer.c.in 7eeaae8b0d7577662acaabbb11107af0659d1b41bc1dfdd4d91422de27127968
-F tool/sqltclsh.c.in 8b2529b6c3cdd8ad6aaff21e80eb58370c428cb207b4607d0da4def064dcec56
+F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898
 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848
 F tool/srcck1.c 371de5363b70154012955544f86fdee8f6e5326f
 F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0a50c9e3bb0dbdaaec819ac6453276ba287b475ea322918ddda1ab3a1ec4b58b
-R 1b4834f0a49c166cef0f6faa748094c7
-U drh
-Z 0323f752d28821f22e44c20f1850c6dd
+P 90cb01d8d6ac12d0b88f2952a75aeefa81ba66f5e4a5377fdd8b9f86aec8e927
+R 76adb99cb262a63e3f8964691f33ecc1
+U mistachkin
+Z 00d4b6b04e6e70aa614f01ae7c47411b
diff --git a/manifest.uuid b/manifest.uuid
index 253d7c078e..d2412dd0a2 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-90cb01d8d6ac12d0b88f2952a75aeefa81ba66f5e4a5377fdd8b9f86aec8e927
\ No newline at end of file
+abb112d9e1a51c1c2f15d1e5f3dc161c9a947841f69a56715f5f1c62603096b2
\ No newline at end of file
diff --git a/tool/sqltclsh.c.in b/tool/sqltclsh.c.in
index 2be05e136e..da354ee935 100644
--- a/tool/sqltclsh.c.in
+++ b/tool/sqltclsh.c.in
@@ -29,15 +29,19 @@
 #define SQLITE_MAX_EXPR_DEPTH 0
 INCLUDE sqlite3.c
 INCLUDE $ROOT/ext/misc/appendvfs.c
+#ifdef SQLITE_HAVE_ZLIB
 INCLUDE $ROOT/ext/misc/zipfile.c
 INCLUDE $ROOT/ext/misc/sqlar.c
+#endif
 INCLUDE $ROOT/src/tclsqlite.c
 
 const char *sqlite3_tclapp_init_proc(Tcl_Interp *interp){
   (void)interp;
   sqlite3_appendvfs_init(0,0,0);
+#ifdef SQLITE_HAVE_ZLIB
   sqlite3_auto_extension((void(*)(void))sqlite3_sqlar_init);
   sqlite3_auto_extension((void(*)(void))sqlite3_zipfile_init);
+#endif
 
   return
 BEGIN_STRING

From a41f2855e9310602d0680fe6056c3653504705a4 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 8 Jan 2018 16:54:26 +0000
Subject: [PATCH 256/488] First code for an auxiliary function that will
 normalize an SQL statement.

FossilOrigin-Name: 84814aac81e54f03430f180926156ab0fc01e22bbce2bb228b698ea0d4a588ea
---
 ext/misc/normalize.c | 639 +++++++++++++++++++++++++++++++++++++++++++
 manifest             |  14 +-
 manifest.uuid        |   2 +-
 3 files changed, 649 insertions(+), 6 deletions(-)
 create mode 100644 ext/misc/normalize.c

diff --git a/ext/misc/normalize.c b/ext/misc/normalize.c
new file mode 100644
index 0000000000..914a154d9a
--- /dev/null
+++ b/ext/misc/normalize.c
@@ -0,0 +1,639 @@
+/*
+** 2018-01-08
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file contains code to implement the sqlite3_normalize() function.
+**
+**    char *sqlite3_normalize(const char *zSql);
+**
+** This function takes an SQL string as input and returns a "normalized"
+** version of that string in memory obtained from sqlite3_malloc64().  The
+** caller is responsible for ensuring that the returned memory is freed.
+**
+** If a memory allocation error occurs, this routine returns NULL.
+**
+** The normalization consists of the following transformations:
+**
+**   (1)  Convert every literal (string, blob literal, numeric constant,
+**        or "NULL" constant) into a ?
+**
+**   (2)  Remove all superfluous whitespace, including comments.  Change
+**        all required whitespace to a single space character.
+**
+**   (3)  Lowercase all ASCII characters.
+**
+**   (4)  If an IN or NOT IN operator is followed by a list of 1 or more
+**        values, convert that list into "(?,?,?)".
+**
+** The purpose of normalization is two-fold:
+**
+**   (1)  Sanitize queries by removing possibly sensitive information contained
+**        in literals.
+**
+**   (2)  Identify structurally identical queries by comparing their
+**        normalized forms.
+*/
+#include <sqlite3.h>
+#include <string.h>
+
+/*
+** Implementation note:
+**
+** Much of the tokenizer logic is copied out of the tokenize.c source file
+** of SQLite.  This logic could be simplified for this particular application,
+** but that would impose a risk of introducing subtle errors.  It is best to
+** keep the code as close to the original as possible.
+*/
+
+
+/* Character classes for tokenizing
+**
+** In the sqlite3GetToken() function, a switch() on aiClass[c] is implemented
+** using a lookup table, whereas a switch() directly on c uses a binary search.
+** The lookup table is much faster.  To maximize speed, and to ensure that
+** a lookup table is used, all of the classes need to be small integers and
+** all of them need to be used within the switch.
+*/
+#define CC_X          0    /* The letter 'x', or start of BLOB literal */
+#define CC_KYWD       1    /* Alphabetics or '_'.  Usable in a keyword */
+#define CC_ID         2    /* unicode characters usable in IDs */
+#define CC_DIGIT      3    /* Digits */
+#define CC_DOLLAR     4    /* '$' */
+#define CC_VARALPHA   5    /* '@', '#', ':'.  Alphabetic SQL variables */
+#define CC_VARNUM     6    /* '?'.  Numeric SQL variables */
+#define CC_SPACE      7    /* Space characters */
+#define CC_QUOTE      8    /* '"', '\'', or '`'.  String literals, quoted ids */
+#define CC_QUOTE2     9    /* '['.   [...] style quoted ids */
+#define CC_PIPE      10    /* '|'.   Bitwise OR or concatenate */
+#define CC_MINUS     11    /* '-'.  Minus or SQL-style comment */
+#define CC_LT        12    /* '<'.  Part of < or <= or <> */
+#define CC_GT        13    /* '>'.  Part of > or >= */
+#define CC_EQ        14    /* '='.  Part of = or == */
+#define CC_BANG      15    /* '!'.  Part of != */
+#define CC_SLASH     16    /* '/'.  / or c-style comment */
+#define CC_LP        17    /* '(' */
+#define CC_RP        18    /* ')' */
+#define CC_SEMI      19    /* ';' */
+#define CC_PLUS      20    /* '+' */
+#define CC_STAR      21    /* '*' */
+#define CC_PERCENT   22    /* '%' */
+#define CC_COMMA     23    /* ',' */
+#define CC_AND       24    /* '&' */
+#define CC_TILDA     25    /* '~' */
+#define CC_DOT       26    /* '.' */
+#define CC_ILLEGAL   27    /* Illegal character */
+
+static const unsigned char aiClass[] = {
+/*         x0  x1  x2  x3  x4  x5  x6  x7  x8  x9  xa  xb  xc  xd  xe  xf */
+/* 0x */   27, 27, 27, 27, 27, 27, 27, 27, 27,  7,  7, 27,  7,  7, 27, 27,
+/* 1x */   27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+/* 2x */    7, 15,  8,  5,  4, 22, 24,  8, 17, 18, 21, 20, 23, 11, 26, 16,
+/* 3x */    3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  5, 19, 12, 14, 13,  6,
+/* 4x */    5,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,
+/* 5x */    1,  1,  1,  1,  1,  1,  1,  1,  0,  1,  1,  9, 27, 27, 27,  1,
+/* 6x */    8,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,
+/* 7x */    1,  1,  1,  1,  1,  1,  1,  1,  0,  1,  1, 27, 10, 27, 25, 27,
+/* 8x */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
+/* 9x */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
+/* Ax */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
+/* Bx */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
+/* Cx */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
+/* Dx */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
+/* Ex */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
+/* Fx */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2
+};
+
+/* An array to map all upper-case characters into their corresponding
+** lower-case character. 
+**
+** SQLite only considers US-ASCII (or EBCDIC) characters.  We do not
+** handle case conversions for the UTF character set since the tables
+** involved are nearly as big or bigger than SQLite itself.
+*/
+const unsigned char sqlite3UpperToLower[] = {
+      0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 16, 17,
+     18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,
+     36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53,
+     54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 97, 98, 99,100,101,102,103,
+    104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,
+    122, 91, 92, 93, 94, 95, 96, 97, 98, 99,100,101,102,103,104,105,106,107,
+    108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,
+    126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,
+    144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,
+    162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,
+    180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,
+    198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,
+    216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,
+    234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,
+    252,253,254,255
+};
+
+/*
+** The following 256 byte lookup table is used to support SQLites built-in
+** equivalents to the following standard library functions:
+**
+**   isspace()                        0x01
+**   isalpha()                        0x02
+**   isdigit()                        0x04
+**   isalnum()                        0x06
+**   isxdigit()                       0x08
+**   toupper()                        0x20
+**   SQLite identifier character      0x40
+**   Quote character                  0x80
+**
+** Bit 0x20 is set if the mapped character requires translation to upper
+** case. i.e. if the character is a lower-case ASCII character.
+** If x is a lower-case ASCII character, then its upper-case equivalent
+** is (x - 0x20). Therefore toupper() can be implemented as:
+**
+**   (x & ~(map[x]&0x20))
+**
+** The equivalent of tolower() is implemented using the sqlite3UpperToLower[]
+** array. tolower() is used more often than toupper() by SQLite.
+**
+** Bit 0x40 is set if the character is non-alphanumeric and can be used in an 
+** SQLite identifier.  Identifiers are alphanumerics, "_", "$", and any
+** non-ASCII UTF character. Hence the test for whether or not a character is
+** part of an identifier is 0x46.
+*/
+const unsigned char sqlite3CtypeMap[256] = {
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,  /* 00..07    ........ */
+  0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00,  /* 08..0f    ........ */
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,  /* 10..17    ........ */
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,  /* 18..1f    ........ */
+  0x01, 0x00, 0x80, 0x00, 0x40, 0x00, 0x00, 0x80,  /* 20..27     !"#$%&' */
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,  /* 28..2f    ()*+,-./ */
+  0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c,  /* 30..37    01234567 */
+  0x0c, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,  /* 38..3f    89:;<=>? */
+
+  0x00, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x02,  /* 40..47    @ABCDEFG */
+  0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,  /* 48..4f    HIJKLMNO */
+  0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,  /* 50..57    PQRSTUVW */
+  0x02, 0x02, 0x02, 0x80, 0x00, 0x00, 0x00, 0x40,  /* 58..5f    XYZ[\]^_ */
+  0x80, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x22,  /* 60..67    `abcdefg */
+  0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22,  /* 68..6f    hijklmno */
+  0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22,  /* 70..77    pqrstuvw */
+  0x22, 0x22, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00,  /* 78..7f    xyz{|}~. */
+
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* 80..87    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* 88..8f    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* 90..97    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* 98..9f    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* a0..a7    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* a8..af    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* b0..b7    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* b8..bf    ........ */
+
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* c0..c7    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* c8..cf    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* d0..d7    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* d8..df    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* e0..e7    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* e8..ef    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* f0..f7    ........ */
+  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40   /* f8..ff    ........ */
+};
+#define sqlite3Toupper(x)   ((x)&~(sqlite3CtypeMap[(unsigned char)(x)]&0x20))
+#define sqlite3Isspace(x)   (sqlite3CtypeMap[(unsigned char)(x)]&0x01)
+#define sqlite3Isalnum(x)   (sqlite3CtypeMap[(unsigned char)(x)]&0x06)
+#define sqlite3Isalpha(x)   (sqlite3CtypeMap[(unsigned char)(x)]&0x02)
+#define sqlite3Isdigit(x)   (sqlite3CtypeMap[(unsigned char)(x)]&0x04)
+#define sqlite3Isxdigit(x)  (sqlite3CtypeMap[(unsigned char)(x)]&0x08)
+#define sqlite3Tolower(x)   (sqlite3UpperToLower[(unsigned char)(x)])
+#define sqlite3Isquote(x)   (sqlite3CtypeMap[(unsigned char)(x)]&0x80)
+
+
+/*
+** If X is a character that can be used in an identifier then
+** IdChar(X) will be true.  Otherwise it is false.
+**
+** For ASCII, any character with the high-order bit set is
+** allowed in an identifier.  For 7-bit characters, 
+** sqlite3IsIdChar[X] must be 1.
+**
+** For EBCDIC, the rules are more complex but have the same
+** end result.
+**
+** Ticket #1066.  the SQL standard does not allow '$' in the
+** middle of identifiers.  But many SQL implementations do. 
+** SQLite will allow '$' in identifiers for compatibility.
+** But the feature is undocumented.
+*/
+#define IdChar(C)  ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0)
+
+/*
+** Ignore testcase() macros
+*/
+#define testcase(X)
+
+/*
+** Token values
+*/
+#define TK_SPACE    0
+#define TK_NAME     1
+#define TK_LITERAL  2
+#define TK_PUNCT    3
+#define TK_ERROR    4
+
+#define TK_MINUS    TK_PUNCT
+#define TK_LP       TK_PUNCT
+#define TK_RP       TK_PUNCT
+#define TK_SEMI     TK_PUNCT
+#define TK_PLUS     TK_PUNCT
+#define TK_STAR     TK_PUNCT
+#define TK_SLASH    TK_PUNCT
+#define TK_REM      TK_PUNCT
+#define TK_EQ       TK_PUNCT
+#define TK_LE       TK_PUNCT
+#define TK_NE       TK_PUNCT
+#define TK_LSHIFT   TK_PUNCT
+#define TK_LT       TK_PUNCT
+#define TK_GE       TK_PUNCT
+#define TK_RSHIFT   TK_PUNCT
+#define TK_GT       TK_PUNCT
+#define TK_GE       TK_PUNCT
+#define TK_BITOR    TK_PUNCT
+#define TK_CONCAT   TK_PUNCT
+#define TK_COMMA    TK_PUNCT
+#define TK_BITAND   TK_PUNCT
+#define TK_BITNOT   TK_PUNCT
+#define TK_STRING   TK_LITERAL
+#define TK_ID       TK_NAME
+#define TK_ILLEGAL  TK_ERROR
+#define TK_DOT      TK_PUNCT
+#define TK_INTEGER  TK_LITERAL
+#define TK_FLOAT    TK_LITERAL
+#define TK_VARIABLE TK_LITERAL
+#define TK_BLOB     TK_LITERAL
+
+/*
+** Return the length (in bytes) of the token that begins at z[0]. 
+** Store the token type in *tokenType before returning.
+*/
+static int sqlite3GetToken(const unsigned char *z, int *tokenType){
+  int i, c;
+  switch( aiClass[*z] ){  /* Switch on the character-class of the first byte
+                          ** of the token. See the comment on the CC_ defines
+                          ** above. */
+    case CC_SPACE: {
+      for(i=1; sqlite3Isspace(z[i]); i++){}
+      *tokenType = TK_SPACE;
+      return i;
+    }
+    case CC_MINUS: {
+      if( z[1]=='-' ){
+        for(i=2; (c=z[i])!=0 && c!='\n'; i++){}
+        *tokenType = TK_SPACE;
+        return i;
+      }
+      *tokenType = TK_MINUS;
+      return 1;
+    }
+    case CC_LP: {
+      *tokenType = TK_LP;
+      return 1;
+    }
+    case CC_RP: {
+      *tokenType = TK_RP;
+      return 1;
+    }
+    case CC_SEMI: {
+      *tokenType = TK_SEMI;
+      return 1;
+    }
+    case CC_PLUS: {
+      *tokenType = TK_PLUS;
+      return 1;
+    }
+    case CC_STAR: {
+      *tokenType = TK_STAR;
+      return 1;
+    }
+    case CC_SLASH: {
+      if( z[1]!='*' || z[2]==0 ){
+        *tokenType = TK_SLASH;
+        return 1;
+      }
+      for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){}
+      if( c ) i++;
+      *tokenType = TK_SPACE;
+      return i;
+    }
+    case CC_PERCENT: {
+      *tokenType = TK_REM;
+      return 1;
+    }
+    case CC_EQ: {
+      *tokenType = TK_EQ;
+      return 1 + (z[1]=='=');
+    }
+    case CC_LT: {
+      if( (c=z[1])=='=' ){
+        *tokenType = TK_LE;
+        return 2;
+      }else if( c=='>' ){
+        *tokenType = TK_NE;
+        return 2;
+      }else if( c=='<' ){
+        *tokenType = TK_LSHIFT;
+        return 2;
+      }else{
+        *tokenType = TK_LT;
+        return 1;
+      }
+    }
+    case CC_GT: {
+      if( (c=z[1])=='=' ){
+        *tokenType = TK_GE;
+        return 2;
+      }else if( c=='>' ){
+        *tokenType = TK_RSHIFT;
+        return 2;
+      }else{
+        *tokenType = TK_GT;
+        return 1;
+      }
+    }
+    case CC_BANG: {
+      if( z[1]!='=' ){
+        *tokenType = TK_ILLEGAL;
+        return 1;
+      }else{
+        *tokenType = TK_NE;
+        return 2;
+      }
+    }
+    case CC_PIPE: {
+      if( z[1]!='|' ){
+        *tokenType = TK_BITOR;
+        return 1;
+      }else{
+        *tokenType = TK_CONCAT;
+        return 2;
+      }
+    }
+    case CC_COMMA: {
+      *tokenType = TK_COMMA;
+      return 1;
+    }
+    case CC_AND: {
+      *tokenType = TK_BITAND;
+      return 1;
+    }
+    case CC_TILDA: {
+      *tokenType = TK_BITNOT;
+      return 1;
+    }
+    case CC_QUOTE: {
+      int delim = z[0];
+      testcase( delim=='`' );
+      testcase( delim=='\'' );
+      testcase( delim=='"' );
+      for(i=1; (c=z[i])!=0; i++){
+        if( c==delim ){
+          if( z[i+1]==delim ){
+            i++;
+          }else{
+            break;
+          }
+        }
+      }
+      if( c=='\'' ){
+        *tokenType = TK_STRING;
+        return i+1;
+      }else if( c!=0 ){
+        *tokenType = TK_ID;
+        return i+1;
+      }else{
+        *tokenType = TK_ILLEGAL;
+        return i;
+      }
+    }
+    case CC_DOT: {
+      if( !sqlite3Isdigit(z[1]) ){
+        *tokenType = TK_DOT;
+        return 1;
+      }
+      /* If the next character is a digit, this is a floating point
+      ** number that begins with ".".  Fall thru into the next case */
+    }
+    case CC_DIGIT: {
+      *tokenType = TK_INTEGER;
+      if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){
+        for(i=3; sqlite3Isxdigit(z[i]); i++){}
+        return i;
+      }
+      for(i=0; sqlite3Isdigit(z[i]); i++){}
+      if( z[i]=='.' ){
+        i++;
+        while( sqlite3Isdigit(z[i]) ){ i++; }
+        *tokenType = TK_FLOAT;
+      }
+      if( (z[i]=='e' || z[i]=='E') &&
+           ( sqlite3Isdigit(z[i+1]) 
+            || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2]))
+           )
+      ){
+        i += 2;
+        while( sqlite3Isdigit(z[i]) ){ i++; }
+        *tokenType = TK_FLOAT;
+      }
+      while( IdChar(z[i]) ){
+        *tokenType = TK_ILLEGAL;
+        i++;
+      }
+      return i;
+    }
+    case CC_QUOTE2: {
+      for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){}
+      *tokenType = c==']' ? TK_ID : TK_ILLEGAL;
+      return i;
+    }
+    case CC_VARNUM: {
+      *tokenType = TK_VARIABLE;
+      for(i=1; sqlite3Isdigit(z[i]); i++){}
+      return i;
+    }
+    case CC_DOLLAR:
+    case CC_VARALPHA: {
+      int n = 0;
+      testcase( z[0]=='$' );  testcase( z[0]=='@' );
+      testcase( z[0]==':' );  testcase( z[0]=='#' );
+      *tokenType = TK_VARIABLE;
+      for(i=1; (c=z[i])!=0; i++){
+        if( IdChar(c) ){
+          n++;
+        }else if( c=='(' && n>0 ){
+          do{
+            i++;
+          }while( (c=z[i])!=0 && !sqlite3Isspace(c) && c!=')' );
+          if( c==')' ){
+            i++;
+          }else{
+            *tokenType = TK_ILLEGAL;
+          }
+          break;
+        }else if( c==':' && z[i+1]==':' ){
+          i++;
+        }else{
+          break;
+        }
+      }
+      if( n==0 ) *tokenType = TK_ILLEGAL;
+      return i;
+    }
+    case CC_KYWD: {
+      for(i=1; aiClass[z[i]]<=CC_KYWD; i++){}
+      if( IdChar(z[i]) ){
+        /* This token started out using characters that can appear in keywords,
+        ** but z[i] is a character not allowed within keywords, so this must
+        ** be an identifier instead */
+        i++;
+        break;
+      }
+      *tokenType = TK_ID;
+      return i;
+    }
+    case CC_X: {
+      testcase( z[0]=='x' ); testcase( z[0]=='X' );
+      if( z[1]=='\'' ){
+        *tokenType = TK_BLOB;
+        for(i=2; sqlite3Isxdigit(z[i]); i++){}
+        if( z[i]!='\'' || i%2 ){
+          *tokenType = TK_ILLEGAL;
+          while( z[i] && z[i]!='\'' ){ i++; }
+        }
+        if( z[i] ) i++;
+        return i;
+      }
+      /* If it is not a BLOB literal, then it must be an ID, since no
+      ** SQL keywords start with the letter 'x'.  Fall through */
+    }
+    case CC_ID: {
+      i = 1;
+      break;
+    }
+    default: {
+      *tokenType = TK_ILLEGAL;
+      return 1;
+    }
+  }
+  while( IdChar(z[i]) ){ i++; }
+  *tokenType = TK_ID;
+  return i;
+}
+
+char *sqlite3_normalize(const char *zSql){
+  char *z;              /* The output string */
+  sqlite3_int64 nZ;     /* Size of the output string in bytes */
+  sqlite3_int64 nSql;   /* Size of the input string in bytes */
+  int i;                /* Next character to read from zSql[] */
+  int j;                /* Next slot to fill in on z[] */
+  int tokenType;        /* Type of the next token */
+  int n;                /* Size of the next token */
+  int k;                /* Loop counter */
+
+  nSql = strlen(zSql);
+  nZ = nSql;
+  z = sqlite3_malloc64( nZ+2 );
+  if( z==0 ) return 0;
+  for(i=j=0; zSql[i]; i += n){
+    n = sqlite3GetToken((unsigned char*)zSql+i, &tokenType);
+    switch( tokenType ){
+      case TK_SPACE: {
+        break;
+      }
+      case TK_ERROR: {
+        sqlite3_free(z);
+        return 0;
+      }
+      case TK_LITERAL: {
+        z[j++] = '?';
+        break;
+      }
+      case TK_PUNCT:
+      case TK_NAME: {
+        if( j>0 && IdChar(z[j-1]) && IdChar(zSql[i]) ) z[j++] = ' ';
+        for(k=0; k<n; k++){
+          z[j++] = sqlite3Tolower(zSql[i+k]);
+        }
+        break;
+      }
+    }
+  }
+  while( j>0 && z[j-1]==' ' ){ j--; }
+  if( i>0 && z[j-1]!=';' ){ z[j++] = ';'; }
+  z[j] = 0;
+  return z;
+}
+
+#ifdef NORMALIZE_TEST
+#include <stdio.h>
+#include <stdlib.h>
+
+static void normalizeFile(char *zIn){
+  int i;
+  if( zIn==0 ) return;
+  for(i=0; zIn[i]; i++){
+    char cSaved;
+    if( zIn[i]!=';' ) continue;
+    cSaved = zIn[i+1];
+    zIn[i+1] = 0;
+    if( sqlite3_complete(zIn) ){
+      char *zOut = sqlite3_normalize(zIn);
+      if( zOut ){
+        printf("%s\n", zOut);
+        sqlite3_free(zOut);
+      }else{
+        fprintf(stderr, "ERROR: %s\n", zIn);
+      }
+      zIn[i+1] = cSaved;
+      zIn += i+1;
+      i = -1;
+    }else{
+      zIn[i+1] = cSaved;
+    }
+  }
+}
+
+int main(int argc, char **argv){
+  int i;
+  FILE *in;
+  char *zBuf = 0;
+  sqlite3_int64 sz, got;
+
+  for(i=1; i<argc; i++){
+    in = fopen(argv[i], "rb");
+    if( in==0 ){
+      fprintf(stderr, "cannot open \"%s\"\n", argv[i]);
+      continue;
+    }
+    fseek(in, 0, SEEK_END);
+    sz = ftell(in);
+    rewind(in);
+    zBuf = sqlite3_realloc64(zBuf, sz+1);
+    if( zBuf==0 ){
+      fprintf(stderr, "failed to malloc for %lld bytes\n", sz);
+      exit(1);
+    }
+    got = fread(zBuf, 1, sz, in);
+    fclose(in);
+    if( got!=sz ){
+      fprintf(stderr, "only able to read %lld of %lld bytes from \"%s\"\n",
+              got, sz, argv[i]);
+    }else{
+      zBuf[got] = 0;
+      normalizeFile(zBuf);
+    }
+  }
+  sqlite3_free(zBuf);
+}
+#endif /* NORMALIZE_TEST */
diff --git a/manifest b/manifest
index 15727841f8..7cd42ecfd6 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Avoid\sthe\suse\sof\sutimensat()\son\solder\sunix\splatforms.
-D 2018-01-07T23:28:10.983
+C First\scode\sfor\san\sauxiliary\sfunction\sthat\swill\snormalize\san\sSQL\sstatement.
+D 2018-01-08T16:54:26.258
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -284,6 +284,7 @@ F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281
 F ext/misc/memvfs.c ab36f49e02ebcdf85a1e08dc4d8599ea8f343e073ac9e0bca18a98b7e1ec9567
 F ext/misc/mmapwarm.c 70b618f2d0bde43fae288ad0b7498a629f2b6f61b50a27e06fae3cd23c83af29
 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342
+F ext/misc/normalize.c 5ca65b1332e41d679d7c07374398b49d020281d4ea30f267c6a6fb500e71f7c7
 F ext/misc/percentile.c 92699c8cd7d517ff610e6037e56506f8904dae2e
 F ext/misc/regexp.c a68d25c659bd2d893cd1215667bbf75ecb9dc7d4
 F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c
@@ -1697,7 +1698,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0a50c9e3bb0dbdaaec819ac6453276ba287b475ea322918ddda1ab3a1ec4b58b
-R 1b4834f0a49c166cef0f6faa748094c7
+P 90cb01d8d6ac12d0b88f2952a75aeefa81ba66f5e4a5377fdd8b9f86aec8e927
+R 9bd921419b4f2c3b5e55f041d45b29ce
+T *branch * normalize
+T *sym-normalize *
+T -sym-trunk *
 U drh
-Z 0323f752d28821f22e44c20f1850c6dd
+Z c8d4afe60d21d90fa2f1cb97ff587c09
diff --git a/manifest.uuid b/manifest.uuid
index 253d7c078e..68c796f48f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-90cb01d8d6ac12d0b88f2952a75aeefa81ba66f5e4a5377fdd8b9f86aec8e927
\ No newline at end of file
+84814aac81e54f03430f180926156ab0fc01e22bbce2bb228b698ea0d4a588ea
\ No newline at end of file

From dcfbff9a86ac17bae4578b944655945c516fd863 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 8 Jan 2018 17:05:32 +0000
Subject: [PATCH 257/488] Fix a malloc/sqlite3_free mismatch in the shell tool
 code.

FossilOrigin-Name: fe053b2ecdc04b918367fa503bf10292fd7cd84a6929670857b2e9153df4d1f9
---
 manifest       | 14 +++++++-------
 manifest.uuid  |  2 +-
 src/shell.c.in |  8 +++++---
 3 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index 43e25b3a81..1215056748 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\scompilation\sof\s'sqltclsh.exe'\sfor\sMSVC.
-D 2018-01-08T14:24:08.629
+C Fix\sa\smalloc/sqlite3_free\smismatch\sin\sthe\sshell\stool\scode.
+D 2018-01-08T17:05:32.727
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -484,7 +484,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in 0e20ac49a6c4458620084bf7715ad39cd9557d9116e447548d2282000e8ca361
+F src/shell.c.in be04d6797b4ee066f064dc370bdcb148c84c6c6979bf12f70025a7a89cf79346
 F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 90cb01d8d6ac12d0b88f2952a75aeefa81ba66f5e4a5377fdd8b9f86aec8e927
-R 76adb99cb262a63e3f8964691f33ecc1
-U mistachkin
-Z 00d4b6b04e6e70aa614f01ae7c47411b
+P abb112d9e1a51c1c2f15d1e5f3dc161c9a947841f69a56715f5f1c62603096b2
+R 97c2ea02cbe64fb40daf371bd5a56946
+U dan
+Z dc8bc5606c8c9f2e7de507a70d755d15
diff --git a/manifest.uuid b/manifest.uuid
index d2412dd0a2..095a867785 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-abb112d9e1a51c1c2f15d1e5f3dc161c9a947841f69a56715f5f1c62603096b2
\ No newline at end of file
+fe053b2ecdc04b918367fa503bf10292fd7cd84a6929670857b2e9153df4d1f9
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index fe4eb624a1..9758d5d180 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -802,8 +802,9 @@ static void shellModuleSchema(
   const char *zName = (const char*)sqlite3_value_text(apVal[0]);
   char *zFake = shellFakeSchema(sqlite3_context_db_handle(pCtx), 0, zName);
   if( zFake ){
-    sqlite3_result_text(pCtx, sqlite3_mprintf("/* %z */", zFake),
+    sqlite3_result_text(pCtx, sqlite3_mprintf("/* %s */", zFake),
                         -1, sqlite3_free);
+    free(zFake);
   }
 }
 
@@ -863,10 +864,11 @@ static void shellAddSchemaName(
          && (zFake = shellFakeSchema(db, zSchema, zName))!=0
         ){
           if( z==0 ){
-            z = sqlite3_mprintf("%s\n/* %z */", zIn, zFake);
+            z = sqlite3_mprintf("%s\n/* %s */", zIn, zFake);
           }else{
-            z = sqlite3_mprintf("%z\n/* %z */", z, zFake);
+            z = sqlite3_mprintf("%z\n/* %s */", z, zFake);
           }
+          free(zFake);
         }
         if( z ){
           sqlite3_result_text(pCtx, z, -1, sqlite3_free);

From 1bc1e2fb5d5d67e6d730a4d061c3dbac50824c92 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Mon, 8 Jan 2018 17:09:05 +0000
Subject: [PATCH 258/488] Remove a superfluous define from the 'fileio'
 extension when compiled for Win32.

FossilOrigin-Name: 6dcba6693f064b0e21360c5eab18abe4a1534f6699b5d7b92e60eab6be14c347
---
 ext/misc/fileio.c |  1 -
 manifest          | 14 +++++++-------
 manifest.uuid     |  2 +-
 3 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c
index bb87f9524d..31af50a0c0 100644
--- a/ext/misc/fileio.c
+++ b/ext/misc/fileio.c
@@ -92,7 +92,6 @@ SQLITE_EXTENSION_INIT1
 #  include <direct.h>
 #  include "test_windirent.h"
 #  define dirent DIRENT
-#  define timespec TIMESPEC
 #  define stat _stat
 #  define mkdir(path,mode) _mkdir(path)
 #  define lstat(path,buf) _stat(path,buf)
diff --git a/manifest b/manifest
index 1215056748..df8f0eb071 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\smalloc/sqlite3_free\smismatch\sin\sthe\sshell\stool\scode.
-D 2018-01-08T17:05:32.727
+C Remove\sa\ssuperfluous\sdefine\sfrom\sthe\s'fileio'\sextension\swhen\scompiled\sfor\sWin32.
+D 2018-01-08T17:09:05.559
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -277,7 +277,7 @@ F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f0
 F ext/misc/csv.c 1a009b93650732e22334edc92459c4630b9fa703397cbb3c8ca279921a36ca11
 F ext/misc/dbdump.c 3509fa6b8932d04e932d6b6b827b6a82ca362781b8e8f3c77336f416793e215e
 F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2
-F ext/misc/fileio.c 672e89e125846848e07e025cbe601dc370a93433d571d9422541a9cefbd12b76
+F ext/misc/fileio.c edf880df28f606f0b00ac872a6a957b544fa5a7edc602ad523a9e94a1a90cce9
 F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25
 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c
 F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281984
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P abb112d9e1a51c1c2f15d1e5f3dc161c9a947841f69a56715f5f1c62603096b2
-R 97c2ea02cbe64fb40daf371bd5a56946
-U dan
-Z dc8bc5606c8c9f2e7de507a70d755d15
+P fe053b2ecdc04b918367fa503bf10292fd7cd84a6929670857b2e9153df4d1f9
+R 976d51d50d3ac4cef737330f069bf895
+U mistachkin
+Z ec6d23a123b55225bd923dacec25128f
diff --git a/manifest.uuid b/manifest.uuid
index 095a867785..26b32bfe54 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-fe053b2ecdc04b918367fa503bf10292fd7cd84a6929670857b2e9153df4d1f9
\ No newline at end of file
+6dcba6693f064b0e21360c5eab18abe4a1534f6699b5d7b92e60eab6be14c347
\ No newline at end of file

From 4064a4cc58d1c96e166174a1346cf4adfac06e30 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 8 Jan 2018 17:12:54 +0000
Subject: [PATCH 259/488] Add an "ifcapable trace {}" block to a test in
 fkey1.test that uses SQL tracing.

FossilOrigin-Name: 7182591d351dde22ed2f6a60521d1d7c10a610d702e79693412efc6938167be0
---
 manifest        | 14 +++++++-------
 manifest.uuid   |  2 +-
 test/fkey1.test | 20 +++++++++++---------
 3 files changed, 19 insertions(+), 17 deletions(-)

diff --git a/manifest b/manifest
index df8f0eb071..c028a332d2 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\sa\ssuperfluous\sdefine\sfrom\sthe\s'fileio'\sextension\swhen\scompiled\sfor\sWin32.
-D 2018-01-08T17:09:05.559
+C Add\san\s"ifcapable\strace\s{}"\sblock\sto\sa\stest\sin\sfkey1.test\sthat\suses\sSQL\ntracing.
+D 2018-01-08T17:12:54.166
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -793,7 +793,7 @@ F test/extraquick.test cb254400bd42bfb777ff675356aabf3287978f79
 F test/fallocate.test 87b5e43c872b7e69cd80b7b8813eb102b571a75d45dda24e38b65537bcc85733
 F test/filectrl.test 6e871c2d35dead1d9a88e176e8d2ca094fec6bb3
 F test/filefmt.test f393e80c4b8d493b7a7f8f3809a8425bbf4292af1f5140f01cb1427798a2bbd4
-F test/fkey1.test 9d7e3a0d409e7f64ab077af3b4fc5e5ce1a4e8d8f1272b65b9d93480aeb1fa2b
+F test/fkey1.test d11dbb8a93ead9b5c46ae5d02da016d61245d47662fb2d844c99214f6163f768
 F test/fkey2.test 155809016fad6b2a1491facf2ac53a551bc57c2c
 F test/fkey3.test 76d475c80b84ee7a5d062e56ccb6ea68882e2b49
 F test/fkey4.test 86446017011273aad8f9a99c1a65019e7bd9ca9d
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P fe053b2ecdc04b918367fa503bf10292fd7cd84a6929670857b2e9153df4d1f9
-R 976d51d50d3ac4cef737330f069bf895
-U mistachkin
-Z ec6d23a123b55225bd923dacec25128f
+P 6dcba6693f064b0e21360c5eab18abe4a1534f6699b5d7b92e60eab6be14c347
+R 409a1d71985cee1ddb2e1600f67ae81d
+U dan
+Z 1ce57652dfb6237638a64015c88854e8
diff --git a/manifest.uuid b/manifest.uuid
index 26b32bfe54..92c91bdd00 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6dcba6693f064b0e21360c5eab18abe4a1534f6699b5d7b92e60eab6be14c347
\ No newline at end of file
+7182591d351dde22ed2f6a60521d1d7c10a610d702e79693412efc6938167be0
\ No newline at end of file
diff --git a/test/fkey1.test b/test/fkey1.test
index 2530327fce..fa87335888 100644
--- a/test/fkey1.test
+++ b/test/fkey1.test
@@ -174,16 +174,18 @@ do_catchsql_test fkey1-5.2 {
 # Make sure sqlite3_trace() output works with triggers used to implement
 # FK constraints
 #
-proc sqltrace {txt} {
-  global traceoutput
-  lappend traceoutput $txt
+ifcapable trace {
+  proc sqltrace {txt} {
+    global traceoutput
+    lappend traceoutput $txt
+  }
+  do_test fkey1-5.2.1 {
+    unset -nocomplain traceoutput
+    db trace sqltrace
+    catch {db eval {INSERT OR REPLACE INTO t11 VALUES(2,3);}}
+    set traceoutput
+  } {{INSERT OR REPLACE INTO t11 VALUES(2,3);} {INSERT OR REPLACE INTO t11 VALUES(2,3);} {INSERT OR REPLACE INTO t11 VALUES(2,3);}}
 }
-do_test fkey1-5.2.1 {
-  unset -nocomplain traceoutput
-  db trace sqltrace
-  catch {db eval {INSERT OR REPLACE INTO t11 VALUES(2,3);}}
-  set traceoutput
-} {{INSERT OR REPLACE INTO t11 VALUES(2,3);} {INSERT OR REPLACE INTO t11 VALUES(2,3);} {INSERT OR REPLACE INTO t11 VALUES(2,3);}}
 
 # A similar test to the above.
 do_execsql_test fkey1-5.3 {

From d4bb7c180bf4532344f4968e4f6b66ae8ccaf277 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 8 Jan 2018 17:34:15 +0000
Subject: [PATCH 260/488] Fix problems in the sqlite3expert.c code revealed by
 -fsanitize.

FossilOrigin-Name: 7a93dd784bfdbf01927979a61643796e0901d9ac285fe4214677838def93a9a4
---
 ext/expert/sqlite3expert.c |  6 +++---
 manifest                   | 12 ++++++------
 manifest.uuid              |  2 +-
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index e4060de681..f347ddea6d 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -495,7 +495,7 @@ static int expertBestIndex(sqlite3_vtab *pVtab, sqlite3_index_info *pIdxInfo){
     }
   }
 
-  pIdxInfo->estimatedCost = 1000000.0 / n;
+  pIdxInfo->estimatedCost = 1000000.0 / (n+1);
   return rc;
 }
 
@@ -754,7 +754,7 @@ static char *idxAppendText(int *pRc, char *zIn, const char *zFmt, ...){
       zRet = (char*)sqlite3_malloc(nIn + nAppend + 1);
     }
     if( zAppend && zRet ){
-      memcpy(zRet, zIn, nIn);
+      if( nIn ) memcpy(zRet, zIn, nIn);
       memcpy(&zRet[nIn], zAppend, nAppend+1);
     }else{
       sqlite3_free(zRet);
@@ -908,7 +908,7 @@ static int idxCreateFromCons(
     char *zCols = 0;
     char *zIdx = 0;
     IdxConstraint *pCons;
-    int h = 0;
+    unsigned int h = 0;
     const char *zFmt;
 
     for(pCons=pEq; pCons; pCons=pCons->pLink){
diff --git a/manifest b/manifest
index c028a332d2..5f5f829f97 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\san\s"ifcapable\strace\s{}"\sblock\sto\sa\stest\sin\sfkey1.test\sthat\suses\sSQL\ntracing.
-D 2018-01-08T17:12:54.166
+C Fix\sproblems\sin\sthe\ssqlite3expert.c\scode\srevealed\sby\s-fsanitize.
+D 2018-01-08T17:34:15.620
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -45,7 +45,7 @@ F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3
 F ext/expert/expert.c 4791c5e064aea81b2b829fa95228b22283380ee370ea88a1e580103b75516ebf
 F ext/expert/expert1.test 0c71a3453ce3a0b4dbe952713aec0ae8d416dd846820dd027b08f305f5278b30
-F ext/expert/sqlite3expert.c 252f3129f12a0e9df094a14711db98265c9c6d7afa033ec906d94e920f5c7ba7
+F ext/expert/sqlite3expert.c 9d352d8693a997402095a16791122ca5c1bff7627faee3625509fcaaef9b30db
 F ext/expert/sqlite3expert.h af6354f8ee5c9e025024e63fec3bd640a802afcc3099a44d804752cf0791d811
 F ext/expert/test_expert.c 85f5c743a899063fa48296d21de2f32c26d09a21c8582b2a0bc482e8de183e7a
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6dcba6693f064b0e21360c5eab18abe4a1534f6699b5d7b92e60eab6be14c347
-R 409a1d71985cee1ddb2e1600f67ae81d
+P 7182591d351dde22ed2f6a60521d1d7c10a610d702e79693412efc6938167be0
+R 49ecd97a6ce816710af5cab9bccdaf02
 U dan
-Z 1ce57652dfb6237638a64015c88854e8
+Z 0d0d547dd2ee63ac3d70f7968bc046aa
diff --git a/manifest.uuid b/manifest.uuid
index 92c91bdd00..0ced8aee6a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7182591d351dde22ed2f6a60521d1d7c10a610d702e79693412efc6938167be0
\ No newline at end of file
+7a93dd784bfdbf01927979a61643796e0901d9ac285fe4214677838def93a9a4
\ No newline at end of file

From ef4248122488372b8aeb7b37d3a06df6a7cf77d2 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 8 Jan 2018 19:18:27 +0000
Subject: [PATCH 261/488] Add IN-operator normalizating and the
 SQLITE_NORMALIZE_CLI compile-time option for generating a stand-alone
 program.

FossilOrigin-Name: d77dbb398afa80c1b3373c55f278491e83d5c80ddc22dbc303876bdcbc127df9
---
 ext/misc/normalize.c | 67 ++++++++++++++++++++++++++++++++++++++++----
 manifest             | 15 ++++------
 manifest.uuid        |  2 +-
 3 files changed, 69 insertions(+), 15 deletions(-)

diff --git a/ext/misc/normalize.c b/ext/misc/normalize.c
index 914a154d9a..fe5fb01b4d 100644
--- a/ext/misc/normalize.c
+++ b/ext/misc/normalize.c
@@ -35,11 +35,19 @@
 **
 ** The purpose of normalization is two-fold:
 **
-**   (1)  Sanitize queries by removing possibly sensitive information contained
-**        in literals.
+**   (1)  Sanitize queries by removing potentially private or sensitive
+**        information contained in literals.
 **
 **   (2)  Identify structurally identical queries by comparing their
 **        normalized forms.
+**
+** Command-Line Utility
+** --------------------
+**
+** This file also contains code for a command-line utility that converts
+** SQL queries in text files into their normalized forms.  To build the
+** command-line program, compile this file with -DSQLITE_NORMALIZE_CLI
+** and link it against the SQLite library.
 */
 #include <sqlite3.h>
 #include <string.h>
@@ -48,9 +56,13 @@
 ** Implementation note:
 **
 ** Much of the tokenizer logic is copied out of the tokenize.c source file
-** of SQLite.  This logic could be simplified for this particular application,
+** of SQLite.  That logic could be simplified for this particular application,
 ** but that would impose a risk of introducing subtle errors.  It is best to
 ** keep the code as close to the original as possible.
+**
+** The tokenize code is in sync with the SQLite core as of 2018-01-08.
+** Any future changes to the core tokenizer might require corresponding
+** adjustments to the tokenizer logic in this module.
 */
 
 
@@ -572,13 +584,54 @@ char *sqlite3_normalize(const char *zSql){
   while( j>0 && z[j-1]==' ' ){ j--; }
   if( i>0 && z[j-1]!=';' ){ z[j++] = ';'; }
   z[j] = 0;
+
+  /* Make a second pass converting "in(...)" where the "..." is not a
+  ** SELECT statement into "in(?,?,?)" */
+  for(i=0; i<j; i=n){
+    char *zIn = strstr(z+i, "in(");
+    int nParen;
+    if( zIn==0 ) break;
+    n = (int)(zIn-z)+3;  /* Index of first char past "in(" */
+    if( n && IdChar(zIn[-1]) ) continue;
+    if( strncmp(zIn, "in(select",9)==0 && !IdChar(zIn[9]) ) continue;
+    if( strncmp(zIn, "in(with",7)==0 && !IdChar(zIn[7]) ) continue;
+    for(nParen=1, k=0; z[n+k]; k++){
+      if( z[n+k]=='(' ) nParen++;
+      if( z[n+k]==')' ){
+        nParen--;
+        if( nParen==0 ) break;
+      }
+    }
+    /* k is the number of bytes in the "..." within "in(...)" */
+    if( k<5 ){
+      z = sqlite3_realloc64(z, j+(5-k)+1);
+      if( z==0 ) return 0;
+      memmove(z+n+5, z+n+k, j-(n+k));
+    }else if( k>5 ){
+      memmove(z+n+5, z+n+k, j-(n+k));
+    }
+    j = j-k+5;
+    z[j] = 0;
+    memcpy(z+n, "?,?,?", 5);
+  }
   return z;
 }
 
-#ifdef NORMALIZE_TEST
+/*
+** For testing purposes, or to build a stand-alone SQL normalizer program,
+** compile this one source file with the -DSQLITE_NORMALIZE_CLI and link
+** it against any SQLite library.  The resulting command-line program will
+** run sqlite3_normalize() over the text of all files named on the command-
+** line and show the result on standard output.
+*/
+#ifdef SQLITE_NORMALIZE_CLI
 #include <stdio.h>
 #include <stdlib.h>
 
+/*
+** Break zIn up into separate SQL statements and run sqlite3_normalize()
+** on each one.  Print the result of each run.
+*/
 static void normalizeFile(char *zIn){
   int i;
   if( zIn==0 ) return;
@@ -604,6 +657,10 @@ static void normalizeFile(char *zIn){
   }
 }
 
+/*
+** The main routine for "sql_normalize".  Read files named on the
+** command-line and run the text of each through sqlite3_normalize().
+*/
 int main(int argc, char **argv){
   int i;
   FILE *in;
@@ -636,4 +693,4 @@ int main(int argc, char **argv){
   }
   sqlite3_free(zBuf);
 }
-#endif /* NORMALIZE_TEST */
+#endif /* SQLITE_NORMALIZE_CLI */
diff --git a/manifest b/manifest
index 7cd42ecfd6..4cc5e208a8 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C First\scode\sfor\san\sauxiliary\sfunction\sthat\swill\snormalize\san\sSQL\sstatement.
-D 2018-01-08T16:54:26.258
+C Add\sIN-operator\snormalizating\sand\sthe\sSQLITE_NORMALIZE_CLI\scompile-time\noption\sfor\sgenerating\sa\sstand-alone\sprogram.
+D 2018-01-08T19:18:27.144
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -284,7 +284,7 @@ F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281
 F ext/misc/memvfs.c ab36f49e02ebcdf85a1e08dc4d8599ea8f343e073ac9e0bca18a98b7e1ec9567
 F ext/misc/mmapwarm.c 70b618f2d0bde43fae288ad0b7498a629f2b6f61b50a27e06fae3cd23c83af29
 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342
-F ext/misc/normalize.c 5ca65b1332e41d679d7c07374398b49d020281d4ea30f267c6a6fb500e71f7c7
+F ext/misc/normalize.c c5f9b3968bf0bfac6ca1c1f802a30ca08e96a9ac7ab3297190b2745046d7d838
 F ext/misc/percentile.c 92699c8cd7d517ff610e6037e56506f8904dae2e
 F ext/misc/regexp.c a68d25c659bd2d893cd1215667bbf75ecb9dc7d4
 F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c
@@ -1698,10 +1698,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 90cb01d8d6ac12d0b88f2952a75aeefa81ba66f5e4a5377fdd8b9f86aec8e927
-R 9bd921419b4f2c3b5e55f041d45b29ce
-T *branch * normalize
-T *sym-normalize *
-T -sym-trunk *
+P 84814aac81e54f03430f180926156ab0fc01e22bbce2bb228b698ea0d4a588ea
+R 52aee785605536a5becd57b22eca4f38
 U drh
-Z c8d4afe60d21d90fa2f1cb97ff587c09
+Z be9e0b00c5811d466485564c364b04ac
diff --git a/manifest.uuid b/manifest.uuid
index 68c796f48f..6aaff97c39 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-84814aac81e54f03430f180926156ab0fc01e22bbce2bb228b698ea0d4a588ea
\ No newline at end of file
+d77dbb398afa80c1b3373c55f278491e83d5c80ddc22dbc303876bdcbc127df9
\ No newline at end of file

From d19866bb98c548049757d4e54efda009badd655c Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 8 Jan 2018 19:29:28 +0000
Subject: [PATCH 262/488] Special handling of the NULL keyword.  Sometimes it
 is a literal, and sometimes it is a keyword.

FossilOrigin-Name: db5d138e97f22ad4d4d11dbef96df93696ba0e557809066bc263ca3c3898f349
---
 ext/misc/normalize.c | 11 +++++++++++
 manifest             | 12 ++++++------
 manifest.uuid        |  2 +-
 3 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/ext/misc/normalize.c b/ext/misc/normalize.c
index fe5fb01b4d..a3c00210f6 100644
--- a/ext/misc/normalize.c
+++ b/ext/misc/normalize.c
@@ -573,6 +573,17 @@ char *sqlite3_normalize(const char *zSql){
       }
       case TK_PUNCT:
       case TK_NAME: {
+        if( n==4 && sqlite3_strnicmp(zSql+i,"NULL",4)==0 ){
+          if( (j>=3 && strncmp(z+j-2,"is",2)==0 && !IdChar(z[j-3]))
+           || (j>=4 && strncmp(z+j-3,"not",3)==0 && !IdChar(z[j-4]))
+          ){
+            /* NULL is a keyword in this case, not a literal value */
+          }else{
+            /* Here the NULL is a literal value */
+            z[j++] = '?';
+            break;
+          }
+        }
         if( j>0 && IdChar(z[j-1]) && IdChar(zSql[i]) ) z[j++] = ' ';
         for(k=0; k<n; k++){
           z[j++] = sqlite3Tolower(zSql[i+k]);
diff --git a/manifest b/manifest
index 4cc5e208a8..2c752a40db 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sIN-operator\snormalizating\sand\sthe\sSQLITE_NORMALIZE_CLI\scompile-time\noption\sfor\sgenerating\sa\sstand-alone\sprogram.
-D 2018-01-08T19:18:27.144
+C Special\shandling\sof\sthe\sNULL\skeyword.\s\sSometimes\sit\sis\sa\sliteral,\sand\ssometimes\nit\sis\sa\skeyword.
+D 2018-01-08T19:29:28.438
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -284,7 +284,7 @@ F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281
 F ext/misc/memvfs.c ab36f49e02ebcdf85a1e08dc4d8599ea8f343e073ac9e0bca18a98b7e1ec9567
 F ext/misc/mmapwarm.c 70b618f2d0bde43fae288ad0b7498a629f2b6f61b50a27e06fae3cd23c83af29
 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342
-F ext/misc/normalize.c c5f9b3968bf0bfac6ca1c1f802a30ca08e96a9ac7ab3297190b2745046d7d838
+F ext/misc/normalize.c f26e69c7539f74dbc0a00468df117bb9085f229ecb64d65c10ebf59a010511bb
 F ext/misc/percentile.c 92699c8cd7d517ff610e6037e56506f8904dae2e
 F ext/misc/regexp.c a68d25c659bd2d893cd1215667bbf75ecb9dc7d4
 F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c
@@ -1698,7 +1698,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 84814aac81e54f03430f180926156ab0fc01e22bbce2bb228b698ea0d4a588ea
-R 52aee785605536a5becd57b22eca4f38
+P d77dbb398afa80c1b3373c55f278491e83d5c80ddc22dbc303876bdcbc127df9
+R 52c05b49206cf2984d560c493371a137
 U drh
-Z be9e0b00c5811d466485564c364b04ac
+Z 26ab19c5b91f60ca62a12106abc7db8e
diff --git a/manifest.uuid b/manifest.uuid
index 6aaff97c39..c338d86fc5 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d77dbb398afa80c1b3373c55f278491e83d5c80ddc22dbc303876bdcbc127df9
\ No newline at end of file
+db5d138e97f22ad4d4d11dbef96df93696ba0e557809066bc263ca3c3898f349
\ No newline at end of file

From 7c15ac1adac225c405d4632d38b41099997e186a Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 8 Jan 2018 19:59:59 +0000
Subject: [PATCH 263/488] Update the zipfile module so that it matches the
 documentation.

FossilOrigin-Name: 7e7e472fa91a2bad2e521d4d67f176c8eb9edc1a07b283e425ea0fa2b6abba1f
---
 ext/misc/zipfile.c | 264 ++++++++++++++++++++++++++-------------------
 manifest           |  16 +--
 manifest.uuid      |   2 +-
 src/shell.c.in     |   4 +-
 test/zipfile.test  |  21 ++--
 5 files changed, 175 insertions(+), 132 deletions(-)

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index d2715c4400..c99bac080e 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -42,17 +42,18 @@ typedef unsigned long u32;
 #define MIN(a,b) ((a)<(b) ? (a) : (b))
 #endif
 
-#define ZIPFILE_SCHEMA "CREATE TABLE y("                           \
-  "name,      /* Name of file in zip archive */"                   \
-  "mode,      /* POSIX mode for file */"                           \
-  "mtime,     /* Last modification time in seconds since epoch */" \
-  "sz,        /* Size of object */"                                \
-  "data,      /* Data stored in zip file (possibly compressed) */" \
-  "method,    /* Compression method (integer) */"                  \
-  "f HIDDEN   /* Name of zip file */"                              \
+#define ZIPFILE_SCHEMA "CREATE TABLE y("                              \
+  "name,      /* 0: Name of file in zip archive */"                   \
+  "mode,      /* 1: POSIX mode for file */"                           \
+  "mtime,     /* 2: Last modification time in seconds since epoch */" \
+  "sz,        /* 3: Size of object */"                                \
+  "rawdata,   /* 4: Raw data */"                                      \
+  "data,      /* 5: Uncompressed data */"                             \
+  "method,    /* 6: Compression method (integer) */"                  \
+  "file HIDDEN   /* Name of zip file */"                              \
 ");"
 
-#define ZIPFILE_F_COLUMN_IDX 6    /* Index of column "f" in the above */
+#define ZIPFILE_F_COLUMN_IDX 7    /* Index of column "f" in the above */
 #define ZIPFILE_BUFFER_SIZE (64*1024)
 
 
@@ -671,6 +672,80 @@ static void zipfileMtimeToDos(ZipfileCDS *pCds, u32 mTime){
     ((res.tm_year-80) << 9));
 }
 
+static void zipfileInflate(
+  sqlite3_context *pCtx,          /* Store error here, if any */
+  const u8 *aIn,                  /* Compressed data */
+  int nIn,                        /* Size of buffer aIn[] in bytes */
+  int nOut                        /* Expected output size */
+){
+  u8 *aRes = sqlite3_malloc(nOut);
+  if( aRes==0 ){
+    sqlite3_result_error_nomem(pCtx);
+  }else{
+    int err;
+    z_stream str;
+    memset(&str, 0, sizeof(str));
+
+    str.next_in = (Byte*)aIn;
+    str.avail_in = nIn;
+    str.next_out = (Byte*)aRes;
+    str.avail_out = nOut;
+
+    err = inflateInit2(&str, -15);
+    if( err!=Z_OK ){
+      zipfileCtxErrorMsg(pCtx, "inflateInit2() failed (%d)", err);
+    }else{
+      err = inflate(&str, Z_NO_FLUSH);
+      if( err!=Z_STREAM_END ){
+        zipfileCtxErrorMsg(pCtx, "inflate() failed (%d)", err);
+      }else{
+        sqlite3_result_blob(pCtx, aRes, nOut, SQLITE_TRANSIENT);
+      }
+    }
+    sqlite3_free(aRes);
+    inflateEnd(&str);
+  }
+}
+
+static int zipfileDeflate(
+  ZipfileTab *pTab,               /* Set error message here */
+  const u8 *aIn, int nIn,         /* Input */
+  u8 **ppOut, int *pnOut          /* Output */
+){
+  int nAlloc = (int)compressBound(nIn);
+  u8 *aOut;
+  int rc;
+
+  aOut = (u8*)sqlite3_malloc(nAlloc);
+  if( aOut==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    int res;
+    z_stream str;
+    memset(&str, 0, sizeof(str));
+    str.next_in = (z_const Bytef*)aIn;
+    str.avail_in = nIn;
+    str.next_out = aOut;
+    str.avail_out = nAlloc;
+
+    deflateInit2(&str, 9, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY);
+    res = deflate(&str, Z_FINISH);
+
+    if( res==Z_STREAM_END ){
+      *ppOut = aOut;
+      *pnOut = (int)str.total_out;
+    }else{
+      sqlite3_free(aOut);
+      pTab->base.zErrMsg = sqlite3_mprintf("zipfile: deflate() error");
+      rc = SQLITE_ERROR;
+    }
+    deflateEnd(&str);
+  }
+
+  return rc;
+}
+
+
 /*
 ** Return values of columns for the row at which the series_cursor
 ** is currently pointing.
@@ -703,26 +778,33 @@ static int zipfileColumn(
       sqlite3_result_int64(ctx, pCsr->cds.szUncompressed);
       break;
     }
-    case 4: { /* data */
-      int sz = pCsr->cds.szCompressed;
-      if( sz>0 ){
-        u8 *aBuf = sqlite3_malloc(sz);
-        if( aBuf==0 ){
-          rc = SQLITE_NOMEM;
-        }else{
-          FILE *pFile = zipfileGetFd(pCsr);
-          rc = zipfileReadData(pFile, aBuf, sz, pCsr->iDataOff,
-              &pCsr->base.pVtab->zErrMsg
-          );
-        }
-        if( rc==SQLITE_OK ){
-          sqlite3_result_blob(ctx, aBuf, sz, SQLITE_TRANSIENT);
-          sqlite3_free(aBuf);
+    case 4:   /* rawdata */
+    case 5: { /* data */
+      if( i==4 || pCsr->cds.iCompression==0 || pCsr->cds.iCompression==8 ){
+        int sz = pCsr->cds.szCompressed;
+        if( sz>0 ){
+          u8 *aBuf = sqlite3_malloc(sz);
+          if( aBuf==0 ){
+            rc = SQLITE_NOMEM;
+          }else{
+            FILE *pFile = zipfileGetFd(pCsr);
+            rc = zipfileReadData(pFile, aBuf, sz, pCsr->iDataOff,
+                &pCsr->base.pVtab->zErrMsg
+            );
+          }
+          if( rc==SQLITE_OK ){
+            if( i==5 && pCsr->cds.iCompression ){
+              zipfileInflate(ctx, aBuf, sz, pCsr->cds.szUncompressed);
+            }else{
+              sqlite3_result_blob(ctx, aBuf, sz, SQLITE_TRANSIENT);
+            }
+            sqlite3_free(aBuf);
+          }
         }
       }
       break;
     }
-    case 5:   /* method */
+    case 6:   /* method */
       sqlite3_result_int(ctx, pCsr->cds.iCompression);
       break;
   }
@@ -1073,7 +1155,9 @@ static int zipfileAppendEntry(
 static int zipfileGetMode(ZipfileTab *pTab, sqlite3_value *pVal, int *pMode){
   const char *z = (const char*)sqlite3_value_text(pVal);
   int mode = 0;
-  if( z==0 || (z[0]>=0 && z[0]<=9) ){
+  if( z==0 ){
+    mode = 33188;                 /* -rw-r--r-- */
+  }else if( z[0]>=0 && z[0]<=9 ){
     mode = sqlite3_value_int(pVal);
   }else{
     const char zTemplate[11] = "-rwxrwxrwx";
@@ -1118,8 +1202,8 @@ static int zipfileUpdate(
   i64 sz;                         /* Uncompressed size */
   const char *zPath;              /* Path for new entry */
   int nPath;                      /* strlen(zPath) */
-  const u8 *pData;                /* Pointer to buffer containing content */
-  int nData;                      /* Size of pData buffer in bytes */
+  const u8 *pData = 0;            /* Pointer to buffer containing content */
+  int nData = 0;                  /* Size of pData buffer in bytes */
   int iMethod = 0;                /* Compression method for new entry */
   u8 *pFree = 0;                  /* Free this */
   ZipfileCDS cds;                 /* New Central Directory Structure entry */
@@ -1143,45 +1227,54 @@ static int zipfileUpdate(
   nPath = (int)strlen(zPath);
   rc = zipfileGetMode(pTab, apVal[3], &mode);
   if( rc!=SQLITE_OK ) return rc;
-  mTime = sqlite3_value_int64(apVal[4]);
-  sz = sqlite3_value_int(apVal[5]);
-  pData = sqlite3_value_blob(apVal[6]);
-  nData = sqlite3_value_bytes(apVal[6]);
-
-  /* If a NULL value is inserted into the 'method' column, do automatic
-  ** compression. */
-  if( nData>0 && sqlite3_value_type(apVal[7])==SQLITE_NULL ){
-    pFree = (u8*)sqlite3_malloc(nData);
-    if( pFree==0 ){
-      rc = SQLITE_NOMEM;
-    }else{
-      int res;
-      z_stream str;
-      memset(&str, 0, sizeof(str));
-      str.next_in = (z_const Bytef*)pData;
-      str.avail_in = nData;
-      str.next_out = pFree;
-      str.avail_out = nData;
-      deflateInit2(&str, 9, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY);
-      res = deflate(&str, Z_FINISH);
-      if( res==Z_STREAM_END ){
-        pData = pFree;
-        nData = str.total_out;
-        iMethod = 8;
-      }else if( res!=Z_OK ){
-        pTab->base.zErrMsg = sqlite3_mprintf("zipfile: deflate() error");
-        rc = SQLITE_ERROR;
-      }
-      deflateEnd(&str);
-    }
+  if( sqlite3_value_type(apVal[4])==SQLITE_NULL ){
+    mTime = (sqlite3_int64)time(0);
   }else{
-    iMethod = sqlite3_value_int(apVal[7]);
+    mTime = sqlite3_value_int64(apVal[4]);
+  }
+
+  if( sqlite3_value_type(apVal[5])==SQLITE_NULL    /* sz */
+   && sqlite3_value_type(apVal[6])==SQLITE_NULL    /* rawdata */
+   && sqlite3_value_type(apVal[7])!=SQLITE_NULL    /* data */
+  ){
+    const u8 *aIn = sqlite3_value_blob(apVal[7]);
+    int nIn = sqlite3_value_bytes(apVal[7]);
+    int bAuto = sqlite3_value_type(apVal[8])==SQLITE_NULL;
+    
+    iMethod = sqlite3_value_int(apVal[8]);
+    sz = nIn;
+    if( iMethod!=0 && iMethod!=8 ){
+      rc = SQLITE_CONSTRAINT;
+    }else if( bAuto || iMethod ){
+      rc = zipfileDeflate(pTab, aIn, nIn, &pFree, &nData);
+      if( rc==SQLITE_OK ){
+        if( iMethod || nData<nIn ){
+          iMethod = 8;
+          pData = pFree;
+        }else{
+          pData = aIn;
+          nData = nIn;
+        }
+      }
+    }
+  }else 
+  if( sqlite3_value_type(apVal[5])!=SQLITE_NULL    /* sz */
+   && sqlite3_value_type(apVal[6])!=SQLITE_NULL    /* rawdata */
+   && sqlite3_value_type(apVal[7])==SQLITE_NULL    /* data */
+   && sqlite3_value_type(apVal[8])!=SQLITE_NULL    /* method */
+  ){
+    pData = sqlite3_value_blob(apVal[6]);
+    nData = sqlite3_value_bytes(apVal[6]);
+    sz = sqlite3_value_int(apVal[5]);
+    iMethod = sqlite3_value_int(apVal[8]);
     if( iMethod<0 || iMethod>65535 ){
       pTab->base.zErrMsg = sqlite3_mprintf(
           "zipfile: invalid compression method: %d", iMethod
       );
       rc = SQLITE_ERROR;
     }
+  }else{
+    rc = SQLITE_CONSTRAINT;
   }
 
   if( rc==SQLITE_OK ){
@@ -1352,52 +1445,6 @@ static int zipfileRegister(sqlite3 *db){
 # define zipfileRegister(x) SQLITE_OK
 #endif
 
-/*
-** zipfile_uncompress(DATA, SZ, METHOD)
-*/
-static void zipfileUncompressFunc(
-  sqlite3_context *context,
-  int argc,
-  sqlite3_value **argv
-){
-  int iMethod;
-
-  iMethod = sqlite3_value_int(argv[2]);
-  if( iMethod==0 ){
-    sqlite3_result_value(context, argv[0]);
-  }else if( iMethod==8 ){
-    Byte *res;
-    int sz = sqlite3_value_int(argv[1]);
-    z_stream str;
-    memset(&str, 0, sizeof(str));
-    str.next_in = (Byte*)sqlite3_value_blob(argv[0]);
-    str.avail_in = sqlite3_value_bytes(argv[0]);
-    res = str.next_out = (Byte*)sqlite3_malloc(sz);
-    if( res==0 ){
-      sqlite3_result_error_nomem(context);
-    }else{
-      int err;
-      str.avail_out = sz;
-
-      err = inflateInit2(&str, -15);
-      if( err!=Z_OK ){
-        zipfileCtxErrorMsg(context, "inflateInit2() failed (%d)", err);
-      }else{
-        err = inflate(&str, Z_NO_FLUSH);
-        if( err!=Z_STREAM_END ){
-          zipfileCtxErrorMsg(context, "inflate() failed (%d)", err);
-        }else{
-          sqlite3_result_blob(context, res, sz, SQLITE_TRANSIENT);
-        }
-      }
-      sqlite3_free(res);
-      inflateEnd(&str);
-    }
-  }else{
-    zipfileCtxErrorMsg(context, "unrecognized compression method: %d", iMethod);
-  }
-}
-
 #ifdef _WIN32
 __declspec(dllexport)
 #endif
@@ -1406,13 +1453,8 @@ int sqlite3_zipfile_init(
   char **pzErrMsg, 
   const sqlite3_api_routines *pApi
 ){
-  int rc = SQLITE_OK;
   SQLITE_EXTENSION_INIT2(pApi);
   (void)pzErrMsg;  /* Unused parameter */
-  rc = sqlite3_create_function(db, "zipfile_uncompress", 3,
-      SQLITE_UTF8, 0, zipfileUncompressFunc, 0, 0
-  );
-  if( rc!=SQLITE_OK ) return rc;
   return zipfileRegister(db);
 }
 
diff --git a/manifest b/manifest
index 5f5f829f97..67ccbcf14d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sproblems\sin\sthe\ssqlite3expert.c\scode\srevealed\sby\s-fsanitize.
-D 2018-01-08T17:34:15.620
+C Update\sthe\szipfile\smodule\sso\sthat\sit\smatches\sthe\sdocumentation.
+D 2018-01-08T19:59:59.813
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -303,7 +303,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c 8075df9296beeebc344567927d114c6d3201110a29110013388d233fa7d4fb2c
+F ext/misc/zipfile.c d576a5b473333f5d14f9380cf005b52a684a1ee46d4a6cfb08802564e6365e5e
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -484,7 +484,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in be04d6797b4ee066f064dc370bdcb148c84c6c6979bf12f70025a7a89cf79346
+F src/shell.c.in 2aa65d155202d1caf457cb7112ec47d1aded1bc54b20e7f8f7cf81ca1dbb43bf
 F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1598,7 +1598,7 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf
 F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc
 F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa
 F test/zerodamage.test e59a56443d6298ecf7435f618f0b27654f0c849e
-F test/zipfile.test a1dd0429294cb9487900fc2b29aa9921329f20a7314aa0921b668246172ac090
+F test/zipfile.test ad4278e1ebb1c7bc0fcd7f9b47df18916b9e8f841165119865a5a6a095a2d0ba
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d
 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 7182591d351dde22ed2f6a60521d1d7c10a610d702e79693412efc6938167be0
-R 49ecd97a6ce816710af5cab9bccdaf02
+P 7a93dd784bfdbf01927979a61643796e0901d9ac285fe4214677838def93a9a4
+R 86cc2218634a3d23586f8875ed6e23c5
 U dan
-Z 0d0d547dd2ee63ac3d70f7968bc046aa
+Z 3ef32c645be21defc25c769ce8063d43
diff --git a/manifest.uuid b/manifest.uuid
index 0ced8aee6a..930322bb0b 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7a93dd784bfdbf01927979a61643796e0901d9ac285fe4214677838def93a9a4
\ No newline at end of file
+7e7e472fa91a2bad2e521d4d67f176c8eb9edc1a07b283e425ea0fa2b6abba1f
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 9758d5d180..aa92d372ab 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -4847,14 +4847,12 @@ static int arExtractCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
 
   const char *azExtraArg[] = { 
     "sqlar_uncompress(data, sz)",
-    "zipfile_uncompress(data, sz, method)"
+    "data"
   };
   const char *azSource[] = {
     "sqlar", "zipfile(?3)"
   };
 
-
-
   sqlite3_stmt *pSql = 0;
   int rc = SQLITE_OK;
   char *zDir = 0;
diff --git a/test/zipfile.test b/test/zipfile.test
index 627945651b..4e0c9b18d1 100644
--- a/test/zipfile.test
+++ b/test/zipfile.test
@@ -28,15 +28,18 @@ do_execsql_test 1.0 {
   1 mode {} 0 {} 0 
   2 mtime {} 0 {} 0 
   3 sz {} 0 {} 0 
-  4 data {} 0 {} 0
-  5 method {} 0 {} 0
+  4 rawdata {} 0 {} 0
+  5 data {} 0 {} 0
+  6 method {} 0 {} 0
 }
 
 do_execsql_test 1.1.1 {
-  INSERT INTO zz VALUES('f.txt', '-rw-r--r--', 1000000000, 5, 'abcde', 0);
+  INSERT INTO zz(name, mode, mtime, sz, rawdata, method) 
+  VALUES('f.txt', '-rw-r--r--', 1000000000, 5, 'abcde', 0);
 }
 do_execsql_test 1.1.2 {
-  INSERT INTO zz VALUES('g.txt', '-rw-r--r--', 1000000002, 5, '12345', 0);
+  INSERT INTO zz(name, mtime, sz, rawdata, method) 
+  VALUES('g.txt', 1000000002, 5, '12345', 0);
 }
 
 do_execsql_test 1.2 {
@@ -47,14 +50,13 @@ do_execsql_test 1.2 {
 }
 
 do_execsql_test 1.3 {
-  INSERT INTO zz VALUES('h.txt', 
-    '-rw-r--r--', 1000000004, 20, 'aaaaaaaaaabbbbbbbbbb', NULL
+  INSERT INTO zz(name, mode, mtime, data) VALUES('h.txt', 
+    '-rw-r--r--', 1000000004, 'aaaaaaaaaabbbbbbbbbb'
   );
 }
 
 do_execsql_test 1.4 {
-  SELECT name, mtime, zipfile_uncompress(data, sz, method), method
-  FROM zipfile('test.zip');
+  SELECT name, mtime, data, method FROM zipfile('test.zip');
 } {
   f.txt 1000000000 abcde 0
   g.txt 1000000002 12345 0
@@ -63,7 +65,8 @@ do_execsql_test 1.4 {
 
 do_execsql_test 1.5.1 {
   BEGIN;
-    INSERT INTO zz VALUES('i.txt', '-rw-r--r--', 1000000006, 5, 'zxcvb', 0);
+    INSERT INTO zz(name, mode, mtime, data, method)
+    VALUES('i.txt', '-rw-r--r--', 1000000006, 'zxcvb', 0);
     SELECT name FROM zz;
   COMMIT;
 } {f.txt g.txt h.txt i.txt}

From 91694dbdd2e9d30c79153922310fe52c7a894d31 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 8 Jan 2018 20:04:57 +0000
Subject: [PATCH 264/488] Test cases for sqlite3_normalize().

FossilOrigin-Name: 658f42257d56a3562dfa8e55023c6e497f55c565cd751d942de3a9e0cb2fa708
---
 Makefile.in          |  1 +
 Makefile.msc         |  1 +
 ext/misc/normalize.c |  4 +--
 main.mk              |  1 +
 manifest             | 21 +++++++------
 manifest.uuid        |  2 +-
 src/test1.c          | 30 ++++++++++++++++++
 test/normalize.test  | 72 ++++++++++++++++++++++++++++++++++++++++++++
 8 files changed, 119 insertions(+), 13 deletions(-)
 create mode 100644 test/normalize.test

diff --git a/Makefile.in b/Makefile.in
index af1bd61548..98655bb820 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -440,6 +440,7 @@ TESTSRC += \
   $(TOP)/ext/misc/ieee754.c \
   $(TOP)/ext/misc/mmapwarm.c \
   $(TOP)/ext/misc/nextchar.c \
+  $(TOP)/ext/misc/normalize.c \
   $(TOP)/ext/misc/percentile.c \
   $(TOP)/ext/misc/regexp.c \
   $(TOP)/ext/misc/remember.c \
diff --git a/Makefile.msc b/Makefile.msc
index 4d6c5c0cc2..640f95deaa 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -1500,6 +1500,7 @@ TESTEXT = \
   $(TOP)\ext\misc\ieee754.c \
   $(TOP)\ext\misc\mmapwarm.c \
   $(TOP)\ext\misc\nextchar.c \
+  $(TOP)\ext\misc\normalize.c \
   $(TOP)\ext\misc\percentile.c \
   $(TOP)\ext\misc\regexp.c \
   $(TOP)\ext\misc\remember.c \
diff --git a/ext/misc/normalize.c b/ext/misc/normalize.c
index a3c00210f6..fd656f1303 100644
--- a/ext/misc/normalize.c
+++ b/ext/misc/normalize.c
@@ -130,7 +130,7 @@ static const unsigned char aiClass[] = {
 ** handle case conversions for the UTF character set since the tables
 ** involved are nearly as big or bigger than SQLite itself.
 */
-const unsigned char sqlite3UpperToLower[] = {
+static const unsigned char sqlite3UpperToLower[] = {
       0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 16, 17,
      18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,
      36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53,
@@ -176,7 +176,7 @@ const unsigned char sqlite3UpperToLower[] = {
 ** non-ASCII UTF character. Hence the test for whether or not a character is
 ** part of an identifier is 0x46.
 */
-const unsigned char sqlite3CtypeMap[256] = {
+static const unsigned char sqlite3CtypeMap[256] = {
   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,  /* 00..07    ........ */
   0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00,  /* 08..0f    ........ */
   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,  /* 10..17    ........ */
diff --git a/main.mk b/main.mk
index fc68d9d307..6271e4b682 100644
--- a/main.mk
+++ b/main.mk
@@ -361,6 +361,7 @@ TESTSRC += \
   $(TOP)/ext/misc/ieee754.c \
   $(TOP)/ext/misc/mmapwarm.c \
   $(TOP)/ext/misc/nextchar.c \
+  $(TOP)/ext/misc/normalize.c \
   $(TOP)/ext/misc/percentile.c \
   $(TOP)/ext/misc/regexp.c \
   $(TOP)/ext/misc/remember.c \
diff --git a/manifest b/manifest
index 2c752a40db..14067d0999 100644
--- a/manifest
+++ b/manifest
@@ -1,10 +1,10 @@
-C Special\shandling\sof\sthe\sNULL\skeyword.\s\sSometimes\sit\sis\sa\sliteral,\sand\ssometimes\nit\sis\sa\skeyword.
-D 2018-01-08T19:29:28.438
+C Test\scases\sfor\ssqlite3_normalize().
+D 2018-01-08T20:04:57.992
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
-F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
+F Makefile.in 822052988bbf6204ca2c3e2665618b5a48acec24105d5afa8c4ebb59cb941d3e
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc b33f630258ce72fb3150c1a6819a484eba3950dc9397571d48672087e12abf4a
+F Makefile.msc 0f2875c7d4cb93ae4dab661f58f9d68699d323cfb87ebd7d8dccbb9949979b83
 F README.md eeae1e552f93ef72ef7c5b8f6647b368a001c28820ad1df179d3dae602bef681
 F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -284,7 +284,7 @@ F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281
 F ext/misc/memvfs.c ab36f49e02ebcdf85a1e08dc4d8599ea8f343e073ac9e0bca18a98b7e1ec9567
 F ext/misc/mmapwarm.c 70b618f2d0bde43fae288ad0b7498a629f2b6f61b50a27e06fae3cd23c83af29
 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342
-F ext/misc/normalize.c f26e69c7539f74dbc0a00468df117bb9085f229ecb64d65c10ebf59a010511bb
+F ext/misc/normalize.c 19262ef3ef29d4de2f281b423326865c8916c63d0cb09f1dc98d24d5c1e8ba64
 F ext/misc/percentile.c 92699c8cd7d517ff610e6037e56506f8904dae2e
 F ext/misc/regexp.c a68d25c659bd2d893cd1215667bbf75ecb9dc7d4
 F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c
@@ -407,7 +407,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 7965d01485f7bec7945407c5238985ea8c7cb2cb686d2bfdbe3d5f79d6fd4eb2
+F main.mk c6b8ee2fa871e01ad6a460d750cd074010292464a60a37a2207930ae0df629dc
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -494,7 +494,7 @@ F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
 F src/tclsqlite.c 1833388c01e3b77f4c712185ee7250b9423ee0981ce6ae7e401e47db0319a696
-F src/test1.c b52f9e7fe62016d357c3266fcfa0793cc1883d3cb2b11dfa39fcba2e70b0305c
+F src/test1.c 1ab7cbbb6693e08364c1a9241e2aee17f8c4925e4cc52396be77ae6845a05828
 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5
 F src/test3.c b8434949dfb8aff8dfa082c8b592109e77844c2135ed3c492113839b6956255b
 F src/test4.c 18ec393bb4d0ad1de729f0b94da7267270f3d8e6
@@ -1093,6 +1093,7 @@ F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660
 F test/nan.test 437d40e6d0778b050d7750726c0cbd2c9936b81962926e8f8c48ca698f00f4d1
 F test/nockpt.test 9a436a7213ba5ef7a32304998d386d3ea3f76c9d
 F test/nolock.test f196cf8b8fbea4e2ca345140a2b3f3b0da45c76e
+F test/normalize.test 501630ab49b0b26b65c74124bf03e3374c1b57fa97aae750f84803609141d167
 F test/notify1.test 669b2b743618efdc18ca4b02f45423d5d2304abf
 F test/notify2.test 2ecabaa1305083856b7c39cf32816b612740c161
 F test/notify3.test 10ff25cde502e72a92053a2f215d64bece4ef934
@@ -1698,7 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d77dbb398afa80c1b3373c55f278491e83d5c80ddc22dbc303876bdcbc127df9
-R 52c05b49206cf2984d560c493371a137
+P db5d138e97f22ad4d4d11dbef96df93696ba0e557809066bc263ca3c3898f349
+R 5585a6831c91176202d5e0f81c59c944
 U drh
-Z 26ab19c5b91f60ca62a12106abc7db8e
+Z ca698384a23dd84f9bb875fa9da10ba1
diff --git a/manifest.uuid b/manifest.uuid
index c338d86fc5..ac2ec7cffb 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-db5d138e97f22ad4d4d11dbef96df93696ba0e557809066bc263ca3c3898f349
\ No newline at end of file
+658f42257d56a3562dfa8e55023c6e497f55c565cd751d942de3a9e0cb2fa708
\ No newline at end of file
diff --git a/src/test1.c b/src/test1.c
index 55d92eb1fd..bc8f389dbd 100644
--- a/src/test1.c
+++ b/src/test1.c
@@ -4559,6 +4559,35 @@ static int SQLITE_TCLAPI test_complete16(
   return TCL_OK;
 }
 
+/*
+** Usage: sqlite3_normalize SQL
+**
+** Return the normalized value for an SQL statement.
+*/
+static int SQLITE_TCLAPI test_normalize(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  char *zSql;
+  char *zNorm;
+  extern char *sqlite3_normalize(const char*);
+
+  if( objc!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "SQL");
+    return TCL_ERROR;
+  }
+
+  zSql = (char*)Tcl_GetString(objv[1]);
+  zNorm = sqlite3_normalize(zSql);
+  if( zNorm ){
+    Tcl_SetObjResult(interp, Tcl_NewStringObj(zNorm, -1));
+    sqlite3_free(zNorm);
+  }
+  return TCL_OK;
+}
+
 /*
 ** Usage: sqlite3_step STMT
 **
@@ -7547,6 +7576,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
      { "sqlite3_open16",                test_open16        ,0 },
      { "sqlite3_open_v2",               test_open_v2       ,0 },
      { "sqlite3_complete16",            test_complete16    ,0 },
+     { "sqlite3_normalize",             test_normalize     ,0 },
 
      { "sqlite3_prepare",               test_prepare       ,0 },
      { "sqlite3_prepare16",             test_prepare16     ,0 },
diff --git a/test/normalize.test b/test/normalize.test
new file mode 100644
index 0000000000..8932650c83
--- /dev/null
+++ b/test/normalize.test
@@ -0,0 +1,72 @@
+# 2018-01-08
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests for the sqlite3_normalize() extension function.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix normalize
+
+foreach {tnum sql norm} {
+  100
+  {SELECT * FROM t1 WHERE a IN (1) AND b=51.42}
+  {select*from t1 where a in(?,?,?)and b=?;}
+
+  110
+  {SELECT a, b+15, c FROM t1 WHERE d NOT IN (SELECT x FROM t2);}
+  {select a,b+?,c from t1 where d not in(select x from t2);}
+
+  120
+  { SELECT NULL, b FROM t1 -- comment text
+     WHERE d IN (WITH t(a) AS (VALUES(5)) /* CTE */
+                 SELECT a FROM t)
+        OR e='hello';
+  }
+  {select?,b from t1 where d in(with t(a)as(values(?))select a from t)or e=?;}
+
+  121
+  {/*Initial comment*/
+   -- another comment line
+   SELECT NULL  /* comment */ , b FROM t1 -- comment text
+     WHERE d IN (WITH t(a) AS (VALUES(5)) /* CTE */
+                 SELECT a FROM t)
+        OR e='hello';
+  }
+  {select?,b from t1 where d in(with t(a)as(values(?))select a from t)or e=?;}
+
+  130
+  {/* Query containing parameters */
+   SELECT x,$::abc(15),y,@abc,z,?99,w FROM t1 /* Trailing comment */}
+  {select x,?,y,?,z,?,w from t1;}
+
+  140
+  {/* Long list on the RHS of IN */
+   SELECT 15 IN (1,2,3,(SELECT * FROM t1),'xyz',x'abcd',22*(x+5),null);}
+  {select?in(?,?,?);}
+
+  150
+  {SELECT x'abc'; -- illegal token}
+  {}
+
+  160
+  {SELECT a,NULL,b FROM t1 WHERE c IS NOT NULL or D is null or e=5}
+  {select a,?,b from t1 where c is not null or d is null or e=?;}
+
+  170
+  {/* IN list exactly 5 bytes long */
+   SELECT * FROM t1 WHERE x IN (1,2,3);}
+  {select*from t1 where x in(?,?,?);}
+} {
+  do_test $tnum [list sqlite3_normalize $sql] $norm
+}
+
+finish_test

From 88a1d6b92c21aa0830e04ce0811118cc0bcc4810 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Tue, 9 Jan 2018 00:26:39 +0000
Subject: [PATCH 265/488] Fix harmless compiler warnings.

FossilOrigin-Name: 1adf4e60391326ba699260402c06604ea25b456e903157ecb05017e7ea954fc4
---
 ext/misc/zipfile.c |  4 ++--
 manifest           | 14 +++++++-------
 manifest.uuid      |  2 +-
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index c99bac080e..b6357545b2 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -714,7 +714,7 @@ static int zipfileDeflate(
 ){
   int nAlloc = (int)compressBound(nIn);
   u8 *aOut;
-  int rc;
+  int rc = SQLITE_OK;
 
   aOut = (u8*)sqlite3_malloc(nAlloc);
   if( aOut==0 ){
@@ -1199,7 +1199,7 @@ static int zipfileUpdate(
 
   int mode;                       /* Mode for new entry */
   i64 mTime;                      /* Modification time for new entry */
-  i64 sz;                         /* Uncompressed size */
+  i64 sz = 0;                     /* Uncompressed size */
   const char *zPath;              /* Path for new entry */
   int nPath;                      /* strlen(zPath) */
   const u8 *pData = 0;            /* Pointer to buffer containing content */
diff --git a/manifest b/manifest
index 67ccbcf14d..4fe1c88833 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\sthe\szipfile\smodule\sso\sthat\sit\smatches\sthe\sdocumentation.
-D 2018-01-08T19:59:59.813
+C Fix\sharmless\scompiler\swarnings.
+D 2018-01-09T00:26:39.624
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -303,7 +303,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c d576a5b473333f5d14f9380cf005b52a684a1ee46d4a6cfb08802564e6365e5e
+F ext/misc/zipfile.c 9c3f5d7ebee44c7d2ee271232fda5e787261fd082f673caea32f14739c782059
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 7a93dd784bfdbf01927979a61643796e0901d9ac285fe4214677838def93a9a4
-R 86cc2218634a3d23586f8875ed6e23c5
-U dan
-Z 3ef32c645be21defc25c769ce8063d43
+P 7e7e472fa91a2bad2e521d4d67f176c8eb9edc1a07b283e425ea0fa2b6abba1f
+R 1c74e64e0b3c653606aafbb1d2f630fb
+U mistachkin
+Z 807ee13ef468838926808e39283b80c4
diff --git a/manifest.uuid b/manifest.uuid
index 930322bb0b..0267b8f46a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7e7e472fa91a2bad2e521d4d67f176c8eb9edc1a07b283e425ea0fa2b6abba1f
\ No newline at end of file
+1adf4e60391326ba699260402c06604ea25b456e903157ecb05017e7ea954fc4
\ No newline at end of file

From 562f0c8e468a5e0bdd216106f28ca6688cd6ce72 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Tue, 9 Jan 2018 00:28:24 +0000
Subject: [PATCH 266/488] Improve portability of compile-time MinGW detection
 for the command line shell.

FossilOrigin-Name: 4a7236140cb0f40fa846da4673a0d56218def1929d58cf016909ce881a681140
---
 manifest       | 12 ++++++------
 manifest.uuid  |  2 +-
 src/shell.c.in |  4 ++--
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index 4fe1c88833..e191942b43 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sharmless\scompiler\swarnings.
-D 2018-01-09T00:26:39.624
+C Improve\sportability\sof\scompile-time\sMinGW\sdetection\sfor\sthe\scommand\sline\sshell.
+D 2018-01-09T00:28:24.035
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -484,7 +484,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in 2aa65d155202d1caf457cb7112ec47d1aded1bc54b20e7f8f7cf81ca1dbb43bf
+F src/shell.c.in fb615970a6ae95be99316d9e4ab14bd2d5c6ecdc15ff905c4107eadfd2e20d0c
 F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 7e7e472fa91a2bad2e521d4d67f176c8eb9edc1a07b283e425ea0fa2b6abba1f
-R 1c74e64e0b3c653606aafbb1d2f630fb
+P 1adf4e60391326ba699260402c06604ea25b456e903157ecb05017e7ea954fc4
+R 5655f2550cccb3efdcb36901ceb7ce1e
 U mistachkin
-Z 807ee13ef468838926808e39283b80c4
+Z 83883ba507a74a46a1e753825276cfba
diff --git a/manifest.uuid b/manifest.uuid
index 0267b8f46a..67d4f93659 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-1adf4e60391326ba699260402c06604ea25b456e903157ecb05017e7ea954fc4
\ No newline at end of file
+4a7236140cb0f40fa846da4673a0d56218def1929d58cf016909ce881a681140
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index aa92d372ab..8ef236054e 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -76,10 +76,10 @@ typedef unsigned char u8;
 #  include <pwd.h>
 # endif
 #endif
-#if (!defined(_WIN32) && !defined(WIN32)) || defined(__MINGW_H)
+#if (!defined(_WIN32) && !defined(WIN32)) || defined(__MINGW32__)
 # include <unistd.h>
 # include <dirent.h>
-# if defined(__MINGW_H)
+# if defined(__MINGW32__)
 #  define DIRENT dirent
 #  ifndef S_ISLNK
 #   define S_ISLNK(mode) (0)

From a8691309ddfc650f7aedc601abb7448d4f4b34a2 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 9 Jan 2018 02:27:13 +0000
Subject: [PATCH 267/488] Avoid a compiler warning when building with newer
 versions of MinGW

FossilOrigin-Name: cba0206a15f30313e16a08634995ebfd5d325d83affb859a215e72509f539b4e
---
 ext/misc/fileio.c |  4 +++-
 manifest          | 14 +++++++-------
 manifest.uuid     |  2 +-
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c
index 31af50a0c0..76d9483624 100644
--- a/ext/misc/fileio.c
+++ b/ext/misc/fileio.c
@@ -92,7 +92,9 @@ SQLITE_EXTENSION_INIT1
 #  include <direct.h>
 #  include "test_windirent.h"
 #  define dirent DIRENT
-#  define stat _stat
+#  ifndef stat
+#    define stat _stat
+#  endif
 #  define mkdir(path,mode) _mkdir(path)
 #  define lstat(path,buf) _stat(path,buf)
 #endif
diff --git a/manifest b/manifest
index e191942b43..9ee3ad94d3 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improve\sportability\sof\scompile-time\sMinGW\sdetection\sfor\sthe\scommand\sline\sshell.
-D 2018-01-09T00:28:24.035
+C Avoid\sa\scompiler\swarning\swhen\sbuilding\swith\snewer\sversions\sof\sMinGW
+D 2018-01-09T02:27:13.505
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -277,7 +277,7 @@ F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f0
 F ext/misc/csv.c 1a009b93650732e22334edc92459c4630b9fa703397cbb3c8ca279921a36ca11
 F ext/misc/dbdump.c 3509fa6b8932d04e932d6b6b827b6a82ca362781b8e8f3c77336f416793e215e
 F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2
-F ext/misc/fileio.c edf880df28f606f0b00ac872a6a957b544fa5a7edc602ad523a9e94a1a90cce9
+F ext/misc/fileio.c 8bc7277143ec73c9f27057f3f38bd912a98d3492346dd688f2f877130f765d02
 F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25
 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c
 F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281984
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 1adf4e60391326ba699260402c06604ea25b456e903157ecb05017e7ea954fc4
-R 5655f2550cccb3efdcb36901ceb7ce1e
-U mistachkin
-Z 83883ba507a74a46a1e753825276cfba
+P 4a7236140cb0f40fa846da4673a0d56218def1929d58cf016909ce881a681140
+R 79b1c4b885f55f7c11d5769bd78fff72
+U drh
+Z 1c8c6d539242ddf8ecab679f4369ea9e
diff --git a/manifest.uuid b/manifest.uuid
index 67d4f93659..900719d377 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4a7236140cb0f40fa846da4673a0d56218def1929d58cf016909ce881a681140
\ No newline at end of file
+cba0206a15f30313e16a08634995ebfd5d325d83affb859a215e72509f539b4e
\ No newline at end of file

From cc234a4b664fc455209ae2b40b8d6cb286b1f882 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 9 Jan 2018 07:16:51 +0000
Subject: [PATCH 268/488] Fix a problem in zipfile.c found by -fsanitize.

FossilOrigin-Name: 4fe697fa6c2b45aec60c33eff1ce2ea97b8a2ca124ef0c0059930269d25cdb2e
---
 ext/misc/zipfile.c | 10 +++++-----
 manifest           | 14 +++++++-------
 manifest.uuid      |  2 +-
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index b6357545b2..f9acdf014a 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -1152,13 +1152,13 @@ static int zipfileAppendEntry(
   return rc;
 }
 
-static int zipfileGetMode(ZipfileTab *pTab, sqlite3_value *pVal, int *pMode){
+static int zipfileGetMode(ZipfileTab *pTab, sqlite3_value *pVal, u32 *pMode){
   const char *z = (const char*)sqlite3_value_text(pVal);
-  int mode = 0;
+  u32 mode = 0;
   if( z==0 ){
-    mode = 33188;                 /* -rw-r--r-- */
+    mode = S_IFREG + 0644;                 /* -rw-r--r-- */
   }else if( z[0]>=0 && z[0]<=9 ){
-    mode = sqlite3_value_int(pVal);
+    mode = (unsigned int)sqlite3_value_int(pVal);
   }else{
     const char zTemplate[11] = "-rwxrwxrwx";
     int i;
@@ -1197,7 +1197,7 @@ static int zipfileUpdate(
   int rc = SQLITE_OK;             /* Return Code */
   ZipfileEntry *pNew = 0;         /* New in-memory CDS entry */
 
-  int mode;                       /* Mode for new entry */
+  u32 mode;                       /* Mode for new entry */
   i64 mTime;                      /* Modification time for new entry */
   i64 sz = 0;                     /* Uncompressed size */
   const char *zPath;              /* Path for new entry */
diff --git a/manifest b/manifest
index 9ee3ad94d3..750572561e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Avoid\sa\scompiler\swarning\swhen\sbuilding\swith\snewer\sversions\sof\sMinGW
-D 2018-01-09T02:27:13.505
+C Fix\sa\sproblem\sin\szipfile.c\sfound\sby\s-fsanitize.
+D 2018-01-09T07:16:51.597
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -303,7 +303,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c 9c3f5d7ebee44c7d2ee271232fda5e787261fd082f673caea32f14739c782059
+F ext/misc/zipfile.c b694574b715dc6ea353c90e0fa31f30faa2121e59bd9e1ae40f874300114a84f
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 4a7236140cb0f40fa846da4673a0d56218def1929d58cf016909ce881a681140
-R 79b1c4b885f55f7c11d5769bd78fff72
-U drh
-Z 1c8c6d539242ddf8ecab679f4369ea9e
+P cba0206a15f30313e16a08634995ebfd5d325d83affb859a215e72509f539b4e
+R 8cb58dd5bd00b76b67672fc7e3030d3c
+U dan
+Z bb6663f307bb629de0b63290f8475ee1
diff --git a/manifest.uuid b/manifest.uuid
index 900719d377..02e21911d2 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-cba0206a15f30313e16a08634995ebfd5d325d83affb859a215e72509f539b4e
\ No newline at end of file
+4fe697fa6c2b45aec60c33eff1ce2ea97b8a2ca124ef0c0059930269d25cdb2e
\ No newline at end of file

From 128011a25846785a472f7c60c082693583f83df5 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 9 Jan 2018 10:29:34 +0000
Subject: [PATCH 269/488] Fix a zipfile problem with adding new directories to
 an archive.

FossilOrigin-Name: 5fed67033c9dd4492bf8cfcf98874284581f448d8cc84fa5470dde239f218375
---
 ext/misc/zipfile.c | 70 +++++++++++++++++++++++++++++++---------------
 manifest           | 14 +++++-----
 manifest.uuid      |  2 +-
 test/zipfile.test  | 22 +++++++++++++++
 4 files changed, 78 insertions(+), 30 deletions(-)

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index f9acdf014a..93b2a6d8bc 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -1152,11 +1152,16 @@ static int zipfileAppendEntry(
   return rc;
 }
 
-static int zipfileGetMode(ZipfileTab *pTab, sqlite3_value *pVal, u32 *pMode){
+static int zipfileGetMode(
+  ZipfileTab *pTab, 
+  sqlite3_value *pVal, 
+  u32 defaultMode,                /* Value to use if pVal IS NULL */
+  u32 *pMode
+){
   const char *z = (const char*)sqlite3_value_text(pVal);
   u32 mode = 0;
   if( z==0 ){
-    mode = S_IFREG + 0644;                 /* -rw-r--r-- */
+    mode = defaultMode;
   }else if( z[0]>=0 && z[0]<=9 ){
     mode = (unsigned int)sqlite3_value_int(pVal);
   }else{
@@ -1208,6 +1213,10 @@ static int zipfileUpdate(
   u8 *pFree = 0;                  /* Free this */
   ZipfileCDS cds;                 /* New Central Directory Structure entry */
 
+  int bIsDir = 0;
+
+  int mNull;
+
   assert( pTab->zFile );
   assert( pTab->pWriteFd );
 
@@ -1223,20 +1232,17 @@ static int zipfileUpdate(
     if( nVal==1 ) return SQLITE_OK;
   }
 
-  zPath = (const char*)sqlite3_value_text(apVal[2]);
-  nPath = (int)strlen(zPath);
-  rc = zipfileGetMode(pTab, apVal[3], &mode);
-  if( rc!=SQLITE_OK ) return rc;
-  if( sqlite3_value_type(apVal[4])==SQLITE_NULL ){
-    mTime = (sqlite3_int64)time(0);
-  }else{
-    mTime = sqlite3_value_int64(apVal[4]);
+  mNull = (sqlite3_value_type(apVal[5])==SQLITE_NULL ? 0x0 : 0x8)  /* sz */
+        + (sqlite3_value_type(apVal[6])==SQLITE_NULL ? 0x0 : 0x4)  /* rawdata */
+        + (sqlite3_value_type(apVal[7])==SQLITE_NULL ? 0x0 : 0x2)  /* data */
+        + (sqlite3_value_type(apVal[8])==SQLITE_NULL ? 0x0 : 0x1); /* method */
+  if( mNull==0x00 ){     
+    /* All four are NULL - this must be a directory */
+    bIsDir = 1;
   }
-
-  if( sqlite3_value_type(apVal[5])==SQLITE_NULL    /* sz */
-   && sqlite3_value_type(apVal[6])==SQLITE_NULL    /* rawdata */
-   && sqlite3_value_type(apVal[7])!=SQLITE_NULL    /* data */
-  ){
+  else if( mNull==0x2 || mNull==0x3 ){
+    /* Value specified for "data", and possibly "method". This must be
+    ** a regular file or a symlink. */
     const u8 *aIn = sqlite3_value_blob(apVal[7]);
     int nIn = sqlite3_value_bytes(apVal[7]);
     int bAuto = sqlite3_value_type(apVal[8])==SQLITE_NULL;
@@ -1257,12 +1263,10 @@ static int zipfileUpdate(
         }
       }
     }
-  }else 
-  if( sqlite3_value_type(apVal[5])!=SQLITE_NULL    /* sz */
-   && sqlite3_value_type(apVal[6])!=SQLITE_NULL    /* rawdata */
-   && sqlite3_value_type(apVal[7])==SQLITE_NULL    /* data */
-   && sqlite3_value_type(apVal[8])!=SQLITE_NULL    /* method */
-  ){
+  }
+  else if( mNull==0x0D ){
+    /* Values specified for "sz", "rawdata" and "method". In other words,
+    ** pre-compressed data is being inserted.  */
     pData = sqlite3_value_blob(apVal[6]);
     nData = sqlite3_value_bytes(apVal[6]);
     sz = sqlite3_value_int(apVal[5]);
@@ -1273,10 +1277,32 @@ static int zipfileUpdate(
       );
       rc = SQLITE_ERROR;
     }
-  }else{
+  }
+  else{
     rc = SQLITE_CONSTRAINT;
   }
 
+  if( rc==SQLITE_OK ){
+    rc = zipfileGetMode(pTab, apVal[3], 
+        (bIsDir ? (S_IFDIR + 0755) : (S_IFREG + 0644)), &mode
+    );
+    if( rc==SQLITE_OK && (bIsDir == ((mode & S_IFDIR)==0)) ){
+      /* The "mode" attribute is a directory, but data has been specified.
+      ** Or vice-versa - no data but "mode" is a file or symlink.  */
+      rc = SQLITE_CONSTRAINT;
+    }
+  }
+
+  if( rc==SQLITE_OK ){
+    zPath = (const char*)sqlite3_value_text(apVal[2]);
+    nPath = (int)strlen(zPath);
+    if( sqlite3_value_type(apVal[4])==SQLITE_NULL ){
+      mTime = (sqlite3_int64)time(0);
+    }else{
+      mTime = sqlite3_value_int64(apVal[4]);
+    }
+  }
+
   if( rc==SQLITE_OK ){
     /* Create the new CDS record. */
     memset(&cds, 0, sizeof(cds));
diff --git a/manifest b/manifest
index 750572561e..309f6238dc 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sproblem\sin\szipfile.c\sfound\sby\s-fsanitize.
-D 2018-01-09T07:16:51.597
+C Fix\sa\szipfile\sproblem\swith\sadding\snew\sdirectories\sto\san\sarchive.
+D 2018-01-09T10:29:34.950
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -303,7 +303,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c b694574b715dc6ea353c90e0fa31f30faa2121e59bd9e1ae40f874300114a84f
+F ext/misc/zipfile.c eaab322dd4b24199b0c6bdfbf866d1d04f7e93f46b4b5ca8436edadece54b03f
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -1598,7 +1598,7 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf
 F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc
 F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa
 F test/zerodamage.test e59a56443d6298ecf7435f618f0b27654f0c849e
-F test/zipfile.test ad4278e1ebb1c7bc0fcd7f9b47df18916b9e8f841165119865a5a6a095a2d0ba
+F test/zipfile.test de2ee377705999bcd7391fb742d6b833ffab4f21dab1d4f484098a66d60eb4fb
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d
 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P cba0206a15f30313e16a08634995ebfd5d325d83affb859a215e72509f539b4e
-R 8cb58dd5bd00b76b67672fc7e3030d3c
+P 4fe697fa6c2b45aec60c33eff1ce2ea97b8a2ca124ef0c0059930269d25cdb2e
+R e226ce112b5f76c5d01abec530de4ed7
 U dan
-Z bb6663f307bb629de0b63290f8475ee1
+Z 162125bc358610b2bc72c2a7b07e618b
diff --git a/manifest.uuid b/manifest.uuid
index 02e21911d2..1a39c77410 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4fe697fa6c2b45aec60c33eff1ce2ea97b8a2ca124ef0c0059930269d25cdb2e
\ No newline at end of file
+5fed67033c9dd4492bf8cfcf98874284581f448d8cc84fa5470dde239f218375
\ No newline at end of file
diff --git a/test/zipfile.test b/test/zipfile.test
index 4e0c9b18d1..4cb6630e09 100644
--- a/test/zipfile.test
+++ b/test/zipfile.test
@@ -79,5 +79,27 @@ do_execsql_test 1.6.0 {
   SELECT name FROM zz;
 } {f.txt h.txt i.txt}
 
+#-------------------------------------------------------------------------
+db close
+forcedelete test.zip
+reset_db
+load_static_extension db zipfile
+do_execsql_test 2.1 {
+  CREATE VIRTUAL TABLE zzz USING zipfile('test.zip');
+  INSERT INTO zzz(name, mode) VALUES('dirname', 'drwxr-xr-x');
+  SELECT name, mode, data FROM zzz;
+} {dirname 16877 {}}
+do_execsql_test 2.2 {
+  INSERT INTO zzz(name, data) VALUES('dirname2', NULL);
+  INSERT INTO zzz(name, data) VALUES('dirname2/file1.txt', 'abcdefghijklmnop');
+  SELECT name, mode, data FROM zzz;
+} {
+  dirname 16877 {}
+  dirname2 16877 {}
+  dirname2/file1.txt 33188 abcdefghijklmnop
+}
+
+
+
 finish_test
 

From 8ee5d0fc053c441c082c218e4b8409f1c461bd3c Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 9 Jan 2018 14:27:58 +0000
Subject: [PATCH 270/488] Attempt to fix the fileio.c extension so that it
 builds on MinGW-64.

FossilOrigin-Name: a7446d3217d39c96c884fbfb294dd320378255f3bfb34e35d15ba6d7c6698f53
---
 ext/misc/fileio.c |  2 +-
 manifest          | 14 +++++++-------
 manifest.uuid     |  2 +-
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c
index 76d9483624..8170e86379 100644
--- a/ext/misc/fileio.c
+++ b/ext/misc/fileio.c
@@ -96,7 +96,7 @@ SQLITE_EXTENSION_INIT1
 #    define stat _stat
 #  endif
 #  define mkdir(path,mode) _mkdir(path)
-#  define lstat(path,buf) _stat(path,buf)
+#  define lstat(path,buf) stat(path,buf)
 #endif
 #include <time.h>
 #include <errno.h>
diff --git a/manifest b/manifest
index 309f6238dc..3e04ddeb45 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\szipfile\sproblem\swith\sadding\snew\sdirectories\sto\san\sarchive.
-D 2018-01-09T10:29:34.950
+C Attempt\sto\sfix\sthe\sfileio.c\sextension\sso\sthat\sit\sbuilds\son\sMinGW-64.
+D 2018-01-09T14:27:58.076
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -277,7 +277,7 @@ F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f0
 F ext/misc/csv.c 1a009b93650732e22334edc92459c4630b9fa703397cbb3c8ca279921a36ca11
 F ext/misc/dbdump.c 3509fa6b8932d04e932d6b6b827b6a82ca362781b8e8f3c77336f416793e215e
 F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2
-F ext/misc/fileio.c 8bc7277143ec73c9f27057f3f38bd912a98d3492346dd688f2f877130f765d02
+F ext/misc/fileio.c 5176893c15421e9c459f25478e3c1b066d19317b676ff0b142862de4e701af82
 F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25
 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c
 F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281984
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 4fe697fa6c2b45aec60c33eff1ce2ea97b8a2ca124ef0c0059930269d25cdb2e
-R e226ce112b5f76c5d01abec530de4ed7
-U dan
-Z 162125bc358610b2bc72c2a7b07e618b
+P 5fed67033c9dd4492bf8cfcf98874284581f448d8cc84fa5470dde239f218375
+R 0ecfe311a33b1c3c21c3c128af696033
+U drh
+Z 00bdd39cf3e409343f20bbdc5609780f
diff --git a/manifest.uuid b/manifest.uuid
index 1a39c77410..ff7756ae2e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5fed67033c9dd4492bf8cfcf98874284581f448d8cc84fa5470dde239f218375
\ No newline at end of file
+a7446d3217d39c96c884fbfb294dd320378255f3bfb34e35d15ba6d7c6698f53
\ No newline at end of file

From f87ae41f52f86c9aa05fd0f0933acdb3431f5f2c Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 9 Jan 2018 14:30:49 +0000
Subject: [PATCH 271/488] Do not attempt to build the code in
 ext/expert/sqlite3expert.c if SQLITE_OMIT_VIRTUALTABLE is defined.

FossilOrigin-Name: ba967ad2e38590fe2a51e814a087140089be5e6a06a795a958e70a8a47f6350f
---
 ext/expert/expert1.test    |  5 +++++
 ext/expert/sqlite3expert.c |  4 ++++
 ext/expert/test_expert.c   |  7 ++++++-
 manifest                   | 18 +++++++++---------
 manifest.uuid              |  2 +-
 5 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/ext/expert/expert1.test b/ext/expert/expert1.test
index 3b3425f07f..3d693ec2e7 100644
--- a/ext/expert/expert1.test
+++ b/ext/expert/expert1.test
@@ -23,6 +23,11 @@ if {![info exists testdir]} {
 source $testdir/tester.tcl
 set testprefix expert1
 
+if {[info commands sqlite3_expert_new]==""} {
+  finish_test
+  return
+}
+
 set CLI [test_binary_name sqlite3]
 set CMD [test_binary_name sqlite3_expert]
 
diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index f347ddea6d..fd09478fb5 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -15,6 +15,8 @@
 #include <string.h>
 #include <stdio.h>
 
+#ifndef SQLITE_OMIT_VIRTUALTABLE 
+
 typedef sqlite3_int64 i64;
 typedef sqlite3_uint64 u64;
 
@@ -1932,3 +1934,5 @@ void sqlite3_expert_destroy(sqlite3expert *p){
     sqlite3_free(p);
   }
 }
+
+#endif /* ifndef SQLITE_OMIT_VIRTUAL_TABLE */
diff --git a/ext/expert/test_expert.c b/ext/expert/test_expert.c
index ad83872f9a..064c1908a9 100644
--- a/ext/expert/test_expert.c
+++ b/ext/expert/test_expert.c
@@ -26,6 +26,8 @@
 #  endif
 #endif
 
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+
 /*
 ** Extract an sqlite3* db handle from the object passed as the second
 ** argument. If successful, set *pDb to point to the db handle and return
@@ -195,7 +197,10 @@ static int SQLITE_TCLAPI test_sqlite3_expert_new(
   return rc;
 }
 
+#endif  /* ifndef SQLITE_OMIT_VIRTUALTABLE */
+
 int TestExpert_Init(Tcl_Interp *interp){
+#ifndef SQLITE_OMIT_VIRTUALTABLE
   struct Cmd {
     const char *zCmd;
     Tcl_ObjCmdProc *xProc;
@@ -208,7 +213,7 @@ int TestExpert_Init(Tcl_Interp *interp){
     struct Cmd *p = &aCmd[i];
     Tcl_CreateObjCommand(interp, p->zCmd, p->xProc, 0, 0);
   }
-
+#endif
   return TCL_OK;
 }
 
diff --git a/manifest b/manifest
index 3e04ddeb45..8fd167235a 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Attempt\sto\sfix\sthe\sfileio.c\sextension\sso\sthat\sit\sbuilds\son\sMinGW-64.
-D 2018-01-09T14:27:58.076
+C Do\snot\sattempt\sto\sbuild\sthe\scode\sin\sext/expert/sqlite3expert.c\sif\nSQLITE_OMIT_VIRTUALTABLE\sis\sdefined.
+D 2018-01-09T14:30:49.074
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -44,10 +44,10 @@ F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74
 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3
 F ext/expert/expert.c 4791c5e064aea81b2b829fa95228b22283380ee370ea88a1e580103b75516ebf
-F ext/expert/expert1.test 0c71a3453ce3a0b4dbe952713aec0ae8d416dd846820dd027b08f305f5278b30
-F ext/expert/sqlite3expert.c 9d352d8693a997402095a16791122ca5c1bff7627faee3625509fcaaef9b30db
+F ext/expert/expert1.test fd21496d8e52c817a7741f467f42b0502c0ac7e07dcdd1d6e15a3e8154ed4e41
+F ext/expert/sqlite3expert.c 9f1b0a5ea34abe1ccd0e9aae6ba5f96865c0f08373a731dd1ed6226074a3d75b
 F ext/expert/sqlite3expert.h af6354f8ee5c9e025024e63fec3bd640a802afcc3099a44d804752cf0791d811
-F ext/expert/test_expert.c 85f5c743a899063fa48296d21de2f32c26d09a21c8582b2a0bc482e8de183e7a
+F ext/expert/test_expert.c d56c194b769bdc90cf829a14c9ecbc1edca9c850b837a4d0b13be14095c32a72
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
 F ext/fts1/ft_hash.c 3927bd880e65329bdc6f506555b228b28924921b
 F ext/fts1/ft_hash.h 06df7bba40dadd19597aa400a875dbc2fed705ea
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5fed67033c9dd4492bf8cfcf98874284581f448d8cc84fa5470dde239f218375
-R 0ecfe311a33b1c3c21c3c128af696033
-U drh
-Z 00bdd39cf3e409343f20bbdc5609780f
+P a7446d3217d39c96c884fbfb294dd320378255f3bfb34e35d15ba6d7c6698f53
+R e70943770a9fc04a23bf01555115b0a6
+U dan
+Z 4badcc0edcd89f4a5802d71ebe94c4d8
diff --git a/manifest.uuid b/manifest.uuid
index ff7756ae2e..272863cff3 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a7446d3217d39c96c884fbfb294dd320378255f3bfb34e35d15ba6d7c6698f53
\ No newline at end of file
+ba967ad2e38590fe2a51e814a087140089be5e6a06a795a958e70a8a47f6350f
\ No newline at end of file

From 6b046be41ac527a19434ed010c8b2627c3cb4931 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 9 Jan 2018 15:25:55 +0000
Subject: [PATCH 272/488] Fix the shell so that it can be built with
 SQLITE_OMIT_VIRTUALTABLE.

FossilOrigin-Name: 931421a22507a5e1edb46e19cb5b3958ec3904807836ab5bcc7a90f37f476e6a
---
 manifest       |  12 ++---
 manifest.uuid  |   2 +-
 src/shell.c.in | 121 ++++++++++++++++++++++++++-----------------------
 3 files changed, 71 insertions(+), 64 deletions(-)

diff --git a/manifest b/manifest
index 8fd167235a..11afed6419 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Do\snot\sattempt\sto\sbuild\sthe\scode\sin\sext/expert/sqlite3expert.c\sif\nSQLITE_OMIT_VIRTUALTABLE\sis\sdefined.
-D 2018-01-09T14:30:49.074
+C Fix\sthe\sshell\sso\sthat\sit\scan\sbe\sbuilt\swith\sSQLITE_OMIT_VIRTUALTABLE.
+D 2018-01-09T15:25:55.758
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -484,7 +484,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in fb615970a6ae95be99316d9e4ab14bd2d5c6ecdc15ff905c4107eadfd2e20d0c
+F src/shell.c.in 2a752aed1dc3727b3845429540543ff7fea383f960c7fa9e3741f16eeaed8686
 F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a7446d3217d39c96c884fbfb294dd320378255f3bfb34e35d15ba6d7c6698f53
-R e70943770a9fc04a23bf01555115b0a6
+P ba967ad2e38590fe2a51e814a087140089be5e6a06a795a958e70a8a47f6350f
+R bfa134ce14c4136d4127894dc7db3a26
 U dan
-Z 4badcc0edcd89f4a5802d71ebe94c4d8
+Z f58bc377dca6569bb98e68d46d49fa3f
diff --git a/manifest.uuid b/manifest.uuid
index 272863cff3..f183a9980d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ba967ad2e38590fe2a51e814a087140089be5e6a06a795a958e70a8a47f6350f
\ No newline at end of file
+931421a22507a5e1edb46e19cb5b3958ec3904807836ab5bcc7a90f37f476e6a
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 8ef236054e..e212510e98 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -2400,6 +2400,7 @@ static void exec_prepared_stmt(
   }
 }
 
+#ifndef SQLITE_OMIT_VIRTUALTABLE
 /*
 ** This function is called to process SQL if the previous shell command
 ** was ".expert". It passes the SQL in the second argument directly to
@@ -2472,6 +2473,63 @@ static int expertFinish(
   return rc;
 }
 
+/*
+** Implementation of ".expert" dot command.
+*/
+static int expertDotCommand(
+  ShellState *pState,             /* Current shell tool state */
+  char **azArg,                   /* Array of arguments passed to dot command */
+  int nArg                        /* Number of entries in azArg[] */
+){
+  int rc = SQLITE_OK;
+  char *zErr = 0;
+  int i;
+  int iSample = 0;
+
+  assert( pState->expert.pExpert==0 );
+  memset(&pState->expert, 0, sizeof(ExpertInfo));
+
+  for(i=1; rc==SQLITE_OK && i<nArg; i++){
+    char *z = azArg[i];
+    int n;
+    if( z[0]=='-' && z[1]=='-' ) z++;
+    n = strlen30(z);
+    if( n>=2 && 0==strncmp(z, "-verbose", n) ){
+      pState->expert.bVerbose = 1;
+    }
+    else if( n>=2 && 0==strncmp(z, "-sample", n) ){
+      if( i==(nArg-1) ){
+        raw_printf(stderr, "option requires an argument: %s\n", z);
+        rc = SQLITE_ERROR;
+      }else{
+        iSample = (int)integerValue(azArg[++i]);
+        if( iSample<0 || iSample>100 ){
+          raw_printf(stderr, "value out of range: %s\n", azArg[i]);
+          rc = SQLITE_ERROR;
+        }
+      }
+    }
+    else{
+      raw_printf(stderr, "unknown option: %s\n", z);
+      rc = SQLITE_ERROR;
+    }
+  }
+
+  if( rc==SQLITE_OK ){
+    pState->expert.pExpert = sqlite3_expert_new(pState->db, &zErr);
+    if( pState->expert.pExpert==0 ){
+      raw_printf(stderr, "sqlite3_expert_new: %s\n", zErr);
+      rc = SQLITE_ERROR;
+    }else{
+      sqlite3_expert_config(
+          pState->expert.pExpert, EXPERT_CONFIG_SAMPLE, iSample
+      );
+    }
+  }
+
+  return rc;
+}
+#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */
 
 /*
 ** Execute a statement or set of statements.  Print
@@ -2499,10 +2557,12 @@ static int shell_exec(
     *pzErrMsg = NULL;
   }
 
+#ifndef SQLITE_OMIT_VIRTUALTABLE
   if( pArg->expert.pExpert ){
     rc = expertHandleSQL(pArg, zSql, pzErrMsg);
     return expertFinish(pArg, (rc!=SQLITE_OK), pzErrMsg);
   }
+#endif
 
   while( zSql[0] && (SQLITE_OK == rc) ){
     static const char *zStmtSql;
@@ -5106,63 +5166,6 @@ static int arDotCommand(
 **********************************************************************************/
 #endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) */
 
-/*
-** Implementation of ".expert" dot command.
-*/
-static int expertDotCommand(
-  ShellState *pState,             /* Current shell tool state */
-  char **azArg,                   /* Array of arguments passed to dot command */
-  int nArg                        /* Number of entries in azArg[] */
-){
-  int rc = SQLITE_OK;
-  char *zErr = 0;
-  int i;
-  int iSample = 0;
-
-  assert( pState->expert.pExpert==0 );
-  memset(&pState->expert, 0, sizeof(ExpertInfo));
-
-  for(i=1; rc==SQLITE_OK && i<nArg; i++){
-    char *z = azArg[i];
-    int n;
-    if( z[0]=='-' && z[1]=='-' ) z++;
-    n = strlen30(z);
-    if( n>=2 && 0==strncmp(z, "-verbose", n) ){
-      pState->expert.bVerbose = 1;
-    }
-    else if( n>=2 && 0==strncmp(z, "-sample", n) ){
-      if( i==(nArg-1) ){
-        raw_printf(stderr, "option requires an argument: %s\n", z);
-        rc = SQLITE_ERROR;
-      }else{
-        iSample = (int)integerValue(azArg[++i]);
-        if( iSample<0 || iSample>100 ){
-          raw_printf(stderr, "value out of range: %s\n", azArg[i]);
-          rc = SQLITE_ERROR;
-        }
-      }
-    }
-    else{
-      raw_printf(stderr, "unknown option: %s\n", z);
-      rc = SQLITE_ERROR;
-    }
-  }
-
-  if( rc==SQLITE_OK ){
-    pState->expert.pExpert = sqlite3_expert_new(pState->db, &zErr);
-    if( pState->expert.pExpert==0 ){
-      raw_printf(stderr, "sqlite3_expert_new: %s\n", zErr);
-      rc = SQLITE_ERROR;
-    }else{
-      sqlite3_expert_config(
-          pState->expert.pExpert, EXPERT_CONFIG_SAMPLE, iSample
-      );
-    }
-  }
-
-  return rc;
-}
-
 
 /*
 ** If an input line begins with "." then invoke this routine to
@@ -5177,9 +5180,11 @@ static int do_meta_command(char *zLine, ShellState *p){
   int rc = 0;
   char *azArg[50];
 
+#ifndef SQLITE_OMIT_VIRTUALTABLE
   if( p->expert.pExpert ){
     expertFinish(p, 1, 0);
   }
+#endif
 
   /* Parse the input line into tokens.
   */
@@ -5544,10 +5549,12 @@ static int do_meta_command(char *zLine, ShellState *p){
     }
   }else
 
+#ifndef SQLITE_OMIT_VIRTUALTABLE
   if( c=='e' && strncmp(azArg[0], "expert", n)==0 ){
     open_db(p, 0);
     expertDotCommand(p, azArg, nArg);
   }else
+#endif
 
   if( c=='f' && strncmp(azArg[0], "fullschema", n)==0 ){
     ShellState data;

From 4297584d66f40b71e71cc1779b08799965124e9f Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 9 Jan 2018 15:28:25 +0000
Subject: [PATCH 273/488] Updates to the README.md file for the ext/misc
 directory.  No code changes.

FossilOrigin-Name: 6b3f4349d7156501f05270d494d6002ee03a0860b6e77b46d17effcab027b753
---
 ext/misc/README.md | 20 ++++++++++++++++++++
 manifest           | 14 +++++++-------
 manifest.uuid      |  2 +-
 3 files changed, 28 insertions(+), 8 deletions(-)

diff --git a/ext/misc/README.md b/ext/misc/README.md
index 970a09c2e9..69cb230255 100644
--- a/ext/misc/README.md
+++ b/ext/misc/README.md
@@ -14,11 +14,20 @@ as follows:
      It is a good example of how to go about implementing a custom
      [table-valued function](https://www.sqlite.org/vtab.html#tabfunc2).
 
+  *  **csv.c** &mdash;  A [virtual table](https://sqlite.org/vtab.html)
+     for reading 
+     [Comma-Separated-Value (CSV) files](https://en.wikipedia.org/wiki/Comma-separated_values).
+
   *  **dbdump.c** &mdash;  This is not actually a loadable extension, but
      rather a library that implements an approximate equivalent to the
      ".dump" command of the
      [command-line shell](https://www.sqlite.org/cli.html).
 
+  *  **json1.c** &mdash;  Various SQL functions and table-valued functions
+     for processing JSON.  This extension is already built into the
+     [SQLite amalgamation](https://sqlite.org/amalgamation.html).  See
+     <https://sqlite.org/json1.html> for additional information.
+
   *  **memvfs.c** &mdash;  This file implements a custom
      [VFS](https://www.sqlite.org/vfs.html) that stores an entire database
      file in a single block of RAM.  It serves as a good example of how
@@ -38,3 +47,14 @@ as follows:
      on the source filename with digits removed, so if we used the name
      "sha3.c" then the entry point would conflict with the prior "sha1.c"
      extension.
+
+  *  **unionvtab.c** &mdash; Implementation of the unionvtab and
+     [swarmvtab](https://sqlite.org/swarmvtab.html) virtual tables.
+     These virtual tables allow a single
+     large table to be spread out across multiple database files.  In the
+     case of swarmvtab, the individual database files can be attached on
+     demand.
+
+  *  **zipfile.c** &mdash;  A [virtual table](https://sqlite.org/vtab.html)
+     that can read and write a 
+     [ZIP archive](https://en.wikipedia.org/wiki/Zip_%28file_format%29).
diff --git a/manifest b/manifest
index 11afed6419..831d360ffb 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sthe\sshell\sso\sthat\sit\scan\sbe\sbuilt\swith\sSQLITE_OMIT_VIRTUALTABLE.
-D 2018-01-09T15:25:55.758
+C Updates\sto\sthe\sREADME.md\sfile\sfor\sthe\sext/misc\sdirectory.\s\sNo\scode\schanges.
+D 2018-01-09T15:28:25.708
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -265,7 +265,7 @@ F ext/lsm1/lsm_win32.c 0a4acbd7e8d136dd3a5753f0a9e7a9802263a9d96cef3278cf120bcaa
 F ext/lsm1/test/lsm1_common.tcl 5ed4bab07c93be2e4f300ebe46007ecf4b3e20bc5fbe1dedaf04a8774a6d8d82
 F ext/lsm1/test/lsm1_simple.test ca949efefa102f4644231dcd9291d8cda7699a4ce1006b26e0e3fcb72233f422
 F ext/lsm1/tool/mklsm1c.tcl f31561bbee5349f0a554d1ad7236ac1991fc09176626f529f6078e07335398b0
-F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601d90b2
+F ext/misc/README.md d6dd0fe1d8af77040216798a6a2b0c46c73054d2f0ea544fbbcdccf6f238c240
 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87
 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb
 F ext/misc/appendvfs.c 4c65f0b79686ae5a483134233d7fd912f0f2d4fd76023404f96f2290fff13b19
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ba967ad2e38590fe2a51e814a087140089be5e6a06a795a958e70a8a47f6350f
-R bfa134ce14c4136d4127894dc7db3a26
-U dan
-Z f58bc377dca6569bb98e68d46d49fa3f
+P 931421a22507a5e1edb46e19cb5b3958ec3904807836ab5bcc7a90f37f476e6a
+R d817db25938f9e9b2bf2929fb858f109
+U drh
+Z 5854e586ea789fbe92ed499c4878d58a
diff --git a/manifest.uuid b/manifest.uuid
index f183a9980d..b72d3fe4f4 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-931421a22507a5e1edb46e19cb5b3958ec3904807836ab5bcc7a90f37f476e6a
\ No newline at end of file
+6b3f4349d7156501f05270d494d6002ee03a0860b6e77b46d17effcab027b753
\ No newline at end of file

From e2d22ffaa34b145013311f9f4061037dc239c1b3 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 9 Jan 2018 15:57:55 +0000
Subject: [PATCH 274/488] Fix another -fsanitize issue in new shell code.

FossilOrigin-Name: 6ee7b54c580de0240e0f49a5fd2d11ddcdbcce8a2ae4b249618d0ab2d54b5e5b
---
 manifest       | 14 +++++++-------
 manifest.uuid  |  2 +-
 src/shell.c.in |  2 ++
 3 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 831d360ffb..b647cd36af 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Updates\sto\sthe\sREADME.md\sfile\sfor\sthe\sext/misc\sdirectory.\s\sNo\scode\schanges.
-D 2018-01-09T15:28:25.708
+C Fix\sanother\s-fsanitize\sissue\sin\snew\sshell\scode.
+D 2018-01-09T15:57:55.704
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -484,7 +484,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in 2a752aed1dc3727b3845429540543ff7fea383f960c7fa9e3741f16eeaed8686
+F src/shell.c.in 4121ecd9a812c9032726aa137a4504251d03f2001dd6c7c110e7cabd3ee1454b
 F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 931421a22507a5e1edb46e19cb5b3958ec3904807836ab5bcc7a90f37f476e6a
-R d817db25938f9e9b2bf2929fb858f109
-U drh
-Z 5854e586ea789fbe92ed499c4878d58a
+P 6b3f4349d7156501f05270d494d6002ee03a0860b6e77b46d17effcab027b753
+R c9d51344455c4168d42531ca44edf723
+U dan
+Z df7d69cf286a4b0474c2af6fdf8505cd
diff --git a/manifest.uuid b/manifest.uuid
index b72d3fe4f4..f31db62ff1 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6b3f4349d7156501f05270d494d6002ee03a0860b6e77b46d17effcab027b753
\ No newline at end of file
+6ee7b54c580de0240e0f49a5fd2d11ddcdbcce8a2ae4b249618d0ab2d54b5e5b
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index e212510e98..d214850aa9 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -2473,6 +2473,8 @@ static int expertFinish(
   return rc;
 }
 
+static sqlite3_int64 integerValue(const char *zArg);
+
 /*
 ** Implementation of ".expert" dot command.
 */

From c48e0271f63cce3a122b4e94f95efc5c3a02556a Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 9 Jan 2018 18:31:11 +0000
Subject: [PATCH 275/488] Fix a potential crash in sqlite3expert.c caused by a
 missing sqlite3_reset() call.

FossilOrigin-Name: f6355970f04a3c4d85801c806366788139d8d5ad29383ec7faa81e2176ffe987
---
 ext/expert/sqlite3expert.c |  1 +
 manifest                   | 18 +++++++++---------
 manifest.uuid              |  2 +-
 test/colname.test          | 10 ++++++----
 test/shell8.test           |  4 ++++
 test/zipfile.test          |  3 +++
 6 files changed, 24 insertions(+), 14 deletions(-)

diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index fd09478fb5..9cafc448b2 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -1525,6 +1525,7 @@ static int idxPopulateOneStat1(
     );
     zOrder = idxAppendText(&rc, zOrder, "%s%d", zComma, ++nCol);
   }
+  sqlite3_reset(pIndexXInfo);
   if( rc==SQLITE_OK ){
     if( p->iSample==100 ){
       zQuery = sqlite3_mprintf(
diff --git a/manifest b/manifest
index b647cd36af..9bd75e6383 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sanother\s-fsanitize\sissue\sin\snew\sshell\scode.
-D 2018-01-09T15:57:55.704
+C Fix\sa\spotential\scrash\sin\ssqlite3expert.c\scaused\sby\sa\smissing\ssqlite3_reset()\ncall.
+D 2018-01-09T18:31:11.283
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -45,7 +45,7 @@ F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3
 F ext/expert/expert.c 4791c5e064aea81b2b829fa95228b22283380ee370ea88a1e580103b75516ebf
 F ext/expert/expert1.test fd21496d8e52c817a7741f467f42b0502c0ac7e07dcdd1d6e15a3e8154ed4e41
-F ext/expert/sqlite3expert.c 9f1b0a5ea34abe1ccd0e9aae6ba5f96865c0f08373a731dd1ed6226074a3d75b
+F ext/expert/sqlite3expert.c 55ea02e9fcc014f4252e8a8c78d2bddc6d7ef62f9cb54ee283a659516eabb711
 F ext/expert/sqlite3expert.h af6354f8ee5c9e025024e63fec3bd640a802afcc3099a44d804752cf0791d811
 F ext/expert/test_expert.c d56c194b769bdc90cf829a14c9ecbc1edca9c850b837a4d0b13be14095c32a72
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
@@ -687,7 +687,7 @@ F test/collate9.test 3adcc799229545940df2f25308dd1ad65869145a
 F test/collateA.test b8218ab90d1fa5c59dcf156efabb1b2599c580d6
 F test/collateB.test 1e68906951b846570f29f20102ed91d29e634854ee47454d725f2151ecac0b95
 F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1
-F test/colname.test 101aa39392a1f6883278f588836a3ab99178f8103f78032433400475cc05109f
+F test/colname.test fb28b3687e03625425bc216edf8b186ce974aa71008e2aa1f426a7dcb75a601d
 F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db
 F test/conflict2.test bb0b94cf7196c64a3cbd815c66d3ee98c2fecd9c
 F test/conflict3.test a83db76a6c3503b2fa057c7bfb08c318d8a422202d8bc5b86226e078e5b49ff9
@@ -1225,7 +1225,7 @@ F test/shell4.test 89ad573879a745974ff2df20ff97c5d6ffffbd5d
 F test/shell5.test 23939a4c51f0421330ea61dbd3c74f9c215f5f8d3d1a94846da6ffc777a35458
 F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3
 F test/shell7.test 115132f66d0463417f408562cc2cf534f6bbc6d83a6d50f0072a9eb171bae97f
-F test/shell8.test 7585847402452d594f0e5f93430d34ed63b2f34ca7e956f63db157f9327c6896
+F test/shell8.test 5a1f2c6d5cc11e7c7d69e960972f447a9b01e80c419ff9ebaf059f94fe5b3ab9
 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3
 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5
 F test/shrink.test 1b4330b1fd9e818c04726d45cb28db73087535ce
@@ -1598,7 +1598,7 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf
 F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc
 F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa
 F test/zerodamage.test e59a56443d6298ecf7435f618f0b27654f0c849e
-F test/zipfile.test de2ee377705999bcd7391fb742d6b833ffab4f21dab1d4f484098a66d60eb4fb
+F test/zipfile.test 2f71a254a9c0fe84362156a8e8cb85188280b95bfc64a26da9179ef5da8557e0
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d
 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6b3f4349d7156501f05270d494d6002ee03a0860b6e77b46d17effcab027b753
-R c9d51344455c4168d42531ca44edf723
+P 6ee7b54c580de0240e0f49a5fd2d11ddcdbcce8a2ae4b249618d0ab2d54b5e5b
+R dc296443bd383bb63a4f4ed70dfa8f45
 U dan
-Z df7d69cf286a4b0474c2af6fdf8505cd
+Z 3dfe6831182d02544e29db133db41f01
diff --git a/manifest.uuid b/manifest.uuid
index f31db62ff1..7a9d58cc88 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6ee7b54c580de0240e0f49a5fd2d11ddcdbcce8a2ae4b249618d0ab2d54b5e5b
\ No newline at end of file
+f6355970f04a3c4d85801c806366788139d8d5ad29383ec7faa81e2176ffe987
\ No newline at end of file
diff --git a/test/colname.test b/test/colname.test
index 358ea77e07..f314f94f6e 100644
--- a/test/colname.test
+++ b/test/colname.test
@@ -393,10 +393,12 @@ do_execsql_test colname-9.300 {
 do_test colname-9.310 {
   execsql2 {SELECT BBb FROM (SELECT aaa AS Bbb FROM t1)}
 } {Bbb 123}
-do_execsql_test colname-9.320 {
-  CREATE TABLE t2 AS SELECT BBb FROM (SELECT aaa AS Bbb FROM t1);
-  SELECT name FROM pragma_table_info('t2');
-} {Bbb}
+ifcapable vtab {
+  do_execsql_test colname-9.320 {
+    CREATE TABLE t2 AS SELECT BBb FROM (SELECT aaa AS Bbb FROM t1);
+    SELECT name FROM pragma_table_info('t2');
+  } {Bbb}
+}
 
 # Issue detected by OSSFuzz on 2017-12-24 (Christmas Eve)
 # caused by check-in https://sqlite.org/src/info/6b2ff26c25
diff --git a/test/shell8.test b/test/shell8.test
index 50e269ef3e..0dcb10d67b 100644
--- a/test/shell8.test
+++ b/test/shell8.test
@@ -15,6 +15,10 @@
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 set testprefix shell8
+
+ifcapable !vtab {
+  finish_test; return
+}
 set CLI [test_find_cli]
 
 proc populate_dir {dirname spec} {
diff --git a/test/zipfile.test b/test/zipfile.test
index 4cb6630e09..9fa38810ff 100644
--- a/test/zipfile.test
+++ b/test/zipfile.test
@@ -14,6 +14,9 @@ set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 set testprefix zipfile
 
+ifcapable !vtab {
+  finish_test; return
+}
 if {[catch {load_static_extension db zipfile} error]} {
   puts "Skipping zipfile tests, hit load error: $error"
   finish_test; return

From 9898c4a0c4ebe63f4ce5bacc8e133c83240c06e6 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 9 Jan 2018 20:34:53 +0000
Subject: [PATCH 276/488] In the os_unix.c implementation of xOpen(), do not
 return SQLITE_READONLY_DIRECTORY in cases where the file cannot be opened for
 reasons other than a readonly directory, such as the process running out of
 file descriptors.

FossilOrigin-Name: fa8b80bb967792de99808712ac03e37ace0f11eb8fbe444aacd3d19184c425ea
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/os_unix.c |  4 +++-
 3 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 9bd75e6383..53754a8904 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\spotential\scrash\sin\ssqlite3expert.c\scaused\sby\sa\smissing\ssqlite3_reset()\ncall.
-D 2018-01-09T18:31:11.283
+C In\sthe\sos_unix.c\simplementation\sof\sxOpen(),\sdo\snot\sreturn\nSQLITE_READONLY_DIRECTORY\sin\scases\swhere\sthe\sfile\scannot\sbe\sopened\sfor\sreasons\nother\sthan\sa\sreadonly\sdirectory,\ssuch\sas\sthe\sprocess\srunning\sout\sof\sfile\ndescriptors.
+D 2018-01-09T20:34:53.247
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -467,7 +467,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104
 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
-F src/os_unix.c 7fc2735390a7809d5d893ed735d994ff12521224b89738226fff6f1a0aa1c932
+F src/os_unix.c a9edddcc35d664c8247a18abd10d73b7a80c0d897a5341de8feea5a47cd57e25
 F src/os_win.c 0a4afa35cc8e812000df3ea2f64b476131b39e29e75d8007d0504726e4761de4
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 9b9cb4e06c03d43d62480a7a685a012d645fcf3a39e7767ccb505fb41ee083ec
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6ee7b54c580de0240e0f49a5fd2d11ddcdbcce8a2ae4b249618d0ab2d54b5e5b
-R dc296443bd383bb63a4f4ed70dfa8f45
+P f6355970f04a3c4d85801c806366788139d8d5ad29383ec7faa81e2176ffe987
+R d7d90fd061094f6c861c9386a57e031e
 U dan
-Z 3dfe6831182d02544e29db133db41f01
+Z 2d88017ad0f4c9b72ee8a5b92f98fadf
diff --git a/manifest.uuid b/manifest.uuid
index 7a9d58cc88..0259e08e45 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f6355970f04a3c4d85801c806366788139d8d5ad29383ec7faa81e2176ffe987
\ No newline at end of file
+fa8b80bb967792de99808712ac03e37ace0f11eb8fbe444aacd3d19184c425ea
\ No newline at end of file
diff --git a/src/os_unix.c b/src/os_unix.c
index 2d377ef56a..3b2b2e2a59 100644
--- a/src/os_unix.c
+++ b/src/os_unix.c
@@ -5917,7 +5917,9 @@ static int unixOpen(
       rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zName);
       /* If unable to create a journal, change the error code to
       ** indicate that the directory permissions are wrong. */
-      if( isNewJrnl && osAccess(zName, F_OK) ) rc = SQLITE_READONLY_DIRECTORY;
+      if( isNewJrnl && errno==EACCES && osAccess(zName, F_OK) ){
+        rc = SQLITE_READONLY_DIRECTORY;
+      }
       goto open_finished;
     }
 

From 2c4df97591568c0f5e0e14a59db7549717a3c481 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 9 Jan 2018 20:44:04 +0000
Subject: [PATCH 277/488] Return SQLITE_CONSTRAINT if a user attempts to update
 a zipfile table.

FossilOrigin-Name: 64c9ccf6c5598a68135e78dad128ec070afa49416e1d1f35745a24276e7aeeea
---
 ext/misc/zipfile.c | 18 +++++++++++-------
 manifest           | 14 +++++++-------
 manifest.uuid      |  2 +-
 test/zipfile.test  | 10 ++++++++++
 4 files changed, 29 insertions(+), 15 deletions(-)

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index 93b2a6d8bc..e4de24af76 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -1221,15 +1221,19 @@ static int zipfileUpdate(
   assert( pTab->pWriteFd );
 
   if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
-    i64 iDelete = sqlite3_value_int64(apVal[0]);
-    ZipfileEntry *p;
-    for(p=pTab->pFirstEntry; p; p=p->pNext){
-      if( p->iRowid==iDelete ){
-        p->bDeleted = 1;
-        break;
+    if( nVal>1 ){
+      return SQLITE_CONSTRAINT;
+    }else{
+      i64 iDelete = sqlite3_value_int64(apVal[0]);
+      ZipfileEntry *p;
+      for(p=pTab->pFirstEntry; p; p=p->pNext){
+        if( p->iRowid==iDelete ){
+          p->bDeleted = 1;
+          break;
+        }
       }
+      return SQLITE_OK;
     }
-    if( nVal==1 ) return SQLITE_OK;
   }
 
   mNull = (sqlite3_value_type(apVal[5])==SQLITE_NULL ? 0x0 : 0x8)  /* sz */
diff --git a/manifest b/manifest
index 53754a8904..2c69e7fdca 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\sthe\sos_unix.c\simplementation\sof\sxOpen(),\sdo\snot\sreturn\nSQLITE_READONLY_DIRECTORY\sin\scases\swhere\sthe\sfile\scannot\sbe\sopened\sfor\sreasons\nother\sthan\sa\sreadonly\sdirectory,\ssuch\sas\sthe\sprocess\srunning\sout\sof\sfile\ndescriptors.
-D 2018-01-09T20:34:53.247
+C Return\sSQLITE_CONSTRAINT\sif\sa\suser\sattempts\sto\supdate\sa\szipfile\stable.
+D 2018-01-09T20:44:04.801
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -303,7 +303,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c eaab322dd4b24199b0c6bdfbf866d1d04f7e93f46b4b5ca8436edadece54b03f
+F ext/misc/zipfile.c be93d2bf946d0ded56cdc3e8ec5b67ebfbcc9da435860a3e46db5a61025dbed7
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -1598,7 +1598,7 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf
 F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc
 F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa
 F test/zerodamage.test e59a56443d6298ecf7435f618f0b27654f0c849e
-F test/zipfile.test 2f71a254a9c0fe84362156a8e8cb85188280b95bfc64a26da9179ef5da8557e0
+F test/zipfile.test 9fb98a24f80fe0d5d09df15cd01bb290777572f45408fdfbe894f2413c9c1222
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d
 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P f6355970f04a3c4d85801c806366788139d8d5ad29383ec7faa81e2176ffe987
-R d7d90fd061094f6c861c9386a57e031e
+P fa8b80bb967792de99808712ac03e37ace0f11eb8fbe444aacd3d19184c425ea
+R 2995a8a6bc93b1ef1a6dc9c0ab114acb
 U dan
-Z 2d88017ad0f4c9b72ee8a5b92f98fadf
+Z 9abd730978f6c4ddfe88986e0606c7d3
diff --git a/manifest.uuid b/manifest.uuid
index 0259e08e45..d0b70fca41 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-fa8b80bb967792de99808712ac03e37ace0f11eb8fbe444aacd3d19184c425ea
\ No newline at end of file
+64c9ccf6c5598a68135e78dad128ec070afa49416e1d1f35745a24276e7aeeea
\ No newline at end of file
diff --git a/test/zipfile.test b/test/zipfile.test
index 9fa38810ff..2a4f9ad96a 100644
--- a/test/zipfile.test
+++ b/test/zipfile.test
@@ -102,6 +102,16 @@ do_execsql_test 2.2 {
   dirname2/file1.txt 33188 abcdefghijklmnop
 }
 
+do_catchsql_test 2.3 {
+  UPDATE zzz SET name = 'dirname3' WHERE name = 'dirname';
+} {1 {constraint failed}}
+do_execsql_test 2.4 {
+  SELECT name, mode, data FROM zzz;
+} {
+  dirname 16877 {}
+  dirname2 16877 {}
+  dirname2/file1.txt 33188 abcdefghijklmnop
+}
 
 
 finish_test

From 7026bd67531ab5fb219ea928be456a9dee9e0937 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Tue, 9 Jan 2018 20:49:33 +0000
Subject: [PATCH 278/488] Fix harmless compiler warnings.

FossilOrigin-Name: 0fb42090cb2c785e044abe273a00da134927db191fea7b0c67ba4028944bce3a
---
 ext/misc/zipfile.c |  8 ++++----
 manifest           | 14 +++++++-------
 manifest.uuid      |  2 +-
 3 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index e4de24af76..9a8d3974c2 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -1202,11 +1202,11 @@ static int zipfileUpdate(
   int rc = SQLITE_OK;             /* Return Code */
   ZipfileEntry *pNew = 0;         /* New in-memory CDS entry */
 
-  u32 mode;                       /* Mode for new entry */
-  i64 mTime;                      /* Modification time for new entry */
+  u32 mode = 0;                   /* Mode for new entry */
+  i64 mTime = 0;                  /* Modification time for new entry */
   i64 sz = 0;                     /* Uncompressed size */
-  const char *zPath;              /* Path for new entry */
-  int nPath;                      /* strlen(zPath) */
+  const char *zPath = 0;          /* Path for new entry */
+  int nPath = 0;                  /* strlen(zPath) */
   const u8 *pData = 0;            /* Pointer to buffer containing content */
   int nData = 0;                  /* Size of pData buffer in bytes */
   int iMethod = 0;                /* Compression method for new entry */
diff --git a/manifest b/manifest
index 2c69e7fdca..d00cf8ce0e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Return\sSQLITE_CONSTRAINT\sif\sa\suser\sattempts\sto\supdate\sa\szipfile\stable.
-D 2018-01-09T20:44:04.801
+C Fix\sharmless\scompiler\swarnings.
+D 2018-01-09T20:49:33.189
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -303,7 +303,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c be93d2bf946d0ded56cdc3e8ec5b67ebfbcc9da435860a3e46db5a61025dbed7
+F ext/misc/zipfile.c 92b840dc168126e192da7d6976f44b4453c40b27ad8f142a7c02dc31aaa31bd8
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P fa8b80bb967792de99808712ac03e37ace0f11eb8fbe444aacd3d19184c425ea
-R 2995a8a6bc93b1ef1a6dc9c0ab114acb
-U dan
-Z 9abd730978f6c4ddfe88986e0606c7d3
+P 64c9ccf6c5598a68135e78dad128ec070afa49416e1d1f35745a24276e7aeeea
+R ec154d3d05b58458e2848c5eff3a9b08
+U mistachkin
+Z cd319f239e5515c89a36567aa5d72543
diff --git a/manifest.uuid b/manifest.uuid
index d0b70fca41..2efa251344 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-64c9ccf6c5598a68135e78dad128ec070afa49416e1d1f35745a24276e7aeeea
\ No newline at end of file
+0fb42090cb2c785e044abe273a00da134927db191fea7b0c67ba4028944bce3a
\ No newline at end of file

From dc006e05aa8ed9181b14924aa742e072dabba79a Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Tue, 9 Jan 2018 22:23:42 +0000
Subject: [PATCH 279/488] When disconnecting from the 'swarmvtab' extension,
 close each database prior to invoking the 'openclose' function on it.

FossilOrigin-Name: 3e5647cb6c4553683e24b9cb62548f16c79c4e2ac9e39cf135ea52a623f7cc33
---
 ext/misc/unionvtab.c |  5 +++--
 manifest             | 14 +++++++-------
 manifest.uuid        |  2 +-
 test/swarmvtab3.test | 19 ++++++++++---------
 4 files changed, 21 insertions(+), 19 deletions(-)

diff --git a/ext/misc/unionvtab.c b/ext/misc/unionvtab.c
index 92d0b833c4..94a5c8f013 100644
--- a/ext/misc/unionvtab.c
+++ b/ext/misc/unionvtab.c
@@ -484,14 +484,15 @@ static int unionDisconnect(sqlite3_vtab *pVtab){
     int i;
     for(i=0; i<pTab->nSrc; i++){
       UnionSrc *pSrc = &pTab->aSrc[i];
-      if( pSrc->db ){
+      int bHaveSrcDb = (pSrc->db!=0);
+      sqlite3_close(pSrc->db);
+      if( bHaveSrcDb ){
         unionInvokeOpenClose(pTab, pSrc, 1, 0);
       }
       sqlite3_free(pSrc->zDb);
       sqlite3_free(pSrc->zTab);
       sqlite3_free(pSrc->zFile);
       sqlite3_free(pSrc->zContext);
-      sqlite3_close(pSrc->db);
     }
     sqlite3_finalize(pTab->pNotFound);
     sqlite3_finalize(pTab->pOpenClose);
diff --git a/manifest b/manifest
index d00cf8ce0e..6b48ede1ff 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sharmless\scompiler\swarnings.
-D 2018-01-09T20:49:33.189
+C When\sdisconnecting\sfrom\sthe\s'swarmvtab'\sextension,\sclose\seach\sdatabase\sprior\sto\sinvoking\sthe\s'openclose'\sfunction\son\sit.
+D 2018-01-09T22:23:42.971
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -297,7 +297,7 @@ F ext/misc/spellfix.c 41cf26c6b89fcaa8798ae10ae64d39c1f1d9d6995152e545bd491c1305
 F ext/misc/sqlar.c 57d5bc45cd5492208e451f697404be88f8612527d64c9d42f96b325b64983d74
 F ext/misc/stmt.c 6f16443abb3551e3f5813bb13ba19a30e7032830015b0f92fe0c0453045c0a11
 F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512
-F ext/misc/unionvtab.c 2aa94902ea646e1aaf6c05eac944a14276cddd67735b2ad856030ffffbb6626c
+F ext/misc/unionvtab.c 0b3173f69b8899da640a13a345dc5ef1400199405f738abe6145b2454195b8ff
 F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95
 F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
@@ -1275,7 +1275,7 @@ F test/subtype1.test 7fe09496352f97053af1437150751be2d0a0cae8
 F test/superlock.test ec94f0556b6488d97f71c79f9061ae08d9ab8f12
 F test/swarmvtab.test 9a3fd5ab3e9b3c976ad1b3d7646aab725114f2ac26b59395d0778b33bab6cdaf
 F test/swarmvtab2.test c948cb2fdfc5b01d85e8f6d6504854202dc1a0782ab2a0ed61538f27cbd0aa5c
-F test/swarmvtab3.test c4c8d09e56ae99b90187ac225458f13f373873ea296fc442c7ad7511f25e7314
+F test/swarmvtab3.test 6cb664669630fcec4102a09333e52068734858fd2761eee3b0465c14cdbcee29
 F test/swarmvtabfault.test 00aec54665909490f5c383f3cae3b5d18bd97c12490b429ff8752a3027acfa42
 F test/symlink.test c9ebe7330d228249e447038276bfc8a7b22f4849
 F test/sync.test 2f84bdbc2b2df1fcb0220575b4b9f8cea94b7529
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 64c9ccf6c5598a68135e78dad128ec070afa49416e1d1f35745a24276e7aeeea
-R ec154d3d05b58458e2848c5eff3a9b08
+P 0fb42090cb2c785e044abe273a00da134927db191fea7b0c67ba4028944bce3a
+R 6a5a34e76be8657e78b0d58d3645af35
 U mistachkin
-Z cd319f239e5515c89a36567aa5d72543
+Z b0c2e831274b963bd4ec05162b124f71
diff --git a/manifest.uuid b/manifest.uuid
index 2efa251344..c9c91c407d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0fb42090cb2c785e044abe273a00da134927db191fea7b0c67ba4028944bce3a
\ No newline at end of file
+3e5647cb6c4553683e24b9cb62548f16c79c4e2ac9e39cf135ea52a623f7cc33
\ No newline at end of file
diff --git a/test/swarmvtab3.test b/test/swarmvtab3.test
index 70d6c7dca0..6e52355d95 100644
--- a/test/swarmvtab3.test
+++ b/test/swarmvtab3.test
@@ -74,7 +74,7 @@ proc check_dbcache {} {
   for {set i 0} {$i<100} {incr i} {
     set exists [file exists test.db$i]
     if {$exists!=($::dbcache(test.db$i)!=0)} {
-      error "inconsistent ::dbcache and disk"
+      error "inconsistent ::dbcache and disk ($i) - $exists"
     }
     incr n $exists
   }
@@ -114,7 +114,7 @@ foreach {tn nMaxOpen cvt} {
 
 } {
   execsql { DROP TABLE IF EXISTS s }
-  
+
   do_execsql_test 1.$tn.1 $cvt
 
   do_execsql_test 1.$tn.2 {
@@ -192,7 +192,7 @@ proc check_dbcache {} {
   foreach k [array names ::dbcache] {
     set exists [file exists $k]
     if {$exists!=($::dbcache($k)!=0)} {
-      error "inconsistent ::dbcache and disk ($k)"
+      error "inconsistent ::dbcache and disk ($k) - $exists"
     }
     incr n $exists
   }
@@ -210,23 +210,24 @@ foreach {tn nMaxOpen cvt} {
   }
 } {
   execsql { DROP TABLE IF EXISTS s }
-  
-  do_execsql_test 1.$tn.1 $cvt
 
-  do_execsql_test 1.$tn.2 {
+  do_execsql_test 3.$tn.1 $cvt
+
+  do_execsql_test 3.$tn.2 {
     SELECT b FROM s WHERE a<10;
   } {0 1 2 3 4 5 6 7 8 9}
 
-  do_test 1.$tn.3 { check_dbcache } $nMaxOpen
+  do_test 3.$tn.3 { check_dbcache } $nMaxOpen
 
-  do_execsql_test 1.$tn.4 {
+  do_execsql_test 3.$tn.4 {
     SELECT b FROM s WHERE (b%10)=0;
   } {0 10 20 30 40 50 60 70 80 90}
 
-  do_test 1.$tn.5 { check_dbcache } $nMaxOpen
+  do_test 3.$tn.5 { check_dbcache } $nMaxOpen
 }
 
 db close
+forcedelete {*}[glob test.db*]
 forcedelete {*}[glob test_remote.db*]
 
 finish_test

From 6116ee4eeefeec9a0fc81ef4784cb774d50e36a2 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 10 Jan 2018 00:40:06 +0000
Subject: [PATCH 280/488] Compute the correct column name even if the column
 identifier is the very last token in the SQL statement.  This fixes a problem
 introduced by check-in [0fdf97efe5df745510c6b] and reported by the community
 during beta-testing.

FossilOrigin-Name: 36b89d728ff13d395fe0e1db8e7c01263f73dccb278b3ece27f6ef78e909b492
---
 manifest          | 32 ++++++++++++++++----------------
 manifest.uuid     |  2 +-
 src/build.c       |  2 +-
 src/parse.y       |  7 +++++--
 src/tokenize.c    |  2 +-
 test/capi2.test   |  2 +-
 test/capi3.test   | 12 ++++++++++++
 test/icu.test     |  4 ++--
 test/main.test    |  4 ++--
 test/select1.test |  2 +-
 test/shell3.test  |  4 ++--
 test/with2.test   |  2 +-
 12 files changed, 45 insertions(+), 30 deletions(-)

diff --git a/manifest b/manifest
index 6b48ede1ff..a8883b6f20 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C When\sdisconnecting\sfrom\sthe\s'swarmvtab'\sextension,\sclose\seach\sdatabase\sprior\sto\sinvoking\sthe\s'openclose'\sfunction\son\sit.
-D 2018-01-09T22:23:42.971
+C Compute\sthe\scorrect\scolumn\sname\seven\sif\sthe\scolumn\sidentifier\sis\sthe\nvery\slast\stoken\sin\sthe\sSQL\sstatement.\s\sThis\sfixes\sa\sproblem\sintroduced\nby\scheck-in\s[0fdf97efe5df745510c6b]\sand\sreported\sby\sthe\scommunity\sduring\nbeta-testing.
+D 2018-01-10T00:40:06.636
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -428,7 +428,7 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
 F src/btree.c 0a1f63b50ab1ac5d4b1637c30cb1ae123fbc162ec8cb6336ddb9491a0bc1e363
 F src/btree.h 0866c0a08255142ea0e754aabd211c843cab32045c978a592a43152405ed0c84
 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc
-F src/build.c a03eb5a1cfff74784c24a4478ba5455711571936f1ac9d43f94fa7df57509977
+F src/build.c 9f9647454f236cab097f266ae970f899b53c71cadab6756c47e2b2e81392c2a1
 F src/callback.c fe677cb5f5abb02f7a772a62a98c2f516426081df68856e8f2d5f950929b966a
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
@@ -472,7 +472,7 @@ F src/os_win.c 0a4afa35cc8e812000df3ea2f64b476131b39e29e75d8007d0504726e4761de4
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 9b9cb4e06c03d43d62480a7a685a012d645fcf3a39e7767ccb505fb41ee083ec
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
-F src/parse.y 44cbbc3e132ea128258eff7be7f6d5c5dfa25522f89ec8b5501276966511bd50
+F src/parse.y 4e750e1b261ff9f1d0b6b5d40a829c66d691899f48953fde839d8b52d41aa148
 F src/pcache.c 7ae91a4557a43d77d449accbfdc68846e6516f8e2eda46e8bbe4536fb669b201
 F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170
 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
@@ -545,7 +545,7 @@ F src/test_windirent.c a895e2c068a06644eef91a7f0a32182445a893b9a0f33d0cdb4283dca
 F src/test_windirent.h 8782864172ba5ae52c5c313c70faeadb324ff74de9c3dcc6b56a557dccaa1de6
 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
-F src/tokenize.c 1003d6d90c6783206c711f0a9397656fa5b055209f4d092caa43bb3bf5215db5
+F src/tokenize.c 5b0c661a85f783d35b9883830736eeb63be4aefc4f6b7d9cd081d48782c041e2
 F src/treeview.c eae35972ff44f67064de2eaf35f04afe94e7aea3271a8b3bcebb3f954880fec3
 F src/trigger.c a34539c69433276d37b0da9a89c117726ff2d292c0902895af1f393a983cd3a1
 F src/update.c 961bd1265d4d1e5cd65c9a54fa5122fb7aefcb003fcf2de0c092fceb7e58972c
@@ -663,8 +663,8 @@ F test/busy.test 510dc6daaad18bcbbc085bcc6217d6dc418def5e73f72ce1475eea0cb783472
 F test/cache.test 13bc046b26210471ca6f2889aceb1ea52dc717de
 F test/cacheflush.test af25bb1509df04c1da10e38d8f322d66eceedf61
 F test/cachespill.test 895997f84a25b323b166aecb69baab2d6380ea98f9e0bcc688c4493c535cfab9
-F test/capi2.test d3267a146df2251e7ad09a672d59e33e836d74c6fbeb843c18e87835ef646c12
-F test/capi3.test 986e57cea8ab423b3fc8c2e3b69330394252d3d2a4496122ff3749e258305695
+F test/capi2.test 34a1a9a96d543a2ec2c209696b11b164444f57253b1f2cba1c2e53fadede6c7b
+F test/capi3.test 36f5c859c91a9bb0075d6ddcfcf2476cad629b09f7bfd135776fb94b06c04706
 F test/capi3b.test efb2b9cfd127efa84433cd7a2d72ce0454ae0dc4
 F test/capi3c.test 7ebed1d8fa2f3190149d556fe8cff5a006be62af437c5c4640db614470126098
 F test/capi3d.test 485048dc5cd07bc68011e4917ad035ad6047ab82
@@ -945,7 +945,7 @@ F test/hexlit.test 4a6a5f46e3c65c4bf1fa06f5dd5a9507a5627751
 F test/hidden.test 23c1393a79e846d68fd902d72c85d5e5dcf98711
 F test/hook.test dbc0b87756e1e20e7497b56889c9e9cd2f8cc2b5
 F test/hook2.test b9ff3b8c6519fb67f33192f1afe86e7782ee4ac8
-F test/icu.test 7fb00edc09e05d51e36be12b33e0af04e3394e3b02dbdcb2eefcb901203e28c4
+F test/icu.test 41aa8847745a879b897a7febea0f8f9efc8e67fe8bf680589b6e07c7b0a1569a
 F test/ieee754.test 806fc0ce7f305f57e3331eaceeddcfec9339e607
 F test/imposter1.test c3f1db2d3db2c24611a6596a3fc0ffc14f1466c8
 F test/in.test 2fa2dfba1afe30eb830f327e7131dfadaa7a701d677de0eb65f9303d99e18fe0
@@ -1032,7 +1032,7 @@ F test/lock6.test ad5b387a3a8096afd3c68a55b9535056431b0cf5
 F test/lock7.test 49f1eaff1cdc491cc5dee3669f3c671d9f172431
 F test/lock_common.tcl 7ffb45accf6ee91c736df9bafe0806a44358f035
 F test/lookaside.test b17c99ae3aef96a8c9fa6f6be33cc75b93d657cb791d3827302b6835b71941f7
-F test/main.test bb75e406c9b64931f3dc7e7f04626633365bb22f
+F test/main.test 6bbb3999fd461eb8fb335cbab97409a3d7f91bbb8da60635e8be3e4a04a77772
 F test/make-where7.tcl 05c16b5d4f5d6512881dfec560cb793915932ef9
 F test/malloc.test 21c213365f2cca95ab2d7dc078dc8525f96065f8
 F test/malloc3.test e3b32c724b5a124b57cb0ed177f675249ad0c66a
@@ -1188,7 +1188,7 @@ F test/schema5.test 29699b4421f183c8f0e88bd28ce7d75d13ea653e
 F test/schema6.test e4bd1f23d368695eb9e7b51ef6e02ca0642ea2ab4a52579959826b5e7dce1f9b
 F test/securedel.test 2f70b2449186a1921bd01ec9da407fbfa98c3a7a5521854c300c194b2ff09384
 F test/securedel2.test 2d54c28e46eb1fd6902089958b20b1b056c6f1c5
-F test/select1.test 460a5824df01575b18f7fa4bd8e40d09de20c542e90c1543e164bc7d3b0a0bb7
+F test/select1.test 2e760bab8f3658b3b97debcf52860d0d2e20aa6cbe8b40e678ddb99871a15491
 F test/select2.test 352480e0e9c66eda9c3044e412abdf5be0215b56
 F test/select3.test 2ce595f8fb8e2ac10071d3b4e424cadd4634a054
 F test/select4.test 5389d9895968d1196c457d59b3ee6515d771d328
@@ -1220,7 +1220,7 @@ F test/shared_err.test 32634e404a3317eeb94abc7a099c556a346fdb8fb3858dbe222a4cbb8
 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304
 F test/shell1.test 6d69e08039aea13f2c42749f162fe05eab7b5c93729f31d49d7d27cf36226e5a
 F test/shell2.test e242a9912f44f4c23c3d1d802a83e934e84c853b
-F test/shell3.test 9b95ba643eaa228376f06a898fb410ee9b6e57c1
+F test/shell3.test ac8c2b744014c3e9a0e26bfd829ab65f00923dc1a91ffd044863e9423cc91494
 F test/shell4.test 89ad573879a745974ff2df20ff97c5d6ffffbd5d
 F test/shell5.test 23939a4c51f0421330ea61dbd3c74f9c215f5f8d3d1a94846da6ffc777a35458
 F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3
@@ -1585,7 +1585,7 @@ F test/win32lock.test fbf107c91d8f5512be5a5b87c4c42ab9fdd54972
 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d
 F test/win32nolock.test ac4f08811a562e45a5755e661f45ca85892bdbbc
 F test/with1.test ca08e291249a810a2ec9b72ceef5575e07d5925b360fcf6652ae6fe06ac4dced
-F test/with2.test 2b40da883658eb74ad8ad06afabe11a408e7fb87
+F test/with2.test e0030e2f0267a910d6c0e4f46f2dfe941c1cc0d4f659ba69b3597728e7e8f1ab
 F test/with3.test e71604a0e53cba82bc04c703987cb1d6751ec0b6
 F test/withM.test 693b61765f2b387b5e3e24a4536e2e82de15ff64
 F test/without_rowid1.test 06b7215130882d6a072233820dd364c874c4fd69221e8fc756ec471009192874
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0fb42090cb2c785e044abe273a00da134927db191fea7b0c67ba4028944bce3a
-R 6a5a34e76be8657e78b0d58d3645af35
-U mistachkin
-Z b0c2e831274b963bd4ec05162b124f71
+P 3e5647cb6c4553683e24b9cb62548f16c79c4e2ac9e39cf135ea52a623f7cc33
+R 13437b6007d6ef55bf4d81b9dff0be71
+U drh
+Z f9f9bd51ed93ca5bce3e08c7f811f3c5
diff --git a/manifest.uuid b/manifest.uuid
index c9c91c407d..3bef9fc4db 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-3e5647cb6c4553683e24b9cb62548f16c79c4e2ac9e39cf135ea52a623f7cc33
\ No newline at end of file
+36b89d728ff13d395fe0e1db8e7c01263f73dccb278b3ece27f6ef78e909b492
\ No newline at end of file
diff --git a/src/build.c b/src/build.c
index 26eb0579e6..58b39d6475 100644
--- a/src/build.c
+++ b/src/build.c
@@ -2124,7 +2124,7 @@ void sqlite3CreateView(
   ** the end.
   */
   sEnd = pParse->sLastToken;
-  assert( sEnd.z[0]!=0 );
+  assert( sEnd.z[0]!=0 || sEnd.n==0 );
   if( sEnd.z[0]!=';' ){
     sEnd.z += sEnd.n;
   }
diff --git a/src/parse.y b/src/parse.y
index e780f8c3ab..d9cf1cb87d 100644
--- a/src/parse.y
+++ b/src/parse.y
@@ -31,8 +31,11 @@
 //
 %syntax_error {
   UNUSED_PARAMETER(yymajor);  /* Silence some compiler warnings */
-  assert( TOKEN.z[0] );  /* The tokenizer always gives us a token */
-  sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &TOKEN);
+  if( TOKEN.z[0] ){
+    sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &TOKEN);
+  }else{
+    sqlite3ErrorMsg(pParse, "incomplete input");
+  }
 }
 %stack_overflow {
   sqlite3ErrorMsg(pParse, "parser stack overflow");
diff --git a/src/tokenize.c b/src/tokenize.c
index 2aab334ae9..e6da3fb547 100644
--- a/src/tokenize.c
+++ b/src/tokenize.c
@@ -526,7 +526,7 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
       }else{
         tokenType = TK_SEMI;
       }
-      zSql -= n;
+      n = 0;
     }
     if( tokenType>=TK_SPACE ){
       assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL );
diff --git a/test/capi2.test b/test/capi2.test
index 5e51871814..0680cf530c 100644
--- a/test/capi2.test
+++ b/test/capi2.test
@@ -163,7 +163,7 @@ do_test capi2-3.2 {
       sqlite3_prepare $DB {select bogus from } -1 TAIL
   } msg]
   lappend rc $msg $TAIL
-} {1 {(1) near " ": syntax error} {}}
+} {1 {(1) incomplete input} {}}
 do_test capi2-3.3 {
   set rc [catch {
       sqlite3_prepare $DB {;;;;select bogus from sqlite_master} -1 TAIL
diff --git a/test/capi3.test b/test/capi3.test
index becf1bf5db..01ce65b6a4 100644
--- a/test/capi3.test
+++ b/test/capi3.test
@@ -649,6 +649,18 @@ do_test capi3-5.33 {
   sqlite3_finalize $STMT
 } SQLITE_OK
 
+# 2018-01-09:  If a column is the last token if a string, the column name
+# was not being set correctly, due to changes in check-in
+# https://sqlite.org/src/info/0fdf97efe5df7455
+#
+# This problem was detected by the community during beta-testing.
+#
+do_test capi3-5.34 {
+  set STMT [sqlite3_prepare $DB {SELECT :a, :b} -1 TAIL]
+  sqlite3_column_count $STMT
+} 2
+check_header $STMT capi-5.35 {:a :b} {{} {}}
+sqlite3_finalize $STMT
 
 set ::ENC [execsql {pragma encoding}]
 db close
diff --git a/test/icu.test b/test/icu.test
index b6d3b7f847..4c4e6d14ec 100644
--- a/test/icu.test
+++ b/test/icu.test
@@ -138,8 +138,8 @@ ifcapable icu {
   do_catchsql_test icu-5.4 { 
     SELECT 'abc' REGEXP 'a[abc]c.*'
   } {0 1}
-  do_catchsql_test icu-5.4 {SELECT 'abc' REGEXP }   {1 {near " ": syntax error}}
-  do_catchsql_test icu-5.5 {SELECT 'abc' REGEXP, 1} {1 {near ",": syntax error}}
+  do_catchsql_test icu-5.5 {SELECT 'abc' REGEXP }   {1 {incomplete input}}
+  do_catchsql_test icu-5.6 {SELECT 'abc' REGEXP, 1} {1 {near ",": syntax error}}
  
   do_malloc_test icu-6.10 -sqlbody {
     SELECT upper(char(0xfb04,0xdf,0xfb04,0xe8,0xfb04));
diff --git a/test/main.test b/test/main.test
index 9346cf6ced..13a385b7c4 100644
--- a/test/main.test
+++ b/test/main.test
@@ -434,7 +434,7 @@ do_test main-3.2.28 {
 } {0 246}
 do_test main-3.2.29 {
   catchsql {select 123/}
-} {1 {near "/": syntax error}}
+} {1 {incomplete input}}
 do_test main-3.2.30 {
   catchsql {select 123--5}
 } {0 123}
@@ -467,7 +467,7 @@ do_test main-3.4 {
 do_test main-3.5 {
   set v [catch {execsql {create}} msg]
   lappend v $msg
-} {1 {near "create": syntax error}}
+} {1 {incomplete input}}
 do_test main-3.6 {
   catchsql {SELECT 'abc' + #9}
 } {1 {near "#9": syntax error}}
diff --git a/test/select1.test b/test/select1.test
index 43b20f6d15..7023a6e65d 100644
--- a/test/select1.test
+++ b/test/select1.test
@@ -688,7 +688,7 @@ do_test select1-7.2 {
 do_test select1-7.3 {
   set v [catch {execsql {SELECT f1 FROM test1 as 'hi', test2 as}} msg]
   lappend v $msg
-} {1 {near "as": syntax error}}
+} {1 {incomplete input}}
 do_test select1-7.4 {
   set v [catch {execsql {
      SELECT f1 FROM test1 ORDER BY;
diff --git a/test/shell3.test b/test/shell3.test
index bb2524c1cc..63c30a2682 100644
--- a/test/shell3.test
+++ b/test/shell3.test
@@ -66,7 +66,7 @@ do_test shell3-1.6 {
 } {0 {}}
 do_test shell3-1.7 {
   catchcmd "foo.db \"CREATE TABLE\""
-} {1 {Error: near "TABLE": syntax error}}
+} {1 {Error: incomplete input}}
 
 #----------------------------------------------------------------------------
 #   shell3-2.*: Basic tests for running SQL file from command line.
@@ -96,6 +96,6 @@ do_test shell3-2.6 {
 } {0 {}}
 do_test shell3-2.7 {
   catchcmd "foo.db" "CREATE TABLE"
-} {1 {Error: near line 1: near "TABLE": syntax error}}
+} {1 {Error: near line 1: incomplete input}}
 
 finish_test
diff --git a/test/with2.test b/test/with2.test
index 02d10b5112..004ec94b97 100644
--- a/test/with2.test
+++ b/test/with2.test
@@ -326,7 +326,7 @@ do_catchsql_test 6.5 {
 
 do_catchsql_test 6.6 { 
   WITH x AS (SELECT * FROM t1) DELETE FROM t2 WHERE
-} {/1 {near .* syntax error}/}
+} {1 {incomplete input}}
 
 do_catchsql_test 6.7 { 
   WITH x AS (SELECT * FROM t1) DELETE FROM t2 WHRE 1;

From 5af06983564e35005dcb550981d8ebceb1b491d6 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 10 Jan 2018 00:53:55 +0000
Subject: [PATCH 281/488] Rearrange some routines in shell.c to avoid the need
 to forward reference a static function.

FossilOrigin-Name: fd7f51a107806666d5c3a3a7a62528ec3e1fb71c4256f49d57b4dcdac4bf8680
---
 manifest       |  12 ++---
 manifest.uuid  |   2 +-
 src/shell.c.in | 128 ++++++++++++++++++++++++-------------------------
 3 files changed, 71 insertions(+), 71 deletions(-)

diff --git a/manifest b/manifest
index a8883b6f20..73254e8c88 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Compute\sthe\scorrect\scolumn\sname\seven\sif\sthe\scolumn\sidentifier\sis\sthe\nvery\slast\stoken\sin\sthe\sSQL\sstatement.\s\sThis\sfixes\sa\sproblem\sintroduced\nby\scheck-in\s[0fdf97efe5df745510c6b]\sand\sreported\sby\sthe\scommunity\sduring\nbeta-testing.
-D 2018-01-10T00:40:06.636
+C Rearrange\ssome\sroutines\sin\sshell.c\sto\savoid\sthe\sneed\sto\sforward\sreference\sa\nstatic\sfunction.
+D 2018-01-10T00:53:55.338
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -484,7 +484,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in 4121ecd9a812c9032726aa137a4504251d03f2001dd6c7c110e7cabd3ee1454b
+F src/shell.c.in 1e50d66dc88bcc61d6300a5b8fb71d2d3821ec2d1418aee69337e391061514dd
 F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3e5647cb6c4553683e24b9cb62548f16c79c4e2ac9e39cf135ea52a623f7cc33
-R 13437b6007d6ef55bf4d81b9dff0be71
+P 36b89d728ff13d395fe0e1db8e7c01263f73dccb278b3ece27f6ef78e909b492
+R c0e5b3be89f9671131b70568ccd2de3b
 U drh
-Z f9f9bd51ed93ca5bce3e08c7f811f3c5
+Z 058862bc11ed7b27f8f707a776eec676
diff --git a/manifest.uuid b/manifest.uuid
index 3bef9fc4db..b802c2b17a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-36b89d728ff13d395fe0e1db8e7c01263f73dccb278b3ece27f6ef78e909b492
\ No newline at end of file
+fd7f51a107806666d5c3a3a7a62528ec3e1fb71c4256f49d57b4dcdac4bf8680
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index d214850aa9..253b1c764d 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -352,6 +352,11 @@ static void endTimer(void){
 */
 #define UNUSED_PARAMETER(x) (void)(x)
 
+/*
+** Number of elements in an array
+*/
+#define ArraySize(X)  (int)(sizeof(X)/sizeof(X[0]))
+
 /*
 ** If the following flag is set, then command execution stops
 ** at an error if we are not interactive.
@@ -624,6 +629,65 @@ static char *one_input_line(FILE *in, char *zPrior, int isContinuation){
   }
   return zResult;
 }
+
+
+/*
+** Return the value of a hexadecimal digit.  Return -1 if the input
+** is not a hex digit.
+*/
+static int hexDigitValue(char c){
+  if( c>='0' && c<='9' ) return c - '0';
+  if( c>='a' && c<='f' ) return c - 'a' + 10;
+  if( c>='A' && c<='F' ) return c - 'A' + 10;
+  return -1;
+}
+
+/*
+** Interpret zArg as an integer value, possibly with suffixes.
+*/
+static sqlite3_int64 integerValue(const char *zArg){
+  sqlite3_int64 v = 0;
+  static const struct { char *zSuffix; int iMult; } aMult[] = {
+    { "KiB", 1024 },
+    { "MiB", 1024*1024 },
+    { "GiB", 1024*1024*1024 },
+    { "KB",  1000 },
+    { "MB",  1000000 },
+    { "GB",  1000000000 },
+    { "K",   1000 },
+    { "M",   1000000 },
+    { "G",   1000000000 },
+  };
+  int i;
+  int isNeg = 0;
+  if( zArg[0]=='-' ){
+    isNeg = 1;
+    zArg++;
+  }else if( zArg[0]=='+' ){
+    zArg++;
+  }
+  if( zArg[0]=='0' && zArg[1]=='x' ){
+    int x;
+    zArg += 2;
+    while( (x = hexDigitValue(zArg[0]))>=0 ){
+      v = (v<<4) + x;
+      zArg++;
+    }
+  }else{
+    while( IsDigit(zArg[0]) ){
+      v = v*10 + zArg[0] - '0';
+      zArg++;
+    }
+  }
+  for(i=0; i<ArraySize(aMult); i++){
+    if( sqlite3_stricmp(aMult[i].zSuffix, zArg)==0 ){
+      v *= aMult[i].iMult;
+      break;
+    }
+  }
+  return isNeg? -v : v;
+}
+
 /*
 ** A variable length string to which one can append text.
 */
@@ -1061,11 +1125,6 @@ static const char *modeDescr[] = {
 #define SEP_Unit      "\x1F"
 #define SEP_Record    "\x1E"
 
-/*
-** Number of elements in an array
-*/
-#define ArraySize(X)  (int)(sizeof(X)/sizeof(X[0]))
-
 /*
 ** A callback for the sqlite3_log() interface.
 */
@@ -2473,8 +2532,6 @@ static int expertFinish(
   return rc;
 }
 
-static sqlite3_int64 integerValue(const char *zArg);
-
 /*
 ** Implementation of ".expert" dot command.
 */
@@ -3383,63 +3440,6 @@ static void resolve_backslashes(char *z){
   if( j<i ) z[j] = 0;
 }
 
-/*
-** Return the value of a hexadecimal digit.  Return -1 if the input
-** is not a hex digit.
-*/
-static int hexDigitValue(char c){
-  if( c>='0' && c<='9' ) return c - '0';
-  if( c>='a' && c<='f' ) return c - 'a' + 10;
-  if( c>='A' && c<='F' ) return c - 'A' + 10;
-  return -1;
-}
-
-/*
-** Interpret zArg as an integer value, possibly with suffixes.
-*/
-static sqlite3_int64 integerValue(const char *zArg){
-  sqlite3_int64 v = 0;
-  static const struct { char *zSuffix; int iMult; } aMult[] = {
-    { "KiB", 1024 },
-    { "MiB", 1024*1024 },
-    { "GiB", 1024*1024*1024 },
-    { "KB",  1000 },
-    { "MB",  1000000 },
-    { "GB",  1000000000 },
-    { "K",   1000 },
-    { "M",   1000000 },
-    { "G",   1000000000 },
-  };
-  int i;
-  int isNeg = 0;
-  if( zArg[0]=='-' ){
-    isNeg = 1;
-    zArg++;
-  }else if( zArg[0]=='+' ){
-    zArg++;
-  }
-  if( zArg[0]=='0' && zArg[1]=='x' ){
-    int x;
-    zArg += 2;
-    while( (x = hexDigitValue(zArg[0]))>=0 ){
-      v = (v<<4) + x;
-      zArg++;
-    }
-  }else{
-    while( IsDigit(zArg[0]) ){
-      v = v*10 + zArg[0] - '0';
-      zArg++;
-    }
-  }
-  for(i=0; i<ArraySize(aMult); i++){
-    if( sqlite3_stricmp(aMult[i].zSuffix, zArg)==0 ){
-      v *= aMult[i].iMult;
-      break;
-    }
-  }
-  return isNeg? -v : v;
-}
-
 /*
 ** Interpret zArg as either an integer or a boolean value.  Return 1 or 0
 ** for TRUE and FALSE.  Return the integer value if appropriate.

From a688ca5ea91e7046e1b88397a6fe0b88fb5b420e Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 10 Jan 2018 11:56:03 +0000
Subject: [PATCH 282/488] Fix a problem in os_unix.c causing it to return
 SQLITE_CANTOPEN instead of SQLITE_READONLY_RECOVERY.

FossilOrigin-Name: 6a16f554f027ba268276b728588b5eaea837cbed85358a06a2f6da3b70e834ad
---
 manifest        | 18 +++++++++---------
 manifest.uuid   |  2 +-
 src/main.c      |  1 +
 src/os_unix.c   | 30 ++++++++++++++++--------------
 test/misc7.test | 38 ++++++++++++++++++++++++++++++++++++--
 5 files changed, 63 insertions(+), 26 deletions(-)

diff --git a/manifest b/manifest
index 73254e8c88..b36b5fd4fb 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Rearrange\ssome\sroutines\sin\sshell.c\sto\savoid\sthe\sneed\sto\sforward\sreference\sa\nstatic\sfunction.
-D 2018-01-10T00:53:55.338
+C Fix\sa\sproblem\sin\sos_unix.c\scausing\sit\sto\sreturn\sSQLITE_CANTOPEN\sinstead\sof\nSQLITE_READONLY_RECOVERY.
+D 2018-01-10T11:56:03.281
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -448,7 +448,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 14686083cedc198540b15a79586cdd4be2acf6d5fa97627e355f817ab07e9fee
 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e
 F src/loadext.c 55bcc3c741059a1056859e8adaf133aa179e22be12215c0936b2f354ef71209b
-F src/main.c 690c4134f944cbd5b71d59dd6e61ce4131f6a50ab774f38108e57d07d79cf876
+F src/main.c 26918d50dd4a61b8f6f210320a522f46b5e7e592335b6aa664ab15b80b7c239b
 F src/malloc.c 6f684fd039f53bf9195193eb0cde731a2954970fabc6ef054ba379b6052dc296
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -467,7 +467,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104
 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
-F src/os_unix.c a9edddcc35d664c8247a18abd10d73b7a80c0d897a5341de8feea5a47cd57e25
+F src/os_unix.c ecdffdc7fc25c07e42908be7c5ea30456fee6263e0d54cdf204557945b445da2
 F src/os_win.c 0a4afa35cc8e812000df3ea2f64b476131b39e29e75d8007d0504726e4761de4
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 9b9cb4e06c03d43d62480a7a685a012d645fcf3a39e7767ccb505fb41ee083ec
@@ -1073,7 +1073,7 @@ F test/misc3.test cf3dda47d5dda3e53fc5804a100d3c82be736c9d
 F test/misc4.test 0d8be3466adf123a7791a66ba2bc8e8d229e87f3
 F test/misc5.test 60e1fc758a93cacd19eb2fafcd1d40d150a05047546c7a92389c98047d621901
 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91
-F test/misc7.test edd0b63e2ee29a256900b0514f6fff27e19e9bb2
+F test/misc7.test 859894e3192257ce2fc4063b5438b220e352286974b387e485050f0ad1f665d6
 F test/misc8.test ba03aaa08f02d62fbb8d3b2f5595c1b33aa9bbc5
 F test/misuse.test 9e7f78402005e833af71dcab32d048003869eca5abcaccc985d4f8dc1d86bcc7
 F test/mjournal.test 9d86e697dcbc5da2c4e8caba9b176b5765fe65e80c88c278b8c09a917e436795
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 36b89d728ff13d395fe0e1db8e7c01263f73dccb278b3ece27f6ef78e909b492
-R c0e5b3be89f9671131b70568ccd2de3b
-U drh
-Z 058862bc11ed7b27f8f707a776eec676
+P fd7f51a107806666d5c3a3a7a62528ec3e1fb71c4256f49d57b4dcdac4bf8680
+R b0aec6844a1353d3e8dcd58b6b9b074d
+U dan
+Z b1844f359ee67c65b8d14dad2349ae2c
diff --git a/manifest.uuid b/manifest.uuid
index b802c2b17a..e566611f17 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-fd7f51a107806666d5c3a3a7a62528ec3e1fb71c4256f49d57b4dcdac4bf8680
\ No newline at end of file
+6a16f554f027ba268276b728588b5eaea837cbed85358a06a2f6da3b70e834ad
\ No newline at end of file
diff --git a/src/main.c b/src/main.c
index 3c8035c120..ed41f98947 100644
--- a/src/main.c
+++ b/src/main.c
@@ -1318,6 +1318,7 @@ const char *sqlite3ErrName(int rc){
       case SQLITE_READONLY_CANTINIT:  zName = "SQLITE_READONLY_CANTINIT"; break;
       case SQLITE_READONLY_ROLLBACK:  zName = "SQLITE_READONLY_ROLLBACK"; break;
       case SQLITE_READONLY_DBMOVED:   zName = "SQLITE_READONLY_DBMOVED";  break;
+      case SQLITE_READONLY_DIRECTORY: zName = "SQLITE_READONLY_DIRECTORY";break;
       case SQLITE_INTERRUPT:          zName = "SQLITE_INTERRUPT";         break;
       case SQLITE_IOERR:              zName = "SQLITE_IOERR";             break;
       case SQLITE_IOERR_READ:         zName = "SQLITE_IOERR_READ";        break;
diff --git a/src/os_unix.c b/src/os_unix.c
index 3b2b2e2a59..d368cb34ea 100644
--- a/src/os_unix.c
+++ b/src/os_unix.c
@@ -5904,22 +5904,24 @@ static int unixOpen(
     fd = robust_open(zName, openFlags, openMode);
     OSTRACE(("OPENX   %-3d %s 0%o\n", fd, zName, openFlags));
     assert( !isExclusive || (openFlags & O_CREAT)!=0 );
-    if( fd<0 && errno!=EISDIR && isReadWrite ){
-      /* Failed to open the file for read/write access. Try read-only. */
-      flags &= ~(SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE);
-      openFlags &= ~(O_RDWR|O_CREAT);
-      flags |= SQLITE_OPEN_READONLY;
-      openFlags |= O_RDONLY;
-      isReadonly = 1;
-      fd = robust_open(zName, openFlags, openMode);
+    if( fd<0 ){
+      if( isNewJrnl && errno==EACCES && osAccess(zName, F_OK) ){
+        /* If unable to create a journal because the directory is not
+        ** writable, change the error code to indicate that. */
+        rc = SQLITE_READONLY_DIRECTORY;
+      }else if( errno!=EISDIR && isReadWrite ){
+        /* Failed to open the file for read/write access. Try read-only. */
+        flags &= ~(SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE);
+        openFlags &= ~(O_RDWR|O_CREAT);
+        flags |= SQLITE_OPEN_READONLY;
+        openFlags |= O_RDONLY;
+        isReadonly = 1;
+        fd = robust_open(zName, openFlags, openMode);
+      }
     }
     if( fd<0 ){
-      rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zName);
-      /* If unable to create a journal, change the error code to
-      ** indicate that the directory permissions are wrong. */
-      if( isNewJrnl && errno==EACCES && osAccess(zName, F_OK) ){
-        rc = SQLITE_READONLY_DIRECTORY;
-      }
+      int rc2 = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zName);
+      if( rc==SQLITE_OK ) rc = rc2;
       goto open_finished;
     }
 
diff --git a/test/misc7.test b/test/misc7.test
index 8fd5fe7546..d0e84dfa47 100644
--- a/test/misc7.test
+++ b/test/misc7.test
@@ -14,6 +14,7 @@
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
+set testprefix misc7
 
 if {[clang_sanitize_address]==0} {
   do_test misc7-1-misuse {
@@ -518,8 +519,41 @@ do_test misc7-22.3 {
 do_test misc7-22.4 { 
   sqlite3_extended_errcode db
 } SQLITE_READONLY_ROLLBACK
-
-db close
+catch { db close }
 forcedelete test.db
 
+if {$::tcl_platform(platform)=="unix"} {
+  reset_db
+  do_execsql_test 23.0 {
+    CREATE TABLE t1(x, y);
+    INSERT INTO t1 VALUES(1, 2);
+  }
+  
+  do_test 23.1 {
+    db close
+    forcedelete tst
+    file mkdir tst
+    forcecopy test.db tst/test.db
+    file attributes tst -permissions r-xr-xr-x
+  } {}
+  
+  sqlite3 db tst/test.db
+  do_execsql_test 23.2 {
+    SELECT * FROM t1;
+  } {1 2}
+  
+  do_catchsql_test 23.3 {
+    INSERT INTO t1 VALUES(3, 4);
+  } {1 {attempt to write a readonly database}}
+  
+  do_test 23.4 {
+    sqlite3_extended_errcode db
+  } {SQLITE_READONLY_DIRECTORY}
+  
+  do_test 23.5 {
+    db close
+    forcedelete tst
+  } {}
+}
+
 finish_test

From b376b3d6aa5648e1ffe9ca5ff09b2d29003834c7 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 10 Jan 2018 13:11:51 +0000
Subject: [PATCH 283/488] Work on the ".archive" command. (1) Add the --dryrun
 option. (2) Do not require --file when open on a ZIP archive. (3)
 Miscellaneous code simplifications. This is an incremental check-in of work
 in progress.

FossilOrigin-Name: a2baada429e84dc4b7243173a056e3c8bc042682f7efb01fdf8d2cc452c97e04
---
 manifest       |  15 +--
 manifest.uuid  |   2 +-
 src/shell.c.in | 269 ++++++++++++++++++++++++++-----------------------
 3 files changed, 152 insertions(+), 134 deletions(-)

diff --git a/manifest b/manifest
index 73254e8c88..2b380e8b60 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Rearrange\ssome\sroutines\sin\sshell.c\sto\savoid\sthe\sneed\sto\sforward\sreference\sa\nstatic\sfunction.
-D 2018-01-10T00:53:55.338
+C Work\son\sthe\s".archive"\scommand.\n(1)\sAdd\sthe\s--dryrun\soption.\n(2)\sDo\snot\srequire\s--file\swhen\sopen\son\sa\sZIP\sarchive.\n(3)\sMiscellaneous\scode\ssimplifications.\nThis\sis\san\sincremental\scheck-in\sof\swork\sin\sprogress.
+D 2018-01-10T13:11:51.661
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -484,7 +484,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in 1e50d66dc88bcc61d6300a5b8fb71d2d3821ec2d1418aee69337e391061514dd
+F src/shell.c.in f86200b08a8201fab1c775edbff4a81cb7b07e5926b1ed1ad7c043453fdfa7b3
 F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1697,7 +1697,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 36b89d728ff13d395fe0e1db8e7c01263f73dccb278b3ece27f6ef78e909b492
-R c0e5b3be89f9671131b70568ccd2de3b
+P fd7f51a107806666d5c3a3a7a62528ec3e1fb71c4256f49d57b4dcdac4bf8680
+R da66d185dcc66f20b0c30262a6b67fe7
+T *branch * archive-improvements
+T *sym-archive-improvements *
+T -sym-trunk *
 U drh
-Z 058862bc11ed7b27f8f707a776eec676
+Z 75859d5801ea825c715cde8533b487b8
diff --git a/manifest.uuid b/manifest.uuid
index b802c2b17a..822237285a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-fd7f51a107806666d5c3a3a7a62528ec3e1fb71c4256f49d57b4dcdac4bf8680
\ No newline at end of file
+a2baada429e84dc4b7243173a056e3c8bc042682f7efb01fdf8d2cc452c97e04
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 253b1c764d..4bfc1c85cc 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -4503,13 +4503,17 @@ static void shellReset(
 */
 typedef struct ArCommand ArCommand;
 struct ArCommand {
-  int eCmd;                       /* An AR_CMD_* value */
+  u8 eCmd;                        /* An AR_CMD_* value */
+  u8 bVerbose;                    /* True if --verbose */
+  u8 bZip;                        /* True if --zip */
+  u8 bDryRun;                     /* True if --dry-run */
+  int nArg;                       /* Number of command arguments */
+  const char *zSrcTable;          /* "sqlar", "zipfile($file)" or "zip" */
   const char *zFile;              /* --file argument, or NULL */
   const char *zDir;               /* --directory argument, or NULL */
-  int bVerbose;                   /* True if --verbose */
-  int bZip;                       /* True if --zip */
-  int nArg;                       /* Number of command arguments */
   char **azArg;                   /* Array of command arguments */
+  ShellState *p;                  /* Shell state */
+  sqlite3 *db;                    /* Database containing the archive */
 };
 
 /*
@@ -4536,6 +4540,8 @@ static int arUsage(FILE *f){
 "  -v, --verbose              Print each filename as it is processed\n"
 "  -f FILE, --file FILE       Operate on archive FILE (default is current db)\n"
 "  -C DIR, --directory DIR    Change to directory DIR to read/extract files\n"
+"  -n, --dryrun               Show the SQL that would have occurred\n"
+"  -z, --zip                  Operate on a ZIP archive instead of an SQLAR\n"
 "\n"
 "See also: http://sqlite.org/cli.html#sqlar_archive_support\n"
 "\n"
@@ -4570,10 +4576,11 @@ static int arErrorMsg(const char *zFmt, ...){
 /*
 ** Other (non-command) switches.
 */
-#define AR_SWITCH_VERBOSE   6
-#define AR_SWITCH_FILE      7
-#define AR_SWITCH_DIRECTORY 8
-#define AR_SWITCH_ZIP       9
+#define AR_SWITCH_VERBOSE     6
+#define AR_SWITCH_FILE        7
+#define AR_SWITCH_DIRECTORY   8
+#define AR_SWITCH_ZIP         9
+#define AR_SWITCH_DRYRUN     10
 
 static int arProcessSwitch(ArCommand *pAr, int eSwitch, const char *zArg){
   switch( eSwitch ){
@@ -4588,6 +4595,9 @@ static int arProcessSwitch(ArCommand *pAr, int eSwitch, const char *zArg){
       pAr->eCmd = eSwitch;
       break;
 
+    case AR_SWITCH_DRYRUN:
+      pAr->bDryRun = 1;
+      break;
     case AR_SWITCH_VERBOSE:
       pAr->bVerbose = 1;
       break;
@@ -4618,20 +4628,21 @@ static int arParseCommand(
   ArCommand *pAr                  /* Populate this object */
 ){
   struct ArSwitch {
-    char cShort;
     const char *zLong;
-    int eSwitch;
-    int bArg;
+    char cShort;
+    u8 eSwitch;
+    u8 bArg;
   } aSwitch[] = {
-    { 'c', "create",    AR_CMD_CREATE, 0 },
-    { 'x', "extract",   AR_CMD_EXTRACT, 0 },
-    { 't', "list",      AR_CMD_LIST, 0 },
-    { 'u', "update",    AR_CMD_UPDATE, 0 },
-    { 'h', "help",      AR_CMD_HELP, 0 },
-    { 'v', "verbose",   AR_SWITCH_VERBOSE, 0 },
-    { 'f', "file",      AR_SWITCH_FILE, 1 },
-    { 'C', "directory", AR_SWITCH_DIRECTORY, 1 },
-    { 'z', "zip",       AR_SWITCH_ZIP, 0 }
+    { "create",    'c', AR_CMD_CREATE,       0 },
+    { "extract",   'x', AR_CMD_EXTRACT,      0 },
+    { "list",      't', AR_CMD_LIST,         0 },
+    { "update",    'u', AR_CMD_UPDATE,       0 },
+    { "help",      'h', AR_CMD_HELP,         0 },
+    { "verbose",   'v', AR_SWITCH_VERBOSE,   0 },
+    { "file",      'f', AR_SWITCH_FILE,      1 },
+    { "directory", 'C', AR_SWITCH_DIRECTORY, 1 },
+    { "zip",       'z', AR_SWITCH_ZIP,       0 },
+    { "dryrun",    'n', AR_SWITCH_DRYRUN,    0 },
   };
   int nSwitch = sizeof(aSwitch) / sizeof(struct ArSwitch);
   struct ArSwitch *pEnd = &aSwitch[nSwitch];
@@ -4758,37 +4769,40 @@ static int arParseCommand(
 ** This is consistent with the way the [tar] command seems to work on
 ** Linux.
 */
-static int arCheckEntries(sqlite3 *db, ArCommand *pAr){
+static int arCheckEntries(ArCommand *pAr){
   int rc = SQLITE_OK;
   if( pAr->nArg ){
-    int i;
+    int i, j;
     sqlite3_stmt *pTest = 0;
 
-    shellPreparePrintf(db, &rc, &pTest, "SELECT name FROM %s WHERE name=?1", 
-        pAr->bZip ? "zipfile(?2)" : "sqlar"
+    shellPreparePrintf(pAr->db, &rc, &pTest,
+        "SELECT name FROM %s WHERE name=$name", 
+        pAr->zSrcTable
     );
-    if( rc==SQLITE_OK && pAr->bZip ){
-      sqlite3_bind_text(pTest, 2, pAr->zFile, -1, SQLITE_TRANSIENT);
+    if( rc==SQLITE_OK
+     && (j = sqlite3_bind_parameter_index(pTest, "$archiveFile"))>0
+    ){
+      sqlite3_bind_text(pTest, j, pAr->zFile, -1, SQLITE_TRANSIENT);
     }
+    j = sqlite3_bind_parameter_index(pTest, "$name");
     for(i=0; i<pAr->nArg && rc==SQLITE_OK; i++){
       char *z = pAr->azArg[i];
       int n = strlen30(z);
       int bOk = 0;
       while( n>0 && z[n-1]=='/' ) n--;
       z[n] = '\0';
-      sqlite3_bind_text(pTest, 1, z, -1, SQLITE_STATIC);
+      sqlite3_bind_text(pTest, j, z, -1, SQLITE_STATIC);
       if( SQLITE_ROW==sqlite3_step(pTest) ){
         bOk = 1;
       }
       shellReset(&rc, pTest);
       if( rc==SQLITE_OK && bOk==0 ){
-        raw_printf(stderr, "not found in archive: %s\n", z);
+        utf8_printf(stderr, "not found in archive: %s\n", z);
         rc = SQLITE_ERROR;
       }
     }
     shellFinalize(&rc, pTest);
   }
-
   return rc;
 }
 
@@ -4814,9 +4828,9 @@ static void arWhereClause(
       for(i=0; i<pAr->nArg; i++){
         const char *z = pAr->azArg[i];
         zWhere = sqlite3_mprintf(
-            "%z%s name = '%q' OR name BETWEEN '%q/' AND '%q0'", 
-            zWhere, zSep, z, z, z
-            );
+          "%z%s name = '%q' OR substr(name,1,%d) = '%q/'", 
+          zWhere, zSep, z, strlen30(z)+1, z
+        );
         if( zWhere==0 ){
           *pRc = SQLITE_NOMEM;
           break;
@@ -4858,9 +4872,8 @@ static void shellModeToString(char *zMode, int mode){
 /*
 ** Implementation of .ar "lisT" command. 
 */
-static int arListCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
+static int arListCommand(ArCommand *pAr){
   const char *zSql = "SELECT %s FROM %s WHERE %s"; 
-  const char *zTbl = (pAr->bZip ? "zipfile(?)" : "sqlar");
   const char *azCols[] = {
     "name",
     "mode, sz, datetime(mtime, 'unixepoch'), name"
@@ -4869,29 +4882,36 @@ static int arListCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
   char *zWhere = 0;
   sqlite3_stmt *pSql = 0;
   int rc;
+  int j;
 
-  rc = arCheckEntries(db, pAr);
+  rc = arCheckEntries(pAr);
   arWhereClause(&rc, pAr, &zWhere);
 
-  shellPreparePrintf(db, &rc, &pSql, zSql, azCols[pAr->bVerbose], zTbl, zWhere);
-  if( rc==SQLITE_OK && pAr->bZip ){
-    sqlite3_bind_text(pSql, 1, pAr->zFile, -1, SQLITE_TRANSIENT);
+  shellPreparePrintf(pAr->db, &rc, &pSql, zSql, azCols[pAr->bVerbose],
+                     pAr->zSrcTable, zWhere);
+  if( rc==SQLITE_OK 
+   && (j = sqlite3_bind_parameter_index(pSql, "$archiveFile"))>0
+  ){
+    sqlite3_bind_text(pSql, j, pAr->zFile, -1, SQLITE_TRANSIENT);
   }
-  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
-    if( pAr->bVerbose ){
-      char zMode[11];
-      shellModeToString(zMode, sqlite3_column_int(pSql, 0));
-
-      raw_printf(p->out, "%s % 10d  %s  %s\n", zMode,
-          sqlite3_column_int(pSql, 1), 
-          sqlite3_column_text(pSql, 2),
-          sqlite3_column_text(pSql, 3)
-      );
-    }else{
-      raw_printf(p->out, "%s\n", sqlite3_column_text(pSql, 0));
+  if( pAr->bDryRun ){
+    utf8_printf(pAr->p->out, "%s\n", sqlite3_sql(pSql));
+  }else{
+    while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
+      if( pAr->bVerbose ){
+        char zMode[11];
+        shellModeToString(zMode, sqlite3_column_int(pSql, 0));
+  
+        utf8_printf(pAr->p->out, "%s % 10d  %s  %s\n", zMode,
+            sqlite3_column_int(pSql, 1), 
+            sqlite3_column_text(pSql, 2),
+            sqlite3_column_text(pSql, 3)
+        );
+      }else{
+        utf8_printf(pAr->p->out, "%s\n", sqlite3_column_text(pSql, 0));
+      }
     }
   }
-
   shellFinalize(&rc, pSql);
   return rc;
 }
@@ -4900,31 +4920,28 @@ static int arListCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
 /*
 ** Implementation of .ar "eXtract" command. 
 */
-static int arExtractCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
+static int arExtractCommand(ArCommand *pAr){
   const char *zSql1 = 
     "SELECT "
-    "  :1 || name, "
-    "  writefile(?1 || name, %s, mode, mtime) "
-    "FROM %s WHERE (%s) AND (data IS NULL OR ?2 = 0)";
+    " ($dir || name),"
+    " writefile(($dir || name), %s, mode, mtime) "
+    "FROM %s WHERE (%s) AND (data IS NULL OR $dirOnly = 0)";
 
   const char *azExtraArg[] = { 
     "sqlar_uncompress(data, sz)",
     "data"
   };
-  const char *azSource[] = {
-    "sqlar", "zipfile(?3)"
-  };
 
   sqlite3_stmt *pSql = 0;
   int rc = SQLITE_OK;
   char *zDir = 0;
   char *zWhere = 0;
-  int i;
+  int i, j;
 
   /* If arguments are specified, check that they actually exist within
   ** the archive before proceeding. And formulate a WHERE clause to
   ** match them.  */
-  rc = arCheckEntries(db, pAr);
+  rc = arCheckEntries(pAr);
   arWhereClause(&rc, pAr, &zWhere);
 
   if( rc==SQLITE_OK ){
@@ -4936,14 +4953,16 @@ static int arExtractCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
     if( zDir==0 ) rc = SQLITE_NOMEM;
   }
 
-  shellPreparePrintf(db, &rc, &pSql, zSql1, 
-      azExtraArg[pAr->bZip], azSource[pAr->bZip], zWhere
+  shellPreparePrintf(pAr->db, &rc, &pSql, zSql1, 
+      azExtraArg[pAr->bZip], pAr->zSrcTable, zWhere
   );
 
   if( rc==SQLITE_OK ){
-    sqlite3_bind_text(pSql, 1, zDir, -1, SQLITE_STATIC);
-    if( pAr->bZip ){
-      sqlite3_bind_text(pSql, 3, pAr->zFile, -1, SQLITE_STATIC);
+    j = sqlite3_bind_parameter_index(pSql, "$dir");
+    sqlite3_bind_text(pSql, j, zDir, -1, SQLITE_STATIC);
+    j = sqlite3_bind_parameter_index(pSql, "$archiveFile");
+    if( j ){
+      sqlite3_bind_text(pSql, j, pAr->zFile, -1, SQLITE_STATIC);
     }
 
     /* Run the SELECT statement twice. The first time, writefile() is called
@@ -4952,10 +4971,15 @@ static int arExtractCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
     ** extracted directories must be reset after they are populated (as
     ** populating them changes the timestamp).  */
     for(i=0; i<2; i++){
-      sqlite3_bind_int(pSql, 2, i);
-      while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
-        if( i==0 && pAr->bVerbose ){
-          raw_printf(p->out, "%s\n", sqlite3_column_text(pSql, 0));
+      j = sqlite3_bind_parameter_index(pSql, "$dirOnly");
+      sqlite3_bind_int(pSql, j, i);
+      if( pAr->bDryRun ){
+        utf8_printf(pAr->p->out, "%s\n", sqlite3_sql(pSql));
+      }else{
+        while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
+          if( i==0 && pAr->bVerbose ){
+            utf8_printf(pAr->p->out, "%s\n", sqlite3_column_text(pSql, 0));
+          }
         }
       }
       shellReset(&rc, pSql);
@@ -4968,6 +4992,20 @@ static int arExtractCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
   return rc;
 }
 
+/*
+** Run the SQL statement in zSql.  Or if doing a --dryrun, merely print it out.
+*/
+static int arExecSql(ArCommand *pAr, const char *zSql){
+  int rc;
+  if( pAr->bDryRun ){
+    utf8_printf(pAr->p->out, "%s\n", zSql);
+    rc = SQLITE_OK;
+  }else{
+    rc = sqlite3_exec(pAr->db, zSql, 0, 0, 0);
+  }
+  return rc;
+}
+
 
 /*
 ** Implementation of .ar "create" and "update" commands.
@@ -4980,11 +5018,9 @@ static int arExtractCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
 ** The create command is the same as update, except that it drops
 ** any existing "sqlar" table before beginning.
 */
-static int arCreateUpdate(
-  ShellState *p,                  /* Shell state pointer */
-  sqlite3 *db,
+static int arCreateOrUpdateCommand(
   ArCommand *pAr,                 /* Command arguments and options */
-  int bUpdate
+  int bUpdate                     /* true for a --create.  false for --update */
 ){
   const char *zSql = "SELECT name, mode, mtime, data FROM fsdir(?, ?)";
   const char *zCreate = 
@@ -5005,17 +5041,17 @@ static int arCreateUpdate(
 
   assert( pAr->bZip==0 );
 
-  rc = sqlite3_exec(db, "SAVEPOINT ar;", 0, 0, 0);
+  rc = arExecSql(pAr, "SAVEPOINT ar;");
   if( rc!=SQLITE_OK ) return rc;
 
   if( bUpdate==0 ){
-    rc = sqlite3_exec(db, zDrop, 0, 0, 0);
+    rc = arExecSql(pAr, zDrop);
     if( rc!=SQLITE_OK ) return rc;
   }
 
-  rc = sqlite3_exec(db, zCreate, 0, 0, 0);
-  shellPrepare(db, &rc, zInsert, &pInsert);
-  shellPrepare(db, &rc, zSql, &pStmt);
+  rc = arExecSql(pAr, zCreate);
+  shellPrepare(pAr->db, &rc, zInsert, &pInsert);
+  shellPrepare(pAr->db, &rc, zSql, &pStmt);
   sqlite3_bind_text(pStmt, 2, pAr->zDir, -1, SQLITE_STATIC);
 
   for(i=0; i<pAr->nArg && rc==SQLITE_OK; i++){
@@ -5027,7 +5063,7 @@ static int arCreateUpdate(
       unsigned int mtime = sqlite3_column_int(pStmt, 2);
 
       if( pAr->bVerbose ){
-        raw_printf(p->out, "%s\n", zName);
+        utf8_printf(pAr->p->out, "%s\n", zName);
       }
 
       sqlite3_bind_text(pInsert, 1, zName, -1, SQLITE_STATIC);
@@ -5047,46 +5083,26 @@ static int arCreateUpdate(
       }
 
       sqlite3_bind_int(pInsert, 4, sz);
-      sqlite3_step(pInsert);
+      if( pAr->bDryRun ){
+        utf8_printf(pAr->p->out, "%s\n", sqlite3_sql(pInsert));
+      }else{
+        sqlite3_step(pInsert);
+      }
       rc = sqlite3_reset(pInsert);
     }
     shellReset(&rc, pStmt);
   }
 
   if( rc!=SQLITE_OK ){
-    sqlite3_exec(db, "ROLLBACK TO ar; RELEASE ar;", 0, 0, 0);
+    arExecSql(pAr, "ROLLBACK TO ar; RELEASE ar;");
   }else{
-    rc = sqlite3_exec(db, "RELEASE ar;", 0, 0, 0);
+    rc = arExecSql(pAr, "RELEASE ar;");
   }
   shellFinalize(&rc, pStmt);
   shellFinalize(&rc, pInsert);
   return rc;
 }
 
-/*
-** Implementation of .ar "Create" command. 
-**
-** Create the "sqlar" table in the database if it does not already exist.
-** Then add each file in the azFile[] array to the archive. Directories
-** are added recursively. If argument bVerbose is non-zero, a message is
-** printed on stdout for each file archived.
-*/
-static int arCreateCommand(
-  ShellState *p,                  /* Shell state pointer */
-  sqlite3 *db,
-  ArCommand *pAr                  /* Command arguments and options */
-){
-  return arCreateUpdate(p, db, pAr, 0);
-}
-
-/*
-** Implementation of .ar "Update" command. 
-*/
-static int arUpdateCmd(ShellState *p, sqlite3 *db, ArCommand *pAr){
-  return arCreateUpdate(p, db, pAr, 1);
-}
-
-
 /*
 ** Implementation of ".ar" dot command.
 */
@@ -5099,18 +5115,19 @@ static int arDotCommand(
   int rc;
   rc = arParseCommand(azArg, nArg, &cmd);
   if( rc==SQLITE_OK ){
-    sqlite3 *db = 0;              /* Database handle to use as archive */
-
+    cmd.p = pState;
+    cmd.db = pState->db;
+    cmd.zSrcTable = "sqlar";
     if( cmd.bZip ){
-      if( cmd.zFile==0 ){
-        raw_printf(stderr, "zip format requires a --file switch\n");
-        return SQLITE_ERROR;
-      }else
+      if( pState->openMode==SHELL_OPEN_ZIPFILE ){
+        cmd.zSrcTable = "zip";
+      }else{
+        cmd.zSrcTable = "zipfile($archiveFile)";
+      }
       if( cmd.eCmd==AR_CMD_CREATE || cmd.eCmd==AR_CMD_UPDATE ){
-        raw_printf(stderr, "zip archives are read-only\n");
+        utf8_printf(stderr, "zip archives are read-only\n");
         return SQLITE_ERROR;
       }
-      db = pState->db;
     }else if( cmd.zFile ){
       int flags;
       if( cmd.eCmd==AR_CMD_CREATE || cmd.eCmd==AR_CMD_UPDATE ){
@@ -5118,33 +5135,31 @@ static int arDotCommand(
       }else{
         flags = SQLITE_OPEN_READONLY;
       }
-      rc = sqlite3_open_v2(cmd.zFile, &db, flags, 0);
+      rc = sqlite3_open_v2(cmd.zFile, &cmd.db, flags, 0);
       if( rc!=SQLITE_OK ){
-        raw_printf(stderr, "cannot open file: %s (%s)\n", 
-            cmd.zFile, sqlite3_errmsg(db)
+        utf8_printf(stderr, "cannot open file: %s (%s)\n", 
+            cmd.zFile, sqlite3_errmsg(cmd.db)
         );
-        sqlite3_close(db);
+        sqlite3_close(cmd.db);
         return rc;
       }
-      sqlite3_fileio_init(db, 0, 0);
+      sqlite3_fileio_init(cmd.db, 0, 0);
 #ifdef SQLITE_HAVE_ZLIB
-      sqlite3_sqlar_init(db, 0, 0);
+      sqlite3_sqlar_init(cmd.db, 0, 0);
 #endif
-    }else{
-      db = pState->db;
     }
 
     switch( cmd.eCmd ){
       case AR_CMD_CREATE:
-        rc = arCreateCommand(pState, db, &cmd);
+        rc = arCreateOrUpdateCommand(&cmd, 0);
         break;
 
       case AR_CMD_EXTRACT:
-        rc = arExtractCommand(pState, db, &cmd);
+        rc = arExtractCommand(&cmd);
         break;
 
       case AR_CMD_LIST:
-        rc = arListCommand(pState, db, &cmd);
+        rc = arListCommand(&cmd);
         break;
 
       case AR_CMD_HELP:
@@ -5153,12 +5168,12 @@ static int arDotCommand(
 
       default:
         assert( cmd.eCmd==AR_CMD_UPDATE );
-        rc = arUpdateCmd(pState, db, &cmd);
+        rc = arCreateOrUpdateCommand(&cmd, 1);
         break;
     }
 
     if( cmd.zFile ){
-      sqlite3_close(db);
+      sqlite3_close(cmd.db);
     }
   }
 

From e75d1f5290361d7a0de2b10c842ce2983c1ce6c2 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 10 Jan 2018 13:58:23 +0000
Subject: [PATCH 284/488] Tag an unreachable branch using ALWAYS().

FossilOrigin-Name: c42c734f11c58724f5d8b32cb1c92e274be350028868d6ed045b2cfd274c64e7
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/malloc.c  |  2 +-
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index b36b5fd4fb..8e88f6ca98 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sproblem\sin\sos_unix.c\scausing\sit\sto\sreturn\sSQLITE_CANTOPEN\sinstead\sof\nSQLITE_READONLY_RECOVERY.
-D 2018-01-10T11:56:03.281
+C Tag\san\sunreachable\sbranch\susing\sALWAYS().
+D 2018-01-10T13:58:23.623
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -449,7 +449,7 @@ F src/insert.c 14686083cedc198540b15a79586cdd4be2acf6d5fa97627e355f817ab07e9fee
 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e
 F src/loadext.c 55bcc3c741059a1056859e8adaf133aa179e22be12215c0936b2f354ef71209b
 F src/main.c 26918d50dd4a61b8f6f210320a522f46b5e7e592335b6aa664ab15b80b7c239b
-F src/malloc.c 6f684fd039f53bf9195193eb0cde731a2954970fabc6ef054ba379b6052dc296
+F src/malloc.c 07295435093ce354c6d9063ac05a2eeae28bd251d2e63c48b3d67c12c76f7e18
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
 F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P fd7f51a107806666d5c3a3a7a62528ec3e1fb71c4256f49d57b4dcdac4bf8680
-R b0aec6844a1353d3e8dcd58b6b9b074d
-U dan
-Z b1844f359ee67c65b8d14dad2349ae2c
+P 6a16f554f027ba268276b728588b5eaea837cbed85358a06a2f6da3b70e834ad
+R 8ba57fadebb1bef28ae4cd97ec4b22a9
+U drh
+Z 3f9f5c48f04edbfb2783cc163a50a4b5
diff --git a/manifest.uuid b/manifest.uuid
index e566611f17..91ba96987f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6a16f554f027ba268276b728588b5eaea837cbed85358a06a2f6da3b70e834ad
\ No newline at end of file
+c42c734f11c58724f5d8b32cb1c92e274be350028868d6ed045b2cfd274c64e7
\ No newline at end of file
diff --git a/src/malloc.c b/src/malloc.c
index ec2d93ac89..d7f9df5efc 100644
--- a/src/malloc.c
+++ b/src/malloc.c
@@ -636,7 +636,7 @@ char *sqlite3DbSpanDup(sqlite3 *db, const char *zStart, const char *zEnd){
   int n;
   while( sqlite3Isspace(zStart[0]) ) zStart++;
   n = (int)(zEnd - zStart);
-  while( n>0 && sqlite3Isspace(zStart[n-1]) ) n--;
+  while( ALWAYS(n>0) && sqlite3Isspace(zStart[n-1]) ) n--;
   return sqlite3DbStrNDup(db, zStart, n);
 }
 

From a82c95b47d17c172077bdc9c1b225a577498efdd Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 10 Jan 2018 14:00:00 +0000
Subject: [PATCH 285/488] Allow the use of ".ar -t" without specifying an
 archive file or the "-z" option when the command-line shell is opened on a
 ZIP archive.

FossilOrigin-Name: 9340a2c145bcb4b38d19276a16264a37341c617f0554d66e1da653f1d9f85163
---
 manifest       | 15 ++++++---------
 manifest.uuid  |  2 +-
 src/shell.c.in | 14 ++++++++++----
 3 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/manifest b/manifest
index 2b380e8b60..96ab066f1b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Work\son\sthe\s".archive"\scommand.\n(1)\sAdd\sthe\s--dryrun\soption.\n(2)\sDo\snot\srequire\s--file\swhen\sopen\son\sa\sZIP\sarchive.\n(3)\sMiscellaneous\scode\ssimplifications.\nThis\sis\san\sincremental\scheck-in\sof\swork\sin\sprogress.
-D 2018-01-10T13:11:51.661
+C Allow\sthe\suse\sof\s".ar\s-t"\swithout\sspecifying\san\sarchive\sfile\sor\sthe\s"-z"\noption\swhen\sthe\scommand-line\sshell\sis\sopened\son\sa\sZIP\sarchive.
+D 2018-01-10T14:00:00.949
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -484,7 +484,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in f86200b08a8201fab1c775edbff4a81cb7b07e5926b1ed1ad7c043453fdfa7b3
+F src/shell.c.in aae06c143e7c70cfe616a96af8b5656f3891916beee172c7a40b084dab221b79
 F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1697,10 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P fd7f51a107806666d5c3a3a7a62528ec3e1fb71c4256f49d57b4dcdac4bf8680
-R da66d185dcc66f20b0c30262a6b67fe7
-T *branch * archive-improvements
-T *sym-archive-improvements *
-T -sym-trunk *
+P a2baada429e84dc4b7243173a056e3c8bc042682f7efb01fdf8d2cc452c97e04
+R 870698cbfc790bad8e7df783afd6e385
 U drh
-Z 75859d5801ea825c715cde8533b487b8
+Z 6cb4ecc388fb1e11ac7a5d118227df45
diff --git a/manifest.uuid b/manifest.uuid
index 822237285a..94454fbf0d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a2baada429e84dc4b7243173a056e3c8bc042682f7efb01fdf8d2cc452c97e04
\ No newline at end of file
+9340a2c145bcb4b38d19276a16264a37341c617f0554d66e1da653f1d9f85163
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 4bfc1c85cc..ad5300336e 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -5118,11 +5118,16 @@ static int arDotCommand(
     cmd.p = pState;
     cmd.db = pState->db;
     cmd.zSrcTable = "sqlar";
-    if( cmd.bZip ){
-      if( pState->openMode==SHELL_OPEN_ZIPFILE ){
+    if( cmd.bZip || pState->openMode==SHELL_OPEN_ZIPFILE ){
+      if( cmd.zFile==0
+       && sqlite3_table_column_metadata(cmd.db,0,"zip","name",0,0,0,0,0)==SQLITE_OK
+      ){
         cmd.zSrcTable = "zip";
-      }else{
+      }else if( cmd.zFile!=0 ){
         cmd.zSrcTable = "zipfile($archiveFile)";
+      }else{
+        utf8_printf(stderr, "no zip archive file specified\n");
+        return SQLITE_ERROR;
       }
       if( cmd.eCmd==AR_CMD_CREATE || cmd.eCmd==AR_CMD_UPDATE ){
         utf8_printf(stderr, "zip archives are read-only\n");
@@ -5135,6 +5140,7 @@ static int arDotCommand(
       }else{
         flags = SQLITE_OPEN_READONLY;
       }
+      cmd.db = 0;
       rc = sqlite3_open_v2(cmd.zFile, &cmd.db, flags, 0);
       if( rc!=SQLITE_OK ){
         utf8_printf(stderr, "cannot open file: %s (%s)\n", 
@@ -5172,7 +5178,7 @@ static int arDotCommand(
         break;
     }
 
-    if( cmd.zFile ){
+    if( cmd.db!=pState->db ){
       sqlite3_close(cmd.db);
     }
   }

From a5676c4d2df3bae5d2da4706e44e6e0ee5955bf0 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 10 Jan 2018 15:17:34 +0000
Subject: [PATCH 286/488] The ".ar" command deduces whether or not the target
 file is a ZIP or SQLAR and does the appropropriate thing.  The "-z" option is
 omitted. The "--append" option is added to open auxiliary databases using
 apndvfs.

FossilOrigin-Name: 430d1a7daa823ae53606b7a158af4e7c16f62ff9b072b90606524e7c3f6131df
---
 manifest         |  14 +++----
 manifest.uuid    |   2 +-
 src/shell.c.in   | 103 +++++++++++++++++++++++++++--------------------
 test/shell8.test |   3 +-
 4 files changed, 68 insertions(+), 54 deletions(-)

diff --git a/manifest b/manifest
index 96ab066f1b..da3f059664 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Allow\sthe\suse\sof\s".ar\s-t"\swithout\sspecifying\san\sarchive\sfile\sor\sthe\s"-z"\noption\swhen\sthe\scommand-line\sshell\sis\sopened\son\sa\sZIP\sarchive.
-D 2018-01-10T14:00:00.949
+C The\s".ar"\scommand\sdeduces\swhether\sor\snot\sthe\starget\sfile\sis\na\sZIP\sor\sSQLAR\sand\sdoes\sthe\sappropropriate\sthing.\s\sThe\s"-z"\soption\sis\somitted.\nThe\s"--append"\soption\sis\sadded\sto\sopen\sauxiliary\sdatabases\susing\sapndvfs.
+D 2018-01-10T15:17:34.832
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -484,7 +484,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in aae06c143e7c70cfe616a96af8b5656f3891916beee172c7a40b084dab221b79
+F src/shell.c.in 7d49ed78c6d8298d38d4733c3cc24f3a4230cd2f151564b7c11ab2e280007f9c
 F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1225,7 +1225,7 @@ F test/shell4.test 89ad573879a745974ff2df20ff97c5d6ffffbd5d
 F test/shell5.test 23939a4c51f0421330ea61dbd3c74f9c215f5f8d3d1a94846da6ffc777a35458
 F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3
 F test/shell7.test 115132f66d0463417f408562cc2cf534f6bbc6d83a6d50f0072a9eb171bae97f
-F test/shell8.test 5a1f2c6d5cc11e7c7d69e960972f447a9b01e80c419ff9ebaf059f94fe5b3ab9
+F test/shell8.test c836470ccde867e1f438a7acad7560805cc04f9dbab84cb55d92925942b76247
 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3
 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5
 F test/shrink.test 1b4330b1fd9e818c04726d45cb28db73087535ce
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a2baada429e84dc4b7243173a056e3c8bc042682f7efb01fdf8d2cc452c97e04
-R 870698cbfc790bad8e7df783afd6e385
+P 9340a2c145bcb4b38d19276a16264a37341c617f0554d66e1da653f1d9f85163
+R 9764fa4832552c458077bab9b9143a51
 U drh
-Z 6cb4ecc388fb1e11ac7a5d118227df45
+Z fa68d24d45da969d4d63bf64e8c4f030
diff --git a/manifest.uuid b/manifest.uuid
index 94454fbf0d..8a5e2767d2 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-9340a2c145bcb4b38d19276a16264a37341c617f0554d66e1da653f1d9f85163
\ No newline at end of file
+430d1a7daa823ae53606b7a158af4e7c16f62ff9b072b90606524e7c3f6131df
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index ad5300336e..83b21b1663 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -4505,10 +4505,11 @@ typedef struct ArCommand ArCommand;
 struct ArCommand {
   u8 eCmd;                        /* An AR_CMD_* value */
   u8 bVerbose;                    /* True if --verbose */
-  u8 bZip;                        /* True if --zip */
+  u8 bZip;                        /* True if the archive is a ZIP */
   u8 bDryRun;                     /* True if --dry-run */
+  u8 bAppend;                     /* True if --append */
   int nArg;                       /* Number of command arguments */
-  const char *zSrcTable;          /* "sqlar", "zipfile($file)" or "zip" */
+  char *zSrcTable;                /* "sqlar", "zipfile($file)" or "zip" */
   const char *zFile;              /* --file argument, or NULL */
   const char *zDir;               /* --directory argument, or NULL */
   char **azArg;                   /* Array of command arguments */
@@ -4541,7 +4542,7 @@ static int arUsage(FILE *f){
 "  -f FILE, --file FILE       Operate on archive FILE (default is current db)\n"
 "  -C DIR, --directory DIR    Change to directory DIR to read/extract files\n"
 "  -n, --dryrun               Show the SQL that would have occurred\n"
-"  -z, --zip                  Operate on a ZIP archive instead of an SQLAR\n"
+"  -a, --append               Append the SQLAR to an existing file\n"
 "\n"
 "See also: http://sqlite.org/cli.html#sqlar_archive_support\n"
 "\n"
@@ -4579,7 +4580,7 @@ static int arErrorMsg(const char *zFmt, ...){
 #define AR_SWITCH_VERBOSE     6
 #define AR_SWITCH_FILE        7
 #define AR_SWITCH_DIRECTORY   8
-#define AR_SWITCH_ZIP         9
+#define AR_SWITCH_APPEND      9
 #define AR_SWITCH_DRYRUN     10
 
 static int arProcessSwitch(ArCommand *pAr, int eSwitch, const char *zArg){
@@ -4601,8 +4602,8 @@ static int arProcessSwitch(ArCommand *pAr, int eSwitch, const char *zArg){
     case AR_SWITCH_VERBOSE:
       pAr->bVerbose = 1;
       break;
-    case AR_SWITCH_ZIP:
-      pAr->bZip = 1;
+    case AR_SWITCH_APPEND:
+      pAr->bAppend = 1;
       break;
 
     case AR_SWITCH_FILE:
@@ -4641,7 +4642,7 @@ static int arParseCommand(
     { "verbose",   'v', AR_SWITCH_VERBOSE,   0 },
     { "file",      'f', AR_SWITCH_FILE,      1 },
     { "directory", 'C', AR_SWITCH_DIRECTORY, 1 },
-    { "zip",       'z', AR_SWITCH_ZIP,       0 },
+    { "append",    'a', AR_SWITCH_APPEND,    0 },
     { "dryrun",    'n', AR_SWITCH_DRYRUN,    0 },
   };
   int nSwitch = sizeof(aSwitch) / sizeof(struct ArSwitch);
@@ -4779,11 +4780,6 @@ static int arCheckEntries(ArCommand *pAr){
         "SELECT name FROM %s WHERE name=$name", 
         pAr->zSrcTable
     );
-    if( rc==SQLITE_OK
-     && (j = sqlite3_bind_parameter_index(pTest, "$archiveFile"))>0
-    ){
-      sqlite3_bind_text(pTest, j, pAr->zFile, -1, SQLITE_TRANSIENT);
-    }
     j = sqlite3_bind_parameter_index(pTest, "$name");
     for(i=0; i<pAr->nArg && rc==SQLITE_OK; i++){
       char *z = pAr->azArg[i];
@@ -4882,18 +4878,12 @@ static int arListCommand(ArCommand *pAr){
   char *zWhere = 0;
   sqlite3_stmt *pSql = 0;
   int rc;
-  int j;
 
   rc = arCheckEntries(pAr);
   arWhereClause(&rc, pAr, &zWhere);
 
   shellPreparePrintf(pAr->db, &rc, &pSql, zSql, azCols[pAr->bVerbose],
                      pAr->zSrcTable, zWhere);
-  if( rc==SQLITE_OK 
-   && (j = sqlite3_bind_parameter_index(pSql, "$archiveFile"))>0
-  ){
-    sqlite3_bind_text(pSql, j, pAr->zFile, -1, SQLITE_TRANSIENT);
-  }
   if( pAr->bDryRun ){
     utf8_printf(pAr->p->out, "%s\n", sqlite3_sql(pSql));
   }else{
@@ -4960,10 +4950,6 @@ static int arExtractCommand(ArCommand *pAr){
   if( rc==SQLITE_OK ){
     j = sqlite3_bind_parameter_index(pSql, "$dir");
     sqlite3_bind_text(pSql, j, zDir, -1, SQLITE_STATIC);
-    j = sqlite3_bind_parameter_index(pSql, "$archiveFile");
-    if( j ){
-      sqlite3_bind_text(pSql, j, pAr->zFile, -1, SQLITE_STATIC);
-    }
 
     /* Run the SELECT statement twice. The first time, writefile() is called
     ** for all archive members that should be extracted. The second time,
@@ -5022,7 +5008,7 @@ static int arCreateOrUpdateCommand(
   ArCommand *pAr,                 /* Command arguments and options */
   int bUpdate                     /* true for a --create.  false for --update */
 ){
-  const char *zSql = "SELECT name, mode, mtime, data FROM fsdir(?, ?)";
+  const char *zSql = "SELECT name, mode, mtime, data FROM fsdir($name, $dir)";
   const char *zCreate = 
       "CREATE TABLE IF NOT EXISTS sqlar(\n"
       "  name TEXT PRIMARY KEY,  -- name of the file\n"
@@ -5037,6 +5023,7 @@ static int arCreateOrUpdateCommand(
   sqlite3_stmt *pStmt = 0;        /* Directory traverser */
   sqlite3_stmt *pInsert = 0;      /* Compilation of zInsert */
   int i;                          /* For iterating through azFile[] */
+  int j;                          /* Parameter index */
   int rc;                         /* Return code */
 
   assert( pAr->bZip==0 );
@@ -5050,12 +5037,19 @@ static int arCreateOrUpdateCommand(
   }
 
   rc = arExecSql(pAr, zCreate);
-  shellPrepare(pAr->db, &rc, zInsert, &pInsert);
+  if( !pAr->bDryRun ){
+    shellPrepare(pAr->db, &rc, zInsert, &pInsert);
+  }
   shellPrepare(pAr->db, &rc, zSql, &pStmt);
-  sqlite3_bind_text(pStmt, 2, pAr->zDir, -1, SQLITE_STATIC);
+  j = sqlite3_bind_parameter_index(pStmt, "$dir");
+  sqlite3_bind_text(pStmt, j, pAr->zDir, -1, SQLITE_STATIC);
+  if( pAr->bDryRun ){
+    utf8_printf(pAr->p->out, "%s;\n", sqlite3_sql(pStmt));
+  }
 
   for(i=0; i<pAr->nArg && rc==SQLITE_OK; i++){
-    sqlite3_bind_text(pStmt, 1, pAr->azArg[i], -1, SQLITE_STATIC);
+    j = sqlite3_bind_parameter_index(pStmt, "$name");
+    sqlite3_bind_text(pStmt, j, pAr->azArg[i], -1, SQLITE_STATIC);
     while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
       int sz;
       const char *zName = (const char*)sqlite3_column_text(pStmt, 0);
@@ -5065,6 +5059,10 @@ static int arCreateOrUpdateCommand(
       if( pAr->bVerbose ){
         utf8_printf(pAr->p->out, "%s\n", zName);
       }
+      if( pAr->bDryRun ){
+        utf8_printf(pAr->p->out, "%s;\n", zInsert);
+        continue;
+      }
 
       sqlite3_bind_text(pInsert, 1, zName, -1, SQLITE_STATIC);
       sqlite3_bind_int(pInsert, 2, mode);
@@ -5115,45 +5113,61 @@ static int arDotCommand(
   int rc;
   rc = arParseCommand(azArg, nArg, &cmd);
   if( rc==SQLITE_OK ){
+    int eDbType = SHELL_OPEN_UNSPEC;
     cmd.p = pState;
     cmd.db = pState->db;
-    cmd.zSrcTable = "sqlar";
-    if( cmd.bZip || pState->openMode==SHELL_OPEN_ZIPFILE ){
-      if( cmd.zFile==0
-       && sqlite3_table_column_metadata(cmd.db,0,"zip","name",0,0,0,0,0)==SQLITE_OK
-      ){
-        cmd.zSrcTable = "zip";
-      }else if( cmd.zFile!=0 ){
-        cmd.zSrcTable = "zipfile($archiveFile)";
+    cmd.zSrcTable = 0;
+    if( cmd.zFile ){
+      eDbType = deduceDatabaseType(cmd.zFile);
+    }else{
+      eDbType = pState->openMode;
+    }
+    if( eDbType==SHELL_OPEN_ZIPFILE ){
+      if( cmd.zFile==0 ){
+        cmd.zSrcTable = sqlite3_mprintf("zip");
       }else{
-        utf8_printf(stderr, "no zip archive file specified\n");
-        return SQLITE_ERROR;
+        cmd.zSrcTable = sqlite3_mprintf("zipfile(%Q)", cmd.zFile);
       }
       if( cmd.eCmd==AR_CMD_CREATE || cmd.eCmd==AR_CMD_UPDATE ){
         utf8_printf(stderr, "zip archives are read-only\n");
-        return SQLITE_ERROR;
+        rc = SQLITE_ERROR;
+        goto end_ar_command;
       }
+      cmd.bZip = 1;
     }else if( cmd.zFile ){
       int flags;
+      if( cmd.bAppend ) eDbType = SHELL_OPEN_APPENDVFS;
       if( cmd.eCmd==AR_CMD_CREATE || cmd.eCmd==AR_CMD_UPDATE ){
         flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE;
       }else{
         flags = SQLITE_OPEN_READONLY;
       }
       cmd.db = 0;
-      rc = sqlite3_open_v2(cmd.zFile, &cmd.db, flags, 0);
+      if( cmd.bDryRun ){
+        utf8_printf(pState->out, "-- open database '%s'%s\n", cmd.zFile,
+             eDbType==SHELL_OPEN_APPENDVFS ? " using 'apndvfs'" : "");
+      }
+      rc = sqlite3_open_v2(cmd.zFile, &cmd.db, flags, 
+             eDbType==SHELL_OPEN_APPENDVFS ? "apndvfs" : 0);
       if( rc!=SQLITE_OK ){
         utf8_printf(stderr, "cannot open file: %s (%s)\n", 
             cmd.zFile, sqlite3_errmsg(cmd.db)
         );
-        sqlite3_close(cmd.db);
-        return rc;
+        goto end_ar_command;
       }
       sqlite3_fileio_init(cmd.db, 0, 0);
 #ifdef SQLITE_HAVE_ZLIB
       sqlite3_sqlar_init(cmd.db, 0, 0);
 #endif
     }
+    if( cmd.zSrcTable==0 ){
+      if( sqlite3_table_column_metadata(cmd.db,0,"sqlar","name",0,0,0,0,0) ){
+        utf8_printf(stderr, "database does not contain an 'sqlar' table\n");
+        rc = SQLITE_ERROR;
+        goto end_ar_command;
+      }
+      cmd.zSrcTable = sqlite3_mprintf("sqlar");
+    }
 
     switch( cmd.eCmd ){
       case AR_CMD_CREATE:
@@ -5177,11 +5191,12 @@ static int arDotCommand(
         rc = arCreateOrUpdateCommand(&cmd, 1);
         break;
     }
-
-    if( cmd.db!=pState->db ){
-      sqlite3_close(cmd.db);
-    }
   }
+end_ar_command:
+  if( cmd.db!=pState->db ){
+    sqlite3_close(cmd.db);
+  }
+  sqlite3_free(cmd.zSrcTable);
 
   return rc;
 }
diff --git a/test/shell8.test b/test/shell8.test
index 0dcb10d67b..de0f237f72 100644
--- a/test/shell8.test
+++ b/test/shell8.test
@@ -101,7 +101,7 @@ foreach {tn tcl} {
     set x2 ".ar --extract --dir ar3"
 
     set c3 ".ar --creat --dir ar1 --file test_xyz.db ."
-    set x3 ".ar --e  --d ar3 --f test_xyz.db"
+    set x3 ".ar --e  --dir ar3 --f test_xyz.db"
   }
 
   4 {
@@ -169,4 +169,3 @@ finish_test
 
 
 finish_test
-

From 6c237b1fa79ae303a76a93878dff71cd2897f163 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 10 Jan 2018 15:53:06 +0000
Subject: [PATCH 287/488] Add the "filetype()" SQL function for interpreting
 file modes to the fileio.c extension.

FossilOrigin-Name: 58c0c74c407d93f48930a4964b6cc48f008b522d193a62a559de0e6a319a8bd0
---
 ext/misc/fileio.c | 28 ++++++++++++++++++++++++++++
 manifest          | 12 ++++++------
 manifest.uuid     |  2 +-
 3 files changed, 35 insertions(+), 7 deletions(-)

diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c
index 8170e86379..8f6de7a80f 100644
--- a/ext/misc/fileio.c
+++ b/ext/misc/fileio.c
@@ -357,6 +357,30 @@ static void writefileFunc(
   }
 }
 
+/*
+** SQL function:   filetype(MODE)
+**
+** Based on the integer mode, return one of "file", "directory", or "symlink".
+*/
+static void fileTypeFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  const char *zMode;
+  int iMode = sqlite3_value_int(argv[0]);
+  if( S_ISLNK(iMode) ){
+    zMode = "symlink";
+  }else if( S_ISREG(iMode) ){
+    zMode = "file";
+  }else if( S_ISDIR(iMode) ){
+    zMode = "directory";
+  }else{
+    zMode = "unknown";
+  }
+  sqlite3_result_text(context, zMode, -1, SQLITE_STATIC);
+}
+
 #ifndef SQLITE_OMIT_VIRTUALTABLE
 
 /* 
@@ -768,6 +792,10 @@ int sqlite3_fileio_init(
     rc = sqlite3_create_function(db, "writefile", -1, SQLITE_UTF8, 0,
                                  writefileFunc, 0, 0);
   }
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_create_function(db, "fileType", 1, SQLITE_UTF8, 0,
+                                 fileTypeFunc, 0, 0);
+  }
   if( rc==SQLITE_OK ){
     rc = fsdirRegister(db);
   }
diff --git a/manifest b/manifest
index da3f059664..2de51ed59b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C The\s".ar"\scommand\sdeduces\swhether\sor\snot\sthe\starget\sfile\sis\na\sZIP\sor\sSQLAR\sand\sdoes\sthe\sappropropriate\sthing.\s\sThe\s"-z"\soption\sis\somitted.\nThe\s"--append"\soption\sis\sadded\sto\sopen\sauxiliary\sdatabases\susing\sapndvfs.
-D 2018-01-10T15:17:34.832
+C Add\sthe\s"filetype()"\sSQL\sfunction\sfor\sinterpreting\sfile\smodes\sto\sthe\nfileio.c\sextension.
+D 2018-01-10T15:53:06.284
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -277,7 +277,7 @@ F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f0
 F ext/misc/csv.c 1a009b93650732e22334edc92459c4630b9fa703397cbb3c8ca279921a36ca11
 F ext/misc/dbdump.c 3509fa6b8932d04e932d6b6b827b6a82ca362781b8e8f3c77336f416793e215e
 F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2
-F ext/misc/fileio.c 5176893c15421e9c459f25478e3c1b066d19317b676ff0b142862de4e701af82
+F ext/misc/fileio.c 19d4e5590a3505dc12e652a64730a935a865bb70eb2bb8bfe75767355a09a6e2
 F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25
 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c
 F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281984
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 9340a2c145bcb4b38d19276a16264a37341c617f0554d66e1da653f1d9f85163
-R 9764fa4832552c458077bab9b9143a51
+P 430d1a7daa823ae53606b7a158af4e7c16f62ff9b072b90606524e7c3f6131df
+R 54a0e19016ae7cc0ce7d6a9a20dca28f
 U drh
-Z fa68d24d45da969d4d63bf64e8c4f030
+Z d67640218b4e3095d4e0ccc9365f61ee
diff --git a/manifest.uuid b/manifest.uuid
index 8a5e2767d2..91f7e865d7 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-430d1a7daa823ae53606b7a158af4e7c16f62ff9b072b90606524e7c3f6131df
\ No newline at end of file
+58c0c74c407d93f48930a4964b6cc48f008b522d193a62a559de0e6a319a8bd0
\ No newline at end of file

From f2ed70e4dea086e0e46d0fe884dd084d0ed70818 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 10 Jan 2018 16:30:40 +0000
Subject: [PATCH 288/488] Have the zipfile module automatically append "/" to
 directory names that do not already end with such a character. This is
 required for info-zip compatibility.

FossilOrigin-Name: 94bc3c60e7d2ec849b90444b06e3057ed645edf3af334f2737252960602868e5
---
 ext/misc/zipfile.c | 15 +++++++++++++++
 manifest           | 16 ++++++++--------
 manifest.uuid      |  2 +-
 test/zipfile.test  | 31 +++++++++++++++++++++++++------
 4 files changed, 49 insertions(+), 15 deletions(-)

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index 9a8d3974c2..8ce2e41801 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -1211,6 +1211,7 @@ static int zipfileUpdate(
   int nData = 0;                  /* Size of pData buffer in bytes */
   int iMethod = 0;                /* Compression method for new entry */
   u8 *pFree = 0;                  /* Free this */
+  char *zFree = 0;                /* Also free this */
   ZipfileCDS cds;                 /* New Central Directory Structure entry */
 
   int bIsDir = 0;
@@ -1307,6 +1308,19 @@ static int zipfileUpdate(
     }
   }
 
+  if( rc==SQLITE_OK && bIsDir ){
+    /* For a directory, check that the last character in the path is a
+    ** '/'. This appears to be required for compatibility with info-zip
+    ** (the unzip command on unix). It does not create directories
+    ** otherwise.  */
+    if( zPath[nPath-1]!='/' ){
+      zFree = sqlite3_mprintf("%s/", zPath);
+      if( zFree==0 ){ rc = SQLITE_NOMEM; }
+      zPath = (const char*)zFree;
+      nPath++;
+    }
+  }
+
   if( rc==SQLITE_OK ){
     /* Create the new CDS record. */
     memset(&cds, 0, sizeof(cds));
@@ -1334,6 +1348,7 @@ static int zipfileUpdate(
   }
 
   sqlite3_free(pFree);
+  sqlite3_free(zFree);
   return rc;
 }
 
diff --git a/manifest b/manifest
index 8e88f6ca98..327ce28b5d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Tag\san\sunreachable\sbranch\susing\sALWAYS().
-D 2018-01-10T13:58:23.623
+C Have\sthe\szipfile\smodule\sautomatically\sappend\s"/"\sto\sdirectory\snames\sthat\sdo\nnot\salready\send\swith\ssuch\sa\scharacter.\sThis\sis\srequired\sfor\sinfo-zip\ncompatibility.
+D 2018-01-10T16:30:40.654
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -303,7 +303,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c 92b840dc168126e192da7d6976f44b4453c40b27ad8f142a7c02dc31aaa31bd8
+F ext/misc/zipfile.c 08ec2ee0093d7a91d66db4ce1d8543cc4c19fc3fb57ee113ef49f95b8613f501
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -1598,7 +1598,7 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf
 F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc
 F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa
 F test/zerodamage.test e59a56443d6298ecf7435f618f0b27654f0c849e
-F test/zipfile.test 9fb98a24f80fe0d5d09df15cd01bb290777572f45408fdfbe894f2413c9c1222
+F test/zipfile.test 355e499ed4e8e0081e136510d8a7895496ee1196be10905e5dd61b0f7fdda39e
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d
 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6a16f554f027ba268276b728588b5eaea837cbed85358a06a2f6da3b70e834ad
-R 8ba57fadebb1bef28ae4cd97ec4b22a9
-U drh
-Z 3f9f5c48f04edbfb2783cc163a50a4b5
+P c42c734f11c58724f5d8b32cb1c92e274be350028868d6ed045b2cfd274c64e7
+R f9e0f45080131d91f8369d2321791b35
+U dan
+Z be2a3cd4c41b45eb12aa75b434fd5e7e
diff --git a/manifest.uuid b/manifest.uuid
index 91ba96987f..839b0e04c8 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c42c734f11c58724f5d8b32cb1c92e274be350028868d6ed045b2cfd274c64e7
\ No newline at end of file
+94bc3c60e7d2ec849b90444b06e3057ed645edf3af334f2737252960602868e5
\ No newline at end of file
diff --git a/test/zipfile.test b/test/zipfile.test
index 2a4f9ad96a..cafbdcefb4 100644
--- a/test/zipfile.test
+++ b/test/zipfile.test
@@ -91,28 +91,47 @@ do_execsql_test 2.1 {
   CREATE VIRTUAL TABLE zzz USING zipfile('test.zip');
   INSERT INTO zzz(name, mode) VALUES('dirname', 'drwxr-xr-x');
   SELECT name, mode, data FROM zzz;
-} {dirname 16877 {}}
+} {dirname/ 16877 {}}
 do_execsql_test 2.2 {
   INSERT INTO zzz(name, data) VALUES('dirname2', NULL);
   INSERT INTO zzz(name, data) VALUES('dirname2/file1.txt', 'abcdefghijklmnop');
   SELECT name, mode, data FROM zzz;
 } {
-  dirname 16877 {}
-  dirname2 16877 {}
+  dirname/ 16877 {}
+  dirname2/ 16877 {}
   dirname2/file1.txt 33188 abcdefghijklmnop
 }
 
 do_catchsql_test 2.3 {
-  UPDATE zzz SET name = 'dirname3' WHERE name = 'dirname';
+  UPDATE zzz SET name = 'dirname3' WHERE name = 'dirname/';
 } {1 {constraint failed}}
 do_execsql_test 2.4 {
   SELECT name, mode, data FROM zzz;
 } {
-  dirname 16877 {}
-  dirname2 16877 {}
+  dirname/ 16877 {}
+  dirname2/ 16877 {}
   dirname2/file1.txt 33188 abcdefghijklmnop
 }
 
+# If on unix, check that the [unzip] utility can unpack our archive.
+#
+if {$::tcl_platform(platform)=="unix"} {
+  do_test 2.5.1 {
+    forcedelete dirname
+      forcedelete dirname2
+      set rc [catch { exec unzip test.zip > /dev/null } msg]
+      list $rc $msg
+  } {0 {}}
+  do_test 2.5.2 { file isdir dirname } 1
+  do_test 2.5.3 { file isdir dirname2 } 1
+  do_test 2.5.4 { file isdir dirname2/file1.txt } 0
+  do_test 2.5.5 { 
+    set fd [open dirname2/file1.txt]
+    set data [read $fd]
+    close $fd
+    set data
+  } {abcdefghijklmnop}
+}
 
 finish_test
 

From 634c70fa1586565fe08a8f208691061cd36602dc Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 10 Jan 2018 16:50:18 +0000
Subject: [PATCH 289/488] Implement the ".ar --create" command using a single
 "REPLACE INTO sqlar SELECT ... FROM fsdir();" statement.  Add the
 shell_putsnl() SQL function for providing --verbose output.

FossilOrigin-Name: 28ab930436fea33c79073e84f39d9e381fa60b4702a5dcbfaaed72baeeae8431
---
 manifest       |  12 +++---
 manifest.uuid  |   2 +-
 src/shell.c.in | 104 +++++++++++++++++++------------------------------
 3 files changed, 46 insertions(+), 72 deletions(-)

diff --git a/manifest b/manifest
index 2de51ed59b..dcc5c90bb1 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\s"filetype()"\sSQL\sfunction\sfor\sinterpreting\sfile\smodes\sto\sthe\nfileio.c\sextension.
-D 2018-01-10T15:53:06.284
+C Implement\sthe\s".ar\s--create"\scommand\susing\sa\ssingle\s\n"REPLACE\sINTO\ssqlar\sSELECT\s...\sFROM\sfsdir();"\sstatement.\s\sAdd\sthe\nshell_putsnl()\sSQL\sfunction\sfor\sproviding\s--verbose\soutput.
+D 2018-01-10T16:50:18.954
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -484,7 +484,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in 7d49ed78c6d8298d38d4733c3cc24f3a4230cd2f151564b7c11ab2e280007f9c
+F src/shell.c.in f8ae3a792ba5ae09d6ad89845fc1cae75eb5b467f8534f654b8107b54e4aa81e
 F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 430d1a7daa823ae53606b7a158af4e7c16f62ff9b072b90606524e7c3f6131df
-R 54a0e19016ae7cc0ce7d6a9a20dca28f
+P 58c0c74c407d93f48930a4964b6cc48f008b522d193a62a559de0e6a319a8bd0
+R 6861922f8b8e02d4e242f42a1464e5df
 U drh
-Z d67640218b4e3095d4e0ccc9365f61ee
+Z dfee26c5de1e816e5523347d081a1560
diff --git a/manifest.uuid b/manifest.uuid
index 91f7e865d7..398fb8a627 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-58c0c74c407d93f48930a4964b6cc48f008b522d193a62a559de0e6a319a8bd0
\ No newline at end of file
+28ab930436fea33c79073e84f39d9e381fa60b4702a5dcbfaaed72baeeae8431
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 83b21b1663..80ca7305ec 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -1135,6 +1135,22 @@ static void shellLog(void *pArg, int iErrCode, const char *zMsg){
   fflush(p->pLog);
 }
 
+/*
+** SQL function:  shell_putsnl(X)
+**
+** Write the text X to the screen (or whatever output is being directed)
+** adding a newline at the end, and then return X.
+*/
+static void shellPutsFunc(
+  sqlite3_context *pCtx,
+  int nVal,
+  sqlite3_value **apVal
+){
+  ShellState *p = (ShellState*)sqlite3_user_data(pCtx);
+  utf8_printf(p->out, "%s\n", sqlite3_value_text(apVal[0]));
+  sqlite3_result_value(pCtx, apVal[0]);
+}
+
 /*
 ** Output the given string as a hex-encoded blob (eg. X'1234' )
 */
@@ -3307,6 +3323,8 @@ static void open_db(ShellState *p, int keepAlive){
                             shellAddSchemaName, 0, 0);
     sqlite3_create_function(p->db, "shell_module_schema", 1, SQLITE_UTF8, 0,
                             shellModuleSchema, 0, 0);
+    sqlite3_create_function(p->db, "shell_putsnl", 1, SQLITE_UTF8, p,
+                            shellPutsFunc, 0, 0);
     if( p->openMode==SHELL_OPEN_ZIPFILE ){
       char *zSql = sqlite3_mprintf(
          "CREATE VIRTUAL TABLE zip USING zipfile(%Q);", p->zDbFilename);
@@ -5008,7 +5026,6 @@ static int arCreateOrUpdateCommand(
   ArCommand *pAr,                 /* Command arguments and options */
   int bUpdate                     /* true for a --create.  false for --update */
 ){
-  const char *zSql = "SELECT name, mode, mtime, data FROM fsdir($name, $dir)";
   const char *zCreate = 
       "CREATE TABLE IF NOT EXISTS sqlar(\n"
       "  name TEXT PRIMARY KEY,  -- name of the file\n"
@@ -5018,86 +5035,41 @@ static int arCreateOrUpdateCommand(
       "  data BLOB               -- compressed content\n"
       ")";
   const char *zDrop = "DROP TABLE IF EXISTS sqlar";
-  const char *zInsert = "REPLACE INTO sqlar VALUES(?,?,?,?,sqlar_compress(?))";
-
-  sqlite3_stmt *pStmt = 0;        /* Directory traverser */
-  sqlite3_stmt *pInsert = 0;      /* Compilation of zInsert */
+  const char *zInsertFmt = 
+     "REPLACE INTO sqlar(name,mode,mtime,sz,data)\n"
+     "  SELECT\n"
+     "    %s,\n"
+     "    mode,\n"
+     "    mtime,\n"
+     "    CASE filetype(mode)\n"
+     "      WHEN 'file' THEN length(data)\n"
+     "      WHEN 'directory' THEN 0\n"
+     "      ELSE -1 END,\n"
+     "    CASE WHEN filetype(mode)<>'directory' THEN data ELSE NULL END\n"
+     "  FROM fsdir(%Q,%Q)\n"
+     "  WHERE filetype(mode)<>'unknown'";
   int i;                          /* For iterating through azFile[] */
-  int j;                          /* Parameter index */
   int rc;                         /* Return code */
 
-  assert( pAr->bZip==0 );
-
   rc = arExecSql(pAr, "SAVEPOINT ar;");
   if( rc!=SQLITE_OK ) return rc;
-
   if( bUpdate==0 ){
     rc = arExecSql(pAr, zDrop);
     if( rc!=SQLITE_OK ) return rc;
   }
-
   rc = arExecSql(pAr, zCreate);
-  if( !pAr->bDryRun ){
-    shellPrepare(pAr->db, &rc, zInsert, &pInsert);
-  }
-  shellPrepare(pAr->db, &rc, zSql, &pStmt);
-  j = sqlite3_bind_parameter_index(pStmt, "$dir");
-  sqlite3_bind_text(pStmt, j, pAr->zDir, -1, SQLITE_STATIC);
-  if( pAr->bDryRun ){
-    utf8_printf(pAr->p->out, "%s;\n", sqlite3_sql(pStmt));
-  }
-
   for(i=0; i<pAr->nArg && rc==SQLITE_OK; i++){
-    j = sqlite3_bind_parameter_index(pStmt, "$name");
-    sqlite3_bind_text(pStmt, j, pAr->azArg[i], -1, SQLITE_STATIC);
-    while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
-      int sz;
-      const char *zName = (const char*)sqlite3_column_text(pStmt, 0);
-      int mode = sqlite3_column_int(pStmt, 1);
-      unsigned int mtime = sqlite3_column_int(pStmt, 2);
-
-      if( pAr->bVerbose ){
-        utf8_printf(pAr->p->out, "%s\n", zName);
-      }
-      if( pAr->bDryRun ){
-        utf8_printf(pAr->p->out, "%s;\n", zInsert);
-        continue;
-      }
-
-      sqlite3_bind_text(pInsert, 1, zName, -1, SQLITE_STATIC);
-      sqlite3_bind_int(pInsert, 2, mode);
-      sqlite3_bind_int64(pInsert, 3, (sqlite3_int64)mtime);
-
-      if( S_ISDIR(mode) ){
-        sz = 0;
-        sqlite3_bind_null(pInsert, 5);
-      }else{
-        sqlite3_bind_value(pInsert, 5, sqlite3_column_value(pStmt, 3));
-        if( S_ISLNK(mode) ){
-          sz = -1;
-        }else{
-          sz = sqlite3_column_bytes(pStmt, 3);
-        }
-      }
-
-      sqlite3_bind_int(pInsert, 4, sz);
-      if( pAr->bDryRun ){
-        utf8_printf(pAr->p->out, "%s\n", sqlite3_sql(pInsert));
-      }else{
-        sqlite3_step(pInsert);
-      }
-      rc = sqlite3_reset(pInsert);
-    }
-    shellReset(&rc, pStmt);
+    char *zSql = sqlite3_mprintf(zInsertFmt,
+        pAr->bVerbose ? "shell_putsnl(name)" : "name",
+        pAr->azArg[i], pAr->zDir);
+    rc = arExecSql(pAr, zSql);
+    sqlite3_free(zSql);
   }
-
   if( rc!=SQLITE_OK ){
     arExecSql(pAr, "ROLLBACK TO ar; RELEASE ar;");
   }else{
     rc = arExecSql(pAr, "RELEASE ar;");
   }
-  shellFinalize(&rc, pStmt);
-  shellFinalize(&rc, pInsert);
   return rc;
 }
 
@@ -5161,7 +5133,9 @@ static int arDotCommand(
 #endif
     }
     if( cmd.zSrcTable==0 ){
-      if( sqlite3_table_column_metadata(cmd.db,0,"sqlar","name",0,0,0,0,0) ){
+      if( cmd.eCmd!=AR_CMD_CREATE
+       && sqlite3_table_column_metadata(cmd.db,0,"sqlar","name",0,0,0,0,0)
+      ){
         utf8_printf(stderr, "database does not contain an 'sqlar' table\n");
         rc = SQLITE_ERROR;
         goto end_ar_command;

From 410cad975f82a286ca3a0256ea865947be0a31cb Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 10 Jan 2018 17:19:16 +0000
Subject: [PATCH 290/488] In the fileio.c extension, change the filetype(MODE)
 function into lsmode(MODE). Use the new lsmode(MODE) function in shell.c.

FossilOrigin-Name: 52d12ba9f33c1f2620776e189c81f3bf991759344ecdd167ea2a6107f0972b9d
---
 ext/misc/fileio.c | 32 +++++++++++++++++++----------
 manifest          | 14 ++++++-------
 manifest.uuid     |  2 +-
 src/shell.c.in    | 52 +++++++++++++----------------------------------
 4 files changed, 43 insertions(+), 57 deletions(-)

diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c
index 8f6de7a80f..f126a2e435 100644
--- a/ext/misc/fileio.c
+++ b/ext/misc/fileio.c
@@ -358,27 +358,37 @@ static void writefileFunc(
 }
 
 /*
-** SQL function:   filetype(MODE)
+** SQL function:   lsmode(MODE)
 **
-** Based on the integer mode, return one of "file", "directory", or "symlink".
+** Given a numberic st_mode from stat(), convert it into a human-readable
+** text string in the style of "ls -l".
 */
-static void fileTypeFunc(
+static void lsModeFunc(
   sqlite3_context *context,
   int argc,
   sqlite3_value **argv
 ){
-  const char *zMode;
+  int i;
   int iMode = sqlite3_value_int(argv[0]);
+  char z[16];
   if( S_ISLNK(iMode) ){
-    zMode = "symlink";
+    z[0] = 'l';
   }else if( S_ISREG(iMode) ){
-    zMode = "file";
+    z[0] = '-';
   }else if( S_ISDIR(iMode) ){
-    zMode = "directory";
+    z[0] = 'd';
   }else{
-    zMode = "unknown";
+    z[0] = '?';
   }
-  sqlite3_result_text(context, zMode, -1, SQLITE_STATIC);
+  for(i=0; i<3; i++){
+    int m = (iMode >> ((2-i)*3));
+    char *a = &z[1 + i*3];
+    a[0] = (m & 0x4) ? 'r' : '-';
+    a[1] = (m & 0x2) ? 'w' : '-';
+    a[2] = (m & 0x1) ? 'x' : '-';
+  }
+  z[10] = '\0';
+  sqlite3_result_text(context, z, -1, SQLITE_TRANSIENT);
 }
 
 #ifndef SQLITE_OMIT_VIRTUALTABLE
@@ -793,8 +803,8 @@ int sqlite3_fileio_init(
                                  writefileFunc, 0, 0);
   }
   if( rc==SQLITE_OK ){
-    rc = sqlite3_create_function(db, "fileType", 1, SQLITE_UTF8, 0,
-                                 fileTypeFunc, 0, 0);
+    rc = sqlite3_create_function(db, "lsmode", 1, SQLITE_UTF8, 0,
+                                 lsModeFunc, 0, 0);
   }
   if( rc==SQLITE_OK ){
     rc = fsdirRegister(db);
diff --git a/manifest b/manifest
index dcc5c90bb1..36a782cc0d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Implement\sthe\s".ar\s--create"\scommand\susing\sa\ssingle\s\n"REPLACE\sINTO\ssqlar\sSELECT\s...\sFROM\sfsdir();"\sstatement.\s\sAdd\sthe\nshell_putsnl()\sSQL\sfunction\sfor\sproviding\s--verbose\soutput.
-D 2018-01-10T16:50:18.954
+C In\sthe\sfileio.c\sextension,\schange\sthe\sfiletype(MODE)\sfunction\sinto\slsmode(MODE).\nUse\sthe\snew\slsmode(MODE)\sfunction\sin\sshell.c.
+D 2018-01-10T17:19:16.761
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -277,7 +277,7 @@ F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f0
 F ext/misc/csv.c 1a009b93650732e22334edc92459c4630b9fa703397cbb3c8ca279921a36ca11
 F ext/misc/dbdump.c 3509fa6b8932d04e932d6b6b827b6a82ca362781b8e8f3c77336f416793e215e
 F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2
-F ext/misc/fileio.c 19d4e5590a3505dc12e652a64730a935a865bb70eb2bb8bfe75767355a09a6e2
+F ext/misc/fileio.c 1194228c96d6b7a374e93602e2ba1899b42d0fc4d4c5253962fb13e1c9ed1f77
 F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25
 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c
 F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281984
@@ -484,7 +484,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in f8ae3a792ba5ae09d6ad89845fc1cae75eb5b467f8534f654b8107b54e4aa81e
+F src/shell.c.in 926858c02fd4f644c79caca8e266bf6391dfc391fc07770d69a9db95c964eded
 F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 58c0c74c407d93f48930a4964b6cc48f008b522d193a62a559de0e6a319a8bd0
-R 6861922f8b8e02d4e242f42a1464e5df
+P 28ab930436fea33c79073e84f39d9e381fa60b4702a5dcbfaaed72baeeae8431
+R 820c3174d3fb9a7cd02211a2c674f634
 U drh
-Z dfee26c5de1e816e5523347d081a1560
+Z 3f66494887095c0b877f4d9025ad2b7a
diff --git a/manifest.uuid b/manifest.uuid
index 398fb8a627..77b89e1854 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-28ab930436fea33c79073e84f39d9e381fa60b4702a5dcbfaaed72baeeae8431
\ No newline at end of file
+52d12ba9f33c1f2620776e189c81f3bf991759344ecdd167ea2a6107f0972b9d
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 80ca7305ec..b6d51e081d 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -4856,33 +4856,6 @@ static void arWhereClause(
   *pzWhere = zWhere;
 }
 
-/*
-** Argument zMode must point to a buffer at least 11 bytes in size. This
-** function populates this buffer with the string interpretation of
-** the unix file mode passed as the second argument (e.g. "drwxr-xr-x").
-*/
-static void shellModeToString(char *zMode, int mode){
-  int i;
-
-  /* Magic numbers copied from [man 2 stat] */
-  if( mode & 0040000 ){
-    zMode[0] = 'd';
-  }else if( (mode & 0120000)==0120000 ){
-    zMode[0] = 'l';
-  }else{
-    zMode[0] = '-';
-  }
-
-  for(i=0; i<3; i++){
-    int m = (mode >> ((2-i)*3));
-    char *a = &zMode[1 + i*3];
-    a[0] = (m & 0x4) ? 'r' : '-';
-    a[1] = (m & 0x2) ? 'w' : '-';
-    a[2] = (m & 0x1) ? 'x' : '-';
-  }
-  zMode[10] = '\0';
-}
-
 /*
 ** Implementation of .ar "lisT" command. 
 */
@@ -4890,7 +4863,7 @@ static int arListCommand(ArCommand *pAr){
   const char *zSql = "SELECT %s FROM %s WHERE %s"; 
   const char *azCols[] = {
     "name",
-    "mode, sz, datetime(mtime, 'unixepoch'), name"
+    "lsmode(mode), sz, datetime(mtime, 'unixepoch'), name"
   };
 
   char *zWhere = 0;
@@ -4907,10 +4880,8 @@ static int arListCommand(ArCommand *pAr){
   }else{
     while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
       if( pAr->bVerbose ){
-        char zMode[11];
-        shellModeToString(zMode, sqlite3_column_int(pSql, 0));
-  
-        utf8_printf(pAr->p->out, "%s % 10d  %s  %s\n", zMode,
+        utf8_printf(pAr->p->out, "%s % 10d  %s  %s\n",
+            sqlite3_column_text(pSql, 0),
             sqlite3_column_int(pSql, 1), 
             sqlite3_column_text(pSql, 2),
             sqlite3_column_text(pSql, 3)
@@ -5005,7 +4976,12 @@ static int arExecSql(ArCommand *pAr, const char *zSql){
     utf8_printf(pAr->p->out, "%s\n", zSql);
     rc = SQLITE_OK;
   }else{
-    rc = sqlite3_exec(pAr->db, zSql, 0, 0, 0);
+    char *zErr = 0;
+    rc = sqlite3_exec(pAr->db, zSql, 0, 0, &zErr);
+    if( zErr ){
+      utf8_printf(stdout, "ERROR: %s\n", zErr);
+      sqlite3_free(zErr);
+    }
   }
   return rc;
 }
@@ -5041,13 +5017,13 @@ static int arCreateOrUpdateCommand(
      "    %s,\n"
      "    mode,\n"
      "    mtime,\n"
-     "    CASE filetype(mode)\n"
-     "      WHEN 'file' THEN length(data)\n"
-     "      WHEN 'directory' THEN 0\n"
+     "    CASE substr(lsmode(mode),1,1)\n"
+     "      WHEN '-' THEN length(data)\n"
+     "      WHEN 'd' THEN 0\n"
      "      ELSE -1 END,\n"
-     "    CASE WHEN filetype(mode)<>'directory' THEN data ELSE NULL END\n"
+     "    CASE WHEN lsmode(mode) LIKE 'd%%' THEN NULL else data END\n"
      "  FROM fsdir(%Q,%Q)\n"
-     "  WHERE filetype(mode)<>'unknown'";
+     "  WHERE lsmode(mode) NOT LIKE '?%%';";
   int i;                          /* For iterating through azFile[] */
   int rc;                         /* Return code */
 

From 4bfd182939211ca2bda599d9da11210cc4454b5d Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 10 Jan 2018 17:37:58 +0000
Subject: [PATCH 291/488] Return an SQLITE_CONSTRAINT error if an attempt is
 made to insert duplicate entries into a zip archive.

FossilOrigin-Name: 1f099b2b45074c89eeff8ff241aa49489c95c2221b25c305fcda670ebf63fb4e
---
 ext/misc/zipfile.c | 25 ++++++++++++++++++++++++-
 manifest           | 14 +++++++-------
 manifest.uuid      |  2 +-
 test/zipfile.test  | 28 ++++++++++++++++++++++++++++
 4 files changed, 60 insertions(+), 9 deletions(-)

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index 8ce2e41801..6785c9610c 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -1033,7 +1033,7 @@ static int zipfileLoadDirectory(ZipfileTab *pTab){
       }else{
         memset(pNew, 0, sizeof(ZipfileEntry));
         pNew->zPath = (char*)&pNew[1];
-        memcpy(pNew->zPath, &aBuf[ZIPFILE_CDS_FIXED_SZ], nFile);
+        memcpy(pNew->zPath, &aRec[ZIPFILE_CDS_FIXED_SZ], nFile);
         pNew->zPath[nFile] = '\0';
         pNew->aCdsEntry = (u8*)&pNew->zPath[nFile+1];
         pNew->nCdsEntry = ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment;
@@ -1189,6 +1189,18 @@ static int zipfileGetMode(
   return SQLITE_ERROR;
 }
 
+/*
+** Both (const char*) arguments point to nul-terminated strings. Argument
+** nB is the value of strlen(zB). This function returns 0 if the strings are
+** identical, ignoring any trailing '/' character in either path.  */
+static int zipfileComparePath(const char *zA, const char *zB, int nB){
+  int nA = strlen(zA);
+  if( zA[nA-1]=='/' ) nA--;
+  if( zB[nB-1]=='/' ) nB--;
+  if( nA==nB && memcmp(zA, zB, nA)==0 ) return 0;
+  return 1;
+}
+
 /*
 ** xUpdate method.
 */
@@ -1321,6 +1333,17 @@ static int zipfileUpdate(
     }
   }
 
+  /* Check that we're not inserting a duplicate entry */
+  if( rc==SQLITE_OK ){
+    ZipfileEntry *p;
+    for(p=pTab->pFirstEntry; p; p=p->pNext){
+      if( zipfileComparePath(p->zPath, zPath, nPath)==0 ){
+        rc = SQLITE_CONSTRAINT;
+        break;
+      }
+    }
+  }
+
   if( rc==SQLITE_OK ){
     /* Create the new CDS record. */
     memset(&cds, 0, sizeof(cds));
diff --git a/manifest b/manifest
index 327ce28b5d..3b5c001489 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Have\sthe\szipfile\smodule\sautomatically\sappend\s"/"\sto\sdirectory\snames\sthat\sdo\nnot\salready\send\swith\ssuch\sa\scharacter.\sThis\sis\srequired\sfor\sinfo-zip\ncompatibility.
-D 2018-01-10T16:30:40.654
+C Return\san\sSQLITE_CONSTRAINT\serror\sif\san\sattempt\sis\smade\sto\sinsert\sduplicate\nentries\sinto\sa\szip\sarchive.
+D 2018-01-10T17:37:58.434
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -303,7 +303,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c 08ec2ee0093d7a91d66db4ce1d8543cc4c19fc3fb57ee113ef49f95b8613f501
+F ext/misc/zipfile.c 00d78e61f0b0a7f51a4d5ffef302ef840336f09d75a97a8e0be1fabf048511b8
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -1598,7 +1598,7 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf
 F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc
 F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa
 F test/zerodamage.test e59a56443d6298ecf7435f618f0b27654f0c849e
-F test/zipfile.test 355e499ed4e8e0081e136510d8a7895496ee1196be10905e5dd61b0f7fdda39e
+F test/zipfile.test e7132ca60031ca5d1df684cf644952bb3f253de3671a5b502780c5de3126a453
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d
 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c42c734f11c58724f5d8b32cb1c92e274be350028868d6ed045b2cfd274c64e7
-R f9e0f45080131d91f8369d2321791b35
+P 94bc3c60e7d2ec849b90444b06e3057ed645edf3af334f2737252960602868e5
+R 9dfa6dcaeef4eb497b2c9753cb4a2823
 U dan
-Z be2a3cd4c41b45eb12aa75b434fd5e7e
+Z 2990c2df8ab5dd5a7f9266ba3fc45276
diff --git a/manifest.uuid b/manifest.uuid
index 839b0e04c8..b72ce7ac03 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-94bc3c60e7d2ec849b90444b06e3057ed645edf3af334f2737252960602868e5
\ No newline at end of file
+1f099b2b45074c89eeff8ff241aa49489c95c2221b25c305fcda670ebf63fb4e
\ No newline at end of file
diff --git a/test/zipfile.test b/test/zipfile.test
index cafbdcefb4..df02797bd8 100644
--- a/test/zipfile.test
+++ b/test/zipfile.test
@@ -133,5 +133,33 @@ if {$::tcl_platform(platform)=="unix"} {
   } {abcdefghijklmnop}
 }
 
+#-------------------------------------------------------------------------
+reset_db
+forcedelete test.zip
+load_static_extension db zipfile
+
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE temp.x1 USING zipfile('test.zip');
+  INSERT INTO x1(name, data) VALUES('dir1/', NULL);
+  INSERT INTO x1(name, data) VALUES('file1', '1234');
+  INSERT INTO x1(name, data) VALUES('dir1/file2', '5678');
+}
+foreach {tn fname} {
+  1 dir1
+  2 file1
+  3 dir1/file2
+} {
+  do_catchsql_test 3.1.$tn.0 {
+    INSERT INTO x1(name, data) VALUES($fname, NULL);
+  } {1 {constraint failed}}
+  do_catchsql_test 3.1.$tn.1 {
+    INSERT INTO x1(name, data) VALUES($fname || '/', NULL);
+  } {1 {constraint failed}}
+  do_catchsql_test 3.1.$tn.2 {
+    INSERT INTO x1(name, data) VALUES($fname, 'abcd');
+  } {1 {constraint failed}}
+}
+
+
 finish_test
 

From 346606494a3a5e176b273faebd8c9b9772e598f3 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 10 Jan 2018 17:39:54 +0000
Subject: [PATCH 292/488] Fix minor problems with the new ".archive" command
 changes.

FossilOrigin-Name: 612b30c95f948438016bd11470e9dd114d7bb064418a57e5954a094d2ca77f69
---
 manifest       | 12 ++++++------
 manifest.uuid  |  2 +-
 src/shell.c.in |  5 ++++-
 3 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 36a782cc0d..7c0f0180e3 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\sthe\sfileio.c\sextension,\schange\sthe\sfiletype(MODE)\sfunction\sinto\slsmode(MODE).\nUse\sthe\snew\slsmode(MODE)\sfunction\sin\sshell.c.
-D 2018-01-10T17:19:16.761
+C Fix\sminor\sproblems\swith\sthe\snew\s".archive"\scommand\schanges.
+D 2018-01-10T17:39:54.594
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -484,7 +484,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in 926858c02fd4f644c79caca8e266bf6391dfc391fc07770d69a9db95c964eded
+F src/shell.c.in 4cb216da4adaac57faf03a502584d44d96f99b3235a6d2956750670c6aabc8b1
 F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 28ab930436fea33c79073e84f39d9e381fa60b4702a5dcbfaaed72baeeae8431
-R 820c3174d3fb9a7cd02211a2c674f634
+P 52d12ba9f33c1f2620776e189c81f3bf991759344ecdd167ea2a6107f0972b9d
+R 1170208662cbaecd416c0d66fbee2d48
 U drh
-Z 3f66494887095c0b877f4d9025ad2b7a
+Z 6adbf06d453c4d240bb3d139f4f72e24
diff --git a/manifest.uuid b/manifest.uuid
index 77b89e1854..c546e6b5ff 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-52d12ba9f33c1f2620776e189c81f3bf991759344ecdd167ea2a6107f0972b9d
\ No newline at end of file
+612b30c95f948438016bd11470e9dd114d7bb064418a57e5954a094d2ca77f69
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index b6d51e081d..f9942446a5 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -5059,12 +5059,12 @@ static int arDotCommand(
 ){
   ArCommand cmd;
   int rc;
+  memset(&cmd, 0, sizeof(cmd));
   rc = arParseCommand(azArg, nArg, &cmd);
   if( rc==SQLITE_OK ){
     int eDbType = SHELL_OPEN_UNSPEC;
     cmd.p = pState;
     cmd.db = pState->db;
-    cmd.zSrcTable = 0;
     if( cmd.zFile ){
       eDbType = deduceDatabaseType(cmd.zFile);
     }else{
@@ -5107,6 +5107,9 @@ static int arDotCommand(
 #ifdef SQLITE_HAVE_ZLIB
       sqlite3_sqlar_init(cmd.db, 0, 0);
 #endif
+      sqlite3_create_function(cmd.db, "shell_putsnl", 1, SQLITE_UTF8, cmd.p,
+                              shellPutsFunc, 0, 0);
+
     }
     if( cmd.zSrcTable==0 ){
       if( cmd.eCmd!=AR_CMD_CREATE

From ca7733b782c3177d0759c09bd69394a178b0cee8 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 10 Jan 2018 18:09:20 +0000
Subject: [PATCH 293/488] Modify the new "--append" option to the .archive so
 that it takes a filename argument, the same as "--file".

FossilOrigin-Name: 58e3b07cc8cb5cc915f7d430483bd455b03f14120e0db23286d2e20dbb5391c5
---
 manifest       | 13 ++++++-------
 manifest.uuid  |  2 +-
 src/shell.c.in |  7 +++----
 3 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/manifest b/manifest
index a7fe4fa39d..747f1758ed 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\slsmode(MODE)\sfunction\sin\sthe\sfileio.c\sextension\sand\suse\sthat\sfunction\nin\sthe\s.archive\simplementation.\s\sAdd\sthe\s--append\sand\s--dryrun\soptions\sto\nthe\s.archive\scommand\sand\sremove\sthe\s--zip\soption,\smaking\sit\sautomatic.
-D 2018-01-10T17:44:03.548
+C Modify\sthe\snew\s"--append"\soption\sto\sthe\s.archive\sso\sthat\sit\stakes\sa\sfilename\nargument,\sthe\ssame\sas\s"--file".
+D 2018-01-10T18:09:20.329
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
@@ -484,7 +484,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in 4cb216da4adaac57faf03a502584d44d96f99b3235a6d2956750670c6aabc8b1
+F src/shell.c.in 2e72b9dc9dabde7e1d26142bf1080dfbc182ac1bf9431f85307b5931062a041b
 F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1697,8 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 1f099b2b45074c89eeff8ff241aa49489c95c2221b25c305fcda670ebf63fb4e 612b30c95f948438016bd11470e9dd114d7bb064418a57e5954a094d2ca77f69
-R 92ad9119ea4a1b866aa95ac38fa72e64
-T +closed 612b30c95f948438016bd11470e9dd114d7bb064418a57e5954a094d2ca77f69
+P 38f28029d16df8489772c27867de8888dd1c26b170861acdc977fd4023939dde
+R 816b51584860870ca4a8d73ef0ed2ba2
 U drh
-Z 0e2d546e002bd2ff2da7a7782cc6fb79
+Z 8ced30e3ccc67e3689656b52b7db29d9
diff --git a/manifest.uuid b/manifest.uuid
index cc72e08c3d..f3f36f5301 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-38f28029d16df8489772c27867de8888dd1c26b170861acdc977fd4023939dde
\ No newline at end of file
+58e3b07cc8cb5cc915f7d430483bd455b03f14120e0db23286d2e20dbb5391c5
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index f9942446a5..e1fca6a072 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -4558,9 +4558,9 @@ static int arUsage(FILE *f){
 "And zero or more optional options:\n"
 "  -v, --verbose              Print each filename as it is processed\n"
 "  -f FILE, --file FILE       Operate on archive FILE (default is current db)\n"
+"  -a FILE, --append FILE     Operate on FILE opened using the apndvfs VFS\n"
 "  -C DIR, --directory DIR    Change to directory DIR to read/extract files\n"
 "  -n, --dryrun               Show the SQL that would have occurred\n"
-"  -a, --append               Append the SQLAR to an existing file\n"
 "\n"
 "See also: http://sqlite.org/cli.html#sqlar_archive_support\n"
 "\n"
@@ -4622,8 +4622,7 @@ static int arProcessSwitch(ArCommand *pAr, int eSwitch, const char *zArg){
       break;
     case AR_SWITCH_APPEND:
       pAr->bAppend = 1;
-      break;
-
+      /* Fall thru into --file */
     case AR_SWITCH_FILE:
       pAr->zFile = zArg;
       break;
@@ -4659,8 +4658,8 @@ static int arParseCommand(
     { "help",      'h', AR_CMD_HELP,         0 },
     { "verbose",   'v', AR_SWITCH_VERBOSE,   0 },
     { "file",      'f', AR_SWITCH_FILE,      1 },
+    { "append",    'a', AR_SWITCH_APPEND,    1 },
     { "directory", 'C', AR_SWITCH_DIRECTORY, 1 },
-    { "append",    'a', AR_SWITCH_APPEND,    0 },
     { "dryrun",    'n', AR_SWITCH_DRYRUN,    0 },
   };
   int nSwitch = sizeof(aSwitch) / sizeof(struct ArSwitch);

From d99c7b7bcb54e7e3531a79c6a86f2cffdb147ef1 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 10 Jan 2018 19:15:21 +0000
Subject: [PATCH 294/488] Include RTREE in the default CLI build.

FossilOrigin-Name: 6a6a3d495633b906ca31f513c30d31e6daf0f0f105be9ba0a0dc07d201d5b630
---
 Makefile.in   |  3 ++-
 Makefile.msc  |  1 +
 main.mk       |  1 +
 manifest      | 16 ++++++++--------
 manifest.uuid |  2 +-
 5 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/Makefile.in b/Makefile.in
index af1bd61548..f340178c7a 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -578,7 +578,8 @@ TESTOPTS = --verbose=file --output=test-out.txt
 # Extra compiler options for various shell tools
 #
 SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4
-# SHELL_OPT += -DSQLITE_ENABLE_FTS5
+#SHELL_OPT += -DSQLITE_ENABLE_FTS5
+SHELL_OPT += -DSQLITE_ENABLE_RTREE
 SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
 SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
 SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB
diff --git a/Makefile.msc b/Makefile.msc
index 6d4538706c..73c8df2e29 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -1606,6 +1606,7 @@ FUZZDATA = \
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_DBSTAT_VTAB
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC -DSQLITE_INTROSPECTION_PRAGMAS
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_RTREE
 !ENDIF
 
 # <<mark>>
diff --git a/main.mk b/main.mk
index fc68d9d307..281de5d335 100644
--- a/main.mk
+++ b/main.mk
@@ -506,6 +506,7 @@ TESTOPTS = --verbose=file --output=test-out.txt
 # Extra compiler options for various shell tools
 #
 SHELL_OPT += -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5
+SHELL_OPT += -DSQLITE_ENABLE_RTREE
 SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
 SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
 SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB
diff --git a/manifest b/manifest
index 747f1758ed..612fc85322 100644
--- a/manifest
+++ b/manifest
@@ -1,10 +1,10 @@
-C Modify\sthe\snew\s"--append"\soption\sto\sthe\s.archive\sso\sthat\sit\stakes\sa\sfilename\nargument,\sthe\ssame\sas\s"--file".
-D 2018-01-10T18:09:20.329
+C Include\sRTREE\sin\sthe\sdefault\sCLI\sbuild.
+D 2018-01-10T19:15:21.479
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
-F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb
+F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 085d3b65cebdebec89dc8c91901c06f18e357eb320a0434bfa67a53e917f10de
+F Makefile.msc ede26e3fb675e0b3b07627640ce5917154a6ee7f8f2c97424eb5ab5f651cbd56
 F README.md eeae1e552f93ef72ef7c5b8f6647b368a001c28820ad1df179d3dae602bef681
 F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -406,7 +406,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 7965d01485f7bec7945407c5238985ea8c7cb2cb686d2bfdbe3d5f79d6fd4eb2
+F main.mk fc0edb268998a049ce70ee3dc056b2a96cc8aa4ef0c6da296700d7081d167627
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 38f28029d16df8489772c27867de8888dd1c26b170861acdc977fd4023939dde
-R 816b51584860870ca4a8d73ef0ed2ba2
+P 58e3b07cc8cb5cc915f7d430483bd455b03f14120e0db23286d2e20dbb5391c5
+R 2e662e06c18ea15ed95bfe93585d99c1
 U drh
-Z 8ced30e3ccc67e3689656b52b7db29d9
+Z 73fb00a74c3f6ec900cd40e3e36e7246
diff --git a/manifest.uuid b/manifest.uuid
index f3f36f5301..3fcaee05db 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-58e3b07cc8cb5cc915f7d430483bd455b03f14120e0db23286d2e20dbb5391c5
\ No newline at end of file
+6a6a3d495633b906ca31f513c30d31e6daf0f0f105be9ba0a0dc07d201d5b630
\ No newline at end of file

From cc9c26a092dbd39949a8e441adcda0aab413ca58 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 10 Jan 2018 19:50:40 +0000
Subject: [PATCH 295/488] Fix a harmless compiler warning in zipfile.c

FossilOrigin-Name: 60c694c1ab26a7a096f17ccea5a93ecda0f9f2113ab5fdc8b17dbffc787724fc
---
 ext/misc/zipfile.c |  3 +--
 manifest           | 12 ++++++------
 manifest.uuid      |  2 +-
 3 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index 6785c9610c..1a65a8896b 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -1194,7 +1194,7 @@ static int zipfileGetMode(
 ** nB is the value of strlen(zB). This function returns 0 if the strings are
 ** identical, ignoring any trailing '/' character in either path.  */
 static int zipfileComparePath(const char *zA, const char *zB, int nB){
-  int nA = strlen(zA);
+  int nA = (int)strlen(zA);
   if( zA[nA-1]=='/' ) nA--;
   if( zB[nB-1]=='/' ) nB--;
   if( nA==nB && memcmp(zA, zB, nA)==0 ) return 0;
@@ -1525,4 +1525,3 @@ int sqlite3_zipfile_init(
   (void)pzErrMsg;  /* Unused parameter */
   return zipfileRegister(db);
 }
-
diff --git a/manifest b/manifest
index 612fc85322..63c68f6f7e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Include\sRTREE\sin\sthe\sdefault\sCLI\sbuild.
-D 2018-01-10T19:15:21.479
+C Fix\sa\sharmless\scompiler\swarning\sin\szipfile.c
+D 2018-01-10T19:50:40.811
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -303,7 +303,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c 00d78e61f0b0a7f51a4d5ffef302ef840336f09d75a97a8e0be1fabf048511b8
+F ext/misc/zipfile.c 0d12ad852af1732c9a4bd0be9f28886b049e0591a871e3246a5347d2cea15430
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 58e3b07cc8cb5cc915f7d430483bd455b03f14120e0db23286d2e20dbb5391c5
-R 2e662e06c18ea15ed95bfe93585d99c1
+P 6a6a3d495633b906ca31f513c30d31e6daf0f0f105be9ba0a0dc07d201d5b630
+R 47ef8e39d746f78565d92ddee819e4f1
 U drh
-Z 73fb00a74c3f6ec900cd40e3e36e7246
+Z 8e9572c27b3beeb784a18b126ea860dc
diff --git a/manifest.uuid b/manifest.uuid
index 3fcaee05db..12625a7794 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6a6a3d495633b906ca31f513c30d31e6daf0f0f105be9ba0a0dc07d201d5b630
\ No newline at end of file
+60c694c1ab26a7a096f17ccea5a93ecda0f9f2113ab5fdc8b17dbffc787724fc
\ No newline at end of file

From 13c209330d03e162aa8cabc196fb1f1f24c24b7c Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 10 Jan 2018 21:41:55 +0000
Subject: [PATCH 296/488] Add support for the ".excel" command (and ".once -e"
 and ".once -x") in the CLI.

FossilOrigin-Name: 23fa7c57c2b204d1ddcc2a939b5271628cf26689ad4ede6976038113095a9801
---
 manifest       | 15 +++++----
 manifest.uuid  |  2 +-
 src/shell.c.in | 90 +++++++++++++++++++++++++++++++++++++++++++++++---
 3 files changed, 95 insertions(+), 12 deletions(-)

diff --git a/manifest b/manifest
index 63c68f6f7e..751835d013 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sharmless\scompiler\swarning\sin\szipfile.c
-D 2018-01-10T19:50:40.811
+C Add\ssupport\sfor\sthe\s".excel"\scommand\s(and\s".once\s-e"\sand\s".once\s-x")\sin\nthe\sCLI.
+D 2018-01-10T21:41:55.211
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -484,7 +484,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in 2e72b9dc9dabde7e1d26142bf1080dfbc182ac1bf9431f85307b5931062a041b
+F src/shell.c.in d1dbc1514d74b425db41be006f2a4f91d26c5e1c2db4ab198b45c12920171e11
 F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1697,7 +1697,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6a6a3d495633b906ca31f513c30d31e6daf0f0f105be9ba0a0dc07d201d5b630
-R 47ef8e39d746f78565d92ddee819e4f1
+P 60c694c1ab26a7a096f17ccea5a93ecda0f9f2113ab5fdc8b17dbffc787724fc
+R bd40eb18946fee4f15d02cd4e13ad69c
+T *branch * excel-shell-cmd
+T *sym-excel-shell-cmd *
+T -sym-trunk *
 U drh
-Z 8e9572c27b3beeb784a18b126ea860dc
+Z f708fb338ffb9679393be576ce4d2140
diff --git a/manifest.uuid b/manifest.uuid
index 12625a7794..5bb31c1d62 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-60c694c1ab26a7a096f17ccea5a93ecda0f9f2113ab5fdc8b17dbffc787724fc
\ No newline at end of file
+23fa7c57c2b204d1ddcc2a939b5271628cf26689ad4ede6976038113095a9801
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index e1fca6a072..81677bf5e8 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -1012,6 +1012,7 @@ struct ShellState {
   u8 statsOn;            /* True to display memory stats before each finalize */
   u8 scanstatsOn;        /* True to display scan stats before each finalize */
   u8 openMode;           /* SHELL_OPEN_NORMAL, _APPENDVFS, or _ZIPFILE */
+  u8 doXdgOpen;          /* Invoke start/open/xdg-open in output_reset() */
   int outCount;          /* Revert to stdout when reaching zero */
   int cnt;               /* Number of records displayed so far */
   FILE *out;             /* Write results here */
@@ -1025,6 +1026,7 @@ struct ShellState {
   int nCheck;            /* Number of ".check" commands run */
   unsigned shellFlgs;    /* Various flags */
   char *zDestTable;      /* Name of destination table when MODE_Insert */
+  char *zTempFile;       /* Temporary file that might need deleting */
   char zTestcase[30];    /* Name of current test case */
   char colSeparator[20]; /* Column separator character for several modes */
   char rowSeparator[20]; /* Row separator character for MODE_Ascii */
@@ -3064,6 +3066,7 @@ static char zHelp[] =
   "                         LIKE pattern TABLE.\n"
   ".echo on|off           Turn command echo on or off\n"
   ".eqp on|off|full       Enable or disable automatic EXPLAIN QUERY PLAN\n"
+  ".excel                 Display the output of next command in a spreadsheet\n"
   ".exit                  Exit this program\n"
   ".expert                EXPERIMENTAL. Suggest indexes for specified queries\n"
 /* Because explain mode comes on automatically now, the ".explain" mode
@@ -3926,7 +3929,11 @@ static void tryToClone(ShellState *p, const char *zNewDb){
 }
 
 /*
-** Change the output file back to stdout
+** Change the output file back to stdout.
+**
+** If the p->doXdgOpen flag is set, that means the output was being
+** redirected to a temporary file named by p->zTempFile.  In that case,
+** launch start/open/xdg-open on that temporary file.
 */
 static void output_reset(ShellState *p){
   if( p->outfile[0]=='|' ){
@@ -3935,6 +3942,22 @@ static void output_reset(ShellState *p){
 #endif
   }else{
     output_file_close(p->out);
+    if( p->doXdgOpen ){
+      const char *zXdgOpenCmd =
+#if defined(_WIN32)
+      "start";
+#elif defined(__APPLE__)
+      "open";
+#else
+      "xdg-open";
+#endif
+      char *zCmd;
+      zCmd = sqlite3_mprintf("%s %s", zXdgOpenCmd, p->zTempFile);
+      system(zCmd);
+      sqlite3_free(zCmd);
+      p->mode = p->doXdgOpen - 1;
+      p->doXdgOpen = 0;
+    }
   }
   p->outfile[0] = 0;
   p->out = stdout;
@@ -4193,6 +4216,38 @@ int shellDeleteFile(const char *zFilename){
   return rc;
 }
 
+/*
+** Try to delete the temporary file (if there is one) and free the
+** memory used to hold the name of the temp file.
+*/
+static void clearTempFile(ShellState *p){
+  if( p->zTempFile==0 ) return;
+  if( shellDeleteFile(p->zTempFile) ) return;
+  sqlite3_free(p->zTempFile);
+  p->zTempFile = 0;
+}
+
+/*
+** Create a new temp file name with the given suffix.
+*/
+static void newTempFile(ShellState *p, const char *zSuffix){
+  clearTempFile(p);
+  sqlite3_free(p->zTempFile);
+  p->zTempFile = 0;
+  sqlite3_file_control(p->db, 0, SQLITE_FCNTL_TEMPFILENAME, &p->zTempFile);
+  if( p->zTempFile==0 ){
+    sqlite3_uint64 r;
+    sqlite3_randomness(sizeof(r), &r);
+    p->zTempFile = sqlite3_mprintf("temp%llx.%s", r, zSuffix);
+  }else{
+    p->zTempFile = sqlite3_mprintf("%z.%s", p->zTempFile, zSuffix);
+  }
+  if( p->zTempFile==0 ){
+    raw_printf(stderr, "out of memory\n");
+    exit(1);
+  }
+}
+
 
 /*
 ** The implementation of SQL scalar function fkey_collate_clause(), used
@@ -5205,6 +5260,7 @@ static int do_meta_command(char *zLine, ShellState *p){
   if( nArg==0 ) return 0; /* no tokens, no error */
   n = strlen30(azArg[0]);
   c = azArg[0][0];
+  clearTempFile(p);
 
 #ifndef SQLITE_OMIT_AUTHORIZATION
   if( c=='a' && strncmp(azArg[0], "auth", n)==0 ){
@@ -6109,18 +6165,26 @@ static int do_meta_command(char *zLine, ShellState *p){
     }
   }else
 
-  if( c=='o'
-   && (strncmp(azArg[0], "output", n)==0 || strncmp(azArg[0], "once", n)==0)
+  if( (c=='o'
+        && (strncmp(azArg[0], "output", n)==0||strncmp(azArg[0], "once", n)==0))
+   || (c=='e' && n==5 && strcmp(azArg[0],"excel")==0)
   ){
     const char *zFile = nArg>=2 ? azArg[1] : "stdout";
+    if( azArg[0][0]=='e' ){
+      /* Transform the ".excel" command into ".once -x" */
+      nArg = 2;
+      azArg[0] = "once";
+      zFile = azArg[1] = "-x";
+      n = 4;
+    }
     if( nArg>2 ){
-      utf8_printf(stderr, "Usage: .%s FILE\n", azArg[0]);
+      utf8_printf(stderr, "Usage: .%s [-e|-x|FILE]\n", azArg[0]);
       rc = 1;
       goto meta_command_exit;
     }
     if( n>1 && strncmp(azArg[0], "once", n)==0 ){
       if( nArg<2 ){
-        raw_printf(stderr, "Usage: .once FILE\n");
+        raw_printf(stderr, "Usage: .once (-e|-x|FILE)\n");
         rc = 1;
         goto meta_command_exit;
       }
@@ -6129,6 +6193,19 @@ static int do_meta_command(char *zLine, ShellState *p){
       p->outCount = 0;
     }
     output_reset(p);
+    if( zFile[0]=='-' && zFile[1]=='-' ) zFile++;
+    if( strcmp(zFile, "-e")==0 || strcmp(zFile, "-x")==0 ){
+      p->doXdgOpen = p->mode + 1;
+      if( zFile[1]=='x' ){
+        newTempFile(p, "csv");
+        p->mode = MODE_Csv;
+        sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma);
+        sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_CrLf);
+      }else{
+        newTempFile(p, "txt");
+      }
+      zFile = p->zTempFile;
+    }
     if( zFile[0]=='|' ){
 #ifdef SQLITE_OMIT_POPEN
       raw_printf(stderr, "Error: pipes are not supported in this OS\n");
@@ -7558,6 +7635,8 @@ static int process_input(ShellState *p, FILE *in){
       if( p->outCount ){
         output_reset(p);
         p->outCount = 0;
+      }else{
+        clearTempFile(p);
       }
     }else if( nSql && _all_whitespace(zSql) ){
       if( ShellHasFlag(p, SHFLG_Echo) ) printf("%s\n", zSql);
@@ -8180,6 +8259,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
   }
   sqlite3_free(data.zFreeOnClose);
   find_home_dir(1);
+  clearTempFile(&data);
 #if !SQLITE_SHELL_IS_UTF8
   for(i=0; i<argc; i++) sqlite3_free(argv[i]);
   sqlite3_free(argv);

From 7f3bf8a9fb9144422cb3d3c1936087f13ab46771 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 10 Jan 2018 21:50:08 +0000
Subject: [PATCH 297/488] Fix a potential SQLITE_MISUSE in the .excel command
 when no database is open.

FossilOrigin-Name: 9b95ff1abfb8d49bbe5a727f5c917a455e4289b4d69196377dc9294409341d70
---
 manifest       | 15 ++++++---------
 manifest.uuid  |  2 +-
 src/shell.c.in |  4 +++-
 3 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index 751835d013..6b08444735 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\ssupport\sfor\sthe\s".excel"\scommand\s(and\s".once\s-e"\sand\s".once\s-x")\sin\nthe\sCLI.
-D 2018-01-10T21:41:55.211
+C Fix\sa\spotential\sSQLITE_MISUSE\sin\sthe\s.excel\scommand\swhen\sno\sdatabase\sis\sopen.
+D 2018-01-10T21:50:08.964
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -484,7 +484,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in d1dbc1514d74b425db41be006f2a4f91d26c5e1c2db4ab198b45c12920171e11
+F src/shell.c.in 362e3af76b80c3bd688f1f6fc8df407e651b71a65f0d919287eb6db5c357260c
 F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1697,10 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 60c694c1ab26a7a096f17ccea5a93ecda0f9f2113ab5fdc8b17dbffc787724fc
-R bd40eb18946fee4f15d02cd4e13ad69c
-T *branch * excel-shell-cmd
-T *sym-excel-shell-cmd *
-T -sym-trunk *
+P 23fa7c57c2b204d1ddcc2a939b5271628cf26689ad4ede6976038113095a9801
+R ba511ddf16c10fcf8fdff3518b957cd4
 U drh
-Z f708fb338ffb9679393be576ce4d2140
+Z 0681d6d553d7112baca91b9b1918eb9d
diff --git a/manifest.uuid b/manifest.uuid
index 5bb31c1d62..4442a7e3cf 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-23fa7c57c2b204d1ddcc2a939b5271628cf26689ad4ede6976038113095a9801
\ No newline at end of file
+9b95ff1abfb8d49bbe5a727f5c917a455e4289b4d69196377dc9294409341d70
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 81677bf5e8..c76b305450 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -4234,7 +4234,9 @@ static void newTempFile(ShellState *p, const char *zSuffix){
   clearTempFile(p);
   sqlite3_free(p->zTempFile);
   p->zTempFile = 0;
-  sqlite3_file_control(p->db, 0, SQLITE_FCNTL_TEMPFILENAME, &p->zTempFile);
+  if( p->db ){
+    sqlite3_file_control(p->db, 0, SQLITE_FCNTL_TEMPFILENAME, &p->zTempFile);
+  }
   if( p->zTempFile==0 ){
     sqlite3_uint64 r;
     sqlite3_randomness(sizeof(r), &r);

From a92a01a77eadf7a582f2973efe2fd371c38fe92b Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 10 Jan 2018 22:15:37 +0000
Subject: [PATCH 298/488] An attempt to get ".once -e" working reliably on
 Windows.

FossilOrigin-Name: 9b97f9d2c876162139dbd9485fcf68412d1572d9ddc179b08938b8a602e895d6
---
 manifest       | 12 ++++++------
 manifest.uuid  |  2 +-
 src/shell.c.in | 18 +++++++++++-------
 3 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/manifest b/manifest
index 6b08444735..acd4d81085 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\spotential\sSQLITE_MISUSE\sin\sthe\s.excel\scommand\swhen\sno\sdatabase\sis\sopen.
-D 2018-01-10T21:50:08.964
+C An\sattempt\sto\sget\s".once\s-e"\sworking\sreliably\son\sWindows.
+D 2018-01-10T22:15:37.810
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -484,7 +484,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in 362e3af76b80c3bd688f1f6fc8df407e651b71a65f0d919287eb6db5c357260c
+F src/shell.c.in 8ac56c0e57d87f04c7c021d17d541063fd4cd9e5cd558caf40a63bf42875b8a9
 F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 23fa7c57c2b204d1ddcc2a939b5271628cf26689ad4ede6976038113095a9801
-R ba511ddf16c10fcf8fdff3518b957cd4
+P 9b95ff1abfb8d49bbe5a727f5c917a455e4289b4d69196377dc9294409341d70
+R 0c911dcb6179b38e957861df043bdcc3
 U drh
-Z 0681d6d553d7112baca91b9b1918eb9d
+Z 8c9fcb0a09d556e95bbd5b8371ae82b1
diff --git a/manifest.uuid b/manifest.uuid
index 4442a7e3cf..e5bf4629ab 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-9b95ff1abfb8d49bbe5a727f5c917a455e4289b4d69196377dc9294409341d70
\ No newline at end of file
+9b97f9d2c876162139dbd9485fcf68412d1572d9ddc179b08938b8a602e895d6
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index c76b305450..fae70a1ece 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -3507,7 +3507,7 @@ static void output_file_close(FILE *f){
 ** recognized and do the right thing.  NULL is returned if the output
 ** filename is "off".
 */
-static FILE *output_file_open(const char *zFile){
+static FILE *output_file_open(const char *zFile, int bTextMode){
   FILE *f;
   if( strcmp(zFile,"stdout")==0 ){
     f = stdout;
@@ -3516,7 +3516,7 @@ static FILE *output_file_open(const char *zFile){
   }else if( strcmp(zFile, "off")==0 ){
     f = 0;
   }else{
-    f = fopen(zFile, "wb");
+    f = fopen(zFile, bTextMode ? "w" : "wb");
     if( f==0 ){
       utf8_printf(stderr, "Error: cannot open \"%s\"\n", zFile);
     }
@@ -3953,7 +3953,9 @@ static void output_reset(ShellState *p){
 #endif
       char *zCmd;
       zCmd = sqlite3_mprintf("%s %s", zXdgOpenCmd, p->zTempFile);
-      system(zCmd);
+      if( system(zCmd) ){
+        utf8_printf(stderr, "Failed: [%s]\n", zCmd);
+      }
       sqlite3_free(zCmd);
       p->mode = p->doXdgOpen - 1;
       p->doXdgOpen = 0;
@@ -6058,7 +6060,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     }else{
       const char *zFile = azArg[1];
       output_file_close(p->pLog);
-      p->pLog = output_file_open(zFile);
+      p->pLog = output_file_open(zFile, 0);
     }
   }else
 
@@ -6172,6 +6174,7 @@ static int do_meta_command(char *zLine, ShellState *p){
    || (c=='e' && n==5 && strcmp(azArg[0],"excel")==0)
   ){
     const char *zFile = nArg>=2 ? azArg[1] : "stdout";
+    int bTxtMode = 0;
     if( azArg[0][0]=='e' ){
       /* Transform the ".excel" command into ".once -x" */
       nArg = 2;
@@ -6205,6 +6208,7 @@ static int do_meta_command(char *zLine, ShellState *p){
         sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_CrLf);
       }else{
         newTempFile(p, "txt");
+        bTxtMode = 1;
       }
       zFile = p->zTempFile;
     }
@@ -6224,7 +6228,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       }
 #endif
     }else{
-      p->out = output_file_open(zFile);
+      p->out = output_file_open(zFile, bTxtMode);
       if( p->out==0 ){
         if( strcmp(zFile,"off")!=0 ){
           utf8_printf(stderr,"Error: cannot write to \"%s\"\n", zFile);
@@ -7101,7 +7105,7 @@ static int do_meta_command(char *zLine, ShellState *p){
   /* Begin redirecting output to the file "testcase-out.txt" */
   if( c=='t' && strcmp(azArg[0],"testcase")==0 ){
     output_reset(p);
-    p->out = output_file_open("testcase-out.txt");
+    p->out = output_file_open("testcase-out.txt", 0);
     if( p->out==0 ){
       raw_printf(stderr, "Error: cannot open 'testcase-out.txt'\n");
     }
@@ -7307,7 +7311,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       goto meta_command_exit;
     }
     output_file_close(p->traceOut);
-    p->traceOut = output_file_open(azArg[1]);
+    p->traceOut = output_file_open(azArg[1], 0);
 #if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT)
     if( p->traceOut==0 ){
       sqlite3_trace_v2(p->db, 0, 0, 0);

From 3c484e8c1f87eddca84af1b63f3751c6265614e8 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 10 Jan 2018 22:27:21 +0000
Subject: [PATCH 299/488] Save and restore the output mode when doing ".once
 -x" or ".excel".

FossilOrigin-Name: f697c164518d36f2a63c87d9f2708d0f9481fad3ded2de61f3f48c393cf7a500
---
 manifest       | 12 ++++++------
 manifest.uuid  |  2 +-
 src/shell.c.in | 22 ++++++++++++++++++++--
 3 files changed, 27 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index acd4d81085..34e956876e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C An\sattempt\sto\sget\s".once\s-e"\sworking\sreliably\son\sWindows.
-D 2018-01-10T22:15:37.810
+C Save\sand\srestore\sthe\soutput\smode\swhen\sdoing\s".once\s-x"\sor\s".excel".
+D 2018-01-10T22:27:21.115
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -484,7 +484,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in 8ac56c0e57d87f04c7c021d17d541063fd4cd9e5cd558caf40a63bf42875b8a9
+F src/shell.c.in 0baa3d017e3e46ed935413f7e8d09b8c77a8f870c84cf8b2d8b81528517bf485
 F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 9b95ff1abfb8d49bbe5a727f5c917a455e4289b4d69196377dc9294409341d70
-R 0c911dcb6179b38e957861df043bdcc3
+P 9b97f9d2c876162139dbd9485fcf68412d1572d9ddc179b08938b8a602e895d6
+R d864377b5e3ba9e6ed72c01bab712abd
 U drh
-Z 8c9fcb0a09d556e95bbd5b8371ae82b1
+Z 372298188cf6cc90b3d7d11dd46f07c5
diff --git a/manifest.uuid b/manifest.uuid
index e5bf4629ab..3358c70b40 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-9b97f9d2c876162139dbd9485fcf68412d1572d9ddc179b08938b8a602e895d6
\ No newline at end of file
+f697c164518d36f2a63c87d9f2708d0f9481fad3ded2de61f3f48c393cf7a500
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index fae70a1ece..e9c268e42a 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -1019,6 +1019,7 @@ struct ShellState {
   FILE *traceOut;        /* Output for sqlite3_trace() */
   int nErr;              /* Number of errors seen */
   int mode;              /* An output mode setting */
+  int modePrior;         /* Saved mode */
   int cMode;             /* temporary output mode for the current query */
   int normalMode;        /* Output mode before ".explain on" */
   int writableSchema;    /* True if PRAGMA writable_schema=ON */
@@ -1030,6 +1031,8 @@ struct ShellState {
   char zTestcase[30];    /* Name of current test case */
   char colSeparator[20]; /* Column separator character for several modes */
   char rowSeparator[20]; /* Row separator character for MODE_Ascii */
+  char colSepPrior[20];  /* Saved column separator */
+  char rowSepPrior[20];  /* Saved row separator */
   int colWidth[100];     /* Requested width of each column when in column mode*/
   int actualWidth[100];  /* Actual width of each column */
   char nullValue[20];    /* The text to print when a NULL comes back from
@@ -1153,6 +1156,20 @@ static void shellPutsFunc(
   sqlite3_result_value(pCtx, apVal[0]);
 }
 
+/*
+** Save or restore the current output mode
+*/
+static void outputModePush(ShellState *p){
+  p->modePrior = p->mode;
+  memcpy(p->colSepPrior, p->colSeparator, sizeof(p->colSeparator));
+  memcpy(p->rowSepPrior, p->rowSeparator, sizeof(p->rowSeparator));
+}
+static void outputModePop(ShellState *p){
+  p->mode = p->modePrior;
+  memcpy(p->colSeparator, p->colSepPrior, sizeof(p->colSeparator));
+  memcpy(p->rowSeparator, p->rowSepPrior, sizeof(p->rowSeparator));
+}
+
 /*
 ** Output the given string as a hex-encoded blob (eg. X'1234' )
 */
@@ -3957,7 +3974,7 @@ static void output_reset(ShellState *p){
         utf8_printf(stderr, "Failed: [%s]\n", zCmd);
       }
       sqlite3_free(zCmd);
-      p->mode = p->doXdgOpen - 1;
+      outputModePop(p);
       p->doXdgOpen = 0;
     }
   }
@@ -6200,7 +6217,8 @@ static int do_meta_command(char *zLine, ShellState *p){
     output_reset(p);
     if( zFile[0]=='-' && zFile[1]=='-' ) zFile++;
     if( strcmp(zFile, "-e")==0 || strcmp(zFile, "-x")==0 ){
-      p->doXdgOpen = p->mode + 1;
+      p->doXdgOpen = 1;
+      outputModePush(p);
       if( zFile[1]=='x' ){
         newTempFile(p, "csv");
         p->mode = MODE_Csv;

From 6c9dcc427d38a3eccf4ed8ae5b4218c981a737d0 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 10 Jan 2018 23:27:30 +0000
Subject: [PATCH 300/488] Update test cases for the new "usage" for .output.

FossilOrigin-Name: fbf5e43c07e7c012cb39b33a74b3fab9e46ba946c48497fbd990110692125f57
---
 manifest         | 12 ++++++------
 manifest.uuid    |  2 +-
 test/shell1.test |  4 ++--
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index 34e956876e..4766b6496a 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Save\sand\srestore\sthe\soutput\smode\swhen\sdoing\s".once\s-x"\sor\s".excel".
-D 2018-01-10T22:27:21.115
+C Update\stest\scases\sfor\sthe\snew\s"usage"\sfor\s.output.
+D 2018-01-10T23:27:30.001
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -1218,7 +1218,7 @@ F test/sharedA.test 0cdf1a76dfa00e6beee66af5b534b1e8df2720f5
 F test/sharedB.test 16cc7178e20965d75278f410943109b77b2e645e
 F test/shared_err.test 32634e404a3317eeb94abc7a099c556a346fdb8fb3858dbe222a4cbb8926a939
 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304
-F test/shell1.test 6d69e08039aea13f2c42749f162fe05eab7b5c93729f31d49d7d27cf36226e5a
+F test/shell1.test 9f8b8da05a79b134e252a5e1d8d411245ad83ac7126c262900b9f42b43108ffd
 F test/shell2.test e242a9912f44f4c23c3d1d802a83e934e84c853b
 F test/shell3.test ac8c2b744014c3e9a0e26bfd829ab65f00923dc1a91ffd044863e9423cc91494
 F test/shell4.test 89ad573879a745974ff2df20ff97c5d6ffffbd5d
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 9b97f9d2c876162139dbd9485fcf68412d1572d9ddc179b08938b8a602e895d6
-R d864377b5e3ba9e6ed72c01bab712abd
+P f697c164518d36f2a63c87d9f2708d0f9481fad3ded2de61f3f48c393cf7a500
+R cbc8182064251451cd032585a1a8098f
 U drh
-Z 372298188cf6cc90b3d7d11dd46f07c5
+Z 0fca8af732176ea4c3e2417e5169029d
diff --git a/manifest.uuid b/manifest.uuid
index 3358c70b40..6ef1621db8 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f697c164518d36f2a63c87d9f2708d0f9481fad3ded2de61f3f48c393cf7a500
\ No newline at end of file
+fbf5e43c07e7c012cb39b33a74b3fab9e46ba946c48497fbd990110692125f57
\ No newline at end of file
diff --git a/test/shell1.test b/test/shell1.test
index 0d03c64f78..81883a8d18 100644
--- a/test/shell1.test
+++ b/test/shell1.test
@@ -495,7 +495,7 @@ do_test shell1-3.15.2 {
 do_test shell1-3.15.3 {
   # too many arguments
   catchcmd "test.db" ".output FOO BAD"
-} {1 {Usage: .output FILE}}
+} {1 {Usage: .output [-e|-x|FILE]}}
 
 # .output stdout         Send output to the screen
 do_test shell1-3.16.1 {
@@ -504,7 +504,7 @@ do_test shell1-3.16.1 {
 do_test shell1-3.16.2 {
   # too many arguments
   catchcmd "test.db" ".output stdout BAD"
-} {1 {Usage: .output FILE}}
+} {1 {Usage: .output [-e|-x|FILE]}}
 
 # .prompt MAIN CONTINUE  Replace the standard prompts
 do_test shell1-3.17.1 {

From 97913134747525664f280839889cd66718752cb3 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 11 Jan 2018 00:04:00 +0000
Subject: [PATCH 301/488] Add the built-in edit(VALUE) and edit(VALUE,EDITOR)
 function to the CLI.

FossilOrigin-Name: ef823c47a0ac4f3b001e6c89c38354b45b5229d872cfc69a29c82e99414b89c8
---
 manifest       |  13 +++--
 manifest.uuid  |   2 +-
 src/shell.c.in | 130 +++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 137 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 18f1e54d50..f1234ab340 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\s".once\s-e"\sand\s".once\s-x"\scommands\sto\sthe\sCLI,\sand\sthe\s".excel"\scommand\nthat\sis\san\salias\sfor\s".once\s-x".
-D 2018-01-10T23:29:42.447
+C Add\sthe\sbuilt-in\sedit(VALUE)\sand\sedit(VALUE,EDITOR)\sfunction\sto\sthe\sCLI.
+D 2018-01-11T00:04:00.842
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -484,7 +484,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in 0baa3d017e3e46ed935413f7e8d09b8c77a8f870c84cf8b2d8b81528517bf485
+F src/shell.c.in 52aef9f2f61915c9226d6ff87444102b744a48e7b001e50d88d262fed3526030
 F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1697,8 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 60c694c1ab26a7a096f17ccea5a93ecda0f9f2113ab5fdc8b17dbffc787724fc fbf5e43c07e7c012cb39b33a74b3fab9e46ba946c48497fbd990110692125f57
-R cbc8182064251451cd032585a1a8098f
-T +closed fbf5e43c07e7c012cb39b33a74b3fab9e46ba946c48497fbd990110692125f57
+P a6ed5ab9bc9741267fd1207e63b4c49d074291085c7e3f25e9f62a82f916a656
+R 178e506824f095711305ed729320460e
 U drh
-Z c9d5887fceb0e0e24e4f6ea11d33b5eb
+Z c1ee7e4e075a7abb60497a235dd7f461
diff --git a/manifest.uuid b/manifest.uuid
index ff97541f24..07d9f13bed 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a6ed5ab9bc9741267fd1207e63b4c49d074291085c7e3f25e9f62a82f916a656
\ No newline at end of file
+ef823c47a0ac4f3b001e6c89c38354b45b5229d872cfc69a29c82e99414b89c8
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index e9c268e42a..e03eebd042 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -1156,6 +1156,132 @@ static void shellPutsFunc(
   sqlite3_result_value(pCtx, apVal[0]);
 }
 
+/*
+** SQL function:   edit(VALUE)
+**                 edit(VALUE,EDITOR)
+**
+** These steps:
+**
+**     (1) Write VALUE into a temporary file.
+**     (2) Run program EDITOR on that temporary file.
+**     (3) Read the temporary file back and return its content as the result.
+**     (4) Delete the temporary file
+**
+** If the EDITOR argument is omitted, use the value in the VISUAL
+** environment variable.  If still there is no EDITOR, through an error.
+**
+** Also throw an error if the EDITOR program returns a non-zero exit code.
+*/
+static void editFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  const char *zEditor;
+  char *zTempFile = 0;
+  sqlite3 *db;
+  char *zCmd = 0;
+  int bBin;
+  int rc;
+  FILE *f = 0;
+  sqlite3_int64 sz;
+  sqlite3_int64 x;
+  unsigned char *p = 0;
+
+  if( argc==2 ){
+    zEditor = (const char*)sqlite3_value_text(argv[1]);
+  }else{
+    zEditor = getenv("VISUAL");
+  }
+  if( zEditor==0 ){
+    sqlite3_result_error(context, "no editor for edit()", -1);
+    return;
+  }
+  if( sqlite3_value_type(argv[0])==SQLITE_NULL ){
+    sqlite3_result_error(context, "NULL input to edit()", -1);
+    return;
+  }
+  db = sqlite3_context_db_handle(context);
+  zTempFile = 0;
+  sqlite3_file_control(db, 0, SQLITE_FCNTL_TEMPFILENAME, &zTempFile);
+  if( zTempFile==0 ){
+    sqlite3_uint64 r = 0;
+    sqlite3_randomness(sizeof(r), &r);
+    zTempFile = sqlite3_mprintf("temp%llx", r);
+    if( zTempFile==0 ){
+      sqlite3_result_error_nomem(context);
+      return;
+    }
+  }
+  bBin = sqlite3_value_type(argv[0])==SQLITE_BLOB;
+  f = fopen(zTempFile, bBin ? "wb" : "w");
+  if( f==0 ){
+    sqlite3_result_error(context, "edit() cannot open temp file", -1);
+    goto edit_func_end;
+  }
+  sz = sqlite3_value_bytes(argv[0]);
+  if( bBin ){
+    x = fwrite(sqlite3_value_blob(argv[0]), 1, sz, f);
+  }else{
+    x = fwrite(sqlite3_value_text(argv[0]), 1, sz, f);
+  }
+  fclose(f);
+  f = 0;
+  if( x!=sz ){
+    sqlite3_result_error(context, "edit() could not write the whole file", -1);
+    goto edit_func_end;
+  }
+  zCmd = sqlite3_mprintf("%s \"%s\"", zEditor, zTempFile);
+  if( zCmd==0 ){
+    sqlite3_result_error_nomem(context);
+    goto edit_func_end;
+  }
+  rc = system(zCmd);
+  sqlite3_free(zCmd);
+  if( rc ){
+    sqlite3_result_error(context, "EDITOR returned non-zero", -1);
+    goto edit_func_end;
+  }
+  f = fopen(zTempFile, bBin ? "rb" : "r");
+  if( f==0 ){
+    sqlite3_result_error(context,
+      "edit() cannot reopen temp file after edit", -1);
+    goto edit_func_end;
+  }
+  fseek(f, 0, SEEK_END);
+  sz = ftell(f);
+  rewind(f);
+  p = sqlite3_malloc64( sz+(bBin==0) );
+  if( p==0 ){
+    sqlite3_result_error_nomem(context);
+    goto edit_func_end;
+  }
+  if( bBin ){
+    x = fread(p, 1, sz, f);
+  }else{
+    x = fread(p, 1, sz, f);
+    p[sz] = 0;
+  }
+  fclose(f);
+  f = 0;
+  if( x!=sz ){
+    sqlite3_result_error(context, "could not read back the whole file", -1);
+    goto edit_func_end;
+  }
+  if( bBin ){
+    sqlite3_result_blob(context, p, sz, sqlite3_free);
+  }else{
+    sqlite3_result_text(context, (const char*)p, sz, sqlite3_free);
+  }
+  p = 0;
+
+edit_func_end:
+  if( f ) fclose(f);
+  unlink(zTempFile);
+  sqlite3_free(zTempFile);
+  sqlite3_free(p);
+}
+
 /*
 ** Save or restore the current output mode
 */
@@ -3345,6 +3471,10 @@ static void open_db(ShellState *p, int keepAlive){
                             shellModuleSchema, 0, 0);
     sqlite3_create_function(p->db, "shell_putsnl", 1, SQLITE_UTF8, p,
                             shellPutsFunc, 0, 0);
+    sqlite3_create_function(p->db, "edit", 1, SQLITE_UTF8, 0,
+                            editFunc, 0, 0);
+    sqlite3_create_function(p->db, "edit", 2, SQLITE_UTF8, 0,
+                            editFunc, 0, 0);
     if( p->openMode==SHELL_OPEN_ZIPFILE ){
       char *zSql = sqlite3_mprintf(
          "CREATE VIRTUAL TABLE zip USING zipfile(%Q);", p->zDbFilename);

From 536c345107d784c85a4976ee576bee4554d35af3 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 11 Jan 2018 00:38:39 +0000
Subject: [PATCH 302/488] Update the .help screen in the CLI.  Make sure the
 temporary files for ".open" are not deleted too soon.

FossilOrigin-Name: b8d92d8dc239597c6f01a6e572b047f98ce374a8f48257683fa839dde3ec993f
---
 manifest       | 12 ++++++------
 manifest.uuid  |  2 +-
 src/shell.c.in |  9 +++++++--
 3 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index f1234ab340..45bcfa6cca 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\sbuilt-in\sedit(VALUE)\sand\sedit(VALUE,EDITOR)\sfunction\sto\sthe\sCLI.
-D 2018-01-11T00:04:00.842
+C Update\sthe\s.help\sscreen\sin\sthe\sCLI.\s\sMake\ssure\sthe\stemporary\sfiles\sfor\n".open"\sare\snot\sdeleted\stoo\ssoon.
+D 2018-01-11T00:38:39.192
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -484,7 +484,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
-F src/shell.c.in 52aef9f2f61915c9226d6ff87444102b744a48e7b001e50d88d262fed3526030
+F src/shell.c.in b87abffd0db09203ad8a133d56fe8f154ace5ec0a14197a153fb7d80b1438c01
 F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a6ed5ab9bc9741267fd1207e63b4c49d074291085c7e3f25e9f62a82f916a656
-R 178e506824f095711305ed729320460e
+P ef823c47a0ac4f3b001e6c89c38354b45b5229d872cfc69a29c82e99414b89c8
+R 3f7dbb61aadad578b23395ee859b8a4c
 U drh
-Z c1ee7e4e075a7abb60497a235dd7f461
+Z c33398289b510440ef343bbf6c1308f4
diff --git a/manifest.uuid b/manifest.uuid
index 07d9f13bed..9cf4eb9896 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ef823c47a0ac4f3b001e6c89c38354b45b5229d872cfc69a29c82e99414b89c8
\ No newline at end of file
+b8d92d8dc239597c6f01a6e572b047f98ce374a8f48257683fa839dde3ec993f
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index e03eebd042..ee21a83d6f 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -3247,10 +3247,12 @@ static char zHelp[] =
   "                         tabs     Tab-separated values\n"
   "                         tcl      TCL list elements\n"
   ".nullvalue STRING      Use STRING in place of NULL values\n"
-  ".once FILENAME         Output for the next SQL command only to FILENAME\n"
+  ".once (-e|-x|FILE)     Output for the next SQL command only to FILE\n"
+  "                         or invoke system text editor (-e) or spreadsheet (-x)\n"
+  "                         on the output.\n"
   ".open ?OPTIONS? ?FILE? Close existing database and reopen FILE\n"
   "                         The --new option starts with an empty file\n"
-  ".output ?FILENAME?     Send output to FILENAME or stdout\n"
+  ".output ?FILE?         Send output to FILE or stdout\n"
   ".print STRING...       Print literal STRING\n"
   ".prompt MAIN CONTINUE  Replace the standard prompts\n"
   ".quit                  Exit this program\n"
@@ -4371,6 +4373,7 @@ int shellDeleteFile(const char *zFilename){
 */
 static void clearTempFile(ShellState *p){
   if( p->zTempFile==0 ) return;
+  if( p->doXdgOpen ) return;
   if( shellDeleteFile(p->zTempFile) ) return;
   sqlite3_free(p->zTempFile);
   p->zTempFile = 0;
@@ -8413,6 +8416,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
   }
   sqlite3_free(data.zFreeOnClose);
   find_home_dir(1);
+  output_reset(&data);
+  data.doXdgOpen = 0;
   clearTempFile(&data);
 #if !SQLITE_SHELL_IS_UTF8
   for(i=0; i<argc; i++) sqlite3_free(argv[i]);

From 24cd20ffa4acbebe5896bc52d927da27710f32ad Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 11 Jan 2018 13:43:04 +0000
Subject: [PATCH 303/488] Fix typo in comment.  No code changes.

FossilOrigin-Name: 2c55571baaae58d6f5b6210cf85d0fa325a9977682bd449e3802191f7a4142de
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/update.c  |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 45bcfa6cca..67e2fbd266 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\sthe\s.help\sscreen\sin\sthe\sCLI.\s\sMake\ssure\sthe\stemporary\sfiles\sfor\n".open"\sare\snot\sdeleted\stoo\ssoon.
-D 2018-01-11T00:38:39.192
+C Fix\stypo\sin\scomment.\s\sNo\scode\schanges.
+D 2018-01-11T13:43:04.788
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -548,7 +548,7 @@ F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
 F src/tokenize.c 5b0c661a85f783d35b9883830736eeb63be4aefc4f6b7d9cd081d48782c041e2
 F src/treeview.c eae35972ff44f67064de2eaf35f04afe94e7aea3271a8b3bcebb3f954880fec3
 F src/trigger.c a34539c69433276d37b0da9a89c117726ff2d292c0902895af1f393a983cd3a1
-F src/update.c 961bd1265d4d1e5cd65c9a54fa5122fb7aefcb003fcf2de0c092fceb7e58972c
+F src/update.c f75a8bf497219fa0c4ba9293022db64cca13e457e33d62f58306f107327506ac
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c 7315e97a8dc2c8e19ca64196c652cf0a65d13fd0a211b2cec082062372dc6261
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ef823c47a0ac4f3b001e6c89c38354b45b5229d872cfc69a29c82e99414b89c8
-R 3f7dbb61aadad578b23395ee859b8a4c
+P b8d92d8dc239597c6f01a6e572b047f98ce374a8f48257683fa839dde3ec993f
+R 7aa5de08d44a1e2c05aadb2e5fa21a20
 U drh
-Z c33398289b510440ef343bbf6c1308f4
+Z af4c8b3a64ae000c2affb5a4b97dbea6
diff --git a/manifest.uuid b/manifest.uuid
index 9cf4eb9896..f5e80a94e6 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b8d92d8dc239597c6f01a6e572b047f98ce374a8f48257683fa839dde3ec993f
\ No newline at end of file
+2c55571baaae58d6f5b6210cf85d0fa325a9977682bd449e3802191f7a4142de
\ No newline at end of file
diff --git a/src/update.c b/src/update.c
index 15e8f4a6ce..3e9d11bbf7 100644
--- a/src/update.c
+++ b/src/update.c
@@ -807,7 +807,7 @@ static void updateVirtualTable(
   int bOnePass;                   /* True to use onepass strategy */
   int addr;                       /* Address of OP_OpenEphemeral */
 
-  /* Allocate nArg registers to martial the arguments to VUpdate. Then
+  /* Allocate nArg registers in which to gather the arguments for VUpdate. Then
   ** create and open the ephemeral table in which the records created from
   ** these arguments will be temporarily stored. */
   assert( v );

From e4185bda9ac72005bae1308665d07596573f7cf5 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 11 Jan 2018 14:50:40 +0000
Subject: [PATCH 304/488] Minor formatting changes in zipfile.c.  No logical
 code differences.

FossilOrigin-Name: 4f68bed3b9a63a349a2a2d7f26609491577e9717034ad86af404cf9eed9d6aaf
---
 ext/misc/zipfile.c | 36 +++++++++++++++++++++++++-----------
 manifest           | 12 ++++++------
 manifest.uuid      |  2 +-
 3 files changed, 32 insertions(+), 18 deletions(-)

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index 1a65a8896b..280bc6a4ed 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -10,6 +10,19 @@
 **
 ******************************************************************************
 **
+** This file implements a virtual table for reading and writing ZIP archive
+** files.
+**
+** Usage example:
+**
+**     SELECT name, sz, datetime(mtime,'unixepoch') FROM zipfile($filename);
+**
+** Current limitations:
+**
+**    *  No support for encryption
+**    *  No support for ZIP archives spanning multiple files
+**    *  No support for zip64 extensions
+**    *  Only the "inflate/deflate" (zlib) compression method is supported
 */
 #include "sqlite3ext.h"
 SQLITE_EXTENSION_INIT1
@@ -42,18 +55,19 @@ typedef unsigned long u32;
 #define MIN(a,b) ((a)<(b) ? (a) : (b))
 #endif
 
-#define ZIPFILE_SCHEMA "CREATE TABLE y("                              \
-  "name,      /* 0: Name of file in zip archive */"                   \
-  "mode,      /* 1: POSIX mode for file */"                           \
-  "mtime,     /* 2: Last modification time in seconds since epoch */" \
-  "sz,        /* 3: Size of object */"                                \
-  "rawdata,   /* 4: Raw data */"                                      \
-  "data,      /* 5: Uncompressed data */"                             \
-  "method,    /* 6: Compression method (integer) */"                  \
-  "file HIDDEN   /* Name of zip file */"                              \
-");"
+static const char ZIPFILE_SCHEMA[] = 
+  "CREATE TABLE y("
+    "name,"              /* 0: Name of file in zip archive */
+    "mode,"              /* 1: POSIX mode for file */
+    "mtime,"             /* 2: Last modification time (secs since 1970)*/
+    "sz,"                /* 3: Size of object */
+    "rawdata,"           /* 4: Raw data */
+    "data,"              /* 5: Uncompressed data */
+    "method,"            /* 6: Compression method (integer) */
+    "file HIDDEN"        /* 7: Name of zip file */
+  ");";
 
-#define ZIPFILE_F_COLUMN_IDX 7    /* Index of column "f" in the above */
+#define ZIPFILE_F_COLUMN_IDX 7    /* Index of column "file" in the above */
 #define ZIPFILE_BUFFER_SIZE (64*1024)
 
 
diff --git a/manifest b/manifest
index 67e2fbd266..ee79e40a2b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\stypo\sin\scomment.\s\sNo\scode\schanges.
-D 2018-01-11T13:43:04.788
+C Minor\sformatting\schanges\sin\szipfile.c.\s\sNo\slogical\scode\sdifferences.
+D 2018-01-11T14:50:40.176
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -303,7 +303,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c 0d12ad852af1732c9a4bd0be9f28886b049e0591a871e3246a5347d2cea15430
+F ext/misc/zipfile.c cbf9dfaf94c25dfd835e79c1474fdc75517a45a37916dec66623b0937af10e51
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -1697,7 +1697,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b8d92d8dc239597c6f01a6e572b047f98ce374a8f48257683fa839dde3ec993f
-R 7aa5de08d44a1e2c05aadb2e5fa21a20
+P 2c55571baaae58d6f5b6210cf85d0fa325a9977682bd449e3802191f7a4142de
+R 33de751c900466ee2a12d260dcb2a3de
 U drh
-Z af4c8b3a64ae000c2affb5a4b97dbea6
+Z 324651d5684ea52cfa823d4ea5809b01
diff --git a/manifest.uuid b/manifest.uuid
index f5e80a94e6..774aa87cbe 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2c55571baaae58d6f5b6210cf85d0fa325a9977682bd449e3802191f7a4142de
\ No newline at end of file
+4f68bed3b9a63a349a2a2d7f26609491577e9717034ad86af404cf9eed9d6aaf
\ No newline at end of file

From 5e9bbe1924b8ba158bb697a3d4568bab09f39f39 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 11 Jan 2018 16:16:03 +0000
Subject: [PATCH 305/488] Add test cases for running multiple RBU operations
 within the same process concurrently.

FossilOrigin-Name: 407b5ed35c178bb0dbc69c8b902652038a0653d55a58a7543f9d4857c6baf3ea
---
 ext/rbu/rbumulti.test | 175 ++++++++++++++++++++++++++++++++++++++++++
 manifest              |  13 ++--
 manifest.uuid         |   2 +-
 3 files changed, 183 insertions(+), 7 deletions(-)
 create mode 100644 ext/rbu/rbumulti.test

diff --git a/ext/rbu/rbumulti.test b/ext/rbu/rbumulti.test
new file mode 100644
index 0000000000..59c6538c6c
--- /dev/null
+++ b/ext/rbu/rbumulti.test
@@ -0,0 +1,175 @@
+# 2018 January 11
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains tests of multiple RBU operations running
+# concurrently within the same process. 
+#
+
+source [file join [file dirname [info script]] rbu_common.tcl]
+set ::testprefix rbumulti
+
+db close
+sqlite3_shutdown
+sqlite3_config_uri 1
+
+autoinstall_test_functions
+
+proc build_db {db} {
+  $db eval {
+    CREATE TABLE t1(a PRIMARY KEY, b, c);
+    CREATE INDEX i1 ON t1(b);
+    CREATE INDEX i2 ON t1(c);
+
+    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<500 )
+    INSERT INTO t1 
+    SELECT randomblob(10), randomblob(100), randomblob(100) FROM s;
+  }
+}
+
+proc build_rbu {db} {
+  $db eval {
+    CREATE TABLE data_t1(a, b, c, rbu_control);
+    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100 )
+    INSERT INTO data_t1 
+    SELECT randomblob(10), randomblob(100), randomblob(100), 0 FROM s;
+  }
+}
+
+proc step_rbu2 {bOpenClose openr1 openr2} {
+
+  forcedelete teststate.db1
+  forcedelete teststate.db2
+
+  if {$bOpenClose!=0 && $bOpenClose!=1} { error $bOpenClose }
+  if {$bOpenClose==0} {
+    eval $openr1 
+    eval $openr2 
+  }
+
+  set b1 0
+  set b2 0
+
+  while {$b1==0 || $b2==0} {
+    if {$bOpenClose==1} {
+      if {$b1==0} { eval $openr1 teststate.db1 }
+      if {$b2==0} { eval $openr2 teststate.db2 }
+    }
+    if {$b1==0} {
+      set rc1 [r1 step]
+      if {$rc1 != "SQLITE_OK"} { set b1 1 }
+    }
+    if {$b2==0} {
+      set rc2 [r2 step]
+      if {$rc2 != "SQLITE_OK"} { set b2 1 }
+    }
+    if {$bOpenClose==1} {
+      if {$b1==0} { r1 close }
+      if {$b2==0} { r2 close }
+    }
+  }
+
+  set rc1 [r1 close]
+  set rc2 [r2 close]
+
+  list $rc1 $rc2
+}
+
+
+for {set i 0} {$i<=3} {incr i} {
+
+  if {$i & 0x01} {
+    sqlite3rbu_create_vfs -default myrbu ""
+  }
+  set bOpenClose [expr $i>>1]
+
+  forcedelete test.db
+  forcedelete test.db2
+  forcedelete rbu.db
+  forcedelete rbu.db2
+  
+  do_test 1.$i.0 {
+    sqlite3 db test.db
+    sqlite3 db2 test.db2
+    build_db db
+    build_db db2
+  
+    sqlite3 rbu1 rbu.db
+    sqlite3 rbu2 rbu.db2
+  
+    build_rbu rbu1
+    build_rbu rbu2
+  
+    rbu1 close
+    rbu2 close
+  } {}
+
+  set m1 [db eval {SELECT md5sum(a, b, c) FROM t1}]
+  set m2 [db2 eval {SELECT md5sum(a, b, c) FROM t1}]
+  
+  do_test 1.$i.1 {
+    step_rbu2 $bOpenClose {
+      sqlite3rbu r1 test.db rbu.db
+    } {
+      sqlite3rbu r2 test.db2 rbu.db2
+    }
+  } {SQLITE_DONE SQLITE_DONE}
+  
+  do_execsql_test -db db  1.$i.2.1 { PRAGMA integrity_check } ok
+  do_execsql_test -db db2 1.$i.2.2 { PRAGMA integrity_check } ok
+
+  do_execsql_test -db db  1.$i.3.1 { SELECT md5sum(a, b, c)==$m1 FROM t1 } 0
+  do_execsql_test -db db2 1.$i.3.2 { SELECT md5sum(a, b, c)==$m2 FROM t1 } 0
+  
+  catch { db close }
+  catch { db2 close }
+  #-----------------------------------------------------------------------
+  forcedelete test.db2
+  forcedelete test.db
+  forcedelete rbu.db2
+  
+  do_test 1.$i.4 {
+    sqlite3 db test.db
+    sqlite3 db2 test.db2
+    build_db db
+    build_db db2
+    sqlite3 rbu2 rbu.db2
+    build_rbu rbu2
+    rbu2 close
+  } {}
+
+  set m1 [db eval {SELECT md5sum(a, b, c) FROM t1}]
+  set m2 [db2 eval {SELECT md5sum(a, b, c) FROM t1}]
+
+  do_test 1.$i.5 {
+    step_rbu2 $bOpenClose {
+      sqlite3rbu_vacuum r1 test.db
+    } {
+      sqlite3rbu r2 test.db2 rbu.db2
+    }
+  } {SQLITE_DONE SQLITE_DONE}
+
+  do_execsql_test -db db  1.$i.6.1 { SELECT md5sum(a, b, c)==$m1 FROM t1 } 1
+  do_execsql_test -db db2 1.$i.6.2 { SELECT md5sum(a, b, c)==$m2 FROM t1 } 0
+
+  do_execsql_test -db db  1.$i.7.1 { PRAGMA integrity_check } ok
+  do_execsql_test -db db2 1.$i.7.2 { PRAGMA integrity_check } ok
+
+  catch { db close }
+  catch { db2 close }
+  if {$i & 0x01} {
+    sqlite3rbu_destroy_vfs myrbu
+  }
+
+}
+
+
+finish_test
+
diff --git a/manifest b/manifest
index ee79e40a2b..fcbf2dda8e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Minor\sformatting\schanges\sin\szipfile.c.\s\sNo\slogical\scode\sdifferences.
-D 2018-01-11T14:50:40.176
+C Add\stest\scases\sfor\srunning\smultiple\sRBU\soperations\swithin\sthe\ssame\sprocess\nconcurrently.
+D 2018-01-11T16:16:03.148
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -330,6 +330,7 @@ F ext/rbu/rbufault2.test 9a7f19edd6ea35c4c9f807d8a3db0a03a5670c06
 F ext/rbu/rbufault3.test 0913c1aeaee266d9c36c33179341a5a504aad7d423d1979cfec43c8346a29899
 F ext/rbu/rbufault4.test 34e70701cbec51571ffbd9fbf9d4e0f2ec495ca7
 F ext/rbu/rbufts.test a2bbd202c9321fba15fb4a62a90add7d70e07bd8404e1e598135adbfff8a0508
+F ext/rbu/rbumulti.test 2cf153ab3d5861ff26517dc6cbaec430787a59f1d50e8771fe7a7529a0551cf1
 F ext/rbu/rbuprogress.test 1849d4e0e50616edf5ce75ce7db86622e656b5cf
 F ext/rbu/rburesume.test 8acb77f4a422ff55acfcfc9cc15a5cb210b1de83
 F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48
@@ -1697,7 +1698,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2c55571baaae58d6f5b6210cf85d0fa325a9977682bd449e3802191f7a4142de
-R 33de751c900466ee2a12d260dcb2a3de
-U drh
-Z 324651d5684ea52cfa823d4ea5809b01
+P 4f68bed3b9a63a349a2a2d7f26609491577e9717034ad86af404cf9eed9d6aaf
+R b4712b12dc6463238ee69f57e3b3a2cb
+U dan
+Z e742283e882b3ab287b40d1d09763224
diff --git a/manifest.uuid b/manifest.uuid
index 774aa87cbe..0d3f74a2d6 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4f68bed3b9a63a349a2a2d7f26609491577e9717034ad86af404cf9eed9d6aaf
\ No newline at end of file
+407b5ed35c178bb0dbc69c8b902652038a0653d55a58a7543f9d4857c6baf3ea
\ No newline at end of file

From 6f390beb7f96774e81420548ef0be7c16567bca1 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 11 Jan 2018 17:04:26 +0000
Subject: [PATCH 306/488] Add the sqlite3_vtab_nochange() method which virtual
 table implementations can use to optimize UPDATEs.

FossilOrigin-Name: d444b1ff39f0a1673a977b8047e1e14a49d461c9934be080d27c2392a830c1c0
---
 manifest        | 23 +++++++++++++----------
 manifest.uuid   |  2 +-
 src/sqlite.h.in | 12 ++++++++++++
 src/update.c    |  2 +-
 src/vdbe.c      | 15 +++++++++++----
 src/vdbeInt.h   |  1 +
 src/vdbeapi.c   | 19 +++++++++++++++++++
 7 files changed, 58 insertions(+), 16 deletions(-)

diff --git a/manifest b/manifest
index ee79e40a2b..977b35bb02 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Minor\sformatting\schanges\sin\szipfile.c.\s\sNo\slogical\scode\sdifferences.
-D 2018-01-11T14:50:40.176
+C Add\sthe\ssqlite3_vtab_nochange()\smethod\swhich\svirtual\stable\simplementations\ncan\suse\sto\soptimize\sUPDATEs.
+D 2018-01-11T17:04:26.957
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -485,7 +485,7 @@ F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
 F src/shell.c.in b87abffd0db09203ad8a133d56fe8f154ace5ec0a14197a153fb7d80b1438c01
-F src/sqlite.h.in 1f1a2da222ec57465794e8984d77f32d0bd0da80cdc136beadda461a0be9d80c
+F src/sqlite.h.in f83e63a48fb31fefc69c83bbe8700b4b44acdd64e440219087b0f14e35eeb8d4
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
 F src/sqliteInt.h fd8702c65994d7de3e2d8f7d85d958731da1ed29476571fdfa2290fd8ec0bf80
@@ -548,14 +548,14 @@ F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
 F src/tokenize.c 5b0c661a85f783d35b9883830736eeb63be4aefc4f6b7d9cd081d48782c041e2
 F src/treeview.c eae35972ff44f67064de2eaf35f04afe94e7aea3271a8b3bcebb3f954880fec3
 F src/trigger.c a34539c69433276d37b0da9a89c117726ff2d292c0902895af1f393a983cd3a1
-F src/update.c f75a8bf497219fa0c4ba9293022db64cca13e457e33d62f58306f107327506ac
+F src/update.c 8bd52c38d6d426925be4488ee106db26d9ee344406315671ed246ddace8d6091
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c 7315e97a8dc2c8e19ca64196c652cf0a65d13fd0a211b2cec082062372dc6261
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
-F src/vdbe.c c70267613df80345ed9bbd9c249c365b53bb9cbbe9d3e78e6d5c049d00ef5501
+F src/vdbe.c 4d0666741d884dfaceb6ac5bbededbd947068e4b742163898feea84a027dace6
 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97
-F src/vdbeInt.h 1fe00770144c12c4913128f35262d11527ef3284561baaab59b947a41c08d0d9
-F src/vdbeapi.c 9c670ca0dcc1cd86373aa353b747b26fe531ca5cd4331690c611d1f03842e2a1
+F src/vdbeInt.h a0969c14950bcc61b2dfa19c6f62b42c5012013fcda342ca927a192ed37e6592
+F src/vdbeapi.c f519346f6db99a1eb3f85ca3ae8aede9ff9c4d90fec809dae7cb946786b2e270
 F src/vdbeaux.c 7ae48b180e5dd5d282e6752d155f1ab7929196d8e6577b82742044188152ca85
 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191
 F src/vdbemem.c 8478f7fb1948bf8fdeec7c2cb59ea58155c31258b9cd43c56d485e03ed40bd07
@@ -1697,7 +1697,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2c55571baaae58d6f5b6210cf85d0fa325a9977682bd449e3802191f7a4142de
-R 33de751c900466ee2a12d260dcb2a3de
+P 4f68bed3b9a63a349a2a2d7f26609491577e9717034ad86af404cf9eed9d6aaf
+R ca3c0ce043db4afb0303f36904592ef8
+T *branch * sqlite3_vtab_nochange
+T *sym-sqlite3_vtab_nochange *
+T -sym-trunk *
 U drh
-Z 324651d5684ea52cfa823d4ea5809b01
+Z 5a280540fff60985c8c72fc104fac16d
diff --git a/manifest.uuid b/manifest.uuid
index 774aa87cbe..3f07edeeb6 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4f68bed3b9a63a349a2a2d7f26609491577e9717034ad86af404cf9eed9d6aaf
\ No newline at end of file
+d444b1ff39f0a1673a977b8047e1e14a49d461c9934be080d27c2392a830c1c0
\ No newline at end of file
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index f63b029312..ba673748ed 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -8297,6 +8297,18 @@ int sqlite3_vtab_config(sqlite3*, int op, ...);
 */
 int sqlite3_vtab_on_conflict(sqlite3 *);
 
+/*
+** CAPI3REF: Determine If Virtual Table Column Access Is For UPDATE
+**
+** If the sqlite3_vtab_nochange(X) routine is called within the [xColumn]
+** method of a [virtual table], then it returns true if and only if the
+** column is being fetched as part of an UPDATE operation during which the
+** column value will not change.  Applications might use this to substitute
+** a lighter-weight value to return that the corresponding [xUpdate] method
+** understands as a "no-change" value.
+*/
+int sqlite3_vtab_nochange(sqlite3_context*);
+
 /*
 ** CAPI3REF: Determine The Collation For a Virtual Table Constraint
 **
diff --git a/src/update.c b/src/update.c
index 3e9d11bbf7..3de36fe211 100644
--- a/src/update.c
+++ b/src/update.c
@@ -827,7 +827,7 @@ static void updateVirtualTable(
     if( aXRef[i]>=0 ){
       sqlite3ExprCode(pParse, pChanges->a[aXRef[i]].pExpr, regArg+2+i);
     }else{
-      sqlite3VdbeAddOp3(v, OP_VColumn, iCsr, i, regArg+2+i);
+      sqlite3VdbeAddOp4Int(v, OP_VColumn, iCsr, i, regArg+2+i, 1);
     }
   }
   if( HasRowid(pTab) ){
diff --git a/src/vdbe.c b/src/vdbe.c
index 4d643d7266..b00174bb10 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -6693,12 +6693,18 @@ case OP_VFilter: {   /* jump */
 #endif /* SQLITE_OMIT_VIRTUALTABLE */
 
 #ifndef SQLITE_OMIT_VIRTUALTABLE
-/* Opcode: VColumn P1 P2 P3 * *
+/* Opcode: VColumn P1 P2 P3 P4 *
 ** Synopsis: r[P3]=vcolumn(P2)
 **
-** Store the value of the P2-th column of
-** the row of the virtual-table that the 
-** P1 cursor is pointing to into register P3.
+** Store in register P3 the value of the P2-th column of
+** the current row of the virtual-table of cursor P1.
+**
+** If the VColumn opcode is being used to fetch the value of
+** an unchanging column during an UPDATE operation, then the P4
+** value is 1.  Otherwise, P4 is 0.  The P4 value is returned
+** by sqlite3_vtab_nochange() routine can can be used
+** by virtual table implementations to return special "no-change"
+** marks which can be more efficient, depending on the virtual table.
 */
 case OP_VColumn: {
   sqlite3_vtab *pVtab;
@@ -6720,6 +6726,7 @@ case OP_VColumn: {
   assert( pModule->xColumn );
   memset(&sContext, 0, sizeof(sContext));
   sContext.pOut = pDest;
+  sContext.bVtabNoChng = pOp->p4.i!=0;
   MemSetTypeFlag(pDest, MEM_Null);
   rc = pModule->xColumn(pCur->uc.pVCur, &sContext, pOp->p2);
   sqlite3VtabImportErrmsg(p, pVtab);
diff --git a/src/vdbeInt.h b/src/vdbeInt.h
index cb783653c0..f646a4036a 100644
--- a/src/vdbeInt.h
+++ b/src/vdbeInt.h
@@ -318,6 +318,7 @@ struct sqlite3_context {
   int isError;            /* Error code returned by the function. */
   u8 skipFlag;            /* Skip accumulator loading if true */
   u8 fErrorOrAux;         /* isError!=0 or pVdbe->pAuxData modified */
+  u8 bVtabNoChng;         /* Fetching an unchanging column in a vtab UPDATE */
   u8 argc;                /* Number of arguments */
   sqlite3_value *argv[1]; /* Argument set */
 };
diff --git a/src/vdbeapi.c b/src/vdbeapi.c
index b9df40b8fd..19aa783bb6 100644
--- a/src/vdbeapi.c
+++ b/src/vdbeapi.c
@@ -745,6 +745,25 @@ sqlite3 *sqlite3_context_db_handle(sqlite3_context *p){
   return p->pOut->db;
 }
 
+/*
+** If this routine is invoked from within an xColumn method of a virtual
+** table, then it returns true if and only if the the call is during an
+** UPDATE operation and the value of the column will not be modified
+** by the UPDATE.
+**
+** If this routine is called from any context other than within the
+** xColumn method of a virtual table, then the return value is meaningless
+** and arbitrary.
+**
+** Virtual table implements might use this routine to optimize their
+** performance by substituting a NULL result, or some other light-weight
+** value, as a signal to the xUpdate routine that the column is unchanged.
+*/
+int sqlite3_vtab_nochange(sqlite3_context *p){
+  assert( p );
+  return p->bVtabNoChng;
+}
+
 /*
 ** Return the current time for a statement.  If the current time
 ** is requested more than once within the same run of a single prepared

From 66a3a91aa6ddc324dd396f0efc475f552c89c9cd Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 11 Jan 2018 17:33:48 +0000
Subject: [PATCH 307/488] Change zipfile to be a WITHOUT ROWID virtual table
 and table-valued function.

FossilOrigin-Name: 931201f64e04247ed613a0301fcc86c3a337c2ed162c6370a80c67a1dd919e7c
---
 ext/misc/zipfile.c | 19 ++++++-------------
 manifest           | 14 +++++++-------
 manifest.uuid      |  2 +-
 test/zipfile.test  |  6 +++++-
 4 files changed, 19 insertions(+), 22 deletions(-)

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index 280bc6a4ed..7ec8a222e7 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -57,7 +57,7 @@ typedef unsigned long u32;
 
 static const char ZIPFILE_SCHEMA[] = 
   "CREATE TABLE y("
-    "name,"              /* 0: Name of file in zip archive */
+    "name PRIMARY KEY,"  /* 0: Name of file in zip archive */
     "mode,"              /* 1: POSIX mode for file */
     "mtime,"             /* 2: Last modification time (secs since 1970)*/
     "sz,"                /* 3: Size of object */
@@ -65,7 +65,7 @@ static const char ZIPFILE_SCHEMA[] =
     "data,"              /* 5: Uncompressed data */
     "method,"            /* 6: Compression method (integer) */
     "file HIDDEN"        /* 7: Name of zip file */
-  ");";
+  ") WITHOUT ROWID;";
 
 #define ZIPFILE_F_COLUMN_IDX 7    /* Index of column "file" in the above */
 #define ZIPFILE_BUFFER_SIZE (64*1024)
@@ -223,7 +223,6 @@ struct ZipfileLFH {
 typedef struct ZipfileEntry ZipfileEntry;
 struct ZipfileEntry {
   char *zPath;               /* Path of zipfile entry */
-  i64 iRowid;                /* Rowid for this value if queried */
   u8 *aCdsEntry;             /* Buffer containing entire CDS entry */
   int nCdsEntry;             /* Size of buffer aCdsEntry[] in bytes */
   int bDeleted;              /* True if entry has been deleted */
@@ -830,12 +829,7 @@ static int zipfileColumn(
 ** Return the rowid for the current row.
 */
 static int zipfileRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
-  ZipfileCsr *pCsr = (ZipfileCsr*)cur;
-  if( pCsr->pCurrent ){
-    *pRowid = pCsr->pCurrent->iRowid;
-  }else{
-    *pRowid = pCsr->cds.iOffset;
-  }
+  assert( 0 );
   return SQLITE_OK;
 }
 
@@ -999,11 +993,9 @@ static void zipfileAddEntry(ZipfileTab *pTab, ZipfileEntry *pNew){
   assert( (pTab->pFirstEntry==0)==(pTab->pLastEntry==0) );
   assert( pNew->pNext==0 );
   if( pTab->pFirstEntry==0 ){
-    pNew->iRowid = 1;
     pTab->pFirstEntry = pTab->pLastEntry = pNew;
   }else{
     assert( pTab->pLastEntry->pNext==0 );
-    pNew->iRowid = pTab->pLastEntry->iRowid+1;
     pTab->pLastEntry->pNext = pNew;
     pTab->pLastEntry = pNew;
   }
@@ -1251,10 +1243,11 @@ static int zipfileUpdate(
     if( nVal>1 ){
       return SQLITE_CONSTRAINT;
     }else{
-      i64 iDelete = sqlite3_value_int64(apVal[0]);
+      const char *zDelete = (const char*)sqlite3_value_text(apVal[0]);
+      int nDelete = strlen(zDelete);
       ZipfileEntry *p;
       for(p=pTab->pFirstEntry; p; p=p->pNext){
-        if( p->iRowid==iDelete ){
+        if( zipfileComparePath(p->zPath, zDelete, nDelete)==0 ){
           p->bDeleted = 1;
           break;
         }
diff --git a/manifest b/manifest
index fcbf2dda8e..c426ee99bf 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\stest\scases\sfor\srunning\smultiple\sRBU\soperations\swithin\sthe\ssame\sprocess\nconcurrently.
-D 2018-01-11T16:16:03.148
+C Change\szipfile\sto\sbe\sa\sWITHOUT\sROWID\svirtual\stable\sand\stable-valued\sfunction.
+D 2018-01-11T17:33:48.340
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -303,7 +303,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c cbf9dfaf94c25dfd835e79c1474fdc75517a45a37916dec66623b0937af10e51
+F ext/misc/zipfile.c e42d3ae79511ee86545bed07e9e5e7321f8139b55cdf73f60849352070bf12ad
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -1599,7 +1599,7 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf
 F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc
 F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa
 F test/zerodamage.test e59a56443d6298ecf7435f618f0b27654f0c849e
-F test/zipfile.test e7132ca60031ca5d1df684cf644952bb3f253de3671a5b502780c5de3126a453
+F test/zipfile.test a5cd98e91aebf343e21a32b97fadd6aefe02d249d5c6a1a3c2e624b88d7bdbe2
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d
 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91
@@ -1698,7 +1698,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 4f68bed3b9a63a349a2a2d7f26609491577e9717034ad86af404cf9eed9d6aaf
-R b4712b12dc6463238ee69f57e3b3a2cb
+P 407b5ed35c178bb0dbc69c8b902652038a0653d55a58a7543f9d4857c6baf3ea
+R 6ae1fe2ba1a1ad2ab58089676c32c82f
 U dan
-Z e742283e882b3ab287b40d1d09763224
+Z df7413e90ffcbb11e6d9a408fc886735
diff --git a/manifest.uuid b/manifest.uuid
index 0d3f74a2d6..91e729e259 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-407b5ed35c178bb0dbc69c8b902652038a0653d55a58a7543f9d4857c6baf3ea
\ No newline at end of file
+931201f64e04247ed613a0301fcc86c3a337c2ed162c6370a80c67a1dd919e7c
\ No newline at end of file
diff --git a/test/zipfile.test b/test/zipfile.test
index df02797bd8..fe1ad89c55 100644
--- a/test/zipfile.test
+++ b/test/zipfile.test
@@ -27,7 +27,7 @@ do_execsql_test 1.0 {
   CREATE VIRTUAL TABLE temp.zz USING zipfile('test.zip');
   PRAGMA table_info(zz);
 } {
-  0 name {} 0 {} 0 
+  0 name {} 1 {} 1 
   1 mode {} 0 {} 0 
   2 mtime {} 0 {} 0 
   3 sz {} 0 {} 0 
@@ -160,6 +160,10 @@ foreach {tn fname} {
   } {1 {constraint failed}}
 }
 
+do_catchsql_test 3.2 {
+  SELECT rowid FROM x1
+} {1 {no such column: rowid}}
+
 
 finish_test
 

From f225059b8b674af30beeaf557873f7bb32d45ad4 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 12 Jan 2018 12:02:02 +0000
Subject: [PATCH 308/488] Add a test to ensure that the
 sqlite3changeset_apply() function ignores tables that do not have the
 expected primary keys.

FossilOrigin-Name: bf2daf06279e46bc37cc92ad1becec1b12d2aa804a14b101fca8b3a7fdb280c3
---
 ext/session/sessionG.test | 34 ++++++++++++++++++++++++++++++++++
 manifest                  | 15 +++++++--------
 manifest.uuid             |  2 +-
 3 files changed, 42 insertions(+), 9 deletions(-)

diff --git a/ext/session/sessionG.test b/ext/session/sessionG.test
index 5c057350e4..2b6c0410a7 100644
--- a/ext/session/sessionG.test
+++ b/ext/session/sessionG.test
@@ -173,5 +173,39 @@ do_test 4.3 {
   compare_db db db2
 } {}
 
+#-------------------------------------------------------------------------
+reset_db 
+catch { db2 close }
+forcedelete test.db2
+sqlite3 db2 test.db2
+
+do_execsql_test 5.0.1 {
+  CREATE TABLE t1(a PRIMARY KEY, b, c);
+  CREATE TABLE t2(a, b, c PRIMARY KEY);
+  CREATE TABLE t3(a, b PRIMARY KEY, c);
+}
+do_execsql_test -db db2 5.0.2 {
+  CREATE TABLE t1(a PRIMARY KEY, b, c);
+  CREATE TABLE t2(a, b, c);
+  CREATE TABLE t3(a, b PRIMARY KEY, c);
+}
+
+do_test 5.1 {
+  do_then_apply_sql {
+    INSERT INTO t1 VALUES(1, 2, 3);
+    INSERT INTO t2 VALUES(4, 5, 6);
+    INSERT INTO t3 VALUES(7, 8, 9);
+  }
+
+  db2 eval {
+    SELECT * FROM t1;
+    SELECT * FROM t2;
+    SELECT * FROM t3;
+  }
+} {1 2 3 7 8 9}
+
+
+
+
 finish_test
 
diff --git a/manifest b/manifest
index e13040d2fc..628128b51f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\ssqlite3_vtab_nochange()\sinterface.\s\sTest\scases\sare\sin\sTH3.
-D 2018-01-11T18:15:40.111
+C Add\sa\stest\sto\sensure\sthat\sthe\ssqlite3changeset_apply()\sfunction\signores\stables\nthat\sdo\snot\shave\sthe\sexpected\sprimary\skeys.
+D 2018-01-12T12:02:02.075
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -390,7 +390,7 @@ F ext/session/sessionC.test 97556f5164ac29f2344b24bd7de6a3a35a95c390
 F ext/session/sessionD.test d4744c78334162851d2a2f285c7e603e31b49aa2
 F ext/session/sessionE.test 0a616c4ad8fd2c05f23217ebb6212ef80b7fef30f5f086a6633a081f93e84637
 F ext/session/sessionF.test c2f178d4dfd723a5fd94a730ea2ccb44c669e3ce
-F ext/session/sessionG.test 01ef705096a9d3984eebdcca79807a211dee1b60
+F ext/session/sessionG.test 63f9a744341d670775af29e4f19c1ef09a4810798400f28cd76704803a2e56ff
 F ext/session/session_common.tcl 7776eda579773113b30c7abfd4545c445228cb73
 F ext/session/session_speed_test.c edc1f96fd5e0e4b16eb03e2a73041013d59e8723
 F ext/session/sessionat.test feb7d22b3124882064b9d9df69f5484a9bb8c123dc9ddc6ffcd357521848139f
@@ -1698,8 +1698,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 931201f64e04247ed613a0301fcc86c3a337c2ed162c6370a80c67a1dd919e7c d444b1ff39f0a1673a977b8047e1e14a49d461c9934be080d27c2392a830c1c0
-R 9b5e5f478942b3c80eb44a5edc905e08
-T +closed d444b1ff39f0a1673a977b8047e1e14a49d461c9934be080d27c2392a830c1c0
-U drh
-Z 90bbe9ce2100212cdce59fb9703ab09f
+P a5d09dfaa337fa51d6e702c6aefe58824ab1e7d221c6e79166e2c6f9c7ab1501
+R 180a0996d52ee7f7f835ebd96e9ccca1
+U dan
+Z 12f762acfd5b059fba2b906c66463308
diff --git a/manifest.uuid b/manifest.uuid
index dea0f276ff..5e594f1a72 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a5d09dfaa337fa51d6e702c6aefe58824ab1e7d221c6e79166e2c6f9c7ab1501
\ No newline at end of file
+bf2daf06279e46bc37cc92ad1becec1b12d2aa804a14b101fca8b3a7fdb280c3
\ No newline at end of file

From 6ac7303e1ab5e0b224124e1c8f180cfa4d491e03 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 12 Jan 2018 14:34:45 +0000
Subject: [PATCH 309/488] Always use utimes() instead of utimensat() since the
 latter is not available even on some recent unix systems.

FossilOrigin-Name: 30ed7a4b6408f0ca921abc4d8b7bb5404fc7708cedcd104b017b361054e7148c
---
 ext/misc/fileio.c |  3 ++-
 manifest          | 14 +++++++-------
 manifest.uuid     |  2 +-
 3 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c
index f126a2e435..7035889482 100644
--- a/ext/misc/fileio.c
+++ b/ext/misc/fileio.c
@@ -86,6 +86,7 @@ SQLITE_EXTENSION_INIT1
 #  include <unistd.h>
 #  include <dirent.h>
 #  include <utime.h>
+#  include <sys/time.h>
 #else
 #  include "windows.h"
 #  include <io.h>
@@ -285,7 +286,7 @@ static int writeFile(
     }else{
       return 1;
     }
-#elif defined(AT_FDCWD)
+#elif defined(AT_FDCWD) && 0 /* utimensat() is not univerally available */
     /* Recent unix */
     struct timespec times[2];
     times[0].tv_nsec = times[1].tv_nsec = 0;
diff --git a/manifest b/manifest
index 628128b51f..59bb960e32 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sa\stest\sto\sensure\sthat\sthe\ssqlite3changeset_apply()\sfunction\signores\stables\nthat\sdo\snot\shave\sthe\sexpected\sprimary\skeys.
-D 2018-01-12T12:02:02.075
+C Always\suse\sutimes()\sinstead\sof\sutimensat()\ssince\sthe\slatter\sis\snot\savailable\neven\son\ssome\srecent\sunix\ssystems.
+D 2018-01-12T14:34:45.816
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -277,7 +277,7 @@ F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f0
 F ext/misc/csv.c 1a009b93650732e22334edc92459c4630b9fa703397cbb3c8ca279921a36ca11
 F ext/misc/dbdump.c 3509fa6b8932d04e932d6b6b827b6a82ca362781b8e8f3c77336f416793e215e
 F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2
-F ext/misc/fileio.c 1194228c96d6b7a374e93602e2ba1899b42d0fc4d4c5253962fb13e1c9ed1f77
+F ext/misc/fileio.c 777c13f00b4505df3bfab602568c98d2b067f7d1d265de88160d0f1ac92f3dcf
 F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25
 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c
 F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281984
@@ -1698,7 +1698,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a5d09dfaa337fa51d6e702c6aefe58824ab1e7d221c6e79166e2c6f9c7ab1501
-R 180a0996d52ee7f7f835ebd96e9ccca1
-U dan
-Z 12f762acfd5b059fba2b906c66463308
+P bf2daf06279e46bc37cc92ad1becec1b12d2aa804a14b101fca8b3a7fdb280c3
+R 2f0bebc7abfba03f2a023e68e468fd71
+U drh
+Z 3cf98027077857228d4585915175b98d
diff --git a/manifest.uuid b/manifest.uuid
index 5e594f1a72..95795ed262 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-bf2daf06279e46bc37cc92ad1becec1b12d2aa804a14b101fca8b3a7fdb280c3
\ No newline at end of file
+30ed7a4b6408f0ca921abc4d8b7bb5404fc7708cedcd104b017b361054e7148c
\ No newline at end of file

From 614efe2b4b42f58968e59a4ae69ff78151adc6d5 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 12 Jan 2018 16:44:29 +0000
Subject: [PATCH 310/488] Experimental change to include changes made to the
 sqlite_stat1 table in changesets generated by the sessions module.
 sqlite_stat1 entries in such changesets are ignored by legacy clients.

FossilOrigin-Name: bd46c4429693545eb16db85692fc591ac529796aa746f5f21df1ce4380619320
---
 ext/session/sessionstat1.test | 125 ++++++++++++++++++++++++++++++++++
 ext/session/sqlite3session.c  |  11 ++-
 manifest                      |  28 ++++----
 manifest.uuid                 |   2 +-
 src/analyze.c                 |  22 ++++++
 src/delete.c                  |   2 +-
 src/vdbe.c                    |   3 +-
 src/vdbe.h                    |   1 +
 src/vdbeaux.c                 |   2 +
 test/hook.test                |  51 ++++++++++++++
 10 files changed, 231 insertions(+), 16 deletions(-)
 create mode 100644 ext/session/sessionstat1.test

diff --git a/ext/session/sessionstat1.test b/ext/session/sessionstat1.test
new file mode 100644
index 0000000000..47bbda15a9
--- /dev/null
+++ b/ext/session/sessionstat1.test
@@ -0,0 +1,125 @@
+# 2018 January 12
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+} 
+source [file join [file dirname [info script]] session_common.tcl]
+source $testdir/tester.tcl
+ifcapable !session {finish_test; return}
+
+set testprefix sessionstat1
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a PRIMARY KEY, b, c);
+  CREATE INDEX t1b ON t1(b);
+  CREATE INDEX t1c ON t1(c);
+
+  WITH s(i) AS (
+    SELECT 0 UNION ALL SELECT i+1 FROM s WHERE (i+1)<32
+  )
+  INSERT INTO t1 SELECT i, i%8, i%2 FROM s;
+}
+
+do_iterator_test 1.1 {} {
+  ANALYZE
+} {
+  {INSERT sqlite_stat1 0 XX. {} {t t1 t sqlite_autoindex_t1_1 t {32 1}}}
+  {INSERT sqlite_stat1 0 XX. {} {t t1 t t1b t {32 4}}} 
+  {INSERT sqlite_stat1 0 XX. {} {t t1 t t1c t {32 16}}}
+}
+
+do_execsql_test 1.2 {
+  WITH s(i) AS (
+    SELECT 32 UNION ALL SELECT i+1 FROM s WHERE (i+1)<64
+  )
+  INSERT INTO t1 SELECT i, i%8, i%2 FROM s;
+}
+
+do_iterator_test 1.3 {} {
+  ANALYZE
+} {
+  {UPDATE sqlite_stat1 0 XX. {t t1 t sqlite_autoindex_t1_1 t {32 1}} {{} {} {} {} t {64 1}}} 
+  {UPDATE sqlite_stat1 0 XX. {t t1 t t1b t {32 4}} {{} {} {} {} t {64 8}}} 
+  {UPDATE sqlite_stat1 0 XX. {t t1 t t1c t {32 16}} {{} {} {} {} t {64 32}}}
+}
+
+do_iterator_test 1.5 {} {
+  DROP INDEX t1b;
+} {
+  {DELETE sqlite_stat1 0 XX. {t t1 t t1b t {64 8}} {}}
+}
+
+do_iterator_test 1.6 {} {
+  DROP TABLE t1;
+} {
+  {DELETE sqlite_stat1 0 XX. {t t1 t sqlite_autoindex_t1_1 t {64 1}} {}}
+  {DELETE sqlite_stat1 0 XX. {t t1 t t1c t {64 32}} {}}
+}
+
+#-------------------------------------------------------------------------
+#
+catch { db2 close }
+forcedelete test.db2
+sqlite3 db2 test.db2
+
+do_test 2.0 {
+  do_common_sql {
+    CREATE TABLE t1(a PRIMARY KEY, b, c);
+    CREATE INDEX t1b ON t1(b);
+    CREATE INDEX t1c ON t1(c);
+    ANALYZE;
+  }
+} {}
+
+do_test 2.1 {
+  do_then_apply_sql {
+    WITH s(i) AS (
+        SELECT 0 UNION ALL SELECT i+1 FROM s WHERE (i+1)<32
+    )
+    INSERT INTO t1 SELECT i, i%8, i%2 FROM s;
+    ANALYZE;
+  }
+} {}
+
+do_execsql_test -db db2 2.2 {
+  SELECT * FROM sqlite_stat1
+} {
+  t1 sqlite_autoindex_t1_1 {32 1} 
+  t1 t1b {32 4} 
+  t1 t1c {32 16}
+}
+
+do_test 2.3 {
+  do_then_apply_sql { DROP INDEX t1c }
+} {}
+
+do_execsql_test -db db2 2.4 {
+  SELECT * FROM sqlite_stat1
+} {
+  t1 sqlite_autoindex_t1_1 {32 1} 
+  t1 t1b {32 4} 
+}
+
+do_test 2.3 {
+  do_then_apply_sql { DROP TABLE t1 }
+} {}
+
+do_execsql_test -db db2 2.4 {
+  SELECT * FROM sqlite_stat1
+} {
+}
+
+do_execsql_test -db db2 2.4 { SELECT count(*) FROM t1 } 32
+
+finish_test
+
diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c
index d5cb467374..ffaa4c98b0 100644
--- a/ext/session/sqlite3session.c
+++ b/ext/session/sqlite3session.c
@@ -944,7 +944,16 @@ static int sessionTableInfo(
   assert( pazCol && pabPK );
 
   nThis = sqlite3Strlen30(zThis);
-  zPragma = sqlite3_mprintf("PRAGMA '%q'.table_info('%q')", zDb, zThis);
+  if( nThis==12 && 0==sqlite3_stricmp("sqlite_stat1", zThis) ){
+    /* For sqlite_stat1, pretend that (tbl,idx) is the PRIMARY KEY. */
+    zPragma = sqlite3_mprintf(
+        "SELECT 0, 'tbl',  '', 0, '', 1     UNION ALL "
+        "SELECT 1, 'idx',  '', 0, '', 2     UNION ALL "
+        "SELECT 2, 'stat', '', 0, '', 0"
+    );
+  }else{
+    zPragma = sqlite3_mprintf("PRAGMA '%q'.table_info('%q')", zDb, zThis);
+  }
   if( !zPragma ) return SQLITE_NOMEM;
 
   rc = sqlite3_prepare_v2(db, zPragma, -1, &pStmt, 0);
diff --git a/manifest b/manifest
index 628128b51f..7e21810e82 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sa\stest\sto\sensure\sthat\sthe\ssqlite3changeset_apply()\sfunction\signores\stables\nthat\sdo\snot\shave\sthe\sexpected\sprimary\skeys.
-D 2018-01-12T12:02:02.075
+C Experimental\schange\sto\sinclude\schanges\smade\sto\sthe\ssqlite_stat1\stable\sin\nchangesets\sgenerated\sby\sthe\ssessions\smodule.\ssqlite_stat1\sentries\sin\ssuch\nchangesets\sare\signored\sby\slegacy\sclients.
+D 2018-01-12T16:44:29.077
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -397,8 +397,9 @@ F ext/session/sessionat.test feb7d22b3124882064b9d9df69f5484a9bb8c123dc9ddc6ffcd
 F ext/session/sessiondiff.test ad13dd65664bae26744e1f18eb3cbd5588349b7e9118851d8f9364248d67bcec
 F ext/session/sessionfault.test da273f2712b6411e85e71465a1733b8501dbf6f7
 F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0
+F ext/session/sessionstat1.test a361997e6ea72d427cb4b1313226b56278b9c74d3045817c33c99de5bef0f2e9
 F ext/session/sessionwor.test 2f3744236dc8b170a695b7d8ddc8c743c7e79fdc
-F ext/session/sqlite3session.c cc127222a9ea6f4eaa31281aa9da924f5244f6099be0ee526c950684fb3513a6
+F ext/session/sqlite3session.c 604aa21fa17c161a982595ec76c533add0e3f02ad7fb48bd4ece59335901a421
 F ext/session/sqlite3session.h cb4d860101ba6d3ac810f18684539b766d24d668fa2436cdde90d711af9464fb
 F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386
 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
@@ -420,7 +421,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b
 F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786
 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a
 F src/alter.c cf7a8af45cb0ace672f47a1b29ab24092a9e8cd8d945a9974e3b5d925f548594
-F src/analyze.c 0d0ccf7520a201d8747ea2f02c92c26e26f801bc161f714f27b9f7630dde0421
+F src/analyze.c f9bfffd0416c547a916cb96793b94684bdb0d26a71542ea31819c6757741c19d
 F src/attach.c 84c477e856b24c2b9a0983b438a707c0cf4d616cee7a425401d418e58afec24c
 F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
@@ -436,7 +437,7 @@ F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
 F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957
 F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
-F src/delete.c 74667ad914ac143731a444a1bacf29ceb18f6eded8a0dd17aafae80baa07f8bb
+F src/delete.c 20c8788451dc737a967c87ea53ad43544d617f5b57d32ccce8bd52a0daf9e89b
 F src/expr.c ad6e7a9c34a4bab9d10cc857d647ae7ce370a633b5c0bfa71f1c29b81ae364b8
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331
@@ -553,11 +554,11 @@ F src/update.c 8bd52c38d6d426925be4488ee106db26d9ee344406315671ed246ddace8d6091
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c 7315e97a8dc2c8e19ca64196c652cf0a65d13fd0a211b2cec082062372dc6261
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
-F src/vdbe.c 4d0666741d884dfaceb6ac5bbededbd947068e4b742163898feea84a027dace6
-F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97
+F src/vdbe.c 5d83f28acbce72f3bb35917c75c34131022010f0a54dbd5ae2cbbf21f08c38c1
+F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a
 F src/vdbeInt.h a0969c14950bcc61b2dfa19c6f62b42c5012013fcda342ca927a192ed37e6592
 F src/vdbeapi.c f519346f6db99a1eb3f85ca3ae8aede9ff9c4d90fec809dae7cb946786b2e270
-F src/vdbeaux.c 7ae48b180e5dd5d282e6752d155f1ab7929196d8e6577b82742044188152ca85
+F src/vdbeaux.c 7d635c30e55196ee8e0af605aceab0c3036ed6d6ed55dd6639c4b2a2a4593b4f
 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191
 F src/vdbemem.c 8478f7fb1948bf8fdeec7c2cb59ea58155c31258b9cd43c56d485e03ed40bd07
 F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f
@@ -944,7 +945,7 @@ F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98
 F test/having.test e4098a4b8962f9596035c3b87a8928a10648acc509f1bb8d6f96413bbf79a1b3
 F test/hexlit.test 4a6a5f46e3c65c4bf1fa06f5dd5a9507a5627751
 F test/hidden.test 23c1393a79e846d68fd902d72c85d5e5dcf98711
-F test/hook.test dbc0b87756e1e20e7497b56889c9e9cd2f8cc2b5
+F test/hook.test 1604b3b2f5931430087540404555c1b6be3618600b81558657c66b533ed70b13
 F test/hook2.test b9ff3b8c6519fb67f33192f1afe86e7782ee4ac8
 F test/icu.test 41aa8847745a879b897a7febea0f8f9efc8e67fe8bf680589b6e07c7b0a1569a
 F test/ieee754.test 806fc0ce7f305f57e3331eaceeddcfec9339e607
@@ -1698,7 +1699,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a5d09dfaa337fa51d6e702c6aefe58824ab1e7d221c6e79166e2c6f9c7ab1501
-R 180a0996d52ee7f7f835ebd96e9ccca1
+P bf2daf06279e46bc37cc92ad1becec1b12d2aa804a14b101fca8b3a7fdb280c3
+R df0683427f9c775029b8cffecc77e29c
+T *branch * sessions-stat1
+T *sym-sessions-stat1 *
+T -sym-trunk *
 U dan
-Z 12f762acfd5b059fba2b906c66463308
+Z e2d0263dd242b14c8f682e74527ab3ca
diff --git a/manifest.uuid b/manifest.uuid
index 5e594f1a72..00b7d3d5a5 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-bf2daf06279e46bc37cc92ad1becec1b12d2aa804a14b101fca8b3a7fdb280c3
\ No newline at end of file
+bd46c4429693545eb16db85692fc591ac529796aa746f5f21df1ce4380619320
\ No newline at end of file
diff --git a/src/analyze.c b/src/analyze.c
index 495cc954ac..85c603ffdf 100644
--- a/src/analyze.c
+++ b/src/analyze.c
@@ -234,6 +234,10 @@ static void openStatTable(
            "DELETE FROM %Q.%s WHERE %s=%Q",
            pDb->zDbSName, zTab, zWhereType, zWhere
         );
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+      }else if( db->xPreUpdateCallback ){
+        sqlite3NestedParse(pParse, "DELETE FROM %Q.%s", pDb->zDbSName, zTab);
+#endif
       }else{
         /* The sqlite_stat[134] table already exists.  Delete all rows. */
         sqlite3VdbeAddOp2(v, OP_Clear, aRoot[i], iDb);
@@ -998,6 +1002,9 @@ static void analyzeOneTable(
   int regIdxname = iMem++;     /* Register containing index name */
   int regStat1 = iMem++;       /* Value for the stat column of sqlite_stat1 */
   int regPrev = iMem;          /* MUST BE LAST (see below) */
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+  Table *pStat1 = 0; 
+#endif
 
   pParse->nMem = MAX(pParse->nMem, iMem);
   v = sqlite3GetVdbe(pParse);
@@ -1023,6 +1030,18 @@ static void analyzeOneTable(
   }
 #endif
 
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+  if( db->xPreUpdateCallback ){
+    pStat1 = (Table*)sqlite3DbMallocZero(db, sizeof(Table) + 13);
+    if( pStat1==0 ) return;
+    pStat1->zName = (char*)&pStat1[1];
+    memcpy(pStat1->zName, "sqlite_stat1", 13);
+    pStat1->nCol = 3;
+    pStat1->iPKey = -1;
+    sqlite3VdbeAddOp4(pParse->pVdbe, OP_Noop, 0, 0, 0,(char*)pStat1,P4_DYNBLOB);
+  }
+#endif
+
   /* Establish a read-lock on the table at the shared-cache level. 
   ** Open a read-only cursor on the table. Also allocate a cursor number
   ** to use for scanning indexes (iIdxCur). No index cursor is opened at
@@ -1224,6 +1243,9 @@ static void analyzeOneTable(
     sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "BBB", 0);
     sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regNewRowid);
     sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regTemp, regNewRowid);
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+    sqlite3VdbeChangeP4(v, -1, (char*)pStat1, P4_TABLE);
+#endif
     sqlite3VdbeChangeP5(v, OPFLAG_APPEND);
 
     /* Add the entries to the stat3 or stat4 table. */
diff --git a/src/delete.c b/src/delete.c
index 5808ac51d4..2b69e4763a 100644
--- a/src/delete.c
+++ b/src/delete.c
@@ -760,7 +760,7 @@ void sqlite3GenerateRowDelete(
     u8 p5 = 0;
     sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur,0,iIdxNoSeek);
     sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, (count?OPFLAG_NCHANGE:0));
-    if( pParse->nested==0 ){
+    if( pParse->nested==0 || 0==sqlite3_stricmp(pTab->zName, "sqlite_stat1") ){
       sqlite3VdbeAppendP4(v, (char*)pTab, P4_TABLE);
     }
     if( eMode!=ONEPASS_OFF ){
diff --git a/src/vdbe.c b/src/vdbe.c
index b00174bb10..f5112e0daa 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -4483,7 +4483,8 @@ case OP_InsertInt: {
 
   /* Invoke the update-hook if required. */
   if( rc ) goto abort_due_to_error;
-  if( db->xUpdateCallback && op ){
+  assert( !op || pTab->aCol || !sqlite3_stricmp(pTab->zName,"sqlite_stat1") );
+  if( db->xUpdateCallback && op && pTab->aCol ){
     db->xUpdateCallback(db->pUpdateArg, op, zDb, pTab->zName, x.nKey);
   }
   break;
diff --git a/src/vdbe.h b/src/vdbe.h
index 3e77eb9db5..f002e05d81 100644
--- a/src/vdbe.h
+++ b/src/vdbe.h
@@ -127,6 +127,7 @@ typedef struct VdbeOpList VdbeOpList;
 #define P4_INT64      (-14) /* P4 is a 64-bit signed integer */
 #define P4_INTARRAY   (-15) /* P4 is a vector of 32-bit integers */
 #define P4_FUNCCTX    (-16) /* P4 is a pointer to an sqlite3_context object */
+#define P4_DYNBLOB    (-17) /* Pointer to memory from sqliteMalloc() */
 
 /* Error message codes for OP_Halt */
 #define P5_ConstraintNotNull 1
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index bc4bbda8e4..1780d37b8c 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -866,6 +866,7 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){
     case P4_REAL:
     case P4_INT64:
     case P4_DYNAMIC:
+    case P4_DYNBLOB:
     case P4_INTARRAY: {
       sqlite3DbFree(db, p4);
       break;
@@ -1407,6 +1408,7 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){
       sqlite3XPrintf(&x, "program");
       break;
     }
+    case P4_DYNBLOB:
     case P4_ADVANCE: {
       zTemp[0] = 0;
       break;
diff --git a/test/hook.test b/test/hook.test
index 9ba220cded..1c9145baef 100644
--- a/test/hook.test
+++ b/test/hook.test
@@ -906,5 +906,56 @@ do_preupdate_test 10.3 {
   DELETE FROM t3 WHERE b=1
 } {DELETE main t3 1 1 0 {} 1}
 
+#-------------------------------------------------------------------------
+# Test that the "update" hook is not fired for operations on the 
+# sqlite_stat1 table performed by ANALYZE, even if a pre-update hook is
+# registered.
+ifcapable analyze {
+  reset_db
+  do_execsql_test 11.1 {
+    CREATE TABLE t1(a, b);
+    CREATE INDEX idx1 ON t1(a);
+    CREATE INDEX idx2 ON t1(b);
+
+    INSERT INTO t1 VALUES(1, 2);
+    INSERT INTO t1 VALUES(3, 4);
+    INSERT INTO t1 VALUES(5, 6);
+    INSERT INTO t1 VALUES(7, 8);
+  }
+
+  db preupdate hook preupdate_cb
+  db update_hook update_cb
+
+  proc preupdate_cb {args} { lappend ::res "preupdate" $args }
+  proc update_cb {args} { lappend ::res "update" $args }
+
+  set ::res [list]
+  do_test 11.2 {
+    execsql ANALYZE
+    set ::res
+  } [list {*}{
+    preupdate {INSERT main sqlite_stat1 1 1}
+    preupdate {INSERT main sqlite_stat1 2 2}
+  }]
+
+  do_execsql_test 11.3 {
+    INSERT INTO t1 VALUES(9, 10);
+    INSERT INTO t1 VALUES(11, 12);
+    INSERT INTO t1 VALUES(13, 14);
+    INSERT INTO t1 VALUES(15, 16);
+  }
+
+  set ::res [list]
+  do_test 11.4 {
+    execsql ANALYZE
+    set ::res
+  } [list {*}{
+    preupdate {DELETE main sqlite_stat1 1 1}
+    preupdate {DELETE main sqlite_stat1 2 2}
+    preupdate {INSERT main sqlite_stat1 1 1}
+    preupdate {INSERT main sqlite_stat1 2 2}
+  }]
+}
+
 
 finish_test

From 3f975373cea89cd3a1fd81a06064a5188e7cfefe Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 12 Jan 2018 18:03:38 +0000
Subject: [PATCH 311/488] Fix obsolete comments.  No changes to code.

FossilOrigin-Name: ec39c99bea024ffce9b87bbf3d80087ff772f43f7fb7e238754c364f7db2a17b
---
 ext/session/sqlite3session.c |  8 ++------
 manifest                     | 15 +++++++--------
 manifest.uuid                |  2 +-
 3 files changed, 10 insertions(+), 15 deletions(-)

diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c
index ffaa4c98b0..de27acd0db 100644
--- a/ext/session/sqlite3session.c
+++ b/ext/session/sqlite3session.c
@@ -894,9 +894,7 @@ static int sessionGrowHash(int bPatchset, SessionTable *pTab){
 
 /*
 ** This function queries the database for the names of the columns of table
-** zThis, in schema zDb. It is expected that the table has nCol columns. If
-** not, SQLITE_SCHEMA is returned and none of the output variables are
-** populated.
+** zThis, in schema zDb.
 **
 ** Otherwise, if they are not NULL, variable *pnCol is set to the number
 ** of columns in the database table and variable *pzTab is set to point to a
@@ -917,9 +915,7 @@ static int sessionGrowHash(int bPatchset, SessionTable *pTab){
 **     *pabPK  = {1, 0, 0, 1}
 **
 ** All returned buffers are part of the same single allocation, which must
-** be freed using sqlite3_free() by the caller. If pazCol was not NULL, then
-** pointer *pazCol should be freed to release all memory. Otherwise, pointer
-** *pabPK. It is illegal for both pazCol and pabPK to be NULL.
+** be freed using sqlite3_free() by the caller
 */
 static int sessionTableInfo(
   sqlite3 *db,                    /* Database connection */
diff --git a/manifest b/manifest
index 1218593ff5..b928840625 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Include\schanges\smade\sto\sthe\ssqlite_stat1\stable\sin\schangesets\ngenerated\sby\sthe\ssessions\smodule.\ssqlite_stat1\sentries\sin\ssuch\schangesets\sare\nignored\sby\slegacy\sclients.
-D 2018-01-12T17:25:25.310
+C Fix\sobsolete\scomments.\s\sNo\schanges\sto\scode.
+D 2018-01-12T18:03:38.307
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -399,7 +399,7 @@ F ext/session/sessionfault.test da273f2712b6411e85e71465a1733b8501dbf6f7
 F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0
 F ext/session/sessionstat1.test a361997e6ea72d427cb4b1313226b56278b9c74d3045817c33c99de5bef0f2e9
 F ext/session/sessionwor.test 2f3744236dc8b170a695b7d8ddc8c743c7e79fdc
-F ext/session/sqlite3session.c 604aa21fa17c161a982595ec76c533add0e3f02ad7fb48bd4ece59335901a421
+F ext/session/sqlite3session.c 0153359f97800a7dddf1c06f4b18d78dd4b233c96bc3e2e63879a71fc80a4b09
 F ext/session/sqlite3session.h cb4d860101ba6d3ac810f18684539b766d24d668fa2436cdde90d711af9464fb
 F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386
 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
@@ -1699,8 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 30ed7a4b6408f0ca921abc4d8b7bb5404fc7708cedcd104b017b361054e7148c bd46c4429693545eb16db85692fc591ac529796aa746f5f21df1ce4380619320
-R 45cb3eae3cd802c11970c064766b4af9
-T +closed bd46c4429693545eb16db85692fc591ac529796aa746f5f21df1ce4380619320
-U dan
-Z f2dd6a8e929126a4d6fd50f30c0d0ed2
+P 2064233533edec0308d0212fb08e72fd14c760a1c88452424cae298e9eaf4af8
+R 3e8aedeebf1827d98e1081cfd8339e33
+U drh
+Z 59c372a6dec67f6ff695d342429f4b78
diff --git a/manifest.uuid b/manifest.uuid
index 72de6258fe..7b4544545f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2064233533edec0308d0212fb08e72fd14c760a1c88452424cae298e9eaf4af8
\ No newline at end of file
+ec39c99bea024ffce9b87bbf3d80087ff772f43f7fb7e238754c364f7db2a17b
\ No newline at end of file

From 84ebe2b303f61d0ee99c560f802aec7cbf8a1cdd Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 12 Jan 2018 18:46:52 +0000
Subject: [PATCH 312/488] Avoid an unnecessary branch when not using pre-update
 hooks.

FossilOrigin-Name: ec96707eb359c7e0597ee22e5ae017774366463dffa6bdb06e4438b09549f5c0
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/vdbe.c    | 17 ++++++++++-------
 3 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/manifest b/manifest
index b928840625..c7f4c5f847 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sobsolete\scomments.\s\sNo\schanges\sto\scode.
-D 2018-01-12T18:03:38.307
+C Avoid\san\sunnecessary\sbranch\swhen\snot\susing\spre-update\shooks.
+D 2018-01-12T18:46:52.041
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -554,7 +554,7 @@ F src/update.c 8bd52c38d6d426925be4488ee106db26d9ee344406315671ed246ddace8d6091
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c 7315e97a8dc2c8e19ca64196c652cf0a65d13fd0a211b2cec082062372dc6261
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
-F src/vdbe.c 5d83f28acbce72f3bb35917c75c34131022010f0a54dbd5ae2cbbf21f08c38c1
+F src/vdbe.c a7d0f8312eea5c2084349f0ea13b25fa7ecdb3606d8aab393d4a98ca7ac4c62b
 F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a
 F src/vdbeInt.h a0969c14950bcc61b2dfa19c6f62b42c5012013fcda342ca927a192ed37e6592
 F src/vdbeapi.c f519346f6db99a1eb3f85ca3ae8aede9ff9c4d90fec809dae7cb946786b2e270
@@ -1699,7 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2064233533edec0308d0212fb08e72fd14c760a1c88452424cae298e9eaf4af8
-R 3e8aedeebf1827d98e1081cfd8339e33
+P ec39c99bea024ffce9b87bbf3d80087ff772f43f7fb7e238754c364f7db2a17b
+R 639f4697487bdd2e7953131ba8eccf21
 U drh
-Z 59c372a6dec67f6ff695d342429f4b78
+Z a0b90d4f07f3f08da8d3bc6f5ca171c6
diff --git a/manifest.uuid b/manifest.uuid
index 7b4544545f..c70f198da6 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ec39c99bea024ffce9b87bbf3d80087ff772f43f7fb7e238754c364f7db2a17b
\ No newline at end of file
+ec96707eb359c7e0597ee22e5ae017774366463dffa6bdb06e4438b09549f5c0
\ No newline at end of file
diff --git a/src/vdbe.c b/src/vdbe.c
index f5112e0daa..5a3908cda6 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -4454,11 +4454,14 @@ case OP_InsertInt: {
 
 #ifdef SQLITE_ENABLE_PREUPDATE_HOOK
   /* Invoke the pre-update hook, if any */
-  if( db->xPreUpdateCallback 
-   && pOp->p4type==P4_TABLE
-   && !(pOp->p5 & OPFLAG_ISUPDATE)
-  ){
-    sqlite3VdbePreUpdateHook(p, pC, SQLITE_INSERT, zDb, pTab, x.nKey, pOp->p2);
+  if( pOp->p4type==P4_TABLE ){
+    if( db->xPreUpdateCallback && !(pOp->p5 & OPFLAG_ISUPDATE) ){
+      sqlite3VdbePreUpdateHook(p, pC, SQLITE_INSERT, zDb, pTab, x.nKey,pOp->p2);
+    }
+    if( op && pTab->aCol==0 ){
+      assert( sqlite3_stricmp(pTab->zName, "sqlite_stat1")==0 );
+      op = 0;
+    }
   }
   if( pOp->p5 & OPFLAG_ISNOOP ) break;
 #endif
@@ -4483,8 +4486,8 @@ case OP_InsertInt: {
 
   /* Invoke the update-hook if required. */
   if( rc ) goto abort_due_to_error;
-  assert( !op || pTab->aCol || !sqlite3_stricmp(pTab->zName,"sqlite_stat1") );
-  if( db->xUpdateCallback && op && pTab->aCol ){
+  if( db->xUpdateCallback && op ){
+    assert( pTab->aCol );
     db->xUpdateCallback(db->pUpdateArg, op, zDb, pTab->zName, x.nKey);
   }
   break;

From 5871318407de833aefbc8721b3a3c1a2e574f512 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 12 Jan 2018 19:20:15 +0000
Subject: [PATCH 313/488] Fix error handling in sqlite3session_diff() when it
 is invoked for table "sqlite_stat1".

FossilOrigin-Name: 874285e477dd9bd164e25ddb08b6b80daaa8cfd111b4180ecce59b1ce26f77a5
---
 ext/session/sessionD.test     | 33 +++++++++++++++++++++++++++++++++
 ext/session/sessionstat1.test |  2 +-
 ext/session/sqlite3session.c  | 20 +++++++++++++-------
 manifest                      | 18 +++++++++---------
 manifest.uuid                 |  2 +-
 5 files changed, 57 insertions(+), 18 deletions(-)

diff --git a/ext/session/sessionD.test b/ext/session/sessionD.test
index b8572782e2..84c31cbc2f 100644
--- a/ext/session/sessionD.test
+++ b/ext/session/sessionD.test
@@ -221,5 +221,38 @@ do_test 4.2.2 {
 } {1 {SQLITE_SCHEMA - table schemas do not match}}
 S delete
 
+do_test 4.3.1 {
+  sqlite3session S db main
+  S attach t4
+  execsql { CREATE TABLE t4(i PRIMARY KEY, b) }
+  list [catch { S diff ixua t4 } msg] $msg
+} {1 {SQLITE_SCHEMA - table schemas do not match}}
+S delete
+do_catchsql_test 4.3.2 {
+  SELECT * FROM ixua.t4;
+} {1 {no such table: ixua.t4}}
+
+do_test 4.4.1 {
+  sqlite3session S db main
+  S attach sqlite_stat1
+  execsql { ANALYZE }
+  execsql { DROP TABLE ixua.sqlite_stat1 }
+  list [catch { S diff ixua sqlite_stat1 } msg] $msg
+} {1 {SQLITE_SCHEMA - table schemas do not match}}
+S delete
+do_catchsql_test 4.4.2 {
+  SELECT * FROM ixua.sqlite_stat1;
+} {1 {no such table: ixua.sqlite_stat1}}
+
+do_test 4.5.1 {
+  sqlite3session S db main
+  S attach t8
+  list [catch { S diff ixua t8 } msg] $msg
+} {0 {}}
+S delete
+do_catchsql_test 4.5.2 {
+  SELECT * FROM ixua.i8;
+} {1 {no such table: ixua.i8}}
+
 finish_test
 
diff --git a/ext/session/sessionstat1.test b/ext/session/sessionstat1.test
index 47bbda15a9..59de1cf1b4 100644
--- a/ext/session/sessionstat1.test
+++ b/ext/session/sessionstat1.test
@@ -119,7 +119,7 @@ do_execsql_test -db db2 2.4 {
 } {
 }
 
-do_execsql_test -db db2 2.4 { SELECT count(*) FROM t1 } 32
+do_execsql_test -db db2 2.5 { SELECT count(*) FROM t1 } 32
 
 finish_test
 
diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c
index de27acd0db..bd83ce87d3 100644
--- a/ext/session/sqlite3session.c
+++ b/ext/session/sqlite3session.c
@@ -941,12 +941,19 @@ static int sessionTableInfo(
 
   nThis = sqlite3Strlen30(zThis);
   if( nThis==12 && 0==sqlite3_stricmp("sqlite_stat1", zThis) ){
-    /* For sqlite_stat1, pretend that (tbl,idx) is the PRIMARY KEY. */
-    zPragma = sqlite3_mprintf(
-        "SELECT 0, 'tbl',  '', 0, '', 1     UNION ALL "
-        "SELECT 1, 'idx',  '', 0, '', 2     UNION ALL "
-        "SELECT 2, 'stat', '', 0, '', 0"
-    );
+    rc = sqlite3_table_column_metadata(db, zDb, zThis, 0, 0, 0, 0, 0, 0);
+    if( rc==SQLITE_OK ){
+      /* For sqlite_stat1, pretend that (tbl,idx) is the PRIMARY KEY. */
+      zPragma = sqlite3_mprintf(
+          "SELECT 0, 'tbl',  '', 0, '', 1     UNION ALL "
+          "SELECT 1, 'idx',  '', 0, '', 2     UNION ALL "
+          "SELECT 2, 'stat', '', 0, '', 0"
+      );
+    }else if( rc==SQLITE_ERROR ){
+      zPragma = sqlite3_mprintf("");
+    }else{
+      return rc;
+    }
   }else{
     zPragma = sqlite3_mprintf("PRAGMA '%q'.table_info('%q')", zDb, zThis);
   }
@@ -1506,7 +1513,6 @@ int sqlite3session_diff(
             if( abPK[i] ) bHasPk = 1;
           }
         }
-
       }
       sqlite3_free((char*)azCol);
       if( bMismatch ){
diff --git a/manifest b/manifest
index c7f4c5f847..c35fcce386 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Avoid\san\sunnecessary\sbranch\swhen\snot\susing\spre-update\shooks.
-D 2018-01-12T18:46:52.041
+C Fix\serror\shandling\sin\ssqlite3session_diff()\swhen\sit\sis\sinvoked\sfor\stable\n"sqlite_stat1".
+D 2018-01-12T19:20:15.804
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -387,7 +387,7 @@ F ext/session/session9.test 5409d90d8141881d08285ed1c2c0d8d10fb92069
 F ext/session/sessionA.test 1feeab0b8e03527f08f2f1defb442da25480138f
 F ext/session/sessionB.test 886252dcb7e692e62ef7e357456200912e367823
 F ext/session/sessionC.test 97556f5164ac29f2344b24bd7de6a3a35a95c390
-F ext/session/sessionD.test d4744c78334162851d2a2f285c7e603e31b49aa2
+F ext/session/sessionD.test d3617e29aa15c9413aee5286d99587633245d58d2ad28f3f331c822735418a22
 F ext/session/sessionE.test 0a616c4ad8fd2c05f23217ebb6212ef80b7fef30f5f086a6633a081f93e84637
 F ext/session/sessionF.test c2f178d4dfd723a5fd94a730ea2ccb44c669e3ce
 F ext/session/sessionG.test 63f9a744341d670775af29e4f19c1ef09a4810798400f28cd76704803a2e56ff
@@ -397,9 +397,9 @@ F ext/session/sessionat.test feb7d22b3124882064b9d9df69f5484a9bb8c123dc9ddc6ffcd
 F ext/session/sessiondiff.test ad13dd65664bae26744e1f18eb3cbd5588349b7e9118851d8f9364248d67bcec
 F ext/session/sessionfault.test da273f2712b6411e85e71465a1733b8501dbf6f7
 F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0
-F ext/session/sessionstat1.test a361997e6ea72d427cb4b1313226b56278b9c74d3045817c33c99de5bef0f2e9
+F ext/session/sessionstat1.test e3a3f5876ce1526b48f6f447ee0b18960ac683e3fc891791e1ca0c08e823d498
 F ext/session/sessionwor.test 2f3744236dc8b170a695b7d8ddc8c743c7e79fdc
-F ext/session/sqlite3session.c 0153359f97800a7dddf1c06f4b18d78dd4b233c96bc3e2e63879a71fc80a4b09
+F ext/session/sqlite3session.c a18bfdab0de090597fb50aae734e2cd41bf5e5d394bbd9a2189308f2bc8525d5
 F ext/session/sqlite3session.h cb4d860101ba6d3ac810f18684539b766d24d668fa2436cdde90d711af9464fb
 F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386
 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
@@ -1699,7 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ec39c99bea024ffce9b87bbf3d80087ff772f43f7fb7e238754c364f7db2a17b
-R 639f4697487bdd2e7953131ba8eccf21
-U drh
-Z a0b90d4f07f3f08da8d3bc6f5ca171c6
+P ec96707eb359c7e0597ee22e5ae017774366463dffa6bdb06e4438b09549f5c0
+R 4a057707a8e14443ce6ba2a239747c3f
+U dan
+Z 642ce6373e880706da1a2277f3f81df3
diff --git a/manifest.uuid b/manifest.uuid
index c70f198da6..660ff79469 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ec96707eb359c7e0597ee22e5ae017774366463dffa6bdb06e4438b09549f5c0
\ No newline at end of file
+874285e477dd9bd164e25ddb08b6b80daaa8cfd111b4180ecce59b1ce26f77a5
\ No newline at end of file

From 4ec6f3a075b8b0214f567e5fab7bf653e42eb31b Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 12 Jan 2018 19:33:18 +0000
Subject: [PATCH 314/488] Simplification to the implementation of OP_Insert.

FossilOrigin-Name: 6acbdba59e9df4313a6232d925a70390acdc43dfa380b4fba7bb8bd442d6e728
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/vdbe.c    | 22 +++++++++++-----------
 3 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/manifest b/manifest
index c35fcce386..d9b6cfb745 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\serror\shandling\sin\ssqlite3session_diff()\swhen\sit\sis\sinvoked\sfor\stable\n"sqlite_stat1".
-D 2018-01-12T19:20:15.804
+C Simplification\sto\sthe\simplementation\sof\sOP_Insert.
+D 2018-01-12T19:33:18.062
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -554,7 +554,7 @@ F src/update.c 8bd52c38d6d426925be4488ee106db26d9ee344406315671ed246ddace8d6091
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c 7315e97a8dc2c8e19ca64196c652cf0a65d13fd0a211b2cec082062372dc6261
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
-F src/vdbe.c a7d0f8312eea5c2084349f0ea13b25fa7ecdb3606d8aab393d4a98ca7ac4c62b
+F src/vdbe.c e660530c799acf41616200373cdaec7af3fce5a25de40d900e8319d50971df73
 F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a
 F src/vdbeInt.h a0969c14950bcc61b2dfa19c6f62b42c5012013fcda342ca927a192ed37e6592
 F src/vdbeapi.c f519346f6db99a1eb3f85ca3ae8aede9ff9c4d90fec809dae7cb946786b2e270
@@ -1699,7 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ec96707eb359c7e0597ee22e5ae017774366463dffa6bdb06e4438b09549f5c0
-R 4a057707a8e14443ce6ba2a239747c3f
-U dan
-Z 642ce6373e880706da1a2277f3f81df3
+P 874285e477dd9bd164e25ddb08b6b80daaa8cfd111b4180ecce59b1ce26f77a5
+R 0ce0db05c5816c8a3f6d5399a0318468
+U drh
+Z 101b1825b326975007694adf2a0d837c
diff --git a/manifest.uuid b/manifest.uuid
index 660ff79469..cf6d922bb8 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-874285e477dd9bd164e25ddb08b6b80daaa8cfd111b4180ecce59b1ce26f77a5
\ No newline at end of file
+6acbdba59e9df4313a6232d925a70390acdc43dfa380b4fba7bb8bd442d6e728
\ No newline at end of file
diff --git a/src/vdbe.c b/src/vdbe.c
index 5a3908cda6..d66387d239 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -4415,10 +4415,8 @@ case OP_InsertInt: {
   int seekResult;   /* Result of prior seek or 0 if no USESEEKRESULT flag */
   const char *zDb;  /* database name - used by the update hook */
   Table *pTab;      /* Table structure - used by update and pre-update hooks */
-  int op;           /* Opcode for update hook: SQLITE_UPDATE or SQLITE_INSERT */
   BtreePayload x;   /* Payload to be inserted */
 
-  op = 0;
   pData = &aMem[pOp->p2];
   assert( pOp->p1>=0 && pOp->p1<p->nCursor );
   assert( memIsValid(pData) );
@@ -4446,21 +4444,20 @@ case OP_InsertInt: {
     zDb = db->aDb[pC->iDb].zDbSName;
     pTab = pOp->p4.pTab;
     assert( (pOp->p5 & OPFLAG_ISNOOP) || HasRowid(pTab) );
-    op = ((pOp->p5 & OPFLAG_ISUPDATE) ? SQLITE_UPDATE : SQLITE_INSERT);
   }else{
-    pTab = 0; /* Not needed.  Silence a compiler warning. */
+    pTab = 0;
     zDb = 0;  /* Not needed.  Silence a compiler warning. */
   }
 
 #ifdef SQLITE_ENABLE_PREUPDATE_HOOK
   /* Invoke the pre-update hook, if any */
-  if( pOp->p4type==P4_TABLE ){
+  if( pTab ){
     if( db->xPreUpdateCallback && !(pOp->p5 & OPFLAG_ISUPDATE) ){
       sqlite3VdbePreUpdateHook(p, pC, SQLITE_INSERT, zDb, pTab, x.nKey,pOp->p2);
     }
-    if( op && pTab->aCol==0 ){
-      assert( sqlite3_stricmp(pTab->zName, "sqlite_stat1")==0 );
-      op = 0;
+    if( db->xUpdateCallback==0 || pTab->aCol==0 ){
+      /* Prevent post-update hook from running in cases when it should not */
+      pTab = 0;
     }
   }
   if( pOp->p5 & OPFLAG_ISNOOP ) break;
@@ -4486,9 +4483,12 @@ case OP_InsertInt: {
 
   /* Invoke the update-hook if required. */
   if( rc ) goto abort_due_to_error;
-  if( db->xUpdateCallback && op ){
-    assert( pTab->aCol );
-    db->xUpdateCallback(db->pUpdateArg, op, zDb, pTab->zName, x.nKey);
+  if( pTab ){
+    assert( db->xUpdateCallback!=0 );
+    assert( pTab->aCol!=0 );
+    db->xUpdateCallback(db->pUpdateArg,
+           (pOp->p5 & OPFLAG_ISUPDATE) ? SQLITE_UPDATE : SQLITE_INSERT,
+           zDb, pTab->zName, x.nKey);
   }
   break;
 }

From ce2fbd1b029477fd8745ca6381cd0b264fd901c5 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 12 Jan 2018 21:00:14 +0000
Subject: [PATCH 315/488] Add the experimental sqlite3_value_nochange()
 interface usable by xUpdate methods of virtual tables to see if a column has
 actually changed values.

FossilOrigin-Name: 8b7be15ece9e2e83959bb0e21e240106fe1077431242c3cc6b81f1baa2382f40
---
 manifest        | 27 +++++++++++++++------------
 manifest.uuid   |  2 +-
 src/sqlite.h.in |  7 +++++++
 src/update.c    |  4 +++-
 src/vdbe.c      | 23 ++++++++++++++++-------
 src/vdbeInt.h   |  2 ++
 src/vdbeapi.c   |  7 ++++++-
 src/vdbeaux.c   |  6 +++++-
 src/vdbemem.c   |  2 +-
 9 files changed, 56 insertions(+), 24 deletions(-)

diff --git a/manifest b/manifest
index d9b6cfb745..c96606d8a9 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Simplification\sto\sthe\simplementation\sof\sOP_Insert.
-D 2018-01-12T19:33:18.062
+C Add\sthe\sexperimental\ssqlite3_value_nochange()\sinterface\susable\sby\sxUpdate\nmethods\sof\svirtual\stables\sto\ssee\sif\sa\scolumn\shas\sactually\schanged\svalues.
+D 2018-01-12T21:00:14.387
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -487,7 +487,7 @@ F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
 F src/shell.c.in b87abffd0db09203ad8a133d56fe8f154ace5ec0a14197a153fb7d80b1438c01
-F src/sqlite.h.in f83e63a48fb31fefc69c83bbe8700b4b44acdd64e440219087b0f14e35eeb8d4
+F src/sqlite.h.in 8c9b7bf8f635c13371d8060061e83d19623c77e7e4430dff7713e47761d1b972
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
 F src/sqliteInt.h fd8702c65994d7de3e2d8f7d85d958731da1ed29476571fdfa2290fd8ec0bf80
@@ -550,17 +550,17 @@ F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
 F src/tokenize.c 5b0c661a85f783d35b9883830736eeb63be4aefc4f6b7d9cd081d48782c041e2
 F src/treeview.c eae35972ff44f67064de2eaf35f04afe94e7aea3271a8b3bcebb3f954880fec3
 F src/trigger.c a34539c69433276d37b0da9a89c117726ff2d292c0902895af1f393a983cd3a1
-F src/update.c 8bd52c38d6d426925be4488ee106db26d9ee344406315671ed246ddace8d6091
+F src/update.c 7736ffc60d9fdfaae4e294d76158c0a9f752b369ae2b22ef9fc800d08e0bf31e
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c 7315e97a8dc2c8e19ca64196c652cf0a65d13fd0a211b2cec082062372dc6261
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
-F src/vdbe.c e660530c799acf41616200373cdaec7af3fce5a25de40d900e8319d50971df73
+F src/vdbe.c 07dd1909ceac146717cc248838ffe2f85165e50e468e6a0c378d840c1caf6e98
 F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a
-F src/vdbeInt.h a0969c14950bcc61b2dfa19c6f62b42c5012013fcda342ca927a192ed37e6592
-F src/vdbeapi.c f519346f6db99a1eb3f85ca3ae8aede9ff9c4d90fec809dae7cb946786b2e270
-F src/vdbeaux.c 7d635c30e55196ee8e0af605aceab0c3036ed6d6ed55dd6639c4b2a2a4593b4f
+F src/vdbeInt.h 5442fc816b6cf19c8801724199fd6b77a02eb31a7a174021713f8c59b30e51fa
+F src/vdbeapi.c 02f773681d06e46454b0606339068d4d4490873dc4a7334bc0c6030552bb2c8c
+F src/vdbeaux.c fc124962c9900b19d201e57084a4302d07136a77f51fdbdfabd625a3ca3b6dc1
 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191
-F src/vdbemem.c 8478f7fb1948bf8fdeec7c2cb59ea58155c31258b9cd43c56d485e03ed40bd07
+F src/vdbemem.c 7548dd5af03d24d534a5dbc41e3bbdf1fab83e9c8856a8d2549ed2ccf33d0e80
 F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a
@@ -1699,7 +1699,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 874285e477dd9bd164e25ddb08b6b80daaa8cfd111b4180ecce59b1ce26f77a5
-R 0ce0db05c5816c8a3f6d5399a0318468
+P 6acbdba59e9df4313a6232d925a70390acdc43dfa380b4fba7bb8bd442d6e728
+R 6f0b4cb15335f84e55c45be4e670230a
+T *branch * sqlite3_value_nochange
+T *sym-sqlite3_value_nochange *
+T -sym-trunk *
 U drh
-Z 101b1825b326975007694adf2a0d837c
+Z 9655b705be92f16dc668ac363b3cc0e3
diff --git a/manifest.uuid b/manifest.uuid
index cf6d922bb8..7fcc0b51be 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6acbdba59e9df4313a6232d925a70390acdc43dfa380b4fba7bb8bd442d6e728
\ No newline at end of file
+8b7be15ece9e2e83959bb0e21e240106fe1077431242c3cc6b81f1baa2382f40
\ No newline at end of file
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index ba673748ed..7579b9926d 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -4847,6 +4847,12 @@ SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int),
 ** then the conversion is performed.  Otherwise no conversion occurs.
 ** The [SQLITE_INTEGER | datatype] after conversion is returned.)^
 **
+** ^Within the [xUpdate] method of a [virtual table], the
+** sqlite3_value_nochange(X) interface returns true if and only if
+** the column corresponding to X is unchanged by the UPDATE operation
+** and the [xColumn] method had previously queried [sqlite3_vtab_nochange()]
+** for that column and returned a NULL as a result.
+**
 ** Please pay particular attention to the fact that the pointer returned
 ** from [sqlite3_value_blob()], [sqlite3_value_text()], or
 ** [sqlite3_value_text16()] can be invalidated by a subsequent call to
@@ -4869,6 +4875,7 @@ int sqlite3_value_bytes(sqlite3_value*);
 int sqlite3_value_bytes16(sqlite3_value*);
 int sqlite3_value_type(sqlite3_value*);
 int sqlite3_value_numeric_type(sqlite3_value*);
+int sqlite3_value_nochange(sqlite3_value*);
 
 /*
 ** CAPI3REF: Finding The Subtype Of SQL Values
diff --git a/src/update.c b/src/update.c
index 3de36fe211..ee920d2a5d 100644
--- a/src/update.c
+++ b/src/update.c
@@ -827,7 +827,8 @@ static void updateVirtualTable(
     if( aXRef[i]>=0 ){
       sqlite3ExprCode(pParse, pChanges->a[aXRef[i]].pExpr, regArg+2+i);
     }else{
-      sqlite3VdbeAddOp4Int(v, OP_VColumn, iCsr, i, regArg+2+i, 1);
+      sqlite3VdbeAddOp3(v, OP_VColumn, iCsr, i, regArg+2+i);
+      sqlite3VdbeChangeP5(v, 1);
     }
   }
   if( HasRowid(pTab) ){
@@ -862,6 +863,7 @@ static void updateVirtualTable(
     /* Create a record from the argument register contents and insert it into
     ** the ephemeral table. */
     sqlite3VdbeAddOp3(v, OP_MakeRecord, regArg, nArg, regRec);
+    sqlite3VdbeChangeP5(v, 2);
     sqlite3VdbeAddOp2(v, OP_NewRowid, ephemTab, regRowid);
     sqlite3VdbeAddOp3(v, OP_Insert, ephemTab, regRec, regRowid);
   }
diff --git a/src/vdbe.c b/src/vdbe.c
index d66387d239..dc058b65ff 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -464,7 +464,7 @@ static void memTracePrint(Mem *p){
   if( p->flags & MEM_Undefined ){
     printf(" undefined");
   }else if( p->flags & MEM_Null ){
-    printf(" NULL");
+    printf(p->flags & MEM_Zero ? " NULL-nochng" : " NULL");
   }else if( (p->flags & (MEM_Int|MEM_Str))==(MEM_Int|MEM_Str) ){
     printf(" si:%lld", p->u.i);
   }else if( p->flags & MEM_Int ){
@@ -2794,7 +2794,11 @@ case OP_MakeRecord: {
     assert( memIsValid(pRec) );
     pRec->uTemp = serial_type = sqlite3VdbeSerialType(pRec, file_format, &len);
     if( pRec->flags & MEM_Zero ){
-      if( nData ){
+      if( serial_type==0 ){
+        assert( pOp->p5==2 || CORRUPT_DB );
+        /* serial_type 10 used internally only */
+        pRec->uTemp = 10;
+      }else if( nData ){
         if( sqlite3VdbeMemExpandBlob(pRec) ) goto no_mem;
       }else{
         nZero += pRec->u.nZero;
@@ -6697,15 +6701,15 @@ case OP_VFilter: {   /* jump */
 #endif /* SQLITE_OMIT_VIRTUALTABLE */
 
 #ifndef SQLITE_OMIT_VIRTUALTABLE
-/* Opcode: VColumn P1 P2 P3 P4 *
+/* Opcode: VColumn P1 P2 P3 * P5
 ** Synopsis: r[P3]=vcolumn(P2)
 **
 ** Store in register P3 the value of the P2-th column of
 ** the current row of the virtual-table of cursor P1.
 **
 ** If the VColumn opcode is being used to fetch the value of
-** an unchanging column during an UPDATE operation, then the P4
-** value is 1.  Otherwise, P4 is 0.  The P4 value is returned
+** an unchanging column during an UPDATE operation, then the P5
+** value is 1.  Otherwise, P5 is 0.  The P5 value is returned
 ** by sqlite3_vtab_nochange() routine can can be used
 ** by virtual table implementations to return special "no-change"
 ** marks which can be more efficient, depending on the virtual table.
@@ -6730,8 +6734,13 @@ case OP_VColumn: {
   assert( pModule->xColumn );
   memset(&sContext, 0, sizeof(sContext));
   sContext.pOut = pDest;
-  sContext.bVtabNoChng = pOp->p4.i!=0;
-  MemSetTypeFlag(pDest, MEM_Null);
+  if( pOp->p5 ){
+    sqlite3VdbeMemSetNull(pDest);
+    pDest->flags = MEM_Null|MEM_Zero;
+    pDest->u.nZero = 0;
+  }else{
+    MemSetTypeFlag(pDest, MEM_Null);
+  }
   rc = pModule->xColumn(pCur->uc.pVCur, &sContext, pOp->p2);
   sqlite3VtabImportErrmsg(p, pVtab);
   if( sContext.isError ){
diff --git a/src/vdbeInt.h b/src/vdbeInt.h
index f646a4036a..b7e324c94b 100644
--- a/src/vdbeInt.h
+++ b/src/vdbeInt.h
@@ -224,6 +224,8 @@ struct sqlite3_value {
 ** If the MEM_Null flag is set, then the value is an SQL NULL value.
 ** For a pointer type created using sqlite3_bind_pointer() or
 ** sqlite3_result_pointer() the MEM_Term and MEM_Subtype flags are also set.
+** If both MEM_Null and MEM_Zero are set, that means that the value is
+** an unchanging column value from VColumn.
 **
 ** If the MEM_Str flag is set then Mem.z points at a string representation.
 ** Usually this is encoded in the same unicode encoding as the main
diff --git a/src/vdbeapi.c b/src/vdbeapi.c
index 19aa783bb6..bdd6d1cd07 100644
--- a/src/vdbeapi.c
+++ b/src/vdbeapi.c
@@ -268,6 +268,11 @@ int sqlite3_value_type(sqlite3_value* pVal){
   return aType[pVal->flags&MEM_AffMask];
 }
 
+/* Return true if a parameter to xUpdate represents an unchanged column */
+int sqlite3_value_nochange(sqlite3_value *pVal){
+  return (pVal->flags&(MEM_Null|MEM_Zero))==(MEM_Null|MEM_Zero);
+}
+
 /* Make a copy of an sqlite3_value object
 */
 sqlite3_value *sqlite3_value_dup(const sqlite3_value *pOrig){
@@ -761,7 +766,7 @@ sqlite3 *sqlite3_context_db_handle(sqlite3_context *p){
 */
 int sqlite3_vtab_nochange(sqlite3_context *p){
   assert( p );
-  return p->bVtabNoChng;
+  return sqlite3_value_nochange(p->pOut);
 }
 
 /*
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index 1780d37b8c..72a5b96bba 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -3454,7 +3454,11 @@ u32 sqlite3VdbeSerialGet(
   Mem *pMem                     /* Memory cell to write value into */
 ){
   switch( serial_type ){
-    case 10:   /* Reserved for future use */
+    case 10: { /* Internal use only: NULL with virtual table
+               ** UPDATE no-change flag set */
+      pMem->flags = MEM_Null|MEM_Zero;
+      break;
+    }
     case 11:   /* Reserved for future use */
     case 0: {  /* Null */
       /* EVIDENCE-OF: R-24078-09375 Value is a NULL. */
diff --git a/src/vdbemem.c b/src/vdbemem.c
index 107d831f4c..d8f1e64328 100644
--- a/src/vdbemem.c
+++ b/src/vdbemem.c
@@ -43,7 +43,7 @@ int sqlite3VdbeCheckMemInvariants(Mem *p){
   if( p->flags & MEM_Null ){
     /* Cannot be both MEM_Null and some other type */
     assert( (p->flags & (MEM_Int|MEM_Real|MEM_Str|MEM_Blob
-                         |MEM_RowSet|MEM_Frame|MEM_Agg|MEM_Zero))==0 );
+                         |MEM_RowSet|MEM_Frame|MEM_Agg))==0 );
 
     /* If MEM_Null is set, then either the value is a pure NULL (the usual
     ** case) or it is a pointer set using sqlite3_bind_pointer() or

From 41fb367a2c5aec8973fa004ba030f2aa55bbad5c Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 12 Jan 2018 23:18:38 +0000
Subject: [PATCH 316/488] Improved comments.  Slightly tighter implementation,
 but no big changes.

FossilOrigin-Name: a1b3f28569f2a8d82b2931527fdfe191b421f3e1ea18ee30e04211e1ad645993
---
 manifest        | 21 +++++++++------------
 manifest.uuid   |  2 +-
 src/sqlite.h.in |  9 +++++++--
 src/sqliteInt.h |  1 +
 src/update.c    |  8 ++++++--
 src/vdbe.c      | 14 ++++++++++----
 6 files changed, 34 insertions(+), 21 deletions(-)

diff --git a/manifest b/manifest
index c96606d8a9..06de123c8e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\sexperimental\ssqlite3_value_nochange()\sinterface\susable\sby\sxUpdate\nmethods\sof\svirtual\stables\sto\ssee\sif\sa\scolumn\shas\sactually\schanged\svalues.
-D 2018-01-12T21:00:14.387
+C Improved\scomments.\s\sSlightly\stighter\simplementation,\sbut\sno\sbig\schanges.
+D 2018-01-12T23:18:38.948
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -487,10 +487,10 @@ F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
 F src/shell.c.in b87abffd0db09203ad8a133d56fe8f154ace5ec0a14197a153fb7d80b1438c01
-F src/sqlite.h.in 8c9b7bf8f635c13371d8060061e83d19623c77e7e4430dff7713e47761d1b972
+F src/sqlite.h.in 86cc0db2b9a55ae7bbde80c9387cdb56aabe24221ee69069ec09626493cff9a8
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
-F src/sqliteInt.h fd8702c65994d7de3e2d8f7d85d958731da1ed29476571fdfa2290fd8ec0bf80
+F src/sqliteInt.h 9c70315598b34810a83e4894455acb18e95cf63ce4e6cbb451ac2d17eabc2544
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -550,11 +550,11 @@ F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
 F src/tokenize.c 5b0c661a85f783d35b9883830736eeb63be4aefc4f6b7d9cd081d48782c041e2
 F src/treeview.c eae35972ff44f67064de2eaf35f04afe94e7aea3271a8b3bcebb3f954880fec3
 F src/trigger.c a34539c69433276d37b0da9a89c117726ff2d292c0902895af1f393a983cd3a1
-F src/update.c 7736ffc60d9fdfaae4e294d76158c0a9f752b369ae2b22ef9fc800d08e0bf31e
+F src/update.c a90a32ffc0100265b0693dbbdbe490756447af181f5ea2c138cce515b08c8795
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c 7315e97a8dc2c8e19ca64196c652cf0a65d13fd0a211b2cec082062372dc6261
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
-F src/vdbe.c 07dd1909ceac146717cc248838ffe2f85165e50e468e6a0c378d840c1caf6e98
+F src/vdbe.c ccc1e17a30325068ae4f0292e8601997946886d23acc989c68f2a261a2795c70
 F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a
 F src/vdbeInt.h 5442fc816b6cf19c8801724199fd6b77a02eb31a7a174021713f8c59b30e51fa
 F src/vdbeapi.c 02f773681d06e46454b0606339068d4d4490873dc4a7334bc0c6030552bb2c8c
@@ -1699,10 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6acbdba59e9df4313a6232d925a70390acdc43dfa380b4fba7bb8bd442d6e728
-R 6f0b4cb15335f84e55c45be4e670230a
-T *branch * sqlite3_value_nochange
-T *sym-sqlite3_value_nochange *
-T -sym-trunk *
+P 8b7be15ece9e2e83959bb0e21e240106fe1077431242c3cc6b81f1baa2382f40
+R f998d1f4e8803d74f5f45b4d8b5e31dd
 U drh
-Z 9655b705be92f16dc668ac363b3cc0e3
+Z 1d40e26fdb71cc6674c4c60bc100f971
diff --git a/manifest.uuid b/manifest.uuid
index 7fcc0b51be..20282b7174 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-8b7be15ece9e2e83959bb0e21e240106fe1077431242c3cc6b81f1baa2382f40
\ No newline at end of file
+a1b3f28569f2a8d82b2931527fdfe191b421f3e1ea18ee30e04211e1ad645993
\ No newline at end of file
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 7579b9926d..cd524a046f 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -4850,8 +4850,13 @@ SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int),
 ** ^Within the [xUpdate] method of a [virtual table], the
 ** sqlite3_value_nochange(X) interface returns true if and only if
 ** the column corresponding to X is unchanged by the UPDATE operation
-** and the [xColumn] method had previously queried [sqlite3_vtab_nochange()]
-** for that column and returned a NULL as a result.
+** that the xUpdate method call was invoked to implement and if
+** and the prior [xColumn] method call that was invoked to extracted
+** the value for that column returned without setting a result (probably
+** because it queried [sqlite3_vtab_nochange()] and found that the column
+** was unchanging).  If sqlite3_value_nochange(X) is invoked anywhere other
+** than within an [xUpdate] method call for an UPDATE statement, then
+** the return value is arbitrary and meaningless.
 **
 ** Please pay particular attention to the fact that the pointer returned
 ** from [sqlite3_value_blob()], [sqlite3_value_text()], or
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 8e79133200..b7b402b8e0 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -3117,6 +3117,7 @@ struct AuthContext {
 #define OPFLAG_PERMUTE       0x01    /* OP_Compare: use the permutation */
 #define OPFLAG_SAVEPOSITION  0x02    /* OP_Delete/Insert: save cursor pos */
 #define OPFLAG_AUXDELETE     0x04    /* OP_Delete: index in a DELETE op */
+#define OPFLAG_NOCHNG_MAGIC  0x6d    /* OP_MakeRecord: serialtype 10 is ok */
 
 /*
  * Each trigger present in the database schema is stored as an instance of
diff --git a/src/update.c b/src/update.c
index ee920d2a5d..32c1a8371a 100644
--- a/src/update.c
+++ b/src/update.c
@@ -828,7 +828,7 @@ static void updateVirtualTable(
       sqlite3ExprCode(pParse, pChanges->a[aXRef[i]].pExpr, regArg+2+i);
     }else{
       sqlite3VdbeAddOp3(v, OP_VColumn, iCsr, i, regArg+2+i);
-      sqlite3VdbeChangeP5(v, 1);
+      sqlite3VdbeChangeP5(v, 1); /* Enable sqlite3_vtab_nochange() */
     }
   }
   if( HasRowid(pTab) ){
@@ -863,7 +863,11 @@ static void updateVirtualTable(
     /* Create a record from the argument register contents and insert it into
     ** the ephemeral table. */
     sqlite3VdbeAddOp3(v, OP_MakeRecord, regArg, nArg, regRec);
-    sqlite3VdbeChangeP5(v, 2);
+#ifdef SQLITE_DEBUG
+    /* Signal an assert() within OP_MakeRecord that it is allowed to
+    ** accept no-change records with serial_type 10 */
+    sqlite3VdbeChangeP5(v, OPFLAG_NOCHNG_MAGIC);
+#endif
     sqlite3VdbeAddOp2(v, OP_NewRowid, ephemTab, regRowid);
     sqlite3VdbeAddOp3(v, OP_Insert, ephemTab, regRec, regRowid);
   }
diff --git a/src/vdbe.c b/src/vdbe.c
index dc058b65ff..d878ca8e31 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -2792,12 +2792,17 @@ case OP_MakeRecord: {
   pRec = pLast;
   do{
     assert( memIsValid(pRec) );
-    pRec->uTemp = serial_type = sqlite3VdbeSerialType(pRec, file_format, &len);
+    serial_type = sqlite3VdbeSerialType(pRec, file_format, &len);
     if( pRec->flags & MEM_Zero ){
       if( serial_type==0 ){
-        assert( pOp->p5==2 || CORRUPT_DB );
-        /* serial_type 10 used internally only */
-        pRec->uTemp = 10;
+        /* Values with MEM_Null and MEM_Zero are created by xColumn virtual
+        ** table methods that never invoke sqlite3_result_xxxxx() while
+        ** computing an unchanging column value in an UPDATE statement.
+        ** Give such values a special internal-use-only serial-type of 10
+        ** so that they can be passed through to xUpdate and have
+        ** a true sqlite3_value_nochange(). */
+        assert( pOp->p5==OPFLAG_NOCHNG_MAGIC || CORRUPT_DB );
+        serial_type = 10;
       }else if( nData ){
         if( sqlite3VdbeMemExpandBlob(pRec) ) goto no_mem;
       }else{
@@ -2809,6 +2814,7 @@ case OP_MakeRecord: {
     testcase( serial_type==127 );
     testcase( serial_type==128 );
     nHdr += serial_type<=127 ? 1 : sqlite3VarintLen(serial_type);
+    pRec->uTemp = serial_type;
     if( pRec==pData0 ) break;
     pRec--;
   }while(1);

From 76506cd3a7b5cb51fe339f753eb6951cc5f3dca3 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 13 Jan 2018 01:53:09 +0000
Subject: [PATCH 317/488] Add the new "nochange" APIs to the extension loading
 mechanism.

FossilOrigin-Name: cd7c42699e73b688dbed29ed5d243d06439d7dbe0ba8c42d8299e04cc0dd4464
---
 manifest         | 15 +++++++--------
 manifest.uuid    |  2 +-
 src/loadext.c    |  5 ++++-
 src/sqlite3ext.h |  5 +++++
 4 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index 8932dc15ad..0237685bce 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\ssqlite3_value_nochange()\sAPI,\susable\sfrom\swithin\sthe\sxUpdate\smethod\nof\sa\svirtual\stable\sto\sdiscover\swhether\sor\snot\sa\scolumn\swas\sunchanged\sat\sthe\nSQL\slevel.
-D 2018-01-12T23:38:10.473
+C Add\sthe\snew\s"nochange"\sAPIs\sto\sthe\sextension\sloading\smechanism.
+D 2018-01-13T01:53:09.144
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -449,7 +449,7 @@ F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 14686083cedc198540b15a79586cdd4be2acf6d5fa97627e355f817ab07e9fee
 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e
-F src/loadext.c 55bcc3c741059a1056859e8adaf133aa179e22be12215c0936b2f354ef71209b
+F src/loadext.c 8d5d9c3ab3c4d600a161f389988bdee6a04dad7c4c1754f04f257734b9d1ce8c
 F src/main.c 26918d50dd4a61b8f6f210320a522f46b5e7e592335b6aa664ab15b80b7c239b
 F src/malloc.c 07295435093ce354c6d9063ac05a2eeae28bd251d2e63c48b3d67c12c76f7e18
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
@@ -489,7 +489,7 @@ F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
 F src/shell.c.in b87abffd0db09203ad8a133d56fe8f154ace5ec0a14197a153fb7d80b1438c01
 F src/sqlite.h.in 9daf78e8f3cecc9ea0c3a82201f75bb74f789ecbfcda28d2e47fa80b3d956961
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
-F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
+F src/sqlite3ext.h 3737a51c5798e47a8ff2af0720f6dbc00fabd5ea401db392d05b1916085857a6
 F src/sqliteInt.h 9c70315598b34810a83e4894455acb18e95cf63ce4e6cbb451ac2d17eabc2544
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
@@ -1699,8 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6acbdba59e9df4313a6232d925a70390acdc43dfa380b4fba7bb8bd442d6e728 a1b3f28569f2a8d82b2931527fdfe191b421f3e1ea18ee30e04211e1ad645993
-R 33c612cb07bc4c3b03c1437b3563b788
-T +closed a1b3f28569f2a8d82b2931527fdfe191b421f3e1ea18ee30e04211e1ad645993
+P dec3ea4e4e6c4b1761ddc883a29eaa50dcd663ce6199667cc0ff82f7849d4f2a
+R 901f5f730b71efe02ff8601865925ddd
 U drh
-Z 1fe0c448d0c954d66825b437c30d556c
+Z db401e20fe7110735f2f14be8ffea4bc
diff --git a/manifest.uuid b/manifest.uuid
index 39a147bce2..cf5a25ebf1 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-dec3ea4e4e6c4b1761ddc883a29eaa50dcd663ce6199667cc0ff82f7849d4f2a
\ No newline at end of file
+cd7c42699e73b688dbed29ed5d243d06439d7dbe0ba8c42d8299e04cc0dd4464
\ No newline at end of file
diff --git a/src/loadext.c b/src/loadext.c
index abc6b3ff1d..80d40a4429 100644
--- a/src/loadext.c
+++ b/src/loadext.c
@@ -430,7 +430,10 @@ static const sqlite3_api_routines sqlite3Apis = {
   sqlite3_prepare16_v3,
   sqlite3_bind_pointer,
   sqlite3_result_pointer,
-  sqlite3_value_pointer
+  sqlite3_value_pointer,
+  /* Version 3.22.0 and later */
+  sqlite3_vtab_nochange,
+  sqlite3_value_nochange
 };
 
 /*
diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h
index d1d2c574ae..4252063909 100644
--- a/src/sqlite3ext.h
+++ b/src/sqlite3ext.h
@@ -292,6 +292,8 @@ struct sqlite3_api_routines {
   int (*bind_pointer)(sqlite3_stmt*,int,void*,const char*,void(*)(void*));
   void (*result_pointer)(sqlite3_context*,void*,const char*,void(*)(void*));
   void *(*value_pointer)(sqlite3_value*,const char*);
+  int (*vtab_nochange)(sqlite3_context*);
+  int (*value_nochange)(sqlite3_value*);
 };
 
 /*
@@ -558,6 +560,9 @@ typedef int (*sqlite3_loadext_entry)(
 #define sqlite3_bind_pointer           sqlite3_api->bind_pointer
 #define sqlite3_result_pointer         sqlite3_api->result_pointer
 #define sqlite3_value_pointer          sqlite3_api->value_pointer
+/* Version 3.22.0 and later */
+#define sqlite3_vtab_nochange          sqlite3_api->vtab_nochange
+#define sqlite3_value_nochange         sqltie3_api->value_nochange
 #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */
 
 #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION)

From 2034353db430f3339c8cd3297730063f881ad0a2 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Sat, 13 Jan 2018 02:07:16 +0000
Subject: [PATCH 318/488] Updates and minor typo fix for the README.

FossilOrigin-Name: bb196fcb677e962db4308c02097968615729a51fe4a1292e468aaf3e4bf522d3
---
 README.md     | 15 +++++++--------
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 3 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/README.md b/README.md
index f70b512d41..e5fa2acb67 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,7 @@
 <h1 align="center">SQLite Source Repository</h1>
 
 This repository contains the complete source code for the SQLite database
-engine.  Some test scripts are also include.  However, many other test scripts
+engine.  Some test scripts are also included.  However, many other test scripts
 and most of the documentation are managed separately.
 
 If you are reading this on a Git mirror someplace, you are doing it wrong.
@@ -104,7 +104,6 @@ recommended.
 SQLite does not require [Tcl](http://www.tcl.tk/) to run, but a Tcl installation
 is required by the makefiles (including those for MSVC).  SQLite contains
 a lot of generated code and Tcl is used to do much of that code generation.
-The makefiles also require AWK.
 
 ## Source Code Tour
 
@@ -116,7 +115,7 @@ The **src/** also contains the "shell.c" file
 which is the main program for the "sqlite3.exe"
 [command-line shell](https://sqlite.org/cli.html) and
 the "tclsqlite.c" file which implements the
-[TCL bindings](https://sqlite.org/tclsqlite.html) for SQLite.
+[Tcl bindings](https://sqlite.org/tclsqlite.html) for SQLite.
 (Historical note:  SQLite began as a Tcl
 extension and only later escaped to the wild as an independent library.)
 
@@ -163,14 +162,14 @@ template for generating its parser.
 
 Lemon also generates the **parse.h** header file, at the same time it
 generates parse.c. But the parse.h header file is
-modified further (to add additional symbols) using the ./addopcodes.awk
-AWK script.
+modified further (to add additional symbols) using the ./addopcodes.tcl
+Tcl script.
 
 The **opcodes.h** header file contains macros that define the numbers
 corresponding to opcodes in the "VDBE" virtual machine.  The opcodes.h
 file is generated by the scanning the src/vdbe.c source file.  The
-AWK script at ./mkopcodeh.awk does this scan and generates opcodes.h.
-A second AWK script, ./mkopcodec.awk, then scans opcodes.h to generate
+Tcl script at ./mkopcodeh.tcl does this scan and generates opcodes.h.
+A second Tcl script, ./mkopcodec.tcl, then scans opcodes.h to generate
 the **opcodes.c** source file, which contains a reverse mapping from
 opcode-number to opcode-name that is used for EXPLAIN output.
 
@@ -280,7 +279,7 @@ Key files:
 
   *  **test*.c** - Files in the src/ folder that begin with "test" go into
      building the "testfixture.exe" program.  The testfixture.exe program is
-     an enhanced TCL shell.  The testfixture.exe program runs scripts in the
+     an enhanced Tcl shell.  The testfixture.exe program runs scripts in the
      test/ folder to validate the core SQLite code.  The testfixture program
      (and some other test programs too) is build and run when you type
      "make test".
diff --git a/manifest b/manifest
index 0237685bce..7b0d218ffc 100644
--- a/manifest
+++ b/manifest
@@ -1,11 +1,11 @@
-C Add\sthe\snew\s"nochange"\sAPIs\sto\sthe\sextension\sloading\smechanism.
-D 2018-01-13T01:53:09.144
+C Updates\sand\sminor\stypo\sfix\sfor\sthe\sREADME.
+D 2018-01-13T02:07:16.057
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc ede26e3fb675e0b3b07627640ce5917154a6ee7f8f2c97424eb5ab5f651cbd56
-F README.md eeae1e552f93ef72ef7c5b8f6647b368a001c28820ad1df179d3dae602bef681
+F README.md d748f58e3ab0fe0307fb4ae0942b415d93dcc4288756e366cc9e7cf8260c093f
 F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2
@@ -1699,7 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P dec3ea4e4e6c4b1761ddc883a29eaa50dcd663ce6199667cc0ff82f7849d4f2a
-R 901f5f730b71efe02ff8601865925ddd
-U drh
-Z db401e20fe7110735f2f14be8ffea4bc
+P cd7c42699e73b688dbed29ed5d243d06439d7dbe0ba8c42d8299e04cc0dd4464
+R 5af93dd5d7e603d352f9007674349f27
+U mistachkin
+Z e985e4fc81de14351414b5000a144309
diff --git a/manifest.uuid b/manifest.uuid
index cf5a25ebf1..b074e985c2 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-cd7c42699e73b688dbed29ed5d243d06439d7dbe0ba8c42d8299e04cc0dd4464
\ No newline at end of file
+bb196fcb677e962db4308c02097968615729a51fe4a1292e468aaf3e4bf522d3
\ No newline at end of file

From 69aedc8db441c25749a0693f42dea0462ba09f9b Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 13 Jan 2018 13:07:49 +0000
Subject: [PATCH 319/488] Fix various problems in test scripts preventing "make
 test" from passing on F2FS file-systems with the "atomic-write" feature.

FossilOrigin-Name: 56d93d070d6b92d8a5a3fec1b09aae8911116c73d072fc5022f0b51668ed996b
---
 manifest             | 58 ++++++++++++++++++++++----------------------
 manifest.uuid        |  2 +-
 test/crash8.test     | 10 ++++++--
 test/delete_db.test  |  5 ++++
 test/exclusive.test  |  7 +++++-
 test/fallocate.test  |  1 +
 test/journal1.test   |  7 +++++-
 test/journal3.test   |  4 ++-
 test/jrnlmode.test   |  4 +++
 test/jrnlmode2.test  |  5 ++++
 test/lock4.test      |  8 ++++++
 test/pager1.test     |  5 ++++
 test/pager3.test     |  4 +++
 test/rollback.test   |  1 +
 test/sharedA.test    |  5 ++++
 test/stmt.test       |  5 ++++
 test/symlink.test    |  2 +-
 test/sync.test       |  4 +++
 test/sync2.test      |  1 +
 test/tempdb.test     |  5 ++++
 test/tkt3457.test    |  4 +++
 test/vacuum5.test    |  8 +++---
 test/wal2.test       | 16 +++++++++---
 test/walmode.test    | 32 ++++++++++++++----------
 test/zerodamage.test |  4 +--
 25 files changed, 149 insertions(+), 58 deletions(-)

diff --git a/manifest b/manifest
index 7b0d218ffc..4999390916 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Updates\sand\sminor\stypo\sfix\sfor\sthe\sREADME.
-D 2018-01-13T02:07:16.057
+C Fix\svarious\sproblems\sin\stest\sscripts\spreventing\s"make\stest"\sfrom\spassing\son\nF2FS\sfile-systems\swith\sthe\s"atomic-write"\sfeature.
+D 2018-01-13T13:07:49.167
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -724,7 +724,7 @@ F test/crash4.test fe2821baf37168dc59dd733dcf7dba2a401487bc
 F test/crash5.test 05dd3aa9dbb751a22d5cdaf22a9c49b6667aa219
 F test/crash6.test 4c56f1e40d0291e1110790a99807aa875b1647ba
 F test/crash7.test 1a194c4900a255258cf94b7fcbfd29536db572df
-F test/crash8.test a63907617d8e74fb54b4bff23eca8a4435625245
+F test/crash8.test 64e464f22883be5184535a9620eb9070027073610d997bd2784227ca9564fb84
 F test/crashM.test d95f59046fa749b0d0822edf18a717788c8f318d
 F test/crashtest1.c 09c1c7d728ccf4feb9e481671e29dda5669bbcc2
 F test/createtab.test b5de160630b209c4b8925bdcbbaf48cc90b67fe8
@@ -744,7 +744,7 @@ F test/delete.test acc38fca8ee4851467705b1c2cfea64cd26667e5
 F test/delete2.test 3a03f2cca1f9a67ec469915cb8babd6485db43fa
 F test/delete3.test 555e84a00a99230b7d049d477a324a631126a6ab
 F test/delete4.test 21d2113217eeaacac2d99defe14fe6611615ae86
-F test/delete_db.test c70a43629dd4d3e1dd03fdaf7a22153af6a69d92
+F test/delete_db.test 096d828493c7907f9ea11a7098ea6a0f73edba89406487d5d6cc2228dc4ab8b0
 F test/descidx1.test 6d03b44c8538fe0eb4924e19fba10cdd8f3c9240
 F test/descidx2.test 9f1a0c83fd57f8667c82310ca21b30a350888b5d
 F test/descidx3.test 09ddbe3f5295f482d2f8b687cf6db8bad7acd9a2
@@ -785,14 +785,14 @@ F test/enc4.test c8f1ce3618508fd0909945beb8b8831feef2c020
 F test/eqp.test 3fe051af50921284189d1970eb653f9fcf5117d2
 F test/errmsg.test eae9f091eb39ce7e20305de45d8e5d115b68fa856fba4ea6757b6ca3705ff7f9
 F test/eval.test a64c9105d6ff163df7cf09d6ac29cdad5922078c
-F test/exclusive.test 9a57bd66e39144b888ca75c309914fcdefb4e3f9
+F test/exclusive.test 1206b87e192497d78c7f35552e86a9d05421498da300fb1cce5ca5351ccde3c3
 F test/exclusive2.test 984090e8e9d1b331d2e8111daf6e5d61dda0bef7
 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7
 F test/exists.test 79a75323c78f02bbe9c251ea502a092f9ef63dac
 F test/expr.test 66a2c9ac34f74f036faa4092f5402c7d3162fc93
 F test/extension01.test 00d13cec817f331a687a243e0e5a2d87b0e358c9
 F test/extraquick.test cb254400bd42bfb777ff675356aabf3287978f79
-F test/fallocate.test 87b5e43c872b7e69cd80b7b8813eb102b571a75d45dda24e38b65537bcc85733
+F test/fallocate.test 07416bd593a116d5893cb244f45a94d5c6fe030561df3bd972e6135f8106e509
 F test/filectrl.test 6e871c2d35dead1d9a88e176e8d2ca094fec6bb3
 F test/filefmt.test f393e80c4b8d493b7a7f8f3809a8425bbf4292af1f5140f01cb1427798a2bbd4
 F test/fkey1.test d11dbb8a93ead9b5c46ae5d02da016d61245d47662fb2d844c99214f6163f768
@@ -1004,11 +1004,11 @@ F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0
 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916
 F test/join5.test bc98ea4b4e5003f5b1453701ebb8cd7d1c01a550
 F test/join6.test cfe6503791ceb0cbb509966740286ec423cbf10b
-F test/journal1.test 69abc726c51b4a0409189f9a85191205297c0577
+F test/journal1.test c7b768041b7f494471531e17abc2f4f5ebf9e5096984f43ed17c4eb80ba34497
 F test/journal2.test 9dac6b4ba0ca79c3b21446bbae993a462c2397c4
-F test/journal3.test ff8af941f9e06161d3db1b46bb9f965ff0e7f307
-F test/jrnlmode.test 7864d59cf7f6e552b9b99ba0f38acd167edc10fa
-F test/jrnlmode2.test 81610545a4e6ed239ea8fa661891893385e23a1d
+F test/journal3.test c9c29883f5bf535ae82ae21c472df6263806a22e467b6db7cd0d6d545305b4f6
+F test/jrnlmode.test a6693f2bed4541a21e703aaa37bb3e10de154130645952933b82b2dec0a8b539
+F test/jrnlmode2.test 8759a1d4657c064637f8b079592651530db738419e1d649c6df7048cd724363d
 F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa
 F test/json101.test d7cdf3e6731d41e0c4bde1c88806abd17f1f478486a1409933c1d8eac9120095
 F test/json102.test eeb54efa221e50b74a2d6fb9259963b48d7414dca3ce2fdfdeed45cb28487bc1
@@ -1028,7 +1028,7 @@ F test/loadext2.test 0408380b57adca04004247179837a18e866a74f7
 F test/lock.test be4fe08118fb988fed741f429b7dd5d65e1c90db
 F test/lock2.test 5242d8ac4e2d59c403aebff606af449b455aceff
 F test/lock3.test f271375930711ae044080f4fe6d6eda930870d00
-F test/lock4.test e175ae13865bc87680607563bafba21f31a26f12
+F test/lock4.test 27143363eda1622f03c133efc8db808fc331afd973486cb571ea71cd717d37b8
 F test/lock5.test c6c5e0ebcb21c61a572870cc86c0cb9f14cede38
 F test/lock6.test ad5b387a3a8096afd3c68a55b9535056431b0cf5
 F test/lock7.test 49f1eaff1cdc491cc5dee3669f3c671d9f172431
@@ -1116,9 +1116,9 @@ F test/oserror.test b32dc34f2363ef18532e3a0a7358e3e7e321974f
 F test/ossfuzz.c 7f5cc87a0280a5854c1bfa7d5c4d07d34731f08ec34dc9c916aa35ed292b1468
 F test/ossshell.c 296ab63067841bd1b1e97b46a0b2af48ee7f69d50d1a723008bee12dd7122622
 F test/ovfl.test 199c482696defceacee8c8e0e0ef36da62726b2f
-F test/pager1.test 8149b2a8986fee667ab6a8171ab310be19e77ae215bebad0e90c857b0df1935c
+F test/pager1.test f596d3bd53ce96e1d87d44d223d2ae6c8867dd782c425e5eb28b5721fa6aaa97
 F test/pager2.test 67b8f40ae98112bcdba1f2b2d03ea83266418c71
-F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f
+F test/pager3.test 4e9a83d6ca0838d7c602c9eb93d1357562d9059c1e02ffb138a8271020838370
 F test/pager4.test a122e9e6925d5b23b31e3dfef8c6a44bbf19590e
 F test/pagerfault.test 263c5442c06caf0b9b9e3fe42acdeb11f254dcebe533f69f401aaef9111eaf20
 F test/pagerfault2.test caf4c7facb914fd3b03a17b31ae2b180c8d6ca1f
@@ -1156,7 +1156,7 @@ F test/regexp2.test 40e894223b3d6672655481493f1be12012f2b33c
 F test/reindex.test 44edd3966b474468b823d481eafef0c305022254
 F test/releasetest.tcl 6aaa853f7a7bbdc458d4cb42c0425228729b0f3e5769e9b41088c08eee999a49 x
 F test/resolver01.test f4022acafda7f4d40eca94dbf16bc5fc4ac30ceb
-F test/rollback.test f580934279800d480a19176c6b44909df31ce7ad45267ea475a541daa522f3d3
+F test/rollback.test 06680159bc6746d0f26276e339e3ae2f951c64812468308838e0a3362d911eaa
 F test/rollback2.test 8435d6ff0f13f51d2a4181c232e706005fa90fc5
 F test/rollbackfault.test 0e646aeab8840c399cfbfa43daab46fd609cf04a
 F test/rowallock.test 3f88ec6819489d0b2341c7a7528ae17c053ab7cc
@@ -1216,7 +1216,7 @@ F test/shared6.test 866bb4982c45ce216c61ded5e8fde4e7e2f3ffa9
 F test/shared7.test a81e99f83e6c51b02ac99c96fb3a2a7b5978c956
 F test/shared8.test 00a07bf5e1337ecf72e94542bdefdc330d7a2538
 F test/shared9.test 5f2a8f79b4d6c7d107a01ffa1ed05ae7e6333e21
-F test/sharedA.test 0cdf1a76dfa00e6beee66af5b534b1e8df2720f5
+F test/sharedA.test 49d87ec54ab640fbbc3786ee3c01de94aaa482a3a9f834ad3fe92770eb69e281
 F test/sharedB.test 16cc7178e20965d75278f410943109b77b2e645e
 F test/shared_err.test 32634e404a3317eeb94abc7a099c556a346fdb8fb3858dbe222a4cbb8926a939
 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304
@@ -1266,7 +1266,7 @@ F test/sqllimits1.test a74ee2a3740b9f9c2437c246d8fb77354862a142
 F test/sqllog.test 6af6cb0b09f4e44e1917e06ce85be7670302517a
 F test/stat.test f8f1279ffffabe6df825723af18cc6e0ae70a893
 F test/statfault.test f525a7bf633e50afd027700e9a486090684b1ac1
-F test/stmt.test 64844332db69cf1a735fcb3e11548557fc95392f
+F test/stmt.test 54ed2cc0764bf3e48a058331813c3dbd19fc1d0827c3d8369914a5d8f564ec75
 F test/stmtvtab1.test 6873dfb24f8e79cbb5b799b95c2e4349060eb7a3b811982749a84b359468e2d5
 F test/subjournal.test 8d4e2572c0ee9a15549f0d8e40863161295107e52f07a3e8012a2e1fdd093c49
 F test/subquery.test d7268d193dd33d5505df965399d3a594e76ae13f
@@ -1279,9 +1279,9 @@ F test/swarmvtab.test 9a3fd5ab3e9b3c976ad1b3d7646aab725114f2ac26b59395d0778b33ba
 F test/swarmvtab2.test c948cb2fdfc5b01d85e8f6d6504854202dc1a0782ab2a0ed61538f27cbd0aa5c
 F test/swarmvtab3.test 6cb664669630fcec4102a09333e52068734858fd2761eee3b0465c14cdbcee29
 F test/swarmvtabfault.test 00aec54665909490f5c383f3cae3b5d18bd97c12490b429ff8752a3027acfa42
-F test/symlink.test c9ebe7330d228249e447038276bfc8a7b22f4849
-F test/sync.test 2f84bdbc2b2df1fcb0220575b4b9f8cea94b7529
-F test/sync2.test 6be8ed007fa063b147773c1982b5bdba97a32badc536bdc6077eff5cf8710ece
+F test/symlink.test 0d816670325536b8973ec08d32b45136baddb80bd45fd178e0ce7a9e8153f3e7
+F test/sync.test 89539f4973c010eda5638407e71ca7fddbcd8e0594f4c9980229f804d4333092
+F test/sync2.test 8f9f7d4f6d5be8ca8941a8dadcc4299e558cb6a1ff653a9469146c7a76ef2039
 F test/syscall.test a39d9a36f852ae6e4800f861bc2f2e83f68bbc2112d9399931ecfadeabd2d69d
 F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04
 F test/tabfunc01.test c47171c36b3d411df2bd49719dcaa5d034f8d277477fd41d253940723b969a51
@@ -1289,7 +1289,7 @@ F test/table.test b708f3e5fa2542fa51dfab21fc07b36ea445cb2f
 F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126
 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930
 F test/tclsqlite.test c3d7ac9449634b9f17fd048a3c0212e88a7448be810a9c5bd051acc1ffa00d2f
-F test/tempdb.test bd92eba8f20e16a9136e434e20b280794de3cdb6
+F test/tempdb.test 4cdaa23ddd8acb4d79cbb1b68ccdfd09b0537aaba909ca69a876157c2a2cbd08
 F test/tempdb2.test 27e41ed540b2f9b056c2e77e9bddc1b875358507
 F test/tempfault.test 0c0d349c9a99bf5f374655742577f8712c647900
 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30
@@ -1416,7 +1416,7 @@ F test/tkt3357.test 77c37c6482b526fe89941ce951c22d011f5922ed
 F test/tkt3419.test 1bbf36d7ea03b638c15804251287c2391f5c1f6b
 F test/tkt3424.test 61f831bd2b071bd128fa5d00fbda57e656ca5812
 F test/tkt3442.test 53840ec5325bb94544792aad4c20476f81dc26b1
-F test/tkt3457.test 44e980fe5334753dcc27b94fa4deabc485a92f74
+F test/tkt3457.test 5651e2cbb94645b677ec663160b9e192b87b7d365aecdfb24e19f749575a6fc2
 F test/tkt3461.test 228ea328a5a21e8663f80ee3d212a6ad92549a19
 F test/tkt3493.test 1686cbde85f8721fc1bdc0ee72f2ef2f63139218
 F test/tkt3508.test d75704db9501625c7f7deec119fcaf1696aefb7d
@@ -1499,7 +1499,7 @@ F test/vacuum.test ce91c39f7f91a4273bf620efad21086b5aa6ef1d
 F test/vacuum2.test aa048abee196c16c9ba308465494009057b79f9b
 F test/vacuum3.test 77ecdd54592b45a0bcb133339f99f1ae0ae94d0d
 F test/vacuum4.test 7ea76b769fffeb41f925303b04cbcf5a5bbeabe55e4c60ae754ff24eeeb7c010
-F test/vacuum5.test c87234e8ca4107f349da4edbeda3e4ea5adc93f3
+F test/vacuum5.test 263b144d537e92ad8e9ca8a73cc6e1583f41cfd0dda9432b87f7806174a2f48c
 F test/vacuummem.test 7b42abb3208bd82dd23a7536588396f295a314f2
 F test/varint.test bbce22cda8fc4d135bcc2b589574be8410614e62
 F test/veryquick.test 57ab846bacf7b90cf4e9a672721ea5c5b669b661
@@ -1526,7 +1526,7 @@ F test/vtab_alter.test 736e66fb5ec7b4fee58229aa3ada2f27ec58bc58c00edae4836890c37
 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8
 F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad
 F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477
-F test/wal2.test 2d81ffe2a02d9e5c7447b266f7153716cfcba7aecda5ed832db4544617399e29
+F test/wal2.test 155b9efa999bdb38ce1cd729b9a4fcdbffd6b88be27f039bad1d2929d287d918
 F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2
 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c
 F test/wal5.test 9c11da7aeccd83a46d79a556ad11a18d3cb15aa9
@@ -1546,7 +1546,7 @@ F test/walcrash3.test e426aa58122d20f2b9fbe9a507f9eb8cab85b8af
 F test/walcrash4.test e7b6e7639a950a0cca8e210e248c8dad4d63bf20
 F test/walfault.test 1f8389f7709877e9b4cc679033d71d6fe529056b
 F test/walhook.test ed00a40ba7255da22d6b66433ab61fab16a63483
-F test/walmode.test 4022fe03ae6e830583672caa101f046438a0473c
+F test/walmode.test cd6e7cff618eaaa5910ce57c3657aa50110397f86213886a2400afb9bfec7b7b
 F test/walnoshm.test 84ca10c544632a756467336b7c3b864d493ee496
 F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03
 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6
@@ -1599,7 +1599,7 @@ F test/without_rowid6.test 1f99644e6508447fb050f73697350c7ceca3392e
 F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf8ac
 F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc
 F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa
-F test/zerodamage.test e59a56443d6298ecf7435f618f0b27654f0c849e
+F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc
 F test/zipfile.test a5cd98e91aebf343e21a32b97fadd6aefe02d249d5c6a1a3c2e624b88d7bdbe2
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d
@@ -1699,7 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P cd7c42699e73b688dbed29ed5d243d06439d7dbe0ba8c42d8299e04cc0dd4464
-R 5af93dd5d7e603d352f9007674349f27
-U mistachkin
-Z e985e4fc81de14351414b5000a144309
+P bb196fcb677e962db4308c02097968615729a51fe4a1292e468aaf3e4bf522d3
+R 3bf69d486c995f84804eb3abae6dd335
+U dan
+Z ecacbd8e6749e1f68b1f30b320420a3c
diff --git a/manifest.uuid b/manifest.uuid
index b074e985c2..a32f986633 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-bb196fcb677e962db4308c02097968615729a51fe4a1292e468aaf3e4bf522d3
\ No newline at end of file
+56d93d070d6b92d8a5a3fec1b09aae8911116c73d072fc5022f0b51668ed996b
\ No newline at end of file
diff --git a/test/crash8.test b/test/crash8.test
index f3b6f6e244..2c6487b38b 100644
--- a/test/crash8.test
+++ b/test/crash8.test
@@ -142,6 +142,7 @@ proc write_file {zFile zData} {
 #    b) Less than 512, or
 #    c) Greater than SQLITE_MAX_PAGE_SIZE
 #
+if {[atomic_batch_write test.db]==0} {
 do_test crash8-3.1 {
   list [file exists test.db-joural] [file exists test.db]
 } {0 1}
@@ -228,6 +229,7 @@ do_test crash8-3.11 {
     PRAGMA integrity_check
   }
 } {6 ok}
+}
 
 
 # If a connection running in persistent-journal mode is part of a 
@@ -266,8 +268,12 @@ ifcapable pragma {
       UPDATE aux.ab SET b = randstr(1000,1000) WHERE a>=1;
       UPDATE ab SET b = randstr(1000,1000) WHERE a>=1;
     }
-    list [file exists test.db-journal] [file exists test2.db-journal]
-  } {1 1}
+  } {persist persist}
+  if {[atomic_batch_write test.db]==0} {
+    do_test crash8.4.1.1 {
+      list [file exists test.db-journal] [file exists test2.db-journal]
+    } [list $bJrnl $bJrnl]
+  }
 
   do_test crash8-4.2 {
     execsql {
diff --git a/test/delete_db.test b/test/delete_db.test
index 09c44ff9f3..6edd9c242e 100644
--- a/test/delete_db.test
+++ b/test/delete_db.test
@@ -17,6 +17,11 @@ set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 set testprefix delete_db
 
+if {[atomic_batch_write test.db]} {
+  finish_test
+  return
+}
+
 proc delete_all {} {
   foreach f [glob -nocomplain test2*] { file delete $f }
   foreach f [glob -nocomplain test3*] { file delete $f }
diff --git a/test/exclusive.test b/test/exclusive.test
index 45f9318205..04de529137 100644
--- a/test/exclusive.test
+++ b/test/exclusive.test
@@ -252,7 +252,9 @@ db2 close
 # opens the journal file for exclusive access, preventing its contents 
 # from being inspected externally.
 #
-if {$tcl_platform(platform) != "windows"} {
+if {$tcl_platform(platform) != "windows"
+ && [atomic_batch_write test.db]==0
+} {
 
   # Return a list of two booleans (either 0 or 1). The first is true
   # if the named file exists. The second is true only if the file
@@ -391,6 +393,7 @@ do_test exclusive-4.5 {
 # Tests exclusive-5.X - test that statement journals are truncated
 # instead of deleted when in exclusive access mode.
 #
+if {[atomic_batch_write test.db]==0} {
 
 # Close and reopen the database so that the temp database is no
 # longer active.
@@ -508,4 +511,6 @@ do_execsql_test exclusive-6.5 {
   SELECT * FROM sqlite_master;
 } {exclusive}
 
+} ;# atomic_batch_write==0
+
 finish_test
diff --git a/test/fallocate.test b/test/fallocate.test
index 63d88ea885..0c971c08c1 100644
--- a/test/fallocate.test
+++ b/test/fallocate.test
@@ -61,6 +61,7 @@ do_test fallocate-1.7 {
   execsql { BEGIN; INSERT INTO t1 VALUES(1, 2); }
   if {[permutation] != "inmemory_journal"
    && [permutation] != "atomic-batch-write"
+   && [atomic_batch_write test.db]==0
   } {
     hexio_get_int [hexio_read test.db-journal 16 4]
   } else {
diff --git a/test/journal1.test b/test/journal1.test
index c89dd2b4c9..bcbafe30f6 100644
--- a/test/journal1.test
+++ b/test/journal1.test
@@ -22,7 +22,12 @@ source $testdir/tester.tcl
 # These tests will not work on windows because windows uses
 # manditory file locking which breaks the copy_file command.
 #
-if {$tcl_platform(platform)=="windows"} {
+# Or with atomic_batch_write systems, as journal files are
+# not created.
+#
+if {$tcl_platform(platform)=="windows"
+ || [atomic_batch_write test.db]
+} {
   finish_test
   return
 }
diff --git a/test/journal3.test b/test/journal3.test
index 939cc27c70..b907352329 100644
--- a/test/journal3.test
+++ b/test/journal3.test
@@ -20,7 +20,9 @@ source $testdir/malloc_common.tcl
 # If a connection is required to create a journal file, it creates it with 
 # the same file-system permissions as the database file itself. Test this.
 #
-if {$::tcl_platform(platform) == "unix"} {
+if {$::tcl_platform(platform) == "unix"
+ && [atomic_batch_write test.db]==0
+} {
 
   # Changed on 2012-02-13:  umask is deliberately ignored for -wal, -journal,
   # and -shm files.
diff --git a/test/jrnlmode.test b/test/jrnlmode.test
index 2ba56f2b00..3112f6184e 100644
--- a/test/jrnlmode.test
+++ b/test/jrnlmode.test
@@ -302,6 +302,7 @@ ifcapable autovacuum&&pragma {
 # The following test caes, jrnlmode-5.*, test the journal_size_limit
 # pragma.
 ifcapable pragma {
+if {[atomic_batch_write test.db]==0} {
   db close
   forcedelete test.db test2.db test3.db
   sqlite3 db test.db
@@ -454,8 +455,10 @@ ifcapable pragma {
     list [file exists test.db-journal] [file size test.db-journal]
   } {1 0}
 }
+}
 
 ifcapable pragma {
+if {[atomic_batch_write test.db]==0} {
   # These tests are not run as part of the "journaltest" permutation,
   # as the test_journal.c layer is incompatible with in-memory journaling.
   if {[permutation] ne "journaltest"} {
@@ -507,6 +510,7 @@ ifcapable pragma {
     } {0}
   }
 }
+}
 
 ifcapable pragma {
   catch { db close }
diff --git a/test/jrnlmode2.test b/test/jrnlmode2.test
index 6ea87d704b..6cc54dc5df 100644
--- a/test/jrnlmode2.test
+++ b/test/jrnlmode2.test
@@ -18,6 +18,11 @@ ifcapable {!pager_pragmas} {
   return
 }
 
+if {[atomic_batch_write test.db]} {
+  finish_test
+  return
+}
+
 #-------------------------------------------------------------------------
 # The tests in this file check that the following two bugs (both now fixed)
 # do not reappear.
diff --git a/test/lock4.test b/test/lock4.test
index b0b1c74fbe..58dd206997 100644
--- a/test/lock4.test
+++ b/test/lock4.test
@@ -17,6 +17,14 @@
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 
+if {[atomic_batch_write test.db]} {
+  # This test uses two processes, one of which blocks until the other
+  # creates a *-journal file. Which doesn't work if atomic writes are
+  # available.
+  finish_test
+  return
+}
+
 do_not_use_codec
 
 # Initialize the test.db database so that it is non-empty
diff --git a/test/pager1.test b/test/pager1.test
index 8451e0b3d2..42edc3d16b 100644
--- a/test/pager1.test
+++ b/test/pager1.test
@@ -17,6 +17,11 @@ source $testdir/malloc_common.tcl
 source $testdir/wal_common.tcl
 set testprefix pager1
 
+if {[atomic_batch_write test.db]} {
+  finish_test
+  return
+}
+
 # Do not use a codec for tests in this file, as the database file is
 # manipulated directly using tcl scripts (using the [hexio_write] command).
 #
diff --git a/test/pager3.test b/test/pager3.test
index 23435a79b7..e815f2788b 100644
--- a/test/pager3.test
+++ b/test/pager3.test
@@ -16,6 +16,10 @@ source $testdir/lock_common.tcl
 source $testdir/malloc_common.tcl
 source $testdir/wal_common.tcl
 
+if {[atomic_batch_write test.db]} {
+  finish_test
+  return
+}
 
 foreach {tn sql res j} {
   1 "PRAGMA journal_mode = DELETE"  delete        0
diff --git a/test/rollback.test b/test/rollback.test
index 60a6190317..423bf20fce 100644
--- a/test/rollback.test
+++ b/test/rollback.test
@@ -83,6 +83,7 @@ if {$tcl_platform(platform) == "unix"
  && [permutation] ne "onefile"
  && [permutation] ne "inmemory_journal"
  && [permutation] ne "atomic-batch-write"
+ && [atomic_batch_write test.db]==0
 } {
   do_test rollback-2.1 {
     execsql {
diff --git a/test/sharedA.test b/test/sharedA.test
index 146fb26be0..55ed5749bb 100644
--- a/test/sharedA.test
+++ b/test/sharedA.test
@@ -19,6 +19,11 @@ if {[run_thread_tests]==0} { finish_test ; return }
 db close
 set ::testprefix sharedA
 
+if {[atomic_batch_write test.db]} {
+  finish_test
+  return
+}
+
 set ::enable_shared_cache [sqlite3_enable_shared_cache 1]
 
 #-------------------------------------------------------------------------
diff --git a/test/stmt.test b/test/stmt.test
index df501f7654..138ce19860 100644
--- a/test/stmt.test
+++ b/test/stmt.test
@@ -16,6 +16,11 @@
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 
+if {[atomic_batch_write test.db]} {
+  finish_test
+  return
+}
+
 do_test stmt-1.1 {
   execsql { CREATE TABLE t1(a integer primary key, b INTEGER NOT NULL) }
 } {}
diff --git a/test/symlink.test b/test/symlink.test
index 949102cf8a..4695b29fa0 100644
--- a/test/symlink.test
+++ b/test/symlink.test
@@ -102,7 +102,7 @@ foreach {tn f} {1 test.db2 2 test.db3} {
         INSERT INTO t1 VALUES(1);
     } db2
     file exists test.db-journal
-  } 1
+  } [expr [atomic_batch_write test.db]==0]
   do_test 2.$tn.3 {
     list [file exists test2.db-journal] [file exists test3.db-journal]
   } {0 0}
diff --git a/test/sync.test b/test/sync.test
index 210039acb5..023425e6b1 100644
--- a/test/sync.test
+++ b/test/sync.test
@@ -26,6 +26,10 @@ ifcapable !pager_pragmas||!attach {
   finish_test
   return
 }
+if {[atomic_batch_write test.db]} {
+  finish_test
+  return
+}
 
 set sqlite_sync_count 0
 proc cond_incr_sync_count {adj} {
diff --git a/test/sync2.test b/test/sync2.test
index 46e8bc72fb..89e66c8455 100644
--- a/test/sync2.test
+++ b/test/sync2.test
@@ -29,6 +29,7 @@ ifcapable !pager_pragmas||!attach||!dirsync {
 if {$::tcl_platform(platform)!="unix" 
   || [permutation] == "journaltest"
   || [permutation] == "inmemory_journal"
+  || [atomic_batch_write test.db] 
 } {
   finish_test
   return
diff --git a/test/tempdb.test b/test/tempdb.test
index 61416ec81c..a32ef06f44 100644
--- a/test/tempdb.test
+++ b/test/tempdb.test
@@ -17,6 +17,11 @@
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 
+if {[atomic_batch_write test.db]} {
+  finish_test
+  return
+}
+
 # Use a temporary database.
 #
 db close
diff --git a/test/tkt3457.test b/test/tkt3457.test
index 0475741322..24b4f0eac0 100644
--- a/test/tkt3457.test
+++ b/test/tkt3457.test
@@ -19,6 +19,10 @@ if {$tcl_platform(platform) != "unix"} {
   finish_test
   return
 }
+if {[atomic_batch_write test.db]} {
+  finish_test
+  return
+}
 
 #-----------------------------------------------------------------------
 # To roll back a hot-journal file, the application needs read and write 
diff --git a/test/vacuum5.test b/test/vacuum5.test
index 8e76a9393c..f203fb87ba 100644
--- a/test/vacuum5.test
+++ b/test/vacuum5.test
@@ -143,9 +143,11 @@ if {$::TEMP_STORE<3 && [permutation]!="inmemory_journal"} {
 
   db close
   tvfs delete
-  do_test 3.2 {
-    lrange $::openfiles 0 4
-  } {test.db test.db-journal test.db-journal {} test.db-journal}
+  if {[atomic_batch_write test.db]==0} {
+    do_test 3.2 {
+      lrange $::openfiles 0 4
+    } {test.db test.db-journal test.db-journal {} test.db-journal}
+  }
 } 
 
 
diff --git a/test/wal2.test b/test/wal2.test
index a9cafec66d..b26f5ca877 100644
--- a/test/wal2.test
+++ b/test/wal2.test
@@ -584,15 +584,23 @@ do_test wal2-6.3.4 {
     BEGIN;
       INSERT INTO t1 VALUES('Groucho');
   }
-  list [file exists test.db-wal] [file exists test.db-journal]
-} {0 1}
+} {}
+if {[atomic_batch_write test.db]==0} {
+  do_test wal2-6.3.4.1 {
+    list [file exists test.db-wal] [file exists test.db-journal]
+  } {0 1}
+}
 do_test wal2-6.3.5 {
   execsql { PRAGMA lock_status }
 } {main exclusive temp closed}
 do_test wal2-6.3.6 {
   execsql { COMMIT }
-  list [file exists test.db-wal] [file exists test.db-journal]
-} {0 1}
+} {}
+if {[atomic_batch_write test.db]==0} {
+  do_test wal2-6.3.6.1 {
+    list [file exists test.db-wal] [file exists test.db-journal]
+  } {0 1}
+}
 do_test wal2-6.3.7 {
   execsql { PRAGMA lock_status }
 } {main exclusive temp closed}
diff --git a/test/walmode.test b/test/walmode.test
index 4e14d54d4f..f760823c8d 100644
--- a/test/walmode.test
+++ b/test/walmode.test
@@ -45,15 +45,17 @@ do_test walmode-1.2 {
   file size test.db
 } {1024}
 
-set expected_sync_count 3
-if {$::tcl_platform(platform)!="windows"} {
-  ifcapable dirsync {
-    incr expected_sync_count
+if {[atomic_batch_write test.db]==0} {
+  set expected_sync_count 3
+  if {$::tcl_platform(platform)!="windows"} {
+    ifcapable dirsync {
+      incr expected_sync_count
+    }
   }
+  do_test walmode-1.3 {
+    set sqlite_sync_count
+  } $expected_sync_count
 }
-do_test walmode-1.3 {
-  set sqlite_sync_count
-} $expected_sync_count
 
 do_test walmode-1.4 {
   file exists test.db-wal
@@ -106,9 +108,11 @@ do_test walmode-4.1 {
   execsql { INSERT INTO t1 VALUES(1, 2) }
   execsql { PRAGMA journal_mode = persist }
 } {persist}
-do_test walmode-4.2 {
-  list [file exists test.db-journal] [file exists test.db-wal]
-} {1 0}
+if {[atomic_batch_write test.db]==0} {
+  do_test walmode-4.2 {
+    list [file exists test.db-journal] [file exists test.db-wal]
+  } {1 0}
+}
 do_test walmode-4.3 {
   execsql { SELECT * FROM t1 }
 } {1 2}
@@ -117,9 +121,11 @@ do_test walmode-4.4 {
   sqlite3 db test.db
   execsql { SELECT * FROM t1 }
 } {1 2}
-do_test walmode-4.5 {
-  list [file exists test.db-journal] [file exists test.db-wal]
-} {1 0}
+if {[atomic_batch_write test.db]==0} {
+  do_test walmode-4.5 {
+    list [file exists test.db-journal] [file exists test.db-wal]
+  } {1 0}
+}
 
 # Test that nothing goes wrong if a connection is prevented from changing
 # from WAL to rollback mode because a second connection has the database
diff --git a/test/zerodamage.test b/test/zerodamage.test
index a87e50b7b5..83bae737df 100644
--- a/test/zerodamage.test
+++ b/test/zerodamage.test
@@ -74,7 +74,7 @@ do_test zerodamage-2.0 {
     UPDATE t1 SET y=randomblob(50) WHERE x=123;
   }
   concat [file_control_powersafe_overwrite db -1] [set ::max_journal_size]
-} {0 1 2576}
+} [list 0 1 [expr ([atomic_batch_write test.db]==0)*2576]]
 
 # Repeat the previous step with zero-damage turned off.  This time the
 # maximum rollback journal size should be much larger.
@@ -87,7 +87,7 @@ do_test zerodamage-2.1 {
     UPDATE t1 SET y=randomblob(50) WHERE x=124;
   }
   concat [file_control_powersafe_overwrite db -1] [set ::max_journal_size]
-} {0 0 24704}
+} [list 0 0 [expr ([atomic_batch_write test.db]==0)*24704]]
 
 if {[wal_is_capable]} {
   # Run a WAL-mode transaction with POWERSAFE_OVERWRITE on to verify that the

From 2483a111dbb53917457dea750e6869a4b9272a80 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 13 Jan 2018 14:01:26 +0000
Subject: [PATCH 320/488] Fix a typo in crash8.test.

FossilOrigin-Name: c3dc7b8d9de94e46fae75fcc96ec68743205e0a2591e14391c394fff203004d1
---
 manifest         | 12 ++++++------
 manifest.uuid    |  2 +-
 test/crash8.test |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 4999390916..a2d500f819 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\svarious\sproblems\sin\stest\sscripts\spreventing\s"make\stest"\sfrom\spassing\son\nF2FS\sfile-systems\swith\sthe\s"atomic-write"\sfeature.
-D 2018-01-13T13:07:49.167
+C Fix\sa\stypo\sin\scrash8.test.
+D 2018-01-13T14:01:26.293
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -724,7 +724,7 @@ F test/crash4.test fe2821baf37168dc59dd733dcf7dba2a401487bc
 F test/crash5.test 05dd3aa9dbb751a22d5cdaf22a9c49b6667aa219
 F test/crash6.test 4c56f1e40d0291e1110790a99807aa875b1647ba
 F test/crash7.test 1a194c4900a255258cf94b7fcbfd29536db572df
-F test/crash8.test 64e464f22883be5184535a9620eb9070027073610d997bd2784227ca9564fb84
+F test/crash8.test 63cd5aea313222d7a69637cf7174c34d151676cc187d57193b66d4c89dedede3
 F test/crashM.test d95f59046fa749b0d0822edf18a717788c8f318d
 F test/crashtest1.c 09c1c7d728ccf4feb9e481671e29dda5669bbcc2
 F test/createtab.test b5de160630b209c4b8925bdcbbaf48cc90b67fe8
@@ -1699,7 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P bb196fcb677e962db4308c02097968615729a51fe4a1292e468aaf3e4bf522d3
-R 3bf69d486c995f84804eb3abae6dd335
+P 56d93d070d6b92d8a5a3fec1b09aae8911116c73d072fc5022f0b51668ed996b
+R 095d7894b995bcb80e7e3cee4fddb19c
 U dan
-Z ecacbd8e6749e1f68b1f30b320420a3c
+Z 4ed30e44a783453de01bc7b75236f201
diff --git a/manifest.uuid b/manifest.uuid
index a32f986633..69eabfe26d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-56d93d070d6b92d8a5a3fec1b09aae8911116c73d072fc5022f0b51668ed996b
\ No newline at end of file
+c3dc7b8d9de94e46fae75fcc96ec68743205e0a2591e14391c394fff203004d1
\ No newline at end of file
diff --git a/test/crash8.test b/test/crash8.test
index 2c6487b38b..7916e9b641 100644
--- a/test/crash8.test
+++ b/test/crash8.test
@@ -272,7 +272,7 @@ ifcapable pragma {
   if {[atomic_batch_write test.db]==0} {
     do_test crash8.4.1.1 {
       list [file exists test.db-journal] [file exists test2.db-journal]
-    } [list $bJrnl $bJrnl]
+    } {1 1}
   }
 
   do_test crash8-4.2 {

From cdb60978b9f2251e903cdd100df8278615628aea Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 13 Jan 2018 14:28:00 +0000
Subject: [PATCH 321/488] Fully initialize the Mem object for serial-type 10,
 in case such a serial-type is found in a corrupt database file.

FossilOrigin-Name: bd70a07d819a54346cb6c40fab681424c5af0dfb6bf29321a3de9fc99d285376
---
 manifest      | 15 +++++++--------
 manifest.uuid |  2 +-
 src/vdbeaux.c |  2 ++
 3 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index 043e201dd1..8cef8f94b4 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\svarious\sproblems\sin\stest\sscripts\spreventing\s"make\stest"\sfrom\spassing\son\nF2FS\sfile-systems\swith\sthe\s"atomic-write"\sfeature.
-D 2018-01-13T14:02:16.056
+C Fully\sinitialize\sthe\sMem\sobject\sfor\sserial-type\s10,\sin\scase\ssuch\sa\nserial-type\sis\sfound\sin\sa\scorrupt\sdatabase\sfile.
+D 2018-01-13T14:28:00.767
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -558,7 +558,7 @@ F src/vdbe.c ccc1e17a30325068ae4f0292e8601997946886d23acc989c68f2a261a2795c70
 F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a
 F src/vdbeInt.h 5442fc816b6cf19c8801724199fd6b77a02eb31a7a174021713f8c59b30e51fa
 F src/vdbeapi.c 02f773681d06e46454b0606339068d4d4490873dc4a7334bc0c6030552bb2c8c
-F src/vdbeaux.c fc124962c9900b19d201e57084a4302d07136a77f51fdbdfabd625a3ca3b6dc1
+F src/vdbeaux.c 2756ac68ac259c416554100598fc291870063288cd7e1af22847f57b3e130e56
 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191
 F src/vdbemem.c 7548dd5af03d24d534a5dbc41e3bbdf1fab83e9c8856a8d2549ed2ccf33d0e80
 F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f
@@ -1699,8 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P bb196fcb677e962db4308c02097968615729a51fe4a1292e468aaf3e4bf522d3 c3dc7b8d9de94e46fae75fcc96ec68743205e0a2591e14391c394fff203004d1
-R 095d7894b995bcb80e7e3cee4fddb19c
-T +closed c3dc7b8d9de94e46fae75fcc96ec68743205e0a2591e14391c394fff203004d1
-U dan
-Z 11c45edcc4119d583f229c1e527d94cf
+P 6bedc7435d26c1f21c0d1b3a52daa0169fa5416b690a99347328dcafdcd78740
+R a7ff4a80a5a33de73167133f07814a11
+U drh
+Z 940a381e739a90351c545fb8067ae8cb
diff --git a/manifest.uuid b/manifest.uuid
index 065366082a..b60772254f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6bedc7435d26c1f21c0d1b3a52daa0169fa5416b690a99347328dcafdcd78740
\ No newline at end of file
+bd70a07d819a54346cb6c40fab681424c5af0dfb6bf29321a3de9fc99d285376
\ No newline at end of file
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index 72a5b96bba..78777bd5a6 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -3457,6 +3457,8 @@ u32 sqlite3VdbeSerialGet(
     case 10: { /* Internal use only: NULL with virtual table
                ** UPDATE no-change flag set */
       pMem->flags = MEM_Null|MEM_Zero;
+      pMem->n = 0;
+      pMem->u.nZero = 0;
       break;
     }
     case 11:   /* Reserved for future use */

From f42884c307241c46994b05915e356c7931c352bc Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 13 Jan 2018 19:08:24 +0000
Subject: [PATCH 322/488] Support UPDATE statements against zipfile virtual
 tables.

FossilOrigin-Name: f2d2a5df4f29b47212fd2411eae6545087b901a270655640c87ceb472e02a24c
---
 ext/misc/zipfile.c | 302 +++++++++++++++++++++++++++------------------
 manifest           |  16 +--
 manifest.uuid      |   2 +-
 test/zipfile.test  |  79 +++++++++++-
 4 files changed, 267 insertions(+), 132 deletions(-)

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index 7ec8a222e7..88b7403f82 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -464,10 +464,47 @@ static u8* zipfileCsrBuffer(ZipfileCsr *pCsr){
 /*
 ** Magic numbers used to read CDS records.
 */
-#define ZIPFILE_CDS_FIXED_SZ  46
-#define ZIPFILE_CDS_NFILE_OFF 28
+#define ZIPFILE_CDS_FIXED_SZ         46
+#define ZIPFILE_CDS_NFILE_OFF        28
 
-static int zipfileReadCDS(ZipfileCsr *pCsr){
+/*
+** Decode the CDS record in buffer aBuf into (*pCDS). Return SQLITE_ERROR
+** if the record is not well-formed, or SQLITE_OK otherwise.
+*/
+static int zipfileReadCDS(u8 *aBuf, ZipfileCDS *pCDS){
+  u8 *aRead = aBuf;
+  u32 sig = zipfileRead32(aRead);
+  int rc = SQLITE_OK;
+  if( sig!=ZIPFILE_SIGNATURE_CDS ){
+    rc = SQLITE_ERROR;
+  }else{
+    pCDS->iVersionMadeBy = zipfileRead16(aRead);
+    pCDS->iVersionExtract = zipfileRead16(aRead);
+    pCDS->flags = zipfileRead16(aRead);
+    pCDS->iCompression = zipfileRead16(aRead);
+    pCDS->mTime = zipfileRead16(aRead);
+    pCDS->mDate = zipfileRead16(aRead);
+    pCDS->crc32 = zipfileRead32(aRead);
+    pCDS->szCompressed = zipfileRead32(aRead);
+    pCDS->szUncompressed = zipfileRead32(aRead);
+    assert( aRead==&aBuf[ZIPFILE_CDS_NFILE_OFF] );
+    pCDS->nFile = zipfileRead16(aRead);
+    pCDS->nExtra = zipfileRead16(aRead);
+    pCDS->nComment = zipfileRead16(aRead);
+    pCDS->iDiskStart = zipfileRead16(aRead);
+    pCDS->iInternalAttr = zipfileRead16(aRead);
+    pCDS->iExternalAttr = zipfileRead32(aRead);
+    pCDS->iOffset = zipfileRead32(aRead);
+    assert( aRead==&aBuf[ZIPFILE_CDS_FIXED_SZ] );
+  }
+
+  return rc;
+}
+
+/*
+** Read the CDS record for the current entry from disk into pCsr->cds.
+*/
+static int zipfileCsrReadCDS(ZipfileCsr *pCsr){
   char **pzErr = &pCsr->base.pVtab->zErrMsg;
   u8 *aRead;
   int rc = SQLITE_OK;
@@ -485,41 +522,19 @@ static int zipfileReadCDS(ZipfileCsr *pCsr){
   }
 
   if( rc==SQLITE_OK ){
-    u32 sig = zipfileRead32(aRead);
-    if( sig!=ZIPFILE_SIGNATURE_CDS ){
+    rc = zipfileReadCDS(aRead, &pCsr->cds);
+    if( rc!=SQLITE_OK ){
       assert( pCsr->pCurrent==0 );
       zipfileSetErrmsg(pCsr,"failed to read CDS at offset %lld",pCsr->iNextOff);
-      rc = SQLITE_ERROR;
     }else{
       int nRead;
-      pCsr->cds.iVersionMadeBy = zipfileRead16(aRead);
-      pCsr->cds.iVersionExtract = zipfileRead16(aRead);
-      pCsr->cds.flags = zipfileRead16(aRead);
-      pCsr->cds.iCompression = zipfileRead16(aRead);
-      pCsr->cds.mTime = zipfileRead16(aRead);
-      pCsr->cds.mDate = zipfileRead16(aRead);
-      pCsr->cds.crc32 = zipfileRead32(aRead);
-      pCsr->cds.szCompressed = zipfileRead32(aRead);
-      pCsr->cds.szUncompressed = zipfileRead32(aRead);
-      assert( pCsr->pCurrent 
-           || aRead==zipfileCsrBuffer(pCsr)+ZIPFILE_CDS_NFILE_OFF 
-      );
-      pCsr->cds.nFile = zipfileRead16(aRead);
-      pCsr->cds.nExtra = zipfileRead16(aRead);
-      pCsr->cds.nComment = zipfileRead16(aRead);
-      pCsr->cds.iDiskStart = zipfileRead16(aRead);
-      pCsr->cds.iInternalAttr = zipfileRead16(aRead);
-      pCsr->cds.iExternalAttr = zipfileRead32(aRead);
-      pCsr->cds.iOffset = zipfileRead32(aRead);
-      assert( pCsr->pCurrent 
-           || aRead==zipfileCsrBuffer(pCsr)+ZIPFILE_CDS_FIXED_SZ
-      );
-
       if( pCsr->pCurrent==0 ){
         nRead = pCsr->cds.nFile + pCsr->cds.nExtra;
         aRead = zipfileCsrBuffer(pCsr);
         pCsr->iNextOff += ZIPFILE_CDS_FIXED_SZ;
         rc = zipfileReadData(pCsr->pFile, aRead, nRead, pCsr->iNextOff, pzErr);
+      }else{
+        aRead = &aRead[ZIPFILE_CDS_FIXED_SZ];
       }
 
       if( rc==SQLITE_OK ){
@@ -570,37 +585,47 @@ static FILE *zipfileGetFd(ZipfileCsr *pCsr){
   return ((ZipfileTab*)(pCsr->base.pVtab))->pWriteFd;
 }
 
-static int zipfileReadLFH(ZipfileCsr *pCsr){
-  FILE *pFile = zipfileGetFd(pCsr);
-  char **pzErr = &pCsr->base.pVtab->zErrMsg;
+static int zipfileReadLFH(
+  FILE *pFd, 
+  i64 iOffset,
+  u8 *aTmp, 
+  ZipfileLFH *pLFH, 
+  char **pzErr
+){
+  u8 *aRead = aTmp;
   static const int szFix = ZIPFILE_LFH_FIXED_SZ;
-  u8 *aRead = zipfileCsrBuffer(pCsr);
   int rc;
 
-  rc = zipfileReadData(pFile, aRead, szFix, pCsr->cds.iOffset, pzErr);
+  rc = zipfileReadData(pFd, aRead, szFix, iOffset, pzErr);
   if( rc==SQLITE_OK ){
     u32 sig = zipfileRead32(aRead);
     if( sig!=ZIPFILE_SIGNATURE_LFH ){
-      zipfileSetErrmsg(pCsr, "failed to read LFH at offset %d", 
-          (int)pCsr->cds.iOffset
-      );
+      *pzErr = sqlite3_mprintf("failed to read LFH at offset %d", (int)iOffset);
       rc = SQLITE_ERROR;
     }else{
-      pCsr->lfh.iVersionExtract = zipfileRead16(aRead);
-      pCsr->lfh.flags = zipfileRead16(aRead);
-      pCsr->lfh.iCompression = zipfileRead16(aRead);
-      pCsr->lfh.mTime = zipfileRead16(aRead);
-      pCsr->lfh.mDate = zipfileRead16(aRead);
-      pCsr->lfh.crc32 = zipfileRead32(aRead);
-      pCsr->lfh.szCompressed = zipfileRead32(aRead);
-      pCsr->lfh.szUncompressed = zipfileRead32(aRead);
-      pCsr->lfh.nFile = zipfileRead16(aRead);
-      pCsr->lfh.nExtra = zipfileRead16(aRead);
-      assert( aRead==zipfileCsrBuffer(pCsr)+szFix );
-      pCsr->iDataOff = pCsr->cds.iOffset+szFix+pCsr->lfh.nFile+pCsr->lfh.nExtra;
+      pLFH->iVersionExtract = zipfileRead16(aRead);
+      pLFH->flags = zipfileRead16(aRead);
+      pLFH->iCompression = zipfileRead16(aRead);
+      pLFH->mTime = zipfileRead16(aRead);
+      pLFH->mDate = zipfileRead16(aRead);
+      pLFH->crc32 = zipfileRead32(aRead);
+      pLFH->szCompressed = zipfileRead32(aRead);
+      pLFH->szUncompressed = zipfileRead32(aRead);
+      pLFH->nFile = zipfileRead16(aRead);
+      pLFH->nExtra = zipfileRead16(aRead);
+      assert( aRead==&aTmp[szFix] );
     }
   }
+  return rc;
+}
 
+static int zipfileCsrReadLFH(ZipfileCsr *pCsr){
+  FILE *pFile = zipfileGetFd(pCsr);
+  char **pzErr = &pCsr->base.pVtab->zErrMsg;
+  u8 *aRead = zipfileCsrBuffer(pCsr);
+  int rc = zipfileReadLFH(pFile, pCsr->cds.iOffset, aRead, &pCsr->lfh, pzErr);
+  pCsr->iDataOff =  pCsr->cds.iOffset + ZIPFILE_LFH_FIXED_SZ;
+  pCsr->iDataOff += pCsr->lfh.nFile+pCsr->lfh.nExtra;
   return rc;
 }
 
@@ -629,9 +654,9 @@ static int zipfileNext(sqlite3_vtab_cursor *cur){
   }
 
   if( pCsr->bEof==0 ){
-    rc = zipfileReadCDS(pCsr);
+    rc = zipfileCsrReadCDS(pCsr);
     if( rc==SQLITE_OK ){
-      rc = zipfileReadLFH(pCsr);
+      rc = zipfileCsrReadLFH(pCsr);
     }
   }
 
@@ -770,6 +795,9 @@ static int zipfileColumn(
 ){
   ZipfileCsr *pCsr = (ZipfileCsr*)cur;
   int rc = SQLITE_OK;
+  if( i>=3 && sqlite3_vtab_nochange(ctx) ){
+    return SQLITE_OK;
+  }
   switch( i ){
     case 0:   /* name */
       sqlite3_result_text(ctx, pCsr->cds.zFile, -1, SQLITE_TRANSIENT);
@@ -989,15 +1017,26 @@ static int zipfileBestIndex(
 ** Add object pNew to the end of the linked list that begins at
 ** ZipfileTab.pFirstEntry and ends with pLastEntry.
 */
-static void zipfileAddEntry(ZipfileTab *pTab, ZipfileEntry *pNew){
+static void zipfileAddEntry(
+  ZipfileTab *pTab, 
+  ZipfileEntry *pBefore, 
+  ZipfileEntry *pNew
+){
   assert( (pTab->pFirstEntry==0)==(pTab->pLastEntry==0) );
   assert( pNew->pNext==0 );
-  if( pTab->pFirstEntry==0 ){
-    pTab->pFirstEntry = pTab->pLastEntry = pNew;
+  if( pBefore==0 ){
+    if( pTab->pFirstEntry==0 ){
+      pTab->pFirstEntry = pTab->pLastEntry = pNew;
+    }else{
+      assert( pTab->pLastEntry->pNext==0 );
+      pTab->pLastEntry->pNext = pNew;
+      pTab->pLastEntry = pNew;
+    }
   }else{
-    assert( pTab->pLastEntry->pNext==0 );
-    pTab->pLastEntry->pNext = pNew;
-    pTab->pLastEntry = pNew;
+    ZipfileEntry **pp;
+    for(pp=&pTab->pFirstEntry; *pp!=pBefore; pp=&((*pp)->pNext));
+    pNew->pNext = pBefore;
+    *pp = pNew;
   }
 }
 
@@ -1044,7 +1083,7 @@ static int zipfileLoadDirectory(ZipfileTab *pTab){
         pNew->aCdsEntry = (u8*)&pNew->zPath[nFile+1];
         pNew->nCdsEntry = ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment;
         memcpy(pNew->aCdsEntry, aRec, pNew->nCdsEntry);
-        zipfileAddEntry(pTab, pNew);
+        zipfileAddEntry(pTab, 0, pNew);
       }
 
       iOff += ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment;
@@ -1168,7 +1207,7 @@ static int zipfileGetMode(
   u32 mode = 0;
   if( z==0 ){
     mode = defaultMode;
-  }else if( z[0]>=0 && z[0]<=9 ){
+  }else if( z[0]>='0' && z[0]<='9' ){
     mode = (unsigned int)sqlite3_value_int(pVal);
   }else{
     const char zTemplate[11] = "-rwxrwxrwx";
@@ -1231,79 +1270,104 @@ static int zipfileUpdate(
   u8 *pFree = 0;                  /* Free this */
   char *zFree = 0;                /* Also free this */
   ZipfileCDS cds;                 /* New Central Directory Structure entry */
-
+  ZipfileEntry *pOld = 0;
   int bIsDir = 0;
 
-  int mNull;
-
   assert( pTab->zFile );
   assert( pTab->pWriteFd );
 
   if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
-    if( nVal>1 ){
-      return SQLITE_CONSTRAINT;
-    }else{
-      const char *zDelete = (const char*)sqlite3_value_text(apVal[0]);
-      int nDelete = strlen(zDelete);
-      ZipfileEntry *p;
-      for(p=pTab->pFirstEntry; p; p=p->pNext){
-        if( zipfileComparePath(p->zPath, zDelete, nDelete)==0 ){
-          p->bDeleted = 1;
-          break;
-        }
+    const char *zDelete = (const char*)sqlite3_value_text(apVal[0]);
+    int nDelete = strlen(zDelete);
+    for(pOld=pTab->pFirstEntry; 1; pOld=pOld->pNext){
+      if( pOld->bDeleted ) continue;
+      if( zipfileComparePath(pOld->zPath, zDelete, nDelete)==0 ){
+        pOld->bDeleted = 1;
+        break;
       }
-      return SQLITE_OK;
+      assert( pOld->pNext );
     }
+    if( nVal==1 ) return SQLITE_OK;
   }
 
-  mNull = (sqlite3_value_type(apVal[5])==SQLITE_NULL ? 0x0 : 0x8)  /* sz */
-        + (sqlite3_value_type(apVal[6])==SQLITE_NULL ? 0x0 : 0x4)  /* rawdata */
-        + (sqlite3_value_type(apVal[7])==SQLITE_NULL ? 0x0 : 0x2)  /* data */
-        + (sqlite3_value_type(apVal[8])==SQLITE_NULL ? 0x0 : 0x1); /* method */
-  if( mNull==0x00 ){     
-    /* All four are NULL - this must be a directory */
-    bIsDir = 1;
-  }
-  else if( mNull==0x2 || mNull==0x3 ){
-    /* Value specified for "data", and possibly "method". This must be
-    ** a regular file or a symlink. */
-    const u8 *aIn = sqlite3_value_blob(apVal[7]);
-    int nIn = sqlite3_value_bytes(apVal[7]);
-    int bAuto = sqlite3_value_type(apVal[8])==SQLITE_NULL;
-    
-    iMethod = sqlite3_value_int(apVal[8]);
-    sz = nIn;
-    if( iMethod!=0 && iMethod!=8 ){
-      rc = SQLITE_CONSTRAINT;
-    }else if( bAuto || iMethod ){
-      rc = zipfileDeflate(pTab, aIn, nIn, &pFree, &nData);
+  if( sqlite3_value_nochange(apVal[5]) && sqlite3_value_nochange(apVal[6])
+   && sqlite3_value_nochange(apVal[7]) && sqlite3_value_nochange(apVal[8])
+  ){
+    /* Reuse the data from the existing entry. */
+    FILE *pFile = pTab->pWriteFd;
+    ZipfileCDS cds;
+    zipfileReadCDS(pOld->aCdsEntry, &cds);
+
+    bIsDir = ((cds.iExternalAttr>>16) & S_IFDIR) ? 1 : 0;
+    sz = cds.szUncompressed;
+    iMethod = cds.iCompression;
+    if( sz>0 ){
+      char **pzErr = &pTab->base.zErrMsg;
+      ZipfileLFH lfh;
+      rc = zipfileReadLFH(pFile, cds.iOffset, pTab->aBuffer, &lfh, pzErr);
       if( rc==SQLITE_OK ){
-        if( iMethod || nData<nIn ){
-          iMethod = 8;
-          pData = pFree;
+        nData = lfh.szCompressed;
+        pData = pFree = sqlite3_malloc(nData);
+        if( pFree==NULL ){
+          rc = SQLITE_NOMEM;
         }else{
-          pData = aIn;
-          nData = nIn;
+          i64 iRead = cds.iOffset + ZIPFILE_LFH_FIXED_SZ + lfh.nFile+lfh.nExtra;
+          rc = zipfileReadData(pFile, pFree, nData, iRead, pzErr);
         }
       }
     }
-  }
-  else if( mNull==0x0D ){
-    /* Values specified for "sz", "rawdata" and "method". In other words,
-    ** pre-compressed data is being inserted.  */
-    pData = sqlite3_value_blob(apVal[6]);
-    nData = sqlite3_value_bytes(apVal[6]);
-    sz = sqlite3_value_int(apVal[5]);
-    iMethod = sqlite3_value_int(apVal[8]);
-    if( iMethod<0 || iMethod>65535 ){
-      pTab->base.zErrMsg = sqlite3_mprintf(
-          "zipfile: invalid compression method: %d", iMethod
-      );
-      rc = SQLITE_ERROR;
+  }else{
+    int mNull;
+    mNull = (sqlite3_value_type(apVal[5])==SQLITE_NULL ? 0x0 : 0x8)  /* sz */
+      + (sqlite3_value_type(apVal[6])==SQLITE_NULL ? 0x0 : 0x4)  /* rawdata */
+      + (sqlite3_value_type(apVal[7])==SQLITE_NULL ? 0x0 : 0x2)  /* data */
+      + (sqlite3_value_type(apVal[8])==SQLITE_NULL ? 0x0 : 0x1); /* method */
+    if( mNull==0x00 ){     
+      /* All four are NULL - this must be a directory */
+      bIsDir = 1;
+    }
+    else if( mNull==0x2 || mNull==0x3 ){
+      /* Value specified for "data", and possibly "method". This must be
+      ** a regular file or a symlink. */
+      const u8 *aIn = sqlite3_value_blob(apVal[7]);
+      int nIn = sqlite3_value_bytes(apVal[7]);
+      int bAuto = sqlite3_value_type(apVal[8])==SQLITE_NULL;
+
+      iMethod = sqlite3_value_int(apVal[8]);
+      sz = nIn;
+      pData = aIn;
+      nData = nIn;
+      if( iMethod!=0 && iMethod!=8 ){
+        rc = SQLITE_CONSTRAINT;
+      }else if( bAuto || iMethod ){
+        int nCmp;
+        rc = zipfileDeflate(pTab, aIn, nIn, &pFree, &nCmp);
+        if( rc==SQLITE_OK ){
+          if( iMethod || nCmp<nIn ){
+            iMethod = 8;
+            pData = pFree;
+            nData = nCmp;
+          }
+        }
+      }
+    }
+    else if( mNull==0x0D ){
+      /* Values specified for "sz", "rawdata" and "method". In other words,
+      ** pre-compressed data is being inserted.  */
+      pData = sqlite3_value_blob(apVal[6]);
+      nData = sqlite3_value_bytes(apVal[6]);
+      sz = sqlite3_value_int(apVal[5]);
+      iMethod = sqlite3_value_int(apVal[8]);
+      if( iMethod<0 || iMethod>65535 ){
+        pTab->base.zErrMsg = sqlite3_mprintf(
+            "zipfile: invalid compression method: %d", iMethod
+            );
+        rc = SQLITE_ERROR;
+      }
+    }
+    else{
+      rc = SQLITE_CONSTRAINT;
     }
-  }
-  else{
-    rc = SQLITE_CONSTRAINT;
   }
 
   if( rc==SQLITE_OK ){
@@ -1344,6 +1408,7 @@ static int zipfileUpdate(
   if( rc==SQLITE_OK ){
     ZipfileEntry *p;
     for(p=pTab->pFirstEntry; p; p=p->pNext){
+      if( p->bDeleted ) continue;
       if( zipfileComparePath(p->zPath, zPath, nPath)==0 ){
         rc = SQLITE_CONSTRAINT;
         break;
@@ -1368,7 +1433,7 @@ static int zipfileUpdate(
     if( pNew==0 ){
       rc = SQLITE_NOMEM;
     }else{
-      zipfileAddEntry(pTab, pNew);
+      zipfileAddEntry(pTab, pOld, pNew);
     }
   }
 
@@ -1377,6 +1442,9 @@ static int zipfileUpdate(
     rc = zipfileAppendEntry(pTab, &cds, zPath, nPath, pData, nData, (u32)mTime);
   }
 
+  if( rc!=SQLITE_OK && pOld ){
+    pOld->bDeleted = 0;
+  }
   sqlite3_free(pFree);
   sqlite3_free(zFree);
   return rc;
diff --git a/manifest b/manifest
index 8cef8f94b4..5633e43082 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fully\sinitialize\sthe\sMem\sobject\sfor\sserial-type\s10,\sin\scase\ssuch\sa\nserial-type\sis\sfound\sin\sa\scorrupt\sdatabase\sfile.
-D 2018-01-13T14:28:00.767
+C Support\sUPDATE\sstatements\sagainst\szipfile\svirtual\stables.
+D 2018-01-13T19:08:24.877
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -303,7 +303,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c e42d3ae79511ee86545bed07e9e5e7321f8139b55cdf73f60849352070bf12ad
+F ext/misc/zipfile.c fd05292abc98fb9ad84ffc24ac8f39a25472c992f81d3ea6f3379e0df36ef4a1
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -1600,7 +1600,7 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf
 F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc
 F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa
 F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc
-F test/zipfile.test a5cd98e91aebf343e21a32b97fadd6aefe02d249d5c6a1a3c2e624b88d7bdbe2
+F test/zipfile.test 61c6daf74f71f6d0c4d925cd7e23eb9ffe642491f8927b26aebba476d3244e50
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d
 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91
@@ -1699,7 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6bedc7435d26c1f21c0d1b3a52daa0169fa5416b690a99347328dcafdcd78740
-R a7ff4a80a5a33de73167133f07814a11
-U drh
-Z 940a381e739a90351c545fb8067ae8cb
+P bd70a07d819a54346cb6c40fab681424c5af0dfb6bf29321a3de9fc99d285376
+R 353db64d08ace436e64659cf45ce290c
+U dan
+Z 6cc524f643589f82fe6965a2f7fe8556
diff --git a/manifest.uuid b/manifest.uuid
index b60772254f..f828d42ddb 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-bd70a07d819a54346cb6c40fab681424c5af0dfb6bf29321a3de9fc99d285376
\ No newline at end of file
+f2d2a5df4f29b47212fd2411eae6545087b901a270655640c87ceb472e02a24c
\ No newline at end of file
diff --git a/test/zipfile.test b/test/zipfile.test
index fe1ad89c55..e766cb31bf 100644
--- a/test/zipfile.test
+++ b/test/zipfile.test
@@ -76,12 +76,79 @@ do_execsql_test 1.5.1 {
 do_execsql_test 1.5.2 {
   SELECT name FROM zz;
 } {f.txt g.txt h.txt i.txt}
+do_execsql_test 1.5.3 {
+  SELECT data FROM zz WHERE name='i.txt';
+} {zxcvb}
 
 do_execsql_test 1.6.0 {
   DELETE FROM zz WHERE name='g.txt';
   SELECT name FROM zz;
 } {f.txt h.txt i.txt}
 
+do_execsql_test 1.6.1 {
+  SELECT name, mode, mtime, data, method FROM zipfile('test.zip');
+} {
+  f.txt 33188 1000000000 abcde 0
+  h.txt 33188 1000000004 aaaaaaaaaabbbbbbbbbb 8
+  i.txt 33188 1000000006 zxcvb 0
+}
+
+do_execsql_test 1.6.2 {
+  UPDATE zz SET mtime=4 WHERE name='i.txt';
+  SELECT name, mode, mtime, data, method FROM zipfile('test.zip');
+} {
+  f.txt 33188 1000000000 abcde 0
+  h.txt 33188 1000000004 aaaaaaaaaabbbbbbbbbb 8
+  i.txt 33188 4 zxcvb 0
+}
+
+do_execsql_test 1.6.3 {
+  UPDATE zz SET mode='-rw-r--r-x' WHERE name='h.txt';
+  SELECT name, mode, mtime, data, method FROM zipfile('test.zip');
+} {
+  f.txt 33188 1000000000 abcde 0
+  h.txt 33189 1000000004 aaaaaaaaaabbbbbbbbbb 8
+  i.txt 33188 4 zxcvb 0
+}
+
+do_execsql_test 1.6.4 {
+  UPDATE zz SET name = 'blue.txt' WHERE name='f.txt';
+  SELECT name, mode, mtime, data, method FROM zipfile('test.zip');
+} {
+  blue.txt 33188 1000000000 abcde 0
+  h.txt 33189 1000000004 aaaaaaaaaabbbbbbbbbb 8
+  i.txt 33188 4 zxcvb 0
+}
+
+do_execsql_test 1.6.5 {
+  UPDATE zz SET data = 'edcba' WHERE name='blue.txt';
+  SELECT name, mode, mtime, data, method FROM zipfile('test.zip');
+} {
+  blue.txt 33188 1000000000 edcba 0
+  h.txt 33189 1000000004 aaaaaaaaaabbbbbbbbbb 8
+  i.txt 33188 4 zxcvb 0
+}
+
+do_execsql_test 1.6.6 {
+  UPDATE zz SET mode=NULL, data = NULL WHERE name='blue.txt';
+  SELECT name, mode, mtime, data, method FROM zipfile('test.zip');
+} {
+  blue.txt/ 16877 1000000000 {} 0
+  h.txt 33189 1000000004 aaaaaaaaaabbbbbbbbbb 8
+  i.txt 33188 4 zxcvb 0
+}
+
+do_catchsql_test 1.6.7 {
+  UPDATE zz SET data=NULL WHERE name='i.txt'
+} {1 {constraint failed}}
+do_execsql_test 1.6.8 {
+  SELECT name, mode, mtime, data, method FROM zipfile('test.zip');
+} {
+  blue.txt/ 16877 1000000000 {} 0
+  h.txt 33189 1000000004 aaaaaaaaaabbbbbbbbbb 8
+  i.txt 33188 4 zxcvb 0
+}
+
 #-------------------------------------------------------------------------
 db close
 forcedelete test.zip
@@ -104,11 +171,11 @@ do_execsql_test 2.2 {
 
 do_catchsql_test 2.3 {
   UPDATE zzz SET name = 'dirname3' WHERE name = 'dirname/';
-} {1 {constraint failed}}
+} {0 {}}
 do_execsql_test 2.4 {
   SELECT name, mode, data FROM zzz;
 } {
-  dirname/ 16877 {}
+  dirname3/ 16877 {}
   dirname2/ 16877 {}
   dirname2/file1.txt 33188 abcdefghijklmnop
 }
@@ -118,11 +185,11 @@ do_execsql_test 2.4 {
 if {$::tcl_platform(platform)=="unix"} {
   do_test 2.5.1 {
     forcedelete dirname
-      forcedelete dirname2
-      set rc [catch { exec unzip test.zip > /dev/null } msg]
-      list $rc $msg
+    forcedelete dirname2
+    set rc [catch { exec unzip test.zip > /dev/null } msg]
+    list $rc $msg
   } {0 {}}
-  do_test 2.5.2 { file isdir dirname } 1
+  do_test 2.5.2 { file isdir dirname3 } 1
   do_test 2.5.3 { file isdir dirname2 } 1
   do_test 2.5.4 { file isdir dirname2/file1.txt } 0
   do_test 2.5.5 { 

From eb545004193ed95aa41213b8aa24496e158085ac Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 13 Jan 2018 23:28:33 +0000
Subject: [PATCH 323/488] Fix harmless compiler warnings in zipfile.c.

FossilOrigin-Name: 8f7a592f8c044d75b4615a95e27454100b10c2b26f4cafee97dec23343821130
---
 ext/misc/zipfile.c |  3 +--
 manifest           | 14 +++++++-------
 manifest.uuid      |  2 +-
 3 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index 88b7403f82..63828e3c09 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -1278,7 +1278,7 @@ static int zipfileUpdate(
 
   if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
     const char *zDelete = (const char*)sqlite3_value_text(apVal[0]);
-    int nDelete = strlen(zDelete);
+    int nDelete = (int)strlen(zDelete);
     for(pOld=pTab->pFirstEntry; 1; pOld=pOld->pNext){
       if( pOld->bDeleted ) continue;
       if( zipfileComparePath(pOld->zPath, zDelete, nDelete)==0 ){
@@ -1295,7 +1295,6 @@ static int zipfileUpdate(
   ){
     /* Reuse the data from the existing entry. */
     FILE *pFile = pTab->pWriteFd;
-    ZipfileCDS cds;
     zipfileReadCDS(pOld->aCdsEntry, &cds);
 
     bIsDir = ((cds.iExternalAttr>>16) & S_IFDIR) ? 1 : 0;
diff --git a/manifest b/manifest
index 5633e43082..296050499c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Support\sUPDATE\sstatements\sagainst\szipfile\svirtual\stables.
-D 2018-01-13T19:08:24.877
+C Fix\sharmless\scompiler\swarnings\sin\szipfile.c.
+D 2018-01-13T23:28:33.571
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -303,7 +303,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c fd05292abc98fb9ad84ffc24ac8f39a25472c992f81d3ea6f3379e0df36ef4a1
+F ext/misc/zipfile.c 7001d7ca733d34a8b4fbacf7f8d322f3b2bf402f41052a021b5eda8227f0f5d4
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -1699,7 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P bd70a07d819a54346cb6c40fab681424c5af0dfb6bf29321a3de9fc99d285376
-R 353db64d08ace436e64659cf45ce290c
-U dan
-Z 6cc524f643589f82fe6965a2f7fe8556
+P f2d2a5df4f29b47212fd2411eae6545087b901a270655640c87ceb472e02a24c
+R 8e0eefc806344b75dcd41a32589e33e5
+U drh
+Z 25a119edfc3f91c75d6d9bd6eb4524cb
diff --git a/manifest.uuid b/manifest.uuid
index f828d42ddb..0bd548e74c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f2d2a5df4f29b47212fd2411eae6545087b901a270655640c87ceb472e02a24c
\ No newline at end of file
+8f7a592f8c044d75b4615a95e27454100b10c2b26f4cafee97dec23343821130
\ No newline at end of file

From b058d05452079aee046a07e94a406b547c02b9f5 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sun, 14 Jan 2018 20:12:23 +0000
Subject: [PATCH 324/488] Avoid excess stack usage when a VALUES clause with
 lots of rows occurs within a scalar expression.  This fixes a problem
 discovered by OSSFuzz.

FossilOrigin-Name: a4fa0581ba7cfd45fabe0198f55b3c2c8ee3ecfd2825aeed91116f44e77d760b
---
 manifest          | 16 ++++++++--------
 manifest.uuid     |  2 +-
 src/expr.c        |  1 -
 src/select.c      | 11 ++++++++---
 test/selectG.test | 20 ++++++++++++++++++++
 5 files changed, 37 insertions(+), 13 deletions(-)

diff --git a/manifest b/manifest
index 296050499c..4a8b555cf8 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sharmless\scompiler\swarnings\sin\szipfile.c.
-D 2018-01-13T23:28:33.571
+C Avoid\sexcess\sstack\susage\swhen\sa\sVALUES\sclause\swith\slots\sof\srows\soccurs\nwithin\sa\sscalar\sexpression.\s\sThis\sfixes\sa\sproblem\sdiscovered\sby\sOSSFuzz.
+D 2018-01-14T20:12:23.878
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -438,7 +438,7 @@ F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957
 F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
 F src/delete.c 20c8788451dc737a967c87ea53ad43544d617f5b57d32ccce8bd52a0daf9e89b
-F src/expr.c ad6e7a9c34a4bab9d10cc857d647ae7ce370a633b5c0bfa71f1c29b81ae364b8
+F src/expr.c 46a7d73d5579feaee7a7274fac0efea0bbae71dd5b107a569501d89e0280c762
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331
 F src/func.c bd528d5ed68ce5cbf78a762e3b735fa75009f7197ff07fab07fd771f35ebaa1b
@@ -485,7 +485,7 @@ F src/printf.c 9506b4b96e59c0467047155f09015750cb2878aeda3d39e5610c1192ddc3c41c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74
+F src/select.c bebe7cce45d899d2237c76bce059d525abf5b861f2fce92f6b53914a961c01ba
 F src/shell.c.in b87abffd0db09203ad8a133d56fe8f154ace5ec0a14197a153fb7d80b1438c01
 F src/sqlite.h.in 9daf78e8f3cecc9ea0c3a82201f75bb74f789ecbfcda28d2e47fa80b3d956961
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
@@ -1205,7 +1205,7 @@ F test/selectC.test e25243f8ca503e06f252eb0218976d07cfeceac3
 F test/selectD.test b0f02a04ef7737decb24e08be2c39b9664b43394
 F test/selectE.test a8730ca330fcf40ace158f134f4fe0eb00c7edbf
 F test/selectF.test 21c94e6438f76537b72532fa9fd4710cdd455fc3
-F test/selectG.test e8600e379589e85e9fefd2fe4d44a4cdd63f6982
+F test/selectG.test 089f7d3d7e6db91566f00b036cb353107a2cca6220eb1cb264085a836dae8840
 F test/server1.test 46803bd3fe8b99b30dbc5ff38ffc756f5c13a118
 F test/session.test 78fa2365e93d3663a6e933f86e7afc395adf18be
 F test/shared.test 1da9dbad400cee0d93f252ccf76e1ae007a63746
@@ -1699,7 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P f2d2a5df4f29b47212fd2411eae6545087b901a270655640c87ceb472e02a24c
-R 8e0eefc806344b75dcd41a32589e33e5
+P 8f7a592f8c044d75b4615a95e27454100b10c2b26f4cafee97dec23343821130
+R 5b961a6038bd65800f58ecff32ab6963
 U drh
-Z 25a119edfc3f91c75d6d9bd6eb4524cb
+Z 3c12035fed53223a4dcc5a3ae6914444
diff --git a/manifest.uuid b/manifest.uuid
index 0bd548e74c..2a881ffea9 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-8f7a592f8c044d75b4615a95e27454100b10c2b26f4cafee97dec23343821130
\ No newline at end of file
+a4fa0581ba7cfd45fabe0198f55b3c2c8ee3ecfd2825aeed91116f44e77d760b
\ No newline at end of file
diff --git a/src/expr.c b/src/expr.c
index 32cc4423fa..a63de5d9fc 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -2764,7 +2764,6 @@ int sqlite3CodeSubselect(
         pSel->pLimit = sqlite3PExpr(pParse, TK_LIMIT, pLimit, 0);
       }
       pSel->iLimit = 0;
-      pSel->selFlags &= ~SF_MultiValue;
       if( sqlite3Select(pParse, pSel, &dest) ){
         return 0;
       }
diff --git a/src/select.c b/src/select.c
index 1a4b0a93af..c3cb4082fe 100644
--- a/src/select.c
+++ b/src/select.c
@@ -2184,9 +2184,14 @@ static int multiSelectOrderBy(
 ** on a VALUES clause.
 **
 ** Because the Select object originates from a VALUES clause:
-**   (1) It has no LIMIT or OFFSET
+**   (1) There is no LIMIT or OFFSET or else there is a LIMIT of exactly 1
 **   (2) All terms are UNION ALL
 **   (3) There is no ORDER BY clause
+**
+** The "LIMIT of exactly 1" case of condition (1) comes about when a VALUES
+** clause occurs within scalar expression (ex: "SELECT (VALUES(1),(2),(3))").
+** The sqlite3CodeSubselect will have added the LIMIT 1 clause in tht case.
+** Since the limit is exactly 1, we only need to evalutes the left-most VALUES.
 */
 static int multiSelectValues(
   Parse *pParse,        /* Parsing context */
@@ -2194,13 +2199,13 @@ static int multiSelectValues(
   SelectDest *pDest     /* What to do with query results */
 ){
   Select *pPrior;
+  Select *pRightmost = p;
   int nRow = 1;
   int rc = 0;
   assert( p->selFlags & SF_MultiValue );
   do{
     assert( p->selFlags & SF_Values );
     assert( p->op==TK_ALL || (p->op==TK_SELECT && p->pPrior==0) );
-    assert( p->pLimit==0 );
     assert( p->pNext==0 || p->pEList->nExpr==p->pNext->pEList->nExpr );
     if( p->pPrior==0 ) break;
     assert( p->pPrior->pNext==p );
@@ -2212,7 +2217,7 @@ static int multiSelectValues(
     p->pPrior = 0;
     rc = sqlite3Select(pParse, p, pDest);
     p->pPrior = pPrior;
-    if( rc ) break;
+    if( rc || pRightmost->pLimit ) break;
     p->nSelectRow = nRow;
     p = p->pNext;
   }
diff --git a/test/selectG.test b/test/selectG.test
index 86d89b121b..fab4c4ed4d 100644
--- a/test/selectG.test
+++ b/test/selectG.test
@@ -36,4 +36,24 @@ do_test 100 {
   }
 } {100000 5000050000 50000.5 1}
   
+# 2018-01-14.  A 100K-entry VALUES clause within a scalar expression does
+# not cause processor stack overflow.
+#
+do_test 110 {
+  set sql "SELECT (VALUES"
+  for {set i 1} {$i<100000} {incr i} {
+    append sql "($i),"
+  }
+  append sql "($i));"
+  db eval $sql
+} {1}
+
+# Only the left-most term of a multi-valued VALUES within a scalar
+# expression is evaluated.
+#
+do_test 120 {
+  set n [llength [split [db eval "explain $sql"] \n]]
+  expr {$n<10}
+} {1}
+
 finish_test

From a9be508a9766d69194212dd89c17d6feaa68cbc3 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 15 Jan 2018 14:32:37 +0000
Subject: [PATCH 325/488] Fix an error in the setDeviceCharacteristics()
 procedure for the (unsupported) QNX code in os_unix.c.

FossilOrigin-Name: 8151913a3987f4dd2d6efee046727f5fa9b6f11d5d3867ea8f512c03a212ac2b
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/os_unix.c |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 4a8b555cf8..687bdf64d0 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Avoid\sexcess\sstack\susage\swhen\sa\sVALUES\sclause\swith\slots\sof\srows\soccurs\nwithin\sa\sscalar\sexpression.\s\sThis\sfixes\sa\sproblem\sdiscovered\sby\sOSSFuzz.
-D 2018-01-14T20:12:23.878
+C Fix\san\serror\sin\sthe\ssetDeviceCharacteristics()\sprocedure\sfor\sthe\n(unsupported)\sQNX\scode\sin\sos_unix.c.
+D 2018-01-15T14:32:37.594
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -469,7 +469,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104
 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
-F src/os_unix.c ecdffdc7fc25c07e42908be7c5ea30456fee6263e0d54cdf204557945b445da2
+F src/os_unix.c a82505be158d8ce42b38dcc9b426187d776904c12cdc68dc8925e1dfcc5cb6ce
 F src/os_win.c 0a4afa35cc8e812000df3ea2f64b476131b39e29e75d8007d0504726e4761de4
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 9b9cb4e06c03d43d62480a7a685a012d645fcf3a39e7767ccb505fb41ee083ec
@@ -1699,7 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 8f7a592f8c044d75b4615a95e27454100b10c2b26f4cafee97dec23343821130
-R 5b961a6038bd65800f58ecff32ab6963
+P a4fa0581ba7cfd45fabe0198f55b3c2c8ee3ecfd2825aeed91116f44e77d760b
+R 755347b860dbdbaa8de782672cfa2550
 U drh
-Z 3c12035fed53223a4dcc5a3ae6914444
+Z bf44e658c85afb034dcb761e50e86217
diff --git a/manifest.uuid b/manifest.uuid
index 2a881ffea9..3e03feb0d3 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a4fa0581ba7cfd45fabe0198f55b3c2c8ee3ecfd2825aeed91116f44e77d760b
\ No newline at end of file
+8151913a3987f4dd2d6efee046727f5fa9b6f11d5d3867ea8f512c03a212ac2b
\ No newline at end of file
diff --git a/src/os_unix.c b/src/os_unix.c
index d368cb34ea..94b1efd87d 100644
--- a/src/os_unix.c
+++ b/src/os_unix.c
@@ -3950,7 +3950,7 @@ static void setDeviceCharacteristics(unixFile *pFile){
     pFile->sectorSize = SQLITE_DEFAULT_SECTOR_SIZE;
     pFile->deviceCharacteristics = 0;
     if( fstatvfs(pFile->h, &fsInfo) == -1 ) {
-      return pFile->sectorSize;
+      return;
     }
 
     if( !strcmp(fsInfo.f_basetype, "tmp") ) {

From 89fa74694149ca5c4f57338f6f161737f991bedc Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 15 Jan 2018 15:49:46 +0000
Subject: [PATCH 326/488] Fix a zipfile problem with extracting zero length
 files compressed using deflate.

FossilOrigin-Name: cf64087224aff1a2fe169d23996d9e5ed8d86459c655eb5d0bace0466a557ec6
---
 ext/misc/zipfile.c | 129 ++++++++++++++++++++++++++++++++++++++++-----
 manifest           |  14 ++---
 manifest.uuid      |   2 +-
 3 files changed, 124 insertions(+), 21 deletions(-)

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index 63828e3c09..27f55f6c14 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -64,7 +64,7 @@ static const char ZIPFILE_SCHEMA[] =
     "rawdata,"           /* 4: Raw data */
     "data,"              /* 5: Uncompressed data */
     "method,"            /* 6: Compression method (integer) */
-    "file HIDDEN"        /* 7: Name of zip file */
+    "z HIDDEN"           /* 7: Name of zip file */
   ") WITHOUT ROWID;";
 
 #define ZIPFILE_F_COLUMN_IDX 7    /* Index of column "file" in the above */
@@ -235,6 +235,7 @@ struct ZipfileEntry {
 typedef struct ZipfileCsr ZipfileCsr;
 struct ZipfileCsr {
   sqlite3_vtab_cursor base;  /* Base class - must be first */
+  i64 iId;                   /* Cursor ID */
   int bEof;                  /* True when at EOF */
 
   /* Used outside of write transactions */
@@ -264,8 +265,10 @@ struct ZipfileTab {
   char *zFile;               /* Zip file this table accesses (may be NULL) */
   u8 *aBuffer;               /* Temporary buffer used for various tasks */
 
-  /* The following are used by write transactions only */
   ZipfileCsr *pCsrList;      /* List of cursors */
+  i64 iNextCsrid;
+
+  /* The following are used by write transactions only */
   ZipfileEntry *pFirstEntry; /* Linked list of all files (if pWriteFd!=0) */
   ZipfileEntry *pLastEntry;  /* Last element in pFirstEntry list */
   FILE *pWriteFd;            /* File handle open on zip archive */
@@ -344,6 +347,7 @@ static int zipfileDisconnect(sqlite3_vtab *pVtab){
 ** Constructor for a new ZipfileCsr object.
 */
 static int zipfileOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){
+  ZipfileTab *pTab = (ZipfileTab*)p;
   ZipfileCsr *pCsr;
   pCsr = sqlite3_malloc(sizeof(*pCsr));
   *ppCsr = (sqlite3_vtab_cursor*)pCsr;
@@ -351,6 +355,9 @@ static int zipfileOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){
     return SQLITE_NOMEM;
   }
   memset(pCsr, 0, sizeof(*pCsr));
+  pCsr->iId = ++pTab->iNextCsrid;
+  pCsr->pCsrNext = pTab->pCsrList;
+  pTab->pCsrList = pCsr;
   return SQLITE_OK;
 }
 
@@ -359,14 +366,6 @@ static int zipfileOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){
 ** by zipfileOpen().
 */
 static void zipfileResetCursor(ZipfileCsr *pCsr){
-  ZipfileTab *pTab = (ZipfileTab*)(pCsr->base.pVtab);
-  ZipfileCsr **pp;
-
-  /* Remove this cursor from the ZipfileTab.pCsrList list. */
-  for(pp=&pTab->pCsrList; *pp; pp=&((*pp)->pCsrNext)){
-    if( *pp==pCsr ) *pp = pCsr->pCsrNext;
-  }
-
   sqlite3_free(pCsr->cds.zFile);
   pCsr->cds.zFile = 0;
   pCsr->bEof = 0;
@@ -381,7 +380,18 @@ static void zipfileResetCursor(ZipfileCsr *pCsr){
 */
 static int zipfileClose(sqlite3_vtab_cursor *cur){
   ZipfileCsr *pCsr = (ZipfileCsr*)cur;
+  ZipfileTab *pTab = (ZipfileTab*)(pCsr->base.pVtab);
+  ZipfileCsr **pp;
   zipfileResetCursor(pCsr);
+
+  /* Remove this cursor from the ZipfileTab.pCsrList list. */
+  for(pp=&pTab->pCsrList; *pp; pp=&((*pp)->pCsrNext)){
+    if( *pp==pCsr ){ 
+      *pp = pCsr->pCsrNext;
+      break;
+    }
+  }
+
   sqlite3_free(pCsr);
   return SQLITE_OK;
 }
@@ -823,7 +833,8 @@ static int zipfileColumn(
     case 5: { /* data */
       if( i==4 || pCsr->cds.iCompression==0 || pCsr->cds.iCompression==8 ){
         int sz = pCsr->cds.szCompressed;
-        if( sz>0 ){
+        int szFinal = pCsr->cds.szUncompressed;
+        if( szFinal>0 ){
           u8 *aBuf = sqlite3_malloc(sz);
           if( aBuf==0 ){
             rc = SQLITE_NOMEM;
@@ -835,12 +846,20 @@ static int zipfileColumn(
           }
           if( rc==SQLITE_OK ){
             if( i==5 && pCsr->cds.iCompression ){
-              zipfileInflate(ctx, aBuf, sz, pCsr->cds.szUncompressed);
+              zipfileInflate(ctx, aBuf, sz, szFinal);
             }else{
               sqlite3_result_blob(ctx, aBuf, sz, SQLITE_TRANSIENT);
             }
             sqlite3_free(aBuf);
           }
+        }else{
+          /* Figure out if this is a directory or a zero-sized file. Consider
+          ** it to be a directory either if the mode suggests so, or if
+          ** the final character in the name is '/'.  */
+          u32 mode = pCsr->cds.iExternalAttr >> 16;
+          if( !(mode & S_IFDIR) && pCsr->cds.zFile[pCsr->cds.nFile-1]!='/' ){
+            sqlite3_result_blob(ctx, "", 0, SQLITE_STATIC);
+          }
         }
       }
       break;
@@ -848,6 +867,9 @@ static int zipfileColumn(
     case 6:   /* method */
       sqlite3_result_int(ctx, pCsr->cds.iCompression);
       break;
+    case 7:   /* z */
+      sqlite3_result_int64(ctx, pCsr->iId);
+      break;
   }
 
   return SQLITE_OK;
@@ -1553,6 +1575,84 @@ static int zipfileRollback(sqlite3_vtab *pVtab){
   return zipfileCommit(pVtab);
 }
 
+static ZipfileCsr *zipfileFindCursor(ZipfileTab *pTab, i64 iId){
+  ZipfileCsr *pCsr;
+  for(pCsr=pTab->pCsrList; pCsr; pCsr=pCsr->pCsrNext){
+    if( iId==pCsr->iId ) break;
+  }
+  return pCsr;
+}
+
+static void zipfileFunctionCds(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  ZipfileCsr *pCsr;
+  ZipfileTab *pTab = (ZipfileTab*)sqlite3_user_data(context);
+  assert( argc>0 );
+
+  pCsr = zipfileFindCursor(pTab, sqlite3_value_int64(argv[0]));
+  if( pCsr ){
+    ZipfileCDS *p = &pCsr->cds;
+    char *zRes = sqlite3_mprintf("{"
+        "\"version-made-by\" : %u, "
+        "\"version-to-extract\" : %u, "
+        "\"flags\" : %u, "
+        "\"compression\" : %u, "
+        "\"time\" : %u, "
+        "\"date\" : %u, "
+        "\"crc32\" : %u, "
+        "\"compressed-size\" : %u, "
+        "\"uncompressed-size\" : %u, "
+        "\"file-name-length\" : %u, "
+        "\"extra-field-length\" : %u, "
+        "\"file-comment-length\" : %u, "
+        "\"disk-number-start\" : %u, "
+        "\"internal-attr\" : %u, "
+        "\"external-attr\" : %u, "
+        "\"offset\" : %u }",
+        (u32)p->iVersionMadeBy, (u32)p->iVersionExtract,
+        (u32)p->flags, (u32)p->iCompression,
+        (u32)p->mTime, (u32)p->mDate,
+        (u32)p->crc32, (u32)p->szCompressed,
+        (u32)p->szUncompressed, (u32)p->nFile,
+        (u32)p->nExtra, (u32)p->nComment,
+        (u32)p->iDiskStart, (u32)p->iInternalAttr,
+        (u32)p->iExternalAttr, (u32)p->iOffset
+    );
+
+    if( zRes==0 ){
+      sqlite3_result_error_nomem(context);
+    }else{
+      sqlite3_result_text(context, zRes, -1, SQLITE_TRANSIENT);
+      sqlite3_free(zRes);
+    }
+  }
+}
+
+
+/*
+** xFindFunction method.
+*/
+static int zipfileFindFunction(
+  sqlite3_vtab *pVtab,            /* Virtual table handle */
+  int nArg,                       /* Number of SQL function arguments */
+  const char *zName,              /* Name of SQL function */
+  void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */
+  void **ppArg                    /* OUT: User data for *pxFunc */
+){
+  if( nArg>0 ){
+    if( sqlite3_stricmp("zipfile_cds", zName)==0 ){
+      *pxFunc = zipfileFunctionCds;
+      *ppArg = (void*)pVtab;
+      return 1;
+    }
+  }
+
+  return 0;
+}
+
 /*
 ** Register the "zipfile" virtual table.
 */
@@ -1576,11 +1676,14 @@ static int zipfileRegister(sqlite3 *db){
     0,                         /* xSync */
     zipfileCommit,             /* xCommit */
     zipfileRollback,           /* xRollback */
-    0,                         /* xFindMethod */
+    zipfileFindFunction,       /* xFindMethod */
     0,                         /* xRename */
   };
 
   int rc = sqlite3_create_module(db, "zipfile"  , &zipfileModule, 0);
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_overload_function(db, "zipfile_cds", -1);
+  }
   return rc;
 }
 #else         /* SQLITE_OMIT_VIRTUALTABLE */
diff --git a/manifest b/manifest
index 687bdf64d0..f7654f5e02 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\san\serror\sin\sthe\ssetDeviceCharacteristics()\sprocedure\sfor\sthe\n(unsupported)\sQNX\scode\sin\sos_unix.c.
-D 2018-01-15T14:32:37.594
+C Fix\sa\szipfile\sproblem\swith\sextracting\szero\slength\sfiles\scompressed\susing\ndeflate.
+D 2018-01-15T15:49:46.517
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -303,7 +303,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c 7001d7ca733d34a8b4fbacf7f8d322f3b2bf402f41052a021b5eda8227f0f5d4
+F ext/misc/zipfile.c 46171a19439d0c76acf48770e736c281536f160d3a5a96a0511e34402e262fac
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -1699,7 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a4fa0581ba7cfd45fabe0198f55b3c2c8ee3ecfd2825aeed91116f44e77d760b
-R 755347b860dbdbaa8de782672cfa2550
-U drh
-Z bf44e658c85afb034dcb761e50e86217
+P 8151913a3987f4dd2d6efee046727f5fa9b6f11d5d3867ea8f512c03a212ac2b
+R d9864f9f3d3219c31b427d2f3aa9a193
+U dan
+Z 4b1456c73d542678caf828eee79b5bff
diff --git a/manifest.uuid b/manifest.uuid
index 3e03feb0d3..19ee4478a0 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-8151913a3987f4dd2d6efee046727f5fa9b6f11d5d3867ea8f512c03a212ac2b
\ No newline at end of file
+cf64087224aff1a2fe169d23996d9e5ed8d86459c655eb5d0bace0466a557ec6
\ No newline at end of file

From 2d620070b945c86515261863a6cbecc11bfd2991 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 15 Jan 2018 19:00:35 +0000
Subject: [PATCH 327/488] Fix a problem in the zipfile module causing it to
 generate incorrect checksums. Remove the ability to insert compressed data
 into a zip archive.

FossilOrigin-Name: b0b7d0363acf38c2178e2d3041d8ce2a0de061a51caa64670dbf539ee6d4356b
---
 ext/misc/zipfile.c | 91 +++++++++++++---------------------------------
 manifest           | 14 +++----
 manifest.uuid      |  2 +-
 test/zipfile.test  | 15 ++++++--
 4 files changed, 46 insertions(+), 76 deletions(-)

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index 27f55f6c14..f17b3735fb 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -805,9 +805,6 @@ static int zipfileColumn(
 ){
   ZipfileCsr *pCsr = (ZipfileCsr*)cur;
   int rc = SQLITE_OK;
-  if( i>=3 && sqlite3_vtab_nochange(ctx) ){
-    return SQLITE_OK;
-  }
   switch( i ){
     case 0:   /* name */
       sqlite3_result_text(ctx, pCsr->cds.zFile, -1, SQLITE_TRANSIENT);
@@ -826,10 +823,13 @@ static int zipfileColumn(
       break;
     }
     case 3: { /* sz */
-      sqlite3_result_int64(ctx, pCsr->cds.szUncompressed);
+      if( sqlite3_vtab_nochange(ctx)==0 ){
+        sqlite3_result_int64(ctx, pCsr->cds.szUncompressed);
+      }
       break;
     }
     case 4:   /* rawdata */
+      if( sqlite3_vtab_nochange(ctx) ) break;
     case 5: { /* data */
       if( i==4 || pCsr->cds.iCompression==0 || pCsr->cds.iCompression==8 ){
         int sz = pCsr->cds.szCompressed;
@@ -872,7 +872,7 @@ static int zipfileColumn(
       break;
   }
 
-  return SQLITE_OK;
+  return rc;
 }
 
 /*
@@ -908,7 +908,8 @@ static int zipfileReadEOCD(
   fseek(pFile, 0, SEEK_END);
   szFile = (i64)ftell(pFile);
   if( szFile==0 ){
-    return SQLITE_EMPTY;
+    memset(pEOCD, 0, sizeof(ZipfileEOCD));
+    return SQLITE_OK;
   }
   nRead = (int)(MIN(szFile, ZIPFILE_BUFFER_SIZE));
   iOff = szFile - nRead;
@@ -986,11 +987,12 @@ static int zipfileFilter(
     }else{
       rc = zipfileReadEOCD(pTab, pCsr->pFile, &pCsr->eocd);
       if( rc==SQLITE_OK ){
-        pCsr->iNextOff = pCsr->eocd.iOffset;
-        rc = zipfileNext(cur);
-      }else if( rc==SQLITE_EMPTY ){
-        rc = SQLITE_OK;
-        pCsr->bEof = 1;
+        if( pCsr->eocd.nEntry==0 ){
+          pCsr->bEof = 1;
+        }else{
+          pCsr->iNextOff = pCsr->eocd.iOffset;
+          rc = zipfileNext(cur);
+        }
       }
     }
   }else{
@@ -1067,7 +1069,7 @@ static int zipfileLoadDirectory(ZipfileTab *pTab){
   int rc;
 
   rc = zipfileReadEOCD(pTab, pTab->pWriteFd, &eocd);
-  if( rc==SQLITE_OK ){
+  if( rc==SQLITE_OK && eocd.nEntry>0 ){
     int i;
     int iOff = 0;
     u8 *aBuf = sqlite3_malloc(eocd.nSize);
@@ -1112,8 +1114,6 @@ static int zipfileLoadDirectory(ZipfileTab *pTab){
     }
 
     sqlite3_free(aBuf);
-  }else if( rc==SQLITE_EMPTY ){
-    rc = SQLITE_OK;
   }
 
   return rc;
@@ -1294,6 +1294,7 @@ static int zipfileUpdate(
   ZipfileCDS cds;                 /* New Central Directory Structure entry */
   ZipfileEntry *pOld = 0;
   int bIsDir = 0;
+  u32 iCrc32 = 0;
 
   assert( pTab->zFile );
   assert( pTab->pWriteFd );
@@ -1312,42 +1313,18 @@ static int zipfileUpdate(
     if( nVal==1 ) return SQLITE_OK;
   }
 
-  if( sqlite3_value_nochange(apVal[5]) && sqlite3_value_nochange(apVal[6])
-   && sqlite3_value_nochange(apVal[7]) && sqlite3_value_nochange(apVal[8])
+  /* Check that "sz" and "rawdata" are both NULL: */
+  if( sqlite3_value_type(apVal[5])!=SQLITE_NULL
+   || sqlite3_value_type(apVal[6])!=SQLITE_NULL
   ){
-    /* Reuse the data from the existing entry. */
-    FILE *pFile = pTab->pWriteFd;
-    zipfileReadCDS(pOld->aCdsEntry, &cds);
+    rc = SQLITE_CONSTRAINT;
+  }
 
-    bIsDir = ((cds.iExternalAttr>>16) & S_IFDIR) ? 1 : 0;
-    sz = cds.szUncompressed;
-    iMethod = cds.iCompression;
-    if( sz>0 ){
-      char **pzErr = &pTab->base.zErrMsg;
-      ZipfileLFH lfh;
-      rc = zipfileReadLFH(pFile, cds.iOffset, pTab->aBuffer, &lfh, pzErr);
-      if( rc==SQLITE_OK ){
-        nData = lfh.szCompressed;
-        pData = pFree = sqlite3_malloc(nData);
-        if( pFree==NULL ){
-          rc = SQLITE_NOMEM;
-        }else{
-          i64 iRead = cds.iOffset + ZIPFILE_LFH_FIXED_SZ + lfh.nFile+lfh.nExtra;
-          rc = zipfileReadData(pFile, pFree, nData, iRead, pzErr);
-        }
-      }
-    }
-  }else{
-    int mNull;
-    mNull = (sqlite3_value_type(apVal[5])==SQLITE_NULL ? 0x0 : 0x8)  /* sz */
-      + (sqlite3_value_type(apVal[6])==SQLITE_NULL ? 0x0 : 0x4)  /* rawdata */
-      + (sqlite3_value_type(apVal[7])==SQLITE_NULL ? 0x0 : 0x2)  /* data */
-      + (sqlite3_value_type(apVal[8])==SQLITE_NULL ? 0x0 : 0x1); /* method */
-    if( mNull==0x00 ){     
-      /* All four are NULL - this must be a directory */
+  if( rc==SQLITE_OK ){
+    if( sqlite3_value_type(apVal[7])==SQLITE_NULL ){
+      /* data=NULL. A directory */
       bIsDir = 1;
-    }
-    else if( mNull==0x2 || mNull==0x3 ){
+    }else{
       /* Value specified for "data", and possibly "method". This must be
       ** a regular file or a symlink. */
       const u8 *aIn = sqlite3_value_blob(apVal[7]);
@@ -1370,25 +1347,9 @@ static int zipfileUpdate(
             nData = nCmp;
           }
         }
+        iCrc32 = crc32(0, aIn, nIn);
       }
     }
-    else if( mNull==0x0D ){
-      /* Values specified for "sz", "rawdata" and "method". In other words,
-      ** pre-compressed data is being inserted.  */
-      pData = sqlite3_value_blob(apVal[6]);
-      nData = sqlite3_value_bytes(apVal[6]);
-      sz = sqlite3_value_int(apVal[5]);
-      iMethod = sqlite3_value_int(apVal[8]);
-      if( iMethod<0 || iMethod>65535 ){
-        pTab->base.zErrMsg = sqlite3_mprintf(
-            "zipfile: invalid compression method: %d", iMethod
-            );
-        rc = SQLITE_ERROR;
-      }
-    }
-    else{
-      rc = SQLITE_CONSTRAINT;
-    }
   }
 
   if( rc==SQLITE_OK ){
@@ -1445,7 +1406,7 @@ static int zipfileUpdate(
     cds.flags = ZIPFILE_NEWENTRY_FLAGS;
     cds.iCompression = (u16)iMethod;
     zipfileMtimeToDos(&cds, (u32)mTime);
-    cds.crc32 = crc32(0, pData, nData);
+    cds.crc32 = iCrc32;
     cds.szCompressed = nData;
     cds.szUncompressed = (u32)sz;
     cds.iExternalAttr = (mode<<16);
diff --git a/manifest b/manifest
index f7654f5e02..47646bc8de 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\szipfile\sproblem\swith\sextracting\szero\slength\sfiles\scompressed\susing\ndeflate.
-D 2018-01-15T15:49:46.517
+C Fix\sa\sproblem\sin\sthe\szipfile\smodule\scausing\sit\sto\sgenerate\sincorrect\nchecksums.\sRemove\sthe\sability\sto\sinsert\scompressed\sdata\sinto\sa\szip\sarchive.
+D 2018-01-15T19:00:35.051
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -303,7 +303,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c 46171a19439d0c76acf48770e736c281536f160d3a5a96a0511e34402e262fac
+F ext/misc/zipfile.c 7e48d2947a1fe71d22e54f3b499d6d0581efa105033683585d3dbde76cffaed7
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -1600,7 +1600,7 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf
 F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc
 F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa
 F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc
-F test/zipfile.test 61c6daf74f71f6d0c4d925cd7e23eb9ffe642491f8927b26aebba476d3244e50
+F test/zipfile.test 71a9d37bb928a1dcee6ab624e8be7bca9f807364440d20fc84363c44ab2f4ac5
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d
 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91
@@ -1699,7 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 8151913a3987f4dd2d6efee046727f5fa9b6f11d5d3867ea8f512c03a212ac2b
-R d9864f9f3d3219c31b427d2f3aa9a193
+P cf64087224aff1a2fe169d23996d9e5ed8d86459c655eb5d0bace0466a557ec6
+R 5e0c54dfbab7bd3d719e333800776169
 U dan
-Z 4b1456c73d542678caf828eee79b5bff
+Z a41cf66d60fe2e5069108d980e3c47d6
diff --git a/manifest.uuid b/manifest.uuid
index 19ee4478a0..78a1e43434 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-cf64087224aff1a2fe169d23996d9e5ed8d86459c655eb5d0bace0466a557ec6
\ No newline at end of file
+b0b7d0363acf38c2178e2d3041d8ce2a0de061a51caa64670dbf539ee6d4356b
\ No newline at end of file
diff --git a/test/zipfile.test b/test/zipfile.test
index e766cb31bf..e246c9ab18 100644
--- a/test/zipfile.test
+++ b/test/zipfile.test
@@ -36,13 +36,22 @@ do_execsql_test 1.0 {
   6 method {} 0 {} 0
 }
 
-do_execsql_test 1.1.1 {
+do_catchsql_test 1.1.0.1 {
   INSERT INTO zz(name, mode, mtime, sz, rawdata, method) 
   VALUES('f.txt', '-rw-r--r--', 1000000000, 5, 'abcde', 0);
-}
-do_execsql_test 1.1.2 {
+} {1 {constraint failed}}
+do_catchsql_test 1.1.0.1 {
   INSERT INTO zz(name, mtime, sz, rawdata, method) 
   VALUES('g.txt', 1000000002, 5, '12345', 0);
+} {1 {constraint failed}}
+
+do_execsql_test 1.1.1 {
+  INSERT INTO zz(name, mode, mtime, data, method) 
+  VALUES('f.txt', '-rw-r--r--', 1000000000, 'abcde', 0);
+}
+do_execsql_test 1.1.2 {
+  INSERT INTO zz(name, mode, mtime, data, method) 
+  VALUES('g.txt', NULL, 1000000002, '12345', 0);
 }
 
 do_execsql_test 1.2 {

From ded2d99d739f3c0b767883e1eea368cf6ecf78f8 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 16 Jan 2018 02:38:35 +0000
Subject: [PATCH 328/488] Disable the ".archive" command tests in shell8.test
 if the CLI is compiled without ZLIB support.

FossilOrigin-Name: ce8bfe6c2b87090a2de1e04bc88fcb878597fe1f4ecd5df6d9d588a65601c901
---
 manifest         | 14 +++++++-------
 manifest.uuid    |  2 +-
 test/shell8.test |  6 ++++++
 3 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 47646bc8de..fe09344538 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sproblem\sin\sthe\szipfile\smodule\scausing\sit\sto\sgenerate\sincorrect\nchecksums.\sRemove\sthe\sability\sto\sinsert\scompressed\sdata\sinto\sa\szip\sarchive.
-D 2018-01-15T19:00:35.051
+C Disable\sthe\s".archive"\scommand\stests\sin\sshell8.test\sif\sthe\sCLI\sis\scompiled\swithout\nZLIB\ssupport.
+D 2018-01-16T02:38:35.189
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -1227,7 +1227,7 @@ F test/shell4.test 89ad573879a745974ff2df20ff97c5d6ffffbd5d
 F test/shell5.test 23939a4c51f0421330ea61dbd3c74f9c215f5f8d3d1a94846da6ffc777a35458
 F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3
 F test/shell7.test 115132f66d0463417f408562cc2cf534f6bbc6d83a6d50f0072a9eb171bae97f
-F test/shell8.test c836470ccde867e1f438a7acad7560805cc04f9dbab84cb55d92925942b76247
+F test/shell8.test 96be02ea0c21f05b24c1883d7b711a1fa8525a68ab7b636aacf6057876941013
 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3
 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5
 F test/shrink.test 1b4330b1fd9e818c04726d45cb28db73087535ce
@@ -1699,7 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P cf64087224aff1a2fe169d23996d9e5ed8d86459c655eb5d0bace0466a557ec6
-R 5e0c54dfbab7bd3d719e333800776169
-U dan
-Z a41cf66d60fe2e5069108d980e3c47d6
+P b0b7d0363acf38c2178e2d3041d8ce2a0de061a51caa64670dbf539ee6d4356b
+R 7e32bbd83b21ce648e53afd2544cab2d
+U drh
+Z 4c459e37e4b5a5b61d3c298abef31f05
diff --git a/manifest.uuid b/manifest.uuid
index 78a1e43434..82ff37d4da 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b0b7d0363acf38c2178e2d3041d8ce2a0de061a51caa64670dbf539ee6d4356b
\ No newline at end of file
+ce8bfe6c2b87090a2de1e04bc88fcb878597fe1f4ecd5df6d9d588a65601c901
\ No newline at end of file
diff --git a/test/shell8.test b/test/shell8.test
index de0f237f72..3658a8ac5d 100644
--- a/test/shell8.test
+++ b/test/shell8.test
@@ -21,6 +21,12 @@ ifcapable !vtab {
 }
 set CLI [test_find_cli]
 
+# Check to make sure the shell has been compiled with ".archive" support.
+#
+if {[string match {*unknown command*} [catchcmd :memory: .archive]]} {
+  finish_test; return
+}
+
 proc populate_dir {dirname spec} {
   # First delete the current tree, if one exists.
   file delete -force $dirname

From 0c2ba13e02348eff515dfc6f9028d562ade8403e Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 16 Jan 2018 13:37:43 +0000
Subject: [PATCH 329/488] Fix a problem causing an infinite loop or other
 malfunction in some UPDATE statements with an OR term in the WHERE clause.

FossilOrigin-Name: feb2c2b6f66b0f45490beb1642d99cdb89fa220e299a8c118929df557c814189
---
 manifest          | 16 ++++++++--------
 manifest.uuid     |  2 +-
 src/where.c       | 25 +++++++++++++++++++++----
 test/update2.test | 15 +++++++++++++++
 4 files changed, 45 insertions(+), 13 deletions(-)

diff --git a/manifest b/manifest
index fe09344538..31ecdf30e1 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Disable\sthe\s".archive"\scommand\stests\sin\sshell8.test\sif\sthe\sCLI\sis\scompiled\swithout\nZLIB\ssupport.
-D 2018-01-16T02:38:35.189
+C Fix\sa\sproblem\scausing\san\sinfinite\sloop\sor\sother\smalfunction\sin\ssome\sUPDATE\nstatements\swith\san\sOR\sterm\sin\sthe\sWHERE\sclause.
+D 2018-01-16T13:37:43.188
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -568,7 +568,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 5a3f464edd64596f601683ed321d12e6fd93c5fb9afdfb3653d6ffd0fee9c48f
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f
-F src/where.c 36b92103f726609cc3dbe07c619426bd6886bede455de56ccff54c8e567f5582
+F src/where.c caf0b6c9d31f22f0b2c91aba723858de52b5d665aaa89034099015aaf9bb8219
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
 F src/wherecode.c af1e79154aaa88cd802d6f2e5b945f67eaca7c958d1525fbf8ee19d5bd7b9020
 F src/whereexpr.c 427ea8e96ec24f2a7814c67b8024ad664a9c7656264c4566c34743cb23186e46
@@ -1490,7 +1490,7 @@ F test/unique2.test 3674e9f2a3f1fbbfd4772ac74b7a97090d0f77d2
 F test/unixexcl.test d936ba2b06794018e136418addd59a2354eeae97
 F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8
 F test/update.test 6c68446b8a0a33d522a7c72b320934596a2d7d32
-F test/update2.test fffc92e72ae568fe048588762e650cd8ccbd8c8b6e4fe9099231766bfe4b51de
+F test/update2.test 5e67667e1c54017d964e626db765cf8bedcf87483c184f4c575bdb8c1dd2313e
 F test/uri.test 3481026f00ade6dfe8adb7acb6e1e47b04369568
 F test/uri2.test 9d3ba7a53ee167572d53a298ee4a5d38ec4a8fb7
 F test/userauth01.test e740a2697a7b40d7c5003a7d7edaee16acd349a9
@@ -1699,7 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b0b7d0363acf38c2178e2d3041d8ce2a0de061a51caa64670dbf539ee6d4356b
-R 7e32bbd83b21ce648e53afd2544cab2d
-U drh
-Z 4c459e37e4b5a5b61d3c298abef31f05
+P ce8bfe6c2b87090a2de1e04bc88fcb878597fe1f4ecd5df6d9d588a65601c901
+R 1aa7995509c0f191863de5587ae93f47
+U dan
+Z 2e5fd108f2c4172f1f8c00e9ff432235
diff --git a/manifest.uuid b/manifest.uuid
index 82ff37d4da..239a9bce67 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ce8bfe6c2b87090a2de1e04bc88fcb878597fe1f4ecd5df6d9d588a65601c901
\ No newline at end of file
+feb2c2b6f66b0f45490beb1642d99cdb89fa220e299a8c118929df557c814189
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index ec53527943..3152c8e9aa 100644
--- a/src/where.c
+++ b/src/where.c
@@ -4801,15 +4801,32 @@ WhereInfo *sqlite3WhereBegin(
 
   /* If the caller is an UPDATE or DELETE statement that is requesting
   ** to use a one-pass algorithm, determine if this is appropriate.
+  **
+  ** A one-pass approach can be used if the caller has requested one
+  ** and either (a) the scan visits at most one row or (b) each
+  ** of the following are true:
+  **
+  **   * the caller has indicated that a one-pass approach can be used
+  **     with multiple rows (by setting WHERE_ONEPASS_MULTIROW), and
+  **   * the table is not a virtual table, and
+  **   * either the scan does not use the OR optimization or the caller
+  **     is a DELETE operation (WHERE_DUPLICATES_OK is only specified
+  **     for DELETE).
+  **
+  ** The last qualification is because an UPDATE statement uses
+  ** WhereInfo.aiCurOnePass[1] to determine whether or not it really can
+  ** use a one-pass approach, and this is not set accurately for scans
+  ** that use the OR optimization.
   */
   assert( (wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || pWInfo->nLevel==1 );
   if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 ){
     int wsFlags = pWInfo->a[0].pWLoop->wsFlags;
     int bOnerow = (wsFlags & WHERE_ONEROW)!=0;
-    if( bOnerow
-     || ((wctrlFlags & WHERE_ONEPASS_MULTIROW)!=0
-           && 0==(wsFlags & WHERE_VIRTUALTABLE))
-    ){
+    if( bOnerow || (
+        0!=(wctrlFlags & WHERE_ONEPASS_MULTIROW)
+     && 0==(wsFlags & WHERE_VIRTUALTABLE)
+     && (0==(wsFlags & WHERE_MULTI_OR) || (wctrlFlags & WHERE_DUPLICATES_OK))
+    )){
       pWInfo->eOnePass = bOnerow ? ONEPASS_SINGLE : ONEPASS_MULTI;
       if( HasRowid(pTabList->a[0].pTab) && (wsFlags & WHERE_IDX_ONLY) ){
         if( wctrlFlags & WHERE_ONEPASS_MULTIROW ){
diff --git a/test/update2.test b/test/update2.test
index af4ce96e7f..a6c3113400 100644
--- a/test/update2.test
+++ b/test/update2.test
@@ -200,5 +200,20 @@ do_test 5.2 {
   set A(NotExists)
 } {1}
 
+#-------------------------------------------------------------------------
+do_execsql_test 6.0 {
+  CREATE TABLE d1(a,b);
+  CREATE INDEX d1b ON d1(a);
+  CREATE INDEX d1c ON d1(b);
+  INSERT INTO d1 VALUES(1,2);
+}
+
+do_execsql_test 6.1 {
+  UPDATE d1 SET a = a+2 WHERE a>0 OR b>0;
+}
+
+do_execsql_test 6.2 {
+  SELECT * FROM d1;
+} {3 2}
 
 finish_test

From 194a17bbb7b58562792c3ed762f4eee6c683064f Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 16 Jan 2018 17:33:09 +0000
Subject: [PATCH 330/488] Change a cat in zipfile.c from (z_const Bytef*) to
 just (Bytef*). This allows the module to build with older versions of zlib.

FossilOrigin-Name: ac9af91d5a2927e71903461e7bbdd2c0168fde9a042853c8bcd4c7ebd0d51e0d
---
 ext/misc/zipfile.c |  2 +-
 manifest           | 12 ++++++------
 manifest.uuid      |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index f17b3735fb..5100bc4f79 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -771,7 +771,7 @@ static int zipfileDeflate(
     int res;
     z_stream str;
     memset(&str, 0, sizeof(str));
-    str.next_in = (z_const Bytef*)aIn;
+    str.next_in = (Bytef*)aIn;
     str.avail_in = nIn;
     str.next_out = aOut;
     str.avail_out = nAlloc;
diff --git a/manifest b/manifest
index 31ecdf30e1..255197fd5b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sproblem\scausing\san\sinfinite\sloop\sor\sother\smalfunction\sin\ssome\sUPDATE\nstatements\swith\san\sOR\sterm\sin\sthe\sWHERE\sclause.
-D 2018-01-16T13:37:43.188
+C Change\sa\scat\sin\szipfile.c\sfrom\s(z_const\sBytef*)\sto\sjust\s(Bytef*).\sThis\sallows\sthe\smodule\sto\sbuild\swith\solder\sversions\sof\szlib.
+D 2018-01-16T17:33:09.985
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -303,7 +303,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c 7e48d2947a1fe71d22e54f3b499d6d0581efa105033683585d3dbde76cffaed7
+F ext/misc/zipfile.c cc12e900e12eec2358c0dc2cb341bf192217aab0932873958c62a4ae7cbf9988
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -1699,7 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ce8bfe6c2b87090a2de1e04bc88fcb878597fe1f4ecd5df6d9d588a65601c901
-R 1aa7995509c0f191863de5587ae93f47
+P feb2c2b6f66b0f45490beb1642d99cdb89fa220e299a8c118929df557c814189
+R 9fb18308fe7bf47a56b5953076efa014
 U dan
-Z 2e5fd108f2c4172f1f8c00e9ff432235
+Z cc16d38e7e88ca94b914298c53e9f0f1
diff --git a/manifest.uuid b/manifest.uuid
index 239a9bce67..653213b754 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-feb2c2b6f66b0f45490beb1642d99cdb89fa220e299a8c118929df557c814189
\ No newline at end of file
+ac9af91d5a2927e71903461e7bbdd2c0168fde9a042853c8bcd4c7ebd0d51e0d
\ No newline at end of file

From fff5367c24875cf771bfcdc0165c387cfdfe8365 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Tue, 16 Jan 2018 19:03:51 +0000
Subject: [PATCH 331/488] Show version of zlib in use when running the shell
 tool in interactive mode.

FossilOrigin-Name: a8906b527a8f7a2cec88cc3b05c063a2785f0d1d141e148d954fc7440972a484
---
 manifest       | 14 +++++++-------
 manifest.uuid  |  2 +-
 src/shell.c.in |  3 +++
 3 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 255197fd5b..9860dc22cc 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Change\sa\scat\sin\szipfile.c\sfrom\s(z_const\sBytef*)\sto\sjust\s(Bytef*).\sThis\sallows\sthe\smodule\sto\sbuild\swith\solder\sversions\sof\szlib.
-D 2018-01-16T17:33:09.985
+C Show\sversion\sof\szlib\sin\suse\swhen\srunning\sthe\sshell\stool\sin\sinteractive\smode.
+D 2018-01-16T19:03:51.951
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -486,7 +486,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c bebe7cce45d899d2237c76bce059d525abf5b861f2fce92f6b53914a961c01ba
-F src/shell.c.in b87abffd0db09203ad8a133d56fe8f154ace5ec0a14197a153fb7d80b1438c01
+F src/shell.c.in 594f9427e519a77b2d6d18e223b370caf9e81e588f47fd5dc463e0910365cdfa
 F src/sqlite.h.in 9daf78e8f3cecc9ea0c3a82201f75bb74f789ecbfcda28d2e47fa80b3d956961
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 3737a51c5798e47a8ff2af0720f6dbc00fabd5ea401db392d05b1916085857a6
@@ -1699,7 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P feb2c2b6f66b0f45490beb1642d99cdb89fa220e299a8c118929df557c814189
-R 9fb18308fe7bf47a56b5953076efa014
-U dan
-Z cc16d38e7e88ca94b914298c53e9f0f1
+P ac9af91d5a2927e71903461e7bbdd2c0168fde9a042853c8bcd4c7ebd0d51e0d
+R 039d269ff5ea61ab17b17f8244c27f58
+U mistachkin
+Z 588ee2b95c490fe538f28f786da31468
diff --git a/manifest.uuid b/manifest.uuid
index 653213b754..6dcfedfad6 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ac9af91d5a2927e71903461e7bbdd2c0168fde9a042853c8bcd4c7ebd0d51e0d
\ No newline at end of file
+a8906b527a8f7a2cec88cc3b05c063a2785f0d1d141e148d954fc7440972a484
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index ee21a83d6f..ce5294de81 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -8380,6 +8380,9 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
         "Enter \".help\" for usage hints.\n",
         sqlite3_libversion(), sqlite3_sourceid()
       );
+#if SQLITE_HAVE_ZLIB
+      printf("Using zlib version %s.\n", zlibVersion());
+#endif
       if( warnInmemoryDb ){
         printf("Connected to a ");
         printBold("transient in-memory database");

From 0ed2fd8a254afeb2b6112ddba474cb21efa32eb0 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 16 Jan 2018 20:05:27 +0000
Subject: [PATCH 332/488] Move the ZLIB version announcement in the CLI out of
 the default banner and put it in the ".version" command.

FossilOrigin-Name: 231679d6f8c0c3d957c8daa88b0743449b60b47159e114f4bc073a8612a73b2a
---
 manifest       | 14 +++++++-------
 manifest.uuid  |  2 +-
 src/shell.c.in | 17 ++++++++++++++---
 3 files changed, 22 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index 9860dc22cc..a5f191ee14 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Show\sversion\sof\szlib\sin\suse\swhen\srunning\sthe\sshell\stool\sin\sinteractive\smode.
-D 2018-01-16T19:03:51.951
+C Move\sthe\sZLIB\sversion\sannouncement\sin\sthe\sCLI\sout\sof\sthe\sdefault\sbanner\nand\sput\sit\sin\sthe\s".version"\scommand.
+D 2018-01-16T20:05:27.199
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -486,7 +486,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c bebe7cce45d899d2237c76bce059d525abf5b861f2fce92f6b53914a961c01ba
-F src/shell.c.in 594f9427e519a77b2d6d18e223b370caf9e81e588f47fd5dc463e0910365cdfa
+F src/shell.c.in 568aacb5b15520fa7981ebcf976ebd99849e6a4777858706762f201a446626f3
 F src/sqlite.h.in 9daf78e8f3cecc9ea0c3a82201f75bb74f789ecbfcda28d2e47fa80b3d956961
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 3737a51c5798e47a8ff2af0720f6dbc00fabd5ea401db392d05b1916085857a6
@@ -1699,7 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ac9af91d5a2927e71903461e7bbdd2c0168fde9a042853c8bcd4c7ebd0d51e0d
-R 039d269ff5ea61ab17b17f8244c27f58
-U mistachkin
-Z 588ee2b95c490fe538f28f786da31468
+P a8906b527a8f7a2cec88cc3b05c063a2785f0d1d141e148d954fc7440972a484
+R 98c56f352f44e579ba7b90a6b8abb9cd
+U drh
+Z e359cfb1d91fd4e09231f1eb354cb112
diff --git a/manifest.uuid b/manifest.uuid
index 6dcfedfad6..1d606c74b9 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a8906b527a8f7a2cec88cc3b05c063a2785f0d1d141e148d954fc7440972a484
\ No newline at end of file
+231679d6f8c0c3d957c8daa88b0743449b60b47159e114f4bc073a8612a73b2a
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index ce5294de81..523d7ef7bf 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -7537,6 +7537,20 @@ static int do_meta_command(char *zLine, ShellState *p){
   if( c=='v' && strncmp(azArg[0], "version", n)==0 ){
     utf8_printf(p->out, "SQLite %s %s\n" /*extra-version-info*/,
         sqlite3_libversion(), sqlite3_sourceid());
+#if SQLITE_HAVE_ZLIB
+    utf8_printf(p->out, "zlib version %s\n", zlibVersion());
+#endif
+#define CTIMEOPT_VAL_(opt) #opt
+#define CTIMEOPT_VAL(opt) CTIMEOPT_VAL_(opt)
+#if defined(__clang__) && defined(__clang_major__)
+    utf8_printf(p->out, "clang-" CTIMEOPT_VAL(__clang_major__) "."
+                    CTIMEOPT_VAL(__clang_minor__) "."
+                    CTIMEOPT_VAL(__clang_patchlevel__) "\n");
+#elif defined(_MSC_VER)
+    utf8_printf(p->out, "msvc-" CTIMEOPT_VAL(_MSC_VER) "\n");
+#elif defined(__GNUC__) && defined(__VERSION__)
+    utf8_printf(p->out, "gcc-" __VERSION__ "\n");
+#endif
   }else
 
   if( c=='v' && strncmp(azArg[0], "vfsinfo", n)==0 ){
@@ -8380,9 +8394,6 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
         "Enter \".help\" for usage hints.\n",
         sqlite3_libversion(), sqlite3_sourceid()
       );
-#if SQLITE_HAVE_ZLIB
-      printf("Using zlib version %s.\n", zlibVersion());
-#endif
       if( warnInmemoryDb ){
         printf("Connected to a ");
         printBold("transient in-memory database");

From 70bc717c0eee621b753f8859c2cd24a1ce20953c Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 16 Jan 2018 20:20:00 +0000
Subject: [PATCH 333/488] Fix a makefile problem causing
 -DSQLITE_ENABLE_STMTVTAB builds to fail.

FossilOrigin-Name: 7f6e5bdf9021c31e8bde82c224bf53be3c93b79f7bb5c2802c54be8031cf89fb
---
 main.mk       |  3 ++-
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 3 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/main.mk b/main.mk
index 281de5d335..504ff34e1c 100644
--- a/main.mk
+++ b/main.mk
@@ -366,6 +366,7 @@ TESTSRC += \
   $(TOP)/ext/misc/remember.c \
   $(TOP)/ext/misc/series.c \
   $(TOP)/ext/misc/spellfix.c \
+  $(TOP)/ext/misc/stmt.c \
   $(TOP)/ext/misc/totype.c \
   $(TOP)/ext/misc/unionvtab.c \
   $(TOP)/ext/misc/wholenumber.c \
@@ -373,7 +374,7 @@ TESTSRC += \
   $(TOP)/ext/misc/zipfile.c \
   $(TOP)/ext/fts5/fts5_tcl.c \
   $(TOP)/ext/fts5/fts5_test_mi.c \
-  $(TOP)/ext/fts5/fts5_test_tok.c 
+  $(TOP)/ext/fts5/fts5_test_tok.c
 
 
 #TESTSRC += $(TOP)/ext/fts2/fts2_tokenizer.c
diff --git a/manifest b/manifest
index a5f191ee14..bc0ce0d4a3 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Move\sthe\sZLIB\sversion\sannouncement\sin\sthe\sCLI\sout\sof\sthe\sdefault\sbanner\nand\sput\sit\sin\sthe\s".version"\scommand.
-D 2018-01-16T20:05:27.199
+C Fix\sa\smakefile\sproblem\scausing\s-DSQLITE_ENABLE_STMTVTAB\sbuilds\sto\sfail.
+D 2018-01-16T20:20:00.594
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -408,7 +408,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk fc0edb268998a049ce70ee3dc056b2a96cc8aa4ef0c6da296700d7081d167627
+F main.mk 69bce07f975e6835a544f43df8d9619ea31d2c8e7b3a8d2cb4c21ac704c9c9c2
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -1699,7 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a8906b527a8f7a2cec88cc3b05c063a2785f0d1d141e148d954fc7440972a484
-R 98c56f352f44e579ba7b90a6b8abb9cd
-U drh
-Z e359cfb1d91fd4e09231f1eb354cb112
+P 231679d6f8c0c3d957c8daa88b0743449b60b47159e114f4bc073a8612a73b2a
+R 6b159b88b50b86cd5639c2b511315c42
+U dan
+Z 8749febdaf33c2d94ed547e3c6b0b72c
diff --git a/manifest.uuid b/manifest.uuid
index 1d606c74b9..53e6471160 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-231679d6f8c0c3d957c8daa88b0743449b60b47159e114f4bc073a8612a73b2a
\ No newline at end of file
+7f6e5bdf9021c31e8bde82c224bf53be3c93b79f7bb5c2802c54be8031cf89fb
\ No newline at end of file

From eb0b6f364e1fc007fde8ebe40a447e098f49df72 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 16 Jan 2018 20:37:56 +0000
Subject: [PATCH 334/488] Remove the unused SQLITE_CANTOPEN_DIRTYWAL result
 code.

FossilOrigin-Name: 3c786305fc6eaa6856ec6c71fc7969f61de266ba1bea0abf0ae133eb94e54b2d
---
 manifest        | 14 +++++++-------
 manifest.uuid   |  2 +-
 src/sqlite.h.in |  1 -
 3 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index bc0ce0d4a3..8e532533b7 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\smakefile\sproblem\scausing\s-DSQLITE_ENABLE_STMTVTAB\sbuilds\sto\sfail.
-D 2018-01-16T20:20:00.594
+C Remove\sthe\sunused\sSQLITE_CANTOPEN_DIRTYWAL\sresult\scode.
+D 2018-01-16T20:37:56.341
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -487,7 +487,7 @@ F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c bebe7cce45d899d2237c76bce059d525abf5b861f2fce92f6b53914a961c01ba
 F src/shell.c.in 568aacb5b15520fa7981ebcf976ebd99849e6a4777858706762f201a446626f3
-F src/sqlite.h.in 9daf78e8f3cecc9ea0c3a82201f75bb74f789ecbfcda28d2e47fa80b3d956961
+F src/sqlite.h.in 840e8aee0d7f8592bd69730797582b342a413eceaff7a220fb5f9db9c412cea9
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 3737a51c5798e47a8ff2af0720f6dbc00fabd5ea401db392d05b1916085857a6
 F src/sqliteInt.h 9c70315598b34810a83e4894455acb18e95cf63ce4e6cbb451ac2d17eabc2544
@@ -1699,7 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 231679d6f8c0c3d957c8daa88b0743449b60b47159e114f4bc073a8612a73b2a
-R 6b159b88b50b86cd5639c2b511315c42
-U dan
-Z 8749febdaf33c2d94ed547e3c6b0b72c
+P 7f6e5bdf9021c31e8bde82c224bf53be3c93b79f7bb5c2802c54be8031cf89fb
+R 0d3241ffa7f896b9008a93ec004c8dbb
+U drh
+Z f5ef3a49cfd0d1f0bb95f024d28d366c
diff --git a/manifest.uuid b/manifest.uuid
index 53e6471160..3971e7693e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7f6e5bdf9021c31e8bde82c224bf53be3c93b79f7bb5c2802c54be8031cf89fb
\ No newline at end of file
+3c786305fc6eaa6856ec6c71fc7969f61de266ba1bea0abf0ae133eb94e54b2d
\ No newline at end of file
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 2661c8f35f..1f7f6a0792 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -510,7 +510,6 @@ int sqlite3_exec(
 #define SQLITE_CANTOPEN_ISDIR          (SQLITE_CANTOPEN | (2<<8))
 #define SQLITE_CANTOPEN_FULLPATH       (SQLITE_CANTOPEN | (3<<8))
 #define SQLITE_CANTOPEN_CONVPATH       (SQLITE_CANTOPEN | (4<<8))
-#define SQLITE_CANTOPEN_DIRTYWAL       (SQLITE_CANTOPEN | (5<<8))
 #define SQLITE_CORRUPT_VTAB            (SQLITE_CORRUPT | (1<<8))
 #define SQLITE_READONLY_RECOVERY       (SQLITE_READONLY | (1<<8))
 #define SQLITE_READONLY_CANTLOCK       (SQLITE_READONLY | (2<<8))

From 4bd387495fac0c8cf3474eac1b5aa388fa34761c Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 16 Jan 2018 20:44:00 +0000
Subject: [PATCH 335/488] Fix a test problem causing an error in
 fts5fault9.test.

FossilOrigin-Name: 8e2048113fc6ed87fc7d5ba470261926aa0bd35474744bb2b99ea596bc468f02
---
 ext/fts5/fts5_tcl.c           |  4 ++--
 ext/fts5/test/fts5fault9.test |  8 ++++++--
 manifest                      | 16 ++++++++--------
 manifest.uuid                 |  2 +-
 4 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c
index 8f79397fbf..c30ed4ae0d 100644
--- a/ext/fts5/fts5_tcl.c
+++ b/ext/fts5/fts5_tcl.c
@@ -482,7 +482,7 @@ static int SQLITE_TCLAPI xF5tApi(
 
       rc = p->pApi->xPhraseFirstColumn(p->pFts, iPhrase, &iter, &iCol);
       if( rc!=SQLITE_OK ){
-        Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+        Tcl_SetResult(interp, sqlite3ErrName(rc), TCL_VOLATILE);
         return TCL_ERROR;
       }
       for( ; iCol>=0; p->pApi->xPhraseNextColumn(p->pFts, &iter, &iCol)){
@@ -924,7 +924,7 @@ static int SQLITE_TCLAPI f5tTokenizerReturn(
 
   rc = p->xToken(p->pCtx, tflags, zToken, nToken, iStart, iEnd);
   Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
-  return TCL_OK;
+  return rc==SQLITE_OK ? TCL_OK : TCL_ERROR;
 
  usage:
   Tcl_WrongNumArgs(interp, 1, objv, "?-colocated? TEXT START END");
diff --git a/ext/fts5/test/fts5fault9.test b/ext/fts5/test/fts5fault9.test
index 1daa5c1cc9..669b13efe7 100644
--- a/ext/fts5/test/fts5fault9.test
+++ b/ext/fts5/test/fts5fault9.test
@@ -24,6 +24,8 @@ ifcapable !fts5 {
 
 foreach_detail_mode $testprefix {
 
+if {"%DETAIL%" != "none"} continue
+
 fts5_aux_test_functions db
 
 do_execsql_test 1.0 {
@@ -98,14 +100,16 @@ do_faultsim_test 4.1 -faults oom-t* -body {
   execsql { SELECT rowid, fts5_test_collist(t4) FROM t4('2') }
 } -test {
   faultsim_test_result \
-      {0 {1 {0.0 0.1 0.2} 2 {0.0 0.1 0.2} 3 {0.0 0.1 0.2}}} {1 SQLITE_NOMEM}
+      {0 {1 {0.0 0.1 0.2} 2 {0.0 0.1 0.2} 3 {0.0 0.1 0.2}}} \
+      {1 SQLITE_NOMEM} {1 SQLITE_ERROR} {1 {SQL logic error}}
 }
 
 do_faultsim_test 4.2 -faults oom-t* -body {
   execsql { SELECT rowid, fts5_test_collist(t4) FROM t4('a5 OR b5 OR c5') }
 } -test {
   faultsim_test_result \
-      {0 {4 {0.0 0.1 0.2} 5 {1.0 1.1 1.2} 6 {2.0 2.1 2.2}}} {1 SQLITE_NOMEM}
+      {0 {4 {0.0 0.1 0.2} 5 {1.0 1.1 1.2} 6 {2.0 2.1 2.2}}} \
+      {1 SQLITE_NOMEM} {1 SQLITE_ERROR} {1 {SQL logic error}}
 }
 
 
diff --git a/manifest b/manifest
index 8e532533b7..84a37b3671 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\sthe\sunused\sSQLITE_CANTOPEN_DIRTYWAL\sresult\scode.
-D 2018-01-16T20:37:56.341
+C Fix\sa\stest\sproblem\scausing\san\serror\sin\sfts5fault9.test.
+D 2018-01-16T20:44:00.735
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -116,7 +116,7 @@ F ext/fts5/fts5_hash.c 32be400cf761868c9db33efe81a06eb19a17c5402ad477ee9efb51301
 F ext/fts5/fts5_index.c 5fe14375a29e8a7aa8f3e863babe180a19269206c254c8f47b216821d4ac1e15
 F ext/fts5/fts5_main.c 24868f88ab2a865defbba7a92eebeb726cc991eb092b71b5f5508f180c72605b
 F ext/fts5/fts5_storage.c fb5ef3c27073f67ade2e1bea08405f9e43f68f5f3676ed0ab7013bce5ba10be6
-F ext/fts5/fts5_tcl.c b470467be4c5cab2d8b026992c05d86cd2293e7d8c4a10ba56d5f4f707981097
+F ext/fts5/fts5_tcl.c a021468dfa204a4a48717d7709503ab062a010e8b79ea73e2a23ba5a0a9a6ec6
 F ext/fts5/fts5_test_mi.c 65864ba1e5c34a61d409c4c587e0bbe0466eb4f8f478d85dc42a92caad1338e6
 F ext/fts5/fts5_test_tok.c ffd657dd67e7fcdb31bf63fb60b6d867299a581d0f46e97086abacd66c2a9b26
 F ext/fts5/fts5_tokenize.c 2ce7b44183538ec46b7907726262ee43ffdd39a8
@@ -168,7 +168,7 @@ F ext/fts5/test/fts5fault5.test a336e4e11847de24c9497f80cce18e00bb3fab7fb11f97d0
 F ext/fts5/test/fts5fault6.test 8a3c61402e36960ba46a419e73121fcefdc9160e0c04b6f5318c7fb0e3180dbc
 F ext/fts5/test/fts5fault7.test 0acbec416edb24b8881f154e99c31e9ccf73f539cfcd164090be139e9e97ed4c
 F ext/fts5/test/fts5fault8.test 318238659d35f82ad215ecb57ca4c87486ea85d45dbeedaee42f148ff5105ee2
-F ext/fts5/test/fts5fault9.test 0111b229388bdf251b91cfead68580227801dd30960a19aa8fe9021a1e73cb6d
+F ext/fts5/test/fts5fault9.test 098e6b894bbdf9b2192f994a30f4043673fb3f338b6b8ab1624c704422f39119
 F ext/fts5/test/fts5faultA.test be4487576bff8c22cee6597d1893b312f306504a8c6ccd3c53ca85af12290c8c
 F ext/fts5/test/fts5faultB.test e6d04f9ea7b21be1d89abb8df2cb4baf65b0453b744d5a805fcd3ef45ff86a7e
 F ext/fts5/test/fts5faultD.test cc5d1225556e356615e719c612e845d41bff7d5a
@@ -1699,7 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 7f6e5bdf9021c31e8bde82c224bf53be3c93b79f7bb5c2802c54be8031cf89fb
-R 0d3241ffa7f896b9008a93ec004c8dbb
-U drh
-Z f5ef3a49cfd0d1f0bb95f024d28d366c
+P 3c786305fc6eaa6856ec6c71fc7969f61de266ba1bea0abf0ae133eb94e54b2d
+R 0c8190aebc59833d33e61d80979c7e84
+U dan
+Z 09329f4d67456c2fa346b8eec49ecdd6
diff --git a/manifest.uuid b/manifest.uuid
index 3971e7693e..0e21792b69 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-3c786305fc6eaa6856ec6c71fc7969f61de266ba1bea0abf0ae133eb94e54b2d
\ No newline at end of file
+8e2048113fc6ed87fc7d5ba470261926aa0bd35474744bb2b99ea596bc468f02
\ No newline at end of file

From 4a4532bbb7515e2e8cacb5acdbc2b126e24b93d5 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 16 Jan 2018 20:50:37 +0000
Subject: [PATCH 336/488] Make the new sqlite3_vtab_collation() interface
 accessible to loadable extensions.

FossilOrigin-Name: f301db3c2343fd2086bc5b69a17b2d226175584ccd09549ebb2cb603590da487
---
 manifest         | 16 ++++++++--------
 manifest.uuid    |  2 +-
 src/loadext.c    |  3 ++-
 src/sqlite3ext.h |  2 ++
 4 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index 84a37b3671..16dea517f1 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\stest\sproblem\scausing\san\serror\sin\sfts5fault9.test.
-D 2018-01-16T20:44:00.735
+C Make\sthe\snew\ssqlite3_vtab_collation()\sinterface\saccessible\sto\sloadable\nextensions.
+D 2018-01-16T20:50:37.717
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -449,7 +449,7 @@ F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 14686083cedc198540b15a79586cdd4be2acf6d5fa97627e355f817ab07e9fee
 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e
-F src/loadext.c 8d5d9c3ab3c4d600a161f389988bdee6a04dad7c4c1754f04f257734b9d1ce8c
+F src/loadext.c f6e4e416a736369f9e80eba609f0acda97148a8b0453784d670c78d3eed2f302
 F src/main.c 26918d50dd4a61b8f6f210320a522f46b5e7e592335b6aa664ab15b80b7c239b
 F src/malloc.c 07295435093ce354c6d9063ac05a2eeae28bd251d2e63c48b3d67c12c76f7e18
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
@@ -489,7 +489,7 @@ F src/select.c bebe7cce45d899d2237c76bce059d525abf5b861f2fce92f6b53914a961c01ba
 F src/shell.c.in 568aacb5b15520fa7981ebcf976ebd99849e6a4777858706762f201a446626f3
 F src/sqlite.h.in 840e8aee0d7f8592bd69730797582b342a413eceaff7a220fb5f9db9c412cea9
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
-F src/sqlite3ext.h 3737a51c5798e47a8ff2af0720f6dbc00fabd5ea401db392d05b1916085857a6
+F src/sqlite3ext.h 99189e7611eb0bf98f21c7835dc74730a84e2e809c98e1e31c33896dee7a2849
 F src/sqliteInt.h 9c70315598b34810a83e4894455acb18e95cf63ce4e6cbb451ac2d17eabc2544
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
@@ -1699,7 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3c786305fc6eaa6856ec6c71fc7969f61de266ba1bea0abf0ae133eb94e54b2d
-R 0c8190aebc59833d33e61d80979c7e84
-U dan
-Z 09329f4d67456c2fa346b8eec49ecdd6
+P 8e2048113fc6ed87fc7d5ba470261926aa0bd35474744bb2b99ea596bc468f02
+R 267bc65de959febfa028d4a40cdbf39c
+U drh
+Z 9b16902ddc51de02e55cd9f83731d91e
diff --git a/manifest.uuid b/manifest.uuid
index 0e21792b69..d7ea59dec0 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-8e2048113fc6ed87fc7d5ba470261926aa0bd35474744bb2b99ea596bc468f02
\ No newline at end of file
+f301db3c2343fd2086bc5b69a17b2d226175584ccd09549ebb2cb603590da487
\ No newline at end of file
diff --git a/src/loadext.c b/src/loadext.c
index 80d40a4429..cf58237bf9 100644
--- a/src/loadext.c
+++ b/src/loadext.c
@@ -433,7 +433,8 @@ static const sqlite3_api_routines sqlite3Apis = {
   sqlite3_value_pointer,
   /* Version 3.22.0 and later */
   sqlite3_vtab_nochange,
-  sqlite3_value_nochange
+  sqlite3_value_nochange,
+  sqlite3_vtab_collation
 };
 
 /*
diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h
index 4252063909..ac92a74901 100644
--- a/src/sqlite3ext.h
+++ b/src/sqlite3ext.h
@@ -294,6 +294,7 @@ struct sqlite3_api_routines {
   void *(*value_pointer)(sqlite3_value*,const char*);
   int (*vtab_nochange)(sqlite3_context*);
   int (*value_nochange)(sqlite3_value*);
+  const char *(*vtab_collation)(sqlite3_index_info*,int);
 };
 
 /*
@@ -563,6 +564,7 @@ typedef int (*sqlite3_loadext_entry)(
 /* Version 3.22.0 and later */
 #define sqlite3_vtab_nochange          sqlite3_api->vtab_nochange
 #define sqlite3_value_nochange         sqltie3_api->value_nochange
+#define sqlite3_vtab_collation         sqltie3_api->vtab_collation
 #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */
 
 #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION)

From 2d9e8c55dab0e7a38a179a65e7c87e8ee94b59d6 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 16 Jan 2018 21:00:58 +0000
Subject: [PATCH 337/488] Fix compiler warning in the FTS5 test interface.

FossilOrigin-Name: 2ab4e8d5d82ab410c00f048c12555f2f40f9fd8bffda64ccbdd3dfe036727315
---
 ext/fts5/fts5_tcl.c |  2 +-
 manifest            | 12 ++++++------
 manifest.uuid       |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c
index c30ed4ae0d..f33dcae033 100644
--- a/ext/fts5/fts5_tcl.c
+++ b/ext/fts5/fts5_tcl.c
@@ -482,7 +482,7 @@ static int SQLITE_TCLAPI xF5tApi(
 
       rc = p->pApi->xPhraseFirstColumn(p->pFts, iPhrase, &iter, &iCol);
       if( rc!=SQLITE_OK ){
-        Tcl_SetResult(interp, sqlite3ErrName(rc), TCL_VOLATILE);
+        Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
         return TCL_ERROR;
       }
       for( ; iCol>=0; p->pApi->xPhraseNextColumn(p->pFts, &iter, &iCol)){
diff --git a/manifest b/manifest
index 16dea517f1..c2ef7d0ba3 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Make\sthe\snew\ssqlite3_vtab_collation()\sinterface\saccessible\sto\sloadable\nextensions.
-D 2018-01-16T20:50:37.717
+C Fix\scompiler\swarning\sin\sthe\sFTS5\stest\sinterface.
+D 2018-01-16T21:00:58.725
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -116,7 +116,7 @@ F ext/fts5/fts5_hash.c 32be400cf761868c9db33efe81a06eb19a17c5402ad477ee9efb51301
 F ext/fts5/fts5_index.c 5fe14375a29e8a7aa8f3e863babe180a19269206c254c8f47b216821d4ac1e15
 F ext/fts5/fts5_main.c 24868f88ab2a865defbba7a92eebeb726cc991eb092b71b5f5508f180c72605b
 F ext/fts5/fts5_storage.c fb5ef3c27073f67ade2e1bea08405f9e43f68f5f3676ed0ab7013bce5ba10be6
-F ext/fts5/fts5_tcl.c a021468dfa204a4a48717d7709503ab062a010e8b79ea73e2a23ba5a0a9a6ec6
+F ext/fts5/fts5_tcl.c 39bcbae507f594aad778172fa914cad0f585bf92fd3b078c686e249282db0d95
 F ext/fts5/fts5_test_mi.c 65864ba1e5c34a61d409c4c587e0bbe0466eb4f8f478d85dc42a92caad1338e6
 F ext/fts5/fts5_test_tok.c ffd657dd67e7fcdb31bf63fb60b6d867299a581d0f46e97086abacd66c2a9b26
 F ext/fts5/fts5_tokenize.c 2ce7b44183538ec46b7907726262ee43ffdd39a8
@@ -1699,7 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 8e2048113fc6ed87fc7d5ba470261926aa0bd35474744bb2b99ea596bc468f02
-R 267bc65de959febfa028d4a40cdbf39c
+P f301db3c2343fd2086bc5b69a17b2d226175584ccd09549ebb2cb603590da487
+R 0b7a4854cd40ff8848724ee789fa5f0a
 U drh
-Z 9b16902ddc51de02e55cd9f83731d91e
+Z a8abeaf4b85a720e69d65b8f52daa4a1
diff --git a/manifest.uuid b/manifest.uuid
index d7ea59dec0..d471974b0f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f301db3c2343fd2086bc5b69a17b2d226175584ccd09549ebb2cb603590da487
\ No newline at end of file
+2ab4e8d5d82ab410c00f048c12555f2f40f9fd8bffda64ccbdd3dfe036727315
\ No newline at end of file

From 8c9e30ba2f1ae1b70129a4a34e3659f177d7d816 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 16 Jan 2018 21:05:10 +0000
Subject: [PATCH 338/488] Remove an unused field from the internal definition
 of the sqlite3_context object.

FossilOrigin-Name: 948a26b5a64d8a89c4ec7546ec4a4bee36e29e82edda677ca75424e7091325d0
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/vdbeInt.h |  1 -
 3 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index c2ef7d0ba3..0b65bd2902 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\scompiler\swarning\sin\sthe\sFTS5\stest\sinterface.
-D 2018-01-16T21:00:58.725
+C Remove\san\sunused\sfield\sfrom\sthe\sinternal\sdefinition\sof\sthe\ssqlite3_context\nobject.
+D 2018-01-16T21:05:10.561
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -556,7 +556,7 @@ F src/util.c 7315e97a8dc2c8e19ca64196c652cf0a65d13fd0a211b2cec082062372dc6261
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
 F src/vdbe.c ccc1e17a30325068ae4f0292e8601997946886d23acc989c68f2a261a2795c70
 F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a
-F src/vdbeInt.h 5442fc816b6cf19c8801724199fd6b77a02eb31a7a174021713f8c59b30e51fa
+F src/vdbeInt.h c8cfbbc28e37e67a493c3f892fb0596add56a31a00e7537a06049af9ef2f51b0
 F src/vdbeapi.c 02f773681d06e46454b0606339068d4d4490873dc4a7334bc0c6030552bb2c8c
 F src/vdbeaux.c 2756ac68ac259c416554100598fc291870063288cd7e1af22847f57b3e130e56
 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191
@@ -1699,7 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P f301db3c2343fd2086bc5b69a17b2d226175584ccd09549ebb2cb603590da487
-R 0b7a4854cd40ff8848724ee789fa5f0a
+P 2ab4e8d5d82ab410c00f048c12555f2f40f9fd8bffda64ccbdd3dfe036727315
+R e4f4080f808433f06cbdcdcd7e3a3a16
 U drh
-Z a8abeaf4b85a720e69d65b8f52daa4a1
+Z f5b1113de44f2b59daf15f747c8d38d4
diff --git a/manifest.uuid b/manifest.uuid
index d471974b0f..63872b4ad3 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2ab4e8d5d82ab410c00f048c12555f2f40f9fd8bffda64ccbdd3dfe036727315
\ No newline at end of file
+948a26b5a64d8a89c4ec7546ec4a4bee36e29e82edda677ca75424e7091325d0
\ No newline at end of file
diff --git a/src/vdbeInt.h b/src/vdbeInt.h
index b7e324c94b..976abb3938 100644
--- a/src/vdbeInt.h
+++ b/src/vdbeInt.h
@@ -320,7 +320,6 @@ struct sqlite3_context {
   int isError;            /* Error code returned by the function. */
   u8 skipFlag;            /* Skip accumulator loading if true */
   u8 fErrorOrAux;         /* isError!=0 or pVdbe->pAuxData modified */
-  u8 bVtabNoChng;         /* Fetching an unchanging column in a vtab UPDATE */
   u8 argc;                /* Number of arguments */
   sqlite3_value *argv[1]; /* Argument set */
 };

From 2879952faac68d9350f7b149fdbfa80a3bee5b9b Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 16 Jan 2018 21:09:00 +0000
Subject: [PATCH 339/488] Fix a problem causing zipfile to store 0 in place of
 the CRC32 value for uncompressed files.

FossilOrigin-Name: ba44724bcca2e87788b7c6d8c5de7fa388360127bd894ee6a171fd66e794fcae
---
 ext/misc/zipfile.c | 18 ++++++++++--------
 manifest           | 16 ++++++++--------
 manifest.uuid      |  2 +-
 test/zipfile.test  |  9 +++++++++
 4 files changed, 28 insertions(+), 17 deletions(-)

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index 5100bc4f79..fa31d62d11 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -1337,14 +1337,16 @@ static int zipfileUpdate(
       nData = nIn;
       if( iMethod!=0 && iMethod!=8 ){
         rc = SQLITE_CONSTRAINT;
-      }else if( bAuto || iMethod ){
-        int nCmp;
-        rc = zipfileDeflate(pTab, aIn, nIn, &pFree, &nCmp);
-        if( rc==SQLITE_OK ){
-          if( iMethod || nCmp<nIn ){
-            iMethod = 8;
-            pData = pFree;
-            nData = nCmp;
+      }else{
+        if( bAuto || iMethod ){
+          int nCmp;
+          rc = zipfileDeflate(pTab, aIn, nIn, &pFree, &nCmp);
+          if( rc==SQLITE_OK ){
+            if( iMethod || nCmp<nIn ){
+              iMethod = 8;
+              pData = pFree;
+              nData = nCmp;
+            }
           }
         }
         iCrc32 = crc32(0, aIn, nIn);
diff --git a/manifest b/manifest
index 0b65bd2902..8d65b34b6c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\san\sunused\sfield\sfrom\sthe\sinternal\sdefinition\sof\sthe\ssqlite3_context\nobject.
-D 2018-01-16T21:05:10.561
+C Fix\sa\sproblem\scausing\szipfile\sto\sstore\s0\sin\splace\sof\sthe\sCRC32\svalue\sfor\nuncompressed\sfiles.
+D 2018-01-16T21:09:00.543
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -303,7 +303,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c cc12e900e12eec2358c0dc2cb341bf192217aab0932873958c62a4ae7cbf9988
+F ext/misc/zipfile.c d99efb67ecdfcae7e1855984c218c8c33d0d46a833eaa4b5a5c3d7a4f6690ce4
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -1600,7 +1600,7 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf
 F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc
 F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa
 F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc
-F test/zipfile.test 71a9d37bb928a1dcee6ab624e8be7bca9f807364440d20fc84363c44ab2f4ac5
+F test/zipfile.test 17e3ed850c7f182a5ce746fadc1a0497495aeacba216d73ece7b3b2c5dfd8867
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d
 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91
@@ -1699,7 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2ab4e8d5d82ab410c00f048c12555f2f40f9fd8bffda64ccbdd3dfe036727315
-R e4f4080f808433f06cbdcdcd7e3a3a16
-U drh
-Z f5b1113de44f2b59daf15f747c8d38d4
+P 948a26b5a64d8a89c4ec7546ec4a4bee36e29e82edda677ca75424e7091325d0
+R fa20b259f24a93ee214da4f46c94ddf6
+U dan
+Z c12f29c5e46aaf9ede6fab6c939cd772
diff --git a/manifest.uuid b/manifest.uuid
index 63872b4ad3..95edffa509 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-948a26b5a64d8a89c4ec7546ec4a4bee36e29e82edda677ca75424e7091325d0
\ No newline at end of file
+ba44724bcca2e87788b7c6d8c5de7fa388360127bd894ee6a171fd66e794fcae
\ No newline at end of file
diff --git a/test/zipfile.test b/test/zipfile.test
index e246c9ab18..c324e0d7c7 100644
--- a/test/zipfile.test
+++ b/test/zipfile.test
@@ -75,6 +75,15 @@ do_execsql_test 1.4 {
   h.txt 1000000004 aaaaaaaaaabbbbbbbbbb 8
 }
 
+do_execsql_test 1.4.1 {
+  SELECT name, json_extract( zipfile_cds(z) , '$.crc32')!=0
+  FROM zipfile('test.zip');
+} {
+  f.txt 1
+  g.txt 1
+  h.txt 1
+}
+
 do_execsql_test 1.5.1 {
   BEGIN;
     INSERT INTO zz(name, mode, mtime, data, method)

From 824f28426acc9f7f34365bd384336bb45d5e1c7f Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Wed, 17 Jan 2018 01:15:08 +0000
Subject: [PATCH 340/488] Fix duplicate test names in 'walfault.test'.

FossilOrigin-Name: 7274d05ff43fc9872f0e4857ae583689e4a6c429b7fa991dcc29744da3048879
---
 manifest           | 14 +++++++-------
 manifest.uuid      |  2 +-
 test/walfault.test |  4 ++--
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index 8d65b34b6c..c0e128cc24 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sproblem\scausing\szipfile\sto\sstore\s0\sin\splace\sof\sthe\sCRC32\svalue\sfor\nuncompressed\sfiles.
-D 2018-01-16T21:09:00.543
+C Fix\sduplicate\stest\snames\sin\s'walfault.test'.
+D 2018-01-17T01:15:08.464
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -1544,7 +1544,7 @@ F test/walcrash.test 21038858cc552077b0522f50b0fa87e38139306a
 F test/walcrash2.test a0edab4e5390f03b99a790de89aad15d6ec70b36
 F test/walcrash3.test e426aa58122d20f2b9fbe9a507f9eb8cab85b8af
 F test/walcrash4.test e7b6e7639a950a0cca8e210e248c8dad4d63bf20
-F test/walfault.test 1f8389f7709877e9b4cc679033d71d6fe529056b
+F test/walfault.test 09b8ad7e52d2f54bce50e31aa7ea51412bb9f70ac13c74e669ddcd8b48b0d98d
 F test/walhook.test ed00a40ba7255da22d6b66433ab61fab16a63483
 F test/walmode.test cd6e7cff618eaaa5910ce57c3657aa50110397f86213886a2400afb9bfec7b7b
 F test/walnoshm.test 84ca10c544632a756467336b7c3b864d493ee496
@@ -1699,7 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 948a26b5a64d8a89c4ec7546ec4a4bee36e29e82edda677ca75424e7091325d0
-R fa20b259f24a93ee214da4f46c94ddf6
-U dan
-Z c12f29c5e46aaf9ede6fab6c939cd772
+P ba44724bcca2e87788b7c6d8c5de7fa388360127bd894ee6a171fd66e794fcae
+R f3a35383cfb8ac5b9a76f4dd696ca9d1
+U mistachkin
+Z 9ba015577ebd6a341cfca228abd228a2
diff --git a/manifest.uuid b/manifest.uuid
index 95edffa509..e4ae655096 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ba44724bcca2e87788b7c6d8c5de7fa388360127bd894ee6a171fd66e794fcae
\ No newline at end of file
+7274d05ff43fc9872f0e4857ae583689e4a6c429b7fa991dcc29744da3048879
\ No newline at end of file
diff --git a/test/walfault.test b/test/walfault.test
index 4e7064d53b..6cb760b258 100644
--- a/test/walfault.test
+++ b/test/walfault.test
@@ -552,7 +552,7 @@ do_faultsim_test walfault-14 -prep {
 #-------------------------------------------------------------------------
 # Test fault-handling when switching out of exclusive-locking mode.
 #
-do_test walfault-14-pre {
+do_test walfault-15-pre {
   faultsim_delete_and_reopen
   execsql {
     PRAGMA auto_vacuum = 0;
@@ -565,7 +565,7 @@ do_test walfault-14-pre {
   }
   faultsim_save_and_close
 } {}
-do_faultsim_test walfault-14 -prep {
+do_faultsim_test walfault-15 -prep {
   faultsim_restore_and_reopen
   execsql {
     SELECT count(*) FROM abc;

From 5685257b83124cffa6e22c4dafedd8ca3060f390 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Wed, 17 Jan 2018 01:26:05 +0000
Subject: [PATCH 341/488] Corrections to error code handling in os_win.c,
 pursuant to walfault.test.

FossilOrigin-Name: 568192228c9578b8ea34c363e10ff28450045cda76248b2f0f89f84b3a57e680
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/os_win.c  | 30 ++++++++++++++++++++----------
 3 files changed, 27 insertions(+), 17 deletions(-)

diff --git a/manifest b/manifest
index c0e128cc24..2670429108 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sduplicate\stest\snames\sin\s'walfault.test'.
-D 2018-01-17T01:15:08.464
+C Corrections\sto\serror\scode\shandling\sin\sos_win.c,\spursuant\sto\swalfault.test.
+D 2018-01-17T01:26:05.915
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -470,7 +470,7 @@ F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
 F src/os_unix.c a82505be158d8ce42b38dcc9b426187d776904c12cdc68dc8925e1dfcc5cb6ce
-F src/os_win.c 0a4afa35cc8e812000df3ea2f64b476131b39e29e75d8007d0504726e4761de4
+F src/os_win.c 196b2b38953a3678e0ae1be40875c9c303ff216ec56ac2a6721e56ce9a9a454e
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 9b9cb4e06c03d43d62480a7a685a012d645fcf3a39e7767ccb505fb41ee083ec
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
@@ -1699,7 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ba44724bcca2e87788b7c6d8c5de7fa388360127bd894ee6a171fd66e794fcae
-R f3a35383cfb8ac5b9a76f4dd696ca9d1
+P 7274d05ff43fc9872f0e4857ae583689e4a6c429b7fa991dcc29744da3048879
+R a85767f9e76828ee76ff125e1a16892d
 U mistachkin
-Z 9ba015577ebd6a341cfca228abd228a2
+Z 21eee94f9fdd383abaac3542e6cde8fb
diff --git a/manifest.uuid b/manifest.uuid
index e4ae655096..f91d56d71c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7274d05ff43fc9872f0e4857ae583689e4a6c429b7fa991dcc29744da3048879
\ No newline at end of file
+568192228c9578b8ea34c363e10ff28450045cda76248b2f0f89f84b3a57e680
\ No newline at end of file
diff --git a/src/os_win.c b/src/os_win.c
index e3243a9c68..6df66f1d7a 100644
--- a/src/os_win.c
+++ b/src/os_win.c
@@ -3899,6 +3899,8 @@ static int winOpenSharedMemory(winFile *pDbFd){
   if( pShmNode ){
     sqlite3_free(pNew);
   }else{
+    int bReadonly;
+
     pShmNode = pNew;
     pNew = 0;
     ((winFile*)(&pShmNode->hFile))->h = INVALID_HANDLE_VALUE;
@@ -3913,7 +3915,9 @@ static int winOpenSharedMemory(winFile *pDbFd){
       }
     }
 
-    if( 0==sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){
+    bReadonly = sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0);
+
+    if( !bReadonly ){
       rc2 = winOpen(pDbFd->pVfs,
                     pShmNode->zFilename,
                     (sqlite3_file*)&pShmNode->hFile,
@@ -3921,13 +3925,13 @@ static int winOpenSharedMemory(winFile *pDbFd){
                     0);
     }
     if( rc2!=SQLITE_OK ){
-      rc2 = winOpen(pDbFd->pVfs,
+      int rc3 = winOpen(pDbFd->pVfs,
                     pShmNode->zFilename,
                     (sqlite3_file*)&pShmNode->hFile,
                     SQLITE_OPEN_WAL|SQLITE_OPEN_READONLY,
                     0);
-      if( rc2!=SQLITE_OK ){
-        rc = winLogError(rc2, osGetLastError(), "winOpenShm",
+      if( rc3!=SQLITE_OK ){
+        rc = winLogError(bReadonly ? rc3 : rc2, osGetLastError(), "winOpenShm",
                          pShmNode->zFilename);
         goto shm_open_err;
       }
@@ -5118,8 +5122,10 @@ static int winOpen(
                         &extendedParameters);
       if( h!=INVALID_HANDLE_VALUE ) break;
       if( isReadWrite ){
-        int isRO = 0;
-        int rc2 = winAccess(pVfs, zName, SQLITE_ACCESS_READ, &isRO);
+        int rc2, isRO = 0;
+        sqlite3BeginBenignMalloc();
+        rc2 = winAccess(pVfs, zName, SQLITE_ACCESS_READ, &isRO);
+        sqlite3EndBenignMalloc();
         if( rc2==SQLITE_OK && isRO ) break;
       }
     }while( winRetryIoerr(&cnt, &lastErrno) );
@@ -5133,8 +5139,10 @@ static int winOpen(
                         NULL);
       if( h!=INVALID_HANDLE_VALUE ) break;
       if( isReadWrite ){
-        int isRO = 0;
-        int rc2 = winAccess(pVfs, zName, SQLITE_ACCESS_READ, &isRO);
+        int rc2, isRO = 0;
+        sqlite3BeginBenignMalloc();
+        rc2 = winAccess(pVfs, zName, SQLITE_ACCESS_READ, &isRO);
+        sqlite3EndBenignMalloc();
         if( rc2==SQLITE_OK && isRO ) break;
       }
     }while( winRetryIoerr(&cnt, &lastErrno) );
@@ -5151,8 +5159,10 @@ static int winOpen(
                         NULL);
       if( h!=INVALID_HANDLE_VALUE ) break;
       if( isReadWrite ){
-        int isRO = 0;
-        int rc2 = winAccess(pVfs, zName, SQLITE_ACCESS_READ, &isRO);
+        int rc2, isRO = 0;
+        sqlite3BeginBenignMalloc();
+        rc2 = winAccess(pVfs, zName, SQLITE_ACCESS_READ, &isRO);
+        sqlite3EndBenignMalloc();
         if( rc2==SQLITE_OK && isRO ) break;
       }
     }while( winRetryIoerr(&cnt, &lastErrno) );

From 98e2cb8bd181cb2f1195b431a4fba9055c459ea6 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Wed, 17 Jan 2018 01:40:57 +0000
Subject: [PATCH 342/488] Simplifications to winOpenSharedMemory in the Win32
 VFS.

FossilOrigin-Name: 3e04999dabb87715de46255b1a9b08d5dfa70d140e0a09a37ea2842d71c77caf
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/os_win.c  | 36 ++++++++++++++----------------------
 3 files changed, 21 insertions(+), 29 deletions(-)

diff --git a/manifest b/manifest
index 2670429108..3faeefe23f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Corrections\sto\serror\scode\shandling\sin\sos_win.c,\spursuant\sto\swalfault.test.
-D 2018-01-17T01:26:05.915
+C Simplifications\sto\swinOpenSharedMemory\sin\sthe\sWin32\sVFS.
+D 2018-01-17T01:40:57.173
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -470,7 +470,7 @@ F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
 F src/os_unix.c a82505be158d8ce42b38dcc9b426187d776904c12cdc68dc8925e1dfcc5cb6ce
-F src/os_win.c 196b2b38953a3678e0ae1be40875c9c303ff216ec56ac2a6721e56ce9a9a454e
+F src/os_win.c 501dde1ee770f4ffa458bfe1cf376a556de3ab00bb8320d659c5984403991d62
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 9b9cb4e06c03d43d62480a7a685a012d645fcf3a39e7767ccb505fb41ee083ec
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
@@ -1699,7 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 7274d05ff43fc9872f0e4857ae583689e4a6c429b7fa991dcc29744da3048879
-R a85767f9e76828ee76ff125e1a16892d
+P 568192228c9578b8ea34c363e10ff28450045cda76248b2f0f89f84b3a57e680
+R 74d0fd9e585002553a9ac7b39220a819
 U mistachkin
-Z 21eee94f9fdd383abaac3542e6cde8fb
+Z 373464fa908aa53424186ef8053b7fb1
diff --git a/manifest.uuid b/manifest.uuid
index f91d56d71c..e6427cfa9a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-568192228c9578b8ea34c363e10ff28450045cda76248b2f0f89f84b3a57e680
\ No newline at end of file
+3e04999dabb87715de46255b1a9b08d5dfa70d140e0a09a37ea2842d71c77caf
\ No newline at end of file
diff --git a/src/os_win.c b/src/os_win.c
index 6df66f1d7a..2b2b8ebd56 100644
--- a/src/os_win.c
+++ b/src/os_win.c
@@ -3865,7 +3865,6 @@ static int winOpenSharedMemory(winFile *pDbFd){
   struct winShm *p;                  /* The connection to be opened */
   winShmNode *pShmNode = 0;          /* The underlying mmapped file */
   int rc = SQLITE_OK;                /* Result code */
-  int rc2 = SQLITE_ERROR;            /* winOpen result code */
   winShmNode *pNew;                  /* Newly allocated winShmNode */
   int nName;                         /* Size of zName in bytes */
 
@@ -3899,7 +3898,8 @@ static int winOpenSharedMemory(winFile *pDbFd){
   if( pShmNode ){
     sqlite3_free(pNew);
   }else{
-    int bReadonly;
+    int inFlags = SQLITE_OPEN_WAL;
+    int outFlags = 0;
 
     pShmNode = pNew;
     pNew = 0;
@@ -3915,28 +3915,20 @@ static int winOpenSharedMemory(winFile *pDbFd){
       }
     }
 
-    bReadonly = sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0);
-
-    if( !bReadonly ){
-      rc2 = winOpen(pDbFd->pVfs,
-                    pShmNode->zFilename,
-                    (sqlite3_file*)&pShmNode->hFile,
-                    SQLITE_OPEN_WAL|SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE,
-                    0);
+    if( 0==sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){
+      inFlags |= SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE;
+    }else{
+      inFlags |= SQLITE_OPEN_READONLY;
     }
-    if( rc2!=SQLITE_OK ){
-      int rc3 = winOpen(pDbFd->pVfs,
-                    pShmNode->zFilename,
-                    (sqlite3_file*)&pShmNode->hFile,
-                    SQLITE_OPEN_WAL|SQLITE_OPEN_READONLY,
-                    0);
-      if( rc3!=SQLITE_OK ){
-        rc = winLogError(bReadonly ? rc3 : rc2, osGetLastError(), "winOpenShm",
-                         pShmNode->zFilename);
-        goto shm_open_err;
-      }
-      pShmNode->isReadonly = 1;
+    rc = winOpen(pDbFd->pVfs, pShmNode->zFilename,
+                 (sqlite3_file*)&pShmNode->hFile,
+                 inFlags, &outFlags);
+    if( rc!=SQLITE_OK ){
+      rc = winLogError(rc, osGetLastError(), "winOpenShm",
+                       pShmNode->zFilename);
+      goto shm_open_err;
     }
+    if( outFlags==SQLITE_OPEN_READONLY ) pShmNode->isReadonly = 1;
 
     rc = winLockSharedMemory(pShmNode);
     if( rc!=SQLITE_OK && rc!=SQLITE_READONLY_CANTINIT ) goto shm_open_err;

From 30fe26c47f6f0d486cad4ba1b438b1b0333fb6e9 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 17 Jan 2018 12:57:07 +0000
Subject: [PATCH 343/488] Update test file fts5fault6.test to account for test
 tokenizers implemented in Tcl returning SQLITE_ERROR instead of SQLITE_NOMEM
 following an OOM error.

FossilOrigin-Name: c232f6424a858ede44940a927fe4e26ee99c6ab614aa6f63e13ba46e88dbb280
---
 ext/fts5/test/fts5fault6.test |  6 +++---
 manifest                      | 14 +++++++-------
 manifest.uuid                 |  2 +-
 3 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/ext/fts5/test/fts5fault6.test b/ext/fts5/test/fts5fault6.test
index 1c1c9f20c1..a39063a356 100644
--- a/ext/fts5/test/fts5fault6.test
+++ b/ext/fts5/test/fts5fault6.test
@@ -253,7 +253,7 @@ do_faultsim_test 5.2 -faults oom* -prep {
     SELECT rowid, mit(matchinfo(t1, 'x')) FROM t1 WHERE t1 MATCH 'a AND c'
   }
 } -test {
-  faultsim_test_result [list 0 $::res]
+  faultsim_test_result [list 0 $::res] {1 {SQL logic error}}
 }
 
 do_faultsim_test 5.3 -faults oom* -prep {
@@ -264,7 +264,7 @@ do_faultsim_test 5.3 -faults oom* -prep {
     SELECT count(*) FROM t1 WHERE t1 MATCH 'd AND e AND f'
   }
 } -test {
-  faultsim_test_result {0 29}
+  faultsim_test_result {0 29} {1 {SQL logic error}}
 }
 
 do_faultsim_test 5.4 -faults oom* -prep {
@@ -275,7 +275,7 @@ do_faultsim_test 5.4 -faults oom* -prep {
     SELECT count(*) FROM t1 WHERE t1 MATCH 'x + e'
   }
 } -test {
-  faultsim_test_result {0 1}
+  faultsim_test_result {0 1} {1 {SQL logic error}}
 }
 
 #-------------------------------------------------------------------------
diff --git a/manifest b/manifest
index 3faeefe23f..65dfc625fb 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Simplifications\sto\swinOpenSharedMemory\sin\sthe\sWin32\sVFS.
-D 2018-01-17T01:40:57.173
+C Update\stest\sfile\sfts5fault6.test\sto\saccount\sfor\stest\stokenizers\simplemented\sin\nTcl\sreturning\sSQLITE_ERROR\sinstead\sof\sSQLITE_NOMEM\sfollowing\san\sOOM\serror.
+D 2018-01-17T12:57:07.649
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -165,7 +165,7 @@ F ext/fts5/test/fts5fault2.test 69c8fdbef830cd0d450908d4504d5bb86609e255af99c421
 F ext/fts5/test/fts5fault3.test da2f9e3e56ff5740d68ebdd6877c97089e7ed28ddff28a0da87a6afea27e5522
 F ext/fts5/test/fts5fault4.test 1c1db5fcfe59401e7833146100f1d8de284a0a686fac31ddac9fb56c459f725b
 F ext/fts5/test/fts5fault5.test a336e4e11847de24c9497f80cce18e00bb3fab7fb11f97d04eb9af898900a762
-F ext/fts5/test/fts5fault6.test 8a3c61402e36960ba46a419e73121fcefdc9160e0c04b6f5318c7fb0e3180dbc
+F ext/fts5/test/fts5fault6.test a0fc0a8f99e4b16500c31dfc7e38e1defe0f1693ac47650517ac7b723b1956f8
 F ext/fts5/test/fts5fault7.test 0acbec416edb24b8881f154e99c31e9ccf73f539cfcd164090be139e9e97ed4c
 F ext/fts5/test/fts5fault8.test 318238659d35f82ad215ecb57ca4c87486ea85d45dbeedaee42f148ff5105ee2
 F ext/fts5/test/fts5fault9.test 098e6b894bbdf9b2192f994a30f4043673fb3f338b6b8ab1624c704422f39119
@@ -1699,7 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 568192228c9578b8ea34c363e10ff28450045cda76248b2f0f89f84b3a57e680
-R 74d0fd9e585002553a9ac7b39220a819
-U mistachkin
-Z 373464fa908aa53424186ef8053b7fb1
+P 3e04999dabb87715de46255b1a9b08d5dfa70d140e0a09a37ea2842d71c77caf
+R 941f33fbf2a23d6c019574c3e00d86d4
+U dan
+Z f10d4208193a5de16aeae87ee4be55df
diff --git a/manifest.uuid b/manifest.uuid
index e6427cfa9a..9d255d110e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-3e04999dabb87715de46255b1a9b08d5dfa70d140e0a09a37ea2842d71c77caf
\ No newline at end of file
+c232f6424a858ede44940a927fe4e26ee99c6ab614aa6f63e13ba46e88dbb280
\ No newline at end of file

From f193937620052be8a23ffced856831668e88d6b3 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 17 Jan 2018 12:58:11 +0000
Subject: [PATCH 344/488] Omit the single test from zipfile.test that uses json
 functionality in non-SQLITE_ENABLE_JSON1 builds.

FossilOrigin-Name: 6bb2a10fad71bf8cec6bca538db7b00be0d26418ab62b83092d3a6a68d1c0d9f
---
 manifest          | 12 ++++++------
 manifest.uuid     |  2 +-
 test/zipfile.test | 16 +++++++++-------
 3 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/manifest b/manifest
index 65dfc625fb..d7a2894498 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\stest\sfile\sfts5fault6.test\sto\saccount\sfor\stest\stokenizers\simplemented\sin\nTcl\sreturning\sSQLITE_ERROR\sinstead\sof\sSQLITE_NOMEM\sfollowing\san\sOOM\serror.
-D 2018-01-17T12:57:07.649
+C Omit\sthe\ssingle\stest\sfrom\szipfile.test\sthat\suses\sjson\sfunctionality\sin\nnon-SQLITE_ENABLE_JSON1\sbuilds.
+D 2018-01-17T12:58:11.883
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -1600,7 +1600,7 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf
 F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc
 F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa
 F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc
-F test/zipfile.test 17e3ed850c7f182a5ce746fadc1a0497495aeacba216d73ece7b3b2c5dfd8867
+F test/zipfile.test cb42e8fa6ba5db4a03ce6baa4401fc6236baf6eb5e62b44f3e463bf6aafd631d
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d
 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91
@@ -1699,7 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3e04999dabb87715de46255b1a9b08d5dfa70d140e0a09a37ea2842d71c77caf
-R 941f33fbf2a23d6c019574c3e00d86d4
+P c232f6424a858ede44940a927fe4e26ee99c6ab614aa6f63e13ba46e88dbb280
+R 1693bc0c1e569048e8eec005916d4449
 U dan
-Z f10d4208193a5de16aeae87ee4be55df
+Z 8dee77e444940dc97285c3ee0298e6bb
diff --git a/manifest.uuid b/manifest.uuid
index 9d255d110e..ac4e921489 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c232f6424a858ede44940a927fe4e26ee99c6ab614aa6f63e13ba46e88dbb280
\ No newline at end of file
+6bb2a10fad71bf8cec6bca538db7b00be0d26418ab62b83092d3a6a68d1c0d9f
\ No newline at end of file
diff --git a/test/zipfile.test b/test/zipfile.test
index c324e0d7c7..07fc80f602 100644
--- a/test/zipfile.test
+++ b/test/zipfile.test
@@ -75,13 +75,15 @@ do_execsql_test 1.4 {
   h.txt 1000000004 aaaaaaaaaabbbbbbbbbb 8
 }
 
-do_execsql_test 1.4.1 {
-  SELECT name, json_extract( zipfile_cds(z) , '$.crc32')!=0
-  FROM zipfile('test.zip');
-} {
-  f.txt 1
-  g.txt 1
-  h.txt 1
+ifcapable json1 {
+  do_execsql_test 1.4.1 {
+    SELECT name, json_extract( zipfile_cds(z) , '$.crc32')!=0
+    FROM zipfile('test.zip');
+  } {
+    f.txt 1
+    g.txt 1
+    h.txt 1
+  }
 }
 
 do_execsql_test 1.5.1 {

From b96851855972a35f30b554fda1c4d1b6f9a14965 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 17 Jan 2018 13:15:23 +0000
Subject: [PATCH 345/488] Fix harmless compiler warnings, mostly unused
 parameters for UDFs in the CLI.

FossilOrigin-Name: bfbeffab7735461acb3773242ba31dae15af9f8b0291c54a6734747bb1b36e66
---
 ext/expert/sqlite3expert.c | 17 +++++++++++++++--
 ext/misc/appendvfs.c       |  2 ++
 ext/misc/fileio.c          | 15 ++++++++++++---
 manifest                   | 22 +++++++++++-----------
 manifest.uuid              |  2 +-
 src/shell.c.in             |  3 +++
 tool/lempar.c              |  2 ++
 7 files changed, 46 insertions(+), 17 deletions(-)

diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c
index 9cafc448b2..e0602bc65e 100644
--- a/ext/expert/sqlite3expert.c
+++ b/ext/expert/sqlite3expert.c
@@ -507,6 +507,10 @@ static int expertUpdate(
   sqlite3_value **azData, 
   sqlite_int64 *pRowid
 ){
+  (void)pVtab;
+  (void)nData;
+  (void)azData;
+  (void)pRowid;
   return SQLITE_OK;
 }
 
@@ -516,6 +520,7 @@ static int expertUpdate(
 static int expertOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
   int rc = SQLITE_OK;
   ExpertCsr *pCsr;
+  (void)pVTab;
   pCsr = idxMalloc(&rc, sizeof(ExpertCsr));
   *ppCursor = (sqlite3_vtab_cursor*)pCsr;
   return rc;
@@ -565,6 +570,7 @@ static int expertNext(sqlite3_vtab_cursor *cur){
 ** Virtual table module xRowid method.
 */
 static int expertRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
+  (void)cur;
   *pRowid = 0;
   return SQLITE_OK;
 }
@@ -595,6 +601,10 @@ static int expertFilter(
   sqlite3expert *pExpert = pVtab->pExpert;
   int rc;
 
+  (void)idxNum;
+  (void)idxStr;
+  (void)argc;
+  (void)argv;
   rc = sqlite3_finalize(pCsr->pData);
   pCsr->pData = 0;
   if( rc==SQLITE_OK ){
@@ -1005,7 +1015,7 @@ static int idxCreateFromWhere(
 ** Create candidate indexes in database [dbm] based on the data in 
 ** linked-list pScan.
 */
-static int idxCreateCandidates(sqlite3expert *p, char **pzErr){
+static int idxCreateCandidates(sqlite3expert *p){
   int rc = SQLITE_OK;
   IdxScan *pIter;
 
@@ -1168,6 +1178,8 @@ static int idxAuthCallback(
   const char *zTrigger
 ){
   int rc = SQLITE_OK;
+  (void)z4;
+  (void)zTrigger;
   if( eOp==SQLITE_INSERT || eOp==SQLITE_UPDATE || eOp==SQLITE_DELETE ){
     if( sqlite3_stricmp(zDb, "main")==0 ){
       sqlite3expert *p = (sqlite3expert*)pCtx;
@@ -1370,6 +1382,7 @@ static void idxSampleFunc(
   struct IdxSampleCtx *p = (struct IdxSampleCtx*)sqlite3_user_data(pCtx);
   int bRet;
 
+  (void)argv;
   assert( argc==0 );
   if( p->nRow==0.0 ){
     bRet = 1;
@@ -1855,7 +1868,7 @@ int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr){
 
   /* Create candidate indexes within the in-memory database file */
   if( rc==SQLITE_OK ){
-    rc = idxCreateCandidates(p, pzErr);
+    rc = idxCreateCandidates(p);
   }
 
   /* Generate the stat1 data */
diff --git a/ext/misc/appendvfs.c b/ext/misc/appendvfs.c
index 1454243057..b224245f3d 100644
--- a/ext/misc/appendvfs.c
+++ b/ext/misc/appendvfs.c
@@ -548,6 +548,8 @@ int sqlite3_appendvfs_init(
   int rc = SQLITE_OK;
   sqlite3_vfs *pOrig;
   SQLITE_EXTENSION_INIT2(pApi);
+  (void)pzErrMsg;
+  (void)db;
   pOrig = sqlite3_vfs_find(0);
   apnd_vfs.iVersion = pOrig->iVersion;
   apnd_vfs.pAppData = pOrig;
diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c
index 7035889482..60a960f310 100644
--- a/ext/misc/fileio.c
+++ b/ext/misc/fileio.c
@@ -372,6 +372,7 @@ static void lsModeFunc(
   int i;
   int iMode = sqlite3_value_int(argv[0]);
   char z[16];
+  (void)argc;
   if( S_ISLNK(iMode) ){
     z[0] = 'l';
   }else if( S_ISREG(iMode) ){
@@ -437,7 +438,10 @@ static int fsdirConnect(
 ){
   fsdir_tab *pNew = 0;
   int rc;
-
+  (void)pAux;
+  (void)argc;
+  (void)argv;
+  (void)pzErr;
   rc = sqlite3_declare_vtab(db, "CREATE TABLE x" FSDIR_SCHEMA);
   if( rc==SQLITE_OK ){
     pNew = (fsdir_tab*)sqlite3_malloc( sizeof(*pNew) );
@@ -461,6 +465,7 @@ static int fsdirDisconnect(sqlite3_vtab *pVtab){
 */
 static int fsdirOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
   fsdir_cursor *pCur;
+  (void)p;
   pCur = sqlite3_malloc( sizeof(*pCur) );
   if( pCur==0 ) return SQLITE_NOMEM;
   memset(pCur, 0, sizeof(*pCur));
@@ -664,7 +669,7 @@ static int fsdirFilter(
 ){
   const char *zDir = 0;
   fsdir_cursor *pCur = (fsdir_cursor*)cur;
-
+  (void)idxStr;
   fsdirResetCursor(pCur);
 
   if( idxNum==0 ){
@@ -722,8 +727,9 @@ static int fsdirBestIndex(
   int i;                 /* Loop over constraints */
   int idx4 = -1;
   int idx5 = -1;
-
   const struct sqlite3_index_constraint *pConstraint;
+
+  (void)tab;
   pConstraint = pIdxInfo->aConstraint;
   for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){
     if( pConstraint->usable==0 ) continue;
@@ -777,6 +783,9 @@ static int fsdirRegister(sqlite3 *db){
     0,                         /* xRollback */
     0,                         /* xFindMethod */
     0,                         /* xRename */
+    0,                         /* xSavepoint */
+    0,                         /* xRelease */
+    0                          /* xRollbackTo */
   };
 
   int rc = sqlite3_create_module(db, "fsdir", &fsdirModule, 0);
diff --git a/manifest b/manifest
index d7a2894498..ca58247446 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Omit\sthe\ssingle\stest\sfrom\szipfile.test\sthat\suses\sjson\sfunctionality\sin\nnon-SQLITE_ENABLE_JSON1\sbuilds.
-D 2018-01-17T12:58:11.883
+C Fix\sharmless\scompiler\swarnings,\smostly\sunused\sparameters\sfor\sUDFs\sin\sthe\sCLI.
+D 2018-01-17T13:15:23.987
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -45,7 +45,7 @@ F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3
 F ext/expert/expert.c 4791c5e064aea81b2b829fa95228b22283380ee370ea88a1e580103b75516ebf
 F ext/expert/expert1.test fd21496d8e52c817a7741f467f42b0502c0ac7e07dcdd1d6e15a3e8154ed4e41
-F ext/expert/sqlite3expert.c 55ea02e9fcc014f4252e8a8c78d2bddc6d7ef62f9cb54ee283a659516eabb711
+F ext/expert/sqlite3expert.c 1dfa561e64dc0f89d56b96e6afda87468c34b43604c2df50c47e3f4362778fb2
 F ext/expert/sqlite3expert.h af6354f8ee5c9e025024e63fec3bd640a802afcc3099a44d804752cf0791d811
 F ext/expert/test_expert.c d56c194b769bdc90cf829a14c9ecbc1edca9c850b837a4d0b13be14095c32a72
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
@@ -268,7 +268,7 @@ F ext/lsm1/tool/mklsm1c.tcl f31561bbee5349f0a554d1ad7236ac1991fc09176626f529f607
 F ext/misc/README.md d6dd0fe1d8af77040216798a6a2b0c46c73054d2f0ea544fbbcdccf6f238c240
 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87
 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb
-F ext/misc/appendvfs.c 4c65f0b79686ae5a483134233d7fd912f0f2d4fd76023404f96f2290fff13b19
+F ext/misc/appendvfs.c 3777f22ec1057dc4e5fd89f2fbddcc7a29fbeef1ad038c736c54411bb1967af7
 F ext/misc/btreeinfo.c d7fd9a2fe2fa33ba28488e2fce703ebecc759219ea9e0bb3b254784866c0a676
 F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005
 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704
@@ -277,7 +277,7 @@ F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f0
 F ext/misc/csv.c 1a009b93650732e22334edc92459c4630b9fa703397cbb3c8ca279921a36ca11
 F ext/misc/dbdump.c 3509fa6b8932d04e932d6b6b827b6a82ca362781b8e8f3c77336f416793e215e
 F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2
-F ext/misc/fileio.c 777c13f00b4505df3bfab602568c98d2b067f7d1d265de88160d0f1ac92f3dcf
+F ext/misc/fileio.c 06bd79dcc43d0887da27ffaadd69b8a698b1bafe203d1d134a3a2964f69368f9
 F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25
 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c
 F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281984
@@ -486,7 +486,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c bebe7cce45d899d2237c76bce059d525abf5b861f2fce92f6b53914a961c01ba
-F src/shell.c.in 568aacb5b15520fa7981ebcf976ebd99849e6a4777858706762f201a446626f3
+F src/shell.c.in 4e1bcf8c70b8fb97c7cbaca6602e2a291d7fe17eff23a5de003d6fabd87f27d1
 F src/sqlite.h.in 840e8aee0d7f8592bd69730797582b342a413eceaff7a220fb5f9db9c412cea9
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 99189e7611eb0bf98f21c7835dc74730a84e2e809c98e1e31c33896dee7a2849
@@ -1619,7 +1619,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
 F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f
 F tool/lemon.c 7f7735326ca9c3b48327b241063cee52d35d44e20ebe1b3624a81658052a4d39
-F tool/lempar.c dddd4f592b8bad36aec4500d456c5db5fe42fefc4ee384913880439d8917f87a
+F tool/lempar.c da840fc8a6fbac23599a65ff075e6e3d01320417c794ff577088e09f5d74b689
 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9
 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca
@@ -1699,7 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c232f6424a858ede44940a927fe4e26ee99c6ab614aa6f63e13ba46e88dbb280
-R 1693bc0c1e569048e8eec005916d4449
-U dan
-Z 8dee77e444940dc97285c3ee0298e6bb
+P 6bb2a10fad71bf8cec6bca538db7b00be0d26418ab62b83092d3a6a68d1c0d9f
+R 3ce13cb6be5e7cd3831ddb44a0279981
+U drh
+Z 43432e1e2cf79a7dd14125e4a977e1e5
diff --git a/manifest.uuid b/manifest.uuid
index ac4e921489..6038ac149a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6bb2a10fad71bf8cec6bca538db7b00be0d26418ab62b83092d3a6a68d1c0d9f
\ No newline at end of file
+bfbeffab7735461acb3773242ba31dae15af9f8b0291c54a6734747bb1b36e66
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 523d7ef7bf..d9c8705e70 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -865,6 +865,7 @@ static void shellModuleSchema(
 ){
   const char *zName = (const char*)sqlite3_value_text(apVal[0]);
   char *zFake = shellFakeSchema(sqlite3_context_db_handle(pCtx), 0, zName);
+  UNUSED_PARAMETER(nVal);
   if( zFake ){
     sqlite3_result_text(pCtx, sqlite3_mprintf("/* %s */", zFake),
                         -1, sqlite3_free);
@@ -909,6 +910,7 @@ static void shellAddSchemaName(
   const char *zSchema = (const char*)sqlite3_value_text(apVal[1]);
   const char *zName = (const char*)sqlite3_value_text(apVal[2]);
   sqlite3 *db = sqlite3_context_db_handle(pCtx);
+  UNUSED_PARAMETER(nVal);
   if( zIn!=0 && strncmp(zIn, "CREATE ", 7)==0 ){
     for(i=0; i<(int)(sizeof(aPrefix)/sizeof(aPrefix[0])); i++){
       int n = strlen30(aPrefix[i]);
@@ -1152,6 +1154,7 @@ static void shellPutsFunc(
   sqlite3_value **apVal
 ){
   ShellState *p = (ShellState*)sqlite3_user_data(pCtx);
+  (void)nVal;
   utf8_printf(p->out, "%s\n", sqlite3_value_text(apVal[0]));
   sqlite3_result_value(pCtx, apVal[0]);
 }
diff --git a/tool/lempar.c b/tool/lempar.c
index 9164eb0c1e..ecc0e6389b 100644
--- a/tool/lempar.c
+++ b/tool/lempar.c
@@ -706,6 +706,8 @@ static void yy_reduce(
   yyStackEntry *yymsp;            /* The top of the parser's stack */
   int yysize;                     /* Amount to pop the stack */
   ParseARG_FETCH;
+  (void)yyLookahead;
+  (void)yyLookaheadToken;
   yymsp = yypParser->yytos;
 #ifndef NDEBUG
   if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){

From 26893c98d21a2d50b521407e255131504ccf742a Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 17 Jan 2018 16:11:26 +0000
Subject: [PATCH 346/488] Fix main.mk so that testfixture can be built either
 from the amalgamation or from individual source files. No changes to code.

FossilOrigin-Name: a8aea925f8fde8f2dc5ff4b744d54aa2bf8916f3ee57f22d77fd1ddb5a35a9cc
---
 main.mk       |  2 +-
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/main.mk b/main.mk
index 504ff34e1c..f2640b4339 100644
--- a/main.mk
+++ b/main.mk
@@ -366,7 +366,6 @@ TESTSRC += \
   $(TOP)/ext/misc/remember.c \
   $(TOP)/ext/misc/series.c \
   $(TOP)/ext/misc/spellfix.c \
-  $(TOP)/ext/misc/stmt.c \
   $(TOP)/ext/misc/totype.c \
   $(TOP)/ext/misc/unionvtab.c \
   $(TOP)/ext/misc/wholenumber.c \
@@ -423,6 +422,7 @@ TESTSRC2 = \
   $(TOP)/ext/fts3/fts3_tokenizer.c \
   $(TOP)/ext/fts3/fts3_write.c \
   $(TOP)/ext/async/sqlite3async.c \
+  $(TOP)/ext/misc/stmt.c \
   $(TOP)/ext/session/sqlite3session.c \
   $(TOP)/ext/session/test_session.c 
 
diff --git a/manifest b/manifest
index ca58247446..90624ae36c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sharmless\scompiler\swarnings,\smostly\sunused\sparameters\sfor\sUDFs\sin\sthe\sCLI.
-D 2018-01-17T13:15:23.987
+C Fix\smain.mk\sso\sthat\stestfixture\scan\sbe\sbuilt\seither\sfrom\sthe\samalgamation\sor\nfrom\sindividual\ssource\sfiles.\sNo\schanges\sto\scode.
+D 2018-01-17T16:11:26.562
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -408,7 +408,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 69bce07f975e6835a544f43df8d9619ea31d2c8e7b3a8d2cb4c21ac704c9c9c2
+F main.mk 4d19895cb268021474ade6244119c80b8a3f1d910cb5b13cf5e04f5f37c3d61b
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -1699,7 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6bb2a10fad71bf8cec6bca538db7b00be0d26418ab62b83092d3a6a68d1c0d9f
-R 3ce13cb6be5e7cd3831ddb44a0279981
-U drh
-Z 43432e1e2cf79a7dd14125e4a977e1e5
+P bfbeffab7735461acb3773242ba31dae15af9f8b0291c54a6734747bb1b36e66
+R ec2f214ab6dea8dc121fe027cc99d1ce
+U dan
+Z d15337a42013c6ddacd72bec21ca6a84
diff --git a/manifest.uuid b/manifest.uuid
index 6038ac149a..505eed63be 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-bfbeffab7735461acb3773242ba31dae15af9f8b0291c54a6734747bb1b36e66
\ No newline at end of file
+a8aea925f8fde8f2dc5ff4b744d54aa2bf8916f3ee57f22d77fd1ddb5a35a9cc
\ No newline at end of file

From 61c758b53a9782d0b31422ad705cc53f672f8e83 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 17 Jan 2018 17:38:18 +0000
Subject: [PATCH 347/488] Fix a problem causing the sessions module to
 occasionally lose track of rows with composite primary keys when there are
 two rows with the same text value in the leftmost column of the PK.

FossilOrigin-Name: 09aed13678374bf22087cd808808b711dc703b7c18bc8aaf704850611e17f5cd
---
 ext/session/sessionH.test    | 39 ++++++++++++++++++++++++++++++++++++
 ext/session/sessionat.test   |  4 ++++
 ext/session/sqlite3session.c |  1 -
 manifest                     | 15 +++++++-------
 manifest.uuid                |  2 +-
 5 files changed, 52 insertions(+), 9 deletions(-)
 create mode 100644 ext/session/sessionH.test

diff --git a/ext/session/sessionH.test b/ext/session/sessionH.test
new file mode 100644
index 0000000000..643fdb3fbe
--- /dev/null
+++ b/ext/session/sessionH.test
@@ -0,0 +1,39 @@
+# 2018 January 18
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+} 
+source [file join [file dirname [info script]] session_common.tcl]
+source $testdir/tester.tcl
+ifcapable !session {finish_test; return}
+set testprefix sessionH
+
+forcedelete test.db2
+sqlite3 db2 test.db2
+
+do_test 1.0 {
+  do_common_sql {
+    CREATE TABLE t1(a, b, c, PRIMARY KEY(a, b));
+  }
+  do_then_apply_sql {
+    WITH s(i) AS (
+      VALUES(1) UNION ALL SELECT i+1 FROM s WHERe i<10000
+    )
+    INSERT INTO t1 SELECT 'abcde', randomblob(16), i FROM s;
+  }
+  compare_db db db2
+} {}
+
+
+finish_test
+
diff --git a/ext/session/sessionat.test b/ext/session/sessionat.test
index f482d01520..4a3f59a441 100644
--- a/ext/session/sessionat.test
+++ b/ext/session/sessionat.test
@@ -241,5 +241,9 @@ eval [string map [list %WR% $trailing] {
 }]
 }
 
+catch { db close }
+catch { db2 close }
+sqlite3_shutdown
+test_sqlite3_log
 
 finish_test
diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c
index bd83ce87d3..6ef90037de 100644
--- a/ext/session/sqlite3session.c
+++ b/ext/session/sqlite3session.c
@@ -838,7 +838,6 @@ static int sessionPreupdateEqual(
         }
         if( memcmp(a, z, n) ) return 0;
         a += n;
-        break;
       }
     }
   }
diff --git a/manifest b/manifest
index 90624ae36c..5e365cf00f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\smain.mk\sso\sthat\stestfixture\scan\sbe\sbuilt\seither\sfrom\sthe\samalgamation\sor\nfrom\sindividual\ssource\sfiles.\sNo\schanges\sto\scode.
-D 2018-01-17T16:11:26.562
+C Fix\sa\sproblem\scausing\sthe\ssessions\smodule\sto\soccasionally\slose\strack\sof\srows\nwith\scomposite\sprimary\skeys\swhen\sthere\sare\stwo\srows\swith\sthe\ssame\stext\svalue\nin\sthe\sleftmost\scolumn\sof\sthe\sPK.
+D 2018-01-17T17:38:18.448
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -391,15 +391,16 @@ F ext/session/sessionD.test d3617e29aa15c9413aee5286d99587633245d58d2ad28f3f331c
 F ext/session/sessionE.test 0a616c4ad8fd2c05f23217ebb6212ef80b7fef30f5f086a6633a081f93e84637
 F ext/session/sessionF.test c2f178d4dfd723a5fd94a730ea2ccb44c669e3ce
 F ext/session/sessionG.test 63f9a744341d670775af29e4f19c1ef09a4810798400f28cd76704803a2e56ff
+F ext/session/sessionH.test 332b60e4c2e0a680105e11936201cabe378216f307e2747803cea56fa7d9ebae
 F ext/session/session_common.tcl 7776eda579773113b30c7abfd4545c445228cb73
 F ext/session/session_speed_test.c edc1f96fd5e0e4b16eb03e2a73041013d59e8723
-F ext/session/sessionat.test feb7d22b3124882064b9d9df69f5484a9bb8c123dc9ddc6ffcd357521848139f
+F ext/session/sessionat.test efe88965e74ff1bc2af9c310b28358c02d420c1fb2705cc7a28f0c1cc142c3ec
 F ext/session/sessiondiff.test ad13dd65664bae26744e1f18eb3cbd5588349b7e9118851d8f9364248d67bcec
 F ext/session/sessionfault.test da273f2712b6411e85e71465a1733b8501dbf6f7
 F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0
 F ext/session/sessionstat1.test e3a3f5876ce1526b48f6f447ee0b18960ac683e3fc891791e1ca0c08e823d498
 F ext/session/sessionwor.test 2f3744236dc8b170a695b7d8ddc8c743c7e79fdc
-F ext/session/sqlite3session.c a18bfdab0de090597fb50aae734e2cd41bf5e5d394bbd9a2189308f2bc8525d5
+F ext/session/sqlite3session.c c9d813ffa8db0670257da0a0c931e7188dfbb0010bf1a38274775c9b6300dd5c
 F ext/session/sqlite3session.h cb4d860101ba6d3ac810f18684539b766d24d668fa2436cdde90d711af9464fb
 F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386
 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
@@ -1699,7 +1700,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P bfbeffab7735461acb3773242ba31dae15af9f8b0291c54a6734747bb1b36e66
-R ec2f214ab6dea8dc121fe027cc99d1ce
+P a8aea925f8fde8f2dc5ff4b744d54aa2bf8916f3ee57f22d77fd1ddb5a35a9cc
+R fffe7fea955e558a2fcc7877bcdc39db
 U dan
-Z d15337a42013c6ddacd72bec21ca6a84
+Z 8da05c1c6303c10ad33f099cfa382df7
diff --git a/manifest.uuid b/manifest.uuid
index 505eed63be..41ac115dac 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a8aea925f8fde8f2dc5ff4b744d54aa2bf8916f3ee57f22d77fd1ddb5a35a9cc
\ No newline at end of file
+09aed13678374bf22087cd808808b711dc703b7c18bc8aaf704850611e17f5cd
\ No newline at end of file

From 3739f29807180878a77c37405f3f166c05dfb086 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 17 Jan 2018 20:57:20 +0000
Subject: [PATCH 348/488] Fix a problem in the sessions module with logging
 sqlite_stat1 rows for which (idx IS NULL) is true.

FossilOrigin-Name: 25bf734be1b3883fccf12ac4d93d50289aa307fb60a52e0e32df12f7ee4edc7a
---
 ext/session/sessionstat1.test | 109 +++++++++++++++++++++
 ext/session/sqlite3session.c  | 176 +++++++++++++++++++++++++---------
 manifest                      |  19 ++--
 manifest.uuid                 |   2 +-
 src/analyze.c                 |   3 +
 5 files changed, 253 insertions(+), 56 deletions(-)

diff --git a/ext/session/sessionstat1.test b/ext/session/sessionstat1.test
index 59de1cf1b4..22b2de8dbc 100644
--- a/ext/session/sessionstat1.test
+++ b/ext/session/sessionstat1.test
@@ -121,5 +121,114 @@ do_execsql_test -db db2 2.4 {
 
 do_execsql_test -db db2 2.5 { SELECT count(*) FROM t1 } 32
 
+#-------------------------------------------------------------------------
+db2 close
+forcedelete test.db2
+reset_db
+sqlite3 db2 test.db2
+
+do_test 3.0 {
+  do_common_sql {
+    CREATE TABLE t1(a, b, c);
+    ANALYZE;
+    DELETE FROM sqlite_stat1;
+  }
+  execsql {
+    INSERT INTO t1 VALUES(1, 1, 1);
+    INSERT INTO t1 VALUES(2, 2, 2);
+    INSERT INTO t1 VALUES(3, 3, 3);
+    INSERT INTO t1 VALUES(4, 4, 4);
+  }
+} {} 
+
+do_iterator_test 3.1 {} {
+  ANALYZE
+} {
+  {INSERT sqlite_stat1 0 XX. {} {t t1 b {} t 4}}
+}
+db null null
+db2 null null
+do_execsql_test 3.2 {
+  SELECT * FROM sqlite_stat1;
+} {t1 null 4}
+do_test 3.3 {
+  execsql { DELETE FROM sqlite_stat1 }
+  do_then_apply_sql { ANALYZE }
+  execsql { SELECT * FROM sqlite_stat1 } db2
+} {t1 null 4}
+do_test 3.4 {
+  execsql { INSERT INTO t1 VALUES(5,5,5) }
+  do_then_apply_sql { ANALYZE }
+  execsql { SELECT * FROM sqlite_stat1 } db2
+} {t1 null 5}
+do_test 3.5 {
+  do_then_apply_sql { DROP TABLE t1 }
+  execsql { SELECT * FROM sqlite_stat1 } db2
+} {}
+
+do_test 3.6.1 {
+  execsql { 
+    CREATE TABLE t1(a, b, c);
+    CREATE TABLE t2(x, y, z);
+    INSERT INTO t1 VALUES(1,1,1), (2,2,2), (3,3,3), (4,4,4), (5,5,5);
+    INSERT INTO t2 SELECT * FROM t1;
+    DELETE FROM sqlite_stat1;
+  }
+  sqlite3session S db main
+  S attach sqlite_stat1
+  execsql { ANALYZE }
+} {}
+do_changeset_test 3.6.2 S {
+  {INSERT sqlite_stat1 0 XX. {} {t t2 b {} t 5}}
+  {INSERT sqlite_stat1 0 XX. {} {t t1 b {} t 5}}
+}
+do_changeset_invert_test 3.6.3 S {
+  {DELETE sqlite_stat1 0 XX. {t t2 b {} t 5} {}}
+  {DELETE sqlite_stat1 0 XX. {t t1 b {} t 5} {}}
+}
+do_test 3.6.4 { S delete } {}
+
+proc sql_changeset_concat {args} {
+  foreach sql $args {
+    sqlite3session S db main
+    S attach sqlite_stat1
+    execsql $sql
+    set change [S changeset]
+    S delete
+
+    if {[info vars ret]!=""} {
+      set ret [sqlite3changeset_concat $ret $change]
+    } else {
+      set ret $change
+    }
+  }
+
+  changeset_to_list $ret
+}
+
+proc do_scc_test {tn args} {
+  uplevel [list \
+    do_test $tn [concat sql_changeset_concat [lrange $args 0 end-1]] \
+    [list {*}[ lindex $args end ]]
+  ]
+}
+
+do_execsql_test 3.7.0 {
+  DELETE FROM sqlite_stat1;
+}
+do_scc_test 3.7.1 {
+  ANALYZE;
+} {
+  INSERT INTO t2 VALUES(6,6,6);
+  ANALYZE;
+} {
+  {INSERT sqlite_stat1 0 XX. {} {t t1 b {} t 5}}
+  {INSERT sqlite_stat1 0 XX. {} {t t2 b {} t 6}}
+}
+
+
+
+
+
 finish_test
 
diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c
index 6ef90037de..cbe75d5667 100644
--- a/ext/session/sqlite3session.c
+++ b/ext/session/sqlite3session.c
@@ -113,6 +113,7 @@ struct SessionTable {
   SessionTable *pNext;
   char *zName;                    /* Local name of table */
   int nCol;                       /* Number of columns in table zName */
+  int bStat1;                     /* True if this is sqlite_stat1 */
   const char **azCol;             /* Column names */
   u8 *abPK;                       /* Array of primary key flags */
   int nEntry;                     /* Total number of entries in hash table */
@@ -472,31 +473,35 @@ static int sessionPreupdateHash(
       if( rc!=SQLITE_OK ) return rc;
 
       eType = sqlite3_value_type(pVal);
-      h = sessionHashAppendType(h, eType);
-      if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){
-        i64 iVal;
-        if( eType==SQLITE_INTEGER ){
-          iVal = sqlite3_value_int64(pVal);
-        }else{
-          double rVal = sqlite3_value_double(pVal);
-          assert( sizeof(iVal)==8 && sizeof(rVal)==8 );
-          memcpy(&iVal, &rVal, 8);
-        }
-        h = sessionHashAppendI64(h, iVal);
-      }else if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){
-        const u8 *z;
-        int n;
-        if( eType==SQLITE_TEXT ){
-          z = (const u8 *)sqlite3_value_text(pVal);
-        }else{
-          z = (const u8 *)sqlite3_value_blob(pVal);
-        }
-        n = sqlite3_value_bytes(pVal);
-        if( !z && (eType!=SQLITE_BLOB || n>0) ) return SQLITE_NOMEM;
-        h = sessionHashAppendBlob(h, n, z);
+      if( pTab->bStat1 && eType==SQLITE_NULL ){
+        h = sessionHashAppendType(h, SQLITE_BLOB);
       }else{
-        assert( eType==SQLITE_NULL );
-        *pbNullPK = 1;
+        h = sessionHashAppendType(h, eType);
+        if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){
+          i64 iVal;
+          if( eType==SQLITE_INTEGER ){
+            iVal = sqlite3_value_int64(pVal);
+          }else{
+            double rVal = sqlite3_value_double(pVal);
+            assert( sizeof(iVal)==8 && sizeof(rVal)==8 );
+            memcpy(&iVal, &rVal, 8);
+          }
+          h = sessionHashAppendI64(h, iVal);
+        }else if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){
+          const u8 *z;
+          int n;
+          if( eType==SQLITE_TEXT ){
+            z = (const u8 *)sqlite3_value_text(pVal);
+          }else{
+            z = (const u8 *)sqlite3_value_blob(pVal);
+          }
+          n = sqlite3_value_bytes(pVal);
+          if( !z && (eType!=SQLITE_BLOB || n>0) ) return SQLITE_NOMEM;
+          h = sessionHashAppendBlob(h, n, z);
+        }else{
+          assert( eType==SQLITE_NULL );
+          *pbNullPK = 1;
+        }
       }
     }
   }
@@ -550,7 +555,7 @@ static unsigned int sessionChangeHash(
          || eType==SQLITE_TEXT || eType==SQLITE_BLOB 
          || eType==SQLITE_NULL || eType==0 
     );
-    assert( !isPK || (eType!=0 && eType!=SQLITE_NULL) );
+    assert( !isPK || (eType!=0 && (pTab->bStat1 || eType!=SQLITE_NULL)) );
 
     if( isPK ){
       a++;
@@ -558,7 +563,7 @@ static unsigned int sessionChangeHash(
       if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){
         h = sessionHashAppendI64(h, sessionGetI64(a));
         a += 8;
-      }else{
+      }else if( eType!=SQLITE_NULL ){
         int n; 
         a += sessionVarintGet(a, &n);
         h = sessionHashAppendBlob(h, n, a);
@@ -794,6 +799,7 @@ static int sessionPreupdateEqual(
       sqlite3_value *pVal;        /* Value returned by preupdate_new/old */
       int rc;                     /* Error code from preupdate_new/old */
       int eType = *a++;           /* Type of value from change record */
+      int eValType;
 
       /* The following calls to preupdate_new() and preupdate_old() can not
       ** fail. This is because they cache their return values, and by the
@@ -808,7 +814,14 @@ static int sessionPreupdateEqual(
         rc = pSession->hook.xOld(pSession->hook.pCtx, iCol, &pVal);
       }
       assert( rc==SQLITE_OK );
-      if( sqlite3_value_type(pVal)!=eType ) return 0;
+      eValType = sqlite3_value_type(pVal);
+      if( eType==SQLITE_BLOB && eValType==SQLITE_NULL && pTab->bStat1 ){
+        int n;
+        a += sessionVarintGet(a, &n);
+        if( n!=0 ) return 0;
+        continue;
+      }
+      if( eValType!=eType ) return 0;
 
       /* A SessionChange object never has a NULL value in a PK column */
       assert( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT
@@ -1047,6 +1060,9 @@ static int sessionInitTable(sqlite3_session *pSession, SessionTable *pTab){
           break;
         }
       }
+      if( 0==sqlite3_stricmp("sqlite_stat1", pTab->zName) ){
+        pTab->bStat1 = 1;
+      }
     }
   }
   return (pSession->rc || pTab->abPK==0);
@@ -1093,7 +1109,7 @@ static void sessionPreupdateOneChange(
   rc = sessionPreupdateHash(pSession, pTab, op==SQLITE_INSERT, &iHash, &bNull);
   if( rc!=SQLITE_OK ) goto error_out;
 
-  if( bNull==0 ){
+  if( bNull==0 || pTab->bStat1 ){
     /* Search the hash table for an existing record for this row. */
     SessionChange *pC;
     for(pC=pTab->apChange[iHash]; pC; pC=pC->pNext){
@@ -1128,6 +1144,7 @@ static void sessionPreupdateOneChange(
         rc = sessionSerializeValue(0, p, &nByte);
         if( rc!=SQLITE_OK ) goto error_out;
       }
+      if( pTab->bStat1 ) nByte += 30;
   
       /* Allocate the change object */
       pChange = (SessionChange *)sqlite3_malloc(nByte);
@@ -1151,7 +1168,12 @@ static void sessionPreupdateOneChange(
         }else if( pTab->abPK[i] ){
           pSession->hook.xNew(pSession->hook.pCtx, i, &p);
         }
-        sessionSerializeValue(&pChange->aRecord[nByte], p, &nByte);
+        if( p && pTab->bStat1 && sqlite3_value_type(p)==SQLITE_NULL ){
+          pChange->aRecord[nByte++] = SQLITE_BLOB;
+          nByte += sessionVarintPut(&pChange->aRecord[nByte], 0);
+        }else{
+          sessionSerializeValue(&pChange->aRecord[nByte], p, &nByte);
+        }
       }
 
       /* Add the change to the hash-table */
@@ -2104,28 +2126,41 @@ static int sessionSelectStmt(
   sqlite3_stmt **ppStmt           /* OUT: Prepared SELECT statement */
 ){
   int rc = SQLITE_OK;
-  int i;
-  const char *zSep = "";
-  SessionBuffer buf = {0, 0, 0};
+  char *zSql = 0;
+  int nSql = -1;
 
-  sessionAppendStr(&buf, "SELECT * FROM ", &rc);
-  sessionAppendIdent(&buf, zDb, &rc);
-  sessionAppendStr(&buf, ".", &rc);
-  sessionAppendIdent(&buf, zTab, &rc);
-  sessionAppendStr(&buf, " WHERE ", &rc);
-  for(i=0; i<nCol; i++){
-    if( abPK[i] ){
-      sessionAppendStr(&buf, zSep, &rc);
-      sessionAppendIdent(&buf, azCol[i], &rc);
-      sessionAppendStr(&buf, " = ?", &rc);
-      sessionAppendInteger(&buf, i+1, &rc);
-      zSep = " AND ";
+  if( 0==sqlite3_stricmp("sqlite_stat1", zTab) ){
+    zSql = sqlite3_mprintf(
+        "SELECT tbl, ?2, stat FROM %Q.sqlite_stat1 WHERE tbl IS ?1 AND "
+        "idx IS (CASE WHEN ?2=X'' THEN NULL ELSE ?2 END)", zDb
+    );
+  }else{
+    int i;
+    const char *zSep = "";
+    SessionBuffer buf = {0, 0, 0};
+
+    sessionAppendStr(&buf, "SELECT * FROM ", &rc);
+    sessionAppendIdent(&buf, zDb, &rc);
+    sessionAppendStr(&buf, ".", &rc);
+    sessionAppendIdent(&buf, zTab, &rc);
+    sessionAppendStr(&buf, " WHERE ", &rc);
+    for(i=0; i<nCol; i++){
+      if( abPK[i] ){
+        sessionAppendStr(&buf, zSep, &rc);
+        sessionAppendIdent(&buf, azCol[i], &rc);
+        sessionAppendStr(&buf, " IS ?", &rc);
+        sessionAppendInteger(&buf, i+1, &rc);
+        zSep = " AND ";
+      }
     }
+    zSql = (char*)buf.aBuf;
+    nSql = buf.nBuf;
   }
+
   if( rc==SQLITE_OK ){
-    rc = sqlite3_prepare_v2(db, (char *)buf.aBuf, buf.nBuf, ppStmt, 0);
+    rc = sqlite3_prepare_v2(db, zSql, nSql, ppStmt, 0);
   }
-  sqlite3_free(buf.aBuf);
+  sqlite3_free(zSql);
   return rc;
 }
 
@@ -2153,7 +2188,7 @@ static int sessionSelectBind(
     switch( eType ){
       case 0:
       case SQLITE_NULL:
-        assert( abPK[i]==0 );
+        /* assert( abPK[i]==0 ); */
         break;
 
       case SQLITE_INTEGER: {
@@ -3464,6 +3499,7 @@ static int sessionUpdateRow(
   return rc;
 }
 
+
 /*
 ** Formulate and prepare an SQL statement to query table zTab by primary
 ** key. Assuming the following table structure:
@@ -3525,6 +3561,47 @@ static int sessionInsertRow(
   return rc;
 }
 
+static int sessionPrepare(sqlite3 *db, sqlite3_stmt **pp, const char *zSql){
+  return sqlite3_prepare_v2(db, zSql, -1, pp, 0);
+}
+
+/*
+** Prepare statements for applying changes to the sqlite_stat1 table.
+** These are similar to those created by sessionSelectRow(),
+** sessionInsertRow(), sessionUpdateRow() and sessionDeleteRow() for 
+** other tables.
+*/
+static int sessionStat1Sql(sqlite3 *db, SessionApplyCtx *p){
+  int rc = sessionSelectRow(db, "sqlite_stat1", p);
+  if( rc==SQLITE_OK ){
+    rc = sessionPrepare(db, &p->pInsert,
+        "INSERT INTO main.sqlite_stat1 VALUES(?1, "
+        "CASE WHEN length(?2)=0 AND typeof(?2)='blob' THEN NULL ELSE ?2 END, "
+        "?3)"
+    );
+  }
+  if( rc==SQLITE_OK ){
+    rc = sessionPrepare(db, &p->pUpdate,
+        "UPDATE main.sqlite_stat1 SET "
+        "tbl = CASE WHEN ?2 THEN ?3 ELSE tbl END, "
+        "idx = CASE WHEN ?5 THEN ?6 ELSE idx END, "
+        "stat = CASE WHEN ?8 THEN ?9 ELSE stat END  "
+        "WHERE tbl=?1 AND idx IS "
+        "CASE WHEN length(?4)=0 AND typeof(?4)='blob' THEN NULL ELSE ?4 END "
+        "AND (?10 OR ?8=0 OR stat IS ?7)"
+    );
+  }
+  if( rc==SQLITE_OK ){
+    rc = sessionPrepare(db, &p->pDelete,
+        "DELETE FROM main.sqlite_stat1 WHERE tbl=?1 AND idx IS "
+        "CASE WHEN length(?2)=0 AND typeof(?2)='blob' THEN NULL ELSE ?2 END "
+        "AND (?4 OR stat IS ?3)"
+    );
+  }
+  assert( rc==SQLITE_OK );
+  return rc;
+}
+
 /*
 ** A wrapper around sqlite3_bind_value() that detects an extra problem. 
 ** See comments in the body of this function for details.
@@ -4092,6 +4169,11 @@ static int sessionChangesetApply(
         }
         else{
           sApply.nCol = nCol;
+          if( 0==sqlite3_stricmp(zTab, "sqlite_stat1") ){
+            if( (rc = sessionStat1Sql(db, &sApply) ) ){
+              break;
+            }
+          }else
           if((rc = sessionSelectRow(db, zTab, &sApply))
           || (rc = sessionUpdateRow(db, zTab, &sApply))
           || (rc = sessionDeleteRow(db, zTab, &sApply))
diff --git a/manifest b/manifest
index 5e365cf00f..ce17059a62 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sproblem\scausing\sthe\ssessions\smodule\sto\soccasionally\slose\strack\sof\srows\nwith\scomposite\sprimary\skeys\swhen\sthere\sare\stwo\srows\swith\sthe\ssame\stext\svalue\nin\sthe\sleftmost\scolumn\sof\sthe\sPK.
-D 2018-01-17T17:38:18.448
+C Fix\sa\sproblem\sin\sthe\ssessions\smodule\swith\slogging\ssqlite_stat1\srows\sfor\swhich\n(idx\sIS\sNULL)\sis\strue.
+D 2018-01-17T20:57:20.602
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -398,9 +398,9 @@ F ext/session/sessionat.test efe88965e74ff1bc2af9c310b28358c02d420c1fb2705cc7a28
 F ext/session/sessiondiff.test ad13dd65664bae26744e1f18eb3cbd5588349b7e9118851d8f9364248d67bcec
 F ext/session/sessionfault.test da273f2712b6411e85e71465a1733b8501dbf6f7
 F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0
-F ext/session/sessionstat1.test e3a3f5876ce1526b48f6f447ee0b18960ac683e3fc891791e1ca0c08e823d498
+F ext/session/sessionstat1.test 16268a9bf62ab19c9bc9e41404bf7a13b5fd37c9cb6cf278a472f0c6c50d7ac1
 F ext/session/sessionwor.test 2f3744236dc8b170a695b7d8ddc8c743c7e79fdc
-F ext/session/sqlite3session.c c9d813ffa8db0670257da0a0c931e7188dfbb0010bf1a38274775c9b6300dd5c
+F ext/session/sqlite3session.c 870d142ec13c0e053139e7f3427aa550e59faf1775ae2bca051766089d9faf7b
 F ext/session/sqlite3session.h cb4d860101ba6d3ac810f18684539b766d24d668fa2436cdde90d711af9464fb
 F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386
 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
@@ -422,7 +422,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b
 F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786
 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a
 F src/alter.c cf7a8af45cb0ace672f47a1b29ab24092a9e8cd8d945a9974e3b5d925f548594
-F src/analyze.c f9bfffd0416c547a916cb96793b94684bdb0d26a71542ea31819c6757741c19d
+F src/analyze.c 6b42e36a5dcc2703a771f2411bd5e99524bd62c7ecde209bb88dfb04c72f046e
 F src/attach.c 84c477e856b24c2b9a0983b438a707c0cf4d616cee7a425401d418e58afec24c
 F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
@@ -1700,7 +1700,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a8aea925f8fde8f2dc5ff4b744d54aa2bf8916f3ee57f22d77fd1ddb5a35a9cc
-R fffe7fea955e558a2fcc7877bcdc39db
+P 09aed13678374bf22087cd808808b711dc703b7c18bc8aaf704850611e17f5cd
+R 99a2e08bb84b46a9c74125d0aa1dbedc
+T *branch * sessions-stat1
+T *sym-sessions-stat1 *
+T -sym-trunk *
 U dan
-Z 8da05c1c6303c10ad33f099cfa382df7
+Z 913b883400e9793131764656e068c1a3
diff --git a/manifest.uuid b/manifest.uuid
index 41ac115dac..a3bd20f4c8 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-09aed13678374bf22087cd808808b711dc703b7c18bc8aaf704850611e17f5cd
\ No newline at end of file
+25bf734be1b3883fccf12ac4d93d50289aa307fb60a52e0e32df12f7ee4edc7a
\ No newline at end of file
diff --git a/src/analyze.c b/src/analyze.c
index 85c603ffdf..0d13d77790 100644
--- a/src/analyze.c
+++ b/src/analyze.c
@@ -1309,6 +1309,9 @@ static void analyzeOneTable(
     sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regNewRowid);
     sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regTemp, regNewRowid);
     sqlite3VdbeChangeP5(v, OPFLAG_APPEND);
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+    sqlite3VdbeChangeP4(v, -1, (char*)pStat1, P4_TABLE);
+#endif
     sqlite3VdbeJumpHere(v, jZeroRows);
   }
 }

From 3dc9727782af8eea33c5b35ce16b6dbe675388c6 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 17 Jan 2018 21:14:17 +0000
Subject: [PATCH 349/488] Alternative implementation for the internal
 sqlite3Pow10() utility for MSVC, which is more accurate on that platform.

FossilOrigin-Name: 469b96be5350ba2291518280ffe179b87aa7fbe701e2813ef63843922771517a
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/util.c    | 21 +++++++++++++++++++++
 3 files changed, 29 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 5e365cf00f..9206bfe480 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sproblem\scausing\sthe\ssessions\smodule\sto\soccasionally\slose\strack\sof\srows\nwith\scomposite\sprimary\skeys\swhen\sthere\sare\stwo\srows\swith\sthe\ssame\stext\svalue\nin\sthe\sleftmost\scolumn\sof\sthe\sPK.
-D 2018-01-17T17:38:18.448
+C Alternative\simplementation\sfor\sthe\sinternal\ssqlite3Pow10()\sutility\sfor\sMSVC,\nwhich\sis\smore\saccurate\son\sthat\splatform.
+D 2018-01-17T21:14:17.292
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -553,7 +553,7 @@ F src/treeview.c eae35972ff44f67064de2eaf35f04afe94e7aea3271a8b3bcebb3f954880fec
 F src/trigger.c a34539c69433276d37b0da9a89c117726ff2d292c0902895af1f393a983cd3a1
 F src/update.c a90a32ffc0100265b0693dbbdbe490756447af181f5ea2c138cce515b08c8795
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
-F src/util.c 7315e97a8dc2c8e19ca64196c652cf0a65d13fd0a211b2cec082062372dc6261
+F src/util.c ef4a5f904d942e660abade7fbf3e6bdb402dabe9e7c27f3361ecf40b945538b5
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
 F src/vdbe.c ccc1e17a30325068ae4f0292e8601997946886d23acc989c68f2a261a2795c70
 F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a
@@ -1700,7 +1700,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a8aea925f8fde8f2dc5ff4b744d54aa2bf8916f3ee57f22d77fd1ddb5a35a9cc
-R fffe7fea955e558a2fcc7877bcdc39db
-U dan
-Z 8da05c1c6303c10ad33f099cfa382df7
+P 09aed13678374bf22087cd808808b711dc703b7c18bc8aaf704850611e17f5cd
+R aea01f6bc87a81119a2fde124523cf0a
+U drh
+Z 99d77d5f6e6371ac4335dbc304fd54d9
diff --git a/manifest.uuid b/manifest.uuid
index 41ac115dac..430019ccf5 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-09aed13678374bf22087cd808808b711dc703b7c18bc8aaf704850611e17f5cd
\ No newline at end of file
+469b96be5350ba2291518280ffe179b87aa7fbe701e2813ef63843922771517a
\ No newline at end of file
diff --git a/src/util.c b/src/util.c
index 75de4b3b30..456317cfa9 100644
--- a/src/util.c
+++ b/src/util.c
@@ -327,6 +327,26 @@ int sqlite3_strnicmp(const char *zLeft, const char *zRight, int N){
 ** This routine only works for values of E between 1 and 341.
 */
 static LONGDOUBLE_TYPE sqlite3Pow10(int E){
+#if defined(_MSC_VER)
+  static const LONGDOUBLE_TYPE x[] = {
+    1.0e+001,
+    1.0e+002,
+    1.0e+004,
+    1.0e+008,
+    1.0e+016,
+    1.0e+032,
+    1.0e+064,
+    1.0e+128,
+    1.0e+256
+  };
+  LONGDOUBLE_TYPE r = 1.0;
+  int i;
+  assert( E>=0 && E<=307 );
+  for(i=0; E!=0; i++, E >>=1){
+    if( E & 1 ) r *= x[i];
+  }
+  return r;
+#else
   LONGDOUBLE_TYPE x = 10.0;
   LONGDOUBLE_TYPE r = 1.0;
   while(1){
@@ -336,6 +356,7 @@ static LONGDOUBLE_TYPE sqlite3Pow10(int E){
     x *= x;
   }
   return r; 
+#endif
 }
 
 /*

From 1611e5a301fdda3843759c335f073142fc2dd271 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 18 Jan 2018 15:06:23 +0000
Subject: [PATCH 350/488] Simplify the sessions preupdate-hook logic for
 transforming NULL to X'' for column sqlite_stat1.idx.

FossilOrigin-Name: 089d7cecaaa47db58320b216a111a5e56123d022008be6c81bc0746148bbdb58
---
 ext/session/sqlite3session.c | 143 +++++++++++++++++++++++------------
 manifest                     |  15 ++--
 manifest.uuid                |   2 +-
 3 files changed, 103 insertions(+), 57 deletions(-)

diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c
index cbe75d5667..fa7dd677a1 100644
--- a/ext/session/sqlite3session.c
+++ b/ext/session/sqlite3session.c
@@ -46,6 +46,7 @@ struct sqlite3_session {
   int rc;                         /* Non-zero if an error has occurred */
   void *pFilterCtx;               /* First argument to pass to xTableFilter */
   int (*xTableFilter)(void *pCtx, const char *zTab);
+  sqlite3_value *pZeroBlob;       /* Value containing X'' */
   sqlite3_session *pNext;         /* Next session object on same db. */
   SessionTable *pTable;           /* List of attached tables */
   SessionHook hook;               /* APIs to grab new and old data with */
@@ -473,35 +474,32 @@ static int sessionPreupdateHash(
       if( rc!=SQLITE_OK ) return rc;
 
       eType = sqlite3_value_type(pVal);
-      if( pTab->bStat1 && eType==SQLITE_NULL ){
-        h = sessionHashAppendType(h, SQLITE_BLOB);
-      }else{
-        h = sessionHashAppendType(h, eType);
-        if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){
-          i64 iVal;
-          if( eType==SQLITE_INTEGER ){
-            iVal = sqlite3_value_int64(pVal);
-          }else{
-            double rVal = sqlite3_value_double(pVal);
-            assert( sizeof(iVal)==8 && sizeof(rVal)==8 );
-            memcpy(&iVal, &rVal, 8);
-          }
-          h = sessionHashAppendI64(h, iVal);
-        }else if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){
-          const u8 *z;
-          int n;
-          if( eType==SQLITE_TEXT ){
-            z = (const u8 *)sqlite3_value_text(pVal);
-          }else{
-            z = (const u8 *)sqlite3_value_blob(pVal);
-          }
-          n = sqlite3_value_bytes(pVal);
-          if( !z && (eType!=SQLITE_BLOB || n>0) ) return SQLITE_NOMEM;
-          h = sessionHashAppendBlob(h, n, z);
+      h = sessionHashAppendType(h, eType);
+      if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){
+        i64 iVal;
+        if( eType==SQLITE_INTEGER ){
+          iVal = sqlite3_value_int64(pVal);
         }else{
-          assert( eType==SQLITE_NULL );
-          *pbNullPK = 1;
+          double rVal = sqlite3_value_double(pVal);
+          assert( sizeof(iVal)==8 && sizeof(rVal)==8 );
+          memcpy(&iVal, &rVal, 8);
         }
+        h = sessionHashAppendI64(h, iVal);
+      }else if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){
+        const u8 *z;
+        int n;
+        if( eType==SQLITE_TEXT ){
+          z = (const u8 *)sqlite3_value_text(pVal);
+        }else{
+          z = (const u8 *)sqlite3_value_blob(pVal);
+        }
+        n = sqlite3_value_bytes(pVal);
+        if( !z && (eType!=SQLITE_BLOB || n>0) ) return SQLITE_NOMEM;
+        h = sessionHashAppendBlob(h, n, z);
+      }else{
+        assert( eType==SQLITE_NULL );
+        assert( pTab->bStat1==0 || i!=1 );
+        *pbNullPK = 1;
       }
     }
   }
@@ -555,7 +553,7 @@ static unsigned int sessionChangeHash(
          || eType==SQLITE_TEXT || eType==SQLITE_BLOB 
          || eType==SQLITE_NULL || eType==0 
     );
-    assert( !isPK || (eType!=0 && (pTab->bStat1 || eType!=SQLITE_NULL)) );
+    assert( !isPK || (eType!=0 && eType!=SQLITE_NULL) );
 
     if( isPK ){
       a++;
@@ -563,7 +561,7 @@ static unsigned int sessionChangeHash(
       if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){
         h = sessionHashAppendI64(h, sessionGetI64(a));
         a += 8;
-      }else if( eType!=SQLITE_NULL ){
+      }else{
         int n; 
         a += sessionVarintGet(a, &n);
         h = sessionHashAppendBlob(h, n, a);
@@ -799,7 +797,6 @@ static int sessionPreupdateEqual(
       sqlite3_value *pVal;        /* Value returned by preupdate_new/old */
       int rc;                     /* Error code from preupdate_new/old */
       int eType = *a++;           /* Type of value from change record */
-      int eValType;
 
       /* The following calls to preupdate_new() and preupdate_old() can not
       ** fail. This is because they cache their return values, and by the
@@ -814,14 +811,7 @@ static int sessionPreupdateEqual(
         rc = pSession->hook.xOld(pSession->hook.pCtx, iCol, &pVal);
       }
       assert( rc==SQLITE_OK );
-      eValType = sqlite3_value_type(pVal);
-      if( eType==SQLITE_BLOB && eValType==SQLITE_NULL && pTab->bStat1 ){
-        int n;
-        a += sessionVarintGet(a, &n);
-        if( n!=0 ) return 0;
-        continue;
-      }
-      if( eValType!=eType ) return 0;
+      if( sqlite3_value_type(pVal)!=eType ) return 0;
 
       /* A SessionChange object never has a NULL value in a PK column */
       assert( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT
@@ -1068,6 +1058,47 @@ static int sessionInitTable(sqlite3_session *pSession, SessionTable *pTab){
   return (pSession->rc || pTab->abPK==0);
 }
 
+/*
+** Versions of the four methods in object SessionHook for use with the
+** sqlite_stat1 table. The purpose of this is to substitute a zero-length
+** blob each time a NULL value is read from the "idx" column of the
+** sqlite_stat1 table.
+*/
+typedef struct SessionStat1Ctx SessionStat1Ctx;
+struct SessionStat1Ctx {
+  SessionHook hook;
+  sqlite3_session *pSession;
+};
+static int sessionStat1Old(void *pCtx, int iCol, sqlite3_value **ppVal){
+  SessionStat1Ctx *p = (SessionStat1Ctx*)pCtx;
+  sqlite3_value *pVal = 0;
+  int rc = p->hook.xOld(p->hook.pCtx, iCol, &pVal);
+  if( rc==SQLITE_OK && iCol==1 && sqlite3_value_type(pVal)==SQLITE_NULL ){
+    pVal = p->pSession->pZeroBlob;
+  }
+  *ppVal = pVal;
+  return rc;
+}
+static int sessionStat1New(void *pCtx, int iCol, sqlite3_value **ppVal){
+  SessionStat1Ctx *p = (SessionStat1Ctx*)pCtx;
+  sqlite3_value *pVal = 0;
+  int rc = p->hook.xNew(p->hook.pCtx, iCol, &pVal);
+  if( rc==SQLITE_OK && iCol==1 && sqlite3_value_type(pVal)==SQLITE_NULL ){
+    pVal = p->pSession->pZeroBlob;
+  }
+  *ppVal = pVal;
+  return rc;
+}
+static int sessionStat1Count(void *pCtx){
+  SessionStat1Ctx *p = (SessionStat1Ctx*)pCtx;
+  return p->hook.xCount(p->hook.pCtx);
+}
+static int sessionStat1Depth(void *pCtx){
+  SessionStat1Ctx *p = (SessionStat1Ctx*)pCtx;
+  return p->hook.xDepth(p->hook.pCtx);
+}
+
+
 /*
 ** This function is only called from with a pre-update-hook reporting a 
 ** change on table pTab (attached to session pSession). The type of change
@@ -1084,6 +1115,7 @@ static void sessionPreupdateOneChange(
   int iHash; 
   int bNull = 0; 
   int rc = SQLITE_OK;
+  SessionStat1Ctx stat1;
 
   if( pSession->rc ) return;
 
@@ -1103,13 +1135,32 @@ static void sessionPreupdateOneChange(
     return;
   }
 
+  if( pTab->bStat1 ){
+    stat1.hook = pSession->hook;
+    stat1.pSession = pSession;
+    pSession->hook.pCtx = (void*)&stat1;
+    pSession->hook.xNew = sessionStat1New;
+    pSession->hook.xOld = sessionStat1Old;
+    pSession->hook.xCount = sessionStat1Count;
+    pSession->hook.xDepth = sessionStat1Depth;
+    if( pSession->pZeroBlob==0 ){
+      sqlite3_value *p = sqlite3ValueNew(0);
+      if( p==0 ){
+        rc = SQLITE_NOMEM;
+        goto error_out;
+      }
+      sqlite3ValueSetStr(p, 0, "", 0, SQLITE_STATIC);
+      pSession->pZeroBlob = p;
+    }
+  }
+
   /* Calculate the hash-key for this change. If the primary key of the row
   ** includes a NULL value, exit early. Such changes are ignored by the
   ** session module. */
   rc = sessionPreupdateHash(pSession, pTab, op==SQLITE_INSERT, &iHash, &bNull);
   if( rc!=SQLITE_OK ) goto error_out;
 
-  if( bNull==0 || pTab->bStat1 ){
+  if( bNull==0 ){
     /* Search the hash table for an existing record for this row. */
     SessionChange *pC;
     for(pC=pTab->apChange[iHash]; pC; pC=pC->pNext){
@@ -1144,7 +1195,6 @@ static void sessionPreupdateOneChange(
         rc = sessionSerializeValue(0, p, &nByte);
         if( rc!=SQLITE_OK ) goto error_out;
       }
-      if( pTab->bStat1 ) nByte += 30;
   
       /* Allocate the change object */
       pChange = (SessionChange *)sqlite3_malloc(nByte);
@@ -1168,12 +1218,7 @@ static void sessionPreupdateOneChange(
         }else if( pTab->abPK[i] ){
           pSession->hook.xNew(pSession->hook.pCtx, i, &p);
         }
-        if( p && pTab->bStat1 && sqlite3_value_type(p)==SQLITE_NULL ){
-          pChange->aRecord[nByte++] = SQLITE_BLOB;
-          nByte += sessionVarintPut(&pChange->aRecord[nByte], 0);
-        }else{
-          sessionSerializeValue(&pChange->aRecord[nByte], p, &nByte);
-        }
+        sessionSerializeValue(&pChange->aRecord[nByte], p, &nByte);
       }
 
       /* Add the change to the hash-table */
@@ -1198,6 +1243,9 @@ static void sessionPreupdateOneChange(
 
   /* If an error has occurred, mark the session object as failed. */
  error_out:
+  if( pTab->bStat1 ){
+    pSession->hook = stat1.hook;
+  }
   if( rc!=SQLITE_OK ){
     pSession->rc = rc;
   }
@@ -1659,6 +1707,7 @@ void sqlite3session_delete(sqlite3_session *pSession){
     }
   }
   sqlite3_mutex_leave(sqlite3_db_mutex(db));
+  sqlite3ValueFree(pSession->pZeroBlob);
 
   /* Delete all attached table objects. And the contents of their 
   ** associated hash-tables. */
@@ -2188,7 +2237,7 @@ static int sessionSelectBind(
     switch( eType ){
       case 0:
       case SQLITE_NULL:
-        /* assert( abPK[i]==0 ); */
+        assert( abPK[i]==0 );
         break;
 
       case SQLITE_INTEGER: {
diff --git a/manifest b/manifest
index ce17059a62..8c535b9409 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sproblem\sin\sthe\ssessions\smodule\swith\slogging\ssqlite_stat1\srows\sfor\swhich\n(idx\sIS\sNULL)\sis\strue.
-D 2018-01-17T20:57:20.602
+C Simplify\sthe\ssessions\spreupdate-hook\slogic\sfor\stransforming\sNULL\sto\sX''\sfor\ncolumn\ssqlite_stat1.idx.
+D 2018-01-18T15:06:23.750
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -400,7 +400,7 @@ F ext/session/sessionfault.test da273f2712b6411e85e71465a1733b8501dbf6f7
 F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0
 F ext/session/sessionstat1.test 16268a9bf62ab19c9bc9e41404bf7a13b5fd37c9cb6cf278a472f0c6c50d7ac1
 F ext/session/sessionwor.test 2f3744236dc8b170a695b7d8ddc8c743c7e79fdc
-F ext/session/sqlite3session.c 870d142ec13c0e053139e7f3427aa550e59faf1775ae2bca051766089d9faf7b
+F ext/session/sqlite3session.c 55f20fa4a9b6acc338e11867439633c7faa3d18809cdd826fde6abf9a75dd7a0
 F ext/session/sqlite3session.h cb4d860101ba6d3ac810f18684539b766d24d668fa2436cdde90d711af9464fb
 F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386
 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
@@ -1700,10 +1700,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 09aed13678374bf22087cd808808b711dc703b7c18bc8aaf704850611e17f5cd
-R 99a2e08bb84b46a9c74125d0aa1dbedc
-T *branch * sessions-stat1
-T *sym-sessions-stat1 *
-T -sym-trunk *
+P 25bf734be1b3883fccf12ac4d93d50289aa307fb60a52e0e32df12f7ee4edc7a
+R f7c998dd2622eae2189b630c53aa0780
 U dan
-Z 913b883400e9793131764656e068c1a3
+Z 3f8bd6ff445f731b364c070e53a9b8ca
diff --git a/manifest.uuid b/manifest.uuid
index a3bd20f4c8..0823a466b2 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-25bf734be1b3883fccf12ac4d93d50289aa307fb60a52e0e32df12f7ee4edc7a
\ No newline at end of file
+089d7cecaaa47db58320b216a111a5e56123d022008be6c81bc0746148bbdb58
\ No newline at end of file

From d1cccf19b5fd00ed45613d4f14dde59024942158 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 18 Jan 2018 16:42:11 +0000
Subject: [PATCH 351/488] Fix sessions module conflict handling for the
 sqlite_stat1 table.

FossilOrigin-Name: f05ee74e05c401eb075a1ba65179662a08a5c3d5b55fc81f2acc841e841dd055
---
 ext/session/sessionstat1.test | 77 +++++++++++++++++++++++++++++++++++
 ext/session/sqlite3session.c  | 41 +++++++++++++------
 manifest                      | 14 +++----
 manifest.uuid                 |  2 +-
 4 files changed, 114 insertions(+), 20 deletions(-)

diff --git a/ext/session/sessionstat1.test b/ext/session/sessionstat1.test
index 22b2de8dbc..082c02bcf1 100644
--- a/ext/session/sessionstat1.test
+++ b/ext/session/sessionstat1.test
@@ -226,9 +226,86 @@ do_scc_test 3.7.1 {
   {INSERT sqlite_stat1 0 XX. {} {t t2 b {} t 6}}
 }
 
+#-------------------------------------------------------------------------
+catch { db2 close }
+reset_db
+forcedelete test.db2
+sqlite3 db2 test.db2
 
+do_test 4.1.0 {
+  do_common_sql {
+    CREATE TABLE t1(a, b);
+    CREATE INDEX i1 ON t1(a);
+    CREATE INDEX i2 ON t1(b);
+    INSERT INTO t1 VALUES(1,1), (2,2);
+    ANALYZE;
+  }
+  execsql { DELETE FROM sqlite_stat1 }
+} {}
 
+do_test 4.1.1 {
+  execsql { INSERT INTO t1 VALUES(3,3); }
+  set C [changeset_from_sql {ANALYZE}]
+  set ::c [list]
+  proc xConflict {args} {
+    lappend ::c $args
+    return "OMIT"
+  }
+  sqlite3changeset_apply db2 $C xConflict
+  set ::c
+} [list {*}{
+  {INSERT sqlite_stat1 CONFLICT {t t1 t i1 t {3 1}} {t t1 t i1 t {2 1}}}
+  {INSERT sqlite_stat1 CONFLICT {t t1 t i2 t {3 1}} {t t1 t i2 t {2 1}}}
+}]
 
+do_execsql_test -db db2 4.1.2 {
+  SELECT * FROM sqlite_stat1 ORDER BY 1,2;
+} {t1 i1 {2 1} t1 i2 {2 1}}
+
+do_test 4.1.3 {
+  proc xConflict {args} {
+    return "REPLACE"
+  }
+  sqlite3changeset_apply db2 $C xConflict
+  execsql { SELECT * FROM sqlite_stat1 ORDER BY 1,2 } db2
+} {t1 i1 {3 1} t1 i2 {3 1}}
+
+do_test 4.2.0 {
+  do_common_sql { 
+    DROP TABLE t1;
+    CREATE TABLE t3(x,y);
+    INSERT INTO t3 VALUES('a','a');
+    INSERT INTO t3 VALUES('b','b');
+    ANALYZE;
+  }
+  execsql { DELETE FROM sqlite_stat1 }
+} {}
+do_test 4.2.1 {
+  execsql { INSERT INTO t3 VALUES('c','c'); }
+  set C [changeset_from_sql {ANALYZE}]
+  set ::c [list]
+  proc xConflict {args} {
+    lappend ::c $args
+    return "OMIT"
+  }
+  sqlite3changeset_apply db2 $C xConflict
+  set ::c
+} [list {*}{
+  {INSERT sqlite_stat1 CONFLICT {t t3 b {} t 3} {t t3 b {} t 2}}
+}]
+
+db2 null null
+do_execsql_test -db db2 4.2.2 {
+  SELECT * FROM sqlite_stat1 ORDER BY 1,2;
+} {t3 null 2}
+
+do_test 4.2.3 {
+  proc xConflict {args} {
+    return "REPLACE"
+  }
+  sqlite3changeset_apply db2 $C xConflict
+  execsql { SELECT * FROM sqlite_stat1 ORDER BY 1,2 } db2
+} {t3 null 3}
 
 finish_test
 
diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c
index fa7dd677a1..ec1a93e08e 100644
--- a/ext/session/sqlite3session.c
+++ b/ext/session/sqlite3session.c
@@ -3378,7 +3378,7 @@ struct SessionApplyCtx {
   int nCol;                       /* Size of azCol[] and abPK[] arrays */
   const char **azCol;             /* Array of column names */
   u8 *abPK;                       /* Boolean array - true if column is in PK */
-
+  int bStat1;                     /* True if table is sqlite_stat1 */
   int bDeferConstraints;          /* True to defer constraints */
   SessionBuffer constraints;      /* Deferred constraints are stored here */
 };
@@ -3981,11 +3981,25 @@ static int sessionApplyOneOp(
 
   }else{
     assert( op==SQLITE_INSERT );
-    rc = sessionBindRow(pIter, sqlite3changeset_new, nCol, 0, p->pInsert);
-    if( rc!=SQLITE_OK ) return rc;
+    if( p->bStat1 ){
+      /* Check if there is a conflicting row. For sqlite_stat1, this needs
+      ** to be done using a SELECT, as there is no PRIMARY KEY in the 
+      ** database schema to throw an exception if a duplicate is inserted.  */
+      rc = sessionSeekToRow(p->db, pIter, p->abPK, p->pSelect);
+      if( rc==SQLITE_ROW ){
+        rc = SQLITE_CONSTRAINT;
+        sqlite3_reset(p->pSelect);
+      }
+    }
+
+    if( rc==SQLITE_OK ){
+      rc = sessionBindRow(pIter, sqlite3changeset_new, nCol, 0, p->pInsert);
+      if( rc!=SQLITE_OK ) return rc;
+
+      sqlite3_step(p->pInsert);
+      rc = sqlite3_reset(p->pInsert);
+    }
 
-    sqlite3_step(p->pInsert);
-    rc = sqlite3_reset(p->pInsert);
     if( (rc&0xff)==SQLITE_CONSTRAINT ){
       rc = sessionConflictHandler(
           SQLITE_CHANGESET_CONFLICT, p, pIter, xConflict, pCtx, pbReplace
@@ -4222,13 +4236,16 @@ static int sessionChangesetApply(
             if( (rc = sessionStat1Sql(db, &sApply) ) ){
               break;
             }
-          }else
-          if((rc = sessionSelectRow(db, zTab, &sApply))
-          || (rc = sessionUpdateRow(db, zTab, &sApply))
-          || (rc = sessionDeleteRow(db, zTab, &sApply))
-          || (rc = sessionInsertRow(db, zTab, &sApply))
-          ){
-            break;
+            sApply.bStat1 = 1;
+          }else{
+            if((rc = sessionSelectRow(db, zTab, &sApply))
+                || (rc = sessionUpdateRow(db, zTab, &sApply))
+                || (rc = sessionDeleteRow(db, zTab, &sApply))
+                || (rc = sessionInsertRow(db, zTab, &sApply))
+              ){
+              break;
+            }
+            sApply.bStat1 = 0;
           }
         }
         nTab = sqlite3Strlen30(zTab);
diff --git a/manifest b/manifest
index 8c535b9409..1dd8d9aec3 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Simplify\sthe\ssessions\spreupdate-hook\slogic\sfor\stransforming\sNULL\sto\sX''\sfor\ncolumn\ssqlite_stat1.idx.
-D 2018-01-18T15:06:23.750
+C Fix\ssessions\smodule\sconflict\shandling\sfor\sthe\ssqlite_stat1\stable.
+D 2018-01-18T16:42:11.152
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -398,9 +398,9 @@ F ext/session/sessionat.test efe88965e74ff1bc2af9c310b28358c02d420c1fb2705cc7a28
 F ext/session/sessiondiff.test ad13dd65664bae26744e1f18eb3cbd5588349b7e9118851d8f9364248d67bcec
 F ext/session/sessionfault.test da273f2712b6411e85e71465a1733b8501dbf6f7
 F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0
-F ext/session/sessionstat1.test 16268a9bf62ab19c9bc9e41404bf7a13b5fd37c9cb6cf278a472f0c6c50d7ac1
+F ext/session/sessionstat1.test 41cd97c2e48619a41cdf8ae749e1b25f34719de638689221aa43971be693bf4e
 F ext/session/sessionwor.test 2f3744236dc8b170a695b7d8ddc8c743c7e79fdc
-F ext/session/sqlite3session.c 55f20fa4a9b6acc338e11867439633c7faa3d18809cdd826fde6abf9a75dd7a0
+F ext/session/sqlite3session.c 989466bba4dff0ede8d4c450b1fc65ca222b87e31193eddbf3931b88bf898a57
 F ext/session/sqlite3session.h cb4d860101ba6d3ac810f18684539b766d24d668fa2436cdde90d711af9464fb
 F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386
 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
@@ -1700,7 +1700,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 25bf734be1b3883fccf12ac4d93d50289aa307fb60a52e0e32df12f7ee4edc7a
-R f7c998dd2622eae2189b630c53aa0780
+P 089d7cecaaa47db58320b216a111a5e56123d022008be6c81bc0746148bbdb58
+R b26168110a43028fdb5773dcda862810
 U dan
-Z 3f8bd6ff445f731b364c070e53a9b8ca
+Z 3667003c7f66cdcf6e4cbccaf82894fb
diff --git a/manifest.uuid b/manifest.uuid
index 0823a466b2..fd362a9c44 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-089d7cecaaa47db58320b216a111a5e56123d022008be6c81bc0746148bbdb58
\ No newline at end of file
+f05ee74e05c401eb075a1ba65179662a08a5c3d5b55fc81f2acc841e841dd055
\ No newline at end of file

From 4b3931ef336a3bbc7cad9290827eb5524b099f63 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 18 Jan 2018 16:52:35 +0000
Subject: [PATCH 352/488] Fix to the documentation for sqlite3_trace_v2().  No
 changes to code.

FossilOrigin-Name: 6fbd0a11a66f8eb4d7820cb49c23bdcb917db98a22c29d76edea1eea6dab0a4e
---
 manifest        | 12 ++++++------
 manifest.uuid   |  2 +-
 src/sqlite.h.in |  4 ++--
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index 9206bfe480..d77626ecd2 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Alternative\simplementation\sfor\sthe\sinternal\ssqlite3Pow10()\sutility\sfor\sMSVC,\nwhich\sis\smore\saccurate\son\sthat\splatform.
-D 2018-01-17T21:14:17.292
+C Fix\sto\sthe\sdocumentation\sfor\ssqlite3_trace_v2().\s\sNo\schanges\sto\scode.
+D 2018-01-18T16:52:35.238
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -488,7 +488,7 @@ F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c bebe7cce45d899d2237c76bce059d525abf5b861f2fce92f6b53914a961c01ba
 F src/shell.c.in 4e1bcf8c70b8fb97c7cbaca6602e2a291d7fe17eff23a5de003d6fabd87f27d1
-F src/sqlite.h.in 840e8aee0d7f8592bd69730797582b342a413eceaff7a220fb5f9db9c412cea9
+F src/sqlite.h.in 959deaad89679e31d7f68fda668b0c5d1f592fffed7a9c1740fb8ded4e4e754a
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 99189e7611eb0bf98f21c7835dc74730a84e2e809c98e1e31c33896dee7a2849
 F src/sqliteInt.h 9c70315598b34810a83e4894455acb18e95cf63ce4e6cbb451ac2d17eabc2544
@@ -1700,7 +1700,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 09aed13678374bf22087cd808808b711dc703b7c18bc8aaf704850611e17f5cd
-R aea01f6bc87a81119a2fde124523cf0a
+P 469b96be5350ba2291518280ffe179b87aa7fbe701e2813ef63843922771517a
+R 2b93eecc1bfa73dda9a58b538de6c0af
 U drh
-Z 99d77d5f6e6371ac4335dbc304fd54d9
+Z f64363038b04686cc1ed3a7aa13703ce
diff --git a/manifest.uuid b/manifest.uuid
index 430019ccf5..9ea54531fe 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-469b96be5350ba2291518280ffe179b87aa7fbe701e2813ef63843922771517a
\ No newline at end of file
+6fbd0a11a66f8eb4d7820cb49c23bdcb917db98a22c29d76edea1eea6dab0a4e
\ No newline at end of file
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 1f7f6a0792..ed1c24de4c 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -2951,8 +2951,8 @@ SQLITE_DEPRECATED void *sqlite3_profile(sqlite3*,
 ** KEYWORDS: SQLITE_TRACE
 **
 ** These constants identify classes of events that can be monitored
-** using the [sqlite3_trace_v2()] tracing logic.  The third argument
-** to [sqlite3_trace_v2()] is an OR-ed combination of one or more of
+** using the [sqlite3_trace_v2()] tracing logic.  The M argument
+** to [sqlite3_trace_v2(D,M,X,P)] is an OR-ed combination of one or more of
 ** the following constants.  ^The first argument to the trace callback
 ** is one of the following constants.
 **

From e3ca3831bae7ae6f7a7ba2d858f8bf95d6e1af83 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 18 Jan 2018 16:53:41 +0000
Subject: [PATCH 353/488] Add comments describing the special sqlite_stat1
 handling to sqlite3session.h.

FossilOrigin-Name: 4431a3256f7436e34b3c33edc1f3e53df7eb3c87daec9bac8d038895d93ca7f2
---
 ext/session/sqlite3session.h | 24 ++++++++++++++++++++++++
 manifest                     | 12 ++++++------
 manifest.uuid                |  2 +-
 3 files changed, 31 insertions(+), 7 deletions(-)

diff --git a/ext/session/sqlite3session.h b/ext/session/sqlite3session.h
index b22df129cb..23d8e689fc 100644
--- a/ext/session/sqlite3session.h
+++ b/ext/session/sqlite3session.h
@@ -147,6 +147,30 @@ int sqlite3session_indirect(sqlite3_session *pSession, int bIndirect);
 **
 ** SQLITE_OK is returned if the call completes without error. Or, if an error 
 ** occurs, an SQLite error code (e.g. SQLITE_NOMEM) is returned.
+**
+** <h3>Special sqlite_stat1 Handling</h3>
+**
+** The "sqlite_stat1" table is an exception to some of the rules above. In
+** SQLite, the schema of sqlite_stat1 is:
+**  <pre>
+**  &nbsp;     CREATE TABLE sqlite_stat1(tbl,idx,stat)  
+**  </pre>
+**
+** Even though sqlite_stat1 does not have a PRIMARY KEY, changes are 
+** recorded for it as if the PRIMARY KEY is (tbl,idx). Additionally, changes 
+** are recorded for rows for which (idx IS NULL) is true. However, for such
+** rows a zero-length blob (SQL value X'') is stored in the changeset or
+** patchset instead of a NULL value. This allows such changesets to be
+** manipulated by legacy implementations of sqlite3changeset_invert(),
+** concat() and similar.
+**
+** The sqlite3changeset_apply() function automatically converts the 
+** zero-length blob back to a NULL value when updating the sqlite_stat1
+** table. However, if the application calls sqlite3changeset_new(),
+** sqlite3changeset_old() or sqlite3changeset_conflict on a changeset 
+** iterator directly (including on a changeset iterator passed to a
+** conflict-handler callback) then the X'' value is returned. The application
+** must translate X'' to NULL itself if required.
 */
 int sqlite3session_attach(
   sqlite3_session *pSession,      /* Session object */
diff --git a/manifest b/manifest
index 1dd8d9aec3..5cb2ef0657 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\ssessions\smodule\sconflict\shandling\sfor\sthe\ssqlite_stat1\stable.
-D 2018-01-18T16:42:11.152
+C Add\scomments\sdescribing\sthe\sspecial\ssqlite_stat1\shandling\sto\ssqlite3session.h.
+D 2018-01-18T16:53:41.289
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -401,7 +401,7 @@ F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0
 F ext/session/sessionstat1.test 41cd97c2e48619a41cdf8ae749e1b25f34719de638689221aa43971be693bf4e
 F ext/session/sessionwor.test 2f3744236dc8b170a695b7d8ddc8c743c7e79fdc
 F ext/session/sqlite3session.c 989466bba4dff0ede8d4c450b1fc65ca222b87e31193eddbf3931b88bf898a57
-F ext/session/sqlite3session.h cb4d860101ba6d3ac810f18684539b766d24d668fa2436cdde90d711af9464fb
+F ext/session/sqlite3session.h 1bf32a9598ae4288bf7ecbda032a0479d6225ab8d2283fd252056a26b1046946
 F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386
 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04
@@ -1700,7 +1700,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 089d7cecaaa47db58320b216a111a5e56123d022008be6c81bc0746148bbdb58
-R b26168110a43028fdb5773dcda862810
+P f05ee74e05c401eb075a1ba65179662a08a5c3d5b55fc81f2acc841e841dd055
+R 62c81de92f4cf24f6460f153aeb30b06
 U dan
-Z 3667003c7f66cdcf6e4cbccaf82894fb
+Z 4f9e979274bbe8dc16dfd6dd12c05029
diff --git a/manifest.uuid b/manifest.uuid
index fd362a9c44..699a6f987c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f05ee74e05c401eb075a1ba65179662a08a5c3d5b55fc81f2acc841e841dd055
\ No newline at end of file
+4431a3256f7436e34b3c33edc1f3e53df7eb3c87daec9bac8d038895d93ca7f2
\ No newline at end of file

From cae5b9feacb655ede761d15cc00d37b82e35b5b2 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 18 Jan 2018 16:56:19 +0000
Subject: [PATCH 354/488] Clarify the handling of the sqlite_stat1 table by
 legacy versions of the sessions module.

FossilOrigin-Name: dc7c48cb4126db9e25c73512cc743155293fe1c4c2516f8c84102228695b6e70
---
 ext/session/sqlite3session.h |  9 +++++++--
 manifest                     | 12 ++++++------
 manifest.uuid                |  2 +-
 3 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/ext/session/sqlite3session.h b/ext/session/sqlite3session.h
index 23d8e689fc..9f33855df0 100644
--- a/ext/session/sqlite3session.h
+++ b/ext/session/sqlite3session.h
@@ -150,8 +150,8 @@ int sqlite3session_indirect(sqlite3_session *pSession, int bIndirect);
 **
 ** <h3>Special sqlite_stat1 Handling</h3>
 **
-** The "sqlite_stat1" table is an exception to some of the rules above. In
-** SQLite, the schema of sqlite_stat1 is:
+** As of SQLite version 3.22.0, the "sqlite_stat1" table is an exception to 
+** some of the rules above. In SQLite, the schema of sqlite_stat1 is:
 **  <pre>
 **  &nbsp;     CREATE TABLE sqlite_stat1(tbl,idx,stat)  
 **  </pre>
@@ -171,6 +171,11 @@ int sqlite3session_indirect(sqlite3_session *pSession, int bIndirect);
 ** iterator directly (including on a changeset iterator passed to a
 ** conflict-handler callback) then the X'' value is returned. The application
 ** must translate X'' to NULL itself if required.
+**
+** Legacy (older than 3.22.0) versions of the sessions module cannot capture
+** changes made to the sqlite_stat1 table. Legacy versions of the
+** sqlite3changeset_apply() function silently ignore any modifications to the
+** sqlite_stat1 table that are part of a changeset or patchset.
 */
 int sqlite3session_attach(
   sqlite3_session *pSession,      /* Session object */
diff --git a/manifest b/manifest
index 5cb2ef0657..44a95be5ce 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\scomments\sdescribing\sthe\sspecial\ssqlite_stat1\shandling\sto\ssqlite3session.h.
-D 2018-01-18T16:53:41.289
+C Clarify\sthe\shandling\sof\sthe\ssqlite_stat1\stable\sby\slegacy\sversions\sof\sthe\nsessions\smodule.
+D 2018-01-18T16:56:19.115
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -401,7 +401,7 @@ F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0
 F ext/session/sessionstat1.test 41cd97c2e48619a41cdf8ae749e1b25f34719de638689221aa43971be693bf4e
 F ext/session/sessionwor.test 2f3744236dc8b170a695b7d8ddc8c743c7e79fdc
 F ext/session/sqlite3session.c 989466bba4dff0ede8d4c450b1fc65ca222b87e31193eddbf3931b88bf898a57
-F ext/session/sqlite3session.h 1bf32a9598ae4288bf7ecbda032a0479d6225ab8d2283fd252056a26b1046946
+F ext/session/sqlite3session.h 01774161cbd328fe3d496323655b9cc142317ff1fb1ae15c1232075ea240e3a4
 F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386
 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04
@@ -1700,7 +1700,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P f05ee74e05c401eb075a1ba65179662a08a5c3d5b55fc81f2acc841e841dd055
-R 62c81de92f4cf24f6460f153aeb30b06
+P 4431a3256f7436e34b3c33edc1f3e53df7eb3c87daec9bac8d038895d93ca7f2
+R 6488443bbec455d1afdebff112c6cb1b
 U dan
-Z 4f9e979274bbe8dc16dfd6dd12c05029
+Z 8531e6daca6e4a95660fec10c24747c3
diff --git a/manifest.uuid b/manifest.uuid
index 699a6f987c..ecf7bc63d6 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4431a3256f7436e34b3c33edc1f3e53df7eb3c87daec9bac8d038895d93ca7f2
\ No newline at end of file
+dc7c48cb4126db9e25c73512cc743155293fe1c4c2516f8c84102228695b6e70
\ No newline at end of file

From 80db2d65212b1d029ea042e7d6350ac313b3b096 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 18 Jan 2018 17:09:26 +0000
Subject: [PATCH 355/488] Update the autoconf configure.ac script and
 Makefile.am templates so that ZLIB is automatically detected and used.

FossilOrigin-Name: 41bfb6b8d61699d09a7e67d2289149abfbb9ce8e75e6ff8560546cad0d2e3f2b
---
 autoconf/Makefile.am  |  2 +-
 autoconf/configure.ac |  3 +++
 manifest              | 17 ++++++++---------
 manifest.uuid         |  2 +-
 4 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/autoconf/Makefile.am b/autoconf/Makefile.am
index 8c046f822d..0f0706873a 100644
--- a/autoconf/Makefile.am
+++ b/autoconf/Makefile.am
@@ -1,5 +1,5 @@
 
-AM_CFLAGS = @THREADSAFE_FLAGS@ @DYNAMIC_EXTENSION_FLAGS@ @FTS5_FLAGS@ @JSON1_FLAGS@ @SESSION_FLAGS@ -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE
+AM_CFLAGS = @THREADSAFE_FLAGS@ @DYNAMIC_EXTENSION_FLAGS@ @FTS5_FLAGS@ @JSON1_FLAGS@ @ZLIB_FLAGS@ @SESSION_FLAGS@ -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE
 
 lib_LTLIBRARIES = libsqlite3.la
 libsqlite3_la_SOURCES = sqlite3.c
diff --git a/autoconf/configure.ac b/autoconf/configure.ac
index c207bdf74d..c1e28043d8 100644
--- a/autoconf/configure.ac
+++ b/autoconf/configure.ac
@@ -164,6 +164,9 @@ AC_SUBST(EXTRA_SHELL_OBJ)
 #-----------------------------------------------------------------------
 
 AC_CHECK_FUNCS(posix_fallocate)
+AC_CHECK_HEADERS(zlib.h)
+AC_SEARCH_LIBS(deflate,z,[ZLIB_FLAGS="-DSQLITE_HAVE_ZLIB"])
+AC_SUBST(ZLIB_FLAGS)
 
 #-----------------------------------------------------------------------
 # UPDATE: Maybe it's better if users just set CFLAGS before invoking
diff --git a/manifest b/manifest
index 3cd31bd5cc..44d3a7e510 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\ssessions\smodule\shandling\sof\ssqlite_stat1\srows\swith\s(idx\sIS\sNULL).
-D 2018-01-18T16:59:52.652
+C Update\sthe\sautoconf\sconfigure.ac\sscript\sand\sMakefile.am\stemplates\sso\sthat\nZLIB\sis\sautomatically\sdetected\sand\sused.
+D 2018-01-18T17:09:26.634
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -12,11 +12,11 @@ F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2
 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90
 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2
 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903
-F autoconf/Makefile.am 66c0befa511f0d95ba229e180067cf0357a9ebf8b3201b06d683c5ba6220fb39
+F autoconf/Makefile.am 6cca3f797c649b40c762484ce26491839fec54de72d376d774969e76ed13931f
 F autoconf/Makefile.msc 2c50a59319af7da4eaca8c13e3240881b1bc245fd175845a055faab7d03d6e67
 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7
 F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1
-F autoconf/configure.ac 8dd08ca564279fff091c9bfdd2599d8f992c9f1f70c5396de2126ad2bd1b3bed
+F autoconf/configure.ac c610cbec367dfdcbbe1341f445699700a0f9e157150254e1d3460fa1602e33c5
 F autoconf/tea/Makefile.in b438a7020446c8a8156e8d97c8914a04833da6fd
 F autoconf/tea/README 3e9a3c060f29a44344ab50aec506f4db903fb873
 F autoconf/tea/aclocal.m4 52c47aac44ce0ddb1f918b6993e8beb8eee88f43
@@ -1700,8 +1700,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6fbd0a11a66f8eb4d7820cb49c23bdcb917db98a22c29d76edea1eea6dab0a4e dc7c48cb4126db9e25c73512cc743155293fe1c4c2516f8c84102228695b6e70
-R e340b654f2343cebf57c71c1eef74e4f
-T +closed dc7c48cb4126db9e25c73512cc743155293fe1c4c2516f8c84102228695b6e70
-U dan
-Z 94f64c633d9eca20229572c5d0ac1d82
+P 0e916416331d7948b312a5dd58ac0c145030bb3b47a37dab2636564397249a86
+R 5056e176b64caf38b7d602040a3b5ee2
+U drh
+Z 40c80df1df52239dfd839811e525ac68
diff --git a/manifest.uuid b/manifest.uuid
index 368a3978bc..6a3e896c14 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0e916416331d7948b312a5dd58ac0c145030bb3b47a37dab2636564397249a86
\ No newline at end of file
+41bfb6b8d61699d09a7e67d2289149abfbb9ce8e75e6ff8560546cad0d2e3f2b
\ No newline at end of file

From 5a7da86f5bc2f5b18320911ae82730a76b459f6c Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 18 Jan 2018 17:46:08 +0000
Subject: [PATCH 356/488] Check for both zlib.h and -lz before enabling zlib
 support in the amalgamation configure script.

FossilOrigin-Name: 8ecd13a1b38e58d4464585c388c1aa27d1f056dba0db239316623e81d3eb5284
---
 autoconf/configure.ac |  5 +++--
 manifest              | 14 +++++++-------
 manifest.uuid         |  2 +-
 3 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/autoconf/configure.ac b/autoconf/configure.ac
index c1e28043d8..19975e96c7 100644
--- a/autoconf/configure.ac
+++ b/autoconf/configure.ac
@@ -164,8 +164,9 @@ AC_SUBST(EXTRA_SHELL_OBJ)
 #-----------------------------------------------------------------------
 
 AC_CHECK_FUNCS(posix_fallocate)
-AC_CHECK_HEADERS(zlib.h)
-AC_SEARCH_LIBS(deflate,z,[ZLIB_FLAGS="-DSQLITE_HAVE_ZLIB"])
+AC_CHECK_HEADERS(zlib.h,[
+  AC_SEARCH_LIBS(deflate,z,[ZLIB_FLAGS="-DSQLITE_HAVE_ZLIB"])
+])
 AC_SUBST(ZLIB_FLAGS)
 
 #-----------------------------------------------------------------------
diff --git a/manifest b/manifest
index 44d3a7e510..6d30478671 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\sthe\sautoconf\sconfigure.ac\sscript\sand\sMakefile.am\stemplates\sso\sthat\nZLIB\sis\sautomatically\sdetected\sand\sused.
-D 2018-01-18T17:09:26.634
+C Check\sfor\sboth\szlib.h\sand\s-lz\sbefore\senabling\szlib\ssupport\sin\sthe\samalgamation\nconfigure\sscript.
+D 2018-01-18T17:46:08.702
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -16,7 +16,7 @@ F autoconf/Makefile.am 6cca3f797c649b40c762484ce26491839fec54de72d376d774969e76e
 F autoconf/Makefile.msc 2c50a59319af7da4eaca8c13e3240881b1bc245fd175845a055faab7d03d6e67
 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7
 F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1
-F autoconf/configure.ac c610cbec367dfdcbbe1341f445699700a0f9e157150254e1d3460fa1602e33c5
+F autoconf/configure.ac aeeed858e5e54e79052ae44ba774e56595dcb787f23a2155aa98a8aa27327b66
 F autoconf/tea/Makefile.in b438a7020446c8a8156e8d97c8914a04833da6fd
 F autoconf/tea/README 3e9a3c060f29a44344ab50aec506f4db903fb873
 F autoconf/tea/aclocal.m4 52c47aac44ce0ddb1f918b6993e8beb8eee88f43
@@ -1700,7 +1700,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0e916416331d7948b312a5dd58ac0c145030bb3b47a37dab2636564397249a86
-R 5056e176b64caf38b7d602040a3b5ee2
-U drh
-Z 40c80df1df52239dfd839811e525ac68
+P 41bfb6b8d61699d09a7e67d2289149abfbb9ce8e75e6ff8560546cad0d2e3f2b
+R 6ef8367290b0a71ac4c6dc7b9f60b8d6
+U dan
+Z 5d28bdaaa8634c1e4ea00e17fbeb8b59
diff --git a/manifest.uuid b/manifest.uuid
index 6a3e896c14..7dab0d2ef4 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-41bfb6b8d61699d09a7e67d2289149abfbb9ce8e75e6ff8560546cad0d2e3f2b
\ No newline at end of file
+8ecd13a1b38e58d4464585c388c1aa27d1f056dba0db239316623e81d3eb5284
\ No newline at end of file

From 1a3a3086163d90e6e71d1c103a2e0dc93438f368 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 18 Jan 2018 19:00:54 +0000
Subject: [PATCH 357/488] Use a loop to avoid recursion in the heightOfSelect()
 function.

FossilOrigin-Name: 86de43595cb2ecebd680fe654affcfb9fbcfff6575c893293ae298124a357bfe
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/expr.c    |  6 +++---
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index 6d30478671..46d6df3929 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Check\sfor\sboth\szlib.h\sand\s-lz\sbefore\senabling\szlib\ssupport\sin\sthe\samalgamation\nconfigure\sscript.
-D 2018-01-18T17:46:08.702
+C Use\sa\sloop\sto\savoid\srecursion\sin\sthe\sheightOfSelect()\sfunction.
+D 2018-01-18T19:00:54.739
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -439,7 +439,7 @@ F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957
 F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
 F src/delete.c 20c8788451dc737a967c87ea53ad43544d617f5b57d32ccce8bd52a0daf9e89b
-F src/expr.c 46a7d73d5579feaee7a7274fac0efea0bbae71dd5b107a569501d89e0280c762
+F src/expr.c 9e06de431c09f144438aa6895ea4d4290fa3c6875bfcc3ba331012ca78deadf0
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331
 F src/func.c bd528d5ed68ce5cbf78a762e3b735fa75009f7197ff07fab07fd771f35ebaa1b
@@ -1700,7 +1700,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 41bfb6b8d61699d09a7e67d2289149abfbb9ce8e75e6ff8560546cad0d2e3f2b
-R 6ef8367290b0a71ac4c6dc7b9f60b8d6
+P 8ecd13a1b38e58d4464585c388c1aa27d1f056dba0db239316623e81d3eb5284
+R 9a4e211a827ab99465bb99e30ad64851
 U dan
-Z 5d28bdaaa8634c1e4ea00e17fbeb8b59
+Z e6a246820ca47f13d6230f5df19d2f34
diff --git a/manifest.uuid b/manifest.uuid
index 7dab0d2ef4..27f50bd5da 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-8ecd13a1b38e58d4464585c388c1aa27d1f056dba0db239316623e81d3eb5284
\ No newline at end of file
+86de43595cb2ecebd680fe654affcfb9fbcfff6575c893293ae298124a357bfe
\ No newline at end of file
diff --git a/src/expr.c b/src/expr.c
index a63de5d9fc..1b87734923 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -658,15 +658,15 @@ static void heightOfExprList(ExprList *p, int *pnHeight){
     }
   }
 }
-static void heightOfSelect(Select *p, int *pnHeight){
-  if( p ){
+static void heightOfSelect(Select *pSelect, int *pnHeight){
+  Select *p;
+  for(p=pSelect; p; p=p->pPrior){
     heightOfExpr(p->pWhere, pnHeight);
     heightOfExpr(p->pHaving, pnHeight);
     heightOfExpr(p->pLimit, pnHeight);
     heightOfExprList(p->pEList, pnHeight);
     heightOfExprList(p->pGroupBy, pnHeight);
     heightOfExprList(p->pOrderBy, pnHeight);
-    heightOfSelect(p->pPrior, pnHeight);
   }
 }
 

From 2f949354094eb96f40d3f5d998e18ea123266791 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 20 Jan 2018 15:48:45 +0000
Subject: [PATCH 358/488] Allow the shell to be built from the configure script
 with SQLITE_OMIT_VIRTUALTABLE defined.

FossilOrigin-Name: 136bf323e42dc90e1780199a381bcbb084b069eca5c7343ee6fc6e2550831536
---
 ext/rtree/rtree.c |  3 ++-
 ext/rtree/rtree.h |  4 ++++
 manifest          | 14 +++++++-------
 manifest.uuid     |  2 +-
 4 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c
index ddabacf408..c0fd8c1819 100644
--- a/ext/rtree/rtree.c
+++ b/ext/rtree/rtree.c
@@ -52,7 +52,8 @@
 **      child page.
 */
 
-#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RTREE)
+#if !defined(SQLITE_CORE) \
+  || (defined(SQLITE_ENABLE_RTREE) && !defined(SQLITE_OMIT_VIRTUALTABLE))
 
 #ifndef SQLITE_CORE
   #include "sqlite3ext.h"
diff --git a/ext/rtree/rtree.h b/ext/rtree/rtree.h
index 1fdbcccc5e..8f41500db1 100644
--- a/ext/rtree/rtree.h
+++ b/ext/rtree/rtree.h
@@ -15,6 +15,10 @@
 */
 #include "sqlite3.h"
 
+#ifdef SQLITE_OMIT_VIRTUALTABLE
+# undef SQLITE_ENABLE_RTREE
+#endif
+
 #ifdef __cplusplus
 extern "C" {
 #endif  /* __cplusplus */
diff --git a/manifest b/manifest
index 46d6df3929..42c7b8baef 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Use\sa\sloop\sto\savoid\srecursion\sin\sthe\sheightOfSelect()\sfunction.
-D 2018-01-18T19:00:54.739
+C Allow\sthe\sshell\sto\sbe\sbuilt\sfrom\sthe\sconfigure\sscript\swith\nSQLITE_OMIT_VIRTUALTABLE\sdefined.
+D 2018-01-20T15:48:45.590
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -350,8 +350,8 @@ F ext/repair/test/checkfreelist01.test 3e8aa6aeb4007680c94a8d07b41c339aa635cc782
 F ext/repair/test/checkindex01.test 6945d0ffc0c1dc993b2ce88036b26e0f5d6fcc65da70fc9df27c2647bb358b0f
 F ext/repair/test/test.tcl 686d76d888dffd021f64260abf29a55c57b2cedfa7fc69150b42b1d6119aac3c
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
-F ext/rtree/rtree.c 2111f685ae07988622c241f819b56fea60782f56e32f97e334473c59f6083481
-F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e
+F ext/rtree/rtree.c d941e44ad901da039caebb9f9fa99d81f2a4fc822e67cafe33fa4f6f789074a0
+F ext/rtree/rtree.h 4a690463901cb5e6127cf05eb8e642f127012fd5003830dbc974eca5802d9412
 F ext/rtree/rtree1.test 82a353747fcab1083d114b2ac84723dfefdbf86c1a6e1df57bf588c7d4285436
 F ext/rtree/rtree2.test 5f25b01acd03470067a2d52783b2eb0a50bf836803d4342d20ca39e541220fe2
 F ext/rtree/rtree3.test 2cafe8265d1ff28f206fce88d114f208349df482
@@ -1700,7 +1700,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 8ecd13a1b38e58d4464585c388c1aa27d1f056dba0db239316623e81d3eb5284
-R 9a4e211a827ab99465bb99e30ad64851
+P 86de43595cb2ecebd680fe654affcfb9fbcfff6575c893293ae298124a357bfe
+R 2aef84e59a8aa705a9da75b37e8e2525
 U dan
-Z e6a246820ca47f13d6230f5df19d2f34
+Z bdb13a1015e901b31413d002f63ee72f
diff --git a/manifest.uuid b/manifest.uuid
index 27f50bd5da..fbb8d7d872 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-86de43595cb2ecebd680fe654affcfb9fbcfff6575c893293ae298124a357bfe
\ No newline at end of file
+136bf323e42dc90e1780199a381bcbb084b069eca5c7343ee6fc6e2550831536
\ No newline at end of file

From 88284b0dc3077939450b4341a0e6ea365f15a8da Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 22 Jan 2018 14:49:15 +0000
Subject: [PATCH 359/488] On the walro2-X.4.2.2 test case, show the size as
 part of the test output.

FossilOrigin-Name: ac5091d7f870f1d0659a576723de454d544e5bfa2bdbba307d7c81eb5ab34e3b
---
 manifest         | 14 +++++++-------
 manifest.uuid    |  2 +-
 test/walro2.test |  1 +
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 42c7b8baef..246b9c257a 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Allow\sthe\sshell\sto\sbe\sbuilt\sfrom\sthe\sconfigure\sscript\swith\nSQLITE_OMIT_VIRTUALTABLE\sdefined.
-D 2018-01-20T15:48:45.590
+C On\sthe\swalro2-X.4.2.2\stest\scase,\sshow\sthe\ssize\sas\spart\sof\sthe\stest\soutput.
+D 2018-01-22T14:49:15.357
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -1553,7 +1553,7 @@ F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03
 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6
 F test/walprotocol.test a112aba0b79e3adeaa485fed09484b32c654e97df58e454aa8489ac2cd57bf84
 F test/walro.test cb438d05ba0d191f10b688e39c4f0cd5b71569a1d1f4440e5bdf3c6880e08c20
-F test/walro2.test 5cd57d192ee334c3894330303b5f8cb6789fef49b2c83ad1b50b9b132d0f7ae1
+F test/walro2.test 71064938e9361bcaefaf5c49bf1be43d7abf44191775eb420e2a1b49dc24824c
 F test/walrofault.test c70cb6e308c443867701856cce92ad8288cd99488fa52afab77cca6cfd51af68
 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417
 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f
@@ -1700,7 +1700,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 86de43595cb2ecebd680fe654affcfb9fbcfff6575c893293ae298124a357bfe
-R 2aef84e59a8aa705a9da75b37e8e2525
-U dan
-Z bdb13a1015e901b31413d002f63ee72f
+P 136bf323e42dc90e1780199a381bcbb084b069eca5c7343ee6fc6e2550831536
+R bce02ef85f0218ef6af27099f7e6faff
+U drh
+Z d37652af866e22c18f57c4be2ebbb16f
diff --git a/manifest.uuid b/manifest.uuid
index fbb8d7d872..ee0b016b91 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-136bf323e42dc90e1780199a381bcbb084b069eca5c7343ee6fc6e2550831536
\ No newline at end of file
+ac5091d7f870f1d0659a576723de454d544e5bfa2bdbba307d7c81eb5ab34e3b
\ No newline at end of file
diff --git a/test/walro2.test b/test/walro2.test
index 876a03196b..14ed050a86 100644
--- a/test/walro2.test
+++ b/test/walro2.test
@@ -263,6 +263,7 @@ do_multiclient_test tn {
   } {500}
   do_test $TN.4.2.2 {
     set sz [file size test.db-wal]
+    puts -nonewline " (sz=$sz)"
     expr {$sz>400000 && $sz<500000}
   } {1}
   do_test $TN.4.2.4 {

From 3a9b9b7749d993f066580b58783009ae6acc1d13 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 22 Jan 2018 15:20:59 +0000
Subject: [PATCH 360/488] Fix a test case in walro2.test so that it works with
 encryption extensions (when each page has a little less usable space
 available than usual).

FossilOrigin-Name: b79521d255a4aa2a7b4424f3498443f27d329ab5428171ba6b87c987f3ccdfc2
---
 manifest         | 14 +++++++-------
 manifest.uuid    |  2 +-
 test/walro2.test |  7 +++----
 3 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/manifest b/manifest
index 246b9c257a..377da8a826 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C On\sthe\swalro2-X.4.2.2\stest\scase,\sshow\sthe\ssize\sas\spart\sof\sthe\stest\soutput.
-D 2018-01-22T14:49:15.357
+C Fix\sa\stest\scase\sin\swalro2.test\sso\sthat\sit\sworks\swith\sencryption\sextensions\n(when\seach\spage\shas\sa\slittle\sless\susable\sspace\savailable\sthan\susual).
+D 2018-01-22T15:20:59.941
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -1553,7 +1553,7 @@ F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03
 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6
 F test/walprotocol.test a112aba0b79e3adeaa485fed09484b32c654e97df58e454aa8489ac2cd57bf84
 F test/walro.test cb438d05ba0d191f10b688e39c4f0cd5b71569a1d1f4440e5bdf3c6880e08c20
-F test/walro2.test 71064938e9361bcaefaf5c49bf1be43d7abf44191775eb420e2a1b49dc24824c
+F test/walro2.test 6c73e8e4b5ccc55f907f4603ba36458b45c085fb6dfb04f30e3c0babbc1c2f41
 F test/walrofault.test c70cb6e308c443867701856cce92ad8288cd99488fa52afab77cca6cfd51af68
 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417
 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f
@@ -1700,7 +1700,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 136bf323e42dc90e1780199a381bcbb084b069eca5c7343ee6fc6e2550831536
-R bce02ef85f0218ef6af27099f7e6faff
-U drh
-Z d37652af866e22c18f57c4be2ebbb16f
+P ac5091d7f870f1d0659a576723de454d544e5bfa2bdbba307d7c81eb5ab34e3b
+R 9a2bdae57204fedd8fbbe0575e4c9803
+U dan
+Z 65b36c011f8ff7d800b7cbb8bc8e96b2
diff --git a/manifest.uuid b/manifest.uuid
index ee0b016b91..89345a02b5 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ac5091d7f870f1d0659a576723de454d544e5bfa2bdbba307d7c81eb5ab34e3b
\ No newline at end of file
+b79521d255a4aa2a7b4424f3498443f27d329ab5428171ba6b87c987f3ccdfc2
\ No newline at end of file
diff --git a/test/walro2.test b/test/walro2.test
index 14ed050a86..1c51e91ec5 100644
--- a/test/walro2.test
+++ b/test/walro2.test
@@ -261,10 +261,9 @@ do_multiclient_test tn {
         SELECT count(*) FROM t2;
     } 
   } {500}
-  do_test $TN.4.2.2 {
-    set sz [file size test.db-wal]
-    puts -nonewline " (sz=$sz)"
-    expr {$sz>400000 && $sz<500000}
+  set sz [file size test.db-wal]
+  do_test $TN.4.2.2.(sz=$sz) {
+    expr {$sz>400000}
   } {1}
   do_test $TN.4.2.4 {
     file_control_persist_wal db 1; db close

From 8bfe66a9c4da70834daca140227f2b3a0460a9ab Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 22 Jan 2018 15:45:12 +0000
Subject: [PATCH 361/488] Avoid unnecessary OOM detection warnings in a
 debugging routine.

FossilOrigin-Name: 395f8ea790e6e295800fa8927f0585b2419b9521ef4fd591d51d2a48db2a90c4
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/btree.c   |  5 ++++-
 3 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index 377da8a826..f11b306145 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\stest\scase\sin\swalro2.test\sso\sthat\sit\sworks\swith\sencryption\sextensions\n(when\seach\spage\shas\sa\slittle\sless\susable\sspace\savailable\sthan\susual).
-D 2018-01-22T15:20:59.941
+C Avoid\sunnecessary\sOOM\sdetection\swarnings\sin\sa\sdebugging\sroutine.
+D 2018-01-22T15:45:12.922
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -428,7 +428,7 @@ F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
-F src/btree.c 0a1f63b50ab1ac5d4b1637c30cb1ae123fbc162ec8cb6336ddb9491a0bc1e363
+F src/btree.c bfc453babec9aa8196ab5db5e588ac5d3f0c398d72faa37296167a84a61c9f2f
 F src/btree.h 0866c0a08255142ea0e754aabd211c843cab32045c978a592a43152405ed0c84
 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc
 F src/build.c 9f9647454f236cab097f266ae970f899b53c71cadab6756c47e2b2e81392c2a1
@@ -1700,7 +1700,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ac5091d7f870f1d0659a576723de454d544e5bfa2bdbba307d7c81eb5ab34e3b
-R 9a2bdae57204fedd8fbbe0575e4c9803
-U dan
-Z 65b36c011f8ff7d800b7cbb8bc8e96b2
+P b79521d255a4aa2a7b4424f3498443f27d329ab5428171ba6b87c987f3ccdfc2
+R e7cccb4f976b9eb07c1586193cc26ec5
+U drh
+Z b64137fc7f67b8d25a73e5d06cbfe718
diff --git a/manifest.uuid b/manifest.uuid
index 89345a02b5..3c8a7f3fb9 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b79521d255a4aa2a7b4424f3498443f27d329ab5428171ba6b87c987f3ccdfc2
\ No newline at end of file
+395f8ea790e6e295800fa8927f0585b2419b9521ef4fd591d51d2a48db2a90c4
\ No newline at end of file
diff --git a/src/btree.c b/src/btree.c
index 8cd5ee6739..3b42f68f9f 100644
--- a/src/btree.c
+++ b/src/btree.c
@@ -123,9 +123,12 @@ int sqlite3_enable_shared_cache(int enable){
 */
 #ifdef SQLITE_DEBUG
 int corruptPageError(int lineno, MemPage *p){
-  char *zMsg = sqlite3_mprintf("database corruption page %d of %s",
+  char *zMsg;
+  sqlite3BeginBenignMalloc();
+  zMsg = sqlite3_mprintf("database corruption page %d of %s",
       (int)p->pgno, sqlite3PagerFilename(p->pBt->pPager, 0)
   );
+  sqlite3EndBenignMalloc();
   if( zMsg ){
     sqlite3ReportError(SQLITE_CORRUPT, lineno, zMsg);
   }

From 138d4f8de22f24317fc77c301d545c43542ae592 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 22 Jan 2018 18:45:57 +0000
Subject: [PATCH 362/488] Version 3.22.0

FossilOrigin-Name: 0c55d179733b46d8d0ba4d88e01a25e10677046ee3da1d5b1581e86726f2171d
---
 manifest      | 11 +++++++----
 manifest.uuid |  2 +-
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/manifest b/manifest
index f11b306145..59cd9b34a1 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Avoid\sunnecessary\sOOM\sdetection\swarnings\sin\sa\sdebugging\sroutine.
-D 2018-01-22T15:45:12.922
+C Version\s3.22.0
+D 2018-01-22T18:45:57.681
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -1700,7 +1700,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b79521d255a4aa2a7b4424f3498443f27d329ab5428171ba6b87c987f3ccdfc2
+P 395f8ea790e6e295800fa8927f0585b2419b9521ef4fd591d51d2a48db2a90c4
 R e7cccb4f976b9eb07c1586193cc26ec5
+T +bgcolor * #d0c0ff
+T +sym-release *
+T +sym-version-3.22.0 *
 U drh
-Z b64137fc7f67b8d25a73e5d06cbfe718
+Z 9c0e858ece53e1ee810a8a479ae4dc82
diff --git a/manifest.uuid b/manifest.uuid
index 3c8a7f3fb9..ed79ed6466 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-395f8ea790e6e295800fa8927f0585b2419b9521ef4fd591d51d2a48db2a90c4
\ No newline at end of file
+0c55d179733b46d8d0ba4d88e01a25e10677046ee3da1d5b1581e86726f2171d
\ No newline at end of file

From b71aa0955bdea9fefdfe991f623da6f5f828bd6c Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Tue, 23 Jan 2018 00:05:18 +0000
Subject: [PATCH 363/488] Fix harmless compiler warnings seen with MSVC.

FossilOrigin-Name: 76a11a80e94d1e9c9645e6e348948f3fd508266d4569c85d80a3879862d0a819
---
 ext/session/sqlite3session.c |  2 +-
 manifest                     | 19 ++++++++-----------
 manifest.uuid                |  2 +-
 src/shell.c.in               | 13 +++++++------
 4 files changed, 17 insertions(+), 19 deletions(-)

diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c
index ec1a93e08e..67915e04f1 100644
--- a/ext/session/sqlite3session.c
+++ b/ext/session/sqlite3session.c
@@ -1115,7 +1115,7 @@ static void sessionPreupdateOneChange(
   int iHash; 
   int bNull = 0; 
   int rc = SQLITE_OK;
-  SessionStat1Ctx stat1;
+  SessionStat1Ctx stat1 = {0, 0};
 
   if( pSession->rc ) return;
 
diff --git a/manifest b/manifest
index 59cd9b34a1..31134887c3 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Version\s3.22.0
-D 2018-01-22T18:45:57.681
+C Fix\sharmless\scompiler\swarnings\sseen\swith\sMSVC.
+D 2018-01-23T00:05:18.184
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -400,7 +400,7 @@ F ext/session/sessionfault.test da273f2712b6411e85e71465a1733b8501dbf6f7
 F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0
 F ext/session/sessionstat1.test 41cd97c2e48619a41cdf8ae749e1b25f34719de638689221aa43971be693bf4e
 F ext/session/sessionwor.test 2f3744236dc8b170a695b7d8ddc8c743c7e79fdc
-F ext/session/sqlite3session.c 989466bba4dff0ede8d4c450b1fc65ca222b87e31193eddbf3931b88bf898a57
+F ext/session/sqlite3session.c 63cfde70155b597480b2d01e65031f8e60d204103db7adf6156fe48e3e379392
 F ext/session/sqlite3session.h 01774161cbd328fe3d496323655b9cc142317ff1fb1ae15c1232075ea240e3a4
 F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386
 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
@@ -487,7 +487,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c bebe7cce45d899d2237c76bce059d525abf5b861f2fce92f6b53914a961c01ba
-F src/shell.c.in 4e1bcf8c70b8fb97c7cbaca6602e2a291d7fe17eff23a5de003d6fabd87f27d1
+F src/shell.c.in 06ffe417973eacbee77a8402e26553b938e7c07f735c811701fc5cd1e642fb5c
 F src/sqlite.h.in 959deaad89679e31d7f68fda668b0c5d1f592fffed7a9c1740fb8ded4e4e754a
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 99189e7611eb0bf98f21c7835dc74730a84e2e809c98e1e31c33896dee7a2849
@@ -1700,10 +1700,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 395f8ea790e6e295800fa8927f0585b2419b9521ef4fd591d51d2a48db2a90c4
-R e7cccb4f976b9eb07c1586193cc26ec5
-T +bgcolor * #d0c0ff
-T +sym-release *
-T +sym-version-3.22.0 *
-U drh
-Z 9c0e858ece53e1ee810a8a479ae4dc82
+P 0c55d179733b46d8d0ba4d88e01a25e10677046ee3da1d5b1581e86726f2171d
+R 8f662370d27c0dbf362ba912c090e456
+U mistachkin
+Z fc242cf748005a9ff5528ff032a48505
diff --git a/manifest.uuid b/manifest.uuid
index ed79ed6466..ae3e0fe06b 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0c55d179733b46d8d0ba4d88e01a25e10677046ee3da1d5b1581e86726f2171d
\ No newline at end of file
+76a11a80e94d1e9c9645e6e348948f3fd508266d4569c85d80a3879862d0a819
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index d9c8705e70..e3a1f30752 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -1272,9 +1272,10 @@ static void editFunc(
     goto edit_func_end;
   }
   if( bBin ){
-    sqlite3_result_blob(context, p, sz, sqlite3_free);
+    sqlite3_result_blob64(context, p, sz, sqlite3_free);
   }else{
-    sqlite3_result_text(context, (const char*)p, sz, sqlite3_free);
+    sqlite3_result_text64(context, (const char*)p, sz,
+                          sqlite3_free, SQLITE_UTF8);
   }
   p = 0;
 
@@ -3435,7 +3436,7 @@ static void open_db(ShellState *p, int keepAlive){
   if( p->db==0 ){
     sqlite3_initialize();
     if( p->openMode==SHELL_OPEN_UNSPEC && access(p->zDbFilename,0)==0 ){
-      p->openMode = deduceDatabaseType(p->zDbFilename);
+      p->openMode = (u8)deduceDatabaseType(p->zDbFilename);
     }
     switch( p->openMode ){
       case SHELL_OPEN_APPENDVFS: {
@@ -5715,7 +5716,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       }else if( strcmp(azArg[1],"trigger")==0 ){
         p->autoEQP = AUTOEQP_trigger;
       }else{
-        p->autoEQP = booleanValue(azArg[1]);
+        p->autoEQP = (u8)booleanValue(azArg[1]);
       }
     }else{
       raw_printf(stderr, "Usage: .eqp off|on|trigger|full\n");
@@ -6488,7 +6489,7 @@ static int do_meta_command(char *zLine, ShellState *p){
 
   if( c=='s' && strncmp(azArg[0], "scanstats", n)==0 ){
     if( nArg==2 ){
-      p->scanstatsOn = booleanValue(azArg[1]);
+      p->scanstatsOn = (u8)booleanValue(azArg[1]);
 #ifndef SQLITE_ENABLE_STMT_SCANSTATUS
       raw_printf(stderr, "Warning: .scanstats not available in this build.\n");
 #endif
@@ -7139,7 +7140,7 @@ static int do_meta_command(char *zLine, ShellState *p){
 
   if( c=='s' && strncmp(azArg[0], "stats", n)==0 ){
     if( nArg==2 ){
-      p->statsOn = booleanValue(azArg[1]);
+      p->statsOn = (u8)booleanValue(azArg[1]);
     }else if( nArg==1 ){
       display_stats(p->db, p, 0);
     }else{

From fa3d7855801803a701daff30ec466a0c4d8ad8e4 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Tue, 23 Jan 2018 00:17:27 +0000
Subject: [PATCH 364/488] Simplify one of the compiler warning fixes from the
 previous check-in.

FossilOrigin-Name: 012d7d49c571207087bbe24af05d639627b96df526bfcb2098a3e975b74d865d
---
 ext/session/sqlite3session.c |  2 +-
 manifest                     | 12 ++++++------
 manifest.uuid                |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c
index 67915e04f1..bdecdd1031 100644
--- a/ext/session/sqlite3session.c
+++ b/ext/session/sqlite3session.c
@@ -1115,7 +1115,7 @@ static void sessionPreupdateOneChange(
   int iHash; 
   int bNull = 0; 
   int rc = SQLITE_OK;
-  SessionStat1Ctx stat1 = {0, 0};
+  SessionStat1Ctx stat1 = {0};
 
   if( pSession->rc ) return;
 
diff --git a/manifest b/manifest
index 31134887c3..6dc303c771 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sharmless\scompiler\swarnings\sseen\swith\sMSVC.
-D 2018-01-23T00:05:18.184
+C Simplify\sone\sof\sthe\scompiler\swarning\sfixes\sfrom\sthe\sprevious\scheck-in.
+D 2018-01-23T00:17:27.230
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -400,7 +400,7 @@ F ext/session/sessionfault.test da273f2712b6411e85e71465a1733b8501dbf6f7
 F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0
 F ext/session/sessionstat1.test 41cd97c2e48619a41cdf8ae749e1b25f34719de638689221aa43971be693bf4e
 F ext/session/sessionwor.test 2f3744236dc8b170a695b7d8ddc8c743c7e79fdc
-F ext/session/sqlite3session.c 63cfde70155b597480b2d01e65031f8e60d204103db7adf6156fe48e3e379392
+F ext/session/sqlite3session.c a5b7aed647abe7e366254b755597fce3f2719d82c98990cb0e1e07a3d203fe2b
 F ext/session/sqlite3session.h 01774161cbd328fe3d496323655b9cc142317ff1fb1ae15c1232075ea240e3a4
 F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386
 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
@@ -1700,7 +1700,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0c55d179733b46d8d0ba4d88e01a25e10677046ee3da1d5b1581e86726f2171d
-R 8f662370d27c0dbf362ba912c090e456
+P 76a11a80e94d1e9c9645e6e348948f3fd508266d4569c85d80a3879862d0a819
+R 47270bb77ce74e463612a801d427faba
 U mistachkin
-Z fc242cf748005a9ff5528ff032a48505
+Z c93f87a6596350b67f49dd4ebf8496b6
diff --git a/manifest.uuid b/manifest.uuid
index ae3e0fe06b..58bcb92a1a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-76a11a80e94d1e9c9645e6e348948f3fd508266d4569c85d80a3879862d0a819
\ No newline at end of file
+012d7d49c571207087bbe24af05d639627b96df526bfcb2098a3e975b74d865d
\ No newline at end of file

From f09ac0b336b7fa8b0205fcd8494992264f4283d6 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 23 Jan 2018 03:44:06 +0000
Subject: [PATCH 365/488] Slightly faster function dispatch in the virtual
 machine by avoiding unnecessary reinitialization of variables that are
 already correctly initialized.

FossilOrigin-Name: edd4e6876cc08ab907bb21f075380de946562d4c56a04923760848b4b11536ac
---
 manifest      | 20 ++++++++++----------
 manifest.uuid |  2 +-
 src/func.c    |  1 +
 src/vdbe.c    | 50 +++++++++++++++++++++++++++++---------------------
 src/vdbeInt.h |  3 ---
 src/vdbeapi.c | 12 ++----------
 6 files changed, 43 insertions(+), 45 deletions(-)

diff --git a/manifest b/manifest
index 6dc303c771..71deaf9781 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Simplify\sone\sof\sthe\scompiler\swarning\sfixes\sfrom\sthe\sprevious\scheck-in.
-D 2018-01-23T00:17:27.230
+C Slightly\sfaster\sfunction\sdispatch\sin\sthe\svirtual\smachine\sby\savoiding\nunnecessary\sreinitialization\sof\svariables\sthat\sare\salready\scorrectly\ninitialized.
+D 2018-01-23T03:44:06.287
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -442,7 +442,7 @@ F src/delete.c 20c8788451dc737a967c87ea53ad43544d617f5b57d32ccce8bd52a0daf9e89b
 F src/expr.c 9e06de431c09f144438aa6895ea4d4290fa3c6875bfcc3ba331012ca78deadf0
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331
-F src/func.c bd528d5ed68ce5cbf78a762e3b735fa75009f7197ff07fab07fd771f35ebaa1b
+F src/func.c 24d4746d347442c99153b5f9ac3204f6932d8d591008a94d6967de6b62fc0102
 F src/global.c ac3094f1dc59fbeb919aef7cc0cc827a8459d1fb1adb7972ef75bd9e0c10b75b
 F src/hash.c a12580e143f10301ed5166ea4964ae2853d3905a511d4e0c44497245c7ce1f7a
 F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4
@@ -555,10 +555,10 @@ F src/update.c a90a32ffc0100265b0693dbbdbe490756447af181f5ea2c138cce515b08c8795
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c ef4a5f904d942e660abade7fbf3e6bdb402dabe9e7c27f3361ecf40b945538b5
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
-F src/vdbe.c ccc1e17a30325068ae4f0292e8601997946886d23acc989c68f2a261a2795c70
+F src/vdbe.c 5aa6fb85281b6af058d2a87b65c1a8a8e0bd2fe554fb3391497ee3702b61e0be
 F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a
-F src/vdbeInt.h c8cfbbc28e37e67a493c3f892fb0596add56a31a00e7537a06049af9ef2f51b0
-F src/vdbeapi.c 02f773681d06e46454b0606339068d4d4490873dc4a7334bc0c6030552bb2c8c
+F src/vdbeInt.h 8d7d07f13cb3c4cbca91e22ba4a1920e542dda7c5d9299920432a0b3d5b009f5
+F src/vdbeapi.c fea41171884a4de119f8b10ab514c788674eeeb7f27218bb6d008e1310bfd07f
 F src/vdbeaux.c 2756ac68ac259c416554100598fc291870063288cd7e1af22847f57b3e130e56
 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191
 F src/vdbemem.c 7548dd5af03d24d534a5dbc41e3bbdf1fab83e9c8856a8d2549ed2ccf33d0e80
@@ -1700,7 +1700,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 76a11a80e94d1e9c9645e6e348948f3fd508266d4569c85d80a3879862d0a819
-R 47270bb77ce74e463612a801d427faba
-U mistachkin
-Z c93f87a6596350b67f49dd4ebf8496b6
+P 012d7d49c571207087bbe24af05d639627b96df526bfcb2098a3e975b74d865d
+R 117511e55335ee1f52b85cfdea90c38c
+U drh
+Z e3904fade688ed958d5e16ed2185daad
diff --git a/manifest.uuid b/manifest.uuid
index 58bcb92a1a..993b1dc61a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-012d7d49c571207087bbe24af05d639627b96df526bfcb2098a3e975b74d865d
\ No newline at end of file
+edd4e6876cc08ab907bb21f075380de946562d4c56a04923760848b4b11536ac
\ No newline at end of file
diff --git a/src/func.c b/src/func.c
index 1076b97161..8f409a0ad1 100644
--- a/src/func.c
+++ b/src/func.c
@@ -35,6 +35,7 @@ static CollSeq *sqlite3GetFuncCollSeq(sqlite3_context *context){
 ** iteration of the aggregate loop.
 */
 static void sqlite3SkipAccumulatorLoad(sqlite3_context *context){
+  if( context->isError==0 ) context->isError = -1;
   context->skipFlag = 1;
 }
 
diff --git a/src/vdbe.c b/src/vdbe.c
index d878ca8e31..8d0b35a6f4 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -6204,12 +6204,17 @@ case OP_AggStep0: {
   assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) );
   assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem+1 - p->nCursor)+1) );
   assert( pOp->p3<pOp->p2 || pOp->p3>=pOp->p2+n );
-  pCtx = sqlite3DbMallocRawNN(db, sizeof(*pCtx) + (n-1)*sizeof(sqlite3_value*));
+  pCtx = sqlite3DbMallocRawNN(db, n*sizeof(sqlite3_value*) +
+               (sizeof(pCtx[0]) + sizeof(Mem) - sizeof(sqlite3_value*)));
   if( pCtx==0 ) goto no_mem;
   pCtx->pMem = 0;
+  pCtx->pOut = (Mem*)&(pCtx->argv[n]);
+  sqlite3VdbeMemInit(pCtx->pOut, db, MEM_Null);
   pCtx->pFunc = pOp->p4.pFunc;
   pCtx->iOp = (int)(pOp - aOp);
   pCtx->pVdbe = p;
+  pCtx->skipFlag = 0;
+  pCtx->isError = 0;
   pCtx->argc = n;
   pOp->p4type = P4_FUNCCTX;
   pOp->p4.pCtx = pCtx;
@@ -6220,7 +6225,6 @@ case OP_AggStep: {
   int i;
   sqlite3_context *pCtx;
   Mem *pMem;
-  Mem t;
 
   assert( pOp->p4type==P4_FUNCCTX );
   pCtx = pOp->p4.pCtx;
@@ -6243,26 +6247,28 @@ case OP_AggStep: {
 #endif
 
   pMem->n++;
-  sqlite3VdbeMemInit(&t, db, MEM_Null);
-  pCtx->pOut = &t;
-  pCtx->fErrorOrAux = 0;
-  pCtx->skipFlag = 0;
+  assert( pCtx->pOut->flags==MEM_Null );
+  assert( pCtx->isError==0 );
+  assert( pCtx->skipFlag==0 );
   (pCtx->pFunc->xSFunc)(pCtx,pCtx->argc,pCtx->argv); /* IMP: R-24505-23230 */
-  if( pCtx->fErrorOrAux ){
-    if( pCtx->isError ){
-      sqlite3VdbeError(p, "%s", sqlite3_value_text(&t));
+  if( pCtx->isError ){
+    if( pCtx->isError>0 ){
+      sqlite3VdbeError(p, "%s", sqlite3_value_text(pCtx->pOut));
       rc = pCtx->isError;
     }
-    sqlite3VdbeMemRelease(&t);
+    if( pCtx->skipFlag ){
+      assert( pOp[-1].opcode==OP_CollSeq );
+      i = pOp[-1].p1;
+      if( i ) sqlite3VdbeMemSetInt64(&aMem[i], 1);
+      pCtx->skipFlag = 0;
+    }
+    sqlite3VdbeMemRelease(pCtx->pOut);
+    pCtx->pOut->flags = MEM_Null;
+    pCtx->isError = 0;
     if( rc ) goto abort_due_to_error;
-  }else{
-    assert( t.flags==MEM_Null );
-  }
-  if( pCtx->skipFlag ){
-    assert( pOp[-1].opcode==OP_CollSeq );
-    i = pOp[-1].p1;
-    if( i ) sqlite3VdbeMemSetInt64(&aMem[i], 1);
   }
+  assert( pCtx->pOut->flags==MEM_Null );
+  assert( pCtx->skipFlag==0 );
   break;
 }
 
@@ -6749,7 +6755,7 @@ case OP_VColumn: {
   }
   rc = pModule->xColumn(pCur->uc.pVCur, &sContext, pOp->p2);
   sqlite3VtabImportErrmsg(p, pVtab);
-  if( sContext.isError ){
+  if( sContext.isError>0 ){
     rc = sContext.isError;
   }
   sqlite3VdbeChangeEncoding(pDest, encoding);
@@ -7014,6 +7020,7 @@ case OP_Function0: {
   pCtx->pFunc = pOp->p4.pFunc;
   pCtx->iOp = (int)(pOp - aOp);
   pCtx->pVdbe = p;
+  pCtx->isError = 0;
   pCtx->argc = n;
   pOp->p4type = P4_FUNCCTX;
   pOp->p4.pCtx = pCtx;
@@ -7048,16 +7055,17 @@ case OP_Function: {
   }
 #endif
   MemSetTypeFlag(pOut, MEM_Null);
-  pCtx->fErrorOrAux = 0;
+  assert( pCtx->isError==0 );
   (*pCtx->pFunc->xSFunc)(pCtx, pCtx->argc, pCtx->argv);/* IMP: R-24505-23230 */
 
   /* If the function returned an error, throw an exception */
-  if( pCtx->fErrorOrAux ){
-    if( pCtx->isError ){
+  if( pCtx->isError ){
+    if( pCtx->isError>0 ){
       sqlite3VdbeError(p, "%s", sqlite3_value_text(pOut));
       rc = pCtx->isError;
     }
     sqlite3VdbeDeleteAuxData(db, &p->pAuxData, pCtx->iOp, pOp->p1);
+    pCtx->isError = 0;
     if( rc ) goto abort_due_to_error;
   }
 
diff --git a/src/vdbeInt.h b/src/vdbeInt.h
index 976abb3938..1e4f615ba2 100644
--- a/src/vdbeInt.h
+++ b/src/vdbeInt.h
@@ -224,8 +224,6 @@ struct sqlite3_value {
 ** If the MEM_Null flag is set, then the value is an SQL NULL value.
 ** For a pointer type created using sqlite3_bind_pointer() or
 ** sqlite3_result_pointer() the MEM_Term and MEM_Subtype flags are also set.
-** If both MEM_Null and MEM_Zero are set, that means that the value is
-** an unchanging column value from VColumn.
 **
 ** If the MEM_Str flag is set then Mem.z points at a string representation.
 ** Usually this is encoded in the same unicode encoding as the main
@@ -319,7 +317,6 @@ struct sqlite3_context {
   int iOp;                /* Instruction number of OP_Function */
   int isError;            /* Error code returned by the function. */
   u8 skipFlag;            /* Skip accumulator loading if true */
-  u8 fErrorOrAux;         /* isError!=0 or pVdbe->pAuxData modified */
   u8 argc;                /* Number of arguments */
   sqlite3_value *argv[1]; /* Argument set */
 };
diff --git a/src/vdbeapi.c b/src/vdbeapi.c
index bdd6d1cd07..dd4a352003 100644
--- a/src/vdbeapi.c
+++ b/src/vdbeapi.c
@@ -372,14 +372,12 @@ void sqlite3_result_double(sqlite3_context *pCtx, double rVal){
 void sqlite3_result_error(sqlite3_context *pCtx, const char *z, int n){
   assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) );
   pCtx->isError = SQLITE_ERROR;
-  pCtx->fErrorOrAux = 1;
   sqlite3VdbeMemSetStr(pCtx->pOut, z, n, SQLITE_UTF8, SQLITE_TRANSIENT);
 }
 #ifndef SQLITE_OMIT_UTF16
 void sqlite3_result_error16(sqlite3_context *pCtx, const void *z, int n){
   assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) );
   pCtx->isError = SQLITE_ERROR;
-  pCtx->fErrorOrAux = 1;
   sqlite3VdbeMemSetStr(pCtx->pOut, z, n, SQLITE_UTF16NATIVE, SQLITE_TRANSIENT);
 }
 #endif
@@ -485,8 +483,7 @@ int sqlite3_result_zeroblob64(sqlite3_context *pCtx, u64 n){
   return SQLITE_OK;
 }
 void sqlite3_result_error_code(sqlite3_context *pCtx, int errCode){
-  pCtx->isError = errCode;
-  pCtx->fErrorOrAux = 1;
+  pCtx->isError = errCode ? errCode : -1;
 #ifdef SQLITE_DEBUG
   if( pCtx->pVdbe ) pCtx->pVdbe->rcApp = errCode;
 #endif
@@ -500,7 +497,6 @@ void sqlite3_result_error_code(sqlite3_context *pCtx, int errCode){
 void sqlite3_result_error_toobig(sqlite3_context *pCtx){
   assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) );
   pCtx->isError = SQLITE_TOOBIG;
-  pCtx->fErrorOrAux = 1;
   sqlite3VdbeMemSetStr(pCtx->pOut, "string or blob too big", -1, 
                        SQLITE_UTF8, SQLITE_STATIC);
 }
@@ -510,7 +506,6 @@ void sqlite3_result_error_nomem(sqlite3_context *pCtx){
   assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) );
   sqlite3VdbeMemSetNull(pCtx->pOut);
   pCtx->isError = SQLITE_NOMEM_BKPT;
-  pCtx->fErrorOrAux = 1;
   sqlite3OomFault(pCtx->pOut->db);
 }
 
@@ -917,10 +912,7 @@ void sqlite3_set_auxdata(
     pAuxData->iAuxArg = iArg;
     pAuxData->pNextAux = pVdbe->pAuxData;
     pVdbe->pAuxData = pAuxData;
-    if( pCtx->fErrorOrAux==0 ){
-      pCtx->isError = 0;
-      pCtx->fErrorOrAux = 1;
-    }
+    if( pCtx->isError==0 ) pCtx->isError = -1;
   }else if( pAuxData->xDeleteAux ){
     pAuxData->xDeleteAux(pAuxData->pAux);
   }

From 7ea3469e539fd296eaf2e5d16d9d1b0796f47bcb Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 23 Jan 2018 04:22:33 +0000
Subject: [PATCH 366/488] Slightly faster implementation of the length() SQL
 function.

FossilOrigin-Name: 9a4199aedb039141a50a68943ef863d7df9637ada318793cce6c12b9d39bcdc5
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/func.c    | 16 +++++++++-------
 3 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/manifest b/manifest
index 71deaf9781..79e0bb8fd9 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Slightly\sfaster\sfunction\sdispatch\sin\sthe\svirtual\smachine\sby\savoiding\nunnecessary\sreinitialization\sof\svariables\sthat\sare\salready\scorrectly\ninitialized.
-D 2018-01-23T03:44:06.287
+C Slightly\sfaster\simplementation\sof\sthe\slength()\sSQL\sfunction.
+D 2018-01-23T04:22:33.527
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -442,7 +442,7 @@ F src/delete.c 20c8788451dc737a967c87ea53ad43544d617f5b57d32ccce8bd52a0daf9e89b
 F src/expr.c 9e06de431c09f144438aa6895ea4d4290fa3c6875bfcc3ba331012ca78deadf0
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331
-F src/func.c 24d4746d347442c99153b5f9ac3204f6932d8d591008a94d6967de6b62fc0102
+F src/func.c 227d17a030b75f7495fc3b8d1aca19850e794397433175c1ba21b713d9833f14
 F src/global.c ac3094f1dc59fbeb919aef7cc0cc827a8459d1fb1adb7972ef75bd9e0c10b75b
 F src/hash.c a12580e143f10301ed5166ea4964ae2853d3905a511d4e0c44497245c7ce1f7a
 F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4
@@ -1700,7 +1700,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 012d7d49c571207087bbe24af05d639627b96df526bfcb2098a3e975b74d865d
-R 117511e55335ee1f52b85cfdea90c38c
+P edd4e6876cc08ab907bb21f075380de946562d4c56a04923760848b4b11536ac
+R aa1994289d6ea2ba21f905f276b2d5c2
 U drh
-Z e3904fade688ed958d5e16ed2185daad
+Z fced45fd12f23e9379e90a75bf8cc2b6
diff --git a/manifest.uuid b/manifest.uuid
index 993b1dc61a..70318058cb 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-edd4e6876cc08ab907bb21f075380de946562d4c56a04923760848b4b11536ac
\ No newline at end of file
+9a4199aedb039141a50a68943ef863d7df9637ada318793cce6c12b9d39bcdc5
\ No newline at end of file
diff --git a/src/func.c b/src/func.c
index 8f409a0ad1..36cfa44230 100644
--- a/src/func.c
+++ b/src/func.c
@@ -102,8 +102,6 @@ static void lengthFunc(
   int argc,
   sqlite3_value **argv
 ){
-  int len;
-
   assert( argc==1 );
   UNUSED_PARAMETER(argc);
   switch( sqlite3_value_type(argv[0]) ){
@@ -115,13 +113,17 @@ static void lengthFunc(
     }
     case SQLITE_TEXT: {
       const unsigned char *z = sqlite3_value_text(argv[0]);
+      const unsigned char *z0;
+      unsigned char c;
       if( z==0 ) return;
-      len = 0;
-      while( *z ){
-        len++;
-        SQLITE_SKIP_UTF8(z);
+      z0 = z;
+      while( (c = *z)!=0 ){
+        z++;
+        if( c>=0xc0 ){
+          while( (*z & 0xc0)==0x80 ){ z++; z0++; }
+        }
       }
-      sqlite3_result_int(context, len);
+      sqlite3_result_int(context, (int)(z-z0));
       break;
     }
     default: {

From 176b3a09b2b2ded371af0a9e078fde1fd03e5bda Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Tue, 23 Jan 2018 07:11:05 +0000
Subject: [PATCH 367/488] Skip defining WIN32_LEAN_AND_MEAN when it is already
 defined.

FossilOrigin-Name: 195f5323df800f7963df2ea251c497c72e4d7ff0b88a70f1fa60f13bb1a9fd3b
---
 manifest             | 16 ++++++++--------
 manifest.uuid        |  2 +-
 src/tclsqlite.c      |  4 +++-
 src/test_windirent.h |  3 +++
 4 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index 79e0bb8fd9..1bb19cf6b3 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Slightly\sfaster\simplementation\sof\sthe\slength()\sSQL\sfunction.
-D 2018-01-23T04:22:33.527
+C Skip\sdefining\sWIN32_LEAN_AND_MEAN\swhen\sit\sis\salready\sdefined.
+D 2018-01-23T07:11:05.143
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -495,7 +495,7 @@ F src/sqliteInt.h 9c70315598b34810a83e4894455acb18e95cf63ce4e6cbb451ac2d17eabc25
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
-F src/tclsqlite.c 1833388c01e3b77f4c712185ee7250b9423ee0981ce6ae7e401e47db0319a696
+F src/tclsqlite.c 11a2618c227fd13ccad73ee02d1199f9880c59db2b3144fd7432db1980a2577d
 F src/test1.c b52f9e7fe62016d357c3266fcfa0793cc1883d3cb2b11dfa39fcba2e70b0305c
 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5
 F src/test3.c b8434949dfb8aff8dfa082c8b592109e77844c2135ed3c492113839b6956255b
@@ -545,7 +545,7 @@ F src/test_thread.c 911d15fb14e19c0c542bdc8aabf981c2f10a4858
 F src/test_vfs.c f0186261a24de2671d080bcd8050732f0cb64f6e
 F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698
 F src/test_windirent.c a895e2c068a06644eef91a7f0a32182445a893b9a0f33d0cdb4283dca2486ac1
-F src/test_windirent.h 8782864172ba5ae52c5c313c70faeadb324ff74de9c3dcc6b56a557dccaa1de6
+F src/test_windirent.h 90dfbe95442c9762357fe128dc7ae3dc199d006de93eb33ba3972e0a90484215
 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
 F src/tokenize.c 5b0c661a85f783d35b9883830736eeb63be4aefc4f6b7d9cd081d48782c041e2
@@ -1700,7 +1700,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P edd4e6876cc08ab907bb21f075380de946562d4c56a04923760848b4b11536ac
-R aa1994289d6ea2ba21f905f276b2d5c2
-U drh
-Z fced45fd12f23e9379e90a75bf8cc2b6
+P 9a4199aedb039141a50a68943ef863d7df9637ada318793cce6c12b9d39bcdc5
+R fc52a77923ed3abf898f25d3a24aab55
+U mistachkin
+Z 3cd93d9679b9898485b5f29a3393d5bf
diff --git a/manifest.uuid b/manifest.uuid
index 70318058cb..3eb8b75e11 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-9a4199aedb039141a50a68943ef863d7df9637ada318793cce6c12b9d39bcdc5
\ No newline at end of file
+195f5323df800f7963df2ea251c497c72e4d7ff0b88a70f1fa60f13bb1a9fd3b
\ No newline at end of file
diff --git a/src/tclsqlite.c b/src/tclsqlite.c
index eed86eee34..252b246e6e 100644
--- a/src/tclsqlite.c
+++ b/src/tclsqlite.c
@@ -64,7 +64,9 @@
 # define GETPID getpid
 #elif !defined(_WIN32_WCE)
 # ifndef SQLITE_AMALGAMATION
-#  define WIN32_LEAN_AND_MEAN
+#  ifndef WIN32_LEAN_AND_MEAN
+#   define WIN32_LEAN_AND_MEAN
+#  endif
 #  include <windows.h>
 # endif
 # define GETPID (int)GetCurrentProcessId
diff --git a/src/test_windirent.h b/src/test_windirent.h
index d71b49f684..ada5322530 100644
--- a/src/test_windirent.h
+++ b/src/test_windirent.h
@@ -20,7 +20,10 @@
 ** We need several data types from the Windows SDK header.
 */
 
+#ifndef WIN32_LEAN_AND_MEAN
 #define WIN32_LEAN_AND_MEAN
+#endif
+
 #include "windows.h"
 
 /*

From 8d7f1630154e98eba7b0364a5cebe759abf90f6e Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 23 Jan 2018 13:30:38 +0000
Subject: [PATCH 368/488] Fix comment typos.  No changes to code.

FossilOrigin-Name: 8e5e74c66b9cdb348392e3db2d9f32cfa20fcec35bf09d9e1f623e7ad875ec97
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/btree.c   |  7 +++----
 3 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/manifest b/manifest
index 1bb19cf6b3..d4c218ede1 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Skip\sdefining\sWIN32_LEAN_AND_MEAN\swhen\sit\sis\salready\sdefined.
-D 2018-01-23T07:11:05.143
+C Fix\scomment\stypos.\s\sNo\schanges\sto\scode.
+D 2018-01-23T13:30:38.622
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -428,7 +428,7 @@ F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
-F src/btree.c bfc453babec9aa8196ab5db5e588ac5d3f0c398d72faa37296167a84a61c9f2f
+F src/btree.c 1beceb1c5f9563271241fd0c294484668e4ad28cf6aa970eab70e50ff6f75e25
 F src/btree.h 0866c0a08255142ea0e754aabd211c843cab32045c978a592a43152405ed0c84
 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc
 F src/build.c 9f9647454f236cab097f266ae970f899b53c71cadab6756c47e2b2e81392c2a1
@@ -1700,7 +1700,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 9a4199aedb039141a50a68943ef863d7df9637ada318793cce6c12b9d39bcdc5
-R fc52a77923ed3abf898f25d3a24aab55
-U mistachkin
-Z 3cd93d9679b9898485b5f29a3393d5bf
+P 195f5323df800f7963df2ea251c497c72e4d7ff0b88a70f1fa60f13bb1a9fd3b
+R 7c865206c0c32ef523d258872610a2c7
+U drh
+Z b85d901c66e70c5307b2badbaae893be
diff --git a/manifest.uuid b/manifest.uuid
index 3eb8b75e11..bd24eaa7d9 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-195f5323df800f7963df2ea251c497c72e4d7ff0b88a70f1fa60f13bb1a9fd3b
\ No newline at end of file
+8e5e74c66b9cdb348392e3db2d9f32cfa20fcec35bf09d9e1f623e7ad875ec97
\ No newline at end of file
diff --git a/src/btree.c b/src/btree.c
index 3b42f68f9f..168f3105fb 100644
--- a/src/btree.c
+++ b/src/btree.c
@@ -6189,9 +6189,8 @@ static void freePage(MemPage *pPage, int *pRC){
 }
 
 /*
-** Free any overflow pages associated with the given Cell.  Write the
-** local Cell size (the number of bytes on the original page, omitting
-** overflow) into *pnSize.
+** Free any overflow pages associated with the given Cell.  Store
+** size information about the cell in pInfo.
 */
 static int clearCell(
   MemPage *pPage,          /* The page that contains the Cell */
@@ -7395,7 +7394,7 @@ static int balance_nonroot(
     }
 
     /* Load b.apCell[] with pointers to all cells in pOld.  If pOld
-    ** constains overflow cells, include them in the b.apCell[] array
+    ** contains overflow cells, include them in the b.apCell[] array
     ** in the correct spot.
     **
     ** Note that when there are multiple overflow cells, it is always the

From b8fff29c68cdb68b472051c6f892216e388196d9 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 23 Jan 2018 14:01:51 +0000
Subject: [PATCH 369/488] In SQLITE_ENABLE_BATCH_ATOMIC_WRITE builds on  F2FS
 file-systems, invoke SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE if an
 SQLITE_FCNTL_COMMIT_ATOMIC_WRITE call fails. Also, do not use an atomic
 transaction to create the initial database. This is because if an error
 occurs while writing to the db file, any changes to the file-size do not seem
 to be rolled back automatically. The only time this matters is when the file
 was 0 bytes in size to start with.

FossilOrigin-Name: b3122db1545aeb48b7c28d480534b4b0fe04e83d5336225714c3cad926e5960e
---
 manifest             | 22 +++++-----
 manifest.uuid        |  2 +-
 src/pager.c          |  7 ++--
 test/malloc3.test    | 11 +++++
 test/misc7.test      | 24 ++++++-----
 test/pagerfault.test | 14 ++++---
 test/walthread.test  | 98 ++++++++++++++++++++++----------------------
 7 files changed, 99 insertions(+), 79 deletions(-)

diff --git a/manifest b/manifest
index d4c218ede1..a1f2ca4ed7 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\scomment\stypos.\s\sNo\schanges\sto\scode.
-D 2018-01-23T13:30:38.622
+C In\sSQLITE_ENABLE_BATCH_ATOMIC_WRITE\sbuilds\son\s\sF2FS\sfile-systems,\sinvoke\nSQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE\sif\san\sSQLITE_FCNTL_COMMIT_ATOMIC_WRITE\scall\nfails.\sAlso,\sdo\snot\suse\san\satomic\stransaction\sto\screate\sthe\sinitial\sdatabase.\nThis\sis\sbecause\sif\san\serror\soccurs\swhile\swriting\sto\sthe\sdb\sfile,\sany\schanges\nto\sthe\sfile-size\sdo\snot\sseem\sto\sbe\srolled\sback\sautomatically.\sThe\sonly\stime\nthis\smatters\sis\swhen\sthe\sfile\swas\s0\sbytes\sin\ssize\sto\sstart\swith.
+D 2018-01-23T14:01:51.708
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -473,7 +473,7 @@ F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
 F src/os_unix.c a82505be158d8ce42b38dcc9b426187d776904c12cdc68dc8925e1dfcc5cb6ce
 F src/os_win.c 501dde1ee770f4ffa458bfe1cf376a556de3ab00bb8320d659c5984403991d62
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
-F src/pager.c 9b9cb4e06c03d43d62480a7a685a012d645fcf3a39e7767ccb505fb41ee083ec
+F src/pager.c cd194a8793ce061e184ddc369fadbc1020c6f431014d22093f6c5e55c9234033
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
 F src/parse.y 4e750e1b261ff9f1d0b6b5d40a829c66d691899f48953fde839d8b52d41aa148
 F src/pcache.c 7ae91a4557a43d77d449accbfdc68846e6516f8e2eda46e8bbe4536fb669b201
@@ -1038,7 +1038,7 @@ F test/lookaside.test b17c99ae3aef96a8c9fa6f6be33cc75b93d657cb791d3827302b6835b7
 F test/main.test 6bbb3999fd461eb8fb335cbab97409a3d7f91bbb8da60635e8be3e4a04a77772
 F test/make-where7.tcl 05c16b5d4f5d6512881dfec560cb793915932ef9
 F test/malloc.test 21c213365f2cca95ab2d7dc078dc8525f96065f8
-F test/malloc3.test e3b32c724b5a124b57cb0ed177f675249ad0c66a
+F test/malloc3.test 6e88bae6312854a4adb4ecc2a6a5ea8c59b4db778b724ba718e1c43fc8c3c136
 F test/malloc4.test 957337613002b7058a85116493a262f679f3a261
 F test/malloc5.test f6eb6eca07a4c75f2897bf43a404689b6295bb95ab2e07d4b52eda743f925a27
 F test/malloc6.test 2f039d9821927eacae43e1831f815e157659a151
@@ -1076,7 +1076,7 @@ F test/misc3.test cf3dda47d5dda3e53fc5804a100d3c82be736c9d
 F test/misc4.test 0d8be3466adf123a7791a66ba2bc8e8d229e87f3
 F test/misc5.test 60e1fc758a93cacd19eb2fafcd1d40d150a05047546c7a92389c98047d621901
 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91
-F test/misc7.test 859894e3192257ce2fc4063b5438b220e352286974b387e485050f0ad1f665d6
+F test/misc7.test 567e223b6497da2226a0340befaf2d663c91ad57a48aede21a35a984a2882d41
 F test/misc8.test ba03aaa08f02d62fbb8d3b2f5595c1b33aa9bbc5
 F test/misuse.test 9e7f78402005e833af71dcab32d048003869eca5abcaccc985d4f8dc1d86bcc7
 F test/mjournal.test 9d86e697dcbc5da2c4e8caba9b176b5765fe65e80c88c278b8c09a917e436795
@@ -1121,7 +1121,7 @@ F test/pager1.test f596d3bd53ce96e1d87d44d223d2ae6c8867dd782c425e5eb28b5721fa6aa
 F test/pager2.test 67b8f40ae98112bcdba1f2b2d03ea83266418c71
 F test/pager3.test 4e9a83d6ca0838d7c602c9eb93d1357562d9059c1e02ffb138a8271020838370
 F test/pager4.test a122e9e6925d5b23b31e3dfef8c6a44bbf19590e
-F test/pagerfault.test 263c5442c06caf0b9b9e3fe42acdeb11f254dcebe533f69f401aaef9111eaf20
+F test/pagerfault.test 63c5da625562c66345ab4528790327ca63db2f6f9cbae2aba8cb7c51de3d1628
 F test/pagerfault2.test caf4c7facb914fd3b03a17b31ae2b180c8d6ca1f
 F test/pagerfault3.test 1003fcda009bf48a8e22a516e193b6ef0dd1bbd8
 F test/pageropt.test 84e4cc5cbca285357f7906e99b21be4f2bf5abc0
@@ -1557,7 +1557,7 @@ F test/walro2.test 6c73e8e4b5ccc55f907f4603ba36458b45c085fb6dfb04f30e3c0babbc1c2
 F test/walrofault.test c70cb6e308c443867701856cce92ad8288cd99488fa52afab77cca6cfd51af68
 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417
 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f
-F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e
+F test/walthread.test 14b20fcfa6ae152f5d8e12f5dc8a8a724b7ef189f5d8ef1e2ceab79f2af51747
 F test/where.test f0c325563acde44f2c4ea6ba348e9e29f7121757
 F test/where2.test 478d2170637b9211f593120648858593bf2445a1
 F test/where3.test 54cdeb02157acc979de41530b804ae7b09552bf1
@@ -1700,7 +1700,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 195f5323df800f7963df2ea251c497c72e4d7ff0b88a70f1fa60f13bb1a9fd3b
-R 7c865206c0c32ef523d258872610a2c7
-U drh
-Z b85d901c66e70c5307b2badbaae893be
+P 8e5e74c66b9cdb348392e3db2d9f32cfa20fcec35bf09d9e1f623e7ad875ec97
+R aeb88c53e35ebccb3804c1a2544a24b3
+U dan
+Z a8da060b421183a740953cd83b7ad3ac
diff --git a/manifest.uuid b/manifest.uuid
index bd24eaa7d9..d3c9f66b6a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-8e5e74c66b9cdb348392e3db2d9f32cfa20fcec35bf09d9e1f623e7ad875ec97
\ No newline at end of file
+b3122db1545aeb48b7c28d480534b4b0fe04e83d5336225714c3cad926e5960e
\ No newline at end of file
diff --git a/src/pager.c b/src/pager.c
index 295cbe04c5..e4680bfbfe 100644
--- a/src/pager.c
+++ b/src/pager.c
@@ -1213,7 +1213,7 @@ static int jrnlBufferSize(Pager *pPager){
 #endif
 
 #ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE
-  if( dc&SQLITE_IOCAP_BATCH_ATOMIC ){
+  if( pPager->dbSize>0 && (dc&SQLITE_IOCAP_BATCH_ATOMIC) ){
     return -1;
   }
 #endif
@@ -6499,8 +6499,9 @@ int sqlite3PagerCommitPhaseOne(
       if( bBatch ){
         if( rc==SQLITE_OK ){
           rc = sqlite3OsFileControl(fd, SQLITE_FCNTL_COMMIT_ATOMIC_WRITE, 0);
-        }else{
-          sqlite3OsFileControl(fd, SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE, 0);
+        }
+        if( rc!=SQLITE_OK ){
+          sqlite3OsFileControlHint(fd, SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE, 0);
         }
       }
 
diff --git a/test/malloc3.test b/test/malloc3.test
index f4a6c3bbe9..b497ab66e9 100644
--- a/test/malloc3.test
+++ b/test/malloc3.test
@@ -27,6 +27,17 @@ if {!$MEMDEBUG} {
    return
 }
 
+# Do not run these tests if F2FS batch writes are supported. In this case,
+# it is possible for a single DML statement in an implicit transaction
+# to fail with SQLITE_NOMEM, but for the transaction to still end up
+# committed to disk. Which confuses the tests in this module.
+#
+if {[atomic_batch_write test.db]} {
+   puts "Skipping malloc3 tests: atomic-batch support"
+   finish_test
+   return
+}
+
 
 # Do not run these tests with an in-memory journal.
 #
diff --git a/test/misc7.test b/test/misc7.test
index d0e84dfa47..8df95575c1 100644
--- a/test/misc7.test
+++ b/test/misc7.test
@@ -43,15 +43,17 @@ do_test misc7-4 {
 
 # Try to open a file with a directory where its journal file should be.
 #
-do_test misc7-5 {
-  delete_file mydir
-  file mkdir mydir-journal
-  sqlite3 db2 ./mydir
-  catchsql {
-    CREATE TABLE abc(a, b, c);
-  } db2
-} {1 {unable to open database file}}
-db2 close
+if {[atomic_batch_write test.db]==0} {
+  do_test misc7-5 {
+    delete_file mydir
+    file mkdir mydir-journal
+    sqlite3 db2 ./mydir
+    catchsql {
+      CREATE TABLE abc(a, b, c);
+    } db2
+  } {1 {unable to open database file}}
+  db2 close
+}
 
 #--------------------------------------------------------------------
 # The following tests, misc7-6.* test the libraries behaviour when
@@ -522,7 +524,9 @@ do_test misc7-22.4 {
 catch { db close }
 forcedelete test.db
 
-if {$::tcl_platform(platform)=="unix"} {
+if {$::tcl_platform(platform)=="unix"
+ && [atomic_batch_write test.db]==0
+} {
   reset_db
   do_execsql_test 23.0 {
     CREATE TABLE t1(x, y);
diff --git a/test/pagerfault.test b/test/pagerfault.test
index 392c1a2b98..3006dad7cc 100644
--- a/test/pagerfault.test
+++ b/test/pagerfault.test
@@ -1203,12 +1203,14 @@ do_faultsim_test pagerfault-26 -prep {
   set contents [db eval {SELECT * FROM t1}]
   if {$contents != "1 2"} { error "Bad database contents ($contents)" }
 
-  set sz [file size test.db]
-  if {$testrc!=0 && $sz!=1024*3 && $sz!=4096*3} { 
-    error "Expected file size to be 3072 or 12288 bytes - actual size $sz bytes"
-  }
-  if {$testrc==0 && $sz!=4096*3} { 
-    error "Expected file size to be 12288 bytes - actual size $sz bytes"
+  if {[atomic_batch_write test.db]==0} {
+    set sz [file size test.db]
+    if {$testrc!=0 && $sz!=1024*3 && $sz!=4096*3} { 
+      error "Expected file size 3072 or 12288 bytes - actual size $sz bytes"
+    }
+    if {$testrc==0 && $sz!=4096*3} { 
+      error "Expected file size to be 12288 bytes - actual size $sz bytes"
+    }
   }
 } 
 
diff --git a/test/walthread.test b/test/walthread.test
index 6249ce11af..8e5df9e589 100644
--- a/test/walthread.test
+++ b/test/walthread.test
@@ -327,59 +327,61 @@ do_thread_test2 walthread-1 -seconds $seconds(walthread-1) -init {
 # the number of write-transactions performed using a rollback journal.
 # For example, "192 w, 185 r".
 #
-do_thread_test2 walthread-2 -seconds $seconds(walthread-2) -init {
-  execsql { CREATE TABLE t1(x INTEGER PRIMARY KEY, y UNIQUE) }
-} -thread RB 2 {
+if {[atomic_batch_write test.db]==0} {
+  do_thread_test2 walthread-2 -seconds $seconds(walthread-2) -init {
+    execsql { CREATE TABLE t1(x INTEGER PRIMARY KEY, y UNIQUE) }
+  } -thread RB 2 {
 
-  db close
-  set nRun 0
-  set nDel 0
-  while {[tt_continue]} {
-    sqlite3 db test.db
-    db busy busyhandler
-    db eval { SELECT * FROM sqlite_master }
-    catch { db eval { PRAGMA journal_mode = DELETE } }
-    db eval {
-      BEGIN;
-      INSERT INTO t1 VALUES(NULL, randomblob(100+$E(pid)));
-    }
-    incr nRun 1
-    incr nDel [file exists test.db-journal]
-    if {[file exists test.db-journal] + [file exists test.db-wal] != 1} {
-      error "File-system looks bad..."
-    }
-    db eval COMMIT
-
-    integrity_check
     db close
-  }
-  list $nRun $nDel
-  set {} "[expr $nRun-$nDel] w, $nDel r"
-
-} -thread WAL 2 {
-  db close
-  set nRun 0
-  set nDel 0
-  while {[tt_continue]} {
-    sqlite3 db test.db
-    db busy busyhandler
-    db eval { SELECT * FROM sqlite_master }
-    catch { db eval { PRAGMA journal_mode = WAL } }
-    db eval {
-      BEGIN;
-      INSERT INTO t1 VALUES(NULL, randomblob(110+$E(pid)));
+    set nRun 0
+    set nDel 0
+    while {[tt_continue]} {
+      sqlite3 db test.db
+      db busy busyhandler
+      db eval { SELECT * FROM sqlite_master }
+      catch { db eval { PRAGMA journal_mode = DELETE } }
+      db eval {
+        BEGIN;
+        INSERT INTO t1 VALUES(NULL, randomblob(100+$E(pid)));
+      }
+      incr nRun 1
+      incr nDel [file exists test.db-journal]
+      if {[file exists test.db-journal] + [file exists test.db-wal] != 1} {
+        error "File-system looks bad..."
+      }
+      db eval COMMIT
+  
+      integrity_check
+      db close
     }
-    incr nRun 1
-    incr nDel [file exists test.db-journal]
-    if {[file exists test.db-journal] + [file exists test.db-wal] != 1} {
-      error "File-system looks bad..."
-    }
-    db eval COMMIT
-
-    integrity_check
+    list $nRun $nDel
+    set {} "[expr $nRun-$nDel] w, $nDel r"
+  
+  } -thread WAL 2 {
     db close
+    set nRun 0
+    set nDel 0
+    while {[tt_continue]} {
+      sqlite3 db test.db
+      db busy busyhandler
+      db eval { SELECT * FROM sqlite_master }
+      catch { db eval { PRAGMA journal_mode = WAL } }
+      db eval {
+        BEGIN;
+        INSERT INTO t1 VALUES(NULL, randomblob(110+$E(pid)));
+      }
+      incr nRun 1
+      incr nDel [file exists test.db-journal]
+      if {[file exists test.db-journal] + [file exists test.db-wal] != 1} {
+        error "File-system looks bad..."
+      }
+      db eval COMMIT
+  
+      integrity_check
+      db close
+    }
+    set {} "[expr $nRun-$nDel] w, $nDel r"
   }
-  set {} "[expr $nRun-$nDel] w, $nDel r"
 }
 
 do_thread_test walthread-3 -seconds $seconds(walthread-3) -init {

From 61d04974f06ad307a56442602c910783f7fa02a0 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 23 Jan 2018 15:26:30 +0000
Subject: [PATCH 370/488] Fix the modification-time setting logic in the
 fileio.c extension on Windows so that it works with utf8 filenames.

FossilOrigin-Name: f785b9041556912edbacdbfb3dfc38705058d7c10d874544295c25db54628bc2
---
 ext/misc/fileio.c |  9 +++++++--
 manifest          | 14 +++++++-------
 manifest.uuid     |  2 +-
 3 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c
index 60a960f310..e51b3e620b 100644
--- a/ext/misc/fileio.c
+++ b/ext/misc/fileio.c
@@ -270,15 +270,20 @@ static int writeFile(
     SYSTEMTIME currentTime;
     LONGLONG intervals;
     HANDLE hFile;
+    LPWSTR zUnicodeName;
+    extern LPWSTR sqlite3_win32_utf8_to_unicode(const char*);
+
     GetSystemTime(&currentTime);
     SystemTimeToFileTime(&currentTime, &lastAccess);
     intervals = Int32x32To64(mtime, 10000000) + 116444736000000000;
     lastWrite.dwLowDateTime = (DWORD)intervals;
     lastWrite.dwHighDateTime = intervals >> 32;
-    hFile = CreateFile(
-      zFile, FILE_WRITE_ATTRIBUTES, 0, NULL, OPEN_EXISTING,
+    zUnicodeName = sqlite3_win32_utf8_to_unicode(zFile);
+    hFile = CreateFileW(
+      zUnicodeName, FILE_WRITE_ATTRIBUTES, 0, NULL, OPEN_EXISTING,
       FILE_FLAG_BACKUP_SEMANTICS, NULL
     );
+    sqlite3_free(zUnicodeName);
     if( hFile!=INVALID_HANDLE_VALUE ){
       BOOL bResult = SetFileTime(hFile, NULL, &lastAccess, &lastWrite);
       CloseHandle(hFile);
diff --git a/manifest b/manifest
index a1f2ca4ed7..c77f83cc4b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\sSQLITE_ENABLE_BATCH_ATOMIC_WRITE\sbuilds\son\s\sF2FS\sfile-systems,\sinvoke\nSQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE\sif\san\sSQLITE_FCNTL_COMMIT_ATOMIC_WRITE\scall\nfails.\sAlso,\sdo\snot\suse\san\satomic\stransaction\sto\screate\sthe\sinitial\sdatabase.\nThis\sis\sbecause\sif\san\serror\soccurs\swhile\swriting\sto\sthe\sdb\sfile,\sany\schanges\nto\sthe\sfile-size\sdo\snot\sseem\sto\sbe\srolled\sback\sautomatically.\sThe\sonly\stime\nthis\smatters\sis\swhen\sthe\sfile\swas\s0\sbytes\sin\ssize\sto\sstart\swith.
-D 2018-01-23T14:01:51.708
+C Fix\sthe\smodification-time\ssetting\slogic\sin\sthe\sfileio.c\sextension\son\sWindows\nso\sthat\sit\sworks\swith\sutf8\sfilenames.
+D 2018-01-23T15:26:30.163
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -277,7 +277,7 @@ F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f0
 F ext/misc/csv.c 1a009b93650732e22334edc92459c4630b9fa703397cbb3c8ca279921a36ca11
 F ext/misc/dbdump.c 3509fa6b8932d04e932d6b6b827b6a82ca362781b8e8f3c77336f416793e215e
 F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2
-F ext/misc/fileio.c 06bd79dcc43d0887da27ffaadd69b8a698b1bafe203d1d134a3a2964f69368f9
+F ext/misc/fileio.c bd2f717be63a9ae9ff85977a756c43a96d47a7763d98ae913636c64b714c232b
 F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25
 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c
 F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281984
@@ -1700,7 +1700,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 8e5e74c66b9cdb348392e3db2d9f32cfa20fcec35bf09d9e1f623e7ad875ec97
-R aeb88c53e35ebccb3804c1a2544a24b3
-U dan
-Z a8da060b421183a740953cd83b7ad3ac
+P b3122db1545aeb48b7c28d480534b4b0fe04e83d5336225714c3cad926e5960e
+R e90c01c3df65c4d4d3208d8c47984c27
+U drh
+Z 0631a5977bb4ccdfaf551e9a8368b5ad
diff --git a/manifest.uuid b/manifest.uuid
index d3c9f66b6a..35210204e2 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b3122db1545aeb48b7c28d480534b4b0fe04e83d5336225714c3cad926e5960e
\ No newline at end of file
+f785b9041556912edbacdbfb3dfc38705058d7c10d874544295c25db54628bc2
\ No newline at end of file

From a916b570137f728a04d4a3be0cb792dd5a22cf63 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 23 Jan 2018 16:38:57 +0000
Subject: [PATCH 371/488] Fix a bug causing spurious "sub-select returns N
 columns expected 1" errors in join queries with a term like "(a, b) IN
 (SELECT ...)" in the WHERE clause.

FossilOrigin-Name: 14dfd96f9bca2df5033b2d894bf63cc8bf450a45ca11df5e3bbb814fdf96b656
---
 manifest           | 16 ++++++++--------
 manifest.uuid      |  2 +-
 src/wherecode.c    |  6 ++++++
 test/rowvalue.test | 45 +++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 60 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index c77f83cc4b..7ea1bf4a4f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sthe\smodification-time\ssetting\slogic\sin\sthe\sfileio.c\sextension\son\sWindows\nso\sthat\sit\sworks\swith\sutf8\sfilenames.
-D 2018-01-23T15:26:30.163
+C Fix\sa\sbug\scausing\sspurious\s"sub-select\sreturns\sN\scolumns\sexpected\s1"\serrors\nin\sjoin\squeries\swith\sa\sterm\slike\s"(a,\sb)\sIN\s(SELECT\s...)"\sin\sthe\sWHERE\sclause.
+D 2018-01-23T16:38:57.346
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -571,7 +571,7 @@ F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f
 F src/where.c caf0b6c9d31f22f0b2c91aba723858de52b5d665aaa89034099015aaf9bb8219
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
-F src/wherecode.c af1e79154aaa88cd802d6f2e5b945f67eaca7c958d1525fbf8ee19d5bd7b9020
+F src/wherecode.c cb360c511f69294ddf00340d7b390e6b1d601a1cfb77b42c4d316fe2f4cd01c3
 F src/whereexpr.c 427ea8e96ec24f2a7814c67b8024ad664a9c7656264c4566c34743cb23186e46
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd
@@ -1163,7 +1163,7 @@ F test/rollbackfault.test 0e646aeab8840c399cfbfa43daab46fd609cf04a
 F test/rowallock.test 3f88ec6819489d0b2341c7a7528ae17c053ab7cc
 F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81
 F test/rowid.test 5b7509f384f4f6fae1af3c8c104c8ca299fea18d
-F test/rowvalue.test 44f3492f415cc9f374e8388a5eb61503eaca5230
+F test/rowvalue.test e1bd1690d891abff6138f9612241615d2de7671cb28eb58ebdd591a5b01089a4
 F test/rowvalue2.test 060d238b7e5639a7c5630cb5e63e311b44efef2b
 F test/rowvalue3.test 3068f508753af69884b12125995f023da0dbb256
 F test/rowvalue4.test 4b556d7de161a0dd8cff095c336e913986398bea
@@ -1700,7 +1700,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b3122db1545aeb48b7c28d480534b4b0fe04e83d5336225714c3cad926e5960e
-R e90c01c3df65c4d4d3208d8c47984c27
-U drh
-Z 0631a5977bb4ccdfaf551e9a8368b5ad
+P f785b9041556912edbacdbfb3dfc38705058d7c10d874544295c25db54628bc2
+R e2c75936c44819f701f68ae9ff6a70d2
+U dan
+Z d0f66b0bfa51e27a550c6dffd5ce24ce
diff --git a/manifest.uuid b/manifest.uuid
index 35210204e2..d83d65e8de 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f785b9041556912edbacdbfb3dfc38705058d7c10d874544295c25db54628bc2
\ No newline at end of file
+14dfd96f9bca2df5033b2d894bf63cc8bf450a45ca11df5e3bbb814fdf96b656
\ No newline at end of file
diff --git a/src/wherecode.c b/src/wherecode.c
index 32dd2048bf..8e8672ff9f 100644
--- a/src/wherecode.c
+++ b/src/wherecode.c
@@ -2170,6 +2170,12 @@ Bitmask sqlite3WhereCodeOneLoopStart(
                     WO_EQ|WO_IN|WO_IS, 0);
     if( pAlt==0 ) continue;
     if( pAlt->wtFlags & (TERM_CODED) ) continue;
+    if( (pAlt->eOperator & WO_IN) 
+     && (pAlt->pExpr->flags & EP_xIsSelect)
+     && (pAlt->pExpr->x.pSelect->pEList->nExpr>1)
+    ){
+      continue;
+    }
     testcase( pAlt->eOperator & WO_EQ );
     testcase( pAlt->eOperator & WO_IS );
     testcase( pAlt->eOperator & WO_IN );
diff --git a/test/rowvalue.test b/test/rowvalue.test
index 5f2701c733..d900e0daa3 100644
--- a/test/rowvalue.test
+++ b/test/rowvalue.test
@@ -394,4 +394,49 @@ do_execsql_test 16.5 {
   3 i ii iii iv
 }
 
+do_execsql_test 17.0 {
+  CREATE TABLE b1(a, b);
+  CREATE TABLE b2(x);
+}
+
+do_execsql_test 17.1 {
+  SELECT * FROM b2 CROSS JOIN b1 
+  WHERE b2.x=b1.a AND (b1.a, 2) 
+  IN (VALUES(1, 2));
+} {}
+
+do_execsql_test 18.0 {
+  CREATE TABLE b3 ( a, b, PRIMARY KEY (a, b) );
+  CREATE TABLE b4 ( a );
+  CREATE TABLE b5 ( a, b );
+  INSERT INTO b3 VALUES (1, 1), (1, 2);
+  INSERT INTO b4 VALUES (1);
+  INSERT INTO b5 VALUES (1, 1), (1, 2);
+}
+
+do_execsql_test 18.1 {
+  SELECT * FROM b3 WHERE (SELECT b3.a, b3.b) IN ( SELECT a, b FROM b5 )
+} {1 1 1 2}
+do_execsql_test 18.2 {
+  SELECT * FROM b3 WHERE (VALUES(b3.a, b3.b)) IN ( SELECT a, b FROM b5 );
+} {1 1 1 2}
+do_execsql_test 18.3 {
+  SELECT * FROM b3 WHERE (b3.a, b3.b) IN ( SELECT a, b FROM b5 ); 
+} {1 1 1 2}
+do_execsql_test 18.4 {
+  SELECT * FROM b3 JOIN b4 ON b4.a = b3.a
+  WHERE (SELECT b3.a, b3.b) IN ( SELECT a, b FROM b5 ); 
+} {1 1 1 1 2 1}
+do_execsql_test 18.5 {
+  SELECT * FROM b3 JOIN b4 ON b4.a = b3.a
+  WHERE (VALUES(b3.a, b3.b)) IN ( SELECT a, b FROM b5 ); 
+} {1 1 1 1 2 1}
+do_execsql_test 18.6 {
+  SELECT * FROM b3 JOIN b4 ON b4.a = b3.a
+  WHERE (b3.a, b3.b) IN ( SELECT a, b FROM b5 ); 
+} {1 1 1 1 2 1}
+
+ 
+finish_test
+
 finish_test

From d15046ac080f4633186fe0e31712525a4326a4bc Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 23 Jan 2018 17:33:42 +0000
Subject: [PATCH 372/488] Work around a problem with GCC on 32-bit machines
 that cause the CAST operator to generate a floating-point result for strings
 that could be represented as very large integers.

FossilOrigin-Name: 1b02731962c21bb097a88801ece76ff441bf882519a821a246da84f4e2a33455
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/vdbemem.c | 14 +++++++++++++-
 3 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index 7ea1bf4a4f..7691a2251e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sbug\scausing\sspurious\s"sub-select\sreturns\sN\scolumns\sexpected\s1"\serrors\nin\sjoin\squeries\swith\sa\sterm\slike\s"(a,\sb)\sIN\s(SELECT\s...)"\sin\sthe\sWHERE\sclause.
-D 2018-01-23T16:38:57.346
+C Work\saround\sa\sproblem\swith\sGCC\son\s32-bit\smachines\sthat\scause\sthe\sCAST\noperator\sto\sgenerate\sa\sfloating-point\sresult\sfor\sstrings\sthat\scould\sbe\nrepresented\sas\svery\slarge\sintegers.
+D 2018-01-23T17:33:42.218
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -561,7 +561,7 @@ F src/vdbeInt.h 8d7d07f13cb3c4cbca91e22ba4a1920e542dda7c5d9299920432a0b3d5b009f5
 F src/vdbeapi.c fea41171884a4de119f8b10ab514c788674eeeb7f27218bb6d008e1310bfd07f
 F src/vdbeaux.c 2756ac68ac259c416554100598fc291870063288cd7e1af22847f57b3e130e56
 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191
-F src/vdbemem.c 7548dd5af03d24d534a5dbc41e3bbdf1fab83e9c8856a8d2549ed2ccf33d0e80
+F src/vdbemem.c 943e41881e6317c9f93c77c1d60d3b37ddc8d26a3f852233ce7423d3e581523e
 F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a
@@ -1700,7 +1700,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P f785b9041556912edbacdbfb3dfc38705058d7c10d874544295c25db54628bc2
-R e2c75936c44819f701f68ae9ff6a70d2
-U dan
-Z d0f66b0bfa51e27a550c6dffd5ce24ce
+P 14dfd96f9bca2df5033b2d894bf63cc8bf450a45ca11df5e3bbb814fdf96b656
+R aad1a01a1cf0d877a8df52447d73431f
+U drh
+Z 57954c94bf10856d82150b78a27b3537
diff --git a/manifest.uuid b/manifest.uuid
index d83d65e8de..8e916bf612 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-14dfd96f9bca2df5033b2d894bf63cc8bf450a45ca11df5e3bbb814fdf96b656
\ No newline at end of file
+1b02731962c21bb097a88801ece76ff441bf882519a821a246da84f4e2a33455
\ No newline at end of file
diff --git a/src/vdbemem.c b/src/vdbemem.c
index d8f1e64328..e2912b3a6f 100644
--- a/src/vdbemem.c
+++ b/src/vdbemem.c
@@ -582,6 +582,18 @@ int sqlite3VdbeMemRealify(Mem *pMem){
   return SQLITE_OK;
 }
 
+/* Compare a floating point value to an integer.  Return true if the two
+** values are the same within the precision of the floating point value.
+**
+** For some versions of GCC on 32-bit machines, if you do the more obvious
+** comparison of "r1==(double)i" you sometimes get an answer of false even
+** though the r1 and (double)i values are bit-for-bit the same.
+*/
+static int sqlite3RealSameAsInt(double r1, sqlite3_int64 i){
+  double r2 = (double)i;
+  return memcmp(&r1, &r2, sizeof(r1))==0;
+}
+
 /*
 ** Convert pMem so that it has types MEM_Real or MEM_Int or both.
 ** Invalidate any prior representations.
@@ -601,7 +613,7 @@ int sqlite3VdbeMemNumerify(Mem *pMem){
     }else{
       i64 i = pMem->u.i;
       sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc);
-      if( rc==1 && pMem->u.r==(double)i ){
+      if( rc==1 && sqlite3RealSameAsInt(pMem->u.r, i) ){
         pMem->u.i = i;
         MemSetTypeFlag(pMem, MEM_Int);
       }else{

From 21d59784227e227674f6a313090e5b84291dfcbe Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 23 Jan 2018 19:24:54 +0000
Subject: [PATCH 373/488] Remove an unreachable branch from
 sqlite3SkipAccumulatorLoad().

FossilOrigin-Name: 8b9c8eab60b692e45c55a718d76b70e63040416040696907463692cbf83fb0c5
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/func.c    |  3 ++-
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 7691a2251e..3592d2f2a5 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Work\saround\sa\sproblem\swith\sGCC\son\s32-bit\smachines\sthat\scause\sthe\sCAST\noperator\sto\sgenerate\sa\sfloating-point\sresult\sfor\sstrings\sthat\scould\sbe\nrepresented\sas\svery\slarge\sintegers.
-D 2018-01-23T17:33:42.218
+C Remove\san\sunreachable\sbranch\sfrom\ssqlite3SkipAccumulatorLoad().
+D 2018-01-23T19:24:54.783
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -442,7 +442,7 @@ F src/delete.c 20c8788451dc737a967c87ea53ad43544d617f5b57d32ccce8bd52a0daf9e89b
 F src/expr.c 9e06de431c09f144438aa6895ea4d4290fa3c6875bfcc3ba331012ca78deadf0
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331
-F src/func.c 227d17a030b75f7495fc3b8d1aca19850e794397433175c1ba21b713d9833f14
+F src/func.c 9970db37cc004136996a5c9b966b86f06583bcf3f275449b977fbb06d75e7300
 F src/global.c ac3094f1dc59fbeb919aef7cc0cc827a8459d1fb1adb7972ef75bd9e0c10b75b
 F src/hash.c a12580e143f10301ed5166ea4964ae2853d3905a511d4e0c44497245c7ce1f7a
 F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4
@@ -1700,7 +1700,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 14dfd96f9bca2df5033b2d894bf63cc8bf450a45ca11df5e3bbb814fdf96b656
-R aad1a01a1cf0d877a8df52447d73431f
+P 1b02731962c21bb097a88801ece76ff441bf882519a821a246da84f4e2a33455
+R 69d183e51c910b309adf151023208c58
 U drh
-Z 57954c94bf10856d82150b78a27b3537
+Z 10ad556e955465f2ef5a7781922bad82
diff --git a/manifest.uuid b/manifest.uuid
index 8e916bf612..4cadc19d58 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-1b02731962c21bb097a88801ece76ff441bf882519a821a246da84f4e2a33455
\ No newline at end of file
+8b9c8eab60b692e45c55a718d76b70e63040416040696907463692cbf83fb0c5
\ No newline at end of file
diff --git a/src/func.c b/src/func.c
index 36cfa44230..4e80535e81 100644
--- a/src/func.c
+++ b/src/func.c
@@ -35,7 +35,8 @@ static CollSeq *sqlite3GetFuncCollSeq(sqlite3_context *context){
 ** iteration of the aggregate loop.
 */
 static void sqlite3SkipAccumulatorLoad(sqlite3_context *context){
-  if( context->isError==0 ) context->isError = -1;
+  assert( context->isError<=0 );
+  context->isError = -1;
   context->skipFlag = 1;
 }
 

From 588049cc997bbd6ea5b62b4ac336916517348a01 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 23 Jan 2018 20:22:15 +0000
Subject: [PATCH 374/488] Increase the version number to 3.23.0 for the next
 development cycle.

FossilOrigin-Name: b58b60b2c0729b73c0ef35bd1a5aa90965d1c01043413ca45b412d8b68f59bce
---
 VERSION       |  2 +-
 configure     | 18 +++++++++---------
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 4 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/VERSION b/VERSION
index a7e7070f80..ee893b7e0e 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-3.22.0
+3.23.0
diff --git a/configure b/configure
index 8e735c6b12..3df19e8ec5 100755
--- a/configure
+++ b/configure
@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for sqlite 3.22.0.
+# Generated by GNU Autoconf 2.69 for sqlite 3.23.0.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -726,8 +726,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='sqlite'
 PACKAGE_TARNAME='sqlite'
-PACKAGE_VERSION='3.22.0'
-PACKAGE_STRING='sqlite 3.22.0'
+PACKAGE_VERSION='3.23.0'
+PACKAGE_STRING='sqlite 3.23.0'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''
 
@@ -1465,7 +1465,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures sqlite 3.22.0 to adapt to many kinds of systems.
+\`configure' configures sqlite 3.23.0 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1530,7 +1530,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of sqlite 3.22.0:";;
+     short | recursive ) echo "Configuration of sqlite 3.23.0:";;
    esac
   cat <<\_ACEOF
 
@@ -1655,7 +1655,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-sqlite configure 3.22.0
+sqlite configure 3.23.0
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2074,7 +2074,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by sqlite $as_me 3.22.0, which was
+It was created by sqlite $as_me 3.23.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -12242,7 +12242,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by sqlite $as_me 3.22.0, which was
+This file was extended by sqlite $as_me 3.23.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -12308,7 +12308,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-sqlite config.status 3.22.0
+sqlite config.status 3.23.0
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 
diff --git a/manifest b/manifest
index 3592d2f2a5..e85d602ab7 100644
--- a/manifest
+++ b/manifest
@@ -1,12 +1,12 @@
-C Remove\san\sunreachable\sbranch\sfrom\ssqlite3SkipAccumulatorLoad().
-D 2018-01-23T19:24:54.783
+C Increase\sthe\sversion\snumber\sto\s3.23.0\sfor\sthe\snext\sdevelopment\scycle.
+D 2018-01-23T20:22:15.707
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc ede26e3fb675e0b3b07627640ce5917154a6ee7f8f2c97424eb5ab5f651cbd56
 F README.md d748f58e3ab0fe0307fb4ae0942b415d93dcc4288756e366cc9e7cf8260c093f
-F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf
+F VERSION cdf91ac446255ecf3d8f6d8c3ee40d64123235ae5b3cef29d344e61b45ec3759
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2
 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90
@@ -32,7 +32,7 @@ F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63
 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977
 F config.h.in 6376abec766e9a0785178b1823b5a587e9f1ccbc
 F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55
-F configure 9af547be0e0e1a8fca8553b82599b5a3be1528a3d78deb68cb49d3b611215cb7 x
+F configure 2c71f331b463e987567a2dd942f728534f1aa7a174551e08a7b31b328e9da4ff x
 F configure.ac d4529ebb26ae046269334f1dac65f2b1d6927c2efe22b2ec24dce24dfe4f83dd
 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad
 F doc/lemon.html 278113807f49d12d04179a93fab92b5b917a08771152ca7949d34e928efa3941
@@ -1700,7 +1700,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 1b02731962c21bb097a88801ece76ff441bf882519a821a246da84f4e2a33455
-R 69d183e51c910b309adf151023208c58
+P 8b9c8eab60b692e45c55a718d76b70e63040416040696907463692cbf83fb0c5
+R 9d516b9e9b7b18aa4204187b9b851494
 U drh
-Z 10ad556e955465f2ef5a7781922bad82
+Z 99c4b4459effdc8153fba53651e59b2d
diff --git a/manifest.uuid b/manifest.uuid
index 4cadc19d58..18af264478 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-8b9c8eab60b692e45c55a718d76b70e63040416040696907463692cbf83fb0c5
\ No newline at end of file
+b58b60b2c0729b73c0ef35bd1a5aa90965d1c01043413ca45b412d8b68f59bce
\ No newline at end of file

From 5347f3c111edac51f199191b1be942327fed08b7 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 24 Jan 2018 01:02:23 +0000
Subject: [PATCH 375/488] Invoke the sqlite3_complete() interface from the
 fuzzer.

FossilOrigin-Name: 332bf84625d2034c9e1c029aa8243aa34088217e52e34e1f5472217743a31af3
---
 manifest       | 12 ++++++------
 manifest.uuid  |  2 +-
 test/ossfuzz.c |  1 +
 3 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index e85d602ab7..a9e8ac9a9c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Increase\sthe\sversion\snumber\sto\s3.23.0\sfor\sthe\snext\sdevelopment\scycle.
-D 2018-01-23T20:22:15.707
+C Invoke\sthe\ssqlite3_complete()\sinterface\sfrom\sthe\sfuzzer.
+D 2018-01-24T01:02:23.624
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -1114,7 +1114,7 @@ F test/orderby7.test 3d1383d52ade5b9eb3a173b3147fdd296f0202da
 F test/orderby8.test 23ef1a5d72bd3adcc2f65561c654295d1b8047bd
 F test/orderby9.test 87fb9548debcc2cd141c5299002dd94672fa76a3
 F test/oserror.test b32dc34f2363ef18532e3a0a7358e3e7e321974f
-F test/ossfuzz.c 7f5cc87a0280a5854c1bfa7d5c4d07d34731f08ec34dc9c916aa35ed292b1468
+F test/ossfuzz.c b4bb024f9713a4c4e6442df1e6882ef53e29b4a3c27190a0562c16edab69052a
 F test/ossshell.c 296ab63067841bd1b1e97b46a0b2af48ee7f69d50d1a723008bee12dd7122622
 F test/ovfl.test 199c482696defceacee8c8e0e0ef36da62726b2f
 F test/pager1.test f596d3bd53ce96e1d87d44d223d2ae6c8867dd782c425e5eb28b5721fa6aaa97
@@ -1700,7 +1700,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 8b9c8eab60b692e45c55a718d76b70e63040416040696907463692cbf83fb0c5
-R 9d516b9e9b7b18aa4204187b9b851494
+P b58b60b2c0729b73c0ef35bd1a5aa90965d1c01043413ca45b412d8b68f59bce
+R 77d670e2fdf91ca12668cb80fdb4cad4
 U drh
-Z 99c4b4459effdc8153fba53651e59b2d
+Z 7daabf9677fc11b22b33a4599e9eabe2
diff --git a/manifest.uuid b/manifest.uuid
index 18af264478..3a0f01961b 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b58b60b2c0729b73c0ef35bd1a5aa90965d1c01043413ca45b412d8b68f59bce
\ No newline at end of file
+332bf84625d2034c9e1c029aa8243aa34088217e52e34e1f5472217743a31af3
\ No newline at end of file
diff --git a/test/ossfuzz.c b/test/ossfuzz.c
index 7b28cf6a7e..50410e4597 100644
--- a/test/ossfuzz.c
+++ b/test/ossfuzz.c
@@ -160,6 +160,7 @@ int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
   /* Run the SQL.  The sqlite_exec() interface expects a zero-terminated
   ** string, so make a copy. */
   zSql = sqlite3_mprintf("%.*s", (int)size, data);
+  sqlite3_complete(zSql);
   sqlite3_exec(cx.db, zSql, exec_handler, (void*)&execCnt, &zErrMsg);
 
   /* Show any errors */

From 56f177460314ceba1ae766ed486e079c71e0be72 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 24 Jan 2018 01:58:49 +0000
Subject: [PATCH 376/488] Make the shell functional even if compiled with
 SQLITE_OMIT_COMPLETE. Omit the sqlite3_complete() call from the fuzzing
 interface if it is compiled with SQLITE_OMIT_COMPLETE.

FossilOrigin-Name: c3e816cca4ddf0967c4c790cdde2345101dde3f2d854e62589f1ac1d7f3c60b8
---
 manifest       | 14 +++++++-------
 manifest.uuid  |  2 +-
 src/shell.c.in | 10 ++++++++++
 test/ossfuzz.c |  2 ++
 4 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index a9e8ac9a9c..c8b0baec36 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Invoke\sthe\ssqlite3_complete()\sinterface\sfrom\sthe\sfuzzer.
-D 2018-01-24T01:02:23.624
+C Make\sthe\sshell\sfunctional\seven\sif\scompiled\swith\sSQLITE_OMIT_COMPLETE.\nOmit\sthe\ssqlite3_complete()\scall\sfrom\sthe\sfuzzing\sinterface\sif\sit\sis\ncompiled\swith\sSQLITE_OMIT_COMPLETE.
+D 2018-01-24T01:58:49.043
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -487,7 +487,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c bebe7cce45d899d2237c76bce059d525abf5b861f2fce92f6b53914a961c01ba
-F src/shell.c.in 06ffe417973eacbee77a8402e26553b938e7c07f735c811701fc5cd1e642fb5c
+F src/shell.c.in 7cea439c3f7f2e4ed6eb4b3a633cd93dccb1349241400de4da0c1291285ed514
 F src/sqlite.h.in 959deaad89679e31d7f68fda668b0c5d1f592fffed7a9c1740fb8ded4e4e754a
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 99189e7611eb0bf98f21c7835dc74730a84e2e809c98e1e31c33896dee7a2849
@@ -1114,7 +1114,7 @@ F test/orderby7.test 3d1383d52ade5b9eb3a173b3147fdd296f0202da
 F test/orderby8.test 23ef1a5d72bd3adcc2f65561c654295d1b8047bd
 F test/orderby9.test 87fb9548debcc2cd141c5299002dd94672fa76a3
 F test/oserror.test b32dc34f2363ef18532e3a0a7358e3e7e321974f
-F test/ossfuzz.c b4bb024f9713a4c4e6442df1e6882ef53e29b4a3c27190a0562c16edab69052a
+F test/ossfuzz.c 3613bc516386234cf2e513fb94dc677ab3862eb7ebc5b3671c319a80f86839fb
 F test/ossshell.c 296ab63067841bd1b1e97b46a0b2af48ee7f69d50d1a723008bee12dd7122622
 F test/ovfl.test 199c482696defceacee8c8e0e0ef36da62726b2f
 F test/pager1.test f596d3bd53ce96e1d87d44d223d2ae6c8867dd782c425e5eb28b5721fa6aaa97
@@ -1700,7 +1700,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b58b60b2c0729b73c0ef35bd1a5aa90965d1c01043413ca45b412d8b68f59bce
-R 77d670e2fdf91ca12668cb80fdb4cad4
+P 332bf84625d2034c9e1c029aa8243aa34088217e52e34e1f5472217743a31af3
+R b1d50b050a44eb29893416acc04ad8bf
 U drh
-Z 7daabf9677fc11b22b33a4599e9eabe2
+Z 6cd3cb0cbe81acd6296afde81215523a
diff --git a/manifest.uuid b/manifest.uuid
index 3a0f01961b..2e7a511d22 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-332bf84625d2034c9e1c029aa8243aa34088217e52e34e1f5472217743a31af3
\ No newline at end of file
+c3e816cca4ddf0967c4c790cdde2345101dde3f2d854e62589f1ac1d7f3c60b8
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index e3a1f30752..a785517503 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -7680,6 +7680,16 @@ static int line_is_command_terminator(const char *zLine){
   return 0;
 }
 
+/*
+** We need a default sqlite3_complete() implementation to use in case
+** the shell is compiled with SQLITE_OMIT_COMPLETE.  The default assumes
+** any arbitrary text is a complete SQL statement.  This is not very
+** user-friendly, but it does seem to work.
+*/
+#ifdef SQLITE_OMIT_COMPLETE
+int sqlite3_complete(const char *zSql){ return 1; }
+#endif
+
 /*
 ** Return true if zSql is a complete SQL statement.  Return false if it
 ** ends in the middle of a string literal or C-style comment.
diff --git a/test/ossfuzz.c b/test/ossfuzz.c
index 50410e4597..fa6e9142fe 100644
--- a/test/ossfuzz.c
+++ b/test/ossfuzz.c
@@ -160,7 +160,9 @@ int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
   /* Run the SQL.  The sqlite_exec() interface expects a zero-terminated
   ** string, so make a copy. */
   zSql = sqlite3_mprintf("%.*s", (int)size, data);
+#ifndef SQLITE_OMIT_COMPLETE
   sqlite3_complete(zSql);
+#endif
   sqlite3_exec(cx.db, zSql, exec_handler, (void*)&execCnt, &zErrMsg);
 
   /* Show any errors */

From 0aac5613d924af3cf62d4ce159ab1d1f8ba932d4 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 24 Jan 2018 06:30:10 +0000
Subject: [PATCH 377/488] Update a couple of test scripts so that they work on
 F2FS file-systems that support atomic transactions.

FossilOrigin-Name: 49e58e645e0c114c71935a3b7fa4771e8a23e28127efd2efe0f2f18813c8f391
---
 manifest         | 16 ++++++++--------
 manifest.uuid    |  2 +-
 test/ioerr.test  |  4 ++--
 test/malloc.test |  2 +-
 4 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/manifest b/manifest
index c8b0baec36..062910f484 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Make\sthe\sshell\sfunctional\seven\sif\scompiled\swith\sSQLITE_OMIT_COMPLETE.\nOmit\sthe\ssqlite3_complete()\scall\sfrom\sthe\sfuzzing\sinterface\sif\sit\sis\ncompiled\swith\sSQLITE_OMIT_COMPLETE.
-D 2018-01-24T01:58:49.043
+C Update\sa\scouple\sof\stest\sscripts\sso\sthat\sthey\swork\son\sF2FS\sfile-systems\sthat\nsupport\satomic\stransactions.
+D 2018-01-24T06:30:10.922
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -993,7 +993,7 @@ F test/interrupt.test 16ea879ec728cb76414c148c5f24afd5d1f91054
 F test/interrupt2.test e4408ca770a6feafbadb0801e54a0dcd1a8d108d
 F test/intpkey.test ac71107a49a06492b69b82aafaf225400598d3c8
 F test/io.test f95bca1783b01ea7761671560d023360d2dfa4cc
-F test/ioerr.test 2a24bd6ed5a8b062e64bfe1f6cf94fb25e92210d
+F test/ioerr.test 470fcc78e9cd352d162baf782fe301ea807d764241f58a48fc58109c2dfcdb6b
 F test/ioerr2.test 2593563599e2cc6b6b4fcf5878b177bdd5d8df26
 F test/ioerr3.test d3cec5e1a11ad6d27527d0d38573fbff14c71bdd
 F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c
@@ -1037,7 +1037,7 @@ F test/lock_common.tcl 7ffb45accf6ee91c736df9bafe0806a44358f035
 F test/lookaside.test b17c99ae3aef96a8c9fa6f6be33cc75b93d657cb791d3827302b6835b71941f7
 F test/main.test 6bbb3999fd461eb8fb335cbab97409a3d7f91bbb8da60635e8be3e4a04a77772
 F test/make-where7.tcl 05c16b5d4f5d6512881dfec560cb793915932ef9
-F test/malloc.test 21c213365f2cca95ab2d7dc078dc8525f96065f8
+F test/malloc.test 18dd1c4188c81ca79cf123527c71b19ee0c31feb9947fdffb0dc6ceb1436816a
 F test/malloc3.test 6e88bae6312854a4adb4ecc2a6a5ea8c59b4db778b724ba718e1c43fc8c3c136
 F test/malloc4.test 957337613002b7058a85116493a262f679f3a261
 F test/malloc5.test f6eb6eca07a4c75f2897bf43a404689b6295bb95ab2e07d4b52eda743f925a27
@@ -1700,7 +1700,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 332bf84625d2034c9e1c029aa8243aa34088217e52e34e1f5472217743a31af3
-R b1d50b050a44eb29893416acc04ad8bf
-U drh
-Z 6cd3cb0cbe81acd6296afde81215523a
+P c3e816cca4ddf0967c4c790cdde2345101dde3f2d854e62589f1ac1d7f3c60b8
+R 43ada520e2e68e9100c12d4e5015a43b
+U dan
+Z 212cc80a5ee1b25bc75e6299ee001b27
diff --git a/manifest.uuid b/manifest.uuid
index 2e7a511d22..071e9f08e1 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c3e816cca4ddf0967c4c790cdde2345101dde3f2d854e62589f1ac1d7f3c60b8
\ No newline at end of file
+49e58e645e0c114c71935a3b7fa4771e8a23e28127efd2efe0f2f18813c8f391
\ No newline at end of file
diff --git a/test/ioerr.test b/test/ioerr.test
index e59647fe50..f42beef5b4 100644
--- a/test/ioerr.test
+++ b/test/ioerr.test
@@ -172,7 +172,7 @@ ifcapable crashtest&&attach {
 # These tests can't be run on windows because the windows version of 
 # SQLite holds a mandatory exclusive lock on journal files it has open.
 #
-if {$tcl_platform(platform)!="windows"} {
+if {$tcl_platform(platform)!="windows" && ![atomic_batch_write test.db]} {
   do_ioerr_test ioerr-7 -tclprep {
     db close
     sqlite3 db2 test2.db
@@ -211,7 +211,7 @@ do_ioerr_test ioerr-8 -ckrefcount true -tclprep {
 
 # For test coverage: Cause an IO error whilst reading the master-journal
 # name from a journal file.
-if {$tcl_platform(platform)=="unix"} {
+if {$tcl_platform(platform)=="unix" && [atomic_batch_write test.db]==0} {
   do_ioerr_test ioerr-9 -ckrefcount true -tclprep {
     execsql {
       CREATE TABLE t1(a,b,c);
diff --git a/test/malloc.test b/test/malloc.test
index dbf4699b27..5e82e8028b 100644
--- a/test/malloc.test
+++ b/test/malloc.test
@@ -329,7 +329,7 @@ ifcapable crashtest&&attach {
   }
 }
 
-if {$tcl_platform(platform)!="windows"} {
+if {$tcl_platform(platform)!="windows" && [atomic_batch_write test.db]==0} {
   do_malloc_test 14 -tclprep {
     catch {db close}
     sqlite3 db2 test2.db

From 9ccc0fd1f90f8872bddcf31adbf915818867d339 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 24 Jan 2018 11:25:59 +0000
Subject: [PATCH 378/488] Fix the sqlite3ext.h header file so that it correctly
 accesses the new sqlite3_value_nochange() and sqlite3_vtab_collation()
 interfaces.

FossilOrigin-Name: 6185d190e250faae0d979a24cb1039c7fce178478ae75df8e4af1d94cd38c18e
---
 manifest         | 14 +++++++-------
 manifest.uuid    |  2 +-
 src/sqlite3ext.h |  4 ++--
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index 062910f484..f9ef0358f5 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\sa\scouple\sof\stest\sscripts\sso\sthat\sthey\swork\son\sF2FS\sfile-systems\sthat\nsupport\satomic\stransactions.
-D 2018-01-24T06:30:10.922
+C Fix\sthe\ssqlite3ext.h\sheader\sfile\sso\sthat\sit\scorrectly\saccesses\sthe\nnew\ssqlite3_value_nochange()\sand\ssqlite3_vtab_collation()\sinterfaces.
+D 2018-01-24T11:25:59.042
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -490,7 +490,7 @@ F src/select.c bebe7cce45d899d2237c76bce059d525abf5b861f2fce92f6b53914a961c01ba
 F src/shell.c.in 7cea439c3f7f2e4ed6eb4b3a633cd93dccb1349241400de4da0c1291285ed514
 F src/sqlite.h.in 959deaad89679e31d7f68fda668b0c5d1f592fffed7a9c1740fb8ded4e4e754a
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
-F src/sqlite3ext.h 99189e7611eb0bf98f21c7835dc74730a84e2e809c98e1e31c33896dee7a2849
+F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d
 F src/sqliteInt.h 9c70315598b34810a83e4894455acb18e95cf63ce4e6cbb451ac2d17eabc2544
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
@@ -1700,7 +1700,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c3e816cca4ddf0967c4c790cdde2345101dde3f2d854e62589f1ac1d7f3c60b8
-R 43ada520e2e68e9100c12d4e5015a43b
-U dan
-Z 212cc80a5ee1b25bc75e6299ee001b27
+P 49e58e645e0c114c71935a3b7fa4771e8a23e28127efd2efe0f2f18813c8f391
+R f4b5f9525aa88d51005feed4871dc3a5
+U drh
+Z 5d5ae0811eec10d9ce697f3704c67092
diff --git a/manifest.uuid b/manifest.uuid
index 071e9f08e1..3af162c09e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-49e58e645e0c114c71935a3b7fa4771e8a23e28127efd2efe0f2f18813c8f391
\ No newline at end of file
+6185d190e250faae0d979a24cb1039c7fce178478ae75df8e4af1d94cd38c18e
\ No newline at end of file
diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h
index ac92a74901..1409370a6f 100644
--- a/src/sqlite3ext.h
+++ b/src/sqlite3ext.h
@@ -563,8 +563,8 @@ typedef int (*sqlite3_loadext_entry)(
 #define sqlite3_value_pointer          sqlite3_api->value_pointer
 /* Version 3.22.0 and later */
 #define sqlite3_vtab_nochange          sqlite3_api->vtab_nochange
-#define sqlite3_value_nochange         sqltie3_api->value_nochange
-#define sqlite3_vtab_collation         sqltie3_api->vtab_collation
+#define sqlite3_value_nochange         sqlite3_api->value_nochange
+#define sqlite3_vtab_collation         sqlite3_api->vtab_collation
 #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */
 
 #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION)

From 93117f0c4ba23d977f143a83cd1a34656b457d47 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 24 Jan 2018 11:29:42 +0000
Subject: [PATCH 379/488] Fix a formatting issue in sqlite3_prepare_v3()
 documentation.  No changes to code.

FossilOrigin-Name: 9e6066de84285252fb8999a8d8e02a46c5bb6c822e4f7421ad3911687357ee5d
---
 manifest        | 12 ++++++------
 manifest.uuid   |  2 +-
 src/sqlite.h.in |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index f9ef0358f5..7a565b0918 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sthe\ssqlite3ext.h\sheader\sfile\sso\sthat\sit\scorrectly\saccesses\sthe\nnew\ssqlite3_value_nochange()\sand\ssqlite3_vtab_collation()\sinterfaces.
-D 2018-01-24T11:25:59.042
+C Fix\sa\sformatting\sissue\sin\ssqlite3_prepare_v3()\sdocumentation.\s\sNo\schanges\nto\scode.
+D 2018-01-24T11:29:42.778
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -488,7 +488,7 @@ F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c bebe7cce45d899d2237c76bce059d525abf5b861f2fce92f6b53914a961c01ba
 F src/shell.c.in 7cea439c3f7f2e4ed6eb4b3a633cd93dccb1349241400de4da0c1291285ed514
-F src/sqlite.h.in 959deaad89679e31d7f68fda668b0c5d1f592fffed7a9c1740fb8ded4e4e754a
+F src/sqlite.h.in 51f9acf52c80113d793ddd38b3940ad6895d97b4752503b19291fb8fcbf54c5e
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d
 F src/sqliteInt.h 9c70315598b34810a83e4894455acb18e95cf63ce4e6cbb451ac2d17eabc2544
@@ -1700,7 +1700,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 49e58e645e0c114c71935a3b7fa4771e8a23e28127efd2efe0f2f18813c8f391
-R f4b5f9525aa88d51005feed4871dc3a5
+P 6185d190e250faae0d979a24cb1039c7fce178478ae75df8e4af1d94cd38c18e
+R 6dab6bedb07a7e4f2ba5c731abd1af01
 U drh
-Z 5d5ae0811eec10d9ce697f3704c67092
+Z b088841a3f96415d8879d1e430cf291c
diff --git a/manifest.uuid b/manifest.uuid
index 3af162c09e..1c96166857 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6185d190e250faae0d979a24cb1039c7fce178478ae75df8e4af1d94cd38c18e
\ No newline at end of file
+9e6066de84285252fb8999a8d8e02a46c5bb6c822e4f7421ad3911687357ee5d
\ No newline at end of file
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index ed1c24de4c..1a0164150b 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -3659,13 +3659,13 @@ int sqlite3_limit(sqlite3*, int id, int newVal);
 ** or [GLOB] operator or if the parameter is compared to an indexed column
 ** and the [SQLITE_ENABLE_STAT3] compile-time option is enabled.
 ** </li>
+** </ol>
 **
 ** <p>^sqlite3_prepare_v3() differs from sqlite3_prepare_v2() only in having
 ** the extra prepFlags parameter, which is a bit array consisting of zero or
 ** more of the [SQLITE_PREPARE_PERSISTENT|SQLITE_PREPARE_*] flags.  ^The
 ** sqlite3_prepare_v2() interface works exactly the same as
 ** sqlite3_prepare_v3() with a zero prepFlags parameter.
-** </ol>
 */
 int sqlite3_prepare(
   sqlite3 *db,            /* Database handle */

From b9f4799b5523e479078db4e23ab73b07a9905ebc Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 24 Jan 2018 12:14:43 +0000
Subject: [PATCH 380/488] Prevent a harmless unused variable warning when
 compiling with SQLITE_OMIT_TRACE.

FossilOrigin-Name: 61a44961b5811c993ae8d4f820a21a0ee739fe5e88cbfe5e4190e0091912ca19
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/vdbe.c    |  4 +++-
 3 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 7a565b0918..70f6b23f88 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sformatting\sissue\sin\ssqlite3_prepare_v3()\sdocumentation.\s\sNo\schanges\nto\scode.
-D 2018-01-24T11:29:42.778
+C Prevent\sa\sharmless\sunused\svariable\swarning\swhen\scompiling\swith\nSQLITE_OMIT_TRACE.
+D 2018-01-24T12:14:43.331
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -555,7 +555,7 @@ F src/update.c a90a32ffc0100265b0693dbbdbe490756447af181f5ea2c138cce515b08c8795
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c ef4a5f904d942e660abade7fbf3e6bdb402dabe9e7c27f3361ecf40b945538b5
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
-F src/vdbe.c 5aa6fb85281b6af058d2a87b65c1a8a8e0bd2fe554fb3391497ee3702b61e0be
+F src/vdbe.c d4684bd5efad095d9e4405e5ca2e1168dc7aff643cae04c424e4bade70641d70
 F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a
 F src/vdbeInt.h 8d7d07f13cb3c4cbca91e22ba4a1920e542dda7c5d9299920432a0b3d5b009f5
 F src/vdbeapi.c fea41171884a4de119f8b10ab514c788674eeeb7f27218bb6d008e1310bfd07f
@@ -1700,7 +1700,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6185d190e250faae0d979a24cb1039c7fce178478ae75df8e4af1d94cd38c18e
-R 6dab6bedb07a7e4f2ba5c731abd1af01
+P 9e6066de84285252fb8999a8d8e02a46c5bb6c822e4f7421ad3911687357ee5d
+R 7e73e9644a1ec009a08fcc00aee933cc
 U drh
-Z b088841a3f96415d8879d1e430cf291c
+Z e8177042bd88031a5c0c9d9fb215dabd
diff --git a/manifest.uuid b/manifest.uuid
index 1c96166857..38eb19867d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-9e6066de84285252fb8999a8d8e02a46c5bb6c822e4f7421ad3911687357ee5d
\ No newline at end of file
+61a44961b5811c993ae8d4f820a21a0ee739fe5e88cbfe5e4190e0091912ca19
\ No newline at end of file
diff --git a/src/vdbe.c b/src/vdbe.c
index 8d0b35a6f4..81a2361a55 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -7107,8 +7107,10 @@ case OP_Function: {
 */
 case OP_Trace:
 case OP_Init: {          /* jump */
-  char *zTrace;
   int i;
+#ifndef SQLITE_OMIT_TRACE
+  char *zTrace;
+#endif
 
   /* If the P4 argument is not NULL, then it must be an SQL comment string.
   ** The "--" string is broken up to prevent false-positives with srcck1.c.

From a87f8ce262cdb1901e4dc8a871dd967f94e33022 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 24 Jan 2018 13:15:56 +0000
Subject: [PATCH 381/488] Improved error message output when the btreeinfo.c
 extension is run in a connection that lacks sqlite_dbpage support.

FossilOrigin-Name: 461b0b81b2acedb1b9ca520657518d6778417c375aeb58c41ead840b75a6d985
---
 ext/misc/btreeinfo.c |  3 ++-
 manifest             | 12 ++++++------
 manifest.uuid        |  2 +-
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/ext/misc/btreeinfo.c b/ext/misc/btreeinfo.c
index d75f06277f..131b210a79 100644
--- a/ext/misc/btreeinfo.c
+++ b/ext/misc/btreeinfo.c
@@ -339,7 +339,8 @@ static int binfoColumn(
     sqlite3 *db = sqlite3_context_db_handle(ctx);
     int rc = binfoCompute(db, pgno, pCsr);
     if( rc ){
-      return rc;
+      pCursor->pVtab->zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
+      return SQLITE_ERROR;
     }
   }
   switch( i ){
diff --git a/manifest b/manifest
index 70f6b23f88..f93da218d6 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Prevent\sa\sharmless\sunused\svariable\swarning\swhen\scompiling\swith\nSQLITE_OMIT_TRACE.
-D 2018-01-24T12:14:43.331
+C Improved\serror\smessage\soutput\swhen\sthe\sbtreeinfo.c\sextension\sis\srun\sin\na\sconnection\sthat\slacks\ssqlite_dbpage\ssupport.
+D 2018-01-24T13:15:56.698
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -269,7 +269,7 @@ F ext/misc/README.md d6dd0fe1d8af77040216798a6a2b0c46c73054d2f0ea544fbbcdccf6f23
 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87
 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb
 F ext/misc/appendvfs.c 3777f22ec1057dc4e5fd89f2fbddcc7a29fbeef1ad038c736c54411bb1967af7
-F ext/misc/btreeinfo.c d7fd9a2fe2fa33ba28488e2fce703ebecc759219ea9e0bb3b254784866c0a676
+F ext/misc/btreeinfo.c 78c8c57d325185ccc04b7679e5b020e34a4d9c87453e6b7ac943d0a26cee3256
 F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005
 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704
 F ext/misc/completion.c 52c3f01523e3e387eb321b4739a89d1fe47cbe6025aa1f2d8d3685e9e365df0f
@@ -1700,7 +1700,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 9e6066de84285252fb8999a8d8e02a46c5bb6c822e4f7421ad3911687357ee5d
-R 7e73e9644a1ec009a08fcc00aee933cc
+P 61a44961b5811c993ae8d4f820a21a0ee739fe5e88cbfe5e4190e0091912ca19
+R 9aa3fb89b7c642ba6cb5e4ab21b67bf7
 U drh
-Z e8177042bd88031a5c0c9d9fb215dabd
+Z 08e16fd59762f7c574401c408bcad9ff
diff --git a/manifest.uuid b/manifest.uuid
index 38eb19867d..a06a90d5fc 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-61a44961b5811c993ae8d4f820a21a0ee739fe5e88cbfe5e4190e0091912ca19
\ No newline at end of file
+461b0b81b2acedb1b9ca520657518d6778417c375aeb58c41ead840b75a6d985
\ No newline at end of file

From 4d6d872c4937bdbb62d4758e8a80e46bfbf02ef9 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 24 Jan 2018 14:39:18 +0000
Subject: [PATCH 382/488] Adjust the fts3rank.test module so that it works on
 big-endian systems (hopefully - I don't have a big-endian machine to test the
 change on.)

FossilOrigin-Name: e4766cabdf64d8e998048ae43154466fc9c3fad0b86102a42b65122abfcbbe55
---
 manifest           | 12 ++++++------
 manifest.uuid      |  2 +-
 test/fts3rank.test | 15 ++++++++++-----
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/manifest b/manifest
index f93da218d6..0d0cc09116 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improved\serror\smessage\soutput\swhen\sthe\sbtreeinfo.c\sextension\sis\srun\sin\na\sconnection\sthat\slacks\ssqlite_dbpage\ssupport.
-D 2018-01-24T13:15:56.698
+C Adjust\sthe\sfts3rank.test\smodule\sso\sthat\sit\sworks\son\sbig-endian\ssystems\n(hopefully\s-\sI\sdon't\shave\sa\sbig-endian\smachine\sto\stest\sthe\schange\son.)
+D 2018-01-24T14:39:18.624
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -894,7 +894,7 @@ F test/fts3offsets.test b85fd382abdc78ebce721d8117bd552dfb75094c
 F test/fts3prefix.test fa794eaab0bdae466494947b0b153d7844478ab2
 F test/fts3prefix2.test e1f0a822ca661dced7f12ce392e14eaf65609dce
 F test/fts3query.test f33eb71a1fe1084ea585eeb7ee76b390729f5170
-F test/fts3rank.test e4d2e16a28c98cae95001a75e2b4b05b19b051ffd6aaab15491c5e0595127b9b
+F test/fts3rank.test cd99bc83a3c923c8d52afd90d86979cf05fc41849f892faeac3988055ef37b99
 F test/fts3rnd.test 1320d8826a845e38a96e769562bf83d7a92a15d0
 F test/fts3shared.test 57e26a801f21027b7530da77db54286a6fe4997e
 F test/fts3snippet.test 01a4231816e03a0660ae53ba2404fe69012fe0db
@@ -1700,7 +1700,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 61a44961b5811c993ae8d4f820a21a0ee739fe5e88cbfe5e4190e0091912ca19
-R 9aa3fb89b7c642ba6cb5e4ab21b67bf7
+P 461b0b81b2acedb1b9ca520657518d6778417c375aeb58c41ead840b75a6d985
+R d7da84a0a0cbf9001c093c5227ed5c74
 U drh
-Z 08e16fd59762f7c574401c408bcad9ff
+Z d9c76d204a4febeaadf3db9edf524e66
diff --git a/manifest.uuid b/manifest.uuid
index a06a90d5fc..a774a7d7aa 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-461b0b81b2acedb1b9ca520657518d6778417c375aeb58c41ead840b75a6d985
\ No newline at end of file
+e4766cabdf64d8e998048ae43154466fc9c3fad0b86102a42b65122abfcbbe55
\ No newline at end of file
diff --git a/test/fts3rank.test b/test/fts3rank.test
index 7ee3143a76..fd1a1c89d7 100644
--- a/test/fts3rank.test
+++ b/test/fts3rank.test
@@ -14,7 +14,7 @@
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
-set testprefix fts3expr5
+set testprefix fts3rank
 
 # If SQLITE_ENABLE_FTS3 is defined, omit this file.
 ifcapable !fts3 {
@@ -56,9 +56,14 @@ do_catchsql_test 1.4 {
   SELECT * FROM t1 ORDER BY rank(x'0000000000000000') DESC, rowid
 } {0 {{one two} one {one two} three {one two} two}}
 
-do_catchsql_test 1.5 {
-  SELECT * FROM t1 ORDER BY rank(x'0100000001000000') DESC, rowid
-} {1 {invalid matchinfo blob passed to function rank()}}
+if {$tcl_platform(byteOrder)=="littleEndian"} {
+  do_catchsql_test 1.5le {
+    SELECT * FROM t1 ORDER BY rank(x'0100000001000000') DESC, rowid
+  } {1 {invalid matchinfo blob passed to function rank()}}
+} else {
+  do_catchsql_test 1.5be {
+    SELECT * FROM t1 ORDER BY rank(x'0000000100000001') DESC, rowid
+  } {1 {invalid matchinfo blob passed to function rank()}}
+}
 
 finish_test
-

From a8519d79b953b4249de75781d11f0cf4aafb57a1 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 24 Jan 2018 14:40:01 +0000
Subject: [PATCH 383/488] Interchange the numeric codes for CURSOR_VALID and
 CURSOR_INVALID to obtain a small size decrease and performance increase.

FossilOrigin-Name: e0f192ea6dda4fa0b243d58c8ce41932519141bcae0689a90318b4f866f54edd
---
 manifest       | 12 ++++++------
 manifest.uuid  |  2 +-
 src/btreeInt.h |  4 ++--
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index 0d0cc09116..ffdd6c23cb 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Adjust\sthe\sfts3rank.test\smodule\sso\sthat\sit\sworks\son\sbig-endian\ssystems\n(hopefully\s-\sI\sdon't\shave\sa\sbig-endian\smachine\sto\stest\sthe\schange\son.)
-D 2018-01-24T14:39:18.624
+C Interchange\sthe\snumeric\scodes\sfor\sCURSOR_VALID\sand\sCURSOR_INVALID\sto\sobtain\na\ssmall\ssize\sdecrease\sand\sperformance\sincrease.
+D 2018-01-24T14:40:01.721
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
@@ -430,7 +430,7 @@ F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
 F src/btree.c 1beceb1c5f9563271241fd0c294484668e4ad28cf6aa970eab70e50ff6f75e25
 F src/btree.h 0866c0a08255142ea0e754aabd211c843cab32045c978a592a43152405ed0c84
-F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc
+F src/btreeInt.h 6e70d15435b60a29386ae9864875618ca225c31bf815038bbd8f510a66bb9cb1
 F src/build.c 9f9647454f236cab097f266ae970f899b53c71cadab6756c47e2b2e81392c2a1
 F src/callback.c fe677cb5f5abb02f7a772a62a98c2f516426081df68856e8f2d5f950929b966a
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
@@ -1700,7 +1700,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 461b0b81b2acedb1b9ca520657518d6778417c375aeb58c41ead840b75a6d985
-R d7da84a0a0cbf9001c093c5227ed5c74
+P e4766cabdf64d8e998048ae43154466fc9c3fad0b86102a42b65122abfcbbe55
+R db620f4cdcf6bbd9d15c47f8bc8f1349
 U drh
-Z d9c76d204a4febeaadf3db9edf524e66
+Z 93685aa7f4c0affe18e092c9d91e885b
diff --git a/manifest.uuid b/manifest.uuid
index a774a7d7aa..e81bb2abcc 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e4766cabdf64d8e998048ae43154466fc9c3fad0b86102a42b65122abfcbbe55
\ No newline at end of file
+e0f192ea6dda4fa0b243d58c8ce41932519141bcae0689a90318b4f866f54edd
\ No newline at end of file
diff --git a/src/btreeInt.h b/src/btreeInt.h
index ac7a3c0a26..87d3247e13 100644
--- a/src/btreeInt.h
+++ b/src/btreeInt.h
@@ -566,8 +566,8 @@ struct BtCursor {
 **   Do nothing else with this cursor.  Any attempt to use the cursor
 **   should return the error code stored in BtCursor.skipNext
 */
-#define CURSOR_INVALID           0
-#define CURSOR_VALID             1
+#define CURSOR_VALID             0
+#define CURSOR_INVALID           1
 #define CURSOR_SKIPNEXT          2
 #define CURSOR_REQUIRESEEK       3
 #define CURSOR_FAULT             4

From da6bc6792fa0a647f11c73b1ee5a57f0355e39b9 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 24 Jan 2018 16:04:21 +0000
Subject: [PATCH 384/488] Rearrange fields of the BtCursor object so that it is
 smaller and requires less initialization, for a small performance
 improvement.

FossilOrigin-Name: 0ddf5292cc0411ec6fcb7399ecf2904c899e0488404d3f65490fbe1db15efdf4
---
 manifest       | 15 +++++++--------
 manifest.uuid  |  2 +-
 src/btree.c    |  7 ++++---
 src/btreeInt.h | 18 +++++++++---------
 4 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/manifest b/manifest
index 57e34466ef..e20e592681 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\snormalize.c\sextension.
-D 2018-01-24T15:07:37.708
+C Rearrange\sfields\sof\sthe\sBtCursor\sobject\sso\sthat\sit\sis\ssmaller\sand\srequires\sless\ninitialization,\sfor\sa\ssmall\sperformance\simprovement.
+D 2018-01-24T16:04:21.587
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -429,9 +429,9 @@ F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
-F src/btree.c 1beceb1c5f9563271241fd0c294484668e4ad28cf6aa970eab70e50ff6f75e25
+F src/btree.c d711228cac336fb35fff21f3f4a0efe2ad58aa9a800dd02929cdf184be1e78a3
 F src/btree.h 0866c0a08255142ea0e754aabd211c843cab32045c978a592a43152405ed0c84
-F src/btreeInt.h 6e70d15435b60a29386ae9864875618ca225c31bf815038bbd8f510a66bb9cb1
+F src/btreeInt.h 620ab4c7235f43572cf3ac2ac8723cbdf68073be4d29da24897c7b77dda5fd96
 F src/build.c 9f9647454f236cab097f266ae970f899b53c71cadab6756c47e2b2e81392c2a1
 F src/callback.c fe677cb5f5abb02f7a772a62a98c2f516426081df68856e8f2d5f950929b966a
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
@@ -1702,8 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e0f192ea6dda4fa0b243d58c8ce41932519141bcae0689a90318b4f866f54edd 658f42257d56a3562dfa8e55023c6e497f55c565cd751d942de3a9e0cb2fa708
-R 01bd832eb21dc9b5a19a52a2b0f11c48
-T +closed 658f42257d56a3562dfa8e55023c6e497f55c565cd751d942de3a9e0cb2fa708
+P 16ebe558d2f0982f630c39b6c1b00d14d27011d454cced213f17c26491a2c06e
+R fc9e02eb09b9540ef2f7fa9944ac0566
 U drh
-Z 043a35201b0c771ecddffe95c53bb65f
+Z c7e32730b85a19ed6c94c7a445674e7d
diff --git a/manifest.uuid b/manifest.uuid
index 498f994dc6..1dd418e847 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-16ebe558d2f0982f630c39b6c1b00d14d27011d454cced213f17c26491a2c06e
\ No newline at end of file
+0ddf5292cc0411ec6fcb7399ecf2904c899e0488404d3f65490fbe1db15efdf4
\ No newline at end of file
diff --git a/src/btree.c b/src/btree.c
index 168f3105fb..33f5842282 100644
--- a/src/btree.c
+++ b/src/btree.c
@@ -4345,7 +4345,7 @@ int sqlite3BtreeCursorSize(void){
 ** of run-time by skipping the initialization of those elements.
 */
 void sqlite3BtreeCursorZero(BtCursor *p){
-  memset(p, 0, offsetof(BtCursor, iPage));
+  memset(p, 0, offsetof(BtCursor, BTCURSOR_FIRST_UNINIT));
 }
 
 /*
@@ -4668,14 +4668,15 @@ static int accessPayload(
     */
     if( (pCur->curFlags & BTCF_ValidOvfl)==0 ){
       int nOvfl = (pCur->info.nPayload-pCur->info.nLocal+ovflSize-1)/ovflSize;
-      if( nOvfl>pCur->nOvflAlloc ){
+      if( pCur->aOverflow==0
+       || nOvfl*sizeof(Pgno) > sqlite3MallocSize(pCur->aOverflow)
+      ){
         Pgno *aNew = (Pgno*)sqlite3Realloc(
             pCur->aOverflow, nOvfl*2*sizeof(Pgno)
         );
         if( aNew==0 ){
           return SQLITE_NOMEM_BKPT;
         }else{
-          pCur->nOvflAlloc = nOvfl*2;
           pCur->aOverflow = aNew;
         }
       }
diff --git a/src/btreeInt.h b/src/btreeInt.h
index 87d3247e13..d1e2c08d18 100644
--- a/src/btreeInt.h
+++ b/src/btreeInt.h
@@ -503,20 +503,20 @@ struct BtCursor {
   u8 curFlags;              /* zero or more BTCF_* flags defined below */
   u8 curPagerFlags;         /* Flags to send to sqlite3PagerGet() */
   u8 hints;                 /* As configured by CursorSetHints() */
-  int nOvflAlloc;           /* Allocated size of aOverflow[] array */
-  Btree *pBtree;            /* The Btree to which this cursor belongs */
-  BtShared *pBt;            /* The BtShared this cursor points to */
-  BtCursor *pNext;          /* Forms a linked list of all cursors */
-  Pgno *aOverflow;          /* Cache of overflow page locations */
-  CellInfo info;            /* A parse of the cell we are pointing at */
-  i64 nKey;                 /* Size of pKey, or last integer key */
-  void *pKey;               /* Saved key that was cursor last known position */
-  Pgno pgnoRoot;            /* The root page of this tree */
   int skipNext;    /* Prev() is noop if negative. Next() is noop if positive.
                    ** Error code if eState==CURSOR_FAULT */
+  Btree *pBtree;            /* The Btree to which this cursor belongs */
+  Pgno *aOverflow;          /* Cache of overflow page locations */
+  void *pKey;               /* Saved key that was cursor last known position */
   /* All fields above are zeroed when the cursor is allocated.  See
   ** sqlite3BtreeCursorZero().  Fields that follow must be manually
   ** initialized. */
+#define BTCURSOR_FIRST_UNINIT pBt   /* Name of first uninitialized field */
+  BtShared *pBt;            /* The BtShared this cursor points to */
+  BtCursor *pNext;          /* Forms a linked list of all cursors */
+  CellInfo info;            /* A parse of the cell we are pointing at */
+  i64 nKey;                 /* Size of pKey, or last integer key */
+  Pgno pgnoRoot;            /* The root page of this tree */
   i8 iPage;                 /* Index of current page in apPage */
   u8 curIntKey;             /* Value of apPage[0]->intKey */
   u16 ix;                   /* Current index for apPage[iPage] */

From 1a0e5b37b242b640c00d876a085ed657fe7b8012 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 24 Jan 2018 18:28:39 +0000
Subject: [PATCH 385/488] Reorganize spellfix.c to make it easier to edit
 automatically (e.g. using a script). No changes to functionality.

FossilOrigin-Name: 090a64faaac579c6ed1cddb02d14e7089468a4aeaeec11e78d461790bcb6eb0a
---
 ext/misc/spellfix.c | 832 ++++++++++++++++++++++----------------------
 manifest            |  14 +-
 manifest.uuid       |   2 +-
 3 files changed, 432 insertions(+), 416 deletions(-)

diff --git a/ext/misc/spellfix.c b/ext/misc/spellfix.c
index 4f17b88e1d..2d7ef1b02b 100644
--- a/ext/misc/spellfix.c
+++ b/ext/misc/spellfix.c
@@ -1200,404 +1200,412 @@ static int utf8Charlen(const char *zIn, int nIn){
   return nChar;
 }
 
+typedef struct Transliteration Transliteration;
+struct Transliteration {
+ unsigned short int cFrom;
+ unsigned char cTo0, cTo1, cTo2, cTo3;
+};
+
 /*
 ** Table of translations from unicode characters into ASCII.
 */
-static const struct {
- unsigned short int cFrom;
- unsigned char cTo0, cTo1;
-} translit[] = {
-  { 0x00A0,  0x20, 0x00 },  /*   to   */
-  { 0x00B5,  0x75, 0x00 },  /* µ to u */
-  { 0x00C0,  0x41, 0x00 },  /* À to A */
-  { 0x00C1,  0x41, 0x00 },  /* Á to A */
-  { 0x00C2,  0x41, 0x00 },  /* Â to A */
-  { 0x00C3,  0x41, 0x00 },  /* Ã to A */
-  { 0x00C4,  0x41, 0x65 },  /* Ä to Ae */
-  { 0x00C5,  0x41, 0x61 },  /* Å to Aa */
-  { 0x00C6,  0x41, 0x45 },  /* Æ to AE */
-  { 0x00C7,  0x43, 0x00 },  /* Ç to C */
-  { 0x00C8,  0x45, 0x00 },  /* È to E */
-  { 0x00C9,  0x45, 0x00 },  /* É to E */
-  { 0x00CA,  0x45, 0x00 },  /* Ê to E */
-  { 0x00CB,  0x45, 0x00 },  /* Ë to E */
-  { 0x00CC,  0x49, 0x00 },  /* Ì to I */
-  { 0x00CD,  0x49, 0x00 },  /* Í to I */
-  { 0x00CE,  0x49, 0x00 },  /* Î to I */
-  { 0x00CF,  0x49, 0x00 },  /* Ï to I */
-  { 0x00D0,  0x44, 0x00 },  /* Ð to D */
-  { 0x00D1,  0x4E, 0x00 },  /* Ñ to N */
-  { 0x00D2,  0x4F, 0x00 },  /* Ò to O */
-  { 0x00D3,  0x4F, 0x00 },  /* Ó to O */
-  { 0x00D4,  0x4F, 0x00 },  /* Ô to O */
-  { 0x00D5,  0x4F, 0x00 },  /* Õ to O */
-  { 0x00D6,  0x4F, 0x65 },  /* Ö to Oe */
-  { 0x00D7,  0x78, 0x00 },  /* × to x */
-  { 0x00D8,  0x4F, 0x00 },  /* Ø to O */
-  { 0x00D9,  0x55, 0x00 },  /* Ù to U */
-  { 0x00DA,  0x55, 0x00 },  /* Ú to U */
-  { 0x00DB,  0x55, 0x00 },  /* Û to U */
-  { 0x00DC,  0x55, 0x65 },  /* Ü to Ue */
-  { 0x00DD,  0x59, 0x00 },  /* Ý to Y */
-  { 0x00DE,  0x54, 0x68 },  /* Þ to Th */
-  { 0x00DF,  0x73, 0x73 },  /* ß to ss */
-  { 0x00E0,  0x61, 0x00 },  /* à to a */
-  { 0x00E1,  0x61, 0x00 },  /* á to a */
-  { 0x00E2,  0x61, 0x00 },  /* â to a */
-  { 0x00E3,  0x61, 0x00 },  /* ã to a */
-  { 0x00E4,  0x61, 0x65 },  /* ä to ae */
-  { 0x00E5,  0x61, 0x61 },  /* å to aa */
-  { 0x00E6,  0x61, 0x65 },  /* æ to ae */
-  { 0x00E7,  0x63, 0x00 },  /* ç to c */
-  { 0x00E8,  0x65, 0x00 },  /* è to e */
-  { 0x00E9,  0x65, 0x00 },  /* é to e */
-  { 0x00EA,  0x65, 0x00 },  /* ê to e */
-  { 0x00EB,  0x65, 0x00 },  /* ë to e */
-  { 0x00EC,  0x69, 0x00 },  /* ì to i */
-  { 0x00ED,  0x69, 0x00 },  /* í to i */
-  { 0x00EE,  0x69, 0x00 },  /* î to i */
-  { 0x00EF,  0x69, 0x00 },  /* ï to i */
-  { 0x00F0,  0x64, 0x00 },  /* ð to d */
-  { 0x00F1,  0x6E, 0x00 },  /* ñ to n */
-  { 0x00F2,  0x6F, 0x00 },  /* ò to o */
-  { 0x00F3,  0x6F, 0x00 },  /* ó to o */
-  { 0x00F4,  0x6F, 0x00 },  /* ô to o */
-  { 0x00F5,  0x6F, 0x00 },  /* õ to o */
-  { 0x00F6,  0x6F, 0x65 },  /* ö to oe */
-  { 0x00F7,  0x3A, 0x00 },  /* ÷ to : */
-  { 0x00F8,  0x6F, 0x00 },  /* ø to o */
-  { 0x00F9,  0x75, 0x00 },  /* ù to u */
-  { 0x00FA,  0x75, 0x00 },  /* ú to u */
-  { 0x00FB,  0x75, 0x00 },  /* û to u */
-  { 0x00FC,  0x75, 0x65 },  /* ü to ue */
-  { 0x00FD,  0x79, 0x00 },  /* ý to y */
-  { 0x00FE,  0x74, 0x68 },  /* þ to th */
-  { 0x00FF,  0x79, 0x00 },  /* ÿ to y */
-  { 0x0100,  0x41, 0x00 },  /* Ā to A */
-  { 0x0101,  0x61, 0x00 },  /* ā to a */
-  { 0x0102,  0x41, 0x00 },  /* Ă to A */
-  { 0x0103,  0x61, 0x00 },  /* ă to a */
-  { 0x0104,  0x41, 0x00 },  /* Ą to A */
-  { 0x0105,  0x61, 0x00 },  /* ą to a */
-  { 0x0106,  0x43, 0x00 },  /* Ć to C */
-  { 0x0107,  0x63, 0x00 },  /* ć to c */
-  { 0x0108,  0x43, 0x68 },  /* Ĉ to Ch */
-  { 0x0109,  0x63, 0x68 },  /* ĉ to ch */
-  { 0x010A,  0x43, 0x00 },  /* Ċ to C */
-  { 0x010B,  0x63, 0x00 },  /* ċ to c */
-  { 0x010C,  0x43, 0x00 },  /* Č to C */
-  { 0x010D,  0x63, 0x00 },  /* č to c */
-  { 0x010E,  0x44, 0x00 },  /* Ď to D */
-  { 0x010F,  0x64, 0x00 },  /* ď to d */
-  { 0x0110,  0x44, 0x00 },  /* Đ to D */
-  { 0x0111,  0x64, 0x00 },  /* đ to d */
-  { 0x0112,  0x45, 0x00 },  /* Ē to E */
-  { 0x0113,  0x65, 0x00 },  /* ē to e */
-  { 0x0114,  0x45, 0x00 },  /* Ĕ to E */
-  { 0x0115,  0x65, 0x00 },  /* ĕ to e */
-  { 0x0116,  0x45, 0x00 },  /* Ė to E */
-  { 0x0117,  0x65, 0x00 },  /* ė to e */
-  { 0x0118,  0x45, 0x00 },  /* Ę to E */
-  { 0x0119,  0x65, 0x00 },  /* ę to e */
-  { 0x011A,  0x45, 0x00 },  /* Ě to E */
-  { 0x011B,  0x65, 0x00 },  /* ě to e */
-  { 0x011C,  0x47, 0x68 },  /* Ĝ to Gh */
-  { 0x011D,  0x67, 0x68 },  /* ĝ to gh */
-  { 0x011E,  0x47, 0x00 },  /* Ğ to G */
-  { 0x011F,  0x67, 0x00 },  /* ğ to g */
-  { 0x0120,  0x47, 0x00 },  /* Ġ to G */
-  { 0x0121,  0x67, 0x00 },  /* ġ to g */
-  { 0x0122,  0x47, 0x00 },  /* Ģ to G */
-  { 0x0123,  0x67, 0x00 },  /* ģ to g */
-  { 0x0124,  0x48, 0x68 },  /* Ĥ to Hh */
-  { 0x0125,  0x68, 0x68 },  /* ĥ to hh */
-  { 0x0126,  0x48, 0x00 },  /* Ħ to H */
-  { 0x0127,  0x68, 0x00 },  /* ħ to h */
-  { 0x0128,  0x49, 0x00 },  /* Ĩ to I */
-  { 0x0129,  0x69, 0x00 },  /* ĩ to i */
-  { 0x012A,  0x49, 0x00 },  /* Ī to I */
-  { 0x012B,  0x69, 0x00 },  /* ī to i */
-  { 0x012C,  0x49, 0x00 },  /* Ĭ to I */
-  { 0x012D,  0x69, 0x00 },  /* ĭ to i */
-  { 0x012E,  0x49, 0x00 },  /* Į to I */
-  { 0x012F,  0x69, 0x00 },  /* į to i */
-  { 0x0130,  0x49, 0x00 },  /* İ to I */
-  { 0x0131,  0x69, 0x00 },  /* ı to i */
-  { 0x0132,  0x49, 0x4A },  /* IJ to IJ */
-  { 0x0133,  0x69, 0x6A },  /* ij to ij */
-  { 0x0134,  0x4A, 0x68 },  /* Ĵ to Jh */
-  { 0x0135,  0x6A, 0x68 },  /* ĵ to jh */
-  { 0x0136,  0x4B, 0x00 },  /* Ķ to K */
-  { 0x0137,  0x6B, 0x00 },  /* ķ to k */
-  { 0x0138,  0x6B, 0x00 },  /* ĸ to k */
-  { 0x0139,  0x4C, 0x00 },  /* Ĺ to L */
-  { 0x013A,  0x6C, 0x00 },  /* ĺ to l */
-  { 0x013B,  0x4C, 0x00 },  /* Ļ to L */
-  { 0x013C,  0x6C, 0x00 },  /* ļ to l */
-  { 0x013D,  0x4C, 0x00 },  /* Ľ to L */
-  { 0x013E,  0x6C, 0x00 },  /* ľ to l */
-  { 0x013F,  0x4C, 0x2E },  /* Ŀ to L. */
-  { 0x0140,  0x6C, 0x2E },  /* ŀ to l. */
-  { 0x0141,  0x4C, 0x00 },  /* Ł to L */
-  { 0x0142,  0x6C, 0x00 },  /* ł to l */
-  { 0x0143,  0x4E, 0x00 },  /* Ń to N */
-  { 0x0144,  0x6E, 0x00 },  /* ń to n */
-  { 0x0145,  0x4E, 0x00 },  /* Ņ to N */
-  { 0x0146,  0x6E, 0x00 },  /* ņ to n */
-  { 0x0147,  0x4E, 0x00 },  /* Ň to N */
-  { 0x0148,  0x6E, 0x00 },  /* ň to n */
-  { 0x0149,  0x27, 0x6E },  /* ʼn to 'n */
-  { 0x014A,  0x4E, 0x47 },  /* Ŋ to NG */
-  { 0x014B,  0x6E, 0x67 },  /* ŋ to ng */
-  { 0x014C,  0x4F, 0x00 },  /* Ō to O */
-  { 0x014D,  0x6F, 0x00 },  /* ō to o */
-  { 0x014E,  0x4F, 0x00 },  /* Ŏ to O */
-  { 0x014F,  0x6F, 0x00 },  /* ŏ to o */
-  { 0x0150,  0x4F, 0x00 },  /* Ő to O */
-  { 0x0151,  0x6F, 0x00 },  /* ő to o */
-  { 0x0152,  0x4F, 0x45 },  /* Πto OE */
-  { 0x0153,  0x6F, 0x65 },  /* œ to oe */
-  { 0x0154,  0x52, 0x00 },  /* Ŕ to R */
-  { 0x0155,  0x72, 0x00 },  /* ŕ to r */
-  { 0x0156,  0x52, 0x00 },  /* Ŗ to R */
-  { 0x0157,  0x72, 0x00 },  /* ŗ to r */
-  { 0x0158,  0x52, 0x00 },  /* Ř to R */
-  { 0x0159,  0x72, 0x00 },  /* ř to r */
-  { 0x015A,  0x53, 0x00 },  /* Ś to S */
-  { 0x015B,  0x73, 0x00 },  /* ś to s */
-  { 0x015C,  0x53, 0x68 },  /* Ŝ to Sh */
-  { 0x015D,  0x73, 0x68 },  /* ŝ to sh */
-  { 0x015E,  0x53, 0x00 },  /* Ş to S */
-  { 0x015F,  0x73, 0x00 },  /* ş to s */
-  { 0x0160,  0x53, 0x00 },  /* Š to S */
-  { 0x0161,  0x73, 0x00 },  /* š to s */
-  { 0x0162,  0x54, 0x00 },  /* Ţ to T */
-  { 0x0163,  0x74, 0x00 },  /* ţ to t */
-  { 0x0164,  0x54, 0x00 },  /* Ť to T */
-  { 0x0165,  0x74, 0x00 },  /* ť to t */
-  { 0x0166,  0x54, 0x00 },  /* Ŧ to T */
-  { 0x0167,  0x74, 0x00 },  /* ŧ to t */
-  { 0x0168,  0x55, 0x00 },  /* Ũ to U */
-  { 0x0169,  0x75, 0x00 },  /* ũ to u */
-  { 0x016A,  0x55, 0x00 },  /* Ū to U */
-  { 0x016B,  0x75, 0x00 },  /* ū to u */
-  { 0x016C,  0x55, 0x00 },  /* Ŭ to U */
-  { 0x016D,  0x75, 0x00 },  /* ŭ to u */
-  { 0x016E,  0x55, 0x00 },  /* Ů to U */
-  { 0x016F,  0x75, 0x00 },  /* ů to u */
-  { 0x0170,  0x55, 0x00 },  /* Ű to U */
-  { 0x0171,  0x75, 0x00 },  /* ű to u */
-  { 0x0172,  0x55, 0x00 },  /* Ų to U */
-  { 0x0173,  0x75, 0x00 },  /* ų to u */
-  { 0x0174,  0x57, 0x00 },  /* Ŵ to W */
-  { 0x0175,  0x77, 0x00 },  /* ŵ to w */
-  { 0x0176,  0x59, 0x00 },  /* Ŷ to Y */
-  { 0x0177,  0x79, 0x00 },  /* ŷ to y */
-  { 0x0178,  0x59, 0x00 },  /* Ÿ to Y */
-  { 0x0179,  0x5A, 0x00 },  /* Ź to Z */
-  { 0x017A,  0x7A, 0x00 },  /* ź to z */
-  { 0x017B,  0x5A, 0x00 },  /* Ż to Z */
-  { 0x017C,  0x7A, 0x00 },  /* ż to z */
-  { 0x017D,  0x5A, 0x00 },  /* Ž to Z */
-  { 0x017E,  0x7A, 0x00 },  /* ž to z */
-  { 0x017F,  0x73, 0x00 },  /* ſ to s */
-  { 0x0192,  0x66, 0x00 },  /* ƒ to f */
-  { 0x0218,  0x53, 0x00 },  /* Ș to S */
-  { 0x0219,  0x73, 0x00 },  /* ș to s */
-  { 0x021A,  0x54, 0x00 },  /* Ț to T */
-  { 0x021B,  0x74, 0x00 },  /* ț to t */
-  { 0x0386,  0x41, 0x00 },  /* Ά to A */
-  { 0x0388,  0x45, 0x00 },  /* Έ to E */
-  { 0x0389,  0x49, 0x00 },  /* Ή to I */
-  { 0x038A,  0x49, 0x00 },  /* Ί to I */
-  { 0x038C,  0x4f, 0x00 },  /* Ό to O */
-  { 0x038E,  0x59, 0x00 },  /* Ύ to Y */
-  { 0x038F,  0x4f, 0x00 },  /* Ώ to O */
-  { 0x0390,  0x69, 0x00 },  /* ΐ to i */
-  { 0x0391,  0x41, 0x00 },  /* Α to A */
-  { 0x0392,  0x42, 0x00 },  /* Β to B */
-  { 0x0393,  0x47, 0x00 },  /* Γ to G */
-  { 0x0394,  0x44, 0x00 },  /* Δ to D */
-  { 0x0395,  0x45, 0x00 },  /* Ε to E */
-  { 0x0396,  0x5a, 0x00 },  /* Ζ to Z */
-  { 0x0397,  0x49, 0x00 },  /* Η to I */
-  { 0x0398,  0x54, 0x68 },  /* Θ to Th */
-  { 0x0399,  0x49, 0x00 },  /* Ι to I */
-  { 0x039A,  0x4b, 0x00 },  /* Κ to K */
-  { 0x039B,  0x4c, 0x00 },  /* Λ to L */
-  { 0x039C,  0x4d, 0x00 },  /* Μ to M */
-  { 0x039D,  0x4e, 0x00 },  /* Ν to N */
-  { 0x039E,  0x58, 0x00 },  /* Ξ to X */
-  { 0x039F,  0x4f, 0x00 },  /* Ο to O */
-  { 0x03A0,  0x50, 0x00 },  /* Π to P */
-  { 0x03A1,  0x52, 0x00 },  /* Ρ to R */
-  { 0x03A3,  0x53, 0x00 },  /* Σ to S */
-  { 0x03A4,  0x54, 0x00 },  /* Τ to T */
-  { 0x03A5,  0x59, 0x00 },  /* Υ to Y */
-  { 0x03A6,  0x46, 0x00 },  /* Φ to F */
-  { 0x03A7,  0x43, 0x68 },  /* Χ to Ch */
-  { 0x03A8,  0x50, 0x73 },  /* Ψ to Ps */
-  { 0x03A9,  0x4f, 0x00 },  /* Ω to O */
-  { 0x03AA,  0x49, 0x00 },  /* Ϊ to I */
-  { 0x03AB,  0x59, 0x00 },  /* Ϋ to Y */
-  { 0x03AC,  0x61, 0x00 },  /* ά to a */
-  { 0x03AD,  0x65, 0x00 },  /* έ to e */
-  { 0x03AE,  0x69, 0x00 },  /* ή to i */
-  { 0x03AF,  0x69, 0x00 },  /* ί to i */
-  { 0x03B1,  0x61, 0x00 },  /* α to a */
-  { 0x03B2,  0x62, 0x00 },  /* β to b */
-  { 0x03B3,  0x67, 0x00 },  /* γ to g */
-  { 0x03B4,  0x64, 0x00 },  /* δ to d */
-  { 0x03B5,  0x65, 0x00 },  /* ε to e */
-  { 0x03B6,  0x7a, 0x00 },  /* ζ to z */
-  { 0x03B7,  0x69, 0x00 },  /* η to i */
-  { 0x03B8,  0x74, 0x68 },  /* θ to th */
-  { 0x03B9,  0x69, 0x00 },  /* ι to i */
-  { 0x03BA,  0x6b, 0x00 },  /* κ to k */
-  { 0x03BB,  0x6c, 0x00 },  /* λ to l */
-  { 0x03BC,  0x6d, 0x00 },  /* μ to m */
-  { 0x03BD,  0x6e, 0x00 },  /* ν to n */
-  { 0x03BE,  0x78, 0x00 },  /* ξ to x */
-  { 0x03BF,  0x6f, 0x00 },  /* ο to o */
-  { 0x03C0,  0x70, 0x00 },  /* π to p */
-  { 0x03C1,  0x72, 0x00 },  /* ρ to r */
-  { 0x03C3,  0x73, 0x00 },  /* σ to s */
-  { 0x03C4,  0x74, 0x00 },  /* τ to t */
-  { 0x03C5,  0x79, 0x00 },  /* υ to y */
-  { 0x03C6,  0x66, 0x00 },  /* φ to f */
-  { 0x03C7,  0x63, 0x68 },  /* χ to ch */
-  { 0x03C8,  0x70, 0x73 },  /* ψ to ps */
-  { 0x03C9,  0x6f, 0x00 },  /* ω to o */
-  { 0x03CA,  0x69, 0x00 },  /* ϊ to i */
-  { 0x03CB,  0x79, 0x00 },  /* ϋ to y */
-  { 0x03CC,  0x6f, 0x00 },  /* ό to o */
-  { 0x03CD,  0x79, 0x00 },  /* ύ to y */
-  { 0x03CE,  0x69, 0x00 },  /* ώ to i */
-  { 0x0400,  0x45, 0x00 },  /* Ѐ to E */
-  { 0x0401,  0x45, 0x00 },  /* Ё to E */
-  { 0x0402,  0x44, 0x00 },  /* Ђ to D */
-  { 0x0403,  0x47, 0x00 },  /* Ѓ to G */
-  { 0x0404,  0x45, 0x00 },  /* Є to E */
-  { 0x0405,  0x5a, 0x00 },  /* Ѕ to Z */
-  { 0x0406,  0x49, 0x00 },  /* І to I */
-  { 0x0407,  0x49, 0x00 },  /* Ї to I */
-  { 0x0408,  0x4a, 0x00 },  /* Ј to J */
-  { 0x0409,  0x49, 0x00 },  /* Љ to I */
-  { 0x040A,  0x4e, 0x00 },  /* Њ to N */
-  { 0x040B,  0x44, 0x00 },  /* Ћ to D */
-  { 0x040C,  0x4b, 0x00 },  /* Ќ to K */
-  { 0x040D,  0x49, 0x00 },  /* Ѝ to I */
-  { 0x040E,  0x55, 0x00 },  /* Ў to U */
-  { 0x040F,  0x44, 0x00 },  /* Џ to D */
-  { 0x0410,  0x41, 0x00 },  /* А to A */
-  { 0x0411,  0x42, 0x00 },  /* Б to B */
-  { 0x0412,  0x56, 0x00 },  /* В to V */
-  { 0x0413,  0x47, 0x00 },  /* Г to G */
-  { 0x0414,  0x44, 0x00 },  /* Д to D */
-  { 0x0415,  0x45, 0x00 },  /* Е to E */
-  { 0x0416,  0x5a, 0x68 },  /* Ж to Zh */
-  { 0x0417,  0x5a, 0x00 },  /* З to Z */
-  { 0x0418,  0x49, 0x00 },  /* И to I */
-  { 0x0419,  0x49, 0x00 },  /* Й to I */
-  { 0x041A,  0x4b, 0x00 },  /* К to K */
-  { 0x041B,  0x4c, 0x00 },  /* Л to L */
-  { 0x041C,  0x4d, 0x00 },  /* М to M */
-  { 0x041D,  0x4e, 0x00 },  /* Н to N */
-  { 0x041E,  0x4f, 0x00 },  /* О to O */
-  { 0x041F,  0x50, 0x00 },  /* П to P */
-  { 0x0420,  0x52, 0x00 },  /* Р to R */
-  { 0x0421,  0x53, 0x00 },  /* С to S */
-  { 0x0422,  0x54, 0x00 },  /* Т to T */
-  { 0x0423,  0x55, 0x00 },  /* У to U */
-  { 0x0424,  0x46, 0x00 },  /* Ф to F */
-  { 0x0425,  0x4b, 0x68 },  /* Х to Kh */
-  { 0x0426,  0x54, 0x63 },  /* Ц to Tc */
-  { 0x0427,  0x43, 0x68 },  /* Ч to Ch */
-  { 0x0428,  0x53, 0x68 },  /* Ш to Sh */
-  { 0x0429,  0x53, 0x68 },  /* Щ to Shch */
-  { 0x042A,  0x61, 0x00 },  /*  to A */
-  { 0x042B,  0x59, 0x00 },  /* Ы to Y */
-  { 0x042C,  0x59, 0x00 },  /*  to Y */
-  { 0x042D,  0x45, 0x00 },  /* Э to E */
-  { 0x042E,  0x49, 0x75 },  /* Ю to Iu */
-  { 0x042F,  0x49, 0x61 },  /* Я to Ia */
-  { 0x0430,  0x61, 0x00 },  /* а to a */
-  { 0x0431,  0x62, 0x00 },  /* б to b */
-  { 0x0432,  0x76, 0x00 },  /* в to v */
-  { 0x0433,  0x67, 0x00 },  /* г to g */
-  { 0x0434,  0x64, 0x00 },  /* д to d */
-  { 0x0435,  0x65, 0x00 },  /* е to e */
-  { 0x0436,  0x7a, 0x68 },  /* ж to zh */
-  { 0x0437,  0x7a, 0x00 },  /* з to z */
-  { 0x0438,  0x69, 0x00 },  /* и to i */
-  { 0x0439,  0x69, 0x00 },  /* й to i */
-  { 0x043A,  0x6b, 0x00 },  /* к to k */
-  { 0x043B,  0x6c, 0x00 },  /* л to l */
-  { 0x043C,  0x6d, 0x00 },  /* м to m */
-  { 0x043D,  0x6e, 0x00 },  /* н to n */
-  { 0x043E,  0x6f, 0x00 },  /* о to o */
-  { 0x043F,  0x70, 0x00 },  /* п to p */
-  { 0x0440,  0x72, 0x00 },  /* р to r */
-  { 0x0441,  0x73, 0x00 },  /* с to s */
-  { 0x0442,  0x74, 0x00 },  /* т to t */
-  { 0x0443,  0x75, 0x00 },  /* у to u */
-  { 0x0444,  0x66, 0x00 },  /* ф to f */
-  { 0x0445,  0x6b, 0x68 },  /* х to kh */
-  { 0x0446,  0x74, 0x63 },  /* ц to tc */
-  { 0x0447,  0x63, 0x68 },  /* ч to ch */
-  { 0x0448,  0x73, 0x68 },  /* ш to sh */
-  { 0x0449,  0x73, 0x68 },  /* щ to shch */
-  { 0x044A,  0x61, 0x00 },  /*  to a */
-  { 0x044B,  0x79, 0x00 },  /* ы to y */
-  { 0x044C,  0x79, 0x00 },  /*  to y */
-  { 0x044D,  0x65, 0x00 },  /* э to e */
-  { 0x044E,  0x69, 0x75 },  /* ю to iu */
-  { 0x044F,  0x69, 0x61 },  /* я to ia */
-  { 0x0450,  0x65, 0x00 },  /* ѐ to e */
-  { 0x0451,  0x65, 0x00 },  /* ё to e */
-  { 0x0452,  0x64, 0x00 },  /* ђ to d */
-  { 0x0453,  0x67, 0x00 },  /* ѓ to g */
-  { 0x0454,  0x65, 0x00 },  /* є to e */
-  { 0x0455,  0x7a, 0x00 },  /* ѕ to z */
-  { 0x0456,  0x69, 0x00 },  /* і to i */
-  { 0x0457,  0x69, 0x00 },  /* ї to i */
-  { 0x0458,  0x6a, 0x00 },  /* ј to j */
-  { 0x0459,  0x69, 0x00 },  /* љ to i */
-  { 0x045A,  0x6e, 0x00 },  /* њ to n */
-  { 0x045B,  0x64, 0x00 },  /* ћ to d */
-  { 0x045C,  0x6b, 0x00 },  /* ќ to k */
-  { 0x045D,  0x69, 0x00 },  /* ѝ to i */
-  { 0x045E,  0x75, 0x00 },  /* ў to u */
-  { 0x045F,  0x64, 0x00 },  /* џ to d */
-  { 0x1E02,  0x42, 0x00 },  /* Ḃ to B */
-  { 0x1E03,  0x62, 0x00 },  /* ḃ to b */
-  { 0x1E0A,  0x44, 0x00 },  /* Ḋ to D */
-  { 0x1E0B,  0x64, 0x00 },  /* ḋ to d */
-  { 0x1E1E,  0x46, 0x00 },  /* Ḟ to F */
-  { 0x1E1F,  0x66, 0x00 },  /* ḟ to f */
-  { 0x1E40,  0x4D, 0x00 },  /* Ṁ to M */
-  { 0x1E41,  0x6D, 0x00 },  /* ṁ to m */
-  { 0x1E56,  0x50, 0x00 },  /* Ṗ to P */
-  { 0x1E57,  0x70, 0x00 },  /* ṗ to p */
-  { 0x1E60,  0x53, 0x00 },  /* Ṡ to S */
-  { 0x1E61,  0x73, 0x00 },  /* ṡ to s */
-  { 0x1E6A,  0x54, 0x00 },  /* Ṫ to T */
-  { 0x1E6B,  0x74, 0x00 },  /* ṫ to t */
-  { 0x1E80,  0x57, 0x00 },  /* Ẁ to W */
-  { 0x1E81,  0x77, 0x00 },  /* ẁ to w */
-  { 0x1E82,  0x57, 0x00 },  /* Ẃ to W */
-  { 0x1E83,  0x77, 0x00 },  /* ẃ to w */
-  { 0x1E84,  0x57, 0x00 },  /* Ẅ to W */
-  { 0x1E85,  0x77, 0x00 },  /* ẅ to w */
-  { 0x1EF2,  0x59, 0x00 },  /* Ỳ to Y */
-  { 0x1EF3,  0x79, 0x00 },  /* ỳ to y */
-  { 0xFB00,  0x66, 0x66 },  /* ff to ff */
-  { 0xFB01,  0x66, 0x69 },  /* fi to fi */
-  { 0xFB02,  0x66, 0x6C },  /* fl to fl */
-  { 0xFB05,  0x73, 0x74 },  /* ſt to st */
-  { 0xFB06,  0x73, 0x74 },  /* st to st */
+static const Transliteration translit[] = {
+  { 0x00A0,  0x20, 0x00, 0x00, 0x00 },  /*   to   */
+  { 0x00B5,  0x75, 0x00, 0x00, 0x00 },  /* µ to u */
+  { 0x00C0,  0x41, 0x00, 0x00, 0x00 },  /* À to A */
+  { 0x00C1,  0x41, 0x00, 0x00, 0x00 },  /* Á to A */
+  { 0x00C2,  0x41, 0x00, 0x00, 0x00 },  /* Â to A */
+  { 0x00C3,  0x41, 0x00, 0x00, 0x00 },  /* Ã to A */
+  { 0x00C4,  0x41, 0x65, 0x00, 0x00 },  /* Ä to Ae */
+  { 0x00C5,  0x41, 0x61, 0x00, 0x00 },  /* Å to Aa */
+  { 0x00C6,  0x41, 0x45, 0x00, 0x00 },  /* Æ to AE */
+  { 0x00C7,  0x43, 0x00, 0x00, 0x00 },  /* Ç to C */
+  { 0x00C8,  0x45, 0x00, 0x00, 0x00 },  /* È to E */
+  { 0x00C9,  0x45, 0x00, 0x00, 0x00 },  /* É to E */
+  { 0x00CA,  0x45, 0x00, 0x00, 0x00 },  /* Ê to E */
+  { 0x00CB,  0x45, 0x00, 0x00, 0x00 },  /* Ë to E */
+  { 0x00CC,  0x49, 0x00, 0x00, 0x00 },  /* Ì to I */
+  { 0x00CD,  0x49, 0x00, 0x00, 0x00 },  /* Í to I */
+  { 0x00CE,  0x49, 0x00, 0x00, 0x00 },  /* Î to I */
+  { 0x00CF,  0x49, 0x00, 0x00, 0x00 },  /* Ï to I */
+  { 0x00D0,  0x44, 0x00, 0x00, 0x00 },  /* Ð to D */
+  { 0x00D1,  0x4E, 0x00, 0x00, 0x00 },  /* Ñ to N */
+  { 0x00D2,  0x4F, 0x00, 0x00, 0x00 },  /* Ò to O */
+  { 0x00D3,  0x4F, 0x00, 0x00, 0x00 },  /* Ó to O */
+  { 0x00D4,  0x4F, 0x00, 0x00, 0x00 },  /* Ô to O */
+  { 0x00D5,  0x4F, 0x00, 0x00, 0x00 },  /* Õ to O */
+  { 0x00D6,  0x4F, 0x65, 0x00, 0x00 },  /* Ö to Oe */
+  { 0x00D7,  0x78, 0x00, 0x00, 0x00 },  /* × to x */
+  { 0x00D8,  0x4F, 0x00, 0x00, 0x00 },  /* Ø to O */
+  { 0x00D9,  0x55, 0x00, 0x00, 0x00 },  /* Ù to U */
+  { 0x00DA,  0x55, 0x00, 0x00, 0x00 },  /* Ú to U */
+  { 0x00DB,  0x55, 0x00, 0x00, 0x00 },  /* Û to U */
+  { 0x00DC,  0x55, 0x65, 0x00, 0x00 },  /* Ü to Ue */
+  { 0x00DD,  0x59, 0x00, 0x00, 0x00 },  /* Ý to Y */
+  { 0x00DE,  0x54, 0x68, 0x00, 0x00 },  /* Þ to Th */
+  { 0x00DF,  0x73, 0x73, 0x00, 0x00 },  /* ß to ss */
+  { 0x00E0,  0x61, 0x00, 0x00, 0x00 },  /* à to a */
+  { 0x00E1,  0x61, 0x00, 0x00, 0x00 },  /* á to a */
+  { 0x00E2,  0x61, 0x00, 0x00, 0x00 },  /* â to a */
+  { 0x00E3,  0x61, 0x00, 0x00, 0x00 },  /* ã to a */
+  { 0x00E4,  0x61, 0x65, 0x00, 0x00 },  /* ä to ae */
+  { 0x00E5,  0x61, 0x61, 0x00, 0x00 },  /* å to aa */
+  { 0x00E6,  0x61, 0x65, 0x00, 0x00 },  /* æ to ae */
+  { 0x00E7,  0x63, 0x00, 0x00, 0x00 },  /* ç to c */
+  { 0x00E8,  0x65, 0x00, 0x00, 0x00 },  /* è to e */
+  { 0x00E9,  0x65, 0x00, 0x00, 0x00 },  /* é to e */
+  { 0x00EA,  0x65, 0x00, 0x00, 0x00 },  /* ê to e */
+  { 0x00EB,  0x65, 0x00, 0x00, 0x00 },  /* ë to e */
+  { 0x00EC,  0x69, 0x00, 0x00, 0x00 },  /* ì to i */
+  { 0x00ED,  0x69, 0x00, 0x00, 0x00 },  /* í to i */
+  { 0x00EE,  0x69, 0x00, 0x00, 0x00 },  /* î to i */
+  { 0x00EF,  0x69, 0x00, 0x00, 0x00 },  /* ï to i */
+  { 0x00F0,  0x64, 0x00, 0x00, 0x00 },  /* ð to d */
+  { 0x00F1,  0x6E, 0x00, 0x00, 0x00 },  /* ñ to n */
+  { 0x00F2,  0x6F, 0x00, 0x00, 0x00 },  /* ò to o */
+  { 0x00F3,  0x6F, 0x00, 0x00, 0x00 },  /* ó to o */
+  { 0x00F4,  0x6F, 0x00, 0x00, 0x00 },  /* ô to o */
+  { 0x00F5,  0x6F, 0x00, 0x00, 0x00 },  /* õ to o */
+  { 0x00F6,  0x6F, 0x65, 0x00, 0x00 },  /* ö to oe */
+  { 0x00F7,  0x3A, 0x00, 0x00, 0x00 },  /* ÷ to : */
+  { 0x00F8,  0x6F, 0x00, 0x00, 0x00 },  /* ø to o */
+  { 0x00F9,  0x75, 0x00, 0x00, 0x00 },  /* ù to u */
+  { 0x00FA,  0x75, 0x00, 0x00, 0x00 },  /* ú to u */
+  { 0x00FB,  0x75, 0x00, 0x00, 0x00 },  /* û to u */
+  { 0x00FC,  0x75, 0x65, 0x00, 0x00 },  /* ü to ue */
+  { 0x00FD,  0x79, 0x00, 0x00, 0x00 },  /* ý to y */
+  { 0x00FE,  0x74, 0x68, 0x00, 0x00 },  /* þ to th */
+  { 0x00FF,  0x79, 0x00, 0x00, 0x00 },  /* ÿ to y */
+  { 0x0100,  0x41, 0x00, 0x00, 0x00 },  /* Ā to A */
+  { 0x0101,  0x61, 0x00, 0x00, 0x00 },  /* ā to a */
+  { 0x0102,  0x41, 0x00, 0x00, 0x00 },  /* Ă to A */
+  { 0x0103,  0x61, 0x00, 0x00, 0x00 },  /* ă to a */
+  { 0x0104,  0x41, 0x00, 0x00, 0x00 },  /* Ą to A */
+  { 0x0105,  0x61, 0x00, 0x00, 0x00 },  /* ą to a */
+  { 0x0106,  0x43, 0x00, 0x00, 0x00 },  /* Ć to C */
+  { 0x0107,  0x63, 0x00, 0x00, 0x00 },  /* ć to c */
+  { 0x0108,  0x43, 0x68, 0x00, 0x00 },  /* Ĉ to Ch */
+  { 0x0109,  0x63, 0x68, 0x00, 0x00 },  /* ĉ to ch */
+  { 0x010A,  0x43, 0x00, 0x00, 0x00 },  /* Ċ to C */
+  { 0x010B,  0x63, 0x00, 0x00, 0x00 },  /* ċ to c */
+  { 0x010C,  0x43, 0x00, 0x00, 0x00 },  /* Č to C */
+  { 0x010D,  0x63, 0x00, 0x00, 0x00 },  /* č to c */
+  { 0x010E,  0x44, 0x00, 0x00, 0x00 },  /* Ď to D */
+  { 0x010F,  0x64, 0x00, 0x00, 0x00 },  /* ď to d */
+  { 0x0110,  0x44, 0x00, 0x00, 0x00 },  /* Đ to D */
+  { 0x0111,  0x64, 0x00, 0x00, 0x00 },  /* đ to d */
+  { 0x0112,  0x45, 0x00, 0x00, 0x00 },  /* Ē to E */
+  { 0x0113,  0x65, 0x00, 0x00, 0x00 },  /* ē to e */
+  { 0x0114,  0x45, 0x00, 0x00, 0x00 },  /* Ĕ to E */
+  { 0x0115,  0x65, 0x00, 0x00, 0x00 },  /* ĕ to e */
+  { 0x0116,  0x45, 0x00, 0x00, 0x00 },  /* Ė to E */
+  { 0x0117,  0x65, 0x00, 0x00, 0x00 },  /* ė to e */
+  { 0x0118,  0x45, 0x00, 0x00, 0x00 },  /* Ę to E */
+  { 0x0119,  0x65, 0x00, 0x00, 0x00 },  /* ę to e */
+  { 0x011A,  0x45, 0x00, 0x00, 0x00 },  /* Ě to E */
+  { 0x011B,  0x65, 0x00, 0x00, 0x00 },  /* ě to e */
+  { 0x011C,  0x47, 0x68, 0x00, 0x00 },  /* Ĝ to Gh */
+  { 0x011D,  0x67, 0x68, 0x00, 0x00 },  /* ĝ to gh */
+  { 0x011E,  0x47, 0x00, 0x00, 0x00 },  /* Ğ to G */
+  { 0x011F,  0x67, 0x00, 0x00, 0x00 },  /* ğ to g */
+  { 0x0120,  0x47, 0x00, 0x00, 0x00 },  /* Ġ to G */
+  { 0x0121,  0x67, 0x00, 0x00, 0x00 },  /* ġ to g */
+  { 0x0122,  0x47, 0x00, 0x00, 0x00 },  /* Ģ to G */
+  { 0x0123,  0x67, 0x00, 0x00, 0x00 },  /* ģ to g */
+  { 0x0124,  0x48, 0x68, 0x00, 0x00 },  /* Ĥ to Hh */
+  { 0x0125,  0x68, 0x68, 0x00, 0x00 },  /* ĥ to hh */
+  { 0x0126,  0x48, 0x00, 0x00, 0x00 },  /* Ħ to H */
+  { 0x0127,  0x68, 0x00, 0x00, 0x00 },  /* ħ to h */
+  { 0x0128,  0x49, 0x00, 0x00, 0x00 },  /* Ĩ to I */
+  { 0x0129,  0x69, 0x00, 0x00, 0x00 },  /* ĩ to i */
+  { 0x012A,  0x49, 0x00, 0x00, 0x00 },  /* Ī to I */
+  { 0x012B,  0x69, 0x00, 0x00, 0x00 },  /* ī to i */
+  { 0x012C,  0x49, 0x00, 0x00, 0x00 },  /* Ĭ to I */
+  { 0x012D,  0x69, 0x00, 0x00, 0x00 },  /* ĭ to i */
+  { 0x012E,  0x49, 0x00, 0x00, 0x00 },  /* Į to I */
+  { 0x012F,  0x69, 0x00, 0x00, 0x00 },  /* į to i */
+  { 0x0130,  0x49, 0x00, 0x00, 0x00 },  /* İ to I */
+  { 0x0131,  0x69, 0x00, 0x00, 0x00 },  /* ı to i */
+  { 0x0132,  0x49, 0x4A, 0x00, 0x00 },  /* IJ to IJ */
+  { 0x0133,  0x69, 0x6A, 0x00, 0x00 },  /* ij to ij */
+  { 0x0134,  0x4A, 0x68, 0x00, 0x00 },  /* Ĵ to Jh */
+  { 0x0135,  0x6A, 0x68, 0x00, 0x00 },  /* ĵ to jh */
+  { 0x0136,  0x4B, 0x00, 0x00, 0x00 },  /* Ķ to K */
+  { 0x0137,  0x6B, 0x00, 0x00, 0x00 },  /* ķ to k */
+  { 0x0138,  0x6B, 0x00, 0x00, 0x00 },  /* ĸ to k */
+  { 0x0139,  0x4C, 0x00, 0x00, 0x00 },  /* Ĺ to L */
+  { 0x013A,  0x6C, 0x00, 0x00, 0x00 },  /* ĺ to l */
+  { 0x013B,  0x4C, 0x00, 0x00, 0x00 },  /* Ļ to L */
+  { 0x013C,  0x6C, 0x00, 0x00, 0x00 },  /* ļ to l */
+  { 0x013D,  0x4C, 0x00, 0x00, 0x00 },  /* Ľ to L */
+  { 0x013E,  0x6C, 0x00, 0x00, 0x00 },  /* ľ to l */
+  { 0x013F,  0x4C, 0x2E, 0x00, 0x00 },  /* Ŀ to L. */
+  { 0x0140,  0x6C, 0x2E, 0x00, 0x00 },  /* ŀ to l. */
+  { 0x0141,  0x4C, 0x00, 0x00, 0x00 },  /* Ł to L */
+  { 0x0142,  0x6C, 0x00, 0x00, 0x00 },  /* ł to l */
+  { 0x0143,  0x4E, 0x00, 0x00, 0x00 },  /* Ń to N */
+  { 0x0144,  0x6E, 0x00, 0x00, 0x00 },  /* ń to n */
+  { 0x0145,  0x4E, 0x00, 0x00, 0x00 },  /* Ņ to N */
+  { 0x0146,  0x6E, 0x00, 0x00, 0x00 },  /* ņ to n */
+  { 0x0147,  0x4E, 0x00, 0x00, 0x00 },  /* Ň to N */
+  { 0x0148,  0x6E, 0x00, 0x00, 0x00 },  /* ň to n */
+  { 0x0149,  0x27, 0x6E, 0x00, 0x00 },  /* ʼn to 'n */
+  { 0x014A,  0x4E, 0x47, 0x00, 0x00 },  /* Ŋ to NG */
+  { 0x014B,  0x6E, 0x67, 0x00, 0x00 },  /* ŋ to ng */
+  { 0x014C,  0x4F, 0x00, 0x00, 0x00 },  /* Ō to O */
+  { 0x014D,  0x6F, 0x00, 0x00, 0x00 },  /* ō to o */
+  { 0x014E,  0x4F, 0x00, 0x00, 0x00 },  /* Ŏ to O */
+  { 0x014F,  0x6F, 0x00, 0x00, 0x00 },  /* ŏ to o */
+  { 0x0150,  0x4F, 0x00, 0x00, 0x00 },  /* Ő to O */
+  { 0x0151,  0x6F, 0x00, 0x00, 0x00 },  /* ő to o */
+  { 0x0152,  0x4F, 0x45, 0x00, 0x00 },  /* Πto OE */
+  { 0x0153,  0x6F, 0x65, 0x00, 0x00 },  /* œ to oe */
+  { 0x0154,  0x52, 0x00, 0x00, 0x00 },  /* Ŕ to R */
+  { 0x0155,  0x72, 0x00, 0x00, 0x00 },  /* ŕ to r */
+  { 0x0156,  0x52, 0x00, 0x00, 0x00 },  /* Ŗ to R */
+  { 0x0157,  0x72, 0x00, 0x00, 0x00 },  /* ŗ to r */
+  { 0x0158,  0x52, 0x00, 0x00, 0x00 },  /* Ř to R */
+  { 0x0159,  0x72, 0x00, 0x00, 0x00 },  /* ř to r */
+  { 0x015A,  0x53, 0x00, 0x00, 0x00 },  /* Ś to S */
+  { 0x015B,  0x73, 0x00, 0x00, 0x00 },  /* ś to s */
+  { 0x015C,  0x53, 0x68, 0x00, 0x00 },  /* Ŝ to Sh */
+  { 0x015D,  0x73, 0x68, 0x00, 0x00 },  /* ŝ to sh */
+  { 0x015E,  0x53, 0x00, 0x00, 0x00 },  /* Ş to S */
+  { 0x015F,  0x73, 0x00, 0x00, 0x00 },  /* ş to s */
+  { 0x0160,  0x53, 0x00, 0x00, 0x00 },  /* Š to S */
+  { 0x0161,  0x73, 0x00, 0x00, 0x00 },  /* š to s */
+  { 0x0162,  0x54, 0x00, 0x00, 0x00 },  /* Ţ to T */
+  { 0x0163,  0x74, 0x00, 0x00, 0x00 },  /* ţ to t */
+  { 0x0164,  0x54, 0x00, 0x00, 0x00 },  /* Ť to T */
+  { 0x0165,  0x74, 0x00, 0x00, 0x00 },  /* ť to t */
+  { 0x0166,  0x54, 0x00, 0x00, 0x00 },  /* Ŧ to T */
+  { 0x0167,  0x74, 0x00, 0x00, 0x00 },  /* ŧ to t */
+  { 0x0168,  0x55, 0x00, 0x00, 0x00 },  /* Ũ to U */
+  { 0x0169,  0x75, 0x00, 0x00, 0x00 },  /* ũ to u */
+  { 0x016A,  0x55, 0x00, 0x00, 0x00 },  /* Ū to U */
+  { 0x016B,  0x75, 0x00, 0x00, 0x00 },  /* ū to u */
+  { 0x016C,  0x55, 0x00, 0x00, 0x00 },  /* Ŭ to U */
+  { 0x016D,  0x75, 0x00, 0x00, 0x00 },  /* ŭ to u */
+  { 0x016E,  0x55, 0x00, 0x00, 0x00 },  /* Ů to U */
+  { 0x016F,  0x75, 0x00, 0x00, 0x00 },  /* ů to u */
+  { 0x0170,  0x55, 0x00, 0x00, 0x00 },  /* Ű to U */
+  { 0x0171,  0x75, 0x00, 0x00, 0x00 },  /* ű to u */
+  { 0x0172,  0x55, 0x00, 0x00, 0x00 },  /* Ų to U */
+  { 0x0173,  0x75, 0x00, 0x00, 0x00 },  /* ų to u */
+  { 0x0174,  0x57, 0x00, 0x00, 0x00 },  /* Ŵ to W */
+  { 0x0175,  0x77, 0x00, 0x00, 0x00 },  /* ŵ to w */
+  { 0x0176,  0x59, 0x00, 0x00, 0x00 },  /* Ŷ to Y */
+  { 0x0177,  0x79, 0x00, 0x00, 0x00 },  /* ŷ to y */
+  { 0x0178,  0x59, 0x00, 0x00, 0x00 },  /* Ÿ to Y */
+  { 0x0179,  0x5A, 0x00, 0x00, 0x00 },  /* Ź to Z */
+  { 0x017A,  0x7A, 0x00, 0x00, 0x00 },  /* ź to z */
+  { 0x017B,  0x5A, 0x00, 0x00, 0x00 },  /* Ż to Z */
+  { 0x017C,  0x7A, 0x00, 0x00, 0x00 },  /* ż to z */
+  { 0x017D,  0x5A, 0x00, 0x00, 0x00 },  /* Ž to Z */
+  { 0x017E,  0x7A, 0x00, 0x00, 0x00 },  /* ž to z */
+  { 0x017F,  0x73, 0x00, 0x00, 0x00 },  /* ſ to s */
+  { 0x0192,  0x66, 0x00, 0x00, 0x00 },  /* ƒ to f */
+  { 0x0218,  0x53, 0x00, 0x00, 0x00 },  /* Ș to S */
+  { 0x0219,  0x73, 0x00, 0x00, 0x00 },  /* ș to s */
+  { 0x021A,  0x54, 0x00, 0x00, 0x00 },  /* Ț to T */
+  { 0x021B,  0x74, 0x00, 0x00, 0x00 },  /* ț to t */
+  { 0x0386,  0x41, 0x00, 0x00, 0x00 },  /* Ά to A */
+  { 0x0388,  0x45, 0x00, 0x00, 0x00 },  /* Έ to E */
+  { 0x0389,  0x49, 0x00, 0x00, 0x00 },  /* Ή to I */
+  { 0x038A,  0x49, 0x00, 0x00, 0x00 },  /* Ί to I */
+  { 0x038C,  0x4f, 0x00, 0x00, 0x00 },  /* Ό to O */
+  { 0x038E,  0x59, 0x00, 0x00, 0x00 },  /* Ύ to Y */
+  { 0x038F,  0x4f, 0x00, 0x00, 0x00 },  /* Ώ to O */
+  { 0x0390,  0x69, 0x00, 0x00, 0x00 },  /* ΐ to i */
+  { 0x0391,  0x41, 0x00, 0x00, 0x00 },  /* Α to A */
+  { 0x0392,  0x42, 0x00, 0x00, 0x00 },  /* Β to B */
+  { 0x0393,  0x47, 0x00, 0x00, 0x00 },  /* Γ to G */
+  { 0x0394,  0x44, 0x00, 0x00, 0x00 },  /* Δ to D */
+  { 0x0395,  0x45, 0x00, 0x00, 0x00 },  /* Ε to E */
+  { 0x0396,  0x5a, 0x00, 0x00, 0x00 },  /* Ζ to Z */
+  { 0x0397,  0x49, 0x00, 0x00, 0x00 },  /* Η to I */
+  { 0x0398,  0x54, 0x68, 0x00, 0x00 },  /* Θ to Th */
+  { 0x0399,  0x49, 0x00, 0x00, 0x00 },  /* Ι to I */
+  { 0x039A,  0x4b, 0x00, 0x00, 0x00 },  /* Κ to K */
+  { 0x039B,  0x4c, 0x00, 0x00, 0x00 },  /* Λ to L */
+  { 0x039C,  0x4d, 0x00, 0x00, 0x00 },  /* Μ to M */
+  { 0x039D,  0x4e, 0x00, 0x00, 0x00 },  /* Ν to N */
+  { 0x039E,  0x58, 0x00, 0x00, 0x00 },  /* Ξ to X */
+  { 0x039F,  0x4f, 0x00, 0x00, 0x00 },  /* Ο to O */
+  { 0x03A0,  0x50, 0x00, 0x00, 0x00 },  /* Π to P */
+  { 0x03A1,  0x52, 0x00, 0x00, 0x00 },  /* Ρ to R */
+  { 0x03A3,  0x53, 0x00, 0x00, 0x00 },  /* Σ to S */
+  { 0x03A4,  0x54, 0x00, 0x00, 0x00 },  /* Τ to T */
+  { 0x03A5,  0x59, 0x00, 0x00, 0x00 },  /* Υ to Y */
+  { 0x03A6,  0x46, 0x00, 0x00, 0x00 },  /* Φ to F */
+  { 0x03A7,  0x43, 0x68, 0x00, 0x00 },  /* Χ to Ch */
+  { 0x03A8,  0x50, 0x73, 0x00, 0x00 },  /* Ψ to Ps */
+  { 0x03A9,  0x4f, 0x00, 0x00, 0x00 },  /* Ω to O */
+  { 0x03AA,  0x49, 0x00, 0x00, 0x00 },  /* Ϊ to I */
+  { 0x03AB,  0x59, 0x00, 0x00, 0x00 },  /* Ϋ to Y */
+  { 0x03AC,  0x61, 0x00, 0x00, 0x00 },  /* ά to a */
+  { 0x03AD,  0x65, 0x00, 0x00, 0x00 },  /* έ to e */
+  { 0x03AE,  0x69, 0x00, 0x00, 0x00 },  /* ή to i */
+  { 0x03AF,  0x69, 0x00, 0x00, 0x00 },  /* ί to i */
+  { 0x03B1,  0x61, 0x00, 0x00, 0x00 },  /* α to a */
+  { 0x03B2,  0x62, 0x00, 0x00, 0x00 },  /* β to b */
+  { 0x03B3,  0x67, 0x00, 0x00, 0x00 },  /* γ to g */
+  { 0x03B4,  0x64, 0x00, 0x00, 0x00 },  /* δ to d */
+  { 0x03B5,  0x65, 0x00, 0x00, 0x00 },  /* ε to e */
+  { 0x03B6,  0x7a, 0x00, 0x00, 0x00 },  /* ζ to z */
+  { 0x03B7,  0x69, 0x00, 0x00, 0x00 },  /* η to i */
+  { 0x03B8,  0x74, 0x68, 0x00, 0x00 },  /* θ to th */
+  { 0x03B9,  0x69, 0x00, 0x00, 0x00 },  /* ι to i */
+  { 0x03BA,  0x6b, 0x00, 0x00, 0x00 },  /* κ to k */
+  { 0x03BB,  0x6c, 0x00, 0x00, 0x00 },  /* λ to l */
+  { 0x03BC,  0x6d, 0x00, 0x00, 0x00 },  /* μ to m */
+  { 0x03BD,  0x6e, 0x00, 0x00, 0x00 },  /* ν to n */
+  { 0x03BE,  0x78, 0x00, 0x00, 0x00 },  /* ξ to x */
+  { 0x03BF,  0x6f, 0x00, 0x00, 0x00 },  /* ο to o */
+  { 0x03C0,  0x70, 0x00, 0x00, 0x00 },  /* π to p */
+  { 0x03C1,  0x72, 0x00, 0x00, 0x00 },  /* ρ to r */
+  { 0x03C3,  0x73, 0x00, 0x00, 0x00 },  /* σ to s */
+  { 0x03C4,  0x74, 0x00, 0x00, 0x00 },  /* τ to t */
+  { 0x03C5,  0x79, 0x00, 0x00, 0x00 },  /* υ to y */
+  { 0x03C6,  0x66, 0x00, 0x00, 0x00 },  /* φ to f */
+  { 0x03C7,  0x63, 0x68, 0x00, 0x00 },  /* χ to ch */
+  { 0x03C8,  0x70, 0x73, 0x00, 0x00 },  /* ψ to ps */
+  { 0x03C9,  0x6f, 0x00, 0x00, 0x00 },  /* ω to o */
+  { 0x03CA,  0x69, 0x00, 0x00, 0x00 },  /* ϊ to i */
+  { 0x03CB,  0x79, 0x00, 0x00, 0x00 },  /* ϋ to y */
+  { 0x03CC,  0x6f, 0x00, 0x00, 0x00 },  /* ό to o */
+  { 0x03CD,  0x79, 0x00, 0x00, 0x00 },  /* ύ to y */
+  { 0x03CE,  0x69, 0x00, 0x00, 0x00 },  /* ώ to i */
+  { 0x0400,  0x45, 0x00, 0x00, 0x00 },  /* Ѐ to E */
+  { 0x0401,  0x45, 0x00, 0x00, 0x00 },  /* Ё to E */
+  { 0x0402,  0x44, 0x00, 0x00, 0x00 },  /* Ђ to D */
+  { 0x0403,  0x47, 0x00, 0x00, 0x00 },  /* Ѓ to G */
+  { 0x0404,  0x45, 0x00, 0x00, 0x00 },  /* Є to E */
+  { 0x0405,  0x5a, 0x00, 0x00, 0x00 },  /* Ѕ to Z */
+  { 0x0406,  0x49, 0x00, 0x00, 0x00 },  /* І to I */
+  { 0x0407,  0x49, 0x00, 0x00, 0x00 },  /* Ї to I */
+  { 0x0408,  0x4a, 0x00, 0x00, 0x00 },  /* Ј to J */
+  { 0x0409,  0x49, 0x00, 0x00, 0x00 },  /* Љ to I */
+  { 0x040A,  0x4e, 0x00, 0x00, 0x00 },  /* Њ to N */
+  { 0x040B,  0x44, 0x00, 0x00, 0x00 },  /* Ћ to D */
+  { 0x040C,  0x4b, 0x00, 0x00, 0x00 },  /* Ќ to K */
+  { 0x040D,  0x49, 0x00, 0x00, 0x00 },  /* Ѝ to I */
+  { 0x040E,  0x55, 0x00, 0x00, 0x00 },  /* Ў to U */
+  { 0x040F,  0x44, 0x00, 0x00, 0x00 },  /* Џ to D */
+  { 0x0410,  0x41, 0x00, 0x00, 0x00 },  /* А to A */
+  { 0x0411,  0x42, 0x00, 0x00, 0x00 },  /* Б to B */
+  { 0x0412,  0x56, 0x00, 0x00, 0x00 },  /* В to V */
+  { 0x0413,  0x47, 0x00, 0x00, 0x00 },  /* Г to G */
+  { 0x0414,  0x44, 0x00, 0x00, 0x00 },  /* Д to D */
+  { 0x0415,  0x45, 0x00, 0x00, 0x00 },  /* Е to E */
+  { 0x0416,  0x5a, 0x68, 0x00, 0x00 },  /* Ж to Zh */
+  { 0x0417,  0x5a, 0x00, 0x00, 0x00 },  /* З to Z */
+  { 0x0418,  0x49, 0x00, 0x00, 0x00 },  /* И to I */
+  { 0x0419,  0x49, 0x00, 0x00, 0x00 },  /* Й to I */
+  { 0x041A,  0x4b, 0x00, 0x00, 0x00 },  /* К to K */
+  { 0x041B,  0x4c, 0x00, 0x00, 0x00 },  /* Л to L */
+  { 0x041C,  0x4d, 0x00, 0x00, 0x00 },  /* М to M */
+  { 0x041D,  0x4e, 0x00, 0x00, 0x00 },  /* Н to N */
+  { 0x041E,  0x4f, 0x00, 0x00, 0x00 },  /* О to O */
+  { 0x041F,  0x50, 0x00, 0x00, 0x00 },  /* П to P */
+  { 0x0420,  0x52, 0x00, 0x00, 0x00 },  /* Р to R */
+  { 0x0421,  0x53, 0x00, 0x00, 0x00 },  /* С to S */
+  { 0x0422,  0x54, 0x00, 0x00, 0x00 },  /* Т to T */
+  { 0x0423,  0x55, 0x00, 0x00, 0x00 },  /* У to U */
+  { 0x0424,  0x46, 0x00, 0x00, 0x00 },  /* Ф to F */
+  { 0x0425,  0x4b, 0x68, 0x00, 0x00 },  /* Х to Kh */
+  { 0x0426,  0x54, 0x63, 0x00, 0x00 },  /* Ц to Tc */
+  { 0x0427,  0x43, 0x68, 0x00, 0x00 },  /* Ч to Ch */
+  { 0x0428,  0x53, 0x68, 0x00, 0x00 },  /* Ш to Sh */
+  { 0x0429,  0x53, 0x68, 0x63, 0x68 },  /* Щ to Shch */
+  { 0x042A,  0x61, 0x00, 0x00, 0x00 },  /*  to A */
+  { 0x042B,  0x59, 0x00, 0x00, 0x00 },  /* Ы to Y */
+  { 0x042C,  0x59, 0x00, 0x00, 0x00 },  /*  to Y */
+  { 0x042D,  0x45, 0x00, 0x00, 0x00 },  /* Э to E */
+  { 0x042E,  0x49, 0x75, 0x00, 0x00 },  /* Ю to Iu */
+  { 0x042F,  0x49, 0x61, 0x00, 0x00 },  /* Я to Ia */
+  { 0x0430,  0x61, 0x00, 0x00, 0x00 },  /* а to a */
+  { 0x0431,  0x62, 0x00, 0x00, 0x00 },  /* б to b */
+  { 0x0432,  0x76, 0x00, 0x00, 0x00 },  /* в to v */
+  { 0x0433,  0x67, 0x00, 0x00, 0x00 },  /* г to g */
+  { 0x0434,  0x64, 0x00, 0x00, 0x00 },  /* д to d */
+  { 0x0435,  0x65, 0x00, 0x00, 0x00 },  /* е to e */
+  { 0x0436,  0x7a, 0x68, 0x00, 0x00 },  /* ж to zh */
+  { 0x0437,  0x7a, 0x00, 0x00, 0x00 },  /* з to z */
+  { 0x0438,  0x69, 0x00, 0x00, 0x00 },  /* и to i */
+  { 0x0439,  0x69, 0x00, 0x00, 0x00 },  /* й to i */
+  { 0x043A,  0x6b, 0x00, 0x00, 0x00 },  /* к to k */
+  { 0x043B,  0x6c, 0x00, 0x00, 0x00 },  /* л to l */
+  { 0x043C,  0x6d, 0x00, 0x00, 0x00 },  /* м to m */
+  { 0x043D,  0x6e, 0x00, 0x00, 0x00 },  /* н to n */
+  { 0x043E,  0x6f, 0x00, 0x00, 0x00 },  /* о to o */
+  { 0x043F,  0x70, 0x00, 0x00, 0x00 },  /* п to p */
+  { 0x0440,  0x72, 0x00, 0x00, 0x00 },  /* р to r */
+  { 0x0441,  0x73, 0x00, 0x00, 0x00 },  /* с to s */
+  { 0x0442,  0x74, 0x00, 0x00, 0x00 },  /* т to t */
+  { 0x0443,  0x75, 0x00, 0x00, 0x00 },  /* у to u */
+  { 0x0444,  0x66, 0x00, 0x00, 0x00 },  /* ф to f */
+  { 0x0445,  0x6b, 0x68, 0x00, 0x00 },  /* х to kh */
+  { 0x0446,  0x74, 0x63, 0x00, 0x00 },  /* ц to tc */
+  { 0x0447,  0x63, 0x68, 0x00, 0x00 },  /* ч to ch */
+  { 0x0448,  0x73, 0x68, 0x00, 0x00 },  /* ш to sh */
+  { 0x0449,  0x73, 0x68, 0x63, 0x68 },  /* щ to shch */
+  { 0x044A,  0x61, 0x00, 0x00, 0x00 },  /*  to a */
+  { 0x044B,  0x79, 0x00, 0x00, 0x00 },  /* ы to y */
+  { 0x044C,  0x79, 0x00, 0x00, 0x00 },  /*  to y */
+  { 0x044D,  0x65, 0x00, 0x00, 0x00 },  /* э to e */
+  { 0x044E,  0x69, 0x75, 0x00, 0x00 },  /* ю to iu */
+  { 0x044F,  0x69, 0x61, 0x00, 0x00 },  /* я to ia */
+  { 0x0450,  0x65, 0x00, 0x00, 0x00 },  /* ѐ to e */
+  { 0x0451,  0x65, 0x00, 0x00, 0x00 },  /* ё to e */
+  { 0x0452,  0x64, 0x00, 0x00, 0x00 },  /* ђ to d */
+  { 0x0453,  0x67, 0x00, 0x00, 0x00 },  /* ѓ to g */
+  { 0x0454,  0x65, 0x00, 0x00, 0x00 },  /* є to e */
+  { 0x0455,  0x7a, 0x00, 0x00, 0x00 },  /* ѕ to z */
+  { 0x0456,  0x69, 0x00, 0x00, 0x00 },  /* і to i */
+  { 0x0457,  0x69, 0x00, 0x00, 0x00 },  /* ї to i */
+  { 0x0458,  0x6a, 0x00, 0x00, 0x00 },  /* ј to j */
+  { 0x0459,  0x69, 0x00, 0x00, 0x00 },  /* љ to i */
+  { 0x045A,  0x6e, 0x00, 0x00, 0x00 },  /* њ to n */
+  { 0x045B,  0x64, 0x00, 0x00, 0x00 },  /* ћ to d */
+  { 0x045C,  0x6b, 0x00, 0x00, 0x00 },  /* ќ to k */
+  { 0x045D,  0x69, 0x00, 0x00, 0x00 },  /* ѝ to i */
+  { 0x045E,  0x75, 0x00, 0x00, 0x00 },  /* ў to u */
+  { 0x045F,  0x64, 0x00, 0x00, 0x00 },  /* џ to d */
+  { 0x1E02,  0x42, 0x00, 0x00, 0x00 },  /* Ḃ to B */
+  { 0x1E03,  0x62, 0x00, 0x00, 0x00 },  /* ḃ to b */
+  { 0x1E0A,  0x44, 0x00, 0x00, 0x00 },  /* Ḋ to D */
+  { 0x1E0B,  0x64, 0x00, 0x00, 0x00 },  /* ḋ to d */
+  { 0x1E1E,  0x46, 0x00, 0x00, 0x00 },  /* Ḟ to F */
+  { 0x1E1F,  0x66, 0x00, 0x00, 0x00 },  /* ḟ to f */
+  { 0x1E40,  0x4D, 0x00, 0x00, 0x00 },  /* Ṁ to M */
+  { 0x1E41,  0x6D, 0x00, 0x00, 0x00 },  /* ṁ to m */
+  { 0x1E56,  0x50, 0x00, 0x00, 0x00 },  /* Ṗ to P */
+  { 0x1E57,  0x70, 0x00, 0x00, 0x00 },  /* ṗ to p */
+  { 0x1E60,  0x53, 0x00, 0x00, 0x00 },  /* Ṡ to S */
+  { 0x1E61,  0x73, 0x00, 0x00, 0x00 },  /* ṡ to s */
+  { 0x1E6A,  0x54, 0x00, 0x00, 0x00 },  /* Ṫ to T */
+  { 0x1E6B,  0x74, 0x00, 0x00, 0x00 },  /* ṫ to t */
+  { 0x1E80,  0x57, 0x00, 0x00, 0x00 },  /* Ẁ to W */
+  { 0x1E81,  0x77, 0x00, 0x00, 0x00 },  /* ẁ to w */
+  { 0x1E82,  0x57, 0x00, 0x00, 0x00 },  /* Ẃ to W */
+  { 0x1E83,  0x77, 0x00, 0x00, 0x00 },  /* ẃ to w */
+  { 0x1E84,  0x57, 0x00, 0x00, 0x00 },  /* Ẅ to W */
+  { 0x1E85,  0x77, 0x00, 0x00, 0x00 },  /* ẅ to w */
+  { 0x1EF2,  0x59, 0x00, 0x00, 0x00 },  /* Ỳ to Y */
+  { 0x1EF3,  0x79, 0x00, 0x00, 0x00 },  /* ỳ to y */
+  { 0xFB00,  0x66, 0x66, 0x00, 0x00 },  /* ff to ff */
+  { 0xFB01,  0x66, 0x69, 0x00, 0x00 },  /* fi to fi */
+  { 0xFB02,  0x66, 0x6C, 0x00, 0x00 },  /* fl to fl */
+  { 0xFB05,  0x73, 0x74, 0x00, 0x00 },  /* ſt to st */
+  { 0xFB06,  0x73, 0x74, 0x00, 0x00 },  /* st to st */
 };
 
+static const Transliteration *spellfixFindTranslit(int c, int *pxTop){
+  *pxTop = (sizeof(translit)/sizeof(translit[0])) - 1;
+  return translit;
+}
+
 /*
 ** Convert the input string from UTF-8 into pure ASCII by converting
 ** all non-ASCII characters to some combination of characters in the
@@ -1621,23 +1629,24 @@ static unsigned char *transliterate(const unsigned char *zIn, int nIn){
       zOut[nOut++] = (unsigned char)c;
     }else{
       int xTop, xBtm, x;
-      xTop = sizeof(translit)/sizeof(translit[0]) - 1;
+      const Transliteration *tbl = spellfixFindTranslit(c, &xTop);
       xBtm = 0;
       while( xTop>=xBtm ){
         x = (xTop + xBtm)/2;
-        if( translit[x].cFrom==c ){
-          zOut[nOut++] = translit[x].cTo0;
-          if( translit[x].cTo1 ){
-            zOut[nOut++] = translit[x].cTo1;
-            /* Add an extra "ch" after the "sh" for Щ and щ */
-            if( c==0x0429 || c== 0x0449 ){
-              zOut[nOut++] = 'c';
-              zOut[nOut++] = 'h';
+        if( tbl[x].cFrom==c ){
+          zOut[nOut++] = tbl[x].cTo0;
+          if( tbl[x].cTo1 ){
+            zOut[nOut++] = tbl[x].cTo1;
+            if( tbl[x].cTo2 ){
+              zOut[nOut++] = tbl[x].cTo2;
+              if( tbl[x].cTo3 ){
+                zOut[nOut++] = tbl[x].cTo3;
+              }
             }
           }
           c = 0;
           break;
-        }else if( translit[x].cFrom>c ){
+        }else if( tbl[x].cFrom>c ){
           xTop = x-1;
         }else{
           xBtm = x+1;
@@ -1668,15 +1677,22 @@ static int translen_to_charlen(const char *zIn, int nIn, int nTrans){
     nOut++;
     if( c>=128 ){
       int xTop, xBtm, x;
-      xTop = sizeof(translit)/sizeof(translit[0]) - 1;
+      const Transliteration *tbl = spellfixFindTranslit(c, &xTop);
       xBtm = 0;
       while( xTop>=xBtm ){
         x = (xTop + xBtm)/2;
-        if( translit[x].cFrom==c ){
-          if( translit[x].cTo1 ) nOut++;
-          if( c==0x0429 || c== 0x0449 ) nOut += 2;
+        if( tbl[x].cFrom==c ){
+          if( tbl[x].cTo1 ){
+            nOut++;
+            if( tbl[x].cTo2 ){
+              nOut++;
+              if( tbl[x].cTo3 ){
+                nOut++;
+              }
+            }
+          }
           break;
-        }else if( translit[x].cFrom>c ){
+        }else if( tbl[x].cFrom>c ){
           xTop = x-1;
         }else{
           xBtm = x+1;
diff --git a/manifest b/manifest
index e20e592681..db881f438c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Rearrange\sfields\sof\sthe\sBtCursor\sobject\sso\sthat\sit\sis\ssmaller\sand\srequires\sless\ninitialization,\sfor\sa\ssmall\sperformance\simprovement.
-D 2018-01-24T16:04:21.587
+C Reorganize\sspellfix.c\sto\smake\sit\seasier\sto\sedit\sautomatically\s(e.g.\susing\sa\s\nscript).\sNo\schanges\sto\sfunctionality.
+D 2018-01-24T18:28:39.575
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -294,7 +294,7 @@ F ext/misc/series.c f3c0dba5c5c749ce1782b53076108f87cf0b71041eb6023f727a9c50681d
 F ext/misc/sha1.c 0b9e9b855354910d3ca467bf39099d570e73db56
 F ext/misc/shathree.c 9e960ba50483214c6a7a4b1517f8d8cef799e9db381195178c3fd3ad207e10c0
 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52
-F ext/misc/spellfix.c 41cf26c6b89fcaa8798ae10ae64d39c1f1d9d6995152e545bd491c13058b8fac
+F ext/misc/spellfix.c 705ce1ed701b38f43c8ee7dd848877ff51c09eda322e39ccac6873adb4b74735
 F ext/misc/sqlar.c 57d5bc45cd5492208e451f697404be88f8612527d64c9d42f96b325b64983d74
 F ext/misc/stmt.c 6f16443abb3551e3f5813bb13ba19a30e7032830015b0f92fe0c0453045c0a11
 F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512
@@ -1702,7 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 16ebe558d2f0982f630c39b6c1b00d14d27011d454cced213f17c26491a2c06e
-R fc9e02eb09b9540ef2f7fa9944ac0566
-U drh
-Z c7e32730b85a19ed6c94c7a445674e7d
+P 0ddf5292cc0411ec6fcb7399ecf2904c899e0488404d3f65490fbe1db15efdf4
+R 9f30c4a7d442a62a313819b7586e7fb7
+U dan
+Z 5b6a5ad739adf30d436b457769ddc2cb
diff --git a/manifest.uuid b/manifest.uuid
index 1dd418e847..795b17508a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0ddf5292cc0411ec6fcb7399ecf2904c899e0488404d3f65490fbe1db15efdf4
\ No newline at end of file
+090a64faaac579c6ed1cddb02d14e7089468a4aeaeec11e78d461790bcb6eb0a
\ No newline at end of file

From 9f8952390a97140d8dffa9be53fd61121726629f Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 24 Jan 2018 20:42:42 +0000
Subject: [PATCH 386/488] When unpacking a sorter record, do so from right to
 left instead of left to right, since by starting with the right-most column,
 the work done by OP_Column opcodes is reduced.

FossilOrigin-Name: 8055e4f42446ceb5bcf752bbf41a73289c3ca759c56c9f779edc3d7f202b7881
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/select.c  |  7 +++++--
 3 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index db881f438c..36be0b5904 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Reorganize\sspellfix.c\sto\smake\sit\seasier\sto\sedit\sautomatically\s(e.g.\susing\sa\s\nscript).\sNo\schanges\sto\sfunctionality.
-D 2018-01-24T18:28:39.575
+C When\sunpacking\sa\ssorter\srecord,\sdo\sso\sfrom\sright\sto\sleft\sinstead\sof\sleft\sto\nright,\ssince\sby\sstarting\swith\sthe\sright-most\scolumn,\sthe\swork\sdone\sby\nOP_Column\sopcodes\sis\sreduced.
+D 2018-01-24T20:42:42.319
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -487,7 +487,7 @@ F src/printf.c 9506b4b96e59c0467047155f09015750cb2878aeda3d39e5610c1192ddc3c41c
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c bebe7cce45d899d2237c76bce059d525abf5b861f2fce92f6b53914a961c01ba
+F src/select.c f02352ac5cbb6fad3804add825743b924cfb6c742ba2e8666d726828a9985d73
 F src/shell.c.in 7cea439c3f7f2e4ed6eb4b3a633cd93dccb1349241400de4da0c1291285ed514
 F src/sqlite.h.in 51f9acf52c80113d793ddd38b3940ad6895d97b4752503b19291fb8fcbf54c5e
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
@@ -1702,7 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0ddf5292cc0411ec6fcb7399ecf2904c899e0488404d3f65490fbe1db15efdf4
-R 9f30c4a7d442a62a313819b7586e7fb7
-U dan
-Z 5b6a5ad739adf30d436b457769ddc2cb
+P 090a64faaac579c6ed1cddb02d14e7089468a4aeaeec11e78d461790bcb6eb0a
+R 36fadb5748a2d2919f267681ce3c3a75
+U drh
+Z 4006c7b9f32358fa0bed65eb183dba8e
diff --git a/manifest.uuid b/manifest.uuid
index 795b17508a..4ff4675403 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-090a64faaac579c6ed1cddb02d14e7089468a4aeaeec11e78d461790bcb6eb0a
\ No newline at end of file
+8055e4f42446ceb5bcf752bbf41a73289c3ca759c56c9f779edc3d7f202b7881
\ No newline at end of file
diff --git a/src/select.c b/src/select.c
index c3cb4082fe..9ac4f9f198 100644
--- a/src/select.c
+++ b/src/select.c
@@ -1266,12 +1266,15 @@ static void generateSortTail(
     iSortTab = iTab;
     bSeq = 1;
   }
-  for(i=0, iCol=nKey+bSeq; i<nSortData; i++){
+  for(i=0, iCol=nKey+bSeq-1; i<nSortData; i++){
+    if( aOutEx[i].u.x.iOrderByCol==0 ) iCol++;
+  }
+  for(i=nSortData-1; i>=0; i--){
     int iRead;
     if( aOutEx[i].u.x.iOrderByCol ){
       iRead = aOutEx[i].u.x.iOrderByCol-1;
     }else{
-      iRead = iCol++;
+      iRead = iCol--;
     }
     sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iRead, regRow+i);
     VdbeComment((v, "%s", aOutEx[i].zName ? aOutEx[i].zName : aOutEx[i].zSpan));

From cad42838d718fdabb18d0ea4fc72ced389489cb0 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 25 Jan 2018 01:20:29 +0000
Subject: [PATCH 387/488] Slightly smaller and faster implementation of OP_If
 and OP_IfNot.

FossilOrigin-Name: 6ab42934e2c4957b5d8927bf4434a9db07ab6078987a6a2d25f35cc468d21203
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/vdbe.c    | 10 +++++-----
 3 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/manifest b/manifest
index 36be0b5904..7d40e3172b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C When\sunpacking\sa\ssorter\srecord,\sdo\sso\sfrom\sright\sto\sleft\sinstead\sof\sleft\sto\nright,\ssince\sby\sstarting\swith\sthe\sright-most\scolumn,\sthe\swork\sdone\sby\nOP_Column\sopcodes\sis\sreduced.
-D 2018-01-24T20:42:42.319
+C Slightly\ssmaller\sand\sfaster\simplementation\sof\sOP_If\sand\sOP_IfNot.
+D 2018-01-25T01:20:29.953
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -556,7 +556,7 @@ F src/update.c a90a32ffc0100265b0693dbbdbe490756447af181f5ea2c138cce515b08c8795
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c ef4a5f904d942e660abade7fbf3e6bdb402dabe9e7c27f3361ecf40b945538b5
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
-F src/vdbe.c d4684bd5efad095d9e4405e5ca2e1168dc7aff643cae04c424e4bade70641d70
+F src/vdbe.c 0e13b8c33a9ecf236b3045f190fb58933049582307b621cbd924b6582a2b705f
 F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a
 F src/vdbeInt.h 8d7d07f13cb3c4cbca91e22ba4a1920e542dda7c5d9299920432a0b3d5b009f5
 F src/vdbeapi.c fea41171884a4de119f8b10ab514c788674eeeb7f27218bb6d008e1310bfd07f
@@ -1702,7 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 090a64faaac579c6ed1cddb02d14e7089468a4aeaeec11e78d461790bcb6eb0a
-R 36fadb5748a2d2919f267681ce3c3a75
+P 8055e4f42446ceb5bcf752bbf41a73289c3ca759c56c9f779edc3d7f202b7881
+R 3a57ffa9e8084df96fa05ea7edc4d068
 U drh
-Z 4006c7b9f32358fa0bed65eb183dba8e
+Z 1f11fb78272f5afcdf3b192bdf3cca0d
diff --git a/manifest.uuid b/manifest.uuid
index 4ff4675403..8b78bd0022 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-8055e4f42446ceb5bcf752bbf41a73289c3ca759c56c9f779edc3d7f202b7881
\ No newline at end of file
+6ab42934e2c4957b5d8927bf4434a9db07ab6078987a6a2d25f35cc468d21203
\ No newline at end of file
diff --git a/src/vdbe.c b/src/vdbe.c
index 81a2361a55..495b56aa1d 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -2289,11 +2289,11 @@ case OP_IfNot: {            /* jump, in1 */
   if( pIn1->flags & MEM_Null ){
     c = pOp->p3;
   }else{
-#ifdef SQLITE_OMIT_FLOATING_POINT
-    c = sqlite3VdbeIntValue(pIn1)!=0;
-#else
-    c = sqlite3VdbeRealValue(pIn1)!=0.0;
-#endif
+    if( pIn1->flags & MEM_Int ){
+      c = pIn1->u.i!=0;
+    }else{
+      c = sqlite3VdbeRealValue(pIn1)!=0.0;
+    }
     if( pOp->opcode==OP_IfNot ) c = !c;
   }
   VdbeBranchTaken(c!=0, 2);

From 8558ef2e199c63455305ea3ffb7f31a56193814d Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 25 Jan 2018 20:50:46 +0000
Subject: [PATCH 388/488] Reorganize zipfile.c code to make it easier to add
 support for in-memory zip archive processing.

FossilOrigin-Name: 30b9258294e3028ef4ea467e332e013995509544e9a23c8bbf5168772a7e895d
---
 ext/misc/zipfile.c | 733 +++++++++++++++++++++++----------------------
 manifest           |  15 +-
 manifest.uuid      |   2 +-
 3 files changed, 382 insertions(+), 368 deletions(-)

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index fa31d62d11..d378606941 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -222,10 +222,9 @@ struct ZipfileLFH {
 
 typedef struct ZipfileEntry ZipfileEntry;
 struct ZipfileEntry {
-  char *zPath;               /* Path of zipfile entry */
-  u8 *aCdsEntry;             /* Buffer containing entire CDS entry */
-  int nCdsEntry;             /* Size of buffer aCdsEntry[] in bytes */
-  int bDeleted;              /* True if entry has been deleted */
+  ZipfileCDS cds;            /* Parsed CDS record */
+  u32 mUnixTime;             /* Modification time, in UNIX format */
+  u8 *aExtra;                /* cds.nExtra+cds.nComment bytes of extra data */
   ZipfileEntry *pNext;       /* Next element in in-memory CDS */
 };
 
@@ -236,28 +235,30 @@ typedef struct ZipfileCsr ZipfileCsr;
 struct ZipfileCsr {
   sqlite3_vtab_cursor base;  /* Base class - must be first */
   i64 iId;                   /* Cursor ID */
-  int bEof;                  /* True when at EOF */
+  u8 bEof;                   /* True when at EOF */
+  u8 bNoop;                  /* If next xNext() call is no-op */
 
   /* Used outside of write transactions */
   FILE *pFile;               /* Zip file */
   i64 iNextOff;              /* Offset of next record in central directory */
   ZipfileEOCD eocd;          /* Parse of central directory record */
 
-  /* Used inside write transactions */
-  ZipfileEntry *pCurrent;
-
-  ZipfileCDS cds;            /* Central Directory Structure */
+  ZipfileEntry *pCurrent;    /* Current entry */
   ZipfileLFH lfh;            /* Local File Header for current entry */
   i64 iDataOff;              /* Offset in zipfile to data */
-  u32 mTime;                 /* Extended mtime value */
-  int flags;                 /* Flags byte (see below for bits) */
   ZipfileCsr *pCsrNext;      /* Next cursor on same virtual table */
 };
 
+/*
+** Possible values for ZipfileCsr.eType. Set in zipfileFilter().
+*/
+#define ZIPFILE_CSR_LIST 1        /* Cursor reads from ZipfileTab.pFirstEntry */
+#define ZIPFILE_CSR_FILE 2        /* Cursor reads from file on disk */
+
 /*
 ** Values for ZipfileCsr.flags.
 */
-#define ZIPFILE_MTIME_VALID 0x0001
+#define ZIPFILE_NOOP_NEXT   0x0002     /* Next xNext() is a no-op */
 
 typedef struct ZipfileTab ZipfileTab;
 struct ZipfileTab {
@@ -335,10 +336,36 @@ static int zipfileConnect(
   return rc;
 }
 
+static void zipfileEntryFree(ZipfileEntry *p){
+  if( p ){
+    sqlite3_free(p->cds.zFile);
+    sqlite3_free(p);
+  }
+}
+
+static void zipfileCleanupTransaction(ZipfileTab *pTab){
+  ZipfileEntry *pEntry;
+  ZipfileEntry *pNext;
+
+  if( pTab->pWriteFd ){
+    fclose(pTab->pWriteFd);
+    pTab->pWriteFd = 0;
+  }
+  for(pEntry=pTab->pFirstEntry; pEntry; pEntry=pNext){
+    pNext = pEntry->pNext;
+    zipfileEntryFree(pEntry);
+  }
+  pTab->pFirstEntry = 0;
+  pTab->pLastEntry = 0;
+  pTab->szCurrent = 0;
+  pTab->szOrig = 0;
+}
+
 /*
 ** This method is the destructor for zipfile vtab objects.
 */
 static int zipfileDisconnect(sqlite3_vtab *pVtab){
+  zipfileCleanupTransaction((ZipfileTab*)pVtab);
   sqlite3_free(pVtab);
   return SQLITE_OK;
 }
@@ -366,12 +393,12 @@ static int zipfileOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){
 ** by zipfileOpen().
 */
 static void zipfileResetCursor(ZipfileCsr *pCsr){
-  sqlite3_free(pCsr->cds.zFile);
-  pCsr->cds.zFile = 0;
   pCsr->bEof = 0;
   if( pCsr->pFile ){
     fclose(pCsr->pFile);
     pCsr->pFile = 0;
+    zipfileEntryFree(pCsr->pCurrent);
+    pCsr->pCurrent = 0;
   }
 }
 
@@ -512,77 +539,117 @@ static int zipfileReadCDS(u8 *aBuf, ZipfileCDS *pCDS){
 }
 
 /*
-** Read the CDS record for the current entry from disk into pCsr->cds.
+** The general format of an extra field is:
+**
+**   Header ID    2 bytes
+**   Data Size    2 bytes
+**   Data         N bytes
 */
-static int zipfileCsrReadCDS(ZipfileCsr *pCsr){
-  char **pzErr = &pCsr->base.pVtab->zErrMsg;
-  u8 *aRead;
-  int rc = SQLITE_OK;
+static int zipfileScanExtra(u8 *aExtra, int nExtra, u32 *pmTime){
+  int ret = 0;
+  u8 *p = aExtra;
+  u8 *pEnd = &aExtra[nExtra];
 
-  sqlite3_free(pCsr->cds.zFile);
-  pCsr->cds.zFile = 0;
+  while( p<pEnd ){
+    u16 id = zipfileRead16(p);
+    u16 nByte = zipfileRead16(p);
 
-  if( pCsr->pCurrent==0 ){
-    aRead = zipfileCsrBuffer(pCsr);
-    rc = zipfileReadData(
-        pCsr->pFile, aRead, ZIPFILE_CDS_FIXED_SZ, pCsr->iNextOff, pzErr
-    );
-  }else{
-    aRead = pCsr->pCurrent->aCdsEntry;
-  }
-
-  if( rc==SQLITE_OK ){
-    rc = zipfileReadCDS(aRead, &pCsr->cds);
-    if( rc!=SQLITE_OK ){
-      assert( pCsr->pCurrent==0 );
-      zipfileSetErrmsg(pCsr,"failed to read CDS at offset %lld",pCsr->iNextOff);
-    }else{
-      int nRead;
-      if( pCsr->pCurrent==0 ){
-        nRead = pCsr->cds.nFile + pCsr->cds.nExtra;
-        aRead = zipfileCsrBuffer(pCsr);
-        pCsr->iNextOff += ZIPFILE_CDS_FIXED_SZ;
-        rc = zipfileReadData(pCsr->pFile, aRead, nRead, pCsr->iNextOff, pzErr);
-      }else{
-        aRead = &aRead[ZIPFILE_CDS_FIXED_SZ];
-      }
-
-      if( rc==SQLITE_OK ){
-        pCsr->cds.zFile = sqlite3_mprintf("%.*s", (int)pCsr->cds.nFile, aRead);
-        pCsr->iNextOff += pCsr->cds.nFile;
-        pCsr->iNextOff += pCsr->cds.nExtra;
-        pCsr->iNextOff += pCsr->cds.nComment;
-      }
-
-      /* Scan the cds.nExtra bytes of "extra" fields for any that can
-      ** be interpreted. The general format of an extra field is:
-      **
-      **   Header ID    2 bytes
-      **   Data Size    2 bytes
-      **   Data         N bytes
-      **
-      */
-      if( rc==SQLITE_OK ){
-        u8 *p = &aRead[pCsr->cds.nFile];
-        u8 *pEnd = &p[pCsr->cds.nExtra];
-
-        while( p<pEnd ){
-          u16 id = zipfileRead16(p);
-          u16 nByte = zipfileRead16(p);
-
-          switch( id ){
-            case ZIPFILE_EXTRA_TIMESTAMP: {
-              u8 b = p[0];
-              if( b & 0x01 ){     /* 0x01 -> modtime is present */
-                pCsr->mTime = zipfileGetU32(&p[1]);
-                pCsr->flags |= ZIPFILE_MTIME_VALID;
-              }
-              break;
-            }
-          }
-
-          p += nByte;
+    switch( id ){
+      case ZIPFILE_EXTRA_TIMESTAMP: {
+        u8 b = p[0];
+        if( b & 0x01 ){     /* 0x01 -> modtime is present */
+          *pmTime = zipfileGetU32(&p[1]);
+          ret = 1;
         }
+        break;
+      }
+    }
+
+    p += nByte;
+  }
+  return ret;
+}
+
+/*
+** "Standard" MS-DOS time format:
+**
+**   File modification time:
+**     Bits 00-04: seconds divided by 2
+**     Bits 05-10: minute
+**     Bits 11-15: hour
+**   File modification date:
+**     Bits 00-04: day
+**     Bits 05-08: month (1-12)
+**     Bits 09-15: years from 1980 
+*/
+static time_t zipfileMtime(ZipfileCDS *pCDS){
+  struct tm t;
+  memset(&t, 0, sizeof(t));
+  t.tm_sec = (pCDS->mTime & 0x1F)*2;
+  t.tm_min = (pCDS->mTime >> 5) & 0x2F;
+  t.tm_hour = (pCDS->mTime >> 11) & 0x1F;
+
+  t.tm_mday = (pCDS->mDate & 0x1F);
+  t.tm_mon = ((pCDS->mDate >> 5) & 0x0F) - 1;
+  t.tm_year = 80 + ((pCDS->mDate >> 9) & 0x7F);
+
+  return mktime(&t);
+}
+
+/*
+** Read a Zip archive CDS header from offset iOff of file pFile. Return
+** SQLITE_OK if successful, or an SQLite error code otherwise.
+*/
+static int zipfileGetEntry(
+  ZipfileTab *pTab,               /* Store any error message here */
+  FILE *pFile,
+  i64 iOff,
+  ZipfileEntry **ppEntry
+){
+  u8 *aRead = pTab->aBuffer;
+  char **pzErr = &pTab->base.zErrMsg;
+  int rc;
+
+  rc = zipfileReadData(pFile, aRead, ZIPFILE_CDS_FIXED_SZ, iOff, pzErr);
+  if( rc==SQLITE_OK ){
+    if( rc!=SQLITE_OK ){
+      *pzErr = sqlite3_mprintf("failed to read CDS at offset %lld", iOff);
+    }else{
+      ZipfileEntry *pNew;
+
+      int nFile = zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF]);
+      int nExtra = zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF+2]);
+      nExtra += zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF+4]);
+
+      pNew = (ZipfileEntry*)sqlite3_malloc(sizeof(ZipfileEntry) + nExtra);
+      if( pNew==0 ){
+        rc = SQLITE_NOMEM;
+      }else{
+        memset(pNew, 0, sizeof(ZipfileEntry));
+        rc = zipfileReadCDS(aRead, &pNew->cds);
+        if( rc!=SQLITE_OK ){
+          *pzErr = sqlite3_mprintf("failed to read CDS at offset %lld", iOff);
+        }else{
+          rc = zipfileReadData(
+              pFile, aRead, nExtra+nFile, iOff+ZIPFILE_CDS_FIXED_SZ, pzErr
+          );
+        }
+      }
+
+      if( rc==SQLITE_OK ){
+        u32 *pt = &pNew->mUnixTime;
+        pNew->cds.zFile = sqlite3_mprintf("%.*s", nFile, aRead); 
+        if( pNew->cds.zFile==0 ){
+          rc = SQLITE_NOMEM;
+        }else if( 0==zipfileScanExtra(&aRead[nFile], pNew->cds.nExtra, pt) ){
+          pNew->mUnixTime = zipfileMtime(&pNew->cds);
+        }
+      }
+
+      if( rc!=SQLITE_OK ){
+        zipfileEntryFree(pNew);
+      }else{
+        *ppEntry = pNew;
       }
     }
   }
@@ -633,8 +700,9 @@ static int zipfileCsrReadLFH(ZipfileCsr *pCsr){
   FILE *pFile = zipfileGetFd(pCsr);
   char **pzErr = &pCsr->base.pVtab->zErrMsg;
   u8 *aRead = zipfileCsrBuffer(pCsr);
-  int rc = zipfileReadLFH(pFile, pCsr->cds.iOffset, aRead, &pCsr->lfh, pzErr);
-  pCsr->iDataOff =  pCsr->cds.iOffset + ZIPFILE_LFH_FIXED_SZ;
+  ZipfileCDS *pCDS = &pCsr->pCurrent->cds;
+  int rc = zipfileReadLFH(pFile, pCDS->iOffset, aRead, &pCsr->lfh, pzErr);
+  pCsr->iDataOff =  pCDS->iOffset + ZIPFILE_LFH_FIXED_SZ;
   pCsr->iDataOff += pCsr->lfh.nFile+pCsr->lfh.nExtra;
   return rc;
 }
@@ -646,59 +714,40 @@ static int zipfileCsrReadLFH(ZipfileCsr *pCsr){
 static int zipfileNext(sqlite3_vtab_cursor *cur){
   ZipfileCsr *pCsr = (ZipfileCsr*)cur;
   int rc = SQLITE_OK;
-  pCsr->flags = 0;
 
-  if( pCsr->pCurrent==0 ){
+  if( pCsr->pFile ){
     i64 iEof = pCsr->eocd.iOffset + pCsr->eocd.nSize;
+    zipfileEntryFree(pCsr->pCurrent);
+    pCsr->pCurrent = 0;
     if( pCsr->iNextOff>=iEof ){
       pCsr->bEof = 1;
+    }else{
+      ZipfileEntry *p = 0;
+      ZipfileTab *pTab = (ZipfileTab*)(cur->pVtab);
+      rc = zipfileGetEntry(pTab, pCsr->pFile, pCsr->iNextOff, &p);
+      if( rc==SQLITE_OK ){
+        pCsr->iNextOff += ZIPFILE_CDS_FIXED_SZ;
+        pCsr->iNextOff += (int)p->cds.nExtra + p->cds.nFile + p->cds.nComment;
+      }
+      pCsr->pCurrent = p;
     }
   }else{
-    assert( pCsr->pFile==0 );
-    do {
+    if( !pCsr->bNoop ){
       pCsr->pCurrent = pCsr->pCurrent->pNext;
-    }while( pCsr->pCurrent && pCsr->pCurrent->bDeleted );
+    }
     if( pCsr->pCurrent==0 ){
       pCsr->bEof = 1;
     }
   }
 
-  if( pCsr->bEof==0 ){
-    rc = zipfileCsrReadCDS(pCsr);
-    if( rc==SQLITE_OK ){
-      rc = zipfileCsrReadLFH(pCsr);
-    }
+  pCsr->bNoop = 0;
+  if( rc==SQLITE_OK && pCsr->bEof==0 ){
+    rc = zipfileCsrReadLFH(pCsr);
   }
 
   return rc;
 }
 
-/*
-** "Standard" MS-DOS time format:
-**
-**   File modification time:
-**     Bits 00-04: seconds divided by 2
-**     Bits 05-10: minute
-**     Bits 11-15: hour
-**   File modification date:
-**     Bits 00-04: day
-**     Bits 05-08: month (1-12)
-**     Bits 09-15: years from 1980 
-*/
-static time_t zipfileMtime(ZipfileCsr *pCsr){
-  struct tm t;
-  memset(&t, 0, sizeof(t));
-  t.tm_sec = (pCsr->cds.mTime & 0x1F)*2;
-  t.tm_min = (pCsr->cds.mTime >> 5) & 0x2F;
-  t.tm_hour = (pCsr->cds.mTime >> 11) & 0x1F;
-
-  t.tm_mday = (pCsr->cds.mDate & 0x1F);
-  t.tm_mon = ((pCsr->cds.mDate >> 5) & 0x0F) - 1;
-  t.tm_year = 80 + ((pCsr->cds.mDate >> 9) & 0x7F);
-
-  return mktime(&t);
-}
-
 static void zipfileMtimeToDos(ZipfileCDS *pCds, u32 mTime){
   time_t t = (time_t)mTime;
   struct tm res;
@@ -804,36 +853,33 @@ static int zipfileColumn(
   int i                       /* Which column to return */
 ){
   ZipfileCsr *pCsr = (ZipfileCsr*)cur;
+  ZipfileCDS *pCDS = &pCsr->pCurrent->cds;
   int rc = SQLITE_OK;
   switch( i ){
     case 0:   /* name */
-      sqlite3_result_text(ctx, pCsr->cds.zFile, -1, SQLITE_TRANSIENT);
+      sqlite3_result_text(ctx, pCDS->zFile, -1, SQLITE_TRANSIENT);
       break;
     case 1:   /* mode */
       /* TODO: Whether or not the following is correct surely depends on
       ** the platform on which the archive was created.  */
-      sqlite3_result_int(ctx, pCsr->cds.iExternalAttr >> 16);
+      sqlite3_result_int(ctx, pCDS->iExternalAttr >> 16);
       break;
     case 2: { /* mtime */
-      if( pCsr->flags & ZIPFILE_MTIME_VALID ){
-        sqlite3_result_int64(ctx, pCsr->mTime);
-      }else{
-        sqlite3_result_int64(ctx, zipfileMtime(pCsr));
-      }
+      sqlite3_result_int64(ctx, pCsr->pCurrent->mUnixTime);
       break;
     }
     case 3: { /* sz */
       if( sqlite3_vtab_nochange(ctx)==0 ){
-        sqlite3_result_int64(ctx, pCsr->cds.szUncompressed);
+        sqlite3_result_int64(ctx, pCDS->szUncompressed);
       }
       break;
     }
     case 4:   /* rawdata */
       if( sqlite3_vtab_nochange(ctx) ) break;
     case 5: { /* data */
-      if( i==4 || pCsr->cds.iCompression==0 || pCsr->cds.iCompression==8 ){
-        int sz = pCsr->cds.szCompressed;
-        int szFinal = pCsr->cds.szUncompressed;
+      if( i==4 || pCDS->iCompression==0 || pCDS->iCompression==8 ){
+        int sz = pCDS->szCompressed;
+        int szFinal = pCDS->szUncompressed;
         if( szFinal>0 ){
           u8 *aBuf = sqlite3_malloc(sz);
           if( aBuf==0 ){
@@ -845,7 +891,7 @@ static int zipfileColumn(
             );
           }
           if( rc==SQLITE_OK ){
-            if( i==5 && pCsr->cds.iCompression ){
+            if( i==5 && pCDS->iCompression ){
               zipfileInflate(ctx, aBuf, sz, szFinal);
             }else{
               sqlite3_result_blob(ctx, aBuf, sz, SQLITE_TRANSIENT);
@@ -856,8 +902,8 @@ static int zipfileColumn(
           /* Figure out if this is a directory or a zero-sized file. Consider
           ** it to be a directory either if the mode suggests so, or if
           ** the final character in the name is '/'.  */
-          u32 mode = pCsr->cds.iExternalAttr >> 16;
-          if( !(mode & S_IFDIR) && pCsr->cds.zFile[pCsr->cds.nFile-1]!='/' ){
+          u32 mode = pCDS->iExternalAttr >> 16;
+          if( !(mode & S_IFDIR) && pCDS->zFile[pCDS->nFile-1]!='/' ){
             sqlite3_result_blob(ctx, "", 0, SQLITE_STATIC);
           }
         }
@@ -865,7 +911,7 @@ static int zipfileColumn(
       break;
     }
     case 6:   /* method */
-      sqlite3_result_int(ctx, pCsr->cds.iCompression);
+      sqlite3_result_int(ctx, pCDS->iCompression);
       break;
     case 7:   /* z */
       sqlite3_result_int64(ctx, pCsr->iId);
@@ -965,6 +1011,7 @@ static int zipfileFilter(
   ZipfileCsr *pCsr = (ZipfileCsr*)cur;
   const char *zFile;              /* Zip file to scan */
   int rc = SQLITE_OK;             /* Return Code */
+  int bInMemory = 0;              /* True for an in-memory zipfile */
 
   zipfileResetCursor(pCsr);
 
@@ -975,11 +1022,12 @@ static int zipfileFilter(
     ** supplied a file name. */
     zipfileSetErrmsg(pCsr, "table function zipfile() requires an argument");
     return SQLITE_ERROR;
+  }else if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){
   }else{
     zFile = (const char*)sqlite3_value_text(argv[0]);
   }
 
-  if( pTab->pWriteFd==0 ){
+  if( 0==pTab->pWriteFd && 0==bInMemory ){
     pCsr->pFile = fopen(zFile, "rb");
     if( pCsr->pFile==0 ){
       zipfileSetErrmsg(pCsr, "cannot open file: %s", zFile);
@@ -996,12 +1044,9 @@ static int zipfileFilter(
       }
     }
   }else{
-    ZipfileEntry e;
-    memset(&e, 0, sizeof(e));
-    e.pNext = pTab->pFirstEntry;
-    pCsr->pCurrent = &e;
+    pCsr->bNoop = 1;
+    pCsr->pCurrent = pTab->pFirstEntry;
     rc = zipfileNext(cur);
-    assert( pCsr->pCurrent!=&e );
   }
 
   return rc;
@@ -1067,110 +1112,35 @@ static void zipfileAddEntry(
 static int zipfileLoadDirectory(ZipfileTab *pTab){
   ZipfileEOCD eocd;
   int rc;
+  int i;
+  i64 iOff;
 
   rc = zipfileReadEOCD(pTab, pTab->pWriteFd, &eocd);
-  if( rc==SQLITE_OK && eocd.nEntry>0 ){
-    int i;
-    int iOff = 0;
-    u8 *aBuf = sqlite3_malloc(eocd.nSize);
-    if( aBuf==0 ){
-      rc = SQLITE_NOMEM;
-    }else{
-      rc = zipfileReadData(
-          pTab->pWriteFd, aBuf, eocd.nSize, eocd.iOffset, &pTab->base.zErrMsg
-      );
+  iOff = eocd.iOffset;
+  for(i=0; rc==SQLITE_OK && i<eocd.nEntry; i++){
+    ZipfileEntry *pNew = 0;
+    rc = zipfileGetEntry(pTab, pTab->pWriteFd, iOff, &pNew);
+
+    if( rc==SQLITE_OK ){
+      zipfileAddEntry(pTab, 0, pNew);
+      iOff += ZIPFILE_CDS_FIXED_SZ;
+      iOff += (int)pNew->cds.nExtra + pNew->cds.nFile + pNew->cds.nComment;
     }
-
-    for(i=0; rc==SQLITE_OK && i<eocd.nEntry; i++){
-      u16 nFile;
-      u16 nExtra;
-      u16 nComment;
-      ZipfileEntry *pNew;
-      u8 *aRec = &aBuf[iOff];
-
-      nFile = zipfileGetU16(&aRec[ZIPFILE_CDS_NFILE_OFF]);
-      nExtra = zipfileGetU16(&aRec[ZIPFILE_CDS_NFILE_OFF+2]);
-      nComment = zipfileGetU16(&aRec[ZIPFILE_CDS_NFILE_OFF+4]);
-
-      pNew = sqlite3_malloc(
-          sizeof(ZipfileEntry) 
-        + nFile+1 
-        + ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment
-      );
-      if( pNew==0 ){
-        rc = SQLITE_NOMEM;
-      }else{
-        memset(pNew, 0, sizeof(ZipfileEntry));
-        pNew->zPath = (char*)&pNew[1];
-        memcpy(pNew->zPath, &aRec[ZIPFILE_CDS_FIXED_SZ], nFile);
-        pNew->zPath[nFile] = '\0';
-        pNew->aCdsEntry = (u8*)&pNew->zPath[nFile+1];
-        pNew->nCdsEntry = ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment;
-        memcpy(pNew->aCdsEntry, aRec, pNew->nCdsEntry);
-        zipfileAddEntry(pTab, 0, pNew);
-      }
-
-      iOff += ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment;
-    }
-
-    sqlite3_free(aBuf);
   }
-
   return rc;
 }
 
-static ZipfileEntry *zipfileNewEntry(
-  ZipfileCDS *pCds,               /* Values for fixed size part of CDS */
-  const char *zPath,              /* Path for new entry */
-  int nPath,                      /* strlen(zPath) */
-  u32 mTime                       /* Modification time (or 0) */
-){
-  u8 *aWrite;
+static ZipfileEntry *zipfileNewEntry(const char *zPath){
   ZipfileEntry *pNew;
-  pCds->nFile = (u16)nPath;
-  pCds->nExtra = mTime ? 9 : 0;
-  pNew = (ZipfileEntry*)sqlite3_malloc(
-    sizeof(ZipfileEntry) + 
-    nPath+1 + 
-    ZIPFILE_CDS_FIXED_SZ + nPath + pCds->nExtra
-  );
-
+  pNew = sqlite3_malloc(sizeof(ZipfileEntry));
   if( pNew ){
     memset(pNew, 0, sizeof(ZipfileEntry));
-    pNew->zPath = (char*)&pNew[1];
-    pNew->aCdsEntry = (u8*)&pNew->zPath[nPath+1];
-    pNew->nCdsEntry = ZIPFILE_CDS_FIXED_SZ + nPath + pCds->nExtra;
-    memcpy(pNew->zPath, zPath, nPath+1);
-
-    aWrite = pNew->aCdsEntry;
-    zipfileWrite32(aWrite, ZIPFILE_SIGNATURE_CDS);
-    zipfileWrite16(aWrite, pCds->iVersionMadeBy);
-    zipfileWrite16(aWrite, pCds->iVersionExtract);
-    zipfileWrite16(aWrite, pCds->flags);
-    zipfileWrite16(aWrite, pCds->iCompression);
-    zipfileWrite16(aWrite, pCds->mTime);
-    zipfileWrite16(aWrite, pCds->mDate);
-    zipfileWrite32(aWrite, pCds->crc32);
-    zipfileWrite32(aWrite, pCds->szCompressed);
-    zipfileWrite32(aWrite, pCds->szUncompressed);
-    zipfileWrite16(aWrite, pCds->nFile);
-    zipfileWrite16(aWrite, pCds->nExtra);
-    zipfileWrite16(aWrite, pCds->nComment);      assert( pCds->nComment==0 );
-    zipfileWrite16(aWrite, pCds->iDiskStart);
-    zipfileWrite16(aWrite, pCds->iInternalAttr);
-    zipfileWrite32(aWrite, pCds->iExternalAttr);
-    zipfileWrite32(aWrite, pCds->iOffset);
-    assert( aWrite==&pNew->aCdsEntry[ZIPFILE_CDS_FIXED_SZ] );
-    memcpy(aWrite, zPath, nPath);
-    if( pCds->nExtra ){
-      aWrite += nPath;
-      zipfileWrite16(aWrite, ZIPFILE_EXTRA_TIMESTAMP);
-      zipfileWrite16(aWrite, 5);
-      *aWrite++ = 0x01;
-      zipfileWrite32(aWrite, mTime);
+    pNew->cds.zFile = sqlite3_mprintf("%s", zPath);
+    if( pNew->cds.zFile==0 ){
+      sqlite3_free(pNew);
+      pNew = 0;
     }
   }
-
   return pNew;
 }
 
@@ -1186,6 +1156,8 @@ static int zipfileAppendEntry(
   u8 *aBuf = pTab->aBuffer;
   int rc;
 
+  pCds->nExtra = 9;
+
   zipfileWrite32(aBuf, ZIPFILE_SIGNATURE_LFH);
   zipfileWrite16(aBuf, pCds->iVersionExtract);
   zipfileWrite16(aBuf, pCds->flags);
@@ -1291,7 +1263,6 @@ static int zipfileUpdate(
   int iMethod = 0;                /* Compression method for new entry */
   u8 *pFree = 0;                  /* Free this */
   char *zFree = 0;                /* Also free this */
-  ZipfileCDS cds;                 /* New Central Directory Structure entry */
   ZipfileEntry *pOld = 0;
   int bIsDir = 0;
   u32 iCrc32 = 0;
@@ -1299,136 +1270,145 @@ static int zipfileUpdate(
   assert( pTab->zFile );
   assert( pTab->pWriteFd );
 
+  /* If this is a DELETE or UPDATE, find the archive entry to delete. */
   if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
     const char *zDelete = (const char*)sqlite3_value_text(apVal[0]);
     int nDelete = (int)strlen(zDelete);
     for(pOld=pTab->pFirstEntry; 1; pOld=pOld->pNext){
-      if( pOld->bDeleted ) continue;
-      if( zipfileComparePath(pOld->zPath, zDelete, nDelete)==0 ){
-        pOld->bDeleted = 1;
+      if( zipfileComparePath(pOld->cds.zFile, zDelete, nDelete)==0 ){
         break;
       }
       assert( pOld->pNext );
     }
-    if( nVal==1 ) return SQLITE_OK;
   }
 
-  /* Check that "sz" and "rawdata" are both NULL: */
-  if( sqlite3_value_type(apVal[5])!=SQLITE_NULL
-   || sqlite3_value_type(apVal[6])!=SQLITE_NULL
-  ){
-    rc = SQLITE_CONSTRAINT;
-  }
-
-  if( rc==SQLITE_OK ){
-    if( sqlite3_value_type(apVal[7])==SQLITE_NULL ){
-      /* data=NULL. A directory */
-      bIsDir = 1;
-    }else{
-      /* Value specified for "data", and possibly "method". This must be
-      ** a regular file or a symlink. */
-      const u8 *aIn = sqlite3_value_blob(apVal[7]);
-      int nIn = sqlite3_value_bytes(apVal[7]);
-      int bAuto = sqlite3_value_type(apVal[8])==SQLITE_NULL;
-
-      iMethod = sqlite3_value_int(apVal[8]);
-      sz = nIn;
-      pData = aIn;
-      nData = nIn;
-      if( iMethod!=0 && iMethod!=8 ){
-        rc = SQLITE_CONSTRAINT;
-      }else{
-        if( bAuto || iMethod ){
-          int nCmp;
-          rc = zipfileDeflate(pTab, aIn, nIn, &pFree, &nCmp);
-          if( rc==SQLITE_OK ){
-            if( iMethod || nCmp<nIn ){
-              iMethod = 8;
-              pData = pFree;
-              nData = nCmp;
-            }
-          }
-        }
-        iCrc32 = crc32(0, aIn, nIn);
-      }
-    }
-  }
-
-  if( rc==SQLITE_OK ){
-    rc = zipfileGetMode(pTab, apVal[3], 
-        (bIsDir ? (S_IFDIR + 0755) : (S_IFREG + 0644)), &mode
-    );
-    if( rc==SQLITE_OK && (bIsDir == ((mode & S_IFDIR)==0)) ){
-      /* The "mode" attribute is a directory, but data has been specified.
-      ** Or vice-versa - no data but "mode" is a file or symlink.  */
+  if( nVal>1 ){
+    /* Check that "sz" and "rawdata" are both NULL: */
+    if( sqlite3_value_type(apVal[5])!=SQLITE_NULL
+     || sqlite3_value_type(apVal[6])!=SQLITE_NULL
+    ){
       rc = SQLITE_CONSTRAINT;
     }
-  }
 
-  if( rc==SQLITE_OK ){
-    zPath = (const char*)sqlite3_value_text(apVal[2]);
-    nPath = (int)strlen(zPath);
-    if( sqlite3_value_type(apVal[4])==SQLITE_NULL ){
-      mTime = (sqlite3_int64)time(0);
-    }else{
-      mTime = sqlite3_value_int64(apVal[4]);
+    if( rc==SQLITE_OK ){
+      if( sqlite3_value_type(apVal[7])==SQLITE_NULL ){
+        /* data=NULL. A directory */
+        bIsDir = 1;
+      }else{
+        /* Value specified for "data", and possibly "method". This must be
+        ** a regular file or a symlink. */
+        const u8 *aIn = sqlite3_value_blob(apVal[7]);
+        int nIn = sqlite3_value_bytes(apVal[7]);
+        int bAuto = sqlite3_value_type(apVal[8])==SQLITE_NULL;
+
+        iMethod = sqlite3_value_int(apVal[8]);
+        sz = nIn;
+        pData = aIn;
+        nData = nIn;
+        if( iMethod!=0 && iMethod!=8 ){
+          rc = SQLITE_CONSTRAINT;
+        }else{
+          if( bAuto || iMethod ){
+            int nCmp;
+            rc = zipfileDeflate(pTab, aIn, nIn, &pFree, &nCmp);
+            if( rc==SQLITE_OK ){
+              if( iMethod || nCmp<nIn ){
+                iMethod = 8;
+                pData = pFree;
+                nData = nCmp;
+              }
+            }
+          }
+          iCrc32 = crc32(0, aIn, nIn);
+        }
+      }
     }
-  }
 
-  if( rc==SQLITE_OK && bIsDir ){
-    /* For a directory, check that the last character in the path is a
-    ** '/'. This appears to be required for compatibility with info-zip
-    ** (the unzip command on unix). It does not create directories
-    ** otherwise.  */
-    if( zPath[nPath-1]!='/' ){
-      zFree = sqlite3_mprintf("%s/", zPath);
-      if( zFree==0 ){ rc = SQLITE_NOMEM; }
-      zPath = (const char*)zFree;
-      nPath++;
-    }
-  }
-
-  /* Check that we're not inserting a duplicate entry */
-  if( rc==SQLITE_OK ){
-    ZipfileEntry *p;
-    for(p=pTab->pFirstEntry; p; p=p->pNext){
-      if( p->bDeleted ) continue;
-      if( zipfileComparePath(p->zPath, zPath, nPath)==0 ){
+    if( rc==SQLITE_OK ){
+      rc = zipfileGetMode(pTab, apVal[3], 
+          (bIsDir ? (S_IFDIR + 0755) : (S_IFREG + 0644)), &mode
+      );
+      if( rc==SQLITE_OK && (bIsDir == ((mode & S_IFDIR)==0)) ){
+        /* The "mode" attribute is a directory, but data has been specified.
+        ** Or vice-versa - no data but "mode" is a file or symlink.  */
         rc = SQLITE_CONSTRAINT;
-        break;
+      }
+    }
+
+    if( rc==SQLITE_OK ){
+      zPath = (const char*)sqlite3_value_text(apVal[2]);
+      nPath = (int)strlen(zPath);
+      if( sqlite3_value_type(apVal[4])==SQLITE_NULL ){
+        mTime = (sqlite3_int64)time(0);
+      }else{
+        mTime = sqlite3_value_int64(apVal[4]);
+      }
+    }
+
+    if( rc==SQLITE_OK && bIsDir ){
+      /* For a directory, check that the last character in the path is a
+      ** '/'. This appears to be required for compatibility with info-zip
+      ** (the unzip command on unix). It does not create directories
+      ** otherwise.  */
+      if( zPath[nPath-1]!='/' ){
+        zFree = sqlite3_mprintf("%s/", zPath);
+        if( zFree==0 ){ rc = SQLITE_NOMEM; }
+        zPath = (const char*)zFree;
+        nPath++;
+      }
+    }
+
+    /* Check that we're not inserting a duplicate entry */
+    if( pOld==0 && rc==SQLITE_OK ){
+      ZipfileEntry *p;
+      for(p=pTab->pFirstEntry; p; p=p->pNext){
+        if( zipfileComparePath(p->cds.zFile, zPath, nPath)==0 ){
+          rc = SQLITE_CONSTRAINT;
+          break;
+        }
+      }
+    }
+
+    if( rc==SQLITE_OK ){
+      /* Create the new CDS record. */
+      pNew = zipfileNewEntry(zPath);
+      if( pNew==0 ){
+        rc = SQLITE_NOMEM;
+      }else{
+        pNew->cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY;
+        pNew->cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED;
+        pNew->cds.flags = ZIPFILE_NEWENTRY_FLAGS;
+        pNew->cds.iCompression = (u16)iMethod;
+        zipfileMtimeToDos(&pNew->cds, (u32)mTime);
+        pNew->cds.crc32 = iCrc32;
+        pNew->cds.szCompressed = nData;
+        pNew->cds.szUncompressed = (u32)sz;
+        pNew->cds.iExternalAttr = (mode<<16);
+        pNew->cds.iOffset = (u32)pTab->szCurrent;
+        pNew->cds.nFile = nPath;
+        pNew->mUnixTime = (u32)mTime;
+        rc = zipfileAppendEntry(
+            pTab, &pNew->cds, zPath, nPath, pData, nData, pNew->mUnixTime
+        );
+        zipfileAddEntry(pTab, pOld, pNew);
       }
     }
   }
 
-  if( rc==SQLITE_OK ){
-    /* Create the new CDS record. */
-    memset(&cds, 0, sizeof(cds));
-    cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY;
-    cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED;
-    cds.flags = ZIPFILE_NEWENTRY_FLAGS;
-    cds.iCompression = (u16)iMethod;
-    zipfileMtimeToDos(&cds, (u32)mTime);
-    cds.crc32 = iCrc32;
-    cds.szCompressed = nData;
-    cds.szUncompressed = (u32)sz;
-    cds.iExternalAttr = (mode<<16);
-    cds.iOffset = (u32)pTab->szCurrent;
-    pNew = zipfileNewEntry(&cds, zPath, nPath, (u32)mTime);
-    if( pNew==0 ){
-      rc = SQLITE_NOMEM;
-    }else{
-      zipfileAddEntry(pTab, pOld, pNew);
+  if( rc==SQLITE_OK && pOld ){
+    ZipfileEntry **pp;
+    ZipfileCsr *pCsr;
+    for(pCsr=pTab->pCsrList; pCsr; pCsr=pCsr->pCsrNext){
+      if( pCsr->pCurrent==pOld ){
+        pCsr->pCurrent = pOld->pNext;
+        pCsr->bNoop = 1;
+      }
     }
+    for(pp=&pTab->pFirstEntry; (*pp)!=pOld; pp=&((*pp)->pNext));
+    *pp = (*pp)->pNext;
+    zipfileEntryFree(pOld);
   }
 
-  /* Append the new header+file to the archive */
-  if( rc==SQLITE_OK ){
-    rc = zipfileAppendEntry(pTab, &cds, zPath, nPath, pData, nData, (u32)mTime);
-  }
-
-  if( rc!=SQLITE_OK && pOld ){
-    pOld->bDeleted = 0;
-  }
   sqlite3_free(pFree);
   sqlite3_free(zFree);
   return rc;
@@ -1450,22 +1430,6 @@ static int zipfileAppendEOCD(ZipfileTab *pTab, ZipfileEOCD *p){
   return zipfileAppendData(pTab, pTab->aBuffer, (int)(aBuf - pTab->aBuffer));
 }
 
-static void zipfileCleanupTransaction(ZipfileTab *pTab){
-  ZipfileEntry *pEntry;
-  ZipfileEntry *pNext;
-
-  for(pEntry=pTab->pFirstEntry; pEntry; pEntry=pNext){
-    pNext = pEntry->pNext;
-    sqlite3_free(pEntry);
-  }
-  pTab->pFirstEntry = 0;
-  pTab->pLastEntry = 0;
-  fclose(pTab->pWriteFd);
-  pTab->pWriteFd = 0;
-  pTab->szCurrent = 0;
-  pTab->szOrig = 0;
-}
-
 static int zipfileBegin(sqlite3_vtab *pVtab){
   ZipfileTab *pTab = (ZipfileTab*)pVtab;
   int rc = SQLITE_OK;
@@ -1504,6 +1468,55 @@ static int zipfileBegin(sqlite3_vtab *pVtab){
   return rc;
 }
 
+/*
+** Serialize the CDS structure into buffer aBuf[]. Return the number
+** of bytes written.
+*/
+static int zipfileSerializeCDS(ZipfileEntry *pEntry, u8 *aBuf){
+  u8 *a = aBuf;
+  ZipfileCDS *pCDS = &pEntry->cds;
+
+  if( pEntry->aExtra==0 ){
+    pCDS->nExtra = 9;
+  }
+
+  zipfileWrite32(a, ZIPFILE_SIGNATURE_CDS);
+  zipfileWrite16(a, pCDS->iVersionMadeBy);
+  zipfileWrite16(a, pCDS->iVersionExtract);
+  zipfileWrite16(a, pCDS->flags);
+  zipfileWrite16(a, pCDS->iCompression);
+  zipfileWrite16(a, pCDS->mTime);
+  zipfileWrite16(a, pCDS->mDate);
+  zipfileWrite32(a, pCDS->crc32);
+  zipfileWrite32(a, pCDS->szCompressed);
+  zipfileWrite32(a, pCDS->szUncompressed);
+  assert( a==&aBuf[ZIPFILE_CDS_NFILE_OFF] );
+  zipfileWrite16(a, pCDS->nFile);
+  zipfileWrite16(a, pCDS->nExtra);
+  zipfileWrite16(a, pCDS->nComment);
+  zipfileWrite16(a, pCDS->iDiskStart);
+  zipfileWrite16(a, pCDS->iInternalAttr);
+  zipfileWrite32(a, pCDS->iExternalAttr);
+  zipfileWrite32(a, pCDS->iOffset);
+
+  memcpy(a, pCDS->zFile, pCDS->nFile);
+  a += pCDS->nFile;
+
+  if( pEntry->aExtra ){
+    int n = (int)pCDS->nExtra + (int)pCDS->nComment;
+    memcpy(a, pEntry->aExtra, n);
+    a += n;
+  }else{
+    assert( pCDS->nExtra==9 );
+    zipfileWrite16(a, ZIPFILE_EXTRA_TIMESTAMP);
+    zipfileWrite16(a, 5);
+    *a++ = 0x01;
+    zipfileWrite32(a, pEntry->mUnixTime);
+  }
+
+  return a-aBuf;
+}
+
 static int zipfileCommit(sqlite3_vtab *pVtab){
   ZipfileTab *pTab = (ZipfileTab*)pVtab;
   int rc = SQLITE_OK;
@@ -1513,10 +1526,10 @@ static int zipfileCommit(sqlite3_vtab *pVtab){
     ZipfileEOCD eocd;
     int nEntry = 0;
 
-    /* Write out all undeleted entries */
+    /* Write out all entries */
     for(p=pTab->pFirstEntry; rc==SQLITE_OK && p; p=p->pNext){
-      if( p->bDeleted ) continue;
-      rc = zipfileAppendData(pTab, p->aCdsEntry, p->nCdsEntry);
+      int n = zipfileSerializeCDS(p, pTab->aBuffer);
+      rc = zipfileAppendData(pTab, pTab->aBuffer, n);
       nEntry++;
     }
 
@@ -1557,7 +1570,7 @@ static void zipfileFunctionCds(
 
   pCsr = zipfileFindCursor(pTab, sqlite3_value_int64(argv[0]));
   if( pCsr ){
-    ZipfileCDS *p = &pCsr->cds;
+    ZipfileCDS *p = &pCsr->pCurrent->cds;
     char *zRes = sqlite3_mprintf("{"
         "\"version-made-by\" : %u, "
         "\"version-to-extract\" : %u, "
diff --git a/manifest b/manifest
index 7d40e3172b..9b84aa5a52 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Slightly\ssmaller\sand\sfaster\simplementation\sof\sOP_If\sand\sOP_IfNot.
-D 2018-01-25T01:20:29.953
+C Reorganize\szipfile.c\scode\sto\smake\sit\seasier\sto\sadd\ssupport\sfor\sin-memory\szip\narchive\sprocessing.
+D 2018-01-25T20:50:46.467
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -304,7 +304,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c d99efb67ecdfcae7e1855984c218c8c33d0d46a833eaa4b5a5c3d7a4f6690ce4
+F ext/misc/zipfile.c 45cc42164444fa49c9a04ba8361e38e4727862a3f2f64a35e4f6f661f8208c5a
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -1702,7 +1702,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 8055e4f42446ceb5bcf752bbf41a73289c3ca759c56c9f779edc3d7f202b7881
-R 3a57ffa9e8084df96fa05ea7edc4d068
-U drh
-Z 1f11fb78272f5afcdf3b192bdf3cca0d
+P 6ab42934e2c4957b5d8927bf4434a9db07ab6078987a6a2d25f35cc468d21203
+R 7248e9fa39da1851c7faf3b8da443f98
+T +closed 469694857a5301e1da8cf067017d400403f3b12882d4265cefb3fd185c536305
+U dan
+Z 84871d5400876eca8ff4f985699fe3ef
diff --git a/manifest.uuid b/manifest.uuid
index 8b78bd0022..ae4e344061 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6ab42934e2c4957b5d8927bf4434a9db07ab6078987a6a2d25f35cc468d21203
\ No newline at end of file
+30b9258294e3028ef4ea467e332e013995509544e9a23c8bbf5168772a7e895d
\ No newline at end of file

From 4eb57ceffc2735a078fd66e58650f10f44e56a63 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 26 Jan 2018 18:37:34 +0000
Subject: [PATCH 389/488] Improve text-to-integer conversion in boundary cases.
  The sqlite3Atoi64() function always returns the minimum or maximum integer
 if the magnitude of the text value is too large.  Trailing whitespace is now
 ignored.

FossilOrigin-Name: ace0644a1a2a42a3ea42d44f00a31915b8a7e56c9ba90f90a6c02001f89f9c86
---
 manifest       | 17 ++++++++---------
 manifest.uuid  |  2 +-
 src/util.c     | 52 +++++++++++++++++++++++++++-----------------------
 test/cast.test | 45 +++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 82 insertions(+), 34 deletions(-)

diff --git a/manifest b/manifest
index 9b84aa5a52..c964e14825 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Reorganize\szipfile.c\scode\sto\smake\sit\seasier\sto\sadd\ssupport\sfor\sin-memory\szip\narchive\sprocessing.
-D 2018-01-25T20:50:46.467
+C Improve\stext-to-integer\sconversion\sin\sboundary\scases.\s\sThe\nsqlite3Atoi64()\sfunction\salways\sreturns\sthe\sminimum\sor\smaximum\sinteger\nif\sthe\smagnitude\sof\sthe\stext\svalue\sis\stoo\slarge.\s\sTrailing\swhitespace\sis\nnow\signored.
+D 2018-01-26T18:37:34.351
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -554,7 +554,7 @@ F src/treeview.c eae35972ff44f67064de2eaf35f04afe94e7aea3271a8b3bcebb3f954880fec
 F src/trigger.c a34539c69433276d37b0da9a89c117726ff2d292c0902895af1f393a983cd3a1
 F src/update.c a90a32ffc0100265b0693dbbdbe490756447af181f5ea2c138cce515b08c8795
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
-F src/util.c ef4a5f904d942e660abade7fbf3e6bdb402dabe9e7c27f3361ecf40b945538b5
+F src/util.c 8628fe1daa9f5db9fa3a70d6fcfd828b73e839feafe7d7faecd7073be56eaec2
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
 F src/vdbe.c 0e13b8c33a9ecf236b3045f190fb58933049582307b621cbd924b6582a2b705f
 F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a
@@ -673,7 +673,7 @@ F test/capi3b.test efb2b9cfd127efa84433cd7a2d72ce0454ae0dc4
 F test/capi3c.test 7ebed1d8fa2f3190149d556fe8cff5a006be62af437c5c4640db614470126098
 F test/capi3d.test 485048dc5cd07bc68011e4917ad035ad6047ab82
 F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe
-F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3
+F test/cast.test 5ceb920718d280b61163500a7d29e0e0a86458b1cbd92d96f962c9d970aa3857
 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef
 F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a
 F test/close.test 799ea4599d2f5704b0a30f477d17c2c760d8523fa5d0c8be4a7df2a8cad787d8
@@ -1702,8 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6ab42934e2c4957b5d8927bf4434a9db07ab6078987a6a2d25f35cc468d21203
-R 7248e9fa39da1851c7faf3b8da443f98
-T +closed 469694857a5301e1da8cf067017d400403f3b12882d4265cefb3fd185c536305
-U dan
-Z 84871d5400876eca8ff4f985699fe3ef
+P 30b9258294e3028ef4ea467e332e013995509544e9a23c8bbf5168772a7e895d
+R a3a69d646870f53631d68482ffc811f2
+U drh
+Z ebcc0484dfae31c55e47d329daf31900
diff --git a/manifest.uuid b/manifest.uuid
index ae4e344061..5fee1327fd 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-30b9258294e3028ef4ea467e332e013995509544e9a23c8bbf5168772a7e895d
\ No newline at end of file
+ace0644a1a2a42a3ea42d44f00a31915b8a7e56c9ba90f90a6c02001f89f9c86
\ No newline at end of file
diff --git a/src/util.c b/src/util.c
index 456317cfa9..9954d709df 100644
--- a/src/util.c
+++ b/src/util.c
@@ -595,7 +595,7 @@ static int compare2pow63(const char *zNum, int incr){
 ** Returns:
 **
 **     0    Successful transformation.  Fits in a 64-bit signed integer.
-**     1    Excess text after the integer value
+**     1    Excess non-space text after the integer value
 **     2    Integer too large for a 64-bit signed integer or is malformed
 **     3    Special case of 9223372036854775808
 **
@@ -638,47 +638,51 @@ int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){
   for(i=0; &zNum[i]<zEnd && (c=zNum[i])>='0' && c<='9'; i+=incr){
     u = u*10 + c - '0';
   }
-  if( u>LARGEST_INT64 ){
-    *pNum = neg ? SMALLEST_INT64 : LARGEST_INT64;
-  }else if( neg ){
+  testcase( i==18*incr );
+  testcase( i==19*incr );
+  testcase( i==20*incr );
+  if( neg ){
     *pNum = -(i64)u;
   }else{
     *pNum = (i64)u;
   }
-  testcase( i==18 );
-  testcase( i==19 );
-  testcase( i==20 );
-  if( &zNum[i]<zEnd              /* Extra bytes at the end */
-   || (i==0 && zStart==zNum)     /* No digits */
+  rc = 0;
+  if( (i==0 && zStart==zNum)     /* No digits */
    || nonNum                     /* UTF16 with high-order bytes non-zero */
   ){
     rc = 1;
-  }else{
-    rc = 0;
+  }else if( &zNum[i]<zEnd ){     /* Extra bytes at the end */
+    int jj = i;
+    do{
+      if( !sqlite3Isspace(zNum[jj]) ){
+        rc = 1;          /* Extra non-space text after the integer */
+        break;
+      }
+      jj += incr;
+    }while( &zNum[jj]<zEnd );
   }
-  if( i>19*incr ){                /* Too many digits */
-    /* zNum is empty or contains non-numeric text or is longer
-    ** than 19 digits (thus guaranteeing that it is too large) */
-    return 2;
-  }else if( i<19*incr ){
+  if( i<19*incr ){
     /* Less than 19 digits, so we know that it fits in 64 bits */
     assert( u<=LARGEST_INT64 );
     return rc;
   }else{
     /* zNum is a 19-digit numbers.  Compare it against 9223372036854775808. */
-    c = compare2pow63(zNum, incr);
+    c = i>19*incr ? 1 : compare2pow63(zNum, incr);
     if( c<0 ){
       /* zNum is less than 9223372036854775808 so it fits */
       assert( u<=LARGEST_INT64 );
       return rc;
-    }else if( c>0 ){
-      /* zNum is greater than 9223372036854775808 so it overflows */
-      return 2;
     }else{
-      /* zNum is exactly 9223372036854775808.  Fits if negative.  The
-      ** special case 2 overflow if positive */
-      assert( u-1==LARGEST_INT64 );
-      return neg ? rc : 3;
+      *pNum = neg ? SMALLEST_INT64 : LARGEST_INT64;
+      if( c>0 ){
+        /* zNum is greater than 9223372036854775808 so it overflows */
+        return 2;
+      }else{
+        /* zNum is exactly 9223372036854775808.  Fits if negative.  The
+        ** special case 2 overflow if positive */
+        assert( u-1==LARGEST_INT64 );
+        return neg ? rc : 3;
+      }
     }
   }
 }
diff --git a/test/cast.test b/test/cast.test
index f47f4bb2bf..f43aa48560 100644
--- a/test/cast.test
+++ b/test/cast.test
@@ -343,4 +343,49 @@ do_test cast-4.4 {
   }
 } {0 abc 0.0 abc}
 
+# Added 2018-01-26
+#
+# EVIDENCE-OF: R-48741-32454 If the prefix integer is greater than
+# +9223372036854775807 then the result of the cast is exactly
+# +9223372036854775807.
+do_execsql_test cast-5.1 {
+  SELECT CAST('9223372036854775808' AS integer);
+  SELECT CAST('  +000009223372036854775808' AS integer);
+  SELECT CAST('12345678901234567890123' AS INTEGER);
+} {9223372036854775807 9223372036854775807 9223372036854775807}
+
+# EVIDENCE-OF: R-06028-16857 Similarly, if the prefix integer is less
+# than -9223372036854775808 then the result of the cast is exactly
+# -9223372036854775808.
+do_execsql_test cast-5.2 {
+  SELECT CAST('-9223372036854775808' AS integer);
+  SELECT CAST('-9223372036854775809' AS integer);
+  SELECT CAST('-12345678901234567890123' AS INTEGER);
+} {-9223372036854775808 -9223372036854775808 -9223372036854775808}
+
+# EVIDENCE-OF: R-33990-33527 When casting to INTEGER, if the text looks
+# like a floating point value with an exponent, the exponent will be
+# ignored because it is no part of the integer prefix.
+# EVIDENCE-OF: R-24225-46995 For example, "(CAST '123e+5' AS INTEGER)"
+# results in 123, not in 12300000.
+do_execsql_test case-5.3 {
+  SELECT CAST('123e+5' AS INTEGER);
+  SELECT CAST('123e+5' AS NUMERIC);
+} {123 12300000.0}
+
+
+# The following does not have anything to do with the CAST operator,
+# but it does deal with affinity transformations.
+#
+do_execsql_test case-6.1 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1(a NUMERIC);
+  INSERT INTO t1 VALUES
+     ('9000000000000000001'),
+     ('9000000000000000001 '),
+     (' 9000000000000000001'),
+     (' 9000000000000000001 ');
+  SELECT * FROM t1;
+} {9000000000000000001 9000000000000000001 9000000000000000001 9000000000000000001}
+
 finish_test

From 8005d60587d572b226dab4d0b7c3ecf441ab595a Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 26 Jan 2018 18:59:25 +0000
Subject: [PATCH 390/488] If the argument to table function zipfile() is a blob
 (not text), assume that it contains a zip file image to interpret, not the
 name of a file on disk.

FossilOrigin-Name: 029ebcd30cb261d949f7587ac54c23d7479796b6716fd4ca7512361b8f32de3e
---
 ext/misc/zipfile.c | 380 +++++++++++++++++++++++++--------------------
 manifest           |  16 +-
 manifest.uuid      |   2 +-
 test/zipfile.test  |  20 +++
 4 files changed, 239 insertions(+), 179 deletions(-)

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index d378606941..c199f8ec1b 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -225,6 +225,8 @@ struct ZipfileEntry {
   ZipfileCDS cds;            /* Parsed CDS record */
   u32 mUnixTime;             /* Modification time, in UNIX format */
   u8 *aExtra;                /* cds.nExtra+cds.nComment bytes of extra data */
+  i64 iDataOff;
+  u8 *aData;                 /* cds.szCompressed bytes of compressed data */
   ZipfileEntry *pNext;       /* Next element in in-memory CDS */
 };
 
@@ -243,9 +245,9 @@ struct ZipfileCsr {
   i64 iNextOff;              /* Offset of next record in central directory */
   ZipfileEOCD eocd;          /* Parse of central directory record */
 
+  ZipfileEntry *pFreeEntry;
+
   ZipfileEntry *pCurrent;    /* Current entry */
-  ZipfileLFH lfh;            /* Local File Header for current entry */
-  i64 iDataOff;              /* Offset in zipfile to data */
   ZipfileCsr *pCsrNext;      /* Next cursor on same virtual table */
 };
 
@@ -393,6 +395,9 @@ static int zipfileOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){
 ** by zipfileOpen().
 */
 static void zipfileResetCursor(ZipfileCsr *pCsr){
+  ZipfileEntry *p;
+  ZipfileEntry *pNext;
+
   pCsr->bEof = 0;
   if( pCsr->pFile ){
     fclose(pCsr->pFile);
@@ -400,6 +405,11 @@ static void zipfileResetCursor(ZipfileCsr *pCsr){
     zipfileEntryFree(pCsr->pCurrent);
     pCsr->pCurrent = 0;
   }
+
+  for(p=pCsr->pFreeEntry; p; p=pNext){
+    pNext = p->pNext;
+    zipfileEntryFree(p);
+  }
 }
 
 /*
@@ -503,6 +513,7 @@ static u8* zipfileCsrBuffer(ZipfileCsr *pCsr){
 */
 #define ZIPFILE_CDS_FIXED_SZ         46
 #define ZIPFILE_CDS_NFILE_OFF        28
+#define ZIPFILE_CDS_SZCOMPRESSED_OFF 20
 
 /*
 ** Decode the CDS record in buffer aBuf into (*pCDS). Return SQLITE_ERROR
@@ -596,62 +607,125 @@ static time_t zipfileMtime(ZipfileCDS *pCDS){
   return mktime(&t);
 }
 
+static int zipfileReadLFH(
+  u8 *aBuffer,
+  ZipfileLFH *pLFH
+){
+  u8 *aRead = aBuffer;
+  int rc = SQLITE_OK;
+
+  u32 sig = zipfileRead32(aRead);
+  if( sig!=ZIPFILE_SIGNATURE_LFH ){
+    rc = SQLITE_ERROR;
+  }else{
+    pLFH->iVersionExtract = zipfileRead16(aRead);
+    pLFH->flags = zipfileRead16(aRead);
+    pLFH->iCompression = zipfileRead16(aRead);
+    pLFH->mTime = zipfileRead16(aRead);
+    pLFH->mDate = zipfileRead16(aRead);
+    pLFH->crc32 = zipfileRead32(aRead);
+    pLFH->szCompressed = zipfileRead32(aRead);
+    pLFH->szUncompressed = zipfileRead32(aRead);
+    pLFH->nFile = zipfileRead16(aRead);
+    pLFH->nExtra = zipfileRead16(aRead);
+  }
+  return rc;
+}
+
 /*
 ** Read a Zip archive CDS header from offset iOff of file pFile. Return
 ** SQLITE_OK if successful, or an SQLite error code otherwise.
 */
 static int zipfileGetEntry(
   ZipfileTab *pTab,               /* Store any error message here */
-  FILE *pFile,
+  const u8 *aBlob,                /* Pointer to in-memory file image */
+  int nBlob,                      /* Size of aBlob[] in bytes */
+  FILE *pFile,                    /* If aBlob==0, read from this file */
   i64 iOff,
   ZipfileEntry **ppEntry
 ){
-  u8 *aRead = pTab->aBuffer;
+  u8 *aRead;
   char **pzErr = &pTab->base.zErrMsg;
-  int rc;
+  int rc = SQLITE_OK;
+
+  if( aBlob==0 ){
+    aRead = pTab->aBuffer;
+    rc = zipfileReadData(pFile, aRead, ZIPFILE_CDS_FIXED_SZ, iOff, pzErr);
+  }else{
+    aRead = (u8*)&aBlob[iOff];
+  }
 
-  rc = zipfileReadData(pFile, aRead, ZIPFILE_CDS_FIXED_SZ, iOff, pzErr);
   if( rc==SQLITE_OK ){
-    if( rc!=SQLITE_OK ){
-      *pzErr = sqlite3_mprintf("failed to read CDS at offset %lld", iOff);
+    int nAlloc;
+    ZipfileEntry *pNew;
+
+    int nFile = zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF]);
+    int nExtra = zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF+2]);
+    nExtra += zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF+4]);
+
+    nAlloc = sizeof(ZipfileEntry) + nExtra;
+    if( aBlob ){
+      nAlloc += zipfileGetU32(&aRead[ZIPFILE_CDS_SZCOMPRESSED_OFF]);
+    }
+
+    pNew = (ZipfileEntry*)sqlite3_malloc(nAlloc);
+    if( pNew==0 ){
+      rc = SQLITE_NOMEM;
     }else{
-      ZipfileEntry *pNew;
-
-      int nFile = zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF]);
-      int nExtra = zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF+2]);
-      nExtra += zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF+4]);
-
-      pNew = (ZipfileEntry*)sqlite3_malloc(sizeof(ZipfileEntry) + nExtra);
-      if( pNew==0 ){
-        rc = SQLITE_NOMEM;
-      }else{
-        memset(pNew, 0, sizeof(ZipfileEntry));
-        rc = zipfileReadCDS(aRead, &pNew->cds);
-        if( rc!=SQLITE_OK ){
-          *pzErr = sqlite3_mprintf("failed to read CDS at offset %lld", iOff);
-        }else{
-          rc = zipfileReadData(
-              pFile, aRead, nExtra+nFile, iOff+ZIPFILE_CDS_FIXED_SZ, pzErr
-          );
-        }
-      }
-
-      if( rc==SQLITE_OK ){
-        u32 *pt = &pNew->mUnixTime;
-        pNew->cds.zFile = sqlite3_mprintf("%.*s", nFile, aRead); 
-        if( pNew->cds.zFile==0 ){
-          rc = SQLITE_NOMEM;
-        }else if( 0==zipfileScanExtra(&aRead[nFile], pNew->cds.nExtra, pt) ){
-          pNew->mUnixTime = zipfileMtime(&pNew->cds);
-        }
-      }
-
+      memset(pNew, 0, sizeof(ZipfileEntry));
+      rc = zipfileReadCDS(aRead, &pNew->cds);
       if( rc!=SQLITE_OK ){
-        zipfileEntryFree(pNew);
+        *pzErr = sqlite3_mprintf("failed to read CDS at offset %lld", iOff);
+      }else if( aBlob==0 ){
+        rc = zipfileReadData(
+            pFile, aRead, nExtra+nFile, iOff+ZIPFILE_CDS_FIXED_SZ, pzErr
+        );
       }else{
-        *ppEntry = pNew;
+        aRead = (u8*)&aBlob[iOff + ZIPFILE_CDS_FIXED_SZ];
       }
     }
+
+    if( rc==SQLITE_OK ){
+      u32 *pt = &pNew->mUnixTime;
+      pNew->cds.zFile = sqlite3_mprintf("%.*s", nFile, aRead); 
+      pNew->aExtra = (u8*)&pNew[1];
+      memcpy(pNew->aExtra, &aRead[nFile], nExtra);
+      if( pNew->cds.zFile==0 ){
+        rc = SQLITE_NOMEM;
+      }else if( 0==zipfileScanExtra(&aRead[nFile], pNew->cds.nExtra, pt) ){
+        pNew->mUnixTime = zipfileMtime(&pNew->cds);
+      }
+    }
+
+    if( rc==SQLITE_OK ){
+      static const int szFix = ZIPFILE_LFH_FIXED_SZ;
+      ZipfileLFH lfh;
+      if( pFile ){
+        rc = zipfileReadData(pFile, aRead, szFix, pNew->cds.iOffset, pzErr);
+      }else{
+        aRead = &aBlob[pNew->cds.iOffset];
+      }
+
+      rc = zipfileReadLFH(aRead, &lfh);
+      if( rc==SQLITE_OK ){
+        pNew->iDataOff =  pNew->cds.iOffset + ZIPFILE_LFH_FIXED_SZ;
+        pNew->iDataOff += lfh.nFile + lfh.nExtra;
+        if( aBlob && pNew->cds.szCompressed ){
+          pNew->aData = &pNew->aExtra[nExtra];
+          memcpy(pNew->aData, &aBlob[pNew->iDataOff], pNew->cds.szCompressed);
+        }
+      }else{
+        *pzErr = sqlite3_mprintf("failed to read LFH at offset %d", 
+            (int)pNew->cds.iOffset
+        );
+      }
+    }
+
+    if( rc!=SQLITE_OK ){
+      zipfileEntryFree(pNew);
+    }else{
+      *ppEntry = pNew;
+    }
   }
 
   return rc;
@@ -662,51 +736,6 @@ static FILE *zipfileGetFd(ZipfileCsr *pCsr){
   return ((ZipfileTab*)(pCsr->base.pVtab))->pWriteFd;
 }
 
-static int zipfileReadLFH(
-  FILE *pFd, 
-  i64 iOffset,
-  u8 *aTmp, 
-  ZipfileLFH *pLFH, 
-  char **pzErr
-){
-  u8 *aRead = aTmp;
-  static const int szFix = ZIPFILE_LFH_FIXED_SZ;
-  int rc;
-
-  rc = zipfileReadData(pFd, aRead, szFix, iOffset, pzErr);
-  if( rc==SQLITE_OK ){
-    u32 sig = zipfileRead32(aRead);
-    if( sig!=ZIPFILE_SIGNATURE_LFH ){
-      *pzErr = sqlite3_mprintf("failed to read LFH at offset %d", (int)iOffset);
-      rc = SQLITE_ERROR;
-    }else{
-      pLFH->iVersionExtract = zipfileRead16(aRead);
-      pLFH->flags = zipfileRead16(aRead);
-      pLFH->iCompression = zipfileRead16(aRead);
-      pLFH->mTime = zipfileRead16(aRead);
-      pLFH->mDate = zipfileRead16(aRead);
-      pLFH->crc32 = zipfileRead32(aRead);
-      pLFH->szCompressed = zipfileRead32(aRead);
-      pLFH->szUncompressed = zipfileRead32(aRead);
-      pLFH->nFile = zipfileRead16(aRead);
-      pLFH->nExtra = zipfileRead16(aRead);
-      assert( aRead==&aTmp[szFix] );
-    }
-  }
-  return rc;
-}
-
-static int zipfileCsrReadLFH(ZipfileCsr *pCsr){
-  FILE *pFile = zipfileGetFd(pCsr);
-  char **pzErr = &pCsr->base.pVtab->zErrMsg;
-  u8 *aRead = zipfileCsrBuffer(pCsr);
-  ZipfileCDS *pCDS = &pCsr->pCurrent->cds;
-  int rc = zipfileReadLFH(pFile, pCDS->iOffset, aRead, &pCsr->lfh, pzErr);
-  pCsr->iDataOff =  pCDS->iOffset + ZIPFILE_LFH_FIXED_SZ;
-  pCsr->iDataOff += pCsr->lfh.nFile+pCsr->lfh.nExtra;
-  return rc;
-}
-
 
 /*
 ** Advance an ZipfileCsr to its next row of output.
@@ -724,7 +753,7 @@ static int zipfileNext(sqlite3_vtab_cursor *cur){
     }else{
       ZipfileEntry *p = 0;
       ZipfileTab *pTab = (ZipfileTab*)(cur->pVtab);
-      rc = zipfileGetEntry(pTab, pCsr->pFile, pCsr->iNextOff, &p);
+      rc = zipfileGetEntry(pTab, 0, 0, pCsr->pFile, pCsr->iNextOff, &p);
       if( rc==SQLITE_OK ){
         pCsr->iNextOff += ZIPFILE_CDS_FIXED_SZ;
         pCsr->iNextOff += (int)p->cds.nExtra + p->cds.nFile + p->cds.nComment;
@@ -741,10 +770,6 @@ static int zipfileNext(sqlite3_vtab_cursor *cur){
   }
 
   pCsr->bNoop = 0;
-  if( rc==SQLITE_OK && pCsr->bEof==0 ){
-    rc = zipfileCsrReadLFH(pCsr);
-  }
-
   return rc;
 }
 
@@ -881,14 +906,22 @@ static int zipfileColumn(
         int sz = pCDS->szCompressed;
         int szFinal = pCDS->szUncompressed;
         if( szFinal>0 ){
-          u8 *aBuf = sqlite3_malloc(sz);
-          if( aBuf==0 ){
-            rc = SQLITE_NOMEM;
+          u8 *aBuf;
+          u8 *aFree = 0;
+          if( pCsr->pCurrent->aData ){
+            aBuf = pCsr->pCurrent->aData;
           }else{
-            FILE *pFile = zipfileGetFd(pCsr);
-            rc = zipfileReadData(pFile, aBuf, sz, pCsr->iDataOff,
-                &pCsr->base.pVtab->zErrMsg
-            );
+            aBuf = aFree = sqlite3_malloc(sz);
+            if( aBuf==0 ){
+              rc = SQLITE_NOMEM;
+            }else{
+              FILE *pFile = zipfileGetFd(pCsr);
+              if( rc==SQLITE_OK ){
+                rc = zipfileReadData(pFile, aBuf, sz, pCsr->pCurrent->iDataOff,
+                    &pCsr->base.pVtab->zErrMsg
+                );
+              }
+            }
           }
           if( rc==SQLITE_OK ){
             if( i==5 && pCDS->iCompression ){
@@ -896,8 +929,8 @@ static int zipfileColumn(
             }else{
               sqlite3_result_blob(ctx, aBuf, sz, SQLITE_TRANSIENT);
             }
-            sqlite3_free(aBuf);
           }
+          sqlite3_free(aFree);
         }else{
           /* Figure out if this is a directory or a zero-sized file. Consider
           ** it to be a directory either if the mode suggests so, or if
@@ -942,25 +975,32 @@ static int zipfileEof(sqlite3_vtab_cursor *cur){
 */
 static int zipfileReadEOCD(
   ZipfileTab *pTab,               /* Return errors here */
-  FILE *pFile,                    /* Read from this file */
+  const u8 *aBlob,                /* Pointer to in-memory file image */
+  int nBlob,                      /* Size of aBlob[] in bytes */
+  FILE *pFile,                    /* Read from this file if aBlob==0 */
   ZipfileEOCD *pEOCD              /* Object to populate */
 ){
   u8 *aRead = pTab->aBuffer;      /* Temporary buffer */
-  i64 szFile;                     /* Total size of file in bytes */
   int nRead;                      /* Bytes to read from file */
-  i64 iOff;                       /* Offset to read from */
-  int rc;
+  int rc = SQLITE_OK;
 
-  fseek(pFile, 0, SEEK_END);
-  szFile = (i64)ftell(pFile);
-  if( szFile==0 ){
-    memset(pEOCD, 0, sizeof(ZipfileEOCD));
-    return SQLITE_OK;
+  if( aBlob==0 ){
+    i64 iOff;                     /* Offset to read from */
+    i64 szFile;                   /* Total size of file in bytes */
+    fseek(pFile, 0, SEEK_END);
+    szFile = (i64)ftell(pFile);
+    if( szFile==0 ){
+      memset(pEOCD, 0, sizeof(ZipfileEOCD));
+      return SQLITE_OK;
+    }
+    nRead = (int)(MIN(szFile, ZIPFILE_BUFFER_SIZE));
+    iOff = szFile - nRead;
+    rc = zipfileReadData(pFile, aRead, nRead, iOff, &pTab->base.zErrMsg);
+  }else{
+    nRead = (int)(MIN(nBlob, ZIPFILE_BUFFER_SIZE));
+    aRead = (u8*)&aBlob[nBlob-nRead];
   }
-  nRead = (int)(MIN(szFile, ZIPFILE_BUFFER_SIZE));
-  iOff = szFile - nRead;
 
-  rc = zipfileReadData(pFile, aRead, nRead, iOff, &pTab->base.zErrMsg);
   if( rc==SQLITE_OK ){
     int i;
 
@@ -986,19 +1026,59 @@ static int zipfileReadEOCD(
     pEOCD->nEntryTotal = zipfileRead16(aRead);
     pEOCD->nSize = zipfileRead32(aRead);
     pEOCD->iOffset = zipfileRead32(aRead);
-
-#if 0
-    printf("iDisk=%d  iFirstDisk=%d  nEntry=%d  "
-           "nEntryTotal=%d  nSize=%d  iOffset=%d", 
-           (int)pEOCD->iDisk, (int)pEOCD->iFirstDisk, (int)pEOCD->nEntry,
-           (int)pEOCD->nEntryTotal, (int)pEOCD->nSize, (int)pEOCD->iOffset
-    );
-#endif
   }
 
   return SQLITE_OK;
 }
 
+/*
+** Add object pNew to the end of the linked list that begins at
+** ZipfileTab.pFirstEntry and ends with pLastEntry.
+*/
+static void zipfileAddEntry(
+  ZipfileTab *pTab, 
+  ZipfileEntry *pBefore, 
+  ZipfileEntry *pNew
+){
+  assert( (pTab->pFirstEntry==0)==(pTab->pLastEntry==0) );
+  assert( pNew->pNext==0 );
+  if( pBefore==0 ){
+    if( pTab->pFirstEntry==0 ){
+      pTab->pFirstEntry = pTab->pLastEntry = pNew;
+    }else{
+      assert( pTab->pLastEntry->pNext==0 );
+      pTab->pLastEntry->pNext = pNew;
+      pTab->pLastEntry = pNew;
+    }
+  }else{
+    ZipfileEntry **pp;
+    for(pp=&pTab->pFirstEntry; *pp!=pBefore; pp=&((*pp)->pNext));
+    pNew->pNext = pBefore;
+    *pp = pNew;
+  }
+}
+
+static int zipfileLoadDirectory(ZipfileTab *pTab, const u8 *aBlob, int nBlob){
+  ZipfileEOCD eocd;
+  int rc;
+  int i;
+  i64 iOff;
+
+  rc = zipfileReadEOCD(pTab, aBlob, nBlob, pTab->pWriteFd, &eocd);
+  iOff = eocd.iOffset;
+  for(i=0; rc==SQLITE_OK && i<eocd.nEntry; i++){
+    ZipfileEntry *pNew = 0;
+    rc = zipfileGetEntry(pTab, aBlob, nBlob, pTab->pWriteFd, iOff, &pNew);
+
+    if( rc==SQLITE_OK ){
+      zipfileAddEntry(pTab, 0, pNew);
+      iOff += ZIPFILE_CDS_FIXED_SZ;
+      iOff += (int)pNew->cds.nExtra + pNew->cds.nFile + pNew->cds.nComment;
+    }
+  }
+  return rc;
+}
+
 /*
 ** xFilter callback.
 */
@@ -1023,6 +1103,14 @@ static int zipfileFilter(
     zipfileSetErrmsg(pCsr, "table function zipfile() requires an argument");
     return SQLITE_ERROR;
   }else if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){
+    const u8 *aBlob = (const u8*)sqlite3_value_blob(argv[0]);
+    int nBlob = sqlite3_value_bytes(argv[0]);
+    assert( pTab->pFirstEntry==0 );
+    rc = zipfileLoadDirectory(pTab, aBlob, nBlob);
+    pCsr->pFreeEntry = pTab->pFirstEntry;
+    pTab->pFirstEntry = pTab->pLastEntry = 0;
+    if( rc!=SQLITE_OK ) return rc;
+    bInMemory = 1;
   }else{
     zFile = (const char*)sqlite3_value_text(argv[0]);
   }
@@ -1033,7 +1121,7 @@ static int zipfileFilter(
       zipfileSetErrmsg(pCsr, "cannot open file: %s", zFile);
       rc = SQLITE_ERROR;
     }else{
-      rc = zipfileReadEOCD(pTab, pCsr->pFile, &pCsr->eocd);
+      rc = zipfileReadEOCD(pTab, 0, 0, pCsr->pFile, &pCsr->eocd);
       if( rc==SQLITE_OK ){
         if( pCsr->eocd.nEntry==0 ){
           pCsr->bEof = 1;
@@ -1045,7 +1133,7 @@ static int zipfileFilter(
     }
   }else{
     pCsr->bNoop = 1;
-    pCsr->pCurrent = pTab->pFirstEntry;
+    pCsr->pCurrent = pCsr->pFreeEntry ? pCsr->pFreeEntry : pTab->pFirstEntry;
     rc = zipfileNext(cur);
   }
 
@@ -1082,54 +1170,6 @@ static int zipfileBestIndex(
   return SQLITE_OK;
 }
 
-/*
-** Add object pNew to the end of the linked list that begins at
-** ZipfileTab.pFirstEntry and ends with pLastEntry.
-*/
-static void zipfileAddEntry(
-  ZipfileTab *pTab, 
-  ZipfileEntry *pBefore, 
-  ZipfileEntry *pNew
-){
-  assert( (pTab->pFirstEntry==0)==(pTab->pLastEntry==0) );
-  assert( pNew->pNext==0 );
-  if( pBefore==0 ){
-    if( pTab->pFirstEntry==0 ){
-      pTab->pFirstEntry = pTab->pLastEntry = pNew;
-    }else{
-      assert( pTab->pLastEntry->pNext==0 );
-      pTab->pLastEntry->pNext = pNew;
-      pTab->pLastEntry = pNew;
-    }
-  }else{
-    ZipfileEntry **pp;
-    for(pp=&pTab->pFirstEntry; *pp!=pBefore; pp=&((*pp)->pNext));
-    pNew->pNext = pBefore;
-    *pp = pNew;
-  }
-}
-
-static int zipfileLoadDirectory(ZipfileTab *pTab){
-  ZipfileEOCD eocd;
-  int rc;
-  int i;
-  i64 iOff;
-
-  rc = zipfileReadEOCD(pTab, pTab->pWriteFd, &eocd);
-  iOff = eocd.iOffset;
-  for(i=0; rc==SQLITE_OK && i<eocd.nEntry; i++){
-    ZipfileEntry *pNew = 0;
-    rc = zipfileGetEntry(pTab, pTab->pWriteFd, iOff, &pNew);
-
-    if( rc==SQLITE_OK ){
-      zipfileAddEntry(pTab, 0, pNew);
-      iOff += ZIPFILE_CDS_FIXED_SZ;
-      iOff += (int)pNew->cds.nExtra + pNew->cds.nFile + pNew->cds.nComment;
-    }
-  }
-  return rc;
-}
-
 static ZipfileEntry *zipfileNewEntry(const char *zPath){
   ZipfileEntry *pNew;
   pNew = sqlite3_malloc(sizeof(ZipfileEntry));
@@ -1458,7 +1498,7 @@ static int zipfileBegin(sqlite3_vtab *pVtab){
   }else{
     fseek(pTab->pWriteFd, 0, SEEK_END);
     pTab->szCurrent = pTab->szOrig = (i64)ftell(pTab->pWriteFd);
-    rc = zipfileLoadDirectory(pTab);
+    rc = zipfileLoadDirectory(pTab, 0, 0);
   }
 
   if( rc!=SQLITE_OK ){
diff --git a/manifest b/manifest
index c964e14825..7ae6ef7c87 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improve\stext-to-integer\sconversion\sin\sboundary\scases.\s\sThe\nsqlite3Atoi64()\sfunction\salways\sreturns\sthe\sminimum\sor\smaximum\sinteger\nif\sthe\smagnitude\sof\sthe\stext\svalue\sis\stoo\slarge.\s\sTrailing\swhitespace\sis\nnow\signored.
-D 2018-01-26T18:37:34.351
+C If\sthe\sargument\sto\stable\sfunction\szipfile()\sis\sa\sblob\s(not\stext),\sassume\sthat\nit\scontains\sa\szip\sfile\simage\sto\sinterpret,\snot\sthe\sname\sof\sa\sfile\son\sdisk.
+D 2018-01-26T18:59:25.439
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -304,7 +304,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c 45cc42164444fa49c9a04ba8361e38e4727862a3f2f64a35e4f6f661f8208c5a
+F ext/misc/zipfile.c 071fb5af4d0e6403f01cfac755d78ed2008b04793ec6f87a1078755a6359a26f
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -1603,7 +1603,7 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf
 F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc
 F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa
 F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc
-F test/zipfile.test cb42e8fa6ba5db4a03ce6baa4401fc6236baf6eb5e62b44f3e463bf6aafd631d
+F test/zipfile.test 0834b33e000991a80d94167e84af346162d212526d8efee0b9d4ba84589fe292
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d
 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91
@@ -1702,7 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 30b9258294e3028ef4ea467e332e013995509544e9a23c8bbf5168772a7e895d
-R a3a69d646870f53631d68482ffc811f2
-U drh
-Z ebcc0484dfae31c55e47d329daf31900
+P ace0644a1a2a42a3ea42d44f00a31915b8a7e56c9ba90f90a6c02001f89f9c86
+R 6f618424b3fb9707a3fb679f6f287314
+U dan
+Z ab7fcdd492fa54ea891500a7623bc431
diff --git a/manifest.uuid b/manifest.uuid
index 5fee1327fd..e4da9d78da 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ace0644a1a2a42a3ea42d44f00a31915b8a7e56c9ba90f90a6c02001f89f9c86
\ No newline at end of file
+029ebcd30cb261d949f7587ac54c23d7479796b6716fd4ca7512361b8f32de3e
\ No newline at end of file
diff --git a/test/zipfile.test b/test/zipfile.test
index 07fc80f602..60fde2ee15 100644
--- a/test/zipfile.test
+++ b/test/zipfile.test
@@ -22,6 +22,20 @@ if {[catch {load_static_extension db zipfile} error]} {
   finish_test; return
 }
 
+proc do_zipfile_blob_test {tn file} {
+  set res1 [
+    db eval { SELECT name,mode,mtime,method,quote(data) FROM zipfile($file) }
+  ]
+
+  set fd [open $file]
+  fconfigure $fd -translation binary -encoding binary
+  set data [read $fd]
+  close $fd
+
+  set res2 [db eval { SELECT name,mode,mtime,method,quote(data) FROM zipfile($data) }]
+  uplevel [list do_test $tn [list set {} $res2] $res1]
+}
+
 forcedelete test.zip
 do_execsql_test 1.0 {
   CREATE VIRTUAL TABLE temp.zz USING zipfile('test.zip');
@@ -60,12 +74,14 @@ do_execsql_test 1.2 {
   f.txt 1000000000 abcde 
   g.txt 1000000002 12345
 }
+do_zipfile_blob_test 1.2.1 test.zip
 
 do_execsql_test 1.3 {
   INSERT INTO zz(name, mode, mtime, data) VALUES('h.txt', 
     '-rw-r--r--', 1000000004, 'aaaaaaaaaabbbbbbbbbb'
   );
 }
+do_zipfile_blob_test 1.3.1 test.zip
 
 do_execsql_test 1.4 {
   SELECT name, mtime, data, method FROM zipfile('test.zip');
@@ -112,6 +128,7 @@ do_execsql_test 1.6.1 {
   h.txt 33188 1000000004 aaaaaaaaaabbbbbbbbbb 8
   i.txt 33188 1000000006 zxcvb 0
 }
+do_zipfile_blob_test 1.6.1a test.zip
 
 do_execsql_test 1.6.2 {
   UPDATE zz SET mtime=4 WHERE name='i.txt';
@@ -130,6 +147,7 @@ do_execsql_test 1.6.3 {
   h.txt 33189 1000000004 aaaaaaaaaabbbbbbbbbb 8
   i.txt 33188 4 zxcvb 0
 }
+do_zipfile_blob_test 1.6.3a test.zip
 
 do_execsql_test 1.6.4 {
   UPDATE zz SET name = 'blue.txt' WHERE name='f.txt';
@@ -139,6 +157,7 @@ do_execsql_test 1.6.4 {
   h.txt 33189 1000000004 aaaaaaaaaabbbbbbbbbb 8
   i.txt 33188 4 zxcvb 0
 }
+do_zipfile_blob_test 1.6.4a test.zip
 
 do_execsql_test 1.6.5 {
   UPDATE zz SET data = 'edcba' WHERE name='blue.txt';
@@ -199,6 +218,7 @@ do_execsql_test 2.4 {
   dirname2/ 16877 {}
   dirname2/file1.txt 33188 abcdefghijklmnop
 }
+do_zipfile_blob_test 2.4.1 test.zip
 
 # If on unix, check that the [unzip] utility can unpack our archive.
 #

From 33f763d1540b07b9e8088b193991d8a436f45f0a Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 26 Jan 2018 22:41:59 +0000
Subject: [PATCH 391/488] Fix the query planner so that it takes into account
 dependencies in the arguments to table-valued functions in subexpressions in
 the WHERE clause. Fix for ticket [80177f0c226ff54f6dd].

FossilOrigin-Name: 7daa687340e475972d6849f84fe1419a1f5700934f53fd80656849adc73d6d04
---
 manifest          | 16 ++++++++--------
 manifest.uuid     |  2 +-
 src/whereexpr.c   |  3 +++
 test/json101.test | 25 ++++++++++++++++++++++++-
 4 files changed, 36 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index 7ae6ef7c87..d9b3091b28 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C If\sthe\sargument\sto\stable\sfunction\szipfile()\sis\sa\sblob\s(not\stext),\sassume\sthat\nit\scontains\sa\szip\sfile\simage\sto\sinterpret,\snot\sthe\sname\sof\sa\sfile\son\sdisk.
-D 2018-01-26T18:59:25.439
+C Fix\sthe\squery\splanner\sso\sthat\sit\stakes\sinto\saccount\sdependencies\sin\sthe\narguments\sto\stable-valued\sfunctions\sin\ssubexpressions\sin\sthe\sWHERE\sclause.\nFix\sfor\sticket\s[80177f0c226ff54f6dd].
+D 2018-01-26T22:41:59.788
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -573,7 +573,7 @@ F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f
 F src/where.c caf0b6c9d31f22f0b2c91aba723858de52b5d665aaa89034099015aaf9bb8219
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
 F src/wherecode.c cb360c511f69294ddf00340d7b390e6b1d601a1cfb77b42c4d316fe2f4cd01c3
-F src/whereexpr.c 427ea8e96ec24f2a7814c67b8024ad664a9c7656264c4566c34743cb23186e46
+F src/whereexpr.c 22dbfd3bf5f6051a61523dd0ebef7a944fb29ee4aa7d2a62feb8aac6ffbbc0eb
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd
 F test/affinity3.test 6a101af2fc945ce2912f6fe54dd646018551710d
@@ -1012,7 +1012,7 @@ F test/journal3.test c9c29883f5bf535ae82ae21c472df6263806a22e467b6db7cd0d6d54530
 F test/jrnlmode.test a6693f2bed4541a21e703aaa37bb3e10de154130645952933b82b2dec0a8b539
 F test/jrnlmode2.test 8759a1d4657c064637f8b079592651530db738419e1d649c6df7048cd724363d
 F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa
-F test/json101.test d7cdf3e6731d41e0c4bde1c88806abd17f1f478486a1409933c1d8eac9120095
+F test/json101.test 24e97954e3bd6404f3715888c7f8f835e36e19c7ae6513b5d9ab2d381498962d
 F test/json102.test eeb54efa221e50b74a2d6fb9259963b48d7414dca3ce2fdfdeed45cb28487bc1
 F test/json103.test c5f6b85e69de05f6b3195f9f9d5ce9cd179099a0
 F test/json104.test 877d5845f6303899b7889ea5dd1bea99076e3100574d5c536082245c5805dcaa
@@ -1702,7 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ace0644a1a2a42a3ea42d44f00a31915b8a7e56c9ba90f90a6c02001f89f9c86
-R 6f618424b3fb9707a3fb679f6f287314
-U dan
-Z ab7fcdd492fa54ea891500a7623bc431
+P 029ebcd30cb261d949f7587ac54c23d7479796b6716fd4ca7512361b8f32de3e
+R 53bd80b0955af3fa0129a178c0e00c7d
+U drh
+Z 5dfc01b033acb2ff4461347ae9c97b23
diff --git a/manifest.uuid b/manifest.uuid
index e4da9d78da..45ee5fd82d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-029ebcd30cb261d949f7587ac54c23d7479796b6716fd4ca7512361b8f32de3e
\ No newline at end of file
+7daa687340e475972d6849f84fe1419a1f5700934f53fd80656849adc73d6d04
\ No newline at end of file
diff --git a/src/whereexpr.c b/src/whereexpr.c
index 58f1908cf8..82cc519650 100644
--- a/src/whereexpr.c
+++ b/src/whereexpr.c
@@ -876,6 +876,9 @@ static Bitmask exprSelectUsage(WhereMaskSet *pMaskSet, Select *pS){
       for(i=0; i<pSrc->nSrc; i++){
         mask |= exprSelectUsage(pMaskSet, pSrc->a[i].pSelect);
         mask |= sqlite3WhereExprUsage(pMaskSet, pSrc->a[i].pOn);
+        if( pSrc->a[i].fg.isTabFunc ){
+          mask |= sqlite3WhereExprListUsage(pMaskSet, pSrc->a[i].u1.pFuncArg);
+        }
       }
     }
     pS = pS->pPrior;
diff --git a/test/json101.test b/test/json101.test
index 4bfcc2d80d..ac4e11ebc2 100644
--- a/test/json101.test
+++ b/test/json101.test
@@ -759,7 +759,30 @@ do_execsql_test json-12.120 {
     FROM t12;
 } {0}
 
-
+# 2018-01-26
+# ticket https://www.sqlite.org/src/tktview/80177f0c226ff54f6ddd41
+# Make sure the query planner knows about the arguments to table-valued functions.
+#
+do_execsql_test json-13.100 {
+  DROP TABLE IF EXISTS t1;
+  DROP TABLE IF EXISTS t2;
+  CREATE TABLE t1(id, json);
+  INSERT INTO t1(id,json) VALUES(1,'{"items":[3,5]}');
+  CREATE TABLE t2(id, json);
+  INSERT INTO t2(id,json) VALUES(2,'{"value":2}');
+  INSERT INTO t2(id,json) VALUES(3,'{"value":3}');
+  INSERT INTO t2(id,json) VALUES(4,'{"value":4}');
+  INSERT INTO t2(id,json) VALUES(5,'{"value":5}');
+  INSERT INTO t2(id,json) VALUES(6,'{"value":6}');
+  SELECT * FROM t1 CROSS JOIN t2
+   WHERE EXISTS(SELECT 1 FROM json_each(t1.json,'$.items') AS Z
+                 WHERE Z.value==t2.id);
+} {1 {{"items":[3,5]}} 3 {{"value":3}} 1 {{"items":[3,5]}} 5 {{"value":5}}}
+do_execsql_test json-13.110 {
+  SELECT * FROM t2 CROSS JOIN t1
+   WHERE EXISTS(SELECT 1 FROM json_each(t1.json,'$.items') AS Z
+                 WHERE Z.value==t2.id);
+} {3 {{"value":3}} 1 {{"items":[3,5]}} 5 {{"value":5}} 1 {{"items":[3,5]}}}
 
 
 finish_test

From c86caa58cd4f101417f6130f58a537c8cbaf6573 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 27 Jan 2018 03:26:10 +0000
Subject: [PATCH 392/488] Fix compiler warnings in zipfile.c.

FossilOrigin-Name: bed610d99b5322ee209b06966c0086e8c55f8a82be8d7693cfdeb4d728679721
---
 ext/misc/zipfile.c |  8 ++------
 manifest           | 12 ++++++------
 manifest.uuid      |  2 +-
 3 files changed, 9 insertions(+), 13 deletions(-)

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index c199f8ec1b..61e60f63d2 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -504,10 +504,6 @@ static void zipfilePutU32(u8 *aBuf, u32 val){
 #define zipfileWrite32(aBuf,val) { zipfilePutU32(aBuf,val); aBuf+=4; }
 #define zipfileWrite16(aBuf,val) { zipfilePutU16(aBuf,val); aBuf+=2; }
 
-static u8* zipfileCsrBuffer(ZipfileCsr *pCsr){
-  return ((ZipfileTab*)(pCsr->base.pVtab))->aBuffer;
-}
-
 /*
 ** Magic numbers used to read CDS records.
 */
@@ -649,7 +645,7 @@ static int zipfileGetEntry(
   int rc = SQLITE_OK;
 
   if( aBlob==0 ){
-    aRead = pTab->aBuffer;
+    aRead = (u8*)pTab->aBuffer;
     rc = zipfileReadData(pFile, aRead, ZIPFILE_CDS_FIXED_SZ, iOff, pzErr);
   }else{
     aRead = (u8*)&aBlob[iOff];
@@ -703,7 +699,7 @@ static int zipfileGetEntry(
       if( pFile ){
         rc = zipfileReadData(pFile, aRead, szFix, pNew->cds.iOffset, pzErr);
       }else{
-        aRead = &aBlob[pNew->cds.iOffset];
+        aRead = (u8*)&aBlob[pNew->cds.iOffset];
       }
 
       rc = zipfileReadLFH(aRead, &lfh);
diff --git a/manifest b/manifest
index d9b3091b28..0e0a71deef 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sthe\squery\splanner\sso\sthat\sit\stakes\sinto\saccount\sdependencies\sin\sthe\narguments\sto\stable-valued\sfunctions\sin\ssubexpressions\sin\sthe\sWHERE\sclause.\nFix\sfor\sticket\s[80177f0c226ff54f6dd].
-D 2018-01-26T22:41:59.788
+C Fix\scompiler\swarnings\sin\szipfile.c.
+D 2018-01-27T03:26:10.457
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -304,7 +304,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c 071fb5af4d0e6403f01cfac755d78ed2008b04793ec6f87a1078755a6359a26f
+F ext/misc/zipfile.c 40195c1cfb43e0ebdf176d37346fea1cb18246e35fbfce477eadc2ff5f7490df
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -1702,7 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 029ebcd30cb261d949f7587ac54c23d7479796b6716fd4ca7512361b8f32de3e
-R 53bd80b0955af3fa0129a178c0e00c7d
+P 7daa687340e475972d6849f84fe1419a1f5700934f53fd80656849adc73d6d04
+R db0d53a567e4f3bfdfebf8c5b2a2a369
 U drh
-Z 5dfc01b033acb2ff4461347ae9c97b23
+Z 1789fd20b7b2fb64dfc10fb2cef79caf
diff --git a/manifest.uuid b/manifest.uuid
index 45ee5fd82d..98506da2d8 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7daa687340e475972d6849f84fe1419a1f5700934f53fd80656849adc73d6d04
\ No newline at end of file
+bed610d99b5322ee209b06966c0086e8c55f8a82be8d7693cfdeb4d728679721
\ No newline at end of file

From 33f10207aa0bc86a190e9ea75298895f508834dd Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 27 Jan 2018 05:40:10 +0000
Subject: [PATCH 393/488] Proposed fix for the query planner problem of ticket
 [ec32177c99ccac2b1].

FossilOrigin-Name: eef8cbef3c49e6331301a8f3f8a5fd12090b1e99e68487c2d405e53ef771f688
---
 manifest         | 14 +++++-----
 manifest.uuid    |  2 +-
 src/where.c      |  1 +
 test/whereF.test | 66 ++++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 75 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 0e0a71deef..374e2479eb 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\scompiler\swarnings\sin\szipfile.c.
-D 2018-01-27T03:26:10.457
+C Proposed\sfix\sfor\sthe\squery\splanner\sproblem\sof\sticket\n[ec32177c99ccac2b1].
+D 2018-01-27T05:40:10.409
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -570,7 +570,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 5a3f464edd64596f601683ed321d12e6fd93c5fb9afdfb3653d6ffd0fee9c48f
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f
-F src/where.c caf0b6c9d31f22f0b2c91aba723858de52b5d665aaa89034099015aaf9bb8219
+F src/where.c 10c7e13c909e4dc93057fae930733f5018cdf1a040f73b9e873a264edfc101e5
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
 F src/wherecode.c cb360c511f69294ddf00340d7b390e6b1d601a1cfb77b42c4d316fe2f4cd01c3
 F src/whereexpr.c 22dbfd3bf5f6051a61523dd0ebef7a944fb29ee4aa7d2a62feb8aac6ffbbc0eb
@@ -1574,7 +1574,7 @@ F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5
 F test/whereC.test cae295158703cb3fc23bf1a108a9ab730efff0f6
 F test/whereD.test 711d4df58d6d4fb9b3f5ce040b818564198be002
 F test/whereE.test b3a055eef928c992b0a33198a7b8dc10eea5ad2f
-F test/whereF.test d44b58338fe5ddd7286023e9bedb255aa264a6c4d2168b49591b167371c675c7
+F test/whereF.test 15552693e5965e096d58a325216bfcf102b6e2b38f4f5a0d64a728c2f3826714
 F test/whereG.test dde4c52a97385a55be6a7cd46be8373f0cf35501
 F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2
 F test/whereI.test eab5b226bbc344ac70d7dc09b963a064860ae6d7
@@ -1702,7 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 7daa687340e475972d6849f84fe1419a1f5700934f53fd80656849adc73d6d04
-R db0d53a567e4f3bfdfebf8c5b2a2a369
+P bed610d99b5322ee209b06966c0086e8c55f8a82be8d7693cfdeb4d728679721
+R 5bfee097e7e4c4077c4171c76898db44
 U drh
-Z 1789fd20b7b2fb64dfc10fb2cef79caf
+Z 3dd05a091b8a00a96f3fbc2c2cb36a31
diff --git a/manifest.uuid b/manifest.uuid
index 98506da2d8..00da401ea7 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-bed610d99b5322ee209b06966c0086e8c55f8a82be8d7693cfdeb4d728679721
\ No newline at end of file
+eef8cbef3c49e6331301a8f3f8a5fd12090b1e99e68487c2d405e53ef771f688
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index 3152c8e9aa..7f45182d9a 100644
--- a/src/where.c
+++ b/src/where.c
@@ -4631,6 +4631,7 @@ WhereInfo *sqlite3WhereBegin(
   */
   for(ii=0; ii<sWLB.pWC->nTerm; ii++){
     WhereTerm *pT = &sWLB.pWC->a[ii];
+    if( pT->wtFlags & TERM_VIRTUAL ) continue;
     if( pT->prereqAll==0 && (nTabList==0 || exprIsDeterministic(pT->pExpr)) ){
       sqlite3ExprIfFalse(pParse, pT->pExpr, pWInfo->iBreak, SQLITE_JUMPIFNULL);
       pT->wtFlags |= TERM_CODED;
diff --git a/test/whereF.test b/test/whereF.test
index 8b272a5c53..e5028969a1 100644
--- a/test/whereF.test
+++ b/test/whereF.test
@@ -215,4 +215,70 @@ ifcapable json1&&vtab {
   } {{{"foo":"meep","other":12345}}}
 }
 
+# 2018-01-27
+# Ticket https://sqlite.org/src/tktview/ec32177c99ccac2b180fd3ea2083
+# Incorrect result when using the new OR clause factoring optimization
+#
+# This is the original test case as reported on the sqlite-users mailing
+# list
+#
+do_execsql_test 7.1 {
+  DROP TABLE IF EXISTS cd;
+  CREATE TABLE cd ( cdid INTEGER PRIMARY KEY NOT NULL, genreid integer );
+  CREATE INDEX cd_idx_genreid ON cd (genreid);
+  INSERT INTO cd  ( cdid, genreid ) VALUES
+                     ( 1,    1 ),
+                     ( 2, NULL ),
+                     ( 3, NULL ),
+                     ( 4, NULL ),
+                     ( 5, NULL );
+  
+  SELECT cdid
+    FROM cd me
+  WHERE 2 > (
+    SELECT COUNT( * )
+      FROM cd rownum__emulation
+    WHERE
+      (
+        me.genreid IS NOT NULL
+          AND
+        rownum__emulation.genreid IS NULL
+      )
+        OR
+      (
+        me.genreid IS NOT NULL
+          AND
+        rownum__emulation.genreid IS NOT NULL
+          AND
+        rownum__emulation.genreid < me.genreid
+      )
+        OR
+      (
+        ( me.genreid = rownum__emulation.genreid OR ( me.genreid IS NULL
+  AND rownum__emulation.genreid IS NULL ) )
+          AND
+        rownum__emulation.cdid > me.cdid
+      )
+  );
+} {4 5}
+
+# Simplified test cases from the ticket
+#
+do_execsql_test 7.2 {
+  DROP TABLE IF EXISTS t1;
+  DROP TABLE IF EXISTS t2;
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
+  INSERT INTO t1(a,b) VALUES(1,1);
+  CREATE TABLE t2(aa INTEGER PRIMARY KEY, bb);
+  INSERT INTO t2(aa,bb) VALUES(1,1),(2,NULL),(3,NULL);
+  SELECT (
+    SELECT COUNT(*) FROM t2
+     WHERE ( t1.b IS NOT NULL AND t2.bb IS NULL )
+        OR ( t2.bb < t1.b )
+        OR ( t1.b IS t2.bb AND t2.aa > t1.a )
+    )
+    FROM t1;
+} {2}
+
+
 finish_test

From c9f3db33d5547187f1ce129452d20e2b0b1e86ff Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 27 Jan 2018 13:55:56 +0000
Subject: [PATCH 394/488] New test case for ticket [ec32177c99ccac2b1] that
 works without the STAT4.

FossilOrigin-Name: 5259d4847f2b73f26b2385f9d8cff8fe0cabc54b4deab8477c87c8d1bb5535b1
---
 manifest         | 12 ++++++------
 manifest.uuid    |  2 +-
 test/whereF.test | 28 ++++++++++++++++++++++++++++
 3 files changed, 35 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index 374e2479eb..b88aeece1d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Proposed\sfix\sfor\sthe\squery\splanner\sproblem\sof\sticket\n[ec32177c99ccac2b1].
-D 2018-01-27T05:40:10.409
+C New\stest\scase\sfor\sticket\s[ec32177c99ccac2b1]\sthat\sworks\swithout\sthe\sSTAT4.
+D 2018-01-27T13:55:56.815
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -1574,7 +1574,7 @@ F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5
 F test/whereC.test cae295158703cb3fc23bf1a108a9ab730efff0f6
 F test/whereD.test 711d4df58d6d4fb9b3f5ce040b818564198be002
 F test/whereE.test b3a055eef928c992b0a33198a7b8dc10eea5ad2f
-F test/whereF.test 15552693e5965e096d58a325216bfcf102b6e2b38f4f5a0d64a728c2f3826714
+F test/whereF.test 3d9412b1199d3e2bed34fcb76b4c48d0bf4df95d27e3f8dd27b6f8b4716d0d89
 F test/whereG.test dde4c52a97385a55be6a7cd46be8373f0cf35501
 F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2
 F test/whereI.test eab5b226bbc344ac70d7dc09b963a064860ae6d7
@@ -1702,7 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P bed610d99b5322ee209b06966c0086e8c55f8a82be8d7693cfdeb4d728679721
-R 5bfee097e7e4c4077c4171c76898db44
+P eef8cbef3c49e6331301a8f3f8a5fd12090b1e99e68487c2d405e53ef771f688
+R a57e94d6d2f924366bc9079a11d7258c
 U drh
-Z 3dd05a091b8a00a96f3fbc2c2cb36a31
+Z 5c67b72d2f152607d54502c89af32425
diff --git a/manifest.uuid b/manifest.uuid
index 00da401ea7..b2d9772040 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-eef8cbef3c49e6331301a8f3f8a5fd12090b1e99e68487c2d405e53ef771f688
\ No newline at end of file
+5259d4847f2b73f26b2385f9d8cff8fe0cabc54b4deab8477c87c8d1bb5535b1
\ No newline at end of file
diff --git a/test/whereF.test b/test/whereF.test
index e5028969a1..121cc3cf22 100644
--- a/test/whereF.test
+++ b/test/whereF.test
@@ -280,5 +280,33 @@ do_execsql_test 7.2 {
     FROM t1;
 } {2}
 
+# The fix for ticket ec32177c99ccac2b180fd3ea2083 only makes a difference
+# in the output when there is a TERM_VNULL entry in the WhereClause array.
+# And TERM_VNULL entries are only generated when compiling with 
+# SQLITE_ENABLE_STAT4.  Nevertheless, it is correct that TERM_VIRTUAL terms
+# should not participate in the factoring optimization.  In all cases other
+# than TERM_VNULL, participation is harmless, but it does consume a few
+# extra CPU cycles.
+#
+# The following test verifies that the TERM_VIRTUAL terms resulting from
+# a GLOB operator do not appear anywhere in the generated code.  This
+# confirms that the problem is fixed, even on builds that omit STAT4.
+#
+do_execsql_test 7.3 {
+  DROP TABLE IF EXISTS t1;
+  DROP TABLE IF EXISTS t2;
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT);
+  INSERT INTO t1(a,b) VALUES(1,'abcxyz');
+  CREATE TABLE t2(aa INTEGER PRIMARY KEY, bb TEXT);
+  INSERT INTO t2(aa,bb) VALUES(1,'abc'),(2,'wxyz'),(3,'xyz');
+  CREATE INDEX t2bb ON t2(bb);
+  EXPLAIN SELECT (
+    SELECT COUNT(*) FROM t2
+     WHERE ( t1.b GLOB 'a*z' AND t2.bb='xyz' )
+        OR ( t2.bb = t1.b )
+        OR ( t2.aa = t1.a )
+    )
+    FROM t1;
+} {~/ (Lt|Ge) /}
 
 finish_test

From 1822ebf9b1997c4bc8ed3371b065a494bac3d6d5 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 27 Jan 2018 14:25:27 +0000
Subject: [PATCH 395/488] Changes to avoid a harmless UB warning from clang.

FossilOrigin-Name: 19f5c1400054df10688ab448e7e23afef97cab4a7c7a3e411f7527509b515dd8
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/util.c    |  8 +++++++-
 3 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index b88aeece1d..faa8c1e356 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C New\stest\scase\sfor\sticket\s[ec32177c99ccac2b1]\sthat\sworks\swithout\sthe\sSTAT4.
-D 2018-01-27T13:55:56.815
+C Changes\sto\savoid\sa\sharmless\sUB\swarning\sfrom\sclang.
+D 2018-01-27T14:25:27.955
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -554,7 +554,7 @@ F src/treeview.c eae35972ff44f67064de2eaf35f04afe94e7aea3271a8b3bcebb3f954880fec
 F src/trigger.c a34539c69433276d37b0da9a89c117726ff2d292c0902895af1f393a983cd3a1
 F src/update.c a90a32ffc0100265b0693dbbdbe490756447af181f5ea2c138cce515b08c8795
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
-F src/util.c 8628fe1daa9f5db9fa3a70d6fcfd828b73e839feafe7d7faecd7073be56eaec2
+F src/util.c d9eb0a6c4aae1b00a7369eadd7ca0bbe946cb4c953b6751aa20d357c2f482157
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
 F src/vdbe.c 0e13b8c33a9ecf236b3045f190fb58933049582307b621cbd924b6582a2b705f
 F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a
@@ -1702,7 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P eef8cbef3c49e6331301a8f3f8a5fd12090b1e99e68487c2d405e53ef771f688
-R a57e94d6d2f924366bc9079a11d7258c
+P 5259d4847f2b73f26b2385f9d8cff8fe0cabc54b4deab8477c87c8d1bb5535b1
+R dc25ffafa3daf6e245ccddfac4936cfd
 U drh
-Z 5c67b72d2f152607d54502c89af32425
+Z 83f6cfb83d43c9463ef36c14a8c84938
diff --git a/manifest.uuid b/manifest.uuid
index b2d9772040..3f53762e60 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5259d4847f2b73f26b2385f9d8cff8fe0cabc54b4deab8477c87c8d1bb5535b1
\ No newline at end of file
+19f5c1400054df10688ab448e7e23afef97cab4a7c7a3e411f7527509b515dd8
\ No newline at end of file
diff --git a/src/util.c b/src/util.c
index 9954d709df..54f9b93887 100644
--- a/src/util.c
+++ b/src/util.c
@@ -641,7 +641,13 @@ int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){
   testcase( i==18*incr );
   testcase( i==19*incr );
   testcase( i==20*incr );
-  if( neg ){
+  if( u>LARGEST_INT64 ){
+    /* This test and assignment is needed only to suppress UB warnings
+    ** from clang and -fsanitize=undefined.  This test and assignment make
+    ** the code a little larger and slower, and no harm comes from omitting
+    ** them, but we must appaise the undefined-behavior pharisees. */
+    *pNum = neg ? SMALLEST_INT64 : LARGEST_INT64;
+  }else if( neg ){
     *pNum = -(i64)u;
   }else{
     *pNum = (i64)u;

From 26333ee3cae6491bf0f4ec2720446d726dfd3e2b Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 27 Jan 2018 16:29:59 +0000
Subject: [PATCH 396/488] If a zipfile virtual table is created with no
 argument - "CREATE VIRTUAL TABLE zzz USING zipfile()" - accumulate data in
 memory. Support "SELECT zipfile_blob(z) FROM zzz LIMIT 1" to retrieve a zip
 archive image.

FossilOrigin-Name: e63185edfe0c316aa60c1fa085d032425ecc7db54536dfa5a977772eaf3c240e
---
 ext/misc/zipfile.c | 257 +++++++++++++++++++++++++++++----------------
 manifest           |  16 +--
 manifest.uuid      |   2 +-
 test/zipfile.test  |  40 ++++++-
 4 files changed, 213 insertions(+), 102 deletions(-)

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index 61e60f63d2..4e59675c1d 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -103,6 +103,8 @@ static const char ZIPFILE_SCHEMA[] =
 #define ZIPFILE_SIGNATURE_EOCD    0x06054b50
 #define ZIPFILE_LFH_FIXED_SZ      30
 
+#define ZIPFILE_EOCD_FIXED_SZ     22
+
 /*
 ** Set the error message contained in context ctx to the results of
 ** vprintf(zFmt, ...).
@@ -645,7 +647,7 @@ static int zipfileGetEntry(
   int rc = SQLITE_OK;
 
   if( aBlob==0 ){
-    aRead = (u8*)pTab->aBuffer;
+    aRead = pTab->aBuffer;
     rc = zipfileReadData(pFile, aRead, ZIPFILE_CDS_FIXED_SZ, iOff, pzErr);
   }else{
     aRead = (u8*)&aBlob[iOff];
@@ -1094,10 +1096,7 @@ static int zipfileFilter(
   if( pTab->zFile ){
     zFile = pTab->zFile;
   }else if( idxNum==0 ){
-    /* Error. This is an eponymous virtual table and the user has not 
-    ** supplied a file name. */
-    zipfileSetErrmsg(pCsr, "table function zipfile() requires an argument");
-    return SQLITE_ERROR;
+    bInMemory = 1;
   }else if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){
     const u8 *aBlob = (const u8*)sqlite3_value_blob(argv[0]);
     int nBlob = sqlite3_value_bytes(argv[0]);
@@ -1166,11 +1165,14 @@ static int zipfileBestIndex(
   return SQLITE_OK;
 }
 
-static ZipfileEntry *zipfileNewEntry(const char *zPath){
+static ZipfileEntry *zipfileNewEntry(const char *zPath, int nData){
   ZipfileEntry *pNew;
-  pNew = sqlite3_malloc(sizeof(ZipfileEntry));
+  pNew = sqlite3_malloc(sizeof(ZipfileEntry) + nData);
   if( pNew ){
     memset(pNew, 0, sizeof(ZipfileEntry));
+    if( nData ){
+      pNew->aData = (u8*)&pNew[1];
+    }
     pNew->cds.zFile = sqlite3_mprintf("%s", zPath);
     if( pNew->cds.zFile==0 ){
       sqlite3_free(pNew);
@@ -1180,46 +1182,51 @@ static ZipfileEntry *zipfileNewEntry(const char *zPath){
   return pNew;
 }
 
-static int zipfileAppendEntry(
-  ZipfileTab *pTab,
-  ZipfileCDS *pCds,
-  const char *zPath,              /* Path for new entry */
-  int nPath,                      /* strlen(zPath) */
-  const u8 *pData,
-  int nData,
-  u32 mTime
-){
-  u8 *aBuf = pTab->aBuffer;
-  int rc;
+static int zipfileSerializeLFH(ZipfileEntry *pEntry, u8 *aBuf){
+  ZipfileCDS *pCds = &pEntry->cds;
+  u8 *a = aBuf;
 
   pCds->nExtra = 9;
 
-  zipfileWrite32(aBuf, ZIPFILE_SIGNATURE_LFH);
-  zipfileWrite16(aBuf, pCds->iVersionExtract);
-  zipfileWrite16(aBuf, pCds->flags);
-  zipfileWrite16(aBuf, pCds->iCompression);
-  zipfileWrite16(aBuf, pCds->mTime);
-  zipfileWrite16(aBuf, pCds->mDate);
-  zipfileWrite32(aBuf, pCds->crc32);
-  zipfileWrite32(aBuf, pCds->szCompressed);
-  zipfileWrite32(aBuf, pCds->szUncompressed);
-  zipfileWrite16(aBuf, (u16)nPath);
-  zipfileWrite16(aBuf, pCds->nExtra);
-  assert( aBuf==&pTab->aBuffer[ZIPFILE_LFH_FIXED_SZ] );
-  rc = zipfileAppendData(pTab, pTab->aBuffer, (int)(aBuf - pTab->aBuffer));
-  if( rc==SQLITE_OK ){
-    rc = zipfileAppendData(pTab, (const u8*)zPath, nPath);
-  }
+  /* Write the LFH itself */
+  zipfileWrite32(a, ZIPFILE_SIGNATURE_LFH);
+  zipfileWrite16(a, pCds->iVersionExtract);
+  zipfileWrite16(a, pCds->flags);
+  zipfileWrite16(a, pCds->iCompression);
+  zipfileWrite16(a, pCds->mTime);
+  zipfileWrite16(a, pCds->mDate);
+  zipfileWrite32(a, pCds->crc32);
+  zipfileWrite32(a, pCds->szCompressed);
+  zipfileWrite32(a, pCds->szUncompressed);
+  zipfileWrite16(a, (u16)pCds->nFile);
+  zipfileWrite16(a, pCds->nExtra);
+  assert( a==&aBuf[ZIPFILE_LFH_FIXED_SZ] );
 
-  if( rc==SQLITE_OK && pCds->nExtra ){
-    aBuf = pTab->aBuffer;
-    zipfileWrite16(aBuf, ZIPFILE_EXTRA_TIMESTAMP);
-    zipfileWrite16(aBuf, 5);
-    *aBuf++ = 0x01;
-    zipfileWrite32(aBuf, mTime);
-    rc = zipfileAppendData(pTab, pTab->aBuffer, 9);
-  }
+  /* Add the file name */
+  memcpy(a, pCds->zFile, (int)pCds->nFile);
+  a += (int)pCds->nFile;
 
+  /* The "extra" data */
+  zipfileWrite16(a, ZIPFILE_EXTRA_TIMESTAMP);
+  zipfileWrite16(a, 5);
+  *a++ = 0x01;
+  zipfileWrite32(a, pEntry->mUnixTime);
+
+  return a-aBuf;
+}
+
+static int zipfileAppendEntry(
+  ZipfileTab *pTab,
+  ZipfileEntry *pEntry,
+  const u8 *pData,
+  int nData
+){
+  u8 *aBuf = pTab->aBuffer;
+  int nBuf;
+  int rc;
+
+  nBuf = zipfileSerializeLFH(pEntry, aBuf);
+  rc = zipfileAppendData(pTab, aBuf, nBuf);
   if( rc==SQLITE_OK ){
     rc = zipfileAppendData(pTab, pData, nData);
   }
@@ -1303,8 +1310,7 @@ static int zipfileUpdate(
   int bIsDir = 0;
   u32 iCrc32 = 0;
 
-  assert( pTab->zFile );
-  assert( pTab->pWriteFd );
+  assert( (pTab->zFile==0)==(pTab->pWriteFd==0) );
 
   /* If this is a DELETE or UPDATE, find the archive entry to delete. */
   if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
@@ -1407,7 +1413,7 @@ static int zipfileUpdate(
 
     if( rc==SQLITE_OK ){
       /* Create the new CDS record. */
-      pNew = zipfileNewEntry(zPath);
+      pNew = zipfileNewEntry(zPath, pTab->zFile ? 0 : (nData+1));
       if( pNew==0 ){
         rc = SQLITE_NOMEM;
       }else{
@@ -1423,9 +1429,11 @@ static int zipfileUpdate(
         pNew->cds.iOffset = (u32)pTab->szCurrent;
         pNew->cds.nFile = nPath;
         pNew->mUnixTime = (u32)mTime;
-        rc = zipfileAppendEntry(
-            pTab, &pNew->cds, zPath, nPath, pData, nData, pNew->mUnixTime
-        );
+        if( pTab->zFile ){
+          rc = zipfileAppendEntry(pTab, pNew, pData, nData);
+        }else{
+          memcpy(pNew->aData, pData, nData);
+        }
         zipfileAddEntry(pTab, pOld, pNew);
       }
     }
@@ -1450,20 +1458,24 @@ static int zipfileUpdate(
   return rc;
 }
 
+static int zipfileSerializeEOCD(ZipfileEOCD *p, u8 *aBuf){
+  u8 *a = aBuf;
+  zipfileWrite32(a, ZIPFILE_SIGNATURE_EOCD);
+  zipfileWrite16(a, p->iDisk);
+  zipfileWrite16(a, p->iFirstDisk);
+  zipfileWrite16(a, p->nEntry);
+  zipfileWrite16(a, p->nEntryTotal);
+  zipfileWrite32(a, p->nSize);
+  zipfileWrite32(a, p->iOffset);
+  zipfileWrite16(a, 0);        /* Size of trailing comment in bytes*/
+
+  return a-aBuf;
+}
+
 static int zipfileAppendEOCD(ZipfileTab *pTab, ZipfileEOCD *p){
-  u8 *aBuf = pTab->aBuffer;
-
-  zipfileWrite32(aBuf, ZIPFILE_SIGNATURE_EOCD);
-  zipfileWrite16(aBuf, p->iDisk);
-  zipfileWrite16(aBuf, p->iFirstDisk);
-  zipfileWrite16(aBuf, p->nEntry);
-  zipfileWrite16(aBuf, p->nEntryTotal);
-  zipfileWrite32(aBuf, p->nSize);
-  zipfileWrite32(aBuf, p->iOffset);
-  zipfileWrite16(aBuf, 0);        /* Size of trailing comment in bytes*/
-
-  assert( (aBuf-pTab->aBuffer)==22 );
-  return zipfileAppendData(pTab, pTab->aBuffer, (int)(aBuf - pTab->aBuffer));
+  int nBuf = zipfileSerializeEOCD(p, pTab->aBuffer);
+  assert( nBuf==ZIPFILE_EOCD_FIXED_SZ );
+  return zipfileAppendData(pTab, pTab->aBuffer, nBuf);
 }
 
 static int zipfileBegin(sqlite3_vtab *pVtab){
@@ -1471,34 +1483,26 @@ static int zipfileBegin(sqlite3_vtab *pVtab){
   int rc = SQLITE_OK;
 
   assert( pTab->pWriteFd==0 );
+  if( pTab->zFile ){
+    /* Open a write fd on the file. Also load the entire central directory
+    ** structure into memory. During the transaction any new file data is 
+    ** appended to the archive file, but the central directory is accumulated
+    ** in main-memory until the transaction is committed.  */
+    pTab->pWriteFd = fopen(pTab->zFile, "ab+");
+    if( pTab->pWriteFd==0 ){
+      pTab->base.zErrMsg = sqlite3_mprintf(
+          "zipfile: failed to open file %s for writing", pTab->zFile
+      );
+      rc = SQLITE_ERROR;
+    }else{
+      fseek(pTab->pWriteFd, 0, SEEK_END);
+      pTab->szCurrent = pTab->szOrig = (i64)ftell(pTab->pWriteFd);
+      rc = zipfileLoadDirectory(pTab, 0, 0);
+    }
 
-  /* This table is only writable if a default archive path was specified 
-  ** as part of the CREATE VIRTUAL TABLE statement. */
-  if( pTab->zFile==0 ){
-    pTab->base.zErrMsg = sqlite3_mprintf(
-        "zipfile: writing requires a default archive"
-    );
-    return SQLITE_ERROR;
-  }
-
-  /* Open a write fd on the file. Also load the entire central directory
-  ** structure into memory. During the transaction any new file data is 
-  ** appended to the archive file, but the central directory is accumulated
-  ** in main-memory until the transaction is committed.  */
-  pTab->pWriteFd = fopen(pTab->zFile, "ab+");
-  if( pTab->pWriteFd==0 ){
-    pTab->base.zErrMsg = sqlite3_mprintf(
-        "zipfile: failed to open file %s for writing", pTab->zFile
-    );
-    rc = SQLITE_ERROR;
-  }else{
-    fseek(pTab->pWriteFd, 0, SEEK_END);
-    pTab->szCurrent = pTab->szOrig = (i64)ftell(pTab->pWriteFd);
-    rc = zipfileLoadDirectory(pTab, 0, 0);
-  }
-
-  if( rc!=SQLITE_OK ){
-    zipfileCleanupTransaction(pTab);
+    if( rc!=SQLITE_OK ){
+      zipfileCleanupTransaction(pTab);
+    }
   }
 
   return rc;
@@ -1643,6 +1647,73 @@ static void zipfileFunctionCds(
   }
 }
 
+static void zipfileFree(void *p) { sqlite3_free(p); }
+
+static void zipfileFunctionBlob(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  ZipfileCsr *pCsr;
+  ZipfileTab *pTab = (ZipfileTab*)sqlite3_user_data(context);
+  ZipfileEntry *p;
+  int nBody = 0;
+  int nCds = 0;
+  int nEocd = ZIPFILE_EOCD_FIXED_SZ;
+  ZipfileEOCD eocd;
+
+  u8 *aZip;
+  int nZip;
+
+  u8 *aBody;
+  u8 *aCds;
+
+  pCsr = zipfileFindCursor(pTab, sqlite3_value_int64(argv[0]));
+  if( pCsr->pFile || pTab->zFile ){
+    sqlite3_result_error(context, "illegal use of zipfile_blob()", -1);
+    return;
+  }
+
+  /* Figure out how large the final file will be */
+  for(p=pTab->pFirstEntry; p; p=p->pNext){
+    nBody += ZIPFILE_LFH_FIXED_SZ + p->cds.nFile + 9 + p->cds.szCompressed;
+    nCds += ZIPFILE_CDS_FIXED_SZ + p->cds.nFile + 9;
+  }
+
+  /* Allocate space to create the serialized file */
+  nZip = nBody + nCds + nEocd;
+  aZip = (u8*)sqlite3_malloc(nZip);
+  if( aZip==0 ){
+    sqlite3_result_error_nomem(context);
+    return;
+  }
+  aBody = aZip;
+  aCds = &aZip[nBody];
+
+  /* Populate the body and CDS */
+  memset(&eocd, 0, sizeof(eocd));
+  for(p=pTab->pFirstEntry; p; p=p->pNext){
+    p->cds.iOffset = (aBody - aZip);
+    aBody += zipfileSerializeLFH(p, aBody);
+    if( p->cds.szCompressed ){
+      memcpy(aBody, p->aData, p->cds.szCompressed);
+      aBody += p->cds.szCompressed;
+    }
+    aCds += zipfileSerializeCDS(p, aCds);
+    eocd.nEntry++;
+  }
+
+  /* Append the EOCD record */
+  assert( aBody==&aZip[nBody] );
+  assert( aCds==&aZip[nBody+nCds] );
+  eocd.nEntryTotal = eocd.nEntry;
+  eocd.nSize = nCds;
+  eocd.iOffset = nBody;
+  zipfileSerializeEOCD(&eocd, aCds);
+
+  sqlite3_result_blob(context, aZip, nZip, zipfileFree);
+}
+
 
 /*
 ** xFindFunction method.
@@ -1660,6 +1731,11 @@ static int zipfileFindFunction(
       *ppArg = (void*)pVtab;
       return 1;
     }
+    if( sqlite3_stricmp("zipfile_blob", zName)==0 ){
+      *pxFunc = zipfileFunctionBlob;
+      *ppArg = (void*)pVtab;
+      return 1;
+    }
   }
 
   return 0;
@@ -1693,9 +1769,8 @@ static int zipfileRegister(sqlite3 *db){
   };
 
   int rc = sqlite3_create_module(db, "zipfile"  , &zipfileModule, 0);
-  if( rc==SQLITE_OK ){
-    rc = sqlite3_overload_function(db, "zipfile_cds", -1);
-  }
+  if( rc==SQLITE_OK ) rc = sqlite3_overload_function(db, "zipfile_cds", -1);
+  if( rc==SQLITE_OK ) rc = sqlite3_overload_function(db, "zipfile_blob", -1);
   return rc;
 }
 #else         /* SQLITE_OMIT_VIRTUALTABLE */
diff --git a/manifest b/manifest
index faa8c1e356..96a7ea5fe4 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Changes\sto\savoid\sa\sharmless\sUB\swarning\sfrom\sclang.
-D 2018-01-27T14:25:27.955
+C If\sa\szipfile\svirtual\stable\sis\screated\swith\sno\sargument\s-\s"CREATE\sVIRTUAL\sTABLE\nzzz\sUSING\szipfile()"\s-\saccumulate\sdata\sin\smemory.\sSupport\s"SELECT\nzipfile_blob(z)\sFROM\szzz\sLIMIT\s1"\sto\sretrieve\sa\szip\sarchive\simage.
+D 2018-01-27T16:29:59.042
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -304,7 +304,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c 40195c1cfb43e0ebdf176d37346fea1cb18246e35fbfce477eadc2ff5f7490df
+F ext/misc/zipfile.c df57128d9ad2a1e60097d7971e787b582fb66ce0926577cb6f5978f7af210b8c
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -1603,7 +1603,7 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf
 F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc
 F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa
 F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc
-F test/zipfile.test 0834b33e000991a80d94167e84af346162d212526d8efee0b9d4ba84589fe292
+F test/zipfile.test 1b213bdc31eddd4a41042875dbdb29b6ea12b7da7a372a8eb07f61d0d76d800f
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d
 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91
@@ -1702,7 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5259d4847f2b73f26b2385f9d8cff8fe0cabc54b4deab8477c87c8d1bb5535b1
-R dc25ffafa3daf6e245ccddfac4936cfd
-U drh
-Z 83f6cfb83d43c9463ef36c14a8c84938
+P 19f5c1400054df10688ab448e7e23afef97cab4a7c7a3e411f7527509b515dd8
+R 8b2b96d9b8d1e07c4df7e0d2971e5e57
+U dan
+Z e468bfc5bd8b08bb5b9404b26ad000bc
diff --git a/manifest.uuid b/manifest.uuid
index 3f53762e60..a8aca2c7e5 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-19f5c1400054df10688ab448e7e23afef97cab4a7c7a3e411f7527509b515dd8
\ No newline at end of file
+e63185edfe0c316aa60c1fa085d032425ecc7db54536dfa5a977772eaf3c240e
\ No newline at end of file
diff --git a/test/zipfile.test b/test/zipfile.test
index 60fde2ee15..e74f81ed1d 100644
--- a/test/zipfile.test
+++ b/test/zipfile.test
@@ -32,8 +32,23 @@ proc do_zipfile_blob_test {tn file} {
   set data [read $fd]
   close $fd
 
-  set res2 [db eval { SELECT name,mode,mtime,method,quote(data) FROM zipfile($data) }]
-  uplevel [list do_test $tn [list set {} $res2] $res1]
+  set res2 [db eval { 
+    SELECT name,mode,mtime,method,quote(data) FROM zipfile($data) 
+  }]
+
+  uplevel [list do_test $tn.1 [list set {} $res2] $res1]
+
+  set T "$file.test_zip"
+  set fd [open $T w]
+  fconfigure $fd -translation binary -encoding binary
+  puts -nonewline $fd $data
+  close $fd
+
+  set res3 [
+    db eval { SELECT name,mode,mtime,method,quote(data) FROM zipfile($T) }
+  ]
+
+  uplevel [list do_test $tn.2 [list set {} $res3] $res1]
 }
 
 forcedelete test.zip
@@ -271,6 +286,27 @@ do_catchsql_test 3.2 {
   SELECT rowid FROM x1
 } {1 {no such column: rowid}}
 
+#-------------------------------------------------------------------------
+reset_db
+forcedelete test.zip
+load_static_extension db zipfile
+
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE x2 USING zipfile();
+  INSERT INTO x2(name, data) VALUES('dir1/', NULL);
+  INSERT INTO x2(name, data) VALUES('file1', '1234');
+  INSERT INTO x2(name, data) VALUES('dir1/file2', '5678');
+  SELECT name, data FROM x2
+} {
+  dir1/ {} file1 1234 dir1/file2 5678
+}
+
+do_test 4.1 {
+  set data [db one {SELECT zipfile_blob(z) FROM x2 LIMIT 1}]
+  db eval { SELECT name, data FROM zipfile($data) }
+} {dir1/ {} file1 1234 dir1/file2 5678}
+
+
 
 finish_test
 

From a07aa8d397bf1cbd490602283ea75385e1e81643 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 27 Jan 2018 18:55:18 +0000
Subject: [PATCH 397/488] Fix missing header comments and other code issues in
 zipfile.c.

FossilOrigin-Name: 6ea8ba312c38365d3e28cfb2a367d729dd2751d1d853843eea0e18126777a320
---
 ext/misc/zipfile.c | 284 +++++++++++++++++++++++++++++----------------
 manifest           |  12 +-
 manifest.uuid      |   2 +-
 3 files changed, 188 insertions(+), 110 deletions(-)

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index 4e59675c1d..efc25927d4 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -93,6 +93,9 @@ static const char ZIPFILE_SCHEMA[] =
 **
 ** ZIPFILE_SIGNATURE_LFH:
 **   First 4 bytes of a valid LFH record.
+**
+** ZIPFILE_SIGNATURE_EOCD
+**   First 4 bytes of a valid EOCD record.
 */
 #define ZIPFILE_EXTRA_TIMESTAMP   0x5455
 #define ZIPFILE_NEWENTRY_MADEBY   ((3<<8) + 30)
@@ -101,24 +104,14 @@ static const char ZIPFILE_SCHEMA[] =
 #define ZIPFILE_SIGNATURE_CDS     0x02014b50
 #define ZIPFILE_SIGNATURE_LFH     0x04034b50
 #define ZIPFILE_SIGNATURE_EOCD    0x06054b50
-#define ZIPFILE_LFH_FIXED_SZ      30
-
-#define ZIPFILE_EOCD_FIXED_SZ     22
 
 /*
-** Set the error message contained in context ctx to the results of
-** vprintf(zFmt, ...).
+** The sizes of the fixed-size part of each of the three main data 
+** structures in a zip archive.
 */
-static void zipfileCtxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){
-  char *zMsg = 0;
-  va_list ap;
-  va_start(ap, zFmt);
-  zMsg = sqlite3_vmprintf(zFmt, ap);
-  sqlite3_result_error(ctx, zMsg, -1);
-  sqlite3_free(zMsg);
-  va_end(ap);
-}
-
+#define ZIPFILE_LFH_FIXED_SZ      30
+#define ZIPFILE_EOCD_FIXED_SZ     22
+#define ZIPFILE_CDS_FIXED_SZ      46
 
 /*
 *** 4.3.16  End of central directory record:
@@ -227,13 +220,13 @@ struct ZipfileEntry {
   ZipfileCDS cds;            /* Parsed CDS record */
   u32 mUnixTime;             /* Modification time, in UNIX format */
   u8 *aExtra;                /* cds.nExtra+cds.nComment bytes of extra data */
-  i64 iDataOff;
+  i64 iDataOff;              /* Offset to data in file (if aData==0) */
   u8 *aData;                 /* cds.szCompressed bytes of compressed data */
   ZipfileEntry *pNext;       /* Next element in in-memory CDS */
 };
 
 /* 
-** Cursor type for recursively iterating through a directory structure.
+** Cursor type for zipfile tables.
 */
 typedef struct ZipfileCsr ZipfileCsr;
 struct ZipfileCsr {
@@ -247,23 +240,11 @@ struct ZipfileCsr {
   i64 iNextOff;              /* Offset of next record in central directory */
   ZipfileEOCD eocd;          /* Parse of central directory record */
 
-  ZipfileEntry *pFreeEntry;
-
+  ZipfileEntry *pFreeEntry;  /* Free this list when cursor is closed or reset */
   ZipfileEntry *pCurrent;    /* Current entry */
   ZipfileCsr *pCsrNext;      /* Next cursor on same virtual table */
 };
 
-/*
-** Possible values for ZipfileCsr.eType. Set in zipfileFilter().
-*/
-#define ZIPFILE_CSR_LIST 1        /* Cursor reads from ZipfileTab.pFirstEntry */
-#define ZIPFILE_CSR_FILE 2        /* Cursor reads from file on disk */
-
-/*
-** Values for ZipfileCsr.flags.
-*/
-#define ZIPFILE_NOOP_NEXT   0x0002     /* Next xNext() is a no-op */
-
 typedef struct ZipfileTab ZipfileTab;
 struct ZipfileTab {
   sqlite3_vtab base;         /* Base class - must be first */
@@ -281,6 +262,24 @@ struct ZipfileTab {
   i64 szOrig;                /* Size of archive at start of transaction */
 };
 
+/*
+** Set the error message contained in context ctx to the results of
+** vprintf(zFmt, ...).
+*/
+static void zipfileCtxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){
+  char *zMsg = 0;
+  va_list ap;
+  va_start(ap, zFmt);
+  zMsg = sqlite3_vmprintf(zFmt, ap);
+  sqlite3_result_error(ctx, zMsg, -1);
+  sqlite3_free(zMsg);
+  va_end(ap);
+}
+
+/*
+** If string zIn is quoted, dequote it in place. Otherwise, if the string
+** is not quoted, do nothing.
+*/
 static void zipfileDequote(char *zIn){
   char q = zIn[0];
   if( q=='"' || q=='\'' || q=='`' || q=='[' ){
@@ -340,6 +339,9 @@ static int zipfileConnect(
   return rc;
 }
 
+/*
+** Free the ZipfileEntry structure indicated by the only argument.
+*/
 static void zipfileEntryFree(ZipfileEntry *p){
   if( p ){
     sqlite3_free(p->cds.zFile);
@@ -347,6 +349,10 @@ static void zipfileEntryFree(ZipfileEntry *p){
   }
 }
 
+/*
+** Release resources that should be freed at the end of a write 
+** transaction.
+*/
 static void zipfileCleanupTransaction(ZipfileTab *pTab){
   ZipfileEntry *pEntry;
   ZipfileEntry *pNext;
@@ -446,6 +452,16 @@ static void zipfileSetErrmsg(ZipfileCsr *pCsr, const char *zFmt, ...){
   va_end(ap);
 }
 
+/*
+** Read nRead bytes of data from offset iOff of file pFile into buffer
+** aRead[]. Return SQLITE_OK if successful, or an SQLite error code
+** otherwise. 
+**
+** If an error does occur, output variable (*pzErrmsg) may be set to point
+** to an English language error message. It is the responsibility of the
+** caller to eventually free this buffer using
+** sqlite3_free().
+*/
 static int zipfileReadData(
   FILE *pFile,                    /* Read from this file */
   u8 *aRead,                      /* Read into this buffer */
@@ -479,9 +495,16 @@ static int zipfileAppendData(
   return SQLITE_OK;
 }
 
+/*
+** Read and return a 16-bit little-endian unsigned integer from buffer aBuf.
+*/
 static u16 zipfileGetU16(const u8 *aBuf){
   return (aBuf[1] << 8) + aBuf[0];
 }
+
+/*
+** Read and return a 32-bit little-endian unsigned integer from buffer aBuf.
+*/
 static u32 zipfileGetU32(const u8 *aBuf){
   return ((u32)(aBuf[3]) << 24)
        + ((u32)(aBuf[2]) << 16)
@@ -489,10 +512,17 @@ static u32 zipfileGetU32(const u8 *aBuf){
        + ((u32)(aBuf[0]) <<  0);
 }
 
+/*
+** Write a 16-bit little endiate integer into buffer aBuf.
+*/
 static void zipfilePutU16(u8 *aBuf, u16 val){
   aBuf[0] = val & 0xFF;
   aBuf[1] = (val>>8) & 0xFF;
 }
+
+/*
+** Write a 32-bit little endiate integer into buffer aBuf.
+*/
 static void zipfilePutU32(u8 *aBuf, u32 val){
   aBuf[0] = val & 0xFF;
   aBuf[1] = (val>>8) & 0xFF;
@@ -509,7 +539,6 @@ static void zipfilePutU32(u8 *aBuf, u32 val){
 /*
 ** Magic numbers used to read CDS records.
 */
-#define ZIPFILE_CDS_FIXED_SZ         46
 #define ZIPFILE_CDS_NFILE_OFF        28
 #define ZIPFILE_CDS_SZCOMPRESSED_OFF 20
 
@@ -548,6 +577,44 @@ static int zipfileReadCDS(u8 *aBuf, ZipfileCDS *pCDS){
 }
 
 /*
+** Decode the LFH record in buffer aBuf into (*pLFH). Return SQLITE_ERROR
+** if the record is not well-formed, or SQLITE_OK otherwise.
+*/
+static int zipfileReadLFH(
+  u8 *aBuffer,
+  ZipfileLFH *pLFH
+){
+  u8 *aRead = aBuffer;
+  int rc = SQLITE_OK;
+
+  u32 sig = zipfileRead32(aRead);
+  if( sig!=ZIPFILE_SIGNATURE_LFH ){
+    rc = SQLITE_ERROR;
+  }else{
+    pLFH->iVersionExtract = zipfileRead16(aRead);
+    pLFH->flags = zipfileRead16(aRead);
+    pLFH->iCompression = zipfileRead16(aRead);
+    pLFH->mTime = zipfileRead16(aRead);
+    pLFH->mDate = zipfileRead16(aRead);
+    pLFH->crc32 = zipfileRead32(aRead);
+    pLFH->szCompressed = zipfileRead32(aRead);
+    pLFH->szUncompressed = zipfileRead32(aRead);
+    pLFH->nFile = zipfileRead16(aRead);
+    pLFH->nExtra = zipfileRead16(aRead);
+  }
+  return rc;
+}
+
+
+/*
+** Buffer aExtra (size nExtra bytes) contains zip archive "extra" fields.
+** Scan through this buffer to find an "extra-timestamp" field. If one
+** exists, extract the 32-bit modification-timestamp from it and store
+** the value in output parameter *pmTime.
+**
+** Zero is returned if no extra-timestamp record could be found (and so
+** *pmTime is left unchanged), or non-zero otherwise.
+**
 ** The general format of an extra field is:
 **
 **   Header ID    2 bytes
@@ -580,6 +647,10 @@ static int zipfileScanExtra(u8 *aExtra, int nExtra, u32 *pmTime){
 }
 
 /*
+** Convert the standard MS-DOS timestamp stored in the mTime and mDate
+** fields of the CDS structure passed as the only argument to a 32-bit
+** UNIX seconds-since-the-epoch timestamp. Return the result.
+**
 ** "Standard" MS-DOS time format:
 **
 **   File modification time:
@@ -590,6 +661,8 @@ static int zipfileScanExtra(u8 *aExtra, int nExtra, u32 *pmTime){
 **     Bits 00-04: day
 **     Bits 05-08: month (1-12)
 **     Bits 09-15: years from 1980 
+**
+** https://msdn.microsoft.com/en-us/library/9kkf9tah.aspx
 */
 static time_t zipfileMtime(ZipfileCDS *pCDS){
   struct tm t;
@@ -605,42 +678,50 @@ static time_t zipfileMtime(ZipfileCDS *pCDS){
   return mktime(&t);
 }
 
-static int zipfileReadLFH(
-  u8 *aBuffer,
-  ZipfileLFH *pLFH
-){
-  u8 *aRead = aBuffer;
-  int rc = SQLITE_OK;
+/*
+** The opposite of zipfileMtime(). This function populates the mTime and
+** mDate fields of the CDS structure passed as the first argument according
+** to the UNIX timestamp value passed as the second.
+*/
+static void zipfileMtimeToDos(ZipfileCDS *pCds, u32 mUnixTime){
+  time_t t = (time_t)mUnixTime;
+  struct tm res;
 
-  u32 sig = zipfileRead32(aRead);
-  if( sig!=ZIPFILE_SIGNATURE_LFH ){
-    rc = SQLITE_ERROR;
-  }else{
-    pLFH->iVersionExtract = zipfileRead16(aRead);
-    pLFH->flags = zipfileRead16(aRead);
-    pLFH->iCompression = zipfileRead16(aRead);
-    pLFH->mTime = zipfileRead16(aRead);
-    pLFH->mDate = zipfileRead16(aRead);
-    pLFH->crc32 = zipfileRead32(aRead);
-    pLFH->szCompressed = zipfileRead32(aRead);
-    pLFH->szUncompressed = zipfileRead32(aRead);
-    pLFH->nFile = zipfileRead16(aRead);
-    pLFH->nExtra = zipfileRead16(aRead);
-  }
-  return rc;
+#if !defined(_WIN32) && !defined(WIN32)
+  localtime_r(&t, &res);
+#else
+  memcpy(&res, localtime(&t), sizeof(struct tm));
+#endif
+
+  pCds->mTime = (u16)(
+    (res.tm_sec / 2) + 
+    (res.tm_min << 5) +
+    (res.tm_hour << 11));
+
+  pCds->mDate = (u16)(
+    (res.tm_mday-1) +
+    ((res.tm_mon+1) << 5) +
+    ((res.tm_year-80) << 9));
 }
 
 /*
-** Read a Zip archive CDS header from offset iOff of file pFile. Return
-** SQLITE_OK if successful, or an SQLite error code otherwise.
+** If aBlob is not NULL, then it is a pointer to a buffer (nBlob bytes in
+** size) containing an entire zip archive image. Or, if aBlob is NULL,
+** then pFile is a file-handle open on a zip file. In either case, this
+** function creates a ZipfileEntry object based on the zip archive entry
+** for which the CDS record is at offset iOff.
+**
+** If successful, SQLITE_OK is returned and (*ppEntry) set to point to
+** the new object. Otherwise, an SQLite error code is returned and the
+** final value of (*ppEntry) undefined.
 */
 static int zipfileGetEntry(
   ZipfileTab *pTab,               /* Store any error message here */
   const u8 *aBlob,                /* Pointer to in-memory file image */
   int nBlob,                      /* Size of aBlob[] in bytes */
   FILE *pFile,                    /* If aBlob==0, read from this file */
-  i64 iOff,
-  ZipfileEntry **ppEntry
+  i64 iOff,                       /* Offset of CDS record */
+  ZipfileEntry **ppEntry          /* OUT: Pointer to new object */
 ){
   u8 *aRead;
   char **pzErr = &pTab->base.zErrMsg;
@@ -729,12 +810,6 @@ static int zipfileGetEntry(
   return rc;
 }
 
-static FILE *zipfileGetFd(ZipfileCsr *pCsr){
-  if( pCsr->pFile ) return pCsr->pFile;
-  return ((ZipfileTab*)(pCsr->base.pVtab))->pWriteFd;
-}
-
-
 /*
 ** Advance an ZipfileCsr to its next row of output.
 */
@@ -771,27 +846,17 @@ static int zipfileNext(sqlite3_vtab_cursor *cur){
   return rc;
 }
 
-static void zipfileMtimeToDos(ZipfileCDS *pCds, u32 mTime){
-  time_t t = (time_t)mTime;
-  struct tm res;
-
-#if !defined(_WIN32) && !defined(WIN32)
-  localtime_r(&t, &res);
-#else
-  memcpy(&res, localtime(&t), sizeof(struct tm));
-#endif
-
-  pCds->mTime = (u16)(
-    (res.tm_sec / 2) + 
-    (res.tm_min << 5) +
-    (res.tm_hour << 11));
-
-  pCds->mDate = (u16)(
-    (res.tm_mday-1) +
-    ((res.tm_mon+1) << 5) +
-    ((res.tm_year-80) << 9));
+static void zipfileFree(void *p) { 
+  sqlite3_free(p); 
 }
 
+/*
+** Buffer aIn (size nIn bytes) contains compressed data. Uncompressed, the
+** size is nOut bytes. This function uncompresses the data and sets the
+** return value in context pCtx to the result (a blob).
+**
+** If an error occurs, an error code is left in pCtx instead.
+*/
 static void zipfileInflate(
   sqlite3_context *pCtx,          /* Store error here, if any */
   const u8 *aIn,                  /* Compressed data */
@@ -819,7 +884,8 @@ static void zipfileInflate(
       if( err!=Z_STREAM_END ){
         zipfileCtxErrorMsg(pCtx, "inflate() failed (%d)", err);
       }else{
-        sqlite3_result_blob(pCtx, aRes, nOut, SQLITE_TRANSIENT);
+        sqlite3_result_blob(pCtx, aRes, nOut, zipfileFree);
+        aRes = 0;
       }
     }
     sqlite3_free(aRes);
@@ -827,6 +893,18 @@ static void zipfileInflate(
   }
 }
 
+/*
+** Buffer aIn (size nIn bytes) contains uncompressed data. This function
+** compresses it and sets (*ppOut) to point to a buffer containing the
+** compressed data. The caller is responsible for eventually calling
+** sqlite3_free() to release buffer (*ppOut). Before returning, (*pnOut) 
+** is set to the size of buffer (*ppOut) in bytes.
+**
+** If no error occurs, SQLITE_OK is returned. Otherwise, an SQLite error
+** code is returned and an error message left in virtual-table handle
+** pTab. The values of (*ppOut) and (*pnOut) are left unchanged in this
+** case.
+*/
 static int zipfileDeflate(
   ZipfileTab *pTab,               /* Set error message here */
   const u8 *aIn, int nIn,         /* Input */
@@ -913,12 +991,13 @@ static int zipfileColumn(
             if( aBuf==0 ){
               rc = SQLITE_NOMEM;
             }else{
-              FILE *pFile = zipfileGetFd(pCsr);
-              if( rc==SQLITE_OK ){
-                rc = zipfileReadData(pFile, aBuf, sz, pCsr->pCurrent->iDataOff,
-                    &pCsr->base.pVtab->zErrMsg
-                );
+              FILE *pFile = pCsr->pFile;
+              if( pFile==0 ){
+                pFile = ((ZipfileTab*)(pCsr->base.pVtab))->pWriteFd;
               }
+              rc = zipfileReadData(pFile, aBuf, sz, pCsr->pCurrent->iDataOff,
+                  &pCsr->base.pVtab->zErrMsg
+              );
             }
           }
           if( rc==SQLITE_OK ){
@@ -953,16 +1032,7 @@ static int zipfileColumn(
 }
 
 /*
-** Return the rowid for the current row.
-*/
-static int zipfileRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
-  assert( 0 );
-  return SQLITE_OK;
-}
-
-/*
-** Return TRUE if the cursor has been moved off of the last
-** row of output.
+** Return TRUE if the cursor is at EOF.
 */
 static int zipfileEof(sqlite3_vtab_cursor *cur){
   ZipfileCsr *pCsr = (ZipfileCsr*)cur;
@@ -970,6 +1040,14 @@ static int zipfileEof(sqlite3_vtab_cursor *cur){
 }
 
 /*
+** If aBlob is not NULL, then it points to a buffer nBlob bytes in size
+** containing an entire zip archive image. Or, if aBlob is NULL, then pFile
+** is guaranteed to be a file-handle open on a zip file.
+**
+** This function attempts to locate the EOCD record within the zip archive
+** and populate *pEOCD with the results of decoding it. SQLITE_OK is
+** returned if successful. Otherwise, an SQLite error code is returned and
+** an English language error message may be left in virtual-table pTab.
 */
 static int zipfileReadEOCD(
   ZipfileTab *pTab,               /* Return errors here */
@@ -1030,8 +1108,10 @@ static int zipfileReadEOCD(
 }
 
 /*
-** Add object pNew to the end of the linked list that begins at
-** ZipfileTab.pFirstEntry and ends with pLastEntry.
+** Add object pNew to the linked list that begins at ZipfileTab.pFirstEntry 
+** and ends with pLastEntry. If argument pBefore is NULL, then pNew is added
+** to the end of the list. Otherwise, it is added to the list immediately
+** before pBefore (which is guaranteed to be a part of said list).
 */
 static void zipfileAddEntry(
   ZipfileTab *pTab, 
@@ -1647,8 +1727,6 @@ static void zipfileFunctionCds(
   }
 }
 
-static void zipfileFree(void *p) { sqlite3_free(p); }
-
 static void zipfileFunctionBlob(
   sqlite3_context *context,
   int argc,
@@ -1758,7 +1836,7 @@ static int zipfileRegister(sqlite3 *db){
     zipfileNext,               /* xNext - advance a cursor */
     zipfileEof,                /* xEof - check for end of scan */
     zipfileColumn,             /* xColumn - read data */
-    zipfileRowid,              /* xRowid - read data */
+    0,                         /* xRowid - read data */
     zipfileUpdate,             /* xUpdate */
     zipfileBegin,              /* xBegin */
     0,                         /* xSync */
diff --git a/manifest b/manifest
index 96a7ea5fe4..e699a399ac 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C If\sa\szipfile\svirtual\stable\sis\screated\swith\sno\sargument\s-\s"CREATE\sVIRTUAL\sTABLE\nzzz\sUSING\szipfile()"\s-\saccumulate\sdata\sin\smemory.\sSupport\s"SELECT\nzipfile_blob(z)\sFROM\szzz\sLIMIT\s1"\sto\sretrieve\sa\szip\sarchive\simage.
-D 2018-01-27T16:29:59.042
+C Fix\smissing\sheader\scomments\sand\sother\scode\sissues\sin\szipfile.c.
+D 2018-01-27T18:55:18.394
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -304,7 +304,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c df57128d9ad2a1e60097d7971e787b582fb66ce0926577cb6f5978f7af210b8c
+F ext/misc/zipfile.c 398f53469ed90076dd5d69494eb7f1bd01a42d402fc5dbe92374178887244805
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -1702,7 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 19f5c1400054df10688ab448e7e23afef97cab4a7c7a3e411f7527509b515dd8
-R 8b2b96d9b8d1e07c4df7e0d2971e5e57
+P e63185edfe0c316aa60c1fa085d032425ecc7db54536dfa5a977772eaf3c240e
+R 347b5f95fee3abc6c3f7640c1a70a4d4
 U dan
-Z e468bfc5bd8b08bb5b9404b26ad000bc
+Z faf05f7eafa9394c044e435efe4a986b
diff --git a/manifest.uuid b/manifest.uuid
index a8aca2c7e5..7c9ca14be3 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e63185edfe0c316aa60c1fa085d032425ecc7db54536dfa5a977772eaf3c240e
\ No newline at end of file
+6ea8ba312c38365d3e28cfb2a367d729dd2751d1d853843eea0e18126777a320
\ No newline at end of file

From 26e731cc8862999c595e31dae1dbcb698d08ebb3 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 29 Jan 2018 16:22:39 +0000
Subject: [PATCH 398/488] Ensure the "unique-not-null" flag is set for
 automatic indexes on columns declared with "col UNIQUE NOT NULL" (where the
 NOT NULL comes after the UNIQUE).

FossilOrigin-Name: 8767f7b880f2e4112f75f0b6ef7be3f50ab1ae20e103e7d03d8bfe77e6c79438
---
 manifest          | 16 ++++++++--------
 manifest.uuid     |  2 +-
 src/build.c       | 20 ++++++++++++++++++--
 src/sqliteInt.h   |  1 +
 test/notnull.test | 45 +++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 73 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index e699a399ac..7a5772a6de 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\smissing\sheader\scomments\sand\sother\scode\sissues\sin\szipfile.c.
-D 2018-01-27T18:55:18.394
+C Ensure\sthe\s"unique-not-null"\sflag\sis\sset\sfor\sautomatic\sindexes\son\scolumns\ndeclared\swith\s"col\sUNIQUE\sNOT\sNULL"\s(where\sthe\sNOT\sNULL\scomes\safter\sthe\nUNIQUE).
+D 2018-01-29T16:22:39.280
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -432,7 +432,7 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
 F src/btree.c d711228cac336fb35fff21f3f4a0efe2ad58aa9a800dd02929cdf184be1e78a3
 F src/btree.h 0866c0a08255142ea0e754aabd211c843cab32045c978a592a43152405ed0c84
 F src/btreeInt.h 620ab4c7235f43572cf3ac2ac8723cbdf68073be4d29da24897c7b77dda5fd96
-F src/build.c 9f9647454f236cab097f266ae970f899b53c71cadab6756c47e2b2e81392c2a1
+F src/build.c 672022c06e1a5c2653f80c77a687de11f7e65ce81d20fe2825aadfa13a875c33
 F src/callback.c fe677cb5f5abb02f7a772a62a98c2f516426081df68856e8f2d5f950929b966a
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0
@@ -492,7 +492,7 @@ F src/shell.c.in 7cea439c3f7f2e4ed6eb4b3a633cd93dccb1349241400de4da0c1291285ed51
 F src/sqlite.h.in 51f9acf52c80113d793ddd38b3940ad6895d97b4752503b19291fb8fcbf54c5e
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d
-F src/sqliteInt.h 9c70315598b34810a83e4894455acb18e95cf63ce4e6cbb451ac2d17eabc2544
+F src/sqliteInt.h 119e240796d23054148005524f1d54a61e27dbef53bef3bc7ab65001405f4751
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -1100,7 +1100,7 @@ F test/normalize.test 501630ab49b0b26b65c74124bf03e3374c1b57fa97aae750f848036091
 F test/notify1.test 669b2b743618efdc18ca4b02f45423d5d2304abf
 F test/notify2.test 2ecabaa1305083856b7c39cf32816b612740c161
 F test/notify3.test 10ff25cde502e72a92053a2f215d64bece4ef934
-F test/notnull.test f8fcf58669ddba79274daa2770d61dfad8274f62
+F test/notnull.test b6999231221df3534827e45e2005dd7a815fdd5f2c2e1afb9be21ead410816f8
 F test/null.test 0dcce4f04284ec66108c503327ad6d224c0752b3
 F test/numcast.test 5d126f7f581432e86a90d1e35cac625164aec4a1
 F test/numindex1.test 20a5450d4b056e48cd5db30e659f13347a099823
@@ -1702,7 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e63185edfe0c316aa60c1fa085d032425ecc7db54536dfa5a977772eaf3c240e
-R 347b5f95fee3abc6c3f7640c1a70a4d4
+P 6ea8ba312c38365d3e28cfb2a367d729dd2751d1d853843eea0e18126777a320
+R 3ff6b96757835180119359cd3166f741
 U dan
-Z faf05f7eafa9394c044e435efe4a986b
+Z 6c5f0f4d34834e8c6ccf2da8ff3a018a
diff --git a/manifest.uuid b/manifest.uuid
index 7c9ca14be3..9eae811f9a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6ea8ba312c38365d3e28cfb2a367d729dd2751d1d853843eea0e18126777a320
\ No newline at end of file
+8767f7b880f2e4112f75f0b6ef7be3f50ab1ae20e103e7d03d8bfe77e6c79438
\ No newline at end of file
diff --git a/src/build.c b/src/build.c
index 58b39d6475..3a656def3e 100644
--- a/src/build.c
+++ b/src/build.c
@@ -1118,10 +1118,24 @@ void sqlite3AddColumn(Parse *pParse, Token *pName, Token *pType){
 */
 void sqlite3AddNotNull(Parse *pParse, int onError){
   Table *p;
+  Column *pCol;
   p = pParse->pNewTable;
   if( p==0 || NEVER(p->nCol<1) ) return;
-  p->aCol[p->nCol-1].notNull = (u8)onError;
+  pCol = &p->aCol[p->nCol-1];
+  pCol->notNull = (u8)onError;
   p->tabFlags |= TF_HasNotNull;
+
+  /* Set the uniqNotNull flag on any UNIQUE or PK indexes already created
+  ** on this column.  */
+  if( pCol->colFlags & COLFLAG_UNIQUE ){
+    Index *pIdx;
+    for(pIdx=p->pIndex; pIdx; pIdx=pIdx->pNext){
+      assert( pIdx->nKeyCol==1 && pIdx->onError!=OE_None );
+      if( pIdx->aiColumn[0]==p->nCol-1 ){
+        pIdx->uniqNotNull = 1;
+      }
+    }
+  }
 }
 
 /*
@@ -3085,7 +3099,9 @@ void sqlite3CreateIndex(
   */
   if( pList==0 ){
     Token prevCol;
-    sqlite3TokenInit(&prevCol, pTab->aCol[pTab->nCol-1].zName);
+    Column *pCol = &pTab->aCol[pTab->nCol-1];
+    pCol->colFlags |= COLFLAG_UNIQUE;
+    sqlite3TokenInit(&prevCol, pCol->zName);
     pList = sqlite3ExprListAppend(pParse, 0,
               sqlite3ExprAlloc(db, TK_ID, &prevCol, 0));
     if( pList==0 ) goto exit_create_index;
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index b7b402b8e0..f408ff62a5 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1754,6 +1754,7 @@ struct Column {
 #define COLFLAG_PRIMKEY  0x0001    /* Column is part of the primary key */
 #define COLFLAG_HIDDEN   0x0002    /* A hidden column in a virtual table */
 #define COLFLAG_HASTYPE  0x0004    /* Type name follows column name */
+#define COLFLAG_UNIQUE   0x0008    /* Column def contains "UNIQUE" or "PK" */
 
 /*
 ** A "Collating Sequence" is defined by an instance of the following
diff --git a/test/notnull.test b/test/notnull.test
index 23fd33d4ba..32d95eaf24 100644
--- a/test/notnull.test
+++ b/test/notnull.test
@@ -561,4 +561,49 @@ do_test notnull-5.5 {
   execsql { SELECT * FROM t1 }
 } {1 2}
 
+#-------------------------------------------------------------------------
+# Check that UNIQUE NOT NULL indexes are always recognized as such.
+#
+proc uses_op_next {sql} {
+  db eval "EXPLAIN $sql" a {
+    if {$a(opcode)=="Next"} { return 1 }
+  }
+  return 0
+}
+
+proc do_uses_op_next_test {tn sql res} {
+  uplevel [list do_test $tn [list uses_op_next $sql] $res]
+}
+
+reset_db
+do_execsql_test notnull-6.0 {
+  CREATE TABLE t1(a UNIQUE);
+  CREATE TABLE t2(a NOT NULL UNIQUE);
+  CREATE TABLE t3(a UNIQUE NOT NULL);
+  CREATE TABLE t4(a NOT NULL);
+  CREATE UNIQUE INDEX t4a ON t4(a);
+
+  CREATE TABLE t5(a PRIMARY KEY);
+  CREATE TABLE t6(a PRIMARY KEY NOT NULL);
+  CREATE TABLE t7(a NOT NULL PRIMARY KEY);
+  CREATE TABLE t8(a PRIMARY KEY) WITHOUT ROWID;
+
+  CREATE TABLE t9(a PRIMARY KEY UNIQUE NOT NULL);
+  CREATE TABLE t10(a UNIQUE PRIMARY KEY NOT NULL);
+}
+
+do_uses_op_next_test notnull-6.1 "SELECT * FROM t1 WHERE a IS ?" 1
+do_uses_op_next_test notnull-6.2 "SELECT * FROM t2 WHERE a IS ?" 0
+do_uses_op_next_test notnull-6.3 "SELECT * FROM t3 WHERE a IS ?" 0
+do_uses_op_next_test notnull-6.4 "SELECT * FROM t4 WHERE a IS ?" 0
+
+do_uses_op_next_test notnull-6.5 "SELECT * FROM t5 WHERE a IS ?" 1
+do_uses_op_next_test notnull-6.6 "SELECT * FROM t6 WHERE a IS ?" 0
+do_uses_op_next_test notnull-6.7 "SELECT * FROM t7 WHERE a IS ?" 0
+do_uses_op_next_test notnull-6.8 "SELECT * FROM t8 WHERE a IS ?" 0
+
+do_uses_op_next_test notnull-6.9 "SELECT * FROM t8 WHERE a IS ?" 0
+do_uses_op_next_test notnull-6.10 "SELECT * FROM t8 WHERE a IS ?" 0
+
 finish_test
+

From 8433e7166b010ebcb727e3a7d85ae3a484fdc321 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 29 Jan 2018 17:08:52 +0000
Subject: [PATCH 399/488] Update the omit-left-join optimization so that it
 works in some cases when the RHS is subject to a UNIQUE but not NOT NULL
 constraint.

FossilOrigin-Name: 88411a402b29f19d69e0d06d9099eabb98c238ddb7c07ce110236316f008753e
---
 manifest        | 17 +++++++------
 manifest.uuid   |  2 +-
 src/where.c     | 10 +++++---
 test/join2.test | 66 +++++++++++++++++++++++++++++++++++++++++++++++--
 4 files changed, 82 insertions(+), 13 deletions(-)

diff --git a/manifest b/manifest
index 7a5772a6de..7c15e392d1 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Ensure\sthe\s"unique-not-null"\sflag\sis\sset\sfor\sautomatic\sindexes\son\scolumns\ndeclared\swith\s"col\sUNIQUE\sNOT\sNULL"\s(where\sthe\sNOT\sNULL\scomes\safter\sthe\nUNIQUE).
-D 2018-01-29T16:22:39.280
+C Update\sthe\somit-left-join\soptimization\sso\sthat\sit\sworks\sin\ssome\scases\swhen\sthe\nRHS\sis\ssubject\sto\sa\sUNIQUE\sbut\snot\sNOT\sNULL\sconstraint.
+D 2018-01-29T17:08:52.190
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -570,7 +570,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 5a3f464edd64596f601683ed321d12e6fd93c5fb9afdfb3653d6ffd0fee9c48f
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f
-F src/where.c 10c7e13c909e4dc93057fae930733f5018cdf1a040f73b9e873a264edfc101e5
+F src/where.c eb37882608c30f7361cf3309b8128c3a2887ff0099886e3685ededb7a7e6b051
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
 F src/wherecode.c cb360c511f69294ddf00340d7b390e6b1d601a1cfb77b42c4d316fe2f4cd01c3
 F src/whereexpr.c 22dbfd3bf5f6051a61523dd0ebef7a944fb29ee4aa7d2a62feb8aac6ffbbc0eb
@@ -1001,7 +1001,7 @@ F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c
 F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4
 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b
 F test/join.test 442c462eea85cf065d70a663c626b780a95af6e11585d909bb63b87598afe678
-F test/join2.test 1a0c26399910b015d9f8f95b884e9a079fd2cfdccd65f7b1603846508cae0dc6
+F test/join2.test 96f56ccf0a8fe6dbd7bb2e1a90bc61540febeaf9b6fdee14506f6fc140b93185
 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0
 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916
 F test/join5.test bc98ea4b4e5003f5b1453701ebb8cd7d1c01a550
@@ -1702,7 +1702,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6ea8ba312c38365d3e28cfb2a367d729dd2751d1d853843eea0e18126777a320
-R 3ff6b96757835180119359cd3166f741
+P 8767f7b880f2e4112f75f0b6ef7be3f50ab1ae20e103e7d03d8bfe77e6c79438
+R f19810b3aa9bfaa7d3b666e31ece61ec
+T *branch * omit-left-join-fix
+T *sym-omit-left-join-fix *
+T -sym-trunk *
 U dan
-Z 6c5f0f4d34834e8c6ccf2da8ff3a018a
+Z ff1cffb196461bca97e3f8e467ebcd67
diff --git a/manifest.uuid b/manifest.uuid
index 9eae811f9a..84c8ec4673 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-8767f7b880f2e4112f75f0b6ef7be3f50ab1ae20e103e7d03d8bfe77e6c79438
\ No newline at end of file
+88411a402b29f19d69e0d06d9099eabb98c238ddb7c07ce110236316f008753e
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index 7f45182d9a..3b18fbf5ff 100644
--- a/src/where.c
+++ b/src/where.c
@@ -2481,10 +2481,13 @@ static int whereLoopAddBtreeIndex(
       if( iCol==XN_ROWID 
        || (iCol>=0 && nInMul==0 && saved_nEq==pProbe->nKeyCol-1)
       ){
-        if( iCol>=0 && pProbe->uniqNotNull==0 ){
-          pNew->wsFlags |= WHERE_UNQ_WANTED;
-        }else{
+        assert( iCol!=XN_ROWID || pProbe->uniqNotNull );
+        if( pProbe->uniqNotNull 
+         || (pProbe->nKeyCol==1 && pProbe->onError && eOp==WO_EQ) 
+        ){
           pNew->wsFlags |= WHERE_ONEROW;
+        }else{
+          pNew->wsFlags |= WHERE_UNQ_WANTED;
         }
       }
     }else if( eOp & WO_ISNULL ){
@@ -2825,6 +2828,7 @@ static int whereLoopAddBtree(
     sPk.onError = OE_Replace;
     sPk.pTable = pTab;
     sPk.szIdxRow = pTab->szTabRow;
+    sPk.uniqNotNull = 1;
     aiRowEstPk[0] = pTab->nRowLogEst;
     aiRowEstPk[1] = 0;
     pFirst = pSrc->pTab->pIndex;
diff --git a/test/join2.test b/test/join2.test
index 76006f1ecd..3dc067e508 100644
--- a/test/join2.test
+++ b/test/join2.test
@@ -155,20 +155,65 @@ do_execsql_test 4.1.4 {
   SELECT v1, v3 FROM c1 LEFT JOIN c2 LEFT JOIN c3 ON (c3.k=v1+1);
 } {2 v3 2 v3 1112 {} 1112 {}}
 
-do_eqp_test 4.2.1 {
+do_eqp_test 4.1.5 {
   SELECT v1, v3 FROM c1 LEFT JOIN c2 ON (c2.k=v1) LEFT JOIN c3 ON (c3.k=v2);
 } {
   0 0 0 {SCAN TABLE c1} 
   0 1 1 {SEARCH TABLE c2 USING INTEGER PRIMARY KEY (rowid=?)}
   0 2 2 {SEARCH TABLE c3 USING INTEGER PRIMARY KEY (rowid=?)}
 }
-do_eqp_test 4.2.2 {
+do_eqp_test 4.1.6 {
   SELECT v1, v3 FROM c1 LEFT JOIN c2 ON (c2.k=v1) LEFT JOIN c3 ON (c3.k=v1+1);
 } {
   0 0 0 {SCAN TABLE c1} 
   0 1 2 {SEARCH TABLE c3 USING INTEGER PRIMARY KEY (rowid=?)}
 }
 
+do_execsql_test 4.2.0 {
+  DROP TABLE c1;
+  DROP TABLE c2;
+  DROP TABLE c3;
+  CREATE TABLE c1(k UNIQUE, v1);
+  CREATE TABLE c2(k UNIQUE, v2);
+  CREATE TABLE c3(k UNIQUE, v3);
+
+  INSERT INTO c1 VALUES(1, 2);
+  INSERT INTO c2 VALUES(2, 3);
+  INSERT INTO c3 VALUES(3, 'v3');
+
+  INSERT INTO c1 VALUES(111, 1112);
+  INSERT INTO c2 VALUES(112, 1113);
+  INSERT INTO c3 VALUES(113, 'v1113');
+}
+do_execsql_test 4.2.1 {
+  SELECT v1, v3 FROM c1 LEFT JOIN c2 ON (c2.k=v1) LEFT JOIN c3 ON (c3.k=v2);
+} {2 v3 1112 {}}
+do_execsql_test 4.2.2 {
+  SELECT v1, v3 FROM c1 LEFT JOIN c2 ON (c2.k=v1) LEFT JOIN c3 ON (c3.k=v1+1);
+} {2 v3 1112 {}}
+
+do_execsql_test 4.2.3 {
+  SELECT DISTINCT v1, v3 FROM c1 LEFT JOIN c2 LEFT JOIN c3 ON (c3.k=v1+1);
+} {2 v3 1112 {}}
+
+do_execsql_test 4.2.4 {
+  SELECT v1, v3 FROM c1 LEFT JOIN c2 LEFT JOIN c3 ON (c3.k=v1+1);
+} {2 v3 2 v3 1112 {} 1112 {}}
+
+do_eqp_test 4.2.5 {
+  SELECT v1, v3 FROM c1 LEFT JOIN c2 ON (c2.k=v1) LEFT JOIN c3 ON (c3.k=v2);
+} {
+  0 0 0 {SCAN TABLE c1} 
+  0 1 1 {SEARCH TABLE c2 USING INDEX sqlite_autoindex_c2_1 (k=?)}
+  0 2 2 {SEARCH TABLE c3 USING INDEX sqlite_autoindex_c3_1 (k=?)}
+}
+do_eqp_test 4.2.6 {
+  SELECT v1, v3 FROM c1 LEFT JOIN c2 ON (c2.k=v1) LEFT JOIN c3 ON (c3.k=v1+1);
+} {
+  0 0 0 {SCAN TABLE c1} 
+  0 1 2 {SEARCH TABLE c3 USING INDEX sqlite_autoindex_c3_1 (k=?)}
+}
+
 # 2017-11-23 (Thanksgiving day)
 # OSSFuzz found an assertion fault in the new LEFT JOIN eliminator code.
 #
@@ -192,4 +237,21 @@ do_execsql_test 4.3.1 {
     LEFT JOIN t2 AS c ON (a.x=c.x);
 } {1 {} 2 {} 3 {} 4 {} 5 {} 6 {} 7 {} 8 {} 9 {} 10 10}
 
+do_execsql_test 5.0 {
+  CREATE TABLE s1 (a INTEGER PRIMARY KEY);
+  CREATE TABLE s2 (a INTEGER PRIMARY KEY);
+  CREATE TABLE s3 (a INTEGER);
+  CREATE UNIQUE INDEX ndx on s3(a);
+}
+do_eqp_test 5.1 {
+  SELECT s1.a FROM s1 left join s2 using (a);
+} {
+  0 0 0 {SCAN TABLE s1}
+}
+do_eqp_test 5.2 {
+  SELECT s1.a FROM s1 left join s3 using (a);
+} {
+  0 0 0 {SCAN TABLE s1}
+}
+
 finish_test

From f8c4b99a0273297cd1779304444b99b79eae6ab4 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 29 Jan 2018 18:41:07 +0000
Subject: [PATCH 400/488] Add aggregate function zipfile() to the zipfile
 extension. For composing new zip archives in memory.

FossilOrigin-Name: e364eeac76a8225146b37d801bc6cabe03e9abede5a1412ebe9d94a32d8838cc
---
 ext/misc/zipfile.c | 282 ++++++++++++++++++++++++++++++++++++++++++---
 manifest           |  14 +--
 manifest.uuid      |   2 +-
 test/zipfile.test  |  48 +++++---
 4 files changed, 302 insertions(+), 44 deletions(-)

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index efc25927d4..f325868149 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -906,9 +906,9 @@ static void zipfileInflate(
 ** case.
 */
 static int zipfileDeflate(
-  ZipfileTab *pTab,               /* Set error message here */
   const u8 *aIn, int nIn,         /* Input */
-  u8 **ppOut, int *pnOut          /* Output */
+  u8 **ppOut, int *pnOut,         /* Output */
+  char **pzErr                    /* OUT: Error message */
 ){
   int nAlloc = (int)compressBound(nIn);
   u8 *aOut;
@@ -934,7 +934,7 @@ static int zipfileDeflate(
       *pnOut = (int)str.total_out;
     }else{
       sqlite3_free(aOut);
-      pTab->base.zErrMsg = sqlite3_mprintf("zipfile: deflate() error");
+      *pzErr = sqlite3_mprintf("zipfile: deflate() error");
       rc = SQLITE_ERROR;
     }
     deflateEnd(&str);
@@ -1315,15 +1315,15 @@ static int zipfileAppendEntry(
 }
 
 static int zipfileGetMode(
-  ZipfileTab *pTab, 
   sqlite3_value *pVal, 
-  u32 defaultMode,                /* Value to use if pVal IS NULL */
-  u32 *pMode
+  int bIsDir,                     /* If true, default to directory */
+  u32 *pMode,                     /* OUT: Mode value */
+  char **pzErr                    /* OUT: Error message */
 ){
   const char *z = (const char*)sqlite3_value_text(pVal);
   u32 mode = 0;
   if( z==0 ){
-    mode = defaultMode;
+    mode = (bIsDir ? (S_IFDIR + 0755) : (S_IFREG + 0644));
   }else if( z[0]>='0' && z[0]<='9' ){
     mode = (unsigned int)sqlite3_value_int(pVal);
   }else{
@@ -1343,11 +1343,16 @@ static int zipfileGetMode(
       else if( z[i]!='-' ) goto parse_error;
     }
   }
+  if( (bIsDir == ((mode & S_IFDIR)==0)) ){
+    /* The "mode" attribute is a directory, but data has been specified.
+    ** Or vice-versa - no data but "mode" is a file or symlink.  */
+    return SQLITE_CONSTRAINT;
+  }
   *pMode = mode;
   return SQLITE_OK;
 
  parse_error:
-  pTab->base.zErrMsg = sqlite3_mprintf("zipfile: parse error in mode: %s", z);
+  *pzErr = sqlite3_mprintf("zipfile: parse error in mode: %s", z);
   return SQLITE_ERROR;
 }
 
@@ -1432,7 +1437,7 @@ static int zipfileUpdate(
         }else{
           if( bAuto || iMethod ){
             int nCmp;
-            rc = zipfileDeflate(pTab, aIn, nIn, &pFree, &nCmp);
+            rc = zipfileDeflate(aIn, nIn, &pFree, &nCmp, &pTab->base.zErrMsg);
             if( rc==SQLITE_OK ){
               if( iMethod || nCmp<nIn ){
                 iMethod = 8;
@@ -1447,14 +1452,7 @@ static int zipfileUpdate(
     }
 
     if( rc==SQLITE_OK ){
-      rc = zipfileGetMode(pTab, apVal[3], 
-          (bIsDir ? (S_IFDIR + 0755) : (S_IFREG + 0644)), &mode
-      );
-      if( rc==SQLITE_OK && (bIsDir == ((mode & S_IFDIR)==0)) ){
-        /* The "mode" attribute is a directory, but data has been specified.
-        ** Or vice-versa - no data but "mode" is a file or symlink.  */
-        rc = SQLITE_CONSTRAINT;
-      }
+      rc = zipfileGetMode(apVal[3], bIsDir, &mode, &pTab->base.zErrMsg);
     }
 
     if( rc==SQLITE_OK ){
@@ -1819,6 +1817,251 @@ static int zipfileFindFunction(
   return 0;
 }
 
+typedef struct ZipfileBuffer ZipfileBuffer;
+struct ZipfileBuffer {
+  u8 *a;                          /* Pointer to buffer */
+  int n;                          /* Size of buffer in bytes */
+  int nAlloc;                     /* Byte allocated at a[] */
+};
+
+typedef struct ZipfileCtx ZipfileCtx;
+struct ZipfileCtx {
+  int nEntry;
+  ZipfileBuffer body;
+  ZipfileBuffer cds;
+};
+
+static int zipfileBufferGrow(ZipfileBuffer *pBuf, int nByte){
+  if( pBuf->n+nByte>pBuf->nAlloc ){
+    u8 *aNew;
+    int nNew = pBuf->n ? pBuf->n*2 : 512;
+    int nReq = pBuf->n + nByte;
+
+    while( nNew<nReq ) nNew = nNew*2;
+    aNew = sqlite3_realloc(pBuf->a, nNew);
+    if( aNew==0 ) return SQLITE_NOMEM;
+    pBuf->a = aNew;
+    pBuf->nAlloc = nNew;
+  }
+  return SQLITE_OK;
+}
+
+/*
+** xStep() callback for the zipfile() aggregate. This can be called in
+** any of the following ways:
+**
+**   SELECT zipfile(name,data) ...
+**   SELECT zipfile(name,mode,mtime,data) ...
+**   SELECT zipfile(name,mode,mtime,data,method) ...
+*/
+void zipfileStep(sqlite3_context *pCtx, int nVal, sqlite3_value **apVal){
+  ZipfileCtx *p;                  /* Aggregate function context */
+  ZipfileEntry e;                 /* New entry to add to zip archive */
+
+  sqlite3_value *pName = 0;
+  sqlite3_value *pMode = 0;
+  sqlite3_value *pMtime = 0;
+  sqlite3_value *pData = 0;
+  sqlite3_value *pMethod = 0;
+
+  int bIsDir = 0;
+  u32 mode;
+  int rc = SQLITE_OK;
+  char *zErr = 0;
+
+  int iMethod = -1;               /* Compression method to use (0 or 8) */
+
+  const u8 *aData = 0;            /* Possibly compressed data for new entry */
+  int nData = 0;                  /* Size of aData[] in bytes */
+  int szUncompressed = 0;         /* Size of data before compression */
+  u8 *aFree = 0;                  /* Free this before returning */
+  u32 iCrc32 = 0;                 /* crc32 of uncompressed data */
+
+  char *zName = 0;                /* Path (name) of new entry */
+  int nName = 0;                  /* Size of zName in bytes */
+  char *zFree = 0;                /* Free this before returning */
+  int nByte;
+
+  memset(&e, 0, sizeof(e));
+  p = (ZipfileCtx*)sqlite3_aggregate_context(pCtx, sizeof(ZipfileCtx));
+  if( p==0 ) return;
+
+  /* Martial the arguments into stack variables */
+  if( nVal!=2 && nVal!=4 && nVal!=5 ){
+    zErr = sqlite3_mprintf("wrong number of arguments to function zipfile()");
+    rc = SQLITE_ERROR;
+    goto zipfile_step_out;
+  }
+  pName = apVal[0];
+  if( nVal==2 ){
+    pData = apVal[1];
+  }else{
+    pMode = apVal[1];
+    pMtime = apVal[2];
+    pData = apVal[3];
+    if( nVal==5 ){
+      pMethod = apVal[4];
+    }
+  }
+
+  /* Check that the 'name' parameter looks ok. */
+  zName = (char*)sqlite3_value_text(pName);
+  nName = sqlite3_value_bytes(pName);
+  if( zName==0 ){
+    zErr = sqlite3_mprintf("first argument to zipfile() must be non-NULL");
+    rc = SQLITE_ERROR;
+    goto zipfile_step_out;
+  }
+
+  /* Inspect the 'method' parameter. This must be either 0 (store), 8 (use
+  ** deflate compression) or NULL (choose automatically).  */
+  if( pMethod && SQLITE_NULL!=sqlite3_value_type(pMethod) ){
+    iMethod = sqlite3_value_int64(pMethod);
+    if( iMethod!=0 && iMethod!=8 ){
+      zErr = sqlite3_mprintf("illegal method value: %d", iMethod);
+      rc = SQLITE_ERROR;
+      goto zipfile_step_out;
+    }
+  }
+
+  /* Now inspect the data. If this is NULL, then the new entry must be a
+  ** directory.  Otherwise, figure out whether or not the data should
+  ** be deflated or simply stored in the zip archive. */
+  if( sqlite3_value_type(pData)==SQLITE_NULL ){
+    bIsDir = 1;
+    iMethod = 0;
+  }else{
+    aData = sqlite3_value_blob(pData);
+    szUncompressed = nData = sqlite3_value_bytes(pData);
+    iCrc32 = crc32(0, aData, nData);
+    if( iMethod<0 || iMethod==8 ){
+      int nOut = 0;
+      rc = zipfileDeflate(aData, nData, &aFree, &nOut, &zErr);
+      if( rc!=SQLITE_OK ){
+        goto zipfile_step_out;
+      }
+      if( iMethod==8 || nOut<nData ){
+        aData = aFree;
+        nData = nOut;
+        iMethod = 8;
+      }else{
+        iMethod = 0;
+      }
+    }
+  }
+
+  /* Decode the "mode" argument. */
+  rc = zipfileGetMode(pMode, bIsDir, &mode, &zErr);
+  if( rc ) goto zipfile_step_out;
+
+  /* Decode the "mtime" argument. */
+  if( pMtime==0 || sqlite3_value_type(pMtime)==SQLITE_NULL ){
+    e.mUnixTime = (u32)time(0);
+  }else{
+    e.mUnixTime = (u32)sqlite3_value_int64(pMtime);
+  }
+
+  /* If this is a directory entry, ensure that there is exactly one '/'
+  ** at the end of the path. Or, if this is not a directory and the path
+  ** ends in '/' it is an error. */
+  if( bIsDir==0 ){
+    if( zName[nName-1]=='/' ){
+      zErr = sqlite3_mprintf("non-directory name must not end with /");
+      rc = SQLITE_ERROR;
+      goto zipfile_step_out;
+    }
+  }else{
+    if( zName[nName-1]!='/' ){
+      zName = zFree = sqlite3_mprintf("%s/", zName);
+      nName++;
+      if( zName==0 ){
+        rc = SQLITE_NOMEM;
+        goto zipfile_step_out;
+      }
+    }else{
+      while( nName>1 && zName[nName-2]=='/' ) nName--;
+    }
+  }
+
+  /* Assemble the ZipfileEntry object for the new zip archive entry */
+  e.cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY;
+  e.cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED;
+  e.cds.flags = ZIPFILE_NEWENTRY_FLAGS;
+  e.cds.iCompression = iMethod;
+  zipfileMtimeToDos(&e.cds, (u32)e.mUnixTime);
+  e.cds.crc32 = iCrc32;
+  e.cds.szCompressed = nData;
+  e.cds.szUncompressed = szUncompressed;
+  e.cds.iExternalAttr = (mode<<16);
+  e.cds.iOffset = p->body.n;
+  e.cds.nFile = nName;
+  e.cds.zFile = zName;
+
+  /* Append the LFH to the body of the new archive */
+  nByte = ZIPFILE_LFH_FIXED_SZ + e.cds.nFile + 9;
+  if( (rc = zipfileBufferGrow(&p->body, nByte)) ) goto zipfile_step_out;
+  p->body.n += zipfileSerializeLFH(&e, &p->body.a[p->body.n]);
+
+  /* Append the data to the body of the new archive */
+  if( (rc = zipfileBufferGrow(&p->body, nData)) ) goto zipfile_step_out;
+  memcpy(&p->body.a[p->body.n], aData, nData);
+  p->body.n += nData;
+
+  /* Append the CDS record to the directory of the new archive */
+  nByte = ZIPFILE_CDS_FIXED_SZ + e.cds.nFile + 9;
+  if( (rc = zipfileBufferGrow(&p->cds, nByte)) ) goto zipfile_step_out;
+  p->cds.n += zipfileSerializeCDS(&e, &p->cds.a[p->cds.n]);
+
+  /* Increment the count of entries in the archive */
+  p->nEntry++;
+
+ zipfile_step_out:
+  sqlite3_free(aFree);
+  sqlite3_free(zFree);
+  if( rc ){
+    if( zErr ){
+      sqlite3_result_error(pCtx, zErr, -1);
+    }else{
+      sqlite3_result_error_code(pCtx, rc);
+    }
+  }
+  sqlite3_free(zErr);
+}
+
+/*
+** xFinalize() callback for zipfile aggregate function.
+*/
+void zipfileFinal(sqlite3_context *pCtx){
+  ZipfileCtx *p;
+  ZipfileEOCD eocd;
+  int nZip;
+  u8 *aZip;
+
+  p = (ZipfileCtx*)sqlite3_aggregate_context(pCtx, sizeof(ZipfileCtx));
+  if( p==0 || p->nEntry==0 ) return;
+
+  memset(&eocd, 0, sizeof(eocd));
+  eocd.nEntry = p->nEntry;
+  eocd.nEntryTotal = p->nEntry;
+  eocd.nSize = p->cds.n;
+  eocd.iOffset = p->body.n;
+
+  nZip = p->body.n + p->cds.n + ZIPFILE_EOCD_FIXED_SZ;
+  aZip = (u8*)sqlite3_malloc(nZip);
+  if( aZip==0 ){
+    sqlite3_result_error_nomem(pCtx);
+  }else{
+    memcpy(aZip, p->body.a, p->body.n);
+    memcpy(&aZip[p->body.n], p->cds.a, p->cds.n);
+    zipfileSerializeEOCD(&eocd, &aZip[p->body.n + p->cds.n]);
+    sqlite3_result_blob(pCtx, aZip, nZip, zipfileFree);
+  }
+
+  sqlite3_free(p->body.a);
+  sqlite3_free(p->cds.a);
+}
+
+
 /*
 ** Register the "zipfile" virtual table.
 */
@@ -1849,6 +2092,11 @@ static int zipfileRegister(sqlite3 *db){
   int rc = sqlite3_create_module(db, "zipfile"  , &zipfileModule, 0);
   if( rc==SQLITE_OK ) rc = sqlite3_overload_function(db, "zipfile_cds", -1);
   if( rc==SQLITE_OK ) rc = sqlite3_overload_function(db, "zipfile_blob", -1);
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_create_function(db, "zipfile", -1, SQLITE_UTF8, 0, 0, 
+        zipfileStep, zipfileFinal
+    );
+  }
   return rc;
 }
 #else         /* SQLITE_OMIT_VIRTUALTABLE */
diff --git a/manifest b/manifest
index 7a5772a6de..09731fddf8 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Ensure\sthe\s"unique-not-null"\sflag\sis\sset\sfor\sautomatic\sindexes\son\scolumns\ndeclared\swith\s"col\sUNIQUE\sNOT\sNULL"\s(where\sthe\sNOT\sNULL\scomes\safter\sthe\nUNIQUE).
-D 2018-01-29T16:22:39.280
+C Add\saggregate\sfunction\szipfile()\sto\sthe\szipfile\sextension.\sFor\scomposing\snew\nzip\sarchives\sin\smemory.
+D 2018-01-29T18:41:07.175
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -304,7 +304,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c 398f53469ed90076dd5d69494eb7f1bd01a42d402fc5dbe92374178887244805
+F ext/misc/zipfile.c 73d932caf69fea8ba42b58ce2cea0fbea2c126a355498fd215008f61651ec661
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -1603,7 +1603,7 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf
 F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc
 F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa
 F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc
-F test/zipfile.test 1b213bdc31eddd4a41042875dbdb29b6ea12b7da7a372a8eb07f61d0d76d800f
+F test/zipfile.test 12199631759785b0185ceb1ff3cffdcc91985ef61475ab1c69139686b09a87c8
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d
 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91
@@ -1702,7 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6ea8ba312c38365d3e28cfb2a367d729dd2751d1d853843eea0e18126777a320
-R 3ff6b96757835180119359cd3166f741
+P 8767f7b880f2e4112f75f0b6ef7be3f50ab1ae20e103e7d03d8bfe77e6c79438
+R 5f1a0bf3bc3038aabd0aee2df4b1bd63
 U dan
-Z 6c5f0f4d34834e8c6ccf2da8ff3a018a
+Z eba1cb16b20a1e223d496488c50326dc
diff --git a/manifest.uuid b/manifest.uuid
index 9eae811f9a..78aea7435e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-8767f7b880f2e4112f75f0b6ef7be3f50ab1ae20e103e7d03d8bfe77e6c79438
\ No newline at end of file
+e364eeac76a8225146b37d801bc6cabe03e9abede5a1412ebe9d94a32d8838cc
\ No newline at end of file
diff --git a/test/zipfile.test b/test/zipfile.test
index e74f81ed1d..04a475244b 100644
--- a/test/zipfile.test
+++ b/test/zipfile.test
@@ -22,33 +22,43 @@ if {[catch {load_static_extension db zipfile} error]} {
   finish_test; return
 }
 
-proc do_zipfile_blob_test {tn file} {
-  set res1 [
-    db eval { SELECT name,mode,mtime,method,quote(data) FROM zipfile($file) }
-  ]
-
-  set fd [open $file]
+proc readfile {f} {
+  set fd [open $f]
   fconfigure $fd -translation binary -encoding binary
   set data [read $fd]
   close $fd
+  set data
+}
 
-  set res2 [db eval { 
-    SELECT name,mode,mtime,method,quote(data) FROM zipfile($data) 
-  }]
+# Argument $file is the name of a zip archive on disk. This function
+# executes test cases to check that the results of each of the following 
+# are the same:
+#
+#         SELECT * FROM zipfile($file)
+#         SELECT * FROM zipfile( readfile($file) )
+#         SELECT * FROM zipfile( 
+#           (SELECT zipfile(name,mode,mtime,data,method) FROM zipfile($file))
+#         )
+#
+proc do_zipfile_blob_test {tn file} {
 
-  uplevel [list do_test $tn.1 [list set {} $res2] $res1]
+  db func r readfile
+  set q1 {SELECT name,mode,mtime,method,quote(data) FROM zipfile($file)}
+  set q2 {SELECT name,mode,mtime,method,quote(data) FROM zipfile( r($file) )}
+  set q3 {SELECT name,mode,mtime,method,quote(data) FROM zipfile(
+    ( SELECT zipfile(name,mode,mtime,data,method) FROM zipfile($file) )
+  )}
 
-  set T "$file.test_zip"
-  set fd [open $T w]
-  fconfigure $fd -translation binary -encoding binary
-  puts -nonewline $fd $data
-  close $fd
 
-  set res3 [
-    db eval { SELECT name,mode,mtime,method,quote(data) FROM zipfile($T) }
-  ]
+  set r1 [db eval $q1]
+  set r2 [db eval $q2]
+  set r3 [db eval $q3]
+  #puts $r1
+  #puts $r2
+  #puts $r3
 
-  uplevel [list do_test $tn.2 [list set {} $res3] $res1]
+  uplevel [list do_test $tn.1 [list set {} $r2] $r1]
+  uplevel [list do_test $tn.1 [list set {} $r3] $r1]
 }
 
 forcedelete test.zip

From ea4125bbefbed96310be4f3d0a742b257b812d8d Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 29 Jan 2018 19:47:32 +0000
Subject: [PATCH 401/488] Add unix-only tests to check that the "unzip" program
 can unpack archives generated by the zipfile extension.

FossilOrigin-Name: 438c5c5237a801ae78809bf324bb9251fb50250addfc1f8e36659442b0e26ab6
---
 ext/misc/fileio.c |  2 +-
 manifest          | 14 ++++-----
 manifest.uuid     |  2 +-
 test/zipfile.test | 75 ++++++++++++++++++++++++++++++++++++++++++-----
 4 files changed, 77 insertions(+), 16 deletions(-)

diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c
index e51b3e620b..7226ff18b3 100644
--- a/ext/misc/fileio.c
+++ b/ext/misc/fileio.c
@@ -491,6 +491,7 @@ static void fsdirResetCursor(fsdir_cursor *pCur){
     sqlite3_free(pLvl->zDir);
   }
   sqlite3_free(pCur->zPath);
+  sqlite3_free(pCur->aLvl);
   pCur->aLvl = 0;
   pCur->zPath = 0;
   pCur->zBase = 0;
@@ -506,7 +507,6 @@ static int fsdirClose(sqlite3_vtab_cursor *cur){
   fsdir_cursor *pCur = (fsdir_cursor*)cur;
 
   fsdirResetCursor(pCur);
-  sqlite3_free(pCur->aLvl);
   sqlite3_free(pCur);
   return SQLITE_OK;
 }
diff --git a/manifest b/manifest
index 09731fddf8..0b73afb532 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\saggregate\sfunction\szipfile()\sto\sthe\szipfile\sextension.\sFor\scomposing\snew\nzip\sarchives\sin\smemory.
-D 2018-01-29T18:41:07.175
+C Add\sunix-only\stests\sto\scheck\sthat\sthe\s"unzip"\sprogram\scan\sunpack\sarchives\ngenerated\sby\sthe\szipfile\sextension.
+D 2018-01-29T19:47:32.262
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -277,7 +277,7 @@ F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f0
 F ext/misc/csv.c 1a009b93650732e22334edc92459c4630b9fa703397cbb3c8ca279921a36ca11
 F ext/misc/dbdump.c 3509fa6b8932d04e932d6b6b827b6a82ca362781b8e8f3c77336f416793e215e
 F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2
-F ext/misc/fileio.c bd2f717be63a9ae9ff85977a756c43a96d47a7763d98ae913636c64b714c232b
+F ext/misc/fileio.c 4cad3a78bfbbea9b1414b4405fd704ee57fbe2621254b011888dae7b0ba8c513
 F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25
 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c
 F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281984
@@ -1603,7 +1603,7 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf
 F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc
 F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa
 F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc
-F test/zipfile.test 12199631759785b0185ceb1ff3cffdcc91985ef61475ab1c69139686b09a87c8
+F test/zipfile.test c40ae3a5d3fd0a31a8c6bdae1dbef55dd7140acb0d3b316c8edb744085ea6134
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d
 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91
@@ -1702,7 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 8767f7b880f2e4112f75f0b6ef7be3f50ab1ae20e103e7d03d8bfe77e6c79438
-R 5f1a0bf3bc3038aabd0aee2df4b1bd63
+P e364eeac76a8225146b37d801bc6cabe03e9abede5a1412ebe9d94a32d8838cc
+R 78b9ab6ccbbf51cef503aff35913fad6
 U dan
-Z eba1cb16b20a1e223d496488c50326dc
+Z 634d51e6967cb83c15fce21d55a23989
diff --git a/manifest.uuid b/manifest.uuid
index 78aea7435e..63ba75784e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e364eeac76a8225146b37d801bc6cabe03e9abede5a1412ebe9d94a32d8838cc
\ No newline at end of file
+438c5c5237a801ae78809bf324bb9251fb50250addfc1f8e36659442b0e26ab6
\ No newline at end of file
diff --git a/test/zipfile.test b/test/zipfile.test
index 04a475244b..d43e685e15 100644
--- a/test/zipfile.test
+++ b/test/zipfile.test
@@ -30,6 +30,24 @@ proc readfile {f} {
   set data
 }
 
+if {$::tcl_platform(platform)=="unix" && [catch {exec unzip}]==0} {
+  set ::UNZIP 1
+  load_static_extension db fileio
+  proc do_unzip {file} {
+    forcedelete test_unzip
+    file mkdir test_unzip
+    exec unzip -d test_unzip $file
+  
+    set res [db eval { 
+      SELECT replace(name,'test_unzip/',''),mode,mtime,data 
+      FROM fsdir('test_unzip') 
+      WHERE name!='test_unzip'
+      ORDER BY name
+    }]
+    set res
+  }
+}
+
 # Argument $file is the name of a zip archive on disk. This function
 # executes test cases to check that the results of each of the following 
 # are the same:
@@ -58,7 +76,49 @@ proc do_zipfile_blob_test {tn file} {
   #puts $r3
 
   uplevel [list do_test $tn.1 [list set {} $r2] $r1]
-  uplevel [list do_test $tn.1 [list set {} $r3] $r1]
+  uplevel [list do_test $tn.2 [list set {} $r3] $r1]
+}
+
+# Argument $file is a zip file on disk. This command runs tests to:
+#
+#   1. Unpack the archive with unix command [unzip] and compare the 
+#      results to reading the same archive using the zipfile() table
+#      valued function.
+#
+#   2. Creates a new archive with the same contents using the zipfile()
+#      aggregate function as follows:
+#
+#      SELECT writefile('test_unzip.zip',
+#          ( SELECT zipfile(name,mode,mtime,data,method) FROM zipfile($file) )
+#      );
+#
+#      Then tests that unpacking the new archive using [unzip] produces
+#      the same results as in (1).
+#
+proc do_unzip_test {tn file} {
+  if {[info vars ::UNZIP]==""} { return }
+  db func sss strip_slash
+
+  db eval {
+    SELECT writefile('test_unzip.zip',
+        ( SELECT zipfile(name,mode,mtime,data,method) FROM zipfile($file) )
+    );
+  }
+
+  set r1 [db eval { 
+    SELECT sss(name),mode,mtime,data FROM zipfile($file) ORDER BY name
+  }]
+  set r2 [do_unzip $file]
+  set r3 [do_unzip test_unzip.zip]
+
+  uplevel [list do_test $tn.1 [list set {} $r2] $r1]
+  uplevel [list do_test $tn.2 [list set {} $r3] $r1]
+}
+proc strip_slash {in} { regsub {/$} $in {} }
+
+proc do_zip_tests {tn file} {
+  uplevel do_zipfile_blob_test $tn.1 $file
+  uplevel do_unzip_test $tn.2 $file
 }
 
 forcedelete test.zip
@@ -99,14 +159,14 @@ do_execsql_test 1.2 {
   f.txt 1000000000 abcde 
   g.txt 1000000002 12345
 }
-do_zipfile_blob_test 1.2.1 test.zip
+do_zip_tests 1.2a test.zip
 
 do_execsql_test 1.3 {
   INSERT INTO zz(name, mode, mtime, data) VALUES('h.txt', 
     '-rw-r--r--', 1000000004, 'aaaaaaaaaabbbbbbbbbb'
   );
 }
-do_zipfile_blob_test 1.3.1 test.zip
+do_zip_tests 1.3a test.zip
 
 do_execsql_test 1.4 {
   SELECT name, mtime, data, method FROM zipfile('test.zip');
@@ -153,7 +213,7 @@ do_execsql_test 1.6.1 {
   h.txt 33188 1000000004 aaaaaaaaaabbbbbbbbbb 8
   i.txt 33188 1000000006 zxcvb 0
 }
-do_zipfile_blob_test 1.6.1a test.zip
+do_zip_tests 1.6.1a test.zip
 
 do_execsql_test 1.6.2 {
   UPDATE zz SET mtime=4 WHERE name='i.txt';
@@ -172,7 +232,7 @@ do_execsql_test 1.6.3 {
   h.txt 33189 1000000004 aaaaaaaaaabbbbbbbbbb 8
   i.txt 33188 4 zxcvb 0
 }
-do_zipfile_blob_test 1.6.3a test.zip
+do_zip_tests 1.6.3a test.zip
 
 do_execsql_test 1.6.4 {
   UPDATE zz SET name = 'blue.txt' WHERE name='f.txt';
@@ -182,7 +242,7 @@ do_execsql_test 1.6.4 {
   h.txt 33189 1000000004 aaaaaaaaaabbbbbbbbbb 8
   i.txt 33188 4 zxcvb 0
 }
-do_zipfile_blob_test 1.6.4a test.zip
+do_zip_tests 1.6.4a test.zip
 
 do_execsql_test 1.6.5 {
   UPDATE zz SET data = 'edcba' WHERE name='blue.txt';
@@ -217,6 +277,7 @@ do_execsql_test 1.6.8 {
 db close
 forcedelete test.zip
 reset_db
+load_static_extension db fileio
 load_static_extension db zipfile
 do_execsql_test 2.1 {
   CREATE VIRTUAL TABLE zzz USING zipfile('test.zip');
@@ -243,7 +304,7 @@ do_execsql_test 2.4 {
   dirname2/ 16877 {}
   dirname2/file1.txt 33188 abcdefghijklmnop
 }
-do_zipfile_blob_test 2.4.1 test.zip
+do_zip_tests 2.4a test.zip
 
 # If on unix, check that the [unzip] utility can unpack our archive.
 #

From fdcd9d4e6c4a08855660df26b84139105f863df3 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 30 Jan 2018 14:07:55 +0000
Subject: [PATCH 402/488] Prevent users from creating zipfile() virtual tables
 without an argument.

FossilOrigin-Name: 81fdbe0cc5a360f818078d47a5888d0a29d555927da279a9a0213702d74ef09a
---
 ext/misc/zipfile.c | 87 ++++++++--------------------------------------
 manifest           | 14 ++++----
 manifest.uuid      |  2 +-
 test/zipfile.test  | 26 +++++---------
 4 files changed, 31 insertions(+), 98 deletions(-)

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index f325868149..247d0b7355 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -318,6 +318,21 @@ static int zipfileConnect(
   ZipfileTab *pNew = 0;
   int rc;
 
+  /* If the table name is not "zipfile", require that the argument be
+  ** specified. This stops zipfile tables from being created as:
+  **
+  **   CREATE VIRTUAL TABLE zzz USING zipfile();
+  **
+  ** It does not prevent:
+  **
+  **   CREATE VIRTUAL TABLE zipfile USING zipfile();
+  */
+  assert( 0==sqlite3_stricmp(argv[0], "zipfile") );
+  if( (0!=sqlite3_stricmp(argv[2], "zipfile") && argc<4) || argc>4 ){
+    *pzErr = sqlite3_mprintf("zipfile constructor requires one argument");
+    return SQLITE_ERROR;
+  }
+
   if( argc>3 ){
     zFile = argv[3];
     nFile = (int)strlen(zFile)+1;
@@ -1725,72 +1740,6 @@ static void zipfileFunctionCds(
   }
 }
 
-static void zipfileFunctionBlob(
-  sqlite3_context *context,
-  int argc,
-  sqlite3_value **argv
-){
-  ZipfileCsr *pCsr;
-  ZipfileTab *pTab = (ZipfileTab*)sqlite3_user_data(context);
-  ZipfileEntry *p;
-  int nBody = 0;
-  int nCds = 0;
-  int nEocd = ZIPFILE_EOCD_FIXED_SZ;
-  ZipfileEOCD eocd;
-
-  u8 *aZip;
-  int nZip;
-
-  u8 *aBody;
-  u8 *aCds;
-
-  pCsr = zipfileFindCursor(pTab, sqlite3_value_int64(argv[0]));
-  if( pCsr->pFile || pTab->zFile ){
-    sqlite3_result_error(context, "illegal use of zipfile_blob()", -1);
-    return;
-  }
-
-  /* Figure out how large the final file will be */
-  for(p=pTab->pFirstEntry; p; p=p->pNext){
-    nBody += ZIPFILE_LFH_FIXED_SZ + p->cds.nFile + 9 + p->cds.szCompressed;
-    nCds += ZIPFILE_CDS_FIXED_SZ + p->cds.nFile + 9;
-  }
-
-  /* Allocate space to create the serialized file */
-  nZip = nBody + nCds + nEocd;
-  aZip = (u8*)sqlite3_malloc(nZip);
-  if( aZip==0 ){
-    sqlite3_result_error_nomem(context);
-    return;
-  }
-  aBody = aZip;
-  aCds = &aZip[nBody];
-
-  /* Populate the body and CDS */
-  memset(&eocd, 0, sizeof(eocd));
-  for(p=pTab->pFirstEntry; p; p=p->pNext){
-    p->cds.iOffset = (aBody - aZip);
-    aBody += zipfileSerializeLFH(p, aBody);
-    if( p->cds.szCompressed ){
-      memcpy(aBody, p->aData, p->cds.szCompressed);
-      aBody += p->cds.szCompressed;
-    }
-    aCds += zipfileSerializeCDS(p, aCds);
-    eocd.nEntry++;
-  }
-
-  /* Append the EOCD record */
-  assert( aBody==&aZip[nBody] );
-  assert( aCds==&aZip[nBody+nCds] );
-  eocd.nEntryTotal = eocd.nEntry;
-  eocd.nSize = nCds;
-  eocd.iOffset = nBody;
-  zipfileSerializeEOCD(&eocd, aCds);
-
-  sqlite3_result_blob(context, aZip, nZip, zipfileFree);
-}
-
-
 /*
 ** xFindFunction method.
 */
@@ -1807,11 +1756,6 @@ static int zipfileFindFunction(
       *ppArg = (void*)pVtab;
       return 1;
     }
-    if( sqlite3_stricmp("zipfile_blob", zName)==0 ){
-      *pxFunc = zipfileFunctionBlob;
-      *ppArg = (void*)pVtab;
-      return 1;
-    }
   }
 
   return 0;
@@ -2091,7 +2035,6 @@ static int zipfileRegister(sqlite3 *db){
 
   int rc = sqlite3_create_module(db, "zipfile"  , &zipfileModule, 0);
   if( rc==SQLITE_OK ) rc = sqlite3_overload_function(db, "zipfile_cds", -1);
-  if( rc==SQLITE_OK ) rc = sqlite3_overload_function(db, "zipfile_blob", -1);
   if( rc==SQLITE_OK ){
     rc = sqlite3_create_function(db, "zipfile", -1, SQLITE_UTF8, 0, 0, 
         zipfileStep, zipfileFinal
diff --git a/manifest b/manifest
index 0b73afb532..f52d1dd656 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sunix-only\stests\sto\scheck\sthat\sthe\s"unzip"\sprogram\scan\sunpack\sarchives\ngenerated\sby\sthe\szipfile\sextension.
-D 2018-01-29T19:47:32.262
+C Prevent\susers\sfrom\screating\szipfile()\svirtual\stables\swithout\san\sargument.
+D 2018-01-30T14:07:55.726
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -304,7 +304,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c 73d932caf69fea8ba42b58ce2cea0fbea2c126a355498fd215008f61651ec661
+F ext/misc/zipfile.c ead25f062cee790b7c764ce8d2c6ad32a7ac82fc31ea80f69be99948f96f2d19
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -1603,7 +1603,7 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf
 F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc
 F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa
 F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc
-F test/zipfile.test c40ae3a5d3fd0a31a8c6bdae1dbef55dd7140acb0d3b316c8edb744085ea6134
+F test/zipfile.test 368a5a0c97be0caaf8c3efa8293bfe18436d546805678fa00b6aa81bc98727ec
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d
 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91
@@ -1702,7 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e364eeac76a8225146b37d801bc6cabe03e9abede5a1412ebe9d94a32d8838cc
-R 78b9ab6ccbbf51cef503aff35913fad6
+P 438c5c5237a801ae78809bf324bb9251fb50250addfc1f8e36659442b0e26ab6
+R 0f8eb13d715c2195926ed8684b288375
 U dan
-Z 634d51e6967cb83c15fce21d55a23989
+Z 6708ef6af1eb582ebfe61496df2754b7
diff --git a/manifest.uuid b/manifest.uuid
index 63ba75784e..2d66824214 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-438c5c5237a801ae78809bf324bb9251fb50250addfc1f8e36659442b0e26ab6
\ No newline at end of file
+81fdbe0cc5a360f818078d47a5888d0a29d555927da279a9a0213702d74ef09a
\ No newline at end of file
diff --git a/test/zipfile.test b/test/zipfile.test
index d43e685e15..76e7d924df 100644
--- a/test/zipfile.test
+++ b/test/zipfile.test
@@ -358,24 +358,14 @@ do_catchsql_test 3.2 {
 } {1 {no such column: rowid}}
 
 #-------------------------------------------------------------------------
-reset_db
-forcedelete test.zip
-load_static_extension db zipfile
-
-do_execsql_test 4.0 {
-  CREATE VIRTUAL TABLE x2 USING zipfile();
-  INSERT INTO x2(name, data) VALUES('dir1/', NULL);
-  INSERT INTO x2(name, data) VALUES('file1', '1234');
-  INSERT INTO x2(name, data) VALUES('dir1/file2', '5678');
-  SELECT name, data FROM x2
-} {
-  dir1/ {} file1 1234 dir1/file2 5678
-}
-
-do_test 4.1 {
-  set data [db one {SELECT zipfile_blob(z) FROM x2 LIMIT 1}]
-  db eval { SELECT name, data FROM zipfile($data) }
-} {dir1/ {} file1 1234 dir1/file2 5678}
+# Test some error conditions.
+#
+do_catchsql_test 4.1 {
+  CREATE VIRTUAL TABLE yyy USING zipfile();
+} {1 {zipfile constructor requires one argument}}
+do_catchsql_test 4.2 {
+  CREATE VIRTUAL TABLE yyy USING zipfile('test.zip', 'test.zip');
+} {1 {zipfile constructor requires one argument}}
 
 
 

From 2f7260deba127ab34f3722035b9a3fb1ff3d0245 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 30 Jan 2018 17:43:22 +0000
Subject: [PATCH 403/488] Add tests cases for the zipfile extension.

FossilOrigin-Name: 13b786dafd308b7aef42328f26fe64981c83d524e4bf82ceb7d58afc7146c7d3
---
 ext/misc/zipfile.c     |  33 ++++----
 manifest               |  14 ++--
 manifest.uuid          |   2 +-
 test/zipfile2.test     | 167 +++++++++++++++++++++++++++++++++++++++++
 test/zipfilefault.test |  67 +++++++++++++++++
 5 files changed, 263 insertions(+), 20 deletions(-)
 create mode 100644 test/zipfile2.test
 create mode 100644 test/zipfilefault.test

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index 247d0b7355..a52033e25b 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -48,13 +48,26 @@ SQLITE_EXTENSION_INIT1
 #ifndef SQLITE_OMIT_VIRTUALTABLE
 
 #ifndef SQLITE_AMALGAMATION
+
 typedef sqlite3_int64 i64;
 typedef unsigned char u8;
 typedef unsigned short u16;
 typedef unsigned long u32;
 #define MIN(a,b) ((a)<(b) ? (a) : (b))
+
+#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST)
+# define ALWAYS(X)      (1)
+# define NEVER(X)       (0)
+#elif !defined(NDEBUG)
+# define ALWAYS(X)      ((X)?1:(assert(0),0))
+# define NEVER(X)       ((X)?(assert(0),1):0)
+#else
+# define ALWAYS(X)      (X)
+# define NEVER(X)       (X)
 #endif
 
+#endif   /* SQLITE_AMALGAMATION */
+
 static const char ZIPFILE_SCHEMA[] = 
   "CREATE TABLE y("
     "name PRIMARY KEY,"  /* 0: Name of file in zip archive */
@@ -283,14 +296,12 @@ static void zipfileCtxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){
 static void zipfileDequote(char *zIn){
   char q = zIn[0];
   if( q=='"' || q=='\'' || q=='`' || q=='[' ){
-    char c;
     int iIn = 1;
     int iOut = 0;
     if( q=='[' ) q = ']';
-    while( (c = zIn[iIn++]) ){
-      if( c==q ){
-        if( zIn[iIn++]!=q ) break;
-      }
+    while( ALWAYS(zIn[iIn]) ){
+      char c = zIn[iIn++];
+      if( c==q && zIn[iIn++]!=q ) break;
       zIn[iOut++] = c;
     }
     zIn[iOut] = '\0';
@@ -445,12 +456,8 @@ static int zipfileClose(sqlite3_vtab_cursor *cur){
   zipfileResetCursor(pCsr);
 
   /* Remove this cursor from the ZipfileTab.pCsrList list. */
-  for(pp=&pTab->pCsrList; *pp; pp=&((*pp)->pCsrNext)){
-    if( *pp==pCsr ){ 
-      *pp = pCsr->pCsrNext;
-      break;
-    }
-  }
+  for(pp=&pTab->pCsrList; *pp!=pCsr; pp=&((*pp)->pCsrNext));
+  *pp = pCsr->pCsrNext;
 
   sqlite3_free(pCsr);
   return SQLITE_OK;
@@ -873,7 +880,7 @@ static void zipfileFree(void *p) {
 ** If an error occurs, an error code is left in pCtx instead.
 */
 static void zipfileInflate(
-  sqlite3_context *pCtx,          /* Store error here, if any */
+  sqlite3_context *pCtx,          /* Store result here */
   const u8 *aIn,                  /* Compressed data */
   int nIn,                        /* Size of buffer aIn[] in bytes */
   int nOut                        /* Expected output size */
@@ -1119,7 +1126,7 @@ static int zipfileReadEOCD(
     pEOCD->iOffset = zipfileRead32(aRead);
   }
 
-  return SQLITE_OK;
+  return rc;
 }
 
 /*
diff --git a/manifest b/manifest
index f52d1dd656..8cbe5d88c3 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Prevent\susers\sfrom\screating\szipfile()\svirtual\stables\swithout\san\sargument.
-D 2018-01-30T14:07:55.726
+C Add\stests\scases\sfor\sthe\szipfile\sextension.
+D 2018-01-30T17:43:22.142
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -304,7 +304,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c ead25f062cee790b7c764ce8d2c6ad32a7ac82fc31ea80f69be99948f96f2d19
+F ext/misc/zipfile.c aa10ec6a235030cd368b511e7c78e40251008dddddeb0eb006ad1f344f78b690
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -1604,6 +1604,8 @@ F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501d
 F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa
 F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc
 F test/zipfile.test 368a5a0c97be0caaf8c3efa8293bfe18436d546805678fa00b6aa81bc98727ec
+F test/zipfile2.test 2855c993641935b51dfb3f9005173114538c7a2eb2a60fe3fc47a802e6794406
+F test/zipfilefault.test df4fa9e16116e0cb21d09037a6b0a7d93fecd8767d82154b66044b4ca43fca58
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d
 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91
@@ -1702,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 438c5c5237a801ae78809bf324bb9251fb50250addfc1f8e36659442b0e26ab6
-R 0f8eb13d715c2195926ed8684b288375
+P 81fdbe0cc5a360f818078d47a5888d0a29d555927da279a9a0213702d74ef09a
+R 63c2b54c1a4defd96e22435ea8f5b625
 U dan
-Z 6708ef6af1eb582ebfe61496df2754b7
+Z 6717becc48c95c87eb42213e9c88ecb3
diff --git a/manifest.uuid b/manifest.uuid
index 2d66824214..97b7247606 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-81fdbe0cc5a360f818078d47a5888d0a29d555927da279a9a0213702d74ef09a
\ No newline at end of file
+13b786dafd308b7aef42328f26fe64981c83d524e4bf82ceb7d58afc7146c7d3
\ No newline at end of file
diff --git a/test/zipfile2.test b/test/zipfile2.test
new file mode 100644
index 0000000000..f3b7a84169
--- /dev/null
+++ b/test/zipfile2.test
@@ -0,0 +1,167 @@
+# 2018 January 30
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix zipfile2
+
+ifcapable !vtab {
+  finish_test; return
+}
+if {[catch {load_static_extension db zipfile} error]} {
+  puts "Skipping zipfile2 tests, hit load error: $error"
+  finish_test; return
+}
+
+proc blobliteral {str} {
+  set concat [string map {" " "" "\n" ""} $str]
+  return "X'$concat'"
+}
+
+proc blob {str} {
+  binary decode hex $str
+}
+
+proc findall {needle haystack} {
+  set L [list]
+  set start 0
+  while { [set idx [string first $needle $haystack $start]]>=0 } {
+    lappend L $idx
+    set start [expr $idx+1]
+  }
+  set L
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE aaa USING zipfile('testzip');
+  CREATE VIRTUAL TABLE bbb USING zipfile("testzip");
+  CREATE VIRTUAL TABLE ccc USING zipfile(`testzip`);
+  CREATE VIRTUAL TABLE ddd USING zipfile([testzip]);
+  CREATE VIRTUAL TABLE eee USING zipfile(testzip);
+  CREATE VIRTUAL TABLE fff USING zipfile('test''zip');
+}
+
+do_test 2.0 {
+  forcedelete testdir
+  file mkdir testdir
+  execsql { CREATE VIRTUAL TABLE hhh USING zipfile('testdir') }
+  catchsql { SELECT * FROM hhh } 
+} {1 {error in fread()}}
+
+
+set archive {
+  504B0304140000080000D4A52BEC09F3B6E0110000001100000005000900612E
+  747874555405000140420F00636F6E74656E7473206F6620612E747874504B03
+  04140000080000D4A52BECD98916A7110000001100000005000900622E747874
+  555405000140420F00636F6E74656E7473206F6620622E747874504B01021E03
+  140000080000D4A52BEC09F3B6E0110000001100000005000900000000000000
+  0000A48100000000612E747874555405000140420F00504B01021E0314000008
+  0000D4A52BECD98916A71100000011000000050009000000000000000000A481
+  3D000000622E747874555405000140420F00504B050600000000020002007800
+  00007A0000000000
+}
+
+do_execsql_test 3.1 {
+  WITH contents(name,mtime,data) AS (
+    VALUES('a.txt', 1000000, 'contents of a.txt') UNION ALL
+    VALUES('b.txt', 1000000, 'contents of b.txt')
+  ) SELECT quote( zipfile(name,NULL,mtime,data) ) FROM contents;
+} [blobliteral $archive]
+
+set blob [blob $archive]
+do_execsql_test 3.2 {
+  SELECT name,mtime,data FROM zipfile($blob)
+} {
+  a.txt 1000000 {contents of a.txt} 
+  b.txt 1000000 {contents of b.txt}
+}
+
+# Corrupt each of the 0x50 0x4B (ascii "PK") headers in the file
+# Test that in each case this causes an error.
+#
+set L [findall 504B $archive]
+for {set i 0} {$i < [llength $L]} {incr i} {
+  set idx [lindex $L $i]
+  set a [string replace $archive $idx [expr $idx+3] 0000]
+  set blob [blob $a]
+  do_catchsql_test 3.3.$i {
+    SELECT name,mtime,data FROM zipfile($blob)
+  } {/1 .*/}
+}
+
+set L [findall 5554 $archive]
+for {set i 0} {$i < [llength $L]} {incr i} {
+  set idx [lindex $L $i]
+  set a [string replace $archive $idx [expr $idx+3] 1234]
+  set blob [blob $a]
+  do_execsql_test 3.4.$i {
+    SELECT name,data FROM zipfile($blob)
+  } {
+    a.txt {contents of a.txt} 
+    b.txt {contents of b.txt}
+  }
+}
+
+for {set i 0} {$i < [llength $L]} {incr i} {
+  set idx [lindex $L $i]
+  set a [string replace $archive [expr $idx+8] [expr $idx+9] 00]
+  set blob [blob $a]
+  do_execsql_test 3.5.$i {
+    SELECT name,data FROM zipfile($blob)
+  } {
+    a.txt {contents of a.txt} 
+    b.txt {contents of b.txt}
+  }
+}
+
+if 0 {
+set blob [db one {
+  WITH contents(name,mtime,data) AS (
+    VALUES('a.txt', 1000000, 'aaaaaaaaaaaaaaaaaaaaaaa')
+  ) SELECT quote( zipfile(name,NULL,mtime,data) ) FROM contents;
+}]
+set blob [string range $blob 2 end]
+set blob [string range $blob 0 end-1]
+while {[string length $blob]>0} {
+  puts [string range $blob 0 63]
+  set blob [string range $blob 64 end]
+}
+exit
+}
+
+set archive2 {
+  504B0304140000080800D4A52BEC08F54C6E050000001700000005000900612E
+  747874555405000140420F004B4CC40A00504B01021E03140000080800D4A52B
+  EC08F54C6E0500000017000000050009000000000000000000A4810000000061
+  2E747874555405000140420F00504B050600000000010001003C000000310000
+  000000
+}
+set blob [blob $archive2]
+do_execsql_test 4.0 {
+  SELECT name,mtime,data,method FROM zipfile($blob)
+} {
+  a.txt 1000000 aaaaaaaaaaaaaaaaaaaaaaa 8
+}
+
+breakpoint
+set L [findall 17000000 $archive2]
+set a $archive2
+foreach i $L { set a [string replace $a $i [expr $i+7] 16000000] }
+set blob [blob $a]
+do_catchsql_test 4.1 {
+  SELECT name,mtime,data,method FROM zipfile($blob)
+} {1 {SQL logic error}}
+
+
+
+finish_test
+
diff --git a/test/zipfilefault.test b/test/zipfilefault.test
new file mode 100644
index 0000000000..9f0f4784dc
--- /dev/null
+++ b/test/zipfilefault.test
@@ -0,0 +1,67 @@
+# 2018 January 30
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/malloc_common.tcl
+set testprefix zipfilefault
+
+ifcapable !vtab {
+  finish_test; return
+}
+if {[catch {load_static_extension db zipfile} error]} {
+  puts "Skipping zipfile2 tests, hit load error: $error"
+  finish_test; return
+}
+
+faultsim_save_and_close
+do_faultsim_test 1 -prep {
+  faultsim_restore_and_reopen
+  load_static_extension db zipfile
+  execsql { DROP TABLE IF EXISTS aaa }
+} -body {
+  execsql { CREATE VIRTUAL TABLE aaa USING zipfile('test.zip') }
+} -test {
+  faultsim_test_result {0 {}} 
+}
+
+forcedelete test.zip
+sqlite3 db test.db
+load_static_extension db zipfile
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE setup USING zipfile('test.zip');
+  INSERT INTO setup(name, data) VALUES('a.txt', '1234567890');
+}
+
+do_faultsim_test 2 -faults oom* -body {
+  execsql { SELECT name,data FROM zipfile('test.zip') }
+} -test {
+  faultsim_test_result {0 {a.txt 1234567890}} 
+}
+
+forcedelete test.zip
+reset_db
+load_static_extension db zipfile
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE setup USING zipfile('test.zip');
+  INSERT INTO setup(name, data) VALUES('a.txt', 'aaaaaaaaaaaaaaaaaaaaaaaaaaaa');
+}
+
+do_faultsim_test 2 -faults oom* -body {
+  execsql { SELECT name,data FROM zipfile('test.zip') }
+} -test {
+  faultsim_test_result {0 {a.txt aaaaaaaaaaaaaaaaaaaaaaaaaaaa}} 
+}
+
+
+finish_test
+

From 099fa84727a06f17809dfb90ad308a500a91ca40 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 30 Jan 2018 18:33:23 +0000
Subject: [PATCH 404/488] If a virtual table xColumn method sets an error
 message using sqlite3_result_error(), use that message in preference to any
 error message left in the sqlite3_vtab object.

FossilOrigin-Name: 71e3b715ad36c2132a587b84221be6359c31d7a17c3fba201b7ed3baf17922f1
---
 manifest           | 14 +++++++-------
 manifest.uuid      |  2 +-
 src/vdbe.c         |  1 +
 test/zipfile2.test |  4 +---
 4 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index 8cbe5d88c3..b8ff099373 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\stests\scases\sfor\sthe\szipfile\sextension.
-D 2018-01-30T17:43:22.142
+C If\sa\svirtual\stable\sxColumn\smethod\ssets\san\serror\smessage\susing\nsqlite3_result_error(),\suse\sthat\smessage\sin\spreference\sto\sany\serror\smessage\nleft\sin\sthe\ssqlite3_vtab\sobject.
+D 2018-01-30T18:33:23.120
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -556,7 +556,7 @@ F src/update.c a90a32ffc0100265b0693dbbdbe490756447af181f5ea2c138cce515b08c8795
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c d9eb0a6c4aae1b00a7369eadd7ca0bbe946cb4c953b6751aa20d357c2f482157
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
-F src/vdbe.c 0e13b8c33a9ecf236b3045f190fb58933049582307b621cbd924b6582a2b705f
+F src/vdbe.c 619d385ff1798fab7c549deb04b078f5a8ec466e2ad315939bc824593a2f9fde
 F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a
 F src/vdbeInt.h 8d7d07f13cb3c4cbca91e22ba4a1920e542dda7c5d9299920432a0b3d5b009f5
 F src/vdbeapi.c fea41171884a4de119f8b10ab514c788674eeeb7f27218bb6d008e1310bfd07f
@@ -1604,7 +1604,7 @@ F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501d
 F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa
 F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc
 F test/zipfile.test 368a5a0c97be0caaf8c3efa8293bfe18436d546805678fa00b6aa81bc98727ec
-F test/zipfile2.test 2855c993641935b51dfb3f9005173114538c7a2eb2a60fe3fc47a802e6794406
+F test/zipfile2.test 1066dd9769028d6f0bf1c87303b24f087cc064d9bbcacefc95613380f642de18
 F test/zipfilefault.test df4fa9e16116e0cb21d09037a6b0a7d93fecd8767d82154b66044b4ca43fca58
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d
@@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 81fdbe0cc5a360f818078d47a5888d0a29d555927da279a9a0213702d74ef09a
-R 63c2b54c1a4defd96e22435ea8f5b625
+P 13b786dafd308b7aef42328f26fe64981c83d524e4bf82ceb7d58afc7146c7d3
+R 48ccd51e910d5d576747af2d5e64e430
 U dan
-Z 6717becc48c95c87eb42213e9c88ecb3
+Z 9d867999d41b646c89b308fd5587d517
diff --git a/manifest.uuid b/manifest.uuid
index 97b7247606..2c1ee2f33c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-13b786dafd308b7aef42328f26fe64981c83d524e4bf82ceb7d58afc7146c7d3
\ No newline at end of file
+71e3b715ad36c2132a587b84221be6359c31d7a17c3fba201b7ed3baf17922f1
\ No newline at end of file
diff --git a/src/vdbe.c b/src/vdbe.c
index 495b56aa1d..7d1cae0caf 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -6756,6 +6756,7 @@ case OP_VColumn: {
   rc = pModule->xColumn(pCur->uc.pVCur, &sContext, pOp->p2);
   sqlite3VtabImportErrmsg(p, pVtab);
   if( sContext.isError>0 ){
+    sqlite3VdbeError(p, "%s", sqlite3_value_text(pDest));
     rc = sContext.isError;
   }
   sqlite3VdbeChangeEncoding(pDest, encoding);
diff --git a/test/zipfile2.test b/test/zipfile2.test
index f3b7a84169..6d20e09301 100644
--- a/test/zipfile2.test
+++ b/test/zipfile2.test
@@ -152,15 +152,13 @@ do_execsql_test 4.0 {
   a.txt 1000000 aaaaaaaaaaaaaaaaaaaaaaa 8
 }
 
-breakpoint
 set L [findall 17000000 $archive2]
 set a $archive2
 foreach i $L { set a [string replace $a $i [expr $i+7] 16000000] }
 set blob [blob $a]
 do_catchsql_test 4.1 {
   SELECT name,mtime,data,method FROM zipfile($blob)
-} {1 {SQL logic error}}
-
+} {1 {inflate() failed (0)}}
 
 
 finish_test

From 04a28c3b6d546a3fd97ee541cfcd0cac2fb60488 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 31 Jan 2018 01:38:44 +0000
Subject: [PATCH 405/488] Changes to the autoconf tarball so that it does not
 try to use system() when building on iOS.

FossilOrigin-Name: 32ed9c106710c74a12d60ec33027fd6a9335627c95642ba608593b2735714da6
---
 autoconf/Makefile.am  |  2 +-
 autoconf/configure.ac |  3 +++
 manifest              | 18 +++++++++---------
 manifest.uuid         |  2 +-
 src/shell.c.in        | 15 ++++++++++++++-
 5 files changed, 28 insertions(+), 12 deletions(-)

diff --git a/autoconf/Makefile.am b/autoconf/Makefile.am
index 0f0706873a..db246924fe 100644
--- a/autoconf/Makefile.am
+++ b/autoconf/Makefile.am
@@ -10,7 +10,7 @@ sqlite3_SOURCES = shell.c sqlite3.h
 EXTRA_sqlite3_SOURCES = sqlite3.c
 sqlite3_LDADD = @EXTRA_SHELL_OBJ@ @READLINE_LIBS@
 sqlite3_DEPENDENCIES = @EXTRA_SHELL_OBJ@
-sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_STMTVTAB -DSQLITE_ENABLE_DBSTAT_VTAB
+sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_STMTVTAB -DSQLITE_ENABLE_DBSTAT_VTAB $(SHELL_CFLAGS)
 
 include_HEADERS = sqlite3.h sqlite3ext.h
 
diff --git a/autoconf/configure.ac b/autoconf/configure.ac
index 19975e96c7..2680becbb0 100644
--- a/autoconf/configure.ac
+++ b/autoconf/configure.ac
@@ -169,6 +169,9 @@ AC_CHECK_HEADERS(zlib.h,[
 ])
 AC_SUBST(ZLIB_FLAGS)
 
+AC_SEARCH_LIBS(system,,,[SHELL_CFLAGS="-DSQLITE_NOHAVE_SYSTEM"])
+AC_SUBST(SHELL_CFLAGS)
+
 #-----------------------------------------------------------------------
 # UPDATE: Maybe it's better if users just set CFLAGS before invoking
 # configure. This option doesn't really add much...
diff --git a/manifest b/manifest
index b8ff099373..2ab0abc923 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C If\sa\svirtual\stable\sxColumn\smethod\ssets\san\serror\smessage\susing\nsqlite3_result_error(),\suse\sthat\smessage\sin\spreference\sto\sany\serror\smessage\nleft\sin\sthe\ssqlite3_vtab\sobject.
-D 2018-01-30T18:33:23.120
+C Changes\sto\sthe\sautoconf\starball\sso\sthat\sit\sdoes\snot\stry\sto\suse\ssystem()\swhen\nbuilding\son\siOS.
+D 2018-01-31T01:38:44.477
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -12,11 +12,11 @@ F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2
 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90
 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2
 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903
-F autoconf/Makefile.am 6cca3f797c649b40c762484ce26491839fec54de72d376d774969e76ed13931f
+F autoconf/Makefile.am 2c274948734e03c51790ff51468f91db8d570bcca864284d9c6d6e777264cd7e
 F autoconf/Makefile.msc 2c50a59319af7da4eaca8c13e3240881b1bc245fd175845a055faab7d03d6e67
 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7
 F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1
-F autoconf/configure.ac aeeed858e5e54e79052ae44ba774e56595dcb787f23a2155aa98a8aa27327b66
+F autoconf/configure.ac 18fca06f884213be062dd5e07c5297079cc45893d9cd3f522ce426e715033e3d
 F autoconf/tea/Makefile.in b438a7020446c8a8156e8d97c8914a04833da6fd
 F autoconf/tea/README 3e9a3c060f29a44344ab50aec506f4db903fb873
 F autoconf/tea/aclocal.m4 52c47aac44ce0ddb1f918b6993e8beb8eee88f43
@@ -488,7 +488,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c f02352ac5cbb6fad3804add825743b924cfb6c742ba2e8666d726828a9985d73
-F src/shell.c.in 7cea439c3f7f2e4ed6eb4b3a633cd93dccb1349241400de4da0c1291285ed514
+F src/shell.c.in ada6d184a5cdaada15a330ca97ad0126125f0aaed40b5c04385278fcaa78aafa
 F src/sqlite.h.in 51f9acf52c80113d793ddd38b3940ad6895d97b4752503b19291fb8fcbf54c5e
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d
@@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 13b786dafd308b7aef42328f26fe64981c83d524e4bf82ceb7d58afc7146c7d3
-R 48ccd51e910d5d576747af2d5e64e430
-U dan
-Z 9d867999d41b646c89b308fd5587d517
+P 71e3b715ad36c2132a587b84221be6359c31d7a17c3fba201b7ed3baf17922f1
+R bcc68c8a2653dd62fee16c20bb3144f0
+U drh
+Z ff1bd702823e7d15c0a453ffcea5582a
diff --git a/manifest.uuid b/manifest.uuid
index 2c1ee2f33c..4422b0dd56 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-71e3b715ad36c2132a587b84221be6359c31d7a17c3fba201b7ed3baf17922f1
\ No newline at end of file
+32ed9c106710c74a12d60ec33027fd6a9335627c95642ba608593b2735714da6
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index a785517503..f683ce9868 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -1175,6 +1175,7 @@ static void shellPutsFunc(
 **
 ** Also throw an error if the EDITOR program returns a non-zero exit code.
 */
+#ifndef SQLITE_NOHAVE_SYSTEM
 static void editFunc(
   sqlite3_context *context,
   int argc,
@@ -1285,6 +1286,7 @@ edit_func_end:
   sqlite3_free(zTempFile);
   sqlite3_free(p);
 }
+#endif /* SQLITE_NOHAVE_SYSTEM */
 
 /*
 ** Save or restore the current output mode
@@ -3273,10 +3275,14 @@ static char zHelp[] =
   ".session CMD ...       Create or control sessions\n"
 #endif
   ".sha3sum ?OPTIONS...?  Compute a SHA3 hash of database content\n"
+#ifndef SQLITE_NOHAVE_SYSTEM
   ".shell CMD ARGS...     Run CMD ARGS... in a system shell\n"
+#endif
   ".show                  Show the current values for various settings\n"
   ".stats ?on|off?        Show stats or turn stats on or off\n"
+#ifndef SQLITE_NOHAVE_SYSTEM
   ".system CMD ARGS...    Run CMD ARGS... in a system shell\n"
+#endif
   ".tables ?TABLE?        List names of tables\n"
   "                         If TABLE specified, only list tables matching\n"
   "                         LIKE pattern TABLE.\n"
@@ -3477,10 +3483,12 @@ static void open_db(ShellState *p, int keepAlive){
                             shellModuleSchema, 0, 0);
     sqlite3_create_function(p->db, "shell_putsnl", 1, SQLITE_UTF8, p,
                             shellPutsFunc, 0, 0);
+#ifndef SQLITE_NOHAVE_SYSTEM
     sqlite3_create_function(p->db, "edit", 1, SQLITE_UTF8, 0,
                             editFunc, 0, 0);
     sqlite3_create_function(p->db, "edit", 2, SQLITE_UTF8, 0,
                             editFunc, 0, 0);
+#endif
     if( p->openMode==SHELL_OPEN_ZIPFILE ){
       char *zSql = sqlite3_mprintf(
          "CREATE VIRTUAL TABLE zip USING zipfile(%Q);", p->zDbFilename);
@@ -4095,6 +4103,7 @@ static void output_reset(ShellState *p){
 #endif
   }else{
     output_file_close(p->out);
+#ifndef SQLITE_NOHAVE_SYSTEM
     if( p->doXdgOpen ){
       const char *zXdgOpenCmd =
 #if defined(_WIN32)
@@ -4113,6 +4122,7 @@ static void output_reset(ShellState *p){
       outputModePop(p);
       p->doXdgOpen = 0;
     }
+#endif /* !defined(SQLITE_NOHAVE_SYSTEM) */
   }
   p->outfile[0] = 0;
   p->out = stdout;
@@ -6353,6 +6363,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     }
     output_reset(p);
     if( zFile[0]=='-' && zFile[1]=='-' ) zFile++;
+#ifndef SQLITE_NOHAVE_SYSTEM
     if( strcmp(zFile, "-e")==0 || strcmp(zFile, "-x")==0 ){
       p->doXdgOpen = 1;
       outputModePush(p);
@@ -6367,6 +6378,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       }
       zFile = p->zTempFile;
     }
+#endif /* SQLITE_NOHAVE_SYSTEM */
     if( zFile[0]=='|' ){
 #ifdef SQLITE_OMIT_POPEN
       raw_printf(stderr, "Error: pipes are not supported in this OS\n");
@@ -6486,7 +6498,6 @@ static int do_meta_command(char *zLine, ShellState *p){
     sqlite3_close(pSrc);
   }else
 
-
   if( c=='s' && strncmp(azArg[0], "scanstats", n)==0 ){
     if( nArg==2 ){
       p->scanstatsOn = (u8)booleanValue(azArg[1]);
@@ -7082,6 +7093,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     sqlite3_free(zSql);
   }else
 
+#ifndef SQLITE_NOHAVE_SYSTEM
   if( c=='s'
    && (strncmp(azArg[0], "shell", n)==0 || strncmp(azArg[0],"system",n)==0)
   ){
@@ -7101,6 +7113,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     sqlite3_free(zCmd);
     if( x ) raw_printf(stderr, "System command returns %d\n", x);
   }else
+#endif /* !defined(SQLITE_NOHAVE_SYSTEM) */
 
   if( c=='s' && strncmp(azArg[0], "show", n)==0 ){
     static const char *azBool[] = { "off", "on", "trigger", "full"};

From 4ea48144f91a65e82900cf8587380d8b451eaf1d Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 31 Jan 2018 14:07:01 +0000
Subject: [PATCH 406/488] Fix a failing assert() in the new code on this
 branch.

FossilOrigin-Name: 74d857d178dfadea7e07ba47439fe11aa9d282b54caf78cd6961e593b37406d0
---
 manifest        | 17 +++++++----------
 manifest.uuid   |  2 +-
 src/where.c     |  4 +---
 test/join2.test | 11 +++++++++++
 4 files changed, 20 insertions(+), 14 deletions(-)

diff --git a/manifest b/manifest
index 7c15e392d1..4b0592f36e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\sthe\somit-left-join\soptimization\sso\sthat\sit\sworks\sin\ssome\scases\swhen\sthe\nRHS\sis\ssubject\sto\sa\sUNIQUE\sbut\snot\sNOT\sNULL\sconstraint.
-D 2018-01-29T17:08:52.190
+C Fix\sa\sfailing\sassert()\sin\sthe\snew\scode\son\sthis\sbranch.
+D 2018-01-31T14:07:01.020
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -570,7 +570,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 5a3f464edd64596f601683ed321d12e6fd93c5fb9afdfb3653d6ffd0fee9c48f
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f
-F src/where.c eb37882608c30f7361cf3309b8128c3a2887ff0099886e3685ededb7a7e6b051
+F src/where.c 7cae47e813393d70c6d327fdf000fcb30f76b1b0b5a5b52ff6402e0c658de32c
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
 F src/wherecode.c cb360c511f69294ddf00340d7b390e6b1d601a1cfb77b42c4d316fe2f4cd01c3
 F src/whereexpr.c 22dbfd3bf5f6051a61523dd0ebef7a944fb29ee4aa7d2a62feb8aac6ffbbc0eb
@@ -1001,7 +1001,7 @@ F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c
 F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4
 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b
 F test/join.test 442c462eea85cf065d70a663c626b780a95af6e11585d909bb63b87598afe678
-F test/join2.test 96f56ccf0a8fe6dbd7bb2e1a90bc61540febeaf9b6fdee14506f6fc140b93185
+F test/join2.test f1b5ba94ba70a791df2d99e61adcebf62edacad1eca57881c97eb3e2ca718595
 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0
 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916
 F test/join5.test bc98ea4b4e5003f5b1453701ebb8cd7d1c01a550
@@ -1702,10 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 8767f7b880f2e4112f75f0b6ef7be3f50ab1ae20e103e7d03d8bfe77e6c79438
-R f19810b3aa9bfaa7d3b666e31ece61ec
-T *branch * omit-left-join-fix
-T *sym-omit-left-join-fix *
-T -sym-trunk *
+P 88411a402b29f19d69e0d06d9099eabb98c238ddb7c07ce110236316f008753e
+R d723a0554461743d1cc2169a56c72b46
 U dan
-Z ff1cffb196461bca97e3f8e467ebcd67
+Z 54f22eae691d89833b1c2792d257ce0d
diff --git a/manifest.uuid b/manifest.uuid
index 84c8ec4673..15554a93f9 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-88411a402b29f19d69e0d06d9099eabb98c238ddb7c07ce110236316f008753e
\ No newline at end of file
+74d857d178dfadea7e07ba47439fe11aa9d282b54caf78cd6961e593b37406d0
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index 3b18fbf5ff..8c1e3cd01f 100644
--- a/src/where.c
+++ b/src/where.c
@@ -2481,8 +2481,7 @@ static int whereLoopAddBtreeIndex(
       if( iCol==XN_ROWID 
        || (iCol>=0 && nInMul==0 && saved_nEq==pProbe->nKeyCol-1)
       ){
-        assert( iCol!=XN_ROWID || pProbe->uniqNotNull );
-        if( pProbe->uniqNotNull 
+        if( iCol==XN_ROWID || pProbe->uniqNotNull 
          || (pProbe->nKeyCol==1 && pProbe->onError && eOp==WO_EQ) 
         ){
           pNew->wsFlags |= WHERE_ONEROW;
@@ -2828,7 +2827,6 @@ static int whereLoopAddBtree(
     sPk.onError = OE_Replace;
     sPk.pTable = pTab;
     sPk.szIdxRow = pTab->szTabRow;
-    sPk.uniqNotNull = 1;
     aiRowEstPk[0] = pTab->nRowLogEst;
     aiRowEstPk[1] = 0;
     pFirst = pSrc->pTab->pIndex;
diff --git a/test/join2.test b/test/join2.test
index 3dc067e508..256846ac91 100644
--- a/test/join2.test
+++ b/test/join2.test
@@ -254,4 +254,15 @@ do_eqp_test 5.2 {
   0 0 0 {SCAN TABLE s1}
 }
 
+do_execsql_test 6.0 {
+  CREATE TABLE u1(a INTEGER PRIMARY KEY, b, c);
+  CREATE TABLE u2(a INTEGER PRIMARY KEY, b, c);
+  CREATE INDEX u1ab ON u1(b, c);
+}
+do_eqp_test 6.1 {
+  SELECT u2.* FROM u2 LEFT JOIN u1 ON( u1.a=u2.a AND u1.b=u2.b AND u1.c=u2.c );
+} {
+  0 0 0 {SCAN TABLE u2}
+}
+
 finish_test

From 44091ed3682ac115c108b3a1d201db76a78caf37 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 31 Jan 2018 19:13:31 +0000
Subject: [PATCH 407/488] When creating a new archive entry, have zipfile store
 UTC instead of local time in the legacy MS-DOS format timestamp field.

FossilOrigin-Name: e2114df18383d111dd5fbac902e08b42a7f4b2b2d6f7bf29574a3722e4a4dad5
---
 ext/misc/zipfile.c | 78 +++++++++++++++++++++++++++++++---------------
 manifest           | 22 +++++++------
 manifest.uuid      |  2 +-
 test/zipfile.test  | 54 ++++++++++++++++++++++++++++++++
 test/zipfile2.test | 45 ++++++++++++++------------
 5 files changed, 145 insertions(+), 56 deletions(-)

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index a52033e25b..067b6d4953 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -686,18 +686,34 @@ static int zipfileScanExtra(u8 *aExtra, int nExtra, u32 *pmTime){
 **
 ** https://msdn.microsoft.com/en-us/library/9kkf9tah.aspx
 */
-static time_t zipfileMtime(ZipfileCDS *pCDS){
-  struct tm t;
-  memset(&t, 0, sizeof(t));
-  t.tm_sec = (pCDS->mTime & 0x1F)*2;
-  t.tm_min = (pCDS->mTime >> 5) & 0x2F;
-  t.tm_hour = (pCDS->mTime >> 11) & 0x1F;
+static u32 zipfileMtime(ZipfileCDS *pCDS){
+  int Y = (1980 + ((pCDS->mDate >> 9) & 0x7F));
+  int M = ((pCDS->mDate >> 5) & 0x0F);
+  int D = (pCDS->mDate & 0x1F);
+  int B = -13;
 
-  t.tm_mday = (pCDS->mDate & 0x1F);
-  t.tm_mon = ((pCDS->mDate >> 5) & 0x0F) - 1;
-  t.tm_year = 80 + ((pCDS->mDate >> 9) & 0x7F);
+  int sec = (pCDS->mTime & 0x1F)*2;
+  int min = (pCDS->mTime >> 5) & 0x3F;
+  int hr = (pCDS->mTime >> 11) & 0x1F;
 
-  return mktime(&t);
+  /* JD = INT(365.25 * (Y+4716)) + INT(30.6001 * (M+1)) + D + B - 1524.5 */
+
+  /* Calculate the JD in seconds for noon on the day in question */
+  if( M<3 ){
+    Y = Y-1;
+    M = M+12;
+  }
+  i64 JD = (i64)(24*60*60) * (
+      (int)(365.25 * (Y + 4716))
+    + (int)(30.6001 * (M + 1))
+    + D + B - 1524
+  );
+
+  /* Correct the JD for the time within the day */
+  JD += (hr-12) * 3600 + min * 60 + sec;
+
+  /* Convert JD to unix timestamp (the JD epoch is 2440587.5) */
+  return (u32)(JD - (i64)(24405875) * 24*60*6);
 }
 
 /*
@@ -706,24 +722,36 @@ static time_t zipfileMtime(ZipfileCDS *pCDS){
 ** to the UNIX timestamp value passed as the second.
 */
 static void zipfileMtimeToDos(ZipfileCDS *pCds, u32 mUnixTime){
-  time_t t = (time_t)mUnixTime;
-  struct tm res;
+  /* Convert unix timestamp to JD (2440588 is noon on 1/1/1970) */
+  i64 JD = (i64)2440588 + mUnixTime / (24*60*60);
 
-#if !defined(_WIN32) && !defined(WIN32)
-  localtime_r(&t, &res);
-#else
-  memcpy(&res, localtime(&t), sizeof(struct tm));
-#endif
+  int A, B, C, D, E;
+  int yr, mon, day;
+  int hr, min, sec;
 
-  pCds->mTime = (u16)(
-    (res.tm_sec / 2) + 
-    (res.tm_min << 5) +
-    (res.tm_hour << 11));
+  A = (int)((JD - 1867216.25)/36524.25);
+  A = JD + 1 + A - (A/4);
+  B = A + 1524;
+  C = (int)((B - 122.1)/365.25);
+  D = (36525*(C&32767))/100;
+  E = (int)((B-D)/30.6001);
 
-  pCds->mDate = (u16)(
-    (res.tm_mday-1) +
-    ((res.tm_mon+1) << 5) +
-    ((res.tm_year-80) << 9));
+  day = B - D - (int)(30.6001*E);
+  mon = (E<14 ? E-1 : E-13);
+  yr = mon>2 ? C-4716 : C-4715;
+
+  hr = (mUnixTime % (24*60*60)) / (60*60);
+  min = (mUnixTime % (60*60)) / 60;
+  sec = (mUnixTime % 60);
+
+  pCds->mDate = (u16)(day + (mon << 5) + ((yr-1980) << 9));
+  pCds->mTime = (u16)(sec/2 + (min<<5) + (hr<<11));
+
+  assert( mUnixTime<315507600 
+       || mUnixTime==zipfileMtime(pCds) 
+       || ((mUnixTime % 2) && mUnixTime-1==zipfileMtime(pCds)) 
+       /* || (mUnixTime % 2) */
+  );
 }
 
 /*
diff --git a/manifest b/manifest
index 5ab4a82cba..96d387ed83 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improve\sthe\somit-left-join\soptimization\sso\sthat\sit\sworks\sin\ssome\scases\s\nwhen\sthe\sRHS\sis\ssubject\sto\sa\sUNIQUE\sbut\snot\sNOT\sNULL\sconstraint.
-D 2018-01-31T16:50:27.186
+C When\screating\sa\snew\sarchive\sentry,\shave\szipfile\sstore\sUTC\sinstead\sof\slocal\ntime\sin\sthe\slegacy\sMS-DOS\sformat\stimestamp\sfield.
+D 2018-01-31T19:13:31.142
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -304,7 +304,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c aa10ec6a235030cd368b511e7c78e40251008dddddeb0eb006ad1f344f78b690
+F ext/misc/zipfile.c 56028f7e74d948b7bef834624d128f563eb63f5950248d538afa9d0f6c3365dc
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -1603,8 +1603,8 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf
 F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc
 F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa
 F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc
-F test/zipfile.test 368a5a0c97be0caaf8c3efa8293bfe18436d546805678fa00b6aa81bc98727ec
-F test/zipfile2.test 1066dd9769028d6f0bf1c87303b24f087cc064d9bbcacefc95613380f642de18
+F test/zipfile.test 3695ab6d731720ca0b542fe21ded39d4a34891f4a590b2e6c909c9ff07e13148
+F test/zipfile2.test 5f93611307c131e83f226a471231d769b794b9e8c6a675cfa3d34b1a79df23fe
 F test/zipfilefault.test df4fa9e16116e0cb21d09037a6b0a7d93fecd8767d82154b66044b4ca43fca58
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d
@@ -1704,8 +1704,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 32ed9c106710c74a12d60ec33027fd6a9335627c95642ba608593b2735714da6 74d857d178dfadea7e07ba47439fe11aa9d282b54caf78cd6961e593b37406d0
-R ee3ce7babbd6fba601d683cb41b035ba
-T +closed 74d857d178dfadea7e07ba47439fe11aa9d282b54caf78cd6961e593b37406d0
-U drh
-Z 20db364b7139e3a10e7c112d48b03b80
+P 02ba8a7ba7ba71cd7abd5dd3093ea486f53a025f6972bb444f2da37e0e2fc3b2
+R 4025431a2ff076723fc2f621d10c0c47
+T *branch * zipfile-timestamp-fix
+T *sym-zipfile-timestamp-fix *
+T -sym-trunk *
+U dan
+Z 7679dfe9f8ef39802f520b9ec5eb4496
diff --git a/manifest.uuid b/manifest.uuid
index 0940797d9a..555963b3cf 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-02ba8a7ba7ba71cd7abd5dd3093ea486f53a025f6972bb444f2da37e0e2fc3b2
\ No newline at end of file
+e2114df18383d111dd5fbac902e08b42a7f4b2b2d6f7bf29574a3722e4a4dad5
\ No newline at end of file
diff --git a/test/zipfile.test b/test/zipfile.test
index 76e7d924df..a6af4f8963 100644
--- a/test/zipfile.test
+++ b/test/zipfile.test
@@ -48,6 +48,18 @@ if {$::tcl_platform(platform)=="unix" && [catch {exec unzip}]==0} {
   }
 }
 
+
+# The argument is a blob (not a hex string) containing a zip archive.
+# This proc removes the extended timestamp fields from the archive
+# and returns the result.
+#
+proc remove_timestamps {blob} {
+  set hex [binary encode hex $blob]
+  set hex [string map {55540500 00000500} $hex]
+  binary decode hex $hex
+}
+
+
 # Argument $file is the name of a zip archive on disk. This function
 # executes test cases to check that the results of each of the following 
 # are the same:
@@ -367,6 +379,48 @@ do_catchsql_test 4.2 {
   CREATE VIRTUAL TABLE yyy USING zipfile('test.zip', 'test.zip');
 } {1 {zipfile constructor requires one argument}}
 
+#--------------------------------------------------------------------------
+
+db func rt remove_timestamps
+do_execsql_test 5.0 {
+  WITH c(name,mtime,data) AS (
+    SELECT 'a.txt', 946684800, 'abc'
+  )
+  SELECT name,mtime,data FROM zipfile(
+    ( SELECT rt( zipfile(name,NULL,mtime,data) ) FROM c )
+  )
+} {
+  a.txt 946684800 abc
+}
+
+if {[info vars ::UNZIP]!=""} { 
+  load_static_extension db fileio
+  forcedelete test.zip
+  do_test 6.0 {
+    execsql {
+      WITH c(name,mtime,data) AS (
+        SELECT 'a.txt', 946684800, 'abc' UNION ALL
+        SELECT 'b.txt', 1000000000, 'abc' UNION ALL
+        SELECT 'c.txt', 1111111000, 'abc'
+      )
+      SELECT writefile('test.zip',
+          ( SELECT rt ( zipfile(name,NULL,mtime,data) ) FROM c )
+      );
+    }
+    forcedelete test_unzip
+    file mkdir test_unzip
+    exec unzip -d test_unzip test.zip
+
+    db eval {
+      SELECT name, mtime FROM fsdir('test_unzip') WHERE name!='test_unzip'
+      ORDER BY name
+    }
+  } [list {*}{
+    test_unzip/a.txt 946684800
+    test_unzip/b.txt 1000000000 
+    test_unzip/c.txt 1111111000 
+  }]
+}
 
 
 finish_test
diff --git a/test/zipfile2.test b/test/zipfile2.test
index 6d20e09301..a6c3e7e3c4 100644
--- a/test/zipfile2.test
+++ b/test/zipfile2.test
@@ -70,12 +70,17 @@ set archive {
   00007A0000000000
 }
 
-do_execsql_test 3.1 {
-  WITH contents(name,mtime,data) AS (
-    VALUES('a.txt', 1000000, 'contents of a.txt') UNION ALL
-    VALUES('b.txt', 1000000, 'contents of b.txt')
-  ) SELECT quote( zipfile(name,NULL,mtime,data) ) FROM contents;
-} [blobliteral $archive]
+if 0 {
+  # This test is broken - the archive generated is slightly different
+  # depending on the zlib version used.
+  do_execsql_test 3.1 {
+    WITH contents(name,mtime,data) AS (
+        VALUES('a.txt', 1000000, 'contents of a.txt') UNION ALL
+        VALUES('b.txt', 1000000, 'contents of b.txt')
+    ) SELECT quote( zipfile(name,NULL,mtime,data) ) FROM contents;
+  } [blobliteral $archive]
+}
+
 
 set blob [blob $archive]
 do_execsql_test 3.2 {
@@ -98,6 +103,7 @@ for {set i 0} {$i < [llength $L]} {incr i} {
   } {/1 .*/}
 }
 
+# Change the "extra info id" for all extended-timestamp fields.
 set L [findall 5554 $archive]
 for {set i 0} {$i < [llength $L]} {incr i} {
   set idx [lindex $L $i]
@@ -123,20 +129,18 @@ for {set i 0} {$i < [llength $L]} {incr i} {
   }
 }
 
-if 0 {
-set blob [db one {
-  WITH contents(name,mtime,data) AS (
-    VALUES('a.txt', 1000000, 'aaaaaaaaaaaaaaaaaaaaaaa')
-  ) SELECT quote( zipfile(name,NULL,mtime,data) ) FROM contents;
-}]
-set blob [string range $blob 2 end]
-set blob [string range $blob 0 end-1]
-while {[string length $blob]>0} {
-  puts [string range $blob 0 63]
-  set blob [string range $blob 64 end]
-}
-exit
-}
+# set blob [db one {
+#   WITH contents(name,mtime,data) AS (
+#     VALUES('a.txt', 1000000, 'aaaaaaaaaaaaaaaaaaaaaaa')
+#   ) SELECT quote( zipfile(name,NULL,mtime,data) ) FROM contents;
+# }]
+# set blob [string range $blob 2 end]
+# set blob [string range $blob 0 end-1]
+# while {[string length $blob]>0} {
+#   puts [string range $blob 0 63]
+#   set blob [string range $blob 64 end]
+# }
+# exit
 
 set archive2 {
   504B0304140000080800D4A52BEC08F54C6E050000001700000005000900612E
@@ -161,5 +165,6 @@ do_catchsql_test 4.1 {
 } {1 {inflate() failed (0)}}
 
 
+
 finish_test
 

From cfaffad65c2e1ae7db956f3b177d55eed3bccb09 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 31 Jan 2018 19:45:58 +0000
Subject: [PATCH 408/488] Fix a test case in zipfile.test.

FossilOrigin-Name: 4eb5b24c64ee5907c18371f563b79fa0caf58285a62b8f09e639a690e6727eaa
---
 manifest          | 15 ++++++---------
 manifest.uuid     |  2 +-
 test/zipfile.test | 31 +++++++++++++++++++++++++------
 3 files changed, 32 insertions(+), 16 deletions(-)

diff --git a/manifest b/manifest
index 96d387ed83..b6843e249b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C When\screating\sa\snew\sarchive\sentry,\shave\szipfile\sstore\sUTC\sinstead\sof\slocal\ntime\sin\sthe\slegacy\sMS-DOS\sformat\stimestamp\sfield.
-D 2018-01-31T19:13:31.142
+C Fix\sa\stest\scase\sin\szipfile.test.
+D 2018-01-31T19:45:58.864
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -1603,7 +1603,7 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf
 F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc
 F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa
 F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc
-F test/zipfile.test 3695ab6d731720ca0b542fe21ded39d4a34891f4a590b2e6c909c9ff07e13148
+F test/zipfile.test a16528022b08789168adc9c79646e41fb55c4f94b26a2aca97085ebc591ca84d
 F test/zipfile2.test 5f93611307c131e83f226a471231d769b794b9e8c6a675cfa3d34b1a79df23fe
 F test/zipfilefault.test df4fa9e16116e0cb21d09037a6b0a7d93fecd8767d82154b66044b4ca43fca58
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
@@ -1704,10 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 02ba8a7ba7ba71cd7abd5dd3093ea486f53a025f6972bb444f2da37e0e2fc3b2
-R 4025431a2ff076723fc2f621d10c0c47
-T *branch * zipfile-timestamp-fix
-T *sym-zipfile-timestamp-fix *
-T -sym-trunk *
+P e2114df18383d111dd5fbac902e08b42a7f4b2b2d6f7bf29574a3722e4a4dad5
+R 1aa884bc5be6161439d6cdee8401a11e
 U dan
-Z 7679dfe9f8ef39802f520b9ec5eb4496
+Z 880157cddf7626c80e702257f31b9f96
diff --git a/manifest.uuid b/manifest.uuid
index 555963b3cf..91dda60143 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e2114df18383d111dd5fbac902e08b42a7f4b2b2d6f7bf29574a3722e4a4dad5
\ No newline at end of file
+4eb5b24c64ee5907c18371f563b79fa0caf58285a62b8f09e639a690e6727eaa
\ No newline at end of file
diff --git a/test/zipfile.test b/test/zipfile.test
index a6af4f8963..92d8501ea8 100644
--- a/test/zipfile.test
+++ b/test/zipfile.test
@@ -394,8 +394,9 @@ do_execsql_test 5.0 {
 }
 
 if {[info vars ::UNZIP]!=""} { 
+ifcapable datetime {
   load_static_extension db fileio
-  forcedelete test.zip
+  forcedelete test1.zip test2.zip
   do_test 6.0 {
     execsql {
       WITH c(name,mtime,data) AS (
@@ -403,16 +404,33 @@ if {[info vars ::UNZIP]!=""} {
         SELECT 'b.txt', 1000000000, 'abc' UNION ALL
         SELECT 'c.txt', 1111111000, 'abc'
       )
-      SELECT writefile('test.zip',
-          ( SELECT rt ( zipfile(name,NULL,mtime,data) ) FROM c )
-      );
+      SELECT writefile('test1.zip', rt( zipfile(name, NULL, mtime, data) ) ),
+             writefile('test2.zip',   ( zipfile(name, NULL, mtime, data) ) ) 
+      FROM c;
     }
     forcedelete test_unzip
     file mkdir test_unzip
-    exec unzip -d test_unzip test.zip
+    exec unzip -d test_unzip test1.zip
 
     db eval {
-      SELECT name, mtime FROM fsdir('test_unzip') WHERE name!='test_unzip'
+      SELECT name, strftime('%s', mtime, 'unixepoch', 'localtime') 
+      FROM fsdir('test_unzip') WHERE name!='test_unzip'
+      ORDER BY name
+    }
+  } [list {*}{
+    test_unzip/a.txt 946684800
+    test_unzip/b.txt 1000000000 
+    test_unzip/c.txt 1111111000 
+  }]
+
+  do_test 6.1 {
+    forcedelete test_unzip
+    file mkdir test_unzip
+    exec unzip -d test_unzip test2.zip
+
+    db eval {
+      SELECT name, mtime 
+      FROM fsdir('test_unzip') WHERE name!='test_unzip'
       ORDER BY name
     }
   } [list {*}{
@@ -421,6 +439,7 @@ if {[info vars ::UNZIP]!=""} {
     test_unzip/c.txt 1111111000 
   }]
 }
+}
 
 
 finish_test

From 06b3bd5b7add86bcf178abe008ba1189c973f6f3 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 1 Feb 2018 01:13:33 +0000
Subject: [PATCH 409/488] When an index is based on a text representation of a
 numeric column in the original table, make sure the indexed value uses the
 canonical text representation of the numeric value in the table. Proposed fix
 for ticket [343634942dd54ab57b70].

FossilOrigin-Name: 88e2ce916791d488076584f3795a89eb4277fcb812af9e4c2f383815d55ff6f1
---
 manifest             | 17 ++++++++---------
 manifest.uuid        |  2 +-
 src/vdbe.c           |  5 +++++
 test/indexexpr1.test | 21 +++++++++++++++++++++
 4 files changed, 35 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index adb223764e..96a5ce3aff 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C When\screating\sa\snew\sarchive\sentry,\shave\szipfile\sstore\sUTC\sinstead\sof\slocal\ntime\sin\sthe\slegacy\sMS-DOS\sformat\stimestamp\sfield.
-D 2018-01-31T20:18:26.779
+C When\san\sindex\sis\sbased\son\sa\stext\srepresentation\sof\sa\snumeric\scolumn\sin\sthe\noriginal\stable,\smake\ssure\sthe\sindexed\svalue\suses\sthe\scanonical\stext\s\nrepresentation\sof\sthe\snumeric\svalue\sin\sthe\stable.\nProposed\sfix\sfor\sticket\s[343634942dd54ab57b70].
+D 2018-02-01T01:13:33.559
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -556,7 +556,7 @@ F src/update.c a90a32ffc0100265b0693dbbdbe490756447af181f5ea2c138cce515b08c8795
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c d9eb0a6c4aae1b00a7369eadd7ca0bbe946cb4c953b6751aa20d357c2f482157
 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
-F src/vdbe.c 619d385ff1798fab7c549deb04b078f5a8ec466e2ad315939bc824593a2f9fde
+F src/vdbe.c a6892805df427bb9d554c37641406438bff568f99d26e72c1fb7e45f2ed6ce5b
 F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a
 F src/vdbeInt.h 8d7d07f13cb3c4cbca91e22ba4a1920e542dda7c5d9299920432a0b3d5b009f5
 F src/vdbeapi.c fea41171884a4de119f8b10ab514c788674eeeb7f27218bb6d008e1310bfd07f
@@ -978,7 +978,7 @@ F test/index7.test 7feababe16f2091b229c22aff2bcc1d4d6b9d2bb
 F test/index8.test bc2e3db70e8e62459aaa1bd7e4a9b39664f8f9d7
 F test/index9.test 0aa3e509dddf81f93380396e40e9bb386904c1054924ba8fa9bcdfe85a8e7721
 F test/indexedby.test faa585e315e868f09bce0eb39c41d6134649b13d2801638294d3ae616edf1609
-F test/indexexpr1.test ace1ad489adc25325ad298434f13b1a515b36bf5dca9fe2a4b66cdf17aea3fa0
+F test/indexexpr1.test 635261197bcdc19b9b2c59bbfa7227d525c00e9587faddb2d293c44d287ce60e
 F test/indexexpr2.test 13247bac49143196556eb3f65e97ef301bd3e993f4511558b5db322ddc370ea6
 F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d
 F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7
@@ -1704,8 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 02ba8a7ba7ba71cd7abd5dd3093ea486f53a025f6972bb444f2da37e0e2fc3b2 4eb5b24c64ee5907c18371f563b79fa0caf58285a62b8f09e639a690e6727eaa
-R 1aa884bc5be6161439d6cdee8401a11e
-T +closed 4eb5b24c64ee5907c18371f563b79fa0caf58285a62b8f09e639a690e6727eaa
-U dan
-Z a1436cfe5518dde187c59fef646e7300
+P b730d187f2202e5b5d31ed6c94c9bb04d7c289f7086a9b44b3d9050ea3586d3a
+R 03c296c6343c0711547e5b14fdc5c828
+U drh
+Z 93daef2f8ab3c187bf34c937a8efe1d7
diff --git a/manifest.uuid b/manifest.uuid
index b0255f791b..c0c07f33f0 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b730d187f2202e5b5d31ed6c94c9bb04d7c289f7086a9b44b3d9050ea3586d3a
\ No newline at end of file
+88e2ce916791d488076584f3795a89eb4277fcb812af9e4c2f383815d55ff6f1
\ No newline at end of file
diff --git a/src/vdbe.c b/src/vdbe.c
index 7d1cae0caf..1bcaedbfee 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -264,6 +264,11 @@ static void applyNumericAffinity(Mem *pRec, int bTryForInt){
     pRec->flags |= MEM_Real;
     if( bTryForInt ) sqlite3VdbeIntegerAffinity(pRec);
   }
+  /* TEXT->NUMERIC is many->one.  Hence, it is important to invalidate the
+  ** string representation after computing a numeric equivalent, because the
+  ** string representation might not be the canonical representation for the
+  ** numeric value.  Ticket [343634942dd54ab57b7024] 2018-01-31. */
+  pRec->flags &= ~MEM_Str;
 }
 
 /*
diff --git a/test/indexexpr1.test b/test/indexexpr1.test
index 28c23b9089..e93dcc0cd1 100644
--- a/test/indexexpr1.test
+++ b/test/indexexpr1.test
@@ -425,4 +425,25 @@ do_execsql_test indexexpr-1510 {
   REPLACE INTO t1 SELECT a, randomblob(a) FROM t1
 } {}
 
+# 2018-01-31 https://www.sqlite.org/src/tktview/343634942dd54ab57b702411
+# When an index on an expression depends on the string representation of
+# a numeric table column, trouble can arise since there are multiple
+# string that can map to the same numeric value.  (Ex: 123, 0123, 000123).
+#
+do_execsql_test indexexpr-1600 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1 (a INTEGER, b);
+  CREATE INDEX idx1 ON t1 (lower(a));
+  INSERT INTO t1 VALUES('0001234',3);
+  PRAGMA integrity_check;
+} {ok}
+do_execsql_test indexexpr-1610 {
+  INSERT INTO t1 VALUES('1234',0),('001234',2),('01234',1);
+  SELECT b FROM t1 WHERE lower(a)='1234' ORDER BY +b;
+} {0 1 2 3}
+do_execsql_test indexexpr-1620 {
+  SELECT b FROM t1 WHERE lower(a)='01234' ORDER BY +b;
+} {}
+
+
 finish_test

From c3ef23a1c9f71ae4eb9fbab9d0fa6da21de4da62 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 1 Feb 2018 15:19:54 +0000
Subject: [PATCH 410/488] Fix a memory leak that could follow an IO error in
 the zipfile extension. And add other tests.

FossilOrigin-Name: e6bb750697c3c7ceb5ce41d216e8ef6a1d556822a3b55e0a007b4a03e194a7d9
---
 ext/misc/zipfile.c     | 33 +++++++++++++++++----------------
 manifest               | 18 +++++++++---------
 manifest.uuid          |  2 +-
 test/zipfile.test      | 37 ++++++++++++++++++++++++++++++++++++-
 test/zipfilefault.test | 15 ++++++++++++++-
 5 files changed, 77 insertions(+), 28 deletions(-)

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index 067b6d4953..dc2d831d37 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -2017,23 +2017,24 @@ void zipfileFinal(sqlite3_context *pCtx){
   u8 *aZip;
 
   p = (ZipfileCtx*)sqlite3_aggregate_context(pCtx, sizeof(ZipfileCtx));
-  if( p==0 || p->nEntry==0 ) return;
+  if( p==0 ) return;
+  if( p->nEntry>0 ){
+    memset(&eocd, 0, sizeof(eocd));
+    eocd.nEntry = p->nEntry;
+    eocd.nEntryTotal = p->nEntry;
+    eocd.nSize = p->cds.n;
+    eocd.iOffset = p->body.n;
 
-  memset(&eocd, 0, sizeof(eocd));
-  eocd.nEntry = p->nEntry;
-  eocd.nEntryTotal = p->nEntry;
-  eocd.nSize = p->cds.n;
-  eocd.iOffset = p->body.n;
-
-  nZip = p->body.n + p->cds.n + ZIPFILE_EOCD_FIXED_SZ;
-  aZip = (u8*)sqlite3_malloc(nZip);
-  if( aZip==0 ){
-    sqlite3_result_error_nomem(pCtx);
-  }else{
-    memcpy(aZip, p->body.a, p->body.n);
-    memcpy(&aZip[p->body.n], p->cds.a, p->cds.n);
-    zipfileSerializeEOCD(&eocd, &aZip[p->body.n + p->cds.n]);
-    sqlite3_result_blob(pCtx, aZip, nZip, zipfileFree);
+    nZip = p->body.n + p->cds.n + ZIPFILE_EOCD_FIXED_SZ;
+    aZip = (u8*)sqlite3_malloc(nZip);
+    if( aZip==0 ){
+      sqlite3_result_error_nomem(pCtx);
+    }else{
+      memcpy(aZip, p->body.a, p->body.n);
+      memcpy(&aZip[p->body.n], p->cds.a, p->cds.n);
+      zipfileSerializeEOCD(&eocd, &aZip[p->body.n + p->cds.n]);
+      sqlite3_result_blob(pCtx, aZip, nZip, zipfileFree);
+    }
   }
 
   sqlite3_free(p->body.a);
diff --git a/manifest b/manifest
index 96a5ce3aff..7f7cb2415b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C When\san\sindex\sis\sbased\son\sa\stext\srepresentation\sof\sa\snumeric\scolumn\sin\sthe\noriginal\stable,\smake\ssure\sthe\sindexed\svalue\suses\sthe\scanonical\stext\s\nrepresentation\sof\sthe\snumeric\svalue\sin\sthe\stable.\nProposed\sfix\sfor\sticket\s[343634942dd54ab57b70].
-D 2018-02-01T01:13:33.559
+C Fix\sa\smemory\sleak\sthat\scould\sfollow\san\sIO\serror\sin\sthe\szipfile\sextension.\sAnd\nadd\sother\stests.
+D 2018-02-01T15:19:54.965
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -304,7 +304,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c 56028f7e74d948b7bef834624d128f563eb63f5950248d538afa9d0f6c3365dc
+F ext/misc/zipfile.c 28f06977290c28361dcb7279016194c8e632c841e8586eb1520bbf494f487a5f
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -1603,9 +1603,9 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf
 F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc
 F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa
 F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc
-F test/zipfile.test a16528022b08789168adc9c79646e41fb55c4f94b26a2aca97085ebc591ca84d
+F test/zipfile.test 37cc584afebc6b64691a5df13deef0623f4bb42be21f1421930c34d9817ba1f2
 F test/zipfile2.test 5f93611307c131e83f226a471231d769b794b9e8c6a675cfa3d34b1a79df23fe
-F test/zipfilefault.test df4fa9e16116e0cb21d09037a6b0a7d93fecd8767d82154b66044b4ca43fca58
+F test/zipfilefault.test e287d6783d95b7bbad3478701834a353675f48502f95d6d58e07d954b8278639
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d
 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91
@@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b730d187f2202e5b5d31ed6c94c9bb04d7c289f7086a9b44b3d9050ea3586d3a
-R 03c296c6343c0711547e5b14fdc5c828
-U drh
-Z 93daef2f8ab3c187bf34c937a8efe1d7
+P 88e2ce916791d488076584f3795a89eb4277fcb812af9e4c2f383815d55ff6f1
+R 59b5173dc10fbd02f5f4a286751c0af4
+U dan
+Z 2f13a257a067155eca38f77c87e4d0a0
diff --git a/manifest.uuid b/manifest.uuid
index c0c07f33f0..1c106ca118 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-88e2ce916791d488076584f3795a89eb4277fcb812af9e4c2f383815d55ff6f1
\ No newline at end of file
+e6bb750697c3c7ceb5ce41d216e8ef6a1d556822a3b55e0a007b4a03e194a7d9
\ No newline at end of file
diff --git a/test/zipfile.test b/test/zipfile.test
index 92d8501ea8..bcfc4b2b1e 100644
--- a/test/zipfile.test
+++ b/test/zipfile.test
@@ -423,7 +423,15 @@ ifcapable datetime {
     test_unzip/c.txt 1111111000 
   }]
 
-  do_test 6.1 {
+  do_execsql_test 6.1 {
+    SELECT name, mtime, data FROM zipfile('test1.zip')
+  } {
+    a.txt 946684800   abc
+    b.txt 1000000000  abc
+    c.txt 1111111000  abc
+  }
+
+  do_test 6.2 {
     forcedelete test_unzip
     file mkdir test_unzip
     exec unzip -d test_unzip test2.zip
@@ -438,9 +446,36 @@ ifcapable datetime {
     test_unzip/b.txt 1000000000 
     test_unzip/c.txt 1111111000 
   }]
+
+  do_execsql_test 6.3 {
+    SELECT name, mtime, data FROM zipfile('test2.zip')
+  } {
+    a.txt 946684800   abc
+    b.txt 1000000000  abc
+    c.txt 1111111000  abc
+  }
 }
 }
 
+#-------------------------------------------------------------------------
+forcedelete test.zip
+do_test 7.0 {
+  execsql {
+    WITH c(name,data) AS (
+        SELECT '1', randomblob(1000000) UNION ALL
+        SELECT '2', randomblob(1000000) UNION ALL
+        SELECT '3', randomblob(1000000) 
+    )
+    SELECT writefile('test.zip', zipfile(name, data) ) FROM c;
+  }
+
+  list [catch {
+    db eval { SELECT name, data FROM zipfile('test.zip') } {
+      if {$name==2} { close [open test.zip w+] }
+    }
+  } msg] $msg
+} {1 {error in fread()}}
+
 
 finish_test
 
diff --git a/test/zipfilefault.test b/test/zipfilefault.test
index 9f0f4784dc..b2c6d7cfc4 100644
--- a/test/zipfilefault.test
+++ b/test/zipfilefault.test
@@ -56,12 +56,25 @@ do_execsql_test 3.0 {
   INSERT INTO setup(name, data) VALUES('a.txt', 'aaaaaaaaaaaaaaaaaaaaaaaaaaaa');
 }
 
-do_faultsim_test 2 -faults oom* -body {
+do_faultsim_test 3 -faults oom* -body {
   execsql { SELECT name,data FROM zipfile('test.zip') }
 } -test {
   faultsim_test_result {0 {a.txt aaaaaaaaaaaaaaaaaaaaaaaaaaaa}} 
 }
 
+do_faultsim_test 4 -faults oom* -body {
+  execsql {
+    WITH c(n, d) AS (
+      SELECT 1, 'aaaaaaaaaaabbbbbbbbbbaaaaaaaaaabbbbbbbbbb'
+    )
+    SELECT name, data FROM zipfile(
+      (SELECT zipfile(n, d) FROM c)
+    );
+  }
+} -test {
+  faultsim_test_result {0 {1 aaaaaaaaaaabbbbbbbbbbaaaaaaaaaabbbbbbbbbb}}
+}
+
 
 finish_test
 

From 563ddbe54d01079afa793a5eb25de2df0f1068ef Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 1 Feb 2018 15:57:00 +0000
Subject: [PATCH 411/488] New assert() statements to help ensure that no other
 errors similar to [343634942dd54ab57b7] ever appear in the code.

FossilOrigin-Name: 5a70af1e9c567f12c997d25d0a305a8d42bf2cc92f2811e9d5fdde720665e213
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/vdbemem.c | 47 +++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 55 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 7f7cb2415b..d213885632 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\smemory\sleak\sthat\scould\sfollow\san\sIO\serror\sin\sthe\szipfile\sextension.\sAnd\nadd\sother\stests.
-D 2018-02-01T15:19:54.965
+C New\sassert()\sstatements\sto\shelp\sensure\sthat\sno\sother\serrors\ssimilar\nto\s[343634942dd54ab57b7]\sever\sappear\sin\sthe\scode.
+D 2018-02-01T15:57:00.046
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -562,7 +562,7 @@ F src/vdbeInt.h 8d7d07f13cb3c4cbca91e22ba4a1920e542dda7c5d9299920432a0b3d5b009f5
 F src/vdbeapi.c fea41171884a4de119f8b10ab514c788674eeeb7f27218bb6d008e1310bfd07f
 F src/vdbeaux.c 2756ac68ac259c416554100598fc291870063288cd7e1af22847f57b3e130e56
 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191
-F src/vdbemem.c 943e41881e6317c9f93c77c1d60d3b37ddc8d26a3f852233ce7423d3e581523e
+F src/vdbemem.c 21c1d00f0b901dda0a2eccf56bf0d86648cae1bde37de0606ae4e7cdb0ff6d3d
 F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a
@@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 88e2ce916791d488076584f3795a89eb4277fcb812af9e4c2f383815d55ff6f1
-R 59b5173dc10fbd02f5f4a286751c0af4
-U dan
-Z 2f13a257a067155eca38f77c87e4d0a0
+P e6bb750697c3c7ceb5ce41d216e8ef6a1d556822a3b55e0a007b4a03e194a7d9
+R 296950b2c8b3ce7a358fc83523609b8d
+U drh
+Z f52624ceb0e0d6c17658803080172034
diff --git a/manifest.uuid b/manifest.uuid
index 1c106ca118..d7e32f11f1 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e6bb750697c3c7ceb5ce41d216e8ef6a1d556822a3b55e0a007b4a03e194a7d9
\ No newline at end of file
+5a70af1e9c567f12c997d25d0a305a8d42bf2cc92f2811e9d5fdde720665e213
\ No newline at end of file
diff --git a/src/vdbemem.c b/src/vdbemem.c
index e2912b3a6f..c02370c738 100644
--- a/src/vdbemem.c
+++ b/src/vdbemem.c
@@ -93,6 +93,51 @@ int sqlite3VdbeCheckMemInvariants(Mem *p){
 }
 #endif
 
+#ifdef SQLITE_DEBUG
+/*
+** Check that string value of pMem agrees with its integer or real value.
+**
+** A single int or real value always converts to the same strings.  But
+** many different strings can be converted into the same int or real.
+** If a table contains a numeric value and an index is based on the
+** corresponding string value, then it is important that the string be
+** derived from the numeric value, not the other way around, to ensure
+** that the index and table are consistent.  See ticket
+** https://www.sqlite.org/src/info/343634942dd54ab (2018-01-31) for
+** an example.
+**
+** This routine looks at pMem to verify that if it has both a numeric
+** representation and a string representation then the string rep has
+** been derived from the numeric and not the other way around.  It returns
+** true if everything is ok and false if there is a problem.
+**
+** This routine is for use inside of assert() statements only.
+*/
+int sqlite3VdbeMemConsistentDualRep(Mem *p){
+  char zBuf[100];
+  char *z;
+  int i, j, incr;
+  if( (p->flags & MEM_Str)==0 ) return 1;
+  if( (p->flags & (MEM_Int|MEM_Real))==0 ) return 1;
+  if( p->flags & MEM_Int ){
+    sqlite3_snprintf(sizeof(zBuf),zBuf,"%lld",p->u.i);
+  }else{
+    sqlite3_snprintf(sizeof(zBuf),zBuf,"%!.15g",p->u.r);
+  }
+  z = p->z;
+  i = j = 0;
+  incr = 1;
+  if( p->enc!=SQLITE_UTF8 ){
+    incr = 2;
+    if( p->enc==SQLITE_UTF16BE ) z++;
+  }
+  while( zBuf[j] ){
+    if( zBuf[j++]!=z[i] ) return 0;
+    i += incr;
+  }
+  return 1;
+}
+#endif /* SQLITE_DEBUG */
 
 /*
 ** If pMem is an object with a valid string representation, this routine
@@ -1096,6 +1141,7 @@ static SQLITE_NOINLINE const void *valueToText(sqlite3_value* pVal, u8 enc){
   assert(pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) || pVal->db==0
               || pVal->db->mallocFailed );
   if( pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) ){
+    assert( sqlite3VdbeMemConsistentDualRep(pVal) );
     return pVal->z;
   }else{
     return 0;
@@ -1118,6 +1164,7 @@ const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){
   assert( (enc&3)==(enc&~SQLITE_UTF16_ALIGNED) );
   assert( (pVal->flags & MEM_RowSet)==0 );
   if( (pVal->flags&(MEM_Str|MEM_Term))==(MEM_Str|MEM_Term) && pVal->enc==enc ){
+    assert( sqlite3VdbeMemConsistentDualRep(pVal) );
     return pVal->z;
   }
   if( pVal->flags&MEM_Null ){

From 15daa6b53ed3dd90b772deba903ab27ff790baf9 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 1 Feb 2018 19:41:23 +0000
Subject: [PATCH 412/488] Fix a problem triggered when a zipfile virtual table
 is created and written to within the same transaction. And add other zipfile
 test.

FossilOrigin-Name: 48f1c556994d7f8f359c649a1da81eec02306106b68946a9a20b276742c4610d
---
 ext/misc/zipfile.c     |  99 +++++++++++++++-----------------
 manifest               |  20 +++----
 manifest.uuid          |   2 +-
 test/zipfile.test      | 127 +++++++++++++++++++++++++++++++++++++++--
 test/zipfile2.test     |  38 ++++++++++++
 test/zipfilefault.test |  54 +++++++++++++++++-
 6 files changed, 270 insertions(+), 70 deletions(-)

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index dc2d831d37..441219565f 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -1073,7 +1073,8 @@ static int zipfileColumn(
     case 6:   /* method */
       sqlite3_result_int(ctx, pCDS->iCompression);
       break;
-    case 7:   /* z */
+    default:  /* z */
+      assert( i==7 );
       sqlite3_result_int64(ctx, pCsr->iId);
       break;
   }
@@ -1217,7 +1218,7 @@ static int zipfileFilter(
 ){
   ZipfileTab *pTab = (ZipfileTab*)cur->pVtab;
   ZipfileCsr *pCsr = (ZipfileCsr*)cur;
-  const char *zFile;              /* Zip file to scan */
+  const char *zFile = 0;          /* Zip file to scan */
   int rc = SQLITE_OK;             /* Return Code */
   int bInMemory = 0;              /* True for an in-memory zipfile */
 
@@ -1226,7 +1227,8 @@ static int zipfileFilter(
   if( pTab->zFile ){
     zFile = pTab->zFile;
   }else if( idxNum==0 ){
-    bInMemory = 1;
+    zipfileSetErrmsg(pCsr, "zipfile() function requires an argument");
+    return SQLITE_ERROR;
   }else if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){
     const u8 *aBlob = (const u8*)sqlite3_value_blob(argv[0]);
     int nBlob = sqlite3_value_bytes(argv[0]);
@@ -1295,14 +1297,11 @@ static int zipfileBestIndex(
   return SQLITE_OK;
 }
 
-static ZipfileEntry *zipfileNewEntry(const char *zPath, int nData){
+static ZipfileEntry *zipfileNewEntry(const char *zPath){
   ZipfileEntry *pNew;
-  pNew = sqlite3_malloc(sizeof(ZipfileEntry) + nData);
+  pNew = sqlite3_malloc(sizeof(ZipfileEntry));
   if( pNew ){
     memset(pNew, 0, sizeof(ZipfileEntry));
-    if( nData ){
-      pNew->aData = (u8*)&pNew[1];
-    }
     pNew->cds.zFile = sqlite3_mprintf("%s", zPath);
     if( pNew->cds.zFile==0 ){
       sqlite3_free(pNew);
@@ -1358,6 +1357,7 @@ static int zipfileAppendEntry(
   nBuf = zipfileSerializeLFH(pEntry, aBuf);
   rc = zipfileAppendData(pTab, aBuf, nBuf);
   if( rc==SQLITE_OK ){
+    pEntry->iDataOff = pTab->szCurrent;
     rc = zipfileAppendData(pTab, pData, nData);
   }
 
@@ -1418,6 +1418,35 @@ static int zipfileComparePath(const char *zA, const char *zB, int nB){
   return 1;
 }
 
+static int zipfileBegin(sqlite3_vtab *pVtab){
+  ZipfileTab *pTab = (ZipfileTab*)pVtab;
+  int rc = SQLITE_OK;
+
+  assert( pTab->pWriteFd==0 );
+
+  /* Open a write fd on the file. Also load the entire central directory
+  ** structure into memory. During the transaction any new file data is 
+  ** appended to the archive file, but the central directory is accumulated
+  ** in main-memory until the transaction is committed.  */
+  pTab->pWriteFd = fopen(pTab->zFile, "ab+");
+  if( pTab->pWriteFd==0 ){
+    pTab->base.zErrMsg = sqlite3_mprintf(
+        "zipfile: failed to open file %s for writing", pTab->zFile
+        );
+    rc = SQLITE_ERROR;
+  }else{
+    fseek(pTab->pWriteFd, 0, SEEK_END);
+    pTab->szCurrent = pTab->szOrig = (i64)ftell(pTab->pWriteFd);
+    rc = zipfileLoadDirectory(pTab, 0, 0);
+  }
+
+  if( rc!=SQLITE_OK ){
+    zipfileCleanupTransaction(pTab);
+  }
+
+  return rc;
+}
+
 /*
 ** xUpdate method.
 */
@@ -1445,7 +1474,10 @@ static int zipfileUpdate(
   int bIsDir = 0;
   u32 iCrc32 = 0;
 
-  assert( (pTab->zFile==0)==(pTab->pWriteFd==0) );
+  if( pTab->pWriteFd==0 ){
+    rc = zipfileBegin(pVtab);
+    if( rc!=SQLITE_OK ) return rc;
+  }
 
   /* If this is a DELETE or UPDATE, find the archive entry to delete. */
   if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
@@ -1541,7 +1573,7 @@ static int zipfileUpdate(
 
     if( rc==SQLITE_OK ){
       /* Create the new CDS record. */
-      pNew = zipfileNewEntry(zPath, pTab->zFile ? 0 : (nData+1));
+      pNew = zipfileNewEntry(zPath);
       if( pNew==0 ){
         rc = SQLITE_NOMEM;
       }else{
@@ -1557,11 +1589,7 @@ static int zipfileUpdate(
         pNew->cds.iOffset = (u32)pTab->szCurrent;
         pNew->cds.nFile = nPath;
         pNew->mUnixTime = (u32)mTime;
-        if( pTab->zFile ){
-          rc = zipfileAppendEntry(pTab, pNew, pData, nData);
-        }else{
-          memcpy(pNew->aData, pData, nData);
-        }
+        rc = zipfileAppendEntry(pTab, pNew, pData, nData);
         zipfileAddEntry(pTab, pOld, pNew);
       }
     }
@@ -1606,36 +1634,6 @@ static int zipfileAppendEOCD(ZipfileTab *pTab, ZipfileEOCD *p){
   return zipfileAppendData(pTab, pTab->aBuffer, nBuf);
 }
 
-static int zipfileBegin(sqlite3_vtab *pVtab){
-  ZipfileTab *pTab = (ZipfileTab*)pVtab;
-  int rc = SQLITE_OK;
-
-  assert( pTab->pWriteFd==0 );
-  if( pTab->zFile ){
-    /* Open a write fd on the file. Also load the entire central directory
-    ** structure into memory. During the transaction any new file data is 
-    ** appended to the archive file, but the central directory is accumulated
-    ** in main-memory until the transaction is committed.  */
-    pTab->pWriteFd = fopen(pTab->zFile, "ab+");
-    if( pTab->pWriteFd==0 ){
-      pTab->base.zErrMsg = sqlite3_mprintf(
-          "zipfile: failed to open file %s for writing", pTab->zFile
-      );
-      rc = SQLITE_ERROR;
-    }else{
-      fseek(pTab->pWriteFd, 0, SEEK_END);
-      pTab->szCurrent = pTab->szOrig = (i64)ftell(pTab->pWriteFd);
-      rc = zipfileLoadDirectory(pTab, 0, 0);
-    }
-
-    if( rc!=SQLITE_OK ){
-      zipfileCleanupTransaction(pTab);
-    }
-  }
-
-  return rc;
-}
-
 /*
 ** Serialize the CDS structure into buffer aBuf[]. Return the number
 ** of bytes written.
@@ -1785,14 +1783,11 @@ static int zipfileFindFunction(
   void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */
   void **ppArg                    /* OUT: User data for *pxFunc */
 ){
-  if( nArg>0 ){
-    if( sqlite3_stricmp("zipfile_cds", zName)==0 ){
-      *pxFunc = zipfileFunctionCds;
-      *ppArg = (void*)pVtab;
-      return 1;
-    }
+  if( sqlite3_stricmp("zipfile_cds", zName)==0 ){
+    *pxFunc = zipfileFunctionCds;
+    *ppArg = (void*)pVtab;
+    return 1;
   }
-
   return 0;
 }
 
diff --git a/manifest b/manifest
index d213885632..e4aa3f1840 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C New\sassert()\sstatements\sto\shelp\sensure\sthat\sno\sother\serrors\ssimilar\nto\s[343634942dd54ab57b7]\sever\sappear\sin\sthe\scode.
-D 2018-02-01T15:57:00.046
+C Fix\sa\sproblem\striggered\swhen\sa\szipfile\svirtual\stable\sis\screated\sand\swritten\sto\nwithin\sthe\ssame\stransaction.\sAnd\sadd\sother\szipfile\stest.
+D 2018-02-01T19:41:23.066
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -304,7 +304,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c 28f06977290c28361dcb7279016194c8e632c841e8586eb1520bbf494f487a5f
+F ext/misc/zipfile.c 2856e99ce4b45669edbdc070649a7c6ea0348979d9297ab682f3c552078d47c7
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -1603,9 +1603,9 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf
 F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc
 F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa
 F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc
-F test/zipfile.test 37cc584afebc6b64691a5df13deef0623f4bb42be21f1421930c34d9817ba1f2
-F test/zipfile2.test 5f93611307c131e83f226a471231d769b794b9e8c6a675cfa3d34b1a79df23fe
-F test/zipfilefault.test e287d6783d95b7bbad3478701834a353675f48502f95d6d58e07d954b8278639
+F test/zipfile.test 1f066994bd77493c87e8ee4bd94db0651d180cff19ffbbe0b70085eb9a2cb34c
+F test/zipfile2.test 67d5f08a202796d4b7a71dfa4b8dcb74aa7a9d1f42c5f17bedff9855c1ba7aa5
+F test/zipfilefault.test 73b08e3d0bbeb275e325ee7e3678ca98781de0737f9153ca23bde1f48a93d728
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d
 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91
@@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e6bb750697c3c7ceb5ce41d216e8ef6a1d556822a3b55e0a007b4a03e194a7d9
-R 296950b2c8b3ce7a358fc83523609b8d
-U drh
-Z f52624ceb0e0d6c17658803080172034
+P 5a70af1e9c567f12c997d25d0a305a8d42bf2cc92f2811e9d5fdde720665e213
+R ec6ac070bff5134b5a74e8c3afc721f5
+U dan
+Z 34f839c0a19e1ff6432f6acef976c132
diff --git a/manifest.uuid b/manifest.uuid
index d7e32f11f1..be5881914d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5a70af1e9c567f12c997d25d0a305a8d42bf2cc92f2811e9d5fdde720665e213
\ No newline at end of file
+48f1c556994d7f8f359c649a1da81eec02306106b68946a9a20b276742c4610d
\ No newline at end of file
diff --git a/test/zipfile.test b/test/zipfile.test
index bcfc4b2b1e..b31077d33c 100644
--- a/test/zipfile.test
+++ b/test/zipfile.test
@@ -151,10 +151,18 @@ do_catchsql_test 1.1.0.1 {
   INSERT INTO zz(name, mode, mtime, sz, rawdata, method) 
   VALUES('f.txt', '-rw-r--r--', 1000000000, 5, 'abcde', 0);
 } {1 {constraint failed}}
-do_catchsql_test 1.1.0.1 {
-  INSERT INTO zz(name, mtime, sz, rawdata, method) 
+do_catchsql_test 1.1.0.2 {
+  INSERT INTO zz(name, mtime, sz, data, method) 
   VALUES('g.txt', 1000000002, 5, '12345', 0);
 } {1 {constraint failed}}
+do_catchsql_test 1.1.0.3 {
+  INSERT INTO zz(name, mtime, rawdata, method) 
+  VALUES('g.txt', 1000000002, '12345', 0);
+} {1 {constraint failed}}
+do_catchsql_test 1.1.0.4 {
+  INSERT INTO zz(name, data, method) 
+  VALUES('g.txt', '12345', 7);
+} {1 {constraint failed}}
 
 do_execsql_test 1.1.1 {
   INSERT INTO zz(name, mode, mtime, data, method) 
@@ -198,6 +206,9 @@ ifcapable json1 {
     h.txt 1
   }
 }
+do_catchsql_test 1.4.2 {
+  SELECT zipfile_cds(mode) FROM zipfile('test.zip');
+} {0 {{} {} {}}}
 
 do_execsql_test 1.5.1 {
   BEGIN;
@@ -285,6 +296,37 @@ do_execsql_test 1.6.8 {
   i.txt 33188 4 zxcvb 0
 }
 
+do_execsql_test 1.6.8 {
+  UPDATE zz SET data = '' WHERE name='i.txt';
+  SELECT name,mode,mtime,data,method from zipfile('test.zip');
+} {
+  blue.txt/ 16877 1000000000 {} 0
+  h.txt 33189 1000000004 aaaaaaaaaabbbbbbbbbb 8
+  i.txt 33188 4 {} 0
+}
+
+do_execsql_test 1.6.9 {
+  SELECT a.name, a.data 
+  FROM zz AS a, zz AS b 
+  WHERE a.name=+b.name AND +a.mode=b.mode
+} {
+  blue.txt/ {}
+  h.txt aaaaaaaaaabbbbbbbbbb
+  i.txt {}
+}
+
+do_execsql_test 1.6.10 {
+  SELECT name, data FROM zz WHERE name LIKE '%txt'
+} {
+  h.txt aaaaaaaaaabbbbbbbbbb
+  i.txt {}
+}
+
+do_execsql_test 1.7 {
+  DELETE FROM zz;
+  SELECT * FROM zz;
+} {}
+
 #-------------------------------------------------------------------------
 db close
 forcedelete test.zip
@@ -379,6 +421,25 @@ do_catchsql_test 4.2 {
   CREATE VIRTUAL TABLE yyy USING zipfile('test.zip', 'test.zip');
 } {1 {zipfile constructor requires one argument}}
 
+do_catchsql_test 4.3 {
+  SELECT * FROM zipfile()
+} {1 {zipfile() function requires an argument}}
+
+do_catchsql_test 4.4 {
+  SELECT * FROM zipfile('/path/that/does/not/exist')
+} {1 {cannot open file: /path/that/does/not/exist}}
+
+foreach {tn mode} {
+  1 abcd
+  2 brwxrwxrwx
+  3 lrwxrrxrwx
+} {
+  do_catchsql_test 4.5.$tn {
+    WITH m(m) AS ( SELECT $mode)
+    SELECT zipfile('a.txt', m, 1000, 'xyz') FROM m
+  } [list 1 "zipfile: parse error in mode: $mode"]
+}
+
 #--------------------------------------------------------------------------
 
 db func rt remove_timestamps
@@ -448,16 +509,18 @@ ifcapable datetime {
   }]
 
   do_execsql_test 6.3 {
-    SELECT name, mtime, data FROM zipfile('test2.zip')
+    SELECT name, mtime, sz, rawdata, data FROM zipfile('test2.zip')
   } {
-    a.txt 946684800   abc
-    b.txt 1000000000  abc
-    c.txt 1111111000  abc
+    a.txt 946684800   3 abc abc
+    b.txt 1000000000  3 abc abc
+    c.txt 1111111000  3 abc abc
   }
 }
 }
 
 #-------------------------------------------------------------------------
+# Force an IO error by truncating the zip archive to zero bytes in size
+# while it is being read.
 forcedelete test.zip
 do_test 7.0 {
   execsql {
@@ -476,6 +539,58 @@ do_test 7.0 {
   } msg] $msg
 } {1 {error in fread()}}
 
+forcedelete test.zip
+do_execsql_test 8.0.1 {
+  CREATE VIRTUAL TABLE zz USING zipfile('test.zip');
+  BEGIN;
+    INSERT INTO zz(name, data) VALUES('a.txt', '1');
+    INSERT INTO zz(name, data) VALUES('b.txt', '2');
+    INSERT INTO zz(name, data) VALUES('c.txt', '1');
+    INSERT INTO zz(name, data) VALUES('d.txt', '2');
+    SELECT name, data FROM zz;
+} {
+  a.txt 1 b.txt 2 c.txt 1 d.txt 2
+}
+do_test 8.0.2 {
+  db eval { SELECT name, data FROM zz } {
+    if { $data=="2" } { db eval { DELETE FROM zz WHERE name=$name } }
+  }
+  execsql { SELECT name, data FROM zz } 
+} {a.txt 1 c.txt 1}
+do_test 8.0.3 {
+  db eval { SELECT name, data FROM zz } {
+    db eval { DELETE FROM zz WHERE name=$name }
+  }
+  execsql { SELECT name, data FROM zz } 
+} {}
+execsql COMMIT
+
+do_execsql_test 8.1.1 {
+  CREATE VIRTUAL TABLE nogood USING zipfile('test_unzip');
+}
+do_catchsql_test 8.1.2 {
+  INSERT INTO nogood(name, data) VALUES('abc', 'def');
+} {1 {zipfile: failed to open file test_unzip for writing}}
+
+do_execsql_test 8.2.1 {
+  DROP TABLE nogood;
+  BEGIN;
+    CREATE VIRTUAL TABLE nogood USING zipfile('test_unzip');
+}
+do_catchsql_test 8.2.2 {
+    INSERT INTO nogood(name, data) VALUES('abc', 'def');
+} {1 {zipfile: failed to open file test_unzip for writing}}
+do_execsql_test 8.2.3 {
+  COMMIT;
+}
+
+forcedelete test.zip
+do_execsql_test 8.3.1 {
+  BEGIN;
+    CREATE VIRTUAL TABLE ok USING zipfile('test.zip');
+    INSERT INTO ok(name, data) VALUES ('sqlite3', 'elf');
+  COMMIT;
+}
 
 finish_test
 
diff --git a/test/zipfile2.test b/test/zipfile2.test
index a6c3e7e3c4..f3509d0a54 100644
--- a/test/zipfile2.test
+++ b/test/zipfile2.test
@@ -164,6 +164,44 @@ do_catchsql_test 4.1 {
   SELECT name,mtime,data,method FROM zipfile($blob)
 } {1 {inflate() failed (0)}}
 
+# Check the response to an unknown compression method (set data to NULL).
+set blob [blob [string map {0800 0900} $archive2]]
+do_execsql_test 4.2 {
+  SELECT name,mtime,data IS NULL,method FROM zipfile($blob)
+} {a.txt 1000000 1 9}
+
+# Corrupt the EOCDS signature bytes in various ways.
+foreach {tn sub} {
+  1 {504B0500}
+  2 {504B0006}
+  3 {50000506}
+  4 {004B0506}
+} {
+  set blob [blob [string map [list 504B0506 $sub] $archive2]]
+  do_catchsql_test 4.3.$tn {
+    SELECT * FROM zipfile($blob)
+  } {1 {cannot find end of central directory record}}
+}
+
+#-------------------------------------------------------------------------
+# Test that a zero-length file with a '/' at the end is treated as
+# a directory (data IS NULL). Even if the mode doesn't indicate
+# that it is a directory.
+
+do_test 5.0 {
+  set blob [db one {
+    WITH c(n, d) AS (
+      SELECT 'notadir', ''
+    )
+    SELECT zipfile(n, d) FROM c
+ }]
+
+  set hex [binary encode hex $blob]
+  set hex [string map {6e6f7461646972 6e6f746164692f} $hex] 
+  set blob2 [binary decode hex $hex]
+
+  execsql { SELECT name, data IS NULL FROM zipfile($blob2) }
+} {notadi/ 1}
 
 
 finish_test
diff --git a/test/zipfilefault.test b/test/zipfilefault.test
index b2c6d7cfc4..a850b5146a 100644
--- a/test/zipfilefault.test
+++ b/test/zipfilefault.test
@@ -42,11 +42,19 @@ do_execsql_test 2.0 {
   INSERT INTO setup(name, data) VALUES('a.txt', '1234567890');
 }
 
-do_faultsim_test 2 -faults oom* -body {
+do_faultsim_test 2.1 -faults oom* -body {
   execsql { SELECT name,data FROM zipfile('test.zip') }
 } -test {
   faultsim_test_result {0 {a.txt 1234567890}} 
 }
+do_faultsim_test 2.2 -faults oom* -body {
+  execsql { 
+    SELECT json_extract( zipfile_cds(z), '$.version-made-by' ) 
+    FROM zipfile('test.zip')
+  }
+} -test {
+  faultsim_test_result {0 798}
+}
 
 forcedelete test.zip
 reset_db
@@ -75,6 +83,50 @@ do_faultsim_test 4 -faults oom* -body {
   faultsim_test_result {0 {1 aaaaaaaaaaabbbbbbbbbbaaaaaaaaaabbbbbbbbbb}}
 }
 
+reset_db
+load_static_extension db zipfile
+
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE setup USING zipfile('test.zip') 
+}
+
+do_faultsim_test 5.1 -faults oom* -prep {
+  forcedelete test.zip
+} -body {
+  execsql {
+    INSERT INTO setup(name, data) 
+    VALUES('a.txt', 'aaaaaaaaaaaaaaaaaaaaaaaaaaaa');
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+do_faultsim_test 5.2 -faults oom* -prep {
+  forcedelete test.zip
+} -body {
+  execsql {
+    INSERT INTO setup(name, data) VALUES('dir', NULL)
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+do_faultsim_test 5.3 -faults oom* -prep {
+  forcedelete test.zip
+  execsql { 
+    DROP TABLE IF EXISTS setup;
+    BEGIN;
+      CREATE VIRTUAL TABLE setup USING zipfile('test.zip') 
+  }
+} -body {
+  execsql {
+    INSERT INTO setup(name, data) VALUES('dir', NULL)
+  }
+} -test {
+  catchsql { COMMIT }
+  faultsim_test_result {0 {}}
+}
+
 
 finish_test
 

From 668845bb0bb831b98577b0261303a376d14ceeb0 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 1 Feb 2018 20:42:23 +0000
Subject: [PATCH 413/488] Add a few more zipfile tests. No changes to code.

FossilOrigin-Name: 3f621545879ea39502cfaf2b61883f92c077070274b4fdf45524ded81cac3e63
---
 manifest               | 14 ++++++-------
 manifest.uuid          |  2 +-
 test/zipfile.test      | 46 +++++++++++++++++++++++++++++++++++++++++-
 test/zipfilefault.test | 33 ++++++++++++++++++++++++++++++
 4 files changed, 86 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index e4aa3f1840..868870f090 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sproblem\striggered\swhen\sa\szipfile\svirtual\stable\sis\screated\sand\swritten\sto\nwithin\sthe\ssame\stransaction.\sAnd\sadd\sother\szipfile\stest.
-D 2018-02-01T19:41:23.066
+C Add\sa\sfew\smore\szipfile\stests.\sNo\schanges\sto\scode.
+D 2018-02-01T20:42:23.543
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -1603,9 +1603,9 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf
 F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc
 F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa
 F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc
-F test/zipfile.test 1f066994bd77493c87e8ee4bd94db0651d180cff19ffbbe0b70085eb9a2cb34c
+F test/zipfile.test 44aa8af115cc3e8c905468768dc761260650a8fdfca57e10f9818f5f8008d340
 F test/zipfile2.test 67d5f08a202796d4b7a71dfa4b8dcb74aa7a9d1f42c5f17bedff9855c1ba7aa5
-F test/zipfilefault.test 73b08e3d0bbeb275e325ee7e3678ca98781de0737f9153ca23bde1f48a93d728
+F test/zipfilefault.test 050be76778d2ec83566a542800fc3967f65ce719671a819016ada8b654bc14f7
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d
 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91
@@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5a70af1e9c567f12c997d25d0a305a8d42bf2cc92f2811e9d5fdde720665e213
-R ec6ac070bff5134b5a74e8c3afc721f5
+P 48f1c556994d7f8f359c649a1da81eec02306106b68946a9a20b276742c4610d
+R cdbed97cc578ccd9e756b9f206f65092
 U dan
-Z 34f839c0a19e1ff6432f6acef976c132
+Z 07c3c59583bca5bfb67bba61fc8753c1
diff --git a/manifest.uuid b/manifest.uuid
index be5881914d..3a4e6a6861 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-48f1c556994d7f8f359c649a1da81eec02306106b68946a9a20b276742c4610d
\ No newline at end of file
+3f621545879ea39502cfaf2b61883f92c077070274b4fdf45524ded81cac3e63
\ No newline at end of file
diff --git a/test/zipfile.test b/test/zipfile.test
index b31077d33c..2bb3f07892 100644
--- a/test/zipfile.test
+++ b/test/zipfile.test
@@ -440,6 +440,37 @@ foreach {tn mode} {
   } [list 1 "zipfile: parse error in mode: $mode"]
 }
 
+do_catchsql_test 4.6 {
+  WITH c(name,data) AS ( SELECT 'a.txt', 'abc')
+  SELECT zipfile(name) FROM c
+} {1 {wrong number of arguments to function zipfile()}}
+
+do_catchsql_test 4.7 {
+  WITH c(name,data) AS ( 
+    SELECT 'a.txt', 'abc' UNION ALL
+    SELECT NULL, 'def'
+  )
+  SELECT zipfile(name,data) FROM c
+} {1 {first argument to zipfile() must be non-NULL}}
+
+do_catchsql_test 4.7 {
+  WITH c(name,data,method) AS ( 
+    SELECT 'a.txt', 'abc', 0
+    UNION SELECT 'b.txt', 'def', 8
+    UNION SELECT 'c.txt', 'ghi', 16
+  )
+  SELECT zipfile(name,NULL,NULL,data,method) FROM c
+} {1 {illegal method value: 16}}
+
+do_catchsql_test 4.8 {
+  WITH c(name,data) AS ( 
+    SELECT 'a.txt', 'abc'
+    UNION SELECT 'b.txt', 'def'
+    UNION SELECT 'c.txt/', 'ghi'
+  )
+  SELECT zipfile(name,NULL,NULL,data) FROM c
+} {1 {non-directory name must not end with /}}
+
 #--------------------------------------------------------------------------
 
 db func rt remove_timestamps
@@ -448,7 +479,7 @@ do_execsql_test 5.0 {
     SELECT 'a.txt', 946684800, 'abc'
   )
   SELECT name,mtime,data FROM zipfile(
-    ( SELECT rt( zipfile(name,NULL,mtime,data) ) FROM c )
+    ( SELECT rt( zipfile(name,NULL,mtime,data,NULL) ) FROM c )
   )
 } {
   a.txt 946684800 abc
@@ -592,5 +623,18 @@ do_execsql_test 8.3.1 {
   COMMIT;
 }
 
+#-------------------------------------------------------------------------
+# Test that the zipfile aggregate correctly adds and removes "/" from
+# the ends of directory file names.
+do_execsql_test 9.0 {
+  WITH src(nm) AS (
+    VALUES('dir1') UNION ALL
+    VALUES('dir2/') UNION ALL
+    VALUES('dir3//') UNION ALL
+    VALUES('dir4///') UNION ALL
+    VALUES('/') 
+  )
+  SELECT name FROM zipfile((SELECT zipfile(nm, NULL) FROM src))
+} {dir1/ dir2/ dir3/ dir4/ /}
 finish_test
 
diff --git a/test/zipfilefault.test b/test/zipfilefault.test
index a850b5146a..158370695e 100644
--- a/test/zipfilefault.test
+++ b/test/zipfilefault.test
@@ -84,6 +84,7 @@ do_faultsim_test 4 -faults oom* -body {
 }
 
 reset_db
+sqlite3_db_config_lookaside db 0 0 0
 load_static_extension db zipfile
 
 do_execsql_test 5.0 {
@@ -127,6 +128,38 @@ do_faultsim_test 5.3 -faults oom* -prep {
   faultsim_test_result {0 {}}
 }
 
+do_faultsim_test 6.1 -faults oom* -body {
+  execsql {
+    WITH c(n, d) AS (
+      VALUES('a.txt', '1234567890') UNION ALL
+      VALUES('dir', NULL)
+    )
+    SELECT zipfile(n, d) IS NULL FROM c;
+  }
+} -test {
+  faultsim_test_result {0 0}
+}
+
+set big [string repeat 0123456789 1000]
+do_faultsim_test 6.2 -faults oom* -body {
+  execsql {
+    WITH c(n, d) AS (
+      VALUES('a.txt', $big)
+    )
+    SELECT zipfile(n, NULL, NULL, d, 0) IS NULL FROM c;
+  }
+} -test {
+  faultsim_test_result {0 0}
+}
+
+do_faultsim_test 7.0 -faults oom* -prep {
+  catch { db close }
+  sqlite3 db ""
+} -body {
+  load_static_extension db zipfile
+} -test {
+}
+
 
 finish_test
 

From 1dff328119f076a1e4d63631fd57baefdc597be1 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 2 Feb 2018 16:20:41 +0000
Subject: [PATCH 414/488] Remove the time() dependency from the zipfile
 extension. Use SQLite VFS methods instead.

FossilOrigin-Name: 0702fb5611c7cf0154c5490eeada66afc733731dff0fe030375bae6a72123a17
---
 ext/misc/zipfile.c | 79 +++++++++++++++++++++++++++++++++-------------
 manifest           | 12 +++----
 manifest.uuid      |  2 +-
 3 files changed, 64 insertions(+), 29 deletions(-)

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index 441219565f..6c3f2b2373 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -30,18 +30,9 @@ SQLITE_EXTENSION_INIT1
 #include <string.h>
 #include <assert.h>
 
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#if !defined(_WIN32) && !defined(WIN32)
-#  include <unistd.h>
-#  include <dirent.h>
-#  include <utime.h>
-#else
+#if defined(_WIN32) || defined(WIN32)
 #  include <io.h>
 #endif
-#include <time.h>
-#include <errno.h>
 
 #include <zlib.h>
 
@@ -68,6 +59,25 @@ typedef unsigned long u32;
 
 #endif   /* SQLITE_AMALGAMATION */
 
+/*
+** Definitions for mode bitmasks S_IFDIR, S_IFREG and S_IFLNK.
+**
+** In some ways it would be better to obtain these values from system 
+** header files. But, the dependency is undesirable and (a) these
+** have been stable for decades, (b) the values are part of POSIX and
+** are also made explicit in [man stat], and (c) are part of the 
+** file format for zip archives.
+*/
+#ifndef S_IFDIR
+# define S_IFDIR 0040000
+#endif
+#ifndef S_IFREG
+# define S_IFREG 0100000
+#endif
+#ifndef S_IFLNK
+# define S_IFLNK 0120000
+#endif
+
 static const char ZIPFILE_SCHEMA[] = 
   "CREATE TABLE y("
     "name PRIMARY KEY,"  /* 0: Name of file in zip archive */
@@ -1447,6 +1457,39 @@ static int zipfileBegin(sqlite3_vtab *pVtab){
   return rc;
 }
 
+/*
+** Return the current time as a 32-bit timestamp in UNIX epoch format (like
+** time(2)).
+*/
+static u32 zipfileTime(void){
+  sqlite3_vfs *pVfs = sqlite3_vfs_find(0);
+  u32 ret;
+  if( pVfs->iVersion>=2 && pVfs->xCurrentTimeInt64 ){
+    i64 ms;
+    pVfs->xCurrentTimeInt64(pVfs, &ms);
+    ret = (u32)((ms/1000) - ((i64)24405875 * 8640));
+  }else{
+    double day;
+    pVfs->xCurrentTime(pVfs, &day);
+    ret = (u32)((day - 2440587.5) * 86400);
+  }
+  return ret;
+}
+
+/*
+** Return a 32-bit timestamp in UNIX epoch format.
+**
+** If the value passed as the only argument is either NULL or an SQL NULL,
+** return the current time. Otherwise, return the value stored in (*pVal)
+** cast to a 32-bit unsigned integer.
+*/
+static u32 zipfileGetTime(sqlite3_value *pVal){
+  if( pVal==0 || sqlite3_value_type(pVal)==SQLITE_NULL ){
+    return zipfileTime();
+  }
+  return (u32)sqlite3_value_int64(pVal);
+}
+
 /*
 ** xUpdate method.
 */
@@ -1461,7 +1504,7 @@ static int zipfileUpdate(
   ZipfileEntry *pNew = 0;         /* New in-memory CDS entry */
 
   u32 mode = 0;                   /* Mode for new entry */
-  i64 mTime = 0;                  /* Modification time for new entry */
+  u32 mTime = 0;                  /* Modification time for new entry */
   i64 sz = 0;                     /* Uncompressed size */
   const char *zPath = 0;          /* Path for new entry */
   int nPath = 0;                  /* strlen(zPath) */
@@ -1540,11 +1583,7 @@ static int zipfileUpdate(
     if( rc==SQLITE_OK ){
       zPath = (const char*)sqlite3_value_text(apVal[2]);
       nPath = (int)strlen(zPath);
-      if( sqlite3_value_type(apVal[4])==SQLITE_NULL ){
-        mTime = (sqlite3_int64)time(0);
-      }else{
-        mTime = sqlite3_value_int64(apVal[4]);
-      }
+      mTime = zipfileGetTime(apVal[4]);
     }
 
     if( rc==SQLITE_OK && bIsDir ){
@@ -1581,7 +1620,7 @@ static int zipfileUpdate(
         pNew->cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED;
         pNew->cds.flags = ZIPFILE_NEWENTRY_FLAGS;
         pNew->cds.iCompression = (u16)iMethod;
-        zipfileMtimeToDos(&pNew->cds, (u32)mTime);
+        zipfileMtimeToDos(&pNew->cds, mTime);
         pNew->cds.crc32 = iCrc32;
         pNew->cds.szCompressed = nData;
         pNew->cds.szUncompressed = (u32)sz;
@@ -1929,11 +1968,7 @@ void zipfileStep(sqlite3_context *pCtx, int nVal, sqlite3_value **apVal){
   if( rc ) goto zipfile_step_out;
 
   /* Decode the "mtime" argument. */
-  if( pMtime==0 || sqlite3_value_type(pMtime)==SQLITE_NULL ){
-    e.mUnixTime = (u32)time(0);
-  }else{
-    e.mUnixTime = (u32)sqlite3_value_int64(pMtime);
-  }
+  e.mUnixTime = zipfileGetTime(pMtime);
 
   /* If this is a directory entry, ensure that there is exactly one '/'
   ** at the end of the path. Or, if this is not a directory and the path
diff --git a/manifest b/manifest
index 868870f090..f4b964ef4f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sa\sfew\smore\szipfile\stests.\sNo\schanges\sto\scode.
-D 2018-02-01T20:42:23.543
+C Remove\sthe\stime()\sdependency\sfrom\sthe\szipfile\sextension.\sUse\sSQLite\sVFS\nmethods\sinstead.
+D 2018-02-02T16:20:41.841
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -304,7 +304,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c 2856e99ce4b45669edbdc070649a7c6ea0348979d9297ab682f3c552078d47c7
+F ext/misc/zipfile.c 8043b5efef0da4abba31ac021a9ea9aaf216c72547fb7f64eed9e08fd96107ff
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 48f1c556994d7f8f359c649a1da81eec02306106b68946a9a20b276742c4610d
-R cdbed97cc578ccd9e756b9f206f65092
+P 3f621545879ea39502cfaf2b61883f92c077070274b4fdf45524ded81cac3e63
+R 0244469f34b504c4ae85aff5c78a3c61
 U dan
-Z 07c3c59583bca5bfb67bba61fc8753c1
+Z ce324d9598828c350892743fba0ec306
diff --git a/manifest.uuid b/manifest.uuid
index 3a4e6a6861..70ec2da9e1 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-3f621545879ea39502cfaf2b61883f92c077070274b4fdf45524ded81cac3e63
\ No newline at end of file
+0702fb5611c7cf0154c5490eeada66afc733731dff0fe030375bae6a72123a17
\ No newline at end of file

From 3944cf8de3073675c0bc30e136652db386584e0e Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 2 Feb 2018 16:28:19 +0000
Subject: [PATCH 415/488] Remove the "include <io.h>" win32 dependency from the
 zipfile extension.

FossilOrigin-Name: cb3feb3bc33718625159eddbf8ef608cd641af84e97d433d6b11ba6da94dad2e
---
 ext/misc/zipfile.c |  4 ----
 manifest           | 13 +++++++------
 manifest.uuid      |  2 +-
 3 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index 6c3f2b2373..a94a8df3ce 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -30,10 +30,6 @@ SQLITE_EXTENSION_INIT1
 #include <string.h>
 #include <assert.h>
 
-#if defined(_WIN32) || defined(WIN32)
-#  include <io.h>
-#endif
-
 #include <zlib.h>
 
 #ifndef SQLITE_OMIT_VIRTUALTABLE
diff --git a/manifest b/manifest
index f4b964ef4f..2443207a1f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\sthe\stime()\sdependency\sfrom\sthe\szipfile\sextension.\sUse\sSQLite\sVFS\nmethods\sinstead.
-D 2018-02-02T16:20:41.841
+C Remove\sthe\s"include\s<io.h>"\swin32\sdependency\sfrom\sthe\szipfile\sextension.
+D 2018-02-02T16:28:19.035
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -304,7 +304,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c 8043b5efef0da4abba31ac021a9ea9aaf216c72547fb7f64eed9e08fd96107ff
+F ext/misc/zipfile.c 8aa0d3da5ccb360dbca578a95597bd4892ccc56b63f01447dcbb0e035720fb9f
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -1704,7 +1704,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3f621545879ea39502cfaf2b61883f92c077070274b4fdf45524ded81cac3e63
-R 0244469f34b504c4ae85aff5c78a3c61
+P 0702fb5611c7cf0154c5490eeada66afc733731dff0fe030375bae6a72123a17
+R 343c0b60986622084d28e8dfa8a41cbf
+T +closed db32506880e727597e8629c21d29a4b5e3360bec7548d98ecdbb09685fd6e7a8
 U dan
-Z ce324d9598828c350892743fba0ec306
+Z 0fa1b90106adf879fb72d19e962c14d7
diff --git a/manifest.uuid b/manifest.uuid
index 70ec2da9e1..866db96ab5 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0702fb5611c7cf0154c5490eeada66afc733731dff0fe030375bae6a72123a17
\ No newline at end of file
+cb3feb3bc33718625159eddbf8ef608cd641af84e97d433d6b11ba6da94dad2e
\ No newline at end of file

From 001f52ed7d2ecd3b9c28a662234193b121318652 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Sun, 4 Feb 2018 01:29:01 +0000
Subject: [PATCH 416/488] Move variable declaration in zipfile extension to fix
 MSVC compiler error.

FossilOrigin-Name: ac6a32482e675871c3ce5ecc727e50e4bba649e80e3c55bf99025fed5667da94
---
 ext/misc/zipfile.c |  3 ++-
 manifest           | 15 +++++++--------
 manifest.uuid      |  2 +-
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index a94a8df3ce..f88b7e22d6 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -701,6 +701,7 @@ static u32 zipfileMtime(ZipfileCDS *pCDS){
   int sec = (pCDS->mTime & 0x1F)*2;
   int min = (pCDS->mTime >> 5) & 0x3F;
   int hr = (pCDS->mTime >> 11) & 0x1F;
+  i64 JD;
 
   /* JD = INT(365.25 * (Y+4716)) + INT(30.6001 * (M+1)) + D + B - 1524.5 */
 
@@ -709,7 +710,7 @@ static u32 zipfileMtime(ZipfileCDS *pCDS){
     Y = Y-1;
     M = M+12;
   }
-  i64 JD = (i64)(24*60*60) * (
+  JD = (i64)(24*60*60) * (
       (int)(365.25 * (Y + 4716))
     + (int)(30.6001 * (M + 1))
     + D + B - 1524
diff --git a/manifest b/manifest
index 2443207a1f..757d26e001 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\sthe\s"include\s<io.h>"\swin32\sdependency\sfrom\sthe\szipfile\sextension.
-D 2018-02-02T16:28:19.035
+C Move\svariable\sdeclaration\sin\szipfile\sextension\sto\sfix\sMSVC\scompiler\serror.
+D 2018-02-04T01:29:01.007
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -304,7 +304,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c 8aa0d3da5ccb360dbca578a95597bd4892ccc56b63f01447dcbb0e035720fb9f
+F ext/misc/zipfile.c 3c3e21d0351a44c34c67ca7e833ba04e0de34421b4c9359aa764919ee36b37ef
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -1704,8 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0702fb5611c7cf0154c5490eeada66afc733731dff0fe030375bae6a72123a17
-R 343c0b60986622084d28e8dfa8a41cbf
-T +closed db32506880e727597e8629c21d29a4b5e3360bec7548d98ecdbb09685fd6e7a8
-U dan
-Z 0fa1b90106adf879fb72d19e962c14d7
+P cb3feb3bc33718625159eddbf8ef608cd641af84e97d433d6b11ba6da94dad2e
+R 8b8d4f678534b218d44a7c7eccfb3100
+U mistachkin
+Z e519063d0405ad8f0532efc454c9e10a
diff --git a/manifest.uuid b/manifest.uuid
index 866db96ab5..984b05d033 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-cb3feb3bc33718625159eddbf8ef608cd641af84e97d433d6b11ba6da94dad2e
\ No newline at end of file
+ac6a32482e675871c3ce5ecc727e50e4bba649e80e3c55bf99025fed5667da94
\ No newline at end of file

From 97f9059eee4c59afe21f9582293b24ad0d50930a Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Sun, 4 Feb 2018 01:30:54 +0000
Subject: [PATCH 417/488] Fix harmless compiler warning seen with MSVC.

FossilOrigin-Name: 63a914410b5750bd08836864ae456ff09549487c5ce49706fedffa3d4f973059
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/btree.c   |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 757d26e001..b4577329bb 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Move\svariable\sdeclaration\sin\szipfile\sextension\sto\sfix\sMSVC\scompiler\serror.
-D 2018-02-04T01:29:01.007
+C Fix\sharmless\scompiler\swarning\sseen\swith\sMSVC.
+D 2018-02-04T01:30:54.482
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -429,7 +429,7 @@ F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
-F src/btree.c d711228cac336fb35fff21f3f4a0efe2ad58aa9a800dd02929cdf184be1e78a3
+F src/btree.c f7bb37c27b3734d58796febb30b5d818b3b4eb592b57b88039e5d92b3ebd9dab
 F src/btree.h 0866c0a08255142ea0e754aabd211c843cab32045c978a592a43152405ed0c84
 F src/btreeInt.h 620ab4c7235f43572cf3ac2ac8723cbdf68073be4d29da24897c7b77dda5fd96
 F src/build.c 672022c06e1a5c2653f80c77a687de11f7e65ce81d20fe2825aadfa13a875c33
@@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P cb3feb3bc33718625159eddbf8ef608cd641af84e97d433d6b11ba6da94dad2e
-R 8b8d4f678534b218d44a7c7eccfb3100
+P ac6a32482e675871c3ce5ecc727e50e4bba649e80e3c55bf99025fed5667da94
+R 996f708603528b0dbd58b1571407dd89
 U mistachkin
-Z e519063d0405ad8f0532efc454c9e10a
+Z c8cfe40d98eaa63e42dbf121b7e80b46
diff --git a/manifest.uuid b/manifest.uuid
index 984b05d033..4640ba7b5d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ac6a32482e675871c3ce5ecc727e50e4bba649e80e3c55bf99025fed5667da94
\ No newline at end of file
+63a914410b5750bd08836864ae456ff09549487c5ce49706fedffa3d4f973059
\ No newline at end of file
diff --git a/src/btree.c b/src/btree.c
index 33f5842282..d3d2a41e26 100644
--- a/src/btree.c
+++ b/src/btree.c
@@ -4669,7 +4669,7 @@ static int accessPayload(
     if( (pCur->curFlags & BTCF_ValidOvfl)==0 ){
       int nOvfl = (pCur->info.nPayload-pCur->info.nLocal+ovflSize-1)/ovflSize;
       if( pCur->aOverflow==0
-       || nOvfl*sizeof(Pgno) > sqlite3MallocSize(pCur->aOverflow)
+       || nOvfl*(int)sizeof(Pgno) > sqlite3MallocSize(pCur->aOverflow)
       ){
         Pgno *aNew = (Pgno*)sqlite3Realloc(
             pCur->aOverflow, nOvfl*2*sizeof(Pgno)

From 4b18c1d53af50abfcbc79aeb29a88df175c58fab Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sun, 4 Feb 2018 20:33:13 +0000
Subject: [PATCH 418/488] In the CLI, make sure sqlite3_initialize() is called
 correctly even for non-UTF8 hosts.

FossilOrigin-Name: a3591fb54178b8f124207ff16e3ef9e58a62585feb3a56ced2b862222aea19d8
---
 manifest       | 14 +++++++-------
 manifest.uuid  |  2 +-
 src/shell.c.in |  2 +-
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index b4577329bb..fd2eb50f21 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sharmless\scompiler\swarning\sseen\swith\sMSVC.
-D 2018-02-04T01:30:54.482
+C In\sthe\sCLI,\nmake\ssure\ssqlite3_initialize()\sis\scalled\scorrectly\seven\sfor\snon-UTF8\shosts.
+D 2018-02-04T20:33:13.165
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -488,7 +488,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c f02352ac5cbb6fad3804add825743b924cfb6c742ba2e8666d726828a9985d73
-F src/shell.c.in ada6d184a5cdaada15a330ca97ad0126125f0aaed40b5c04385278fcaa78aafa
+F src/shell.c.in 0b2c8d55e3703723a22610d75ccd4037962c3bcb5ad5570efd781e8c521811c4
 F src/sqlite.h.in 51f9acf52c80113d793ddd38b3940ad6895d97b4752503b19291fb8fcbf54c5e
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d
@@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ac6a32482e675871c3ce5ecc727e50e4bba649e80e3c55bf99025fed5667da94
-R 996f708603528b0dbd58b1571407dd89
-U mistachkin
-Z c8cfe40d98eaa63e42dbf121b7e80b46
+P 63a914410b5750bd08836864ae456ff09549487c5ce49706fedffa3d4f973059
+R e1e5372ebf3735de49eb34feee943e70
+U drh
+Z 839d4cdabfa93aa0b085a28f8c75bf19
diff --git a/manifest.uuid b/manifest.uuid
index 4640ba7b5d..80271b1207 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-63a914410b5750bd08836864ae456ff09549487c5ce49706fedffa3d4f973059
\ No newline at end of file
+a3591fb54178b8f124207ff16e3ef9e58a62585feb3a56ced2b862222aea19d8
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index f683ce9868..3e3f9ac8f9 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -8094,8 +8094,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
   }
 #endif
   main_init(&data);
-#if !SQLITE_SHELL_IS_UTF8
   sqlite3_initialize();
+#if !SQLITE_SHELL_IS_UTF8
   argv = sqlite3_malloc64(sizeof(argv[0])*argc);
   if( argv==0 ){
     raw_printf(stderr, "out of memory\n");

From 60d72b9baba2ee0938324fb5e2d68d97dab27ffb Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 5 Feb 2018 13:28:51 +0000
Subject: [PATCH 419/488] Update test file walro2.test to account for systems
 with a page-size (getpagesize()) larger than 32KB.

FossilOrigin-Name: d9e59cfb8476e1ec1ca458b8382172526c0557ab785f41e31971d03045c9344c
---
 manifest         | 14 +++++++-------
 manifest.uuid    |  2 +-
 test/walro2.test | 18 +++++++++++++++---
 3 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index fd2eb50f21..53c8e27ad1 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\sthe\sCLI,\nmake\ssure\ssqlite3_initialize()\sis\scalled\scorrectly\seven\sfor\snon-UTF8\shosts.
-D 2018-02-04T20:33:13.165
+C Update\stest\sfile\swalro2.test\sto\saccount\sfor\ssystems\swith\sa\spage-size\n(getpagesize())\slarger\sthan\s32KB.
+D 2018-02-05T13:28:51.262
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -1555,7 +1555,7 @@ F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03
 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6
 F test/walprotocol.test a112aba0b79e3adeaa485fed09484b32c654e97df58e454aa8489ac2cd57bf84
 F test/walro.test cb438d05ba0d191f10b688e39c4f0cd5b71569a1d1f4440e5bdf3c6880e08c20
-F test/walro2.test 6c73e8e4b5ccc55f907f4603ba36458b45c085fb6dfb04f30e3c0babbc1c2f41
+F test/walro2.test ff7e493ce65a4dadf878d89aab4c385f8ddd3c1919333a18890013fce3feeb68
 F test/walrofault.test c70cb6e308c443867701856cce92ad8288cd99488fa52afab77cca6cfd51af68
 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417
 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f
@@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 63a914410b5750bd08836864ae456ff09549487c5ce49706fedffa3d4f973059
-R e1e5372ebf3735de49eb34feee943e70
-U drh
-Z 839d4cdabfa93aa0b085a28f8c75bf19
+P a3591fb54178b8f124207ff16e3ef9e58a62585feb3a56ced2b862222aea19d8
+R 6c73de726d0398e2d66d996a80d76264
+U dan
+Z ca72fcd07de53b3665abd69681f9d461
diff --git a/manifest.uuid b/manifest.uuid
index 80271b1207..486bb16f75 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a3591fb54178b8f124207ff16e3ef9e58a62585feb3a56ced2b862222aea19d8
\ No newline at end of file
+d9e59cfb8476e1ec1ca458b8382172526c0557ab785f41e31971d03045c9344c
\ No newline at end of file
diff --git a/test/walro2.test b/test/walro2.test
index 1c51e91ec5..248acf86cb 100644
--- a/test/walro2.test
+++ b/test/walro2.test
@@ -39,6 +39,18 @@ proc copy_to_test2 {bZeroShm} {
   }
 }
 
+# Most systems allocate the *-shm file in 32KB trunks. But on UNIX systems
+# for which the getpagesize() call returns greater than 32K, the *-shm
+# file is allocated in page-sized units (since you cannot mmap part of
+# a page). The following code sets variable $MINSHMSZ to the smallest
+# possible *-shm file (i.e. the greater of 32KB and the system page-size).
+#
+do_execsql_test 0.0 {
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t1(x);
+}
+set MINSHMSZ [file size test.db-shm]
+
 foreach bZeroShm {0 1} {
 set TN [expr $bZeroShm+1]
 do_multiclient_test tn {
@@ -169,7 +181,7 @@ do_multiclient_test tn {
   } {a b c d e f g h 1 2}
   do_test $TN.3.2.2 {
     list [file size test.db-wal] [file size test.db-shm]
-  } {0 32768}
+  } [list 0 $MINSHMSZ]
 
   do_test $TN.3.3.0 {
     code2 { sqlite3 db2 test.db }
@@ -182,7 +194,7 @@ do_multiclient_test tn {
     code2 { db2 close }
     code1 { db close }
     list [file size test.db-wal] [file size test.db-shm]
-  } [list [wal_file_size 4 1024] 32768]
+  } [list [wal_file_size 4 1024] $MINSHMSZ]
   do_test $TN.3.3.1 {
     code1 { sqlite3 db file:test.db?readonly_shm=1 }
     sql1 { SELECT * FROM t1 }
@@ -196,7 +208,7 @@ do_multiclient_test tn {
     }
     code2 { db2 close }
     list [file size test.db-wal] [file size test.db-shm]
-  } [list [wal_file_size 4 1024] 32768]
+  } [list [wal_file_size 4 1024] $MINSHMSZ]
   do_test $TN.3.3.3 {
     sql1 { SELECT * FROM t1 }
   } {i ii}

From d1317095b58e87e9543fc35b418b59c70003deb0 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 5 Feb 2018 13:42:45 +0000
Subject: [PATCH 420/488] Fix another minor problem in walro2.test.

FossilOrigin-Name: ba0631de60ca38bf7efa6dbd86ec8774bf6f438c804155968e97f17eabe3b20a
---
 manifest         | 12 ++++++------
 manifest.uuid    |  2 +-
 test/walro2.test |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 53c8e27ad1..5a2f7c7320 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\stest\sfile\swalro2.test\sto\saccount\sfor\ssystems\swith\sa\spage-size\n(getpagesize())\slarger\sthan\s32KB.
-D 2018-02-05T13:28:51.262
+C Fix\sanother\sminor\sproblem\sin\swalro2.test.
+D 2018-02-05T13:42:45.777
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -1555,7 +1555,7 @@ F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03
 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6
 F test/walprotocol.test a112aba0b79e3adeaa485fed09484b32c654e97df58e454aa8489ac2cd57bf84
 F test/walro.test cb438d05ba0d191f10b688e39c4f0cd5b71569a1d1f4440e5bdf3c6880e08c20
-F test/walro2.test ff7e493ce65a4dadf878d89aab4c385f8ddd3c1919333a18890013fce3feeb68
+F test/walro2.test 0e79dd15cbdb4f482c01ea248373669c732414a726b357d04846a816afafb768
 F test/walrofault.test c70cb6e308c443867701856cce92ad8288cd99488fa52afab77cca6cfd51af68
 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417
 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f
@@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a3591fb54178b8f124207ff16e3ef9e58a62585feb3a56ced2b862222aea19d8
-R 6c73de726d0398e2d66d996a80d76264
+P d9e59cfb8476e1ec1ca458b8382172526c0557ab785f41e31971d03045c9344c
+R f3585ddda6ecfb38609942de6c6ce298
 U dan
-Z ca72fcd07de53b3665abd69681f9d461
+Z 93b8cc01dad51cf808b345d2ca0944f2
diff --git a/manifest.uuid b/manifest.uuid
index 486bb16f75..b8f8b2f481 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d9e59cfb8476e1ec1ca458b8382172526c0557ab785f41e31971d03045c9344c
\ No newline at end of file
+ba0631de60ca38bf7efa6dbd86ec8774bf6f438c804155968e97f17eabe3b20a
\ No newline at end of file
diff --git a/test/walro2.test b/test/walro2.test
index 248acf86cb..34408c1695 100644
--- a/test/walro2.test
+++ b/test/walro2.test
@@ -48,7 +48,7 @@ proc copy_to_test2 {bZeroShm} {
 do_execsql_test 0.0 {
   PRAGMA journal_mode = wal;
   CREATE TABLE t1(x);
-}
+} {wal}
 set MINSHMSZ [file size test.db-shm]
 
 foreach bZeroShm {0 1} {

From 561158937bab6dcd057026dc554d28a35c7e0bd6 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 5 Feb 2018 16:39:12 +0000
Subject: [PATCH 421/488] Allocation the mutex used by the unix VFS only once
 at initialization, instead of every time it is needed.

FossilOrigin-Name: 5764dc160783f5c4017204b3e26a89d31240c868484ced8214c9ad872bd77bd4
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/os_unix.c | 10 ++++++----
 3 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/manifest b/manifest
index 5a2f7c7320..265850e6d9 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sanother\sminor\sproblem\sin\swalro2.test.
-D 2018-02-05T13:42:45.777
+C Allocation\sthe\smutex\sused\sby\sthe\sunix\sVFS\sonly\sonce\sat\sinitialization,\sinstead\nof\severy\stime\sit\sis\sneeded.
+D 2018-02-05T16:39:12.754
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -471,7 +471,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104
 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
-F src/os_unix.c a82505be158d8ce42b38dcc9b426187d776904c12cdc68dc8925e1dfcc5cb6ce
+F src/os_unix.c ce491421b3a54b63094a155eeac668a3bc8e5b86a5a58551d906e5b5affb443f
 F src/os_win.c 501dde1ee770f4ffa458bfe1cf376a556de3ab00bb8320d659c5984403991d62
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c cd194a8793ce061e184ddc369fadbc1020c6f431014d22093f6c5e55c9234033
@@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d9e59cfb8476e1ec1ca458b8382172526c0557ab785f41e31971d03045c9344c
-R f3585ddda6ecfb38609942de6c6ce298
-U dan
-Z 93b8cc01dad51cf808b345d2ca0944f2
+P ba0631de60ca38bf7efa6dbd86ec8774bf6f438c804155968e97f17eabe3b20a
+R 07658fd89f10e428329340f4fdba337c
+U drh
+Z f93cbf0dc0e4b2bfb043a63cd29cee7e
diff --git a/manifest.uuid b/manifest.uuid
index b8f8b2f481..2da9ca7d0a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ba0631de60ca38bf7efa6dbd86ec8774bf6f438c804155968e97f17eabe3b20a
\ No newline at end of file
+5764dc160783f5c4017204b3e26a89d31240c868484ced8214c9ad872bd77bd4
\ No newline at end of file
diff --git a/src/os_unix.c b/src/os_unix.c
index 94b1efd87d..b24c6861d3 100644
--- a/src/os_unix.c
+++ b/src/os_unix.c
@@ -696,15 +696,16 @@ static int robust_open(const char *z, int f, mode_t m){
 **     assert( unixMutexHeld() );
 **   unixEnterLeave()
 */
+static sqlite3_mutex *unixBigLock = 0;
 static void unixEnterMutex(void){
-  sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1));
+  sqlite3_mutex_enter(unixBigLock);
 }
 static void unixLeaveMutex(void){
-  sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1));
+  sqlite3_mutex_leave(unixBigLock);
 }
 #ifdef SQLITE_DEBUG
 static int unixMutexHeld(void) {
-  return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1));
+  return sqlite3_mutex_held(unixBigLock);
 }
 #endif
 
@@ -5846,7 +5847,6 @@ static int unixOpen(
     randomnessPid = osGetpid(0);
     sqlite3_randomness(0,0);
   }
-
   memset(p, 0, sizeof(unixFile));
 
   if( eType==SQLITE_OPEN_MAIN_DB ){
@@ -7721,6 +7721,7 @@ int sqlite3_os_init(void){
   for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){
     sqlite3_vfs_register(&aVfs[i], i==0);
   }
+  unixBigLock = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1);
   return SQLITE_OK; 
 }
 
@@ -7732,6 +7733,7 @@ int sqlite3_os_init(void){
 ** This routine is a no-op for unix.
 */
 int sqlite3_os_end(void){ 
+  unixBigLock = 0;
   return SQLITE_OK; 
 }
  

From 435666e26d906ed2b8c1022d63f714f8143be8bb Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Mon, 5 Feb 2018 20:42:50 +0000
Subject: [PATCH 422/488] Allocation the mutex used by the Win32 VFS only once
 at initialization, instead of every time it is needed.

FossilOrigin-Name: 535ed0ac5e8728ec91fc0a4cb54b820923d161cfd4e0e6aed6df6cdae365bc7d
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/os_win.c  |  9 ++++++---
 3 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index 265850e6d9..5469fa43ad 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Allocation\sthe\smutex\sused\sby\sthe\sunix\sVFS\sonly\sonce\sat\sinitialization,\sinstead\nof\severy\stime\sit\sis\sneeded.
-D 2018-02-05T16:39:12.754
+C Allocation\sthe\smutex\sused\sby\sthe\sWin32\sVFS\sonly\sonce\sat\sinitialization,\sinstead\sof\severy\stime\sit\sis\sneeded.
+D 2018-02-05T20:42:50.153
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -472,7 +472,7 @@ F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
 F src/os_unix.c ce491421b3a54b63094a155eeac668a3bc8e5b86a5a58551d906e5b5affb443f
-F src/os_win.c 501dde1ee770f4ffa458bfe1cf376a556de3ab00bb8320d659c5984403991d62
+F src/os_win.c ca6b6a557b9015d9ecf77003482558b04bbbec86d5065af30e89295dfc01fbb7
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c cd194a8793ce061e184ddc369fadbc1020c6f431014d22093f6c5e55c9234033
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
@@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ba0631de60ca38bf7efa6dbd86ec8774bf6f438c804155968e97f17eabe3b20a
-R 07658fd89f10e428329340f4fdba337c
-U drh
-Z f93cbf0dc0e4b2bfb043a63cd29cee7e
+P 5764dc160783f5c4017204b3e26a89d31240c868484ced8214c9ad872bd77bd4
+R 7ec707deb63ac6534c16562af3ea60e2
+U mistachkin
+Z 87b212ba57f1d2b4c01281b9a95b7258
diff --git a/manifest.uuid b/manifest.uuid
index 2da9ca7d0a..12329d932f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5764dc160783f5c4017204b3e26a89d31240c868484ced8214c9ad872bd77bd4
\ No newline at end of file
+535ed0ac5e8728ec91fc0a4cb54b820923d161cfd4e0e6aed6df6cdae365bc7d
\ No newline at end of file
diff --git a/src/os_win.c b/src/os_win.c
index 2b2b8ebd56..6132d321a1 100644
--- a/src/os_win.c
+++ b/src/os_win.c
@@ -3631,15 +3631,16 @@ static SYSTEM_INFO winSysInfo;
 **     assert( winShmMutexHeld() );
 **   winShmLeaveMutex()
 */
+static sqlite3_mutex *winBigLock = 0;
 static void winShmEnterMutex(void){
-  sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1));
+  sqlite3_mutex_enter(winBigLock);
 }
 static void winShmLeaveMutex(void){
-  sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1));
+  sqlite3_mutex_leave(winBigLock);
 }
 #ifndef NDEBUG
 static int winShmMutexHeld(void) {
-  return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1));
+  return sqlite3_mutex_held(winBigLock);
 }
 #endif
 
@@ -6062,6 +6063,7 @@ int sqlite3_os_init(void){
   sqlite3_vfs_register(&winLongPathNolockVfs, 0);
 #endif
 
+  winBigLock = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1);
   return SQLITE_OK;
 }
 
@@ -6072,6 +6074,7 @@ int sqlite3_os_end(void){
     sleepObj = NULL;
   }
 #endif
+  winBigLock = 0;
   return SQLITE_OK;
 }
 

From bc6b8d73592ad72e674b10152012170a01c31c62 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Mon, 5 Feb 2018 21:02:47 +0000
Subject: [PATCH 423/488] Adjust the previous check-in, which modified the
 Win32 VFS, so that it works with SQLITE_OMIT_WAL.

FossilOrigin-Name: 36c2e67e82626f8d0a187c6c286c133ed659889e3b577469261b9dcd3b3ab75b
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/os_win.c  |  7 +++++++
 3 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index 5469fa43ad..48ba691a5f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Allocation\sthe\smutex\sused\sby\sthe\sWin32\sVFS\sonly\sonce\sat\sinitialization,\sinstead\sof\severy\stime\sit\sis\sneeded.
-D 2018-02-05T20:42:50.153
+C Adjust\sthe\sprevious\scheck-in,\swhich\smodified\sthe\sWin32\sVFS,\sso\sthat\sit\sworks\swith\sSQLITE_OMIT_WAL.
+D 2018-02-05T21:02:47.090
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -472,7 +472,7 @@ F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
 F src/os_unix.c ce491421b3a54b63094a155eeac668a3bc8e5b86a5a58551d906e5b5affb443f
-F src/os_win.c ca6b6a557b9015d9ecf77003482558b04bbbec86d5065af30e89295dfc01fbb7
+F src/os_win.c eb03c6d52f893bcd7fdd4c6006674c13c1b5e49543fec98d605201af2997171c
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c cd194a8793ce061e184ddc369fadbc1020c6f431014d22093f6c5e55c9234033
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
@@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5764dc160783f5c4017204b3e26a89d31240c868484ced8214c9ad872bd77bd4
-R 7ec707deb63ac6534c16562af3ea60e2
+P 535ed0ac5e8728ec91fc0a4cb54b820923d161cfd4e0e6aed6df6cdae365bc7d
+R 610345e12e18345d79d861c16dfe01d9
 U mistachkin
-Z 87b212ba57f1d2b4c01281b9a95b7258
+Z 7a54a037e46133481e56c9143fb3122e
diff --git a/manifest.uuid b/manifest.uuid
index 12329d932f..899bc00cb8 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-535ed0ac5e8728ec91fc0a4cb54b820923d161cfd4e0e6aed6df6cdae365bc7d
\ No newline at end of file
+36c2e67e82626f8d0a187c6c286c133ed659889e3b577469261b9dcd3b3ab75b
\ No newline at end of file
diff --git a/src/os_win.c b/src/os_win.c
index 6132d321a1..534426977f 100644
--- a/src/os_win.c
+++ b/src/os_win.c
@@ -6063,7 +6063,10 @@ int sqlite3_os_init(void){
   sqlite3_vfs_register(&winLongPathNolockVfs, 0);
 #endif
 
+#ifndef SQLITE_OMIT_WAL
   winBigLock = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1);
+#endif
+
   return SQLITE_OK;
 }
 
@@ -6074,7 +6077,11 @@ int sqlite3_os_end(void){
     sleepObj = NULL;
   }
 #endif
+
+#ifndef SQLITE_OMIT_WAL
   winBigLock = 0;
+#endif
+
   return SQLITE_OK;
 }
 

From fa68815fa326af4b9bcdc61df39185170985f078 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 7 Feb 2018 16:14:41 +0000
Subject: [PATCH 424/488] When the final connection disconnects from a wal mode
 database, check that the database file has not been moved or unlinked before
 deleting the wal and shm files.

FossilOrigin-Name: 4761db83b6d3d57f281370899403c102e39ad0021d315dd6a6912d250436782a
---
 manifest         | 16 +++++-----
 manifest.uuid    |  2 +-
 src/pager.c      | 67 ++++++++++++++++++++++-------------------
 test/nockpt.test | 78 ++++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 123 insertions(+), 40 deletions(-)

diff --git a/manifest b/manifest
index 48ba691a5f..80740ef60b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Adjust\sthe\sprevious\scheck-in,\swhich\smodified\sthe\sWin32\sVFS,\sso\sthat\sit\sworks\swith\sSQLITE_OMIT_WAL.
-D 2018-02-05T21:02:47.090
+C When\sthe\sfinal\sconnection\sdisconnects\sfrom\sa\swal\smode\sdatabase,\scheck\sthat\sthe\ndatabase\sfile\shas\snot\sbeen\smoved\sor\sunlinked\sbefore\sdeleting\sthe\swal\sand\sshm\nfiles.
+D 2018-02-07T16:14:41.573
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -474,7 +474,7 @@ F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
 F src/os_unix.c ce491421b3a54b63094a155eeac668a3bc8e5b86a5a58551d906e5b5affb443f
 F src/os_win.c eb03c6d52f893bcd7fdd4c6006674c13c1b5e49543fec98d605201af2997171c
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
-F src/pager.c cd194a8793ce061e184ddc369fadbc1020c6f431014d22093f6c5e55c9234033
+F src/pager.c a3834a40acc2f3ab247d846f850d8c9313587d9c99c57a4dc194f2d4d7bf9d15
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
 F src/parse.y 4e750e1b261ff9f1d0b6b5d40a829c66d691899f48953fde839d8b52d41aa148
 F src/pcache.c 7ae91a4557a43d77d449accbfdc68846e6516f8e2eda46e8bbe4536fb669b201
@@ -1094,7 +1094,7 @@ F test/multiplex4.test e8ae4c4bd70606a5727743241f13b5701990abe4
 F test/mutex1.test ea2cc74d97f077b9e74c84cbd024f14d79a8126f
 F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660
 F test/nan.test 437d40e6d0778b050d7750726c0cbd2c9936b81962926e8f8c48ca698f00f4d1
-F test/nockpt.test 9a436a7213ba5ef7a32304998d386d3ea3f76c9d
+F test/nockpt.test d291d618c934a453683cb2eff95f633d406f7147fa0403e10055db19dcc3842a
 F test/nolock.test f196cf8b8fbea4e2ca345140a2b3f3b0da45c76e
 F test/normalize.test 501630ab49b0b26b65c74124bf03e3374c1b57fa97aae750f84803609141d167
 F test/notify1.test 669b2b743618efdc18ca4b02f45423d5d2304abf
@@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 535ed0ac5e8728ec91fc0a4cb54b820923d161cfd4e0e6aed6df6cdae365bc7d
-R 610345e12e18345d79d861c16dfe01d9
-U mistachkin
-Z 7a54a037e46133481e56c9143fb3122e
+P 36c2e67e82626f8d0a187c6c286c133ed659889e3b577469261b9dcd3b3ab75b
+R ed31a2562d55c76884009a210d222cf0
+U dan
+Z 0e612251c296ff4ce7a47fbd872de74d
diff --git a/manifest.uuid b/manifest.uuid
index 899bc00cb8..f1767d6dc9 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-36c2e67e82626f8d0a187c6c286c133ed659889e3b577469261b9dcd3b3ab75b
\ No newline at end of file
+4761db83b6d3d57f281370899403c102e39ad0021d315dd6a6912d250436782a
\ No newline at end of file
diff --git a/src/pager.c b/src/pager.c
index e4680bfbfe..93a9b3bd3e 100644
--- a/src/pager.c
+++ b/src/pager.c
@@ -4102,6 +4102,30 @@ static void pagerFreeMapHdrs(Pager *pPager){
   }
 }
 
+/* Verify that the database file has not be deleted or renamed out from
+** under the pager.  Return SQLITE_OK if the database is still were it ought
+** to be on disk.  Return non-zero (SQLITE_READONLY_DBMOVED or some other error
+** code from sqlite3OsAccess()) if the database has gone missing.
+*/
+static int databaseIsUnmoved(Pager *pPager){
+  int bHasMoved = 0;
+  int rc;
+
+  if( pPager->tempFile ) return SQLITE_OK;
+  if( pPager->dbSize==0 ) return SQLITE_OK;
+  assert( pPager->zFilename && pPager->zFilename[0] );
+  rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_HAS_MOVED, &bHasMoved);
+  if( rc==SQLITE_NOTFOUND ){
+    /* If the HAS_MOVED file-control is unimplemented, assume that the file
+    ** has not been moved.  That is the historical behavior of SQLite: prior to
+    ** version 3.8.3, it never checked */
+    rc = SQLITE_OK;
+  }else if( rc==SQLITE_OK && bHasMoved ){
+    rc = SQLITE_READONLY_DBMOVED;
+  }
+  return rc;
+}
+
 
 /*
 ** Shutdown the page cache.  Free all memory and close all files.
@@ -4118,8 +4142,7 @@ static void pagerFreeMapHdrs(Pager *pPager){
 ** to the caller.
 */
 int sqlite3PagerClose(Pager *pPager, sqlite3 *db){
-  u8 *pTmp = (u8 *)pPager->pTmpSpace;
-
+  u8 *pTmp = (u8*)pPager->pTmpSpace;
   assert( db || pagerUseWal(pPager)==0 );
   assert( assert_pager_state(pPager) );
   disable_simulated_io_errors();
@@ -4128,11 +4151,17 @@ int sqlite3PagerClose(Pager *pPager, sqlite3 *db){
   /* pPager->errCode = 0; */
   pPager->exclusiveMode = 0;
 #ifndef SQLITE_OMIT_WAL
-  assert( db || pPager->pWal==0 );
-  sqlite3WalClose(pPager->pWal, db, pPager->walSyncFlags, pPager->pageSize,
-      (db && (db->flags & SQLITE_NoCkptOnClose) ? 0 : pTmp)
-  );
-  pPager->pWal = 0;
+  {
+    u8 *a = 0;
+    assert( db || pPager->pWal==0 );
+    if( db && 0==(db->flags & SQLITE_NoCkptOnClose) 
+     && SQLITE_OK==databaseIsUnmoved(pPager)
+    ){
+      a = pTmp;
+    }
+    sqlite3WalClose(pPager->pWal, db, pPager->walSyncFlags, pPager->pageSize,a);
+    pPager->pWal = 0;
+  }
 #endif
   pager_reset(pPager);
   if( MEMDB ){
@@ -4967,30 +4996,6 @@ act_like_temp_file:
 }
 
 
-/* Verify that the database file has not be deleted or renamed out from
-** under the pager.  Return SQLITE_OK if the database is still were it ought
-** to be on disk.  Return non-zero (SQLITE_READONLY_DBMOVED or some other error
-** code from sqlite3OsAccess()) if the database has gone missing.
-*/
-static int databaseIsUnmoved(Pager *pPager){
-  int bHasMoved = 0;
-  int rc;
-
-  if( pPager->tempFile ) return SQLITE_OK;
-  if( pPager->dbSize==0 ) return SQLITE_OK;
-  assert( pPager->zFilename && pPager->zFilename[0] );
-  rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_HAS_MOVED, &bHasMoved);
-  if( rc==SQLITE_NOTFOUND ){
-    /* If the HAS_MOVED file-control is unimplemented, assume that the file
-    ** has not been moved.  That is the historical behavior of SQLite: prior to
-    ** version 3.8.3, it never checked */
-    rc = SQLITE_OK;
-  }else if( rc==SQLITE_OK && bHasMoved ){
-    rc = SQLITE_READONLY_DBMOVED;
-  }
-  return rc;
-}
-
 
 /*
 ** This function is called after transitioning from PAGER_UNLOCK to
diff --git a/test/nockpt.test b/test/nockpt.test
index bd3953f1ee..8f6b5e3be4 100644
--- a/test/nockpt.test
+++ b/test/nockpt.test
@@ -61,6 +61,84 @@ do_test 1.14 { sqlite3_db_config db NO_CKPT_ON_CLOSE 1 } {1}
 do_execsql_test 1.14 { PRAGMA main.journal_mode = delete } {delete}
 do_test 1.15 { file exists test.db-wal } {0}
 
+#-------------------------------------------------------------------------
+# Test an unusual scenario:
+#
+#   1. A wal mode db is opened and written. Then sqlite3_close_v2() used
+#      to close the db handle while there is still an unfinalized
+#      statement (so the db handle stays open).
+#
+#   2. The db, wal and *-shm files are deleted from the file system.
+#
+#   3. Another connection creates a new wal mode db at the same file-system
+#      location as the previous one.
+#
+#   4. The statement left unfinalized in (1) is finalized.
+#
+# The test is to ensure that the connection left open in step (1) does
+# not try to delete the wal file from the file-system as part of step
+# 4.
+#
+reset_db
+db close
+
+# Open a connection on a wal database. Write to it a bit. Then prepare
+# a statement and call sqlite3_close_v2() (so that the statement handle
+# holds the db connection open).
+#
+set ::db1 [sqlite3_open_v2 test.db SQLITE_OPEN_READWRITE ""]
+do_test 2.0 {
+  lindex [
+    sqlite3_exec $::db1 {
+      PRAGMA journal_mode = wal;
+      CREATE TABLE t1(x PRIMARY KEY, y UNIQUE, z);
+      INSERT INTO t1 VALUES(1, 2, 3);
+      PRAGMA wal_checkpoint;
+    }] 0
+} {0}
+set ::stmt [sqlite3_prepare $::db1 "SELECT * FROM t1" -1 dummy]
+sqlite3_close_v2 $::db1
+
+# Delete the database, wal and shm files.
+#
+forcedelete test.db test.db-wal test.db-shm
+
+# Open and populate a new database file at the same file-system location
+# as the one just deleted. Contrive a partial checkpoint on it.
+#
+sqlite3 db  test.db
+sqlite3 db2 test.db
+do_execsql_test 2.1 {
+  PRAGMA journal_mode = wal;
+  CREATE TABLE y1(a PRIMARY KEY, b UNIQUE, c);
+  INSERT INTO y1 VALUES('a', 'b', 'c');
+  INSERT INTO y1 VALUES('d', 'e', 'f');
+} {wal}
+do_execsql_test -db db2 2.2 {
+  BEGIN;
+    SELECT * FROM y1;
+} {a b c d e f}
+do_execsql_test 2.3 {
+  UPDATE y1 SET c='g' WHERE a='d';
+  PRAGMA wal_checkpoint;
+} {0 11 10}
+do_execsql_test -db db2 2.4 {
+  COMMIT
+}
+
+# Finalize the statement handle, causing the first connection to be
+# closed. Test that this has not corrupted the database file by 
+# deleting the new wal file from the file-system. If it has, this
+# test should fail with an IO or corruption error.
+#
+do_test 2.5 {
+  sqlite3_finalize $::stmt
+  sqlite3 db3 test.db
+  execsql { 
+    PRAGMA integrity_check; 
+    SELECT * FROM y1;
+  } db3
+} {ok a b c d e g}
 
 
 finish_test

From eab0e103042143fe5712ea067c86f7b8acd545df Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 7 Feb 2018 18:02:50 +0000
Subject: [PATCH 425/488] In extensions rtree, fts3 and fts5, ensure that when
 dynamic buffers are bound to persistent SQL statements using SQLITE_STATIC,
 the binding is replaced with an SQL NULL before the buffer is freed.
 Otherwise, a user may obtain a pointer to the persistent statement using
 sqlite3_next_stmt() and attempt to access the freed buffer using
 sqlite3_expanded_sql() or similar.

FossilOrigin-Name: 2a5f813bc61f9e780f2ccbda425611f65ad523b6d486a1e5e2b9d5e9f1d260a2
---
 ext/fts3/fts3_write.c     |  5 +++++
 ext/fts5/fts5_index.c     |  4 ++++
 ext/fts5/fts5_storage.c   |  2 ++
 ext/fts5/test/fts5aa.test |  2 +-
 ext/rtree/rtree.c         |  1 +
 ext/rtree/rtree1.test     |  1 +
 ext/rtree/rtree4.test     |  1 +
 ext/rtree/rtree5.test     |  1 +
 ext/rtree/rtree6.test     |  2 +-
 ext/rtree/rtreeG.test     |  1 +
 manifest                  | 34 +++++++++++++++++-----------------
 manifest.uuid             |  2 +-
 test/fts3aa.test          |  1 +
 test/tester.tcl           | 10 ++++++++++
 14 files changed, 47 insertions(+), 20 deletions(-)

diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c
index daf3399a43..5bca766c20 100644
--- a/ext/fts3/fts3_write.c
+++ b/ext/fts3/fts3_write.c
@@ -1908,6 +1908,7 @@ static int fts3WriteSegment(
     sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC);
     sqlite3_step(pStmt);
     rc = sqlite3_reset(pStmt);
+    sqlite3_bind_null(pStmt, 2);
   }
   return rc;
 }
@@ -1964,6 +1965,7 @@ static int fts3WriteSegdir(
     sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC);
     sqlite3_step(pStmt);
     rc = sqlite3_reset(pStmt);
+    sqlite3_bind_null(pStmt, 6);
   }
   return rc;
 }
@@ -3443,6 +3445,7 @@ static void fts3UpdateDocTotals(
   sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, SQLITE_STATIC);
   sqlite3_step(pStmt);
   *pRC = sqlite3_reset(pStmt);
+  sqlite3_bind_null(pStmt, 2);
   sqlite3_free(a);
 }
 
@@ -4631,6 +4634,7 @@ static int fts3TruncateSegment(
       sqlite3_bind_int(pChomp, 4, iIdx);
       sqlite3_step(pChomp);
       rc = sqlite3_reset(pChomp);
+      sqlite3_bind_null(pChomp, 2);
     }
   }
 
@@ -4710,6 +4714,7 @@ static int fts3IncrmergeHintStore(Fts3Table *p, Blob *pHint){
     sqlite3_bind_blob(pReplace, 2, pHint->a, pHint->n, SQLITE_STATIC);
     sqlite3_step(pReplace);
     rc = sqlite3_reset(pReplace);
+    sqlite3_bind_null(pReplace, 2);
   }
 
   return rc;
diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index a75bf0fd42..412a04faca 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -758,6 +758,7 @@ static void fts5DataWrite(Fts5Index *p, i64 iRowid, const u8 *pData, int nData){
   sqlite3_bind_blob(p->pWriter, 2, pData, nData, SQLITE_STATIC);
   sqlite3_step(p->pWriter);
   p->rc = sqlite3_reset(p->pWriter);
+  sqlite3_bind_null(p->pWriter, 2);
 }
 
 /*
@@ -2386,6 +2387,7 @@ static void fts5SegIterSeekInit(
     bDlidx = (val & 0x0001);
   }
   p->rc = sqlite3_reset(pIdxSelect);
+  sqlite3_bind_null(pIdxSelect, 2);
 
   if( iPg<pSeg->pgnoFirst ){
     iPg = pSeg->pgnoFirst;
@@ -3598,6 +3600,7 @@ static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){
           sqlite3_bind_blob(pIdxSelect, 2, aBlob, 2, SQLITE_STATIC);
           assert( sqlite3_step(pIdxSelect)!=SQLITE_ROW );
           p->rc = sqlite3_reset(pIdxSelect);
+          sqlite3_bind_null(pIdxSelect, 2);
         }
       }
 #endif
@@ -3724,6 +3727,7 @@ static void fts5WriteFlushBtree(Fts5Index *p, Fts5SegWriter *pWriter){
     sqlite3_bind_int64(p->pIdxWriter, 3, bFlag + ((i64)pWriter->iBtPage<<1));
     sqlite3_step(p->pIdxWriter);
     p->rc = sqlite3_reset(p->pIdxWriter);
+    sqlite3_bind_null(p->pIdxWriter, 2);
   }
   pWriter->iBtPage = 0;
 }
diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c
index 59336fc7ac..70d7135113 100644
--- a/ext/fts5/fts5_storage.c
+++ b/ext/fts5/fts5_storage.c
@@ -458,6 +458,7 @@ static int fts5StorageInsertDocsize(
       sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
       sqlite3_step(pReplace);
       rc = sqlite3_reset(pReplace);
+      sqlite3_bind_null(pReplace, 2);
     }
   }
   return rc;
@@ -1118,6 +1119,7 @@ int sqlite3Fts5StorageConfigValue(
     }
     sqlite3_step(pReplace);
     rc = sqlite3_reset(pReplace);
+    sqlite3_bind_null(pReplace, 1);
   }
   if( rc==SQLITE_OK && pVal ){
     int iNew = p->pConfig->iCookie + 1;
diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test
index a3ea0afc28..67cb62012a 100644
--- a/ext/fts5/test/fts5aa.test
+++ b/ext/fts5/test/fts5aa.test
@@ -593,5 +593,5 @@ do_execsql_test 22.1 {
 
 }
 
-
+expand_all_sql db
 finish_test
diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c
index c0fd8c1819..00513d4005 100644
--- a/ext/rtree/rtree.c
+++ b/ext/rtree/rtree.c
@@ -785,6 +785,7 @@ static int nodeWrite(Rtree *pRtree, RtreeNode *pNode){
     sqlite3_step(p);
     pNode->isDirty = 0;
     rc = sqlite3_reset(p);
+    sqlite3_bind_null(p, 2);
     if( pNode->iNode==0 && rc==SQLITE_OK ){
       pNode->iNode = sqlite3_last_insert_rowid(pRtree->db);
       nodeHashInsert(pRtree, pNode);
diff --git a/ext/rtree/rtree1.test b/ext/rtree/rtree1.test
index 0deee6635b..ac6e8d9d96 100644
--- a/ext/rtree/rtree1.test
+++ b/ext/rtree/rtree1.test
@@ -609,4 +609,5 @@ do_execsql_test 15.2 {
   COMMIT;
 }
 
+expand_all_sql db
 finish_test
diff --git a/ext/rtree/rtree4.test b/ext/rtree/rtree4.test
index af3f8d3995..a73921d8d5 100644
--- a/ext/rtree/rtree4.test
+++ b/ext/rtree/rtree4.test
@@ -250,4 +250,5 @@ for {set nDim 1} {$nDim<=5} {incr nDim} {
   do_rtree_integrity_test rtree4-$nDim.3 rx
 }
 
+expand_all_sql db
 finish_test
diff --git a/ext/rtree/rtree5.test b/ext/rtree/rtree5.test
index 749385e882..92bb6905c7 100644
--- a/ext/rtree/rtree5.test
+++ b/ext/rtree/rtree5.test
@@ -79,4 +79,5 @@ do_test rtree5-1.13 {
 } {2 2147483643 2147483647 -2147483648 -2147483643}
 do_rtree_integrity_test rtree5-1.14 t1
 
+expand_all_sql db
 finish_test
diff --git a/ext/rtree/rtree6.test b/ext/rtree/rtree6.test
index c9c87e8ad9..406604810b 100644
--- a/ext/rtree/rtree6.test
+++ b/ext/rtree/rtree6.test
@@ -158,5 +158,5 @@ do_execsql_test rtree6-3.5 {
     x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>1.1
 } {}
 
-
+expand_all_sql db
 finish_test
diff --git a/ext/rtree/rtreeG.test b/ext/rtree/rtreeG.test
index 3bef89c8e7..12225d5832 100644
--- a/ext/rtree/rtreeG.test
+++ b/ext/rtree/rtreeG.test
@@ -59,6 +59,7 @@ do_test rtreeG-1.4log {
   set ::log
 } {}
 
+expand_all_sql db
 db close
 sqlite3_shutdown
 test_sqlite3_log
diff --git a/manifest b/manifest
index 80740ef60b..047f9dc234 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C When\sthe\sfinal\sconnection\sdisconnects\sfrom\sa\swal\smode\sdatabase,\scheck\sthat\sthe\ndatabase\sfile\shas\snot\sbeen\smoved\sor\sunlinked\sbefore\sdeleting\sthe\swal\sand\sshm\nfiles.
-D 2018-02-07T16:14:41.573
+C In\sextensions\srtree,\sfts3\sand\sfts5,\sensure\sthat\swhen\sdynamic\sbuffers\sare\sbound\nto\spersistent\sSQL\sstatements\susing\sSQLITE_STATIC,\sthe\sbinding\sis\sreplaced\swith\nan\sSQL\sNULL\sbefore\sthe\sbuffer\sis\sfreed.\sOtherwise,\sa\suser\smay\sobtain\sa\spointer\nto\sthe\spersistent\sstatement\susing\ssqlite3_next_stmt()\sand\sattempt\sto\saccess\nthe\sfreed\sbuffer\susing\ssqlite3_expanded_sql()\sor\ssimilar.
+D 2018-02-07T18:02:50.375
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -96,7 +96,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3
 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004
 F ext/fts3/fts3_unicode.c 525a3bd9a7564603c5c061b7de55403a565307758a94600e8a2f6b00d1c40d9d
 F ext/fts3/fts3_unicode2.c cc04fc672bfd42b1e650398cb0bf71f64f9aae032cfe75bbcfe75b9cf966029c
-F ext/fts3/fts3_write.c a3f7bf869622d1d0aa66661ba71d88e6f9646d69a2c335f40a0addf25974db47
+F ext/fts3/fts3_write.c 7a7cf93c02ebe0ee4211e4aa07da77586c2dcf7d381c1382f81d29c9aa8cae8c
 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
 F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87fea4ac01b73
@@ -113,9 +113,9 @@ F ext/fts5/fts5_buffer.c 1dd1ec0446b3acfc2d7d407eb894762a461613e2695273f48e449bf
 F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857
 F ext/fts5/fts5_expr.c 01048018d21524e2c302b063ff5c3cdcf546e03297215e577205d85b47499deb
 F ext/fts5/fts5_hash.c 32be400cf761868c9db33efe81a06eb19a17c5402ad477ee9efb51301546dd55
-F ext/fts5/fts5_index.c 5fe14375a29e8a7aa8f3e863babe180a19269206c254c8f47b216821d4ac1e15
+F ext/fts5/fts5_index.c 22b71d0e9e4b3ddd123a39ae27174e0012da2806f91b64087a68584f13f189de
 F ext/fts5/fts5_main.c 24868f88ab2a865defbba7a92eebeb726cc991eb092b71b5f5508f180c72605b
-F ext/fts5/fts5_storage.c fb5ef3c27073f67ade2e1bea08405f9e43f68f5f3676ed0ab7013bce5ba10be6
+F ext/fts5/fts5_storage.c 4bec8a1b3905978b22a67bca5f4a3cfdb94af234cf51efb36f4f2d733d278634
 F ext/fts5/fts5_tcl.c 39bcbae507f594aad778172fa914cad0f585bf92fd3b078c686e249282db0d95
 F ext/fts5/fts5_test_mi.c 65864ba1e5c34a61d409c4c587e0bbe0466eb4f8f478d85dc42a92caad1338e6
 F ext/fts5/fts5_test_tok.c ffd657dd67e7fcdb31bf63fb60b6d867299a581d0f46e97086abacd66c2a9b26
@@ -126,7 +126,7 @@ F ext/fts5/fts5_vocab.c 1cd79854cb21543e66507b25b0578bc1b20aa6a1349b7feceb8e8fed
 F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05
 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
 F ext/fts5/test/fts5_common.tcl b01c584144b5064f30e6c648145a2dd6bc440841
-F ext/fts5/test/fts5aa.test cba3fae6466446980caf1b9f5f26df77f95a999d35db7d932d6e82ae7ba0ede9
+F ext/fts5/test/fts5aa.test 6e2fdb0ee667c05f41921e7ec345cae874be651670900918e9ccc539514b9356
 F ext/fts5/test/fts5ab.test 9205c839332c908aaad2b01ab8670ece8b161e8f2ec8a9fabf18ca9385880bb7
 F ext/fts5/test/fts5ac.test a7aa7e1fefc6e1918aa4d3111d5c44a09177168e962c5fd2cca9620de8a7ed6d
 F ext/fts5/test/fts5ad.test e8cf959dfcd57c8e46d6f5f25665686f3b6627130a9a981371dafdf6482790de
@@ -351,14 +351,14 @@ F ext/repair/test/checkfreelist01.test 3e8aa6aeb4007680c94a8d07b41c339aa635cc782
 F ext/repair/test/checkindex01.test 6945d0ffc0c1dc993b2ce88036b26e0f5d6fcc65da70fc9df27c2647bb358b0f
 F ext/repair/test/test.tcl 686d76d888dffd021f64260abf29a55c57b2cedfa7fc69150b42b1d6119aac3c
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
-F ext/rtree/rtree.c d941e44ad901da039caebb9f9fa99d81f2a4fc822e67cafe33fa4f6f789074a0
+F ext/rtree/rtree.c bc61010e978b5b8ae6dbb90274a2fbb5db5ff5e2880b5c6e8abd48eea77264db
 F ext/rtree/rtree.h 4a690463901cb5e6127cf05eb8e642f127012fd5003830dbc974eca5802d9412
-F ext/rtree/rtree1.test 82a353747fcab1083d114b2ac84723dfefdbf86c1a6e1df57bf588c7d4285436
+F ext/rtree/rtree1.test 47e2095bebea6813754fd7afa6a20e2b7b4ebcd5cb7dbcb6932b6c9f86bbf972
 F ext/rtree/rtree2.test 5f25b01acd03470067a2d52783b2eb0a50bf836803d4342d20ca39e541220fe2
 F ext/rtree/rtree3.test 2cafe8265d1ff28f206fce88d114f208349df482
-F ext/rtree/rtree4.test 67b021858ba4334c8d49b3449476942c2ce0e5ef7123538f2e9dd508ed03a12d
-F ext/rtree/rtree5.test 8aaa4bcdc42f718fe165572f5623e4732831aca95a2bc32482d33d4d2cf1325d
-F ext/rtree/rtree6.test 773a90db2dce6a8353dd0d5b64bca69b29761196
+F ext/rtree/rtree4.test 304de65d484540111b896827e4261815e5dca4ce28eeecd58be648cd73452c4b
+F ext/rtree/rtree5.test 49c9041d713d54560b315c2c7ef7207ee287eba1b20f8266968a06f2e55d3142
+F ext/rtree/rtree6.test 916a641d2beac01b9880871ff07612d56c1e466190a27c82ab36ffd58be03b9f
 F ext/rtree/rtree7.test c8fb2e555b128dd0f0bdb520c61380014f497f8a23c40f2e820acc9f9e4fdce5
 F ext/rtree/rtree8.test 649f5a37ec656028a4a32674b9b1183104285a7625a09d2a8f52a1cef72c93f2
 F ext/rtree/rtree9.test c646f12c8c1c68ef015c6c043d86a0c42488e2e68ed1bb1b0771a7ca246cbabf
@@ -368,7 +368,7 @@ F ext/rtree/rtreeC.test d9d06dda1aee68b4dc227dfcc899f335f8b621e9d1920ee3d4e5dab8
 F ext/rtree/rtreeD.test fe46aa7f012e137bd58294409b16c0d43976c3bb92c8f710481e577c4a1100dc
 F ext/rtree/rtreeE.test e65d3fc625da1800b412fc8785817327d43ccfec5f5973912d8c9e471928caa9
 F ext/rtree/rtreeF.test 81ffa7ef51c4e4618d497a57328c265bf576990c7070633b623b23cd450ed331
-F ext/rtree/rtreeG.test fd3af1ca944a0bdb0cbb5455a4905c9f012e2fffcab6b791f07afa0dcbbcae0e
+F ext/rtree/rtreeG.test 1b9ca6e3effb48f4161edaa463ddeaa8fca4b2526d084f9cbf5dbe4e0184939c
 F ext/rtree/rtree_perf.tcl 6c18c1f23cd48e0f948930c98dfdd37dfccb5195
 F ext/rtree/rtree_util.tcl db734b4c5e75fed6acc56d9701f2235345acfdec750b5fc7b587936f5f6bceed
 F ext/rtree/rtreecheck.test 4d29103d1e16fcbf90135d1c637b833688492b063b2971dfb5dc6ba76555cfee
@@ -845,7 +845,7 @@ F test/fts2r.test b154c30b63061d8725e320fba1a39e2201cadd5e
 F test/fts2token.test d8070b241a15ff13592a9ae4a8b7c171af6f445a
 F test/fts3.test 672a040ea57036fb4b6fdc09027c18d7d24ab654
 F test/fts3_common.tcl 99cf6659b87c0f74f55963c2aea03b3a7d66ceb0
-F test/fts3aa.test 39b65c11913d277c91d7426c62cfc1d147d1b4e9a48fecd9e38f60d0b5a5f505
+F test/fts3aa.test f267fcd6aca30fc70b81e5d82b68b34b38f581896020b57ed49e9777c7ebd85f
 F test/fts3ab.test 7f6cf260ae80dda064023df8e8e503e9a412b91f
 F test/fts3ac.test 636ed7486043055d4f126a0e385f2d5a82ebbf63
 F test/fts3ad.test e40570cb6f74f059129ad48bcef3d7cbc20dda49
@@ -1299,7 +1299,7 @@ F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30
 F test/temptable2.test cd396beb41117a5302fff61767c35fa4270a0d5e
 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637
 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc
-F test/tester.tcl 3ed81b9e1d9718a8d9603596c8a877793d054294053c4277a3d3897eabab3866
+F test/tester.tcl 94901a4625d9a2229666dd5c44120ddf7f0fb639470710ef74a4cefc7b039e07
 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5
 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58
 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7
@@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 36c2e67e82626f8d0a187c6c286c133ed659889e3b577469261b9dcd3b3ab75b
-R ed31a2562d55c76884009a210d222cf0
+P 4761db83b6d3d57f281370899403c102e39ad0021d315dd6a6912d250436782a
+R 43f41f7fb61c314e4ad3b5665d35c1a5
 U dan
-Z 0e612251c296ff4ce7a47fbd872de74d
+Z 58e2db6aa4428ce3cec59e835a85dfcd
diff --git a/manifest.uuid b/manifest.uuid
index f1767d6dc9..6993509eea 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4761db83b6d3d57f281370899403c102e39ad0021d315dd6a6912d250436782a
\ No newline at end of file
+2a5f813bc61f9e780f2ccbda425611f65ad523b6d486a1e5e2b9d5e9f1d260a2
\ No newline at end of file
diff --git a/test/fts3aa.test b/test/fts3aa.test
index 10ec273cbf..d5f96d81a7 100644
--- a/test/fts3aa.test
+++ b/test/fts3aa.test
@@ -250,4 +250,5 @@ do_execsql_test 9.2 {
   CREATE VIRTUAL TABLE t10 USING fts3(<, b, c);
 }
 
+expand_all_sql db
 finish_test
diff --git a/test/tester.tcl b/test/tester.tcl
index d0d6c92a60..6021ce72be 100644
--- a/test/tester.tcl
+++ b/test/tester.tcl
@@ -2309,6 +2309,16 @@ proc test_find_sqldiff {} {
   return $prog
 }
 
+# Call sqlite3_expanded_sql() on all statements associated with database
+# connection $db. This sometimes finds use-after-free bugs if run with
+# valgrind or address-sanitizer.
+proc expand_all_sql {db} {
+  set stmt ""
+  while {[set stmt [sqlite3_next_stmt $db $stmt]]!=""} {
+    sqlite3_expanded_sql $stmt
+  }
+}
+
 
 # If the library is compiled with the SQLITE_DEFAULT_AUTOVACUUM macro set
 # to non-zero, then set the global variable $AUTOVACUUM to 1.

From b189e4101c2a8a6651222901f41afc00ff41d33f Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Wed, 7 Feb 2018 18:45:00 +0000
Subject: [PATCH 426/488] Fix typo in comment.  Skip tests added by check-in
 [4761db83b6] when running on Windows.

FossilOrigin-Name: 468a389cdc8b6f92c77b11f682e210ad8cce7e3dbc0df308a898f4115e85fce2
---
 manifest         | 16 ++++++++--------
 manifest.uuid    |  2 +-
 src/pager.c      |  2 +-
 test/nockpt.test |  2 ++
 4 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index 047f9dc234..a9179281b9 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\sextensions\srtree,\sfts3\sand\sfts5,\sensure\sthat\swhen\sdynamic\sbuffers\sare\sbound\nto\spersistent\sSQL\sstatements\susing\sSQLITE_STATIC,\sthe\sbinding\sis\sreplaced\swith\nan\sSQL\sNULL\sbefore\sthe\sbuffer\sis\sfreed.\sOtherwise,\sa\suser\smay\sobtain\sa\spointer\nto\sthe\spersistent\sstatement\susing\ssqlite3_next_stmt()\sand\sattempt\sto\saccess\nthe\sfreed\sbuffer\susing\ssqlite3_expanded_sql()\sor\ssimilar.
-D 2018-02-07T18:02:50.375
+C Fix\stypo\sin\scomment.\s\sSkip\stests\sadded\sby\scheck-in\s[4761db83b6]\swhen\srunning\son\sWindows.
+D 2018-02-07T18:45:00.417
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -474,7 +474,7 @@ F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
 F src/os_unix.c ce491421b3a54b63094a155eeac668a3bc8e5b86a5a58551d906e5b5affb443f
 F src/os_win.c eb03c6d52f893bcd7fdd4c6006674c13c1b5e49543fec98d605201af2997171c
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
-F src/pager.c a3834a40acc2f3ab247d846f850d8c9313587d9c99c57a4dc194f2d4d7bf9d15
+F src/pager.c 0b6bd5442733b2e08d0673de6cdafe3e7ab0b5715e4844ac836ab346b1d9ed89
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
 F src/parse.y 4e750e1b261ff9f1d0b6b5d40a829c66d691899f48953fde839d8b52d41aa148
 F src/pcache.c 7ae91a4557a43d77d449accbfdc68846e6516f8e2eda46e8bbe4536fb669b201
@@ -1094,7 +1094,7 @@ F test/multiplex4.test e8ae4c4bd70606a5727743241f13b5701990abe4
 F test/mutex1.test ea2cc74d97f077b9e74c84cbd024f14d79a8126f
 F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660
 F test/nan.test 437d40e6d0778b050d7750726c0cbd2c9936b81962926e8f8c48ca698f00f4d1
-F test/nockpt.test d291d618c934a453683cb2eff95f633d406f7147fa0403e10055db19dcc3842a
+F test/nockpt.test fd5473e30a84848b25ab06524750c05095383e191be83ccb4e6951c8beddfb5b
 F test/nolock.test f196cf8b8fbea4e2ca345140a2b3f3b0da45c76e
 F test/normalize.test 501630ab49b0b26b65c74124bf03e3374c1b57fa97aae750f84803609141d167
 F test/notify1.test 669b2b743618efdc18ca4b02f45423d5d2304abf
@@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 4761db83b6d3d57f281370899403c102e39ad0021d315dd6a6912d250436782a
-R 43f41f7fb61c314e4ad3b5665d35c1a5
-U dan
-Z 58e2db6aa4428ce3cec59e835a85dfcd
+P 2a5f813bc61f9e780f2ccbda425611f65ad523b6d486a1e5e2b9d5e9f1d260a2
+R c941c05feb1d90fa13b7661ff1c5c31d
+U mistachkin
+Z 34b282be56160e3c52f85502f9e831c9
diff --git a/manifest.uuid b/manifest.uuid
index 6993509eea..56adc03264 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2a5f813bc61f9e780f2ccbda425611f65ad523b6d486a1e5e2b9d5e9f1d260a2
\ No newline at end of file
+468a389cdc8b6f92c77b11f682e210ad8cce7e3dbc0df308a898f4115e85fce2
\ No newline at end of file
diff --git a/src/pager.c b/src/pager.c
index 93a9b3bd3e..b98487f333 100644
--- a/src/pager.c
+++ b/src/pager.c
@@ -4103,7 +4103,7 @@ static void pagerFreeMapHdrs(Pager *pPager){
 }
 
 /* Verify that the database file has not be deleted or renamed out from
-** under the pager.  Return SQLITE_OK if the database is still were it ought
+** under the pager.  Return SQLITE_OK if the database is still where it ought
 ** to be on disk.  Return non-zero (SQLITE_READONLY_DBMOVED or some other error
 ** code from sqlite3OsAccess()) if the database has gone missing.
 */
diff --git a/test/nockpt.test b/test/nockpt.test
index 8f6b5e3be4..6da3313ad8 100644
--- a/test/nockpt.test
+++ b/test/nockpt.test
@@ -61,6 +61,7 @@ do_test 1.14 { sqlite3_db_config db NO_CKPT_ON_CLOSE 1 } {1}
 do_execsql_test 1.14 { PRAGMA main.journal_mode = delete } {delete}
 do_test 1.15 { file exists test.db-wal } {0}
 
+if {$::tcl_platform(platform)!="windows"} {
 #-------------------------------------------------------------------------
 # Test an unusual scenario:
 #
@@ -139,6 +140,7 @@ do_test 2.5 {
     SELECT * FROM y1;
   } db3
 } {ok a b c d e g}
+}
 
 
 finish_test

From 2a86110a6c9b409be5f465717d3a7cd3519aedfe Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Thu, 8 Feb 2018 01:00:11 +0000
Subject: [PATCH 427/488] Enhance sqlite3ErrStr() to include several more error
 codes.

FossilOrigin-Name: ad5d3bdc739a0997786f94fb5789b726b9f53ff883226093924338fe5000922b
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/main.c    | 10 ++++++++++
 3 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index a9179281b9..84c8db0cbd 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\stypo\sin\scomment.\s\sSkip\stests\sadded\sby\scheck-in\s[4761db83b6]\swhen\srunning\son\sWindows.
-D 2018-02-07T18:45:00.417
+C Enhance\ssqlite3ErrStr()\sto\sinclude\sseveral\smore\serror\scodes.
+D 2018-02-08T01:00:11.430
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -452,7 +452,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 14686083cedc198540b15a79586cdd4be2acf6d5fa97627e355f817ab07e9fee
 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e
 F src/loadext.c f6e4e416a736369f9e80eba609f0acda97148a8b0453784d670c78d3eed2f302
-F src/main.c 26918d50dd4a61b8f6f210320a522f46b5e7e592335b6aa664ab15b80b7c239b
+F src/main.c dc75b3585bab31a88962eda2abe35a38b58c2e4d648e1a11e249fe43174c783f
 F src/malloc.c 07295435093ce354c6d9063ac05a2eeae28bd251d2e63c48b3d67c12c76f7e18
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2a5f813bc61f9e780f2ccbda425611f65ad523b6d486a1e5e2b9d5e9f1d260a2
-R c941c05feb1d90fa13b7661ff1c5c31d
+P 468a389cdc8b6f92c77b11f682e210ad8cce7e3dbc0df308a898f4115e85fce2
+R 1a5e19bb7fec633fe466598d51b2a7e2
 U mistachkin
-Z 34b282be56160e3c52f85502f9e831c9
+Z 7e5222f5afb9dd9d9137e335880f687f
diff --git a/manifest.uuid b/manifest.uuid
index 56adc03264..e5baaa9133 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-468a389cdc8b6f92c77b11f682e210ad8cce7e3dbc0df308a898f4115e85fce2
\ No newline at end of file
+ad5d3bdc739a0997786f94fb5789b726b9f53ff883226093924338fe5000922b
\ No newline at end of file
diff --git a/src/main.c b/src/main.c
index ed41f98947..4534631da7 100644
--- a/src/main.c
+++ b/src/main.c
@@ -1438,6 +1438,8 @@ const char *sqlite3ErrStr(int rc){
     /* SQLITE_FORMAT      */ 0,
     /* SQLITE_RANGE       */ "column index out of range",
     /* SQLITE_NOTADB      */ "file is not a database",
+    /* SQLITE_NOTICE      */ "notification message",
+    /* SQLITE_WARNING     */ "warning message",
   };
   const char *zErr = "unknown error";
   switch( rc ){
@@ -1445,6 +1447,14 @@ const char *sqlite3ErrStr(int rc){
       zErr = "abort due to ROLLBACK";
       break;
     }
+    case SQLITE_ROW: {
+      zErr = "another row available";
+      break;
+    }
+    case SQLITE_DONE: {
+      zErr = "no more rows available";
+      break;
+    }
     default: {
       rc &= 0xff;
       if( ALWAYS(rc>=0) && rc<ArraySize(aMsg) && aMsg[rc]!=0 ){

From 54cfb492b758e6679ca7511babe88c99bd105ba4 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 9 Feb 2018 15:04:51 +0000
Subject: [PATCH 428/488] Fix a harmless compiler warning.

FossilOrigin-Name: a6c3115483d597fc77ab19fdcfd1d3437cad7e467081ad8c5315fb98c115eed9
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 tool/lempar.c |  3 ++-
 3 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index 84c8db0cbd..7bf30b74e1 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Enhance\ssqlite3ErrStr()\sto\sinclude\sseveral\smore\serror\scodes.
-D 2018-02-08T01:00:11.430
+C Fix\sa\sharmless\scompiler\swarning.
+D 2018-02-09T15:04:51.897
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -1624,7 +1624,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
 F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f
 F tool/lemon.c 7f7735326ca9c3b48327b241063cee52d35d44e20ebe1b3624a81658052a4d39
-F tool/lempar.c da840fc8a6fbac23599a65ff075e6e3d01320417c794ff577088e09f5d74b689
+F tool/lempar.c 468a155e8729cfbccfe1d85bf60d064f1dab76167a51149ec5c7928a2de63953
 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9
 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca
@@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 468a389cdc8b6f92c77b11f682e210ad8cce7e3dbc0df308a898f4115e85fce2
-R 1a5e19bb7fec633fe466598d51b2a7e2
-U mistachkin
-Z 7e5222f5afb9dd9d9137e335880f687f
+P ad5d3bdc739a0997786f94fb5789b726b9f53ff883226093924338fe5000922b
+R b0cb95e907260417df10582a6a66e391
+U drh
+Z cf86aa36db1eb294d8bf4ad196d065d9
diff --git a/manifest.uuid b/manifest.uuid
index e5baaa9133..f234bbcf32 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ad5d3bdc739a0997786f94fb5789b726b9f53ff883226093924338fe5000922b
\ No newline at end of file
+a6c3115483d597fc77ab19fdcfd1d3437cad7e467081ad8c5315fb98c115eed9
\ No newline at end of file
diff --git a/tool/lempar.c b/tool/lempar.c
index ecc0e6389b..1ade666916 100644
--- a/tool/lempar.c
+++ b/tool/lempar.c
@@ -511,7 +511,8 @@ static unsigned int yy_find_shift_action(
 #endif
   do{
     i = yy_shift_ofst[stateno];
-    assert( i>=0 && i+YYNTOKEN<=sizeof(yy_lookahead)/sizeof(yy_lookahead[0]) );
+    assert( i>=0 );
+    assert( i+YYNTOKEN<=(int)sizeof(yy_lookahead)/sizeof(yy_lookahead[0]) );
     assert( iLookAhead!=YYNOCODE );
     assert( iLookAhead < YYNTOKEN );
     i += iLookAhead;

From 91347c066b96a9704b4c1a01c1a91b7a5a0765e6 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 9 Feb 2018 15:42:40 +0000
Subject: [PATCH 429/488] Make the tests in func6.test more robust against
 implementation changes.

FossilOrigin-Name: b685d3231097fb90e7d61d9ac01cc560e8bf2671d49390ae7af5bfdbd6d04f11
---
 manifest        |  14 ++---
 manifest.uuid   |   2 +-
 test/func6.test | 147 +++++++++++++++++++++++++++++++++++-------------
 3 files changed, 117 insertions(+), 46 deletions(-)

diff --git a/manifest b/manifest
index 7bf30b74e1..a0ff58b5eb 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sharmless\scompiler\swarning.
-D 2018-02-09T15:04:51.897
+C Make\sthe\stests\sin\sfunc6.test\smore\srobust\sagainst\simplementation\schanges.
+D 2018-02-09T15:42:40.632
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -926,7 +926,7 @@ F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f
 F test/func3.test d202a7606d23f90988a664e88e268aed1087c11c
 F test/func4.test 6beacdfcb0e18c358e6c2dcacf1b65d1fa80955f
 F test/func5.test cdd224400bc3e48d891827cc913a57051a426fa4
-F test/func6.test a4281c8fcd42b56f7a60f28e8e4d444e8b2256f9e82658b7ab87699f8318f564
+F test/func6.test 612311a51adad23326d15353eea8d90394798acb26dd1f7c7cad59bd8ac982d2
 F test/fuzz-oss1.test e58330d01cbbd8215ee636b17a03fe220b37dbfa
 F test/fuzz.test 96083052bf5765e4518c1ba686ce2bab785670d1
 F test/fuzz2.test 76dc35b32b6d6f965259508508abce75a6c4d7e1
@@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ad5d3bdc739a0997786f94fb5789b726b9f53ff883226093924338fe5000922b
-R b0cb95e907260417df10582a6a66e391
-U drh
-Z cf86aa36db1eb294d8bf4ad196d065d9
+P a6c3115483d597fc77ab19fdcfd1d3437cad7e467081ad8c5315fb98c115eed9
+R f6c1b3a83bf1fd9e6dbd920e348c4245
+U dan
+Z baeba07272c3f575c3e3036aa614a6dd
diff --git a/manifest.uuid b/manifest.uuid
index f234bbcf32..3dcc2798f8 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a6c3115483d597fc77ab19fdcfd1d3437cad7e467081ad8c5315fb98c115eed9
\ No newline at end of file
+b685d3231097fb90e7d61d9ac01cc560e8bf2671d49390ae7af5bfdbd6d04f11
\ No newline at end of file
diff --git a/test/func6.test b/test/func6.test
index 02e1998cd0..50415e7963 100644
--- a/test/func6.test
+++ b/test/func6.test
@@ -33,54 +33,125 @@ do_execsql_test func6-100 {
   CREATE TABLE t2(x TEXT PRIMARY KEY, y) WITHOUT ROWID;
   INSERT INTO t2(x,y) SELECT a, b FROM t1;
 }
+
+# Load the contents of $file from disk and return it encoded as a hex
+# string.
+proc loadhex {file} {
+  set fd [open $file]
+  fconfigure $fd -translation binary -encoding binary
+  set data [read $fd]
+  close $fd
+  binary encode hex $data 
+}
+
+# Each argument is either an integer between 0 and 65535, a text value, or
+# an empty string representing an SQL NULL. This command builds an SQLite
+# record containing the values passed as arguments and returns it encoded
+# as a hex string.
+proc hexrecord {args} {
+  set hdr ""
+  set body ""
+  foreach x $args {
+    if {$x==""} {
+      append hdr 00
+    } elseif {[string is integer $x]==0} {
+      set n [string length $x]
+      append hdr [format %02x [expr $n*2 + 13]]
+      append body [binary encode hex $x]
+    } elseif {$x == 0} {
+      append hdr 08
+    } elseif {$x == 1} {
+      append hdr 09
+    } elseif {$x <= 127} {
+      append hdr 01
+      append body [format %02x $x]
+    } else {
+      append hdr 02
+      append body [format %04x $x]
+    }
+  }
+  set res [format %02x [expr 1 + [string length $hdr]/2]]
+  append res $hdr
+  append res $body
+}
+
+# Argument $off is an offset into the database image encoded as a hex string
+# in argument $hexdb. This command returns 0 if the offset contains the hex
+# $hexrec, or throws an exception otherwise.
+#
+proc offset_contains_record {off hexdb hexrec} {
+  set n [string length $hexrec]
+  set off [expr $off*2]
+  if { [string compare $hexrec [string range $hexdb $off [expr $off+$n-1]]] } {
+    error "record not found!"
+  }
+  return 0
+}
+
+# This command is the implementation of SQL function "offrec()". The first
+# argument to this is an offset value. The remaining values are used to
+# formulate an SQLite record. If database file test.db does not contain
+# an equivalent record at the specified offset, an exception is thrown.
+# Otherwise, 0 is returned.
+#
+proc offrec {args} {
+  set offset [lindex $args 0]
+  set rec [hexrecord {*}[lrange $args 1 end]]
+  offset_contains_record $offset $::F $rec
+}
+set F [loadhex test.db]
+db func offrec offrec
+
+# Test the sanity of the tests.
+do_execsql_test func6-105 {
+  SELECT sqlite_offset(d) FROM t1 ORDER BY rowid LIMIT 1;
+} {8179}
+do_test func6-106 {
+  set r [hexrecord abc001 1 999 {}]
+  offset_contains_record 8179 $F $r
+} 0
+
+set z100 [string trim [string repeat "0 " 100]]
+
+# Test offsets within table b-tree t1.
 do_execsql_test func6-110 {
-  SELECT a, sqlite_offset(d)/4096 + 1,
-            sqlite_offset(d)%4096 FROM t1
-   ORDER BY rowid LIMIT 2;
-} {abc001 2 4084 abc002 2 4069}
+  SELECT offrec(sqlite_offset(d), a, b, c, d) FROM t1 ORDER BY rowid
+} $z100
+
 do_execsql_test func6-120 {
   SELECT a, typeof(sqlite_offset(+a)) FROM t1
    ORDER BY rowid LIMIT 2;
 } {abc001 null abc002 null}
+
+# Test offsets within index b-tree t1a.
 do_execsql_test func6-130 {
-  SELECT a, sqlite_offset(a)/4096+1, 
-         sqlite_offset(a)%4096
-   FROM t1
-   ORDER BY a LIMIT 2;
-} {abc001 3 4087 abc002 3 4076}
+  SELECT offrec(sqlite_offset(a), a, rowid) FROM t1 ORDER BY a
+} $z100
+
+# Test offsets within table b-tree t1 with a temp b-tree ORDER BY.
 do_execsql_test func6-140 {
-  SELECT a, sqlite_offset(d)/4096+1, 
-         sqlite_offset(d)%4096
-   FROM t1
-   ORDER BY a LIMIT 2;
-} {abc001 2 4084 abc002 2 4069}
+  SELECT offrec(sqlite_offset(d), a, b, c, d) FROM t1 ORDER BY a
+} $z100
+
+# Test offsets from both index t1a and table t1 in the same query.
 do_execsql_test func6-150 {
-  SELECT a,
-         sqlite_offset(a)/4096+1, 
-         sqlite_offset(a)%4096,
-         sqlite_offset(d)/4096+1, 
-         sqlite_offset(d)%4096
-   FROM t1
-   ORDER BY a LIMIT 2;
-} {abc001 3 4087 2 4084 abc002 3 4076 2 4069}
+  SELECT offrec(sqlite_offset(a), a, rowid),
+         offrec(sqlite_offset(d), a, b, c, d)
+  FROM t1 ORDER BY a
+} [concat $z100 $z100]
+
+# Test offsets from both index t1bc and table t1 in the same query.
 do_execsql_test func6-160 {
-  SELECT b,
-         sqlite_offset(b)/4096+1, 
-         sqlite_offset(b)%4096,
-         sqlite_offset(c)/4096+1, 
-         sqlite_offset(c)%4096,
-         sqlite_offset(d)/4096+1, 
-         sqlite_offset(d)%4096
-   FROM t1
-   ORDER BY b LIMIT 2;
-} {1 4 4090 4 4090 2 4084 2 4 4081 4 4081 2 4069}
-
+  SELECT offrec(sqlite_offset(b), b, c, rowid),
+         offrec(sqlite_offset(c), b, c, rowid),
+         offrec(sqlite_offset(d), a, b, c, d)
+  FROM t1
+  ORDER BY b
+} [concat $z100 $z100 $z100]
 
+# Test offsets in WITHOUT ROWID table t2.
 do_execsql_test func6-200 {
-  SELECT y, sqlite_offset(y)/4096+1,
-         sqlite_offset(y)%4096
-   FROM t2
-   ORDER BY x LIMIT 2;
-} {1 5 4087 2 5 4076}
+  SELECT offrec( sqlite_offset(y), x, y ) FROM t2 ORDER BY x
+} $z100
 
 finish_test

From 510fea885e5b4841a57d13c21958ce650029f408 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 9 Feb 2018 20:49:15 +0000
Subject: [PATCH 430/488] Add the zorder.c extension implementing zorder() and
 unzorder() SQL functions.

FossilOrigin-Name: a57a77dc0cc9fbaa9d5b134422f7a8cc8d4c2851ed3c2bdd449800c6a5d2aae0
---
 ext/misc/zorder.c | 102 ++++++++++++++++++++++++++++++++++++++++++++++
 manifest          |  13 +++---
 manifest.uuid     |   2 +-
 3 files changed, 110 insertions(+), 7 deletions(-)
 create mode 100644 ext/misc/zorder.c

diff --git a/ext/misc/zorder.c b/ext/misc/zorder.c
new file mode 100644
index 0000000000..c385d3c3c3
--- /dev/null
+++ b/ext/misc/zorder.c
@@ -0,0 +1,102 @@
+/*
+** 2018-02-09
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** SQL functions for z-order (Morton code) transformations.
+**
+**      zorder(X0,X0,..,xN)      Generate an N+1 dimension Morton code
+**
+**      unzorder(Z,N,I)          Extract the I-th dimension from N-dimensional
+**                               Morton code Z.
+*/
+#include "sqlite3ext.h"
+SQLITE_EXTENSION_INIT1
+#include <assert.h>
+#include <string.h>
+
+/*
+** Functions:     zorder(X0,X1,....)
+**
+** Convert integers X0, X1, ... into morton code.
+**
+** The output is a signed 64-bit integer.  If any argument is too large,
+** an error is thrown.
+*/
+static void zorderFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  sqlite3_int64 z, x[63];
+  int i, j;
+  z = 0;
+  for(i=0; i<argc; i++){
+    x[i] = sqlite3_value_int64(argv[i]);
+  }
+  if( argc>0 ){
+    for(i=0; i<63; i++){
+      j = i%argc;
+      z |= (x[j]&1)<<i;
+      x[j] >>= 1;
+    }
+  }
+  sqlite3_result_int64(context, z);
+  for(i=0; i<argc; i++){
+    if( x[i] ){
+      sqlite3_result_error(context, "parameter too large", -1);
+    }
+  }
+}
+
+
+/*
+** Functions:     unzorder(Z,N,I)
+**
+** Assuming that Z is an N-dimensional Morton code, extract the I-th
+** dimension.
+*/
+static void unzorderFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  sqlite3_int64 z, n, i, x;
+  int j, k;
+  z = sqlite3_value_int64(argv[0]);
+  n = sqlite3_value_int64(argv[1]);
+  i = sqlite3_value_int64(argv[2]);
+  x = 0;
+  for(k=0, j=i; j<63; j+=n, k++){
+    x |= ((z>>j)&1)<<k;
+  }
+  sqlite3_result_int64(context, x);
+}
+
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_zorder_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  int rc = SQLITE_OK;
+  SQLITE_EXTENSION_INIT2(pApi);
+  (void)pzErrMsg;  /* Unused parameter */
+  rc = sqlite3_create_function(db, "zorder", -1, SQLITE_UTF8, 0,
+                               zorderFunc, 0, 0);
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_create_function(db, "unzorder", 3, SQLITE_UTF8, 0,
+                               unzorderFunc, 0, 0);
+  }
+  return rc;
+}
diff --git a/manifest b/manifest
index a0ff58b5eb..6cda78885d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Make\sthe\stests\sin\sfunc6.test\smore\srobust\sagainst\simplementation\schanges.
-D 2018-02-09T15:42:40.632
+C Add\sthe\szorder.c\sextension\simplementing\szorder()\sand\sunzorder()\sSQL\sfunctions.
+D 2018-02-09T20:49:15.905
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -305,6 +305,7 @@ F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
 F ext/misc/zipfile.c 3c3e21d0351a44c34c67ca7e833ba04e0de34421b4c9359aa764919ee36b37ef
+F ext/misc/zorder.c b0ff58fa643afa1d846786d51ea8d5c4b6b35aa0254ab5a82617db92f3adda64
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee
@@ -1704,7 +1705,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a6c3115483d597fc77ab19fdcfd1d3437cad7e467081ad8c5315fb98c115eed9
-R f6c1b3a83bf1fd9e6dbd920e348c4245
-U dan
-Z baeba07272c3f575c3e3036aa614a6dd
+P b685d3231097fb90e7d61d9ac01cc560e8bf2671d49390ae7af5bfdbd6d04f11
+R 574de61b6452baa1e71eeeef413a6e94
+U drh
+Z 5935918b7bd1bd1e30b211bf68154f29
diff --git a/manifest.uuid b/manifest.uuid
index 3dcc2798f8..c0133280c0 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b685d3231097fb90e7d61d9ac01cc560e8bf2671d49390ae7af5bfdbd6d04f11
\ No newline at end of file
+a57a77dc0cc9fbaa9d5b134422f7a8cc8d4c2851ed3c2bdd449800c6a5d2aae0
\ No newline at end of file

From f313952097725eaf25f80b3431035d19bb1c5c72 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 9 Feb 2018 23:25:14 +0000
Subject: [PATCH 431/488] Improve the performance of the built-in REPLACE()
 function in cases where it does many substitutions that make the string
 larger.  OSSFuzz is reporting intermittant timeouts when running a test where
 it does a REPLACE() on a 930KB random blob. Perhaps this enhancement will fix
 that.

FossilOrigin-Name: fab2c2b07b5d3cd851db3e6f5c8a44155e32b0df22905ea33412b153b825a928
---
 manifest       | 14 +++++++-------
 manifest.uuid  |  2 +-
 src/func.c     | 43 ++++++++++++++++++++++++++-----------------
 test/func.test | 11 +++++++++++
 4 files changed, 45 insertions(+), 25 deletions(-)

diff --git a/manifest b/manifest
index 6cda78885d..e41ea214cc 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\szorder.c\sextension\simplementing\szorder()\sand\sunzorder()\sSQL\sfunctions.
-D 2018-02-09T20:49:15.905
+C Improve\sthe\sperformance\sof\sthe\sbuilt-in\sREPLACE()\sfunction\sin\scases\swhere\nit\sdoes\smany\ssubstitutions\sthat\smake\sthe\sstring\slarger.\s\sOSSFuzz\sis\sreporting\nintermittant\stimeouts\swhen\srunning\sa\stest\swhere\sit\sdoes\sa\sREPLACE()\son\sa\s\n930KB\srandom\sblob.\sPerhaps\sthis\senhancement\swill\sfix\sthat.
+D 2018-02-09T23:25:14.764
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -444,7 +444,7 @@ F src/delete.c 20c8788451dc737a967c87ea53ad43544d617f5b57d32ccce8bd52a0daf9e89b
 F src/expr.c 9e06de431c09f144438aa6895ea4d4290fa3c6875bfcc3ba331012ca78deadf0
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331
-F src/func.c 9970db37cc004136996a5c9b966b86f06583bcf3f275449b977fbb06d75e7300
+F src/func.c 385b9b01851f55c6547f2592ac378572298eb979b02516387ec6f21379d85507
 F src/global.c ac3094f1dc59fbeb919aef7cc0cc827a8459d1fb1adb7972ef75bd9e0c10b75b
 F src/hash.c a12580e143f10301ed5166ea4964ae2853d3905a511d4e0c44497245c7ce1f7a
 F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4
@@ -922,7 +922,7 @@ F test/fts4onepass.test 7319d61a2ed1325fc54afd0c060a0513b462303a
 F test/fts4opt.test fd6a11684b965e1999564ae763797b7fb9e34c96
 F test/fts4unicode.test ceca76422abc251818cb25dabe33d3c3970da5f7c90e1540f190824e6b3a7c95
 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d
-F test/func.test ae97561957aba6ca9e3a7b8a13aac41830d701ef
+F test/func.test 09dda479bcfc568f99f3070413e9672a8eeedc1be9c5d819bf55d4788c2583b7
 F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f
 F test/func3.test d202a7606d23f90988a664e88e268aed1087c11c
 F test/func4.test 6beacdfcb0e18c358e6c2dcacf1b65d1fa80955f
@@ -1705,7 +1705,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b685d3231097fb90e7d61d9ac01cc560e8bf2671d49390ae7af5bfdbd6d04f11
-R 574de61b6452baa1e71eeeef413a6e94
+P a57a77dc0cc9fbaa9d5b134422f7a8cc8d4c2851ed3c2bdd449800c6a5d2aae0
+R adb83e680df996ddba44e43de1a05c6b
 U drh
-Z 5935918b7bd1bd1e30b211bf68154f29
+Z 6ead8882aba40c327286a8b049f64ee5
diff --git a/manifest.uuid b/manifest.uuid
index c0133280c0..fcc9af0088 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a57a77dc0cc9fbaa9d5b134422f7a8cc8d4c2851ed3c2bdd449800c6a5d2aae0
\ No newline at end of file
+fab2c2b07b5d3cd851db3e6f5c8a44155e32b0df22905ea33412b153b825a928
\ No newline at end of file
diff --git a/src/func.c b/src/func.c
index 4e80535e81..399f6dba35 100644
--- a/src/func.c
+++ b/src/func.c
@@ -1199,6 +1199,8 @@ static void replaceFunc(
   i64 nOut;                /* Maximum size of zOut */
   int loopLimit;           /* Last zStr[] that might match zPattern[] */
   int i, j;                /* Loop counters */
+  unsigned cntExpand;      /* Number zOut expansions */
+  sqlite3 *db = sqlite3_context_db_handle(context);
 
   assert( argc==3 );
   UNUSED_PARAMETER(argc);
@@ -1230,33 +1232,40 @@ static void replaceFunc(
     return;
   }
   loopLimit = nStr - nPattern;  
+  cntExpand = 0;
   for(i=j=0; i<=loopLimit; i++){
     if( zStr[i]!=zPattern[0] || memcmp(&zStr[i], zPattern, nPattern) ){
       zOut[j++] = zStr[i];
     }else{
-      u8 *zOld;
-      sqlite3 *db = sqlite3_context_db_handle(context);
-      nOut += nRep - nPattern;
-      testcase( nOut-1==db->aLimit[SQLITE_LIMIT_LENGTH] );
-      testcase( nOut-2==db->aLimit[SQLITE_LIMIT_LENGTH] );
-      if( nOut-1>db->aLimit[SQLITE_LIMIT_LENGTH] ){
-        sqlite3_result_error_toobig(context);
-        sqlite3_free(zOut);
-        return;
-      }
-      zOld = zOut;
-      zOut = sqlite3_realloc64(zOut, (int)nOut);
-      if( zOut==0 ){
-        sqlite3_result_error_nomem(context);
-        sqlite3_free(zOld);
-        return;
+      if( nRep>nPattern ){
+        nOut += nRep - nPattern;
+        if( nOut-1>db->aLimit[SQLITE_LIMIT_LENGTH] ){
+          sqlite3_result_error_toobig(context);
+          sqlite3_free(zOut);
+          return;
+        }
+        testcase( nOut-1==db->aLimit[SQLITE_LIMIT_LENGTH] );
+        testcase( nOut-2==db->aLimit[SQLITE_LIMIT_LENGTH] );
+        cntExpand++;
+        if( (cntExpand&(cntExpand-1))==0 ){
+          /* Grow the size of the output buffer only on substitutions
+          ** whose index is a power of two: 1, 2, 4, 8, 16, 32, ... */
+          u8 *zOld;
+          zOld = zOut;
+          zOut = sqlite3_realloc64(zOut, (int)nOut + (nOut - nStr - 1));
+          if( zOut==0 ){
+            sqlite3_result_error_nomem(context);
+            sqlite3_free(zOld);
+            return;
+          }
+        }
       }
       memcpy(&zOut[j], zRep, nRep);
       j += nRep;
       i += nPattern-1;
     }
   }
-  assert( j+nStr-i+1==nOut );
+  assert( j+nStr-i+1<=nOut );
   memcpy(&zOut[j], &zStr[i], nStr-i);
   j += nStr - i;
   assert( j<=nOut );
diff --git a/test/func.test b/test/func.test
index 98ae8ddeb5..23a3ae4392 100644
--- a/test/func.test
+++ b/test/func.test
@@ -507,6 +507,17 @@ if {$encoding=="UTF-16le"} {
     execsql {SELECT hex(replace('aabcdefg','a','aaa'))}
   } {616161616161626364656667}
 }
+do_execsql_test func-9.14 {
+  WITH RECURSIVE c(x) AS (
+     VALUES(1)
+     UNION ALL
+     SELECT x+1 FROM c WHERE x<1040
+  )
+  SELECT 
+    count(*),
+    sum(length(replace(printf('abc%.*cxyz',x,'m'),'m','nnnn'))-(6+x*4))
+  FROM c;
+} {1040 0}
   
 # Use the "sqlite_register_test_function" TCL command which is part of
 # the text fixture in order to verify correct operation of some of

From c86d82f27d3dc1450a73a7eae83c9cb69a9ca0c7 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 10 Feb 2018 02:31:30 +0000
Subject: [PATCH 432/488] Fix misplaced testcase() macros from the previous
 check-in.

FossilOrigin-Name: 3aed949a18a251c5795f21f0385c205a127502b7e9cf06bc7f4c763951cd7984
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/func.c    |  4 ++--
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index e41ea214cc..bb81d9a18e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improve\sthe\sperformance\sof\sthe\sbuilt-in\sREPLACE()\sfunction\sin\scases\swhere\nit\sdoes\smany\ssubstitutions\sthat\smake\sthe\sstring\slarger.\s\sOSSFuzz\sis\sreporting\nintermittant\stimeouts\swhen\srunning\sa\stest\swhere\sit\sdoes\sa\sREPLACE()\son\sa\s\n930KB\srandom\sblob.\sPerhaps\sthis\senhancement\swill\sfix\sthat.
-D 2018-02-09T23:25:14.764
+C Fix\smisplaced\stestcase()\smacros\sfrom\sthe\sprevious\scheck-in.
+D 2018-02-10T02:31:30.872
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -444,7 +444,7 @@ F src/delete.c 20c8788451dc737a967c87ea53ad43544d617f5b57d32ccce8bd52a0daf9e89b
 F src/expr.c 9e06de431c09f144438aa6895ea4d4290fa3c6875bfcc3ba331012ca78deadf0
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331
-F src/func.c 385b9b01851f55c6547f2592ac378572298eb979b02516387ec6f21379d85507
+F src/func.c 94f42cba2cc1c34aeaa441022ba0170ec3fec4bba54db4e0ded085c6dc0fdc51
 F src/global.c ac3094f1dc59fbeb919aef7cc0cc827a8459d1fb1adb7972ef75bd9e0c10b75b
 F src/hash.c a12580e143f10301ed5166ea4964ae2853d3905a511d4e0c44497245c7ce1f7a
 F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4
@@ -1705,7 +1705,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a57a77dc0cc9fbaa9d5b134422f7a8cc8d4c2851ed3c2bdd449800c6a5d2aae0
-R adb83e680df996ddba44e43de1a05c6b
+P fab2c2b07b5d3cd851db3e6f5c8a44155e32b0df22905ea33412b153b825a928
+R ec6c39c5f311f4ee60e0b56f01a82df2
 U drh
-Z 6ead8882aba40c327286a8b049f64ee5
+Z 0f7daa30b8fefb9cfc3cc8e0bb044dd7
diff --git a/manifest.uuid b/manifest.uuid
index fcc9af0088..611a8eb076 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-fab2c2b07b5d3cd851db3e6f5c8a44155e32b0df22905ea33412b153b825a928
\ No newline at end of file
+3aed949a18a251c5795f21f0385c205a127502b7e9cf06bc7f4c763951cd7984
\ No newline at end of file
diff --git a/src/func.c b/src/func.c
index 399f6dba35..deb7e74928 100644
--- a/src/func.c
+++ b/src/func.c
@@ -1239,13 +1239,13 @@ static void replaceFunc(
     }else{
       if( nRep>nPattern ){
         nOut += nRep - nPattern;
+        testcase( nOut-1==db->aLimit[SQLITE_LIMIT_LENGTH] );
+        testcase( nOut-2==db->aLimit[SQLITE_LIMIT_LENGTH] );
         if( nOut-1>db->aLimit[SQLITE_LIMIT_LENGTH] ){
           sqlite3_result_error_toobig(context);
           sqlite3_free(zOut);
           return;
         }
-        testcase( nOut-1==db->aLimit[SQLITE_LIMIT_LENGTH] );
-        testcase( nOut-2==db->aLimit[SQLITE_LIMIT_LENGTH] );
         cntExpand++;
         if( (cntExpand&(cntExpand-1))==0 ){
           /* Grow the size of the output buffer only on substitutions

From 00c3ef2df906a6e05ce7635f65a49c67a5d391f3 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 12 Feb 2018 13:30:34 +0000
Subject: [PATCH 433/488] Fix a possible infinite loop in VACUUM that can come
 up when the schema has been carefully corrupted.  Problem discovered by
 OSSFuzz. Test cases in TH3.

FossilOrigin-Name: 27754b74ddf6463c6f9c48ea0df3637945b5ea086b92a41ddeb55b9aaefea4c2
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/vacuum.c  |  4 ++--
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index bb81d9a18e..6364a68c58 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\smisplaced\stestcase()\smacros\sfrom\sthe\sprevious\scheck-in.
-D 2018-02-10T02:31:30.872
+C Fix\sa\spossible\sinfinite\sloop\sin\sVACUUM\sthat\scan\scome\sup\swhen\sthe\sschema\nhas\sbeen\scarefully\scorrupted.\s\sProblem\sdiscovered\sby\sOSSFuzz.\sTest\scases\nin\sTH3.
+D 2018-02-12T13:30:34.512
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -556,7 +556,7 @@ F src/trigger.c a34539c69433276d37b0da9a89c117726ff2d292c0902895af1f393a983cd3a1
 F src/update.c a90a32ffc0100265b0693dbbdbe490756447af181f5ea2c138cce515b08c8795
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c d9eb0a6c4aae1b00a7369eadd7ca0bbe946cb4c953b6751aa20d357c2f482157
-F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739
+F src/vacuum.c 762ee9bbf8733d87d8cd06f58d950e881982e416f8c767334a40ffd341b6bff5
 F src/vdbe.c a6892805df427bb9d554c37641406438bff568f99d26e72c1fb7e45f2ed6ce5b
 F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a
 F src/vdbeInt.h 8d7d07f13cb3c4cbca91e22ba4a1920e542dda7c5d9299920432a0b3d5b009f5
@@ -1705,7 +1705,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P fab2c2b07b5d3cd851db3e6f5c8a44155e32b0df22905ea33412b153b825a928
-R ec6c39c5f311f4ee60e0b56f01a82df2
+P 3aed949a18a251c5795f21f0385c205a127502b7e9cf06bc7f4c763951cd7984
+R fd19a34320fb110422297cf4ffe4adf5
 U drh
-Z 0f7daa30b8fefb9cfc3cc8e0bb044dd7
+Z e49790afb4374ff1f3bcb3262a0392c1
diff --git a/manifest.uuid b/manifest.uuid
index 611a8eb076..01c0f92c3d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-3aed949a18a251c5795f21f0385c205a127502b7e9cf06bc7f4c763951cd7984
\ No newline at end of file
+27754b74ddf6463c6f9c48ea0df3637945b5ea086b92a41ddeb55b9aaefea4c2
\ No newline at end of file
diff --git a/src/vacuum.c b/src/vacuum.c
index fde08ddc2a..c7a14a675d 100644
--- a/src/vacuum.c
+++ b/src/vacuum.c
@@ -39,8 +39,8 @@ static int execSql(sqlite3 *db, char **pzErrMsg, const char *zSql){
   while( SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
     const char *zSubSql = (const char*)sqlite3_column_text(pStmt,0);
     assert( sqlite3_strnicmp(zSql,"SELECT",6)==0 );
-    if( zSubSql ){
-      assert( zSubSql[0]!='S' );
+    assert( sqlite3_strnicmp(zSubSql,"SELECT",6)!=0 || CORRUPT_DB );
+    if( zSubSql && zSubSql[0]!='S' ){
       rc = execSql(db, pzErrMsg, zSubSql);
       if( rc!=SQLITE_OK ) break;
     }

From 1db7db1f03303b798ac572dbc2de8a2d7beea745 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 12 Feb 2018 15:27:32 +0000
Subject: [PATCH 434/488] Update test file func6.test so that it works with
 SQLITE_ENABLE_NULL_TRIM builds.

FossilOrigin-Name: e128d72f1a565551d3b6fdb3807fcb1c2a2bf355f0c09d04562b62f49d01fdbc
---
 manifest          | 16 ++++++++--------
 manifest.uuid     |  2 +-
 src/test_config.c |  6 ++++++
 test/func6.test   | 21 +++++++++++++++++++--
 4 files changed, 34 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index 6364a68c58..8f42c5f8a4 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\spossible\sinfinite\sloop\sin\sVACUUM\sthat\scan\scome\sup\swhen\sthe\sschema\nhas\sbeen\scarefully\scorrupted.\s\sProblem\sdiscovered\sby\sOSSFuzz.\sTest\scases\nin\sTH3.
-D 2018-02-12T13:30:34.512
+C Update\stest\sfile\sfunc6.test\sso\sthat\sit\sworks\swith\sSQLITE_ENABLE_NULL_TRIM\nbuilds.
+D 2018-02-12T15:27:32.327
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -513,7 +513,7 @@ F src/test_backup.c bf5da90c9926df0a4b941f2d92825a01bbe090a0
 F src/test_bestindex.c 78809f11026f18a93fcfd798d9479cba37e1201c830260bf1edc674b2fa9b857
 F src/test_blob.c ae4a0620b478548afb67963095a7417cd06a4ec0a56adb453542203bfdcb31ce
 F src/test_btree.c 8b2dc8b8848cf3a4db93f11578f075e82252a274
-F src/test_config.c cc8a1d44648d9392a14f4ecfc841d027daaf61f952b9f70792edf11373aaa3dd
+F src/test_config.c 03df8eb9dc8d5ee96588dfa34fd989c75625d5a0daf59491e32b966da4010870
 F src/test_delete.c e2fe07646dff6300b48d49b2fee2fe192ed389e834dd635e3b3bac0ce0bf9f8f
 F src/test_demovfs.c a0c3bdd45ed044115c2c9f7779e56eafff18741e
 F src/test_devsym.c 1960abbb234b97e9b920f07e99503fc04b443f62bbc3c6ff2c2cea2133e3b8a2
@@ -927,7 +927,7 @@ F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f
 F test/func3.test d202a7606d23f90988a664e88e268aed1087c11c
 F test/func4.test 6beacdfcb0e18c358e6c2dcacf1b65d1fa80955f
 F test/func5.test cdd224400bc3e48d891827cc913a57051a426fa4
-F test/func6.test 612311a51adad23326d15353eea8d90394798acb26dd1f7c7cad59bd8ac982d2
+F test/func6.test 90e42b64c4f9fb6f04f44cb8a1da586c8542502e926b19c76504fe74ff2a9b7c
 F test/fuzz-oss1.test e58330d01cbbd8215ee636b17a03fe220b37dbfa
 F test/fuzz.test 96083052bf5765e4518c1ba686ce2bab785670d1
 F test/fuzz2.test 76dc35b32b6d6f965259508508abce75a6c4d7e1
@@ -1705,7 +1705,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3aed949a18a251c5795f21f0385c205a127502b7e9cf06bc7f4c763951cd7984
-R fd19a34320fb110422297cf4ffe4adf5
-U drh
-Z e49790afb4374ff1f3bcb3262a0392c1
+P 27754b74ddf6463c6f9c48ea0df3637945b5ea086b92a41ddeb55b9aaefea4c2
+R 6efccf0da31b2c149ed049f6fe56fc5d
+U dan
+Z 5bfb70e60829ab8825e3358e0b864cd4
diff --git a/manifest.uuid b/manifest.uuid
index 01c0f92c3d..029b7dbbe1 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-27754b74ddf6463c6f9c48ea0df3637945b5ea086b92a41ddeb55b9aaefea4c2
\ No newline at end of file
+e128d72f1a565551d3b6fdb3807fcb1c2a2bf355f0c09d04562b62f49d01fdbc
\ No newline at end of file
diff --git a/src/test_config.c b/src/test_config.c
index ad63016baf..c3961ad928 100644
--- a/src/test_config.c
+++ b/src/test_config.c
@@ -501,6 +501,12 @@ Tcl_SetVar2(interp, "sqlite_options", "long_double",
 
 Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY);
 
+#ifdef SQLITE_ENABLE_NULL_TRIM
+  Tcl_SetVar2(interp, "sqlite_options", "null_trim", "1", TCL_GLOBAL_ONLY);
+#else
+  Tcl_SetVar2(interp, "sqlite_options", "null_trim", "0", TCL_GLOBAL_ONLY);
+#endif
+
 #ifdef SQLITE_OMIT_OR_OPTIMIZATION
   Tcl_SetVar2(interp, "sqlite_options", "or_opt", "0", TCL_GLOBAL_ONLY);
 #else
diff --git a/test/func6.test b/test/func6.test
index 50415e7963..1e16a7ca38 100644
--- a/test/func6.test
+++ b/test/func6.test
@@ -22,6 +22,11 @@ ifcapable !offset_sql_func {
   return
 }
 
+set bNullTrim 0
+ifcapable null_trim {
+  set bNullTrim 1
+}
+
 do_execsql_test func6-100 {
   PRAGMA page_size=4096;
   PRAGMA auto_vacuum=NONE;
@@ -51,6 +56,13 @@ proc loadhex {file} {
 proc hexrecord {args} {
   set hdr ""
   set body ""
+
+  if {$::bNullTrim} {
+    while {[llength $args] && [lindex $args end]=={}} {
+      set args [lrange $args 0 end-1]
+    }
+  }
+
   foreach x $args {
     if {$x==""} {
       append hdr 00
@@ -103,12 +115,17 @@ set F [loadhex test.db]
 db func offrec offrec
 
 # Test the sanity of the tests.
+if {$bNullTrim} {
+  set offset 8180
+} else {
+  set offset 8179
+}
 do_execsql_test func6-105 {
   SELECT sqlite_offset(d) FROM t1 ORDER BY rowid LIMIT 1;
-} {8179}
+} $offset
 do_test func6-106 {
   set r [hexrecord abc001 1 999 {}]
-  offset_contains_record 8179 $F $r
+  offset_contains_record $offset $F $r
 } 0
 
 set z100 [string trim [string repeat "0 " 100]]

From 35043cc32a4717edcc36153b2a1ced67d8750232 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 12 Feb 2018 20:27:34 +0000
Subject: [PATCH 435/488] When compiling with VDBE_PROFILE, add the
 sqlite3NProfileCnt global variable which can be used to measure per-opcode
 values other than elapse time, if non-zero.

FossilOrigin-Name: d44d5936428859b6d3e089d9835acb8ca1f3c0a85b59eeaa619170623e7dcc30
---
 manifest        | 18 +++++++++---------
 manifest.uuid   |  2 +-
 src/global.c    |  7 +++++++
 src/sqliteInt.h |  3 +++
 src/vdbe.c      |  4 ++--
 5 files changed, 22 insertions(+), 12 deletions(-)

diff --git a/manifest b/manifest
index 8f42c5f8a4..b9699c0d2d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\stest\sfile\sfunc6.test\sso\sthat\sit\sworks\swith\sSQLITE_ENABLE_NULL_TRIM\nbuilds.
-D 2018-02-12T15:27:32.327
+C When\scompiling\swith\sVDBE_PROFILE,\sadd\sthe\ssqlite3NProfileCnt\sglobal\svariable\nwhich\scan\sbe\sused\sto\smeasure\sper-opcode\svalues\sother\sthan\selapse\stime,\sif\nnon-zero.
+D 2018-02-12T20:27:34.197
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -445,7 +445,7 @@ F src/expr.c 9e06de431c09f144438aa6895ea4d4290fa3c6875bfcc3ba331012ca78deadf0
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331
 F src/func.c 94f42cba2cc1c34aeaa441022ba0170ec3fec4bba54db4e0ded085c6dc0fdc51
-F src/global.c ac3094f1dc59fbeb919aef7cc0cc827a8459d1fb1adb7972ef75bd9e0c10b75b
+F src/global.c 01506976bd75e5e7b977207a6a05062e2dd0050012f8071be06bbea22ec6d69a
 F src/hash.c a12580e143f10301ed5166ea4964ae2853d3905a511d4e0c44497245c7ce1f7a
 F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4
 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da
@@ -493,7 +493,7 @@ F src/shell.c.in 0b2c8d55e3703723a22610d75ccd4037962c3bcb5ad5570efd781e8c521811c
 F src/sqlite.h.in 51f9acf52c80113d793ddd38b3940ad6895d97b4752503b19291fb8fcbf54c5e
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d
-F src/sqliteInt.h 119e240796d23054148005524f1d54a61e27dbef53bef3bc7ab65001405f4751
+F src/sqliteInt.h fd8943e9d6b8bd04b3a22ef806b843c2c06f022db3f37f1436ac3a9b04e5b8a1
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -557,7 +557,7 @@ F src/update.c a90a32ffc0100265b0693dbbdbe490756447af181f5ea2c138cce515b08c8795
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c d9eb0a6c4aae1b00a7369eadd7ca0bbe946cb4c953b6751aa20d357c2f482157
 F src/vacuum.c 762ee9bbf8733d87d8cd06f58d950e881982e416f8c767334a40ffd341b6bff5
-F src/vdbe.c a6892805df427bb9d554c37641406438bff568f99d26e72c1fb7e45f2ed6ce5b
+F src/vdbe.c 72f5a4b95be486170104779aeef000423acabf55ac0c661afdae889be72c8922
 F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a
 F src/vdbeInt.h 8d7d07f13cb3c4cbca91e22ba4a1920e542dda7c5d9299920432a0b3d5b009f5
 F src/vdbeapi.c fea41171884a4de119f8b10ab514c788674eeeb7f27218bb6d008e1310bfd07f
@@ -1705,7 +1705,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 27754b74ddf6463c6f9c48ea0df3637945b5ea086b92a41ddeb55b9aaefea4c2
-R 6efccf0da31b2c149ed049f6fe56fc5d
-U dan
-Z 5bfb70e60829ab8825e3358e0b864cd4
+P e128d72f1a565551d3b6fdb3807fcb1c2a2bf355f0c09d04562b62f49d01fdbc
+R ea7b0f8bed7883637ff159eaee70912b
+U drh
+Z 7df15e7989dc925e831e6a33868c90fa
diff --git a/manifest.uuid b/manifest.uuid
index 029b7dbbe1..c9cd92d93c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e128d72f1a565551d3b6fdb3807fcb1c2a2bf355f0c09d04562b62f49d01fdbc
\ No newline at end of file
+d44d5936428859b6d3e089d9835acb8ca1f3c0a85b59eeaa619170623e7dcc30
\ No newline at end of file
diff --git a/src/global.c b/src/global.c
index e66be81495..04a3d185a8 100644
--- a/src/global.c
+++ b/src/global.c
@@ -258,6 +258,13 @@ const Token sqlite3IntTokens[] = {
    { "1", 1 }
 };
 
+#ifdef VDBE_PROFILE
+/*
+** The following performance counter can be used in place of
+** sqlite3Hwtime() for profiling.  This is a no-op on standard builds.
+*/
+sqlite3_uint64 sqlite3NProfileCnt = 0;
+#endif
 
 /*
 ** The value of the "pending" byte must be 0x40000000 (1 byte past the
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index f408ff62a5..f4c82cdb68 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -4069,6 +4069,9 @@ extern FuncDefHash sqlite3BuiltinFunctions;
 extern int sqlite3PendingByte;
 #endif
 #endif
+#ifdef VDBE_PROFILE
+extern sqlite3_uint64 sqlite3NProfileCnt;
+#endif
 void sqlite3RootPageMoved(sqlite3*, int, int, int);
 void sqlite3Reindex(Parse*, Token*, Token*);
 void sqlite3AlterFunctions(void);
diff --git a/src/vdbe.c b/src/vdbe.c
index 1bcaedbfee..ebf221d02f 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -648,7 +648,7 @@ int sqlite3VdbeExec(
 
     assert( pOp>=aOp && pOp<&aOp[p->nOp]);
 #ifdef VDBE_PROFILE
-    start = sqlite3Hwtime();
+    start = sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime();
 #endif
     nVmStep++;
 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
@@ -7233,7 +7233,7 @@ default: {          /* This is really OP_Noop and OP_Explain */
 
 #ifdef VDBE_PROFILE
     {
-      u64 endTime = sqlite3Hwtime();
+      u64 endTime = sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime();
       if( endTime>start ) pOrigOp->cycles += endTime - start;
       pOrigOp->cnt++;
     }

From c083cb4422a602f1c1a5c08ec3d9d2460ee8716a Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 12 Feb 2018 20:50:28 +0000
Subject: [PATCH 436/488] Updates to the speed-check.sh script for improvements
 to the --vdbeprofile option.

FossilOrigin-Name: f89c8f8ecd67f72ee730a2a334dff69b406a3dd00f50abbb54c4dc86cfab1bd1
---
 manifest            | 12 ++++++------
 manifest.uuid       |  2 +-
 tool/speed-check.sh | 11 ++++++++++-
 3 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index b9699c0d2d..240d519aa1 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C When\scompiling\swith\sVDBE_PROFILE,\sadd\sthe\ssqlite3NProfileCnt\sglobal\svariable\nwhich\scan\sbe\sused\sto\smeasure\sper-opcode\svalues\sother\sthan\selapse\stime,\sif\nnon-zero.
-D 2018-02-12T20:27:34.197
+C Updates\sto\sthe\sspeed-check.sh\sscript\sfor\simprovements\sto\sthe\s--vdbeprofile\noption.
+D 2018-02-12T20:50:28.825
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -1663,7 +1663,7 @@ F tool/showstat4.c 0682ebea7abf4d3657f53c4a243f2e7eab48eab344ed36a94bb75dcd19a5c
 F tool/showwal.c ad9d768f96ca6199ad3a8c9562d679680bd032dd01204ea3e5ea6fb931d81847
 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe
 F tool/spaceanal.tcl 4bfd19aad7eb3ce0372ef0255f58035e0bba4ff5e9acfd763a10c6fb365c8dec
-F tool/speed-check.sh 9ae425da8819e54e780cf494fc6d8175dfb16e109ae3214a45a5c9bb2b74e2c4
+F tool/speed-check.sh c6239c71ba6951ac5d093bbea55aadf019246c8e7a0c079422e62312a6ab1247
 F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355
 F tool/speedtest16.c ecb6542862151c3e6509bbc00509b234562ae81e
 F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
@@ -1705,7 +1705,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e128d72f1a565551d3b6fdb3807fcb1c2a2bf355f0c09d04562b62f49d01fdbc
-R ea7b0f8bed7883637ff159eaee70912b
+P d44d5936428859b6d3e089d9835acb8ca1f3c0a85b59eeaa619170623e7dcc30
+R ed478e21f56ba8798873dd31125e1435
 U drh
-Z 7df15e7989dc925e831e6a33868c90fa
+Z b17fd438653dec89029ec54019e16fa1
diff --git a/manifest.uuid b/manifest.uuid
index c9cd92d93c..a9e9aba772 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d44d5936428859b6d3e089d9835acb8ca1f3c0a85b59eeaa619170623e7dcc30
\ No newline at end of file
+f89c8f8ecd67f72ee730a2a334dff69b406a3dd00f50abbb54c4dc86cfab1bd1
\ No newline at end of file
diff --git a/tool/speed-check.sh b/tool/speed-check.sh
index 6cc3018981..cc31e6aa0a 100644
--- a/tool/speed-check.sh
+++ b/tool/speed-check.sh
@@ -39,6 +39,7 @@ LEAN_OPTS="$LEAN_OPTS -DSQLITE_USE_ALLOCA"
 BASELINE="trunk"
 doExplain=0
 doCachegrind=1
+doVdbeProfile=0
 while test "$1" != ""; do
   case $1 in
     --reprepare)
@@ -78,6 +79,7 @@ while test "$1" != ""; do
         rm -f vdbe_profile.out
         CC_OPTS="$CC_OPTS -DVDBE_PROFILE"
         doCachegrind=0
+        doVdbeProfile=1
         ;;
     --lean)
         CC_OPTS="$CC_OPTS $LEAN_OPTS"
@@ -137,6 +139,9 @@ echo "NAME           = $NAME" | tee summary-$NAME.txt
 echo "SPEEDTEST_OPTS = $SPEEDTEST_OPTS" | tee -a summary-$NAME.txt
 echo "CC_OPTS        = $CC_OPTS" | tee -a summary-$NAME.txt
 rm -f cachegrind.out.* speedtest1 speedtest1.db sqlite3.o
+if test $doVdbeProfile -eq 1; then
+  rm -f vdbe_profile.out
+fi
 $CC -g -Os -Wall -I. $CC_OPTS -c sqlite3.c
 size sqlite3.o | tee -a summary-$NAME.txt
 if test $doExplain -eq 1; then
@@ -163,6 +168,10 @@ fi
 if test $doExplain -eq 1; then
   ./speedtest1 --explain $SPEEDTEST_OPTS | ./sqlite3 >explain-$NAME.txt
 fi
-if test "$NAME" != "$BASELINE"; then
+if test $doVdbeProfile -eq 1; then
+  tclsh ../sqlite/tool/vdbe_profile.tcl >vdbeprofile-$NAME.txt
+  open vdbeprofile-$NAME.txt
+fi
+if test "$NAME" != "$BASELINE" -a $doVdbeProfile -ne 1; then
   fossil test-diff --tk -c 20 cout-$BASELINE.txt cout-$NAME.txt
 fi

From e84c498be19511245f2acd5c6f71dad441d908fb Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 13 Feb 2018 16:21:32 +0000
Subject: [PATCH 437/488] Ensure that multiple updates of the same FTS4 row
 (i.e. the row with the same rowid) within a single transaction are written to
 separate segments. Fix for [d6ec09ec].

FossilOrigin-Name: d2a81a3752a8298ee65fddcf32dee2147ba6467f770315a41966b3533977aa64
---
 ext/fts3/fts3_write.c |  2 +-
 manifest              | 16 ++++++++--------
 manifest.uuid         |  2 +-
 test/fts4onepass.test | 14 ++++++++++++++
 4 files changed, 24 insertions(+), 10 deletions(-)

diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c
index 5bca766c20..792a498a6f 100644
--- a/ext/fts3/fts3_write.c
+++ b/ext/fts3/fts3_write.c
@@ -5639,7 +5639,7 @@ int sqlite3Fts3UpdateMethod(
         rc = FTS_CORRUPT_VTAB;
       }
     }
-    if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){
+    if( rc==SQLITE_OK ){
       rc = fts3PendingTermsDocid(p, 0, iLangid, *pRowid);
     }
     if( rc==SQLITE_OK ){
diff --git a/manifest b/manifest
index 240d519aa1..ecd5e1242b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Updates\sto\sthe\sspeed-check.sh\sscript\sfor\simprovements\sto\sthe\s--vdbeprofile\noption.
-D 2018-02-12T20:50:28.825
+C Ensure\sthat\smultiple\supdates\sof\sthe\ssame\sFTS4\srow\s(i.e.\sthe\srow\swith\sthe\ssame\nrowid)\swithin\sa\ssingle\stransaction\sare\swritten\sto\sseparate\ssegments.\sFix\sfor\n[d6ec09ec].
+D 2018-02-13T16:21:32.960
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -96,7 +96,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3
 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004
 F ext/fts3/fts3_unicode.c 525a3bd9a7564603c5c061b7de55403a565307758a94600e8a2f6b00d1c40d9d
 F ext/fts3/fts3_unicode2.c cc04fc672bfd42b1e650398cb0bf71f64f9aae032cfe75bbcfe75b9cf966029c
-F ext/fts3/fts3_write.c 7a7cf93c02ebe0ee4211e4aa07da77586c2dcf7d381c1382f81d29c9aa8cae8c
+F ext/fts3/fts3_write.c 434fe5eaf6b848e9fa18dd269ab2450732f776ffb4d8051cfd0eef2c510ec5da
 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
 F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87fea4ac01b73
@@ -918,7 +918,7 @@ F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891
 F test/fts4merge3.test 8d9ccb4a3d41c4c617a149d6c4b13ad02de797d0
 F test/fts4merge4.test d895b1057a7798b67e03455d0fa50e9ea836c47b
 F test/fts4noti.test 5553d7bb2e20bf4a06b23e849352efc022ce6309
-F test/fts4onepass.test 7319d61a2ed1325fc54afd0c060a0513b462303a
+F test/fts4onepass.test d69ddc4ee3415e40b0c5d1d0408488a87614d4f63ba9c44f3e52db541d6b7cc7
 F test/fts4opt.test fd6a11684b965e1999564ae763797b7fb9e34c96
 F test/fts4unicode.test ceca76422abc251818cb25dabe33d3c3970da5f7c90e1540f190824e6b3a7c95
 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d
@@ -1705,7 +1705,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d44d5936428859b6d3e089d9835acb8ca1f3c0a85b59eeaa619170623e7dcc30
-R ed478e21f56ba8798873dd31125e1435
-U drh
-Z b17fd438653dec89029ec54019e16fa1
+P f89c8f8ecd67f72ee730a2a334dff69b406a3dd00f50abbb54c4dc86cfab1bd1
+R 6201bc04be1c8ea59cb7f6218616fce1
+U dan
+Z a105e3a27f3cee541040c34c5a8f6a03
diff --git a/manifest.uuid b/manifest.uuid
index a9e9aba772..a809b05e72 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f89c8f8ecd67f72ee730a2a334dff69b406a3dd00f50abbb54c4dc86cfab1bd1
\ No newline at end of file
+d2a81a3752a8298ee65fddcf32dee2147ba6467f770315a41966b3533977aa64
\ No newline at end of file
diff --git a/test/fts4onepass.test b/test/fts4onepass.test
index 46cb4b794b..344be4b1df 100644
--- a/test/fts4onepass.test
+++ b/test/fts4onepass.test
@@ -143,4 +143,18 @@ foreach {tn tcl1 tcl2}  {
   eval $tcl2
 }
 
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE zt USING fts4(a, b);
+  INSERT INTO zt(rowid, a, b) VALUES(1, 'unus duo', NULL);
+  INSERT INTO zt(rowid, a, b) VALUES(2, NULL, NULL);
+
+  BEGIN;
+    UPDATE zt SET b='septum' WHERE rowid = 1;
+    UPDATE zt SET b='octo' WHERE rowid = 1;
+  COMMIT;
+
+  SELECT count(*) FROM zt_segdir;
+} {3}
+
+
 finish_test

From d446e79545c8cf260869e2436a85a43ec4722a40 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 13 Feb 2018 16:30:19 +0000
Subject: [PATCH 438/488] Remove an unused variable from FTS4.

FossilOrigin-Name: 27ea783b281606196ad1e7e6e6dcbd80c75d4bafcf3e58756a71c213320e4db2
---
 ext/fts3/fts3_write.c |  2 --
 manifest              | 14 +++++++-------
 manifest.uuid         |  2 +-
 3 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c
index 792a498a6f..0baf82b76e 100644
--- a/ext/fts3/fts3_write.c
+++ b/ext/fts3/fts3_write.c
@@ -5529,7 +5529,6 @@ int sqlite3Fts3UpdateMethod(
 ){
   Fts3Table *p = (Fts3Table *)pVtab;
   int rc = SQLITE_OK;             /* Return Code */
-  int isRemove = 0;               /* True for an UPDATE or DELETE */
   u32 *aSzIns = 0;                /* Sizes of inserted documents */
   u32 *aSzDel = 0;                /* Sizes of deleted documents */
   int nChng = 0;                  /* Net change in number of documents */
@@ -5627,7 +5626,6 @@ int sqlite3Fts3UpdateMethod(
   if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
     assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER );
     rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel);
-    isRemove = 1;
   }
   
   /* If this is an INSERT or UPDATE operation, insert the new record. */
diff --git a/manifest b/manifest
index ecd5e1242b..a075d14e40 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Ensure\sthat\smultiple\supdates\sof\sthe\ssame\sFTS4\srow\s(i.e.\sthe\srow\swith\sthe\ssame\nrowid)\swithin\sa\ssingle\stransaction\sare\swritten\sto\sseparate\ssegments.\sFix\sfor\n[d6ec09ec].
-D 2018-02-13T16:21:32.960
+C Remove\san\sunused\svariable\sfrom\sFTS4.
+D 2018-02-13T16:30:19.084
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -96,7 +96,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3
 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004
 F ext/fts3/fts3_unicode.c 525a3bd9a7564603c5c061b7de55403a565307758a94600e8a2f6b00d1c40d9d
 F ext/fts3/fts3_unicode2.c cc04fc672bfd42b1e650398cb0bf71f64f9aae032cfe75bbcfe75b9cf966029c
-F ext/fts3/fts3_write.c 434fe5eaf6b848e9fa18dd269ab2450732f776ffb4d8051cfd0eef2c510ec5da
+F ext/fts3/fts3_write.c b583dede85eb0c3c3026f8d7ccb781ea4e845ae583754fecb2ca425b5907d87d
 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
 F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87fea4ac01b73
@@ -1705,7 +1705,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P f89c8f8ecd67f72ee730a2a334dff69b406a3dd00f50abbb54c4dc86cfab1bd1
-R 6201bc04be1c8ea59cb7f6218616fce1
-U dan
-Z a105e3a27f3cee541040c34c5a8f6a03
+P d2a81a3752a8298ee65fddcf32dee2147ba6467f770315a41966b3533977aa64
+R 24596befc5c0ec11ef119fbee4890efb
+U drh
+Z ec945a84da77652b5b990130e3b70870
diff --git a/manifest.uuid b/manifest.uuid
index a809b05e72..80c33318ca 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d2a81a3752a8298ee65fddcf32dee2147ba6467f770315a41966b3533977aa64
\ No newline at end of file
+27ea783b281606196ad1e7e6e6dcbd80c75d4bafcf3e58756a71c213320e4db2
\ No newline at end of file

From 4d1c6845ff65486beff0d35255694664dcd50686 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 13 Feb 2018 18:48:08 +0000
Subject: [PATCH 439/488] Fix an incorrect table lookup used to find the
 appropriate search operator for a WHERE clause on a row-value inequality. 
 The incorrect table lookup was causing an incorrect answer for the less-than
 operator. Fix for ticket [f484b65f3d6230593c34f11]

FossilOrigin-Name: f3112e67cdb27c1aec8d2cee3cb91ade061d093e13505894698e26336898b336
---
 manifest           |  14 +++---
 manifest.uuid      |   2 +-
 src/wherecode.c    |  10 ++++-
 test/rowvalue.test | 103 ++++++++++++++++++++++++++++++++++++++++++++-
 4 files changed, 119 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index a075d14e40..378392f9f0 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\san\sunused\svariable\sfrom\sFTS4.
-D 2018-02-13T16:30:19.084
+C Fix\san\sincorrect\stable\slookup\sused\sto\sfind\sthe\sappropriate\ssearch\soperator\nfor\sa\sWHERE\sclause\son\sa\srow-value\sinequality.\s\sThe\sincorrect\stable\slookup\nwas\scausing\san\sincorrect\sanswer\sfor\sthe\sless-than\soperator.\nFix\sfor\sticket\s[f484b65f3d6230593c34f11]
+D 2018-02-13T18:48:08.593
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -573,7 +573,7 @@ F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f
 F src/where.c 7cae47e813393d70c6d327fdf000fcb30f76b1b0b5a5b52ff6402e0c658de32c
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
-F src/wherecode.c cb360c511f69294ddf00340d7b390e6b1d601a1cfb77b42c4d316fe2f4cd01c3
+F src/wherecode.c e1aaadd8fec650037cfbf27d1b3470338fb3b58fec34d11082df16fe9a08fbd7
 F src/whereexpr.c 22dbfd3bf5f6051a61523dd0ebef7a944fb29ee4aa7d2a62feb8aac6ffbbc0eb
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd
@@ -1166,7 +1166,7 @@ F test/rollbackfault.test 0e646aeab8840c399cfbfa43daab46fd609cf04a
 F test/rowallock.test 3f88ec6819489d0b2341c7a7528ae17c053ab7cc
 F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81
 F test/rowid.test 5b7509f384f4f6fae1af3c8c104c8ca299fea18d
-F test/rowvalue.test e1bd1690d891abff6138f9612241615d2de7671cb28eb58ebdd591a5b01089a4
+F test/rowvalue.test a786af333ab4d64e2ac91e4a234851a959ab61ea74b4fc9db333f5f8a9cb24f0
 F test/rowvalue2.test 060d238b7e5639a7c5630cb5e63e311b44efef2b
 F test/rowvalue3.test 3068f508753af69884b12125995f023da0dbb256
 F test/rowvalue4.test 4b556d7de161a0dd8cff095c336e913986398bea
@@ -1705,7 +1705,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d2a81a3752a8298ee65fddcf32dee2147ba6467f770315a41966b3533977aa64
-R 24596befc5c0ec11ef119fbee4890efb
+P 27ea783b281606196ad1e7e6e6dcbd80c75d4bafcf3e58756a71c213320e4db2
+R db63889646c3d3e78e93aa0d3ea314fa
 U drh
-Z ec945a84da77652b5b990130e3b70870
+Z e60bc3e5bf72f574703087920d14128f
diff --git a/manifest.uuid b/manifest.uuid
index 80c33318ca..4a351d48d9 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-27ea783b281606196ad1e7e6e6dcbd80c75d4bafcf3e58756a71c213320e4db2
\ No newline at end of file
+f3112e67cdb27c1aec8d2cee3cb91ade061d093e13505894698e26336898b336
\ No newline at end of file
diff --git a/src/wherecode.c b/src/wherecode.c
index 8e8672ff9f..e40a940ac1 100644
--- a/src/wherecode.c
+++ b/src/wherecode.c
@@ -1395,7 +1395,15 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       if( sqlite3ExprIsVector(pX->pRight) ){
         r1 = rTemp = sqlite3GetTempReg(pParse);
         codeExprOrVector(pParse, pX->pRight, r1, 1);
-        op = aMoveOp[(pX->op - TK_GT) | 0x0001];
+        testcase( pX->op==TK_GT );
+        testcase( pX->op==TK_GE );
+        testcase( pX->op==TK_LT );
+        testcase( pX->op==TK_LE );
+        op = aMoveOp[((pX->op - TK_GT - 1) & 0x3) | 0x1];
+        assert( pX->op!=TK_GT || op==OP_SeekGE );
+        assert( pX->op!=TK_GE || op==OP_SeekGE );
+        assert( pX->op!=TK_LT || op==OP_SeekLE );
+        assert( pX->op!=TK_LE || op==OP_SeekLE );
       }else{
         r1 = sqlite3ExprCodeTemp(pParse, pX->pRight, &rTemp);
         disableTerm(pLevel, pStart);
diff --git a/test/rowvalue.test b/test/rowvalue.test
index d900e0daa3..8a5f6f3ef4 100644
--- a/test/rowvalue.test
+++ b/test/rowvalue.test
@@ -437,6 +437,107 @@ do_execsql_test 18.6 {
 } {1 1 1 1 2 1}
 
  
-finish_test
+# 2018-02-13 Ticket https://www.sqlite.org/src/tktview/f484b65f3d6230593c3
+# Incorrect result from a row-value comparison in the WHERE clause.
+#
+do_execsql_test 19.1 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1(a INTEGER PRIMARY KEY,b);
+  INSERT INTO t1(a,b) VALUES(1,11),(2,22),(3,33),(4,44);
+  SELECT * FROM t1 WHERE (a,b)>(0,0) ORDER BY +a;
+} {1 11 2 22 3 33 4 44}
+do_execsql_test 19.2 {
+  SELECT * FROM t1 WHERE (a,b)>=(0,0) ORDER BY +a;
+} {1 11 2 22 3 33 4 44}
+do_execsql_test 19.3 {
+  SELECT * FROM t1 WHERE (a,b)<(5,0) ORDER BY +a;
+} {1 11 2 22 3 33 4 44}
+do_execsql_test 19.4 {
+  SELECT * FROM t1 WHERE (a,b)<=(5,0) ORDER BY +a;
+} {1 11 2 22 3 33 4 44}
+do_execsql_test 19.5 {
+  SELECT * FROM t1 WHERE (a,b)>(3,0) ORDER BY +a;
+} {3 33 4 44}
+do_execsql_test 19.6 {
+  SELECT * FROM t1 WHERE (a,b)>=(3,0) ORDER BY +a;
+} {3 33 4 44}
+do_execsql_test 19.7 {
+  SELECT * FROM t1 WHERE (a,b)<(3,0) ORDER BY +a;
+} {1 11 2 22}
+do_execsql_test 19.8 {
+  SELECT * FROM t1 WHERE (a,b)<=(3,0) ORDER BY +a;
+} {1 11 2 22}
+do_execsql_test 19.9 {
+  SELECT * FROM t1 WHERE (a,b)>(3,32) ORDER BY +a;
+} {3 33 4 44}
+do_execsql_test 19.10 {
+  SELECT * FROM t1 WHERE (a,b)>(3,33) ORDER BY +a;
+} {4 44}
+do_execsql_test 19.11 {
+  SELECT * FROM t1 WHERE (a,b)>=(3,33) ORDER BY +a;
+} {3 33 4 44}
+do_execsql_test 19.12 {
+  SELECT * FROM t1 WHERE (a,b)>=(3,34) ORDER BY +a;
+} {4 44}
+do_execsql_test 19.13 {
+  SELECT * FROM t1 WHERE (a,b)<(3,34) ORDER BY +a;
+} {1 11 2 22 3 33}
+do_execsql_test 19.14 {
+  SELECT * FROM t1 WHERE (a,b)<(3,33) ORDER BY +a;
+} {1 11 2 22}
+do_execsql_test 19.15 {
+  SELECT * FROM t1 WHERE (a,b)<=(3,33) ORDER BY +a;
+} {1 11 2 22 3 33}
+do_execsql_test 19.16 {
+  SELECT * FROM t1 WHERE (a,b)<=(3,32) ORDER BY +a;
+} {1 11 2 22}
+do_execsql_test 19.21 {
+  SELECT * FROM t1 WHERE (0,0)<(a,b) ORDER BY +a;
+} {1 11 2 22 3 33 4 44}
+do_execsql_test 19.22 {
+  SELECT * FROM t1 WHERE (0,0)<=(a,b) ORDER BY +a;
+} {1 11 2 22 3 33 4 44}
+do_execsql_test 19.23 {
+  SELECT * FROM t1 WHERE (5,0)>(a,b) ORDER BY +a;
+} {1 11 2 22 3 33 4 44}
+do_execsql_test 19.24 {
+  SELECT * FROM t1 WHERE (5,0)>=(a,b) ORDER BY +a;
+} {1 11 2 22 3 33 4 44}
+do_execsql_test 19.25 {
+  SELECT * FROM t1 WHERE (3,0)<(a,b) ORDER BY +a;
+} {3 33 4 44}
+do_execsql_test 19.26 {
+  SELECT * FROM t1 WHERE (3,0)<=(a,b) ORDER BY +a;
+} {3 33 4 44}
+do_execsql_test 19.27 {
+  SELECT * FROM t1 WHERE (3,0)>(a,b) ORDER BY +a;
+} {1 11 2 22}
+do_execsql_test 19.28 {
+  SELECT * FROM t1 WHERE (3,0)>=(a,b) ORDER BY +a;
+} {1 11 2 22}
+do_execsql_test 19.29 {
+  SELECT * FROM t1 WHERE (3,32)<(a,b) ORDER BY +a;
+} {3 33 4 44}
+do_execsql_test 19.30 {
+  SELECT * FROM t1 WHERE (3,33)<(a,b) ORDER BY +a;
+} {4 44}
+do_execsql_test 19.31 {
+  SELECT * FROM t1 WHERE (3,33)<=(a,b) ORDER BY +a;
+} {3 33 4 44}
+do_execsql_test 19.32 {
+  SELECT * FROM t1 WHERE (3,34)<=(a,b) ORDER BY +a;
+} {4 44}
+do_execsql_test 19.33 {
+  SELECT * FROM t1 WHERE (3,34)>(a,b) ORDER BY +a;
+} {1 11 2 22 3 33}
+do_execsql_test 19.34 {
+  SELECT * FROM t1 WHERE (3,33)>(a,b) ORDER BY +a;
+} {1 11 2 22}
+do_execsql_test 19.35 {
+  SELECT * FROM t1 WHERE (3,33)>=(a,b) ORDER BY +a;
+} {1 11 2 22 3 33}
+do_execsql_test 19.36 {
+  SELECT * FROM t1 WHERE (3,32)>=(a,b) ORDER BY +a;
+} {1 11 2 22}
 
 finish_test

From a9abfb1e6c4f6d7a37613c52da9e2316a5333341 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 13 Feb 2018 19:13:05 +0000
Subject: [PATCH 440/488] Improved test cases for the [f484b65f3d6230593c34f]
 bug.

FossilOrigin-Name: 1f300514e6cdd32c755dc1b05dfa125b10eda2eb55a1024b66f66dbf5cf96289
---
 manifest           | 12 +++---
 manifest.uuid      |  2 +-
 test/rowvalue.test | 96 +++++++++++++++++++++++-----------------------
 3 files changed, 55 insertions(+), 55 deletions(-)

diff --git a/manifest b/manifest
index 378392f9f0..7a723a9a49 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\san\sincorrect\stable\slookup\sused\sto\sfind\sthe\sappropriate\ssearch\soperator\nfor\sa\sWHERE\sclause\son\sa\srow-value\sinequality.\s\sThe\sincorrect\stable\slookup\nwas\scausing\san\sincorrect\sanswer\sfor\sthe\sless-than\soperator.\nFix\sfor\sticket\s[f484b65f3d6230593c34f11]
-D 2018-02-13T18:48:08.593
+C Improved\stest\scases\sfor\sthe\s[f484b65f3d6230593c34f]\sbug.
+D 2018-02-13T19:13:05.932
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -1166,7 +1166,7 @@ F test/rollbackfault.test 0e646aeab8840c399cfbfa43daab46fd609cf04a
 F test/rowallock.test 3f88ec6819489d0b2341c7a7528ae17c053ab7cc
 F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81
 F test/rowid.test 5b7509f384f4f6fae1af3c8c104c8ca299fea18d
-F test/rowvalue.test a786af333ab4d64e2ac91e4a234851a959ab61ea74b4fc9db333f5f8a9cb24f0
+F test/rowvalue.test 0bc33483f2ef5e69ff4bdd2ae58e36fc598bfd1605fb718c8329bcfc0c10cfd1
 F test/rowvalue2.test 060d238b7e5639a7c5630cb5e63e311b44efef2b
 F test/rowvalue3.test 3068f508753af69884b12125995f023da0dbb256
 F test/rowvalue4.test 4b556d7de161a0dd8cff095c336e913986398bea
@@ -1705,7 +1705,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 27ea783b281606196ad1e7e6e6dcbd80c75d4bafcf3e58756a71c213320e4db2
-R db63889646c3d3e78e93aa0d3ea314fa
+P f3112e67cdb27c1aec8d2cee3cb91ade061d093e13505894698e26336898b336
+R 5562ebb89ed1db5735eb0527fd19eb1a
 U drh
-Z e60bc3e5bf72f574703087920d14128f
+Z 5c7cba3bc5ff7309961a0180cc01b405
diff --git a/manifest.uuid b/manifest.uuid
index 4a351d48d9..f101211b79 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f3112e67cdb27c1aec8d2cee3cb91ade061d093e13505894698e26336898b336
\ No newline at end of file
+1f300514e6cdd32c755dc1b05dfa125b10eda2eb55a1024b66f66dbf5cf96289
\ No newline at end of file
diff --git a/test/rowvalue.test b/test/rowvalue.test
index 8a5f6f3ef4..5805178301 100644
--- a/test/rowvalue.test
+++ b/test/rowvalue.test
@@ -444,100 +444,100 @@ do_execsql_test 19.1 {
   DROP TABLE IF EXISTS t1;
   CREATE TABLE t1(a INTEGER PRIMARY KEY,b);
   INSERT INTO t1(a,b) VALUES(1,11),(2,22),(3,33),(4,44);
-  SELECT * FROM t1 WHERE (a,b)>(0,0) ORDER BY +a;
+  SELECT * FROM t1 WHERE (a,b)>(0,0) ORDER BY a;
 } {1 11 2 22 3 33 4 44}
 do_execsql_test 19.2 {
-  SELECT * FROM t1 WHERE (a,b)>=(0,0) ORDER BY +a;
+  SELECT * FROM t1 WHERE (a,b)>=(0,0) ORDER BY a;
 } {1 11 2 22 3 33 4 44}
 do_execsql_test 19.3 {
-  SELECT * FROM t1 WHERE (a,b)<(5,0) ORDER BY +a;
-} {1 11 2 22 3 33 4 44}
+  SELECT * FROM t1 WHERE (a,b)<(5,0) ORDER BY a DESC;
+} {4 44 3 33 2 22 1 11}
 do_execsql_test 19.4 {
-  SELECT * FROM t1 WHERE (a,b)<=(5,0) ORDER BY +a;
-} {1 11 2 22 3 33 4 44}
+  SELECT * FROM t1 WHERE (a,b)<=(5,0) ORDER BY a DESC;
+} {4 44 3 33 2 22 1 11}
 do_execsql_test 19.5 {
-  SELECT * FROM t1 WHERE (a,b)>(3,0) ORDER BY +a;
+  SELECT * FROM t1 WHERE (a,b)>(3,0) ORDER BY a;
 } {3 33 4 44}
 do_execsql_test 19.6 {
-  SELECT * FROM t1 WHERE (a,b)>=(3,0) ORDER BY +a;
+  SELECT * FROM t1 WHERE (a,b)>=(3,0) ORDER BY a;
 } {3 33 4 44}
 do_execsql_test 19.7 {
-  SELECT * FROM t1 WHERE (a,b)<(3,0) ORDER BY +a;
-} {1 11 2 22}
+  SELECT * FROM t1 WHERE (a,b)<(3,0) ORDER BY a DESC;
+} {2 22 1 11}
 do_execsql_test 19.8 {
-  SELECT * FROM t1 WHERE (a,b)<=(3,0) ORDER BY +a;
-} {1 11 2 22}
+  SELECT * FROM t1 WHERE (a,b)<=(3,0) ORDER BY a DESC;
+} {2 22 1 11}
 do_execsql_test 19.9 {
-  SELECT * FROM t1 WHERE (a,b)>(3,32) ORDER BY +a;
+  SELECT * FROM t1 WHERE (a,b)>(3,32) ORDER BY a;
 } {3 33 4 44}
 do_execsql_test 19.10 {
-  SELECT * FROM t1 WHERE (a,b)>(3,33) ORDER BY +a;
+  SELECT * FROM t1 WHERE (a,b)>(3,33) ORDER BY a;
 } {4 44}
 do_execsql_test 19.11 {
-  SELECT * FROM t1 WHERE (a,b)>=(3,33) ORDER BY +a;
+  SELECT * FROM t1 WHERE (a,b)>=(3,33) ORDER BY a;
 } {3 33 4 44}
 do_execsql_test 19.12 {
-  SELECT * FROM t1 WHERE (a,b)>=(3,34) ORDER BY +a;
+  SELECT * FROM t1 WHERE (a,b)>=(3,34) ORDER BY a;
 } {4 44}
 do_execsql_test 19.13 {
-  SELECT * FROM t1 WHERE (a,b)<(3,34) ORDER BY +a;
-} {1 11 2 22 3 33}
+  SELECT * FROM t1 WHERE (a,b)<(3,34) ORDER BY a DESC;
+} {3 33 2 22 1 11}
 do_execsql_test 19.14 {
-  SELECT * FROM t1 WHERE (a,b)<(3,33) ORDER BY +a;
-} {1 11 2 22}
+  SELECT * FROM t1 WHERE (a,b)<(3,33) ORDER BY a DESC;
+} {2 22 1 11}
 do_execsql_test 19.15 {
-  SELECT * FROM t1 WHERE (a,b)<=(3,33) ORDER BY +a;
-} {1 11 2 22 3 33}
+  SELECT * FROM t1 WHERE (a,b)<=(3,33) ORDER BY a DESC;
+} {3 33 2 22 1 11}
 do_execsql_test 19.16 {
-  SELECT * FROM t1 WHERE (a,b)<=(3,32) ORDER BY +a;
-} {1 11 2 22}
+  SELECT * FROM t1 WHERE (a,b)<=(3,32) ORDER BY a DESC;
+} {2 22 1 11}
 do_execsql_test 19.21 {
-  SELECT * FROM t1 WHERE (0,0)<(a,b) ORDER BY +a;
+  SELECT * FROM t1 WHERE (0,0)<(a,b) ORDER BY a;
 } {1 11 2 22 3 33 4 44}
 do_execsql_test 19.22 {
-  SELECT * FROM t1 WHERE (0,0)<=(a,b) ORDER BY +a;
+  SELECT * FROM t1 WHERE (0,0)<=(a,b) ORDER BY a;
 } {1 11 2 22 3 33 4 44}
 do_execsql_test 19.23 {
-  SELECT * FROM t1 WHERE (5,0)>(a,b) ORDER BY +a;
-} {1 11 2 22 3 33 4 44}
+  SELECT * FROM t1 WHERE (5,0)>(a,b) ORDER BY a DESC;
+} {4 44 3 33 2 22 1 11}
 do_execsql_test 19.24 {
-  SELECT * FROM t1 WHERE (5,0)>=(a,b) ORDER BY +a;
-} {1 11 2 22 3 33 4 44}
+  SELECT * FROM t1 WHERE (5,0)>=(a,b) ORDER BY a DESC;
+} {4 44 3 33 2 22 1 11}
 do_execsql_test 19.25 {
-  SELECT * FROM t1 WHERE (3,0)<(a,b) ORDER BY +a;
+  SELECT * FROM t1 WHERE (3,0)<(a,b) ORDER BY a;
 } {3 33 4 44}
 do_execsql_test 19.26 {
-  SELECT * FROM t1 WHERE (3,0)<=(a,b) ORDER BY +a;
+  SELECT * FROM t1 WHERE (3,0)<=(a,b) ORDER BY a;
 } {3 33 4 44}
 do_execsql_test 19.27 {
-  SELECT * FROM t1 WHERE (3,0)>(a,b) ORDER BY +a;
-} {1 11 2 22}
+  SELECT * FROM t1 WHERE (3,0)>(a,b) ORDER BY a DESC;
+} {2 22 1 11}
 do_execsql_test 19.28 {
-  SELECT * FROM t1 WHERE (3,0)>=(a,b) ORDER BY +a;
-} {1 11 2 22}
+  SELECT * FROM t1 WHERE (3,0)>=(a,b) ORDER BY a DESC;
+} {2 22 1 11}
 do_execsql_test 19.29 {
-  SELECT * FROM t1 WHERE (3,32)<(a,b) ORDER BY +a;
+  SELECT * FROM t1 WHERE (3,32)<(a,b) ORDER BY a;
 } {3 33 4 44}
 do_execsql_test 19.30 {
-  SELECT * FROM t1 WHERE (3,33)<(a,b) ORDER BY +a;
+  SELECT * FROM t1 WHERE (3,33)<(a,b) ORDER BY a;
 } {4 44}
 do_execsql_test 19.31 {
-  SELECT * FROM t1 WHERE (3,33)<=(a,b) ORDER BY +a;
+  SELECT * FROM t1 WHERE (3,33)<=(a,b) ORDER BY a;
 } {3 33 4 44}
 do_execsql_test 19.32 {
-  SELECT * FROM t1 WHERE (3,34)<=(a,b) ORDER BY +a;
+  SELECT * FROM t1 WHERE (3,34)<=(a,b) ORDER BY a;
 } {4 44}
 do_execsql_test 19.33 {
-  SELECT * FROM t1 WHERE (3,34)>(a,b) ORDER BY +a;
-} {1 11 2 22 3 33}
+  SELECT * FROM t1 WHERE (3,34)>(a,b) ORDER BY a DESC;
+} {3 33 2 22 1 11}
 do_execsql_test 19.34 {
-  SELECT * FROM t1 WHERE (3,33)>(a,b) ORDER BY +a;
-} {1 11 2 22}
+  SELECT * FROM t1 WHERE (3,33)>(a,b) ORDER BY a DESC;
+} {2 22 1 11}
 do_execsql_test 19.35 {
-  SELECT * FROM t1 WHERE (3,33)>=(a,b) ORDER BY +a;
-} {1 11 2 22 3 33}
+  SELECT * FROM t1 WHERE (3,33)>=(a,b) ORDER BY a DESC;
+} {3 33 2 22 1 11}
 do_execsql_test 19.36 {
-  SELECT * FROM t1 WHERE (3,32)>=(a,b) ORDER BY +a;
-} {1 11 2 22}
+  SELECT * FROM t1 WHERE (3,32)>=(a,b) ORDER BY a DESC;
+} {2 22 1 11}
 
 finish_test

From e2d27e02590f1ab0fb10681e5e5425e59d81b593 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 14 Feb 2018 14:13:09 +0000
Subject: [PATCH 441/488] Try to optimize spellfix1 by storing a NULL in the k1
 column of %_vocab if it would otherwise have the same value as the word
 column.

FossilOrigin-Name: b76ec7cb5841f9ad02b342271e91c5f5a598006f69df7953a6213c706e01f7a2
---
 ext/misc/spellfix.c | 14 +++++++-------
 manifest            | 14 +++++++-------
 manifest.uuid       |  2 +-
 test/spellfix.test  |  2 +-
 4 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/ext/misc/spellfix.c b/ext/misc/spellfix.c
index 2d7ef1b02b..6e3a3bca16 100644
--- a/ext/misc/spellfix.c
+++ b/ext/misc/spellfix.c
@@ -2492,7 +2492,7 @@ static int spellfix1FilterForMatch(
   nPattern = (int)strlen(zPattern);
   if( zPattern[nPattern-1]=='*' ) nPattern--;
   zSql = sqlite3_mprintf(
-     "SELECT id, word, rank, k1"
+     "SELECT id, word, rank, coalesce(k1,word)"
      "  FROM \"%w\".\"%w_vocab\""
      " WHERE langid=%d AND k2>=?1 AND k2<?2",
      p->zDbName, p->zTableName, iLang
@@ -2826,17 +2826,17 @@ static int spellfix1Update(
       if( sqlite3_value_type(argv[1])==SQLITE_NULL ){
         spellfix1DbExec(&rc, db,
                "INSERT INTO \"%w\".\"%w_vocab\"(rank,langid,word,k1,k2) "
-               "VALUES(%d,%d,%Q,%Q,%Q)",
+               "VALUES(%d,%d,%Q,nullif(%Q,%Q),%Q)",
                p->zDbName, p->zTableName,
-               iRank, iLang, zWord, zK1, zK2
+               iRank, iLang, zWord, zK1, zWord, zK2
         );
       }else{
         newRowid = sqlite3_value_int64(argv[1]);
         spellfix1DbExec(&rc, db,
             "INSERT OR %s INTO \"%w\".\"%w_vocab\"(id,rank,langid,word,k1,k2) "
-            "VALUES(%lld,%d,%d,%Q,%Q,%Q)",
+            "VALUES(%lld,%d,%d,%Q,nullif(%Q,%Q),%Q)",
             zConflict, p->zDbName, p->zTableName,
-            newRowid, iRank, iLang, zWord, zK1, zK2
+            newRowid, iRank, iLang, zWord, zK1, zWord, zK2
         );
       }
       *pRowid = sqlite3_last_insert_rowid(db);
@@ -2845,9 +2845,9 @@ static int spellfix1Update(
       newRowid = *pRowid = sqlite3_value_int64(argv[1]);
       spellfix1DbExec(&rc, db,
              "UPDATE OR %s \"%w\".\"%w_vocab\" SET id=%lld, rank=%d, langid=%d,"
-             " word=%Q, k1=%Q, k2=%Q WHERE id=%lld",
+             " word=%Q, k1=nullif(%Q,%Q), k2=%Q WHERE id=%lld",
              zConflict, p->zDbName, p->zTableName, newRowid, iRank, iLang,
-             zWord, zK1, zK2, rowid
+             zWord, zK1, zWord, zK2, rowid
       );
     }
     sqlite3_free(zK1);
diff --git a/manifest b/manifest
index 7a723a9a49..8bac7cb68b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improved\stest\scases\sfor\sthe\s[f484b65f3d6230593c34f]\sbug.
-D 2018-02-13T19:13:05.932
+C Try\sto\soptimize\sspellfix1\sby\sstoring\sa\sNULL\sin\sthe\sk1\scolumn\sof\s%_vocab\nif\sit\swould\sotherwise\shave\sthe\ssame\svalue\sas\sthe\sword\scolumn.
+D 2018-02-14T14:13:09.238
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -294,7 +294,7 @@ F ext/misc/series.c f3c0dba5c5c749ce1782b53076108f87cf0b71041eb6023f727a9c50681d
 F ext/misc/sha1.c 0b9e9b855354910d3ca467bf39099d570e73db56
 F ext/misc/shathree.c 9e960ba50483214c6a7a4b1517f8d8cef799e9db381195178c3fd3ad207e10c0
 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52
-F ext/misc/spellfix.c 705ce1ed701b38f43c8ee7dd848877ff51c09eda322e39ccac6873adb4b74735
+F ext/misc/spellfix.c efe188bdd84d283e1c82e4bcf7058ffc1856176d9ba2ba14fd5efc4aa2d8661f
 F ext/misc/sqlar.c 57d5bc45cd5492208e451f697404be88f8612527d64c9d42f96b325b64983d74
 F ext/misc/stmt.c 6f16443abb3551e3f5813bb13ba19a30e7032830015b0f92fe0c0453045c0a11
 F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512
@@ -1262,7 +1262,7 @@ F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715
 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa
 F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b
 F test/speedtest1.c a5faf4cbe5769eee4b721b3875cb3f12520a9b99d9026b1063b47c39603375b8
-F test/spellfix.test f9c1f431e2c096c8775fec032952320c0e4700db
+F test/spellfix.test 951a6405d49d1a23d6b78027d3877b4a33eeb8221dcab5704b499755bb4f552e
 F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3
 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33
 F test/sqldiff1.test 28cd737cf1b0078b1ec1bbf425e674c47785835e
@@ -1705,7 +1705,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P f3112e67cdb27c1aec8d2cee3cb91ade061d093e13505894698e26336898b336
-R 5562ebb89ed1db5735eb0527fd19eb1a
+P 1f300514e6cdd32c755dc1b05dfa125b10eda2eb55a1024b66f66dbf5cf96289
+R 255e72ba75e81cf4ca1d4ef198b4e2a9
 U drh
-Z 5c7cba3bc5ff7309961a0180cc01b405
+Z 21f5c49e8ec82780f55a5436de6c983b
diff --git a/manifest.uuid b/manifest.uuid
index f101211b79..91ef23e4ef 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-1f300514e6cdd32c755dc1b05dfa125b10eda2eb55a1024b66f66dbf5cf96289
\ No newline at end of file
+b76ec7cb5841f9ad02b342271e91c5f5a598006f69df7953a6213c706e01f7a2
\ No newline at end of file
diff --git a/test/spellfix.test b/test/spellfix.test
index 8128bb59d2..68bcfd5adb 100644
--- a/test/spellfix.test
+++ b/test/spellfix.test
@@ -279,7 +279,7 @@ ifcapable trace {
   do_tracesql_test 6.2.3 {
     SELECT word, distance FROM t3 WHERE rowid = 10 AND word MATCH 'kiiner';
   } {keener 300
-    {SELECT id, word, rank, k1  FROM "main"."t3_vocab" WHERE langid=0 AND k2>=?1 AND k2<?2}
+    {SELECT id, word, rank, coalesce(k1,word)  FROM "main"."t3_vocab" WHERE langid=0 AND k2>=?1 AND k2<?2}
   }
 }
 

From 4e6e5cef6ce48fd188a59c8344ba20cd67e73d24 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 14 Feb 2018 15:30:48 +0000
Subject: [PATCH 442/488] New test cases for the editdist3() function in the
 spellfix extension.

FossilOrigin-Name: 2d3f1f22d53034576a0ec8d9c360fb19efbb05b2e95ad1b09d120fb1b7da33fd
---
 manifest            |  11 +-
 manifest.uuid       |   2 +-
 test/spellfix4.test | 346 ++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 353 insertions(+), 6 deletions(-)
 create mode 100644 test/spellfix4.test

diff --git a/manifest b/manifest
index 8bac7cb68b..0b9be9c6dd 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Try\sto\soptimize\sspellfix1\sby\sstoring\sa\sNULL\sin\sthe\sk1\scolumn\sof\s%_vocab\nif\sit\swould\sotherwise\shave\sthe\ssame\svalue\sas\sthe\sword\scolumn.
-D 2018-02-14T14:13:09.238
+C New\stest\scases\sfor\sthe\seditdist3()\sfunction\sin\sthe\sspellfix\sextension.
+D 2018-02-14T15:30:48.110
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -1265,6 +1265,7 @@ F test/speedtest1.c a5faf4cbe5769eee4b721b3875cb3f12520a9b99d9026b1063b47c396033
 F test/spellfix.test 951a6405d49d1a23d6b78027d3877b4a33eeb8221dcab5704b499755bb4f552e
 F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3
 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33
+F test/spellfix4.test eaae7a334516e09896a684fb552d338fa2844c65db9817d900701fe7598327e6
 F test/sqldiff1.test 28cd737cf1b0078b1ec1bbf425e674c47785835e
 F test/sqllimits1.test a74ee2a3740b9f9c2437c246d8fb77354862a142
 F test/sqllog.test 6af6cb0b09f4e44e1917e06ce85be7670302517a
@@ -1705,7 +1706,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 1f300514e6cdd32c755dc1b05dfa125b10eda2eb55a1024b66f66dbf5cf96289
-R 255e72ba75e81cf4ca1d4ef198b4e2a9
+P b76ec7cb5841f9ad02b342271e91c5f5a598006f69df7953a6213c706e01f7a2
+R 955df0b8bc6bab58cd406ae281f344ae
 U drh
-Z 21f5c49e8ec82780f55a5436de6c983b
+Z d19bf6cab5bd800e73ca3fefa0386288
diff --git a/manifest.uuid b/manifest.uuid
index 91ef23e4ef..ebe643ee66 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b76ec7cb5841f9ad02b342271e91c5f5a598006f69df7953a6213c706e01f7a2
\ No newline at end of file
+2d3f1f22d53034576a0ec8d9c360fb19efbb05b2e95ad1b09d120fb1b7da33fd
\ No newline at end of file
diff --git a/test/spellfix4.test b/test/spellfix4.test
new file mode 100644
index 0000000000..7054bb98ef
--- /dev/null
+++ b/test/spellfix4.test
@@ -0,0 +1,346 @@
+# 2018-02-14
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Test cases for the editdist3() function in the spellfix extension.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix spellfix4
+
+ifcapable !vtab { finish_test ; return }
+
+load_static_extension db spellfix
+
+do_execsql_test 100 {
+  CREATE TABLE cost1(iLang, cFrom, cTo, iCost);
+  INSERT INTO cost1 VALUES
+    (0, '', '?',  97),
+    (0, '?', '',  98),
+    (0, '?', '?', 99),
+    (0, 'm', 'n', 50),
+    (0, 'n', 'm', 50)
+  ;
+  SELECT editdist3('cost1');
+  SELECT editdist3('anchor','amchor');
+} {{} 50}
+do_execsql_test 110 {
+  SELECT editdist3('anchor','anchoxr');
+} {97}
+do_execsql_test 111 {
+  SELECT editdist3('anchor','xanchor');
+} {97}
+do_execsql_test 112 {
+  SELECT editdist3('anchor','anchorx');
+} {97}
+do_execsql_test 120 {
+  SELECT editdist3('anchor','anchr');
+} {98}
+do_execsql_test 121 {
+  SELECT editdist3('anchor','ancho');
+} {98}
+do_execsql_test 122 {
+  SELECT editdist3('anchor','nchor');
+} {98}
+do_execsql_test 130 {
+  SELECT editdist3('anchor','anchur');
+} {99}
+do_execsql_test 131 {
+  SELECT editdist3('anchor','onchor');
+} {99}
+do_execsql_test 132 {
+  SELECT editdist3('anchor','anchot');
+} {99}
+do_execsql_test 140 {
+  SELECT editdist3('anchor','omchor');
+} {149}
+
+do_execsql_test 200 {
+  INSERT INTO cost1 VALUES
+    (0, 'a', 'ä', 5),
+    (0, 'ss', 'ß', 8)
+  ;
+  SELECT editdist3('cost1');
+  SELECT editdist3('strasse','straße');
+  SELECT editdist3('straße','strasse');
+} {{} 8 196}
+do_execsql_test 210 {
+  SELECT editdist3('baume','bäume');
+} {5}
+do_execsql_test 220 {
+  SELECT editdist3('baum','bäume');
+} {102}
+do_execsql_test 230 {
+  INSERT INTO cost1 VALUES
+    (0, 'ä', 'a', 5),
+    (0, 'ß', 'ss', 8)
+  ;
+  SELECT editdist3('cost1');
+  SELECT editdist3('strasse','straße');
+  SELECT editdist3('straße','strasse');
+} {{} 8 8}
+
+do_execsql_test 300 {
+  DELETE FROM cost1;
+  INSERT INTO cost1 VALUES
+    (0, '', '?',  97),
+    (0, '?', '',  98),
+    (0, '?', '?', 99),
+    (0, 'a', 'e', 50),
+    (0, 'a', 'i', 70),
+    (0, 'a', 'o', 75),
+    (0, 'a', 'u', 85),
+    (0, 'e', 'a', 50),
+    (0, 'e', 'i', 50),
+    (0, 'e', 'o', 75),
+    (0, 'e', 'u', 85),
+    (0, 'i', 'a', 70),
+    (0, 'i', 'e', 50),
+    (0, 'i', 'o', 75),
+    (0, 'i', 'u', 85),
+    (0, 'o', 'a', 75),
+    (0, 'o', 'e', 75),
+    (0, 'o', 'i', 75),
+    (0, 'o', 'u', 40),
+    (0, 'u', 'a', 85),
+    (0, 'u', 'e', 85),
+    (0, 'u', 'i', 85),
+    (0, 'u', 'o', 40),
+    (0, 'm', 'n', 45),
+    (0, 'n', 'm', 45)
+  ;
+  CREATE TABLE words(x TEXT);
+  INSERT INTO words VALUES
+   ('abraham'),
+   ('action'),
+   ('africa'),
+   ('aladdin'),
+   ('alert'),
+   ('alien'),
+   ('amazon'),
+   ('analog'),
+   ('animal'),
+   ('apollo'),
+   ('archive'),
+   ('arnold'),
+   ('aspirin'),
+   ('august'),
+   ('average'),
+   ('bahama'),
+   ('bambino'),
+   ('barcode'),
+   ('bazooka'),
+   ('belgium'),
+   ('between'),
+   ('biology'),
+   ('blonde'),
+   ('border'),
+   ('brave'),
+   ('british'),
+   ('bucket'),
+   ('button'),
+   ('caesar'),
+   ('camilla'),
+   ('cannon'),
+   ('caramel'),
+   ('carpet'),
+   ('catalog'),
+   ('century'),
+   ('chaos'),
+   ('chef'),
+   ('china'),
+   ('circus'),
+   ('classic'),
+   ('clinic'),
+   ('coconut'),
+   ('combine'),
+   ('complex'),
+   ('congo'),
+   ('convert'),
+   ('cosmos'),
+   ('crack'),
+   ('crown'),
+   ('cyclone'),
+   ('deal'),
+   ('delete'),
+   ('denver'),
+   ('detail'),
+   ('diana'),
+   ('direct'),
+   ('dolby'),
+   ('double'),
+   ('dublin'),
+   ('echo'),
+   ('edition'),
+   ('electra'),
+   ('emotion'),
+   ('enjoy'),
+   ('escape'),
+   ('everest'),
+   ('exile'),
+   ('express'),
+   ('family'),
+   ('ferrari'),
+   ('filter'),
+   ('fish'),
+   ('florida'),
+   ('ford'),
+   ('forum'),
+   ('frank'),
+   ('frozen'),
+   ('gallery'),
+   ('garlic'),
+   ('geneva'),
+   ('gibson'),
+   ('gloria'),
+   ('gordon'),
+   ('gravity'),
+   ('ground'),
+   ('habitat'),
+   ('harlem'),
+   ('hazard'),
+   ('herbert'),
+   ('hobby'),
+   ('house'),
+   ('icon'),
+   ('immune'),
+   ('india'),
+   ('inside'),
+   ('isotope'),
+   ('jamaica'),
+   ('jazz'),
+   ('joker'),
+   ('juliet'),
+   ('jupiter'),
+   ('kevin'),
+   ('korea'),
+   ('latin'),
+   ('legal'),
+   ('lexicon'),
+   ('limbo'),
+   ('lithium'),
+   ('logo'),
+   ('lucas'),
+   ('madrid'),
+   ('major'),
+   ('manual'),
+   ('mars'),
+   ('maximum'),
+   ('medical'),
+   ('mental'),
+   ('meter'),
+   ('miguel'),
+   ('mimosa'),
+   ('miranda'),
+   ('modern'),
+   ('money'),
+   ('morgan'),
+   ('motor'),
+   ('mystic'),
+   ('nebula'),
+   ('network'),
+   ('nice'),
+   ('nitro'),
+   ('norway'),
+   ('nurse'),
+   ('octavia'),
+   ('olympic'),
+   ('opus'),
+   ('orient'),
+   ('othello'),
+   ('pacific'),
+   ('panama'),
+   ('paper'),
+   ('parking'),
+   ('pasta'),
+   ('paul'),
+   ('people'),
+   ('permit'),
+   ('phrase'),
+   ('pilgrim'),
+   ('planet'),
+   ('pocket'),
+   ('police'),
+   ('popular'),
+   ('prefer'),
+   ('presto'),
+   ('private'),
+   ('project'),
+   ('proxy'),
+   ('python'),
+   ('quota'),
+   ('rainbow'),
+   ('raymond'),
+   ('region'),
+   ('report'),
+   ('reward'),
+   ('risk'),
+   ('robot'),
+   ('rose'),
+   ('russian'),
+   ('sailor'),
+   ('salt'),
+   ('saturn'),
+   ('scorpio'),
+   ('second'),
+   ('seminar'),
+   ('shadow'),
+   ('shave'),
+   ('shock'),
+   ('silence'),
+   ('sinatra'),
+   ('sleep'),
+   ('social'),
+   ('sonata'),
+   ('spain'),
+   ('sphere'),
+   ('spray'),
+   ('state'),
+   ('stone'),
+   ('strong'),
+   ('sugar'),
+   ('supreme'),
+   ('swing'),
+   ('talent'),
+   ('telecom'),
+   ('thermos'),
+   ('tina'),
+   ('tommy'),
+   ('torso'),
+   ('trade'),
+   ('trick'),
+   ('tropic'),
+   ('turtle'),
+   ('uniform'),
+   ('user'),
+   ('vega'),
+   ('vertigo'),
+   ('village'),
+   ('visible'),
+   ('vocal'),
+   ('voyage'),
+   ('weekend'),
+   ('winter'),
+   ('year'),
+   ('zipper')
+  ;
+  SELECT editdist3('cost1');
+} {{}}
+do_execsql_test 310 {
+  SELECT editdist3(a.x,b.x), a.x, b.x
+    FROM words a, words b
+   WHERE a.x<b.x
+   ORDER BY 1, 2
+   LIMIT 20
+} {139 bucket pocket 149 manual mental 150 meter motor 169 crack trick 173 sinatra sonata 174 edition emotion 174 major motor 174 risk rose 174 state stone 194 deal detail 196 alert talent 196 analog catalog 196 deal legal 196 ford forum 196 risk trick 196 stone strong 197 china tina 197 congo logo 197 diana tina 197 florida gloria}
+
+
+finish_test

From 0fae06fcb6bb4d0e0f75d83759fcc0699a533e9a Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 14 Feb 2018 20:25:33 +0000
Subject: [PATCH 443/488] Disable assert() in the spellfix extension if not
 compiled with SQLITE_DEBUG.

FossilOrigin-Name: 3c53ee0fdea4cbf2590e2b289b021b0ef8b7ead2945db1bdfc767432d9447acb
---
 ext/misc/spellfix.c |  6 ++++++
 manifest            | 12 ++++++------
 manifest.uuid       |  2 +-
 3 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/ext/misc/spellfix.c b/ext/misc/spellfix.c
index 6e3a3bca16..1966d554e7 100644
--- a/ext/misc/spellfix.c
+++ b/ext/misc/spellfix.c
@@ -18,6 +18,12 @@
 SQLITE_EXTENSION_INIT1
 
 #ifndef SQLITE_AMALGAMATION
+# if !defined(NDEBUG) && !defined(SQLITE_DEBUG)
+#  define NDEBUG 1
+# endif
+# if defined(NDEBUG) && defined(SQLITE_DEBUG)
+#  undef NDEBUG
+# endif
 # include <string.h>
 # include <stdio.h>
 # include <stdlib.h>
diff --git a/manifest b/manifest
index 0b9be9c6dd..09759a8b0f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C New\stest\scases\sfor\sthe\seditdist3()\sfunction\sin\sthe\sspellfix\sextension.
-D 2018-02-14T15:30:48.110
+C Disable\sassert()\sin\sthe\sspellfix\sextension\sif\snot\scompiled\swith\sSQLITE_DEBUG.
+D 2018-02-14T20:25:33.844
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -294,7 +294,7 @@ F ext/misc/series.c f3c0dba5c5c749ce1782b53076108f87cf0b71041eb6023f727a9c50681d
 F ext/misc/sha1.c 0b9e9b855354910d3ca467bf39099d570e73db56
 F ext/misc/shathree.c 9e960ba50483214c6a7a4b1517f8d8cef799e9db381195178c3fd3ad207e10c0
 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52
-F ext/misc/spellfix.c efe188bdd84d283e1c82e4bcf7058ffc1856176d9ba2ba14fd5efc4aa2d8661f
+F ext/misc/spellfix.c bb73be37491fd5007efab8dfd8ac416315d47f8aba99179b8adb4255e85c7ff5
 F ext/misc/sqlar.c 57d5bc45cd5492208e451f697404be88f8612527d64c9d42f96b325b64983d74
 F ext/misc/stmt.c 6f16443abb3551e3f5813bb13ba19a30e7032830015b0f92fe0c0453045c0a11
 F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512
@@ -1706,7 +1706,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b76ec7cb5841f9ad02b342271e91c5f5a598006f69df7953a6213c706e01f7a2
-R 955df0b8bc6bab58cd406ae281f344ae
+P 2d3f1f22d53034576a0ec8d9c360fb19efbb05b2e95ad1b09d120fb1b7da33fd
+R 4aff58b3b0f19d8824c9d78629ce8a88
 U drh
-Z d19bf6cab5bd800e73ca3fefa0386288
+Z bdffc86a194b9f42419156834dd68b1f
diff --git a/manifest.uuid b/manifest.uuid
index ebe643ee66..129d553977 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2d3f1f22d53034576a0ec8d9c360fb19efbb05b2e95ad1b09d120fb1b7da33fd
\ No newline at end of file
+3c53ee0fdea4cbf2590e2b289b021b0ef8b7ead2945db1bdfc767432d9447acb
\ No newline at end of file

From d9274a8ae35a5f195a5d4a3629ea74978e69cf13 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 14 Feb 2018 20:58:36 +0000
Subject: [PATCH 444/488] Performance optimizations to the editdist3() function
 in the spellfix extension.

FossilOrigin-Name: 70d304dcbac4c3fd5e3b96108bffea2ce6c0db19c847397d5c5e268bb90a981d
---
 ext/misc/spellfix.c | 14 ++++++--------
 manifest            | 12 ++++++------
 manifest.uuid       |  2 +-
 3 files changed, 13 insertions(+), 15 deletions(-)

diff --git a/ext/misc/spellfix.c b/ext/misc/spellfix.c
index 1966d554e7..28374b8cc2 100644
--- a/ext/misc/spellfix.c
+++ b/ext/misc/spellfix.c
@@ -691,6 +691,7 @@ static int editDist3ConfigLoad(
     assert( zTo!=0 || nTo==0 );
     if( nFrom>100 || nTo>100 ) continue;
     if( iCost<0 ) continue;
+    if( iCost>10000 ) continue;   /* Costs above 10K are considered infinite */
     if( pLang==0 || iLang!=iLangPrev ){
       EditDist3Lang *pNew;
       pNew = sqlite3_realloc64(p->a, (p->nLang+1)*sizeof(p->a[0]));
@@ -782,7 +783,7 @@ static int matchFromTo(
 ){
   int b1 = pStr->a[n1].nByte;
   if( b1>n2 ) return 0;
-  if( memcmp(pStr->z+n1, z2, b1)!=0 ) return 0;
+  if( strncmp(pStr->z+n1, z2, b1)!=0 ) return 0;
   return 1;
 }
 
@@ -864,9 +865,6 @@ static EditDist3FromString *editDist3FromStringNew(
 /*
 ** Update entry m[i] such that it is the minimum of its current value
 ** and m[j]+iCost.
-**
-** If the iCost is 1,000,000 or greater, then consider the cost to be
-** infinite and skip the update.
 */
 static void updateCost(
   unsigned int *m,
@@ -874,11 +872,11 @@ static void updateCost(
   int j,
   int iCost
 ){
+  unsigned int b;
   assert( iCost>=0 );
-  if( iCost<10000 ){
-    unsigned int b = m[j] + iCost;
-    if( b<m[i] ) m[i] = b;
-  }
+  assert( iCost<10000 );
+  b = m[j] + iCost;
+  if( b<m[i] ) m[i] = b;
 }
 
 /*
diff --git a/manifest b/manifest
index 09759a8b0f..13a742a2be 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Disable\sassert()\sin\sthe\sspellfix\sextension\sif\snot\scompiled\swith\sSQLITE_DEBUG.
-D 2018-02-14T20:25:33.844
+C Performance\soptimizations\sto\sthe\seditdist3()\sfunction\sin\sthe\sspellfix\nextension.
+D 2018-02-14T20:58:36.311
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -294,7 +294,7 @@ F ext/misc/series.c f3c0dba5c5c749ce1782b53076108f87cf0b71041eb6023f727a9c50681d
 F ext/misc/sha1.c 0b9e9b855354910d3ca467bf39099d570e73db56
 F ext/misc/shathree.c 9e960ba50483214c6a7a4b1517f8d8cef799e9db381195178c3fd3ad207e10c0
 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52
-F ext/misc/spellfix.c bb73be37491fd5007efab8dfd8ac416315d47f8aba99179b8adb4255e85c7ff5
+F ext/misc/spellfix.c fcf23b0746bfefbec46a54cfb2cd0720c442a6e382247b4049748f882ad26b94
 F ext/misc/sqlar.c 57d5bc45cd5492208e451f697404be88f8612527d64c9d42f96b325b64983d74
 F ext/misc/stmt.c 6f16443abb3551e3f5813bb13ba19a30e7032830015b0f92fe0c0453045c0a11
 F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512
@@ -1706,7 +1706,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2d3f1f22d53034576a0ec8d9c360fb19efbb05b2e95ad1b09d120fb1b7da33fd
-R 4aff58b3b0f19d8824c9d78629ce8a88
+P 3c53ee0fdea4cbf2590e2b289b021b0ef8b7ead2945db1bdfc767432d9447acb
+R 52a6749d61b6207535a693308aa98655
 U drh
-Z bdffc86a194b9f42419156834dd68b1f
+Z c6d9011f82c1274dfef70231bb03befb
diff --git a/manifest.uuid b/manifest.uuid
index 129d553977..8a40d27ca4 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-3c53ee0fdea4cbf2590e2b289b021b0ef8b7ead2945db1bdfc767432d9447acb
\ No newline at end of file
+70d304dcbac4c3fd5e3b96108bffea2ce6c0db19c847397d5c5e268bb90a981d
\ No newline at end of file

From ee269a60c9a4749d1c54913854f5a671aaf22e04 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 14 Feb 2018 23:27:43 +0000
Subject: [PATCH 445/488] Add the --readonly option to the ".open" command in
 the CLI.

FossilOrigin-Name: 06870bb15656b50b0e14d4364bb21afac76500e313ecf67aaef3688d603fd076
---
 manifest       | 12 ++++++------
 manifest.uuid  |  2 +-
 src/shell.c.in | 11 +++++++++++
 3 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index 13a742a2be..a6b76d1492 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Performance\soptimizations\sto\sthe\seditdist3()\sfunction\sin\sthe\sspellfix\nextension.
-D 2018-02-14T20:58:36.311
+C Add\sthe\s--readonly\soption\sto\sthe\s".open"\scommand\sin\sthe\sCLI.
+D 2018-02-14T23:27:43.700
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -489,7 +489,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c f02352ac5cbb6fad3804add825743b924cfb6c742ba2e8666d726828a9985d73
-F src/shell.c.in 0b2c8d55e3703723a22610d75ccd4037962c3bcb5ad5570efd781e8c521811c4
+F src/shell.c.in 6fdccd0a2879dab2e0fc2ffedccddebc1f84bd88a806d28c2774ec5e4feca479
 F src/sqlite.h.in 51f9acf52c80113d793ddd38b3940ad6895d97b4752503b19291fb8fcbf54c5e
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d
@@ -1706,7 +1706,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3c53ee0fdea4cbf2590e2b289b021b0ef8b7ead2945db1bdfc767432d9447acb
-R 52a6749d61b6207535a693308aa98655
+P 70d304dcbac4c3fd5e3b96108bffea2ce6c0db19c847397d5c5e268bb90a981d
+R 2fdde9b3aa61b61468b1828a4e04c2be
 U drh
-Z c6d9011f82c1274dfef70231bb03befb
+Z 881e2b520c52d585df5898197d96b448
diff --git a/manifest.uuid b/manifest.uuid
index 8a40d27ca4..10c7702d81 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-70d304dcbac4c3fd5e3b96108bffea2ce6c0db19c847397d5c5e268bb90a981d
\ No newline at end of file
+06870bb15656b50b0e14d4364bb21afac76500e313ecf67aaef3688d603fd076
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 3e3f9ac8f9..1ef414f4c2 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -1069,6 +1069,7 @@ struct ShellState {
 #define SHELL_OPEN_NORMAL     1      /* Normal database file */
 #define SHELL_OPEN_APPENDVFS  2      /* Use appendvfs */
 #define SHELL_OPEN_ZIPFILE    3      /* Use the zipfile virtual table */
+#define SHELL_OPEN_READONLY   4      /* Open a normal database read-only */
 
 /*
 ** These are the allowed shellFlgs values
@@ -3258,6 +3259,7 @@ static char zHelp[] =
   "                         on the output.\n"
   ".open ?OPTIONS? ?FILE? Close existing database and reopen FILE\n"
   "                         The --new option starts with an empty file\n"
+  "                         Other options: --readonly --append --zip\n"
   ".output ?FILE?         Send output to FILE or stdout\n"
   ".print STRING...       Print literal STRING\n"
   ".prompt MAIN CONTINUE  Replace the standard prompts\n"
@@ -3454,6 +3456,10 @@ static void open_db(ShellState *p, int keepAlive){
         sqlite3_open(":memory:", &p->db);
         break;
       }
+      case SHELL_OPEN_READONLY: {
+        sqlite3_open_v2(p->zDbFilename, &p->db, SQLITE_OPEN_READONLY, 0);
+        break;
+      }
       case SHELL_OPEN_UNSPEC:
       case SHELL_OPEN_NORMAL: {
         sqlite3_open(p->zDbFilename, &p->db);
@@ -6307,6 +6313,8 @@ static int do_meta_command(char *zLine, ShellState *p){
 #endif
       }else if( optionMatch(z, "append") ){
         p->openMode = SHELL_OPEN_APPENDVFS;
+      }else if( optionMatch(z, "readonly") ){
+        p->openMode = SHELL_OPEN_READONLY;
       }else if( z[0]=='-' ){
         utf8_printf(stderr, "unknown option: %s\n", z);
         rc = 1;
@@ -7985,6 +7993,7 @@ static const char zOptions[] =
   "   -nullvalue TEXT      set text string for NULL values. Default ''\n"
   "   -pagecache SIZE N    use N slots of SZ bytes each for page cache memory\n"
   "   -quote               set output mode to 'quote'\n"
+  "   -readonly            open the database read-only\n"
   "   -separator SEP       set output column separator. Default: '|'\n"
   "   -stats               print memory stats before each finalize\n"
   "   -version             show SQLite version\n"
@@ -8234,6 +8243,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
 #endif
     }else if( strcmp(z,"-append")==0 ){
       data.openMode = SHELL_OPEN_APPENDVFS;
+    }else if( strcmp(z,"-readonly")==0 ){
+      data.openMode = SHELL_OPEN_READONLY;
     }
   }
   if( data.zDbFilename==0 ){

From 501ea056309a623303e5cfdbeda6c55d5fadfe0b Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 15 Feb 2018 01:03:37 +0000
Subject: [PATCH 446/488] Fix the --heap, --pagecache, --lookaside and other
 options to the command-line shell that invoke sqlite3_config() so that they
 work again.

FossilOrigin-Name: 00707f2f2f746a6421f3e2de995e68cc8adba7225a04db6b28db52944e7e988e
---
 manifest       | 12 ++++++------
 manifest.uuid  |  2 +-
 src/shell.c.in | 24 +++++++++++++++++++++---
 3 files changed, 28 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index a6b76d1492..98d58723e4 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\s--readonly\soption\sto\sthe\s".open"\scommand\sin\sthe\sCLI.
-D 2018-02-14T23:27:43.700
+C Fix\sthe\s--heap,\s--pagecache,\s--lookaside\sand\sother\soptions\sto\sthe\scommand-line\nshell\sthat\sinvoke\ssqlite3_config()\sso\sthat\sthey\swork\sagain.
+D 2018-02-15T01:03:37.596
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -489,7 +489,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c f02352ac5cbb6fad3804add825743b924cfb6c742ba2e8666d726828a9985d73
-F src/shell.c.in 6fdccd0a2879dab2e0fc2ffedccddebc1f84bd88a806d28c2774ec5e4feca479
+F src/shell.c.in c1b14bc0bc1fe73a97dfaad55ca09f5d6e44f6019977d94e7e3d1861616035e1
 F src/sqlite.h.in 51f9acf52c80113d793ddd38b3940ad6895d97b4752503b19291fb8fcbf54c5e
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d
@@ -1706,7 +1706,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 70d304dcbac4c3fd5e3b96108bffea2ce6c0db19c847397d5c5e268bb90a981d
-R 2fdde9b3aa61b61468b1828a4e04c2be
+P 06870bb15656b50b0e14d4364bb21afac76500e313ecf67aaef3688d603fd076
+R 4e71ff1498e1cda7e0d47e08c283419f
 U drh
-Z 881e2b520c52d585df5898197d96b448
+Z 76196b3bed6ed8589ed1fea48a14df1f
diff --git a/manifest.uuid b/manifest.uuid
index 10c7702d81..4756fcb048 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-06870bb15656b50b0e14d4364bb21afac76500e313ecf67aaef3688d603fd076
\ No newline at end of file
+00707f2f2f746a6421f3e2de995e68cc8adba7225a04db6b28db52944e7e988e
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 1ef414f4c2..c074b5ca7b 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -8103,21 +8103,39 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
   }
 #endif
   main_init(&data);
-  sqlite3_initialize();
+
+  /* On Windows, we must translate command-line arguments into UTF-8.
+  ** The SQLite memory allocator subsystem has to be enabled in order to
+  ** do this.  But we want to run an sqlite3_shutdown() afterwards so that
+  ** subsequent sqlite3_config() calls will work.  So copy all results into
+  ** memory that does not come from the SQLite memory allocator.
+  */
 #if !SQLITE_SHELL_IS_UTF8
-  argv = sqlite3_malloc64(sizeof(argv[0])*argc);
+  sqlite3_initialize();
+  argv = malloc(sizeof(argv[0])*argc);
   if( argv==0 ){
     raw_printf(stderr, "out of memory\n");
     exit(1);
   }
   for(i=0; i<argc; i++){
-    argv[i] = sqlite3_win32_unicode_to_utf8(wargv[i]);
+    char *z = sqlite3_win32_unicode_to_utf8(wargv[i]);
+    int n;
+    if( z==0 ){
+      raw_printf(stderr, "out of memory\n");
+      exit(1);
+    }
+    n = (int)strlen(z);
+    argv[i] = malloc( n+1 );
     if( argv[i]==0 ){
       raw_printf(stderr, "out of memory\n");
       exit(1);
     }
+    memcpy(argv[i], z, n+1);
+    sqlite3_free(z);
   }
+  sqlite3_shutdown();
 #endif
+
   assert( argc>=1 && argv && argv[0] );
   Argv0 = argv[0];
 

From 46e835a2c42f8322e27fe00d4d6bd4525e207712 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 15 Feb 2018 03:05:56 +0000
Subject: [PATCH 447/488] Reduce the number of calls to strncmp() required to
 run editDist3Core().

FossilOrigin-Name: afd6fbc01052ccfc9bd29fb8f934b291b8f56af44fcae870da7e1355fe95c29a
---
 ext/misc/spellfix.c |  3 +++
 manifest            | 12 ++++++------
 manifest.uuid       |  2 +-
 3 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/ext/misc/spellfix.c b/ext/misc/spellfix.c
index 28374b8cc2..8b528562a8 100644
--- a/ext/misc/spellfix.c
+++ b/ext/misc/spellfix.c
@@ -756,6 +756,7 @@ static int utf8Len(unsigned char c, int N){
 ** the given string.
 */
 static int matchTo(EditDist3Cost *p, const char *z, int n){
+  if( p->a[p->nFrom]!=z[0] ) return 0;
   if( p->nTo>n ) return 0;
   if( strncmp(p->a+p->nFrom, z, p->nTo)!=0 ) return 0;
   return 1;
@@ -767,6 +768,7 @@ static int matchTo(EditDist3Cost *p, const char *z, int n){
 */
 static int matchFrom(EditDist3Cost *p, const char *z, int n){
   assert( p->nFrom<=n );
+  if( p->a[0]!=z[0] ) return 0;
   if( strncmp(p->a, z, p->nFrom)!=0 ) return 0;
   return 1;
 }
@@ -783,6 +785,7 @@ static int matchFromTo(
 ){
   int b1 = pStr->a[n1].nByte;
   if( b1>n2 ) return 0;
+  if( pStr->z[n1]!=z2[0] ) return 0;
   if( strncmp(pStr->z+n1, z2, b1)!=0 ) return 0;
   return 1;
 }
diff --git a/manifest b/manifest
index 98d58723e4..411cad769d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sthe\s--heap,\s--pagecache,\s--lookaside\sand\sother\soptions\sto\sthe\scommand-line\nshell\sthat\sinvoke\ssqlite3_config()\sso\sthat\sthey\swork\sagain.
-D 2018-02-15T01:03:37.596
+C Reduce\sthe\snumber\sof\scalls\sto\sstrncmp()\srequired\sto\srun\seditDist3Core().
+D 2018-02-15T03:05:56.872
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -294,7 +294,7 @@ F ext/misc/series.c f3c0dba5c5c749ce1782b53076108f87cf0b71041eb6023f727a9c50681d
 F ext/misc/sha1.c 0b9e9b855354910d3ca467bf39099d570e73db56
 F ext/misc/shathree.c 9e960ba50483214c6a7a4b1517f8d8cef799e9db381195178c3fd3ad207e10c0
 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52
-F ext/misc/spellfix.c fcf23b0746bfefbec46a54cfb2cd0720c442a6e382247b4049748f882ad26b94
+F ext/misc/spellfix.c ad435d8d84a64a7287f67ba618d5fc4a775e73d8489b1c7ae85a339ed5d20c34
 F ext/misc/sqlar.c 57d5bc45cd5492208e451f697404be88f8612527d64c9d42f96b325b64983d74
 F ext/misc/stmt.c 6f16443abb3551e3f5813bb13ba19a30e7032830015b0f92fe0c0453045c0a11
 F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512
@@ -1706,7 +1706,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 06870bb15656b50b0e14d4364bb21afac76500e313ecf67aaef3688d603fd076
-R 4e71ff1498e1cda7e0d47e08c283419f
+P 00707f2f2f746a6421f3e2de995e68cc8adba7225a04db6b28db52944e7e988e
+R bf16a0c80a0f655a42b9a2dc09fe803f
 U drh
-Z 76196b3bed6ed8589ed1fea48a14df1f
+Z 2ab397c1e9ab0ece50ec14a24b630ce7
diff --git a/manifest.uuid b/manifest.uuid
index 4756fcb048..df37324255 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-00707f2f2f746a6421f3e2de995e68cc8adba7225a04db6b28db52944e7e988e
\ No newline at end of file
+afd6fbc01052ccfc9bd29fb8f934b291b8f56af44fcae870da7e1355fe95c29a
\ No newline at end of file

From f4bc6c43d7e45580f5ee4cc40586829890afa155 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 15 Feb 2018 03:56:33 +0000
Subject: [PATCH 448/488] Improve performance of editdist3() by keeping the
 costs in sorted order. Also add a new regression test to editdist3().

FossilOrigin-Name: dc734c5b61464dfd6bfa7963f2ecce32e405a0c2ba1ef6f453ec9389da080256
---
 ext/misc/spellfix.c | 82 ++++++++++++++++++++++++++++++++++++++++++++-
 manifest            | 14 ++++----
 manifest.uuid       |  2 +-
 test/spellfix4.test | 31 ++++++++++-------
 4 files changed, 108 insertions(+), 21 deletions(-)

diff --git a/ext/misc/spellfix.c b/ext/misc/spellfix.c
index 8b528562a8..8651bb7a9d 100644
--- a/ext/misc/spellfix.c
+++ b/ext/misc/spellfix.c
@@ -658,6 +658,79 @@ static void editDist3ConfigDelete(void *pIn){
   sqlite3_free(p);
 }
 
+/* Compare the FROM values of two EditDist3Cost objects, for sorting.
+** Return negative, zero, or positive if the A is less than, equal to,
+** or greater than B.
+*/
+static int editDist3CostCompare(EditDist3Cost *pA, EditDist3Cost *pB){
+  int n = pA->nFrom;
+  int rc;
+  if( n>pB->nFrom ) n = pB->nFrom;
+  rc = strncmp(pA->a, pB->a, n);
+  if( rc==0 ) rc = pA->nFrom - pB->nFrom;
+  return rc;
+}
+
+/*
+** Merge together two sorted lists of EditDist3Cost objects, in order
+** of increasing FROM.
+*/
+static EditDist3Cost *editDist3CostMerge(
+  EditDist3Cost *pA,
+  EditDist3Cost *pB
+){
+  EditDist3Cost *pHead = 0;
+  EditDist3Cost **ppTail = &pHead;
+  EditDist3Cost *p;
+  while( pA && pB ){
+    if( editDist3CostCompare(pA,pB)<=0 ){
+      p = pA;
+      pA = pA->pNext;
+    }else{
+      p = pB;
+      pB = pB->pNext;
+    }
+    *ppTail = p;
+    ppTail =  &p->pNext;
+  }
+  if( pA ){
+    *ppTail = pA;
+  }else{
+    *ppTail = pB;
+  }
+  return pHead;
+}
+
+/*
+** Sort a list of EditDist3Cost objects into order of increasing FROM
+*/
+static EditDist3Cost *editDist3CostSort(EditDist3Cost *pList){
+  EditDist3Cost *ap[60], *p;
+  int i;
+  int mx = 0;
+  ap[0] = 0;
+  ap[1] = 0;
+  while( pList ){
+    p = pList;
+    pList = p->pNext;
+    p->pNext = 0;
+    for(i=0; ap[i]; i++){
+      p = editDist3CostMerge(ap[i],p);
+      ap[i] = 0;
+    }
+    ap[i] = p;
+    if( i>mx ){
+      mx = i;
+      ap[i+1] = 0;
+    }
+  }
+  p = 0;
+  for(i=0; i<=mx; i++){
+    if( ap[i] ) p = editDist3CostMerge(p,ap[i]);
+  }
+  return p;
+}
+
 /*
 ** Load all edit-distance weights from a table.
 */
@@ -729,6 +802,12 @@ static int editDist3ConfigLoad(
   }
   rc2 = sqlite3_finalize(pStmt);
   if( rc==SQLITE_OK ) rc = rc2;
+  if( rc==SQLITE_OK ){
+    int iLang;
+    for(iLang=0; iLang<p->nLang; iLang++){
+      p->a[iLang].pCost = editDist3CostSort(p->a[iLang].pCost);
+    }
+  }
   return rc;
 }
 
@@ -943,8 +1022,9 @@ static int editDist3Core(
     a2[i2].nByte = utf8Len((unsigned char)z2[i2], n2-i2);
     for(p=pLang->pCost; p; p=p->pNext){
       EditDist3Cost **apNew;
-      if( p->nFrom>0 ) continue;
+      if( p->nFrom>0 ) break;
       if( i2+p->nTo>n2 ) continue;
+      if( p->a[0]>z2[i2] ) break;
       if( matchTo(p, z2+i2, n2-i2)==0 ) continue;
       a2[i2].nIns++;
       apNew = sqlite3_realloc64(a2[i2].apIns, sizeof(*apNew)*a2[i2].nIns);
diff --git a/manifest b/manifest
index 411cad769d..dc00c62eff 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Reduce\sthe\snumber\sof\scalls\sto\sstrncmp()\srequired\sto\srun\seditDist3Core().
-D 2018-02-15T03:05:56.872
+C Improve\sperformance\sof\seditdist3()\sby\skeeping\sthe\scosts\sin\ssorted\sorder.\nAlso\sadd\sa\snew\sregression\stest\sto\seditdist3().
+D 2018-02-15T03:56:33.574
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -294,7 +294,7 @@ F ext/misc/series.c f3c0dba5c5c749ce1782b53076108f87cf0b71041eb6023f727a9c50681d
 F ext/misc/sha1.c 0b9e9b855354910d3ca467bf39099d570e73db56
 F ext/misc/shathree.c 9e960ba50483214c6a7a4b1517f8d8cef799e9db381195178c3fd3ad207e10c0
 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52
-F ext/misc/spellfix.c ad435d8d84a64a7287f67ba618d5fc4a775e73d8489b1c7ae85a339ed5d20c34
+F ext/misc/spellfix.c b3a644285cb008f3c10ed4cf04e17adcbc7d62c3911c79d786dfc91c177534f0
 F ext/misc/sqlar.c 57d5bc45cd5492208e451f697404be88f8612527d64c9d42f96b325b64983d74
 F ext/misc/stmt.c 6f16443abb3551e3f5813bb13ba19a30e7032830015b0f92fe0c0453045c0a11
 F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512
@@ -1265,7 +1265,7 @@ F test/speedtest1.c a5faf4cbe5769eee4b721b3875cb3f12520a9b99d9026b1063b47c396033
 F test/spellfix.test 951a6405d49d1a23d6b78027d3877b4a33eeb8221dcab5704b499755bb4f552e
 F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3
 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33
-F test/spellfix4.test eaae7a334516e09896a684fb552d338fa2844c65db9817d900701fe7598327e6
+F test/spellfix4.test 51c7c26514ade169855c66bcf130bd5acfb4d7fd090cc624645ab275ae6a41fb
 F test/sqldiff1.test 28cd737cf1b0078b1ec1bbf425e674c47785835e
 F test/sqllimits1.test a74ee2a3740b9f9c2437c246d8fb77354862a142
 F test/sqllog.test 6af6cb0b09f4e44e1917e06ce85be7670302517a
@@ -1706,7 +1706,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 00707f2f2f746a6421f3e2de995e68cc8adba7225a04db6b28db52944e7e988e
-R bf16a0c80a0f655a42b9a2dc09fe803f
+P afd6fbc01052ccfc9bd29fb8f934b291b8f56af44fcae870da7e1355fe95c29a
+R 8c5288c6ebac589573d48d7520074806
 U drh
-Z 2ab397c1e9ab0ece50ec14a24b630ce7
+Z 65b4bdcfd9180aa4d82e9c5cf2ce0fa5
diff --git a/manifest.uuid b/manifest.uuid
index df37324255..a1afa09049 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-afd6fbc01052ccfc9bd29fb8f934b291b8f56af44fcae870da7e1355fe95c29a
\ No newline at end of file
+dc734c5b61464dfd6bfa7963f2ecce32e405a0c2ba1ef6f453ec9389da080256
\ No newline at end of file
diff --git a/test/spellfix4.test b/test/spellfix4.test
index 7054bb98ef..caf6d5139a 100644
--- a/test/spellfix4.test
+++ b/test/spellfix4.test
@@ -97,22 +97,22 @@ do_execsql_test 300 {
     (0, 'a', 'e', 50),
     (0, 'a', 'i', 70),
     (0, 'a', 'o', 75),
-    (0, 'a', 'u', 85),
+    (0, 'a', 'u', 81),
     (0, 'e', 'a', 50),
-    (0, 'e', 'i', 50),
-    (0, 'e', 'o', 75),
-    (0, 'e', 'u', 85),
+    (0, 'e', 'i', 52),
+    (0, 'e', 'o', 72),
+    (0, 'e', 'u', 82),
     (0, 'i', 'a', 70),
-    (0, 'i', 'e', 50),
+    (0, 'i', 'e', 52),
     (0, 'i', 'o', 75),
-    (0, 'i', 'u', 85),
+    (0, 'i', 'u', 83),
     (0, 'o', 'a', 75),
-    (0, 'o', 'e', 75),
+    (0, 'o', 'e', 72),
     (0, 'o', 'i', 75),
     (0, 'o', 'u', 40),
-    (0, 'u', 'a', 85),
-    (0, 'u', 'e', 85),
-    (0, 'u', 'i', 85),
+    (0, 'u', 'a', 81),
+    (0, 'u', 'e', 82),
+    (0, 'u', 'i', 83),
     (0, 'u', 'o', 40),
     (0, 'm', 'n', 45),
     (0, 'n', 'm', 45)
@@ -340,7 +340,14 @@ do_execsql_test 310 {
    WHERE a.x<b.x
    ORDER BY 1, 2
    LIMIT 20
-} {139 bucket pocket 149 manual mental 150 meter motor 169 crack trick 173 sinatra sonata 174 edition emotion 174 major motor 174 risk rose 174 state stone 194 deal detail 196 alert talent 196 analog catalog 196 deal legal 196 ford forum 196 risk trick 196 stone strong 197 china tina 197 congo logo 197 diana tina 197 florida gloria}
-
+} {139 bucket pocket 144 meter motor 149 manual mental 169 crack trick 173 sinatra sonata 174 edition emotion 174 major motor 174 risk rose 174 state stone 194 deal detail 196 alert talent 196 analog catalog 196 deal legal 196 ford forum 196 risk trick 196 stone strong 197 china tina 197 congo logo 197 diana tina 197 florida gloria}
+do_execsql_test 320 {
+  SELECT md5sum(ed||'/'||sx||'/'||sy||',') FROM (
+      SELECT editdist3(a.x,b.x) AS ed, a.x AS sx, b.x AS sy
+        FROM words a, words b
+       WHERE a.x<b.x
+       ORDER BY 1, 2
+  )
+} {69d0a31872203a775e19325ea98cd053}
 
 finish_test

From 713f34ace9b9e0771ba51053425aa89ea3f5714a Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 15 Feb 2018 21:00:37 +0000
Subject: [PATCH 449/488] Do not allow parameters or schema references inside
 of WITH clause of triggers and views.  This fixes a bug discovered by OSSFuzz
 and present since common-table-expressions were first added in 2014-02-03.

FossilOrigin-Name: b918d4b4e546d3903ff20efc3c8ca26dd9761cc8ed9ef7d2799b66ff9ae1ae51
---
 manifest        | 13 +++++++------
 manifest.uuid   |  2 +-
 src/attach.c    | 10 ++++++++++
 test/with4.test | 52 +++++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 70 insertions(+), 7 deletions(-)
 create mode 100644 test/with4.test

diff --git a/manifest b/manifest
index dc00c62eff..b3d50466f6 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improve\sperformance\sof\seditdist3()\sby\skeeping\sthe\scosts\sin\ssorted\sorder.\nAlso\sadd\sa\snew\sregression\stest\sto\seditdist3().
-D 2018-02-15T03:56:33.574
+C Do\snot\sallow\sparameters\sor\sschema\sreferences\sinside\sof\sWITH\sclause\sof\ntriggers\sand\sviews.\s\sThis\sfixes\sa\sbug\sdiscovered\sby\sOSSFuzz\sand\spresent\nsince\scommon-table-expressions\swere\sfirst\sadded\sin\s2014-02-03.
+D 2018-02-15T21:00:37.044
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -425,7 +425,7 @@ F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786
 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a
 F src/alter.c cf7a8af45cb0ace672f47a1b29ab24092a9e8cd8d945a9974e3b5d925f548594
 F src/analyze.c 6b42e36a5dcc2703a771f2411bd5e99524bd62c7ecde209bb88dfb04c72f046e
-F src/attach.c 84c477e856b24c2b9a0983b438a707c0cf4d616cee7a425401d418e58afec24c
+F src/attach.c e1c0d72dd0747da84aa1352f8729906b1d8d2a162ba7f6c475980be202b37c46
 F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
@@ -1594,6 +1594,7 @@ F test/win32nolock.test ac4f08811a562e45a5755e661f45ca85892bdbbc
 F test/with1.test ca08e291249a810a2ec9b72ceef5575e07d5925b360fcf6652ae6fe06ac4dced
 F test/with2.test e0030e2f0267a910d6c0e4f46f2dfe941c1cc0d4f659ba69b3597728e7e8f1ab
 F test/with3.test e71604a0e53cba82bc04c703987cb1d6751ec0b6
+F test/with4.test 257be66c0c67fee1defbbac0f685c3465e2cad037f21ce65f23f86084f198205
 F test/withM.test 693b61765f2b387b5e3e24a4536e2e82de15ff64
 F test/without_rowid1.test 06b7215130882d6a072233820dd364c874c4fd69221e8fc756ec471009192874
 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99
@@ -1706,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P afd6fbc01052ccfc9bd29fb8f934b291b8f56af44fcae870da7e1355fe95c29a
-R 8c5288c6ebac589573d48d7520074806
+P dc734c5b61464dfd6bfa7963f2ecce32e405a0c2ba1ef6f453ec9389da080256
+R 8b09aca00d46d50dd4b70270c44a2f05
 U drh
-Z 65b4bdcfd9180aa4d82e9c5cf2ce0fa5
+Z bc5ab0c5d5dc21d5926f6e1994fc1380
diff --git a/manifest.uuid b/manifest.uuid
index a1afa09049..636ea46456 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-dc734c5b61464dfd6bfa7963f2ecce32e405a0c2ba1ef6f453ec9389da080256
\ No newline at end of file
+b918d4b4e546d3903ff20efc3c8ca26dd9761cc8ed9ef7d2799b66ff9ae1ae51
\ No newline at end of file
diff --git a/src/attach.c b/src/attach.c
index fa38e84159..67fc4afa82 100644
--- a/src/attach.c
+++ b/src/attach.c
@@ -504,6 +504,16 @@ int sqlite3FixSelect(
     if( sqlite3FixExpr(pFix, pSelect->pLimit) ){
       return 1;
     }
+#if 1
+    if( pSelect->pWith ){
+      int i;
+      for(i=0; i<pSelect->pWith->nCte; i++){
+        if( sqlite3FixSelect(pFix, pSelect->pWith->a[i].pSelect) ){
+          return 1;
+        }
+      }
+    }
+#endif
     pSelect = pSelect->pPrior;
   }
   return 0;
diff --git a/test/with4.test b/test/with4.test
new file mode 100644
index 0000000000..b0eeba6d14
--- /dev/null
+++ b/test/with4.test
@@ -0,0 +1,52 @@
+# 2018-02-15
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the WITH clause in TRIGGERs and VIEWs.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix with4
+
+ifcapable {!cte} {
+  finish_test
+  return
+}
+
+do_execsql_test 100 {
+  ATTACH ':memory:' AS aux;
+  CREATE TABLE main.t1(a,b);
+  CREATE TABLE aux.t2(x,y);
+  INSERT INTO t1 VALUES(1,2);
+  INSERT INTO t2 VALUES(3,4);
+} {}
+do_catchsql_test 110 {
+  CREATE VIEW v1 AS SELECT * FROM t1, aux.t2;
+} {1 {view v1 cannot reference objects in database aux}}
+do_catchsql_test 120 {
+  CREATE VIEW v2 AS WITH v(m,n) AS (SELECT x,y FROM aux.t2) SELECT * FROM t1, v;
+} {1 {view v2 cannot reference objects in database aux}}
+do_catchsql_test 130 {
+  CREATE VIEW v2 AS WITH v(m,n) AS (SELECT 5,?2) SELECT * FROM t1, v;
+} {1 {parameters are not allowed in views}}
+
+do_catchsql_test 200 {
+  CREATE TRIGGER r1 AFTER INSERT ON t1 BEGIN
+     WITH v(m,n) AS (SELECT x,y FROM aux.t2) SELECT * FROM t1, v;
+  END;
+} {1 {trigger r1 cannot reference objects in database aux}}
+do_catchsql_test 210 {
+  CREATE TRIGGER r1 AFTER INSERT ON t1 BEGIN
+     WITH v(m,n) AS (SELECT 5,?2) SELECT * FROM t1, v;
+  END;
+} {1 {trigger cannot use variables}}
+
+finish_test

From c5515508081c94e96d3e9adb9ac8579e4dc9335b Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 17 Feb 2018 07:38:50 +0000
Subject: [PATCH 450/488] Fix a data race causing a tsan complaint with
 SQLITE_ENABLE_API_ARMOR builds on unix. The race condition is not dangerous
 in practice, it just upsets tsan.

FossilOrigin-Name: f53b8a573bfbb48780243d56ca8372165adb1b979731c43d46cd1f8eb7e593f3
---
 manifest         | 14 +++++++-------
 manifest.uuid    |  2 +-
 src/mutex_unix.c | 39 +++++++++++++++++++++++----------------
 3 files changed, 31 insertions(+), 24 deletions(-)

diff --git a/manifest b/manifest
index b3d50466f6..3d3100acc3 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Do\snot\sallow\sparameters\sor\sschema\sreferences\sinside\sof\sWITH\sclause\sof\ntriggers\sand\sviews.\s\sThis\sfixes\sa\sbug\sdiscovered\sby\sOSSFuzz\sand\spresent\nsince\scommon-table-expressions\swere\sfirst\sadded\sin\s2014-02-03.
-D 2018-02-15T21:00:37.044
+C Fix\sa\sdata\srace\scausing\sa\stsan\scomplaint\swith\sSQLITE_ENABLE_API_ARMOR\sbuilds\non\sunix.\sThe\srace\scondition\sis\snot\sdangerous\sin\spractice,\sit\sjust\supsets\stsan.
+D 2018-02-17T07:38:50.155
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -465,7 +465,7 @@ F src/msvc.h 4942752b6a253116baaa8de75256c51a459a5e81
 F src/mutex.c b021263554c8a3995e9d53193b8194b96d1ed28e06c3b532dd7f7d29cf0c7d53
 F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85
 F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4
-F src/mutex_unix.c 27bb6cc49485ee46711a6580ab7b3f1402211d23
+F src/mutex_unix.c aaf9ebc3f89df28483c52208497a99a02cc3650011422fc9d4c57e4392f7fe58
 F src/mutex_w32.c a898fa969823b100c0f5fdc57e54c9a1e419ab4d
 F src/notify.c 9711a7575036f0d3040ba61bc6e217f13a9888e7
 F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104
@@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P dc734c5b61464dfd6bfa7963f2ecce32e405a0c2ba1ef6f453ec9389da080256
-R 8b09aca00d46d50dd4b70270c44a2f05
-U drh
-Z bc5ab0c5d5dc21d5926f6e1994fc1380
+P b918d4b4e546d3903ff20efc3c8ca26dd9761cc8ed9ef7d2799b66ff9ae1ae51
+R f2c9c538fa5937452e735a3a9fd538f8
+U dan
+Z 36b8ef4562da3a84a51e82bfb896975f
diff --git a/manifest.uuid b/manifest.uuid
index 636ea46456..caa3d48cb7 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b918d4b4e546d3903ff20efc3c8ca26dd9761cc8ed9ef7d2799b66ff9ae1ae51
\ No newline at end of file
+f53b8a573bfbb48780243d56ca8372165adb1b979731c43d46cd1f8eb7e593f3
\ No newline at end of file
diff --git a/src/mutex_unix.c b/src/mutex_unix.c
index 55d08c8052..9282d28016 100644
--- a/src/mutex_unix.c
+++ b/src/mutex_unix.c
@@ -50,11 +50,12 @@ struct sqlite3_mutex {
 #endif
 };
 #if SQLITE_MUTEX_NREF
-#define SQLITE3_MUTEX_INITIALIZER {PTHREAD_MUTEX_INITIALIZER,0,0,(pthread_t)0,0}
+# define SQLITE3_MUTEX_INITIALIZER(id) \
+     {PTHREAD_MUTEX_INITIALIZER,id,0,(pthread_t)0,0}
 #elif defined(SQLITE_ENABLE_API_ARMOR)
-#define SQLITE3_MUTEX_INITIALIZER { PTHREAD_MUTEX_INITIALIZER, 0 }
+# define SQLITE3_MUTEX_INITIALIZER(id) { PTHREAD_MUTEX_INITIALIZER, id }
 #else
-#define SQLITE3_MUTEX_INITIALIZER { PTHREAD_MUTEX_INITIALIZER }
+#define SQLITE3_MUTEX_INITIALIZER(id) { PTHREAD_MUTEX_INITIALIZER }
 #endif
 
 /*
@@ -151,18 +152,18 @@ static int pthreadMutexEnd(void){ return SQLITE_OK; }
 */
 static sqlite3_mutex *pthreadMutexAlloc(int iType){
   static sqlite3_mutex staticMutexes[] = {
-    SQLITE3_MUTEX_INITIALIZER,
-    SQLITE3_MUTEX_INITIALIZER,
-    SQLITE3_MUTEX_INITIALIZER,
-    SQLITE3_MUTEX_INITIALIZER,
-    SQLITE3_MUTEX_INITIALIZER,
-    SQLITE3_MUTEX_INITIALIZER,
-    SQLITE3_MUTEX_INITIALIZER,
-    SQLITE3_MUTEX_INITIALIZER,
-    SQLITE3_MUTEX_INITIALIZER,
-    SQLITE3_MUTEX_INITIALIZER,
-    SQLITE3_MUTEX_INITIALIZER,
-    SQLITE3_MUTEX_INITIALIZER
+    SQLITE3_MUTEX_INITIALIZER(2),
+    SQLITE3_MUTEX_INITIALIZER(3),
+    SQLITE3_MUTEX_INITIALIZER(4),
+    SQLITE3_MUTEX_INITIALIZER(5),
+    SQLITE3_MUTEX_INITIALIZER(6),
+    SQLITE3_MUTEX_INITIALIZER(7),
+    SQLITE3_MUTEX_INITIALIZER(8),
+    SQLITE3_MUTEX_INITIALIZER(9),
+    SQLITE3_MUTEX_INITIALIZER(10),
+    SQLITE3_MUTEX_INITIALIZER(11),
+    SQLITE3_MUTEX_INITIALIZER(12),
+    SQLITE3_MUTEX_INITIALIZER(13)
   };
   sqlite3_mutex *p;
   switch( iType ){
@@ -180,6 +181,9 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){
         pthread_mutexattr_settype(&recursiveAttr, PTHREAD_MUTEX_RECURSIVE);
         pthread_mutex_init(&p->mutex, &recursiveAttr);
         pthread_mutexattr_destroy(&recursiveAttr);
+#endif
+#if SQLITE_MUTEX_NREF || defined(SQLITE_ENABLE_API_ARMOR)
+        p->id = SQLITE_MUTEX_RECURSIVE;
 #endif
       }
       break;
@@ -188,6 +192,9 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){
       p = sqlite3MallocZero( sizeof(*p) );
       if( p ){
         pthread_mutex_init(&p->mutex, 0);
+#if SQLITE_MUTEX_NREF || defined(SQLITE_ENABLE_API_ARMOR)
+        p->id = SQLITE_MUTEX_FAST;
+#endif
       }
       break;
     }
@@ -203,7 +210,7 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){
     }
   }
 #if SQLITE_MUTEX_NREF || defined(SQLITE_ENABLE_API_ARMOR)
-  if( p ) p->id = iType;
+  assert( p==0 || p->id==iType );
 #endif
   return p;
 }

From 091881bbb030581af513b879bb8adc2878d63592 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Sun, 18 Feb 2018 00:54:06 +0000
Subject: [PATCH 451/488] Port mutex enhancments from check-in [f53b8a573b] to
 the Win32 implementation.

FossilOrigin-Name: 74bb7225d132c80fd5758bb8c120448e3b3e951d0ca2fa0c57cac0a9c6c27045
---
 manifest        | 14 +++++++-------
 manifest.uuid   |  2 +-
 src/mutex_w32.c | 34 +++++++++++++++++-----------------
 3 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/manifest b/manifest
index 3d3100acc3..7d33baf3f3 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sdata\srace\scausing\sa\stsan\scomplaint\swith\sSQLITE_ENABLE_API_ARMOR\sbuilds\non\sunix.\sThe\srace\scondition\sis\snot\sdangerous\sin\spractice,\sit\sjust\supsets\stsan.
-D 2018-02-17T07:38:50.155
+C Port\smutex\senhancments\sfrom\scheck-in\s[f53b8a573b]\sto\sthe\sWin32\simplementation.
+D 2018-02-18T00:54:06.860
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -466,7 +466,7 @@ F src/mutex.c b021263554c8a3995e9d53193b8194b96d1ed28e06c3b532dd7f7d29cf0c7d53
 F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85
 F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4
 F src/mutex_unix.c aaf9ebc3f89df28483c52208497a99a02cc3650011422fc9d4c57e4392f7fe58
-F src/mutex_w32.c a898fa969823b100c0f5fdc57e54c9a1e419ab4d
+F src/mutex_w32.c 7670d770c94bbfe8289bec9d7f1394c5a00a57c37f892aab6b6612d085255235
 F src/notify.c 9711a7575036f0d3040ba61bc6e217f13a9888e7
 F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104
 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
@@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b918d4b4e546d3903ff20efc3c8ca26dd9761cc8ed9ef7d2799b66ff9ae1ae51
-R f2c9c538fa5937452e735a3a9fd538f8
-U dan
-Z 36b8ef4562da3a84a51e82bfb896975f
+P f53b8a573bfbb48780243d56ca8372165adb1b979731c43d46cd1f8eb7e593f3
+R 46ee6e1b6f9dc8c46dcea8bb3cc2eb52
+U mistachkin
+Z 55a8fbd8babcd295b13a70346a882792
diff --git a/manifest.uuid b/manifest.uuid
index caa3d48cb7..28ef8a23c6 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f53b8a573bfbb48780243d56ca8372165adb1b979731c43d46cd1f8eb7e593f3
\ No newline at end of file
+74bb7225d132c80fd5758bb8c120448e3b3e951d0ca2fa0c57cac0a9c6c27045
\ No newline at end of file
diff --git a/src/mutex_w32.c b/src/mutex_w32.c
index 9da93cf319..8a8ae289ba 100644
--- a/src/mutex_w32.c
+++ b/src/mutex_w32.c
@@ -40,7 +40,7 @@ struct sqlite3_mutex {
 #ifdef SQLITE_DEBUG
   volatile int nRef;         /* Number of enterances */
   volatile DWORD owner;      /* Thread holding this mutex */
-  volatile int trace;        /* True to trace changes */
+  volatile LONG trace;       /* True to trace changes */
 #endif
 };
 
@@ -52,10 +52,10 @@ struct sqlite3_mutex {
 #define SQLITE_W32_MUTEX_INITIALIZER { 0 }
 
 #ifdef SQLITE_DEBUG
-#define SQLITE3_MUTEX_INITIALIZER { SQLITE_W32_MUTEX_INITIALIZER, 0, \
+#define SQLITE3_MUTEX_INITIALIZER(id) { SQLITE_W32_MUTEX_INITIALIZER, id, \
                                     0L, (DWORD)0, 0 }
 #else
-#define SQLITE3_MUTEX_INITIALIZER { SQLITE_W32_MUTEX_INITIALIZER, 0 }
+#define SQLITE3_MUTEX_INITIALIZER(id) { SQLITE_W32_MUTEX_INITIALIZER, id }
 #endif
 
 #ifdef SQLITE_DEBUG
@@ -98,18 +98,18 @@ void sqlite3MemoryBarrier(void){
 ** Initialize and deinitialize the mutex subsystem.
 */
 static sqlite3_mutex winMutex_staticMutexes[] = {
-  SQLITE3_MUTEX_INITIALIZER,
-  SQLITE3_MUTEX_INITIALIZER,
-  SQLITE3_MUTEX_INITIALIZER,
-  SQLITE3_MUTEX_INITIALIZER,
-  SQLITE3_MUTEX_INITIALIZER,
-  SQLITE3_MUTEX_INITIALIZER,
-  SQLITE3_MUTEX_INITIALIZER,
-  SQLITE3_MUTEX_INITIALIZER,
-  SQLITE3_MUTEX_INITIALIZER,
-  SQLITE3_MUTEX_INITIALIZER,
-  SQLITE3_MUTEX_INITIALIZER,
-  SQLITE3_MUTEX_INITIALIZER
+  SQLITE3_MUTEX_INITIALIZER(2),
+  SQLITE3_MUTEX_INITIALIZER(3),
+  SQLITE3_MUTEX_INITIALIZER(4),
+  SQLITE3_MUTEX_INITIALIZER(5),
+  SQLITE3_MUTEX_INITIALIZER(6),
+  SQLITE3_MUTEX_INITIALIZER(7),
+  SQLITE3_MUTEX_INITIALIZER(8),
+  SQLITE3_MUTEX_INITIALIZER(9),
+  SQLITE3_MUTEX_INITIALIZER(10),
+  SQLITE3_MUTEX_INITIALIZER(11),
+  SQLITE3_MUTEX_INITIALIZER(12),
+  SQLITE3_MUTEX_INITIALIZER(13)
 };
 
 static int winMutex_isInit = 0;
@@ -239,15 +239,15 @@ static sqlite3_mutex *winMutexAlloc(int iType){
       }
 #endif
       p = &winMutex_staticMutexes[iType-2];
-      p->id = iType;
 #ifdef SQLITE_DEBUG
 #ifdef SQLITE_WIN32_MUTEX_TRACE_STATIC
-      p->trace = 1;
+      InterlockedCompareExchange(&p->trace, 1, 0);
 #endif
 #endif
       break;
     }
   }
+  assert( p==0 || p->id==iType );
   return p;
 }
 

From e28eb64d09e3615d2695c2f6972d0c01b472bc2f Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sun, 18 Feb 2018 17:50:03 +0000
Subject: [PATCH 452/488] Fix a memory leak in the processing of nested row
 values.  This problem has existed every since row values support was added
 (version 3.15.0, 2016-10-14) but was only just now detected by OSSFuzz.

FossilOrigin-Name: 2df6bbf1b8ca881c8a465d6624de66fde4c5975ccae6b2f2dda392b137f577de
---
 manifest           | 16 ++++++++--------
 manifest.uuid      |  2 +-
 src/whereexpr.c    |  2 +-
 test/rowvalue.test |  6 ++++++
 4 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index 7d33baf3f3..646fefe6c5 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Port\smutex\senhancments\sfrom\scheck-in\s[f53b8a573b]\sto\sthe\sWin32\simplementation.
-D 2018-02-18T00:54:06.860
+C Fix\sa\smemory\sleak\sin\sthe\sprocessing\sof\snested\srow\svalues.\s\sThis\sproblem\shas\nexisted\severy\ssince\srow\svalues\ssupport\swas\sadded\s(version\s3.15.0,\s2016-10-14)\nbut\swas\sonly\sjust\snow\sdetected\sby\sOSSFuzz.
+D 2018-02-18T17:50:03.135
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -574,7 +574,7 @@ F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f
 F src/where.c 7cae47e813393d70c6d327fdf000fcb30f76b1b0b5a5b52ff6402e0c658de32c
 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971
 F src/wherecode.c e1aaadd8fec650037cfbf27d1b3470338fb3b58fec34d11082df16fe9a08fbd7
-F src/whereexpr.c 22dbfd3bf5f6051a61523dd0ebef7a944fb29ee4aa7d2a62feb8aac6ffbbc0eb
+F src/whereexpr.c 53532be687e12f3cd314f1e204cd4fbdac7ad250e918a182b048121e16e828ae
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd
 F test/affinity3.test 6a101af2fc945ce2912f6fe54dd646018551710d
@@ -1166,7 +1166,7 @@ F test/rollbackfault.test 0e646aeab8840c399cfbfa43daab46fd609cf04a
 F test/rowallock.test 3f88ec6819489d0b2341c7a7528ae17c053ab7cc
 F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81
 F test/rowid.test 5b7509f384f4f6fae1af3c8c104c8ca299fea18d
-F test/rowvalue.test 0bc33483f2ef5e69ff4bdd2ae58e36fc598bfd1605fb718c8329bcfc0c10cfd1
+F test/rowvalue.test 32861d6a933ded868035f2ec79aeb993a2a46eb7a6d282ae13415a4c2e369463
 F test/rowvalue2.test 060d238b7e5639a7c5630cb5e63e311b44efef2b
 F test/rowvalue3.test 3068f508753af69884b12125995f023da0dbb256
 F test/rowvalue4.test 4b556d7de161a0dd8cff095c336e913986398bea
@@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P f53b8a573bfbb48780243d56ca8372165adb1b979731c43d46cd1f8eb7e593f3
-R 46ee6e1b6f9dc8c46dcea8bb3cc2eb52
-U mistachkin
-Z 55a8fbd8babcd295b13a70346a882792
+P 74bb7225d132c80fd5758bb8c120448e3b3e951d0ca2fa0c57cac0a9c6c27045
+R 31bbf28c288f447f1b599c906d481780
+U drh
+Z 61189c02e04bfbbf029a1ddaa686693e
diff --git a/manifest.uuid b/manifest.uuid
index 28ef8a23c6..cc0ffaa4fc 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-74bb7225d132c80fd5758bb8c120448e3b3e951d0ca2fa0c57cac0a9c6c27045
\ No newline at end of file
+2df6bbf1b8ca881c8a465d6624de66fde4c5975ccae6b2f2dda392b137f577de
\ No newline at end of file
diff --git a/src/whereexpr.c b/src/whereexpr.c
index 82cc519650..313c5ee9bc 100644
--- a/src/whereexpr.c
+++ b/src/whereexpr.c
@@ -1291,7 +1291,7 @@ static void exprAnalyze(
       exprAnalyze(pSrc, pWC, idxNew);
     }
     pTerm = &pWC->a[idxTerm];
-    pTerm->wtFlags = TERM_CODED|TERM_VIRTUAL;  /* Disable the original */
+    pTerm->wtFlags |= TERM_CODED|TERM_VIRTUAL;  /* Disable the original */
     pTerm->eOperator = 0;
   }
 
diff --git a/test/rowvalue.test b/test/rowvalue.test
index 5805178301..00d9395331 100644
--- a/test/rowvalue.test
+++ b/test/rowvalue.test
@@ -540,4 +540,10 @@ do_execsql_test 19.36 {
   SELECT * FROM t1 WHERE (3,32)>=(a,b) ORDER BY a DESC;
 } {2 22 1 11}
 
+# 2018-02-18: Memory leak nexted row-value.  Detected by OSSFuzz.
+#
+do_catchsql_test 20.1 {
+  SELECT 1 WHERE (2,(2,0)) IS (2,(2,0));
+} {0 1}
+
 finish_test

From a224ee2a1b12450014aa4ae1e0e85316efcf39c4 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 19 Feb 2018 13:53:56 +0000
Subject: [PATCH 453/488] Fix an assert so that it compares two CellInfo
 objects field by field instead of using memcmp().  Memcmp() does not work on
 x86 because of uninitialized padding bytes.

FossilOrigin-Name: 88258770adead70fa101c74e266a37bb9aaffac0ba738a4b345617feb8c46477
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/btree.c   | 10 +++++++++-
 3 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 646fefe6c5..d73a577eff 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\smemory\sleak\sin\sthe\sprocessing\sof\snested\srow\svalues.\s\sThis\sproblem\shas\nexisted\severy\ssince\srow\svalues\ssupport\swas\sadded\s(version\s3.15.0,\s2016-10-14)\nbut\swas\sonly\sjust\snow\sdetected\sby\sOSSFuzz.
-D 2018-02-18T17:50:03.135
+C Fix\san\sassert\sso\sthat\sit\scompares\stwo\sCellInfo\sobjects\sfield\sby\sfield\ninstead\sof\susing\smemcmp().\s\sMemcmp()\sdoes\snot\swork\son\sx86\nbecause\sof\suninitialized\spadding\sbytes.
+D 2018-02-19T13:53:56.396
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -430,7 +430,7 @@ F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
-F src/btree.c f7bb37c27b3734d58796febb30b5d818b3b4eb592b57b88039e5d92b3ebd9dab
+F src/btree.c 18a53540aa35dbdf77f715ea928422a4ed9011dc16ea7b50f803fd1617fcc4f5
 F src/btree.h 0866c0a08255142ea0e754aabd211c843cab32045c978a592a43152405ed0c84
 F src/btreeInt.h 620ab4c7235f43572cf3ac2ac8723cbdf68073be4d29da24897c7b77dda5fd96
 F src/build.c 672022c06e1a5c2653f80c77a687de11f7e65ce81d20fe2825aadfa13a875c33
@@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 74bb7225d132c80fd5758bb8c120448e3b3e951d0ca2fa0c57cac0a9c6c27045
-R 31bbf28c288f447f1b599c906d481780
+P 2df6bbf1b8ca881c8a465d6624de66fde4c5975ccae6b2f2dda392b137f577de
+R e03e92857318b640fae3aae1181693fa
 U drh
-Z 61189c02e04bfbbf029a1ddaa686693e
+Z 8b8a951294db4a887bc90d59e44c7680
diff --git a/manifest.uuid b/manifest.uuid
index cc0ffaa4fc..d969c3165e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2df6bbf1b8ca881c8a465d6624de66fde4c5975ccae6b2f2dda392b137f577de
\ No newline at end of file
+88258770adead70fa101c74e266a37bb9aaffac0ba738a4b345617feb8c46477
\ No newline at end of file
diff --git a/src/btree.c b/src/btree.c
index d3d2a41e26..22f8a50e59 100644
--- a/src/btree.c
+++ b/src/btree.c
@@ -4388,11 +4388,19 @@ int sqlite3BtreeCloseCursor(BtCursor *pCur){
 ** Using this cache reduces the number of calls to btreeParseCell().
 */
 #ifndef NDEBUG
+  static int cellInfoEqual(CellInfo *a, CellInfo *b){
+    if( a->nKey!=b->nKey ) return 0;
+    if( a->pPayload!=b->pPayload ) return 0;
+    if( a->nPayload!=b->nPayload ) return 0;
+    if( a->nLocal!=b->nLocal ) return 0;
+    if( a->nSize!=b->nSize ) return 0;
+    return 1;
+  }
   static void assertCellInfo(BtCursor *pCur){
     CellInfo info;
     memset(&info, 0, sizeof(info));
     btreeParseCell(pCur->pPage, pCur->ix, &info);
-    assert( CORRUPT_DB || memcmp(&info, &pCur->info, sizeof(info))==0 );
+    assert( CORRUPT_DB || cellInfoEqual(&info, &pCur->info) );
   }
 #else
   #define assertCellInfo(x)

From 26f625fbf4b293012f2a3ff4b8127faadc787c66 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 19 Feb 2018 16:34:31 +0000
Subject: [PATCH 454/488] Only try to use the geteuid() interface on unix if
 HAVE_FCHOWN is defined. This fixes the build for vxWorks, we are told.

FossilOrigin-Name: 38f654dc117bb11d61407327884b5c913cfaf920547bebc4ce5a4cf983e5c20e
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/os_unix.c |  4 ++++
 3 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index d73a577eff..ea203fe2f3 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\san\sassert\sso\sthat\sit\scompares\stwo\sCellInfo\sobjects\sfield\sby\sfield\ninstead\sof\susing\smemcmp().\s\sMemcmp()\sdoes\snot\swork\son\sx86\nbecause\sof\suninitialized\spadding\sbytes.
-D 2018-02-19T13:53:56.396
+C Only\stry\sto\suse\sthe\sgeteuid()\sinterface\son\sunix\sif\sHAVE_FCHOWN\sis\sdefined.\nThis\sfixes\sthe\sbuild\sfor\svxWorks,\swe\sare\stold.
+D 2018-02-19T16:34:31.891
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -472,7 +472,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104
 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
-F src/os_unix.c ce491421b3a54b63094a155eeac668a3bc8e5b86a5a58551d906e5b5affb443f
+F src/os_unix.c 9e02186f0dcde77d3ec27d4409664eb692c4e8ea0257e30c0b7f25969fa69fec
 F src/os_win.c eb03c6d52f893bcd7fdd4c6006674c13c1b5e49543fec98d605201af2997171c
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 0b6bd5442733b2e08d0673de6cdafe3e7ab0b5715e4844ac836ab346b1d9ed89
@@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2df6bbf1b8ca881c8a465d6624de66fde4c5975ccae6b2f2dda392b137f577de
-R e03e92857318b640fae3aae1181693fa
+P 88258770adead70fa101c74e266a37bb9aaffac0ba738a4b345617feb8c46477
+R 2e4bdb0c045376ff7154d29ff239bf61
 U drh
-Z 8b8a951294db4a887bc90d59e44c7680
+Z 37b9f8fa92af1d03e78cb9f20faaf7e7
diff --git a/manifest.uuid b/manifest.uuid
index d969c3165e..f3e38fcfac 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-88258770adead70fa101c74e266a37bb9aaffac0ba738a4b345617feb8c46477
\ No newline at end of file
+38f654dc117bb11d61407327884b5c913cfaf920547bebc4ce5a4cf983e5c20e
\ No newline at end of file
diff --git a/src/os_unix.c b/src/os_unix.c
index b24c6861d3..cb2ae33d0a 100644
--- a/src/os_unix.c
+++ b/src/os_unix.c
@@ -468,7 +468,11 @@ static struct unix_syscall {
 #endif
 #define osFchown    ((int(*)(int,uid_t,gid_t))aSyscall[20].pCurrent)
 
+#if defined(HAVE_FCHOWN)
   { "geteuid",      (sqlite3_syscall_ptr)geteuid,         0 },
+#else
+  { "geteuid",      (sqlite3_syscall_ptr)0,               0 },
+#endif
 #define osGeteuid   ((uid_t(*)(void))aSyscall[21].pCurrent)
 
 #if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0

From 62856465f2350ccba3aa95798e6af543e466f521 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 19 Feb 2018 17:03:23 +0000
Subject: [PATCH 455/488] In the printf() library, measure width and precision
 in characters rather than bytes if the "!" (alternate-form-2) flag is present
 on a %s or %z substitution.

FossilOrigin-Name: ca31c6630422fca70e626dd38aae96296bd8535d491ca52391624a5e7e663636
---
 manifest      | 15 +++++++++------
 manifest.uuid |  2 +-
 src/printf.c  | 24 +++++++++++++++++++++---
 3 files changed, 31 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index ea203fe2f3..51334ef416 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Only\stry\sto\suse\sthe\sgeteuid()\sinterface\son\sunix\sif\sHAVE_FCHOWN\sis\sdefined.\nThis\sfixes\sthe\sbuild\sfor\svxWorks,\swe\sare\stold.
-D 2018-02-19T16:34:31.891
+C In\sthe\sprintf()\slibrary,\smeasure\swidth\sand\sprecision\sin\scharacters\srather\nthan\sbytes\sif\sthe\s"!"\s(alternate-form-2)\sflag\sis\spresent\son\sa\s%s\sor\s%z\nsubstitution.
+D 2018-02-19T17:03:23.575
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -484,7 +484,7 @@ F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
 F src/pragma.c bea56df3ae0637768c0da4fbbb8f2492f780980d95000034a105ff291bf7ca69
 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
 F src/prepare.c 259f4e7960c47082c9653f3d5f0c294abd68bb9c3aab86de7630700cba1c20fb
-F src/printf.c 9506b4b96e59c0467047155f09015750cb2878aeda3d39e5610c1192ddc3c41c
+F src/printf.c a94897345b3228d97c6aa06297890b7c62bc47acc86e34d3634d125426d78695
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
@@ -1707,7 +1707,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 88258770adead70fa101c74e266a37bb9aaffac0ba738a4b345617feb8c46477
-R 2e4bdb0c045376ff7154d29ff239bf61
+P 38f654dc117bb11d61407327884b5c913cfaf920547bebc4ce5a4cf983e5c20e
+R 07cd50bae7c4080dd411c728ce9cf3a0
+T *branch * printf-enhancement
+T *sym-printf-enhancement *
+T -sym-trunk *
 U drh
-Z 37b9f8fa92af1d03e78cb9f20faaf7e7
+Z 9d6ee38fe0b61a32cc5a7309de027572
diff --git a/manifest.uuid b/manifest.uuid
index f3e38fcfac..29c0e9958e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-38f654dc117bb11d61407327884b5c913cfaf920547bebc4ce5a4cf983e5c20e
\ No newline at end of file
+ca31c6630422fca70e626dd38aae96296bd8535d491ca52391624a5e7e663636
\ No newline at end of file
diff --git a/src/printf.c b/src/printf.c
index ca8d26e4f9..f6980c90db 100644
--- a/src/printf.c
+++ b/src/printf.c
@@ -654,10 +654,25 @@ void sqlite3VXPrintf(
           zExtra = bufpt;
         }
         if( precision>=0 ){
-          for(length=0; length<precision && bufpt[length]; length++){}
+          if( flag_altform2 ){
+            /* Set length to the number of bytes needed in order to display
+            ** precision characters */
+            unsigned char *z = (unsigned char*)bufpt;
+            while( precision-- > 0 && z[0] ){
+              SQLITE_SKIP_UTF8(z);
+            }
+            length = (int)(z - (unsigned char*)bufpt);
+          }else{
+            for(length=0; length<precision && bufpt[length]; length++){}
+          }
         }else{
           length = 0x7fffffff & (int)strlen(bufpt);
         }
+        if( flag_altform2 && width>0 ){
+          /* Adjust width to account for extra bytes in UTF-8 characters */
+          int ii = length - 1;
+          while( ii>=0 ) if( (bufpt[ii--] & 0xc0)==0x80 ) width++;
+        }
         break;
       case etSQLESCAPE:           /* Escape ' characters */
       case etSQLESCAPE2:          /* Escape ' and enclose in '...' */
@@ -700,7 +715,7 @@ void sqlite3VXPrintf(
         if( needQuote ) bufpt[j++] = q;
         bufpt[j] = 0;
         length = j;
-        /* The precision in %q and %Q means how many input characters to
+        /* The precision in %q and %Q means how many input bytes to
         ** consume, not the length of the output...
         ** if( precision>=0 && precision<length ) length = precision; */
         break;
@@ -742,7 +757,10 @@ void sqlite3VXPrintf(
     /*
     ** The text of the conversion is pointed to by "bufpt" and is
     ** "length" characters long.  The field width is "width".  Do
-    ** the output.
+    ** the output.  Both length and width are in bytes, not characters,
+    ** at this point.  If the "!" flag was present on string conversions
+    ** indicating that width and precision should be expressed in characters,
+    ** then the values have been translated prior to reaching this point.
     */
     width -= length;
     if( width>0 ){

From 57e3ba7623c1d819399f779e83372fea1727b9bf Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 19 Feb 2018 18:03:10 +0000
Subject: [PATCH 456/488] Make the alternate-form-2 flag ("!") change the
 meaning of width and precision from bytes to characters for the %q, %Q, and
 %w extensions of printf().

FossilOrigin-Name: 391540acbea34eb88c75b1152b458d8936b3fdfff47633b9ec897775b015ba4c
---
 manifest      | 15 ++++++---------
 manifest.uuid |  2 +-
 src/printf.c  | 20 +++++++++++++-------
 3 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/manifest b/manifest
index 51334ef416..867a07179e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\sthe\sprintf()\slibrary,\smeasure\swidth\sand\sprecision\sin\scharacters\srather\nthan\sbytes\sif\sthe\s"!"\s(alternate-form-2)\sflag\sis\spresent\son\sa\s%s\sor\s%z\nsubstitution.
-D 2018-02-19T17:03:23.575
+C Make\sthe\salternate-form-2\sflag\s("!")\schange\sthe\smeaning\sof\swidth\sand\sprecision\nfrom\sbytes\sto\scharacters\sfor\sthe\s%q,\s%Q,\sand\s%w\sextensions\sof\sprintf().
+D 2018-02-19T18:03:10.959
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -484,7 +484,7 @@ F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
 F src/pragma.c bea56df3ae0637768c0da4fbbb8f2492f780980d95000034a105ff291bf7ca69
 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
 F src/prepare.c 259f4e7960c47082c9653f3d5f0c294abd68bb9c3aab86de7630700cba1c20fb
-F src/printf.c a94897345b3228d97c6aa06297890b7c62bc47acc86e34d3634d125426d78695
+F src/printf.c ab91b63394bf07a430af0c644f8e42ca136c015e2533544f585dab8b0a408b84
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
@@ -1707,10 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 38f654dc117bb11d61407327884b5c913cfaf920547bebc4ce5a4cf983e5c20e
-R 07cd50bae7c4080dd411c728ce9cf3a0
-T *branch * printf-enhancement
-T *sym-printf-enhancement *
-T -sym-trunk *
+P ca31c6630422fca70e626dd38aae96296bd8535d491ca52391624a5e7e663636
+R d4aa6a8346f9339a7533b828b4137a72
 U drh
-Z 9d6ee38fe0b61a32cc5a7309de027572
+Z bc6f530d4e962e8540ac43cd25813033
diff --git a/manifest.uuid b/manifest.uuid
index 29c0e9958e..50989e81be 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ca31c6630422fca70e626dd38aae96296bd8535d491ca52391624a5e7e663636
\ No newline at end of file
+391540acbea34eb88c75b1152b458d8936b3fdfff47633b9ec897775b015ba4c
\ No newline at end of file
diff --git a/src/printf.c b/src/printf.c
index f6980c90db..ff948ff5a4 100644
--- a/src/printf.c
+++ b/src/printf.c
@@ -668,15 +668,16 @@ void sqlite3VXPrintf(
         }else{
           length = 0x7fffffff & (int)strlen(bufpt);
         }
+      adjust_width_for_utf8:
         if( flag_altform2 && width>0 ){
           /* Adjust width to account for extra bytes in UTF-8 characters */
           int ii = length - 1;
           while( ii>=0 ) if( (bufpt[ii--] & 0xc0)==0x80 ) width++;
         }
         break;
-      case etSQLESCAPE:           /* Escape ' characters */
-      case etSQLESCAPE2:          /* Escape ' and enclose in '...' */
-      case etSQLESCAPE3: {        /* Escape " characters */
+      case etSQLESCAPE:           /* %q: Escape ' characters */
+      case etSQLESCAPE2:          /* %Q: Escape ' and enclose in '...' */
+      case etSQLESCAPE3: {        /* %w: Escape " characters */
         int i, j, k, n, isnull;
         int needQuote;
         char ch;
@@ -690,9 +691,17 @@ void sqlite3VXPrintf(
         }
         isnull = escarg==0;
         if( isnull ) escarg = (xtype==etSQLESCAPE2 ? "NULL" : "(NULL)");
+        /* For %q, %Q, and %w, the precision is the number of byte (or
+        ** characters if the ! flags is present) to use from the input.
+        ** Because of the extra quoting characters inserted, the number
+        ** of output characters may be larger than the precision.
+        */
         k = precision;
         for(i=n=0; k!=0 && (ch=escarg[i])!=0; i++, k--){
           if( ch==q )  n++;
+          if( flag_altform2 && (ch&0xc0)==0xc0 ){
+            while( (escarg[i+1]&0xc0)==0x80 ){ i++; }
+          }
         }
         needQuote = !isnull && xtype==etSQLESCAPE2;
         n += i + 3;
@@ -715,10 +724,7 @@ void sqlite3VXPrintf(
         if( needQuote ) bufpt[j++] = q;
         bufpt[j] = 0;
         length = j;
-        /* The precision in %q and %Q means how many input bytes to
-        ** consume, not the length of the output...
-        ** if( precision>=0 && precision<length ) length = precision; */
-        break;
+        goto adjust_width_for_utf8;
       }
       case etTOKEN: {
         Token *pToken;

From 136102bedab5a5a0a180e73388f9934d175d6717 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 19 Feb 2018 18:56:52 +0000
Subject: [PATCH 457/488] Fix the %c substitution in printf() so that it works
 with unicode characters.

FossilOrigin-Name: c35be1d9e9132fbadc74d207c1088e5a710a151c0835c38a63bf3c8e6fd711a0
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/printf.c  | 38 +++++++++++++++++++++++++++++++++-----
 3 files changed, 40 insertions(+), 12 deletions(-)

diff --git a/manifest b/manifest
index 867a07179e..151cc25b6f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Make\sthe\salternate-form-2\sflag\s("!")\schange\sthe\smeaning\sof\swidth\sand\sprecision\nfrom\sbytes\sto\scharacters\sfor\sthe\s%q,\s%Q,\sand\s%w\sextensions\sof\sprintf().
-D 2018-02-19T18:03:10.959
+C Fix\sthe\s%c\ssubstitution\sin\sprintf()\sso\sthat\sit\sworks\swith\sunicode\scharacters.
+D 2018-02-19T18:56:52.191
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -484,7 +484,7 @@ F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
 F src/pragma.c bea56df3ae0637768c0da4fbbb8f2492f780980d95000034a105ff291bf7ca69
 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
 F src/prepare.c 259f4e7960c47082c9653f3d5f0c294abd68bb9c3aab86de7630700cba1c20fb
-F src/printf.c ab91b63394bf07a430af0c644f8e42ca136c015e2533544f585dab8b0a408b84
+F src/printf.c 721a26a112e895ec7c7388fab9e8a1eb1c3fe55493cd14a19eb26ffd8ea9798e
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
@@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ca31c6630422fca70e626dd38aae96296bd8535d491ca52391624a5e7e663636
-R d4aa6a8346f9339a7533b828b4137a72
+P 391540acbea34eb88c75b1152b458d8936b3fdfff47633b9ec897775b015ba4c
+R 71fe5339e526e4b9a6fb2ef9bc7b6d4e
 U drh
-Z bc6f530d4e962e8540ac43cd25813033
+Z 27f720e6a7a3028447e586595bfbe60c
diff --git a/manifest.uuid b/manifest.uuid
index 50989e81be..34e844ad2e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-391540acbea34eb88c75b1152b458d8936b3fdfff47633b9ec897775b015ba4c
\ No newline at end of file
+c35be1d9e9132fbadc74d207c1088e5a710a151c0835c38a63bf3c8e6fd711a0
\ No newline at end of file
diff --git a/src/printf.c b/src/printf.c
index ff948ff5a4..ada582ebdf 100644
--- a/src/printf.c
+++ b/src/printf.c
@@ -624,9 +624,37 @@ void sqlite3VXPrintf(
       case etCHARX:
         if( bArgList ){
           bufpt = getTextArg(pArgList);
-          c = bufpt ? bufpt[0] : 0;
+          length = 0;
+          if( bufpt ){
+            buf[0] = c = *(bufpt++);
+            length = 1;
+            if( (c&0xc0)==0xc0 ){
+              while( length<4 && (bufpt[0]&0xc0)==0x80 ){
+                buf[length++] = *(bufpt++);
+              }
+            }
+          }
         }else{
-          c = va_arg(ap,int);
+          unsigned int ch = va_arg(ap,unsigned int);
+          if( ch<0x00080 ){
+            buf[0] = ch & 0xff;
+            length = 1;
+          }else if( ch<0x00800 ){
+            buf[0] = 0xc0 + (u8)((ch>>6)&0x1f);
+            buf[1] = 0x80 + (u8)(ch & 0x3f);
+            length = 2;
+          }else if( ch<0x10000 ){
+            buf[0] = 0xe0 + (u8)((ch>>12)&0x0f);
+            buf[1] = 0x80 + (u8)((ch>>6) & 0x3f);
+            buf[2] = 0x80 + (u8)(ch & 0x3f);
+            length = 3;
+          }else{
+            buf[0] = 0xf0 + (u8)((ch>>18) & 0x07);
+            buf[1] = 0x80 + (u8)((ch>>12) & 0x3f);
+            buf[2] = 0x80 + (u8)((ch>>6) & 0x3f);
+            buf[3] = 0x80 + (u8)(ch & 0x3f);
+            length = 4;
+          }
         }
         if( precision>1 ){
           width -= precision-1;
@@ -634,10 +662,10 @@ void sqlite3VXPrintf(
             sqlite3AppendChar(pAccum, width-1, ' ');
             width = 0;
           }
-          sqlite3AppendChar(pAccum, precision-1, c);
+          while( precision-- > 1 ){
+            sqlite3StrAccumAppend(pAccum, buf, length);
+          }
         }
-        length = 1;
-        buf[0] = c;
         bufpt = buf;
         break;
       case etSTRING:

From cf7c8370d19b10b23e6b2360137490a3eddddc02 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 19 Feb 2018 20:23:20 +0000
Subject: [PATCH 458/488] Test cases and a bug fix on the new unicode handling
 in %c.

FossilOrigin-Name: e41d64e95b9b6fec5dc329553822925e7d648a41912b420cfa1ba198736e6bab
---
 manifest          | 14 +++++++-------
 manifest.uuid     |  2 +-
 src/printf.c      |  3 ++-
 test/printf2.test | 25 +++++++++++++++++++++++++
 4 files changed, 35 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index 151cc25b6f..e688a8efc4 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sthe\s%c\ssubstitution\sin\sprintf()\sso\sthat\sit\sworks\swith\sunicode\scharacters.
-D 2018-02-19T18:56:52.191
+C Test\scases\sand\sa\sbug\sfix\son\sthe\snew\sunicode\shandling\sin\s%c.
+D 2018-02-19T20:23:20.986
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -484,7 +484,7 @@ F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
 F src/pragma.c bea56df3ae0637768c0da4fbbb8f2492f780980d95000034a105ff291bf7ca69
 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
 F src/prepare.c 259f4e7960c47082c9653f3d5f0c294abd68bb9c3aab86de7630700cba1c20fb
-F src/printf.c 721a26a112e895ec7c7388fab9e8a1eb1c3fe55493cd14a19eb26ffd8ea9798e
+F src/printf.c 1c8986a3ab20f0cdc7bdf1b7f6ea7b9857acdd34eba5c82e0be70fed825d2f9f
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
@@ -1141,7 +1141,7 @@ F test/pragma4.test 3046501bee2f652dc2a4f9c87781e2741361d6864439c8381aba6c3b774b
 F test/pragma5.test 824ce6ced5d6b7ec71abe37fc6005ff836fe39d638273dc5192b39864b9ee983
 F test/pragmafault.test 275edaf3161771d37de60e5c2b412627ac94cef11739236bec12ed1258b240f8
 F test/printf.test b3ff34e73d59124140eaf89f7672e21bc2ca5fcc
-F test/printf2.test 9e6db85f81c63f2367c34a9d7db384088bd374ad
+F test/printf2.test e2bb746fdc8bbf45bbd495ba6f3a8b2256a47428a333274ae01b30bfeb938ca1
 F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb
 F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc
 F test/pushdown.test 5e72c51c5e33253ed639ccee1e01ce62d62b6eee5ca893cd82334e4ee7b1d7fc
@@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 391540acbea34eb88c75b1152b458d8936b3fdfff47633b9ec897775b015ba4c
-R 71fe5339e526e4b9a6fb2ef9bc7b6d4e
+P c35be1d9e9132fbadc74d207c1088e5a710a151c0835c38a63bf3c8e6fd711a0
+R fe2c99c67a288621be87056c5aacba15
 U drh
-Z 27f720e6a7a3028447e586595bfbe60c
+Z fd62d4a92d36a20a423e3f08ea4c3a69
diff --git a/manifest.uuid b/manifest.uuid
index 34e844ad2e..6a39fe7287 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c35be1d9e9132fbadc74d207c1088e5a710a151c0835c38a63bf3c8e6fd711a0
\ No newline at end of file
+e41d64e95b9b6fec5dc329553822925e7d648a41912b420cfa1ba198736e6bab
\ No newline at end of file
diff --git a/src/printf.c b/src/printf.c
index ada582ebdf..725caa2d5e 100644
--- a/src/printf.c
+++ b/src/printf.c
@@ -667,7 +667,8 @@ void sqlite3VXPrintf(
           }
         }
         bufpt = buf;
-        break;
+        flag_altform2 = 1;
+        goto adjust_width_for_utf8;
       case etSTRING:
       case etDYNSTRING:
         if( bArgList ){
diff --git a/test/printf2.test b/test/printf2.test
index d30966d167..c73727d005 100644
--- a/test/printf2.test
+++ b/test/printf2.test
@@ -148,6 +148,31 @@ do_execsql_test printf2-4.10 {
   SELECT printf('|%,d|%,d|',1234567890,-1234567890);
 } {|1,234,567,890|-1,234,567,890|}
 
+# 2018-02-19.  Unicode characters with %c
+do_execsql_test printf2-5.100 {
+  SELECT printf('(%8c)',char(11106));
+} {{(       ⭢)}}
+do_execsql_test printf2-5.101 {
+  SELECT printf('(%-8c)',char(11106));
+} {{(⭢       )}}
+do_execsql_test printf2-5.102 {
+  SELECT printf('(%5.3c)',char(1492));
+} {{(  ההה)}}
+do_execsql_test printf2-5.103 {
+  SELECT printf('(%-5.3c)',char(1492));
+} {{(ההה  )}}
+do_execsql_test printf2-5.104 {
+  SELECT printf('(%3.3c)',char(1492));
+} {{(ההה)}}
+do_execsql_test printf2-5.105 {
+  SELECT printf('(%-3.3c)',char(1492));
+} {{(ההה)}}
+do_execsql_test printf2-5.104 {
+  SELECT printf('(%2c)',char(1513));
+} {{( ש)}}
+do_execsql_test printf2-5.106 {
+  SELECT printf('(%-2c)',char(1513));
+} {{(ש )}}
 
 
 finish_test

From 57f29b7bfbc0bc69f2c2f0b73e22a5aa0664d13c Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 19 Feb 2018 20:41:01 +0000
Subject: [PATCH 459/488] More test cases for unicode handling in printf.

FossilOrigin-Name: ac188211119bfd77f43e4aba8acab733a56aac90b4c8a11f7ef8e6388582438f
---
 manifest          | 12 ++++++------
 manifest.uuid     |  2 +-
 test/printf2.test | 32 ++++++++++++++++++++++++++++++++
 3 files changed, 39 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index e688a8efc4..a521ef217b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Test\scases\sand\sa\sbug\sfix\son\sthe\snew\sunicode\shandling\sin\s%c.
-D 2018-02-19T20:23:20.986
+C More\stest\scases\sfor\sunicode\shandling\sin\sprintf.
+D 2018-02-19T20:41:01.120
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -1141,7 +1141,7 @@ F test/pragma4.test 3046501bee2f652dc2a4f9c87781e2741361d6864439c8381aba6c3b774b
 F test/pragma5.test 824ce6ced5d6b7ec71abe37fc6005ff836fe39d638273dc5192b39864b9ee983
 F test/pragmafault.test 275edaf3161771d37de60e5c2b412627ac94cef11739236bec12ed1258b240f8
 F test/printf.test b3ff34e73d59124140eaf89f7672e21bc2ca5fcc
-F test/printf2.test e2bb746fdc8bbf45bbd495ba6f3a8b2256a47428a333274ae01b30bfeb938ca1
+F test/printf2.test 30b5dd0b4b992dc5626496846ecce17ff592cacbcb11c3e589f3ac4d7e129dae
 F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb
 F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc
 F test/pushdown.test 5e72c51c5e33253ed639ccee1e01ce62d62b6eee5ca893cd82334e4ee7b1d7fc
@@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c35be1d9e9132fbadc74d207c1088e5a710a151c0835c38a63bf3c8e6fd711a0
-R fe2c99c67a288621be87056c5aacba15
+P e41d64e95b9b6fec5dc329553822925e7d648a41912b420cfa1ba198736e6bab
+R c809a7387d4329cd8d06217ca9198d46
 U drh
-Z fd62d4a92d36a20a423e3f08ea4c3a69
+Z c256c2f346298486ba50767fa31b18cb
diff --git a/manifest.uuid b/manifest.uuid
index 6a39fe7287..4a4024e523 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e41d64e95b9b6fec5dc329553822925e7d648a41912b420cfa1ba198736e6bab
\ No newline at end of file
+ac188211119bfd77f43e4aba8acab733a56aac90b4c8a11f7ef8e6388582438f
\ No newline at end of file
diff --git a/test/printf2.test b/test/printf2.test
index c73727d005..998038f88e 100644
--- a/test/printf2.test
+++ b/test/printf2.test
@@ -174,5 +174,37 @@ do_execsql_test printf2-5.106 {
   SELECT printf('(%-2c)',char(1513));
 } {{(ש )}}
 
+# 2018-02-19.  Unicode characters with the "!" flag in %s and friends.
+do_execsql_test printf2-6.100 {
+  SELECT printf('(%!.3s)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד');
+} {(הנה)}
+do_execsql_test printf2-6.101 {
+  SELECT printf('(%.6s)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד');
+} {(הנה)}
+do_execsql_test printf2-6.102 {
+  SELECT printf('(%!5.3s)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד');
+} {{(  הנה)}}
+do_execsql_test printf2-6.103 {
+  SELECT printf('(%8.6s)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד');
+} {{(  הנה)}}
+do_execsql_test printf2-6.104 {
+  SELECT printf('(%!-5.3s)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד');
+} {{(הנה  )}}
+do_execsql_test printf2-6.105 {
+  SELECT printf('(%-8.6s)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד');
+} {{(הנה  )}}
+do_execsql_test printf2-6.106 {
+  SELECT printf('(%!.3Q)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד');
+} {('הנה')}
+do_execsql_test printf2-6.107 {
+  SELECT printf('(%.6Q)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד');
+} {('הנה')}
+do_execsql_test printf2-6.108 {
+  SELECT printf('(%!7.3Q)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד');
+} {{(  'הנה')}}
+do_execsql_test printf2-6.109 {
+  SELECT printf('(%10.6Q)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד');
+} {{(  'הנה')}}
+
 
 finish_test

From a15a7c352ce0aad7143e589e32a4d9e60f5ce9e2 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 19 Feb 2018 21:58:16 +0000
Subject: [PATCH 460/488] Printing a value of 0 using %c terminates the string.

FossilOrigin-Name: 255612f0a131f2f522cbca3cc5a1edcf7e38938abca25ba421e7e38a422db9c9
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/printf.c  |  5 +++--
 3 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index a521ef217b..5fae45a4a2 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C More\stest\scases\sfor\sunicode\shandling\sin\sprintf.
-D 2018-02-19T20:41:01.120
+C Printing\sa\svalue\sof\s0\susing\s%c\sterminates\sthe\sstring.
+D 2018-02-19T21:58:16.694
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -484,7 +484,7 @@ F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
 F src/pragma.c bea56df3ae0637768c0da4fbbb8f2492f780980d95000034a105ff291bf7ca69
 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
 F src/prepare.c 259f4e7960c47082c9653f3d5f0c294abd68bb9c3aab86de7630700cba1c20fb
-F src/printf.c 1c8986a3ab20f0cdc7bdf1b7f6ea7b9857acdd34eba5c82e0be70fed825d2f9f
+F src/printf.c 07244a9e1755247c9543db551acd2f3cce461b28d13e9f3ab87053c35fd62717
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
@@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e41d64e95b9b6fec5dc329553822925e7d648a41912b420cfa1ba198736e6bab
-R c809a7387d4329cd8d06217ca9198d46
+P ac188211119bfd77f43e4aba8acab733a56aac90b4c8a11f7ef8e6388582438f
+R 508c3db0e2913d65a43c01bb9b80f5ab
 U drh
-Z c256c2f346298486ba50767fa31b18cb
+Z 626afeea462f05f2075e81e36cf30b70
diff --git a/manifest.uuid b/manifest.uuid
index 4a4024e523..3b60577eae 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ac188211119bfd77f43e4aba8acab733a56aac90b4c8a11f7ef8e6388582438f
\ No newline at end of file
+255612f0a131f2f522cbca3cc5a1edcf7e38938abca25ba421e7e38a422db9c9
\ No newline at end of file
diff --git a/src/printf.c b/src/printf.c
index 725caa2d5e..11d5b82828 100644
--- a/src/printf.c
+++ b/src/printf.c
@@ -624,15 +624,16 @@ void sqlite3VXPrintf(
       case etCHARX:
         if( bArgList ){
           bufpt = getTextArg(pArgList);
-          length = 0;
+          length = 1;
           if( bufpt ){
             buf[0] = c = *(bufpt++);
-            length = 1;
             if( (c&0xc0)==0xc0 ){
               while( length<4 && (bufpt[0]&0xc0)==0x80 ){
                 buf[length++] = *(bufpt++);
               }
             }
+          }else{
+            buf[0] = 0;
           }
         }else{
           unsigned int ch = va_arg(ap,unsigned int);

From b0b6f8783c8a80fbf68e38a3230182edeacef306 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 20 Feb 2018 13:46:20 +0000
Subject: [PATCH 461/488] Remove a lot of the text describing extended format
 options from the documentation on sqlite3_mprintf() and friends, since that
 information is now covered by the separate printf.html document.  Provide
 links to that other document.  No changes to code.

FossilOrigin-Name: 99eec556f065ad19548e48d1f4ae0a3767b4e82e1c83fa2365062e3c5e0071fb
---
 manifest        | 13 ++++-----
 manifest.uuid   |  2 +-
 src/sqlite.h.in | 78 +++++--------------------------------------------
 3 files changed, 14 insertions(+), 79 deletions(-)

diff --git a/manifest b/manifest
index 2e00962c43..9788e54316 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Enhance\sthe\sstring\sformatter\s(used\sby\sprintf())\sso\sthat\sthe\swidth\sand\nprecision\sof\sstring\ssubstitution\soperators\srefer\sto\scharacters\sinstead\sof\nbytes\swhen\sthe\salternate-form-2\sflag\s("!")\sis\sused.\s\sAlso\sfix\sthe\s%c\nsubstition\sto\salways\swork\swithin\sunicode,\sregardless\sof\sthe\s\nalternate-form-2\sflag.
-D 2018-02-19T22:46:19.020
+C Remove\sa\slot\sof\sthe\stext\sdescribing\sextended\sformat\soptions\sfrom\sthe\ndocumentation\son\ssqlite3_mprintf()\sand\sfriends,\ssince\sthat\sinformation\sis\nnow\scovered\sby\sthe\sseparate\sprintf.html\sdocument.\s\sProvide\slinks\sto\sthat\nother\sdocument.\s\sNo\schanges\sto\scode.
+D 2018-02-20T13:46:20.902
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -490,7 +490,7 @@ F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c f02352ac5cbb6fad3804add825743b924cfb6c742ba2e8666d726828a9985d73
 F src/shell.c.in c1b14bc0bc1fe73a97dfaad55ca09f5d6e44f6019977d94e7e3d1861616035e1
-F src/sqlite.h.in 51f9acf52c80113d793ddd38b3940ad6895d97b4752503b19291fb8fcbf54c5e
+F src/sqlite.h.in 235e942dd8c01b414c2996828b4d0d2500faf8850f8b24ae17d31c172f519e69
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d
 F src/sqliteInt.h fd8943e9d6b8bd04b3a22ef806b843c2c06f022db3f37f1436ac3a9b04e5b8a1
@@ -1707,8 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 38f654dc117bb11d61407327884b5c913cfaf920547bebc4ce5a4cf983e5c20e 255612f0a131f2f522cbca3cc5a1edcf7e38938abca25ba421e7e38a422db9c9
-R 508c3db0e2913d65a43c01bb9b80f5ab
-T +closed 255612f0a131f2f522cbca3cc5a1edcf7e38938abca25ba421e7e38a422db9c9
+P c883c4d33f4cd722982aca17eeb355d19eeed793991d923b5494d173b1d0c73a
+R 2031e9cc56213989aa9a3f7b2af81256
 U drh
-Z 1886f73e61ed30038df9856ac6df181f
+Z 242cc6a18e31e977a6ec01c768dc69ec
diff --git a/manifest.uuid b/manifest.uuid
index 25a24484fc..736ace92ca 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c883c4d33f4cd722982aca17eeb355d19eeed793991d923b5494d173b1d0c73a
\ No newline at end of file
+99eec556f065ad19548e48d1f4ae0a3767b4e82e1c83fa2365062e3c5e0071fb
\ No newline at end of file
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 1a0164150b..a4344935da 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -2496,16 +2496,16 @@ void sqlite3_free_table(char **result);
 **
 ** These routines are work-alikes of the "printf()" family of functions
 ** from the standard C library.
-** These routines understand most of the common K&R formatting options,
-** plus some additional non-standard formats, detailed below.
-** Note that some of the more obscure formatting options from recent
-** C-library standards are omitted from this implementation.
+** These routines understand most of the common formatting options from
+** the standard library printf() 
+** plus some additional non-standard formats ([%q], [%Q], [%w], and [%z]).
+** See the [built-in printf()] documentation for details.
 **
 ** ^The sqlite3_mprintf() and sqlite3_vmprintf() routines write their
-** results into memory obtained from [sqlite3_malloc()].
+** results into memory obtained from [sqlite3_malloc64()].
 ** The strings returned by these two routines should be
 ** released by [sqlite3_free()].  ^Both routines return a
-** NULL pointer if [sqlite3_malloc()] is unable to allocate enough
+** NULL pointer if [sqlite3_malloc64()] is unable to allocate enough
 ** memory to hold the resulting string.
 **
 ** ^(The sqlite3_snprintf() routine is similar to "snprintf()" from
@@ -2529,71 +2529,7 @@ void sqlite3_free_table(char **result);
 **
 ** ^The sqlite3_vsnprintf() routine is a varargs version of sqlite3_snprintf().
 **
-** These routines all implement some additional formatting
-** options that are useful for constructing SQL statements.
-** All of the usual printf() formatting options apply.  In addition, there
-** is are "%q", "%Q", "%w" and "%z" options.
-**
-** ^(The %q option works like %s in that it substitutes a nul-terminated
-** string from the argument list.  But %q also doubles every '\'' character.
-** %q is designed for use inside a string literal.)^  By doubling each '\''
-** character it escapes that character and allows it to be inserted into
-** the string.
-**
-** For example, assume the string variable zText contains text as follows:
-**
-** <blockquote><pre>
-**  char *zText = "It's a happy day!";
-** </pre></blockquote>
-**
-** One can use this text in an SQL statement as follows:
-**
-** <blockquote><pre>
-**  char *zSQL = sqlite3_mprintf("INSERT INTO table VALUES('%q')", zText);
-**  sqlite3_exec(db, zSQL, 0, 0, 0);
-**  sqlite3_free(zSQL);
-** </pre></blockquote>
-**
-** Because the %q format string is used, the '\'' character in zText
-** is escaped and the SQL generated is as follows:
-**
-** <blockquote><pre>
-**  INSERT INTO table1 VALUES('It''s a happy day!')
-** </pre></blockquote>
-**
-** This is correct.  Had we used %s instead of %q, the generated SQL
-** would have looked like this:
-**
-** <blockquote><pre>
-**  INSERT INTO table1 VALUES('It's a happy day!');
-** </pre></blockquote>
-**
-** This second example is an SQL syntax error.  As a general rule you should
-** always use %q instead of %s when inserting text into a string literal.
-**
-** ^(The %Q option works like %q except it also adds single quotes around
-** the outside of the total string.  Additionally, if the parameter in the
-** argument list is a NULL pointer, %Q substitutes the text "NULL" (without
-** single quotes).)^  So, for example, one could say:
-**
-** <blockquote><pre>
-**  char *zSQL = sqlite3_mprintf("INSERT INTO table VALUES(%Q)", zText);
-**  sqlite3_exec(db, zSQL, 0, 0, 0);
-**  sqlite3_free(zSQL);
-** </pre></blockquote>
-**
-** The code above will render a correct SQL statement in the zSQL
-** variable even if the zText variable is a NULL pointer.
-**
-** ^(The "%w" formatting option is like "%q" except that it expects to
-** be contained within double-quotes instead of single quotes, and it
-** escapes the double-quote character instead of the single-quote
-** character.)^  The "%w" formatting option is intended for safely inserting
-** table and column names into a constructed SQL statement.
-**
-** ^(The "%z" formatting option works like "%s" but with the
-** addition that after the string has been read and copied into
-** the result, [sqlite3_free()] is called on the input string.)^
+** See also:  [built-in printf()], [printf() SQL function]
 */
 char *sqlite3_mprintf(const char*,...);
 char *sqlite3_vmprintf(const char*, va_list);

From cc398969e058b006bf87e35700b366977fc8682c Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 20 Feb 2018 15:23:37 +0000
Subject: [PATCH 462/488] Optimize calls to sqlite3_mprintf("%z...") so that
 they attempt to append text onto the end of the existing memory allocation
 rather than reallocating and copying.

FossilOrigin-Name: 4bc8a48e644562f6e6192f4c6fc4a70f6bb59f8126ed6c6dc876bedf65d74cda
---
 Makefile.in   |  1 +
 Makefile.msc  |  2 +-
 main.mk       |  1 +
 manifest      | 18 +++++++++---------
 manifest.uuid |  2 +-
 src/printf.c  | 17 +++++++++++++++++
 6 files changed, 30 insertions(+), 11 deletions(-)

diff --git a/Makefile.in b/Makefile.in
index 7bc413e843..1766ca580e 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -591,6 +591,7 @@ SHELL_OPT += -DSQLITE_INTROSPECTION_PRAGMAS
 FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
 FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
+FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000
 FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c
 DBFUZZ_OPT = 
 
diff --git a/Makefile.msc b/Makefile.msc
index 875aeedeab..af053cd615 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -1615,7 +1615,7 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_RTREE
 #
 MPTESTER_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS5
 FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1
-FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -DSQLITE_MAX_MEMORY=50000000
+FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -DSQLITE_MAX_MEMORY=50000000 -DSQLITE_PRINTF_PRECISION_LIMIT=1000
 FUZZCHECK_SRC = $(TOP)\test\fuzzcheck.c $(TOP)\test\ossfuzz.c
 OSSSHELL_SRC = $(TOP)\test\ossshell.c $(TOP)\test\ossfuzz.c
 DBFUZZ_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION
diff --git a/main.mk b/main.mk
index 2eddfc6f27..c5a0aa6cd3 100644
--- a/main.mk
+++ b/main.mk
@@ -519,6 +519,7 @@ SHELL_OPT += -DSQLITE_INTROSPECTION_PRAGMAS
 FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5
 FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
+FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000
 DBFUZZ_OPT =
 KV_OPT = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ
 ST_OPT = -DSQLITE_THREADSAFE=0
diff --git a/manifest b/manifest
index 9788e54316..76a5ca6866 100644
--- a/manifest
+++ b/manifest
@@ -1,10 +1,10 @@
-C Remove\sa\slot\sof\sthe\stext\sdescribing\sextended\sformat\soptions\sfrom\sthe\ndocumentation\son\ssqlite3_mprintf()\sand\sfriends,\ssince\sthat\sinformation\sis\nnow\scovered\sby\sthe\sseparate\sprintf.html\sdocument.\s\sProvide\slinks\sto\sthat\nother\sdocument.\s\sNo\schanges\sto\scode.
-D 2018-02-20T13:46:20.902
+C Optimize\scalls\sto\ssqlite3_mprintf("%z...")\sso\sthat\sthey\sattempt\sto\sappend\ntext\sonto\sthe\send\sof\sthe\sexisting\smemory\sallocation\srather\sthan\sreallocating\nand\scopying.
+D 2018-02-20T15:23:37.534
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
-F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
+F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc b5e9eb7212c5a61d4800172c710376d1158fd5eea2edc0be8e05c0c372e767e4
+F Makefile.msc b53ebf2538324c914e4308a6cd44610eb1ae50bdbd17d7ee65f680918e54da88
 F README.md d748f58e3ab0fe0307fb4ae0942b415d93dcc4288756e366cc9e7cf8260c093f
 F VERSION cdf91ac446255ecf3d8f6d8c3ee40d64123235ae5b3cef29d344e61b45ec3759
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -411,7 +411,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk c8c473bd91d553acab3fb0608ddb69fc769c7bcf6d9e258800504bfda86c792b
+F main.mk 04969462bfd32c9f08d4a6d40622e8c43c8c5ecfb2ee52ffb5737c5eca4b0c03
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -484,7 +484,7 @@ F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
 F src/pragma.c bea56df3ae0637768c0da4fbbb8f2492f780980d95000034a105ff291bf7ca69
 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
 F src/prepare.c 259f4e7960c47082c9653f3d5f0c294abd68bb9c3aab86de7630700cba1c20fb
-F src/printf.c 07244a9e1755247c9543db551acd2f3cce461b28d13e9f3ab87053c35fd62717
+F src/printf.c d3b7844ddeb11fbbdd38dd84d09c9c1ac171d21fb038473c3aa97981201cc660
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
@@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c883c4d33f4cd722982aca17eeb355d19eeed793991d923b5494d173b1d0c73a
-R 2031e9cc56213989aa9a3f7b2af81256
+P 99eec556f065ad19548e48d1f4ae0a3767b4e82e1c83fa2365062e3c5e0071fb
+R 7c8988fb832914bacef8f6a6a302789a
 U drh
-Z 242cc6a18e31e977a6ec01c768dc69ec
+Z 0178b35e80c45b1a39d6c8cc991625e8
diff --git a/manifest.uuid b/manifest.uuid
index 736ace92ca..9279bc5dbb 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-99eec556f065ad19548e48d1f4ae0a3767b4e82e1c83fa2365062e3c5e0071fb
\ No newline at end of file
+4bc8a48e644562f6e6192f4c6fc4a70f6bb59f8126ed6c6dc876bedf65d74cda
\ No newline at end of file
diff --git a/src/printf.c b/src/printf.c
index 11d5b82828..fcbd7fdbe0 100644
--- a/src/printf.c
+++ b/src/printf.c
@@ -206,6 +206,11 @@ void sqlite3VXPrintf(
   PrintfArguments *pArgList = 0; /* Arguments for SQLITE_PRINTF_SQLFUNC */
   char buf[etBUFSIZE];       /* Conversion buffer */
 
+  /* pAccum never starts out with an empty buffer that was obtained from 
+  ** malloc().  This precondition is required by the mprintf("%z...")
+  ** optimization. */
+  assert( pAccum->nChar>0 || (pAccum->printfFlags&SQLITE_PRINTF_MALLOCED)==0 );
+
   bufpt = 0;
   if( (pAccum->printfFlags & SQLITE_PRINTF_SQLFUNC)!=0 ){
     pArgList = va_arg(ap, PrintfArguments*);
@@ -681,6 +686,18 @@ void sqlite3VXPrintf(
         if( bufpt==0 ){
           bufpt = "";
         }else if( xtype==etDYNSTRING ){
+          if( pAccum->nChar==0 && pAccum->mxAlloc && width==0 && precision<0 ){
+            /* Special optimization for sqlite3_mprintf("%z..."):
+            ** Extend an existing memory allocation rather than creating
+            ** a new one. */
+            assert( (pAccum->printfFlags&SQLITE_PRINTF_MALLOCED)==0 );
+            pAccum->zText = bufpt;
+            pAccum->nAlloc = sqlite3DbMallocSize(pAccum->db, bufpt);
+            pAccum->nChar = 0x7fffffff & (int)strlen(bufpt);
+            pAccum->printfFlags |= SQLITE_PRINTF_MALLOCED;
+            length = 0;
+            break;
+          }
           zExtra = bufpt;
         }
         if( precision>=0 ){

From 3993678d3d5b27dbeb3d9298c71cbda17f74d699 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 20 Feb 2018 16:39:11 +0000
Subject: [PATCH 463/488] Remove a superfluous "#if 1".  No logic changes.

FossilOrigin-Name: caaffd32001ff44c31f2d6dbe921f9d294f869c2c4a81e943fdf3f43f8d9313c
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/attach.c  |  2 --
 3 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index 76a5ca6866..26878a1bec 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Optimize\scalls\sto\ssqlite3_mprintf("%z...")\sso\sthat\sthey\sattempt\sto\sappend\ntext\sonto\sthe\send\sof\sthe\sexisting\smemory\sallocation\srather\sthan\sreallocating\nand\scopying.
-D 2018-02-20T15:23:37.534
+C Remove\sa\ssuperfluous\s"#if\s1".\s\sNo\slogic\schanges.
+D 2018-02-20T16:39:11.853
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737
@@ -425,7 +425,7 @@ F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786
 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a
 F src/alter.c cf7a8af45cb0ace672f47a1b29ab24092a9e8cd8d945a9974e3b5d925f548594
 F src/analyze.c 6b42e36a5dcc2703a771f2411bd5e99524bd62c7ecde209bb88dfb04c72f046e
-F src/attach.c e1c0d72dd0747da84aa1352f8729906b1d8d2a162ba7f6c475980be202b37c46
+F src/attach.c 79cb6b365d79b96b15dd429aa3e4c3b99335039e059baca8df28f1708fbff261
 F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
@@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 99eec556f065ad19548e48d1f4ae0a3767b4e82e1c83fa2365062e3c5e0071fb
-R 7c8988fb832914bacef8f6a6a302789a
+P 4bc8a48e644562f6e6192f4c6fc4a70f6bb59f8126ed6c6dc876bedf65d74cda
+R b395f52b369116e5af9544f274df7b1d
 U drh
-Z 0178b35e80c45b1a39d6c8cc991625e8
+Z 0f63b7ed04628440113fb474f826bbce
diff --git a/manifest.uuid b/manifest.uuid
index 9279bc5dbb..b912daee1d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4bc8a48e644562f6e6192f4c6fc4a70f6bb59f8126ed6c6dc876bedf65d74cda
\ No newline at end of file
+caaffd32001ff44c31f2d6dbe921f9d294f869c2c4a81e943fdf3f43f8d9313c
\ No newline at end of file
diff --git a/src/attach.c b/src/attach.c
index 67fc4afa82..f3d68553b6 100644
--- a/src/attach.c
+++ b/src/attach.c
@@ -504,7 +504,6 @@ int sqlite3FixSelect(
     if( sqlite3FixExpr(pFix, pSelect->pLimit) ){
       return 1;
     }
-#if 1
     if( pSelect->pWith ){
       int i;
       for(i=0; i<pSelect->pWith->nCte; i++){
@@ -513,7 +512,6 @@ int sqlite3FixSelect(
         }
       }
     }
-#endif
     pSelect = pSelect->pPrior;
   }
   return 0;

From 7436f1e491c7df4a438a1b9d73f32243362224a0 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 20 Feb 2018 17:02:31 +0000
Subject: [PATCH 464/488] In the speed-check.sh script, make WAL mode the
 default.  Add the --legacy option to do DELETE mode.  Continue to use DELETE
 mode for long-term performance graphs since WAL mode was not always
 available, but use WAL mode for day-to-day performance measurements.

FossilOrigin-Name: cd81613850c0458b4b5b2460b9b38ce7ff6e557f7add9615058bb96f5a344ccd
---
 manifest            | 12 ++++++------
 manifest.uuid       |  2 +-
 tool/speed-check.sh |  9 ++++++++-
 3 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 26878a1bec..d34c77f35d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\sa\ssuperfluous\s"#if\s1".\s\sNo\slogic\schanges.
-D 2018-02-20T16:39:11.853
+C In\sthe\sspeed-check.sh\sscript,\smake\sWAL\smode\sthe\sdefault.\s\sAdd\sthe\s--legacy\noption\sto\sdo\sDELETE\smode.\s\sContinue\sto\suse\sDELETE\smode\sfor\slong-term\sperformance\ngraphs\ssince\sWAL\smode\swas\snot\salways\savailable,\sbut\suse\sWAL\smode\sfor\s\nday-to-day\sperformance\smeasurements.
+D 2018-02-20T17:02:31.731
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737
@@ -1665,7 +1665,7 @@ F tool/showstat4.c 0682ebea7abf4d3657f53c4a243f2e7eab48eab344ed36a94bb75dcd19a5c
 F tool/showwal.c ad9d768f96ca6199ad3a8c9562d679680bd032dd01204ea3e5ea6fb931d81847
 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe
 F tool/spaceanal.tcl 4bfd19aad7eb3ce0372ef0255f58035e0bba4ff5e9acfd763a10c6fb365c8dec
-F tool/speed-check.sh c6239c71ba6951ac5d093bbea55aadf019246c8e7a0c079422e62312a6ab1247
+F tool/speed-check.sh 4ff9b095cf1a7643f0264e7fb7d23f0b12b7cce587a9de315877c378e90eeaf4
 F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355
 F tool/speedtest16.c ecb6542862151c3e6509bbc00509b234562ae81e
 F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
@@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 4bc8a48e644562f6e6192f4c6fc4a70f6bb59f8126ed6c6dc876bedf65d74cda
-R b395f52b369116e5af9544f274df7b1d
+P caaffd32001ff44c31f2d6dbe921f9d294f869c2c4a81e943fdf3f43f8d9313c
+R fbade0c89f4079ed24b0fd1c0a4dd6f0
 U drh
-Z 0f63b7ed04628440113fb474f826bbce
+Z 870fe7a9ea537077e2a70e18f17526cf
diff --git a/manifest.uuid b/manifest.uuid
index b912daee1d..d2359b7ccc 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-caaffd32001ff44c31f2d6dbe921f9d294f869c2c4a81e943fdf3f43f8d9313c
\ No newline at end of file
+cd81613850c0458b4b5b2460b9b38ce7ff6e557f7add9615058bb96f5a344ccd
\ No newline at end of file
diff --git a/tool/speed-check.sh b/tool/speed-check.sh
index cc31e6aa0a..fc05ac98ef 100644
--- a/tool/speed-check.sh
+++ b/tool/speed-check.sh
@@ -40,6 +40,7 @@ BASELINE="trunk"
 doExplain=0
 doCachegrind=1
 doVdbeProfile=0
+doWal=1
 while test "$1" != ""; do
   case $1 in
     --reprepare)
@@ -63,8 +64,11 @@ while test "$1" != ""; do
     --temp)
         SPEEDTEST_OPTS="$SPEEDTEST_OPTS --temp 6"
         ;;
+    --legacy)
+	doWal=0
+        ;;
     --wal)
-        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --journal wal"
+        doWal=1
         ;;
     --size)
         shift; SIZE=$1
@@ -134,6 +138,9 @@ while test "$1" != ""; do
   esac
   shift
 done
+if test $doWal -eq 1; then
+  SPEEDTEST_OPTS="$SPEEDTEST_OPTS --journal wal"
+fi
 SPEEDTEST_OPTS="$SPEEDTEST_OPTS --size $SIZE"
 echo "NAME           = $NAME" | tee summary-$NAME.txt
 echo "SPEEDTEST_OPTS = $SPEEDTEST_OPTS" | tee -a summary-$NAME.txt

From 2e178d7321265e21fcec226345267d57a61560ca Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 20 Feb 2018 22:20:57 +0000
Subject: [PATCH 465/488] Make the walIndexPage() routine about 3x faster by
 factoring out the seldom used reallocation logic into a separate subroutine.

FossilOrigin-Name: e2b107141cd97bd4ab240748a9ce43fc2ec950ea74610697a4a7a3d7a6441e6b
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/wal.c     | 16 +++++++++++++++-
 3 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index d34c77f35d..2255c10023 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\sthe\sspeed-check.sh\sscript,\smake\sWAL\smode\sthe\sdefault.\s\sAdd\sthe\s--legacy\noption\sto\sdo\sDELETE\smode.\s\sContinue\sto\suse\sDELETE\smode\sfor\slong-term\sperformance\ngraphs\ssince\sWAL\smode\swas\snot\salways\savailable,\sbut\suse\sWAL\smode\sfor\s\nday-to-day\sperformance\smeasurements.
-D 2018-02-20T17:02:31.731
+C Make\sthe\swalIndexPage()\sroutine\sabout\s3x\sfaster\sby\sfactoring\sout\sthe\sseldom\nused\sreallocation\slogic\sinto\sa\sseparate\ssubroutine.
+D 2018-02-20T22:20:57.397
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737
@@ -568,7 +568,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c 5a3f464edd64596f601683ed321d12e6fd93c5fb9afdfb3653d6ffd0fee9c48f
+F src/wal.c fff06582a9611cd472fe67ba2b37da1ba052b48a41e3ee60ad55e87faf724c75
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f
 F src/where.c 7cae47e813393d70c6d327fdf000fcb30f76b1b0b5a5b52ff6402e0c658de32c
@@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P caaffd32001ff44c31f2d6dbe921f9d294f869c2c4a81e943fdf3f43f8d9313c
-R fbade0c89f4079ed24b0fd1c0a4dd6f0
+P cd81613850c0458b4b5b2460b9b38ce7ff6e557f7add9615058bb96f5a344ccd
+R 1787f3579805a4f44923446da8d3d679
 U drh
-Z 870fe7a9ea537077e2a70e18f17526cf
+Z 192c58e6de32e2b3cf41a7ef956f2494
diff --git a/manifest.uuid b/manifest.uuid
index d2359b7ccc..83c33c8bfe 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-cd81613850c0458b4b5b2460b9b38ce7ff6e557f7add9615058bb96f5a344ccd
\ No newline at end of file
+e2b107141cd97bd4ab240748a9ce43fc2ec950ea74610697a4a7a3d7a6441e6b
\ No newline at end of file
diff --git a/src/wal.c b/src/wal.c
index 7b49f3dab4..99832dff3d 100644
--- a/src/wal.c
+++ b/src/wal.c
@@ -554,7 +554,11 @@ struct WalIterator {
 ** page and SQLITE_OK is returned. If an error (an OOM or VFS error) occurs,
 ** then an SQLite error code is returned and *ppPage is set to 0.
 */
-static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){
+static SQLITE_NOINLINE int walIndexPageRealloc(
+  Wal *pWal,               /* The WAL context */
+  int iPage,               /* The page we seek */
+  volatile u32 **ppPage    /* Write the page pointer here */
+){
   int rc = SQLITE_OK;
 
   /* Enlarge the pWal->apWiData[] array if required */
@@ -596,6 +600,16 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){
   assert( iPage==0 || *ppPage || rc!=SQLITE_OK );
   return rc;
 }
+static int walIndexPage(
+  Wal *pWal,               /* The WAL context */
+  int iPage,               /* The page we seek */
+  volatile u32 **ppPage    /* Write the page pointer here */
+){
+  if( pWal->nWiData<=iPage || (*ppPage = pWal->apWiData[iPage])==0 ){
+    return walIndexPageRealloc(pWal, iPage, ppPage);
+  }
+  return SQLITE_OK;
+}
 
 /*
 ** Return a pointer to the WalCkptInfo structure in the wal-index.

From 4f4e49a98b8d3a28384e4a14df23ae4b98e63868 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 21 Feb 2018 00:48:02 +0000
Subject: [PATCH 466/488] Remove an unnecessary memset().

FossilOrigin-Name: e7b7f5884a78630c6ba6a217c36247397e62f1928d4b480df929efcfc4a01893
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/os_unix.c |  2 --
 3 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index 2255c10023..671a63c223 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Make\sthe\swalIndexPage()\sroutine\sabout\s3x\sfaster\sby\sfactoring\sout\sthe\sseldom\nused\sreallocation\slogic\sinto\sa\sseparate\ssubroutine.
-D 2018-02-20T22:20:57.397
+C Remove\san\sunnecessary\smemset().
+D 2018-02-21T00:48:02.760
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737
@@ -472,7 +472,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104
 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
-F src/os_unix.c 9e02186f0dcde77d3ec27d4409664eb692c4e8ea0257e30c0b7f25969fa69fec
+F src/os_unix.c 928586cbde31890538854f1657463392d63b2d7794f577774573ae53ec283d7b
 F src/os_win.c eb03c6d52f893bcd7fdd4c6006674c13c1b5e49543fec98d605201af2997171c
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 0b6bd5442733b2e08d0673de6cdafe3e7ab0b5715e4844ac836ab346b1d9ed89
@@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P cd81613850c0458b4b5b2460b9b38ce7ff6e557f7add9615058bb96f5a344ccd
-R 1787f3579805a4f44923446da8d3d679
+P e2b107141cd97bd4ab240748a9ce43fc2ec950ea74610697a4a7a3d7a6441e6b
+R 59bb924e6fde113e9c73dbca193c7d8f
 U drh
-Z 192c58e6de32e2b3cf41a7ef956f2494
+Z 888a0e04ef92ada18bb625445f9ec36e
diff --git a/manifest.uuid b/manifest.uuid
index 83c33c8bfe..de2a013349 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e2b107141cd97bd4ab240748a9ce43fc2ec950ea74610697a4a7a3d7a6441e6b
\ No newline at end of file
+e7b7f5884a78630c6ba6a217c36247397e62f1928d4b480df929efcfc4a01893
\ No newline at end of file
diff --git a/src/os_unix.c b/src/os_unix.c
index cb2ae33d0a..5a1d396ce2 100644
--- a/src/os_unix.c
+++ b/src/os_unix.c
@@ -4180,12 +4180,10 @@ static int unixShmSystemLock(
 
   if( pShmNode->h>=0 ){
     /* Initialize the locking parameters */
-    memset(&f, 0, sizeof(f));
     f.l_type = lockType;
     f.l_whence = SEEK_SET;
     f.l_start = ofst;
     f.l_len = n;
-
     rc = osFcntl(pShmNode->h, F_SETLK, &f);
     rc = (rc!=(-1)) ? SQLITE_OK : SQLITE_BUSY;
   }

From 8d3e15eec45466b1c89117af5f8d8a0bb6216512 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 21 Feb 2018 01:05:37 +0000
Subject: [PATCH 467/488] Small performance improvement in
 sqltie3WalFindFrame().

FossilOrigin-Name: 52013cad0e6ce2d694f25e2984a76d438cef724f0e07c8bb9d3dce8a3caf4350
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/wal.c     |  3 ++-
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 671a63c223..1939578313 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\san\sunnecessary\smemset().
-D 2018-02-21T00:48:02.760
+C Small\sperformance\simprovement\sin\ssqltie3WalFindFrame().
+D 2018-02-21T01:05:37.077
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737
@@ -568,7 +568,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c fff06582a9611cd472fe67ba2b37da1ba052b48a41e3ee60ad55e87faf724c75
+F src/wal.c 634908f6bc2b5147c1a4218381393fc5f19cfc16bef2d9fa8029c2a7f31b73ae
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f
 F src/where.c 7cae47e813393d70c6d327fdf000fcb30f76b1b0b5a5b52ff6402e0c658de32c
@@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e2b107141cd97bd4ab240748a9ce43fc2ec950ea74610697a4a7a3d7a6441e6b
-R 59bb924e6fde113e9c73dbca193c7d8f
+P e7b7f5884a78630c6ba6a217c36247397e62f1928d4b480df929efcfc4a01893
+R cec4fbff2931f49a6ea05bd82279c171
 U drh
-Z 888a0e04ef92ada18bb625445f9ec36e
+Z ba266545389d418145bdf93b79a10d36
diff --git a/manifest.uuid b/manifest.uuid
index de2a013349..bd3bf9c056 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e7b7f5884a78630c6ba6a217c36247397e62f1928d4b480df929efcfc4a01893
\ No newline at end of file
+52013cad0e6ce2d694f25e2984a76d438cef724f0e07c8bb9d3dce8a3caf4350
\ No newline at end of file
diff --git a/src/wal.c b/src/wal.c
index 99832dff3d..72bb600f5c 100644
--- a/src/wal.c
+++ b/src/wal.c
@@ -2882,7 +2882,7 @@ int sqlite3WalFindFrame(
   **     table after the current read-transaction had started.
   */
   iMinHash = walFramePage(pWal->minFrame);
-  for(iHash=walFramePage(iLast); iHash>=iMinHash && iRead==0; iHash--){
+  for(iHash=walFramePage(iLast); iHash>=iMinHash; iHash--){
     volatile ht_slot *aHash;      /* Pointer to hash table */
     volatile u32 *aPgno;          /* Pointer to array of page numbers */
     u32 iZero;                    /* Frame number corresponding to aPgno[0] */
@@ -2905,6 +2905,7 @@ int sqlite3WalFindFrame(
         return SQLITE_CORRUPT_BKPT;
       }
     }
+    if( iRead ) break;
   }
 
 #ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT

From c0ec2f77627556ea88fe2b4f9ffaf74b33ad9091 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 21 Feb 2018 01:48:22 +0000
Subject: [PATCH 468/488] Remove an unreachable branch in the walIndexPage()
 logic.

FossilOrigin-Name: 4e61a9731f5949dcf05c541106b56be16add04a7730613481fc8519d3e4f493b
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/wal.c     | 29 ++++++++++++++---------------
 3 files changed, 21 insertions(+), 22 deletions(-)

diff --git a/manifest b/manifest
index 1939578313..80d8dc6b2b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Small\sperformance\simprovement\sin\ssqltie3WalFindFrame().
-D 2018-02-21T01:05:37.077
+C Remove\san\sunreachable\sbranch\sin\sthe\swalIndexPage()\slogic.
+D 2018-02-21T01:48:22.912
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737
@@ -568,7 +568,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c 634908f6bc2b5147c1a4218381393fc5f19cfc16bef2d9fa8029c2a7f31b73ae
+F src/wal.c 0ba9af2589b67c4bc60c773c7d2080491099037af21e73545f932bb29bf70b65
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f
 F src/where.c 7cae47e813393d70c6d327fdf000fcb30f76b1b0b5a5b52ff6402e0c658de32c
@@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e7b7f5884a78630c6ba6a217c36247397e62f1928d4b480df929efcfc4a01893
-R cec4fbff2931f49a6ea05bd82279c171
+P 52013cad0e6ce2d694f25e2984a76d438cef724f0e07c8bb9d3dce8a3caf4350
+R 85b71d55df961ca31201d9b641eb44c9
 U drh
-Z ba266545389d418145bdf93b79a10d36
+Z e9ca7c00b9c8d598dec87551eaa9cc3d
diff --git a/manifest.uuid b/manifest.uuid
index bd3bf9c056..9ccc6e494d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-52013cad0e6ce2d694f25e2984a76d438cef724f0e07c8bb9d3dce8a3caf4350
\ No newline at end of file
+4e61a9731f5949dcf05c541106b56be16add04a7730613481fc8519d3e4f493b
\ No newline at end of file
diff --git a/src/wal.c b/src/wal.c
index 72bb600f5c..d56946e309 100644
--- a/src/wal.c
+++ b/src/wal.c
@@ -577,21 +577,20 @@ static SQLITE_NOINLINE int walIndexPageRealloc(
   }
 
   /* Request a pointer to the required page from the VFS */
-  if( pWal->apWiData[iPage]==0 ){
-    if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){
-      pWal->apWiData[iPage] = (u32 volatile *)sqlite3MallocZero(WALINDEX_PGSZ);
-      if( !pWal->apWiData[iPage] ) rc = SQLITE_NOMEM_BKPT;
-    }else{
-      rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, 
-          pWal->writeLock, (void volatile **)&pWal->apWiData[iPage]
-      );
-      assert( pWal->apWiData[iPage]!=0 || rc!=SQLITE_OK || pWal->writeLock==0 );
-      testcase( pWal->apWiData[iPage]==0 && rc==SQLITE_OK );
-      if( (rc&0xff)==SQLITE_READONLY ){
-        pWal->readOnly |= WAL_SHM_RDONLY;
-        if( rc==SQLITE_READONLY ){
-          rc = SQLITE_OK;
-        }
+  assert( pWal->apWiData[iPage]==0 );
+  if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){
+    pWal->apWiData[iPage] = (u32 volatile *)sqlite3MallocZero(WALINDEX_PGSZ);
+    if( !pWal->apWiData[iPage] ) rc = SQLITE_NOMEM_BKPT;
+  }else{
+    rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, 
+        pWal->writeLock, (void volatile **)&pWal->apWiData[iPage]
+    );
+    assert( pWal->apWiData[iPage]!=0 || rc!=SQLITE_OK || pWal->writeLock==0 );
+    testcase( pWal->apWiData[iPage]==0 && rc==SQLITE_OK );
+    if( (rc&0xff)==SQLITE_READONLY ){
+      pWal->readOnly |= WAL_SHM_RDONLY;
+      if( rc==SQLITE_READONLY ){
+        rc = SQLITE_OK;
       }
     }
   }

From d6ec7a6ab3e9a8ea2df0cf3375a4e2334311a3eb Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 22 Feb 2018 18:28:19 +0000
Subject: [PATCH 469/488] Updates to the README.md file.  Mention the ability
 to use SQLite-archives for download.

FossilOrigin-Name: ab9a7ae4eec0aaa5e0b472998f6de7212055d93c163f0b66da60b956bcc45add
---
 README.md     | 16 +++++++++-------
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 3 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/README.md b/README.md
index e5fa2acb67..50954b6cde 100644
--- a/README.md
+++ b/README.md
@@ -14,15 +14,17 @@ SQLite sources are managed using the
 [Fossil](https://www.fossil-scm.org/), a distributed version control system
 that was specifically designed to support SQLite development.
 If you do not want to use Fossil, you can download tarballs or ZIP
-archives as follows:
+archives or [SQLite archives](https://sqlite.org/sqlar) as follows:
 
-  *  Lastest trunk check-in:
-     <https://www.sqlite.org/src/tarball/sqlite.tar.gz> or
-     <https://www.sqlite.org/src/zip/sqlite.zip>.
+  *  Lastest trunk check-in as
+     [Tarball](https://www.sqlite.org/src/tarball/sqlite.tar.gz),
+     [ZIP-archive](https://www.sqlite.org/src/zip/sqlite.zip), or
+     [SQLite-archive](https://www.sqlite.org/src/sqlar/sqlite.sqlar).
 
-  *  Latest release:
-     <https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=release> or
-     <https://www.sqlite.org/src/zip/sqlite.zip?r=release>.
+  *  Latest release as
+     [Tarball](https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=release),
+     [ZIP-archive](https://www.sqlite.org/src/zip/sqlite.zip?r=release), or
+     [SQLite-archive](https://www.sqlite.org/src/sqlar/sqlite.sqlar?r=release).
 
   *  For other check-ins, substitute an appropriate branch name or
      tag or hash prefix for "release" in the URLs of the previous
diff --git a/manifest b/manifest
index 80d8dc6b2b..e3dde34935 100644
--- a/manifest
+++ b/manifest
@@ -1,11 +1,11 @@
-C Remove\san\sunreachable\sbranch\sin\sthe\swalIndexPage()\slogic.
-D 2018-02-21T01:48:22.912
+C Updates\sto\sthe\sREADME.md\sfile.\s\sMention\sthe\sability\sto\suse\sSQLite-archives\sfor\ndownload.
+D 2018-02-22T18:28:19.974
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc b53ebf2538324c914e4308a6cd44610eb1ae50bdbd17d7ee65f680918e54da88
-F README.md d748f58e3ab0fe0307fb4ae0942b415d93dcc4288756e366cc9e7cf8260c093f
+F README.md 7c0991bd30eab5f377cd1508a619fc7ccd8014897da31e57f60d9675c0779abd
 F VERSION cdf91ac446255ecf3d8f6d8c3ee40d64123235ae5b3cef29d344e61b45ec3759
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2
@@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 52013cad0e6ce2d694f25e2984a76d438cef724f0e07c8bb9d3dce8a3caf4350
-R 85b71d55df961ca31201d9b641eb44c9
+P 4e61a9731f5949dcf05c541106b56be16add04a7730613481fc8519d3e4f493b
+R aa9874414cd1a4e6c530b39e5a39ec09
 U drh
-Z e9ca7c00b9c8d598dec87551eaa9cc3d
+Z 856f42b132f15d59c7f2cba2fefa2460
diff --git a/manifest.uuid b/manifest.uuid
index 9ccc6e494d..db728d9dc2 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4e61a9731f5949dcf05c541106b56be16add04a7730613481fc8519d3e4f493b
\ No newline at end of file
+ab9a7ae4eec0aaa5e0b472998f6de7212055d93c163f0b66da60b956bcc45add
\ No newline at end of file

From 4f46b58a2d9483b921132977c818c1b6951030a0 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 22 Feb 2018 18:30:53 +0000
Subject: [PATCH 470/488] For the "SQLite archive" link in the README.md file,
 point to the CLI .archive command documentation, not the original "sqlar"
 website.

FossilOrigin-Name: 4290faf5f7223ebe35b8ed5c4f0ba2444c1764501cabc09a79000f9c1d0890c7
---
 README.md     |  2 +-
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/README.md b/README.md
index 50954b6cde..a92e5ae24e 100644
--- a/README.md
+++ b/README.md
@@ -14,7 +14,7 @@ SQLite sources are managed using the
 [Fossil](https://www.fossil-scm.org/), a distributed version control system
 that was specifically designed to support SQLite development.
 If you do not want to use Fossil, you can download tarballs or ZIP
-archives or [SQLite archives](https://sqlite.org/sqlar) as follows:
+archives or [SQLite archives](https://sqlite.org/cli.html#sqlar) as follows:
 
   *  Lastest trunk check-in as
      [Tarball](https://www.sqlite.org/src/tarball/sqlite.tar.gz),
diff --git a/manifest b/manifest
index e3dde34935..f9ea1ef022 100644
--- a/manifest
+++ b/manifest
@@ -1,11 +1,11 @@
-C Updates\sto\sthe\sREADME.md\sfile.\s\sMention\sthe\sability\sto\suse\sSQLite-archives\sfor\ndownload.
-D 2018-02-22T18:28:19.974
+C For\sthe\s"SQLite\sarchive"\slink\sin\sthe\sREADME.md\sfile,\spoint\sto\sthe\nCLI\s.archive\scommand\sdocumentation,\snot\sthe\soriginal\s"sqlar"\swebsite.
+D 2018-02-22T18:30:53.011
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc b53ebf2538324c914e4308a6cd44610eb1ae50bdbd17d7ee65f680918e54da88
-F README.md 7c0991bd30eab5f377cd1508a619fc7ccd8014897da31e57f60d9675c0779abd
+F README.md 1d5342ebda97420f114283e604e5fe99b0da939d63b76d492eabbaae23488276
 F VERSION cdf91ac446255ecf3d8f6d8c3ee40d64123235ae5b3cef29d344e61b45ec3759
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2
@@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 4e61a9731f5949dcf05c541106b56be16add04a7730613481fc8519d3e4f493b
-R aa9874414cd1a4e6c530b39e5a39ec09
+P ab9a7ae4eec0aaa5e0b472998f6de7212055d93c163f0b66da60b956bcc45add
+R 3cb70dc7ae645278be920e32ff6e511f
 U drh
-Z 856f42b132f15d59c7f2cba2fefa2460
+Z c1c7fcd61cd44109ceb9fbe2b0c63c1e
diff --git a/manifest.uuid b/manifest.uuid
index db728d9dc2..4c38faf3fa 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ab9a7ae4eec0aaa5e0b472998f6de7212055d93c163f0b66da60b956bcc45add
\ No newline at end of file
+4290faf5f7223ebe35b8ed5c4f0ba2444c1764501cabc09a79000f9c1d0890c7
\ No newline at end of file

From fda0867873cc3cf5823416a10eb10c69da761af6 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Fri, 23 Feb 2018 13:38:20 +0000
Subject: [PATCH 471/488] Enable the JSON1 extension when compiling the test
 fixture with MSVC.

FossilOrigin-Name: e9e9f27b3d496531905ad7459cf60366ad3798f507266134dd6388cdec50fdc4
---
 Makefile.msc  |  1 +
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/Makefile.msc b/Makefile.msc
index af053cd615..1b2bcb55ac 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -2252,6 +2252,7 @@ TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_DEFAULT_PAGE_SIZE=1024
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB
+TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_JSON1
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) $(TEST_CCONV_OPTS)
 
 TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2)
diff --git a/manifest b/manifest
index f9ea1ef022..8c403db8d8 100644
--- a/manifest
+++ b/manifest
@@ -1,10 +1,10 @@
-C For\sthe\s"SQLite\sarchive"\slink\sin\sthe\sREADME.md\sfile,\spoint\sto\sthe\nCLI\s.archive\scommand\sdocumentation,\snot\sthe\soriginal\s"sqlar"\swebsite.
-D 2018-02-22T18:30:53.011
+C Enable\sthe\sJSON1\sextension\swhen\scompiling\sthe\stest\sfixture\swith\sMSVC.
+D 2018-02-23T13:38:20.020
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc b53ebf2538324c914e4308a6cd44610eb1ae50bdbd17d7ee65f680918e54da88
+F Makefile.msc bf19d3a0eb849bd3b114653b0e455aa5b2799a96f413287a5866013db0e47f30
 F README.md 1d5342ebda97420f114283e604e5fe99b0da939d63b76d492eabbaae23488276
 F VERSION cdf91ac446255ecf3d8f6d8c3ee40d64123235ae5b3cef29d344e61b45ec3759
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ab9a7ae4eec0aaa5e0b472998f6de7212055d93c163f0b66da60b956bcc45add
-R 3cb70dc7ae645278be920e32ff6e511f
-U drh
-Z c1c7fcd61cd44109ceb9fbe2b0c63c1e
+P 4290faf5f7223ebe35b8ed5c4f0ba2444c1764501cabc09a79000f9c1d0890c7
+R 3d31ca3626fb7de7ae6ca98028d90678
+U mistachkin
+Z 415294341d4b98b1263aa22d9cb61bd0
diff --git a/manifest.uuid b/manifest.uuid
index 4c38faf3fa..db4761886f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4290faf5f7223ebe35b8ed5c4f0ba2444c1764501cabc09a79000f9c1d0890c7
\ No newline at end of file
+e9e9f27b3d496531905ad7459cf60366ad3798f507266134dd6388cdec50fdc4
\ No newline at end of file

From 5facffbc70893cea166235f87ac3c2ae7ee6fbe1 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Fri, 23 Feb 2018 13:38:54 +0000
Subject: [PATCH 472/488] Fix harmless compiler warnings in the zipfile
 extension seen with MSVC.

FossilOrigin-Name: 15c587cf6ffc920dc37f67cacb4f0db6a603fb998a22a639f755387910516414
---
 ext/misc/zipfile.c | 14 +++++++-------
 manifest           | 12 ++++++------
 manifest.uuid      |  2 +-
 3 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c
index f88b7e22d6..a400ae30e1 100644
--- a/ext/misc/zipfile.c
+++ b/ext/misc/zipfile.c
@@ -737,7 +737,7 @@ static void zipfileMtimeToDos(ZipfileCDS *pCds, u32 mUnixTime){
   int hr, min, sec;
 
   A = (int)((JD - 1867216.25)/36524.25);
-  A = JD + 1 + A - (A/4);
+  A = (int)(JD + 1 + A - (A/4));
   B = A + 1524;
   C = (int)((B - 122.1)/365.25);
   D = (36525*(C&32767))/100;
@@ -1623,7 +1623,7 @@ static int zipfileUpdate(
         pNew->cds.szUncompressed = (u32)sz;
         pNew->cds.iExternalAttr = (mode<<16);
         pNew->cds.iOffset = (u32)pTab->szCurrent;
-        pNew->cds.nFile = nPath;
+        pNew->cds.nFile = (u16)nPath;
         pNew->mUnixTime = (u32)mTime;
         rc = zipfileAppendEntry(pTab, pNew, pData, nData);
         zipfileAddEntry(pTab, pOld, pNew);
@@ -1926,7 +1926,7 @@ void zipfileStep(sqlite3_context *pCtx, int nVal, sqlite3_value **apVal){
   /* Inspect the 'method' parameter. This must be either 0 (store), 8 (use
   ** deflate compression) or NULL (choose automatically).  */
   if( pMethod && SQLITE_NULL!=sqlite3_value_type(pMethod) ){
-    iMethod = sqlite3_value_int64(pMethod);
+    iMethod = (int)sqlite3_value_int64(pMethod);
     if( iMethod!=0 && iMethod!=8 ){
       zErr = sqlite3_mprintf("illegal method value: %d", iMethod);
       rc = SQLITE_ERROR;
@@ -1993,14 +1993,14 @@ void zipfileStep(sqlite3_context *pCtx, int nVal, sqlite3_value **apVal){
   e.cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY;
   e.cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED;
   e.cds.flags = ZIPFILE_NEWENTRY_FLAGS;
-  e.cds.iCompression = iMethod;
+  e.cds.iCompression = (u16)iMethod;
   zipfileMtimeToDos(&e.cds, (u32)e.mUnixTime);
   e.cds.crc32 = iCrc32;
   e.cds.szCompressed = nData;
   e.cds.szUncompressed = szUncompressed;
   e.cds.iExternalAttr = (mode<<16);
   e.cds.iOffset = p->body.n;
-  e.cds.nFile = nName;
+  e.cds.nFile = (u16)nName;
   e.cds.zFile = zName;
 
   /* Append the LFH to the body of the new archive */
@@ -2047,8 +2047,8 @@ void zipfileFinal(sqlite3_context *pCtx){
   if( p==0 ) return;
   if( p->nEntry>0 ){
     memset(&eocd, 0, sizeof(eocd));
-    eocd.nEntry = p->nEntry;
-    eocd.nEntryTotal = p->nEntry;
+    eocd.nEntry = (u16)p->nEntry;
+    eocd.nEntryTotal = (u16)p->nEntry;
     eocd.nSize = p->cds.n;
     eocd.iOffset = p->body.n;
 
diff --git a/manifest b/manifest
index 8c403db8d8..a1b9650113 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Enable\sthe\sJSON1\sextension\swhen\scompiling\sthe\stest\sfixture\swith\sMSVC.
-D 2018-02-23T13:38:20.020
+C Fix\sharmless\scompiler\swarnings\sin\sthe\szipfile\sextension\sseen\swith\sMSVC.
+D 2018-02-23T13:38:54.742
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737
@@ -304,7 +304,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178
 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c 3c3e21d0351a44c34c67ca7e833ba04e0de34421b4c9359aa764919ee36b37ef
+F ext/misc/zipfile.c bc88dbf9aa26c24595d158c1398208786ef34e7aee19ea29e8f0110c752fca03
 F ext/misc/zorder.c b0ff58fa643afa1d846786d51ea8d5c4b6b35aa0254ab5a82617db92f3adda64
 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e
 F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842
@@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 4290faf5f7223ebe35b8ed5c4f0ba2444c1764501cabc09a79000f9c1d0890c7
-R 3d31ca3626fb7de7ae6ca98028d90678
+P e9e9f27b3d496531905ad7459cf60366ad3798f507266134dd6388cdec50fdc4
+R 654cd747b1d6661b630741ec0eabab23
 U mistachkin
-Z 415294341d4b98b1263aa22d9cb61bd0
+Z f6be006b93932e4aec25faabbec16d8c
diff --git a/manifest.uuid b/manifest.uuid
index db4761886f..3218ef0aa6 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e9e9f27b3d496531905ad7459cf60366ad3798f507266134dd6388cdec50fdc4
\ No newline at end of file
+15c587cf6ffc920dc37f67cacb4f0db6a603fb998a22a639f755387910516414
\ No newline at end of file

From 007c843b0f9b832c21eac4274b361173544c9bb7 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 26 Feb 2018 03:20:18 +0000
Subject: [PATCH 473/488] Experimental implementation of IS TRUE and IS FALSE
 operators.  All TRUE and FALSE to act like constants if the names do not
 resolve to a column name.

FossilOrigin-Name: 40314bc999af08ab10e654241208842b4bb95b19858d11249444372250ea4160
---
 manifest            | 23 +++++++++++++----------
 manifest.uuid       |  2 +-
 src/expr.c          | 25 +++++++++++++++++++++++++
 src/resolve.c       | 36 +++++++++++++++++++++++++++++++-----
 src/vdbe.c          |  6 +++---
 tool/addopcodes.tcl |  3 +++
 6 files changed, 76 insertions(+), 19 deletions(-)

diff --git a/manifest b/manifest
index a1b9650113..3417c2a6b1 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sharmless\scompiler\swarnings\sin\sthe\szipfile\sextension\sseen\swith\sMSVC.
-D 2018-02-23T13:38:54.742
+C Experimental\simplementation\sof\sIS\sTRUE\sand\sIS\sFALSE\soperators.\s\sAll\sTRUE\sand\nFALSE\sto\sact\slike\sconstants\sif\sthe\snames\sdo\snot\sresolve\sto\sa\scolumn\sname.
+D 2018-02-26T03:20:18.624
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737
@@ -441,7 +441,7 @@ F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957
 F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
 F src/delete.c 20c8788451dc737a967c87ea53ad43544d617f5b57d32ccce8bd52a0daf9e89b
-F src/expr.c 9e06de431c09f144438aa6895ea4d4290fa3c6875bfcc3ba331012ca78deadf0
+F src/expr.c 56bfb9937e14c8daf3e94d23ee9c516f35573afcbf973dc2fae2d60405e18862
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331
 F src/func.c 94f42cba2cc1c34aeaa441022ba0170ec3fec4bba54db4e0ded085c6dc0fdc51
@@ -486,7 +486,7 @@ F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
 F src/prepare.c 259f4e7960c47082c9653f3d5f0c294abd68bb9c3aab86de7630700cba1c20fb
 F src/printf.c d3b7844ddeb11fbbdd38dd84d09c9c1ac171d21fb038473c3aa97981201cc660
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
-F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
+F src/resolve.c e3555b156d60407c23692c1e99c05cdf8257f1b7530684dce85247012e58bd8a
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c f02352ac5cbb6fad3804add825743b924cfb6c742ba2e8666d726828a9985d73
 F src/shell.c.in c1b14bc0bc1fe73a97dfaad55ca09f5d6e44f6019977d94e7e3d1861616035e1
@@ -557,7 +557,7 @@ F src/update.c a90a32ffc0100265b0693dbbdbe490756447af181f5ea2c138cce515b08c8795
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c d9eb0a6c4aae1b00a7369eadd7ca0bbe946cb4c953b6751aa20d357c2f482157
 F src/vacuum.c 762ee9bbf8733d87d8cd06f58d950e881982e416f8c767334a40ffd341b6bff5
-F src/vdbe.c 72f5a4b95be486170104779aeef000423acabf55ac0c661afdae889be72c8922
+F src/vdbe.c d221ab77e0997d410004d46cb373f3b848d994b64b439dc69db9d7d43b255397
 F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a
 F src/vdbeInt.h 8d7d07f13cb3c4cbca91e22ba4a1920e542dda7c5d9299920432a0b3d5b009f5
 F src/vdbeapi.c fea41171884a4de119f8b10ab514c788674eeeb7f27218bb6d008e1310bfd07f
@@ -1612,7 +1612,7 @@ F test/zipfilefault.test 050be76778d2ec83566a542800fc3967f65ce719671a819016ada8b
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d
 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91
-F tool/addopcodes.tcl 7181c041d495e3f26acc36d15c86923ed722285f9015f017f41a3efdb9a0dab4
+F tool/addopcodes.tcl 92514b0db662868f1ec803ea95d8f5cf31213ba022fdfd5b7d289518d2e1eecf
 F tool/build-all-msvc.bat c12328d06c45fec8baada5949e3d5af54bf8c887 x
 F tool/build-shell.sh 950f47c6174f1eea171319438b93ba67ff5bf367
 F tool/cg_anno.tcl f95b0006c52cf7f0496b506343415b6ee3cdcdd3 x
@@ -1707,7 +1707,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e9e9f27b3d496531905ad7459cf60366ad3798f507266134dd6388cdec50fdc4
-R 654cd747b1d6661b630741ec0eabab23
-U mistachkin
-Z f6be006b93932e4aec25faabbec16d8c
+P 15c587cf6ffc920dc37f67cacb4f0db6a603fb998a22a639f755387910516414
+R b02e960cb9fe764e3ce324741c78d420
+T *branch * is-true-operator
+T *sym-is-true-operator *
+T -sym-trunk *
+U drh
+Z 2f211764040206a38839d885e0d3c8ec
diff --git a/manifest.uuid b/manifest.uuid
index 3218ef0aa6..8ab9156881 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-15c587cf6ffc920dc37f67cacb4f0db6a603fb998a22a639f755387910516414
\ No newline at end of file
+40314bc999af08ab10e654241208842b4bb95b19858d11249444372250ea4160
\ No newline at end of file
diff --git a/src/expr.c b/src/expr.c
index 1b87734923..5079035ca1 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -3543,6 +3543,14 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){
       codeInteger(pParse, pExpr, 0, target);
       return target;
     }
+    case TK_TRUE: {
+      sqlite3VdbeAddOp2(v, OP_Integer, 1, target);
+      return target;
+    }
+    case TK_FALSE: {
+      sqlite3VdbeAddOp2(v, OP_Integer, 0, target);
+      return target;
+    }
 #ifndef SQLITE_OMIT_FLOATING_POINT
     case TK_FLOAT: {
       assert( !ExprHasProperty(pExpr, EP_IntValue) );
@@ -3698,6 +3706,13 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){
       sqlite3VdbeAddOp2(v, op, r1, inReg);
       break;
     }
+    case TK_ISTRUE: {
+      r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, &regFree1);
+      testcase( regFree1==0 );
+      sqlite3VdbeAddOp2(v, OP_Not, r1, inReg);
+      sqlite3VdbeAddOp2(v, OP_Not, inReg, inReg);
+      break;
+    }
     case TK_ISNULL:
     case TK_NOTNULL: {
       int addr;
@@ -4473,6 +4488,11 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){
       sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull);
       break;
     }
+    case TK_ISTRUE: {
+      testcase( jumpIfNull==0 );
+      sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull);
+      break;
+    }
     case TK_IS:
     case TK_ISNOT:
       testcase( op==TK_IS );
@@ -4627,6 +4647,11 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){
       sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull);
       break;
     }
+    case TK_ISTRUE: {
+      testcase( jumpIfNull==0 );
+      sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull);
+      break;
+    }
     case TK_IS:
     case TK_ISNOT:
       testcase( pExpr->op==TK_IS );
diff --git a/src/resolve.c b/src/resolve.c
index f735fffa02..fb00788dd0 100644
--- a/src/resolve.c
+++ b/src/resolve.c
@@ -431,10 +431,22 @@ static int lookupName(
   ** Because no reference was made to outer contexts, the pNC->nRef
   ** fields are not changed in any context.
   */
-  if( cnt==0 && zTab==0 && ExprHasProperty(pExpr,EP_DblQuoted) ){
-    pExpr->op = TK_STRING;
-    pExpr->pTab = 0;
-    return WRC_Prune;
+  if( cnt==0 && zTab==0 ){
+    if( ExprHasProperty(pExpr,EP_DblQuoted) ){
+      pExpr->op = TK_STRING;
+      pExpr->pTab = 0;
+      return WRC_Prune;
+    }
+    if( sqlite3StrICmp(zCol, "true")==0 ){
+      pExpr->op = TK_TRUE;
+      pExpr->pTab = 0;
+      return WRC_Prune;
+    }
+    if( sqlite3StrICmp(zCol, "false")==0 ){
+      pExpr->op = TK_FALSE;
+      pExpr->pTab = 0;
+      return WRC_Prune;
+    }
   }
 
   /*
@@ -783,6 +795,21 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){
       notValid(pParse, pNC, "parameters", NC_IsCheck|NC_PartIdx|NC_IdxExpr);
       break;
     }
+    case TK_IS:
+      /* Handle special cases of "x IS TRUE" and "x IS FALSE".  The first
+      ** is transformed into "+x" and the second into "NOT x". */
+      if( pExpr->pRight->op==TK_ID ){
+        int rc = resolveExprStep(pWalker, pExpr->pRight);
+        if( rc==WRC_Abort ) return WRC_Abort;
+        if( pExpr->pRight->op==TK_TRUE ){
+          pExpr->op = TK_ISTRUE;
+          return WRC_Continue;
+        }else if( pExpr->pRight->op==TK_FALSE ){
+          pExpr->op = TK_NOT;
+          return WRC_Continue;
+        }
+      }
+      /* Fall thru */
     case TK_BETWEEN:
     case TK_EQ:
     case TK_NE:
@@ -790,7 +817,6 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){
     case TK_LE:
     case TK_GT:
     case TK_GE:
-    case TK_IS:
     case TK_ISNOT: {
       int nLeft, nRight;
       if( pParse->db->mallocFailed ) break;
diff --git a/src/vdbe.c b/src/vdbe.c
index ebf221d02f..7115c8cf7d 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -2211,10 +2211,10 @@ case OP_Or: {             /* same as TK_OR, in1, in2, out3 */
 case OP_Not: {                /* same as TK_NOT, in1, out2 */
   pIn1 = &aMem[pOp->p1];
   pOut = &aMem[pOp->p2];
-  sqlite3VdbeMemSetNull(pOut);
   if( (pIn1->flags & MEM_Null)==0 ){
-    pOut->flags = MEM_Int;
-    pOut->u.i = !sqlite3VdbeIntValue(pIn1);
+    sqlite3VdbeMemSetInt64(pOut, !sqlite3VdbeIntValue(pIn1));
+  }else{
+    sqlite3VdbeMemSetNull(pOut);
   }
   break;
 }
diff --git a/tool/addopcodes.tcl b/tool/addopcodes.tcl
index dfd7fac88c..7059d71363 100644
--- a/tool/addopcodes.tcl
+++ b/tool/addopcodes.tcl
@@ -22,6 +22,8 @@ close $in
 # ILLEGAL *must* be the last two token codes and they must be in that order.
 #
 set extras {
+  TRUE
+  FALSE
   ISNOT
   FUNCTION
   COLUMN
@@ -29,6 +31,7 @@ set extras {
   AGG_COLUMN
   UMINUS
   UPLUS
+  ISTRUE
   REGISTER
   VECTOR
   SELECT_COLUMN

From 1fcfa7240f9ab22d11f12d92378c64e89fbf36fb Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 26 Feb 2018 15:27:31 +0000
Subject: [PATCH 474/488] Always interpret non-zero floating-point values as
 true even if their integer part is zero.  Fix for ticket
 [36fae083b450e3af857a459e20].

FossilOrigin-Name: a983fa857048c748112c2f805f13aadeae43b43b3d27fa55433a3699b51c9a0e
---
 manifest       | 20 ++++++++++----------
 manifest.uuid  |  2 +-
 src/vdbe.c     | 41 +++++++++++++----------------------------
 src/vdbeInt.h  |  1 +
 src/vdbemem.c  | 10 ++++++++++
 test/expr.test | 22 ++++++++++++++++++++++
 6 files changed, 57 insertions(+), 39 deletions(-)

diff --git a/manifest b/manifest
index a1b9650113..007a40eb4c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sharmless\scompiler\swarnings\sin\sthe\szipfile\sextension\sseen\swith\sMSVC.
-D 2018-02-23T13:38:54.742
+C Always\sinterpret\snon-zero\sfloating-point\svalues\sas\strue\seven\sif\stheir\ninteger\spart\sis\szero.\s\sFix\sfor\sticket\s[36fae083b450e3af857a459e20].
+D 2018-02-26T15:27:31.204
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737
@@ -557,13 +557,13 @@ F src/update.c a90a32ffc0100265b0693dbbdbe490756447af181f5ea2c138cce515b08c8795
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c d9eb0a6c4aae1b00a7369eadd7ca0bbe946cb4c953b6751aa20d357c2f482157
 F src/vacuum.c 762ee9bbf8733d87d8cd06f58d950e881982e416f8c767334a40ffd341b6bff5
-F src/vdbe.c 72f5a4b95be486170104779aeef000423acabf55ac0c661afdae889be72c8922
+F src/vdbe.c bf51a88d09f3a3f22fec7e81fc5a1a6b72a7bc9a1863c8ab48d4983916e25fbd
 F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a
-F src/vdbeInt.h 8d7d07f13cb3c4cbca91e22ba4a1920e542dda7c5d9299920432a0b3d5b009f5
+F src/vdbeInt.h 95f7adfdc5c8f1353321f55a6c5ec00a90877e3b85af5159e393afb41ff54110
 F src/vdbeapi.c fea41171884a4de119f8b10ab514c788674eeeb7f27218bb6d008e1310bfd07f
 F src/vdbeaux.c 2756ac68ac259c416554100598fc291870063288cd7e1af22847f57b3e130e56
 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191
-F src/vdbemem.c 21c1d00f0b901dda0a2eccf56bf0d86648cae1bde37de0606ae4e7cdb0ff6d3d
+F src/vdbemem.c 414e28d3a7e2a8bee2bb247de115dcbc68e3cbac284d5862d077002f7a93bce1
 F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f
 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c
 F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a
@@ -792,7 +792,7 @@ F test/exclusive.test 1206b87e192497d78c7f35552e86a9d05421498da300fb1cce5ca5351c
 F test/exclusive2.test 984090e8e9d1b331d2e8111daf6e5d61dda0bef7
 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7
 F test/exists.test 79a75323c78f02bbe9c251ea502a092f9ef63dac
-F test/expr.test 66a2c9ac34f74f036faa4092f5402c7d3162fc93
+F test/expr.test 4d34609c823fb0b856a6f71ffd75a2d03e7ccc366f8d182ae901b4467abb9caa
 F test/extension01.test 00d13cec817f331a687a243e0e5a2d87b0e358c9
 F test/extraquick.test cb254400bd42bfb777ff675356aabf3287978f79
 F test/fallocate.test 07416bd593a116d5893cb244f45a94d5c6fe030561df3bd972e6135f8106e509
@@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e9e9f27b3d496531905ad7459cf60366ad3798f507266134dd6388cdec50fdc4
-R 654cd747b1d6661b630741ec0eabab23
-U mistachkin
-Z f6be006b93932e4aec25faabbec16d8c
+P 15c587cf6ffc920dc37f67cacb4f0db6a603fb998a22a639f755387910516414
+R ff7cf472c64b844ed90baa6da5e6de1d
+U drh
+Z 4ad98123770e35c50780573ab58f9c3a
diff --git a/manifest.uuid b/manifest.uuid
index 3218ef0aa6..48e20093ff 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-15c587cf6ffc920dc37f67cacb4f0db6a603fb998a22a639f755387910516414
\ No newline at end of file
+a983fa857048c748112c2f805f13aadeae43b43b3d27fa55433a3699b51c9a0e
\ No newline at end of file
diff --git a/src/vdbe.c b/src/vdbe.c
index ebf221d02f..b4c96b498f 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -2172,18 +2172,8 @@ case OP_Or: {             /* same as TK_OR, in1, in2, out3 */
   int v1;    /* Left operand:  0==FALSE, 1==TRUE, 2==UNKNOWN or NULL */
   int v2;    /* Right operand: 0==FALSE, 1==TRUE, 2==UNKNOWN or NULL */
 
-  pIn1 = &aMem[pOp->p1];
-  if( pIn1->flags & MEM_Null ){
-    v1 = 2;
-  }else{
-    v1 = sqlite3VdbeIntValue(pIn1)!=0;
-  }
-  pIn2 = &aMem[pOp->p2];
-  if( pIn2->flags & MEM_Null ){
-    v2 = 2;
-  }else{
-    v2 = sqlite3VdbeIntValue(pIn2)!=0;
-  }
+  v1 = sqlite3VdbeBooleanValue(&aMem[pOp->p1], 2);
+  v2 = sqlite3VdbeBooleanValue(&aMem[pOp->p2], 2);
   if( pOp->opcode==OP_And ){
     static const unsigned char and_logic[] = { 0, 0, 0, 0, 1, 2, 0, 2, 2 };
     v1 = and_logic[v1*3+v2];
@@ -2214,7 +2204,7 @@ case OP_Not: {                /* same as TK_NOT, in1, out2 */
   sqlite3VdbeMemSetNull(pOut);
   if( (pIn1->flags & MEM_Null)==0 ){
     pOut->flags = MEM_Int;
-    pOut->u.i = !sqlite3VdbeIntValue(pIn1);
+    pOut->u.i = !sqlite3VdbeBooleanValue(pIn1, 0);
   }
   break;
 }
@@ -2281,30 +2271,25 @@ case OP_Once: {             /* jump */
 ** is considered true if it is numeric and non-zero.  If the value
 ** in P1 is NULL then take the jump if and only if P3 is non-zero.
 */
+case OP_If:  {               /* jump, in1 */
+  int c;
+  c = sqlite3VdbeBooleanValue(&aMem[pOp->p1], pOp->p3);
+  VdbeBranchTaken(c!=0, 2);
+  if( c ) goto jump_to_p2;
+  break;
+}
+
 /* Opcode: IfNot P1 P2 P3 * *
 **
 ** Jump to P2 if the value in register P1 is False.  The value
 ** is considered false if it has a numeric value of zero.  If the value
 ** in P1 is NULL then take the jump if and only if P3 is non-zero.
 */
-case OP_If:                 /* jump, in1 */
 case OP_IfNot: {            /* jump, in1 */
   int c;
-  pIn1 = &aMem[pOp->p1];
-  if( pIn1->flags & MEM_Null ){
-    c = pOp->p3;
-  }else{
-    if( pIn1->flags & MEM_Int ){
-      c = pIn1->u.i!=0;
-    }else{
-      c = sqlite3VdbeRealValue(pIn1)!=0.0;
-    }
-    if( pOp->opcode==OP_IfNot ) c = !c;
-  }
+  c = !sqlite3VdbeBooleanValue(&aMem[pOp->p1], !pOp->p3);
   VdbeBranchTaken(c!=0, 2);
-  if( c ){
-    goto jump_to_p2;
-  }
+  if( c ) goto jump_to_p2;
   break;
 }
 
diff --git a/src/vdbeInt.h b/src/vdbeInt.h
index 1e4f615ba2..44f901abf7 100644
--- a/src/vdbeInt.h
+++ b/src/vdbeInt.h
@@ -486,6 +486,7 @@ int sqlite3VdbeMemStringify(Mem*, u8, u8);
 i64 sqlite3VdbeIntValue(Mem*);
 int sqlite3VdbeMemIntegerify(Mem*);
 double sqlite3VdbeRealValue(Mem*);
+int sqlite3VdbeBooleanValue(Mem*, int ifNull);
 void sqlite3VdbeIntegerAffinity(Mem*);
 int sqlite3VdbeMemRealify(Mem*);
 int sqlite3VdbeMemNumerify(Mem*);
diff --git a/src/vdbemem.c b/src/vdbemem.c
index c02370c738..8df3c0d53b 100644
--- a/src/vdbemem.c
+++ b/src/vdbemem.c
@@ -572,6 +572,16 @@ double sqlite3VdbeRealValue(Mem *pMem){
   }
 }
 
+/*
+** Return 1 if pMem represents true, and return 0 if pMem represents false.
+** Return the value ifNull if pMem is NULL.  
+*/
+int sqlite3VdbeBooleanValue(Mem *pMem, int ifNull){
+  if( pMem->flags & MEM_Int ) return pMem->u.i!=0;
+  if( pMem->flags & MEM_Null ) return ifNull;
+  return sqlite3VdbeRealValue(pMem)!=0.0;
+}
+
 /*
 ** The MEM structure is already a MEM_Real.  Try to also make it a
 ** MEM_Int if we can.
diff --git a/test/expr.test b/test/expr.test
index 7a6d477259..edef3486af 100644
--- a/test/expr.test
+++ b/test/expr.test
@@ -977,6 +977,28 @@ do_execsql_test expr-13.9 {
   SELECT '' <= "";
 } {1}
 
+# 2018-02-26. Ticket https://www.sqlite.org/src/tktview/36fae083b450e3af85
+# 
+do_execsql_test expr-14.1 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1(x);
+  INSERT INTO t1 VALUES(0),(1),(NULL),(0.5),('1x'),('0x');
+  SELECT count(*) FROM t1
+   WHERE (x OR (8==9)) != (CASE WHEN x THEN 1 ELSE 0 END);
+} {0}
+do_execsql_test expr-14.2 {
+  SELECT count(*) FROM t1
+   WHERE (x OR (8==9)) != (NOT NOT x);
+} {0}
+do_execsql_test expr-14.3 {
+  SELECT sum(NOT x) FROM t1
+   WHERE x
+} {0}
+do_execsql_test expr-14.4 {
+  SELECT sum(CASE WHEN x THEN 0 ELSE 1 END) FROM t1
+   WHERE x
+} {0}
+
 
 
 finish_test

From f7e867c347d274f3feb08452afa4053048c7cb04 Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Mon, 26 Feb 2018 16:49:20 +0000
Subject: [PATCH 475/488] Fix heap issue in the shell on Win32 caused by
 allocator mismatch.

FossilOrigin-Name: 7b3a75519aa72a6b49a125ee84946ba2c0722dd6644ee0c80ba567eb85daf5c9
---
 manifest       | 14 +++++++-------
 manifest.uuid  |  2 +-
 src/shell.c.in |  4 ++--
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index 007a40eb4c..b60224db6e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Always\sinterpret\snon-zero\sfloating-point\svalues\sas\strue\seven\sif\stheir\ninteger\spart\sis\szero.\s\sFix\sfor\sticket\s[36fae083b450e3af857a459e20].
-D 2018-02-26T15:27:31.204
+C Fix\sheap\sissue\sin\sthe\sshell\son\sWin32\scaused\sby\sallocator\smismatch.
+D 2018-02-26T16:49:20.522
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737
@@ -489,7 +489,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c f02352ac5cbb6fad3804add825743b924cfb6c742ba2e8666d726828a9985d73
-F src/shell.c.in c1b14bc0bc1fe73a97dfaad55ca09f5d6e44f6019977d94e7e3d1861616035e1
+F src/shell.c.in 47dac081e15544f4d4373a837215391459971b312c8bed15903e7e9045c3e652
 F src/sqlite.h.in 235e942dd8c01b414c2996828b4d0d2500faf8850f8b24ae17d31c172f519e69
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d
@@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 15c587cf6ffc920dc37f67cacb4f0db6a603fb998a22a639f755387910516414
-R ff7cf472c64b844ed90baa6da5e6de1d
-U drh
-Z 4ad98123770e35c50780573ab58f9c3a
+P a983fa857048c748112c2f805f13aadeae43b43b3d27fa55433a3699b51c9a0e
+R 1433684d91ba65f3b7614e9fa9c71d0a
+U mistachkin
+Z b329981237c7dadda32eba65e163ae21
diff --git a/manifest.uuid b/manifest.uuid
index 48e20093ff..f5717e276e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a983fa857048c748112c2f805f13aadeae43b43b3d27fa55433a3699b51c9a0e
\ No newline at end of file
+7b3a75519aa72a6b49a125ee84946ba2c0722dd6644ee0c80ba567eb85daf5c9
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index c074b5ca7b..121a6c3506 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -8490,8 +8490,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
   data.doXdgOpen = 0;
   clearTempFile(&data);
 #if !SQLITE_SHELL_IS_UTF8
-  for(i=0; i<argc; i++) sqlite3_free(argv[i]);
-  sqlite3_free(argv);
+  for(i=0; i<argc; i++) free(argv[i]);
+  free(argv);
 #endif
   return rc;
 }

From 6bd4140d33c3d8daba8688837207e8a4dd19af0c Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Mon, 26 Feb 2018 17:03:03 +0000
Subject: [PATCH 476/488] More tests pursuant to ticket
 [36fae083b450e3af857a459e20].

FossilOrigin-Name: c26f236da61aef7c7a8fe38f1c7f33eea894c8fae38de6a085d4cce933a70502
---
 manifest       | 12 ++++++------
 manifest.uuid  |  2 +-
 test/expr.test | 35 +++++++++++++++++++++++++++++++++++
 3 files changed, 42 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index b60224db6e..983bc3f72d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sheap\sissue\sin\sthe\sshell\son\sWin32\scaused\sby\sallocator\smismatch.
-D 2018-02-26T16:49:20.522
+C More\stests\spursuant\sto\sticket\s[36fae083b450e3af857a459e20].
+D 2018-02-26T17:03:03.328
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737
@@ -792,7 +792,7 @@ F test/exclusive.test 1206b87e192497d78c7f35552e86a9d05421498da300fb1cce5ca5351c
 F test/exclusive2.test 984090e8e9d1b331d2e8111daf6e5d61dda0bef7
 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7
 F test/exists.test 79a75323c78f02bbe9c251ea502a092f9ef63dac
-F test/expr.test 4d34609c823fb0b856a6f71ffd75a2d03e7ccc366f8d182ae901b4467abb9caa
+F test/expr.test 7cb55e80aeb41d65fec968c08212505123063fea60bdc355d764d747670e9eea
 F test/extension01.test 00d13cec817f331a687a243e0e5a2d87b0e358c9
 F test/extraquick.test cb254400bd42bfb777ff675356aabf3287978f79
 F test/fallocate.test 07416bd593a116d5893cb244f45a94d5c6fe030561df3bd972e6135f8106e509
@@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a983fa857048c748112c2f805f13aadeae43b43b3d27fa55433a3699b51c9a0e
-R 1433684d91ba65f3b7614e9fa9c71d0a
+P 7b3a75519aa72a6b49a125ee84946ba2c0722dd6644ee0c80ba567eb85daf5c9
+R 54e3679053cfb34f03d5fc5a51c25e95
 U mistachkin
-Z b329981237c7dadda32eba65e163ae21
+Z 67d82893997d32bf8b3a9fa69332f798
diff --git a/manifest.uuid b/manifest.uuid
index f5717e276e..5e91c27a63 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7b3a75519aa72a6b49a125ee84946ba2c0722dd6644ee0c80ba567eb85daf5c9
\ No newline at end of file
+c26f236da61aef7c7a8fe38f1c7f33eea894c8fae38de6a085d4cce933a70502
\ No newline at end of file
diff --git a/test/expr.test b/test/expr.test
index edef3486af..3cdc9180e8 100644
--- a/test/expr.test
+++ b/test/expr.test
@@ -1000,5 +1000,40 @@ do_execsql_test expr-14.4 {
 } {0}
 
 
+foreach {tn val} [list 1 NaN 2 -NaN 3 NaN0 4 -NaN0 5 Inf 6 -Inf] {
+  do_execsql_test expr-15.$tn.1 {
+    DROP TABLE IF EXISTS t1;
+    CREATE TABLE t1(x);
+    INSERT INTO t1 VALUES(0),(1),(NULL),(0.5),('1x'),('0x');
+  }
+
+  do_test expr-15.$tn.2 {
+    set ::STMT [sqlite3_prepare db "INSERT INTO t1 VALUES(?)" -1 TAIL]
+    sqlite3_bind_double $::STMT 1 $val
+    sqlite3_step $::STMT
+    sqlite3_reset $::STMT
+    sqlite3_finalize $::STMT
+  } {SQLITE_OK}
+
+  do_execsql_test expr-15.$tn.3 {
+    SELECT count(*) FROM t1
+     WHERE (x OR (8==9)) != (CASE WHEN x THEN 1 ELSE 0 END);
+  } {0}
+
+  do_execsql_test expr-15.$tn.4 {
+    SELECT count(*) FROM t1
+     WHERE (x OR (8==9)) != (NOT NOT x);
+  } {0}
+
+  do_execsql_test expr-15.$tn.5 {
+    SELECT sum(NOT x) FROM t1
+     WHERE x
+  } {0}
+
+  do_execsql_test expr-15.$tn.6 {
+    SELECT sum(CASE WHEN x THEN 0 ELSE 1 END) FROM t1
+     WHERE x
+  } {0}
+}
 
 finish_test

From 8abed7b90769a6f63ef2681cfd4ae92d617b883f Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 26 Feb 2018 18:49:05 +0000
Subject: [PATCH 477/488] Refactor for correct NULL handling in the IS TRUE, IS
 FALSE, IS NOT TRUE, and IS NOT FALSE operators.

FossilOrigin-Name: cf2abd59be9971a55bd3d6c5df374c6aaa23bf81819482b42f01ee2484dcd739
---
 manifest            | 21 ++++++------
 manifest.uuid       |  2 +-
 src/expr.c          | 53 +++++++++++++++++++++--------
 src/resolve.c       | 34 +++++++++----------
 src/sqliteInt.h     |  3 +-
 src/vdbe.c          | 28 ++++++++++++++++
 test/istrue.test    | 81 +++++++++++++++++++++++++++++++++++++++++++++
 tool/addopcodes.tcl |  5 ++-
 8 files changed, 181 insertions(+), 46 deletions(-)
 create mode 100644 test/istrue.test

diff --git a/manifest b/manifest
index afbeea2798..d23a52f63f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\sthe\sfix\sfor\sdetermining\struth\sof\sfloating\spoint\svalues\sfrom\strunk.
-D 2018-02-26T15:31:39.919
+C Refactor\sfor\scorrect\sNULL\shandling\sin\sthe\sIS\sTRUE,\sIS\sFALSE,\sIS\sNOT\sTRUE,\nand\sIS\sNOT\sFALSE\soperators.
+D 2018-02-26T18:49:05.668
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737
@@ -441,7 +441,7 @@ F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957
 F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
 F src/delete.c 20c8788451dc737a967c87ea53ad43544d617f5b57d32ccce8bd52a0daf9e89b
-F src/expr.c 56bfb9937e14c8daf3e94d23ee9c516f35573afcbf973dc2fae2d60405e18862
+F src/expr.c 4344e70a158c0092e0c14c564d352546259297996cba250f1d6d9dbf830a415d
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331
 F src/func.c 94f42cba2cc1c34aeaa441022ba0170ec3fec4bba54db4e0ded085c6dc0fdc51
@@ -486,14 +486,14 @@ F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
 F src/prepare.c 259f4e7960c47082c9653f3d5f0c294abd68bb9c3aab86de7630700cba1c20fb
 F src/printf.c d3b7844ddeb11fbbdd38dd84d09c9c1ac171d21fb038473c3aa97981201cc660
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
-F src/resolve.c e3555b156d60407c23692c1e99c05cdf8257f1b7530684dce85247012e58bd8a
+F src/resolve.c 7e6a9702fe3bbf556b1105497ba7a51c24bf606bcc2a7c329122b31e04149005
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c f02352ac5cbb6fad3804add825743b924cfb6c742ba2e8666d726828a9985d73
 F src/shell.c.in c1b14bc0bc1fe73a97dfaad55ca09f5d6e44f6019977d94e7e3d1861616035e1
 F src/sqlite.h.in 235e942dd8c01b414c2996828b4d0d2500faf8850f8b24ae17d31c172f519e69
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d
-F src/sqliteInt.h fd8943e9d6b8bd04b3a22ef806b843c2c06f022db3f37f1436ac3a9b04e5b8a1
+F src/sqliteInt.h d38064626689d9a832f17cfc0feef250e34a9f6f2589805e8cef5e85c38c18fa
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -557,7 +557,7 @@ F src/update.c a90a32ffc0100265b0693dbbdbe490756447af181f5ea2c138cce515b08c8795
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c d9eb0a6c4aae1b00a7369eadd7ca0bbe946cb4c953b6751aa20d357c2f482157
 F src/vacuum.c 762ee9bbf8733d87d8cd06f58d950e881982e416f8c767334a40ffd341b6bff5
-F src/vdbe.c 2d1338273ff6d6430c0d7b88afbd8f4683df0d23980ab1e638a6b490e9766e80
+F src/vdbe.c 39675332be215b4672f8800b9355e190476ff5fe35fe71f2c9944a3f9dce17b7
 F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a
 F src/vdbeInt.h 95f7adfdc5c8f1353321f55a6c5ec00a90877e3b85af5159e393afb41ff54110
 F src/vdbeapi.c fea41171884a4de119f8b10ab514c788674eeeb7f27218bb6d008e1310bfd07f
@@ -1001,6 +1001,7 @@ F test/ioerr3.test d3cec5e1a11ad6d27527d0d38573fbff14c71bdd
 F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c
 F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4
 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b
+F test/istrue.test e95d4366a705232d81781545f314457f4cc407c8a53b5eac96703f220a74303a
 F test/join.test 442c462eea85cf065d70a663c626b780a95af6e11585d909bb63b87598afe678
 F test/join2.test f1b5ba94ba70a791df2d99e61adcebf62edacad1eca57881c97eb3e2ca718595
 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0
@@ -1612,7 +1613,7 @@ F test/zipfilefault.test 050be76778d2ec83566a542800fc3967f65ce719671a819016ada8b
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d
 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91
-F tool/addopcodes.tcl 92514b0db662868f1ec803ea95d8f5cf31213ba022fdfd5b7d289518d2e1eecf
+F tool/addopcodes.tcl 0288d5b26b9b35f4cb5affb76eec63f1dfce117bbc2020066708069ef60b86ff
 F tool/build-all-msvc.bat c12328d06c45fec8baada5949e3d5af54bf8c887 x
 F tool/build-shell.sh 950f47c6174f1eea171319438b93ba67ff5bf367
 F tool/cg_anno.tcl f95b0006c52cf7f0496b506343415b6ee3cdcdd3 x
@@ -1707,7 +1708,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 40314bc999af08ab10e654241208842b4bb95b19858d11249444372250ea4160 a983fa857048c748112c2f805f13aadeae43b43b3d27fa55433a3699b51c9a0e
-R 38dbfe1b01398cfde35ddf8702196d69
+P 003dc140536d0dd4384252ae1b82827b5d8b2b399520f99549b9266a482009fb
+R e29768d7d377928f9be17fdcc6f50028
 U drh
-Z bda364b06b801f317fc0151c8a36021e
+Z 99658c0596100aeaf925626dd549f8e1
diff --git a/manifest.uuid b/manifest.uuid
index 4d9aad0fd3..d9fe0c4992 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-003dc140536d0dd4384252ae1b82827b5d8b2b399520f99549b9266a482009fb
\ No newline at end of file
+cf2abd59be9971a55bd3d6c5df374c6aaa23bf81819482b42f01ee2484dcd739
\ No newline at end of file
diff --git a/src/expr.c b/src/expr.c
index 5079035ca1..f32a03a6d7 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -3543,12 +3543,8 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){
       codeInteger(pParse, pExpr, 0, target);
       return target;
     }
-    case TK_TRUE: {
-      sqlite3VdbeAddOp2(v, OP_Integer, 1, target);
-      return target;
-    }
-    case TK_FALSE: {
-      sqlite3VdbeAddOp2(v, OP_Integer, 0, target);
+    case TK_TRUEFALSE: {
+      sqlite3VdbeAddOp2(v, OP_Integer, pExpr->iTable, target);
       return target;
     }
 #ifndef SQLITE_OMIT_FLOATING_POINT
@@ -3706,11 +3702,14 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){
       sqlite3VdbeAddOp2(v, op, r1, inReg);
       break;
     }
-    case TK_ISTRUE: {
+    case TK_TRUTH: {
+      assert( pExpr->pRight->op==TK_TRUEFALSE );
+      assert( pExpr->pRight->iTable==0 || pExpr->pRight->iTable==1 );
+      assert( pExpr->op2==TK_IS || pExpr->op2==TK_ISNOT );
       r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, &regFree1);
       testcase( regFree1==0 );
-      sqlite3VdbeAddOp2(v, OP_Not, r1, inReg);
-      sqlite3VdbeAddOp2(v, OP_Not, inReg, inReg);
+      sqlite3VdbeAddOp4Int(v, OP_IsTrue, r1, inReg, !pExpr->pRight->iTable,
+                           pExpr->pRight->iTable ^ (pExpr->op2==TK_IS));
       break;
     }
     case TK_ISNULL:
@@ -4488,9 +4487,21 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){
       sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull);
       break;
     }
-    case TK_ISTRUE: {
+    case TK_TRUTH: {
+      int isNot;
       testcase( jumpIfNull==0 );
-      sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull);
+      assert( pExpr->pRight->op==TK_TRUEFALSE );
+      assert( pExpr->pRight->iTable==0 || pExpr->pRight->iTable==1 );
+      testcase( pExpr->pRight->iTable==0 );
+      assert( pExpr->op2==TK_IS || pExpr->op2==TK_ISNOT );
+      isNot = pExpr->op2==TK_ISNOT;
+      if( pExpr->pRight->iTable ^ isNot ){
+        sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest,
+                          isNot ? SQLITE_JUMPIFNULL : 0);
+      }else{
+        sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest,
+                           isNot ? SQLITE_JUMPIFNULL : 0);
+      }
       break;
     }
     case TK_IS:
@@ -4647,9 +4658,25 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){
       sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull);
       break;
     }
-    case TK_ISTRUE: {
+    case TK_TRUTH: {
       testcase( jumpIfNull==0 );
-      sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull);
+      int isNot;
+      testcase( jumpIfNull==0 );
+      assert( pExpr->pRight->op==TK_TRUEFALSE );
+      assert( pExpr->pRight->iTable==0 || pExpr->pRight->iTable==1 );
+      testcase( pExpr->pRight->iTable==0 );
+      assert( pExpr->op2==TK_IS || pExpr->op2==TK_ISNOT );
+      isNot = pExpr->op2==TK_ISNOT;
+      if( pExpr->pRight->iTable ^ isNot ){
+        /* IS TRUE and IS NOT FALSE */
+        sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest,
+                           isNot ? 0 : SQLITE_JUMPIFNULL);
+
+      }else{
+        /* IS FALSE and IS NOT TRUE */
+        sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest,
+                          isNot ? 0: SQLITE_JUMPIFNULL);
+      }
       break;
     }
     case TK_IS:
diff --git a/src/resolve.c b/src/resolve.c
index fb00788dd0..c6b086c3ad 100644
--- a/src/resolve.c
+++ b/src/resolve.c
@@ -437,13 +437,9 @@ static int lookupName(
       pExpr->pTab = 0;
       return WRC_Prune;
     }
-    if( sqlite3StrICmp(zCol, "true")==0 ){
-      pExpr->op = TK_TRUE;
-      pExpr->pTab = 0;
-      return WRC_Prune;
-    }
-    if( sqlite3StrICmp(zCol, "false")==0 ){
-      pExpr->op = TK_FALSE;
+    if( sqlite3StrICmp(zCol, "true")==0 || sqlite3StrICmp(zCol, "false")==0 ){
+      pExpr->op = TK_TRUEFALSE;
+      pExpr->iTable = zCol[4]==0;
       pExpr->pTab = 0;
       return WRC_Prune;
     }
@@ -796,28 +792,30 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){
       break;
     }
     case TK_IS:
-      /* Handle special cases of "x IS TRUE" and "x IS FALSE".  The first
-      ** is transformed into "+x" and the second into "NOT x". */
-      if( pExpr->pRight->op==TK_ID ){
-        int rc = resolveExprStep(pWalker, pExpr->pRight);
+    case TK_ISNOT: {
+      Expr *pRight;
+      assert( !ExprHasProperty(pExpr, EP_Reduced) );
+      /* Handle special cases of "x IS TRUE", "x IS FALSE", "x IS NOT TRUE",
+      ** and "x IS NOT FALSE". */
+      if( (pRight = pExpr->pRight)->op==TK_ID ){
+        int rc = resolveExprStep(pWalker, pRight);
         if( rc==WRC_Abort ) return WRC_Abort;
-        if( pExpr->pRight->op==TK_TRUE ){
-          pExpr->op = TK_ISTRUE;
-          return WRC_Continue;
-        }else if( pExpr->pRight->op==TK_FALSE ){
-          pExpr->op = TK_NOT;
+        if( pRight->op==TK_TRUEFALSE ){
+          assert( pRight->iTable==0 || pRight->iTable==1 );
+          pExpr->op2 = pExpr->op;
+          pExpr->op = TK_TRUTH;
           return WRC_Continue;
         }
       }
       /* Fall thru */
+    }
     case TK_BETWEEN:
     case TK_EQ:
     case TK_NE:
     case TK_LT:
     case TK_LE:
     case TK_GT:
-    case TK_GE:
-    case TK_ISNOT: {
+    case TK_GE: {
       int nLeft, nRight;
       if( pParse->db->mallocFailed ) break;
       assert( pExpr->pLeft!=0 );
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index f4c82cdb68..6da57c25f6 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -2387,7 +2387,8 @@ struct Expr {
                          ** TK_REGISTER: register number
                          ** TK_TRIGGER: 1 -> new, 0 -> old
                          ** EP_Unlikely:  134217728 times likelihood
-                         ** TK_SELECT: 1st register of result vector */
+                         ** TK_SELECT: 1st register of result vector
+                         ** TK_TRUEFALSE: 1 for true, 0 for false */
   ynVar iColumn;         /* TK_COLUMN: column index.  -1 for rowid.
                          ** TK_VARIABLE: variable number (always >= 1).
                          ** TK_SELECT_COLUMN: column of the result vector */
diff --git a/src/vdbe.c b/src/vdbe.c
index 61fc55e4de..cdc16292b8 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -2191,6 +2191,34 @@ case OP_Or: {             /* same as TK_OR, in1, in2, out3 */
   break;
 }
 
+/* Opcode: IsTrue P1 P2 P3 P4 *
+** Synopsis: r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4
+**
+** This opcode implements the IS TRUE, IS FALSE, IS NOT TRUE, and
+** IS NOT FALSE operators.
+**
+** Interpret the value in register P1 as a boolean value.  Store the that
+** boolean (a 0 or 1) in register P2.  Or if the value in register P1 is 
+** NULL, then the P3 is stored in register P2.  Invert the answer if P4
+** is 1.
+**
+** The logic is summarized like this:
+**
+** <ul> 
+** <li> P3==0, P4==0  &rarr;  r[P2] = r[P1] IS TRUE
+** <li> P3==1, P4==1  &rarr;  r[P2] = r[P1] IS FALSE
+** <li> P3==0, P4==1  &rarr;  r[P2] = r[P1] IS NOT TRUE
+** <li> P3==1, P4==0  &rarr;  r[P2] = r[P1] IS NOT FALSE
+** </ul>
+*/
+case OP_IsTrue: {               /* in1, out2 */
+  assert( pOp->p4type==P4_INT32 );
+  assert( pOp->p4.i==0 || pOp->p4.i==1 );
+  sqlite3VdbeMemSetInt64(&aMem[pOp->p2],
+      sqlite3VdbeBooleanValue(&aMem[pOp->p1], pOp->p3) ^ pOp->p4.i);
+  break;
+}
+
 /* Opcode: Not P1 P2 * * *
 ** Synopsis: r[P2]= !r[P1]
 **
diff --git a/test/istrue.test b/test/istrue.test
new file mode 100644
index 0000000000..cb6bfc52dc
--- /dev/null
+++ b/test/istrue.test
@@ -0,0 +1,81 @@
+# 2018-02-26
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing expressions of the form
+#
+#        x IS TRUE
+#        x IS FALSE
+#        x IS NOT TRUE
+#        x IS NOT FALSE
+#
+# Tests are also included for the use of TRUE and FALSE as
+# literal values.
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+do_execsql_test istrue-100 {
+  CREATE TABLE t1(x INTEGER PRIMARY KEY, y BOOLEAN);
+  INSERT INTO t1 VALUES(1, true),(2, false),(3, null);
+  SELECT x FROM t1 WHERE y IS TRUE;
+} {1}
+do_execsql_test istrue-110 {
+  SELECT x FROM t1 WHERE y IS FALSE;
+} {2}
+do_execsql_test istrue-120 {
+  SELECT x FROM t1 WHERE y IS NULL;
+} {3}
+do_execsql_test istrue-130 {
+  SELECT x FROM t1 WHERE y IS NOT TRUE;
+} {2 3}
+do_execsql_test istrue-140 {
+  SELECT x FROM t1 WHERE y IS NOT FALSE;
+} {1 3}
+do_execsql_test istrue-150 {
+  SELECT x FROM t1 WHERE y IS NOT NULL;
+} {1 2}
+unset -nocomplain X
+set X 9
+do_execsql_test istrue-160 {
+  SELECT x FROM t1 WHERE y IS TRUE OR (8==$X)
+} {1}
+do_execsql_test istrue-170 {
+  SELECT x FROM t1 WHERE y IS FALSE OR (8==$X)
+} {2}
+do_execsql_test istrue-180 {
+  SELECT x FROM t1 WHERE y IS NULL OR (8==$X);
+} {3}
+do_execsql_test istrue-190 {
+  SELECT x FROM t1 WHERE y IS NOT TRUE OR (8==$X);
+} {2 3}
+do_execsql_test istrue-200 {
+  SELECT x FROM t1 WHERE y IS NOT FALSE OR (8==$X);
+} {1 3}
+do_execsql_test istrue-210 {
+  SELECT x FROM t1 WHERE y IS NOT NULL OR (8==$X);
+} {1 2}
+
+do_execsql_test istrue-300 {
+  SELECT x,
+         y IS TRUE, y IS FALSE, y is NULL,
+         y IS NOT TRUE, y IS NOT FALSE, y IS NOT NULL, '|'
+    FROM t1 ORDER BY x;
+} {1 1 0 0 0 1 1 | 2 0 1 0 1 0 1 | 3 0 0 1 1 1 0 |}
+
+do_execsql_test istrue-400 {
+  SELECT x FROM t1 WHERE true;
+} {1 2 3}
+do_execsql_test istrue-410 {
+  SELECT x FROM t1 WHERE false;
+} {}
+
+
+finish_test
diff --git a/tool/addopcodes.tcl b/tool/addopcodes.tcl
index 7059d71363..070779029b 100644
--- a/tool/addopcodes.tcl
+++ b/tool/addopcodes.tcl
@@ -22,8 +22,7 @@ close $in
 # ILLEGAL *must* be the last two token codes and they must be in that order.
 #
 set extras {
-  TRUE
-  FALSE
+  TRUEFALSE
   ISNOT
   FUNCTION
   COLUMN
@@ -31,7 +30,7 @@ set extras {
   AGG_COLUMN
   UMINUS
   UPLUS
-  ISTRUE
+  TRUTH
   REGISTER
   VECTOR
   SELECT_COLUMN

From 3432821192f3c0ddcb5588f401daf661060ea124 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 26 Feb 2018 19:03:25 +0000
Subject: [PATCH 478/488] Enhance TreeView so that it can display the new IS
 TRUE expression trees.

FossilOrigin-Name: 7e38305e6f71c3353df3d11d099e255eefbacceafd7ed2ccaba257fe2e7b8c38
---
 manifest       | 12 ++++++------
 manifest.uuid  |  2 +-
 src/treeview.c | 18 ++++++++++++++++++
 3 files changed, 25 insertions(+), 7 deletions(-)

diff --git a/manifest b/manifest
index d23a52f63f..7117845e2a 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Refactor\sfor\scorrect\sNULL\shandling\sin\sthe\sIS\sTRUE,\sIS\sFALSE,\sIS\sNOT\sTRUE,\nand\sIS\sNOT\sFALSE\soperators.
-D 2018-02-26T18:49:05.668
+C Enhance\sTreeView\sso\sthat\sit\scan\sdisplay\sthe\snew\sIS\sTRUE\sexpression\strees.
+D 2018-02-26T19:03:25.495
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737
@@ -551,7 +551,7 @@ F src/test_windirent.h 90dfbe95442c9762357fe128dc7ae3dc199d006de93eb33ba3972e0a9
 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
 F src/tokenize.c 5b0c661a85f783d35b9883830736eeb63be4aefc4f6b7d9cd081d48782c041e2
-F src/treeview.c eae35972ff44f67064de2eaf35f04afe94e7aea3271a8b3bcebb3f954880fec3
+F src/treeview.c bdc8ca85b6f9b56b10381801b656f0ec0f4a86a3f5006cd3f9faa24fc15ea443
 F src/trigger.c a34539c69433276d37b0da9a89c117726ff2d292c0902895af1f393a983cd3a1
 F src/update.c a90a32ffc0100265b0693dbbdbe490756447af181f5ea2c138cce515b08c8795
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
@@ -1708,7 +1708,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 003dc140536d0dd4384252ae1b82827b5d8b2b399520f99549b9266a482009fb
-R e29768d7d377928f9be17fdcc6f50028
+P cf2abd59be9971a55bd3d6c5df374c6aaa23bf81819482b42f01ee2484dcd739
+R 6f6e40c4ec384ce48a27d1e80052a614
 U drh
-Z 99658c0596100aeaf925626dd549f8e1
+Z deaee9f4cadd8927720fc9b75136faa8
diff --git a/manifest.uuid b/manifest.uuid
index d9fe0c4992..46f8b1b4b2 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-cf2abd59be9971a55bd3d6c5df374c6aaa23bf81819482b42f01ee2484dcd739
\ No newline at end of file
+7e38305e6f71c3353df3d11d099e255eefbacceafd7ed2ccaba257fe2e7b8c38
\ No newline at end of file
diff --git a/src/treeview.c b/src/treeview.c
index 6dd386ffe6..ea91710cfd 100644
--- a/src/treeview.c
+++ b/src/treeview.c
@@ -292,6 +292,10 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){
       sqlite3TreeViewLine(pView,"NULL");
       break;
     }
+    case TK_TRUEFALSE: {
+      sqlite3TreeViewLine(pView, pExpr->iTable ? "TRUE":"FALSE");
+      break;
+    }
 #ifndef SQLITE_OMIT_BLOB_LITERAL
     case TK_BLOB: {
       sqlite3TreeViewLine(pView,"%s", pExpr->u.zToken);
@@ -348,6 +352,20 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){
     case TK_ISNULL:  zUniOp = "ISNULL"; break;
     case TK_NOTNULL: zUniOp = "NOTNULL"; break;
 
+    case TK_TRUTH: {
+      assert( pExpr->op2==TK_IS || pExpr->op2==TK_ISNOT );
+      assert( pExpr->pRight );
+      assert( pExpr->pRight->op==TK_TRUEFALSE );
+      assert( pExpr->pRight->iTable==0 || pExpr->pRight->iTable==1 );
+      switch( (pExpr->op2==TK_ISNOT)*2 + pExpr->pRight->iTable ){
+        case 0: zUniOp = "IS-FALSE";     break;
+        case 1: zUniOp = "IS-TRUE";      break;
+        case 2: zUniOp = "IS-NOT-FALSE"; break;
+        case 3: zUniOp = "IS-NOT-TRUE";  break;
+      }
+      break;
+    }
+
     case TK_SPAN: {
       sqlite3TreeViewLine(pView, "SPAN %Q", pExpr->u.zToken);
       sqlite3TreeViewExpr(pView, pExpr->pLeft, 0);

From 171d16bb2187b41636980b809d04d923e2811523 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 26 Feb 2018 20:15:54 +0000
Subject: [PATCH 479/488] Get the "DEFAULT true" and "DEFAULT false" phrases
 working correctly in CREATE TABLE.

FossilOrigin-Name: 8002f87d96b3f885cd208e7d204907a33ba60c4057ce2338b71e2de41215b0e5
---
 manifest         | 20 ++++++++++----------
 manifest.uuid    |  2 +-
 src/expr.c       | 28 +++++++++++++++++++++++++++-
 src/parse.y      |  2 ++
 src/resolve.c    |  6 ++----
 src/sqliteInt.h  |  1 +
 test/istrue.test | 11 +++++++++++
 7 files changed, 54 insertions(+), 16 deletions(-)

diff --git a/manifest b/manifest
index 7117845e2a..39f7ed6fcb 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Enhance\sTreeView\sso\sthat\sit\scan\sdisplay\sthe\snew\sIS\sTRUE\sexpression\strees.
-D 2018-02-26T19:03:25.495
+C Get\sthe\s"DEFAULT\strue"\sand\s"DEFAULT\sfalse"\sphrases\sworking\scorrectly\sin\nCREATE\sTABLE.
+D 2018-02-26T20:15:54.167
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737
@@ -441,7 +441,7 @@ F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957
 F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
 F src/delete.c 20c8788451dc737a967c87ea53ad43544d617f5b57d32ccce8bd52a0daf9e89b
-F src/expr.c 4344e70a158c0092e0c14c564d352546259297996cba250f1d6d9dbf830a415d
+F src/expr.c 2662b7cb76943b5865fe5e62f21615e7f86928b98f0b908b339f84a91ace8f8e
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331
 F src/func.c 94f42cba2cc1c34aeaa441022ba0170ec3fec4bba54db4e0ded085c6dc0fdc51
@@ -477,7 +477,7 @@ F src/os_win.c eb03c6d52f893bcd7fdd4c6006674c13c1b5e49543fec98d605201af2997171c
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 0b6bd5442733b2e08d0673de6cdafe3e7ab0b5715e4844ac836ab346b1d9ed89
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
-F src/parse.y 4e750e1b261ff9f1d0b6b5d40a829c66d691899f48953fde839d8b52d41aa148
+F src/parse.y 6319911fb7ab37907fb1ea14ddfea429b2c42707276eaa388b6c402891dfc091
 F src/pcache.c 7ae91a4557a43d77d449accbfdc68846e6516f8e2eda46e8bbe4536fb669b201
 F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170
 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
@@ -486,14 +486,14 @@ F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
 F src/prepare.c 259f4e7960c47082c9653f3d5f0c294abd68bb9c3aab86de7630700cba1c20fb
 F src/printf.c d3b7844ddeb11fbbdd38dd84d09c9c1ac171d21fb038473c3aa97981201cc660
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
-F src/resolve.c 7e6a9702fe3bbf556b1105497ba7a51c24bf606bcc2a7c329122b31e04149005
+F src/resolve.c 6f8a45063c693019734887b5f95e860137960fae324ef8b6db36f06e8c53240e
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c f02352ac5cbb6fad3804add825743b924cfb6c742ba2e8666d726828a9985d73
 F src/shell.c.in c1b14bc0bc1fe73a97dfaad55ca09f5d6e44f6019977d94e7e3d1861616035e1
 F src/sqlite.h.in 235e942dd8c01b414c2996828b4d0d2500faf8850f8b24ae17d31c172f519e69
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d
-F src/sqliteInt.h d38064626689d9a832f17cfc0feef250e34a9f6f2589805e8cef5e85c38c18fa
+F src/sqliteInt.h a6856d3d1aefd594b2c8c6245c6fbcba3d691e38555e22910ee6e7a839bc3897
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -1001,7 +1001,7 @@ F test/ioerr3.test d3cec5e1a11ad6d27527d0d38573fbff14c71bdd
 F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c
 F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4
 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b
-F test/istrue.test e95d4366a705232d81781545f314457f4cc407c8a53b5eac96703f220a74303a
+F test/istrue.test c15523b11ea315e7c2855e12f39ddd9df8852d23b8caafceb53a5f5dca907c91
 F test/join.test 442c462eea85cf065d70a663c626b780a95af6e11585d909bb63b87598afe678
 F test/join2.test f1b5ba94ba70a791df2d99e61adcebf62edacad1eca57881c97eb3e2ca718595
 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0
@@ -1708,7 +1708,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P cf2abd59be9971a55bd3d6c5df374c6aaa23bf81819482b42f01ee2484dcd739
-R 6f6e40c4ec384ce48a27d1e80052a614
+P 7e38305e6f71c3353df3d11d099e255eefbacceafd7ed2ccaba257fe2e7b8c38
+R 8bbc8431995f4fe7e9fb21f05431fedd
 U drh
-Z deaee9f4cadd8927720fc9b75136faa8
+Z 0df480b7d34c4b4166a378f5cc8b7dd8
diff --git a/manifest.uuid b/manifest.uuid
index 46f8b1b4b2..4d5c7fa522 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7e38305e6f71c3353df3d11d099e255eefbacceafd7ed2ccaba257fe2e7b8c38
\ No newline at end of file
+8002f87d96b3f885cd208e7d204907a33ba60c4057ce2338b71e2de41215b0e5
\ No newline at end of file
diff --git a/src/expr.c b/src/expr.c
index f32a03a6d7..75d8db7e42 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -1123,7 +1123,7 @@ static int dupedExprStructSize(Expr *p, int flags){
   assert( flags==EXPRDUP_REDUCE || flags==0 ); /* Only one flag value allowed */
   assert( EXPR_FULLSIZE<=0xfff );
   assert( (0xfff & (EP_Reduced|EP_TokenOnly))==0 );
-  if( 0==flags || p->op==TK_SELECT_COLUMN ){
+  if( 0==flags || p->op==TK_SELECT_COLUMN || p->op==TK_TRUEFALSE ){
     nSize = EXPR_FULLSIZE;
   }else{
     assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) );
@@ -1732,6 +1732,26 @@ int sqlite3SelectWalkFail(Walker *pWalker, Select *NotUsed){
   return WRC_Abort;
 }
 
+/*
+** If the input expression is an ID with the name "true" or "false"
+** then convert it into an appropriate TK_TRUEFALSE term.  Return true
+** if a conversion occurred, and false if the expression is unaltered.
+*/
+int sqlite3ExprIdToTrueFalse(Expr *pExpr){
+  assert( pExpr->op==TK_ID || pExpr->op==TK_STRING );
+  if( sqlite3StrICmp(pExpr->u.zToken, "true")==0
+   || sqlite3StrICmp(pExpr->u.zToken, "false")==0
+  ){
+    pExpr->op = TK_TRUEFALSE;
+    pExpr->iTable = pExpr->u.zToken[4]==0;
+    pExpr->pTab = 0;
+    ExprSetProperty(pExpr, EP_NoReduce);
+    return 1;
+  }
+  return 0;
+}
+
+
 /*
 ** These routines are Walker callbacks used to check expressions to
 ** see if they are "constant" for some definition of constant.  The
@@ -1779,6 +1799,12 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){
         return WRC_Abort;
       }
     case TK_ID:
+      /* Convert "true" or "false" in a DEFAULT clause into the
+      ** appropriate TK_TRUEFALSE operator */
+      if( pWalker->eCode>=4 && sqlite3ExprIdToTrueFalse(pExpr) ){
+        return WRC_Prune;
+      }
+      /* Fall thru */
     case TK_COLUMN:
     case TK_AGG_FUNCTION:
     case TK_AGG_COLUMN:
diff --git a/src/parse.y b/src/parse.y
index d9cf1cb87d..ff2e74af01 100644
--- a/src/parse.y
+++ b/src/parse.y
@@ -313,6 +313,8 @@ ccons ::= DEFAULT MINUS(A) term(X) scanpt(Z).      {
 }
 ccons ::= DEFAULT scanpt id(X).       {
   Expr *p = tokenExpr(pParse, TK_STRING, X);
+  sqlite3ExprIdToTrueFalse(p);
+  testcase( p->op==TK_TRUEFALSE && p->iTable==0 );
   sqlite3AddDefaultValue(pParse,p,X.z,X.z+X.n);
 }
 
diff --git a/src/resolve.c b/src/resolve.c
index c6b086c3ad..33dd4dcb7b 100644
--- a/src/resolve.c
+++ b/src/resolve.c
@@ -432,15 +432,13 @@ static int lookupName(
   ** fields are not changed in any context.
   */
   if( cnt==0 && zTab==0 ){
+    assert( pExpr->op==TK_ID );
     if( ExprHasProperty(pExpr,EP_DblQuoted) ){
       pExpr->op = TK_STRING;
       pExpr->pTab = 0;
       return WRC_Prune;
     }
-    if( sqlite3StrICmp(zCol, "true")==0 || sqlite3StrICmp(zCol, "false")==0 ){
-      pExpr->op = TK_TRUEFALSE;
-      pExpr->iTable = zCol[4]==0;
-      pExpr->pTab = 0;
+    if( sqlite3ExprIdToTrueFalse(pExpr) ){
       return WRC_Prune;
     }
   }
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 6da57c25f6..09413de004 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -3840,6 +3840,7 @@ void sqlite3EndTransaction(Parse*,int);
 void sqlite3Savepoint(Parse*, int, Token*);
 void sqlite3CloseSavepoints(sqlite3 *);
 void sqlite3LeaveMutexAndCloseZombie(sqlite3*);
+int sqlite3ExprIdToTrueFalse(Expr*);
 int sqlite3ExprIsConstant(Expr*);
 int sqlite3ExprIsConstantNotJoin(Expr*);
 int sqlite3ExprIsConstantOrFunction(Expr*, u8);
diff --git a/test/istrue.test b/test/istrue.test
index cb6bfc52dc..5ffc7f5f4f 100644
--- a/test/istrue.test
+++ b/test/istrue.test
@@ -77,5 +77,16 @@ do_execsql_test istrue-410 {
   SELECT x FROM t1 WHERE false;
 } {}
 
+do_execsql_test istrue-500 {
+  CREATE TABLE t2(
+     a INTEGER PRIMARY KEY,
+     b BOOLEAN DEFAULT true,
+     c BOOLEAN DEFAULT(true),
+     d BOOLEAN DEFAULT false,
+     e BOOLEAN DEFAULT(false)
+  );
+  INSERT INTO t2 DEFAULT VALUES;
+  SELECT * FROM t2;
+} {1 1 1 0 0}
 
 finish_test

From 43c4ac8bfecba96e4db6d075c5fa743a4fe25ea5 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Mon, 26 Feb 2018 21:26:27 +0000
Subject: [PATCH 480/488] Code simplifications.  New test cases.

FossilOrigin-Name: 57508518ef9d003d259ba98dcc32e5104aca26731a7161808741fe10bc0830d0
---
 manifest         | 22 +++++++++++-----------
 manifest.uuid    |  2 +-
 src/expr.c       | 47 +++++++++++++++++++++++++++--------------------
 src/parse.y      |  2 +-
 src/resolve.c    |  1 -
 src/sqliteInt.h  |  4 ++--
 src/treeview.c   | 16 ++++++++--------
 test/istrue.test | 34 ++++++++++++++++++++++++++++++++++
 8 files changed, 84 insertions(+), 44 deletions(-)

diff --git a/manifest b/manifest
index 39f7ed6fcb..96e1edd95d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Get\sthe\s"DEFAULT\strue"\sand\s"DEFAULT\sfalse"\sphrases\sworking\scorrectly\sin\nCREATE\sTABLE.
-D 2018-02-26T20:15:54.167
+C Code\ssimplifications.\s\sNew\stest\scases.
+D 2018-02-26T21:26:27.326
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737
@@ -441,7 +441,7 @@ F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957
 F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
 F src/delete.c 20c8788451dc737a967c87ea53ad43544d617f5b57d32ccce8bd52a0daf9e89b
-F src/expr.c 2662b7cb76943b5865fe5e62f21615e7f86928b98f0b908b339f84a91ace8f8e
+F src/expr.c ee29e9c2af181dd8fe4476f7e983aa13aa1634d00dd955a90ffbeb4732d7bc39
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331
 F src/func.c 94f42cba2cc1c34aeaa441022ba0170ec3fec4bba54db4e0ded085c6dc0fdc51
@@ -477,7 +477,7 @@ F src/os_win.c eb03c6d52f893bcd7fdd4c6006674c13c1b5e49543fec98d605201af2997171c
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 0b6bd5442733b2e08d0673de6cdafe3e7ab0b5715e4844ac836ab346b1d9ed89
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
-F src/parse.y 6319911fb7ab37907fb1ea14ddfea429b2c42707276eaa388b6c402891dfc091
+F src/parse.y b875c74b72497f3cf23d9c6227ec0c859824ac9a5b23380cc1ed8adcff4857a0
 F src/pcache.c 7ae91a4557a43d77d449accbfdc68846e6516f8e2eda46e8bbe4536fb669b201
 F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170
 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
@@ -486,14 +486,14 @@ F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
 F src/prepare.c 259f4e7960c47082c9653f3d5f0c294abd68bb9c3aab86de7630700cba1c20fb
 F src/printf.c d3b7844ddeb11fbbdd38dd84d09c9c1ac171d21fb038473c3aa97981201cc660
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
-F src/resolve.c 6f8a45063c693019734887b5f95e860137960fae324ef8b6db36f06e8c53240e
+F src/resolve.c 66c73fcb7719b8ff0e841b58338f13604ff3e2b50a723f9b8f383595735262f6
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c f02352ac5cbb6fad3804add825743b924cfb6c742ba2e8666d726828a9985d73
 F src/shell.c.in c1b14bc0bc1fe73a97dfaad55ca09f5d6e44f6019977d94e7e3d1861616035e1
 F src/sqlite.h.in 235e942dd8c01b414c2996828b4d0d2500faf8850f8b24ae17d31c172f519e69
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d
-F src/sqliteInt.h a6856d3d1aefd594b2c8c6245c6fbcba3d691e38555e22910ee6e7a839bc3897
+F src/sqliteInt.h ce6d2c08df3216a69d3f7fb740e8ddb7ef94169277776778e7cfce05ab245d10
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -551,7 +551,7 @@ F src/test_windirent.h 90dfbe95442c9762357fe128dc7ae3dc199d006de93eb33ba3972e0a9
 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
 F src/tokenize.c 5b0c661a85f783d35b9883830736eeb63be4aefc4f6b7d9cd081d48782c041e2
-F src/treeview.c bdc8ca85b6f9b56b10381801b656f0ec0f4a86a3f5006cd3f9faa24fc15ea443
+F src/treeview.c b2470f160658181f5d1d7672144d5e7a2635a398a2f062842b7b4e0c766c4158
 F src/trigger.c a34539c69433276d37b0da9a89c117726ff2d292c0902895af1f393a983cd3a1
 F src/update.c a90a32ffc0100265b0693dbbdbe490756447af181f5ea2c138cce515b08c8795
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
@@ -1001,7 +1001,7 @@ F test/ioerr3.test d3cec5e1a11ad6d27527d0d38573fbff14c71bdd
 F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c
 F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4
 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b
-F test/istrue.test c15523b11ea315e7c2855e12f39ddd9df8852d23b8caafceb53a5f5dca907c91
+F test/istrue.test 2d89873d72248b7a0ef7b5457120100cce75a19181bcf4b30e90a3f96a5c5558
 F test/join.test 442c462eea85cf065d70a663c626b780a95af6e11585d909bb63b87598afe678
 F test/join2.test f1b5ba94ba70a791df2d99e61adcebf62edacad1eca57881c97eb3e2ca718595
 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0
@@ -1708,7 +1708,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 7e38305e6f71c3353df3d11d099e255eefbacceafd7ed2ccaba257fe2e7b8c38
-R 8bbc8431995f4fe7e9fb21f05431fedd
+P 8002f87d96b3f885cd208e7d204907a33ba60c4057ce2338b71e2de41215b0e5
+R 5cfd707edda06e1acf2e2d72bf60da27
 U drh
-Z 0df480b7d34c4b4166a378f5cc8b7dd8
+Z d672c3ecc0520c79d23db3920b6bc66e
diff --git a/manifest.uuid b/manifest.uuid
index 4d5c7fa522..16a67a9163 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-8002f87d96b3f885cd208e7d204907a33ba60c4057ce2338b71e2de41215b0e5
\ No newline at end of file
+57508518ef9d003d259ba98dcc32e5104aca26731a7161808741fe10bc0830d0
\ No newline at end of file
diff --git a/src/expr.c b/src/expr.c
index 75d8db7e42..fae2c11d9c 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -1123,7 +1123,7 @@ static int dupedExprStructSize(Expr *p, int flags){
   assert( flags==EXPRDUP_REDUCE || flags==0 ); /* Only one flag value allowed */
   assert( EXPR_FULLSIZE<=0xfff );
   assert( (0xfff & (EP_Reduced|EP_TokenOnly))==0 );
-  if( 0==flags || p->op==TK_SELECT_COLUMN || p->op==TK_TRUEFALSE ){
+  if( 0==flags || p->op==TK_SELECT_COLUMN ){
     nSize = EXPR_FULLSIZE;
   }else{
     assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) );
@@ -1743,14 +1743,22 @@ int sqlite3ExprIdToTrueFalse(Expr *pExpr){
    || sqlite3StrICmp(pExpr->u.zToken, "false")==0
   ){
     pExpr->op = TK_TRUEFALSE;
-    pExpr->iTable = pExpr->u.zToken[4]==0;
-    pExpr->pTab = 0;
-    ExprSetProperty(pExpr, EP_NoReduce);
     return 1;
   }
   return 0;
 }
 
+/*
+** The argument is one of a TK_TRUEFALSE term.  Return 1 if it is TRUE
+** and 0 if it is FALSE.
+*/
+int sqlite3ExprTruthOperand(const Expr *pExpr){
+  assert( pExpr->op==TK_TRUEFALSE );
+  assert( sqlite3StrICmp(pExpr->u.zToken,"true")==0
+       || sqlite3StrICmp(pExpr->u.zToken,"false")==0 );
+  return pExpr->u.zToken[4]==0;
+}
+
 
 /*
 ** These routines are Walker callbacks used to check expressions to
@@ -3570,7 +3578,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){
       return target;
     }
     case TK_TRUEFALSE: {
-      sqlite3VdbeAddOp2(v, OP_Integer, pExpr->iTable, target);
+      sqlite3VdbeAddOp2(v, OP_Integer, sqlite3ExprTruthOperand(pExpr), target);
       return target;
     }
 #ifndef SQLITE_OMIT_FLOATING_POINT
@@ -3729,13 +3737,12 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){
       break;
     }
     case TK_TRUTH: {
-      assert( pExpr->pRight->op==TK_TRUEFALSE );
-      assert( pExpr->pRight->iTable==0 || pExpr->pRight->iTable==1 );
-      assert( pExpr->op2==TK_IS || pExpr->op2==TK_ISNOT );
+      int isTrue;
       r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, &regFree1);
       testcase( regFree1==0 );
-      sqlite3VdbeAddOp4Int(v, OP_IsTrue, r1, inReg, !pExpr->pRight->iTable,
-                           pExpr->pRight->iTable ^ (pExpr->op2==TK_IS));
+      isTrue = sqlite3ExprTruthOperand(pExpr->pRight);
+      sqlite3VdbeAddOp4Int(v, OP_IsTrue, r1, inReg, !isTrue,
+                           isTrue ^ (pExpr->op2==TK_IS));
       break;
     }
     case TK_ISNULL:
@@ -4515,13 +4522,13 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){
     }
     case TK_TRUTH: {
       int isNot;
+      int isTrue;
       testcase( jumpIfNull==0 );
-      assert( pExpr->pRight->op==TK_TRUEFALSE );
-      assert( pExpr->pRight->iTable==0 || pExpr->pRight->iTable==1 );
-      testcase( pExpr->pRight->iTable==0 );
-      assert( pExpr->op2==TK_IS || pExpr->op2==TK_ISNOT );
       isNot = pExpr->op2==TK_ISNOT;
-      if( pExpr->pRight->iTable ^ isNot ){
+      isTrue = sqlite3ExprTruthOperand(pExpr->pRight);
+      testcase( isTrue && isNot );
+      testcase( isTrue && !isNot );
+      if( isTrue ^ isNot ){
         sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest,
                           isNot ? SQLITE_JUMPIFNULL : 0);
       }else{
@@ -4685,15 +4692,15 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){
       break;
     }
     case TK_TRUTH: {
-      testcase( jumpIfNull==0 );
       int isNot;
+      int isTrue;
       testcase( jumpIfNull==0 );
-      assert( pExpr->pRight->op==TK_TRUEFALSE );
-      assert( pExpr->pRight->iTable==0 || pExpr->pRight->iTable==1 );
-      testcase( pExpr->pRight->iTable==0 );
       assert( pExpr->op2==TK_IS || pExpr->op2==TK_ISNOT );
       isNot = pExpr->op2==TK_ISNOT;
-      if( pExpr->pRight->iTable ^ isNot ){
+      isTrue = sqlite3ExprTruthOperand(pExpr->pRight);
+      testcase( isTrue && isNot );
+      testcase( isTrue && !isNot );
+      if( isTrue ^ isNot ){
         /* IS TRUE and IS NOT FALSE */
         sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest,
                            isNot ? 0 : SQLITE_JUMPIFNULL);
diff --git a/src/parse.y b/src/parse.y
index ff2e74af01..81e4fdf1e0 100644
--- a/src/parse.y
+++ b/src/parse.y
@@ -314,7 +314,7 @@ ccons ::= DEFAULT MINUS(A) term(X) scanpt(Z).      {
 ccons ::= DEFAULT scanpt id(X).       {
   Expr *p = tokenExpr(pParse, TK_STRING, X);
   sqlite3ExprIdToTrueFalse(p);
-  testcase( p->op==TK_TRUEFALSE && p->iTable==0 );
+  testcase( p->op==TK_TRUEFALSE && sqlite3ExprTruthOperand(p) );
   sqlite3AddDefaultValue(pParse,p,X.z,X.z+X.n);
 }
 
diff --git a/src/resolve.c b/src/resolve.c
index 33dd4dcb7b..7ae49bde57 100644
--- a/src/resolve.c
+++ b/src/resolve.c
@@ -799,7 +799,6 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){
         int rc = resolveExprStep(pWalker, pRight);
         if( rc==WRC_Abort ) return WRC_Abort;
         if( pRight->op==TK_TRUEFALSE ){
-          assert( pRight->iTable==0 || pRight->iTable==1 );
           pExpr->op2 = pExpr->op;
           pExpr->op = TK_TRUTH;
           return WRC_Continue;
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 09413de004..4dd58d3370 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -2387,8 +2387,7 @@ struct Expr {
                          ** TK_REGISTER: register number
                          ** TK_TRIGGER: 1 -> new, 0 -> old
                          ** EP_Unlikely:  134217728 times likelihood
-                         ** TK_SELECT: 1st register of result vector
-                         ** TK_TRUEFALSE: 1 for true, 0 for false */
+                         ** TK_SELECT: 1st register of result vector */
   ynVar iColumn;         /* TK_COLUMN: column index.  -1 for rowid.
                          ** TK_VARIABLE: variable number (always >= 1).
                          ** TK_SELECT_COLUMN: column of the result vector */
@@ -3841,6 +3840,7 @@ void sqlite3Savepoint(Parse*, int, Token*);
 void sqlite3CloseSavepoints(sqlite3 *);
 void sqlite3LeaveMutexAndCloseZombie(sqlite3*);
 int sqlite3ExprIdToTrueFalse(Expr*);
+int sqlite3ExprTruthOperand(const Expr*);
 int sqlite3ExprIsConstant(Expr*);
 int sqlite3ExprIsConstantNotJoin(Expr*);
 int sqlite3ExprIsConstantOrFunction(Expr*, u8);
diff --git a/src/treeview.c b/src/treeview.c
index ea91710cfd..cfe818268f 100644
--- a/src/treeview.c
+++ b/src/treeview.c
@@ -293,7 +293,8 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){
       break;
     }
     case TK_TRUEFALSE: {
-      sqlite3TreeViewLine(pView, pExpr->iTable ? "TRUE":"FALSE");
+      sqlite3TreeViewLine(pView,
+         sqlite3ExprTruthOperand(pExpr) ? "TRUE" : "FALSE");
       break;
     }
 #ifndef SQLITE_OMIT_BLOB_LITERAL
@@ -353,16 +354,15 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){
     case TK_NOTNULL: zUniOp = "NOTNULL"; break;
 
     case TK_TRUTH: {
+      int x;
+      const char *azOp[] = {
+         "IS-FALSE", "IS-TRUE", "IS-NOT-FALSE", "IS-NOT-TRUE"
+      };
       assert( pExpr->op2==TK_IS || pExpr->op2==TK_ISNOT );
       assert( pExpr->pRight );
       assert( pExpr->pRight->op==TK_TRUEFALSE );
-      assert( pExpr->pRight->iTable==0 || pExpr->pRight->iTable==1 );
-      switch( (pExpr->op2==TK_ISNOT)*2 + pExpr->pRight->iTable ){
-        case 0: zUniOp = "IS-FALSE";     break;
-        case 1: zUniOp = "IS-TRUE";      break;
-        case 2: zUniOp = "IS-NOT-FALSE"; break;
-        case 3: zUniOp = "IS-NOT-TRUE";  break;
-      }
+      x = (pExpr->op2==TK_ISNOT)*2 + sqlite3ExprTruthOperand(pExpr->pRight);
+      zUniOp = azOp[x];
       break;
     }
 
diff --git a/test/istrue.test b/test/istrue.test
index 5ffc7f5f4f..250f1f9d5d 100644
--- a/test/istrue.test
+++ b/test/istrue.test
@@ -88,5 +88,39 @@ do_execsql_test istrue-500 {
   INSERT INTO t2 DEFAULT VALUES;
   SELECT * FROM t2;
 } {1 1 1 0 0}
+do_execsql_test istrue-510 {
+  DROP TABLE t2;
+  CREATE TABLE t2(
+     a INTEGER PRIMARY KEY,
+     b BOOLEAN DEFAULT(not true),
+     c BOOLEAN DEFAULT(not false)
+  );
+  INSERT INTO t2(a) VALUES(99);
+  SELECT * FROM t2;
+} {99 0 1}
+do_execsql_test istrue-520 {
+  DROP TABLE t2;
+  CREATE TABLE t2(
+     a INTEGER PRIMARY KEY,
+     b BOOLEAN CHECK(b IS TRUE),
+     c BOOLEAN CHECK(c IS FALSE),
+     d BOOLEAN CHECK(d IS NOT TRUE),
+     e BOOLEAN CHECK(e IS NOT FALSE)
+  );
+  INSERT INTO t2 VALUES(1,true,false,null,null);
+  SELECT * FROM t2;
+} {1 1 0 {} {}}
+do_catchsql_test istrue-521 {
+  INSERT INTO t2 VALUES(2,false,false,null,null);
+} {1 {CHECK constraint failed: t2}}
+do_catchsql_test istrue-522 {
+  INSERT INTO t2 VALUES(2,true,true,null,null);
+} {1 {CHECK constraint failed: t2}}
+do_catchsql_test istrue-523 {
+  INSERT INTO t2 VALUES(2,true,false,true,null);
+} {1 {CHECK constraint failed: t2}}
+do_catchsql_test istrue-524 {
+  INSERT INTO t2 VALUES(2,true,false,null,false);
+} {1 {CHECK constraint failed: t2}}
 
 finish_test

From e39ef31cd68218eaf39bf02cdb365b936158baef Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 27 Feb 2018 00:58:13 +0000
Subject: [PATCH 481/488] Remove an unnecessary decision.

FossilOrigin-Name: adcb4665490404e8da2de128dd5df9aeed8e08893649dedb6788ebb5d957b400
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/expr.c    |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 96e1edd95d..7f3497c990 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Code\ssimplifications.\s\sNew\stest\scases.
-D 2018-02-26T21:26:27.326
+C Remove\san\sunnecessary\sdecision.
+D 2018-02-27T00:58:13.219
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737
@@ -441,7 +441,7 @@ F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957
 F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
 F src/delete.c 20c8788451dc737a967c87ea53ad43544d617f5b57d32ccce8bd52a0daf9e89b
-F src/expr.c ee29e9c2af181dd8fe4476f7e983aa13aa1634d00dd955a90ffbeb4732d7bc39
+F src/expr.c 03975fff227cd24b17e86b5bcc64e0fb9deecccd1b50d23c5f1f8a6852b99064
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331
 F src/func.c 94f42cba2cc1c34aeaa441022ba0170ec3fec4bba54db4e0ded085c6dc0fdc51
@@ -1708,7 +1708,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 8002f87d96b3f885cd208e7d204907a33ba60c4057ce2338b71e2de41215b0e5
-R 5cfd707edda06e1acf2e2d72bf60da27
+P 57508518ef9d003d259ba98dcc32e5104aca26731a7161808741fe10bc0830d0
+R 8ef9da63ad321cc3596e92a715293098
 U drh
-Z d672c3ecc0520c79d23db3920b6bc66e
+Z 8fed3cfabeba6ea8fc4880d89a25745b
diff --git a/manifest.uuid b/manifest.uuid
index 16a67a9163..622a0019fa 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-57508518ef9d003d259ba98dcc32e5104aca26731a7161808741fe10bc0830d0
\ No newline at end of file
+adcb4665490404e8da2de128dd5df9aeed8e08893649dedb6788ebb5d957b400
\ No newline at end of file
diff --git a/src/expr.c b/src/expr.c
index fae2c11d9c..e1959761f4 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -1809,7 +1809,7 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){
     case TK_ID:
       /* Convert "true" or "false" in a DEFAULT clause into the
       ** appropriate TK_TRUEFALSE operator */
-      if( pWalker->eCode>=4 && sqlite3ExprIdToTrueFalse(pExpr) ){
+      if( sqlite3ExprIdToTrueFalse(pExpr) ){
         return WRC_Prune;
       }
       /* Fall thru */

From 96acafbea7bc8b7c688f0a39b98bcd0ed00dc35c Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Tue, 27 Feb 2018 14:49:25 +0000
Subject: [PATCH 482/488] Clean up comments and variable names prior to merge.

FossilOrigin-Name: 6445519e91c4f98b4a9a45d5091d733c31497ebf0eb23a76edce3091f626035d
---
 manifest        | 20 ++++++++++----------
 manifest.uuid   |  2 +-
 src/expr.c      | 40 +++++++++++++++++++++-------------------
 src/parse.y     |  2 +-
 src/sqliteInt.h |  2 +-
 src/treeview.c  |  4 ++--
 src/vdbe.c      | 11 ++++++-----
 7 files changed, 42 insertions(+), 39 deletions(-)

diff --git a/manifest b/manifest
index 7f3497c990..bdef730d7d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\san\sunnecessary\sdecision.
-D 2018-02-27T00:58:13.219
+C Clean\sup\scomments\sand\svariable\snames\sprior\sto\smerge.
+D 2018-02-27T14:49:25.328
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737
@@ -441,7 +441,7 @@ F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957
 F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6
 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720
 F src/delete.c 20c8788451dc737a967c87ea53ad43544d617f5b57d32ccce8bd52a0daf9e89b
-F src/expr.c 03975fff227cd24b17e86b5bcc64e0fb9deecccd1b50d23c5f1f8a6852b99064
+F src/expr.c 904f85e5f5fa232488cbb662fcf3306ae203728751468670fb9840a9b1886abc
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331
 F src/func.c 94f42cba2cc1c34aeaa441022ba0170ec3fec4bba54db4e0ded085c6dc0fdc51
@@ -477,7 +477,7 @@ F src/os_win.c eb03c6d52f893bcd7fdd4c6006674c13c1b5e49543fec98d605201af2997171c
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 0b6bd5442733b2e08d0673de6cdafe3e7ab0b5715e4844ac836ab346b1d9ed89
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
-F src/parse.y b875c74b72497f3cf23d9c6227ec0c859824ac9a5b23380cc1ed8adcff4857a0
+F src/parse.y 97b31a676ec1cb049970883a317efcd07ca8c7daded950c8de49199cdfdb1dfa
 F src/pcache.c 7ae91a4557a43d77d449accbfdc68846e6516f8e2eda46e8bbe4536fb669b201
 F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170
 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
@@ -493,7 +493,7 @@ F src/shell.c.in c1b14bc0bc1fe73a97dfaad55ca09f5d6e44f6019977d94e7e3d1861616035e
 F src/sqlite.h.in 235e942dd8c01b414c2996828b4d0d2500faf8850f8b24ae17d31c172f519e69
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d
-F src/sqliteInt.h ce6d2c08df3216a69d3f7fb740e8ddb7ef94169277776778e7cfce05ab245d10
+F src/sqliteInt.h 6c321e404f97117c9b676ac1a8591a22d095bf5ab730624becea3ad93067c89c
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -551,13 +551,13 @@ F src/test_windirent.h 90dfbe95442c9762357fe128dc7ae3dc199d006de93eb33ba3972e0a9
 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
 F src/tokenize.c 5b0c661a85f783d35b9883830736eeb63be4aefc4f6b7d9cd081d48782c041e2
-F src/treeview.c b2470f160658181f5d1d7672144d5e7a2635a398a2f062842b7b4e0c766c4158
+F src/treeview.c e0d62678314abf0e1e6d09ea405f4de3f404a17b4641c2169c304b5edf509320
 F src/trigger.c a34539c69433276d37b0da9a89c117726ff2d292c0902895af1f393a983cd3a1
 F src/update.c a90a32ffc0100265b0693dbbdbe490756447af181f5ea2c138cce515b08c8795
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c d9eb0a6c4aae1b00a7369eadd7ca0bbe946cb4c953b6751aa20d357c2f482157
 F src/vacuum.c 762ee9bbf8733d87d8cd06f58d950e881982e416f8c767334a40ffd341b6bff5
-F src/vdbe.c 39675332be215b4672f8800b9355e190476ff5fe35fe71f2c9944a3f9dce17b7
+F src/vdbe.c 99a0f275c83c01332b9af6179f2ee1e5d24ad81f90f17f8364397f8a3d20fb40
 F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a
 F src/vdbeInt.h 95f7adfdc5c8f1353321f55a6c5ec00a90877e3b85af5159e393afb41ff54110
 F src/vdbeapi.c fea41171884a4de119f8b10ab514c788674eeeb7f27218bb6d008e1310bfd07f
@@ -1708,7 +1708,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 57508518ef9d003d259ba98dcc32e5104aca26731a7161808741fe10bc0830d0
-R 8ef9da63ad321cc3596e92a715293098
+P adcb4665490404e8da2de128dd5df9aeed8e08893649dedb6788ebb5d957b400
+R 691db3d03f207528a0777702d3649318
 U drh
-Z 8fed3cfabeba6ea8fc4880d89a25745b
+Z 3708f8d55564f3165287f25ec6417cc9
diff --git a/manifest.uuid b/manifest.uuid
index 622a0019fa..d25d53f33e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-adcb4665490404e8da2de128dd5df9aeed8e08893649dedb6788ebb5d957b400
\ No newline at end of file
+6445519e91c4f98b4a9a45d5091d733c31497ebf0eb23a76edce3091f626035d
\ No newline at end of file
diff --git a/src/expr.c b/src/expr.c
index e1959761f4..d2fad0ffc3 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -1734,8 +1734,8 @@ int sqlite3SelectWalkFail(Walker *pWalker, Select *NotUsed){
 
 /*
 ** If the input expression is an ID with the name "true" or "false"
-** then convert it into an appropriate TK_TRUEFALSE term.  Return true
-** if a conversion occurred, and false if the expression is unaltered.
+** then convert it into an TK_TRUEFALSE term.  Return non-zero if
+** the conversion happened, and zero if the expression is unaltered.
 */
 int sqlite3ExprIdToTrueFalse(Expr *pExpr){
   assert( pExpr->op==TK_ID || pExpr->op==TK_STRING );
@@ -1749,10 +1749,10 @@ int sqlite3ExprIdToTrueFalse(Expr *pExpr){
 }
 
 /*
-** The argument is one of a TK_TRUEFALSE term.  Return 1 if it is TRUE
+** The argument must be a TK_TRUEFALSE Expr node.  Return 1 if it is TRUE
 ** and 0 if it is FALSE.
 */
-int sqlite3ExprTruthOperand(const Expr *pExpr){
+int sqlite3ExprTruthValue(const Expr *pExpr){
   assert( pExpr->op==TK_TRUEFALSE );
   assert( sqlite3StrICmp(pExpr->u.zToken,"true")==0
        || sqlite3StrICmp(pExpr->u.zToken,"false")==0 );
@@ -3578,7 +3578,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){
       return target;
     }
     case TK_TRUEFALSE: {
-      sqlite3VdbeAddOp2(v, OP_Integer, sqlite3ExprTruthOperand(pExpr), target);
+      sqlite3VdbeAddOp2(v, OP_Integer, sqlite3ExprTruthValue(pExpr), target);
       return target;
     }
 #ifndef SQLITE_OMIT_FLOATING_POINT
@@ -3737,12 +3737,15 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){
       break;
     }
     case TK_TRUTH: {
-      int isTrue;
+      int isTrue;    /* IS TRUE or IS NOT TRUE */
+      int bNormal;   /* IS TRUE or IS FALSE */
       r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, &regFree1);
       testcase( regFree1==0 );
-      isTrue = sqlite3ExprTruthOperand(pExpr->pRight);
-      sqlite3VdbeAddOp4Int(v, OP_IsTrue, r1, inReg, !isTrue,
-                           isTrue ^ (pExpr->op2==TK_IS));
+      isTrue = sqlite3ExprTruthValue(pExpr->pRight);
+      bNormal = pExpr->op2==TK_IS;
+      testcase( isTrue && bNormal);
+      testcase( !isTrue && bNormal);
+      sqlite3VdbeAddOp4Int(v, OP_IsTrue, r1, inReg, !isTrue, isTrue ^ bNormal);
       break;
     }
     case TK_ISNULL:
@@ -4521,13 +4524,13 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){
       break;
     }
     case TK_TRUTH: {
-      int isNot;
-      int isTrue;
+      int isNot;      /* IS NOT TRUE or IS NOT FALSE */
+      int isTrue;     /* IS TRUE or IS NOT TRUE */
       testcase( jumpIfNull==0 );
       isNot = pExpr->op2==TK_ISNOT;
-      isTrue = sqlite3ExprTruthOperand(pExpr->pRight);
+      isTrue = sqlite3ExprTruthValue(pExpr->pRight);
       testcase( isTrue && isNot );
-      testcase( isTrue && !isNot );
+      testcase( !isTrue && isNot );
       if( isTrue ^ isNot ){
         sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest,
                           isNot ? SQLITE_JUMPIFNULL : 0);
@@ -4692,14 +4695,13 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){
       break;
     }
     case TK_TRUTH: {
-      int isNot;
-      int isTrue;
+      int isNot;   /* IS NOT TRUE or IS NOT FALSE */
+      int isTrue;  /* IS TRUE or IS NOT TRUE */
       testcase( jumpIfNull==0 );
-      assert( pExpr->op2==TK_IS || pExpr->op2==TK_ISNOT );
       isNot = pExpr->op2==TK_ISNOT;
-      isTrue = sqlite3ExprTruthOperand(pExpr->pRight);
+      isTrue = sqlite3ExprTruthValue(pExpr->pRight);
       testcase( isTrue && isNot );
-      testcase( isTrue && !isNot );
+      testcase( !isTrue && isNot );
       if( isTrue ^ isNot ){
         /* IS TRUE and IS NOT FALSE */
         sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest,
@@ -4708,7 +4710,7 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){
       }else{
         /* IS FALSE and IS NOT TRUE */
         sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest,
-                          isNot ? 0: SQLITE_JUMPIFNULL);
+                          isNot ? 0 : SQLITE_JUMPIFNULL);
       }
       break;
     }
diff --git a/src/parse.y b/src/parse.y
index 81e4fdf1e0..49cc1cb0d4 100644
--- a/src/parse.y
+++ b/src/parse.y
@@ -314,7 +314,7 @@ ccons ::= DEFAULT MINUS(A) term(X) scanpt(Z).      {
 ccons ::= DEFAULT scanpt id(X).       {
   Expr *p = tokenExpr(pParse, TK_STRING, X);
   sqlite3ExprIdToTrueFalse(p);
-  testcase( p->op==TK_TRUEFALSE && sqlite3ExprTruthOperand(p) );
+  testcase( p->op==TK_TRUEFALSE && sqlite3ExprTruthValue(p) );
   sqlite3AddDefaultValue(pParse,p,X.z,X.z+X.n);
 }
 
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 4dd58d3370..eba52b6770 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -3840,7 +3840,7 @@ void sqlite3Savepoint(Parse*, int, Token*);
 void sqlite3CloseSavepoints(sqlite3 *);
 void sqlite3LeaveMutexAndCloseZombie(sqlite3*);
 int sqlite3ExprIdToTrueFalse(Expr*);
-int sqlite3ExprTruthOperand(const Expr*);
+int sqlite3ExprTruthValue(const Expr*);
 int sqlite3ExprIsConstant(Expr*);
 int sqlite3ExprIsConstantNotJoin(Expr*);
 int sqlite3ExprIsConstantOrFunction(Expr*, u8);
diff --git a/src/treeview.c b/src/treeview.c
index cfe818268f..e33fe6e227 100644
--- a/src/treeview.c
+++ b/src/treeview.c
@@ -294,7 +294,7 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){
     }
     case TK_TRUEFALSE: {
       sqlite3TreeViewLine(pView,
-         sqlite3ExprTruthOperand(pExpr) ? "TRUE" : "FALSE");
+         sqlite3ExprTruthValue(pExpr) ? "TRUE" : "FALSE");
       break;
     }
 #ifndef SQLITE_OMIT_BLOB_LITERAL
@@ -361,7 +361,7 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){
       assert( pExpr->op2==TK_IS || pExpr->op2==TK_ISNOT );
       assert( pExpr->pRight );
       assert( pExpr->pRight->op==TK_TRUEFALSE );
-      x = (pExpr->op2==TK_ISNOT)*2 + sqlite3ExprTruthOperand(pExpr->pRight);
+      x = (pExpr->op2==TK_ISNOT)*2 + sqlite3ExprTruthValue(pExpr->pRight);
       zUniOp = azOp[x];
       break;
     }
diff --git a/src/vdbe.c b/src/vdbe.c
index cdc16292b8..10715a8232 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -2197,7 +2197,7 @@ case OP_Or: {             /* same as TK_OR, in1, in2, out3 */
 ** This opcode implements the IS TRUE, IS FALSE, IS NOT TRUE, and
 ** IS NOT FALSE operators.
 **
-** Interpret the value in register P1 as a boolean value.  Store the that
+** Interpret the value in register P1 as a boolean value.  Store that
 ** boolean (a 0 or 1) in register P2.  Or if the value in register P1 is 
 ** NULL, then the P3 is stored in register P2.  Invert the answer if P4
 ** is 1.
@@ -2205,15 +2205,16 @@ case OP_Or: {             /* same as TK_OR, in1, in2, out3 */
 ** The logic is summarized like this:
 **
 ** <ul> 
-** <li> P3==0, P4==0  &rarr;  r[P2] = r[P1] IS TRUE
-** <li> P3==1, P4==1  &rarr;  r[P2] = r[P1] IS FALSE
-** <li> P3==0, P4==1  &rarr;  r[P2] = r[P1] IS NOT TRUE
-** <li> P3==1, P4==0  &rarr;  r[P2] = r[P1] IS NOT FALSE
+** <li> If P3==0 and P4==0  then  r[P2] := r[P1] IS TRUE
+** <li> If P3==1 and P4==1  then  r[P2] := r[P1] IS FALSE
+** <li> If P3==0 and P4==1  then  r[P2] := r[P1] IS NOT TRUE
+** <li> If P3==1 and P4==0  then  r[P2] := r[P1] IS NOT FALSE
 ** </ul>
 */
 case OP_IsTrue: {               /* in1, out2 */
   assert( pOp->p4type==P4_INT32 );
   assert( pOp->p4.i==0 || pOp->p4.i==1 );
+  assert( pOp->p3==0 || pOp->p3==1 );
   sqlite3VdbeMemSetInt64(&aMem[pOp->p2],
       sqlite3VdbeBooleanValue(&aMem[pOp->p1], pOp->p3) ^ pOp->p4.i);
   break;

From d7fd899eeb8d29b0bc59727a2e137d4c341100cb Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 28 Feb 2018 04:30:55 +0000
Subject: [PATCH 483/488] Fix a potential NULL pointer deref following OOM in
 the new IS TRUE logic.

FossilOrigin-Name: 48775ec693c21927ff26448acc614dd67d56240fa263499da2a9fe405c546c32
---
 manifest      | 13 ++++++-------
 manifest.uuid |  2 +-
 src/parse.y   |  6 ++++--
 3 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/manifest b/manifest
index 50048f9068..be35bd4d18 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\ssupport\sfor\sTRUE\sand\sFALSE\skeywords\sand\sfor\soperators\sIS\sTRUE,\sIS\sFALSE,\nIS\sNOT\sTRUE,\sand\sIS\sNOT\sFALSE.\s\sIf\sthere\sis\sare\scolumns\snamed\sTRUE\sor\sFALSE,\nthen\sthe\skeywords\sresolve\sto\sthe\scolumn\snames,\sfor\scompatibility.\s\sThe\nbehavior\sof\sthe\s"DEFAULT\strue"\sphrase\sis\schanged\sto\smean\swhat\sit\ssays,\nrather\sthan\sbeing\san\salias\sfor\s"DEFAULT\s'true'".
-D 2018-02-27T15:40:59.030
+C Fix\sa\spotential\sNULL\spointer\sderef\sfollowing\sOOM\sin\sthe\snew\sIS\sTRUE\nlogic.
+D 2018-02-28T04:30:55.210
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737
@@ -477,7 +477,7 @@ F src/os_win.c eb03c6d52f893bcd7fdd4c6006674c13c1b5e49543fec98d605201af2997171c
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 0b6bd5442733b2e08d0673de6cdafe3e7ab0b5715e4844ac836ab346b1d9ed89
 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a
-F src/parse.y 97b31a676ec1cb049970883a317efcd07ca8c7daded950c8de49199cdfdb1dfa
+F src/parse.y 3be4b2b5e33ef7dab3128b765898ee786fc6cd698a7f053f756a68f2f238f0aa
 F src/pcache.c 7ae91a4557a43d77d449accbfdc68846e6516f8e2eda46e8bbe4536fb669b201
 F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170
 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
@@ -1708,8 +1708,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c26f236da61aef7c7a8fe38f1c7f33eea894c8fae38de6a085d4cce933a70502 6445519e91c4f98b4a9a45d5091d733c31497ebf0eb23a76edce3091f626035d
-R 74016598c506c5fb19fa13d7ec93e943
-T +closed 6445519e91c4f98b4a9a45d5091d733c31497ebf0eb23a76edce3091f626035d
+P 9a7f02c50e6c8a3dc3addf4e51b7e04bd31670d6ce58d26d55273c43b5ecc8b0
+R 8cff6bf49e22739f377b0c86155b1f52
 U drh
-Z 4b60eba7ac8a552c151e343e027ad980
+Z 08a4a2d4426a2c0bef4340ae5149b6ad
diff --git a/manifest.uuid b/manifest.uuid
index 50dab302b5..8f7e16b28e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-9a7f02c50e6c8a3dc3addf4e51b7e04bd31670d6ce58d26d55273c43b5ecc8b0
\ No newline at end of file
+48775ec693c21927ff26448acc614dd67d56240fa263499da2a9fe405c546c32
\ No newline at end of file
diff --git a/src/parse.y b/src/parse.y
index 49cc1cb0d4..9c41484ab2 100644
--- a/src/parse.y
+++ b/src/parse.y
@@ -313,8 +313,10 @@ ccons ::= DEFAULT MINUS(A) term(X) scanpt(Z).      {
 }
 ccons ::= DEFAULT scanpt id(X).       {
   Expr *p = tokenExpr(pParse, TK_STRING, X);
-  sqlite3ExprIdToTrueFalse(p);
-  testcase( p->op==TK_TRUEFALSE && sqlite3ExprTruthValue(p) );
+  if( p ){
+    sqlite3ExprIdToTrueFalse(p);
+    testcase( p->op==TK_TRUEFALSE && sqlite3ExprTruthValue(p) );
+  }
   sqlite3AddDefaultValue(pParse,p,X.z,X.z+X.n);
 }
 

From 6a8a629e6bf8886e1f5b43f4cdcdd850f96b3bd3 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 28 Feb 2018 21:50:00 +0000
Subject: [PATCH 484/488] Use &lt;pre&gt; around code snippets in the
 documentation for sessions interfaces.  This is a documentation change only
 with no changes to code.

FossilOrigin-Name: c949b915e893e917315ce21092d4c4bbd3e1b88d5326928f71dcc2f18f300702
---
 ext/session/sqlite3session.h |  4 ++++
 manifest                     | 12 ++++++------
 manifest.uuid                |  2 +-
 3 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/ext/session/sqlite3session.h b/ext/session/sqlite3session.h
index 9f33855df0..f9e371fb9a 100644
--- a/ext/session/sqlite3session.h
+++ b/ext/session/sqlite3session.h
@@ -678,6 +678,7 @@ int sqlite3changeset_fk_conflicts(
 ** to that error is returned by this function. Otherwise, SQLITE_OK is
 ** returned. This is to allow the following pattern (pseudo-code):
 **
+** <pre>
 **   sqlite3changeset_start();
 **   while( SQLITE_ROW==sqlite3changeset_next() ){
 **     // Do something with change.
@@ -686,6 +687,7 @@ int sqlite3changeset_fk_conflicts(
 **   if( rc!=SQLITE_OK ){
 **     // An error has occurred 
 **   }
+** </pre>
 */
 int sqlite3changeset_finalize(sqlite3_changeset_iter *pIter);
 
@@ -733,6 +735,7 @@ int sqlite3changeset_invert(
 ** sqlite3_changegroup object. Calling it produces similar results as the
 ** following code fragment:
 **
+** <pre>
 **   sqlite3_changegroup *pGrp;
 **   rc = sqlite3_changegroup_new(&pGrp);
 **   if( rc==SQLITE_OK ) rc = sqlite3changegroup_add(pGrp, nA, pA);
@@ -743,6 +746,7 @@ int sqlite3changeset_invert(
 **     *ppOut = 0;
 **     *pnOut = 0;
 **   }
+** </pre>
 **
 ** Refer to the sqlite3_changegroup documentation below for details.
 */
diff --git a/manifest b/manifest
index be35bd4d18..20de1e13a0 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\spotential\sNULL\spointer\sderef\sfollowing\sOOM\sin\sthe\snew\sIS\sTRUE\nlogic.
-D 2018-02-28T04:30:55.210
+C Use\s&lt;pre&gt;\saround\scode\ssnippets\sin\sthe\sdocumentation\sfor\ssessions\ninterfaces.\s\sThis\sis\sa\sdocumentation\schange\sonly\swith\sno\schanges\sto\scode.
+D 2018-02-28T21:50:00.053
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737
@@ -403,7 +403,7 @@ F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0
 F ext/session/sessionstat1.test 41cd97c2e48619a41cdf8ae749e1b25f34719de638689221aa43971be693bf4e
 F ext/session/sessionwor.test 2f3744236dc8b170a695b7d8ddc8c743c7e79fdc
 F ext/session/sqlite3session.c a5b7aed647abe7e366254b755597fce3f2719d82c98990cb0e1e07a3d203fe2b
-F ext/session/sqlite3session.h 01774161cbd328fe3d496323655b9cc142317ff1fb1ae15c1232075ea240e3a4
+F ext/session/sqlite3session.h 357b889d5d1be268bd39bce9df8d28fb4afc88c35b5ebf62dad6aeb725d3f37e
 F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386
 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04
@@ -1708,7 +1708,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 9a7f02c50e6c8a3dc3addf4e51b7e04bd31670d6ce58d26d55273c43b5ecc8b0
-R 8cff6bf49e22739f377b0c86155b1f52
+P 48775ec693c21927ff26448acc614dd67d56240fa263499da2a9fe405c546c32
+R 534465c4041022fe3e1312b87af4cd46
 U drh
-Z 08a4a2d4426a2c0bef4340ae5149b6ad
+Z d5651a3a39c99623b3bbaa536731261b
diff --git a/manifest.uuid b/manifest.uuid
index 8f7e16b28e..33f071d785 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-48775ec693c21927ff26448acc614dd67d56240fa263499da2a9fe405c546c32
\ No newline at end of file
+c949b915e893e917315ce21092d4c4bbd3e1b88d5326928f71dcc2f18f300702
\ No newline at end of file

From bda30ce4d82fa2f26309028e93dbacf2e7612818 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Wed, 28 Feb 2018 22:21:29 +0000
Subject: [PATCH 485/488] Enhance the sessions documentation to show the
 methods of the various objects.

FossilOrigin-Name: e01177754ad6d9e2d38adddddd2e2e212094dac1154bda5fcee61ca8b678ae0f
---
 ext/session/sqlite3session.h | 33 ++++++++++++++++++++++++++++++++-
 manifest                     | 12 ++++++------
 manifest.uuid                |  2 +-
 3 files changed, 39 insertions(+), 8 deletions(-)

diff --git a/ext/session/sqlite3session.h b/ext/session/sqlite3session.h
index f9e371fb9a..81b1ed06d6 100644
--- a/ext/session/sqlite3session.h
+++ b/ext/session/sqlite3session.h
@@ -13,16 +13,23 @@ extern "C" {
 
 /*
 ** CAPI3REF: Session Object Handle
+**
+** An instance of this object is a [session] that can be used to
+** record changes to a database.
 */
 typedef struct sqlite3_session sqlite3_session;
 
 /*
 ** CAPI3REF: Changeset Iterator Handle
+**
+** An instance of this object is as as a cursor for iterating
+** over the elements of a [changeset] or [patchset].
 */
 typedef struct sqlite3_changeset_iter sqlite3_changeset_iter;
 
 /*
 ** CAPI3REF: Create A New Session Object
+** CONSTRUCTOR: sqlite3_session
 **
 ** Create a new session object attached to database handle db. If successful,
 ** a pointer to the new object is written to *ppSession and SQLITE_OK is
@@ -59,6 +66,7 @@ int sqlite3session_create(
 
 /*
 ** CAPI3REF: Delete A Session Object
+** DESTRUCTOR: sqlite3_session
 **
 ** Delete a session object previously allocated using 
 ** [sqlite3session_create()]. Once a session object has been deleted, the
@@ -74,6 +82,7 @@ void sqlite3session_delete(sqlite3_session *pSession);
 
 /*
 ** CAPI3REF: Enable Or Disable A Session Object
+** METHOD: sqlite3_session
 **
 ** Enable or disable the recording of changes by a session object. When
 ** enabled, a session object records changes made to the database. When
@@ -93,6 +102,7 @@ int sqlite3session_enable(sqlite3_session *pSession, int bEnable);
 
 /*
 ** CAPI3REF: Set Or Clear the Indirect Change Flag
+** METHOD: sqlite3_session
 **
 ** Each change recorded by a session object is marked as either direct or
 ** indirect. A change is marked as indirect if either:
@@ -122,6 +132,7 @@ int sqlite3session_indirect(sqlite3_session *pSession, int bIndirect);
 
 /*
 ** CAPI3REF: Attach A Table To A Session Object
+** METHOD: sqlite3_session
 **
 ** If argument zTab is not NULL, then it is the name of a table to attach
 ** to the session object passed as the first argument. All subsequent changes 
@@ -184,6 +195,7 @@ int sqlite3session_attach(
 
 /*
 ** CAPI3REF: Set a table filter on a Session Object.
+** METHOD: sqlite3_session
 **
 ** The second argument (xFilter) is the "filter callback". For changes to rows 
 ** in tables that are not attached to the Session object, the filter is called
@@ -202,6 +214,7 @@ void sqlite3session_table_filter(
 
 /*
 ** CAPI3REF: Generate A Changeset From A Session Object
+** METHOD: sqlite3_session
 **
 ** Obtain a changeset containing changes to the tables attached to the 
 ** session object passed as the first argument. If successful, 
@@ -311,7 +324,8 @@ int sqlite3session_changeset(
 );
 
 /*
-** CAPI3REF: Load The Difference Between Tables Into A Session 
+** CAPI3REF: Load The Difference Between Tables Into A Session
+** METHOD: sqlite3_session
 **
 ** If it is not already attached to the session object passed as the first
 ** argument, this function attaches table zTbl in the same manner as the
@@ -376,6 +390,7 @@ int sqlite3session_diff(
 
 /*
 ** CAPI3REF: Generate A Patchset From A Session Object
+** METHOD: sqlite3_session
 **
 ** The differences between a patchset and a changeset are that:
 **
@@ -427,6 +442,7 @@ int sqlite3session_isempty(sqlite3_session *pSession);
 
 /*
 ** CAPI3REF: Create An Iterator To Traverse A Changeset 
+** CONSTRUCTOR: sqlite3_changeset_iter
 **
 ** Create an iterator used to iterate through the contents of a changeset.
 ** If successful, *pp is set to point to the iterator handle and SQLITE_OK
@@ -467,6 +483,7 @@ int sqlite3changeset_start(
 
 /*
 ** CAPI3REF: Advance A Changeset Iterator
+** METHOD: sqlite3_changeset_iter
 **
 ** This function may only be used with iterators created by function
 ** [sqlite3changeset_start()]. If it is called on an iterator passed to
@@ -491,6 +508,7 @@ int sqlite3changeset_next(sqlite3_changeset_iter *pIter);
 
 /*
 ** CAPI3REF: Obtain The Current Operation From A Changeset Iterator
+** METHOD: sqlite3_changeset_iter
 **
 ** The pIter argument passed to this function may either be an iterator
 ** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator
@@ -525,6 +543,7 @@ int sqlite3changeset_op(
 
 /*
 ** CAPI3REF: Obtain The Primary Key Definition Of A Table
+** METHOD: sqlite3_changeset_iter
 **
 ** For each modified table, a changeset includes the following:
 **
@@ -556,6 +575,7 @@ int sqlite3changeset_pk(
 
 /*
 ** CAPI3REF: Obtain old.* Values From A Changeset Iterator
+** METHOD: sqlite3_changeset_iter
 **
 ** The pIter argument passed to this function may either be an iterator
 ** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator
@@ -586,6 +606,7 @@ int sqlite3changeset_old(
 
 /*
 ** CAPI3REF: Obtain new.* Values From A Changeset Iterator
+** METHOD: sqlite3_changeset_iter
 **
 ** The pIter argument passed to this function may either be an iterator
 ** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator
@@ -619,6 +640,7 @@ int sqlite3changeset_new(
 
 /*
 ** CAPI3REF: Obtain Conflicting Row Values From A Changeset Iterator
+** METHOD: sqlite3_changeset_iter
 **
 ** This function should only be used with iterator objects passed to a
 ** conflict-handler callback by [sqlite3changeset_apply()] with either
@@ -646,6 +668,7 @@ int sqlite3changeset_conflict(
 
 /*
 ** CAPI3REF: Determine The Number Of Foreign Key Constraint Violations
+** METHOD: sqlite3_changeset_iter
 **
 ** This function may only be called with an iterator passed to an
 ** SQLITE_CHANGESET_FOREIGN_KEY conflict handler callback. In this case
@@ -662,6 +685,7 @@ int sqlite3changeset_fk_conflicts(
 
 /*
 ** CAPI3REF: Finalize A Changeset Iterator
+** METHOD: sqlite3_changeset_iter
 **
 ** This function is used to finalize an iterator allocated with
 ** [sqlite3changeset_start()].
@@ -762,11 +786,15 @@ int sqlite3changeset_concat(
 
 /*
 ** CAPI3REF: Changegroup Handle
+**
+** A changegroup is an object used to combine two or more 
+** [changesets] or [patchsets]
 */
 typedef struct sqlite3_changegroup sqlite3_changegroup;
 
 /*
 ** CAPI3REF: Create A New Changegroup Object
+** CONSTRUCTOR: sqlite3_changegroup
 **
 ** An sqlite3_changegroup object is used to combine two or more changesets
 ** (or patchsets) into a single changeset (or patchset). A single changegroup
@@ -804,6 +832,7 @@ int sqlite3changegroup_new(sqlite3_changegroup **pp);
 
 /*
 ** CAPI3REF: Add A Changeset To A Changegroup
+** METHOD: sqlite3_changegroup
 **
 ** Add all changes within the changeset (or patchset) in buffer pData (size
 ** nData bytes) to the changegroup. 
@@ -881,6 +910,7 @@ int sqlite3changegroup_add(sqlite3_changegroup*, int nData, void *pData);
 
 /*
 ** CAPI3REF: Obtain A Composite Changeset From A Changegroup
+** METHOD: sqlite3_changegroup
 **
 ** Obtain a buffer containing a changeset (or patchset) representing the
 ** current contents of the changegroup. If the inputs to the changegroup
@@ -911,6 +941,7 @@ int sqlite3changegroup_output(
 
 /*
 ** CAPI3REF: Delete A Changegroup Object
+** DESTRUCTOR: sqlite3_changegroup
 */
 void sqlite3changegroup_delete(sqlite3_changegroup*);
 
diff --git a/manifest b/manifest
index 20de1e13a0..07d15b424e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Use\s&lt;pre&gt;\saround\scode\ssnippets\sin\sthe\sdocumentation\sfor\ssessions\ninterfaces.\s\sThis\sis\sa\sdocumentation\schange\sonly\swith\sno\schanges\sto\scode.
-D 2018-02-28T21:50:00.053
+C Enhance\sthe\ssessions\sdocumentation\sto\sshow\sthe\smethods\sof\sthe\svarious\sobjects.
+D 2018-02-28T22:21:29.549
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737
@@ -403,7 +403,7 @@ F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0
 F ext/session/sessionstat1.test 41cd97c2e48619a41cdf8ae749e1b25f34719de638689221aa43971be693bf4e
 F ext/session/sessionwor.test 2f3744236dc8b170a695b7d8ddc8c743c7e79fdc
 F ext/session/sqlite3session.c a5b7aed647abe7e366254b755597fce3f2719d82c98990cb0e1e07a3d203fe2b
-F ext/session/sqlite3session.h 357b889d5d1be268bd39bce9df8d28fb4afc88c35b5ebf62dad6aeb725d3f37e
+F ext/session/sqlite3session.h b4de978c24a48a0d9b3b92ddfb749f4b07461766325ee950f5ecb8384c10606f
 F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386
 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04
@@ -1708,7 +1708,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 48775ec693c21927ff26448acc614dd67d56240fa263499da2a9fe405c546c32
-R 534465c4041022fe3e1312b87af4cd46
+P c949b915e893e917315ce21092d4c4bbd3e1b88d5326928f71dcc2f18f300702
+R 6f69334171b7b3454d79aa33bdfe9b61
 U drh
-Z d5651a3a39c99623b3bbaa536731261b
+Z 67038639e12c985c1bd40cee53ce76b7
diff --git a/manifest.uuid b/manifest.uuid
index 33f071d785..36563f126d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c949b915e893e917315ce21092d4c4bbd3e1b88d5326928f71dcc2f18f300702
\ No newline at end of file
+e01177754ad6d9e2d38adddddd2e2e212094dac1154bda5fcee61ca8b678ae0f
\ No newline at end of file

From dd8a4af8e77795f8d0982db57e3e959b7ba108ec Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 1 Mar 2018 12:05:51 +0000
Subject: [PATCH 486/488] Fix some crashes in the sqlite3changeset_apply()
 function that could be caused by corrupt changeset blobs.

FossilOrigin-Name: 745a9a7fef0f28a57ea3f44899058993f6ecdedda52c81a09a4a9ce09c9004d6
---
 ext/session/session4.test    | 25 ++++++++++++++++++++
 ext/session/sqlite3session.c | 46 ++++++++++++++++++++++++++----------
 manifest                     | 16 ++++++-------
 manifest.uuid                |  2 +-
 4 files changed, 67 insertions(+), 22 deletions(-)

diff --git a/ext/session/session4.test b/ext/session/session4.test
index 120a230b57..d07c57ffd6 100644
--- a/ext/session/session4.test
+++ b/ext/session/session4.test
@@ -64,4 +64,29 @@ do_test 1.3 {
   list [catch { sqlite3changeset_apply db $x xConflict } msg] $msg
 } {1 SQLITE_CORRUPT}
 
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 2.0 {
+  CREATE TABLE t1(a INTEGER PRIMARY KEY,b,c,d);
+  CREATE TABLE t2(e TEXT PRIMARY KEY NOT NULL,f,g);
+  CREATE TABLE t3(w REAL PRIMARY KEY NOT NULL,x,y);
+  CREATE TABLE t4(z PRIMARY KEY) WITHOUT ROWID;
+}
+
+foreach {tn blob} {
+  1 {54010174340012000000}
+  2 {54fefe8bcb0012000300}
+  3 {5480809280808001017434001200fb}
+  4 {50af9c939c9c9cb09c9c6400b09c9c6400}
+  5 {12000300}
+  6 {09847304}
+  7 {5401017434001208}
+} {
+if {$tn==7} breakpoint
+  do_test 2.$tn {
+    set changeset [binary decode hex $blob]
+    list [catch { sqlite3changeset_apply db $changeset xConflict } msg] $msg
+  } {1 SQLITE_CORRUPT}
+}
+
 finish_test
diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c
index bdecdd1031..2a130c7e9e 100644
--- a/ext/session/sqlite3session.c
+++ b/ext/session/sqlite3session.c
@@ -2718,15 +2718,18 @@ static int sessionReadRecord(
   for(i=0; i<nCol && rc==SQLITE_OK; i++){
     int eType = 0;                /* Type of value (SQLITE_NULL, TEXT etc.) */
     if( abPK && abPK[i]==0 ) continue;
-    rc = sessionInputBuffer(pIn, 9);
+    if( pIn->iNext>=pIn->nData ){
+      rc = SQLITE_CORRUPT;
+    }else{
+      rc = sessionInputBuffer(pIn, 9);
+    }
     if( rc==SQLITE_OK ){
       eType = pIn->aData[pIn->iNext++];
-    }
-
-    assert( apOut[i]==0 );
-    if( eType ){
-      apOut[i] = sqlite3ValueNew(0);
-      if( !apOut[i] ) rc = SQLITE_NOMEM;
+      assert( apOut[i]==0 );
+      if( eType ){
+        apOut[i] = sqlite3ValueNew(0);
+        if( !apOut[i] ) rc = SQLITE_NOMEM;
+      }
     }
 
     if( rc==SQLITE_OK ){
@@ -2857,11 +2860,15 @@ static int sessionChangesetReadTblhdr(sqlite3_changeset_iter *p){
     int nByte;
     int nVarint;
     nVarint = sessionVarintGet(&p->in.aData[p->in.iNext], &p->nCol);
-    nCopy -= nVarint;
-    p->in.iNext += nVarint;
-    nByte = p->nCol * sizeof(sqlite3_value*) * 2 + nCopy;
-    p->tblhdr.nBuf = 0;
-    sessionBufferGrow(&p->tblhdr, nByte, &rc);
+    if( p->nCol>0 ){
+      nCopy -= nVarint;
+      p->in.iNext += nVarint;
+      nByte = p->nCol * sizeof(sqlite3_value*) * 2 + nCopy;
+      p->tblhdr.nBuf = 0;
+      sessionBufferGrow(&p->tblhdr, nByte, &rc);
+    }else{
+      rc = SQLITE_CORRUPT;
+    }
   }
 
   if( rc==SQLITE_OK ){
@@ -2939,6 +2946,13 @@ static int sessionChangesetNext(
     op = p->in.aData[p->in.iNext++];
   }
 
+  if( p->zTab==0 ){
+    /* The first record in the changeset is not a table header. Must be a
+    ** corrupt changeset. */
+    assert( p->in.iNext==1 );
+    return (p->rc = SQLITE_CORRUPT_BKPT);
+  }
+
   p->op = op;
   p->bIndirect = p->in.aData[p->in.iNext++];
   if( p->op!=SQLITE_UPDATE && p->op!=SQLITE_DELETE && p->op!=SQLITE_INSERT ){
@@ -3708,7 +3722,13 @@ static int sessionBindRow(
     if( !abPK || abPK[i] ){
       sqlite3_value *pVal;
       (void)xValue(pIter, i, &pVal);
-      rc = sessionBindValue(pStmt, i+1, pVal);
+      if( pVal==0 ){
+        /* The value in the changeset was "undefined". This indicates a
+        ** corrupt changeset blob.  */
+        rc = SQLITE_CORRUPT;
+      }else{
+        rc = sessionBindValue(pStmt, i+1, pVal);
+      }
     }
   }
   return rc;
diff --git a/manifest b/manifest
index 07d15b424e..f09706c7be 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Enhance\sthe\ssessions\sdocumentation\sto\sshow\sthe\smethods\sof\sthe\svarious\sobjects.
-D 2018-02-28T22:21:29.549
+C Fix\ssome\scrashes\sin\sthe\ssqlite3changeset_apply()\sfunction\sthat\scould\sbe\scaused\nby\scorrupt\schangeset\sblobs.
+D 2018-03-01T12:05:51.293
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737
@@ -381,7 +381,7 @@ F ext/session/changeset.c 4ccbaa4531944c24584bf6a61ba3a39c62b6267a
 F ext/session/session1.test 736d7ff178662f0b717c37f46531b84a5ce0210ccb0c4edf629c55dbcbbc3ea1
 F ext/session/session2.test 284de45abae4cc1082bc52012ee81521d5ac58e0
 F ext/session/session3.test ce9ce3dfa489473987f899e9f6a0f2db9bde3479
-F ext/session/session4.test 457b02bdc349eb01151e54de014df77abd3c08c8
+F ext/session/session4.test 488539cee1d2510b415236fac2727575f4110e7609a500eb21c8d9e517dfff38
 F ext/session/session5.test 716bc6fafd625ce60dfa62ae128971628c1a1169
 F ext/session/session6.test 443789bc2fca12e4f7075cf692c60b8a2bea1a26
 F ext/session/session8.test 8e194b3f655d861ca36de5d4de53f702751bab3b
@@ -402,7 +402,7 @@ F ext/session/sessionfault.test da273f2712b6411e85e71465a1733b8501dbf6f7
 F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0
 F ext/session/sessionstat1.test 41cd97c2e48619a41cdf8ae749e1b25f34719de638689221aa43971be693bf4e
 F ext/session/sessionwor.test 2f3744236dc8b170a695b7d8ddc8c743c7e79fdc
-F ext/session/sqlite3session.c a5b7aed647abe7e366254b755597fce3f2719d82c98990cb0e1e07a3d203fe2b
+F ext/session/sqlite3session.c bd8e52f8b4adef1d886564655030d5a7675baf59d52542c8f7eda99971048e13
 F ext/session/sqlite3session.h b4de978c24a48a0d9b3b92ddfb749f4b07461766325ee950f5ecb8384c10606f
 F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386
 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
@@ -1708,7 +1708,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c949b915e893e917315ce21092d4c4bbd3e1b88d5326928f71dcc2f18f300702
-R 6f69334171b7b3454d79aa33bdfe9b61
-U drh
-Z 67038639e12c985c1bd40cee53ce76b7
+P e01177754ad6d9e2d38adddddd2e2e212094dac1154bda5fcee61ca8b678ae0f
+R 72900d8c116195630f7d9bfcd1153ca1
+U dan
+Z e1e8f32ef49baada43e1fbc29b2fefd6
diff --git a/manifest.uuid b/manifest.uuid
index 36563f126d..bad79d7fbb 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e01177754ad6d9e2d38adddddd2e2e212094dac1154bda5fcee61ca8b678ae0f
\ No newline at end of file
+745a9a7fef0f28a57ea3f44899058993f6ecdedda52c81a09a4a9ce09c9004d6
\ No newline at end of file

From e341ec699f0eda5d52e73dd7b6b3ca90a579e626 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 1 Mar 2018 15:20:33 +0000
Subject: [PATCH 487/488] Fix further crashes in sqlite3changeset_apply()
 caused by corrupt changeset blobs.

FossilOrigin-Name: 2c01c72e357296d761493aac47329f15d87afcada2b39606fc018af0e0011bb5
---
 ext/session/session4.test    | 67 +++++++++++++++++++++++++++++++-----
 ext/session/sqlite3session.c | 44 +++++++++++++----------
 manifest                     | 14 ++++----
 manifest.uuid                |  2 +-
 4 files changed, 93 insertions(+), 34 deletions(-)

diff --git a/ext/session/session4.test b/ext/session/session4.test
index d07c57ffd6..bf55e362c6 100644
--- a/ext/session/session4.test
+++ b/ext/session/session4.test
@@ -74,17 +74,68 @@ do_execsql_test 2.0 {
 }
 
 foreach {tn blob} {
-  1 {54010174340012000000}
-  2 {54fefe8bcb0012000300}
-  3 {5480809280808001017434001200fb}
-  4 {50af9c939c9c9cb09c9c6400b09c9c6400}
-  5 {12000300}
-  6 {09847304}
-  7 {5401017434001208}
+  1 54010174340012000000
+  2 54fefe8bcb0012000300
+  3 5480809280808001017434001200fb
+  4 50af9c939c9c9cb09c9c6400b09c9c6400
+  5 12000300
+  6 09847304
+  7 5401017434001208
+  8 54010174340012fc0386868600
+  9 54010174340012FC0386868600
+  10 548894FEFE
+  11 54010171340012E703ABFA7433FD1200
+  12 540101743400120003FFED00010000000000000002120002400C00000000000054040100000074310017000100000000000000050100000000000000030100000000000000040000010000000000000004010000000000000003001700010000000000000007030378797A01000000000000000F000001000000000000000F030378797A005403010000743200090003037838790100000000800000000200000000000000000900030378327902400C0000000000000304666F7572
+  13 540101743400120003001200010000000000000002120002400C0000000000005404010000007431001700010000000000000005010000000000000003010000000000000004000001000000000000000401000000000000000300170001000000000000000703FC87797A01000000000000000F000001000000000000000F030378797A005403010000743200090003037838790100000000800000000200000000000000000900030378327902400C0000000000000304666F7572
+  14 540101743400120003001200010000000000000002120002400C00000000000054040100000074310017000100000000000000050100000000000000030100000000000000040000010000000000000004010000000000000003001700010000000000000007030378797A01000000000000000F000001000000000000000F03FC87797A005403010000743200090003037838790100000000800000000200000000000000000900030378327902400C0000000000000304666F7572
+  15 540101743400120003001200010000000000000002120002400C00000000000054040100000074310017000100000000000000050100000000000000030100000000000000040000010000000000000004010000000000000003001700010000000000000007030378797A01000000000000000F000001000000000000000F030378797A005403010000743200090003FC8738790100000000800000000200000000000000000900030378327902400C0000000000000304666F7572
+  16 540101743400120003001200010000000000000002120002400C00000000000054040100000074310017000100000000000000050100000000000000030100000000000000040000010000000000000004010000000000000003001700010000000000000007030378797A01000000000000000F000001000000000000000F030378797A00540301000074320009000303783879010000000080000000020000000000000000090003FC87327902400C0000000000000304666F7572
+  17 540101743400120003FFE3000412F7010000E600000000021202120002400C0000000000005B0401000000743100171C0304646F750002400C000000000000540401000000D3310017000100000000000000050100000000000378797A405403000002F10100000100000000000004090001000100000007030378797A0100000000000D0007000001000000002300000F1B0378797A405403013900743200090003038C3879010000000000000000000002120002400C0000000000005B0401000000743117170003047C5E00FF
+  18 54010174340012000300120001000000E6FF100000120002401E00000000000054040100000074310017000100040000010000000000000004FFFF7FFF0000000000010000010000001000000007030378797A01000000000000000F000000000000FA0304666F7572
+  19 540101743400120003001200010000000000000002121B02400C00000000000054040000000074310017000100000000000000050100000000000000030100000000000000040000010000000000000004010000000000000003001700010000000000000007030378817A01000000000000000F000001000000000100000F030378797A005403010000743200090003FFE809000303780000000000000304666F7572
+  20 5401017D3400120003001200010000000000000002120002400CFC00000000005404010000007431001700010000000000000005010000000000000003010000000000000004000001000000000000000401000000000000000300170001000000000000000703FFFF797A01000000000000000F000001000000000000000F030378797A005403010000743200090003037838790100000000800000000200000000000000000900030378326C02400C0000000000000304666F7572
+  21 5401017434001200030012000100FFE20000000002120002400C00000000000054040100E0007431001700010000E99D000000020000000003FFE70009000303783279020004000001030000000000002117000003001700012701000100000000743100000100000000008000090003037F387901000000008000000002000000000400000009005303010A00FF7FFFFF00000000000304664F6572
+  22 540101743400120003FFFF7FFF0000000000000002120002400C00000000000054040100000074310017000100000000000000050100000000000000030100010000000000000000040000010000000000000004010000000000000003001700010000000000000007030378797A01000000000000000F000001000000000000000F030378797A005403010000743200090003037838790100000000800000000200000000000000000900030378327902400C0000000000000304666F7572
+  23 540101742700120100120003F5FF0300
+  24 5401017434E312540101743400120003FFFC00
+  25 540101743400540101743D3D3D3D3D3D3D3D3D3D3D3D3D3400120003FFED000300
+  26 5401017446EA5301743D1D3D3D01743D1D3D3DCF3D3D3D1A3D3D3D3D3400120003FFFF000000
+  27 540101743400540101743D3D3D3D3D3D3D3D3D3D251000120003FF81000000000000
+  28 540101340012000397FF3D7F3D3400120003001200540101743D3D3D3D3D3D393D3D3D12000300
+  29 500174340050010F74340012000300120003FFE5
+  30 5004007233E900177FEF0054257F0002EF001200031E12000300
+  31 5001015001015252525250010174340012EF039A9A0100E351525D52525252525252525252525252525252525250010174340012EF039A0100009A9A9A9A9A9BA3B200120003010040743400
+  32 5401017400123400120003FFFC00
+  33 540101743400120003001200010000000000004002120002400C0000000000005404010000007431001700010000000000000005010000000000000003010000000000000004000001000000000000000401000000000000000300170001000000000000000703FC87797A01000000000000000F000001000000000000000F030378797A005403010000743200090003037838790100000000800000000200000000000000000900030378327902400C0000000000000304666F7572
+  34 54040100000074310017000100000002000015050100000000000000030100000000140000040000010000000000000004010000000000000003001700010000000000000007030378797A01000000000000000F000001000000000000000F030378797A0054030100007432000900030378387901000000008E000000020000000000000000090003FFFF000002400C0000000000000304666F7572
+  35 540101743400120003001200010000000000000002120002400C00000000000050060100000074310017000100000000000000050100000000000000030100000003001700010000666F7572
+  36 540101743400120003001200010000000000000002120002400C00000000000050050100000074310017000100000000000000050100000000000000030100000003001700010000666F7572
+  37 540101743400120003001200010000000000000002120002400C00000000000050040100008074310017000100000000000000050100000000000000030100000003001700010000666F7572
+  38 540101743400120003001200010000000000000002120002400C00000000000050040100000074310017000000000000000000050100000000000000030100000003001700010000666F7572
+  39 540101743400120003001200010000000000000002120002400C00000000000050040100018074310017000100000000000000050100000000000000030100000003001700010000666F7572
+  40 540101743400120003001200010000000000000002120002400C0000000000005004FEFFFFFF74310017000100000000000000050100000000000000030100000003001700010000666F7572
+  41 540101743400120003001200010000000000000002120002400C00000000000050040100000074310017000004000000000000050100000000000000030100000003001700010000666F7572
+  42 540101743400120003001200010000000000000002120002400C0000000000005005FFFF050074310017000100000000000000050100000000000000030100000003001700010000666F7572
+  43 540101743400120003001200010000000000000002120002400C000000000000500401006E0074310017000300000000001221050100000000000000030100000003001700010000666F7572
+  44 540101743400120003001200010000000000020000120002400C00000000000050050100000074310017000100000000000000050100004000000000030100000025001700010000666F7572
+  45 540101743400120003001200010000000000ECFF02120002400C000000000000500401F9FF00743100170001000000000000000500E1000000000000030100000003000000000000666F7572
+  46 54010174340B0B0B0B0B0B0B0B0B0B0B0B0B0B0B00120003001200010000000000000002120002400C00000000000050040100000074310017010000000000000000050100FFE900000000030100000003007F00000000666F7572
+  47 54010103001200010000000000020002120002400C0000000000005004010000F374310017000100000000000000050100000000000000030100000003001700010000666F8E72
+  48 540101743400120003001200010000000000000002120002400C00000000000050030012000174310017000700000000000000050100002000000001000000000003001700010000666F7572
+  49 540101743400120004001200010000000000000002120002400C0000000000005004010000FC733100170001000000000000000501000000000000000301000000F6FF17000100007C6F7572
+  50 54010174FFDDFF8003001200010000100000000002120002400C000000000000500401000000743100170000000005010000000000000000000003010072
+  51 540101743200120003001200010000000000000002120002400C00000000000050040100001074310017000000000003010000120300170100000000000000050100000000000000030100000003001700010000666F7572
+  52 540101745401017434001200010000000000001702120002400C00000000000050040100001A74310017000100000000000100000100000000000000030100000003001700010000666F7572
+  53 540101743400120003001200010000000000000002120002400C000000000000500401000000743100170001000002400C00000000000050040110000074310017000000000000050100000000000000030100000003001700010000666F7572
+  54 540101743400120003001200010000000000000002120002400C000000000002120002400C00000000000050040100000074310017FF0050040100000074310017FF7F00000000000000050100000000000000030100000003001700010000666F7572
+  55 540101743400120003001200010000000000000002120002400C00000000000050040100000074310017000100010080000001000000020003010100000300170100000003001700010000666F7572
 } {
-if {$tn==7} breakpoint
   do_test 2.$tn {
     set changeset [binary decode hex $blob]
+#set fd [open x.change w+]
+#fconfigure $fd -encoding binary -translation binary
+#puts -nonewline $fd $changeset
+#close $fd
     list [catch { sqlite3changeset_apply db $changeset xConflict } msg] $msg
   } {1 SQLITE_CORRUPT}
 }
diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c
index 2a130c7e9e..04ca5b2b30 100644
--- a/ext/session/sqlite3session.c
+++ b/ext/session/sqlite3session.c
@@ -2718,17 +2718,17 @@ static int sessionReadRecord(
   for(i=0; i<nCol && rc==SQLITE_OK; i++){
     int eType = 0;                /* Type of value (SQLITE_NULL, TEXT etc.) */
     if( abPK && abPK[i]==0 ) continue;
-    if( pIn->iNext>=pIn->nData ){
-      rc = SQLITE_CORRUPT;
-    }else{
-      rc = sessionInputBuffer(pIn, 9);
-    }
+    rc = sessionInputBuffer(pIn, 9);
     if( rc==SQLITE_OK ){
-      eType = pIn->aData[pIn->iNext++];
-      assert( apOut[i]==0 );
-      if( eType ){
-        apOut[i] = sqlite3ValueNew(0);
-        if( !apOut[i] ) rc = SQLITE_NOMEM;
+      if( pIn->iNext>=pIn->nData ){
+        rc = SQLITE_CORRUPT_BKPT;
+      }else{
+        eType = pIn->aData[pIn->iNext++];
+        assert( apOut[i]==0 );
+        if( eType ){
+          apOut[i] = sqlite3ValueNew(0);
+          if( !apOut[i] ) rc = SQLITE_NOMEM;
+        }
       }
     }
 
@@ -2739,10 +2739,14 @@ static int sessionReadRecord(
         pIn->iNext += sessionVarintGet(aVal, &nByte);
         rc = sessionInputBuffer(pIn, nByte);
         if( rc==SQLITE_OK ){
-          u8 enc = (eType==SQLITE_TEXT ? SQLITE_UTF8 : 0);
-          rc = sessionValueSetStr(apOut[i],&pIn->aData[pIn->iNext],nByte,enc);
+          if( nByte<0 || nByte>pIn->nData-pIn->iNext ){
+            rc = SQLITE_CORRUPT_BKPT;
+          }else{
+            u8 enc = (eType==SQLITE_TEXT ? SQLITE_UTF8 : 0);
+            rc = sessionValueSetStr(apOut[i],&pIn->aData[pIn->iNext],nByte,enc);
+            pIn->iNext += nByte;
+          }
         }
-        pIn->iNext += nByte;
       }
       if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){
         sqlite3_int64 v = sessionGetI64(aVal);
@@ -2782,8 +2786,12 @@ static int sessionChangesetBufferTblhdr(SessionInput *pIn, int *pnByte){
   rc = sessionInputBuffer(pIn, 9);
   if( rc==SQLITE_OK ){
     nRead += sessionVarintGet(&pIn->aData[pIn->iNext + nRead], &nCol);
-    rc = sessionInputBuffer(pIn, nRead+nCol+100);
-    nRead += nCol;
+    if( nCol<0 ){
+      rc = SQLITE_CORRUPT_BKPT;
+    }else{
+      rc = sessionInputBuffer(pIn, nRead+nCol+100);
+      nRead += nCol;
+    }
   }
 
   while( rc==SQLITE_OK ){
@@ -2867,7 +2875,7 @@ static int sessionChangesetReadTblhdr(sqlite3_changeset_iter *p){
       p->tblhdr.nBuf = 0;
       sessionBufferGrow(&p->tblhdr, nByte, &rc);
     }else{
-      rc = SQLITE_CORRUPT;
+      rc = SQLITE_CORRUPT_BKPT;
     }
   }
 
@@ -2995,9 +3003,9 @@ static int sessionChangesetNext(
       ** new.* to old.*, to accommodate the code that reads these arrays.  */
       for(i=0; i<p->nCol; i++){
         assert( p->apValue[i]==0 );
-        assert( p->abPK[i]==0 || p->apValue[i+p->nCol] );
         if( p->abPK[i] ){
           p->apValue[i] = p->apValue[i+p->nCol];
+          if( p->apValue[i]==0 ) return (p->rc = SQLITE_CORRUPT_BKPT);
           p->apValue[i+p->nCol] = 0;
         }
       }
@@ -3725,7 +3733,7 @@ static int sessionBindRow(
       if( pVal==0 ){
         /* The value in the changeset was "undefined". This indicates a
         ** corrupt changeset blob.  */
-        rc = SQLITE_CORRUPT;
+        rc = SQLITE_CORRUPT_BKPT;
       }else{
         rc = sessionBindValue(pStmt, i+1, pVal);
       }
diff --git a/manifest b/manifest
index f09706c7be..a8dbce79c3 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\ssome\scrashes\sin\sthe\ssqlite3changeset_apply()\sfunction\sthat\scould\sbe\scaused\nby\scorrupt\schangeset\sblobs.
-D 2018-03-01T12:05:51.293
+C Fix\sfurther\scrashes\sin\ssqlite3changeset_apply()\scaused\sby\scorrupt\schangeset\nblobs.
+D 2018-03-01T15:20:33.861
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737
@@ -381,7 +381,7 @@ F ext/session/changeset.c 4ccbaa4531944c24584bf6a61ba3a39c62b6267a
 F ext/session/session1.test 736d7ff178662f0b717c37f46531b84a5ce0210ccb0c4edf629c55dbcbbc3ea1
 F ext/session/session2.test 284de45abae4cc1082bc52012ee81521d5ac58e0
 F ext/session/session3.test ce9ce3dfa489473987f899e9f6a0f2db9bde3479
-F ext/session/session4.test 488539cee1d2510b415236fac2727575f4110e7609a500eb21c8d9e517dfff38
+F ext/session/session4.test efd7a46ed6a954d51ab00bdc4d656d2bc31e46be64393224cf6acf1319fbd32c
 F ext/session/session5.test 716bc6fafd625ce60dfa62ae128971628c1a1169
 F ext/session/session6.test 443789bc2fca12e4f7075cf692c60b8a2bea1a26
 F ext/session/session8.test 8e194b3f655d861ca36de5d4de53f702751bab3b
@@ -402,7 +402,7 @@ F ext/session/sessionfault.test da273f2712b6411e85e71465a1733b8501dbf6f7
 F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0
 F ext/session/sessionstat1.test 41cd97c2e48619a41cdf8ae749e1b25f34719de638689221aa43971be693bf4e
 F ext/session/sessionwor.test 2f3744236dc8b170a695b7d8ddc8c743c7e79fdc
-F ext/session/sqlite3session.c bd8e52f8b4adef1d886564655030d5a7675baf59d52542c8f7eda99971048e13
+F ext/session/sqlite3session.c 0b7f1b8eb5b5a83fd96127b93139eadd2f2e2915c1eaceab4f5d771719c0c22f
 F ext/session/sqlite3session.h b4de978c24a48a0d9b3b92ddfb749f4b07461766325ee950f5ecb8384c10606f
 F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386
 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
@@ -1708,7 +1708,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e01177754ad6d9e2d38adddddd2e2e212094dac1154bda5fcee61ca8b678ae0f
-R 72900d8c116195630f7d9bfcd1153ca1
+P 745a9a7fef0f28a57ea3f44899058993f6ecdedda52c81a09a4a9ce09c9004d6
+R 6fcbb31fa405c62d7397cf36ed752e3f
 U dan
-Z e1e8f32ef49baada43e1fbc29b2fefd6
+Z 16996797a4de130c696d538f1e3a40ad
diff --git a/manifest.uuid b/manifest.uuid
index bad79d7fbb..896e1fd2d4 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-745a9a7fef0f28a57ea3f44899058993f6ecdedda52c81a09a4a9ce09c9004d6
\ No newline at end of file
+2c01c72e357296d761493aac47329f15d87afcada2b39606fc018af0e0011bb5
\ No newline at end of file

From 5e9825ecc4cbafa141c3993c249a381b4b3240df Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Thu, 1 Mar 2018 18:09:02 +0000
Subject: [PATCH 488/488] Typo fixes in comments.  No changes to code.

FossilOrigin-Name: 1293d4f64f322ee57510bdff497c54e0be2c9205217e1fd96887bba1f36bec40
---
 ext/session/sqlite3session.h |  2 +-
 manifest                     | 16 ++++++++--------
 manifest.uuid                |  2 +-
 src/vdbe.c                   |  2 +-
 4 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/ext/session/sqlite3session.h b/ext/session/sqlite3session.h
index 81b1ed06d6..407ec6a62b 100644
--- a/ext/session/sqlite3session.h
+++ b/ext/session/sqlite3session.h
@@ -22,7 +22,7 @@ typedef struct sqlite3_session sqlite3_session;
 /*
 ** CAPI3REF: Changeset Iterator Handle
 **
-** An instance of this object is as as a cursor for iterating
+** An instance of this object acts as a cursor for iterating
 ** over the elements of a [changeset] or [patchset].
 */
 typedef struct sqlite3_changeset_iter sqlite3_changeset_iter;
diff --git a/manifest b/manifest
index a8dbce79c3..1072fdeb6d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sfurther\scrashes\sin\ssqlite3changeset_apply()\scaused\sby\scorrupt\schangeset\nblobs.
-D 2018-03-01T15:20:33.861
+C Typo\sfixes\sin\scomments.\s\sNo\schanges\sto\scode.
+D 2018-03-01T18:09:02.014
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737
@@ -403,7 +403,7 @@ F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0
 F ext/session/sessionstat1.test 41cd97c2e48619a41cdf8ae749e1b25f34719de638689221aa43971be693bf4e
 F ext/session/sessionwor.test 2f3744236dc8b170a695b7d8ddc8c743c7e79fdc
 F ext/session/sqlite3session.c 0b7f1b8eb5b5a83fd96127b93139eadd2f2e2915c1eaceab4f5d771719c0c22f
-F ext/session/sqlite3session.h b4de978c24a48a0d9b3b92ddfb749f4b07461766325ee950f5ecb8384c10606f
+F ext/session/sqlite3session.h 2e1584b030fbd841cefdce15ba984871978d305f586da2d1972f6e1958fa10b1
 F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386
 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04
@@ -557,7 +557,7 @@ F src/update.c a90a32ffc0100265b0693dbbdbe490756447af181f5ea2c138cce515b08c8795
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c d9eb0a6c4aae1b00a7369eadd7ca0bbe946cb4c953b6751aa20d357c2f482157
 F src/vacuum.c 762ee9bbf8733d87d8cd06f58d950e881982e416f8c767334a40ffd341b6bff5
-F src/vdbe.c 99a0f275c83c01332b9af6179f2ee1e5d24ad81f90f17f8364397f8a3d20fb40
+F src/vdbe.c 88d8e0797bf49624d056014f34f302a7370cc119915cc0ece8f2b7bf5e7c30fb
 F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a
 F src/vdbeInt.h 95f7adfdc5c8f1353321f55a6c5ec00a90877e3b85af5159e393afb41ff54110
 F src/vdbeapi.c fea41171884a4de119f8b10ab514c788674eeeb7f27218bb6d008e1310bfd07f
@@ -1708,7 +1708,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 745a9a7fef0f28a57ea3f44899058993f6ecdedda52c81a09a4a9ce09c9004d6
-R 6fcbb31fa405c62d7397cf36ed752e3f
-U dan
-Z 16996797a4de130c696d538f1e3a40ad
+P 2c01c72e357296d761493aac47329f15d87afcada2b39606fc018af0e0011bb5
+R 74d9f04f790ae61bbdb4bfdda923d919
+U mistachkin
+Z 6963c71440aaaba3f2df4c41ff5161b5
diff --git a/manifest.uuid b/manifest.uuid
index 896e1fd2d4..168b5b04bf 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2c01c72e357296d761493aac47329f15d87afcada2b39606fc018af0e0011bb5
\ No newline at end of file
+1293d4f64f322ee57510bdff497c54e0be2c9205217e1fd96887bba1f36bec40
\ No newline at end of file
diff --git a/src/vdbe.c b/src/vdbe.c
index 10715a8232..4a506dd592 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -2379,7 +2379,7 @@ case OP_IfNullRow: {         /* jump */
 ** P2 is the column number for the argument to the sqlite_offset() function.
 ** This opcode does not use P2 itself, but the P2 value is used by the
 ** code generator.  The P1, P2, and P3 operands to this opcode are the
-** as as for OP_Column.
+** same as for OP_Column.
 **
 ** This opcode is only available if SQLite is compiled with the
 ** -DSQLITE_ENABLE_OFFSET_SQL_FUNC option.