diff --git a/Makefile.in b/Makefile.in
index d2f9710059..028f615cd7 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -1306,6 +1306,9 @@ showwal$(TEXE):	$(TOP)/tool/showwal.c sqlite3.lo
 showshm$(TEXE):	$(TOP)/tool/showshm.c
 	$(LTLINK) -o $@ $(TOP)/tool/showshm.c
 
+index_usage$(TEXE): $(TOP)/tool/index_usage.c sqlite3.lo
+	$(LTLINK) -o $@ $(TOP)/tool/index_usage.c sqlite3.lo $(TLIBS)
+
 changeset$(TEXE):	$(TOP)/ext/session/changeset.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/ext/session/changeset.c sqlite3.lo $(TLIBS)
 
diff --git a/Makefile.msc b/Makefile.msc
index e3feffab35..3b95d088d2 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -2444,6 +2444,10 @@ showwal.exe:	$(TOP)\tool\showwal.c $(SQLITE3C) $(SQLITE3H)
 showshm.exe:	$(TOP)\tool\showshm.c
 	$(LTLINK) $(NO_WARN)	$(TOP)\tool\showshm.c /link $(LDFLAGS) $(LTLINKOPTS)
 
+index_usage.exe: $(TOP)\tool\index_usage.c $(SQLITE3C) $(SQLITE3H)
+	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
+		$(TOP)\tool\index_usage.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
+
 changeset.exe:	$(TOP)\ext\session\changeset.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
 		-DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 \
diff --git a/autoconf/Makefile.msc b/autoconf/Makefile.msc
index 270c83c230..a131d4c6e7 100644
--- a/autoconf/Makefile.msc
+++ b/autoconf/Makefile.msc
@@ -283,6 +283,7 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBSTAT_VTAB=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_INTROSPECTION_PRAGMAS=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DESERIALIZE=1
 !ENDIF
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
 !ENDIF
@@ -937,6 +938,7 @@ LIBRESOBJS =
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DESERIALIZE=1
 !ENDIF
 
 
diff --git a/ext/fts3/fts3_unicode.c b/ext/fts3/fts3_unicode.c
index dfb2680c50..c02ea3990c 100644
--- a/ext/fts3/fts3_unicode.c
+++ b/ext/fts3/fts3_unicode.c
@@ -82,7 +82,7 @@ typedef struct unicode_cursor unicode_cursor;
 
 struct unicode_tokenizer {
   sqlite3_tokenizer base;
-  int bRemoveDiacritic;
+  int eRemoveDiacritic;
   int nException;
   int *aiException;
 };
@@ -227,17 +227,20 @@ static int unicodeCreate(
   pNew = (unicode_tokenizer *) sqlite3_malloc(sizeof(unicode_tokenizer));
   if( pNew==NULL ) return SQLITE_NOMEM;
   memset(pNew, 0, sizeof(unicode_tokenizer));
-  pNew->bRemoveDiacritic = 1;
+  pNew->eRemoveDiacritic = 1;
 
   for(i=0; rc==SQLITE_OK && i<nArg; i++){
     const char *z = azArg[i];
     int n = (int)strlen(z);
 
     if( n==19 && memcmp("remove_diacritics=1", z, 19)==0 ){
-      pNew->bRemoveDiacritic = 1;
+      pNew->eRemoveDiacritic = 1;
     }
     else if( n==19 && memcmp("remove_diacritics=0", z, 19)==0 ){
-      pNew->bRemoveDiacritic = 0;
+      pNew->eRemoveDiacritic = 0;
+    }
+    else if( n==19 && memcmp("remove_diacritics=2", z, 19)==0 ){
+      pNew->eRemoveDiacritic = 2;
     }
     else if( n>=11 && memcmp("tokenchars=", z, 11)==0 ){
       rc = unicodeAddExceptions(pNew, 1, &z[11], n-11);
@@ -350,7 +353,7 @@ static int unicodeNext(
 
     /* Write the folded case of the last character read to the output */
     zEnd = z;
-    iOut = sqlite3FtsUnicodeFold((int)iCode, p->bRemoveDiacritic);
+    iOut = sqlite3FtsUnicodeFold((int)iCode, p->eRemoveDiacritic);
     if( iOut ){
       WRITE_UTF8(zOut, iOut);
     }
diff --git a/ext/fts3/fts3_unicode2.c b/ext/fts3/fts3_unicode2.c
index da7251ed0c..41027b2546 100644
--- a/ext/fts3/fts3_unicode2.c
+++ b/ext/fts3/fts3_unicode2.c
@@ -159,32 +159,47 @@ int sqlite3FtsUnicodeIsalnum(int c){
 ** E"). The resuls of passing a codepoint that corresponds to an
 ** uppercase letter are undefined.
 */
-static int remove_diacritic(int c){
+static int remove_diacritic(int c, int bComplex){
   unsigned short aDia[] = {
         0,  1797,  1848,  1859,  1891,  1928,  1940,  1995, 
      2024,  2040,  2060,  2110,  2168,  2206,  2264,  2286, 
      2344,  2383,  2472,  2488,  2516,  2596,  2668,  2732, 
      2782,  2842,  2894,  2954,  2984,  3000,  3028,  3336, 
-     3456,  3696,  3712,  3728,  3744,  3896,  3912,  3928, 
-     3968,  4008,  4040,  4106,  4138,  4170,  4202,  4234, 
-     4266,  4296,  4312,  4344,  4408,  4424,  4472,  4504, 
-     6148,  6198,  6264,  6280,  6360,  6429,  6505,  6529, 
-    61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, 
-    61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, 
-    62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, 
-    62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, 
-    62924, 63050, 63082, 63274, 63390, 
+     3456,  3696,  3712,  3728,  3744,  3766,  3832,  3896, 
+     3912,  3928,  3944,  3968,  4008,  4040,  4056,  4106, 
+     4138,  4170,  4202,  4234,  4266,  4296,  4312,  4344, 
+     4408,  4424,  4442,  4472,  4488,  4504,  6148,  6198, 
+     6264,  6280,  6360,  6429,  6505,  6529, 61448, 61468, 
+    61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, 
+    61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, 
+    61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, 
+    62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, 
+    62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, 
+    62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, 
+    63182, 63242, 63274, 63310, 63368, 63390, 
   };
   char aChar[] = {
-    '\0', 'a',  'c',  'e',  'i',  'n',  'o',  'u',  'y',  'y',  'a',  'c',  
-    'd',  'e',  'e',  'g',  'h',  'i',  'j',  'k',  'l',  'n',  'o',  'r',  
-    's',  't',  'u',  'u',  'w',  'y',  'z',  'o',  'u',  'a',  'i',  'o',  
-    'u',  'g',  'k',  'o',  'j',  'g',  'n',  'a',  'e',  'i',  'o',  'r',  
-    'u',  's',  't',  'h',  'a',  'e',  'o',  'y',  '\0', '\0', '\0', '\0', 
-    '\0', '\0', '\0', '\0', 'a',  'b',  'd',  'd',  'e',  'f',  'g',  'h',  
-    'h',  'i',  'k',  'l',  'l',  'm',  'n',  'p',  'r',  'r',  's',  't',  
-    'u',  'v',  'w',  'w',  'x',  'y',  'z',  'h',  't',  'w',  'y',  'a',  
-    'e',  'i',  'o',  'u',  'y',  
+    '\0',      'a'|0x00,  'c'|0x00,  'e'|0x00,  'i'|0x00,  'n'|0x00,  
+    'o'|0x00,  'u'|0x00,  'y'|0x00,  'y'|0x00,  'a'|0x00,  'c'|0x00,  
+    'd'|0x00,  'e'|0x00,  'e'|0x00,  'g'|0x00,  'h'|0x00,  'i'|0x00,  
+    'j'|0x00,  'k'|0x00,  'l'|0x00,  'n'|0x00,  'o'|0x00,  'r'|0x00,  
+    's'|0x00,  't'|0x00,  'u'|0x00,  'u'|0x00,  'w'|0x00,  'y'|0x00,  
+    'z'|0x00,  'o'|0x00,  'u'|0x00,  'a'|0x00,  'i'|0x00,  'o'|0x00,  
+    'u'|0x00,  'u'|0x80,  'a'|0x80,  'g'|0x00,  'k'|0x00,  'o'|0x00,  
+    'o'|0x80,  'j'|0x00,  'g'|0x00,  'n'|0x00,  'a'|0x80,  'a'|0x00,  
+    'e'|0x00,  'i'|0x00,  'o'|0x00,  'r'|0x00,  'u'|0x00,  's'|0x00,  
+    't'|0x00,  'h'|0x00,  'a'|0x00,  'e'|0x00,  'o'|0x80,  'o'|0x00,  
+    'o'|0x80,  'y'|0x00,  '\0',      '\0',      '\0',      '\0',      
+    '\0',      '\0',      '\0',      '\0',      'a'|0x00,  'b'|0x00,  
+    'c'|0x80,  'd'|0x00,  'd'|0x00,  'e'|0x80,  'e'|0x00,  'e'|0x80,  
+    'f'|0x00,  'g'|0x00,  'h'|0x00,  'h'|0x00,  'i'|0x00,  'i'|0x80,  
+    'k'|0x00,  'l'|0x00,  'l'|0x80,  'l'|0x00,  'm'|0x00,  'n'|0x00,  
+    'o'|0x80,  'p'|0x00,  'r'|0x00,  'r'|0x80,  'r'|0x00,  's'|0x00,  
+    's'|0x80,  't'|0x00,  'u'|0x00,  'u'|0x80,  'v'|0x00,  'w'|0x00,  
+    'w'|0x00,  'x'|0x00,  'y'|0x00,  'z'|0x00,  'h'|0x00,  't'|0x00,  
+    'w'|0x00,  'y'|0x00,  'a'|0x00,  'a'|0x80,  'a'|0x80,  'a'|0x80,  
+    'e'|0x00,  'e'|0x80,  'e'|0x80,  'i'|0x00,  'o'|0x00,  'o'|0x80,  
+    'o'|0x80,  'o'|0x80,  'u'|0x00,  'u'|0x80,  'u'|0x80,  'y'|0x00,  
   };
 
   unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
@@ -201,7 +216,8 @@ static int remove_diacritic(int c){
     }
   }
   assert( key>=aDia[iRes] );
-  return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);
+  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
+  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);
 }
 
 
@@ -228,7 +244,7 @@ int sqlite3FtsUnicodeIsdiacritic(int c){
 ** The results are undefined if the value passed to this function
 ** is less than zero.
 */
-int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){
+int sqlite3FtsUnicodeFold(int c, int eRemoveDiacritic){
   /* Each entry in the following array defines a rule for folding a range
   ** of codepoints to lower case. The rule applies to a range of nRange
   ** codepoints starting at codepoint iCode.
@@ -351,7 +367,9 @@ int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){
       assert( ret>0 );
     }
 
-    if( bRemoveDiacritic ) ret = remove_diacritic(ret);
+    if( eRemoveDiacritic ){
+      ret = remove_diacritic(ret, eRemoveDiacritic==2);
+    }
   }
   
   else if( c>=66560 && c<66600 ){
diff --git a/ext/fts3/unicode/mkunicode.tcl b/ext/fts3/unicode/mkunicode.tcl
index 84b8ddc80b..8465262033 100644
--- a/ext/fts3/unicode/mkunicode.tcl
+++ b/ext/fts3/unicode/mkunicode.tcl
@@ -9,11 +9,12 @@ proc print_rd {map} {
   set nRange 1
   set iFirst  [lindex $map 0 0]
   set cPrev   [lindex $map 0 1]
+  set fPrev   [lindex $map 0 2]
 
   foreach m [lrange $map 1 end] {
-    foreach {i c} $m {}
+    foreach {i c f} $m {}
 
-    if {$cPrev == $c} {
+    if {$cPrev == $c && $fPrev==$f} {
       for {set j [expr $iFirst+$nRange]} {$j<$i} {incr j} {
         if {[info exists tl_lookup_table($j)]==0} break
       }
@@ -29,13 +30,16 @@ proc print_rd {map} {
 
     lappend lRange [list $iFirst $nRange]
     lappend aChar  $cPrev
+    lappend aFlag  $fPrev
 
     set iFirst $i
     set cPrev  $c
+    set fPrev  $f
     set nRange 1
   }
   lappend lRange [list $iFirst $nRange]
   lappend aChar $cPrev
+  lappend aFlag $fPrev
 
   puts "/*"
   puts "** If the argument is a codepoint corresponding to a lowercase letter"
@@ -45,7 +49,7 @@ proc print_rd {map} {
   puts "** E\"). The resuls of passing a codepoint that corresponds to an"
   puts "** uppercase letter are undefined."
   puts "*/"
-  puts "static int ${::remove_diacritic}(int c)\{"
+  puts "static int ${::remove_diacritic}(int c, int bComplex)\{"
   puts "  unsigned short aDia\[\] = \{"
   puts -nonewline "        0, "
   set i 1
@@ -60,13 +64,17 @@ proc print_rd {map} {
   puts ""
   puts "  \};"
   puts "  char aChar\[\] = \{"
-  puts -nonewline "    '\\0', "
+  puts -nonewline "    '\\0',      "
   set i 1
-  foreach c $aChar {
-    set str "'$c',  "
-    if {$c == ""} { set str "'\\0', " }
+  foreach c $aChar f $aFlag {
+    if { $f } {
+      set str "'$c'|0x80,  "
+    } else {
+      set str "'$c'|0x00,  "
+    }
+    if {$c == ""} { set str "'\\0',      " }
 
-    if {($i % 12)==0} {puts "" ; puts -nonewline "    " }
+    if {($i % 6)==0} {puts "" ; puts -nonewline "    " }
     incr i
     puts -nonewline "$str"
   }
@@ -87,7 +95,8 @@ proc print_rd {map} {
     }
   }
   assert( key>=aDia[iRes] );
-  return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);}
+  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
+  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);}
   puts "\}"
 }
 
@@ -95,7 +104,8 @@ proc print_isdiacritic {zFunc map} {
 
   set lCode [list]
   foreach m $map {
-    foreach {code char} $m {}
+    foreach {code char flag} $m {}
+    if {$flag} continue
     if {$code && $char == ""} { lappend lCode $code }
   }
   set lCode [lsort -integer $lCode]
@@ -472,7 +482,7 @@ proc print_fold {zFunc} {
   puts "** The results are undefined if the value passed to this function"
   puts "** is less than zero."
   puts "*/"
-  puts "int ${zFunc}\(int c, int bRemoveDiacritic)\{"
+  puts "int ${zFunc}\(int c, int eRemoveDiacritic)\{"
 
   set liOff [tl_generate_ioff_table $lRecord]
   tl_print_table_header
@@ -516,7 +526,9 @@ proc print_fold {zFunc} {
       assert( ret>0 );
     }
 
-    if( bRemoveDiacritic ) ret = ${::remove_diacritic}(ret);
+    if( eRemoveDiacritic ){
+      ret = ${::remove_diacritic}(ret, eRemoveDiacritic==2);
+    }
   }
   }]
 
@@ -605,10 +617,6 @@ proc print_categories {lMap} {
 
   set nCat [expr [llength [array names C]] + 1]
   puts [code {
-    int sqlite3Fts5UnicodeNCat(void) { 
-      return $nCat;
-    }
-
     int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ 
       aArray[0] = 1;
       switch( zCat[0] ){
diff --git a/ext/fts3/unicode/parseunicode.tcl b/ext/fts3/unicode/parseunicode.tcl
index 966d7bdd3a..7c246a4a09 100644
--- a/ext/fts3/unicode/parseunicode.tcl
+++ b/ext/fts3/unicode/parseunicode.tcl
@@ -7,12 +7,24 @@
 # character that it should be replaced with, or an empty string if the 
 # codepoint should simply be removed from the input. Examples:
 #
-#   { 224 a  }     (replace codepoint 224 to "a")
-#   { 769 "" }     (remove codepoint 769 from input)
+#   { 224 a  0 }     (replace codepoint 224 to "a")
+#   { 769 "" 0 }     (remove codepoint 769 from input)
 #
 # Mappings are only returned for non-upper case codepoints. It is assumed
 # that the input has already been folded to lower case.
 #
+# The third value in the list is always either 0 or 1. 0 if the 
+# UnicodeData.txt file maps the codepoint to a single ASCII character and
+# a diacritic, or 1 if the mapping is indirect. For example, consider the 
+# two entries:
+#
+# 1ECD;LATIN SMALL LETTER O WITH DOT BELOW;Ll;0;L;006F 0323;;;;N;;;1ECC;;1ECC
+# 1ED9;LATIN SMALL LETTER O WITH CIRCUMFLEX AND DOT BELOW;Ll;0;L;1ECD 0302;;;;N;;;1ED8;;1ED8
+#
+# The first codepoint is a direct mapping (as 006F is ASCII and 0323 is a 
+# diacritic). The second is an indirect mapping, as it maps to the
+# first codepoint plus 0302 (a diacritic).
+#
 proc rd_load_unicodedata_text {zName} {
   global tl_lookup_table
 
@@ -53,18 +65,29 @@ proc rd_load_unicodedata_text {zName} {
     set iAscii [expr "0x[lindex $character_decomposition_mapping 0]"]
     set iDia   [expr "0x[lindex $character_decomposition_mapping 1]"]
 
+    # Filter out upper-case characters, as they will be mapped to their
+    # lower-case equivalents before this data is used.
     if {[info exists tl_lookup_table($iCode)]} continue
 
+    # Check if this is an indirect mapping. If so, set bIndirect to true
+    # and change $iAscii to the indirectly mappped ASCII character.
+    set bIndirect 0
+    if {[info exists dia($iDia)] && [info exists mapping($iAscii)]} {
+      set iAscii $mapping($iAscii)
+      set bIndirect 1
+    }
+
     if { ($iAscii >= 97 && $iAscii <= 122)
       || ($iAscii >= 65 && $iAscii <= 90)
     } {
-      lappend lRet [list $iCode [string tolower [format %c $iAscii]]]
+      lappend lRet [list $iCode [string tolower [format %c $iAscii]] $bIndirect]
+      set mapping($iCode) $iAscii
       set dia($iDia) 1
     }
   }
 
   foreach d [array names dia] {
-    lappend lRet [list $d ""]
+    lappend lRet [list $d "" 0]
   }
   set lRet [lsort -integer -index 0 $lRet]
 
diff --git a/ext/fts5/fts5_tokenize.c b/ext/fts5/fts5_tokenize.c
index af2bc222f2..35526a3898 100644
--- a/ext/fts5/fts5_tokenize.c
+++ b/ext/fts5/fts5_tokenize.c
@@ -234,13 +234,18 @@ struct Unicode61Tokenizer {
   unsigned char aTokenChar[128];  /* ASCII range token characters */
   char *aFold;                    /* Buffer to fold text into */
   int nFold;                      /* Size of aFold[] in bytes */
-  int bRemoveDiacritic;           /* True if remove_diacritics=1 is set */
+  int eRemoveDiacritic;           /* True if remove_diacritics=1 is set */
   int nException;
   int *aiException;
 
   unsigned char aCategory[32];    /* True for token char categories */
 };
 
+/* Values for eRemoveDiacritic (must match internals of fts5_unicode2.c) */
+#define FTS5_REMOVE_DIACRITICS_NONE    0
+#define FTS5_REMOVE_DIACRITICS_SIMPLE  1
+#define FTS5_REMOVE_DIACRITICS_COMPLEX 2
+
 static int fts5UnicodeAddExceptions(
   Unicode61Tokenizer *p,          /* Tokenizer object */
   const char *z,                  /* Characters to treat as exceptions */
@@ -361,7 +366,7 @@ static int fts5UnicodeCreate(
       int i;
       memset(p, 0, sizeof(Unicode61Tokenizer));
 
-      p->bRemoveDiacritic = 1;
+      p->eRemoveDiacritic = FTS5_REMOVE_DIACRITICS_SIMPLE;
       p->nFold = 64;
       p->aFold = sqlite3_malloc(p->nFold * sizeof(char));
       if( p->aFold==0 ){
@@ -382,10 +387,15 @@ static int fts5UnicodeCreate(
       for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
         const char *zArg = azArg[i+1];
         if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
-          if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
+          if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
             rc = SQLITE_ERROR;
+          }else{
+            p->eRemoveDiacritic = (zArg[0] - '0');
+            assert( p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_NONE
+                 || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_SIMPLE
+                 || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_COMPLEX
+            );
           }
-          p->bRemoveDiacritic = (zArg[0]=='1');
         }else
         if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){
           rc = fts5UnicodeAddExceptions(p, zArg, 1);
@@ -499,7 +509,7 @@ static int fts5UnicodeTokenize(
         READ_UTF8(zCsr, zTerm, iCode);
         if( fts5UnicodeIsAlnum(p,iCode)||sqlite3Fts5UnicodeIsdiacritic(iCode) ){
  non_ascii_tokenchar:
-          iCode = sqlite3Fts5UnicodeFold(iCode, p->bRemoveDiacritic);
+          iCode = sqlite3Fts5UnicodeFold(iCode, p->eRemoveDiacritic);
           if( iCode ) WRITE_UTF8(zOut, iCode);
         }else{
           break;
diff --git a/ext/fts5/fts5_unicode2.c b/ext/fts5/fts5_unicode2.c
index 8c48aaa49b..9d3ffdc972 100644
--- a/ext/fts5/fts5_unicode2.c
+++ b/ext/fts5/fts5_unicode2.c
@@ -28,32 +28,47 @@
 ** E"). The resuls of passing a codepoint that corresponds to an
 ** uppercase letter are undefined.
 */
-static int fts5_remove_diacritic(int c){
+static int fts5_remove_diacritic(int c, int bComplex){
   unsigned short aDia[] = {
         0,  1797,  1848,  1859,  1891,  1928,  1940,  1995, 
      2024,  2040,  2060,  2110,  2168,  2206,  2264,  2286, 
      2344,  2383,  2472,  2488,  2516,  2596,  2668,  2732, 
      2782,  2842,  2894,  2954,  2984,  3000,  3028,  3336, 
-     3456,  3696,  3712,  3728,  3744,  3896,  3912,  3928, 
-     3968,  4008,  4040,  4106,  4138,  4170,  4202,  4234, 
-     4266,  4296,  4312,  4344,  4408,  4424,  4472,  4504, 
-     6148,  6198,  6264,  6280,  6360,  6429,  6505,  6529, 
-    61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, 
-    61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, 
-    62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, 
-    62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, 
-    62924, 63050, 63082, 63274, 63390, 
+     3456,  3696,  3712,  3728,  3744,  3766,  3832,  3896, 
+     3912,  3928,  3944,  3968,  4008,  4040,  4056,  4106, 
+     4138,  4170,  4202,  4234,  4266,  4296,  4312,  4344, 
+     4408,  4424,  4442,  4472,  4488,  4504,  6148,  6198, 
+     6264,  6280,  6360,  6429,  6505,  6529, 61448, 61468, 
+    61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, 
+    61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, 
+    61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, 
+    62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, 
+    62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, 
+    62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, 
+    63182, 63242, 63274, 63310, 63368, 63390, 
   };
   char aChar[] = {
-    '\0', 'a',  'c',  'e',  'i',  'n',  'o',  'u',  'y',  'y',  'a',  'c',  
-    'd',  'e',  'e',  'g',  'h',  'i',  'j',  'k',  'l',  'n',  'o',  'r',  
-    's',  't',  'u',  'u',  'w',  'y',  'z',  'o',  'u',  'a',  'i',  'o',  
-    'u',  'g',  'k',  'o',  'j',  'g',  'n',  'a',  'e',  'i',  'o',  'r',  
-    'u',  's',  't',  'h',  'a',  'e',  'o',  'y',  '\0', '\0', '\0', '\0', 
-    '\0', '\0', '\0', '\0', 'a',  'b',  'd',  'd',  'e',  'f',  'g',  'h',  
-    'h',  'i',  'k',  'l',  'l',  'm',  'n',  'p',  'r',  'r',  's',  't',  
-    'u',  'v',  'w',  'w',  'x',  'y',  'z',  'h',  't',  'w',  'y',  'a',  
-    'e',  'i',  'o',  'u',  'y',  
+    '\0',      'a'|0x00,  'c'|0x00,  'e'|0x00,  'i'|0x00,  'n'|0x00,  
+    'o'|0x00,  'u'|0x00,  'y'|0x00,  'y'|0x00,  'a'|0x00,  'c'|0x00,  
+    'd'|0x00,  'e'|0x00,  'e'|0x00,  'g'|0x00,  'h'|0x00,  'i'|0x00,  
+    'j'|0x00,  'k'|0x00,  'l'|0x00,  'n'|0x00,  'o'|0x00,  'r'|0x00,  
+    's'|0x00,  't'|0x00,  'u'|0x00,  'u'|0x00,  'w'|0x00,  'y'|0x00,  
+    'z'|0x00,  'o'|0x00,  'u'|0x00,  'a'|0x00,  'i'|0x00,  'o'|0x00,  
+    'u'|0x00,  'u'|0x80,  'a'|0x80,  'g'|0x00,  'k'|0x00,  'o'|0x00,  
+    'o'|0x80,  'j'|0x00,  'g'|0x00,  'n'|0x00,  'a'|0x80,  'a'|0x00,  
+    'e'|0x00,  'i'|0x00,  'o'|0x00,  'r'|0x00,  'u'|0x00,  's'|0x00,  
+    't'|0x00,  'h'|0x00,  'a'|0x00,  'e'|0x00,  'o'|0x80,  'o'|0x00,  
+    'o'|0x80,  'y'|0x00,  '\0',      '\0',      '\0',      '\0',      
+    '\0',      '\0',      '\0',      '\0',      'a'|0x00,  'b'|0x00,  
+    'c'|0x80,  'd'|0x00,  'd'|0x00,  'e'|0x80,  'e'|0x00,  'e'|0x80,  
+    'f'|0x00,  'g'|0x00,  'h'|0x00,  'h'|0x00,  'i'|0x00,  'i'|0x80,  
+    'k'|0x00,  'l'|0x00,  'l'|0x80,  'l'|0x00,  'm'|0x00,  'n'|0x00,  
+    'o'|0x80,  'p'|0x00,  'r'|0x00,  'r'|0x80,  'r'|0x00,  's'|0x00,  
+    's'|0x80,  't'|0x00,  'u'|0x00,  'u'|0x80,  'v'|0x00,  'w'|0x00,  
+    'w'|0x00,  'x'|0x00,  'y'|0x00,  'z'|0x00,  'h'|0x00,  't'|0x00,  
+    'w'|0x00,  'y'|0x00,  'a'|0x00,  'a'|0x80,  'a'|0x80,  'a'|0x80,  
+    'e'|0x00,  'e'|0x80,  'e'|0x80,  'i'|0x00,  'o'|0x00,  'o'|0x80,  
+    'o'|0x80,  'o'|0x80,  'u'|0x00,  'u'|0x80,  'u'|0x80,  'y'|0x00,  
   };
 
   unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
@@ -70,7 +85,8 @@ static int fts5_remove_diacritic(int c){
     }
   }
   assert( key>=aDia[iRes] );
-  return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);
+  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
+  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);
 }
 
 
@@ -97,7 +113,7 @@ int sqlite3Fts5UnicodeIsdiacritic(int c){
 ** The results are undefined if the value passed to this function
 ** is less than zero.
 */
-int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){
+int sqlite3Fts5UnicodeFold(int c, int eRemoveDiacritic){
   /* Each entry in the following array defines a rule for folding a range
   ** of codepoints to lower case. The rule applies to a range of nRange
   ** codepoints starting at codepoint iCode.
@@ -220,7 +236,9 @@ int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){
       assert( ret>0 );
     }
 
-    if( bRemoveDiacritic ) ret = fts5_remove_diacritic(ret);
+    if( eRemoveDiacritic ){
+      ret = fts5_remove_diacritic(ret, eRemoveDiacritic==2);
+    }
   }
   
   else if( c>=66560 && c<66600 ){
@@ -230,13 +248,6 @@ int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){
   return ret;
 }
 
-
-#if 0
-int sqlite3Fts5UnicodeNCat(void) { 
-  return 32;
-}
-#endif
-
 int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ 
   aArray[0] = 1;
   switch( zCat[0] ){
@@ -756,9 +767,8 @@ void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){
     int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ];
     int n = (aFts5UnicodeData[iTbl] >> 5) + i;
     for(; i<128 && i<n; i++){
-      aAscii[i] = (u8)bToken;
+      aAscii[i] = bToken;
     }
     iTbl++;
   }
 }
-
diff --git a/ext/fts5/test/fts5tokenizer.test b/ext/fts5/test/fts5tokenizer.test
index e71979ecab..27370657ed 100644
--- a/ext/fts5/test/fts5tokenizer.test
+++ b/ext/fts5/test/fts5tokenizer.test
@@ -189,7 +189,7 @@ do_catchsql_test 6.2 {
 } {1 {error in tokenizer constructor}}
 do_catchsql_test 6.3 {
   CREATE VIRTUAL TABLE a3 USING fts5(
-    x, y, tokenize = 'unicode61 remove_diacritics 2'
+    x, y, tokenize = 'unicode61 remove_diacritics 3'
   );
 } {1 {error in tokenizer constructor}}
 do_catchsql_test 6.4 {
diff --git a/ext/fts5/test/fts5umlaut.test b/ext/fts5/test/fts5umlaut.test
new file mode 100644
index 0000000000..fa948107a3
--- /dev/null
+++ b/ext/fts5/test/fts5umlaut.test
@@ -0,0 +1,65 @@
+# 2014 June 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5umlaut
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x);
+  CREATE VIRTUAL TABLE t2 USING fts5(
+      x, 
+      tokenize="unicode61 remove_diacritics 2"
+  );
+}
+
+foreach {tn q res1 res2} {
+  1 "Hà Nội"                  0 1
+  2 "Hà Noi"                  1 1
+  3 "Ha Noi"                  1 1
+  4 "Ha N\u1ed9i"             0 1
+  5 "Ha N\u006fi"             1 1
+  6 "Ha N\u006f\u0302i"       1 1
+  7 "Ha N\u006f\u0323\u0302i" 1 1
+} {
+  do_execsql_test 1.$tn.1 {
+    DELETE FROM t1;
+    INSERT INTO t1(rowid, x) VALUES (1, 'Ha Noi');
+    SELECT count(*) FROM t1($q)
+  } $res1
+  do_execsql_test 1.$tn.2 {
+    DELETE FROM t1;
+    INSERT INTO t1(rowid, x) VALUES (1, $q);
+    SELECT count(*) FROM t1('Ha Noi')
+  } $res1
+
+  do_execsql_test 1.$tn.2 {
+    DELETE FROM t2;
+    INSERT INTO t2(rowid, x) VALUES (1, 'Ha Noi');
+    SELECT count(*) FROM t2($q)
+  } $res2
+  do_execsql_test 1.$tn.2 {
+    DELETE FROM t2;
+    INSERT INTO t2(rowid, x) VALUES (1, $q);
+    SELECT count(*) FROM t2('Ha Noi')
+  } $res2
+}
+
+finish_test
+
diff --git a/ext/fts5/test/fts5unicode3.test b/ext/fts5/test/fts5unicode3.test
index 99f9c8c953..ed89624467 100644
--- a/ext/fts5/test/fts5unicode3.test
+++ b/ext/fts5/test/fts5unicode3.test
@@ -36,24 +36,26 @@ foreach x [an_load_unicodedata_text $UD] {
 }
 
 foreach {y} [rd_load_unicodedata_text $UD] {
-  foreach {code ascii} $y {}
+  foreach {code ascii f} $y {}
   if {$ascii==""} {
     set int 0
   } else {
     binary scan $ascii c int
   }
-  set aDiacritic($code) $int
+  set aDiacritic($code,$f) $int
+  if {$f==0} { set aDiacritic($code,1) $int }
 }
 
 proc tcl_fold {i {bRemoveDiacritic 0}} {
   global tl_lookup_table
   global aDiacritic
+  set f [expr $bRemoveDiacritic==2]
 
   if {[info exists tl_lookup_table($i)]} {
     set i $tl_lookup_table($i)
   }
-  if {$bRemoveDiacritic && [info exists aDiacritic($i)]} {
-    set i $aDiacritic($i)
+  if {$bRemoveDiacritic && [info exists aDiacritic($i,$f)]} {
+    set i $aDiacritic($i,$f)
   }
   expr $i
 }
@@ -85,7 +87,7 @@ do_execsql_test 1.1 {
   SELECT count(*), min(i) FROM ii WHERE fts5_fold(i)!=CAST(tcl_fold(i) AS int);
 } {0 {}}
 
-do_execsql_test 1.2 {
+do_execsql_test 1.2.1 {
   WITH ii(i) AS (
     SELECT -1
     UNION ALL
@@ -95,6 +97,16 @@ do_execsql_test 1.2 {
   WHERE fts5_fold(i,1)!=CAST(tcl_fold(i,1) AS int);
 } {0 {}}
 
+do_execsql_test 1.2.2 {
+  WITH ii(i) AS (
+    SELECT -1
+    UNION ALL
+    SELECT i+1 FROM ii WHERE i<100000
+  )
+  SELECT count(*), min(i) FROM ii 
+  WHERE fts5_fold(i,2)!=CAST(tcl_fold(i,2) AS int);
+} {0 {}}
+
 do_execsql_test 1.3 {
   WITH ii(i) AS (
     SELECT -1
diff --git a/main.mk b/main.mk
index ca7df9f8eb..270f9c48b4 100644
--- a/main.mk
+++ b/main.mk
@@ -999,6 +999,10 @@ showwal$(EXE):	$(TOP)/tool/showwal.c sqlite3.o
 showshm$(EXE):	$(TOP)/tool/showshm.c
 	$(TCC) -o showshm$(EXE) $(TOP)/tool/showshm.c
 
+index_usage$(EXE): $(TOP)/tool/index_usage.c sqlite3.o
+	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_DEPRECATED -o index_usage$(EXE) \
+		$(TOP)/tool/index_usage.c sqlite3.o $(THREADLIB)
+
 changeset$(EXE):	$(TOP)/ext/session/changeset.c sqlite3.o
 	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o changeset$(EXE) \
 		$(TOP)/ext/session/changeset.c sqlite3.o $(THREADLIB)
diff --git a/manifest b/manifest
index 1a49faf5b8..666b467b8b 100644
--- a/manifest
+++ b/manifest
@@ -1,10 +1,10 @@
-C Merge\sin\sall\schanges\sfor\srelease\s3.26.0.
-D 2018-12-03T18:24:31.174
+C Merge\senhancements\sfrom\strunk,\sespecially\sthe\ssqlite3_normalized_sql()\nenhancement.
+D 2018-12-05T13:49:23.731
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
-F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c
+F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 0d6831ff7951b302e888d86d4c469e2ec3c22f59eba4118b8c38d5a51d9e2d4f
+F Makefile.msc b7d4a710fa3f0b8cfc532ff195b85dc1ba2a8ad34343cb3d67639f28f0a24306
 F README.md 377233394b905d3b2e2b33741289e093bc93f2e7adbe00923b2c5958c9a9edee
 F VERSION 654da1d4053fb09ffc33a3910e6d427182a7dcdc67e934fa83de2849ac83fccb
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -14,7 +14,7 @@ F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2
 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903
 F autoconf/Makefile.am e14b629addaa1ce372b72043f28f40de2e32b7e211b6e0fc18dbb87989197e40
 F autoconf/Makefile.fallback 22fe523eb36dfce31e0f6349f782eb084e86a5620b2b0b4f84a2d6133f53f5ac
-F autoconf/Makefile.msc 9e73d9abaadb7a4951ddd0e947c5c791770f23bb1e04bfa50b43c01bee0575f2
+F autoconf/Makefile.msc ee8ab2b6b5d712f68acc4c186e3fad7b831b87a2b118e0a123043ea8a2c8b016
 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7
 F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1
 F autoconf/configure.ac 308de24343e76ecfbe9a67f8fcd4c5216b790d230c5d9ce10210b7d5965d6192
@@ -96,8 +96,8 @@ F ext/fts3/fts3_tokenize_vtab.c a47c2a33de6db00816704315ac0a9afdfa1c71fa5b99f791
 F ext/fts3/fts3_tokenizer.c a22bf311a71f3efa9d7012d8cc48fc9b0f3dace7
 F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3
 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004
-F ext/fts3/fts3_unicode.c 525a3bd9a7564603c5c061b7de55403a565307758a94600e8a2f6b00d1c40d9d
-F ext/fts3/fts3_unicode2.c cc04fc672bfd42b1e650398cb0bf71f64f9aae032cfe75bbcfe75b9cf966029c
+F ext/fts3/fts3_unicode.c b1902e9ad47a6569fbb8ecb5ce52f20fe59b590d5c5e3bbdd56b10b03bdf632b
+F ext/fts3/fts3_unicode2.c 90e65f4291c8ecceee284ecc8d5d48734e95ecd4b008e06f36f14e77f93d655f
 F ext/fts3/fts3_write.c a85bc4885fde7f1b44c9de013b62f7cd3332dc59e208053d878729b1d04745bc
 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
@@ -105,8 +105,8 @@ F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87
 F ext/fts3/tool/fts3view.c 202801a2056995b763864d60c2dee744d46f1677
 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c
 F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
-F ext/fts3/unicode/mkunicode.tcl 0069320b64db6ee269c5e95f1f150d070fbf0a863fc7b3549d7e52bd068fb118
-F ext/fts3/unicode/parseunicode.tcl 024ae0bdd96309d7b8fc479148191e9b3001dc74017a3f65f9a27de3b3ff968b
+F ext/fts3/unicode/mkunicode.tcl 2ea30d8122ccf1e33142c9cc8913d8cad9eb6668db359a228f10aeb37e2ab863
+F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h 5edc74ca603d71284d475886e6e91b5c5cf2e8e93e9ba3a36ba2f2440ee97948
 F ext/fts5/fts5Int.h 39f12034b598df4e0f59bbe6cf03af03a905a534b04f182d155641a04e1eb797
@@ -121,8 +121,8 @@ F ext/fts5/fts5_storage.c 4bec8a1b3905978b22a67bca5f4a3cfdb94af234cf51efb36f4f2d
 F ext/fts5/fts5_tcl.c 39bcbae507f594aad778172fa914cad0f585bf92fd3b078c686e249282db0d95
 F ext/fts5/fts5_test_mi.c 65864ba1e5c34a61d409c4c587e0bbe0466eb4f8f478d85dc42a92caad1338e6
 F ext/fts5/fts5_test_tok.c 80de1a4b1a3caa216c3be8862440f0117a8357dd9b7cfc5a2a2ce11fe6eb64ae
-F ext/fts5/fts5_tokenize.c ebd13d034f3dc7c841e1c32c364a4fca5cc2e05a0b91682a93fa1e6defcd4292
-F ext/fts5/fts5_unicode2.c 543cf0987c27ad59e5a7a6222480b917b5431009b7b139027c9581a63e39e37e
+F ext/fts5/fts5_tokenize.c ca2b6a033794945ac505241a86b0aa978709c23aa2e6121984d3e3ede96003c8
+F ext/fts5/fts5_unicode2.c 26519f47d567c5a9b08c376edd38f1b8a773b45ff29f1ffd81d9edfe20f96e18
 F ext/fts5/fts5_varint.c a5aceacda04dafcbae725413d7a16818ecd65738
 F ext/fts5/fts5_vocab.c fbe38044889b2d2d99babeeef239c620fb0332bb928a84506ac748d81500b354
 F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05
@@ -206,10 +206,11 @@ F ext/fts5/test/fts5synonym.test 1651815b8008de170e8e600dcacc17521d765482ea8f074
 F ext/fts5/test/fts5synonym2.test b54cce5c34ec08ed616f646635538ae82e34a0e28f947ec60b6fadbc4b3fb17a
 F ext/fts5/test/fts5tok1.test ce6551e41ff56f30b69963577324624733bed0d1753589f06120d664d9cd45c9
 F ext/fts5/test/fts5tok2.test dcacb32d4a2a3f0dd3215d4a3987f78ae4be21a2
-F ext/fts5/test/fts5tokenizer.test 6aeb5e8061ffc0ff9a5299f27beaee3b2b4b8b336d4f107262bca338bea8f8e9
+F ext/fts5/test/fts5tokenizer.test ac3c9112b263a639fb0508ae73a3ee886bf4866d2153771a8e8a20c721305a43
+F ext/fts5/test/fts5umlaut.test a42fe2fe6387c40c49ab27ccbd070e1ae38e07f38d05926482cc0bccac9ad602
 F ext/fts5/test/fts5unicode.test 17056f4efe6b0a5d4f41fdf7a7dc9af2873004562eaa899d40633b93dc95f5a9
 F ext/fts5/test/fts5unicode2.test 9b3df486de05fb4bde4aa7ee8de2e6dae1df6eb90e3f2e242c9383b95d314e3e
-F ext/fts5/test/fts5unicode3.test c3caecbe8264629ffe653b43ca5790b9793eba4422f92203e5247558e5a534e7
+F ext/fts5/test/fts5unicode3.test 9cbc82e2b02e2e3b7504103580c90f095e07fe8230b1951a9ed7558717b5feb7
 F ext/fts5/test/fts5unicode4.test 6463301d669f963c83988017aa354108be0b947d325aef58d3abddf27147b687
 F ext/fts5/test/fts5unindexed.test 9021af86a0fb9fc616f7a69a996db0116e7936d0db63892db6bafabbec21af4d
 F ext/fts5/test/fts5update.test 0737876e20e97a6a6abf45de19fc99315727bcee6a83fadcada1cc080b9aa8f0
@@ -432,7 +433,7 @@ F ext/userauth/userauth.c f81aa5a3ecacf406f170c62a144405858f6f6de51dbdc0920134e6
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk c4070221684a71edd3846c05a5cfc648762f3044ef4d91648d33f1f692e15c3c
+F main.mk 7c4484f59dc2ccf831c9786b9eb4e8131d63b80328e1d92ddefaee6284f26386
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -454,7 +455,7 @@ F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6
 F src/btree.c 4a2184be69d491d4b0228d4e397d67cb0802bbec06e7615b485ea1af69a131f6
 F src/btree.h 1ed41c71481a1196a520064f2282bc13d768bbd8ae2850e319a3048f8ee7cb3d
 F src/btreeInt.h 6c65e6c96f561596f6870c79a64d4706af81613881d7947e3f063e923f14115f
-F src/build.c 5e04fb8528a4a915ed9af94b5cd068e53f2cfa9824d37464c4059279ea9bc8a0
+F src/build.c 7d86c455d226f5ccef7103c2a0c75ea71fe6cd6206311bd9d02d700c2b2c2565
 F src/callback.c 789bd33d188146f66c0dd8306472a72d1c05f71924b24a91caf6bd45cf9aba73
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c 109e58d00f62e8e71ee1eb5944ac18b90171c928ab2e082e058056e1137cc20b
@@ -467,14 +468,14 @@ F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c 972a4ba14296bef2303a0abbad1e3d82bc3c61f9e6ce4e8e9528bdee68748812
 F src/func.c 8efa2c813b3f6a831a070311b5bcbc97993b79cbcd274bdb49bde56ccd3d37bc
 F src/global.c 8291eee0782b83124de14ec0389ec9fd6ae1873358a6b0d9469fe17a46ad803b
-F src/hash.c 931ec82d7e070654a8facb42549bbb3a25720171d73ba94c3d3160580d01ef1f
-F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4
+F src/hash.c 6d2f67276469384fb8784fb8e962deeaae6832955626468325d705a01b999594
+F src/hash.h eebf2250e56b5d5353b873406557e373d2888cf51f111e28917666456d479e85
 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
-F src/insert.c 6b81aae27b196925d8ff78824f4bbd435d6a40cd38dc324685e21735bb402109
+F src/insert.c f12f27eb606d601825be9a229a7390a8d64d40226697883f96de8e088d620055
 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e
 F src/loadext.c 9050dd153b5583804184be9c9dee9ebb554178d6db1f8ac280899e8aad9060e6
-F src/main.c 974f09b62f97fd7b3da63e7dce2c70f19ed1a6cca5efba25ea0d9f03f9fb49a9
+F src/main.c 423fce7ca924c8aa04d2844de4775129109172aeb912eeb4606fb1157301297e
 F src/malloc.c 07295435093ce354c6d9063ac05a2eeae28bd251d2e63c48b3d67c12c76f7e18
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -499,23 +500,23 @@ F src/os_win.c 85d9e532d0444ab6c16d7431490c2e279e282aa0917b0e988996b1ae0de5c5a0
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 79b8ced46a3c32be19ab4b00d8836292e460411d283a333cb008923c69fbca6a
 F src/pager.h 389ba8f526d13026aa7081dc581aa742eb7207e3277e7106c522c5b65ad92590
-F src/parse.y 9e69c380ac16423a1f373cde66d1be0d14a789f93464705a48dcc812d04d9210
+F src/parse.y 9281e9941d5df0f7978c4bfe1521090ba7363ac4535c79f57531ba0194904d43
 F src/pcache.c 696a01f1a6370c1b50a09c15972bc3bee3333f8fcd1f2da8e9a76b1b062c59ee
 F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586
-F src/pcache1.c bf9fcea656dce1cd2cca6b77a1d1d3552050d55a31c98bf0d9f405930a83bc95
+F src/pcache1.c ad0ffc5b35b0280d045ac569d34d4b842e3e6a4a118f6396b320987a0957afcc
 F src/pragma.c 4e056f042683b99c4ea0db395f68d051b1a95833ab40951c40d3ef7e1fee1354
 F src/pragma.h 7003ea8e45e5da0a7cd6d35846214f9ae9ecf5be66b268415ceea5855324af11
-F src/prepare.c f81f8d707e583192c28fea0b2e19385415b7d188123b23f49b038076408d7a69
+F src/prepare.c 277c8af17124bd4f67ae6e1a8e795a91f12bf7ce3a4b344c7eac46d360250c20
 F src/printf.c 0f1177cf1dd4d7827bf64d840768514ec76409abecaca9e8b577dbd065150381
 F src/random.c f27af4099afaea7284ade5c206224dcfdb2334cfd119d018b470d46356b3f27d
-F src/resolve.c 4cfc44def0f0690ceaab8f6481f5d76284d7f9509aab6e218a679b4836a54614
+F src/resolve.c 976e7879286a1eecdc71ceff64f6d1b3f58c8f8096537ba668b3dc0887f410c1
 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93
 F src/select.c e1cad752987c476a9d1d053682558caec048ea799ef7ba1b6e38e2266451010d
-F src/shell.c.in 482e23a370cbe5b0d4c73a0f0f5fce34f7caa08a14a8d75e12f0225c4e14915c
+F src/shell.c.in 1f0819e69fb1ebd2eb44695530dc43936608bf9b752981a0ffd4e2e4a9e3883d
 F src/sqlite.h.in 0880d9e366c2b0e1cbc40d32c2671e67c7bc7e9cb7f7298956c254ca8845c702
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 960f1b86c3610fa23cb6a267572a97dcf286e77aa0dd3b9b23292ffaa1ea8683
-F src/sqliteInt.h ab9fab2d16c97ea3d73c45e8eb2e44056d6a4d698df7722353e61093664432d6
+F src/sqliteInt.h 399dc79d2fd00a060294d2d637c5065da511f552c26c274136a6b1d653dec07d
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -584,15 +585,15 @@ F src/vacuum.c 8747a99e0687ae5cb3515b70a9d82bbd20370de9f097c7bd93a392ece3dea03c
 F src/vdbe.c d2672a54cc283e1425cc8c7c45271530b3686b8b5f06bf4426d9a5812e951abe
 F src/vdbe.h 5081dcc497777efe5e9ebe7330d283a044a005e4bdda2e2e984f03bf89a0d907
 F src/vdbeInt.h 437e6c6af679fdf157867eb83a8adc6cf5145d6774453c2214cfd0bd01d92980
-F src/vdbeapi.c ecccfce6f614c33a95952efeec969d163e8349eac314ee2b7b163eda921b5eb0
-F src/vdbeaux.c 51a6b2b75c63865e3b6cd30371878a73039c4d7b1263cf07166ed23f5212b442
+F src/vdbeapi.c 9ac7e3946a2762b79c314922c84c7de30731e0ed1d7c94ac82266b795221d9d7
+F src/vdbeaux.c b3c364b1cfb9cda13bf86d571c572ff89ffefa89bbfc9a506731214a004789a8
 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191
 F src/vdbemem.c 7b3305bc4a5139f4536ac9b5f61da0f915e49d2e3fdfa87dfdfa9d7aba8bc1e9
 F src/vdbesort.c 90aad5a92608f2dd771c96749beabdb562c9d881131a860a7a5bccf66dc3be7f
 F src/vdbetrace.c 79d6dbbc479267b255a7de8080eee6e729928a0ef93ed9b0bfa5618875b48392
 F src/vtab.c 70188a745dc4e57d26e942681ff4b2912b7c8249ad5de3f60f0677b4337bcfaa
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c f53ac2b6ec6f23960471b4402632b8276d8b80c1ac31a28ea08500c56a9cc5c9
+F src/wal.c 59b20f8fd0d7420e8896a1ef7cbd5db7352f383235ec25db08d9c22b55f4b603
 F src/wal.h f325a5856b669f5ba449157485915816103857c8574efc746ac55eba3335c5e0
 F src/walker.c fb94aadc9099ff9c6506d0a8b88d51266005bcaa265403f3d7caf732a562eb66
 F src/where.c 3818e8a736a05d2cb194e64399af707e367fbcc5c251d785804d02eaf121288e
@@ -969,6 +970,7 @@ F test/fts4merge4.test d895b1057a7798b67e03455d0fa50e9ea836c47b
 F test/fts4noti.test 5553d7bb2e20bf4a06b23e849352efc022ce6309
 F test/fts4onepass.test d69ddc4ee3415e40b0c5d1d0408488a87614d4f63ba9c44f3e52db541d6b7cc7
 F test/fts4opt.test 0fd0cc84000743ff2a883b9b84b4a5be07249f0ba790c8848a757164cdd46b2a
+F test/fts4umlaut.test 1d28e2a2dffa794e15babadebdece091431b09d740be79c08eb6aba1173a8c84
 F test/fts4unicode.test ceca76422abc251818cb25dabe33d3c3970da5f7c90e1540f190824e6b3a7c95
 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d
 F test/func.test 09dda479bcfc568f99f3070413e9672a8eeedc1be9c5d819bf55d4788c2583b7
@@ -1066,7 +1068,7 @@ F test/journal3.test c9c29883f5bf535ae82ae21c472df6263806a22e467b6db7cd0d6d54530
 F test/jrnlmode.test a6693f2bed4541a21e703aaa37bb3e10de154130645952933b82b2dec0a8b539
 F test/jrnlmode2.test 8759a1d4657c064637f8b079592651530db738419e1d649c6df7048cd724363d
 F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa
-F test/json101.test b40a9f5395d8e669b0bc3eb550ad2ae9e5ada01fbce23c446c2a30a305a6d575
+F test/json101.test 8f8977b00ba02f9a26c1d1f52f29f540f6d5eb162cbd5eb78bb805366d4ab26d
 F test/json102.test eeb54efa221e50b74a2d6fb9259963b48d7414dca3ce2fdfdeed45cb28487bc1
 F test/json103.test aff6b7a4c17d5a20b487a7bc1a274bfdc63b829413bdfb83bedac42ec7f67e3b
 F test/json104.test 877d5845f6303899b7889ea5dd1bea99076e3100574d5c536082245c5805dcaa
@@ -1290,7 +1292,7 @@ F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304
 F test/shell1.test d2bf5daeb6f449f0169c9ef3094db17a16a02199c5dcf1a635a3e79b07eb0edd
 F test/shell2.test e242a9912f44f4c23c3d1d802a83e934e84c853b
 F test/shell3.test ac8c2b744014c3e9a0e26bfd829ab65f00923dc1a91ffd044863e9423cc91494
-F test/shell4.test 89ad573879a745974ff2df20ff97c5d6ffffbd5d
+F test/shell4.test a6881d0ae226ded0df8ebdfa574c5aa6dc28d6884ccba1089dc56ed08b9e5ef4
 F test/shell5.test 23939a4c51f0421330ea61dbd3c74f9c215f5f8d3d1a94846da6ffc777a35458
 F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3
 F test/shell7.test 115132f66d0463417f408562cc2cf534f6bbc6d83a6d50f0072a9eb171bae97f
@@ -1355,7 +1357,7 @@ F test/sync.test 89539f4973c010eda5638407e71ca7fddbcd8e0594f4c9980229f804d433309
 F test/sync2.test 8f9f7d4f6d5be8ca8941a8dadcc4299e558cb6a1ff653a9469146c7a76ef2039
 F test/syscall.test a39d9a36f852ae6e4800f861bc2f2e83f68bbc2112d9399931ecfadeabd2d69d
 F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04
-F test/tabfunc01.test 54300134f76db817685194d2f0e63e3fbf7380b45e0d426e00a9aee752497cfb
+F test/tabfunc01.test 20e98ffe55f35d8d33fd834ca8bf9d4b637e560af8fcd00464b4154d90a4db45
 F test/table.test eb3463b7add9f16a5bb836badf118cf391b809d09fdccd1f79684600d07ec132
 F test/tableapi.test ecbcc29c4ab62c1912c3717c48ea5c5e59f7d64e4a91034e6148bd2b82f177f4
 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930
@@ -1710,9 +1712,10 @@ F tool/fuzzershell.c e1d90a03ca790d7c331c2aae08ca46ff435f1ae1faa6cb9cc48f4687c18
 F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4
 F tool/genfkey.test b6afd7b825d797a1e1274f519ab5695373552ecad5cd373530c63533638a5a4f
 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
+F tool/index_usage.c 8fd515b97522ae4b1aa6ca9847439f005d4cbaf6eb0eb416b694dba77c0263f0
 F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f
-F tool/lemon.c 60d1e1eb0f7ebae709f68f1472d77fbf291c5345cd98ff417219da7e74fd09e9
-F tool/lempar.c 452f12d40229847634a160e5666b6c4ec4392fd81941c3443861b48d497054cc
+F tool/lemon.c c9ba01f6729c892ae3e0f55c8a2d694a7e6ec3dd3aa280b411a008ef69b410cd
+F tool/lempar.c 61af95b8fac2bfd59c09d55330e78f3f5e352d7aa80bf37404b96ef795be3fdc
 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9
 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca
@@ -1792,7 +1795,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 81e626f4a44be75425cf916ec61b6b36df0907ebac4adbf6786f87ad4f3a0674 f0ddb358cc68e5ec6d9e758893ab3da058a3b2e705124a7449279c992e672a4a
-R cff54908fb8658b7788ed6a5676c7d42
+P 85fd92c71fd5b29bd45bb1717a5be761600fc23bed12eaff9eed708701a0fdf7 47b73f6bfee8c5e41c408fb70dff0e4596c0a3eb0aeba40ad232a6cb1fe75532
+R f5c4a734b93f0cfb5050e78ecedd5cc2
 U drh
-Z 7fad9191e7df60cbd073aacab8209c36
+Z 9ac2790769c3800c64245b58c8bae081
diff --git a/manifest.uuid b/manifest.uuid
index e4481dad94..ecb1c1ee63 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-85fd92c71fd5b29bd45bb1717a5be761600fc23bed12eaff9eed708701a0fdf7
\ No newline at end of file
+342c9538d9c6a993ac0acaa6f74ad58886bcef7bb53783d053f9b24c131aec5d
\ No newline at end of file
diff --git a/src/build.c b/src/build.c
index 8d1e2bdd99..9c60899c94 100644
--- a/src/build.c
+++ b/src/build.c
@@ -229,7 +229,7 @@ void sqlite3FinishCoding(Parse *pParse){
   if( v && pParse->nErr==0 && !db->mallocFailed ){
     /* A minimum of one cursor is required if autoincrement is used
     *  See ticket [a696379c1f08866] */
-    if( pParse->pAinc!=0 && pParse->nTab==0 ) pParse->nTab = 1;
+    assert( pParse->pAinc==0 || pParse->nTab>0 );
     sqlite3VdbeMakeReady(v, pParse);
     pParse->rc = SQLITE_DONE;
   }else{
diff --git a/src/hash.c b/src/hash.c
index fba9dc9f80..e725f2aa5e 100644
--- a/src/hash.c
+++ b/src/hash.c
@@ -72,7 +72,7 @@ static unsigned int strHashN(const char *z, int n){
     /* Knuth multiplicative hashing.  (Sorting & Searching, p. 510).
     ** 0x9e3779b1 is 2654435761 which is the closest prime number to
     ** (2**32)*golden_ratio, where golden_ratio = (sqrt(5) - 1)/2. */
-    h += sqlite3UpperToLower[z[i]];
+    h += sqlite3UpperToLower[(unsigned char)z[i]];
     h *= 0x9e3779b1;
   }
   return h;
diff --git a/src/hash.h b/src/hash.h
index 90540eda50..cf773663b7 100644
--- a/src/hash.h
+++ b/src/hash.h
@@ -68,6 +68,9 @@ struct HashElem {
 void sqlite3HashInit(Hash*);
 void *sqlite3HashInsert(Hash*, const char *pKey, void *pData);
 void *sqlite3HashFind(const Hash*, const char *pKey);
+#ifdef SQLITE_ENABLE_NORMALIZE
+void *sqlite3HashFindN(const Hash *pH, const char *pKey, int nKey);
+#endif
 void sqlite3HashClear(Hash*);
 
 /*
diff --git a/src/insert.c b/src/insert.c
index 7a9413901f..0c036e494f 100644
--- a/src/insert.c
+++ b/src/insert.c
@@ -319,6 +319,7 @@ void sqlite3AutoincrementBegin(Parse *pParse){
     aOp[7].p2 = memId+2;
     aOp[7].p1 = memId;
     aOp[10].p2 = memId;
+    if( pParse->nTab==0 ) pParse->nTab = 1;
   }
 }
 
diff --git a/src/main.c b/src/main.c
index 110c4d2a1a..42d67b1302 100644
--- a/src/main.c
+++ b/src/main.c
@@ -1996,6 +1996,7 @@ void *sqlite3_trace(sqlite3 *db, void(*xTrace)(void*,const char*), void *pArg){
   sqlite3_mutex_enter(db->mutex);
   pOld = db->pTraceArg;
   db->mTrace = xTrace ? SQLITE_TRACE_LEGACY : 0;
+  if( db->xProfile ) db->mTrace |= SQLITE_TRACE_XPROFILE;
   db->xTrace = (int(*)(u32,void*,void*,void*))xTrace;
   db->pTraceArg = pArg;
   sqlite3_mutex_leave(db->mutex);
@@ -2020,6 +2021,9 @@ int sqlite3_trace_v2(
   if( mTrace==0 ) xTrace = 0;
   if( xTrace==0 ) mTrace = 0;
   db->mTrace = mTrace;
+#ifndef SQLITE_OMIT_DEPRECATED
+  if( db->xProfile ) db->mTrace |= SQLITE_TRACE_XPROFILE;
+#endif
   db->xTrace = xTrace;
   db->pTraceArg = pArg;
   sqlite3_mutex_leave(db->mutex);
@@ -2052,6 +2056,8 @@ void *sqlite3_profile(
   pOld = db->pProfileArg;
   db->xProfile = xProfile;
   db->pProfileArg = pArg;
+  db->mTrace &= SQLITE_TRACE_NONLEGACY_MASK;
+  if( db->xProfile ) db->mTrace |= SQLITE_TRACE_XPROFILE;
   sqlite3_mutex_leave(db->mutex);
   return pOld;
 }
diff --git a/src/parse.y b/src/parse.y
index 42fd181909..5aae351bc7 100644
--- a/src/parse.y
+++ b/src/parse.y
@@ -679,6 +679,12 @@ seltablist(A) ::= stl_prefix(A) nm(Y) dbnm(D) LP exprlist(E) RP as(Z)
         pNew->zName = pOld->zName;
         pNew->zDatabase = pOld->zDatabase;
         pNew->pSelect = pOld->pSelect;
+        if( pOld->fg.isTabFunc ){
+          pNew->u1.pFuncArg = pOld->u1.pFuncArg;
+          pOld->u1.pFuncArg = 0;
+          pOld->fg.isTabFunc = 0;
+          pNew->fg.isTabFunc = 1;
+        }
         pOld->zName = pOld->zDatabase = 0;
         pOld->pSelect = 0;
       }
diff --git a/src/pcache1.c b/src/pcache1.c
index 2880c2c5e6..6df0f15d13 100644
--- a/src/pcache1.c
+++ b/src/pcache1.c
@@ -102,6 +102,7 @@ struct PgHdr1 {
   PCache1 *pCache;               /* Cache that currently owns this page */
   PgHdr1 *pLruNext;              /* Next in LRU list of unpinned pages */
   PgHdr1 *pLruPrev;              /* Previous in LRU list of unpinned pages */
+                                 /* NB: pLruPrev is only valid if pLruNext!=0 */
 };
 
 /*
@@ -570,7 +571,8 @@ static PgHdr1 *pcache1PinPage(PgHdr1 *pPage){
   pPage->pLruPrev->pLruNext = pPage->pLruNext;
   pPage->pLruNext->pLruPrev = pPage->pLruPrev;
   pPage->pLruNext = 0;
-  pPage->pLruPrev = 0;
+  /* pPage->pLruPrev = 0;
+  ** No need to clear pLruPrev as it is never accessed if pLruNext is 0 */
   assert( pPage->isAnchor==0 );
   assert( pPage->pCache->pGroup->lru.isAnchor==1 );
   pPage->pCache->nRecyclable--;
@@ -908,8 +910,9 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2(
     pPage->iKey = iKey;
     pPage->pNext = pCache->apHash[h];
     pPage->pCache = pCache;
-    pPage->pLruPrev = 0;
     pPage->pLruNext = 0;
+    /* pPage->pLruPrev = 0;
+    ** No need to clear pLruPrev since it is not accessed when pLruNext==0 */
     *(void **)pPage->page.pExtra = 0;
     pCache->apHash[h] = pPage;
     if( iKey>pCache->iMaxKey ){
@@ -1069,7 +1072,7 @@ static void pcache1Unpin(
   /* It is an error to call this function if the page is already 
   ** part of the PGroup LRU list.
   */
-  assert( pPage->pLruPrev==0 && pPage->pLruNext==0 );
+  assert( pPage->pLruNext==0 );
   assert( PAGE_IS_PINNED(pPage) );
 
   if( reuseUnlikely || pGroup->nPurgeable>pGroup->nMaxPage ){
diff --git a/src/prepare.c b/src/prepare.c
index 4d33f0b1e1..86afac5350 100644
--- a/src/prepare.c
+++ b/src/prepare.c
@@ -791,8 +791,7 @@ done:
 */
 static int estimateNormalizedSize(
   const char *zSql, /* The original SQL string */
-  int nSql,         /* Length of original SQL string */
-  u8 prepFlags      /* The flags passed to sqlite3_prepare_v3() */
+  int nSql          /* Length of original SQL string */
 ){
   int nOut = nSql + 4;
   const char *z = zSql;
@@ -847,18 +846,14 @@ static void copyNormalizedToken(
 }
 
 /*
-** Perform normalization of the SQL contained in the prepared statement and
-** store the result in the zNormSql field.  The schema for the associated
-** databases are consulted while performing the normalization in order to
-** determine if a token appears to be an identifier.  All identifiers are
-** left intact in the normalized SQL and all literals are replaced with a
-** single '?'.
+** Compute a normalization of the SQL given by zSql[0..nSql-1].  Return
+** the normalization in space obtained from sqlite3DbMalloc().  Or return
+** NULL if anything goes wrong or if zSql is NULL.
 */
-void sqlite3Normalize(
+char *sqlite3Normalize(
   Vdbe *pVdbe,      /* VM being reprepared */
   const char *zSql, /* The original SQL string */
-  int nSql,         /* Size of the input string in bytes */
-  u8 prepFlags      /* The flags passed to sqlite3_prepare_v3() */
+  int nSql          /* Size of the input string in bytes */
 ){
   sqlite3 *db;           /* Database handle. */
   char *z;               /* The output string */
@@ -873,11 +868,10 @@ void sqlite3Normalize(
 
   db = sqlite3VdbeDb(pVdbe);
   assert( db!=0 );
-  assert( pVdbe->zNormSql==0 );
-  if( zSql==0 ) return;
-  nZ = estimateNormalizedSize(zSql, nSql, prepFlags);
+  if( zSql==0 ) return 0;
+  nZ = estimateNormalizedSize(zSql, nSql);
   z = sqlite3DbMallocRawNN(db, nZ);
-  if( z==0 ) return;
+  if( z==0 ) goto normalizeError;
   sqlite3HashInit(&inHash);
   for(i=j=0; i<nSql && zSql[i]; i+=n){
     int flags = 0;
@@ -888,9 +882,7 @@ void sqlite3Normalize(
         break;
       }
       case TK_ILLEGAL: {
-        sqlite3DbFree(db, z);
-        sqlite3HashClear(&inHash);
-        return;
+        goto normalizeError;
       }
       case TK_STRING:
       case TK_INTEGER:
@@ -971,11 +963,7 @@ void sqlite3Normalize(
           }
           if( flags&SQLITE_TOKEN_QUOTED ){ i2++; n2-=2; }
           if( shouldTreatAsIdentifier(db, zSql+i2, n2, &rc)==0 ){
-            if( rc!=SQLITE_OK ){
-              sqlite3DbFree(db, z);
-              sqlite3HashClear(&inHash);
-              return;
-            }
+            if( rc!=SQLITE_OK ) goto normalizeError;
             if( sqlite3_keyword_check(zSql+i2, n2)==0 ){
               z[j++] = '?';
               break;
@@ -992,8 +980,13 @@ void sqlite3Normalize(
   if( j>0 && z[j-1]!=';' ){ z[j++] = ';'; }
   z[j] = 0;
   assert( j<nZ && "two" );
-  pVdbe->zNormSql = z;
   sqlite3HashClear(&inHash);
+  return z;
+
+normalizeError:
+  sqlite3DbFree(db, z);
+  sqlite3HashClear(&inHash);
+  return 0;
 }
 #endif /* SQLITE_ENABLE_NORMALIZE */
 
diff --git a/src/resolve.c b/src/resolve.c
index 0c7dfc0b25..effbe646fe 100644
--- a/src/resolve.c
+++ b/src/resolve.c
@@ -80,7 +80,7 @@ static void resolveAlias(
     if( pExpr->op==TK_COLLATE ){
       pDup = sqlite3ExprAddCollateString(pParse, pDup, pExpr->u.zToken);
     }
-    ExprSetProperty(pDup, EP_Alias);
+//    ExprSetProperty(pDup, EP_Alias);
 
     /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This 
     ** prevents ExprDelete() from deleting the Expr structure itself,
diff --git a/src/shell.c.in b/src/shell.c.in
index ad5a1498be..887c2fcf59 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -1007,6 +1007,7 @@ struct ShellState {
   u8 openMode;           /* SHELL_OPEN_NORMAL, _APPENDVFS, or _ZIPFILE */
   u8 doXdgOpen;          /* Invoke start/open/xdg-open in output_reset() */
   u8 nEqpLevel;          /* Depth of the EQP output graph */
+  u8 eTraceType;         /* SHELL_TRACE_* value for type of trace */
   unsigned mEqpLines;    /* Mask of veritical lines in the EQP output graph */
   int outCount;          /* Revert to stdout when reaching zero */
   int cnt;               /* Number of records displayed so far */
@@ -1066,6 +1067,12 @@ struct ShellState {
 #define SHELL_OPEN_READONLY    4      /* Open a normal database read-only */
 #define SHELL_OPEN_DESERIALIZE 5      /* Open using sqlite3_deserialize() */
 
+/* Allowed values for ShellState.eTraceType
+*/
+#define SHELL_TRACE_PLAIN      0      /* Show input SQL text */
+#define SHELL_TRACE_EXPANDED   1      /* Show expanded SQL text */
+#define SHELL_TRACE_NORMALIZED 2      /* Show normalized SQL text */
+
 /*
 ** These are the allowed shellFlgs values
 */
@@ -3492,7 +3499,22 @@ static const char *(azHelp[]) = {
   ".testcase NAME           Begin redirecting output to 'testcase-out.txt'",
   ".timeout MS              Try opening locked tables for MS milliseconds",
   ".timer on|off            Turn SQL timer on or off",
-  ".trace FILE|off          Output each SQL statement as it is run",
+#ifndef SQLITE_OMIT_TRACE
+  ".trace ?OPTIONS?         Output each SQL statement as it is run",
+  "    FILE                    Send output to FILE",
+  "    stdout                  Send output to stdout",
+  "    stderr                  Send output to stderr",
+  "    off                     Disable tracing",
+  "    --expanded              Expand query parameters",
+#ifdef SQLITE_ENABLE_NORMALIZE
+  "    --normalized            Normal the SQL statements",
+#endif
+  "    --plain                 Show SQL as it is input",
+  "    --stmt                  Trace statement execution (SQLITE_TRACE_STMT)",
+  "    --profile               Profile statements (SQLITE_TRACE_PROFILE)",
+  "    --row                   Trace each row (SQLITE_TRACE_ROW)",
+  "    --close                 Trace connection close (SQLITE_TRACE_CLOSE)",
+#endif /* SQLITE_OMIT_TRACE */
   ".vfsinfo ?AUX?           Information about the top-level VFS",
   ".vfslist                 List all available VFSes",
   ".vfsname ?AUX?           Print the name of the VFS stack",
@@ -3999,24 +4021,60 @@ static FILE *output_file_open(const char *zFile, int bTextMode){
   return f;
 }
 
-#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT)
+#ifndef SQLITE_OMIT_TRACE
 /*
 ** A routine for handling output from sqlite3_trace().
 */
 static int sql_trace_callback(
-  unsigned mType,
-  void *pArg,
-  void *pP,
-  void *pX
+  unsigned mType,         /* The trace type */
+  void *pArg,             /* The ShellState pointer */
+  void *pP,               /* Usually a pointer to sqlite_stmt */
+  void *pX                /* Auxiliary output */
 ){
-  FILE *f = (FILE*)pArg;
-  UNUSED_PARAMETER(mType);
-  UNUSED_PARAMETER(pP);
-  if( f ){
-    const char *z = (const char*)pX;
-    int i = strlen30(z);
-    while( i>0 && z[i-1]==';' ){ i--; }
-    utf8_printf(f, "%.*s;\n", i, z);
+  ShellState *p = (ShellState*)pArg;
+  sqlite3_stmt *pStmt;
+  const char *zSql;
+  int nSql;
+  if( p->traceOut==0 ) return 0;
+  if( mType==SQLITE_TRACE_CLOSE ){
+    utf8_printf(p->traceOut, "-- closing database connection\n");
+    return 0;
+  }
+  if( mType!=SQLITE_TRACE_ROW && ((const char*)pX)[0]=='-' ){
+    zSql = (const char*)pX;
+  }else{
+    pStmt = (sqlite3_stmt*)pP;
+    switch( p->eTraceType ){
+      case SHELL_TRACE_EXPANDED: {
+        zSql = sqlite3_expanded_sql(pStmt);
+        break;
+      }
+#ifdef SQLITE_ENABLE_NORMALIZE
+      case SHELL_TRACE_NORMALIZED: {
+        zSql = sqlite3_normalized_sql(pStmt);
+        break;
+      }
+#endif
+      default: {
+        zSql = sqlite3_sql(pStmt);
+        break;
+      }
+    }
+  }
+  if( zSql==0 ) return 0;
+  nSql = strlen30(zSql);
+  while( nSql>0 && zSql[nSql-1]==';' ){ nSql--; }
+  switch( mType ){
+    case SQLITE_TRACE_ROW:
+    case SQLITE_TRACE_STMT: {
+      utf8_printf(p->traceOut, "%.*s;\n", nSql, zSql);
+      break;
+    }
+    case SQLITE_TRACE_PROFILE: {
+      sqlite3_int64 nNanosec = *(sqlite3_int64*)pX;
+      utf8_printf(p->traceOut, "%.*s; -- %lld ns\n", nSql, zSql, nNanosec);
+      break;
+    }
   }
   return 0;
 }
@@ -7838,23 +7896,55 @@ static int do_meta_command(char *zLine, ShellState *p){
     }
   }else
 
+#ifndef SQLITE_OMIT_TRACE
   if( c=='t' && strncmp(azArg[0], "trace", n)==0 ){
+    int mType = 0;
+    int jj;
     open_db(p, 0);
-    if( nArg!=2 ){
-      raw_printf(stderr, "Usage: .trace FILE|off\n");
-      rc = 1;
-      goto meta_command_exit;
+    for(jj=1; jj<nArg; jj++){
+      const char *z = azArg[jj];
+      if( z[0]=='-' ){
+        if( optionMatch(z, "expanded") ){
+          p->eTraceType = SHELL_TRACE_EXPANDED;
+        }
+#ifdef SQLITE_ENABLE_NORMALIZE
+        else if( optionMatch(z, "normalized") ){
+          p->eTraceType = SHELL_TRACE_NORMALIZED;
+        }
+#endif
+        else if( optionMatch(z, "plain") ){
+          p->eTraceType = SHELL_TRACE_PLAIN;
+        }
+        else if( optionMatch(z, "profile") ){
+          mType |= SQLITE_TRACE_PROFILE;
+        }
+        else if( optionMatch(z, "row") ){
+          mType |= SQLITE_TRACE_ROW;
+        }
+        else if( optionMatch(z, "stmt") ){
+          mType |= SQLITE_TRACE_STMT;
+        }
+        else if( optionMatch(z, "close") ){
+          mType |= SQLITE_TRACE_CLOSE;
+        }
+        else {
+          raw_printf(stderr, "Unknown option \"%s\" on \".trace\"\n", z);
+          rc = 1;
+          goto meta_command_exit;
+        }
+      }else{
+        output_file_close(p->traceOut);
+        p->traceOut = output_file_open(azArg[1], 0);
+      }
     }
-    output_file_close(p->traceOut);
-    p->traceOut = output_file_open(azArg[1], 0);
-#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT)
     if( p->traceOut==0 ){
       sqlite3_trace_v2(p->db, 0, 0, 0);
     }else{
-      sqlite3_trace_v2(p->db, SQLITE_TRACE_STMT, sql_trace_callback,p->traceOut);
+      if( mType==0 ) mType = SQLITE_TRACE_STMT;
+      sqlite3_trace_v2(p->db, mType, sql_trace_callback, p);
     }
-#endif
   }else
+#endif /* !defined(SQLITE_OMIT_TRACE) */
 
 #if SQLITE_USER_AUTHENTICATION
   if( c=='u' && strncmp(azArg[0], "user", n)==0 ){
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 52a111555d..7c43a1c667 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1365,10 +1365,13 @@ void sqlite3CryptFunc(sqlite3_context*,int,sqlite3_value**);
 /* This is an extra SQLITE_TRACE macro that indicates "legacy" tracing
 ** in the style of sqlite3_trace()
 */
-#define SQLITE_TRACE_LEGACY  0x80
+#define SQLITE_TRACE_LEGACY          0x40     /* Use the legacy xTrace */
+#define SQLITE_TRACE_XPROFILE        0x80     /* Use the legacy xProfile */
 #else
-#define SQLITE_TRACE_LEGACY  0
+#define SQLITE_TRACE_LEGACY          0
+#define SQLITE_TRACE_XPROFILE        0
 #endif /* SQLITE_OMIT_DEPRECATED */
+#define SQLITE_TRACE_NONLEGACY_MASK  0x0f     /* Normal flags */
 
 
 /*
@@ -1429,8 +1432,10 @@ struct sqlite3 {
   void **aExtension;            /* Array of shared library handles */
   int (*xTrace)(u32,void*,void*,void*);     /* Trace function */
   void *pTraceArg;                          /* Argument to the trace function */
+#ifndef SQLITE_OMIT_DEPRECATED
   void (*xProfile)(void*,const char*,u64);  /* Profiling function */
   void *pProfileArg;                        /* Argument to profile function */
+#endif
   void *pCommitArg;                 /* Argument to xCommitCallback() */
   int (*xCommitCallback)(void*);    /* Invoked at every commit. */
   void *pRollbackArg;               /* Argument to xRollbackCallback() */
@@ -4433,7 +4438,7 @@ int sqlite3VdbeParameterIndex(Vdbe*, const char*, int);
 int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *);
 void sqlite3ParserReset(Parse*);
 #ifdef SQLITE_ENABLE_NORMALIZE
-void sqlite3Normalize(Vdbe*, const char*, int, u8);
+char *sqlite3Normalize(Vdbe*, const char*, int);
 #endif
 int sqlite3Reprepare(Vdbe*);
 void sqlite3ExprListCheckLength(Parse*, ExprList*, const char*);
diff --git a/src/vdbeapi.c b/src/vdbeapi.c
index 59327bed38..ca073589f7 100644
--- a/src/vdbeapi.c
+++ b/src/vdbeapi.c
@@ -62,14 +62,16 @@ static SQLITE_NOINLINE void invokeProfileCallback(sqlite3 *db, Vdbe *p){
   sqlite3_int64 iNow;
   sqlite3_int64 iElapse;
   assert( p->startTime>0 );
-  assert( db->xProfile!=0 || (db->mTrace & SQLITE_TRACE_PROFILE)!=0 );
+  assert( (db->mTrace & (SQLITE_TRACE_PROFILE|SQLITE_TRACE_XPROFILE))!=0 );
   assert( db->init.busy==0 );
   assert( p->zSql!=0 );
   sqlite3OsCurrentTimeInt64(db->pVfs, &iNow);
   iElapse = (iNow - p->startTime)*1000000;
+#ifndef SQLITE_OMIT_DEPRECATED  	
   if( db->xProfile ){
     db->xProfile(db->pProfileArg, p->zSql, iElapse);
   }
+#endif
   if( db->mTrace & SQLITE_TRACE_PROFILE ){
     db->xTrace(SQLITE_TRACE_PROFILE, db->pTraceArg, p, (void*)&iElapse);
   }
@@ -602,7 +604,7 @@ static int sqlite3Step(Vdbe *p){
     );
 
 #ifndef SQLITE_OMIT_TRACE
-    if( (db->xProfile || (db->mTrace & SQLITE_TRACE_PROFILE)!=0)
+    if( (db->mTrace & (SQLITE_TRACE_PROFILE|SQLITE_TRACE_XPROFILE))!=0
         && !db->init.busy && p->zSql ){
       sqlite3OsCurrentTimeInt64(db->pVfs, &p->startTime);
     }else{
@@ -629,16 +631,18 @@ static int sqlite3Step(Vdbe *p){
     db->nVdbeExec--;
   }
 
+  if( rc!=SQLITE_ROW ){
 #ifndef SQLITE_OMIT_TRACE
-  /* If the statement completed successfully, invoke the profile callback */
-  if( rc!=SQLITE_ROW ) checkProfileCallback(db, p);
+    /* If the statement completed successfully, invoke the profile callback */
+    checkProfileCallback(db, p);
 #endif
 
-  if( rc==SQLITE_DONE && db->autoCommit ){
-    assert( p->rc==SQLITE_OK );
-    p->rc = doWalCallbacks(db);
-    if( p->rc!=SQLITE_OK ){
-      rc = SQLITE_ERROR;
+    if( rc==SQLITE_DONE && db->autoCommit ){
+      assert( p->rc==SQLITE_OK );
+      p->rc = doWalCallbacks(db);
+      if( p->rc!=SQLITE_OK ){
+        rc = SQLITE_ERROR;
+      }
     }
   }
 
@@ -1708,7 +1712,11 @@ char *sqlite3_expanded_sql(sqlite3_stmt *pStmt){
 */
 const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt){
   Vdbe *p = (Vdbe *)pStmt;
-  return p ? p->zNormSql : 0;
+  if( p==0 ) return 0;
+  if( p->zNormSql==0 && p->zSql!=0 ){
+    p->zNormSql = sqlite3Normalize(p, p->zSql, sqlite3Strlen30(p->zSql));
+  }
+  return p->zNormSql;
 }
 #endif /* SQLITE_ENABLE_NORMALIZE */
 
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index 85f0f7980f..ba8d2b11c2 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -67,7 +67,7 @@ void sqlite3VdbeSetSql(Vdbe *p, const char *z, int n, u8 prepFlags){
 #ifdef SQLITE_ENABLE_NORMALIZE
   assert( p->zNormSql==0 );
   if( p->zSql && (prepFlags & SQLITE_PREPARE_NORMALIZE)!=0 ){
-    sqlite3Normalize(p, p->zSql, n, prepFlags);
+    p->zNormSql = sqlite3Normalize(p, p->zSql, n);
     assert( p->zNormSql!=0 || p->db->mallocFailed );
   }
 #endif
diff --git a/src/wal.c b/src/wal.c
index c67bd9dcac..4b22ba42ef 100644
--- a/src/wal.c
+++ b/src/wal.c
@@ -2856,19 +2856,37 @@ void sqlite3WalEndReadTransaction(Wal *pWal){
 }
 
 /*
-** Search the hash tables for an entry matching page number pgno. Ignore
-** any entries that lie after frame iLast within the wal file.
+** Search the wal file for page pgno. If found, set *piRead to the frame that
+** contains the page. Otherwise, if pgno is not in the wal file, set *piRead
+** to zero.
+**
+** Return SQLITE_OK if successful, or an error code if an error occurs. If an
+** error does occur, the final value of *piRead is undefined.
 */
-static int walFindFrame(
-  Wal *pWal, 
-  Pgno pgno, 
-  u32 iLast, 
-  u32 *piRead
+int sqlite3WalFindFrame(
+  Wal *pWal,                      /* WAL handle */
+  Pgno pgno,                      /* Database page number to read data for */
+  u32 *piRead                     /* OUT: Frame number (or zero) */
 ){
+  u32 iRead = 0;                  /* If !=0, WAL frame to return data from */
+  u32 iLast = pWal->hdr.mxFrame;  /* Last page in WAL for this reader */
   int iHash;                      /* Used to loop through N hash tables */
-  u32 iRead = 0;
   int iMinHash;
 
+  /* This routine is only be called from within a read transaction. */
+  assert( pWal->readLock>=0 || pWal->lockError );
+
+  /* If the "last page" field of the wal-index header snapshot is 0, then
+  ** no data will be read from the wal under any circumstances. Return early
+  ** in this case as an optimization.  Likewise, if pWal->readLock==0, 
+  ** then the WAL is ignored by the reader so return early, as if the 
+  ** WAL were empty.
+  */
+  if( iLast==0 || (pWal->readLock==0 && pWal->bShmUnreliable==0) ){
+    *piRead = 0;
+    return SQLITE_OK;
+  }
+
   /* Each iteration of the following for() loop searches one
   ** hash table (each hash table indexes up to HASHTABLE_NPAGE frames).
   **
@@ -2919,48 +2937,11 @@ static int walFindFrame(
     if( iRead ) break;
   }
 
-  *piRead = iRead;
-  return SQLITE_OK;
-}
-
-/*
-** Search the wal file for page pgno. If found, set *piRead to the frame that
-** contains the page. Otherwise, if pgno is not in the wal file, set *piRead
-** to zero.
-**
-** Return SQLITE_OK if successful, or an error code if an error occurs. If an
-** error does occur, the final value of *piRead is undefined.
-*/
-int sqlite3WalFindFrame(
-  Wal *pWal,                      /* WAL handle */
-  Pgno pgno,                      /* Database page number to read data for */
-  u32 *piRead                     /* OUT: Frame number (or zero) */
-){
-  u32 iRead = 0;                  /* If !=0, WAL frame to return data from */
-  u32 iLast = pWal->hdr.mxFrame;  /* Last page in WAL for this reader */
-  int rc;
-
-  /* This routine is only be called from within a read transaction. */
-  assert( pWal->readLock>=0 || pWal->lockError );
-
-  /* If the "last page" field of the wal-index header snapshot is 0, then
-  ** no data will be read from the wal under any circumstances. Return early
-  ** in this case as an optimization.  Likewise, if pWal->readLock==0, 
-  ** then the WAL is ignored by the reader so return early, as if the 
-  ** WAL were empty.
-  */
-  if( iLast==0 || (pWal->readLock==0 && pWal->bShmUnreliable==0) ){
-    *piRead = 0;
-    return SQLITE_OK;
-  }
-
-  rc = walFindFrame(pWal, pgno, iLast, &iRead);
-
 #ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT
   /* If expensive assert() statements are available, do a linear search
   ** of the wal-index file content. Make sure the results agree with the
   ** result obtained using the hash indexes above.  */
-  if( rc==SQLITE_OK ){
+  {
     u32 iRead2 = 0;
     u32 iTest;
     assert( pWal->bShmUnreliable || pWal->minFrame>0 );
@@ -2975,7 +2956,7 @@ int sqlite3WalFindFrame(
 #endif
 
   *piRead = iRead;
-  return rc;
+  return SQLITE_OK;
 }
 
 /*
diff --git a/test/fts4umlaut.test b/test/fts4umlaut.test
new file mode 100644
index 0000000000..4dd96b220c
--- /dev/null
+++ b/test/fts4umlaut.test
@@ -0,0 +1,65 @@
+# 2018 December 3
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix fts4umlaut
+
+ifcapable !fts3 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x);
+  CREATE VIRTUAL TABLE t2 USING fts4(
+      x, 
+      tokenize=unicode61 "remove_diacritics=2"
+  );
+}
+
+foreach {tn q res1 res2} {
+  1 "Hà Nội"                  0 1
+  2 "Hà Noi"                  1 1
+  3 "Ha Noi"                  1 1
+  4 "Ha N\u1ed9i"             0 1
+  5 "Ha N\u006fi"             1 1
+  6 "Ha N\u006f\u0302i"       1 1
+  7 "Ha N\u006f\u0323\u0302i" 1 1
+} {
+  do_execsql_test 1.$tn.1 {
+    DELETE FROM t1;
+    INSERT INTO t1(rowid, x) VALUES (1, 'Ha Noi');
+    SELECT count(*) FROM t1 WHERE t1 MATCH $q
+  } $res1
+  do_execsql_test 1.$tn.2 {
+    DELETE FROM t1;
+    INSERT INTO t1(rowid, x) VALUES (1, $q);
+    SELECT count(*) FROM t1 WHERE t1 MATCH 'Ha Noi'
+  } $res1
+
+  do_execsql_test 1.$tn.2 {
+    DELETE FROM t2;
+    INSERT INTO t2(rowid, x) VALUES (1, 'Ha Noi');
+    SELECT count(*) FROM t2 WHERE t2 MATCH $q
+  } $res2
+  do_execsql_test 1.$tn.2 {
+    DELETE FROM t2;
+    INSERT INTO t2(rowid, x) VALUES (1, $q);
+    SELECT count(*) FROM t2 WHERE t2 MATCH 'Ha Noi'
+  } $res2
+}
+
+finish_test
+
diff --git a/test/json101.test b/test/json101.test
index 9a93ee739f..534478df93 100644
--- a/test/json101.test
+++ b/test/json101.test
@@ -813,6 +813,23 @@ do_execsql_test json-14.170 {
   SELECT fullkey FROM json_tree('null');
 } {$}
 
-
+# 2018-12-03
+# Make sure the table-valued functions contained within parentheses
+# work correctly.
+#
+# Bug reported via private email. See TH3 for more information.
+#
+do_execsql_test json-15.100 {
+  SELECT * FROM JSON_EACH('{"a":1, "b":2}');
+} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
+do_execsql_test json-15.110 {
+  SELECT xyz.* FROM JSON_EACH('{"a":1, "b":2}') AS xyz;
+} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
+do_execsql_test json-15.120 {
+  SELECT * FROM (JSON_EACH('{"a":1, "b":2}'));
+} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
+do_execsql_test json-15.130 {
+  SELECT xyz.* FROM (JSON_EACH('{"a":1, "b":2}')) AS xyz;
+} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
 
 finish_test
diff --git a/test/shell4.test b/test/shell4.test
index 88e5e69a28..2210e74a67 100644
--- a/test/shell4.test
+++ b/test/shell4.test
@@ -107,14 +107,14 @@ SELECT 1;
 } {1 1 1}
 
 do_test shell4-2.1 {
-  catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace"
-} {1 {Usage: .trace FILE|off}}
+  catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace --unknown"
+} {1 {Unknown option "--unknown" on ".trace"}}
 do_test shell4-2.2 {
   catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace off\n.trace off\n"
 } {0 {}}
 do_test shell4-2.3 {
-  catchcmd ":memory:" ".trace stdout\n.trace\n.trace off\n.dump\n"
-} {/^1 {PRAGMA.*Usage:.*}$/}
+  catchcmd ":memory:" ".trace stdout\n.dump\n.trace off\n"
+} {/^0 {PRAGMA.*}$/}
 ifcapable trace {
 do_test shell4-2.4 {
   catchcmd ":memory:" ".trace stdout\nCREATE TABLE t1(x);SELECT * FROM t1;"
diff --git a/test/tabfunc01.test b/test/tabfunc01.test
index dfe3190b52..49f0df889e 100644
--- a/test/tabfunc01.test
+++ b/test/tabfunc01.test
@@ -125,6 +125,11 @@ do_execsql_test tabfunc01-4.3 {
   SELECT * FROM aux1.generate_series(1,4)
 } {1 2 3 4}
 
+# 2018-12-03: Fix bug reported by by private email.
+do_execsql_test tabfunc01-4.4 {
+  SELECT * FROM (generate_series(1,5,2)) AS x LIMIT 10;
+} {1 3 5}
+
 # The next series of tests is verifying that virtual table are able
 # to optimize the IN operator, even on terms that are not marked "omit".
 # When the generate_series virtual table is compiled for the testfixture,
diff --git a/tool/index_usage.c b/tool/index_usage.c
new file mode 100644
index 0000000000..a86202425e
--- /dev/null
+++ b/tool/index_usage.c
@@ -0,0 +1,164 @@
+/*
+** 2018-12-04
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** 
+** This file implements a utility program used to help determine which
+** indexes in a database schema are used and unused, and how often specific
+** indexes are used.
+*/
+#include "sqlite3.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <string.h>
+
+static void usage(const char *argv0){
+  printf("Usage: %s DATABASE LOG\n\n", argv0);
+  printf(
+    "DATABASE is an SQLite database against which various statements\n"
+    "have been run.  The SQL text is stored in LOG.  LOG is an SQLite\n"
+    "database with this schema:\n"
+    "\n"
+    "    CREATE TABLE sqllog(sql TEXT);\n"
+    "\n"
+    "This utility program analyzes statements contained in LOG and prints\n"
+    "a report showing how many times each index in DATABASE is used by the\n"
+    "statements in LOG.\n"
+    "\n"
+    "DATABASE only needs to contain the schema used by the statements in\n"
+    "LOG. The content can be removed from DATABASE.\n"
+  );
+  printf("\nAnalysis will be done by SQLite version %s dated %.20s\n"
+         "checkin number %.40s. Different versions\n"
+         "of SQLite might use different indexes.\n",
+         sqlite3_libversion(), sqlite3_sourceid(), sqlite3_sourceid()+21);
+  exit(1);
+}
+
+int main(int argc, char **argv){
+  sqlite3 *db = 0;          /* The main database */
+  sqlite3_stmt *pStmt = 0;  /* a query */
+  char *zSql;
+  int nErr = 0;
+  int rc;
+
+  if( argc!=3 ) usage(argv[0]);
+  rc = sqlite3_open_v2(argv[1], &db, SQLITE_OPEN_READONLY, 0);
+  if( rc ){
+    printf("Cannot open \"%s\" for reading: %s\n", argv[1], sqlite3_errmsg(db));
+    goto errorOut;
+  }
+  rc = sqlite3_prepare_v2(db, "SELECT * FROM sqlite_master", -1, &pStmt, 0);
+  if( rc ){
+    printf("Cannot read the schema from \"%s\" - %s\n", argv[1],
+           sqlite3_errmsg(db));
+    goto errorOut;
+  }
+  sqlite3_finalize(pStmt);
+  pStmt = 0;
+  rc = sqlite3_exec(db, 
+     "CREATE TABLE temp.idxu(\n"
+     "  tbl TEXT,\n"
+     "  idx TEXT,\n"
+     "  cnt INT,\n"
+     "  PRIMARY KEY(idx)\n"
+     ") WITHOUT ROWID;", 0, 0, 0);
+  if( rc ){
+    printf("Cannot create the result table - %s\n",
+           sqlite3_errmsg(db));
+    goto errorOut;
+  }
+  rc = sqlite3_exec(db,
+     "INSERT INTO temp.idxu(tbl,idx,cnt)"
+     " SELECT tbl_name, name, 0 FROM sqlite_master"
+     " WHERE type='index' AND sql IS NOT NULL", 0, 0, 0);
+
+  /* Open the LOG database */
+  zSql = sqlite3_mprintf("ATTACH %Q AS log", argv[2]);
+  rc = sqlite3_exec(db, zSql, 0, 0, 0);
+  sqlite3_free(zSql);
+  if( rc ){
+    printf("Cannot open the LOG database \"%s\" - %s\n",
+           argv[2], sqlite3_errmsg(db));
+    goto errorOut;
+  }
+  rc = sqlite3_prepare_v2(db, "SELECT sql, rowid FROM log.sqllog",
+                          -1, &pStmt, 0);
+  if( rc ){
+    printf("Cannot read the SQLLOG table in the LOG database \"%s\" - %s\n",
+           argv[2], sqlite3_errmsg(db));
+    goto errorOut;
+  }
+
+  /* Update the counts based on LOG */
+  while( sqlite3_step(pStmt)==SQLITE_ROW ){
+    const char *zLog = (const char*)sqlite3_column_text(pStmt, 0);
+    sqlite3_stmt *pS2;
+    if( zLog==0 ) continue;
+    zSql = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", zLog);
+    rc = sqlite3_prepare_v2(db, zSql, -1, &pS2, 0);
+    sqlite3_free(zSql);
+    if( rc ){
+      printf("Cannot compile LOG entry %d (%s): %s\n",
+             sqlite3_column_int(pStmt, 1), zLog, sqlite3_errmsg(db));
+      nErr++;
+    }else{
+      while( sqlite3_step(pS2)==SQLITE_ROW ){
+        const char *zExplain = (const char*)sqlite3_column_text(pS2,3);
+        const char *z1, *z2;
+        int n;
+        /* printf("EXPLAIN: %s\n", zExplain); */
+        z1 = strstr(zExplain, " USING INDEX ");
+        if( z1==0 ) continue;
+        z1 += 13;
+        for(z2=z1+1; z2[1] && z2[1]!='('; z2++){}
+        n = z2 - z1;
+        zSql = sqlite3_mprintf(
+          "UPDATE temp.idxu SET cnt=cnt+1 WHERE idx='%.*q'", n, z1
+        );
+        /* printf("sql: %s\n", zSql); */
+        sqlite3_exec(db, zSql, 0, 0, 0);
+        sqlite3_free(zSql);
+      }
+    }
+    sqlite3_finalize(pS2);
+  }
+  sqlite3_finalize(pStmt);
+
+  /* Generate the report */
+  rc = sqlite3_prepare_v2(db,
+     "SELECT tbl, idx, cnt, "
+     "   (SELECT group_concat(name,',') FROM pragma_index_info(idx))"
+     " FROM temp.idxu, main.sqlite_master"
+     " WHERE temp.idxu.tbl=main.sqlite_master.tbl_name"
+     "   AND temp.idxu.idx=main.sqlite_master.name"
+     " ORDER BY cnt DESC, tbl, idx",
+     -1, &pStmt, 0);
+  if( rc ){
+    printf("Cannot query the result table - %s\n",
+           sqlite3_errmsg(db));
+    goto errorOut;
+  }
+  while( sqlite3_step(pStmt)==SQLITE_ROW ){
+    printf("%10d %s on %s(%s)\n", 
+       sqlite3_column_int(pStmt, 2),
+       sqlite3_column_text(pStmt, 1),
+       sqlite3_column_text(pStmt, 0),
+       sqlite3_column_text(pStmt, 3));
+  }
+  sqlite3_finalize(pStmt);
+  pStmt = 0;
+
+errorOut:
+  sqlite3_finalize(pStmt);
+  sqlite3_close(db);
+  return nErr;
+}
diff --git a/tool/lemon.c b/tool/lemon.c
index 1fca8b9755..7f0e557535 100644
--- a/tool/lemon.c
+++ b/tool/lemon.c
@@ -4590,13 +4590,20 @@ void ReportTable(
   tplt_print(out,lemp,lemp->overflow,&lineno);
   tplt_xfer(lemp->name,in,out,&lineno);
 
-  /* Generate the table of rule information
+  /* Generate the tables of rule information.  yyRuleInfoLhs[] and
+  ** yyRuleInfoNRhs[].
   **
   ** Note: This code depends on the fact that rules are number
   ** sequentually beginning with 0.
   */
   for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){
-    fprintf(out,"  { %4d, %4d }, /* (%d) ",rp->lhs->index,-rp->nrhs,i);
+    fprintf(out,"  %4d,  /* (%d) ", rp->lhs->index, i);
+     rule_print(out, rp);
+    fprintf(out," */\n"); lineno++;
+  }
+  tplt_xfer(lemp->name,in,out,&lineno);
+  for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){
+    fprintf(out,"  %3d,  /* (%d) ", -rp->nrhs, i);
     rule_print(out, rp);
     fprintf(out," */\n"); lineno++;
   }
diff --git a/tool/lempar.c b/tool/lempar.c
index 325b0e5418..94c0a3162a 100644
--- a/tool/lempar.c
+++ b/tool/lempar.c
@@ -686,13 +686,15 @@ static void yy_shift(
   yyTraceShift(yypParser, yyNewState, "Shift");
 }
 
-/* The following table contains information about every rule that
-** is used during the reduce.
-*/
-static const struct {
-  YYCODETYPE lhs;       /* Symbol on the left-hand side of the rule */
-  signed char nrhs;     /* Negative of the number of RHS symbols in the rule */
-} yyRuleInfo[] = {
+/* For rule J, yyRuleInfoLhs[J] contains the symbol on the left-hand side
+** of that rule */
+static const YYCODETYPE yyRuleInfoLhs[] = {
+%%
+};
+
+/* For rule J, yyRuleInfoNRhs[J] contains the negative of the number
+** of symbols on the right-hand side of that rule. */
+static const signed char yyRuleInfoNRhs[] = {
 %%
 };
 
@@ -725,7 +727,7 @@ static YYACTIONTYPE yy_reduce(
   yymsp = yypParser->yytos;
 #ifndef NDEBUG
   if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){
-    yysize = yyRuleInfo[yyruleno].nrhs;
+    yysize = yyRuleInfoNRhs[yyruleno];
     if( yysize ){
       fprintf(yyTraceFILE, "%sReduce %d [%s], go to state %d.\n",
         yyTracePrompt,
@@ -740,7 +742,7 @@ static YYACTIONTYPE yy_reduce(
   /* Check that the stack is large enough to grow by a single entry
   ** if the RHS of the rule is empty.  This ensures that there is room
   ** enough on the stack to push the LHS value */
-  if( yyRuleInfo[yyruleno].nrhs==0 ){
+  if( yyRuleInfoNRhs[yyruleno]==0 ){
 #ifdef YYTRACKMAXSTACKDEPTH
     if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){
       yypParser->yyhwm++;
@@ -782,9 +784,9 @@ static YYACTIONTYPE yy_reduce(
 %%
 /********** End reduce actions ************************************************/
   };
-  assert( yyruleno<sizeof(yyRuleInfo)/sizeof(yyRuleInfo[0]) );
-  yygoto = yyRuleInfo[yyruleno].lhs;
-  yysize = yyRuleInfo[yyruleno].nrhs;
+  assert( yyruleno<sizeof(yyRuleInfoLhs)/sizeof(yyRuleInfoLhs[0]) );
+  yygoto = yyRuleInfoLhs[yyruleno];
+  yysize = yyRuleInfoNRhs[yyruleno];
   yyact = yy_find_reduce_action(yymsp[yysize].stateno,(YYCODETYPE)yygoto);
 
   /* There are no SHIFTREDUCE actions on nonterminals because the table