A new approach for UTF-8 translation. (CVS 4004)

FossilOrigin-Name: 6c8ad2790eaede90b3f1ef62614e667178b2a8c4

manifest

@@ -1,5 +1,5 @@
-C Do\snot\sallow\sbytes\s0x80\sthrough\s0xBF\sto\sbe\sthe\sfirst\scharacter\sof\san\nidentifer\sbecause\sno\svalid\sUTF-8\scharacter\scan\sbegin\swith\sthose\sbytes.\nIf\swe\sallowed\san\sidentifier\sto\sbegin\swith\sone\sof\sthose\sbytes,\sthen\sthe\nsubstr()\sfunction\sin\sALTER\sTABLE\swill\snot\swork\scorrectly.\s(CVS\s4003)
+C A\snew\sapproach\sfor\sUTF-8\stranslation.\s(CVS\s4004)
-D 2007-05-15T09:00:15
+D 2007-05-15T11:55:09
 F Makefile.in 87b200ad9970907f76df734d29dff3d294c10935
 F Makefile.linux-gcc 2d8574d1ba75f129aba2019f0b959db380a90935
 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028
@@ -71,7 +71,7 @@ F src/date.c 6049db7d5a8fdf2c677ff7d58fa31d4f6593c988
 F src/delete.c 5c0d89b3ef7d48fe1f5124bfe8341f982747fe29
 F src/experimental.c 1b2d1a6cd62ecc39610e97670332ca073c50792b
 F src/expr.c 436f1d3e5addf95c195016b518cd2f44a6f5f081
-F src/func.c cdbe36f2a71b7e6835189414c2fb626a339e300f
+F src/func.c fc1524fd6097b19c54cc4555e3ea724eec628e2c
 F src/hash.c 67b23e14f0257b69a3e8aa663e4eeadc1a2b6fd5
 F src/hash.h 1b3f7e2609141fd571f62199fc38687d262e9564
 F src/insert.c e595ca26805dfb3a9ebaabc28e7947c479f3b14d
@@ -104,7 +104,7 @@ F src/server.c 087b92a39d883e3fa113cae259d64e4c7438bc96
 F src/shell.c d07ae326b3815d80f71c69b3c7584382e47f6447
 F src/sqlite.h.in aa3c0c02883663944db09ee79f3165d3c4ad7c47
 F src/sqlite3ext.h 7d0d363ea7327e817ef0dfe1b7eee1f171b72890
-F src/sqliteInt.h c31c9526bc602c3c71ddc45c548e987530826f11
+F src/sqliteInt.h 591f857fbf89a2a30f1d7c30018abb4596c4fc71
 F src/table.c a8de75bcedf84d4060d804264b067ab3b1a3561d
 F src/tclsqlite.c f425c7583665ef78dd8397b2de0b8e0028e80ce2
 F src/test1.c 84c841e1088f743200b87581506e93f70344bd32
@@ -129,7 +129,7 @@ F src/test_tclvar.c 315e77c17f128ff8c06b38c08617fd07c825a95b
 F src/tokenize.c 9aa8e3f06f56a700ef498582dae431be3d5c4f4c
 F src/trigger.c 420192efe3e6f03addf7897c60c3c8bf913d3493
 F src/update.c 3359041db390a8f856d67272f299600e2104f350
-F src/utf.c be7c64eed83fa3c01e0c42905e1c311dcd1be704
+F src/utf.c 3fe21361e83cfcf5c18764d8d5c59fffdf9144db
 F src/util.c 4f6bbcec2b2b1884d652b82c9f8949ede4618d68
 F src/vacuum.c 8bd895d29e7074e78d4e80f948e35ddc9cf2beef
 F src/vdbe.c 5deb4cdccd57065ccf8a2e5c704e8473c90d204b
@@ -491,7 +491,7 @@ F www/tclsqlite.tcl bb0d1357328a42b1993d78573e587c6dcbc964b9
 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0
 F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b
 F www/whentouse.tcl fc46eae081251c3c181bd79c5faef8195d7991a5
-P 51eeae7b7ad5a0fbdd9d4418120dbdb6cd577cd5
+P 252810424d8c4dcd19b369d62027094df7cf0bcc
-R 710324b40f5e54ea717a1562c7c07a4d
+R 8b16cbfea1aee4c8ac49aefcccb7ea15
 U drh
-Z 6fd77513c09320f6a39989f1ee4db43a
+Z 1d3ef6d9d156a8b50e6d1607d4475644

manifest.uuid

@@ -1 +1 @@
-252810424d8c4dcd19b369d62027094df7cf0bcc
+6c8ad2790eaede90b3f1ef62614e667178b2a8c4

src/func.c

@@ -16,7 +16,7 @@
 ** sqliteRegisterBuildinFunctions() found at the bottom of the file.
 ** All other code has file scope.
 **
-** $Id: func.c,v 1.156 2007/05/15 01:13:47 drh Exp $
+** $Id: func.c,v 1.157 2007/05/15 11:55:09 drh Exp $
 */
 #include "sqliteInt.h"
 #include <ctype.h>
@@ -103,7 +103,11 @@ static void lengthFunc(
     case SQLITE_TEXT: {
       const unsigned char *z = sqlite3_value_text(argv[0]);
       if( z==0 ) return;
-      for(len=0; *z; z++){ if( (0xc0&*z)!=0x80 ) len++; }
+      len = 0;
+      while( *z ){
+        len++;
+        SQLITE_SKIP_UTF8(z);
+      }
       sqlite3_result_int(context, len);
       break;
     }
@@ -162,7 +166,6 @@ static void substrFunc(
 ){
   const unsigned char *z;
   const unsigned char *z2;
-  int i;
   int len;
   int p0type;
   i64 p1, p2;
@@ -176,7 +179,10 @@ static void substrFunc(
   }else{
     z = sqlite3_value_text(argv[0]);
     if( z==0 ) return;
-    for(len=0, z2=z; *z2; z2++){ if( (0xc0&*z2)!=0x80 ) len++; }
+    len = 0;
+    for(z2=z; *z2; len++){
+      SQLITE_SKIP_UTF8(z2);
+    }
   }
   p1 = sqlite3_value_int(argv[1]);
   p2 = sqlite3_value_int(argv[2]);
@@ -193,16 +199,14 @@ static void substrFunc(
     p2 = len-p1;
   }
   if( p0type!=SQLITE_BLOB ){
-    for(i=0; i<p1 && z[i]; i++){
+    while( *z && p1 ){
-      if( (z[i]&0xc0)==0x80 ) p1++;
+      SQLITE_SKIP_UTF8(z);
+      p1--;
     }
-    while( z[i] && (z[i]&0xc0)==0x80 ){ i++; p1++; }
+    for(z2=z; *z2 && p2; p2--){
-    for(; i<p1+p2 && z[i]; i++){
+      SQLITE_SKIP_UTF8(z2);
-      if( (z[i]&0xc0)==0x80 ) p2++;
     }
-    while( z[i] && (z[i]&0xc0)==0x80 ){ i++; p2++; }
+    sqlite3_result_text(context, (char*)z, z2-z, SQLITE_TRANSIENT);
-    if( p2<0 ) p2 = 0;
-    sqlite3_result_text(context, (char*)&z[p1], p2, SQLITE_TRANSIENT);
   }else{
     if( p2<0 ) p2 = 0;
     sqlite3_result_blob(context, (char*)&z[p1], p2, SQLITE_TRANSIENT);
@@ -389,13 +393,13 @@ static const struct compareInfo likeInfoNorm = { '%', '_',   0, 1 };
 static const struct compareInfo likeInfoAlt = { '%', '_',   0, 0 };
 
 /*
-** X is a pointer to the first byte of a UTF-8 character.  Increment
+** Read a single UTF-8 character and return its value.
-** X so that it points to the next character.  This only works right
-** if X points to a well-formed UTF-8 string.
 */
-#define sqliteNextChar(X)  while( (0xc0&*++(X))==0x80 ){}
+u32 sqlite3ReadUtf8(const unsigned char *z){
-#define sqliteCharVal(X)   sqlite3ReadUtf8(X)
+  u32 c;
-
+  SQLITE_READ_UTF8(z, c);
+  return c;
+}
 
 /*
 ** Compare two UTF-8 strings for equality where the first string can
@@ -446,20 +450,20 @@ static int patternCompare(
       while( (c=zPattern[1]) == matchAll || c == matchOne ){
         if( c==matchOne ){
           if( *zString==0 ) return 0;
-          sqliteNextChar(zString);
+          SQLITE_SKIP_UTF8(zString);
         }
         zPattern++;
       }
       if( c && esc && sqlite3ReadUtf8(&zPattern[1])==esc ){
         u8 const *zTemp = &zPattern[1];
-        sqliteNextChar(zTemp);
+        SQLITE_SKIP_UTF8(zTemp);
         c = *zTemp;
       }
       if( c==0 ) return 1;
       if( c==matchSet ){
         assert( esc==0 );   /* This is GLOB, not LIKE */
         while( *zString && patternCompare(&zPattern[1],zString,pInfo,esc)==0 ){
-          sqliteNextChar(zString);
+          SQLITE_SKIP_UTF8(zString);
         }
         return *zString!=0;
       }else{
@@ -473,20 +477,20 @@ static int patternCompare(
           }
           if( c2==0 ) return 0;
           if( patternCompare(&zPattern[1],zString,pInfo,esc) ) return 1;
-          sqliteNextChar(zString);
+          SQLITE_SKIP_UTF8(zString);
         }
         return 0;
       }
     }else if( !prevEscape && c==matchOne ){
       if( *zString==0 ) return 0;
-      sqliteNextChar(zString);
+      SQLITE_SKIP_UTF8(zString);
       zPattern++;
     }else if( c==matchSet ){
       int prior_c = 0;
       assert( esc==0 );    /* This only occurs for GLOB, not LIKE */
       seen = 0;
       invert = 0;
-      c = sqliteCharVal(zString);
+      c = sqlite3ReadUtf8(zString);
       if( c==0 ) return 0;
       c2 = *++zPattern;
       if( c2=='^' ){ invert = 1; c2 = *++zPattern; }
@@ -494,10 +498,10 @@ static int patternCompare(
         if( c==']' ) seen = 1;
         c2 = *++zPattern;
       }
-      while( (c2 = sqliteCharVal(zPattern))!=0 && c2!=']' ){
+      while( (c2 = sqlite3ReadUtf8(zPattern))!=0 && c2!=']' ){
         if( c2=='-' && zPattern[1]!=']' && zPattern[1]!=0 && prior_c>0 ){
           zPattern++;
-          c2 = sqliteCharVal(zPattern);
+          c2 = sqlite3ReadUtf8(zPattern);
           if( c>=prior_c && c<=c2 ) seen = 1;
           prior_c = 0;
         }else if( c==c2 ){
@@ -506,14 +510,14 @@ static int patternCompare(
         }else{
           prior_c = c2;
         }
-        sqliteNextChar(zPattern);
+        SQLITE_SKIP_UTF8(zPattern);
       }
       if( c2==0 || (seen ^ invert)==0 ) return 0;
-      sqliteNextChar(zString);
+      SQLITE_SKIP_UTF8(zString);
       zPattern++;
     }else if( esc && !prevEscape && sqlite3ReadUtf8(zPattern)==esc){
       prevEscape = 1;
-      sqliteNextChar(zPattern);
+      SQLITE_SKIP_UTF8(zPattern);
     }else{
       if( noCase ){
         if( sqlite3UpperToLower[c] != sqlite3UpperToLower[*zString] ) return 0;
@@ -851,7 +855,7 @@ static void trimFunc(
   }else{
     const unsigned char *z;
     for(z=zCharSet, nChar=0; *z; nChar++){
-      sqliteNextChar(z);
+      SQLITE_SKIP_UTF8(z);
     }
     if( nChar>0 ){
       azChar = sqlite3_malloc( nChar*(sizeof(char*)+1) );
@@ -861,7 +865,7 @@ static void trimFunc(
       aLen = (unsigned char*)&azChar[nChar];
       for(z=zCharSet, nChar=0; *z; nChar++){
         azChar[nChar] = z;
-        sqliteNextChar(z);
+        SQLITE_SKIP_UTF8(z);
         aLen[nChar] = z - azChar[nChar];
       }
     }

src/sqliteInt.h

@@ -11,7 +11,7 @@
 *************************************************************************
 ** Internal interface definitions for SQLite.
 **
-** @(#) $Id: sqliteInt.h,v 1.566 2007/05/12 12:08:51 drh Exp $
+** @(#) $Id: sqliteInt.h,v 1.567 2007/05/15 11:55:09 drh Exp $
 */
 #ifndef _SQLITEINT_H_
 #define _SQLITEINT_H_
@@ -1536,6 +1536,63 @@ typedef struct {
  */
 extern int sqlite3_always_code_trigger_setup;
 
+/*
+** A lookup table used by the SQLITE_READ_UTF8 macro.  The definition
+** is in utf.c.
+*/
+extern const unsigned char sqlite3UtfTrans1[];
+
+/*
+** Macros for reading UTF8 characters.
+**
+** SQLITE_READ_UTF8(x,c) reads a single UTF8 value out of x and writes
+** that value into c.  The type of x must be unsigned char*.  The type
+** of c must be unsigned int.
+**
+** SQLITE_SKIP_UTF8(x) advances x forward by one character.  The type of
+** x must be unsigned char*.
+**
+** Notes On Invalid UTF-8:
+**
+**  *  These macros never allow a 7-bit character (0x00 through 0x7f) to
+**     be encoded as a multi-byte character.  Any multi-byte character that
+**     attempts to encode a value between 0x00 and 0x7f is rendered as 0xfffd.
+**
+**  *  These macros never allow a UTF16 surragate value to be encoded.
+**     If a multi-byte character attempts to encode a value between
+**     0xd800 and 0xe000 then it is rendered as 0xfffd.
+**
+**  *  Bytes in the range of 0x80 through 0xbf which occur as the first
+**     byte of a character are interpreted as single-byte characters
+**     and rendered as themselves even though they are technically
+**     invalid characters.
+**
+**  *  These routines accept an infinite number of different UTF8 encodings
+**     for unicode values 0x80 and greater.  They do not change over-length
+**     encodings to 0xfffd as some systems recommend.
+** 
+*/
+#define SQLITE_READ_UTF8(zIn, c) {                     \
+  c = *(zIn++);                                        \
+  if( c>=0xc0 ){                                       \
+    c = sqlite3UtfTrans1[c-0xc0];                      \
+    while( (*zIn & 0xc0)==0x80 ){                      \
+      c = (c<<6) + (0x3f & *(zIn++));                  \
+    }                                                  \
+    if( c<0x80                                         \
+        || (c&0xFFFFF800)==0xD800                      \
+        || (c&0xFFFFFFFE)==0xFFFE ){  c = 0xFFFD; }    \
+  }                                                    \
+}
+#define SQLITE_SKIP_UTF8(zIn) {                        \
+  if( (*(zIn++))>=0xc0 ){                              \
+    while( (*zIn & 0xc0)==0x80 ){ zIn++; }             \
+  }                                                    \
+}
+
+
+
+
 /*
 ** The SQLITE_CORRUPT_BKPT macro can be either a constant (for production
 ** builds) or a function call (for debugging).  If it is a function call,
@@ -1753,7 +1810,7 @@ int sqlite3GetInt32(const char *, int*);
 int sqlite3FitsIn64Bits(const char *);
 int sqlite3Utf16ByteLen(const void *pData, int nChar);
 int sqlite3Utf8CharLen(const char *pData, int nByte);
-int sqlite3ReadUtf8(const unsigned char *);
+u32 sqlite3ReadUtf8(const unsigned char *);
 int sqlite3PutVarint(unsigned char *, u64);
 int sqlite3GetVarint(const unsigned char *, u64 *);
 int sqlite3GetVarint32(const unsigned char *, u32 *);

src/utf.c

@@ -12,7 +12,7 @@
 ** This file contains routines used to translate between UTF-8, 
 ** UTF-16, UTF-16BE, and UTF-16LE.
 **
-** $Id: utf.c,v 1.46 2007/05/10 17:23:12 drh Exp $
+** $Id: utf.c,v 1.47 2007/05/15 11:55:09 drh Exp $
 **
 ** Notes on UTF-8:
 **
@@ -34,29 +34,6 @@
 **     0xff 0xfe   little-endian utf-16 follows
 **     0xfe 0xff   big-endian utf-16 follows
 **
-**
-** Handling of malformed strings:
-**
-** SQLite accepts and processes malformed strings without an error wherever
-** possible. However this is not possible when converting between UTF-8 and
-** UTF-16.
-**
-** When converting malformed UTF-8 strings to UTF-16, one instance of the
-** replacement character U+FFFD for each byte that cannot be interpeted as
-** part of a valid unicode character.
-**
-** When converting malformed UTF-16 strings to UTF-8, one instance of the
-** replacement character U+FFFD for each pair of bytes that cannot be
-** interpeted as part of a valid unicode character.
-**
-** This file contains the following public routines:
-**
-** sqlite3VdbeMemTranslate() - Translate the encoding used by a Mem* string.
-** sqlite3VdbeMemHandleBom() - Handle byte-order-marks in UTF16 Mem* strings.
-** sqlite3Utf16ByteLen()     - Calculate byte-length of a void* UTF16 string.
-** sqlite3Utf8CharLen()      - Calculate char-length of a char* UTF8 string.
-** sqlite3Utf8LikeCompare()  - Do a LIKE match given two UTF8 char* strings.
-**
 */
 #include "sqliteInt.h"
 #include <assert.h>
@@ -69,89 +46,20 @@
 const int sqlite3one = 1;
 
 /*
-** This table maps from the first byte of a UTF-8 character to the number
+** This lookup table is used to help decode the first byte of
-** of trailing bytes expected. A value '4' indicates that the table key
+** a multi-byte UTF8 character.
-** is not a legal first byte for a UTF-8 character.
 */
-static const u8 xtra_utf8_bytes[256]  = {
+const unsigned char sqlite3UtfTrans1[] = {
-/* 0xxxxxxx */
+  0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
-0, 0, 0, 0, 0, 0, 0, 0,     0, 0, 0, 0, 0, 0, 0, 0,
+  0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
-0, 0, 0, 0, 0, 0, 0, 0,     0, 0, 0, 0, 0, 0, 0, 0,
+  0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
-0, 0, 0, 0, 0, 0, 0, 0,     0, 0, 0, 0, 0, 0, 0, 0,
+  0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
-0, 0, 0, 0, 0, 0, 0, 0,     0, 0, 0, 0, 0, 0, 0, 0,
+  0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
-0, 0, 0, 0, 0, 0, 0, 0,     0, 0, 0, 0, 0, 0, 0, 0,
+  0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
-0, 0, 0, 0, 0, 0, 0, 0,     0, 0, 0, 0, 0, 0, 0, 0,
+  0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
-0, 0, 0, 0, 0, 0, 0, 0,     0, 0, 0, 0, 0, 0, 0, 0,
+  0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00,
-0, 0, 0, 0, 0, 0, 0, 0,     0, 0, 0, 0, 0, 0, 0, 0,
-
-/* 10wwwwww */
-4, 4, 4, 4, 4, 4, 4, 4,     4, 4, 4, 4, 4, 4, 4, 4,
-4, 4, 4, 4, 4, 4, 4, 4,     4, 4, 4, 4, 4, 4, 4, 4,
-4, 4, 4, 4, 4, 4, 4, 4,     4, 4, 4, 4, 4, 4, 4, 4,
-4, 4, 4, 4, 4, 4, 4, 4,     4, 4, 4, 4, 4, 4, 4, 4,
-
-/* 110yyyyy */
-1, 1, 1, 1, 1, 1, 1, 1,     1, 1, 1, 1, 1, 1, 1, 1,
-1, 1, 1, 1, 1, 1, 1, 1,     1, 1, 1, 1, 1, 1, 1, 1,
-
-/* 1110zzzz */
-2, 2, 2, 2, 2, 2, 2, 2,     2, 2, 2, 2, 2, 2, 2, 2,
-
-/* 11110yyy */
-3, 3, 3, 3, 3, 3, 3, 3,     4, 4, 4, 4, 4, 4, 4, 4,
 };
 
-/*
-** This table maps from the number of trailing bytes in a UTF-8 character
-** to an integer constant that is effectively calculated for each character
-** read by a naive implementation of a UTF-8 character reader. The code
-** in the READ_UTF8 macro explains things best.
-*/
-static const int xtra_utf8_bits[] =  {
-  0,
-  12416,          /* (0xC0 << 6) + (0x80) */
-  925824,         /* (0xE0 << 12) + (0x80 << 6) + (0x80) */
-  63447168        /* (0xF0 << 18) + (0x80 << 12) + (0x80 << 6) + 0x80 */
-};
-
-/*
-** If a UTF-8 character contains N bytes extra bytes (N bytes follow
-** the initial byte so that the total character length is N+1) then
-** masking the character with utf8_mask[N] must produce a non-zero
-** result.  Otherwise, we have an (illegal) overlong encoding.
-*/
-static const int utf_mask[] = {
-  0x00000000,
-  0xffffff80,
-  0xfffff800,
-  0xffff0000,
-};
-
-#define READ_UTF8(zIn, c) { \
-  int xtra;                                            \
-  c = *(zIn)++;                                        \
-  xtra = xtra_utf8_bytes[c];                           \
-  switch( xtra ){                                      \
-    case 4: c = (int)0xFFFD; break;                    \
-    case 3: c = (c<<6) + *(zIn)++;                     \
-    case 2: c = (c<<6) + *(zIn)++;                     \
-    case 1: c = (c<<6) + *(zIn)++;                     \
-    c -= xtra_utf8_bits[xtra];                         \
-    if( (utf_mask[xtra]&c)==0                          \
-        || (c&0xFFFFF800)==0xD800                      \
-        || (c&0xFFFFFFFE)==0xFFFE ){  c = 0xFFFD; }    \
-  }                                                    \
-}
-int sqlite3ReadUtf8(const unsigned char *z){
-  int c;
-  READ_UTF8(z, c);
-  return c;
-}
-
-#define SKIP_UTF8(zIn) {                               \
-  zIn += (xtra_utf8_bytes[*(u8 *)zIn] + 1);            \
-}
-
 #define WRITE_UTF8(zOut, c) {                          \
   if( c<0x00080 ){                                     \
     *zOut++ = (c&0xFF);                                \
@@ -346,14 +254,14 @@ int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){
     if( desiredEnc==SQLITE_UTF16LE ){
       /* UTF-8 -> UTF-16 Little-endian */
       while( zIn<zTerm ){
-        READ_UTF8(zIn, c); 
+        SQLITE_READ_UTF8(zIn, c); 
         WRITE_UTF16LE(z, c);
       }
     }else{
       assert( desiredEnc==SQLITE_UTF16BE );
       /* UTF-8 -> UTF-16 Big-endian */
       while( zIn<zTerm ){
-        READ_UTF8(zIn, c); 
+        SQLITE_READ_UTF8(zIn, c); 
         WRITE_UTF16BE(z, c);
       }
     }
@@ -457,17 +365,18 @@ int sqlite3VdbeMemHandleBom(Mem *pMem){
 ** number of unicode characters in the first nByte of pZ (or up to 
 ** the first 0x00, whichever comes first).
 */
-int sqlite3Utf8CharLen(const char *z, int nByte){
+int sqlite3Utf8CharLen(const char *zIn, int nByte){
   int r = 0;
-  const char *zTerm;
+  const u8 *z = (const u8*)zIn;
+  const u8 *zTerm;
   if( nByte>=0 ){
     zTerm = &z[nByte];
   }else{
-    zTerm = (const char *)(-1);
+    zTerm = (const u8*)(-1);
   }
   assert( z<=zTerm );
   while( *z!=0 && z<zTerm ){
-    SKIP_UTF8(z);
+    SQLITE_SKIP_UTF8(z);
     r++;
   }
   return r;
@@ -589,7 +498,7 @@ int sqlite3Utf8To8(unsigned char *zIn){
   int c;
 
   while(1){
-    READ_UTF8(zIn, c);
+    SQLITE_READ_UTF8(zIn, c);
     if( c==0 ) break;
     if( c!=0xfffd ){
       WRITE_UTF8(zOut, c);
@@ -617,8 +526,9 @@ void sqlite3UtfSelfTest(){
     z = zBuf;
     WRITE_UTF8(z, i);
     n = z-zBuf;
+    z[0] = 0;
     z = zBuf;
-    READ_UTF8(z, c);
+    SQLITE_READ_UTF8(z, c);
     t = i;
     if( i>=0xD800 && i<=0xDFFF ) t = 0xFFFD;
     if( (i&0xFFFFFFFE)==0xFFFE ) t = 0xFFFD;
@@ -630,6 +540,7 @@ void sqlite3UtfSelfTest(){
     z = zBuf;
     WRITE_UTF16LE(z, i);
     n = z-zBuf;
+    z[0] = 0;
     z = zBuf;
     READ_UTF16LE(z, c);
     assert( c==i );
@@ -640,6 +551,7 @@ void sqlite3UtfSelfTest(){
     z = zBuf;
     WRITE_UTF16BE(z, i);
     n = z-zBuf;
+    z[0] = 0;
     z = zBuf;
     READ_UTF16BE(z, c);
     assert( c==i );