Improved handling of UTF8 by the command-line shell.

FossilOrigin-Name: 4534ebf15fbcd6fe2028957b7aa591b6cd5da95f

manifest

@@ -1,5 +1,5 @@
-C Add\sthe\s".auth\sON|OFF"\scommand\sto\sthe\scommand-line\sshell.
+C Improved\shandling\sof\sUTF8\sby\sthe\scommand-line\sshell.
-D 2016-04-04T17:23:10.395
+D 2016-04-04T17:34:54.498
 F Makefile.in e812bb732d7af01baa09f1278bd4f4a2e3a09449
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc fe57d7e3e74fa383fd01ced796c0ffd966fc094a
@@ -360,7 +360,7 @@ F src/os.h 91ff889115ecd01f436d3611f7f5ea4dc12d92f1
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa
 F src/os_unix.c b1ccb273771f41dbdbe0ba7c1ad63c38ad5972ec
-F src/os_win.c ff870d89f4cb088a04cbf5ea0cbd9ff1b089ff4a
+F src/os_win.c b3ba9573d8d893e70a6a8015bbee572ecf7ffbef
 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca
 F src/pager.c 38718a019ca762ba4f6795425d5a54db70d1790d
 F src/pager.h e1d38a2f14849e219df0f91f8323504d134c8a56
@@ -376,7 +376,7 @@ F src/random.c ba2679f80ec82c4190062d756f22d0c358180696
 F src/resolve.c b8f7174e5f8c33c44ded3a25a973d0bb89228c20
 F src/rowset.c 9fe4b3ad7cc00944386bb600233d8f523de07a6e
 F src/select.c 7849cee0a01952a9c93cd28989daedfa57731143
-F src/shell.c e0996a0be612c8d2630fdf8bcedf4c4260a29734
+F src/shell.c b7922fa264f8c8d72a5ec6dd0b091e15a93c4de5
 F src/sqlite.h.in c46a7b85d3f37371cacea8f98ec825f5e52c420c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 98f72cbfe00169c39089115427d06ea05fe4b4a2
@@ -1051,7 +1051,7 @@ F test/sharedA.test 0cdf1a76dfa00e6beee66af5b534b1e8df2720f5
 F test/sharedB.test 16cc7178e20965d75278f410943109b77b2e645e
 F test/shared_err.test 2f2aee20db294b9924e81f6ccbe60f19e21e8506
 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304
-F test/shell1.test dff5b20ad989770aface6d714491121172dfe8b0
+F test/shell1.test a216486c6aeae4d3e75e37d64e4b6c8974108750
 F test/shell2.test e242a9912f44f4c23c3d1d802a83e934e84c853b
 F test/shell3.test da513d522ef6f01cee8475dcf8332bff8982b3dd
 F test/shell4.test 69995ee1cc278eb149aa8746ce1f935f4eaf98b9
@@ -1482,7 +1482,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 8627a4cd6d64bd076b56c1e8ccc3b1dfc1b4c07d
+P 65c7bcc42786a254966c531ba9062abb8fc8c5bf f4cbd18db47af4af990d7891dcc831e92b3f17e0
-R 327fee759e5533f8bd09762659f3f70b
+R 6a3470f790b8def5155325c3cfdf4cee
+T +closed f4cbd18db47af4af990d7891dcc831e92b3f17e0
 U drh
-Z ff39f0b57606b297cab9fa07bc83cd18
+Z cd8719efec0532e424e0fef6159fe981

manifest.uuid

@@ -1 +1 @@
-65c7bcc42786a254966c531ba9062abb8fc8c5bf
+4534ebf15fbcd6fe2028957b7aa591b6cd5da95f

src/os_win.c

@@ -1306,6 +1306,12 @@ void sqlite3_win32_write_debug(const char *zBuf, int nBuf){
   int nMin = MIN(nBuf, (SQLITE_WIN32_DBG_BUF_SIZE - 1)); /* may be negative. */
   if( nMin<-1 ) nMin = -1; /* all negative values become -1. */
   assert( nMin==-1 || nMin==0 || nMin<SQLITE_WIN32_DBG_BUF_SIZE );
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( !zBuf ){
+    (void)SQLITE_MISUSE_BKPT;
+    return;
+  }
+#endif
 #if defined(SQLITE_WIN32_HAS_ANSI)
   if( nMin>0 ){
     memset(zDbgBuf, 0, SQLITE_WIN32_DBG_BUF_SIZE);
@@ -1631,147 +1637,244 @@ void sqlite3MemSetDefault(void){
 #endif /* SQLITE_WIN32_MALLOC */
 
 /*
-** Convert a UTF-8 string to Microsoft Unicode (UTF-16?).
+** Convert a UTF-8 string to Microsoft Unicode.
 **
-** Space to hold the returned string is obtained from malloc.
+** Space to hold the returned string is obtained from sqlite3_malloc().
 */
-static LPWSTR winUtf8ToUnicode(const char *zFilename){
+static LPWSTR winUtf8ToUnicode(const char *zText){
   int nChar;
-  LPWSTR zWideFilename;
+  LPWSTR zWideText;
 
-  nChar = osMultiByteToWideChar(CP_UTF8, 0, zFilename, -1, NULL, 0);
+  nChar = osMultiByteToWideChar(CP_UTF8, 0, zText, -1, NULL, 0);
   if( nChar==0 ){
     return 0;
   }
-  zWideFilename = sqlite3MallocZero( nChar*sizeof(zWideFilename[0]) );
+  zWideText = sqlite3MallocZero( nChar*sizeof(WCHAR) );
-  if( zWideFilename==0 ){
+  if( zWideText==0 ){
     return 0;
   }
-  nChar = osMultiByteToWideChar(CP_UTF8, 0, zFilename, -1, zWideFilename,
+  nChar = osMultiByteToWideChar(CP_UTF8, 0, zText, -1, zWideText,
                                 nChar);
   if( nChar==0 ){
-    sqlite3_free(zWideFilename);
+    sqlite3_free(zWideText);
-    zWideFilename = 0;
+    zWideText = 0;
   }
-  return zWideFilename;
+  return zWideText;
 }
 
 /*
-** Convert Microsoft Unicode to UTF-8.  Space to hold the returned string is
+** Convert a Microsoft Unicode string to UTF-8.
-** obtained from sqlite3_malloc().
+**
+** Space to hold the returned string is obtained from sqlite3_malloc().
 */
-static char *winUnicodeToUtf8(LPCWSTR zWideFilename){
+static char *winUnicodeToUtf8(LPCWSTR zWideText){
   int nByte;
-  char *zFilename;
+  char *zText;
 
-  nByte = osWideCharToMultiByte(CP_UTF8, 0, zWideFilename, -1, 0, 0, 0, 0);
+  nByte = osWideCharToMultiByte(CP_UTF8, 0, zWideText, -1, 0, 0, 0, 0);
   if( nByte == 0 ){
     return 0;
   }
-  zFilename = sqlite3MallocZero( nByte );
+  zText = sqlite3MallocZero( nByte );
-  if( zFilename==0 ){
+  if( zText==0 ){
     return 0;
   }
-  nByte = osWideCharToMultiByte(CP_UTF8, 0, zWideFilename, -1, zFilename, nByte,
+  nByte = osWideCharToMultiByte(CP_UTF8, 0, zWideText, -1, zText, nByte,
                                 0, 0);
   if( nByte == 0 ){
-    sqlite3_free(zFilename);
+    sqlite3_free(zText);
-    zFilename = 0;
+    zText = 0;
   }
-  return zFilename;
+  return zText;
 }
 
 /*
-** Convert an ANSI string to Microsoft Unicode, based on the
+** Convert an ANSI string to Microsoft Unicode, using the ANSI or OEM
-** current codepage settings for file apis.
+** code page.
 **
-** Space to hold the returned string is obtained
+** Space to hold the returned string is obtained from sqlite3_malloc().
-** from sqlite3_malloc.
 */
-static LPWSTR winMbcsToUnicode(const char *zFilename){
+static LPWSTR winMbcsToUnicode(const char *zText, int useAnsi){
   int nByte;
-  LPWSTR zMbcsFilename;
+  LPWSTR zMbcsText;
-  int codepage = osAreFileApisANSI() ? CP_ACP : CP_OEMCP;
+  int codepage = useAnsi ? CP_ACP : CP_OEMCP;
 
-  nByte = osMultiByteToWideChar(codepage, 0, zFilename, -1, NULL,
+  nByte = osMultiByteToWideChar(codepage, 0, zText, -1, NULL,
                                 0)*sizeof(WCHAR);
   if( nByte==0 ){
     return 0;
   }
-  zMbcsFilename = sqlite3MallocZero( nByte*sizeof(zMbcsFilename[0]) );
+  zMbcsText = sqlite3MallocZero( nByte*sizeof(WCHAR) );
-  if( zMbcsFilename==0 ){
+  if( zMbcsText==0 ){
     return 0;
   }
-  nByte = osMultiByteToWideChar(codepage, 0, zFilename, -1, zMbcsFilename,
+  nByte = osMultiByteToWideChar(codepage, 0, zText, -1, zMbcsText,
                                 nByte);
   if( nByte==0 ){
-    sqlite3_free(zMbcsFilename);
+    sqlite3_free(zMbcsText);
-    zMbcsFilename = 0;
+    zMbcsText = 0;
   }
-  return zMbcsFilename;
+  return zMbcsText;
 }
 
 /*
-** Convert Microsoft Unicode to multi-byte character string, based on the
+** Convert a Microsoft Unicode string to a multi-byte character string,
-** user's ANSI codepage.
+** using the ANSI or OEM code page.
 **
-** Space to hold the returned string is obtained from
+** Space to hold the returned string is obtained from sqlite3_malloc().
-** sqlite3_malloc().
 */
-static char *winUnicodeToMbcs(LPCWSTR zWideFilename){
+static char *winUnicodeToMbcs(LPCWSTR zWideText, int useAnsi){
   int nByte;
-  char *zFilename;
+  char *zText;
-  int codepage = osAreFileApisANSI() ? CP_ACP : CP_OEMCP;
+  int codepage = useAnsi ? CP_ACP : CP_OEMCP;
 
-  nByte = osWideCharToMultiByte(codepage, 0, zWideFilename, -1, 0, 0, 0, 0);
+  nByte = osWideCharToMultiByte(codepage, 0, zWideText, -1, 0, 0, 0, 0);
   if( nByte == 0 ){
     return 0;
   }
-  zFilename = sqlite3MallocZero( nByte );
+  zText = sqlite3MallocZero( nByte );
-  if( zFilename==0 ){
+  if( zText==0 ){
     return 0;
   }
-  nByte = osWideCharToMultiByte(codepage, 0, zWideFilename, -1, zFilename,
+  nByte = osWideCharToMultiByte(codepage, 0, zWideText, -1, zText,
                                 nByte, 0, 0);
   if( nByte == 0 ){
-    sqlite3_free(zFilename);
+    sqlite3_free(zText);
-    zFilename = 0;
+    zText = 0;
   }
-  return zFilename;
+  return zText;
 }
 
 /*
-** Convert multibyte character string to UTF-8.  Space to hold the
+** Convert a multi-byte character string to UTF-8.
-** returned string is obtained from sqlite3_malloc().
+**
+** Space to hold the returned string is obtained from sqlite3_malloc().
 */
-char *sqlite3_win32_mbcs_to_utf8(const char *zFilename){
+static char *winMbcsToUtf8(const char *zText, int useAnsi){
-  char *zFilenameUtf8;
+  char *zTextUtf8;
   LPWSTR zTmpWide;
 
-  zTmpWide = winMbcsToUnicode(zFilename);
+  zTmpWide = winMbcsToUnicode(zText, useAnsi);
   if( zTmpWide==0 ){
     return 0;
   }
-  zFilenameUtf8 = winUnicodeToUtf8(zTmpWide);
+  zTextUtf8 = winUnicodeToUtf8(zTmpWide);
   sqlite3_free(zTmpWide);
-  return zFilenameUtf8;
+  return zTextUtf8;
 }
 
 /*
-** Convert UTF-8 to multibyte character string.  Space to hold the
+** Convert a UTF-8 string to a multi-byte character string.
-** returned string is obtained from sqlite3_malloc().
+**
+** Space to hold the returned string is obtained from sqlite3_malloc().
 */
-char *sqlite3_win32_utf8_to_mbcs(const char *zFilename){
+static char *winUtf8ToMbcs(const char *zText, int useAnsi){
-  char *zFilenameMbcs;
+  char *zTextMbcs;
   LPWSTR zTmpWide;
 
-  zTmpWide = winUtf8ToUnicode(zFilename);
+  zTmpWide = winUtf8ToUnicode(zText);
   if( zTmpWide==0 ){
     return 0;
   }
-  zFilenameMbcs = winUnicodeToMbcs(zTmpWide);
+  zTextMbcs = winUnicodeToMbcs(zTmpWide, useAnsi);
   sqlite3_free(zTmpWide);
-  return zFilenameMbcs;
+  return zTextMbcs;
+}
+
+/*
+** This is a public wrapper for the winUtf8ToUnicode() function.
+*/
+LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText){
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( !zText ){
+    (void)SQLITE_MISUSE_BKPT;
+    return 0;
+  }
+#endif
+#ifndef SQLITE_OMIT_AUTOINIT
+  if( sqlite3_initialize() ) return 0;
+#endif
+  return winUtf8ToUnicode(zText);
+}
+
+/*
+** This is a public wrapper for the winUnicodeToUtf8() function.
+*/
+char *sqlite3_win32_unicode_to_utf8(LPCWSTR zWideText){
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( !zWideText ){
+    (void)SQLITE_MISUSE_BKPT;
+    return 0;
+  }
+#endif
+#ifndef SQLITE_OMIT_AUTOINIT
+  if( sqlite3_initialize() ) return 0;
+#endif
+  return winUnicodeToUtf8(zWideText);
+}
+
+/*
+** This is a public wrapper for the winMbcsToUtf8() function.
+*/
+char *sqlite3_win32_mbcs_to_utf8(const char *zText){
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( !zText ){
+    (void)SQLITE_MISUSE_BKPT;
+    return 0;
+  }
+#endif
+#ifndef SQLITE_OMIT_AUTOINIT
+  if( sqlite3_initialize() ) return 0;
+#endif
+  return winMbcsToUtf8(zText, osAreFileApisANSI());
+}
+
+/*
+** This is a public wrapper for the winMbcsToUtf8() function.
+*/
+char *sqlite3_win32_mbcs_to_utf8_v2(const char *zText, int useAnsi){
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( !zText ){
+    (void)SQLITE_MISUSE_BKPT;
+    return 0;
+  }
+#endif
+#ifndef SQLITE_OMIT_AUTOINIT
+  if( sqlite3_initialize() ) return 0;
+#endif
+  return winMbcsToUtf8(zText, useAnsi);
+}
+
+/*
+** This is a public wrapper for the winUtf8ToMbcs() function.
+*/
+char *sqlite3_win32_utf8_to_mbcs(const char *zText){
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( !zText ){
+    (void)SQLITE_MISUSE_BKPT;
+    return 0;
+  }
+#endif
+#ifndef SQLITE_OMIT_AUTOINIT
+  if( sqlite3_initialize() ) return 0;
+#endif
+  return winUtf8ToMbcs(zText, osAreFileApisANSI());
+}
+
+/*
+** This is a public wrapper for the winUtf8ToMbcs() function.
+*/
+char *sqlite3_win32_utf8_to_mbcs_v2(const char *zText, int useAnsi){
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( !zText ){
+    (void)SQLITE_MISUSE_BKPT;
+    return 0;
+  }
+#endif
+#ifndef SQLITE_OMIT_AUTOINIT
+  if( sqlite3_initialize() ) return 0;
+#endif
+  return winUtf8ToMbcs(zText, useAnsi);
 }
 
 /*
@@ -1873,7 +1976,7 @@ static int winGetLastErrorMsg(DWORD lastErrno, int nBuf, char *zBuf){
     if( dwLen > 0 ){
       /* allocate a buffer and convert to UTF8 */
       sqlite3BeginBenignMalloc();
-      zOut = sqlite3_win32_mbcs_to_utf8(zTemp);
+      zOut = winMbcsToUtf8(zTemp, osAreFileApisANSI());
       sqlite3EndBenignMalloc();
       /* free the system buffer allocated by FormatMessage */
       osLocalFree(zTemp);
@@ -4318,7 +4421,7 @@ static char *winConvertToUtf8Filename(const void *zFilename){
   }
 #ifdef SQLITE_WIN32_HAS_ANSI
   else{
-    zConverted = sqlite3_win32_mbcs_to_utf8(zFilename);
+    zConverted = winMbcsToUtf8(zFilename, osAreFileApisANSI());
   }
 #endif
   /* caller will handle out of memory */
@@ -4339,7 +4442,7 @@ static void *winConvertFromUtf8Filename(const char *zFilename){
   }
 #ifdef SQLITE_WIN32_HAS_ANSI
   else{
-    zConverted = sqlite3_win32_utf8_to_mbcs(zFilename);
+    zConverted = winUtf8ToMbcs(zFilename, osAreFileApisANSI());
   }
 #endif
   /* caller will handle out of memory */
@@ -4540,7 +4643,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){
       return winLogError(SQLITE_IOERR_GETTEMPPATH, osGetLastError(),
                          "winGetTempname3", 0);
     }
-    zUtf8 = sqlite3_win32_mbcs_to_utf8(zMbcsPath);
+    zUtf8 = winMbcsToUtf8(zMbcsPath, osAreFileApisANSI());
     if( zUtf8 ){
       sqlite3_snprintf(nMax, zBuf, "%s", zUtf8);
       sqlite3_free(zUtf8);
@@ -5318,7 +5421,7 @@ static int winFullPathname(
                          "winFullPathname4", zRelative);
     }
     sqlite3_free(zConverted);
-    zOut = sqlite3_win32_mbcs_to_utf8(zTemp);
+    zOut = winMbcsToUtf8(zTemp, osAreFileApisANSI());
     sqlite3_free(zTemp);
   }
 #endif

src/shell.c

@@ -136,6 +136,15 @@
 #define IsDigit(X)  isdigit((unsigned char)X)
 #define ToLower(X)  (char)tolower((unsigned char)X)
 
+#if defined(_WIN32) || defined(WIN32)
+#include <windows.h>
+
+/* string conversion routines only needed on Win32 */
+extern char *sqlite3_win32_unicode_to_utf8(LPCWSTR);
+extern char *sqlite3_win32_mbcs_to_utf8_v2(const char *, int);
+extern char *sqlite3_win32_utf8_to_mbcs_v2(const char *, int);
+#endif
+
 /* On Windows, we normally run with output mode of TEXT so that \n characters
 ** are automatically translated into \r\n.  However, this behavior needs
 ** to be disabled in some cases (ex: when generating CSV output and when
@@ -143,17 +152,17 @@
 ** routines take care of that.
 */
 #if defined(_WIN32) || defined(WIN32)
-static void setBinaryMode(FILE *out){
+static void setBinaryMode(FILE *file, int isOutput){
-  fflush(out);
+  if( isOutput ) fflush(file);
-  _setmode(_fileno(out), _O_BINARY);
+  _setmode(_fileno(file), _O_BINARY);
 }
-static void setTextMode(FILE *out){
+static void setTextMode(FILE *file, int isOutput){
-  fflush(out);
+  if( isOutput ) fflush(file);
-  _setmode(_fileno(out), _O_TEXT);
+  _setmode(_fileno(file), _O_TEXT);
 }
 #else
-# define setBinaryMode(X)
+# define setBinaryMode(X,Y)
-# define setTextMode(X)
+# define setTextMode(X,Y)
 #endif
 
 
@@ -229,8 +238,6 @@ static void endTimer(void){
 
 #elif (defined(_WIN32) || defined(WIN32))
 
-#include <windows.h>
-
 /* Saved resource information for the beginning of an operation */
 static HANDLE hProcess;
 static FILETIME ftKernelBegin;
@@ -361,6 +368,38 @@ static char *Argv0;
 static char mainPrompt[20];     /* First line prompt. default: "sqlite> "*/
 static char continuePrompt[20]; /* Continuation prompt. default: "   ...> " */
 
+/*
+** Render output like fprintf().  Except, if the output is going to the
+** console and if this is running on a Windows machine, translate the
+** output from UTF-8 into MBCS.
+*/
+#if defined(_WIN32) || defined(WIN32)
+void utf8_printf(FILE *out, const char *zFormat, ...){
+  va_list ap;
+  va_start(ap, zFormat);
+  if( stdout_is_console && (out==stdout || out==stderr) ){
+    char *z1 = sqlite3_vmprintf(zFormat, ap);
+    char *z2 = sqlite3_win32_utf8_to_mbcs_v2(z1, 0);
+    sqlite3_free(z1);
+    fputs(z2, out);
+    sqlite3_free(z2);
+  }else{
+    vfprintf(out, zFormat, ap);
+  }
+  va_end(ap);
+}
+#elif !defined(utf8_printf)
+# define utf8_printf fprintf
+#endif
+
+/*
+** Render output like fprintf().  This should not be used on anything that
+** includes string formatting (e.g. "%s").
+*/
+#if !defined(raw_printf)
+# define raw_printf fprintf
+#endif
+
 /*
 ** Write I/O traces to the following stream.
 */
@@ -382,7 +421,7 @@ static void SQLITE_CDECL iotracePrintf(const char *zFormat, ...){
   va_start(ap, zFormat);
   z = sqlite3_vmprintf(zFormat, ap);
   va_end(ap);
-  fprintf(iotrace, "%s", z);
+  utf8_printf(iotrace, "%s", z);
   sqlite3_free(z);
 }
 #endif
@@ -486,8 +525,7 @@ static char *local_getline(char *zLine, FILE *in){
   /* For interactive input on Windows systems, translate the
   ** multi-byte characterset characters into UTF-8. */
   if( stdin_is_interactive ){
-    extern char *sqlite3_win32_mbcs_to_utf8(const char*);
+    char *zTrans = sqlite3_win32_mbcs_to_utf8_v2(zLine, 0);
-    char *zTrans = sqlite3_win32_mbcs_to_utf8(zLine);
     if( zTrans ){
       int nTrans = strlen30(zTrans)+1;
       if( nTrans>nLine ){
@@ -552,39 +590,6 @@ struct OpenSession {
 };
 #endif
 
-/*
-** Render output like fprintf().  Except, if the output is going to the
-** console and if this is running on a Windows machine, translate the
-** output from UTF-8 into MBCS.
-*/
-#if defined(_WIN32) || defined(WIN32)
-void utf8_printf(FILE *out, const char *zFormat, ...){
-  va_list ap;
-  va_start(ap, zFormat);
-  if( stdout_is_console && (out==stdout || out==stderr) ){
-    extern char *sqlite3_win32_utf8_to_mbcs(const char*);
-    char *z1 = sqlite3_vmprintf(zFormat, ap);
-    char *z2 = sqlite3_win32_utf8_to_mbcs(z1);
-    sqlite3_free(z1);
-    fputs(z2, out);
-    sqlite3_free(z2);
-  }else{
-    vfprintf(out, zFormat, ap);
-  }
-  va_end(ap);
-}
-#elif !defined(utf8_printf)
-# define utf8_printf fprintf
-#endif
-
-/*
-** Render output like fprintf().  This should not be used on anything that
-** includes string formatting (e.g. "%s").
-*/
-#if !defined(raw_printf)
-# define raw_printf fprintf
-#endif
-
 /*
 ** Shell output mode information from before ".explain on",
 ** saved so that it can be restored by ".explain off"
@@ -723,7 +728,7 @@ static void output_hex_blob(FILE *out, const void *pBlob, int nBlob){
 static void output_quoted_string(FILE *out, const char *z){
   int i;
   int nSingle = 0;
-  setBinaryMode(out);
+  setBinaryMode(out, 1);
   for(i=0; z[i]; i++){
     if( z[i]=='\'' ) nSingle++;
   }
@@ -746,7 +751,7 @@ static void output_quoted_string(FILE *out, const char *z){
     }
     raw_printf(out,"'");
   }
-  setTextMode(out);
+  setTextMode(out, 1);
 }
 
 /*
@@ -1109,7 +1114,7 @@ static int shell_callback(
       break;
     }
     case MODE_Csv: {
-      setBinaryMode(p->out);
+      setBinaryMode(p->out, 1);
       if( p->cnt++==0 && p->showHeader ){
         for(i=0; i<nArg; i++){
           output_csv(p, azCol[i] ? azCol[i] : "", i<nArg-1);
@@ -1122,7 +1127,7 @@ static int shell_callback(
         }
         utf8_printf(p->out, "%s", p->rowSeparator);
       }
-      setTextMode(p->out);
+      setTextMode(p->out, 1);
       break;
     }
     case MODE_Insert: {
@@ -2085,7 +2090,7 @@ static char zHelp[] =
 ** Print help information for the ".sessions" command
 */
 void session_help(ShellState *p){
-  fprintf(p->out,
+  raw_printf(p->out,
     ".session ?NAME? SUBCOMMAND ?ARGS...?\n"
     "If ?NAME? is omitted, the first defined session is used.\n"
     "Subcommands:\n"
@@ -3053,9 +3058,9 @@ static int do_meta_command(char *zLine, ShellState *p){
   if( c=='b' && n>=3 && strncmp(azArg[0], "binary", n)==0 ){
     if( nArg==2 ){
       if( booleanValue(azArg[1]) ){
-        setBinaryMode(p->out);
+        setBinaryMode(p->out, 1);
       }else{
-        setTextMode(p->out);
+        setTextMode(p->out, 1);
       }
     }else{
       raw_printf(stderr, "Usage: .binary on|off\n");
@@ -3966,11 +3971,11 @@ static int do_meta_command(char *zLine, ShellState *p){
       if( nCmd!=2 ) goto session_syntax_error;
       if( pSession->p==0 ){
         session_not_open:
-        fprintf(stderr, "ERROR: No sessions are open\n");
+        raw_printf(stderr, "ERROR: No sessions are open\n");
       }else{
         rc = sqlite3session_attach(pSession->p, azCmd[1]);
         if( rc ){
-          fprintf(stderr, "ERROR: sqlite3session_attach() returns %d\n", rc);
+          raw_printf(stderr, "ERROR: sqlite3session_attach() returns %d\n", rc);
           rc = 0;
         }
       }
@@ -3986,7 +3991,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       if( pSession->p==0 ) goto session_not_open;
       out = fopen(azCmd[1], "wb");
       if( out==0 ){
-        fprintf(stderr, "ERROR: cannot open \"%s\" for writing\n", azCmd[1]);
+        utf8_printf(stderr, "ERROR: cannot open \"%s\" for writing\n", azCmd[1]);
       }else{
         int szChng;
         void *pChng;
@@ -4001,7 +4006,7 @@ static int do_meta_command(char *zLine, ShellState *p){
         }
         if( pChng
           && fwrite(pChng, szChng, 1, out)!=1 ){
-          fprintf(stderr, "ERROR: Failed to write entire %d-byte output\n",
+          raw_printf(stderr, "ERROR: Failed to write entire %d-byte output\n",
                   szChng);
         }
         sqlite3_free(pChng);
@@ -4029,7 +4034,8 @@ static int do_meta_command(char *zLine, ShellState *p){
       ii = nCmd==1 ? -1 : booleanValue(azCmd[1]);
       if( p->nSession ){
         ii = sqlite3session_enable(pSession->p, ii);
-        fprintf(p->out, "session %s enable flag = %d\n", pSession->zName, ii);
+        utf8_printf(p->out, "session %s enable flag = %d\n",
+                    pSession->zName, ii);
       }
     }else
 
@@ -4047,7 +4053,7 @@ static int do_meta_command(char *zLine, ShellState *p){
         nByte = sizeof(pSession->azFilter[0])*(nCmd-1);
         pSession->azFilter = sqlite3_malloc( nByte );
         if( pSession->azFilter==0 ){
-          fprintf(stderr, "Error: out or memory\n");
+          raw_printf(stderr, "Error: out or memory\n");
           exit(1);
         }
         for(ii=1; ii<nCmd; ii++){
@@ -4066,7 +4072,8 @@ static int do_meta_command(char *zLine, ShellState *p){
       ii = nCmd==1 ? -1 : booleanValue(azCmd[1]);
       if( p->nSession ){
         ii = sqlite3session_indirect(pSession->p, ii);
-        fprintf(p->out, "session %s indirect flag = %d\n", pSession->zName,ii);
+        utf8_printf(p->out, "session %s indirect flag = %d\n",
+                    pSession->zName, ii);
       }
     }else
 
@@ -4078,7 +4085,8 @@ static int do_meta_command(char *zLine, ShellState *p){
       if( nCmd!=1 ) goto session_syntax_error;
       if( p->nSession ){
         ii = sqlite3session_isempty(pSession->p);
-        fprintf(p->out, "session %s isempty flag = %d\n", pSession->zName, ii);
+        utf8_printf(p->out, "session %s isempty flag = %d\n",
+                    pSession->zName, ii);
       }
     }else
 
@@ -4087,7 +4095,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     */
     if( strcmp(azCmd[0],"list")==0 ){
       for(i=0; i<p->nSession; i++){
-        fprintf(p->out, "%d %s\n", i, p->aSession[i].zName);
+        utf8_printf(p->out, "%d %s\n", i, p->aSession[i].zName);
       }
     }else
 
@@ -4102,18 +4110,18 @@ static int do_meta_command(char *zLine, ShellState *p){
       if( zName[0]==0 ) goto session_syntax_error;
       for(i=0; i<p->nSession; i++){
         if( strcmp(p->aSession[i].zName,zName)==0 ){
-          fprintf(stderr, "Session \"%s\" already exists\n", zName);
+          utf8_printf(stderr, "Session \"%s\" already exists\n", zName);
           goto meta_command_exit;
         }
       }
       if( p->nSession>=ArraySize(p->aSession) ){
-        fprintf(stderr, "Maximum of %d sessions\n", ArraySize(p->aSession));
+        raw_printf(stderr, "Maximum of %d sessions\n", ArraySize(p->aSession));
         goto meta_command_exit;
       }
       pSession = &p->aSession[p->nSession];
       rc = sqlite3session_create(p->db, azCmd[1], &pSession->p);
       if( rc ){
-        fprintf(stderr, "Cannot open session: error code=%d\n", rc);
+        raw_printf(stderr, "Cannot open session: error code=%d\n", rc);
         rc = 0;
         goto meta_command_exit;
       }
@@ -5043,7 +5051,20 @@ static char *cmdline_option_value(int argc, char **argv, int i){
   return argv[i];
 }
 
+#ifndef SQLITE_SHELL_IS_UTF8
+#  if (defined(_WIN32) || defined(WIN32)) && defined(_MSC_VER)
+#    define SQLITE_SHELL_IS_UTF8          (0)
+#  else
+#    define SQLITE_SHELL_IS_UTF8          (1)
+#  endif
+#endif
+
+#if SQLITE_SHELL_IS_UTF8
 int SQLITE_CDECL main(int argc, char **argv){
+#else
+int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
+  char **argv;
+#endif
   char *zErrMsg = 0;
   ShellState data;
   const char *zInitFile = 0;
@@ -5054,6 +5075,11 @@ int SQLITE_CDECL main(int argc, char **argv){
   int nCmd = 0;
   char **azCmd = 0;
 
+  setBinaryMode(stdin, 0);
+  setvbuf(stderr, 0, _IONBF, 0); /* Make sure stderr is unbuffered */
+  stdin_is_interactive = isatty(0);
+  stdout_is_console = isatty(1);
+
 #if USE_SYSTEM_SQLITE+0!=1
   if( strcmp(sqlite3_sourceid(),SQLITE_SOURCE_ID)!=0 ){
     utf8_printf(stderr, "SQLite header and source version mismatch\n%s\n%s\n",
@@ -5061,12 +5087,24 @@ int SQLITE_CDECL main(int argc, char **argv){
     exit(1);
   }
 #endif
-  setBinaryMode(stdin);
-  setvbuf(stderr, 0, _IONBF, 0); /* Make sure stderr is unbuffered */
-  Argv0 = argv[0];
   main_init(&data);
-  stdin_is_interactive = isatty(0);
+#if !SQLITE_SHELL_IS_UTF8
-  stdout_is_console = isatty(1);
+  sqlite3_initialize();
+  argv = sqlite3_malloc64(sizeof(argv[0])*argc);
+  if( argv==0 ){
+    raw_printf(stderr, "out of memory\n");
+    exit(1);
+  }
+  for(i=0; i<argc; i++){
+    argv[i] = sqlite3_win32_unicode_to_utf8(wargv[i]);
+    if( argv[i]==0 ){
+      raw_printf(stderr, "out of memory\n");
+      exit(1);
+    }
+  }
+#endif
+  assert( argc>=1 && argv && argv[0] );
+  Argv0 = argv[0];
 
   /* Make sure we have a valid signal handler early, before anything
   ** else is done.
@@ -5398,5 +5436,9 @@ int SQLITE_CDECL main(int argc, char **argv){
     sqlite3_close(data.db);
   }
   sqlite3_free(data.zFreeOnClose);
+#if !SQLITE_SHELL_IS_UTF8
+  for(i=0; i<argc; i++) sqlite3_free(argv[i]);
+  sqlite3_free(argv);
+#endif
   return rc;
 }

test/shell1.test

@@ -910,4 +910,41 @@ do_test shell1-5.0 {
   }
 } {}
 
+# The string used here is the word "test" in Chinese.
+# In UTF-8, it is encoded as: \xE6\xB5\x8B\xE8\xAF\x95
+set test \u6D4B\u8BD5
+
+do_test shell1-6.0 {
+  set fileName $test; append fileName .db
+  catch {forcedelete $fileName}
+  set x [catchcmdex $fileName "CREATE TABLE t1(x);\n.schema\n"]
+  set code [lindex $x 0]
+  set res [string trim [lindex $x 1]]
+  if {$code ne "0"} {
+    error "failed with error: $res"
+  }
+  if {$res ne "CREATE TABLE t1(x);"} {
+    error "failed with mismatch: $res"
+  }
+  if {![file exists $fileName]} {
+    error "file \"$fileName\" (Unicode) does not exist"
+  }
+  forcedelete $fileName
+} {}
+
+do_test shell1-6.1 {
+  catch {forcedelete test3.db}
+  set x [catchcmdex test3.db \
+      "CREATE TABLE [encoding convertto utf-8 $test](x);\n.schema\n"]
+  set code [lindex $x 0]
+  set res [string trim [lindex $x 1]]
+  if {$code ne "0"} {
+    error "failed with error: $res"
+  }
+  if {$res ne "CREATE TABLE ${test}(x);"} {
+    error "failed with mismatch: $res"
+  }
+  forcedelete test3.db
+} {}
+
 finish_test