MDEV-26713 Windows - improve utf8 support for command line tools

client/mysql.cc

@@ -88,9 +88,7 @@ extern "C" {
 #endif /* defined(HAVE_CURSES_H) && defined(HAVE_TERM_H) */
 
 #undef bcmp				// Fix problem with new readline
-#if defined(_WIN32)
+#if !defined(_WIN32)
-#include <conio.h>
-#else
 # ifdef __APPLE__
 #  include <editline/readline.h>
 # else
@@ -104,6 +102,101 @@ extern "C" {
 #endif
 }
 
+
+#if defined(_WIN32)
+/*
+  Set console mode for the whole duration of the client session.
+
+  We need for input
+    - line input (i.e read lines from console)
+    - echo typed characters
+    - "cooked" mode, i.e we do not want to handle all keystrokes,
+      like DEL etc ourselves, yet. We might want handle keystrokes
+      in the future, to implement tab completion, and better
+      (multiline) history.
+
+ Disable VT escapes for the output.We do not know what kind of escapes SELECT would return.
+*/
+struct Console_mode
+{
+  HANDLE in= GetStdHandle(STD_INPUT_HANDLE);
+  HANDLE out= GetStdHandle(STD_OUTPUT_HANDLE);
+  DWORD mode_in=0;
+  DWORD mode_out=0;
+
+  enum {STDIN_CHANGED = 1, STDOUT_CHANGED = 2};
+  int changes=0;
+
+  Console_mode()
+  {
+    if (in && in != INVALID_HANDLE_VALUE && GetConsoleMode(in, &mode_in))
+    {
+      SetConsoleMode(in, ENABLE_ECHO_INPUT|ENABLE_LINE_INPUT|ENABLE_PROCESSED_INPUT);
+      changes |= STDIN_CHANGED;
+    }
+
+    if (out && out != INVALID_HANDLE_VALUE && GetConsoleMode(out, &mode_out))
+    {
+#ifdef ENABLE_VIRTUAL_TERMINAL_INPUT
+      SetConsoleMode(out, mode_out & ~ENABLE_VIRTUAL_TERMINAL_INPUT);
+      changes |= STDOUT_CHANGED;
+#endif
+    }
+  }
+
+  ~Console_mode()
+  {
+    if (changes & STDIN_CHANGED)
+      SetConsoleMode(in, mode_in);
+
+    if(changes & STDOUT_CHANGED)
+      SetConsoleMode(out, mode_out);
+  }
+};
+
+static Console_mode my_conmode;
+
+#define MAX_CGETS_LINE_LEN 65535
+/** Read line from console in ANSI codepage, chomp EOL*/
+static char *win_readline()
+{
+  static wchar_t wstrbuf[MAX_CGETS_LINE_LEN];
+  static char strbuf[MAX_CGETS_LINE_LEN*2];
+
+  DWORD nchars= 0;
+  SetLastError(0);
+  if (!ReadConsoleW(GetStdHandle(STD_INPUT_HANDLE), wstrbuf, MAX_CGETS_LINE_LEN-1,
+                    &nchars, NULL))
+    goto err;
+  if (nchars == 0 && GetLastError() == ERROR_OPERATION_ABORTED)
+    goto err;
+
+  while (nchars > 0)
+  {
+    if (wstrbuf[nchars - 1] != '\n' && wstrbuf[nchars - 1] != '\r')
+      break;
+    wstrbuf[--nchars]= 0;
+  }
+
+  wstrbuf[nchars]= 0;
+  if (nchars > 0)
+  {
+    int len = WideCharToMultiByte(GetConsoleCP(), 0, wstrbuf, nchars + 1,
+                               strbuf, sizeof(strbuf),NULL, NULL);
+    if (len < 1)
+      strbuf[0]= 0;
+  }
+  else
+  {
+    strbuf[0]= 0;
+  }
+  return strbuf;
+err:
+  return NULL;
+}
+#endif
+
+
 #ifdef HAVE_VIDATTR
 static int have_curses= 0;
 static void my_vidattr(chtype attrs)
@@ -1719,8 +1812,10 @@ static void usage(int version)
 	 my_progname, VER, MYSQL_SERVER_VERSION, SYSTEM_TYPE, MACHINE_TYPE,
          readline, rl_library_version);
 #else
-  printf("%s  Ver %s Distrib %s, for %s (%s), source revision %s\n", my_progname, VER,
+  printf("%s  Ver %s Distrib %s, for %s (%s), source revision %s"
-	MYSQL_SERVER_VERSION, SYSTEM_TYPE, MACHINE_TYPE,SOURCE_REVISION);
+         IF_WIN(", codepage %u",) "\n", my_progname, VER,
+         MYSQL_SERVER_VERSION, SYSTEM_TYPE, MACHINE_TYPE,SOURCE_REVISION,
+         IF_WIN(GetConsoleCP(),));
 #endif
 
   if (version)
@@ -2115,26 +2210,7 @@ static int read_and_execute(bool interactive)
 
 #if defined(_WIN32)
       tee_fputs(prompt, stdout);
-      if (!tmpbuf.is_alloced())
+      line= win_readline();
-        tmpbuf.alloc(65535);
-      tmpbuf.length(0);
-      buffer.length(0);
-      size_t clen;
-      do
-      {
-	line= my_cgets((char*)tmpbuf.ptr(), tmpbuf.alloced_length()-1, &clen);
-        buffer.append(line, clen);
-        /* 
-           if we got buffer fully filled than there is a chance that
-           something else is still in console input buffer
-        */
-      } while (tmpbuf.alloced_length() <= clen);
-      /* 
-        An empty line is returned from my_cgets when there's error reading :
-        Ctrl-c for example
-      */
-      if (line)
-        line= buffer.c_ptr();
 #else
       if (opt_outfile)
 	fputs(prompt, OUTFILE);

cmake/win_compatibility.manifest

@@ -19,4 +19,9 @@
 
     </application>
   </compatibility>
+  <application>
+    <windowsSettings>
+      <activeCodePage xmlns="http://schemas.microsoft.com/SMI/2019/WindowsSettings">UTF-8</activeCodePage>
+    </windowsSettings>
+  </application>
 </asmv1:assembly>

mysql-test/main/charset_client_win.test

@@ -1,2 +1,2 @@
 --source include/windows.inc
---exec chcp 1257 > NUL && $MYSQL --default-character-set=auto -e "select @@character_set_client"
+--exec set MARIADB_DISABLE_UTF8_CONSOLE=1 && chcp 1257 > NUL && $MYSQL --default-character-set=auto -e "select @@character_set_client"

mysql-test/main/charset_client_win_utf8mb4.result

@@ -0,0 +1,2 @@
+@@character_set_client
+utf8mb4

mysql-test/main/charset_client_win_utf8mb4.test

@@ -0,0 +1,2 @@
+--source include/windows.inc
+--exec $MYSQL --default-character-set=auto -e "select @@character_set_client"

mysql-test/main/grant_utf8_cli.test

@@ -1,5 +1,3 @@
- # UTF8 parameters to mysql client do not work on Windows
---source include/not_windows.inc
 --source include/not_embedded.inc
 
 #

mysql-test/suite.pm

@@ -87,6 +87,23 @@ sub skip_combinations {
   $skip{'main/ssl_verify_ip.test'} = 'x509v3 support required'
     unless $openssl_ver ge "1.0.2";
 
+  sub utf8_command_line_ok() {
+   if (IS_WINDOWS) {
+     # Can use UTF8 on command line since Windows 10 1903 (10.0.18362)
+     my($os_name, $os_major, $os_minor, $os_build, $os_id) = Win32::GetOSVersion();
+     if($os_major lt 10){
+       return 0;
+     } elsif($os_major gt 10 or $os_minor gt 0 or $os_build ge 18362) {
+       return 1;
+     }
+     return 0;
+   }
+   return 1;
+  }
+
+  $skip{'main/charset_client_win_utf8mb4.test'} =
+  $skip{'main/grant_utf8_cli.test'} = 'No utf8 command line support'
+    unless utf8_command_line_ok();
 
   %skip;
 }

mysys/charset.c

@@ -1393,7 +1393,7 @@ static const MY_CSET_OS_NAME charsets[] =
 #ifdef UNCOMMENT_THIS_WHEN_WL_WL_4024_IS_DONE
   {"cp54936",        "gb18030",  my_cs_exact},
 #endif
-  {"cp65001",        "utf8",     my_cs_exact},
+  {"cp65001",        "utf8mb4",  my_cs_exact},
 
 #else /* not Windows */
 

mysys/get_password.c

@@ -62,34 +62,40 @@
 
 char *get_tty_password(const char *opt_message)
 {
-  char to[80];
+  wchar_t wbuf[80];
-  char *pos=to,*end=to+sizeof(to)-1;
+  char to[80*3]; /* there is at most 3 bytes per wchar, in any codepage */
+  wchar_t *pos=wbuf,*end=wbuf + array_elements(wbuf)-1;
   DBUG_ENTER("get_tty_password");
   _cputs(opt_message ? opt_message : "Enter password: ");
   for (;;)
   {
-    char tmp;
+    int wc;
-    tmp=_getch();
+    wc=_getwch();
-    if (tmp == '\b' || (int) tmp == 127)
+    if (wc == '\b' || wc == 127)
     {
-      if (pos != to)
+      if (pos != wbuf)
       {
         _cputs("\b \b");
         pos--;
         continue;
       }
     }
-    if (tmp == '\n' || tmp == '\r' || tmp == 3)
+    if (wc == '\n' || wc == '\r' || wc == 3 || pos == end)
       break;
-    if (iscntrl(tmp) || pos == end)
+    if (iswcntrl(wc))
       continue;
+
+    /* Do not print '*' for half-unicode char(high surrogate)*/
+    if (wc < 0xD800 || wc > 0xDBFF)
+    {
       _cputs("*");
-    *(pos++) = tmp;
     }
-  while (pos != to && isspace(pos[-1]) == ' ')
+    *(pos++)= (wchar_t)wc;
-    pos--;					/* Allow dummy space at end */
+  }
   *pos=0;
   _cputs("\n");
+  if (!WideCharToMultiByte(GetConsoleCP(), 0 , wbuf , -1 , to, sizeof(to), NULL, NULL))
+    to[0]=0;
   DBUG_RETURN(my_strdup(PSI_INSTRUMENT_ME, to,MYF(MY_FAE)));
 }
 

mysys/my_init.c

@@ -67,6 +67,38 @@ static ulong atoi_octal(const char *str)
 MYSQL_FILE *mysql_stdin= NULL;
 static MYSQL_FILE instrumented_stdin;
 
+#ifdef _WIN32
+UINT orig_console_cp, orig_console_output_cp;
+
+static void reset_console_cp(void)
+{
+  SetConsoleCP(orig_console_cp);
+  SetConsoleOutputCP(orig_console_output_cp);
+}
+
+/*
+    The below fixes discrepancies in console output and
+    command line parameter encoding. command line is in
+    ANSI codepage, output to console by default is in OEM, but
+    we like them to be in the same encoding.
+
+    We do this only if current codepage is UTF8, i.e when we
+    know we're on Windows that can handle UTF8 well.
+  */
+static void set_console_cp()
+{
+  UINT acp= GetACP();
+  if (acp == CP_UTF8 && !getenv("MARIADB_DISABLE_UTF8_CONSOLE"))
+  {
+    orig_console_cp= GetConsoleCP();
+    SetConsoleCP(acp);
+    orig_console_output_cp= GetConsoleOutputCP();
+    SetConsoleOutputCP(acp);
+    atexit(reset_console_cp);
+  }
+}
+#endif
+
 /**
   Initialize my_sys functions, resources and variables
 
@@ -131,6 +163,7 @@ my_bool my_init(void)
 #ifdef _WIN32
     if (win32_init_tcp_ip())
       DBUG_RETURN(1);
+    set_console_cp();
 #endif
 #ifdef CHECK_UNLIKELY
     init_my_likely();

win/packaging/extra.wxs.in

@@ -461,7 +461,7 @@
             Section="mysqld"
             Name="my.ini"
             Key="character-set-server"
-            Value="utf8" />
+            Value="utf8mb4" />
       </Component>
 
       <!-- Shortcuts in program menu (mysql client etc) -->