MDEV-17088 Provide tools to encode/decode mysql-encoded file system names

The original patch was made by Takashi Sasaki <tsasaki609@gmail.com>.

.gitignore

@@ -39,6 +39,7 @@ client/mysqlimport
 client/mysqlshow
 client/mysqlslap
 client/mysqltest
+client/mariadb-conv
 cmake_install.cmake
 dbug/*.r
 dbug/factorial

client/CMakeLists.txt

@@ -80,6 +80,9 @@ MYSQL_ADD_EXECUTABLE(mysqlslap mysqlslap.c)
 SET_SOURCE_FILES_PROPERTIES(mysqlslap.c PROPERTIES COMPILE_FLAGS "-DTHREADS")
 TARGET_LINK_LIBRARIES(mysqlslap ${CLIENT_LIB})
 
+MYSQL_ADD_EXECUTABLE(mariadb-conv mariadb-conv.cc)
+TARGET_LINK_LIBRARIES(mariadb-conv ${CLIENT_LIB} strings)
+
 # "WIN32" also covers 64 bit. "echo" is used in some files below "mysql-test/".
 IF(WIN32)
   MYSQL_ADD_EXECUTABLE(echo echo.c COMPONENT Junk)

client/mariadb-conv.cc

@@ -0,0 +1,209 @@
+/*
+   Copyright (c) 2001, 2013, Oracle and/or its affiliates.
+   Copyright (c) 2010, 2019, MariaDB
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA
+*/
+
+/*
+   Character set conversion utility
+*/
+
+#include "mariadb.h"
+#include "client_priv.h"
+
+#define CONV_VERSION "1.0"
+
+
+class CmdOpt
+{
+public:
+  const char *m_charset_from;
+  const char *m_charset_to;
+  my_bool m_continue;
+  CmdOpt()
+   :m_charset_from("latin1"),
+    m_charset_to("latin1"),
+    m_continue(FALSE)
+  { }
+  static CHARSET_INFO *csinfo_by_name(const char *csname)
+  {
+    return get_charset_by_csname(csname, MY_CS_PRIMARY, MYF(0));
+  }
+  CHARSET_INFO *csinfo_from() const
+  {
+    return m_charset_from ? csinfo_by_name(m_charset_from) : NULL;
+  }
+  CHARSET_INFO *csinfo_to() const
+  {
+    return m_charset_to ? csinfo_by_name(m_charset_to) : NULL;
+  }
+};
+
+
+static CmdOpt opt;
+
+
+static struct my_option long_options[] =
+{
+  {"from", 'f', "Specifies the encoding of the input.", &opt.m_charset_from,
+   &opt.m_charset_from, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+  {"to", 't', "Specifies the encoding of the output.", &opt.m_charset_to,
+   &opt.m_charset_to, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+  {"continue", 'c', "When this option is given, characters that cannot be "
+   "converted are silently discarded, instead of leading to a conversion error.",
+   &opt.m_continue, &opt.m_continue, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+  {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
+};
+
+
+my_bool
+get_one_option(const struct my_option *opt,
+               char *value, const char *filename)
+{
+  return 0;
+}
+
+
+class Conv
+{
+  CHARSET_INFO *m_tocs;
+  CHARSET_INFO *m_fromcs;
+  bool m_continue;
+public:
+  Conv(CHARSET_INFO *tocs, CHARSET_INFO *fromcs, bool opt_continue)
+   :m_tocs(tocs), m_fromcs(fromcs), m_continue(opt_continue)
+  { }
+  bool convert_file(FILE *infile) const;
+  bool convert_file_by_name(const char *filename) const;
+};
+
+
+bool Conv::convert_file(FILE *infile) const
+{
+  char from[FN_REFLEN + 1], to[FN_REFLEN + 2];
+
+  while (fgets(from, sizeof(from), infile) != NULL)
+  {
+    uint errors;
+    size_t length= 0;
+    for (char *s= from; s < from + sizeof(from); s++)
+    {
+      if (*s == '\0' || *s == '\r' || *s == '\n')
+      {
+        *s= '\0';
+        length= s - from;
+        break;
+      }
+    }
+
+    if (!length)
+    {
+      puts("");
+      continue;
+    }
+
+    length= my_convert(to, (uint32) (sizeof(to) - 1), m_tocs,
+                       from, (uint32) length, m_fromcs,
+                       &errors);
+    to[length]= '\0';
+    if (unlikely(!length || errors) && !m_continue)
+      return true;
+    else
+      puts(to);
+  }
+
+  return false;
+} /* convert */
+
+
+bool Conv::convert_file_by_name(const char *filename) const
+{
+  FILE *fp;
+  if ((fp= fopen(filename, "r")) == NULL)
+  {
+    printf("can't open file %s", filename);
+    return 1;
+  }
+  bool rc= convert_file(fp);
+  fclose(fp);
+  return rc;
+}
+
+
+class Session
+{
+public:
+  Session(const char *prog)
+  {
+    MY_INIT(prog);
+  }
+  ~Session()
+  {
+    my_end(0);
+  }
+  void usage(void)
+  {
+    printf("%s Ver %s Distrib %s for %s on %s\n", my_progname, CONV_VERSION,
+      MYSQL_SERVER_VERSION, SYSTEM_TYPE, MACHINE_TYPE);
+    puts("Character set conversion utility for MariaDB");
+    puts("Usage:");
+    printf("%s [-f encoding] [-t encoding] [inputfile]...\n", my_progname);
+    my_print_help(long_options);
+  }
+};
+
+
+int main(int argc, char *argv[])
+{
+  Session session(argv[0]);
+  CHARSET_INFO *charset_info_from= NULL;
+  CHARSET_INFO *charset_info_to= NULL;
+
+  if (handle_options(&argc, &argv, long_options, get_one_option))
+  {
+    session.usage();
+    return 1;
+  }
+
+  if (!(charset_info_from= opt.csinfo_from()))
+  {
+    fprintf(stderr, "Character set %s is not supported\n", opt.m_charset_from);
+    return 1;
+  }
+
+  if (!(charset_info_to= opt.csinfo_to()))
+  {
+    fprintf(stderr, "Character set %s is not supported\n", opt.m_charset_to);
+    return 1;
+  }
+
+  Conv conv(charset_info_to, charset_info_from, opt.m_continue);
+
+  if (argc == 0)
+  {
+    if (conv.convert_file(stdin))
+      return 1;
+  }
+  else
+  {
+    for (int i= 0; i < argc; i++)
+    {
+      if (conv.convert_file_by_name(argv[i]))
+       return 1;
+    }
+  }
+
+  return 0;
+} /* main */

debian/mariadb-client-10.5.install

@@ -1,5 +1,6 @@
 debian/additions/innotop/innotop usr/bin/
 debian/additions/mysqlreport usr/bin/
+usr/bin/mariadb-conv
 usr/bin/mysql_find_rows
 usr/bin/mysql_fix_extensions
 usr/bin/mysql_waitpid

mysql-test/mysql-test-run.pl

@@ -176,6 +176,7 @@ my @DEFAULT_SUITES= qw(
     archive-
     binlog-
     binlog_encryption-
+    client-
     csv-
     compat/oracle-
     compat/mssql-
@@ -218,6 +219,7 @@ our $exe_mysqladmin;
 our $exe_mysqltest;
 our $exe_libtool;
 our $exe_mysql_embedded;
+our $exe_mariadb_conv;
 
 our $opt_big_test= 0;
 our $opt_staging_run= 0;
@@ -2185,6 +2187,7 @@ sub executable_setup () {
   $exe_mysqladmin=     mtr_exe_exists("$path_client_bindir/mysqladmin");
   $exe_mysql=          mtr_exe_exists("$path_client_bindir/mysql");
   $exe_mysql_plugin=   mtr_exe_exists("$path_client_bindir/mysql_plugin");
+  $exe_mariadb_conv=   mtr_exe_exists("$path_client_bindir/mariadb-conv");
 
   $exe_mysql_embedded= mtr_exe_maybe_exists("$basedir/libmysqld/examples/mysql_embedded");
 
@@ -2486,6 +2489,7 @@ sub environment_setup {
   $ENV{'EXE_MYSQL'}=                $exe_mysql;
   $ENV{'MYSQL_PLUGIN'}=             $exe_mysql_plugin;
   $ENV{'MYSQL_EMBEDDED'}=           $exe_mysql_embedded;
+  $ENV{'MARIADB_CONV'}=             $exe_mariadb_conv;
   if(IS_WINDOWS)
   {
      $ENV{'MYSQL_INSTALL_DB_EXE'}=  mtr_exe_exists("$bindir/sql$opt_vs_config/mysql_install_db");

mysql-test/suite/client/mariadb-conv-cp932.result

@@ -0,0 +1,47 @@
+#
+# MDEV-17088 Provide tools to encode/decode mysql-encoded file system names
+#
+SET NAMES cp932;
+@6e2c@8a66@8cc7@6599
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
+@6e2c@8a66@8cc7@6599
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
+@6e2c@8a66@8cc7@6599@5eab
+# bulk convert with pipe
+CREATE TABLE t1 (id SERIAL, a VARCHAR(64) CHARACTER SET cp932);
+INSERT INTO t1 (a) VALUES ('<27><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>'), ('<27><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>2');
+a
+@6e2c@8a66@8cc7@6599
+@6e2c@8a66@8cc7@65992
+BINARY CONVERT(a USING filename)
+@6e2c@8a66@8cc7@6599
+@6e2c@8a66@8cc7@65992
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>2
+test/<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>.frm
+test/<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>2.frm
+DROP TABLE t1;
+# bulk convert with file
+# --- Start of mariadb-conv for mysql-conv-test-cp932.txt ---
+
+@6e2c@8a66@8cc7@6599
+
+@6e2c@8a66@8cc7@65992
+@6e2c@8a66@8cc7@65993
+
+# --- End of mariadb-conv for mysql-conv-test-cp932.txt ---
+# --- Start of mariadb-conv for mysql-conv-test-cp932.txt and mysql-conv-test-cp932-2.txt ---
+
+@6e2c@8a66@8cc7@6599
+
+@6e2c@8a66@8cc7@65992
+@6e2c@8a66@8cc7@65993
+
+
+@6e2c@8a66@8cc7@6599
+
+@6e2c@8a66@8cc7@65992
+@6e2c@8a66@8cc7@65993
+
+# --- Start of mariadb-conv for mysql-conv-test-cp932.txt and mysql-conv-test-cp932-2.txt ---

mysql-test/suite/client/mariadb-conv-cp932.test

@@ -0,0 +1,54 @@
+--echo #
+--echo # MDEV-17088 Provide tools to encode/decode mysql-encoded file system names
+--echo #
+
+--character_set cp932
+SET NAMES cp932;
+
+--let $MYSQLD_DATADIR= `select @@datadir`
+
+# simple I/O
+--exec echo "<22><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>" | $MARIADB_CONV -f cp932 -t filename
+--exec echo "@6e2c@8a66@8cc7@6599@5eab" | $MARIADB_CONV -f filename -t cp932
+
+# undo query result
+--let $query_result=`SELECT CONVERT(CONVERT('<27><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>' USING filename) USING binary);`
+--echo $query_result
+--exec echo $query_result | $MARIADB_CONV -f filename -t cp932
+
+--let $reverse_query_result=`SELECT CONVERT(_filename '@6e2c@8a66@8cc7@6599@5eab' USING cp932);`
+--echo $reverse_query_result
+--exec echo $reverse_query_result | $MARIADB_CONV -f cp932 -t filename
+
+--echo # bulk convert with pipe
+
+CREATE TABLE t1 (id SERIAL, a VARCHAR(64) CHARACTER SET cp932);
+INSERT INTO t1 (a) VALUES ('<27><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>'), ('<27><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>2');
+--exec $MYSQL -Dtest --default-character-set=cp932 -e "SELECT a FROM t1 ORDER BY id" | $MARIADB_CONV -f cp932 -t filename
+--exec $MYSQL -Dtest --default-character-set=cp932 -e "SELECT BINARY CONVERT(a USING filename) FROM t1 ORDER BY id"
+--exec $MYSQL -Dtest --default-character-set=cp932 --column-names=0 -e "SELECT BINARY CONVERT(a USING filename) FROM t1 ORDER BY id" | $MARIADB_CONV -f filename -t cp932
+--exec $MYSQL -Dtest --default-character-set=cp932 --column-names=0 -e "SELECT CONCAT('test/', BINARY CONVERT(a USING filename),'.frm') FROM t1 ORDER BY id" | $REPLACE "/" "@002f" "." "@002e"| $MARIADB_CONV -f filename -t cp932
+DROP TABLE t1;
+
+
+--echo # bulk convert with file
+--write_file $MYSQL_TMP_DIR/mysql-conv-test-cp932.txt
+
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
+
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>2
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>3
+
+EOF
+
+--echo # --- Start of mariadb-conv for mysql-conv-test-cp932.txt ---
+--exec $MARIADB_CONV -f cp932 -t filename $MYSQL_TMP_DIR/mysql-conv-test-cp932.txt
+--echo # --- End of mariadb-conv for mysql-conv-test-cp932.txt ---
+
+--copy_file $MYSQL_TMP_DIR/mysql-conv-test-cp932.txt $MYSQL_TMP_DIR/mysql-conv-test-cp932-2.txt
+--echo # --- Start of mariadb-conv for mysql-conv-test-cp932.txt and mysql-conv-test-cp932-2.txt ---
+--exec $MARIADB_CONV -f cp932 -t filename $MYSQL_TMP_DIR/mysql-conv-test-cp932.txt $MYSQL_TMP_DIR/mysql-conv-test-cp932-2.txt
+--echo # --- Start of mariadb-conv for mysql-conv-test-cp932.txt and mysql-conv-test-cp932-2.txt ---
+
+--remove_file $MYSQL_TMP_DIR/mysql-conv-test-cp932.txt
+--remove_file $MYSQL_TMP_DIR/mysql-conv-test-cp932-2.txt

mysql-test/suite/client/mariadb-conv-utf8.result

@@ -0,0 +1,47 @@
+#
+# MDEV-17088 Provide tools to encode/decode mysql-encoded file system names
+#
+SET NAMES utf8;
+@6e2c@8a66@8cc7@6599
+測試資料庫
+@6e2c@8a66@8cc7@6599
+測試資料
+測試資料庫
+@6e2c@8a66@8cc7@6599@5eab
+# bulk convert with pipe
+CREATE TABLE t1 (id SERIAL, a VARCHAR(64) CHARACTER SET utf8);
+INSERT INTO t1 (a) VALUES ('測試資料'), ('測試資料2');
+a
+@6e2c@8a66@8cc7@6599
+@6e2c@8a66@8cc7@65992
+BINARY CONVERT(a USING filename)
+@6e2c@8a66@8cc7@6599
+@6e2c@8a66@8cc7@65992
+測試資料
+測試資料2
+test/測試資料.frm
+test/測試資料2.frm
+DROP TABLE t1;
+# bulk convert with file
+# --- Start of mariadb-conv for mysql-conv-test-utf8.txt ---
+
+@6e2c@8a66@8cc7@6599
+
+@6e2c@8a66@8cc7@65992
+@6e2c@8a66@8cc7@65993
+
+# --- End of mariadb-conv for mysql-conv-test-utf8.txt ---
+# --- Start of mariadb-conv for mysql-conv-test-utf8.txt and mysql-conv-test-utf8-2.txt ---
+
+@6e2c@8a66@8cc7@6599
+
+@6e2c@8a66@8cc7@65992
+@6e2c@8a66@8cc7@65993
+
+
+@6e2c@8a66@8cc7@6599
+
+@6e2c@8a66@8cc7@65992
+@6e2c@8a66@8cc7@65993
+
+# --- Start of mariadb-conv for mysql-conv-test-utf8.txt and mysql-conv-test-utf8-2.txt ---

mysql-test/suite/client/mariadb-conv-utf8.test

@@ -0,0 +1,54 @@
+--echo #
+--echo # MDEV-17088 Provide tools to encode/decode mysql-encoded file system names
+--echo #
+
+--character_set utf8
+SET NAMES utf8;
+
+--let $MYSQLD_DATADIR= `select @@datadir`
+
+# simple I/O
+--exec echo "測試資料" | $MARIADB_CONV -f utf8 -t filename
+--exec echo "@6e2c@8a66@8cc7@6599@5eab" | $MARIADB_CONV -f filename -t utf8
+
+# undo query result
+--let $query_result=`SELECT CONVERT(CONVERT('測試資料' USING filename) USING binary);`
+--echo $query_result
+--exec echo $query_result | $MARIADB_CONV -f filename -t utf8
+
+--let $reverse_query_result=`SELECT CONVERT(_filename '@6e2c@8a66@8cc7@6599@5eab' USING utf8);`
+--echo $reverse_query_result
+--exec echo $reverse_query_result | $MARIADB_CONV -f utf8 -t filename
+
+--echo # bulk convert with pipe
+
+CREATE TABLE t1 (id SERIAL, a VARCHAR(64) CHARACTER SET utf8);
+INSERT INTO t1 (a) VALUES ('測試資料'), ('測試資料2');
+--exec $MYSQL -Dtest --default-character-set=utf8 -e "SELECT a FROM t1 ORDER BY id" | $MARIADB_CONV -f utf8 -t filename
+--exec $MYSQL -Dtest --default-character-set=utf8 -e "SELECT BINARY CONVERT(a USING filename) FROM t1 ORDER BY id"
+--exec $MYSQL -Dtest --default-character-set=utf8 --column-names=0 -e "SELECT BINARY CONVERT(a USING filename) FROM t1 ORDER BY id" | $MARIADB_CONV -f filename -t utf8
+--exec $MYSQL -Dtest --default-character-set=utf8 --column-names=0 -e "SELECT CONCAT('test/', BINARY CONVERT(a USING filename),'.frm') FROM t1 ORDER BY id" | $REPLACE "/" "@002f" "." "@002e"| $MARIADB_CONV -f filename -t utf8
+DROP TABLE t1;
+
+
+--echo # bulk convert with file
+--write_file $MYSQL_TMP_DIR/mysql-conv-test-utf8.txt
+
+測試資料
+
+測試資料2
+測試資料3
+
+EOF
+
+--echo # --- Start of mariadb-conv for mysql-conv-test-utf8.txt ---
+--exec $MARIADB_CONV -f utf8 -t filename $MYSQL_TMP_DIR/mysql-conv-test-utf8.txt
+--echo # --- End of mariadb-conv for mysql-conv-test-utf8.txt ---
+
+--copy_file $MYSQL_TMP_DIR/mysql-conv-test-utf8.txt $MYSQL_TMP_DIR/mysql-conv-test-utf8-2.txt
+--echo # --- Start of mariadb-conv for mysql-conv-test-utf8.txt and mysql-conv-test-utf8-2.txt ---
+--exec $MARIADB_CONV -f utf8 -t filename $MYSQL_TMP_DIR/mysql-conv-test-utf8.txt $MYSQL_TMP_DIR/mysql-conv-test-utf8-2.txt
+--echo # --- Start of mariadb-conv for mysql-conv-test-utf8.txt and mysql-conv-test-utf8-2.txt ---
+
+--remove_file $MYSQL_TMP_DIR/mysql-conv-test-utf8.txt
+--remove_file $MYSQL_TMP_DIR/mysql-conv-test-utf8-2.txt

mysql-test/suite/client/mariadb-conv.result

@@ -0,0 +1,13 @@
+#
+# MDEV-17088 Provide tools to encode/decode mysql-encoded file system names
+#
+# default encoding
+t1
+t1
+t1
+# invalid option
+mariadb-conv: unknown option '-r'
+# unknown "to" character set
+Character set unknown-cs is not supported
+# unknown "from" character set
+Character set unknown-cs is not supported

mysql-test/suite/client/mariadb-conv.test

@@ -0,0 +1,25 @@
+--echo #
+--echo # MDEV-17088 Provide tools to encode/decode mysql-encoded file system names
+--echo #
+
+--character_set latin1
+
+--echo # default encoding
+--exec echo "t1" | $MARIADB_CONV
+--exec echo "t1" | $MARIADB_CONV -f filename
+--exec echo "t1" | $MARIADB_CONV -t filename
+
+--echo # invalid option
+--replace_regex /.*mariadb-conv.*: unknown/mariadb-conv: unknown/
+--error 1
+--exec echo "t1" | $MARIADB_CONV -f filename -r latin1  2>&1 > /dev/null
+
+--echo # unknown "to" character set
+--replace_regex /.*mariadb-conv.*: unknown/mariadb-conv: unknown/
+--error 1
+--exec echo "t1" | $MARIADB_CONV -f filename -t unknown-cs  2>&1 > /dev/null
+
+--echo # unknown "from" character set
+--replace_regex /.*mariadb-conv.*: unknown/mariadb-conv: unknown/
+--error 1
+--exec echo "t1" | $MARIADB_CONV -f unknown-cs -t latin1 2>&1 > /dev/null