diff --git a/.gitignore b/.gitignore index 69fc59429fd..4e74d862823 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,7 @@ client/mysqlimport client/mysqlshow client/mysqlslap client/mysqltest +client/mariadb-conv cmake_install.cmake dbug/*.r dbug/factorial diff --git a/client/CMakeLists.txt b/client/CMakeLists.txt index 63e4cf36b00..8f8f1046874 100644 --- a/client/CMakeLists.txt +++ b/client/CMakeLists.txt @@ -80,6 +80,9 @@ MYSQL_ADD_EXECUTABLE(mysqlslap mysqlslap.c) SET_SOURCE_FILES_PROPERTIES(mysqlslap.c PROPERTIES COMPILE_FLAGS "-DTHREADS") TARGET_LINK_LIBRARIES(mysqlslap ${CLIENT_LIB}) +MYSQL_ADD_EXECUTABLE(mariadb-conv mariadb-conv.cc) +TARGET_LINK_LIBRARIES(mariadb-conv ${CLIENT_LIB} strings) + # "WIN32" also covers 64 bit. "echo" is used in some files below "mysql-test/". IF(WIN32) MYSQL_ADD_EXECUTABLE(echo echo.c COMPONENT Junk) diff --git a/client/mariadb-conv.cc b/client/mariadb-conv.cc new file mode 100644 index 00000000000..a4517e2ad25 --- /dev/null +++ b/client/mariadb-conv.cc @@ -0,0 +1,209 @@ +/* + Copyright (c) 2001, 2013, Oracle and/or its affiliates. + Copyright (c) 2010, 2019, MariaDB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +/* + Character set conversion utility +*/ + +#include "mariadb.h" +#include "client_priv.h" + +#define CONV_VERSION "1.0" + + +class CmdOpt +{ +public: + const char *m_charset_from; + const char *m_charset_to; + my_bool m_continue; + CmdOpt() + :m_charset_from("latin1"), + m_charset_to("latin1"), + m_continue(FALSE) + { } + static CHARSET_INFO *csinfo_by_name(const char *csname) + { + return get_charset_by_csname(csname, MY_CS_PRIMARY, MYF(0)); + } + CHARSET_INFO *csinfo_from() const + { + return m_charset_from ? csinfo_by_name(m_charset_from) : NULL; + } + CHARSET_INFO *csinfo_to() const + { + return m_charset_to ? csinfo_by_name(m_charset_to) : NULL; + } +}; + + +static CmdOpt opt; + + +static struct my_option long_options[] = +{ + {"from", 'f', "Specifies the encoding of the input.", &opt.m_charset_from, + &opt.m_charset_from, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"to", 't', "Specifies the encoding of the output.", &opt.m_charset_to, + &opt.m_charset_to, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"continue", 'c', "When this option is given, characters that cannot be " + "converted are silently discarded, instead of leading to a conversion error.", + &opt.m_continue, &opt.m_continue, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} +}; + + +my_bool +get_one_option(const struct my_option *opt, + char *value, const char *filename) +{ + return 0; +} + + +class Conv +{ + CHARSET_INFO *m_tocs; + CHARSET_INFO *m_fromcs; + bool m_continue; +public: + Conv(CHARSET_INFO *tocs, CHARSET_INFO *fromcs, bool opt_continue) + :m_tocs(tocs), m_fromcs(fromcs), m_continue(opt_continue) + { } + bool convert_file(FILE *infile) const; + bool convert_file_by_name(const char *filename) const; +}; + + +bool Conv::convert_file(FILE *infile) const +{ + char from[FN_REFLEN + 1], to[FN_REFLEN + 2]; + + while (fgets(from, sizeof(from), infile) != NULL) + { + uint errors; + size_t length= 0; + for (char *s= from; s < from + sizeof(from); s++) + { + if (*s == '\0' || *s == '\r' || *s == '\n') + { + *s= '\0'; + length= s - from; + break; + } + } + + if (!length) + { + puts(""); + continue; + } + + length= my_convert(to, (uint32) (sizeof(to) - 1), m_tocs, + from, (uint32) length, m_fromcs, + &errors); + to[length]= '\0'; + if (unlikely(!length || errors) && !m_continue) + return true; + else + puts(to); + } + + return false; +} /* convert */ + + +bool Conv::convert_file_by_name(const char *filename) const +{ + FILE *fp; + if ((fp= fopen(filename, "r")) == NULL) + { + printf("can't open file %s", filename); + return 1; + } + bool rc= convert_file(fp); + fclose(fp); + return rc; +} + + +class Session +{ +public: + Session(const char *prog) + { + MY_INIT(prog); + } + ~Session() + { + my_end(0); + } + void usage(void) + { + printf("%s Ver %s Distrib %s for %s on %s\n", my_progname, CONV_VERSION, + MYSQL_SERVER_VERSION, SYSTEM_TYPE, MACHINE_TYPE); + puts("Character set conversion utility for MariaDB"); + puts("Usage:"); + printf("%s [-f encoding] [-t encoding] [inputfile]...\n", my_progname); + my_print_help(long_options); + } +}; + + +int main(int argc, char *argv[]) +{ + Session session(argv[0]); + CHARSET_INFO *charset_info_from= NULL; + CHARSET_INFO *charset_info_to= NULL; + + if (handle_options(&argc, &argv, long_options, get_one_option)) + { + session.usage(); + return 1; + } + + if (!(charset_info_from= opt.csinfo_from())) + { + fprintf(stderr, "Character set %s is not supported\n", opt.m_charset_from); + return 1; + } + + if (!(charset_info_to= opt.csinfo_to())) + { + fprintf(stderr, "Character set %s is not supported\n", opt.m_charset_to); + return 1; + } + + Conv conv(charset_info_to, charset_info_from, opt.m_continue); + + if (argc == 0) + { + if (conv.convert_file(stdin)) + return 1; + } + else + { + for (int i= 0; i < argc; i++) + { + if (conv.convert_file_by_name(argv[i])) + return 1; + } + } + + return 0; +} /* main */ diff --git a/debian/mariadb-client-10.5.install b/debian/mariadb-client-10.5.install index 4cd0af71c24..4d848364f23 100644 --- a/debian/mariadb-client-10.5.install +++ b/debian/mariadb-client-10.5.install @@ -1,5 +1,6 @@ debian/additions/innotop/innotop usr/bin/ debian/additions/mysqlreport usr/bin/ +usr/bin/mariadb-conv usr/bin/mysql_find_rows usr/bin/mysql_fix_extensions usr/bin/mysql_waitpid diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index 64e2436b1f4..98a8e90556e 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -176,6 +176,7 @@ my @DEFAULT_SUITES= qw( archive- binlog- binlog_encryption- + client- csv- compat/oracle- compat/mssql- @@ -218,6 +219,7 @@ our $exe_mysqladmin; our $exe_mysqltest; our $exe_libtool; our $exe_mysql_embedded; +our $exe_mariadb_conv; our $opt_big_test= 0; our $opt_staging_run= 0; @@ -2185,6 +2187,7 @@ sub executable_setup () { $exe_mysqladmin= mtr_exe_exists("$path_client_bindir/mysqladmin"); $exe_mysql= mtr_exe_exists("$path_client_bindir/mysql"); $exe_mysql_plugin= mtr_exe_exists("$path_client_bindir/mysql_plugin"); + $exe_mariadb_conv= mtr_exe_exists("$path_client_bindir/mariadb-conv"); $exe_mysql_embedded= mtr_exe_maybe_exists("$basedir/libmysqld/examples/mysql_embedded"); @@ -2486,6 +2489,7 @@ sub environment_setup { $ENV{'EXE_MYSQL'}= $exe_mysql; $ENV{'MYSQL_PLUGIN'}= $exe_mysql_plugin; $ENV{'MYSQL_EMBEDDED'}= $exe_mysql_embedded; + $ENV{'MARIADB_CONV'}= $exe_mariadb_conv; if(IS_WINDOWS) { $ENV{'MYSQL_INSTALL_DB_EXE'}= mtr_exe_exists("$bindir/sql$opt_vs_config/mysql_install_db"); diff --git a/mysql-test/suite/client/mariadb-conv-cp932.result b/mysql-test/suite/client/mariadb-conv-cp932.result new file mode 100644 index 00000000000..e18e36b81c9 --- /dev/null +++ b/mysql-test/suite/client/mariadb-conv-cp932.result @@ -0,0 +1,47 @@ +# +# MDEV-17088 Provide tools to encode/decode mysql-encoded file system names +# +SET NAMES cp932; +@6e2c@8a66@8cc7@6599 +‘ͺŽŽŽ‘—ΏŒΙ +@6e2c@8a66@8cc7@6599 +‘ͺŽŽŽ‘—Ώ +‘ͺŽŽŽ‘—ΏŒΙ +@6e2c@8a66@8cc7@6599@5eab +# bulk convert with pipe +CREATE TABLE t1 (id SERIAL, a VARCHAR(64) CHARACTER SET cp932); +INSERT INTO t1 (a) VALUES ('‘ͺŽŽŽ‘—Ώ'), ('‘ͺŽŽŽ‘—Ώ2'); +a +@6e2c@8a66@8cc7@6599 +@6e2c@8a66@8cc7@65992 +BINARY CONVERT(a USING filename) +@6e2c@8a66@8cc7@6599 +@6e2c@8a66@8cc7@65992 +‘ͺŽŽŽ‘—Ώ +‘ͺŽŽŽ‘—Ώ2 +test/‘ͺŽŽŽ‘—Ώ.frm +test/‘ͺŽŽŽ‘—Ώ2.frm +DROP TABLE t1; +# bulk convert with file +# --- Start of mariadb-conv for mysql-conv-test-cp932.txt --- + +@6e2c@8a66@8cc7@6599 + +@6e2c@8a66@8cc7@65992 +@6e2c@8a66@8cc7@65993 + +# --- End of mariadb-conv for mysql-conv-test-cp932.txt --- +# --- Start of mariadb-conv for mysql-conv-test-cp932.txt and mysql-conv-test-cp932-2.txt --- + +@6e2c@8a66@8cc7@6599 + +@6e2c@8a66@8cc7@65992 +@6e2c@8a66@8cc7@65993 + + +@6e2c@8a66@8cc7@6599 + +@6e2c@8a66@8cc7@65992 +@6e2c@8a66@8cc7@65993 + +# --- Start of mariadb-conv for mysql-conv-test-cp932.txt and mysql-conv-test-cp932-2.txt --- diff --git a/mysql-test/suite/client/mariadb-conv-cp932.test b/mysql-test/suite/client/mariadb-conv-cp932.test new file mode 100644 index 00000000000..abbaa2bae3d --- /dev/null +++ b/mysql-test/suite/client/mariadb-conv-cp932.test @@ -0,0 +1,54 @@ +--echo # +--echo # MDEV-17088 Provide tools to encode/decode mysql-encoded file system names +--echo # + +--character_set cp932 +SET NAMES cp932; + +--let $MYSQLD_DATADIR= `select @@datadir` + +# simple I/O +--exec echo "‘ͺŽŽŽ‘—Ώ" | $MARIADB_CONV -f cp932 -t filename +--exec echo "@6e2c@8a66@8cc7@6599@5eab" | $MARIADB_CONV -f filename -t cp932 + +# undo query result +--let $query_result=`SELECT CONVERT(CONVERT('‘ͺŽŽŽ‘—Ώ' USING filename) USING binary);` +--echo $query_result +--exec echo $query_result | $MARIADB_CONV -f filename -t cp932 + +--let $reverse_query_result=`SELECT CONVERT(_filename '@6e2c@8a66@8cc7@6599@5eab' USING cp932);` +--echo $reverse_query_result +--exec echo $reverse_query_result | $MARIADB_CONV -f cp932 -t filename + +--echo # bulk convert with pipe + +CREATE TABLE t1 (id SERIAL, a VARCHAR(64) CHARACTER SET cp932); +INSERT INTO t1 (a) VALUES ('‘ͺŽŽŽ‘—Ώ'), ('‘ͺŽŽŽ‘—Ώ2'); +--exec $MYSQL -Dtest --default-character-set=cp932 -e "SELECT a FROM t1 ORDER BY id" | $MARIADB_CONV -f cp932 -t filename +--exec $MYSQL -Dtest --default-character-set=cp932 -e "SELECT BINARY CONVERT(a USING filename) FROM t1 ORDER BY id" +--exec $MYSQL -Dtest --default-character-set=cp932 --column-names=0 -e "SELECT BINARY CONVERT(a USING filename) FROM t1 ORDER BY id" | $MARIADB_CONV -f filename -t cp932 +--exec $MYSQL -Dtest --default-character-set=cp932 --column-names=0 -e "SELECT CONCAT('test/', BINARY CONVERT(a USING filename),'.frm') FROM t1 ORDER BY id" | $REPLACE "/" "@002f" "." "@002e"| $MARIADB_CONV -f filename -t cp932 +DROP TABLE t1; + + +--echo # bulk convert with file +--write_file $MYSQL_TMP_DIR/mysql-conv-test-cp932.txt + +‘ͺŽŽŽ‘—Ώ + +‘ͺŽŽŽ‘—Ώ2 +‘ͺŽŽŽ‘—Ώ3 + +EOF + +--echo # --- Start of mariadb-conv for mysql-conv-test-cp932.txt --- +--exec $MARIADB_CONV -f cp932 -t filename $MYSQL_TMP_DIR/mysql-conv-test-cp932.txt +--echo # --- End of mariadb-conv for mysql-conv-test-cp932.txt --- + +--copy_file $MYSQL_TMP_DIR/mysql-conv-test-cp932.txt $MYSQL_TMP_DIR/mysql-conv-test-cp932-2.txt +--echo # --- Start of mariadb-conv for mysql-conv-test-cp932.txt and mysql-conv-test-cp932-2.txt --- +--exec $MARIADB_CONV -f cp932 -t filename $MYSQL_TMP_DIR/mysql-conv-test-cp932.txt $MYSQL_TMP_DIR/mysql-conv-test-cp932-2.txt +--echo # --- Start of mariadb-conv for mysql-conv-test-cp932.txt and mysql-conv-test-cp932-2.txt --- + +--remove_file $MYSQL_TMP_DIR/mysql-conv-test-cp932.txt +--remove_file $MYSQL_TMP_DIR/mysql-conv-test-cp932-2.txt diff --git a/mysql-test/suite/client/mariadb-conv-utf8.result b/mysql-test/suite/client/mariadb-conv-utf8.result new file mode 100644 index 00000000000..b38cb118c72 --- /dev/null +++ b/mysql-test/suite/client/mariadb-conv-utf8.result @@ -0,0 +1,47 @@ +# +# MDEV-17088 Provide tools to encode/decode mysql-encoded file system names +# +SET NAMES utf8; +@6e2c@8a66@8cc7@6599 +測試資料庫 +@6e2c@8a66@8cc7@6599 +測試資料 +測試資料庫 +@6e2c@8a66@8cc7@6599@5eab +# bulk convert with pipe +CREATE TABLE t1 (id SERIAL, a VARCHAR(64) CHARACTER SET utf8); +INSERT INTO t1 (a) VALUES ('測試資料'), ('測試資料2'); +a +@6e2c@8a66@8cc7@6599 +@6e2c@8a66@8cc7@65992 +BINARY CONVERT(a USING filename) +@6e2c@8a66@8cc7@6599 +@6e2c@8a66@8cc7@65992 +測試資料 +測試資料2 +test/測試資料.frm +test/測試資料2.frm +DROP TABLE t1; +# bulk convert with file +# --- Start of mariadb-conv for mysql-conv-test-utf8.txt --- + +@6e2c@8a66@8cc7@6599 + +@6e2c@8a66@8cc7@65992 +@6e2c@8a66@8cc7@65993 + +# --- End of mariadb-conv for mysql-conv-test-utf8.txt --- +# --- Start of mariadb-conv for mysql-conv-test-utf8.txt and mysql-conv-test-utf8-2.txt --- + +@6e2c@8a66@8cc7@6599 + +@6e2c@8a66@8cc7@65992 +@6e2c@8a66@8cc7@65993 + + +@6e2c@8a66@8cc7@6599 + +@6e2c@8a66@8cc7@65992 +@6e2c@8a66@8cc7@65993 + +# --- Start of mariadb-conv for mysql-conv-test-utf8.txt and mysql-conv-test-utf8-2.txt --- diff --git a/mysql-test/suite/client/mariadb-conv-utf8.test b/mysql-test/suite/client/mariadb-conv-utf8.test new file mode 100644 index 00000000000..cbfc38fa60b --- /dev/null +++ b/mysql-test/suite/client/mariadb-conv-utf8.test @@ -0,0 +1,54 @@ +--echo # +--echo # MDEV-17088 Provide tools to encode/decode mysql-encoded file system names +--echo # + +--character_set utf8 +SET NAMES utf8; + +--let $MYSQLD_DATADIR= `select @@datadir` + +# simple I/O +--exec echo "測試資料" | $MARIADB_CONV -f utf8 -t filename +--exec echo "@6e2c@8a66@8cc7@6599@5eab" | $MARIADB_CONV -f filename -t utf8 + +# undo query result +--let $query_result=`SELECT CONVERT(CONVERT('測試資料' USING filename) USING binary);` +--echo $query_result +--exec echo $query_result | $MARIADB_CONV -f filename -t utf8 + +--let $reverse_query_result=`SELECT CONVERT(_filename '@6e2c@8a66@8cc7@6599@5eab' USING utf8);` +--echo $reverse_query_result +--exec echo $reverse_query_result | $MARIADB_CONV -f utf8 -t filename + +--echo # bulk convert with pipe + +CREATE TABLE t1 (id SERIAL, a VARCHAR(64) CHARACTER SET utf8); +INSERT INTO t1 (a) VALUES ('測試資料'), ('測試資料2'); +--exec $MYSQL -Dtest --default-character-set=utf8 -e "SELECT a FROM t1 ORDER BY id" | $MARIADB_CONV -f utf8 -t filename +--exec $MYSQL -Dtest --default-character-set=utf8 -e "SELECT BINARY CONVERT(a USING filename) FROM t1 ORDER BY id" +--exec $MYSQL -Dtest --default-character-set=utf8 --column-names=0 -e "SELECT BINARY CONVERT(a USING filename) FROM t1 ORDER BY id" | $MARIADB_CONV -f filename -t utf8 +--exec $MYSQL -Dtest --default-character-set=utf8 --column-names=0 -e "SELECT CONCAT('test/', BINARY CONVERT(a USING filename),'.frm') FROM t1 ORDER BY id" | $REPLACE "/" "@002f" "." "@002e"| $MARIADB_CONV -f filename -t utf8 +DROP TABLE t1; + + +--echo # bulk convert with file +--write_file $MYSQL_TMP_DIR/mysql-conv-test-utf8.txt + +測試資料 + +測試資料2 +測試資料3 + +EOF + +--echo # --- Start of mariadb-conv for mysql-conv-test-utf8.txt --- +--exec $MARIADB_CONV -f utf8 -t filename $MYSQL_TMP_DIR/mysql-conv-test-utf8.txt +--echo # --- End of mariadb-conv for mysql-conv-test-utf8.txt --- + +--copy_file $MYSQL_TMP_DIR/mysql-conv-test-utf8.txt $MYSQL_TMP_DIR/mysql-conv-test-utf8-2.txt +--echo # --- Start of mariadb-conv for mysql-conv-test-utf8.txt and mysql-conv-test-utf8-2.txt --- +--exec $MARIADB_CONV -f utf8 -t filename $MYSQL_TMP_DIR/mysql-conv-test-utf8.txt $MYSQL_TMP_DIR/mysql-conv-test-utf8-2.txt +--echo # --- Start of mariadb-conv for mysql-conv-test-utf8.txt and mysql-conv-test-utf8-2.txt --- + +--remove_file $MYSQL_TMP_DIR/mysql-conv-test-utf8.txt +--remove_file $MYSQL_TMP_DIR/mysql-conv-test-utf8-2.txt diff --git a/mysql-test/suite/client/mariadb-conv.result b/mysql-test/suite/client/mariadb-conv.result new file mode 100644 index 00000000000..1677a13de1d --- /dev/null +++ b/mysql-test/suite/client/mariadb-conv.result @@ -0,0 +1,13 @@ +# +# MDEV-17088 Provide tools to encode/decode mysql-encoded file system names +# +# default encoding +t1 +t1 +t1 +# invalid option +mariadb-conv: unknown option '-r' +# unknown "to" character set +Character set unknown-cs is not supported +# unknown "from" character set +Character set unknown-cs is not supported diff --git a/mysql-test/suite/client/mariadb-conv.test b/mysql-test/suite/client/mariadb-conv.test new file mode 100644 index 00000000000..2046b876b69 --- /dev/null +++ b/mysql-test/suite/client/mariadb-conv.test @@ -0,0 +1,25 @@ +--echo # +--echo # MDEV-17088 Provide tools to encode/decode mysql-encoded file system names +--echo # + +--character_set latin1 + +--echo # default encoding +--exec echo "t1" | $MARIADB_CONV +--exec echo "t1" | $MARIADB_CONV -f filename +--exec echo "t1" | $MARIADB_CONV -t filename + +--echo # invalid option +--replace_regex /.*mariadb-conv.*: unknown/mariadb-conv: unknown/ +--error 1 +--exec echo "t1" | $MARIADB_CONV -f filename -r latin1 2>&1 > /dev/null + +--echo # unknown "to" character set +--replace_regex /.*mariadb-conv.*: unknown/mariadb-conv: unknown/ +--error 1 +--exec echo "t1" | $MARIADB_CONV -f filename -t unknown-cs 2>&1 > /dev/null + +--echo # unknown "from" character set +--replace_regex /.*mariadb-conv.*: unknown/mariadb-conv: unknown/ +--error 1 +--exec echo "t1" | $MARIADB_CONV -f unknown-cs -t latin1 2>&1 > /dev/null