From a50e6c9eb126cbb1d46735dade6c9006f1a5a764 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 13 May 2020 13:10:35 +0200 Subject: [PATCH 01/19] MDEV-17153 server crash on repair table ... use_frm data_file_length == 0 in mi_repair() is normal for REPAIR ... USE_FRM. But in-file links (for blocks and deleted chain) must be compared with the real file length to avoid spurious "link points outside datafile" warnings and arbitrary block skipping. --- mysql-test/r/repair.result | 21 +++++++++++++++++++++ mysql-test/t/repair.test | 21 +++++++++++++++++++++ storage/myisam/mi_check.c | 6 ++++++ 3 files changed, 48 insertions(+) diff --git a/mysql-test/r/repair.result b/mysql-test/r/repair.result index 171ef836bda..749b8a74b4f 100644 --- a/mysql-test/r/repair.result +++ b/mysql-test/r/repair.result @@ -233,3 +233,24 @@ i UNLOCK TABLES; DROP TABLE t1; # End of 10.0 tests +create table t1 (a int, b varchar(200)); +insert t1 select seq, repeat(200, seq) from seq_1_to_30; +delete from t1 where a % 13 = 0; +repair table t1 use_frm; +Table Op Msg_type Msg_text +test.t1 repair warning Number of rows changed from 0 to 28 +test.t1 repair status OK +delete from t1 where a % 11 = 0; +repair table t1 extended use_frm; +Table Op Msg_type Msg_text +test.t1 repair warning Number of rows changed from 0 to 26 +test.t1 repair status OK +delete from t1 where a % 7 = 0; +set myisam_repair_threads = 2; +repair table t1 use_frm; +Table Op Msg_type Msg_text +test.t1 repair warning Number of rows changed from 0 to 22 +test.t1 repair status OK +set myisam_repair_threads = default; +drop table t1; +# End of 10.2 tests diff --git a/mysql-test/t/repair.test b/mysql-test/t/repair.test index 75978ac482b..435eb99683a 100644 --- a/mysql-test/t/repair.test +++ b/mysql-test/t/repair.test @@ -1,6 +1,7 @@ # # Test of repair table # +--source include/have_sequence.inc --source include/default_charset.inc call mtr.add_suppression("character set is multi-byte"); @@ -246,3 +247,23 @@ UNLOCK TABLES; DROP TABLE t1; --echo # End of 10.0 tests + +# +# MDEV-17153 server crash on repair table ... use_frm +# +# Note, this test case doesn't crash, but shows spurios warnings +# unless the bug is fixed +# +create table t1 (a int, b varchar(200)); +insert t1 select seq, repeat(200, seq) from seq_1_to_30; +delete from t1 where a % 13 = 0; +repair table t1 use_frm; +delete from t1 where a % 11 = 0; +repair table t1 extended use_frm; +delete from t1 where a % 7 = 0; +set myisam_repair_threads = 2; +repair table t1 use_frm; +set myisam_repair_threads = default; +drop table t1; + +--echo # End of 10.2 tests diff --git a/storage/myisam/mi_check.c b/storage/myisam/mi_check.c index b18ffb99a11..d9b9bb5af4a 100644 --- a/storage/myisam/mi_check.c +++ b/storage/myisam/mi_check.c @@ -1586,6 +1586,8 @@ int mi_repair(HA_CHECK *param, register MI_INFO *info, sort_param.filepos=new_header_length; param->read_cache.end_of_file=sort_info.filelength= mysql_file_seek(info->dfile, 0L, MY_SEEK_END, MYF(0)); + if (info->state->data_file_length == 0) + info->state->data_file_length= sort_info.filelength; sort_info.dupp=0; sort_param.fix_datafile= (my_bool) (! rep_quick); sort_param.master=1; @@ -2290,6 +2292,8 @@ int mi_repair_by_sort(HA_CHECK *param, register MI_INFO *info, sort_info.buff=0; param->read_cache.end_of_file=sort_info.filelength= mysql_file_seek(param->read_cache.file, 0L, MY_SEEK_END, MYF(0)); + if (info->state->data_file_length == 0) + info->state->data_file_length= sort_info.filelength; sort_param.wordlist=NULL; init_alloc_root(&sort_param.wordroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0, @@ -2757,6 +2761,8 @@ int mi_repair_parallel(HA_CHECK *param, register MI_INFO *info, sort_info.buff=0; param->read_cache.end_of_file=sort_info.filelength= mysql_file_seek(param->read_cache.file, 0L, MY_SEEK_END, MYF(0)); + if (info->state->data_file_length == 0) + info->state->data_file_length= sort_info.filelength; if (share->data_file_type == DYNAMIC_RECORD) rec_length=MY_MAX(share->base.min_pack_length+1,share->base.min_block_length); From 39c141b4ae9e6381f9efd4c566e61b91371a3d63 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 13 May 2020 19:48:23 +0200 Subject: [PATCH 02/19] don't include .git files in source packages the existing ".git/" rule only filters out .git directories, but as we have submodules now, we have to filter out .git files too. --- cmake/cpack_source_ignore_files.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/cpack_source_ignore_files.cmake b/cmake/cpack_source_ignore_files.cmake index 5b103501a99..eb1ec191d17 100644 --- a/cmake/cpack_source_ignore_files.cmake +++ b/cmake/cpack_source_ignore_files.cmake @@ -15,6 +15,7 @@ SET(CPACK_SOURCE_IGNORE_FILES \\\\.git/ +\\\\.git$ \\\\.gitignore$ \\\\.gitattributes$ CMakeCache\\\\.txt$ From b01c8a6cc8b361c6c8ecd887f1958c1f0b2e9811 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 14 May 2020 16:18:01 +0200 Subject: [PATCH 03/19] MDEV-22558 wrong error for invalid utf8 table comment --- mysql-test/r/comment_table.result | 5 ++++- mysql-test/t/comment_table.test | 11 ++++++++--- sql/sql_table.cc | 19 +++++++++++++++++++ 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/mysql-test/r/comment_table.result b/mysql-test/r/comment_table.result index 99ecb04a362..d0af71b4540 100644 --- a/mysql-test/r/comment_table.result +++ b/mysql-test/r/comment_table.result @@ -1,4 +1,3 @@ -DROP TABLE IF EXISTS t1; create table t1 (c1 VARCHAR(10) NOT NULL COMMENT 'c1 comment', c2 INTEGER,c3 INTEGER COMMENT '012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789', c4 INTEGER, c5 INTEGER, c6 INTEGER, c7 INTEGER, INDEX i1 (c1) COMMENT 'i1 comment',INDEX i2(c2) ) COMMENT='abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcd'; SELECT table_comment,char_length(table_comment) FROM information_schema.tables WHERE table_name='t1'; @@ -125,3 +124,7 @@ SELECT table_comment,char_length(table_comment) FROM information_schema.tables W table_comment char_length(table_comment) SELECT column_comment,char_length(column_comment) FROM information_schema.columns WHERE table_name='t1'; column_comment char_length(column_comment) +set names utf8; +create table t1 (x int comment 'a’'); +ERROR HY000: Invalid utf8 character string: 'a' +set names latin1; diff --git a/mysql-test/t/comment_table.test b/mysql-test/t/comment_table.test index 2a85fbac571..db8bdf54ba6 100644 --- a/mysql-test/t/comment_table.test +++ b/mysql-test/t/comment_table.test @@ -1,6 +1,3 @@ ---disable_warnings -DROP TABLE IF EXISTS t1; ---enable_warnings #1024 bytes create table t1 (c1 VARCHAR(10) NOT NULL COMMENT 'c1 comment', c2 INTEGER,c3 INTEGER COMMENT '012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789', c4 INTEGER, c5 INTEGER, c6 INTEGER, c7 INTEGER, INDEX i1 (c1) COMMENT 'i1 comment',INDEX i2(c2) ) COMMENT='abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcd'; @@ -53,3 +50,11 @@ create table t1 (c1 VARCHAR(10) NOT NULL COMMENT 'c1 comment', c2 INTEGER,c3 INT ) COMMENT='abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcde'; SELECT table_comment,char_length(table_comment) FROM information_schema.tables WHERE table_name='t1'; SELECT column_comment,char_length(column_comment) FROM information_schema.columns WHERE table_name='t1'; + +# +# MDEV-22558 wrong error for invalid utf8 table comment +# +set names utf8; +--error ER_INVALID_CHARACTER_STRING +create table t1 (x int comment 'a’'); +set names latin1; diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 8d81911bf29..87d84b1abc6 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -4294,6 +4294,25 @@ bool validate_comment_length(THD *thd, LEX_STRING *comment, size_t max_len, Well_formed_prefix(system_charset_info, *comment, max_len).length(); if (tmp_len < comment->length) { +#if MARIADB_VERSION_ID < 100500 + if (comment->length <= max_len) + { + if (thd->is_strict_mode()) + { + my_error(ER_INVALID_CHARACTER_STRING, MYF(0), + system_charset_info->csname, comment->str); + DBUG_RETURN(true); + } + push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, + ER_INVALID_CHARACTER_STRING, + ER_THD(thd, ER_INVALID_CHARACTER_STRING), + system_charset_info->csname, comment->str); + comment->length= tmp_len; + DBUG_RETURN(false); + } +#else +#error do it in TEXT_STRING_sys +#endif if (thd->is_strict_mode()) { my_error(err_code, MYF(0), name, static_cast(max_len)); From 1e951155bddd42b84e1c4ed949a85d62829e696f Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 20 May 2020 17:02:47 +0200 Subject: [PATCH 04/19] bugfix: use THD::main_mem_root for kill error message cannot use the current THD::mem_root, because it can be temporarily reassigned to something with a very different life time (e.g. to TABLE::mem_root or range optimizer mem_root). --- sql/sql_class.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sql/sql_class.h b/sql/sql_class.h index 80aa5710b62..838998af94f 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -3729,7 +3729,8 @@ public: The worst things that can happen is that we get a suboptimal error message. */ - if ((killed_err= (err_info*) alloc(sizeof(*killed_err)))) + killed_err= (err_info*) alloc_root(&main_mem_root, sizeof(*killed_err)); + if (killed_err) { killed_err->no= killed_errno_arg; ::strmake((char*) killed_err->msg, killed_err_msg_arg, From ad772478660a67c1c93c22725e25b168ec512f4d Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 20 May 2020 17:04:22 +0200 Subject: [PATCH 05/19] MDEV-21958 Query having many NOT-IN clauses running forever and causing available free memory to use completely let thd->killed to abort range optimizer --- sql/opt_range.h | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/opt_range.h b/sql/opt_range.h index dff50252e1a..1e6aca540d8 100644 --- a/sql/opt_range.h +++ b/sql/opt_range.h @@ -656,6 +656,7 @@ public: bool statement_should_be_aborted() const { return + thd->killed || thd->is_fatal_error || thd->is_error() || alloced_sel_args > SEL_ARG::MAX_SEL_ARGS; From 6af37ba881fee7e6f651d5e0730c9374337ad1b4 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 21 May 2020 01:31:17 +0200 Subject: [PATCH 06/19] fix rocksdb zstd detection WITH_ROCKSDB_ZSTD must use the same capitalization as in the foreach loop --- storage/rocksdb/build_rocksdb.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/rocksdb/build_rocksdb.cmake b/storage/rocksdb/build_rocksdb.cmake index ca61842fbb8..3f3dca7e990 100644 --- a/storage/rocksdb/build_rocksdb.cmake +++ b/storage/rocksdb/build_rocksdb.cmake @@ -67,7 +67,7 @@ if(SNAPPY_FOUND AND (NOT WITH_ROCKSDB_snappy STREQUAL "OFF")) endif() include(CheckFunctionExists) -if(ZSTD_FOUND AND (NOT WITH_ROCKSDB_zstd STREQUAL "OFF")) +if(ZSTD_FOUND AND (NOT WITH_ROCKSDB_ZSTD STREQUAL "OFF")) SET(CMAKE_REQUIRED_LIBRARIES zstd) CHECK_FUNCTION_EXISTS(ZDICT_trainFromBuffer ZSTD_VALID) UNSET(CMAKE_REQUIRED_LIBRARIES) From cceb965a7918c1e3a29965fe0b440cf051764358 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 22 May 2020 12:05:53 +0200 Subject: [PATCH 07/19] Revert "MDEV-12445 : Rocksdb does not shutdown worker threads and aborts in memleak check on server shutdown" This reverts commit 6f1f9114971c3c158e9ac97313c92a24f6554415. because it doesn't do anything now (the server doesn't check my_disable_leak_check) and it never did anything before (because without `extern` it simply created a local instance of my_disable_leak_check, did not affect server's my_disable_leak_check). --- sql/mysqld.cc | 1 - storage/rocksdb/ha_rocksdb.cc | 8 -------- 2 files changed, 9 deletions(-) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 5878f4a2286..e40d5c38233 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -365,7 +365,6 @@ static bool volatile select_thread_in_use, signal_thread_in_use; static volatile bool ready_to_exit; static my_bool opt_debugging= 0, opt_external_locking= 0, opt_console= 0; static my_bool opt_short_log_format= 0, opt_silent_startup= 0; -bool my_disable_leak_check= false; uint kill_cached_threads; static uint wake_thread; diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index bf9183d7dea..5cbe4c162a8 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -114,7 +114,6 @@ int thd_binlog_format(const MYSQL_THD thd); bool thd_binlog_filter_ok(const MYSQL_THD thd); } -MYSQL_PLUGIN_IMPORT bool my_disable_leak_check; extern my_bool opt_core_file; // Needed in rocksdb_init_func @@ -5688,13 +5687,6 @@ static int rocksdb_init_func(void *const p) { } #endif - /** - Rocksdb does not always shutdown its threads, when - plugin is shut down. Disable server's leak check - at exit to avoid crash. - */ - my_disable_leak_check = true; - err = my_error_register(rdb_get_error_messages, HA_ERR_ROCKSDB_FIRST, HA_ERR_ROCKSDB_LAST); if (err != 0) { From 8cf589218f1b6ac5f8775068873357bd0dc8acbb Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 22 May 2020 19:50:22 +0200 Subject: [PATCH 08/19] optimize performance of the build in a fresh clone don't fetch the complete history of all submodules, it's rarely needed --- cmake/submodules.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/submodules.cmake b/cmake/submodules.cmake index 34d1f37c956..6e039bfeae3 100644 --- a/cmake/submodules.cmake +++ b/cmake/submodules.cmake @@ -19,11 +19,11 @@ IF(GIT_EXECUTABLE AND EXISTS "${CMAKE_SOURCE_DIR}/.git") SET(update_result 0) ELSEIF (cmake_update_submodules MATCHES force) MESSAGE(STATUS "Updating submodules (forced)") - EXECUTE_PROCESS(COMMAND "${GIT_EXECUTABLE}" submodule update --init --force + EXECUTE_PROCESS(COMMAND "${GIT_EXECUTABLE}" submodule update --init --force --depth=1 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" RESULT_VARIABLE update_result) ELSEIF (cmake_update_submodules MATCHES yes) - EXECUTE_PROCESS(COMMAND "${GIT_EXECUTABLE}" submodule update --init + EXECUTE_PROCESS(COMMAND "${GIT_EXECUTABLE}" submodule update --init --depth=1 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" RESULT_VARIABLE update_result) ELSE() From 04726f292088cf00c1ac9ed7543535e7a091810a Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sat, 23 May 2020 18:35:42 +0200 Subject: [PATCH 09/19] get rid of cmake warning --- cmake/build_configurations/mysql_release.cmake | 2 +- .../tokudb/PerconaFT/cmake_modules/TokuFeatureDetection.cmake | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/build_configurations/mysql_release.cmake b/cmake/build_configurations/mysql_release.cmake index b19a9437495..1609c45a632 100644 --- a/cmake/build_configurations/mysql_release.cmake +++ b/cmake/build_configurations/mysql_release.cmake @@ -136,7 +136,7 @@ IF(UNIX) RedHat/Fedora/Oracle Linux: yum install libaio-devel SuSE: zypper install libaio-devel - If you really do not want it, pass -DIGNORE_AIO_CHECK to cmake. + If you really do not want it, pass -DIGNORE_AIO_CHECK=ON to cmake. ") ENDIF() diff --git a/storage/tokudb/PerconaFT/cmake_modules/TokuFeatureDetection.cmake b/storage/tokudb/PerconaFT/cmake_modules/TokuFeatureDetection.cmake index 2f04a33558a..88210868eda 100644 --- a/storage/tokudb/PerconaFT/cmake_modules/TokuFeatureDetection.cmake +++ b/storage/tokudb/PerconaFT/cmake_modules/TokuFeatureDetection.cmake @@ -134,4 +134,4 @@ static __thread int tlsvar = 0; int main(void) { return tlsvar; }" HAVE_GNU_TLS) ## set TOKUDB_REVISION -set(CMAKE_TOKUDB_REVISION 0 CACHE INTEGER "Revision of tokudb.") +set(CMAKE_TOKUDB_REVISION 0 CACHE STRING "Revision of tokudb.") From e64dc07125c8143e27565947e71db76d597dfaf8 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 24 May 2020 21:10:41 +0200 Subject: [PATCH 10/19] assert(a && b); -> assert(a); assert(b); --- mysys/my_bitmap.c | 56 +++++++++++++++++++++++++++------------------ sql/ha_partition.cc | 17 ++++++++------ 2 files changed, 44 insertions(+), 29 deletions(-) diff --git a/mysys/my_bitmap.c b/mysys/my_bitmap.c index c79c5a2d324..f857101af02 100644 --- a/mysys/my_bitmap.c +++ b/mysys/my_bitmap.c @@ -255,7 +255,8 @@ my_bool bitmap_fast_test_and_set(MY_BITMAP *map, uint bitmap_bit) my_bool bitmap_test_and_set(MY_BITMAP *map, uint bitmap_bit) { my_bool res; - DBUG_ASSERT(map->bitmap && bitmap_bit < map->n_bits); + DBUG_ASSERT(map->bitmap); + DBUG_ASSERT(bitmap_bit < map->n_bits); bitmap_lock(map); res= bitmap_fast_test_and_set(map, bitmap_bit); bitmap_unlock(map); @@ -288,7 +289,8 @@ my_bool bitmap_fast_test_and_clear(MY_BITMAP *map, uint bitmap_bit) my_bool bitmap_test_and_clear(MY_BITMAP *map, uint bitmap_bit) { my_bool res; - DBUG_ASSERT(map->bitmap && bitmap_bit < map->n_bits); + DBUG_ASSERT(map->bitmap); + DBUG_ASSERT(bitmap_bit < map->n_bits); bitmap_lock(map); res= bitmap_fast_test_and_clear(map, bitmap_bit); bitmap_unlock(map); @@ -317,8 +319,8 @@ void bitmap_set_prefix(MY_BITMAP *map, uint prefix_size) uint prefix_bytes, prefix_bits, d; uchar *m= (uchar *)map->bitmap; - DBUG_ASSERT(map->bitmap && - (prefix_size <= map->n_bits || prefix_size == (uint) ~0)); + DBUG_ASSERT(map->bitmap); + DBUG_ASSERT(prefix_size <= map->n_bits || prefix_size == (uint) ~0); set_if_smaller(prefix_size, map->n_bits); if ((prefix_bytes= prefix_size / 8)) memset(m, 0xff, prefix_bytes); @@ -340,7 +342,8 @@ my_bool bitmap_is_prefix(const MY_BITMAP *map, uint prefix_size) uchar *m= (uchar*) map->bitmap; uchar *end_prefix= m+(prefix_size-1)/8; uchar *end; - DBUG_ASSERT(m && prefix_size <= map->n_bits); + DBUG_ASSERT(m); + DBUG_ASSERT(prefix_size <= map->n_bits); /* Empty prefix is always true */ if (!prefix_size) @@ -393,8 +396,8 @@ my_bool bitmap_is_subset(const MY_BITMAP *map1, const MY_BITMAP *map2) { my_bitmap_map *m1= map1->bitmap, *m2= map2->bitmap, *end; - DBUG_ASSERT(map1->bitmap && map2->bitmap && - map1->n_bits==map2->n_bits); + DBUG_ASSERT(map1->bitmap && map2->bitmap); + DBUG_ASSERT(map1->n_bits==map2->n_bits); end= map1->last_word_ptr; while (m1 < end) @@ -412,8 +415,9 @@ my_bool bitmap_is_overlapping(const MY_BITMAP *map1, const MY_BITMAP *map2) { my_bitmap_map *m1= map1->bitmap, *m2= map2->bitmap, *end; - DBUG_ASSERT(map1->bitmap && map2->bitmap && - map1->n_bits==map2->n_bits); + DBUG_ASSERT(map1->bitmap); + DBUG_ASSERT(map2->bitmap); + DBUG_ASSERT(map1->n_bits==map2->n_bits); end= map1->last_word_ptr; while (m1 < end) @@ -431,7 +435,8 @@ void bitmap_intersect(MY_BITMAP *map, const MY_BITMAP *map2) my_bitmap_map *to= map->bitmap, *from= map2->bitmap, *end; uint len= no_words_in_map(map), len2 = no_words_in_map(map2); - DBUG_ASSERT(map->bitmap && map2->bitmap); + DBUG_ASSERT(map->bitmap); + DBUG_ASSERT(map2->bitmap); end= to+MY_MIN(len,len2); while (to < end) @@ -476,7 +481,8 @@ my_bool bitmap_exists_intersection(const MY_BITMAP **bitmap_array, uint i, j, start_idx, end_idx; my_bitmap_map cur_res; - DBUG_ASSERT(bitmap_count && end_bit >= start_bit); + DBUG_ASSERT(bitmap_count); + DBUG_ASSERT(end_bit >= start_bit); for (j= 0; j < bitmap_count; j++) DBUG_ASSERT(end_bit < bitmap_array[j]->n_bits); @@ -504,8 +510,9 @@ my_bool bitmap_union_is_set_all(const MY_BITMAP *map1, const MY_BITMAP *map2) { my_bitmap_map *m1= map1->bitmap, *m2= map2->bitmap, *end; - DBUG_ASSERT(map1->bitmap && map2->bitmap && - map1->n_bits==map2->n_bits); + DBUG_ASSERT(map1->bitmap); + DBUG_ASSERT(map2->bitmap); + DBUG_ASSERT(map1->n_bits==map2->n_bits); end= map1->last_word_ptr; while ( m1 < end) if ((*m1++ | *m2++) != 0xFFFFFFFF) @@ -550,8 +557,9 @@ void bitmap_set_above(MY_BITMAP *map, uint from_byte, uint use_bit) void bitmap_subtract(MY_BITMAP *map, const MY_BITMAP *map2) { my_bitmap_map *to= map->bitmap, *from= map2->bitmap, *end; - DBUG_ASSERT(map->bitmap && map2->bitmap && - map->n_bits==map2->n_bits); + DBUG_ASSERT(map->bitmap); + DBUG_ASSERT(map2->bitmap); + DBUG_ASSERT(map->n_bits==map2->n_bits); end= map->last_word_ptr; @@ -564,8 +572,9 @@ void bitmap_union(MY_BITMAP *map, const MY_BITMAP *map2) { my_bitmap_map *to= map->bitmap, *from= map2->bitmap, *end; - DBUG_ASSERT(map->bitmap && map2->bitmap && - map->n_bits==map2->n_bits); + DBUG_ASSERT(map->bitmap); + DBUG_ASSERT(map2->bitmap); + DBUG_ASSERT(map->n_bits == map2->n_bits); end= map->last_word_ptr; while (to <= end) @@ -576,8 +585,9 @@ void bitmap_union(MY_BITMAP *map, const MY_BITMAP *map2) void bitmap_xor(MY_BITMAP *map, const MY_BITMAP *map2) { my_bitmap_map *to= map->bitmap, *from= map2->bitmap, *end= map->last_word_ptr; - DBUG_ASSERT(map->bitmap && map2->bitmap && - map->n_bits==map2->n_bits); + DBUG_ASSERT(map->bitmap); + DBUG_ASSERT(map2->bitmap); + DBUG_ASSERT(map->n_bits == map2->n_bits); while (to <= end) *to++ ^= *from++; } @@ -614,8 +624,9 @@ void bitmap_copy(MY_BITMAP *map, const MY_BITMAP *map2) { my_bitmap_map *to= map->bitmap, *from= map2->bitmap, *end; - DBUG_ASSERT(map->bitmap && map2->bitmap && - map->n_bits==map2->n_bits); + DBUG_ASSERT(map->bitmap); + DBUG_ASSERT(map2->bitmap); + DBUG_ASSERT(map->n_bits == map2->n_bits); end= map->last_word_ptr; while (to <= end) @@ -744,7 +755,8 @@ uint bitmap_lock_set_next(MY_BITMAP *map) void bitmap_lock_clear_bit(MY_BITMAP *map, uint bitmap_bit) { bitmap_lock(map); - DBUG_ASSERT(map->bitmap && bitmap_bit < map->n_bits); + DBUG_ASSERT(map->bitmap); + DBUG_ASSERT(bitmap_bit < map->n_bits); bitmap_clear_bit(map, bitmap_bit); bitmap_unlock(map); } diff --git a/sql/ha_partition.cc b/sql/ha_partition.cc index 15e3a765075..453a1c9ff89 100644 --- a/sql/ha_partition.cc +++ b/sql/ha_partition.cc @@ -2183,7 +2183,8 @@ void ha_partition::update_create_info(HA_CREATE_INFO *create_info) sub_elem= subpart_it++; DBUG_ASSERT(sub_elem); part= i * num_subparts + j; - DBUG_ASSERT(part < m_file_tot_parts && m_file[part]); + DBUG_ASSERT(part < m_file_tot_parts); + DBUG_ASSERT(m_file[part]); dummy_info.data_file_name= dummy_info.index_file_name = NULL; m_file[part]->update_create_info(&dummy_info); sub_elem->data_file_name = (char*) dummy_info.data_file_name; @@ -3770,7 +3771,8 @@ int ha_partition::external_lock(THD *thd, int lock_type) MY_BITMAP *used_partitions; DBUG_ENTER("ha_partition::external_lock"); - DBUG_ASSERT(!auto_increment_lock && !auto_increment_safe_stmt_log_lock); + DBUG_ASSERT(!auto_increment_lock); + DBUG_ASSERT(!auto_increment_safe_stmt_log_lock); if (lock_type == F_UNLCK) used_partitions= &m_locked_partitions; @@ -4034,8 +4036,8 @@ void ha_partition::unlock_row() bool ha_partition::was_semi_consistent_read() { DBUG_ENTER("ha_partition::was_semi_consistent_read"); - DBUG_ASSERT(m_last_part < m_tot_parts && - bitmap_is_set(&(m_part_info->read_partitions), m_last_part)); + DBUG_ASSERT(m_last_part < m_tot_parts); + DBUG_ASSERT(bitmap_is_set(&(m_part_info->read_partitions), m_last_part)); DBUG_RETURN(m_file[m_last_part]->was_semi_consistent_read()); } @@ -5910,8 +5912,8 @@ int ha_partition::partition_scan_set_up(uchar * buf, bool idx_read_flag) DBUG_ASSERT(m_part_spec.start_part < m_tot_parts); m_ordered_scan_ongoing= m_ordered; } - DBUG_ASSERT(m_part_spec.start_part < m_tot_parts && - m_part_spec.end_part < m_tot_parts); + DBUG_ASSERT(m_part_spec.start_part < m_tot_parts); + DBUG_ASSERT(m_part_spec.end_part < m_tot_parts); DBUG_RETURN(0); } @@ -8653,7 +8655,8 @@ void ha_partition::get_auto_increment(ulonglong offset, ulonglong increment, DBUG_PRINT("info", ("offset: %lu inc: %lu desired_values: %lu " "first_value: %lu", (ulong) offset, (ulong) increment, (ulong) nb_desired_values, (ulong) *first_value)); - DBUG_ASSERT(increment && nb_desired_values); + DBUG_ASSERT(increment); + DBUG_ASSERT(nb_desired_values); *first_value= 0; if (table->s->next_number_keypart) { From 9fd8f1b26406747f82913cf97034d9a4eb3d0a2e Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 24 May 2020 17:30:57 +0200 Subject: [PATCH 11/19] mtr: update titlebar when the test ends, not when it starts otherwise it reaches "0 tests left" state and then waits for a few minutes for all workers to complete their tests. show failures. account for retries. --- mysql-test/lib/mtr_report.pm | 60 +++++++++++++++++++++++++++--------- mysql-test/mysql-test-run.pl | 50 +----------------------------- 2 files changed, 46 insertions(+), 64 deletions(-) diff --git a/mysql-test/lib/mtr_report.pm b/mysql-test/lib/mtr_report.pm index e553b0305b4..a58fb369a13 100644 --- a/mysql-test/lib/mtr_report.pm +++ b/mysql-test/lib/mtr_report.pm @@ -37,8 +37,20 @@ use My::Platform; use POSIX qw[ _exit ]; use IO::Handle qw[ flush ]; use mtr_results; - use Term::ANSIColor; +use English; + +my $tot_real_time= 0; +my $tests_done= 0; +my $tests_failed= 0; + +our $timestamp= 0; +our $timediff= 0; +our $name; +our $verbose; +our $verbose_restart= 0; +our $timer= 1; +our $tests_total; my %color_map = qw/pass green retry-pass green @@ -47,20 +59,39 @@ my %color_map = qw/pass green disabled bright_black skipped yellow reset reset/; -sub xterm_color { - if (-t STDOUT and defined $ENV{TERM} and $ENV{TERM} =~ /xterm/) { - syswrite STDOUT, color($color_map{$_[0]}); + +my $set_titlebar; +my $set_color= sub { }; + +if (-t STDOUT) { + if (IS_WINDOWS) { + eval { + require Win32::Console; + $set_titlebar = sub { &Win32::Console::Title($_[0]);}; + } + } elsif ($ENV{TERM} =~ /xterm/) { + $set_titlebar = sub { syswrite STDOUT, "\e]0;$_[0]\a"; }; + $set_color = sub { syswrite STDOUT, color($color_map{$_[0]}); } } } -my $tot_real_time= 0; +sub titlebar_stat($) { -our $timestamp= 0; -our $timediff= 0; -our $name; -our $verbose; -our $verbose_restart= 0; -our $timer= 1; + sub time_format($) { + sprintf '%d:%02d:%02d', $_[0]/3600, ($_[0]/60)%60, $_[0]%60; + } + + $tests_done++; + $tests_failed++ if $_[0] =~ /fail/; + $tests_total++ if $_[0] =~ /retry/; + + my $spent = time - $BASETIME; + my $left = $tests_total - $tests_done; + + &$set_titlebar(sprintf "mtr: spent %s on %d tests. %s (%d tests) left, %d failed", + time_format($spent), $tests_done, + time_format($spent/$tests_done * $left), $left, $tests_failed); +} sub report_option { my ($opt, $value)= @_; @@ -321,8 +352,6 @@ sub mtr_report_stats ($$$$) { if ( $timer ) { - use English; - mtr_report("Spent", sprintf("%.3f", $tot_real_time),"of", time - $BASETIME, "seconds executing testcases"); } @@ -620,10 +649,11 @@ sub mtr_report (@) { my @s = split /\[ (\S+) \]/, _name() . "@_\n"; if (@s > 1) { print $s[0]; - xterm_color($s[1]); + &$set_color($s[1]); print "[ $s[1] ]"; - xterm_color('reset'); + &$set_color('reset'); print $s[2]; + titlebar_stat($s[1]) if $set_titlebar; } else { print $s[0]; } diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index d229390cbb5..29294590ae3 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -369,32 +369,6 @@ my $opt_stop_keep_alive= $ENV{MTR_STOP_KEEP_ALIVE}; select(STDOUT); $| = 1; # Automatically flush STDOUT -my $set_titlebar; - - - BEGIN { - if (IS_WINDOWS) { - my $have_win32_console= 0; - eval { - require Win32::Console; - Win32::Console->import(); - $have_win32_console = 1; - }; - eval 'sub HAVE_WIN32_CONSOLE { $have_win32_console }'; - } else { - eval 'sub HAVE_WIN32_CONSOLE { 0 }'; - } -} - -if (-t STDOUT) { - if (IS_WINDOWS and HAVE_WIN32_CONSOLE) { - $set_titlebar = sub {Win32::Console::Title $_[0];}; - } elsif (defined $ENV{TERM} and $ENV{TERM} =~ /xterm/) { - $set_titlebar = sub { syswrite STDOUT, "\e];$_[0]\a"; }; - } -} - - main(); sub have_wsrep() { @@ -632,7 +606,7 @@ sub main { } ####################################################################### - my $num_tests= @$tests; + my $num_tests= $mtr_report::tests_total= @$tests; if ( $opt_parallel eq "auto" ) { # Try to find a suitable value for number of workers if (IS_WINDOWS) @@ -1073,8 +1047,6 @@ sub run_test_server ($$$) { delete $next->{reserved}; } - titlebar_stat(scalar(@$tests)) if $set_titlebar; - if ($next) { # We don't need this any more delete $next->{criteria}; @@ -6666,23 +6638,3 @@ sub list_options ($) { exit(1); } - -sub time_format($) { - sprintf '%d:%02d:%02d', $_[0]/3600, ($_[0]/60)%60, $_[0]%60; -} - -our $num_tests; - -sub titlebar_stat { - my ($left) = @_; - - # 2.5 -> best by test - $num_tests = $left + 2.5 unless $num_tests; - - my $done = $num_tests - $left; - my $spent = time - $^T; - - &$set_titlebar(sprintf "mtr: spent %s on %d tests. %s (%d tests) left", - time_format($spent), $done, - time_format($spent/$done * $left), $left); -} From 2c9c9acbfc8740dfacc2c0ce88717dfa203b479c Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 26 May 2020 12:11:24 +0200 Subject: [PATCH 12/19] bintars should use bundled PCRE --- cmake/build_configurations/mysql_release.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/build_configurations/mysql_release.cmake b/cmake/build_configurations/mysql_release.cmake index 1609c45a632..9d3785e152b 100644 --- a/cmake/build_configurations/mysql_release.cmake +++ b/cmake/build_configurations/mysql_release.cmake @@ -104,6 +104,7 @@ ELSEIF(DEB) SET(HAVE_EMBEDDED_PRIVILEGE_CONTROL ON) ELSE() SET(WITH_SSL bundled CACHE STRING "") + SET(WITH_PCRE bundled CACHE STRING "") SET(WITH_ZLIB bundled CACHE STRING "") SET(WITH_JEMALLOC static CACHE STRING "") ENDIF() From 8afcc37c685a0b78c3da2b8aa2e5fc70e33e9787 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 26 May 2020 14:11:41 +0200 Subject: [PATCH 13/19] update C/C --- libmariadb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmariadb b/libmariadb index cdfecebc993..ce74fd0c400 160000 --- a/libmariadb +++ b/libmariadb @@ -1 +1 @@ -Subproject commit cdfecebc9932a0dd5516c10505bfe78d79132e7d +Subproject commit ce74fd0c4009ed9f4bcbdb4a01e96c823e961dc3 From e2d7da498231a303854a45455b0b9753af54e3be Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 28 May 2020 01:43:21 +0200 Subject: [PATCH 14/19] Remove unused WiX source file --- win/packaging/custom_ui.wxs | 183 ------------------------------------ 1 file changed, 183 deletions(-) delete mode 100644 win/packaging/custom_ui.wxs diff --git a/win/packaging/custom_ui.wxs b/win/packaging/custom_ui.wxs deleted file mode 100644 index 70fa3ba3abd..00000000000 --- a/win/packaging/custom_ui.wxs +++ /dev/null @@ -1,183 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 1 - - - - - - - 1 - - - - Create default [ProductName] instance - - - - {\WixUI_Font_Title}Default instance properties - - - - - - - Service '[SERVICENAME]' will be removed - - - - Remove default database directory '[DATABASELOCATION]' - - - - - 1 - - - 1 - - - 1 - - - - Remove default [ProductName] database - - - - {\WixUI_Font_Title}Default instance properties - - - - - - - - - - 1 - 1 - - - - WixUI_InstallMode = "Change" - !DBInstance=3 - WixUI_InstallMode = "Remove" - - - - - - - - - - - - - - - - - - SERVICENAME - - - - - - - - - - - - Running mysql_install_db.exe - - - - - - - - - - - - - - - - - - - - - - - - CMDLINE_SERVICENAME - - CMDLINE_DATABASELOCATION - - - - CMDLINE_SERVICENAME - - CMDLINE_DATABASELOCATION - - - \ No newline at end of file From ff72f36948985a0dcd2811e4d8dd66e600badc0e Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 28 May 2020 02:06:23 +0200 Subject: [PATCH 15/19] MSI installer : Use CAQuietExec64 on Win64 , not CAQuietExec It works, but irritates people who look into the log and see traces of 32bit custom action server. --- win/packaging/create_msi.cmake.in | 2 ++ win/packaging/extra.wxs.in | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/win/packaging/create_msi.cmake.in b/win/packaging/create_msi.cmake.in index 21c59dd4f38..c589f98becc 100644 --- a/win/packaging/create_msi.cmake.in +++ b/win/packaging/create_msi.cmake.in @@ -70,10 +70,12 @@ IF(CMAKE_SIZEOF_VOID_P EQUAL 8) SET(Win64 " Win64='yes'") SET(Platform x64) SET(PlatformProgramFilesFolder ProgramFiles64Folder) + SET(CA_QUIET_EXEC CAQuietExec64) ELSE() SET(CANDLE_ARCH -arch x86) SET(Platform x86) SET(PlatformProgramFilesFolder ProgramFilesFolder) + SET(CA_QUIET_EXEC CAQuietExec) SET(Win64) ENDIF() diff --git a/win/packaging/extra.wxs.in b/win/packaging/extra.wxs.in index 3aba129fdf3..4c8e1a5a973 100644 --- a/win/packaging/extra.wxs.in +++ b/win/packaging/extra.wxs.in @@ -666,7 +666,7 @@ - From b00cd3e453f9986d0290b4a7c0ca48c509bdb0ef Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 28 May 2020 15:02:10 +0200 Subject: [PATCH 16/19] MDEV-22743 Windows 10 MSI installer : port in use is not determined when checking for free port, use the same logic (IPv6 socket address / dual socket), like the server would. Previous solution for testing whether port is free was trying to bind IPv4 socket on INADDR_ANY. This not work now on some reason, that attempt succeeds, even if there is an existing IPv6-dual socket listening on 0.0.0.0:3306 --- win/packaging/ca/CustomAction.cpp | 97 ++++++++++++++++++++++++------- 1 file changed, 76 insertions(+), 21 deletions(-) diff --git a/win/packaging/ca/CustomAction.cpp b/win/packaging/ca/CustomAction.cpp index 4005bd6e73b..48d56ba1183 100644 --- a/win/packaging/ca/CustomAction.cpp +++ b/win/packaging/ca/CustomAction.cpp @@ -20,6 +20,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ #undef NOMINMAX #include +#include #include #include #include @@ -33,6 +34,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ #include #include + #define ONE_MB 1048576 UINT ExecRemoveDataDirectory(wchar_t *dir) { @@ -255,36 +257,89 @@ bool ExecRemoveService(const wchar_t *name) return ret; } -/* - Check if port is free by trying to bind to the port -*/ -bool IsPortFree(short port) +/* Find whether TCP port is in use by trying to bind to the port. */ +static bool IsPortInUse(unsigned short port) { - WORD wVersionRequested; - WSADATA wsaData; + struct addrinfo* ai, * a; + struct addrinfo hints {}; - wVersionRequested = MAKEWORD(2, 2); + char port_buf[NI_MAXSERV]; + SOCKET ip_sock = INVALID_SOCKET; + hints.ai_flags = AI_PASSIVE; + hints.ai_socktype = SOCK_STREAM; + hints.ai_family = AF_UNSPEC; + snprintf(port_buf, NI_MAXSERV, "%u", (unsigned)port); - WSAStartup(wVersionRequested, &wsaData); - - struct sockaddr_in sin; - SOCKET sock; - sock = socket(AF_INET, SOCK_STREAM, 0); - if(sock == -1) + if (getaddrinfo(NULL, port_buf, &hints, &ai)) { return false; } - sin.sin_port = htons(port); - sin.sin_addr.s_addr = 0; - sin.sin_addr.s_addr = INADDR_ANY; - sin.sin_family = AF_INET; - if(bind(sock, (struct sockaddr *)&sin,sizeof(struct sockaddr_in) ) == -1) + + /* + Prefer IPv6 socket to IPv4, since we'll use IPv6 dual socket, + which coveres both IP versions. + */ + for (a = ai; a; a = a->ai_next) + { + if (a->ai_family == AF_INET6 && + (ip_sock = socket(a->ai_family, a->ai_socktype, a->ai_protocol)) != INVALID_SOCKET) + { + break; + } + } + + if (ip_sock == INVALID_SOCKET) + { + for (a = ai; a; a = a->ai_next) + { + if (ai->ai_family == AF_INET && + (ip_sock = socket(a->ai_family, a->ai_socktype, a->ai_protocol)) != INVALID_SOCKET) + { + break; + } + } + } + + if (ip_sock == INVALID_SOCKET) { return false; } - closesocket(sock); + + /* Use SO_EXCLUSIVEADDRUSE to prevent multiple binding. */ + int arg = 1; + setsockopt(ip_sock, SOL_SOCKET, SO_EXCLUSIVEADDRUSE, (char*)&arg, sizeof(arg)); + + /* Allow dual socket, so that IPv4 and IPv6 are both covered.*/ + if (a->ai_family == AF_INET6) + { + arg = 0; + setsockopt(ip_sock, IPPROTO_IPV6, IPV6_V6ONLY, (char*)&arg, sizeof(arg)); + } + + bool in_use = false; + if (bind(ip_sock, a->ai_addr, a->ai_addrlen) == SOCKET_ERROR) + { + DWORD last_error = WSAGetLastError(); + in_use = (last_error == WSAEADDRINUSE || last_error == WSAEACCES); + } + + freeaddrinfo(ai); + closesocket(ip_sock); + return in_use; +} + + +/* + Check if TCP port is free +*/ +bool IsPortFree(unsigned short port) +{ + WORD wVersionRequested = MAKEWORD(2, 2); + WSADATA wsaData; + WSAStartup(wVersionRequested, &wsaData); + bool in_use = IsPortInUse(port); WSACleanup(); - return true; + return !in_use; } @@ -634,7 +689,7 @@ extern "C" UINT __stdcall CheckDatabaseProperties (MSIHANDLE hInstall) goto LExit; } - short port = (short)_wtoi(Port); + unsigned short port = (unsigned short)_wtoi(Port); if (!IsPortFree(port)) { ErrorMsg = From 069200267db4cb79bbd55d06cd7ea9e1acd86ce8 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 28 May 2020 15:03:28 +0200 Subject: [PATCH 17/19] Fix cmake warning - custom command succeeds without creating own OUTPUT. --- sql/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 19fdd788207..17779adcff8 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -404,6 +404,7 @@ IF(WIN32 AND MYSQLD_EXECUTABLE) COMMAND ${CMAKE_COMMAND} -E make_directory data COMMAND ${CMAKE_COMMAND} -E chdir data ${CMAKE_COMMAND} ${CONFIG_PARAM} -P ${CMAKE_CURRENT_BINARY_DIR}/create_initial_db.cmake + COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/initdb.dep WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/ DEPENDS mysqld ) From 6e6a4227c050eefda70fd5d4f906e9274cb12f39 Mon Sep 17 00:00:00 2001 From: Kentoku SHIBA Date: Thu, 28 May 2020 10:21:46 +0900 Subject: [PATCH 18/19] Add grn_db_fin_mecab_tokenizer to finalyze mecab tokenizer --- storage/mroonga/vendor/groonga/lib/db.c | 3 ++ .../mroonga/vendor/groonga/lib/tokenizers.c | 30 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/storage/mroonga/vendor/groonga/lib/db.c b/storage/mroonga/vendor/groonga/lib/db.c index cba22aa0e64..f3769f9aa4c 100644 --- a/storage/mroonga/vendor/groonga/lib/db.c +++ b/storage/mroonga/vendor/groonga/lib/db.c @@ -445,6 +445,9 @@ grn_db_close(grn_ctx *ctx, grn_obj *db) ctx_used_db = ctx->impl && ctx->impl->db == db; if (ctx_used_db) { +#ifdef GRN_WITH_MECAB + grn_db_fin_mecab_tokenizer(ctx); +#endif grn_ctx_loader_clear(ctx); if (ctx->impl->parser) { grn_expr_parser_close(ctx); diff --git a/storage/mroonga/vendor/groonga/lib/tokenizers.c b/storage/mroonga/vendor/groonga/lib/tokenizers.c index 11f274e72db..3daacce7ef9 100644 --- a/storage/mroonga/vendor/groonga/lib/tokenizers.c +++ b/storage/mroonga/vendor/groonga/lib/tokenizers.c @@ -797,6 +797,36 @@ grn_db_init_mecab_tokenizer(grn_ctx *ctx) } } +void +grn_db_fin_mecab_tokenizer(grn_ctx *ctx) +{ + switch (GRN_CTX_GET_ENCODING(ctx)) { + case GRN_ENC_EUC_JP : + case GRN_ENC_UTF8 : + case GRN_ENC_SJIS : +#if defined(GRN_EMBEDDED) && defined(GRN_WITH_MECAB) + { + GRN_PLUGIN_DECLARE_FUNCTIONS(tokenizers_mecab); + GRN_PLUGIN_IMPL_NAME_TAGGED(fin, tokenizers_mecab)(ctx); + } +#else /* defined(GRN_EMBEDDED) && defined(GRN_WITH_MECAB) */ + { + const char *mecab_plugin_name = "tokenizers/mecab"; + char *path; + path = grn_plugin_find_path(ctx, mecab_plugin_name); + if (path) { + GRN_FREE(path); + grn_plugin_unregister(ctx, mecab_plugin_name); + } + } +#endif /* defined(GRN_EMBEDDED) && defined(GRN_WITH_MECAB) */ + break; + default : + break; + } + return; +} + #define DEF_TOKENIZER(name, init, next, fin, vars)\ (grn_proc_create(ctx, (name), (sizeof(name) - 1),\ GRN_PROC_TOKENIZER, (init), (next), (fin), 3, (vars))) From 38ea795bb622cce6f7178291ed737ca7396a124a Mon Sep 17 00:00:00 2001 From: Kentoku SHIBA Date: Fri, 29 May 2020 01:51:30 +0900 Subject: [PATCH 19/19] Add a counter to avoid multiple initialization of Groonga mecab tokenizer --- storage/mroonga/vendor/groonga/lib/grn_tokenizers.h | 1 + .../mroonga/vendor/groonga/plugins/tokenizers/mecab.c | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/storage/mroonga/vendor/groonga/lib/grn_tokenizers.h b/storage/mroonga/vendor/groonga/lib/grn_tokenizers.h index e90dbfc0b31..81ac2ab6c46 100644 --- a/storage/mroonga/vendor/groonga/lib/grn_tokenizers.h +++ b/storage/mroonga/vendor/groonga/lib/grn_tokenizers.h @@ -30,6 +30,7 @@ grn_rc grn_tokenizers_init(void); grn_rc grn_tokenizers_fin(void); grn_rc grn_db_init_mecab_tokenizer(grn_ctx *ctx); +void grn_db_fin_mecab_tokenizer(grn_ctx *ctx); grn_rc grn_db_init_builtin_tokenizers(grn_ctx *ctx); #ifdef __cplusplus diff --git a/storage/mroonga/vendor/groonga/plugins/tokenizers/mecab.c b/storage/mroonga/vendor/groonga/plugins/tokenizers/mecab.c index 3dd969a89c5..cabf2c94e53 100644 --- a/storage/mroonga/vendor/groonga/plugins/tokenizers/mecab.c +++ b/storage/mroonga/vendor/groonga/plugins/tokenizers/mecab.c @@ -31,6 +31,7 @@ #include #include +static unsigned int sole_mecab_init_counter = 0; static mecab_t *sole_mecab = NULL; static grn_plugin_mutex *sole_mecab_mutex = NULL; static grn_encoding sole_mecab_encoding = GRN_ENC_NONE; @@ -563,6 +564,11 @@ check_mecab_dictionary_encoding(grn_ctx *ctx) grn_rc GRN_PLUGIN_INIT(grn_ctx *ctx) { + ++sole_mecab_init_counter; + if (sole_mecab_init_counter > 1) + { + return GRN_SUCCESS; + } { char env[GRN_ENV_BUFFER_SIZE]; @@ -636,6 +642,11 @@ GRN_PLUGIN_REGISTER(grn_ctx *ctx) grn_rc GRN_PLUGIN_FIN(grn_ctx *ctx) { + --sole_mecab_init_counter; + if (sole_mecab_init_counter > 0) + { + return GRN_SUCCESS; + } if (sole_mecab) { mecab_destroy(sole_mecab); sole_mecab = NULL;