From a93ac8d95fd504412bd819325bfaa816e317786c Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 13 Nov 2018 13:10:32 +0100 Subject: [PATCH] MDEV-16448 mysql_upgrade_service remove my.ini variables that are no more valid MDEV-16447 incorporate Innodb slow shutdown into mysql_upgrade_service.exe --- sql/CMakeLists.txt | 4 +- sql/mysql_upgrade_service.cc | 211 +++++++++++++++++++++--------- sql/upgrade_conf_file.cc | 177 +++++++++++++++++++++++++ win/upgrade_wizard/upgradeDlg.cpp | 25 ++-- 4 files changed, 344 insertions(+), 73 deletions(-) create mode 100644 sql/upgrade_conf_file.cc diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index d85e588abeb..c6910f469f9 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -489,9 +489,11 @@ IF(WIN32) ADD_LIBRARY(winservice STATIC winservice.c) TARGET_LINK_LIBRARIES(winservice shell32) + MYSQL_ADD_EXECUTABLE(mysql_upgrade_service mysql_upgrade_service.cc - COMPONENT Server) + upgrade_conf_file.cc + COMPONENT Server) TARGET_LINK_LIBRARIES(mysql_upgrade_service mysys winservice) ENDIF(WIN32) diff --git a/sql/mysql_upgrade_service.cc b/sql/mysql_upgrade_service.cc index 1a3df4f5536..9ea78accf44 100644 --- a/sql/mysql_upgrade_service.cc +++ b/sql/mysql_upgrade_service.cc @@ -29,6 +29,9 @@ #include #include +#include + +extern int upgrade_config_file(const char *myini_path); /* We're using version APIs */ #pragma comment(lib, "version") @@ -47,6 +50,8 @@ static char mysqlupgrade_path[MAX_PATH]; static char defaults_file_param[MAX_PATH + 16]; /*--defaults-file= */ static char logfile_path[MAX_PATH]; +char my_ini_bck[MAX_PATH]; +mysqld_service_properties service_properties; static char *opt_service; static SC_HANDLE service; static SC_HANDLE scm; @@ -59,7 +64,7 @@ HANDLE logfile_handle; Maybe,they can be made parameters */ static unsigned int startup_timeout= 60; -static unsigned int shutdown_timeout= 60; +static unsigned int shutdown_timeout= 60*60; static struct my_option my_long_options[]= { @@ -112,6 +117,7 @@ static void die(const char *fmt, ...) fprintf(stderr, "FATAL ERROR: "); vfprintf(stderr, fmt, args); + fputc('\n', stderr); if (logfile_path[0]) { fprintf(stderr, "Additional information can be found in the log file %s", @@ -122,6 +128,11 @@ static void die(const char *fmt, ...) fflush(stdout); /* Cleanup */ + if (my_ini_bck[0]) + { + MoveFileEx(my_ini_bck, service_properties.inifile,MOVEFILE_REPLACE_EXISTING); + } + /* Stop service that we started, if it was not initally running at program start. @@ -309,77 +320,76 @@ void initiate_mysqld_shutdown() } } +static void get_service_config() +{ + scm = OpenSCManager(NULL, NULL, SC_MANAGER_ALL_ACCESS); + if (!scm) + die("OpenSCManager failed with %u", GetLastError()); + service = OpenService(scm, opt_service, SERVICE_ALL_ACCESS); + if (!service) + die("OpenService failed with %u", GetLastError()); + BYTE config_buffer[8 * 1024]; + LPQUERY_SERVICE_CONFIGW config = (LPQUERY_SERVICE_CONFIGW)config_buffer; + DWORD size = sizeof(config_buffer); + DWORD needed; + if (!QueryServiceConfigW(service, config, size, &needed)) + die("QueryServiceConfig failed with %u", GetLastError()); + + if (get_mysql_service_properties(config->lpBinaryPathName, &service_properties)) + { + die("Not a valid MySQL service"); + } + + int my_major = MYSQL_VERSION_ID / 10000; + int my_minor = (MYSQL_VERSION_ID % 10000) / 100; + int my_patch = MYSQL_VERSION_ID % 100; + + if (my_major < service_properties.version_major || + (my_major == service_properties.version_major && my_minor < service_properties.version_minor)) + { + die("Can not downgrade, the service is currently running as version %d.%d.%d" + ", my version is %d.%d.%d", service_properties.version_major, service_properties.version_minor, + service_properties.version_patch, my_major, my_minor, my_patch); + } + if (service_properties.inifile[0] == 0) + { + /* + Weird case, no --defaults-file in service definition, need to create one. + */ + sprintf_s(service_properties.inifile, MAX_PATH, "%s\\my.ini", service_properties.datadir); + } + sprintf(defaults_file_param, "--defaults-file=%s", service_properties.inifile); +} /* Change service configuration (binPath) to point to mysqld from this installation. */ static void change_service_config() { - char defaults_file[MAX_PATH]; char default_character_set[64]; char buf[MAX_PATH]; - char commandline[3*MAX_PATH + 19]; + char commandline[3 * MAX_PATH + 19]; int i; - scm= OpenSCManager(NULL, NULL, SC_MANAGER_ALL_ACCESS); - if(!scm) - die("OpenSCManager failed with %u", GetLastError()); - service= OpenService(scm, opt_service, SERVICE_ALL_ACCESS); - if (!service) - die("OpenService failed with %u", GetLastError()); - - BYTE config_buffer[8*1024]; - LPQUERY_SERVICE_CONFIGW config= (LPQUERY_SERVICE_CONFIGW)config_buffer; - DWORD size= sizeof(config_buffer); - DWORD needed; - if (!QueryServiceConfigW(service, config, size, &needed)) - die("QueryServiceConfig failed with %u", GetLastError()); - - mysqld_service_properties props; - if (get_mysql_service_properties(config->lpBinaryPathName, &props)) - { - die("Not a valid MySQL service"); - } - - int my_major= MYSQL_VERSION_ID/10000; - int my_minor= (MYSQL_VERSION_ID %10000)/100; - int my_patch= MYSQL_VERSION_ID%100; - - if(my_major < props.version_major || - (my_major == props.version_major && my_minor < props.version_minor)) - { - die("Can not downgrade, the service is currently running as version %d.%d.%d" - ", my version is %d.%d.%d", props.version_major, props.version_minor, - props.version_patch, my_major, my_minor, my_patch); - } - - if(props.inifile[0] == 0) - { - /* - Weird case, no --defaults-file in service definition, need to create one. - */ - sprintf_s(props.inifile, MAX_PATH, "%s\\my.ini", props.datadir); - } - /* Write datadir to my.ini, after converting backslashes to unix style slashes. */ - strcpy_s(buf, MAX_PATH, props.datadir); + strcpy_s(buf, MAX_PATH, service_properties.datadir); for(i= 0; buf[i]; i++) { if (buf[i] == '\\') buf[i]= '/'; } - WritePrivateProfileString("mysqld", "datadir",buf, props.inifile); + WritePrivateProfileString("mysqld", "datadir",buf, service_properties.inifile); /* Remove basedir from defaults file, otherwise the service wont come up in the new version, and will complain about mismatched message file. */ - WritePrivateProfileString("mysqld", "basedir",NULL, props.inifile); + WritePrivateProfileString("mysqld", "basedir",NULL, service_properties.inifile); /* Replace default-character-set with character-set-server, to avoid @@ -397,7 +407,7 @@ static void change_service_config() default_character_set, defaults_file); } - sprintf(defaults_file_param,"--defaults-file=%s", props.inifile); + sprintf(defaults_file_param,"--defaults-file=%s", service_properties.inifile); sprintf_s(commandline, "\"%s\" \"%s\" \"%s\"", mysqld_path, defaults_file_param, opt_service); if (!ChangeServiceConfig(service, SERVICE_NO_CHANGE, SERVICE_NO_CHANGE, @@ -449,23 +459,97 @@ int main(int argc, char **argv) reads them from pipe and uses as progress indicator. */ setvbuf(stdout, NULL, _IONBF, 0); + int phase = 0; + int max_phases=10; + get_service_config(); - log("Phase 1/8: Changing service configuration"); - change_service_config(); + bool my_ini_exists; + bool old_mysqld_exe_exists; - log("Phase 2/8: Stopping service"); + log("Phase %d/%d: Stopping service", ++phase,max_phases); stop_mysqld_service(); + my_ini_exists = (GetFileAttributes(service_properties.inifile) != INVALID_FILE_ATTRIBUTES); + if (!my_ini_exists) + { + HANDLE h = CreateFile(service_properties.inifile, GENERIC_WRITE, FILE_SHARE_READ|FILE_SHARE_WRITE, + 0, CREATE_NEW, 0 ,0); + if (h != INVALID_HANDLE_VALUE) + { + CloseHandle(h); + } + else if (GetLastError() != ERROR_FILE_EXISTS) + { + die("Can't create ini file %s, last error %u", service_properties.inifile, GetLastError()); + } + } + + old_mysqld_exe_exists = (GetFileAttributes(service_properties.mysqld_exe) != INVALID_FILE_ATTRIBUTES); + log("Phase %d/%d: Fixing server config file%s", ++phase, max_phases, my_ini_exists ? "" : "(skipped)"); + + snprintf(my_ini_bck, sizeof(my_ini_bck), "%s.BCK", service_properties.inifile); + CopyFile(service_properties.inifile, my_ini_bck, FALSE); + upgrade_config_file(service_properties.inifile); + + log("Phase %d/%d: Ensuring innodb slow shutdown%s", ++phase, max_phases, + old_mysqld_exe_exists?",this can take some time":"(skipped)"); + + char socket_param[FN_REFLEN]; + sprintf_s(socket_param, "--socket=mysql_upgrade_service_%d", + GetCurrentProcessId()); + + DWORD start_duration_ms = 0; + + if (old_mysqld_exe_exists) + { + /* Start/stop server with --loose-innodb-fast-shutdown=0 */ + mysqld_process = (HANDLE)run_tool(P_NOWAIT, service_properties.mysqld_exe, + defaults_file_param, "--loose-innodb-fast-shutdown=0", "--skip-networking", + "--enable-named-pipe", socket_param, "--skip-slave-start", NULL); + + if (mysqld_process == INVALID_HANDLE_VALUE) + { + die("Cannot start mysqld.exe process, last error =%u", GetLastError()); + } + char pipe_name[64]; + snprintf(pipe_name, sizeof(pipe_name), "\\\\.\\pipe\\mysql_upgrade_service_%u", + GetCurrentProcessId()); + for (;;) + { + if (WaitForSingleObject(mysqld_process, 0) != WAIT_TIMEOUT) + die("mysqld.exe did not start"); + + if (WaitNamedPipe(pipe_name, 0)) + { + // Server started, shut it down. + initiate_mysqld_shutdown(); + if (WaitForSingleObject((HANDLE)mysqld_process, shutdown_timeout * 1000) != WAIT_OBJECT_0) + { + die("Could not shutdown server started with '--innodb-fast-shutdown=0'"); + } + DWORD exit_code; + if (!GetExitCodeProcess((HANDLE)mysqld_process, &exit_code)) + { + die("Could not get mysqld's exit code"); + } + if (exit_code) + { + die("Could not get successfully shutdown mysqld"); + } + CloseHandle(mysqld_process); + break; + } + Sleep(500); + start_duration_ms += 500; + } + } /* Start mysqld.exe as non-service skipping privileges (so we do not care about the password). But disable networking and enable pipe for communication, for security reasons. */ - char socket_param[FN_REFLEN]; - sprintf_s(socket_param,"--socket=mysql_upgrade_service_%d", - GetCurrentProcessId()); - log("Phase 3/8: Starting mysqld for upgrade"); + log("Phase %d/%d: Starting mysqld for upgrade",++phase,max_phases); mysqld_process= (HANDLE)run_tool(P_NOWAIT, mysqld_path, defaults_file_param, "--skip-networking", "--skip-grant-tables", "--enable-named-pipe", socket_param,"--skip-slave-start", NULL); @@ -475,8 +559,8 @@ int main(int argc, char **argv) die("Cannot start mysqld.exe process, errno=%d", errno); } - log("Phase 4/8: Waiting for startup to complete"); - DWORD start_duration_ms= 0; + log("Phase %d/%d: Waiting for startup to complete",++phase,max_phases); + start_duration_ms= 0; for(;;) { if (WaitForSingleObject(mysqld_process, 0) != WAIT_TIMEOUT) @@ -493,7 +577,7 @@ int main(int argc, char **argv) start_duration_ms+= 500; } - log("Phase 5/8: Running mysql_upgrade"); + log("Phase %d/%d: Running mysql_upgrade",++phase,max_phases); int upgrade_err= (int) run_tool(P_WAIT, mysqlupgrade_path, "--protocol=pipe", "--force", socket_param, NULL); @@ -501,10 +585,13 @@ int main(int argc, char **argv) if (upgrade_err) die("mysql_upgrade failed with error code %d\n", upgrade_err); - log("Phase 6/8: Initiating server shutdown"); + log("Phase %d/%d: Changing service configuration", ++phase, max_phases); + change_service_config(); + + log("Phase %d/%d: Initiating server shutdown",++phase, max_phases); initiate_mysqld_shutdown(); - log("Phase 7/8: Waiting for shutdown to complete"); + log("Phase %d/%d: Waiting for shutdown to complete",++phase, max_phases); if (WaitForSingleObject(mysqld_process, shutdown_timeout*1000) != WAIT_OBJECT_0) { @@ -514,7 +601,7 @@ int main(int argc, char **argv) CloseHandle(mysqld_process); mysqld_process= NULL; - log("Phase 8/8: Starting service%s", + log("Phase %d/%d: Starting service%s",++phase,max_phases, (initial_service_state == SERVICE_RUNNING)?"":" (skipped)"); if (initial_service_state == SERVICE_RUNNING) { @@ -527,6 +614,10 @@ int main(int argc, char **argv) CloseServiceHandle(scm); if (logfile_handle) CloseHandle(logfile_handle); + if(my_ini_bck[0]) + { + DeleteFile(my_ini_bck); + } my_end(0); exit(0); } diff --git a/sql/upgrade_conf_file.cc b/sql/upgrade_conf_file.cc new file mode 100644 index 00000000000..4e167f0263f --- /dev/null +++ b/sql/upgrade_conf_file.cc @@ -0,0 +1,177 @@ +/* + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ + + +/* + Variables that were present in older releases, but are now removed. + to get the list of variables that are present in current release + execute + + SELECT LOWER(variable_name) from INFORMATION_SCHEMA.GLOBAL_VARIABLES ORDER BY 1 + + Compare the list between releases to figure out which variables have gone. + + Note : the list below only includes the default-compiled server and none of the + loadable plugins. +*/ +#include +#include +#include +#include +#include + +static const char *removed_variables[] = +{ +"aria_recover", +"debug_crc_break", +"engine_condition_pushdown", +"have_csv", +"have_innodb", +"have_ndbcluster", +"have_partitioning", +"innodb_adaptive_flushing_method", +"innodb_adaptive_hash_index_partitions", +"innodb_additional_mem_pool_size", +"innodb_api_bk_commit_interval", +"innodb_api_disable_rowlock", +"innodb_api_enable_binlog", +"innodb_api_enable_mdl", +"innodb_api_trx_level", +"innodb_blocking_buffer_pool_restore", +"innodb_buffer_pool_populate", +"innodb_buffer_pool_restore_at_startup", +"innodb_buffer_pool_shm_checksum", +"innodb_buffer_pool_shm_key", +"innodb_checkpoint_age_target", +"innodb_cleaner_eviction_factor", +"innodb_cleaner_flush_chunk_size", +"innodb_cleaner_free_list_lwm", +"innodb_cleaner_lru_chunk_size", +"innodb_cleaner_lsn_age_factor", +"innodb_cleaner_max_flush_time", +"innodb_cleaner_max_lru_time", +"innodb_corrupt_table_action", +"innodb_dict_size_limit", +"innodb_doublewrite_file", +"innodb_empty_free_list_algorithm", +"innodb_fake_changes", +"innodb_fast_checksum", +"innodb_file_format", +"innodb_file_format_check", +"innodb_file_format_max", +"innodb_flush_neighbor_pages", +"innodb_foreground_preflush", +"innodb_ibuf_accel_rate", +"innodb_ibuf_active_contract", +"innodb_ibuf_max_size", +"innodb_import_table_from_xtrabackup", +"innodb_instrument_semaphores", +"innodb_kill_idle_transaction", +"innodb_large_prefix", +"innodb_lazy_drop_table", +"innodb_locking_fake_changes", +"innodb_log_arch_dir", +"innodb_log_arch_expire_sec", +"innodb_log_archive", +"innodb_log_block_size", +"innodb_log_checksum_algorithm", +"innodb_max_bitmap_file_size", +"innodb_max_changed_pages", +"innodb_merge_sort_block_size", +"innodb_mirrored_log_groups", +"innodb_mtflush_threads", +"innodb_persistent_stats_root_page", +"innodb_print_lock_wait_timeout_info", +"innodb_purge_run_now", +"innodb_purge_stop_now", +"innodb_read_ahead", +"innodb_recovery_stats", +"innodb_recovery_update_relay_log", +"innodb_show_locks_held", +"innodb_show_verbose_locks", +"innodb_stats_auto_update", +"innodb_stats_update_need_lock", +"innodb_support_xa", +"innodb_thread_concurrency_timer_based", +"innodb_track_changed_pages", +"innodb_track_redo_log_now", +"innodb_use_fallocate", +"innodb_use_global_flush_log_at_trx_commit", +"innodb_use_mtflush", +"innodb_use_stacktrace", +"innodb_use_sys_malloc", +"innodb_use_sys_stats_table", +"innodb_use_trim", +"log", +"log_slow_queries", +"rpl_recovery_rank", +"sql_big_tables", +"sql_low_priority_updates", +"sql_max_join_size" +}; + + +static int cmp_strings(const void* a, const void *b) +{ + return strcmp((const char *)a, *(const char **)b); +} + +/** + Convert file from a previous version, by removing +*/ +int upgrade_config_file(const char *myini_path) +{ +#define MY_INI_SECTION_SIZE 32*1024 +3 + static char section_data[MY_INI_SECTION_SIZE]; + for (const char *section_name : { "mysqld","server","mariadb" }) + { + DWORD size = GetPrivateProfileSection(section_name, section_data, MY_INI_SECTION_SIZE, myini_path); + if (size == MY_INI_SECTION_SIZE - 2) + { + return -1; + } + + for (char *keyval = section_data; *keyval; keyval += strlen(keyval) + 1) + { + char varname[256]; + char *key_end = strchr(keyval, '='); + if (!key_end) + key_end = keyval+ strlen(keyval); + + if (key_end - keyval > sizeof(varname)) + continue; + // copy and normalize (convert dash to underscore) to variable names + for (char *p = keyval, *q = varname;; p++,q++) + { + if (p == key_end) + { + *q = 0; + break; + } + *q = (*p == '-') ? '_' : *p; + } + const char *v = (const char *)bsearch(varname, removed_variables, sizeof(removed_variables) / sizeof(removed_variables[0]), + sizeof(char *), cmp_strings); + + if (v) + { + fprintf(stdout, "Removing variable '%s' from config file\n", varname); + // delete variable + *key_end = 0; + WritePrivateProfileString(section_name, keyval, 0, myini_path); + } + } + } + return 0; +} diff --git a/win/upgrade_wizard/upgradeDlg.cpp b/win/upgrade_wizard/upgradeDlg.cpp index d996c0ebe5d..793e89886d6 100644 --- a/win/upgrade_wizard/upgradeDlg.cpp +++ b/win/upgrade_wizard/upgradeDlg.cpp @@ -422,21 +422,22 @@ void CUpgradeDlg::UpgradeOneService(const string& servicename) { allMessages[lines%MAX_MESSAGES] = output_line; m_DataDir.SetWindowText(allMessages[lines%MAX_MESSAGES].c_str()); - output_line.clear(); lines++; - /* - Updating progress dialog.There are currently 9 messages from - mysql_upgrade_service (actually it also writes Phase N/M but - we do not parse the output right now). - */ -#define EXPRECTED_MYSQL_UPGRADE_MESSAGES 9 + int curPhase, numPhases; - int stepsTotal= m_ProgressTotal*EXPRECTED_MYSQL_UPGRADE_MESSAGES; - int stepsCurrent= m_ProgressCurrent*EXPRECTED_MYSQL_UPGRADE_MESSAGES - + lines; - int percentDone= stepsCurrent*100/stepsTotal; - m_Progress.SetPos(percentDone); + // Parse output line to update progress indicator + if (strncmp(output_line.c_str(),"Phase ",6) == 0 && + sscanf(output_line.c_str() +6 ,"%d/%d",&curPhase,&numPhases) == 2 + && numPhases > 0 ) + { + int stepsTotal= m_ProgressTotal*numPhases; + int stepsCurrent= m_ProgressCurrent*numPhases+ curPhase; + int percentDone= stepsCurrent*100/stepsTotal; + m_Progress.SetPos(percentDone); + m_Progress.SetPos(stepsCurrent * 100 / stepsTotal); + } + output_line.clear(); } else {