From 636bb59034fd13d2e1b27af42adbb01ddeb3bd82 Mon Sep 17 00:00:00 2001 From: Monty Date: Thu, 28 Apr 2016 17:15:38 +0300 Subject: [PATCH] Final fixes for Memory_used - Change some static variables to dynamic to ensure that we don't do any memory allocations before server starts or stops - Print more memory information on SIGHUP. Fixed output. - Write out if memory was lost if run with --debug-at-exit - Fixed wrong #ifdef in sql_cache.cc --- plugin/wsrep_info/plugin.cc | 14 ++++----- sql/log.cc | 6 ++++ sql/log.h | 1 + sql/mysqld.cc | 14 ++++++--- sql/mysqld.h | 1 + sql/rpl_gtid.cc | 3 +- sql/rpl_gtid.h | 3 +- sql/rpl_rli.cc | 1 + sql/sql_cache.cc | 2 +- sql/sql_class.cc | 1 - sql/sql_repl.cc | 1 + sql/sql_test.cc | 60 +++++++++++++++++++++---------------- sql/wsrep_mysqld.cc | 15 +++++++--- sql/wsrep_sst.cc | 2 +- sql/wsrep_utils.h | 2 +- 15 files changed, 78 insertions(+), 48 deletions(-) diff --git a/plugin/wsrep_info/plugin.cc b/plugin/wsrep_info/plugin.cc index b8aaee5a132..f9843cb6f39 100644 --- a/plugin/wsrep_info/plugin.cc +++ b/plugin/wsrep_info/plugin.cc @@ -120,10 +120,10 @@ static int wsrep_memb_fill_table(THD *thd, TABLE_LIST *tables, COND *cond) if (check_global_access(thd, SUPER_ACL, true)) return rc; - wsrep_config_state.lock(); + wsrep_config_state->lock(); Dynamic_array *memb_arr= - wsrep_config_state.get_member_info(); + wsrep_config_state->get_member_info(); TABLE *table= tables->table; @@ -151,7 +151,7 @@ static int wsrep_memb_fill_table(THD *thd, TABLE_LIST *tables, COND *cond) } end: - wsrep_config_state.unlock(); + wsrep_config_state->unlock(); return rc; } @@ -175,10 +175,10 @@ static int wsrep_status_fill_table(THD *thd, TABLE_LIST *tables, COND *cond) if (check_global_access(thd, SUPER_ACL, true)) return rc; - wsrep_config_state.lock(); + wsrep_config_state->lock(); - wsrep_view_info_t view= wsrep_config_state.get_view_info(); - wsrep_member_status_t status= wsrep_config_state.get_status(); + wsrep_view_info_t view= wsrep_config_state->get_view_info(); + wsrep_member_status_t status= wsrep_config_state->get_status(); TABLE *table= tables->table; @@ -210,7 +210,7 @@ static int wsrep_status_fill_table(THD *thd, TABLE_LIST *tables, COND *cond) if (schema_table_store_record(thd, table)) rc= 1; - wsrep_config_state.unlock(); + wsrep_config_state->unlock(); return rc; } diff --git a/sql/log.cc b/sql/log.cc index dc8c08bfd36..f04b1432229 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -139,6 +139,12 @@ static bool start_binlog_background_thread(); static rpl_binlog_state rpl_global_gtid_binlog_state; +void setup_log_handling() +{ + rpl_global_gtid_binlog_state.init(); +} + + /** purge logs, master and slave sides both, related error code convertor. diff --git a/sql/log.h b/sql/log.h index 9bf80d6e603..e556ef91399 100644 --- a/sql/log.h +++ b/sql/log.h @@ -26,6 +26,7 @@ class Relay_log_info; class Format_description_log_event; +void setup_log_handling(); bool trans_has_updated_trans_table(const THD* thd); bool stmt_has_updated_trans_table(const THD *thd); bool use_trans_cache(const THD* thd, bool is_transactional); diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 9843a1bf67c..845d114bc7a 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -367,7 +367,7 @@ static my_bool opt_short_log_format= 0, opt_silent_startup= 0; uint kill_cached_threads; static uint wake_thread; ulong max_used_connections; -static volatile ulong cached_thread_count= 0; +volatile ulong cached_thread_count= 0; static char *mysqld_user, *mysqld_chroot; static char *default_character_set_name; static char *character_set_filesystem_name; @@ -1987,8 +1987,6 @@ static void __cdecl kill_server(int sig_ptr) if (wsrep_inited == 1) wsrep_deinit(true); - wsrep_thr_deinit(); - if (sig != MYSQL_KILL_SIGNAL && sig != 0) unireg_abort(1); /* purecov: inspected */ @@ -2144,6 +2142,9 @@ static void mysqld_exit(int exit_code) set_malloc_size_cb(NULL); cleanup_tls(); DBUG_LEAVE; + if (opt_endinfo && global_status_var.global_memory_used) + fprintf(stderr, "Warning: Memory not freed: %ld\n", + (long) global_status_var.global_memory_used); sd_notify(0, "STATUS=MariaDB server is down"); exit(exit_code); /* purecov: inspected */ } @@ -2213,11 +2214,12 @@ void clean_up(bool print_message) free_global_client_stats(); free_global_table_stats(); free_global_index_stats(); - delete_dynamic(&all_options); + delete_dynamic(&all_options); // This should be empty free_all_rpl_filters(); #ifdef HAVE_REPLICATION end_slave_list(); #endif + wsrep_thr_deinit(); my_uuid_end(); delete binlog_filter; delete global_rpl_filter; @@ -5032,6 +5034,8 @@ static int init_server_components() /* Setup logs */ + setup_log_handling(); + /* Enable old-fashioned error log, except when the user has requested help information. Since the implementation of plugin server @@ -5202,7 +5206,9 @@ static int init_server_components() /* It's now safe to use thread specific memory */ mysqld_server_initialized= 1; +#ifndef EMBEDDED_LIBRARY wsrep_thr_init(); +#endif if (WSREP_ON && !wsrep_recovery && !opt_abort) /* WSREP BEFORE SE */ { diff --git a/sql/mysqld.h b/sql/mysqld.h index ef4a0d6a47a..e538cbd724e 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -203,6 +203,7 @@ extern LEX_CSTRING reason_slave_blocked; extern ulong slave_trans_retries; extern uint slave_net_timeout; extern int max_user_connections; +extern volatile ulong cached_thread_count; extern ulong what_to_log,flush_time; extern ulong max_prepared_stmt_count, prepared_stmt_count; extern ulong open_files_limit; diff --git a/sql/rpl_gtid.cc b/sql/rpl_gtid.cc index f54ef2b0081..5a94e1c5b54 100644 --- a/sql/rpl_gtid.cc +++ b/sql/rpl_gtid.cc @@ -1127,7 +1127,7 @@ rpl_slave_state::is_empty() } -rpl_binlog_state::rpl_binlog_state() +void rpl_binlog_state::init() { my_hash_init(&hash, &my_charset_bin, 32, offsetof(element, domain_id), sizeof(uint32), NULL, my_free, HASH_UNIQUE); @@ -1137,7 +1137,6 @@ rpl_binlog_state::rpl_binlog_state() initialized= 1; } - void rpl_binlog_state::reset_nolock() { diff --git a/sql/rpl_gtid.h b/sql/rpl_gtid.h index ece6effbef6..f638a084e38 100644 --- a/sql/rpl_gtid.h +++ b/sql/rpl_gtid.h @@ -231,9 +231,10 @@ struct rpl_binlog_state /* Auxiliary buffer to sort gtid list. */ DYNAMIC_ARRAY gtid_sort_array; - rpl_binlog_state(); + rpl_binlog_state() :initialized(0) {} ~rpl_binlog_state(); + void init(); void reset_nolock(); void reset(); void free(); diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc index 2dd6f7d7afc..f9d0910a2a7 100644 --- a/sql/rpl_rli.cc +++ b/sql/rpl_rli.cc @@ -70,6 +70,7 @@ Relay_log_info::Relay_log_info(bool is_slave_recovery) DBUG_ENTER("Relay_log_info::Relay_log_info"); relay_log.is_relay_log= TRUE; + relay_log_state.init(); #ifdef HAVE_PSI_INTERFACE relay_log.set_psi_keys(key_RELAYLOG_LOCK_index, key_RELAYLOG_update_cond, diff --git a/sql/sql_cache.cc b/sql/sql_cache.cc index 91dd8ad7325..42b80d9b143 100644 --- a/sql/sql_cache.cc +++ b/sql/sql_cache.cc @@ -354,7 +354,7 @@ const uchar *query_state_map; #include "emb_qcache.h" #endif -#if !defined(EXTRA_DBUG) && !defined(DBUG_OFF) +#if defined(EXTRA_DEBUG) && !defined(DBUG_OFF) #define RW_WLOCK(M) {DBUG_PRINT("lock", ("rwlock wlock 0x%lx",(ulong)(M))); \ if (!mysql_rwlock_wrlock(M)) DBUG_PRINT("lock", ("rwlock wlock ok")); \ else DBUG_PRINT("lock", ("rwlock wlock FAILED %d", errno)); } diff --git a/sql/sql_class.cc b/sql/sql_class.cc index e189fd7e582..b1217cb1f9f 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -1733,7 +1733,6 @@ THD::~THD() other types are handled explicitely */ - void add_to_status(STATUS_VAR *to_var, STATUS_VAR *from_var) { ulong *end= (ulong*) ((uchar*) to_var + diff --git a/sql/sql_repl.cc b/sql/sql_repl.cc index c9e2b3a586d..36f0cd84cbf 100644 --- a/sql/sql_repl.cc +++ b/sql/sql_repl.cc @@ -182,6 +182,7 @@ struct binlog_send_info { { error_text[0] = 0; bzero(&error_gtid, sizeof(error_gtid)); + until_binlog_state.init(); } }; diff --git a/sql/sql_test.cc b/sql/sql_test.cc index 642cf208908..86f7dc268b9 100644 --- a/sql/sql_test.cc +++ b/sql/sql_test.cc @@ -54,7 +54,7 @@ static const char *lock_descriptions[] = }; -#ifndef DBUG_OFF +#ifdef EXTRA_DEBUG void print_where(COND *cond,const char *info, enum_query_type query_type) @@ -559,16 +559,19 @@ C_MODE_END void mysql_print_status() { char current_dir[FN_REFLEN]; + char llbuff[10][22]; STATUS_VAR tmp; + uint count; - calc_sum_of_all_status(&tmp); + count= calc_sum_of_all_status(&tmp); printf("\nStatus information:\n\n"); (void) my_getwd(current_dir, sizeof(current_dir),MYF(0)); printf("Current dir: %s\n", current_dir); - printf("Running threads: %d Stack size: %ld\n", thread_count, + printf("Running threads: %d Cached threads: %lu Stack size: %ld\n", + count, cached_thread_count, (long) my_thread_stack_size); +#ifdef EXTRA_DEBUG thr_print_locks(); // Write some debug info -#ifndef DBUG_OFF print_cached_tables(); #endif /* Print key cache status */ @@ -614,28 +617,33 @@ Next alarm time: %lu\n", #ifdef HAVE_MALLINFO struct mallinfo info= mallinfo(); printf("\nMemory status:\n\ -Non-mmapped space allocated from system: %d\n\ -Number of free chunks: %d\n\ -Number of fastbin blocks: %d\n\ -Number of mmapped regions: %d\n\ -Space in mmapped regions: %d\n\ -Maximum total allocated space: %d\n\ -Space available in freed fastbin blocks: %d\n\ -Total allocated space: %d\n\ -Total free space: %d\n\ -Top-most, releasable space: %d\n\ -Estimated memory (with thread stack): %ld\n", - (int) info.arena , - (int) info.ordblks, - (int) info.smblks, - (int) info.hblks, - (int) info.hblkhd, - (int) info.usmblks, - (int) info.fsmblks, - (int) info.uordblks, - (int) info.fordblks, - (int) info.keepcost, - (long) (thread_count * my_thread_stack_size + info.hblkhd + info.arena)); +Non-mmapped space allocated from system: %s\n\ +Number of free chunks: %lu\n\ +Number of fastbin blocks: %lu\n\ +Number of mmapped regions: %lu\n\ +Space in mmapped regions: %s\n\ +Maximum total allocated space: %s\n\ +Space available in freed fastbin blocks: %s\n\ +Total allocated space: %s\n\ +Total free space: %s\n\ +Top-most, releasable space: %s\n\ +Estimated memory (with thread stack): %s\n\ +Global memory allocated by server: %s\n\ +Memory allocated by threads: %s\n", + llstr(info.arena, llbuff[0]), + (ulong) info.ordblks, + (ulong) info.smblks, + (ulong) info.hblks, + llstr(info.hblkhd, llbuff[1]), + llstr(info.usmblks, llbuff[2]), + llstr(info.fsmblks, llbuff[3]), + llstr(info.uordblks, llbuff[4]), + llstr(info.fordblks, llbuff[5]), + llstr(info.keepcost, llbuff[6]), + llstr((count + cached_thread_count)* my_thread_stack_size + info.hblkhd + info.arena, llbuff[7]), + llstr(tmp.global_memory_used, llbuff[8]), + llstr(tmp.local_memory_used, llbuff[9])); + #endif #ifdef HAVE_EVENT_SCHEDULER diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc index f84ebe4dbb9..3cdf95e2bf2 100644 --- a/sql/wsrep_mysqld.cc +++ b/sql/wsrep_mysqld.cc @@ -202,7 +202,7 @@ wsrep_uuid_t local_uuid = WSREP_UUID_UNDEFINED; wsrep_seqno_t local_seqno = WSREP_SEQNO_UNDEFINED; long wsrep_protocol_version = 3; -wsp::Config_state wsrep_config_state; +wsp::Config_state *wsrep_config_state; // Boolean denoting if server is in initial startup phase. This is needed // to make sure that main thread waiting in wsrep_sst_wait() is signaled @@ -281,7 +281,7 @@ wsrep_view_handler_cb (void* app_ctx, *sst_req = NULL; *sst_req_len = 0; - wsrep_member_status_t memb_status= wsrep_config_state.get_status(); + wsrep_member_status_t memb_status= wsrep_config_state->get_status(); if (memcmp(&cluster_uuid, &view->state_id.uuid, sizeof(wsrep_uuid_t))) { @@ -442,7 +442,7 @@ wsrep_view_handler_cb (void* app_ctx, out: if (view->status == WSREP_VIEW_PRIMARY) wsrep_startup= FALSE; - wsrep_config_state.set(memb_status, view); + wsrep_config_state->set(memb_status, view); return WSREP_CB_SUCCESS; } @@ -484,7 +484,7 @@ static void wsrep_synced_cb(void* app_ctx) signal_main= true; } - wsrep_config_state.set(WSREP_MEMBER_SYNCED); + wsrep_config_state->set(WSREP_MEMBER_SYNCED); mysql_mutex_unlock (&LOCK_wsrep_ready); if (signal_main) @@ -753,6 +753,8 @@ done: /* Initialize wsrep thread LOCKs and CONDs */ void wsrep_thr_init() { + DBUG_ENTER("wsrep_thr_init"); + wsrep_config_state = new wsp::Config_state; #ifdef HAVE_PSI_INTERFACE mysql_mutex_register("sql", wsrep_mutexes, array_elements(wsrep_mutexes)); mysql_cond_register("sql", wsrep_conds, array_elements(wsrep_conds)); @@ -772,6 +774,7 @@ void wsrep_thr_init() mysql_mutex_init(key_LOCK_wsrep_slave_threads, &LOCK_wsrep_slave_threads, MY_MUTEX_INIT_FAST); mysql_mutex_init(key_LOCK_wsrep_desync, &LOCK_wsrep_desync, MY_MUTEX_INIT_FAST); mysql_mutex_init(key_LOCK_wsrep_config_state, &LOCK_wsrep_config_state, MY_MUTEX_INIT_FAST); + DBUG_VOID_RETURN; } @@ -819,6 +822,8 @@ void wsrep_deinit(bool free_options) /* Destroy wsrep thread LOCKs and CONDs */ void wsrep_thr_deinit() { + if (!wsrep_config_state) + return; // Never initialized mysql_mutex_destroy(&LOCK_wsrep_ready); mysql_cond_destroy(&COND_wsrep_ready); mysql_mutex_destroy(&LOCK_wsrep_sst); @@ -832,6 +837,8 @@ void wsrep_thr_deinit() mysql_mutex_destroy(&LOCK_wsrep_slave_threads); mysql_mutex_destroy(&LOCK_wsrep_desync); mysql_mutex_destroy(&LOCK_wsrep_config_state); + delete wsrep_config_state; + wsrep_config_state= 0; // Safety } void wsrep_recover() diff --git a/sql/wsrep_sst.cc b/sql/wsrep_sst.cc index 6d04527cbcb..562bc7effb4 100644 --- a/sql/wsrep_sst.cc +++ b/sql/wsrep_sst.cc @@ -1256,7 +1256,7 @@ wsrep_cb_status_t wsrep_sst_donate_cb (void* app_ctx, void* recv_ctx, /* This will be reset when sync callback is called. * Should we set wsrep_ready to FALSE here too? */ - wsrep_config_state.set(WSREP_MEMBER_DONOR); + wsrep_config_state->set(WSREP_MEMBER_DONOR); const char* method = (char*)msg; size_t method_len = strlen (method); diff --git a/sql/wsrep_utils.h b/sql/wsrep_utils.h index ed699eabec9..54235cf5273 100644 --- a/sql/wsrep_utils.h +++ b/sql/wsrep_utils.h @@ -233,7 +233,7 @@ private: } /* namespace wsp */ -extern wsp::Config_state wsrep_config_state; +extern wsp::Config_state *wsrep_config_state; namespace wsp { /* a class to manage env vars array */