From 9c3a2a0d8e492f6595d82290fc17f2279dd9707a Mon Sep 17 00:00:00 2001 From: "monty@mysql.com/nosik.monty.fi" <> Date: Tue, 28 Nov 2006 00:47:21 +0200 Subject: [PATCH 1/2] Increased heap max length to > 4G for 64 bit machines Initialize key_part->type on open. This caused key_copy() to fail for bit_fields. (key_copy is used in HANDLER and opt_range) --- include/heap.h | 8 ++++---- mysql-test/r/show_check.result | 2 +- mysql-test/r/type_bit.result | 8 ++++++++ mysql-test/t/type_bit.test | 11 +++++++++++ sql/ha_heap.cc | 2 +- sql/item_sum.cc | 4 ++-- sql/mysqld.cc | 15 +++++++++++---- sql/set_var.cc | 4 ++-- sql/sql_class.h | 17 +++++++++-------- sql/sql_select.cc | 2 +- sql/table.cc | 1 + sql/uniques.cc | 4 ++-- 12 files changed, 53 insertions(+), 25 deletions(-) diff --git a/include/heap.h b/include/heap.h index cfbb6113f86..4255ee52b18 100644 --- a/include/heap.h +++ b/include/heap.h @@ -46,8 +46,8 @@ typedef struct st_heapinfo /* Struct from heap_info */ ulong records; /* Records in database */ ulong deleted; /* Deleted records in database */ ulong max_records; - ulong data_length; - ulong index_length; + ulonglong data_length; + ulonglong index_length; uint reclength; /* Length of one record */ int errkey; ulonglong auto_increment; @@ -135,7 +135,7 @@ typedef struct st_heap_share HP_BLOCK block; HP_KEYDEF *keydef; ulong min_records,max_records; /* Params to open */ - ulong data_length,index_length,max_table_size; + ulonglong data_length,index_length,max_table_size; uint key_stat_version; /* version to indicate insert/delete */ uint records; /* records */ uint blength; /* records rounded up to 2^n */ @@ -187,7 +187,7 @@ typedef struct st_heap_create_info { uint auto_key; /* keynr [1 - maxkey] for auto key */ uint auto_key_type; - ulong max_table_size; + ulonglong max_table_size; ulonglong auto_increment; my_bool with_auto_increment; } HP_CREATE_INFO; diff --git a/mysql-test/r/show_check.result b/mysql-test/r/show_check.result index 28cbdb16919..0dfb7c95f9a 100644 --- a/mysql-test/r/show_check.result +++ b/mysql-test/r/show_check.result @@ -296,7 +296,7 @@ SET sql_quote_show_create= @old_sql_quote_show_create; SET sql_mode= @old_sql_mode; select @@max_heap_table_size; @@max_heap_table_size -1047552 +1048576 CREATE TABLE t1 ( a int(11) default NULL, KEY a USING BTREE (a) diff --git a/mysql-test/r/type_bit.result b/mysql-test/r/type_bit.result index bd58e83bb3f..f3883077919 100644 --- a/mysql-test/r/type_bit.result +++ b/mysql-test/r/type_bit.result @@ -610,4 +610,12 @@ select hex(a), b from t1; hex(a) b 1 2 drop table t1; +create table t1(bit_field bit(2), int_field int, key a(bit_field)); +insert into t1 values (1,2); +handler t1 open as t1; +handler t1 read a=(1); +bit_field int_field + 2 +handler t1 close; +drop table t1; End of 5.0 tests diff --git a/mysql-test/t/type_bit.test b/mysql-test/t/type_bit.test index d46ba667665..48ad24ff6b7 100644 --- a/mysql-test/t/type_bit.test +++ b/mysql-test/t/type_bit.test @@ -261,4 +261,15 @@ insert into t1 (b, a) values ('2', '1'); select hex(a), b from t1; drop table t1; +# +# type was not properly initalized, which caused key_copy to fail +# + +create table t1(bit_field bit(2), int_field int, key a(bit_field)); +insert into t1 values (1,2); +handler t1 open as t1; +handler t1 read a=(1); +handler t1 close; +drop table t1; + --echo End of 5.0 tests diff --git a/sql/ha_heap.cc b/sql/ha_heap.cc index 34ef888a029..3cf7593bd8a 100644 --- a/sql/ha_heap.cc +++ b/sql/ha_heap.cc @@ -631,7 +631,7 @@ int ha_heap::create(const char *name, TABLE *table_arg, } mem_per_row+= MY_ALIGN(share->reclength + 1, sizeof(char*)); max_rows = (ha_rows) (table->in_use->variables.max_heap_table_size / - mem_per_row); + (ulonglong) mem_per_row); if (table_arg->found_next_number_field) { keydef[share->next_number_index].flag|= HA_AUTO_KEY; diff --git a/sql/item_sum.cc b/sql/item_sum.cc index f1e0001c994..de0f713f454 100644 --- a/sql/item_sum.cc +++ b/sql/item_sum.cc @@ -3388,8 +3388,8 @@ bool Item_func_group_concat::setup(THD *thd) duplicate values (according to the syntax of this function). If there is no DISTINCT or ORDER BY clauses, we don't create this tree. */ - init_tree(tree, min(thd->variables.max_heap_table_size, - thd->variables.sortbuff_size/16), 0, + init_tree(tree, (uint) min(thd->variables.max_heap_table_size, + thd->variables.sortbuff_size/16), 0, tree_key_length, compare_key, 0, NULL, (void*) this); } diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 36a5605b507..dd483b0b158 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -70,6 +70,12 @@ #define IF_PURIFY(A,B) (B) #endif +#if SIZEOF_CHARP == 4 +#define MAX_MEM_TABLE_SIZE ~(ulong) 0 +#else +#define MAX_MEM_TABLE_SIZE ~(ulonglong) 0 +#endif + /* stack traces are only supported on linux intel */ #if defined(__linux__) && defined(__i386__) && defined(USE_PSTACK) #define HAVE_STACK_TRACE_ON_SEGV @@ -5718,8 +5724,9 @@ The minimum value for this variable is 4096.", {"max_heap_table_size", OPT_MAX_HEP_TABLE_SIZE, "Don't allow creation of heap tables bigger than this.", (gptr*) &global_system_variables.max_heap_table_size, - (gptr*) &max_system_variables.max_heap_table_size, 0, GET_ULONG, - REQUIRED_ARG, 16*1024*1024L, 16384, ~0L, MALLOC_OVERHEAD, 1024, 0}, + (gptr*) &max_system_variables.max_heap_table_size, 0, GET_ULL, + REQUIRED_ARG, 16*1024*1024L, 16384, MAX_MEM_TABLE_SIZE, + MALLOC_OVERHEAD, 1024, 0}, {"max_join_size", OPT_MAX_JOIN_SIZE, "Joins that are probably going to read more than max_join_size records return an error.", (gptr*) &global_system_variables.max_join_size, @@ -5994,8 +6001,8 @@ The minimum value for this variable is 4096.", {"tmp_table_size", OPT_TMP_TABLE_SIZE, "If an in-memory temporary table exceeds this size, MySQL will automatically convert it to an on-disk MyISAM table.", (gptr*) &global_system_variables.tmp_table_size, - (gptr*) &max_system_variables.tmp_table_size, 0, GET_ULONG, - REQUIRED_ARG, 32*1024*1024L, 1024, ~0L, 0, 1, 0}, + (gptr*) &max_system_variables.tmp_table_size, 0, GET_ULL, + REQUIRED_ARG, 32*1024*1024L, 1024, MAX_MEM_TABLE_SIZE, 0, 1, 0}, {"transaction_alloc_block_size", OPT_TRANS_ALLOC_BLOCK_SIZE, "Allocation block size for transactions to be stored in binary log", (gptr*) &global_system_variables.trans_alloc_block_size, diff --git a/sql/set_var.cc b/sql/set_var.cc index 55c62a9a5a5..c3e93aae619 100644 --- a/sql/set_var.cc +++ b/sql/set_var.cc @@ -248,7 +248,7 @@ sys_var_thd_ulong sys_max_delayed_threads("max_delayed_threads", fix_max_connections); sys_var_thd_ulong sys_max_error_count("max_error_count", &SV::max_error_count); -sys_var_thd_ulong sys_max_heap_table_size("max_heap_table_size", +sys_var_thd_ulonglong sys_max_heap_table_size("max_heap_table_size", &SV::max_heap_table_size); sys_var_thd_ulong sys_pseudo_thread_id("pseudo_thread_id", &SV::pseudo_thread_id, @@ -415,7 +415,7 @@ sys_var_thd_enum sys_tx_isolation("tx_isolation", &SV::tx_isolation, &tx_isolation_typelib, fix_tx_isolation); -sys_var_thd_ulong sys_tmp_table_size("tmp_table_size", +sys_var_thd_ulonglong sys_tmp_table_size("tmp_table_size", &SV::tmp_table_size); sys_var_bool_ptr sys_timed_mutexes("timed_mutexes", &timed_mutexes); diff --git a/sql/sql_class.h b/sql/sql_class.h index 2467385b679..e3ae682edae 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -495,6 +495,8 @@ struct system_variables { ulonglong myisam_max_extra_sort_file_size; ulonglong myisam_max_sort_file_size; + ulonglong max_heap_table_size; + ulonglong tmp_table_size; ha_rows select_limit; ha_rows max_join_size; ulong auto_increment_increment, auto_increment_offset; @@ -503,7 +505,6 @@ struct system_variables ulong long_query_time; ulong max_allowed_packet; ulong max_error_count; - ulong max_heap_table_size; ulong max_length_for_sort_data; ulong max_sort_length; ulong max_tmp_tables; @@ -527,7 +528,6 @@ struct system_variables ulong div_precincrement; ulong sortbuff_size; ulong table_type; - ulong tmp_table_size; ulong tx_isolation; ulong completion_type; /* Determines which non-standard SQL behaviour should be enabled */ @@ -2068,7 +2068,8 @@ class user_var_entry class Unique :public Sql_alloc { DYNAMIC_ARRAY file_ptrs; - ulong max_elements, max_in_memory_size; + ulong max_elements; + ulonglong max_in_memory_size; IO_CACHE file; TREE tree; byte *record_pointers; @@ -2078,7 +2079,7 @@ class Unique :public Sql_alloc public: ulong elements; Unique(qsort_cmp2 comp_func, void *comp_func_fixed_arg, - uint size_arg, ulong max_in_memory_size_arg); + uint size_arg, ulonglong max_in_memory_size_arg); ~Unique(); ulong elements_in_tree() { return tree.elements_in_tree; } inline bool unique_add(void *ptr) @@ -2092,13 +2093,13 @@ public: bool get(TABLE *table); static double get_use_cost(uint *buffer, uint nkeys, uint key_size, - ulong max_in_memory_size); + ulonglong max_in_memory_size); inline static int get_cost_calc_buff_size(ulong nkeys, uint key_size, - ulong max_in_memory_size) + ulonglong max_in_memory_size) { - register ulong max_elems_in_tree= + register ulonglong max_elems_in_tree= (1 + max_in_memory_size / ALIGN_SIZE(sizeof(TREE_ELEMENT)+key_size)); - return sizeof(uint)*(1 + nkeys/max_elems_in_tree); + return (int) (sizeof(uint)*(1 + nkeys/max_elems_in_tree)); } void reset(); diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 1d89cac584a..4758371f2e0 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -9254,7 +9254,7 @@ create_tmp_table(THD *thd,TMP_TABLE_PARAM *param,List &fields, param->recinfo=recinfo; store_record(table,s->default_values); // Make empty default record - if (thd->variables.tmp_table_size == ~(ulong) 0) // No limit + if (thd->variables.tmp_table_size == ~ (ulonglong) 0) // No limit table->s->max_rows= ~(ha_rows) 0; else table->s->max_rows= (((table->s->db_type == DB_TYPE_HEAP) ? diff --git a/sql/table.cc b/sql/table.cc index 5a2281775b1..eed90eb411f 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -730,6 +730,7 @@ int openfrm(THD *thd, const char *name, const char *alias, uint db_stat, if (key_part->fieldnr) { // Should always be true ! Field *field=key_part->field=outparam->field[key_part->fieldnr-1]; + key_part->type= field->key_type(); if (field->null_ptr) { key_part->null_offset=(uint) ((byte*) field->null_ptr - diff --git a/sql/uniques.cc b/sql/uniques.cc index ad074f8b2b0..ac603791376 100644 --- a/sql/uniques.cc +++ b/sql/uniques.cc @@ -55,7 +55,7 @@ int unique_write_to_ptrs(gptr key, element_count count, Unique *unique) } Unique::Unique(qsort_cmp2 comp_func, void * comp_func_fixed_arg, - uint size_arg, ulong max_in_memory_size_arg) + uint size_arg, ulonglong max_in_memory_size_arg) :max_in_memory_size(max_in_memory_size_arg), size(size_arg), elements(0) { my_b_clear(&file); @@ -260,7 +260,7 @@ static double get_merge_many_buffs_cost(uint *buffer, */ double Unique::get_use_cost(uint *buffer, uint nkeys, uint key_size, - ulong max_in_memory_size) + ulonglong max_in_memory_size) { ulong max_elements_in_tree; ulong last_tree_elems; From 0bee93e038edf9a70ffe9d10cd93883a3288802f Mon Sep 17 00:00:00 2001 From: "monty@mysql.com/narttu.mysql.fi" <> Date: Wed, 29 Nov 2006 22:14:08 +0200 Subject: [PATCH 2/2] Ignore some generated files Don't return from my_thread_global_end() until all threads have called my_thread_end() Bug#24387: Valgrind: my_thread_init (handle_sl sql, handle_one_conn, handle_slave_io) --- .bzrignore | 7 +++++++ include/my_pthread.h | 3 +-- mysys/my_thr_init.c | 50 +++++++++++++++++++++++++++++++++++--------- 3 files changed, 48 insertions(+), 12 deletions(-) diff --git a/.bzrignore b/.bzrignore index d6086ba4a76..8455c05b220 100644 --- a/.bzrignore +++ b/.bzrignore @@ -1323,3 +1323,10 @@ win/vs8cache.txt zlib/*.ds? zlib/*.vcproj mysql-test/r/*.warnings +bdb/dist/db.h +bdb/dist/db_config.h +bdb/dist/db_cxx.h +bdb/dist/db_int.h +bdb/dist/include.tcl +*.gcda +*.gcno diff --git a/include/my_pthread.h b/include/my_pthread.h index 3e4388413e0..300291610d3 100644 --- a/include/my_pthread.h +++ b/include/my_pthread.h @@ -677,14 +677,13 @@ struct st_my_thread_var }; extern struct st_my_thread_var *_my_thread_var(void) __attribute__ ((const)); +extern uint my_thread_end_wait_time; #define my_thread_var (_my_thread_var()) #define my_errno my_thread_var->thr_errno /* Keep track of shutdown,signal, and main threads so that my_end() will not report errors with them */ -extern pthread_t shutdown_th, main_th, signal_th; - /* statistics_xxx functions are for not essential statistic */ #ifndef thread_safe_increment diff --git a/mysys/my_thr_init.c b/mysys/my_thr_init.c index 4d23d01cd82..7800edd700d 100644 --- a/mysys/my_thr_init.c +++ b/mysys/my_thr_init.c @@ -30,7 +30,10 @@ pthread_key(struct st_my_thread_var, THR_KEY_mysys); #endif /* USE_TLS */ pthread_mutex_t THR_LOCK_malloc,THR_LOCK_open, THR_LOCK_lock,THR_LOCK_isam,THR_LOCK_myisam,THR_LOCK_heap, - THR_LOCK_net, THR_LOCK_charset; + THR_LOCK_net, THR_LOCK_charset, THR_LOCK_threads; +pthread_cond_t THR_COND_threads; +uint THR_thread_count= 0; +uint my_thread_end_wait_time= 5; #if !defined(HAVE_LOCALTIME_R) || !defined(HAVE_GMTIME_R) pthread_mutex_t LOCK_localtime_r; #endif @@ -79,7 +82,7 @@ my_bool my_thread_global_init(void) #endif #ifdef PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP /* - Set mutex type to "errorcheck" a.k.a "adaptive" + Set mutex type to "errorcheck" */ pthread_mutexattr_init(&my_errorcheck_mutexattr); pthread_mutexattr_settype(&my_errorcheck_mutexattr, @@ -94,6 +97,8 @@ my_bool my_thread_global_init(void) pthread_mutex_init(&THR_LOCK_heap,MY_MUTEX_INIT_FAST); pthread_mutex_init(&THR_LOCK_net,MY_MUTEX_INIT_FAST); pthread_mutex_init(&THR_LOCK_charset,MY_MUTEX_INIT_FAST); + pthread_mutex_init(&THR_LOCK_threads,MY_MUTEX_INIT_FAST); + pthread_cond_init (&THR_COND_threads, NULL); #if defined( __WIN__) || defined(OS2) win_pthread_init(); #endif @@ -114,6 +119,25 @@ my_bool my_thread_global_init(void) void my_thread_global_end(void) { + struct timespec abstime; + set_timespec(abstime, my_thread_end_wait_time); + my_bool all_threads_killed= 1; + + pthread_mutex_lock(&THR_LOCK_threads); + while (THR_thread_count) + { + int error= pthread_cond_timedwait(&THR_COND_threads, &THR_LOCK_threads, + &abstime); + if (error == ETIMEDOUT || error == ETIME) + { + if (THR_thread_count) + fprintf(stderr,"error in my_thread_global_end(): %d threads didn't exit\n", + THR_thread_count); + all_threads_killed= 0; + } + } + pthread_mutex_unlock(&THR_LOCK_threads); + pthread_key_delete(THR_KEY_mysys); #ifdef PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP pthread_mutexattr_destroy(&my_fast_mutexattr); @@ -129,6 +153,11 @@ void my_thread_global_end(void) pthread_mutex_destroy(&THR_LOCK_heap); pthread_mutex_destroy(&THR_LOCK_net); pthread_mutex_destroy(&THR_LOCK_charset); + if (all_threads_killed) + { + pthread_mutex_destroy(&THR_LOCK_threads); + pthread_cond_destroy (&THR_COND_threads); + } #if !defined(HAVE_LOCALTIME_R) || !defined(HAVE_GMTIME_R) pthread_mutex_destroy(&LOCK_localtime_r); #endif @@ -154,9 +183,6 @@ my_bool my_thread_init(void) #ifdef EXTRA_DEBUG_THREADS fprintf(stderr,"my_thread_init(): thread_id=%ld\n",pthread_self()); #endif -#if !defined(__WIN__) || defined(USE_TLS) || ! defined(SAFE_MUTEX) - pthread_mutex_lock(&THR_LOCK_lock); -#endif #if !defined(__WIN__) || defined(USE_TLS) if (my_pthread_getspecific(struct st_my_thread_var *,THR_KEY_mysys)) @@ -174,7 +200,7 @@ my_bool my_thread_init(void) } pthread_setspecific(THR_KEY_mysys,tmp); -#else +#else /* defined(__WIN__) && !(defined(USE_TLS) */ /* Skip initialization if the thread specific variable is already initialized */ @@ -182,7 +208,6 @@ my_bool my_thread_init(void) goto end; tmp= &THR_KEY_mysys; #endif - tmp->id= ++thread_id; #if defined(__WIN__) && defined(EMBEDDED_LIBRARY) tmp->thread_self= (pthread_t)getpid(); #endif @@ -190,10 +215,11 @@ my_bool my_thread_init(void) pthread_cond_init(&tmp->suspend, NULL); tmp->init= 1; + pthread_mutex_lock(&THR_LOCK_threads); + tmp->id= ++thread_id; + ++THR_thread_count; + pthread_mutex_unlock(&THR_LOCK_threads); end: -#if !defined(__WIN__) || defined(USE_TLS) || ! defined(SAFE_MUTEX) - pthread_mutex_unlock(&THR_LOCK_lock); -#endif return error; } @@ -232,6 +258,10 @@ void my_thread_end(void) #if (!defined(__WIN__) && !defined(OS2)) || defined(USE_TLS) pthread_setspecific(THR_KEY_mysys,0); #endif + pthread_mutex_lock(&THR_LOCK_threads); + if (--THR_thread_count == 0) + pthread_cond_signal(&THR_COND_threads); + pthread_mutex_unlock(&THR_LOCK_threads); } struct st_my_thread_var *_my_thread_var(void)