From 7c6cf7fefe68a1a3f68e7d6436da4689ec302bca Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 25 Jan 2018 14:25:48 +0100 Subject: [PATCH 01/31] bug: ha_heap was unilaterally increasing reclength proper fix replacing the hack from b80fa4000d6 don't confuse length of the data area (reclength) with the offset to the "deleted" mark. --- include/heap.h | 1 + storage/heap/_check.c | 2 +- storage/heap/ha_heap.cc | 11 +---------- storage/heap/hp_create.c | 8 +++++--- storage/heap/hp_delete.c | 2 +- storage/heap/hp_rrnd.c | 4 ++-- storage/heap/hp_rsame.c | 2 +- storage/heap/hp_scan.c | 2 +- storage/heap/hp_write.c | 4 ++-- 9 files changed, 15 insertions(+), 21 deletions(-) diff --git a/include/heap.h b/include/heap.h index 2b7fd28699d..eecb7084e66 100644 --- a/include/heap.h +++ b/include/heap.h @@ -144,6 +144,7 @@ typedef struct st_heap_share uint key_version; /* Updated on key change */ uint file_version; /* Update on clear */ uint reclength; /* Length of one record */ + uint visible; /* Offset to the visible/deleted mark */ uint changed; uint keys,max_key_length; uint currently_disabled_keys; /* saved value from "keys" when disabled */ diff --git a/storage/heap/_check.c b/storage/heap/_check.c index b64c9ab1831..f2fd8ab94be 100644 --- a/storage/heap/_check.c +++ b/storage/heap/_check.c @@ -79,7 +79,7 @@ int heap_check_heap(HP_INFO *info, my_bool print_status) } hp_find_record(info,pos); - if (!info->current_ptr[share->reclength]) + if (!info->current_ptr[share->visible]) deleted++; else records++; diff --git a/storage/heap/ha_heap.cc b/storage/heap/ha_heap.cc index 259e54bfc59..ec76d08bf97 100644 --- a/storage/heap/ha_heap.cc +++ b/storage/heap/ha_heap.cc @@ -100,15 +100,6 @@ const char **ha_heap::bas_ext() const int ha_heap::open(const char *name, int mode, uint test_if_locked) { - if (table->s->reclength < sizeof (char*)) - { - MEM_UNDEFINED(table->s->default_values + table->s->reclength, - sizeof(char*) - table->s->reclength); - table->s->reclength= sizeof(char*); - MEM_UNDEFINED(table->record[0], table->s->reclength); - MEM_UNDEFINED(table->record[1], table->s->reclength); - } - internal_table= test(test_if_locked & HA_OPEN_INTERNAL_TABLE); if (internal_table || (!(file= heap_open(name, mode)) && my_errno == ENOENT)) { @@ -736,7 +727,7 @@ heap_prepare_hp_create_info(TABLE *table_arg, bool internal_table, } } } - mem_per_row+= MY_ALIGN(share->reclength + 1, sizeof(char*)); + mem_per_row+= MY_ALIGN(max(share->reclength, sizeof(char*)) + 1, sizeof(char*)); if (table_arg->found_next_number_field) { keydef[share->next_number_index].flag|= HA_AUTO_KEY; diff --git a/storage/heap/hp_create.c b/storage/heap/hp_create.c index 0b5dc841ada..1daca0beeb5 100644 --- a/storage/heap/hp_create.c +++ b/storage/heap/hp_create.c @@ -33,6 +33,7 @@ int heap_create(const char *name, HP_CREATE_INFO *create_info, uint keys= create_info->keys; ulong min_records= create_info->min_records; ulong max_records= create_info->max_records; + uint visible_offset; DBUG_ENTER("heap_create"); if (!create_info->internal_table) @@ -58,9 +59,9 @@ int heap_create(const char *name, HP_CREATE_INFO *create_info, /* We have to store sometimes uchar* del_link in records, - so the record length should be at least sizeof(uchar*) + so the visible_offset must be least at sizeof(uchar*) */ - set_if_bigger(reclength, sizeof (uchar*)); + visible_offset= max(reclength, sizeof (char*)); for (i= key_segs= max_length= 0, keyinfo= keydef; i < keys; i++, keyinfo++) { @@ -152,7 +153,7 @@ int heap_create(const char *name, HP_CREATE_INFO *create_info, share->keydef= (HP_KEYDEF*) (share + 1); share->key_stat_version= 1; keyseg= (HA_KEYSEG*) (share->keydef + keys); - init_block(&share->block, reclength + 1, min_records, max_records); + init_block(&share->block, visible_offset + 1, min_records, max_records); /* Fix keys */ memcpy(share->keydef, keydef, (size_t) (sizeof(keydef[0]) * keys)); for (i= 0, keyinfo= share->keydef; i < keys; i++, keyinfo++) @@ -192,6 +193,7 @@ int heap_create(const char *name, HP_CREATE_INFO *create_info, share->max_table_size= create_info->max_table_size; share->data_length= share->index_length= 0; share->reclength= reclength; + share->visible= visible_offset; share->blength= 1; share->keys= keys; share->max_key_length= max_length; diff --git a/storage/heap/hp_delete.c b/storage/heap/hp_delete.c index 1cbfe7408d4..eb9749601aa 100644 --- a/storage/heap/hp_delete.c +++ b/storage/heap/hp_delete.c @@ -45,7 +45,7 @@ int heap_delete(HP_INFO *info, const uchar *record) info->update=HA_STATE_DELETED; *((uchar**) pos)=share->del_link; share->del_link=pos; - pos[share->reclength]=0; /* Record deleted */ + pos[share->visible]=0; /* Record deleted */ share->deleted++; share->key_version++; #if !defined(DBUG_OFF) && defined(EXTRA_HEAP_DEBUG) diff --git a/storage/heap/hp_rrnd.c b/storage/heap/hp_rrnd.c index 8e0d51a78ca..d105c5c2e13 100644 --- a/storage/heap/hp_rrnd.c +++ b/storage/heap/hp_rrnd.c @@ -37,7 +37,7 @@ int heap_rrnd(register HP_INFO *info, uchar *record, uchar *pos) info->update= 0; DBUG_RETURN(my_errno= HA_ERR_END_OF_FILE); } - if (!info->current_ptr[share->reclength]) + if (!info->current_ptr[share->visible]) { info->update= HA_STATE_PREV_FOUND | HA_STATE_NEXT_FOUND; DBUG_RETURN(my_errno=HA_ERR_RECORD_DELETED); @@ -91,7 +91,7 @@ int heap_rrnd_old(register HP_INFO *info, uchar *record, ulong pos) hp_find_record(info, pos); end: - if (!info->current_ptr[share->reclength]) + if (!info->current_ptr[share->visible]) { info->update= HA_STATE_PREV_FOUND | HA_STATE_NEXT_FOUND; DBUG_RETURN(my_errno=HA_ERR_RECORD_DELETED); diff --git a/storage/heap/hp_rsame.c b/storage/heap/hp_rsame.c index 40c5a18f974..19767fd752b 100644 --- a/storage/heap/hp_rsame.c +++ b/storage/heap/hp_rsame.c @@ -32,7 +32,7 @@ int heap_rsame(register HP_INFO *info, uchar *record, int inx) DBUG_ENTER("heap_rsame"); test_active(info); - if (info->current_ptr[share->reclength]) + if (info->current_ptr[share->visible]) { if (inx < -1 || inx >= (int) share->keys) { diff --git a/storage/heap/hp_scan.c b/storage/heap/hp_scan.c index 39a6f2082a3..65726c37e1f 100644 --- a/storage/heap/hp_scan.c +++ b/storage/heap/hp_scan.c @@ -62,7 +62,7 @@ int heap_scan(register HP_INFO *info, uchar *record) } hp_find_record(info, pos); } - if (!info->current_ptr[share->reclength]) + if (!info->current_ptr[share->visible]) { DBUG_PRINT("warning",("Found deleted record")); info->update= HA_STATE_PREV_FOUND | HA_STATE_NEXT_FOUND; diff --git a/storage/heap/hp_write.c b/storage/heap/hp_write.c index e45c78b75f1..4a9c1ba59c0 100644 --- a/storage/heap/hp_write.c +++ b/storage/heap/hp_write.c @@ -54,7 +54,7 @@ int heap_write(HP_INFO *info, const uchar *record) } memcpy(pos,record,(size_t) share->reclength); - pos[share->reclength]=1; /* Mark record as not deleted */ + pos[share->visible]= 1; /* Mark record as not deleted */ if (++share->records == share->blength) share->blength+= share->blength; info->s->key_version++; @@ -92,7 +92,7 @@ err: share->deleted++; *((uchar**) pos)=share->del_link; share->del_link=pos; - pos[share->reclength]=0; /* Record deleted */ + pos[share->visible]= 0; /* Record deleted */ DBUG_RETURN(my_errno); } /* heap_write */ From c8afe7daaced3339f24b88d63f6bcf8477dfaf8c Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 5 Feb 2018 14:13:26 +0100 Subject: [PATCH 02/31] cleanup: remove a duplicated test case --- mysql-test/r/view.result | 108 ------------------------------------- mysql-test/t/view.test | 112 --------------------------------------- 2 files changed, 220 deletions(-) diff --git a/mysql-test/r/view.result b/mysql-test/r/view.result index 8bc33a8860b..1d12c50954f 100644 --- a/mysql-test/r/view.result +++ b/mysql-test/r/view.result @@ -5236,114 +5236,6 @@ execute stmt1; deallocate prepare stmt1; drop view v1,v2; drop table t1,t2; -# -# MDEV-6251: SIGSEGV in query optimizer (in set_check_materialized -# with MERGE view) -# -CREATE TABLE t1 (a1 INT(11) NOT NULL DEFAULT NULL AUTO_INCREMENT PRIMARY KEY); -CREATE TABLE t2 (b1 INT(11) NOT NULL DEFAULT NULL AUTO_INCREMENT PRIMARY KEY); -CREATE TABLE t3 (c1 INT(11) NOT NULL DEFAULT NULL AUTO_INCREMENT PRIMARY KEY); -CREATE TABLE t4 (d1 INT(11) NOT NULL DEFAULT NULL AUTO_INCREMENT PRIMARY KEY); -CREATE TABLE t5 (e1 INT(11) NOT NULL DEFAULT NULL AUTO_INCREMENT PRIMARY KEY); -CREATE TABLE t6 (f1 INT(11) NOT NULL DEFAULT NULL AUTO_INCREMENT PRIMARY KEY); -CREATE OR REPLACE view v1 AS -SELECT 1 -FROM t1 a_alias_1 -LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 -LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 -LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 -LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -; -SELECT 1 -FROM (( SELECT 1 -FROM t1 a_alias_1 -LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 -LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 -LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 -LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -) t1) -LEFT OUTER JOIN (( SELECT 1 -FROM t1 a_alias_1 -LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 -LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 -LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 -LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -) t2) ON 1=1 -LEFT OUTER JOIN (( SELECT 1 -FROM t1 a_alias_1 -LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 -LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 -LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 -LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -) t3) ON 1=1 -LEFT OUTER JOIN (( SELECT 1 -FROM t1 a_alias_1 -LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 -LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 -LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 -LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -) t4) ON 1=1 -LEFT OUTER JOIN (( SELECT 1 -FROM t1 a_alias_1 -LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 -LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 -LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 -LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -) t5) ON 1=1 -LEFT OUTER JOIN (( SELECT 1 -FROM t1 a_alias_1 -LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 -LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 -LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 -LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -) t6) ON 1=1 -LEFT OUTER JOIN (( SELECT 1 -FROM t1 a_alias_1 -LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 -LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 -LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 -LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -) t7) ON 1=1 -LEFT OUTER JOIN (( SELECT 1 -FROM t1 a_alias_1 -LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 -LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 -LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 -LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -) t8) ON 1=1 -; -1 -SELECT 1 -FROM (v1 t1) -LEFT OUTER JOIN (v1 t2) ON 1=1 -LEFT OUTER JOIN (v1 t3) ON 1=1 -LEFT OUTER JOIN (v1 t4) ON 1=1 -LEFT OUTER JOIN (v1 t5) ON 1=1 -LEFT OUTER JOIN (v1 t6) ON 1=1 -LEFT OUTER JOIN (v1 t7) ON 1=1 -LEFT OUTER JOIN (v1 t8) ON 1=1 -; -1 -drop view v1; -drop table t1,t2,t3,t4,t5,t6; # ----------------------------------------------------------------- # -- End of 5.3 tests. # ----------------------------------------------------------------- diff --git a/mysql-test/t/view.test b/mysql-test/t/view.test index 0fc7cb6adf7..4b3537df2de 100644 --- a/mysql-test/t/view.test +++ b/mysql-test/t/view.test @@ -5169,118 +5169,6 @@ deallocate prepare stmt1; drop view v1,v2; drop table t1,t2; ---echo # ---echo # MDEV-6251: SIGSEGV in query optimizer (in set_check_materialized ---echo # with MERGE view) ---echo # - -CREATE TABLE t1 (a1 INT(11) NOT NULL DEFAULT NULL AUTO_INCREMENT PRIMARY KEY); -CREATE TABLE t2 (b1 INT(11) NOT NULL DEFAULT NULL AUTO_INCREMENT PRIMARY KEY); -CREATE TABLE t3 (c1 INT(11) NOT NULL DEFAULT NULL AUTO_INCREMENT PRIMARY KEY); -CREATE TABLE t4 (d1 INT(11) NOT NULL DEFAULT NULL AUTO_INCREMENT PRIMARY KEY); -CREATE TABLE t5 (e1 INT(11) NOT NULL DEFAULT NULL AUTO_INCREMENT PRIMARY KEY); -CREATE TABLE t6 (f1 INT(11) NOT NULL DEFAULT NULL AUTO_INCREMENT PRIMARY KEY); - -CREATE OR REPLACE view v1 AS - SELECT 1 - FROM t1 a_alias_1 - LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 - LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 - LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 - LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -; - -SELECT 1 -FROM (( SELECT 1 - FROM t1 a_alias_1 - LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 - LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 - LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 - LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -) t1) -LEFT OUTER JOIN (( SELECT 1 - FROM t1 a_alias_1 - LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 - LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 - LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 - LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -) t2) ON 1=1 -LEFT OUTER JOIN (( SELECT 1 - FROM t1 a_alias_1 - LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 - LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 - LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 - LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -) t3) ON 1=1 -LEFT OUTER JOIN (( SELECT 1 - FROM t1 a_alias_1 - LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 - LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 - LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 - LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -) t4) ON 1=1 -LEFT OUTER JOIN (( SELECT 1 - FROM t1 a_alias_1 - LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 - LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 - LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 - LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -) t5) ON 1=1 -LEFT OUTER JOIN (( SELECT 1 - FROM t1 a_alias_1 - LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 - LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 - LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 - LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -) t6) ON 1=1 -LEFT OUTER JOIN (( SELECT 1 - FROM t1 a_alias_1 - LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 - LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 - LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 - LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -) t7) ON 1=1 -LEFT OUTER JOIN (( SELECT 1 - FROM t1 a_alias_1 - LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 - LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 - LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 - LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -) t8) ON 1=1 -; - -SELECT 1 -FROM (v1 t1) -LEFT OUTER JOIN (v1 t2) ON 1=1 -LEFT OUTER JOIN (v1 t3) ON 1=1 -LEFT OUTER JOIN (v1 t4) ON 1=1 -LEFT OUTER JOIN (v1 t5) ON 1=1 -LEFT OUTER JOIN (v1 t6) ON 1=1 -LEFT OUTER JOIN (v1 t7) ON 1=1 -LEFT OUTER JOIN (v1 t8) ON 1=1 -; - -drop view v1; -drop table t1,t2,t3,t4,t5,t6; - --echo # ----------------------------------------------------------------- --echo # -- End of 5.3 tests. --echo # ----------------------------------------------------------------- From 2709380587bbcbc7abda77f11ee0abd207c65027 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 14 Feb 2018 18:14:24 +0100 Subject: [PATCH 03/31] MDEV-13748 Assertion `status_var.local_memory_used == 0 || !debug_assert_on_not_freed_memory' failed in virtual THD::~THD after query with INTERSECT my_safe_alloca()/my_safe_afree() work as alloca() or malloc()/free() depending on the memory size to allocate, that is, depending on reclength here. They only work correctly if reclength doesn't change in the middle. --- mysql-test/suite/maria/dynamic.result | 4 ++++ mysql-test/suite/maria/dynamic.test | 7 +++++++ storage/maria/ma_dynrec.c | 10 +++++----- 3 files changed, 16 insertions(+), 5 deletions(-) create mode 100644 mysql-test/suite/maria/dynamic.result create mode 100644 mysql-test/suite/maria/dynamic.test diff --git a/mysql-test/suite/maria/dynamic.result b/mysql-test/suite/maria/dynamic.result new file mode 100644 index 00000000000..1e87010e9ca --- /dev/null +++ b/mysql-test/suite/maria/dynamic.result @@ -0,0 +1,4 @@ +create table t1 (a blob, b varchar(20000)) engine=aria row_format=dynamic; +insert t1 (b) values (repeat('a', 20000)); +update t1 set b='b'; +drop table t1; diff --git a/mysql-test/suite/maria/dynamic.test b/mysql-test/suite/maria/dynamic.test new file mode 100644 index 00000000000..f8a1e98cd41 --- /dev/null +++ b/mysql-test/suite/maria/dynamic.test @@ -0,0 +1,7 @@ +# +# MDEV-13748 Assertion `status_var.local_memory_used == 0 || !debug_assert_on_not_freed_memory' failed in virtual THD::~THD after query with INTERSECT +# +create table t1 (a blob, b varchar(20000)) engine=aria row_format=dynamic; +insert t1 (b) values (repeat('a', 20000)); +update t1 set b='b'; +drop table t1; diff --git a/storage/maria/ma_dynrec.c b/storage/maria/ma_dynrec.c index c47da42b555..35a5040f09d 100644 --- a/storage/maria/ma_dynrec.c +++ b/storage/maria/ma_dynrec.c @@ -276,7 +276,7 @@ my_bool _ma_update_blob_record(MARIA_HA *info, MARIA_RECORD_POS pos, { uchar *rec_buff; int error; - ulong reclength,extra; + ulong reclength,reclength2,extra; extra= (ALIGN_SIZE(MARIA_MAX_DYN_BLOCK_HEADER)+MARIA_SPLIT_LENGTH+ MARIA_DYN_DELETE_BLOCK_HEADER); @@ -289,17 +289,17 @@ my_bool _ma_update_blob_record(MARIA_HA *info, MARIA_RECORD_POS pos, return 1; } #endif - if (!(rec_buff=(uchar*) my_safe_alloca(reclength, - MARIA_MAX_RECORD_ON_STACK))) + if (!(rec_buff=(uchar*) my_safe_alloca(reclength, MARIA_MAX_RECORD_ON_STACK))) { my_errno= HA_ERR_OUT_OF_MEM; /* purecov: inspected */ return(1); } - reclength= _ma_rec_pack(info,rec_buff+ALIGN_SIZE(MARIA_MAX_DYN_BLOCK_HEADER), + reclength2= _ma_rec_pack(info,rec_buff+ALIGN_SIZE(MARIA_MAX_DYN_BLOCK_HEADER), record); + DBUG_ASSERT(reclength2 <= reclength); error=update_dynamic_record(info,pos, rec_buff+ALIGN_SIZE(MARIA_MAX_DYN_BLOCK_HEADER), - reclength); + reclength2); my_safe_afree(rec_buff, reclength, MARIA_MAX_RECORD_ON_STACK); return(error != 0); } From 03de234baf423eaefe3d0b768dccb719b1298ff6 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 14 Feb 2018 19:12:23 +0100 Subject: [PATCH 04/31] MDEV-13982 Server crashes in in ha_partition::engine_name use the correct handlerton when looking for TRANSACTIONAL=1 support --- mysql-test/suite/parts/r/partition_alter_maria.result | 9 +++++++++ mysql-test/suite/parts/t/partition_alter_maria.test | 7 +++++++ sql/sql_table.cc | 2 +- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/mysql-test/suite/parts/r/partition_alter_maria.result b/mysql-test/suite/parts/r/partition_alter_maria.result index 460d20b9255..d79bc0a41fe 100644 --- a/mysql-test/suite/parts/r/partition_alter_maria.result +++ b/mysql-test/suite/parts/r/partition_alter_maria.result @@ -16,6 +16,15 @@ select * from t1; pk dt 1 2017-09-28 15:12:00 drop table t1; +create table t1 (a int) engine=Aria transactional=1 partition by hash(a) partitions 2; +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL +) ENGINE=Aria DEFAULT CHARSET=latin1 TRANSACTIONAL=1 +/*!50100 PARTITION BY HASH (a) +PARTITIONS 2 */ +drop table t1; # # MDEV-13788 Server crash when issuing bad SQL partition syntax # diff --git a/mysql-test/suite/parts/t/partition_alter_maria.test b/mysql-test/suite/parts/t/partition_alter_maria.test index e21f0dfab82..e0b9256391d 100644 --- a/mysql-test/suite/parts/t/partition_alter_maria.test +++ b/mysql-test/suite/parts/t/partition_alter_maria.test @@ -17,5 +17,12 @@ alter table t1 drop partition p20181231; select * from t1; drop table t1; +# +# MDEV-13982 Server crashes in in ha_partition::engine_name +# +create table t1 (a int) engine=Aria transactional=1 partition by hash(a) partitions 2; +show create table t1; +drop table t1; + --let $engine=Aria --source inc/part_alter_values.inc diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 19093d9b2ca..f56781faf39 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -4397,7 +4397,7 @@ bool mysql_create_table_no_lock(THD *thd, /* Give warnings for not supported table options */ #if defined(WITH_ARIA_STORAGE_ENGINE) extern handlerton *maria_hton; - if (file->ht != maria_hton) + if (file->partition_ht() != maria_hton) #endif if (create_info->transactional) push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_WARN, From 7bd258c44c00f232ecf4ec448179f114b878227b Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 15 Feb 2018 10:06:14 +0100 Subject: [PATCH 05/31] fix plugins.server_audit test for --ps --- mysql-test/suite/plugins/t/server_audit.test | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mysql-test/suite/plugins/t/server_audit.test b/mysql-test/suite/plugins/t/server_audit.test index 9be0d5556f0..6c5eaffd9a2 100644 --- a/mysql-test/suite/plugins/t/server_audit.test +++ b/mysql-test/suite/plugins/t/server_audit.test @@ -42,8 +42,10 @@ select 1, 3; insert into t2 values (1), (2); select * from t2; +--disable_ps_protocol --error ER_NO_SUCH_TABLE select * from t_doesnt_exist; +--enable_ps_protocol --error 1064 syntax_error_query; drop table renamed_t1, t2; From f853b8ed26623200405abbc943e82295ad9f6ab3 Mon Sep 17 00:00:00 2001 From: Monty Date: Sat, 17 Feb 2018 17:47:18 +0200 Subject: [PATCH 06/31] partition_alter_myisam produces warning if no symlink support --- mysql-test/suite/parts/t/partition_alter_myisam.test | 1 + 1 file changed, 1 insertion(+) diff --git a/mysql-test/suite/parts/t/partition_alter_myisam.test b/mysql-test/suite/parts/t/partition_alter_myisam.test index b2bd0e72e4c..7e984921c1c 100644 --- a/mysql-test/suite/parts/t/partition_alter_myisam.test +++ b/mysql-test/suite/parts/t/partition_alter_myisam.test @@ -1,4 +1,5 @@ --source include/have_partition.inc +--source include/have_symlink.inc --let $engine=MyISAM --source inc/part_alter_values.inc From 965e16376c251010f6c4d7182159acacfe198ff8 Mon Sep 17 00:00:00 2001 From: Monty Date: Sat, 17 Feb 2018 17:48:23 +0200 Subject: [PATCH 07/31] TokuDB didn't compile with valgrind TokuDB uses USE_VALGRIND while MariaDB uses HAVE_valgrind Fixed by defining USE_VALGRIND in TokuDB if HAVE_valgrind is used --- storage/tokudb/PerconaFT/portability/toku_race_tools.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/storage/tokudb/PerconaFT/portability/toku_race_tools.h b/storage/tokudb/PerconaFT/portability/toku_race_tools.h index 8482a164fb8..9ed46ec909d 100644 --- a/storage/tokudb/PerconaFT/portability/toku_race_tools.h +++ b/storage/tokudb/PerconaFT/portability/toku_race_tools.h @@ -40,6 +40,11 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #include +#ifdef HAVE_valgrind +#undef USE_VALGRIND +#define USE_VALGRIND 1 +#endif + #if defined(__linux__) && USE_VALGRIND # include From 55bc3f1dd9daf50fb3bda53cd09dbfdc5a60f977 Mon Sep 17 00:00:00 2001 From: Monty Date: Sat, 17 Feb 2018 18:04:06 +0200 Subject: [PATCH 08/31] Fixed performance problem with Aria in find_head() For some simple benchmarks, a majority of time was spend in find_head() which tries to find the best place to put the record. The result of this patch is a 2x or more speedup for inserts without keys for format PAGE. All changes are only related to how rows are stored Should fix some of the problems mentioned in: MDEV-8132 Temporary tables using Aria with very poor performance MDEV-9079 Aria very slow for internal temporary tables MDEV-5841 Mariadb very poor temporary performance The following changes where done: - For rows with a small row length that fits into a page (818 bytes with 8192 pages), stop as soon as we hit a match. - Added markers full_head_size and full_tail_size that tells us where to start searching on the bitmap page - Ensure that page->used_size is correctly updated when bitmap grows. This allows us to stop searching at used_size - Added code to check that the bitmap variables are correct. - Fixed a wrong test where we set "first_bitmap_with_space". This shouldn't have caused any notable problems. --- storage/maria/ma_bitmap.c | 241 +++++++++++++++++++++++++++++++++----- storage/maria/maria_def.h | 3 + 2 files changed, 212 insertions(+), 32 deletions(-) diff --git a/storage/maria/ma_bitmap.c b/storage/maria/ma_bitmap.c index 9ab5533fdbd..e8e2bc56b25 100644 --- a/storage/maria/ma_bitmap.c +++ b/storage/maria/ma_bitmap.c @@ -145,6 +145,11 @@ static my_bool _ma_bitmap_create_missing(MARIA_HA *info, MARIA_FILE_BITMAP *bitmap, pgcache_page_no_t page); static void _ma_bitmap_unpin_all(MARIA_SHARE *share); +#ifndef DBUG_OFF +static void _ma_check_bitmap(MARIA_FILE_BITMAP *bitmap); +#else +#define _ma_check_bitmap(A) do { } while(0) +#endif /* Write bitmap page to key cache */ @@ -267,6 +272,13 @@ my_bool _ma_bitmap_init(MARIA_SHARE *share, File file, bitmap->sizes[6]= max_page_size - max_page_size * 80 / 100; bitmap->sizes[7]= 0; + /* + If a record size will fit into the smallest empty page, return first + found page in find_head() + */ + if (bitmap->sizes[3] >= share->base.max_pack_length) + bitmap->return_first_match= 1; + mysql_mutex_init(key_SHARE_BITMAP_lock, &share->bitmap.bitmap_lock, MY_MUTEX_INIT_SLOW); mysql_cond_init(key_SHARE_BITMAP_cond, @@ -677,7 +689,8 @@ void _ma_bitmap_delete_all(MARIA_SHARE *share) bzero(bitmap->map, bitmap->block_size); bitmap->changed= 1; bitmap->page= 0; - bitmap->used_size= bitmap->total_size= bitmap->max_total_size; + bitmap->used_size= bitmap->full_tail_size= bitmap->full_head_size= 0; + bitmap->total_size= bitmap->max_total_size; } DBUG_VOID_RETURN; } @@ -715,6 +728,7 @@ void _ma_bitmap_reset_cache(MARIA_SHARE *share) */ bitmap->page= ((pgcache_page_no_t) 0) - bitmap->pages_covered; bitmap->used_size= bitmap->total_size= bitmap->max_total_size; + bitmap->full_head_size= bitmap->full_tail_size= bitmap->max_total_size; bfill(bitmap->map, share->block_size, 255); #ifndef DBUG_OFF memcpy(bitmap->map + bitmap->block_size, bitmap->map, bitmap->block_size); @@ -1016,9 +1030,6 @@ static void adjust_total_size(MARIA_HA *info, pgcache_page_no_t page) bitmap Bitmap handler page Page to read - TODO - Update 'bitmap->used_size' to real size of used bitmap - NOTE We don't always have share->bitmap.bitmap_lock here (when called from_ma_check_bitmap_data() for example). @@ -1035,6 +1046,9 @@ static my_bool _ma_read_bitmap_page(MARIA_HA *info, MARIA_SHARE *share= info->s; my_bool res; DBUG_ENTER("_ma_read_bitmap_page"); + DBUG_PRINT("enter", ("page: %lld data_file_length: %lld", + (longlong) page, + (longlong) share->state.state.data_file_length)); DBUG_ASSERT(page % bitmap->pages_covered == 0); DBUG_ASSERT(!bitmap->changed); @@ -1049,13 +1063,22 @@ static my_bool _ma_read_bitmap_page(MARIA_HA *info, } adjust_total_size(info, page); - bitmap->used_size= bitmap->total_size; + bitmap->full_head_size= bitmap->full_tail_size= 0; DBUG_ASSERT(share->pagecache->block_size == bitmap->block_size); res= pagecache_read(share->pagecache, &bitmap->file, page, 0, bitmap->map, PAGECACHE_PLAIN_PAGE, PAGECACHE_LOCK_LEFT_UNLOCKED, 0) == NULL; + if (!res) + { + /* Calculate used_size */ + const uchar *data, *end= bitmap->map; + for (data= bitmap->map + bitmap->total_size; --data >= end && *data == 0; ) + {} + bitmap->used_size= (uint) ((data + 1) - end); + DBUG_ASSERT(bitmap->used_size <= bitmap->total_size); + } /* We can't check maria_bitmap_marker here as if the bitmap page previously had a true checksum and the user switched mode to not checksum @@ -1067,7 +1090,10 @@ static my_bool _ma_read_bitmap_page(MARIA_HA *info, #ifndef DBUG_OFF if (!res) + { memcpy(bitmap->map + bitmap->block_size, bitmap->map, bitmap->block_size); + _ma_check_bitmap(bitmap); + } #endif DBUG_RETURN(res); } @@ -1097,6 +1123,8 @@ static my_bool _ma_change_bitmap_page(MARIA_HA *info, { DBUG_ENTER("_ma_change_bitmap_page"); + _ma_check_bitmap(bitmap); + /* We have to mark the file changed here, as otherwise the following read/write to pagecache may force a page out from this file, which would @@ -1228,6 +1256,9 @@ static void fill_block(MARIA_FILE_BITMAP *bitmap, This is defined as the first page of the set of pages with the smallest free space that can hold 'size'. + NOTES + Updates bitmap->full_head_size while scanning data + RETURN 0 ok (block is updated) 1 error (no space in bitmap; block is not touched) @@ -1238,10 +1269,11 @@ static my_bool allocate_head(MARIA_FILE_BITMAP *bitmap, uint size, MARIA_BITMAP_BLOCK *block) { uint min_bits= size_to_head_pattern(bitmap, size); - uchar *data= bitmap->map, *end= data + bitmap->used_size; + uchar *data, *end; uchar *best_data= 0; uint best_bits= (uint) -1, UNINIT_VAR(best_pos); - uint first_pattern= 0; /* if doing insert_order */ + my_bool first_pattern= 0; /* if doing insert_order */ + my_bool first_found= 1; MARIA_SHARE *share= bitmap->share; my_bool insert_order= MY_TEST(share->base.extra_options & MA_EXTRA_OPTIONS_INSERT_ORDER); @@ -1249,16 +1281,19 @@ static my_bool allocate_head(MARIA_FILE_BITMAP *bitmap, uint size, DBUG_ASSERT(size <= FULL_PAGE_SIZE(share)); + end= bitmap->map + bitmap->used_size; if (insert_order && bitmap->page == share->last_insert_bitmap) { uint last_insert_page= share->last_insert_page; uint byte= 6 * (last_insert_page / 16); first_pattern= last_insert_page % 16; - DBUG_ASSERT(data + byte < end); - data+= byte; + data= bitmap->map+byte; + DBUG_ASSERT(data <= end); } + else + data= bitmap->map + (bitmap->full_head_size/6)*6; - for (; data < end; data+= 6) + for (; data < end; data+= 6, first_pattern= 0) { ulonglong bits= uint6korr(data); /* 6 bytes = 6*8/3= 16 patterns */ uint i; @@ -1271,17 +1306,24 @@ static my_bool allocate_head(MARIA_FILE_BITMAP *bitmap, uint size, */ if ((!bits && best_data) || ((bits & 04444444444444444LL) == 04444444444444444LL)) - { - first_pattern= 0; // always restart from 0 when moving to new 6-byte continue; - } + for (i= first_pattern, bits >>= (3 * first_pattern); i < 16 ; i++, bits >>= 3) { uint pattern= (uint) (bits & 7); + + if (pattern <= 3) /* Room for more data */ + { + if (first_found) + { + first_found= 0; + bitmap->full_head_size= (data - bitmap->map); + } + } if (pattern <= min_bits) { - /* There is enough space here */ + /* There is enough space here, check if we have found better */ if ((int) pattern > (int) best_bits) { /* @@ -1292,23 +1334,32 @@ static my_bool allocate_head(MARIA_FILE_BITMAP *bitmap, uint size, best_bits= pattern; best_data= data; best_pos= i; - if (pattern == min_bits) + if (pattern == min_bits || bitmap->return_first_match) goto found; /* Best possible match */ } } } - first_pattern= 0; // always restart from 0 when moving to new 6-byte } if (!best_data) /* Found no place */ { if (data >= bitmap->map + bitmap->total_size) DBUG_RETURN(1); /* No space in bitmap */ + DBUG_ASSERT(uint6korr(data) == 0); /* Allocate data at end of bitmap */ - bitmap->used_size+= 6; - set_if_smaller(bitmap->used_size, bitmap->total_size); + bitmap->used_size= (uint) (data - bitmap->map) + 6; best_data= data; best_pos= best_bits= 0; } + else + { + /* + This is not stricly needed as used_size should be alligned on 6, + but for easier debugging lets try to keep it more accurate + */ + uint position= (uint) (best_data - bitmap->map) + 6; + set_if_bigger(bitmap->used_size, position); + } + DBUG_ASSERT(bitmap->used_size <= bitmap->total_size); found: if (insert_order) @@ -1341,12 +1392,15 @@ static my_bool allocate_tail(MARIA_FILE_BITMAP *bitmap, uint size, MARIA_BITMAP_BLOCK *block) { uint min_bits= size_to_tail_pattern(bitmap, size); - uchar *data= bitmap->map, *end= data + bitmap->used_size; - uchar *best_data= 0; + uchar *data, *end, *best_data= 0; + my_bool first_found= 1; uint best_bits= (uint) -1, UNINIT_VAR(best_pos); DBUG_ENTER("allocate_tail"); DBUG_PRINT("enter", ("size: %u", size)); + data= bitmap->map + (bitmap->full_tail_size/6)*6; + end= bitmap->map + bitmap->used_size; + /* We have to add DIR_ENTRY_SIZE here as this is not part of the data size See call to allocate_tail() in find_tail(). @@ -1375,7 +1429,19 @@ static my_bool allocate_tail(MARIA_FILE_BITMAP *bitmap, uint size, for (i= 0; i < 16; i++, bits >>= 3) { uint pattern= (uint) (bits & 7); - if (pattern <= min_bits && (!pattern || pattern >= 5)) + + if (pattern == 0 || + (pattern > FULL_HEAD_PAGE && pattern < FULL_TAIL_PAGE)) + { + /* There is room for tail data */ + if (first_found) + { + first_found= 0; + bitmap->full_tail_size= (data - bitmap->map); + } + } + + if (pattern <= min_bits && (!pattern || pattern > FULL_HEAD_PAGE)) { if ((int) pattern > (int) best_bits) { @@ -1392,10 +1458,11 @@ static my_bool allocate_tail(MARIA_FILE_BITMAP *bitmap, uint size, { if (data >= bitmap->map + bitmap->total_size) DBUG_RETURN(1); + DBUG_ASSERT(uint6korr(data) == 0); /* Allocate data at end of bitmap */ best_data= data; - bitmap->used_size+= 6; - set_if_smaller(bitmap->used_size, bitmap->total_size); + bitmap->used_size= (uint) (data - bitmap->map) + 6; + DBUG_ASSERT(bitmap->used_size <= bitmap->total_size); best_pos= best_bits= 0; } @@ -1434,8 +1501,7 @@ static ulong allocate_full_pages(MARIA_FILE_BITMAP *bitmap, ulong pages_needed, MARIA_BITMAP_BLOCK *block, my_bool full_page) { - uchar *data= bitmap->map, *data_end= data + bitmap->used_size; - uchar *page_end= data + bitmap->total_size; + uchar *data, *data_end, *page_end; uchar *best_data= 0; uint min_size; uint best_area_size, UNINIT_VAR(best_prefix_area_size); @@ -1449,6 +1515,10 @@ static ulong allocate_full_pages(MARIA_FILE_BITMAP *bitmap, min_size= BLOB_SEGMENT_MIN_SIZE; best_area_size= ~(uint) 0; + data= bitmap->map + (bitmap->full_head_size/6)*6; + data_end= bitmap->map + bitmap->used_size; + page_end= bitmap->map + bitmap->total_size; + for (; data < page_end; data+= 6) { ulonglong bits= uint6korr(data); /* 6 bytes = 6*8/3= 16 patterns */ @@ -1466,6 +1536,12 @@ static ulong allocate_full_pages(MARIA_FILE_BITMAP *bitmap, if ((bits= uint6korr(data))) break; } + /* + Check if we are end of bitmap. In this case we know that + the rest of the bitmap is usable + */ + if (data >= data_end) + data= page_end; area_size= (uint) (data - data_start) / 6 * 16; if (area_size >= best_area_size) continue; @@ -1823,7 +1899,7 @@ static my_bool allocate_blobs(MARIA_HA *info, MARIA_ROW *row) /* - Store in the bitmap the new size for a head page + Reserve the current head page SYNOPSIS use_head() @@ -2225,7 +2301,7 @@ static my_bool set_page_bits(MARIA_HA *info, MARIA_FILE_BITMAP *bitmap, pgcache_page_no_t page, uint fill_pattern) { pgcache_page_no_t bitmap_page; - uint offset_page, offset, tmp, org_tmp; + uint offset_page, offset, tmp, org_tmp, used_offset; uchar *data; DBUG_ENTER("set_page_bits"); DBUG_ASSERT(fill_pattern <= 7); @@ -2237,6 +2313,7 @@ static my_bool set_page_bits(MARIA_HA *info, MARIA_FILE_BITMAP *bitmap, /* Find page number from start of bitmap */ offset_page= (uint) (page - bitmap->page - 1); + /* Mark place used by reading/writing 2 bytes at a time to handle bitmaps in overlapping bytes @@ -2248,11 +2325,37 @@ static my_bool set_page_bits(MARIA_HA *info, MARIA_FILE_BITMAP *bitmap, tmp= (tmp & ~(7 << offset)) | (fill_pattern << offset); if (tmp == org_tmp) DBUG_RETURN(0); /* No changes */ - int2store(data, tmp); + /* + Take care to not write bytes outside of bitmap. + fill_pattern is 3 bits, so we need to write two bytes + if bit position we write to is > (8-3) + */ + if (offset > 5) + int2store(data, tmp); + else + data[0]= tmp; + + /* + Reset full_head_size or full_tail_size if we are releasing data before + it. Increase used_size if we are allocating data. + */ + used_offset= (uint) (data - bitmap->map); + if (fill_pattern < 4) + set_if_smaller(bitmap->full_head_size, used_offset); + if (fill_pattern == 0 || (fill_pattern > 4 && fill_pattern < 7)) + set_if_smaller(bitmap->full_tail_size, used_offset); + if (fill_pattern != 0) + { + /* Calulcate which was the last changed byte */ + used_offset+= offset > 5 ? 2 : 1; + set_if_bigger(bitmap->used_size, used_offset); + } + + _ma_check_bitmap(bitmap); bitmap->changed= 1; DBUG_EXECUTE("bitmap", _ma_print_bitmap_changes(bitmap);); - if (fill_pattern != 3 && fill_pattern != 7) + if (fill_pattern != FULL_HEAD_PAGE && fill_pattern != FULL_TAIL_PAGE) set_if_smaller(info->s->state.first_bitmap_with_space, bitmap_page); /* Note that if the condition above is false (page is full), and all pages of @@ -2345,7 +2448,7 @@ my_bool _ma_bitmap_reset_full_page_bits(MARIA_HA *info, uint page_count) { ulonglong bitmap_page; - uint offset, bit_start, bit_count, tmp; + uint offset, bit_start, bit_count, tmp, byte_offset; uchar *data; DBUG_ENTER("_ma_bitmap_reset_full_page_bits"); DBUG_PRINT("enter", ("page: %lu page_count: %u", (ulong) page, page_count)); @@ -2365,7 +2468,8 @@ my_bool _ma_bitmap_reset_full_page_bits(MARIA_HA *info, bit_start= offset * 3; bit_count= page_count * 3; - data= bitmap->map + bit_start / 8; + byte_offset= bit_start/8; + data= bitmap->map + byte_offset; offset= bit_start & 7; tmp= (255 << offset); /* Bits to keep */ @@ -2376,6 +2480,9 @@ my_bool _ma_bitmap_reset_full_page_bits(MARIA_HA *info, } *data&= ~tmp; + set_if_smaller(bitmap->full_head_size, byte_offset); + set_if_smaller(bitmap->full_tail_size, byte_offset); + if ((int) (bit_count-= (8 - offset)) > 0) { uint fill; @@ -2477,6 +2584,8 @@ my_bool _ma_bitmap_set_full_page_bits(MARIA_HA *info, tmp= (1 << bit_count) - 1; *data|= tmp; } + set_if_bigger(bitmap->used_size, (uint) (data - bitmap->map) + 1); + _ma_check_bitmap(bitmap); bitmap->changed= 1; DBUG_EXECUTE("bitmap", _ma_print_bitmap_changes(bitmap);); DBUG_RETURN(0); @@ -2835,6 +2944,72 @@ my_bool _ma_check_bitmap_data(MARIA_HA *info, enum en_page_type page_type, return (bitmap_pattern != bits); } +/** + Check that bitmap looks correct + + - All data before full_head_size and full_tail_size are allocated + - There is no allocated data after used_size + All of the above need to be correct only according to 6 byte + alignment as all loops reads 6 bytes at a time and we check both + start and end position according to the current 6 byte position. +*/ + +#ifndef DBUG_OFF +static void _ma_check_bitmap(MARIA_FILE_BITMAP *bitmap) +{ + uchar *data= bitmap->map; + uchar *end= bitmap->map + bitmap->total_size; + uchar *full_head_end=0, *full_tail_end=0, *first_empty= bitmap->map; + + for (; data < end; data+= 6) + { + ulonglong bits= uint6korr(data); /* 6 bytes = 6*8/3= 16 patterns */ + uint i; + + if (bits == 04444444444444444LL || bits == 0xffffffffffffLL) + { + first_empty= data + 6; + continue; /* block fully used */ + } + if (bits == 0) + { + if (!full_head_end) + full_head_end= data; + if (!full_tail_end) + full_tail_end= data; + continue; + } + + first_empty= data + 6; + if (!full_head_end || !full_tail_end) + { + for (i= 0, bits >>= 0; i < 16 ; i++, bits >>= 3) + { + uint pattern= (uint) (bits & 7); + if (pattern == FULL_HEAD_PAGE || pattern == FULL_TAIL_PAGE) + continue; + + if (pattern < 4 && !full_head_end) + full_head_end= data; + if ((pattern == 0 || (pattern > 4 && pattern < 7)) && !full_tail_end) + full_tail_end= data; + } + } + } + if (!full_head_end) + full_head_end= data; + if (!full_tail_end) + full_tail_end= data; + + /* used_size must point after the last byte that had some data) */ + DBUG_ASSERT(bitmap->used_size <= bitmap->total_size); + DBUG_ASSERT((bitmap->map + (bitmap->used_size+5)/6*6) >= first_empty); + /* full_xxxx_size can't point after the first block that has free data */ + DBUG_ASSERT((bitmap->map + (bitmap->full_head_size/6*6)) <= full_head_end); + DBUG_ASSERT((bitmap->map + (bitmap->full_tail_size/6*6)) <= full_tail_end); +} +#endif + /* Check if the page type matches the one that we have in the bitmap @@ -3072,6 +3247,7 @@ static my_bool _ma_bitmap_create_missing(MARIA_HA *info, pgcache_page_no_t from, to; my_off_t data_file_length= share->state.state.data_file_length; DBUG_ENTER("_ma_bitmap_create_missing"); + DBUG_PRINT("enter", ("page: %lld", (longlong) page)); /* First (in offset order) bitmap page to create */ if (data_file_length < block_size) @@ -3124,7 +3300,8 @@ static my_bool _ma_bitmap_create_missing(MARIA_HA *info, only later as we are going to modify it very soon. */ bzero(bitmap->map, bitmap->block_size); - bitmap->used_size= 0; + bitmap->used_size= bitmap->full_head_size= bitmap->full_tail_size= 0; + bitmap->changed=1; #ifndef DBUG_OFF /* Make a copy of the page to be able to print out bitmap changes during diff --git a/storage/maria/maria_def.h b/storage/maria/maria_def.h index d57a429365a..6cd6cda4b2f 100644 --- a/storage/maria/maria_def.h +++ b/storage/maria/maria_def.h @@ -331,7 +331,10 @@ typedef struct st_maria_file_bitmap pgcache_page_no_t last_bitmap_page; /* Last possible bitmap page */ my_bool changed; /* 1 if page needs to be written */ my_bool changed_not_flushed; /* 1 if some bitmap is not flushed */ + my_bool return_first_match; /* Shortcut find_head() */ uint used_size; /* Size of bitmap head that is not 0 */ + uint full_head_size; /* Where to start search for head */ + uint full_tail_size; /* Where to start search for tail */ uint flush_all_requested; /**< If _ma_bitmap_flush_all waiting */ uint waiting_for_flush_all_requested; /* If someone is waiting for above */ uint non_flushable; /**< 0 if bitmap and log are in sync */ From 0e8cb572f11a767849d97776c0fb4a0a0eb9c490 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Sat, 17 Feb 2018 15:18:14 +0200 Subject: [PATCH 09/31] Fix innodb_encryption-page-compression test by force flushing dirty pages. --- .../innodb_encryption-page-compression.result | 12 ++++++++++++ .../t/innodb_encryption-page-compression.test | 19 ++++++++++++++++--- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/mysql-test/suite/encryption/r/innodb_encryption-page-compression.result b/mysql-test/suite/encryption/r/innodb_encryption-page-compression.result index 88437408c32..584f8293f5d 100644 --- a/mysql-test/suite/encryption/r/innodb_encryption-page-compression.result +++ b/mysql-test/suite/encryption/r/innodb_encryption-page-compression.result @@ -135,6 +135,12 @@ count(*) select count(*) from innodb_page_compressed9 where c1 < 500000; count(*) 2000 +flush tables innodb_page_compressed1, innodb_page_compressed2, +innodb_page_compressed3, innodb_page_compressed4, +innodb_page_compressed5, innodb_page_compressed6, +innodb_page_compressed7, innodb_page_compressed8, +innodb_page_compressed9 for export; +unlock tables; SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; variable_value > 0 1 @@ -152,6 +158,12 @@ update innodb_page_compressed6 set c1 = c1 + 1; update innodb_page_compressed7 set c1 = c1 + 1; update innodb_page_compressed8 set c1 = c1 + 1; update innodb_page_compressed9 set c1 = c1 + 1; +flush tables innodb_page_compressed1, innodb_page_compressed2, +innodb_page_compressed3, innodb_page_compressed4, +innodb_page_compressed5, innodb_page_compressed6, +innodb_page_compressed7, innodb_page_compressed8, +innodb_page_compressed9 for export; +unlock tables; SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; variable_value > 0 1 diff --git a/mysql-test/suite/encryption/t/innodb_encryption-page-compression.test b/mysql-test/suite/encryption/t/innodb_encryption-page-compression.test index 0c18b97bfd6..ef9e3d6e7f4 100644 --- a/mysql-test/suite/encryption/t/innodb_encryption-page-compression.test +++ b/mysql-test/suite/encryption/t/innodb_encryption-page-compression.test @@ -81,6 +81,14 @@ select count(*) from innodb_page_compressed7 where c1 < 500000; select count(*) from innodb_page_compressed8 where c1 < 500000; select count(*) from innodb_page_compressed9 where c1 < 500000; +flush tables innodb_page_compressed1, innodb_page_compressed2, +innodb_page_compressed3, innodb_page_compressed4, +innodb_page_compressed5, innodb_page_compressed6, +innodb_page_compressed7, innodb_page_compressed8, +innodb_page_compressed9 for export; + +unlock tables; + let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_PAGE_COMPRESSED'; --source include/wait_condition.inc let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_ENCRYPTED'; @@ -104,9 +112,14 @@ update innodb_page_compressed7 set c1 = c1 + 1; update innodb_page_compressed8 set c1 = c1 + 1; update innodb_page_compressed9 set c1 = c1 + 1; -let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_ENCRYPTED'; ---source include/wait_condition.inc -let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_PAGE_COMPRESSED'; +flush tables innodb_page_compressed1, innodb_page_compressed2, +innodb_page_compressed3, innodb_page_compressed4, +innodb_page_compressed5, innodb_page_compressed6, +innodb_page_compressed7, innodb_page_compressed8, +innodb_page_compressed9 for export; + +unlock tables; + --source include/wait_condition.inc SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; From 9ea3ad6d7541035f514a6212d9b72ee165f52fb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Sun, 18 Feb 2018 07:32:19 +0200 Subject: [PATCH 10/31] Disable failing test. --- mysql-test/suite/encryption/disabled.def | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql-test/suite/encryption/disabled.def b/mysql-test/suite/encryption/disabled.def index abbb82d51f6..acb936fa6f7 100644 --- a/mysql-test/suite/encryption/disabled.def +++ b/mysql-test/suite/encryption/disabled.def @@ -12,4 +12,4 @@ innodb_scrub : MDEV-8139 scrubbing does not work reliably innodb_scrub_background : MDEV-8139 scrubbing does not work reliably - +innodb_encryption-page-compression : MDEV-14814 encryption.innodb_encryption-page-compression failed in buildbot with timeout on wait condition From 7a84688e2c30a95297246b5dc3005854bcbd5bf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Sun, 18 Feb 2018 07:47:47 +0200 Subject: [PATCH 11/31] MDEV-14814: encryption.innodb_encryption-page-compression failed in buildbot with timeout on wait condition Test changes only. --- mysql-test/suite/encryption/disabled.def | 2 +- .../r/innodb_encryption-page-compression.result | 6 +++--- .../encryption/t/innodb_encryption-page-compression.test | 8 ++++++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/mysql-test/suite/encryption/disabled.def b/mysql-test/suite/encryption/disabled.def index acb936fa6f7..abbb82d51f6 100644 --- a/mysql-test/suite/encryption/disabled.def +++ b/mysql-test/suite/encryption/disabled.def @@ -12,4 +12,4 @@ innodb_scrub : MDEV-8139 scrubbing does not work reliably innodb_scrub_background : MDEV-8139 scrubbing does not work reliably -innodb_encryption-page-compression : MDEV-14814 encryption.innodb_encryption-page-compression failed in buildbot with timeout on wait condition + diff --git a/mysql-test/suite/encryption/r/innodb_encryption-page-compression.result b/mysql-test/suite/encryption/r/innodb_encryption-page-compression.result index 584f8293f5d..14933e526c4 100644 --- a/mysql-test/suite/encryption/r/innodb_encryption-page-compression.result +++ b/mysql-test/suite/encryption/r/innodb_encryption-page-compression.result @@ -141,6 +141,7 @@ innodb_page_compressed5, innodb_page_compressed6, innodb_page_compressed7, innodb_page_compressed8, innodb_page_compressed9 for export; unlock tables; +# Wait until dirty pages are compressed and encrypted SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; variable_value > 0 1 @@ -164,9 +165,8 @@ innodb_page_compressed5, innodb_page_compressed6, innodb_page_compressed7, innodb_page_compressed8, innodb_page_compressed9 for export; unlock tables; -SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; -variable_value > 0 -1 +# Wait until dirty pages are compressed and encrypted 2 +unlock tables; SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted'; variable_value > 0 1 diff --git a/mysql-test/suite/encryption/t/innodb_encryption-page-compression.test b/mysql-test/suite/encryption/t/innodb_encryption-page-compression.test index ef9e3d6e7f4..fe132c0c59b 100644 --- a/mysql-test/suite/encryption/t/innodb_encryption-page-compression.test +++ b/mysql-test/suite/encryption/t/innodb_encryption-page-compression.test @@ -89,6 +89,7 @@ innodb_page_compressed9 for export; unlock tables; +--echo # Wait until dirty pages are compressed and encrypted let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_PAGE_COMPRESSED'; --source include/wait_condition.inc let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_ENCRYPTED'; @@ -117,12 +118,15 @@ innodb_page_compressed3, innodb_page_compressed4, innodb_page_compressed5, innodb_page_compressed6, innodb_page_compressed7, innodb_page_compressed8, innodb_page_compressed9 for export; - unlock tables; +--echo # Wait until dirty pages are compressed and encrypted 2 +let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_PAGE_COMPRESSED'; +--source include/wait_condition.inc +unlock tables; +let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_DECRYPTED'; --source include/wait_condition.inc -SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted'; SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_compressed'; SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_decompressed'; From b697f213a7bb90737e41ae09a5bfb08c6f47c0f6 Mon Sep 17 00:00:00 2001 From: Sergey Vojtovich Date: Mon, 19 Feb 2018 13:53:35 +0400 Subject: [PATCH 12/31] MDEV-14541 - Workaround GCC ICE on ARM64 Added missing include. --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index b4de70a283d..8d2efac89cb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -162,6 +162,7 @@ INCLUDE(plugin) INCLUDE(install_macros) INCLUDE(systemd) INCLUDE(mysql_add_executable) +INCLUDE(compile_flags) # Handle options OPTION(DISABLE_SHARED From 83b471348d6fc4e9bb4bd2e5e23b304979708a8e Mon Sep 17 00:00:00 2001 From: Sergey Vojtovich Date: Mon, 19 Feb 2018 16:51:15 +0400 Subject: [PATCH 13/31] MDEV-14318 - cmake updates to build on arm64 Added more files affected by GCC ICE: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=67143 --- storage/innobase/CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index 0033cdbae35..c5f9abdaa86 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -175,7 +175,13 @@ IF(CMAKE_COMPILER_IS_GNUCXX AND CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64") btr/btr0btr.cc btr/btr0cur.cc buf/buf0buf.cc + fts/fts0fts.cc gis/gis0sea.cc + handler/handler0alter.cc + mtr/mtr0mtr.cc + row/row0merge.cc + row/row0mysql.cc + srv/srv0srv.cc COMPILE_FLAGS "-O0" ) ENDIF() From 852c35f571b1b7454aab5768899e2faee8f941d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 19 Feb 2018 14:58:05 +0200 Subject: [PATCH 14/31] MDEV-11581 follow-up fix: Correct a condition fsp_fill_free_list(): Correctly determine whether the temporary tablespace file should be extended in order to respond to a page allocation request. The inverted condition was noticed by Thiru when he analyzed MDEV-13013. --- storage/innobase/fsp/fsp0fsp.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc index 2531a9fb5aa..dd0148a3275 100644 --- a/storage/innobase/fsp/fsp0fsp.cc +++ b/storage/innobase/fsp/fsp0fsp.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1108,7 +1108,7 @@ fsp_fill_free_list( skip_resize = !srv_sys_space.can_auto_extend_last_file(); break; case SRV_TMP_SPACE_ID: - skip_resize = srv_tmp_space.can_auto_extend_last_file(); + skip_resize = !srv_tmp_space.can_auto_extend_last_file(); break; } From 0ea45725d85c8426c62f4d6707ae2029b49ceb3d Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Mon, 19 Feb 2018 15:21:54 +0000 Subject: [PATCH 15/31] Fix 2 more VS2015 warnings --- sql/slave.cc | 19 +++++++++---------- win/upgrade_wizard/CMakeLists.txt | 2 +- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/sql/slave.cc b/sql/slave.cc index 9c6e8ebc200..5534ddb1b49 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -2626,7 +2626,6 @@ bool show_master_info(THD *thd, Master_info *mi, bool full) void show_master_info_get_fields(THD *thd, List *field_list, bool full, size_t gtid_pos_length) { - Master_info *mi; MEM_ROOT *mem_root= thd->mem_root; DBUG_ENTER("show_master_info_get_fields"); @@ -2645,10 +2644,10 @@ void show_master_info_get_fields(THD *thd, List *field_list, Item_empty_string(thd, "Slave_IO_State", 30), mem_root); field_list->push_back(new (mem_root) - Item_empty_string(thd, "Master_Host", sizeof(mi->host)), + Item_empty_string(thd, "Master_Host", sizeof(Master_info::host)), mem_root); field_list->push_back(new (mem_root) - Item_empty_string(thd, "Master_User", sizeof(mi->user)), + Item_empty_string(thd, "Master_User", sizeof(Master_info::user)), mem_root); field_list->push_back(new (mem_root) Item_return_int(thd, "Master_Port", 7, MYSQL_TYPE_LONG), @@ -2733,23 +2732,23 @@ void show_master_info_get_fields(THD *thd, List *field_list, mem_root); field_list->push_back(new (mem_root) Item_empty_string(thd, "Master_SSL_CA_File", - sizeof(mi->ssl_ca)), + sizeof(Master_info::ssl_ca)), mem_root); field_list->push_back(new (mem_root) Item_empty_string(thd, "Master_SSL_CA_Path", - sizeof(mi->ssl_capath)), + sizeof(Master_info::ssl_capath)), mem_root); field_list->push_back(new (mem_root) Item_empty_string(thd, "Master_SSL_Cert", - sizeof(mi->ssl_cert)), + sizeof(Master_info::ssl_cert)), mem_root); field_list->push_back(new (mem_root) Item_empty_string(thd, "Master_SSL_Cipher", - sizeof(mi->ssl_cipher)), + sizeof(Master_info::ssl_cipher)), mem_root); field_list->push_back(new (mem_root) Item_empty_string(thd, "Master_SSL_Key", - sizeof(mi->ssl_key)), + sizeof(Master_info::ssl_key)), mem_root); field_list->push_back(new (mem_root) Item_return_int(thd, "Seconds_Behind_Master", 10, @@ -2783,11 +2782,11 @@ void show_master_info_get_fields(THD *thd, List *field_list, mem_root); field_list->push_back(new (mem_root) Item_empty_string(thd, "Master_SSL_Crl", - sizeof(mi->ssl_crl)), + sizeof(Master_info::ssl_crl)), mem_root); field_list->push_back(new (mem_root) Item_empty_string(thd, "Master_SSL_Crlpath", - sizeof(mi->ssl_crlpath)), + sizeof(Master_info::ssl_crlpath)), mem_root); field_list->push_back(new (mem_root) Item_empty_string(thd, "Using_Gtid", diff --git a/win/upgrade_wizard/CMakeLists.txt b/win/upgrade_wizard/CMakeLists.txt index dc4ef67387d..5f415e4960d 100644 --- a/win/upgrade_wizard/CMakeLists.txt +++ b/win/upgrade_wizard/CMakeLists.txt @@ -24,7 +24,7 @@ ELSE() SET(CMAKE_MFC_FLAG 1) ENDIF() # Enable exception handling (avoids warnings) -SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc") +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc -DNO_WARN_MBCS_MFC_DEPRECATION") INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql) MYSQL_ADD_EXECUTABLE(mysql_upgrade_wizard From 84e8e07e8e185fe98ad2932bd32e94424fb9383a Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Mon, 19 Feb 2018 16:37:08 +0000 Subject: [PATCH 16/31] Revert "Fix 2 more VS2015 warnings" This reverts commit 0ea45725d85c8426c62f4d6707ae2029b49ceb3d, since it breaks clang --- sql/slave.cc | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/sql/slave.cc b/sql/slave.cc index 5534ddb1b49..9c6e8ebc200 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -2626,6 +2626,7 @@ bool show_master_info(THD *thd, Master_info *mi, bool full) void show_master_info_get_fields(THD *thd, List *field_list, bool full, size_t gtid_pos_length) { + Master_info *mi; MEM_ROOT *mem_root= thd->mem_root; DBUG_ENTER("show_master_info_get_fields"); @@ -2644,10 +2645,10 @@ void show_master_info_get_fields(THD *thd, List *field_list, Item_empty_string(thd, "Slave_IO_State", 30), mem_root); field_list->push_back(new (mem_root) - Item_empty_string(thd, "Master_Host", sizeof(Master_info::host)), + Item_empty_string(thd, "Master_Host", sizeof(mi->host)), mem_root); field_list->push_back(new (mem_root) - Item_empty_string(thd, "Master_User", sizeof(Master_info::user)), + Item_empty_string(thd, "Master_User", sizeof(mi->user)), mem_root); field_list->push_back(new (mem_root) Item_return_int(thd, "Master_Port", 7, MYSQL_TYPE_LONG), @@ -2732,23 +2733,23 @@ void show_master_info_get_fields(THD *thd, List *field_list, mem_root); field_list->push_back(new (mem_root) Item_empty_string(thd, "Master_SSL_CA_File", - sizeof(Master_info::ssl_ca)), + sizeof(mi->ssl_ca)), mem_root); field_list->push_back(new (mem_root) Item_empty_string(thd, "Master_SSL_CA_Path", - sizeof(Master_info::ssl_capath)), + sizeof(mi->ssl_capath)), mem_root); field_list->push_back(new (mem_root) Item_empty_string(thd, "Master_SSL_Cert", - sizeof(Master_info::ssl_cert)), + sizeof(mi->ssl_cert)), mem_root); field_list->push_back(new (mem_root) Item_empty_string(thd, "Master_SSL_Cipher", - sizeof(Master_info::ssl_cipher)), + sizeof(mi->ssl_cipher)), mem_root); field_list->push_back(new (mem_root) Item_empty_string(thd, "Master_SSL_Key", - sizeof(Master_info::ssl_key)), + sizeof(mi->ssl_key)), mem_root); field_list->push_back(new (mem_root) Item_return_int(thd, "Seconds_Behind_Master", 10, @@ -2782,11 +2783,11 @@ void show_master_info_get_fields(THD *thd, List *field_list, mem_root); field_list->push_back(new (mem_root) Item_empty_string(thd, "Master_SSL_Crl", - sizeof(Master_info::ssl_crl)), + sizeof(mi->ssl_crl)), mem_root); field_list->push_back(new (mem_root) Item_empty_string(thd, "Master_SSL_Crlpath", - sizeof(Master_info::ssl_crlpath)), + sizeof(mi->ssl_crlpath)), mem_root); field_list->push_back(new (mem_root) Item_empty_string(thd, "Using_Gtid", From a128fe4346a466aa8e059baa620a3b689fa21241 Mon Sep 17 00:00:00 2001 From: Galina Shalygina Date: Tue, 20 Feb 2018 01:16:50 +0200 Subject: [PATCH 17/31] MDEV-14297: Lost name of a explicitly named CTE column used in the non-recursive CTE via prepared statement The problem appears as the column names of the CTE were allocated on the wrong MEMROOT and after the preparation of the statement they disappear. To fix it in the procedure With_element::rename_columns_of_derived_unit the CTE column names are now allocated in the permanent MEMROOT for the prepared statements and stored procedures. --- mysql-test/r/cte_nonrecursive.result | 27 +++++++++++++++++++++++++++ mysql-test/t/cte_nonrecursive.test | 25 +++++++++++++++++++++++++ sql/sql_cte.cc | 7 +++++++ 3 files changed, 59 insertions(+) diff --git a/mysql-test/r/cte_nonrecursive.result b/mysql-test/r/cte_nonrecursive.result index 33e14af900a..f2bfce9f6ba 100644 --- a/mysql-test/r/cte_nonrecursive.result +++ b/mysql-test/r/cte_nonrecursive.result @@ -1398,3 +1398,30 @@ i 2 3 DROP TABLE t1; +# +# MDEV-14297: Lost name of a explicitly named CTE column used in +# the non-recursive CTE via prepared statement +# +CREATE TABLE t1 (i int); +INSERT INTO t1 VALUES (1),(2),(3); +PREPARE stmt FROM "WITH cte(a) AS (SELECT 1) SELECT * FROM cte"; +EXECUTE stmt; +a +1 +DEALLOCATE PREPARE stmt; +PREPARE stmt FROM "CREATE VIEW v1 AS WITH cte(a) AS (SELECT 1) SELECT * FROM cte"; +EXECUTE stmt; +SELECT * FROM v1; +a +1 +DEALLOCATE PREPARE stmt; +PREPARE stmt FROM "CREATE VIEW v2 AS WITH cte(a) AS (SELECT * FROM t1) SELECT * FROM cte"; +EXECUTE stmt; +SELECT * FROM v2; +a +1 +2 +3 +DEALLOCATE PREPARE stmt; +DROP TABLE t1; +DROP VIEW v1,v2; diff --git a/mysql-test/t/cte_nonrecursive.test b/mysql-test/t/cte_nonrecursive.test index 05de03fc7af..3d073183877 100644 --- a/mysql-test/t/cte_nonrecursive.test +++ b/mysql-test/t/cte_nonrecursive.test @@ -964,3 +964,28 @@ WITH RECURSIVE c1 AS (WITH c3 AS (SELECT * FROM c2) SELECT * FROM c3), SELECT * FROM c1; DROP TABLE t1; + +--echo # +--echo # MDEV-14297: Lost name of a explicitly named CTE column used in +--echo # the non-recursive CTE via prepared statement +--echo # + +CREATE TABLE t1 (i int); +INSERT INTO t1 VALUES (1),(2),(3); + +PREPARE stmt FROM "WITH cte(a) AS (SELECT 1) SELECT * FROM cte"; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +PREPARE stmt FROM "CREATE VIEW v1 AS WITH cte(a) AS (SELECT 1) SELECT * FROM cte"; +EXECUTE stmt; +SELECT * FROM v1; +DEALLOCATE PREPARE stmt; + +PREPARE stmt FROM "CREATE VIEW v2 AS WITH cte(a) AS (SELECT * FROM t1) SELECT * FROM cte"; +EXECUTE stmt; +SELECT * FROM v2; +DEALLOCATE PREPARE stmt; + +DROP TABLE t1; +DROP VIEW v1,v2; diff --git a/sql/sql_cte.cc b/sql/sql_cte.cc index ac27f1713f9..dd46295d799 100644 --- a/sql/sql_cte.cc +++ b/sql/sql_cte.cc @@ -910,12 +910,19 @@ With_element::rename_columns_of_derived_unit(THD *thd, my_error(ER_WITH_COL_WRONG_LIST, MYF(0)); return true; } + + Query_arena *arena, backup; + arena= thd->activate_stmt_arena_if_needed(&backup); + /* Rename the columns of the first select in the unit */ while ((item= it++, name= nm++)) { item->set_name(thd, name->str, (uint) name->length, system_charset_info); item->is_autogenerated_name= false; } + + if (arena) + thd->restore_active_arena(arena, &backup); } else make_valid_column_names(thd, select->item_list); From 3a7126cbb7ade5c736ec4e124704faaecd5f80f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Tue, 20 Feb 2018 09:17:29 +0200 Subject: [PATCH 18/31] Disable galera_gtid until 1047: WSREP has not yet prepared node for application use is fixed. --- mysql-test/suite/galera/disabled.def | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def index b67dea0f2f4..742917dc122 100644 --- a/mysql-test/suite/galera/disabled.def +++ b/mysql-test/suite/galera/disabled.def @@ -58,4 +58,5 @@ MW-328C: MDEV-13549 Galera test failures 10.1 MW-328A: MDEV-13549 Galera test failures 10.1 MW-328B: MDEV-13549 Galera test failures 10.1 MW-328: MDEV-13549 Galera test failures 10.1 -galera_suspend_slave: MDEV-13549 Galera test failures 10.1 \ No newline at end of file +galera_suspend_slave: MDEV-13549 Galera test failures 10.1 +galera_gtid: MDEV-13549 Galera test failures 10.1 \ No newline at end of file From c3a3b77f9a1c4bc320e071a187e454a62fe70366 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Tue, 20 Feb 2018 09:27:31 +0200 Subject: [PATCH 19/31] Disable failing Galera tests: * galera_gtid_slave * galera_unicode_identifiers --- mysql-test/suite/galera/disabled.def | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def index 742917dc122..d82d5dd2023 100644 --- a/mysql-test/suite/galera/disabled.def +++ b/mysql-test/suite/galera/disabled.def @@ -54,9 +54,11 @@ galera_pc_ignore_sb : MDEV-13549 Galera test failures 10.1 galera_lock_table : MDEV-13549 Galera test failures 10.1 MW-284 : MDEV-13549 Galera test failures 10.1 galera_as_slave : MDEV-13549 Galera test failures 10.1 -MW-328C: MDEV-13549 Galera test failures 10.1 -MW-328A: MDEV-13549 Galera test failures 10.1 -MW-328B: MDEV-13549 Galera test failures 10.1 -MW-328: MDEV-13549 Galera test failures 10.1 -galera_suspend_slave: MDEV-13549 Galera test failures 10.1 -galera_gtid: MDEV-13549 Galera test failures 10.1 \ No newline at end of file +MW-328C : MDEV-13549 Galera test failures 10.1 +MW-328A : MDEV-13549 Galera test failures 10.1 +MW-328B : MDEV-13549 Galera test failures 10.1 +MW-328 : MDEV-13549 Galera test failures 10.1 +galera_suspend_slave : MDEV-13549 Galera test failures 10.1 +galera_gtid : MDEV-13549 Galera test failures 10.1 +galera_gtid_slave : MDEV-13549 Galera test failures 10.1 +galera_unicode_identifiers : MDEV-13549 Galera test failures 10.1 From 56d6776524909b5bd413996cb703e099024105a5 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 20 Feb 2018 20:03:23 +0000 Subject: [PATCH 20/31] Windows : remove freopen(), "to keep fd = 0 busy". a) We do not need this on Windows, and it is not clear what it does,inside service. b) It hinders debugging. mysql-test-run.pl --debugger=vsjitdebugger more often than , would stop here throwing "invalid handle" exception. --- sql/mysqld.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 0de9d7a9f0b..3cb8e41f94e 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -5871,12 +5871,14 @@ int mysqld_main(int argc, char **argv) mysqld_port, MYSQL_COMPILATION_COMMENT); +#ifndef _WIN32 // try to keep fd=0 busy - if (!freopen(IF_WIN("NUL","/dev/null"), "r", stdin)) + if (!freopen("/dev/null", "r", stdin)) { // fall back on failure fclose(stdin); } +#endif #if defined(_WIN32) && !defined(EMBEDDED_LIBRARY) Service.SetRunning(); From 01e89d6a86ddc26da29ad530e74c20cee0529fa5 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Wed, 21 Feb 2018 15:42:34 +0300 Subject: [PATCH 21/31] MDEV-15372: Parallel slave speedup very limited when log_slave_updates=OFF Make MyRocks' non-XA commit path to first do the commit without syncing and then sync. --- storage/rocksdb/ha_rocksdb.cc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index 4a0a3eddb1a..0231dce7cf5 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -3154,10 +3154,20 @@ static int rocksdb_commit(handlerton* hton, THD* thd, bool commit_tx) - For a COMMIT statement that finishes a multi-statement transaction - For a statement that has its own transaction */ + + // First, commit without syncing. This establishes the commit order + tx->set_sync(false); if (tx->commit()) { DBUG_RETURN(HA_ERR_ROCKSDB_COMMIT_FAILED); } thd_wakeup_subsequent_commits(thd, 0); + + if (rocksdb_flush_log_at_trx_commit == FLUSH_LOG_SYNC) + { + rocksdb::Status s= rdb->FlushWAL(true); + if (!s.ok()) + DBUG_RETURN(HA_ERR_INTERNAL_ERROR); + } } else { /* We get here when committing a statement within a transaction. From 88d1c1c5514dc922699e69d64877b7788c74ffcc Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 12 Feb 2018 15:12:49 +0100 Subject: [PATCH 22/31] MDEV-15288 Configure errors when building without INNOBASE --- storage/innobase/CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index 048a1286961..18efa58b2a3 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -489,4 +489,6 @@ MYSQL_ADD_PLUGIN(innobase ${INNOBASE_SOURCES} STORAGE_ENGINE MODULE_OUTPUT_NAME ha_innodb LINK_LIBRARIES ${ZLIB_LIBRARY} ${LINKER_SCRIPT}) -ADD_DEPENDENCIES(innobase GenError) +IF(TARGET innobase) + ADD_DEPENDENCIES(innobase GenError) +ENDIF() From 00a556c0c20f58f06c95b7976aef0b174abb30ff Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Wed, 21 Feb 2018 17:00:03 +0300 Subject: [PATCH 23/31] MDEV-15372: Parallel slave speedup very limited when log_slave_updates=OFF Part #2: some transactions have m_rocksdb_tx==NULL (and most functions of Rdb_transction_impl handle this case. Do like they do) --- storage/rocksdb/ha_rocksdb.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index 0231dce7cf5..bc15e948d41 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -2240,7 +2240,8 @@ public: } void set_sync(bool sync) override { - m_rocksdb_tx->GetWriteOptions()->sync = sync; + if (m_rocksdb_tx) + m_rocksdb_tx->GetWriteOptions()->sync = sync; } void release_lock(rocksdb::ColumnFamilyHandle *const column_family, From 4e6dab94d0931eafba502f5a91da29a54e75bb33 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 21 Feb 2018 19:38:57 +0530 Subject: [PATCH 24/31] MDEV-10.1.31 does not join an existing cluster with SST xtrabackup-v2 Analysis:- The problem is the change in the implementation of wait_for_listen in wsrep_sst_xtrabackup-v2.sh. The new script uses lsof which will always exit with an error code if it can't find all the items, and because the script has the -e option set in the hashbang line (#!/bin/bash -ue), the script will abort right after running lsof if lsof can't find even a single item among all the items listed in its arguments. This will happen even if socat is running and listening, because it can't find nc. The loop in wait_for_listen will therefore always quit after one iteration without writing the "ready" line to signal the parent. Solution:- We will or the lsof with true. Patch Credit :Daniel Black and David Wang --- scripts/wsrep_sst_xtrabackup-v2.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh index 64dd182e2f2..9104daf19bc 100644 --- a/scripts/wsrep_sst_xtrabackup-v2.sh +++ b/scripts/wsrep_sst_xtrabackup-v2.sh @@ -644,7 +644,7 @@ wait_for_listen() for i in {1..300} do - LSOF_OUT=$(lsof -sTCP:LISTEN -i TCP:${PORT} -a -c nc -c socat -F c) + LSOF_OUT=$(lsof -sTCP:LISTEN -i TCP:${PORT} -a -c nc -c socat -F c 2> /dev/null || :) [ -n "${LSOF_OUT}" ] && break sleep 0.2 done From db0484f355dae8d3a4e4aab731522521e8ff976e Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Wed, 21 Feb 2018 17:27:46 +0300 Subject: [PATCH 25/31] Change MyRocks Maturity Level from Beta to Gamma (RC) --- storage/rocksdb/ha_rocksdb.h | 2 +- .../rocksdb/r/mariadb_port_fixes.result | 24 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/storage/rocksdb/ha_rocksdb.h b/storage/rocksdb/ha_rocksdb.h index 4432a4de8d1..1769072722a 100644 --- a/storage/rocksdb/ha_rocksdb.h +++ b/storage/rocksdb/ha_rocksdb.h @@ -1412,6 +1412,6 @@ private: Rdb_inplace_alter_ctx &operator=(const Rdb_inplace_alter_ctx &); }; -const int MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL= MariaDB_PLUGIN_MATURITY_BETA; +const int MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL= MariaDB_PLUGIN_MATURITY_GAMMA; } // namespace myrocks diff --git a/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result b/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result index 021373be02a..27b1fd1e9af 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result @@ -69,15 +69,15 @@ set global rocksdb_strict_collation_check=@tmp_rscc; # select plugin_name, plugin_maturity from information_schema.plugins where plugin_name like '%rocksdb%'; plugin_name plugin_maturity -ROCKSDB Beta -ROCKSDB_CFSTATS Beta -ROCKSDB_DBSTATS Beta -ROCKSDB_PERF_CONTEXT Beta -ROCKSDB_PERF_CONTEXT_GLOBAL Beta -ROCKSDB_CF_OPTIONS Beta -ROCKSDB_COMPACTION_STATS Beta -ROCKSDB_GLOBAL_INFO Beta -ROCKSDB_DDL Beta -ROCKSDB_INDEX_FILE_MAP Beta -ROCKSDB_LOCKS Beta -ROCKSDB_TRX Beta +ROCKSDB Gamma +ROCKSDB_CFSTATS Gamma +ROCKSDB_DBSTATS Gamma +ROCKSDB_PERF_CONTEXT Gamma +ROCKSDB_PERF_CONTEXT_GLOBAL Gamma +ROCKSDB_CF_OPTIONS Gamma +ROCKSDB_COMPACTION_STATS Gamma +ROCKSDB_GLOBAL_INFO Gamma +ROCKSDB_DDL Gamma +ROCKSDB_INDEX_FILE_MAP Gamma +ROCKSDB_LOCKS Gamma +ROCKSDB_TRX Gamma From 18455ec3f1a9c22977f0ed87233852813b53eb49 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 14 Feb 2018 12:35:12 +0100 Subject: [PATCH 26/31] fix compilation wih -DPLUGIN_PARTITION=NO --- sql/partition_info.cc | 33 ++++++++++++++++----------------- storage/connect/ha_connect.cc | 16 +++++----------- 2 files changed, 21 insertions(+), 28 deletions(-) diff --git a/sql/partition_info.cc b/sql/partition_info.cc index 14f301e2d85..df0a733f121 100644 --- a/sql/partition_info.cc +++ b/sql/partition_info.cc @@ -2744,23 +2744,6 @@ end: } -bool partition_info::error_if_requires_values() const -{ - switch (part_type) { - case NOT_A_PARTITION: - case HASH_PARTITION: - break; - case RANGE_PARTITION: - my_error(ER_PARTITION_REQUIRES_VALUES_ERROR, MYF(0), "RANGE", "LESS THAN"); - return true; - case LIST_PARTITION: - my_error(ER_PARTITION_REQUIRES_VALUES_ERROR, MYF(0), "LIST", "IN"); - return true; - } - return false; -} - - /** Fix partition data from parser. @@ -3211,3 +3194,19 @@ bool check_partition_dirs(partition_info *part_info) } #endif /* WITH_PARTITION_STORAGE_ENGINE */ + +bool partition_info::error_if_requires_values() const +{ + switch (part_type) { + case NOT_A_PARTITION: + case HASH_PARTITION: + break; + case RANGE_PARTITION: + my_error(ER_PARTITION_REQUIRES_VALUES_ERROR, MYF(0), "RANGE", "LESS THAN"); + return true; + case LIST_PARTITION: + my_error(ER_PARTITION_REQUIRES_VALUES_ERROR, MYF(0), "LIST", "IN"); + return true; + } + return false; +} diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index 5964b1d049f..08a61a75048 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -1754,11 +1754,13 @@ bool ha_connect::CheckVirtualIndex(TABLE_SHARE *s) bool ha_connect::IsPartitioned(void) { +#ifdef WITH_PARTITION_STORAGE_ENGINE if (tshp) return tshp->partition_info_str_len > 0; else if (table && table->part_info) return true; else +#endif return false; } // end of IsPartitioned @@ -6165,8 +6167,10 @@ int ha_connect::create(const char *name, TABLE *table_arg, TABLE *st= table; // Probably unuseful THD *thd= ha_thd(); LEX_STRING cnc = table_arg->s->connect_string; -#if defined(WITH_PARTITION_STORAGE_ENGINE) +#ifdef WITH_PARTITION_STORAGE_ENGINE partition_info *part_info= table_arg->part_info; +#else +#define part_info 0 #endif // WITH_PARTITION_STORAGE_ENGINE xp= GetUser(thd, xp); PGLOBAL g= xp->g; @@ -6268,9 +6272,7 @@ int ha_connect::create(const char *name, TABLE *table_arg, // fall through case TAB_MYSQL: -#if defined(WITH_PARTITION_STORAGE_ENGINE) if (!part_info) -#endif // WITH_PARTITION_STORAGE_ENGINE {const char *src= options->srcdef; PCSZ host, db, tab= options->tabname; int port; @@ -6534,7 +6536,6 @@ int ha_connect::create(const char *name, TABLE *table_arg, } else lwt[i]= tolower(options->type[i]); -#if defined(WITH_PARTITION_STORAGE_ENGINE) if (part_info) { char *p; @@ -6544,7 +6545,6 @@ int ha_connect::create(const char *name, TABLE *table_arg, strcat(strcat(strcpy(buf, p), "."), lwt); *p= 0; } else { -#endif // WITH_PARTITION_STORAGE_ENGINE strcat(strcat(strcpy(buf, GetTableName()), "."), lwt); sprintf(g->Message, "No file name. Table will use %s", buf); @@ -6552,9 +6552,7 @@ int ha_connect::create(const char *name, TABLE *table_arg, push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 0, g->Message); strcat(strcat(strcpy(dbpath, "./"), table->s->db.str), "/"); -#if defined(WITH_PARTITION_STORAGE_ENGINE) } // endif part_info -#endif // WITH_PARTITION_STORAGE_ENGINE PlugSetPath(fn, buf, dbpath); @@ -6619,11 +6617,9 @@ int ha_connect::create(const char *name, TABLE *table_arg, push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 0, "Unexpected command in create, please contact CONNECT team"); -#if defined(WITH_PARTITION_STORAGE_ENGINE) if (part_info && !inward) strncpy(partname, decode(g, strrchr(name, '#') + 1), sizeof(partname) - 1); // strcpy(partname, part_info->curr_part_elem->partition_name); -#endif // WITH_PARTITION_STORAGE_ENGINE if (g->Alchecked == 0 && (!IsFileType(type) || FileExists(options->filename, false))) { @@ -6659,12 +6655,10 @@ int ha_connect::create(const char *name, TABLE *table_arg, my_message(ER_UNKNOWN_ERROR, g->Message, MYF(0)); rc = HA_ERR_INTERNAL_ERROR; } else if (cat) { -#if defined(WITH_PARTITION_STORAGE_ENGINE) if (part_info) strncpy(partname, decode(g, strrchr(name, (inward ? slash : '#')) + 1), sizeof(partname) - 1); -#endif // WITH_PARTITION_STORAGE_ENGINE if ((rc= optimize(table->in_use, NULL))) { htrc("Create rc=%d %s\n", rc, g->Message); From 50359719f0dd1f8f6d3d256a0db107654f8d39f1 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 14 Feb 2018 12:35:47 +0100 Subject: [PATCH 27/31] fix compilation -DWITH_PERFSCHEMA=NO tokudb apparently requires perfschema now --- storage/tokudb/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/storage/tokudb/CMakeLists.txt b/storage/tokudb/CMakeLists.txt index 7ea98728034..1cd86eb5808 100644 --- a/storage/tokudb/CMakeLists.txt +++ b/storage/tokudb/CMakeLists.txt @@ -4,6 +4,8 @@ IF(CMAKE_VERSION VERSION_LESS "2.8.9") MESSAGE(STATUS "CMake 2.8.9 or higher is required by TokuDB") ELSEIF(NOT HAVE_DLOPEN) MESSAGE(STATUS "dlopen is required by TokuDB") +ELSEIF(NOT TARGET perfschema) + MESSAGE(STATUS "Performance Schema is required by TokuDB") ELSEIF(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64") # tokudb requires F_NOCACHE or O_DIRECT, and designated initializers From 5d8ac1ece1687278cbc860e97eae9c7cc163ee73 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 14 Feb 2018 13:30:52 +0100 Subject: [PATCH 28/31] log all mtr output in vardir/log/stdout.log despite the name, it logs both stdout and stderr --- mysql-test/lib/My/SafeProcess/Base.pm | 12 ++++++++---- mysql-test/lib/My/Tee.pm | 23 +++++++++++++++++++++++ mysql-test/mysql-test-run.pl | 9 ++++++++- 3 files changed, 39 insertions(+), 5 deletions(-) create mode 100644 mysql-test/lib/My/Tee.pm diff --git a/mysql-test/lib/My/SafeProcess/Base.pm b/mysql-test/lib/My/SafeProcess/Base.pm index 1ac0120a735..c5ac2ab51c2 100644 --- a/mysql-test/lib/My/SafeProcess/Base.pm +++ b/mysql-test/lib/My/SafeProcess/Base.pm @@ -186,8 +186,10 @@ sub create_process { # it and any childs(that hasn't changed group themself) setpgrp(0,0) if $opts{setpgrp}; - if ( $output and !open(STDOUT, $open_mode, $output) ) { - croak("can't redirect STDOUT to '$output': $!"); + if ( $output ) { + close STDOUT; + open(STDOUT, $open_mode, $output) + or croak "can't redirect STDOUT to '$output': $!"; } if ( $error ) { @@ -196,8 +198,10 @@ sub create_process { croak("can't dup STDOUT: $!"); } } - elsif ( ! open(STDERR, $open_mode, $error) ) { - croak("can't redirect STDERR to '$error': $!"); + else { + close STDERR; + open(STDERR, $open_mode, $error) + or croak "can't redirect STDERR to '$error': $!"; } } diff --git a/mysql-test/lib/My/Tee.pm b/mysql-test/lib/My/Tee.pm new file mode 100644 index 00000000000..ee82e6f45ae --- /dev/null +++ b/mysql-test/lib/My/Tee.pm @@ -0,0 +1,23 @@ +package My::Tee; + +# see PerlIO::via + +our $copyfh; + +sub PUSHED +{ + open($copyfh, '>', "$::opt_vardir/log/stdout.log") + or die "open(>$::opt_vardir/log/stdout.log): $!" + unless $copyfh; + bless { }, shift; +} + +sub WRITE +{ + my ($obj, $buf, $fh) = @_; + print $fh $buf; + print $copyfh $buf; + return length($buf); +} + +1; diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index eaec51b82b4..2581eb7b503 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -91,6 +91,7 @@ use My::Platform; use My::SafeProcess; use My::ConfigFactory; use My::Options; +use My::Tee; use My::Find; use My::SysInfo; use My::CoreDump; @@ -392,6 +393,11 @@ sub main { initialize_servers(); init_timers(); + unless (IS_WINDOWS) { + binmode(STDOUT,":via(My::Tee)") or die "binmode(STDOUT, :via(My::Tee)):$!"; + binmode(STDERR,":via(My::Tee)") or die "binmode(STDERR, :via(My::Tee)):$!"; + } + mtr_report("Checking supported features..."); executable_setup(); @@ -6280,7 +6286,8 @@ sub xterm_stat { my $done = $num_tests - $left; my $spent = time - $^T; - printf "\e];mtr: spent %s on %d tests. %s (%d tests) left\a", + syswrite STDOUT, sprintf + "\e];mtr: spent %s on %d tests. %s (%d tests) left\a", time_format($spent), $done, time_format($spent/$done * $left), $left; } From e119799a920e8096886a51ded305bf06609438a4 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 22 Feb 2018 08:40:54 +0100 Subject: [PATCH 29/31] fix compilation wih -DPLUGIN_PARTITION=NO rocksdb and spider --- storage/rocksdb/ha_rocksdb.cc | 4 ++-- storage/rocksdb/rdb_datadic.cc | 2 +- storage/spider/CMakeLists.txt | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index bc15e948d41..5e4152f7dcf 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -6156,7 +6156,7 @@ int ha_rocksdb::create_cfs( // Internal consistency check to make sure that data in TABLE and // Rdb_tbl_def structures matches. Either both are missing or both are // specified. Yes, this is critical enough to make it into SHIP_ASSERT. - SHIP_ASSERT(!table_arg->part_info == tbl_def_arg->base_partition().empty()); + SHIP_ASSERT(IF_PARTITIONING(!table_arg->part_info,true) == tbl_def_arg->base_partition().empty()); // Generate the name for the column family to use. bool per_part_match_found = false; @@ -8395,7 +8395,7 @@ const std::string ha_rocksdb::generate_cf_name(const uint index, key_comment, table_arg, tbl_def_arg, per_part_match_found, RDB_CF_NAME_QUALIFIER); - if (table_arg->part_info != nullptr && !*per_part_match_found) { + if (IF_PARTITIONING(table_arg->part_info,nullptr) != nullptr && !*per_part_match_found) { // At this point we tried to search for a custom CF name for a partition, // but none was specified. Therefore default one will be used. return ""; diff --git a/storage/rocksdb/rdb_datadic.cc b/storage/rocksdb/rdb_datadic.cc index b2f5af705a3..68ad8e091b2 100644 --- a/storage/rocksdb/rdb_datadic.cc +++ b/storage/rocksdb/rdb_datadic.cc @@ -538,7 +538,7 @@ const std::string Rdb_key_def::parse_comment_for_qualifier( // NOTE: this means if you specify a qualifier for a specific partition it // will take precedence the 'table level' qualifier if one exists. std::string search_str_part; - if (table_arg->part_info != nullptr) { + if (IF_PARTITIONING(table_arg->part_info,nullptr) != nullptr) { std::string partition_name = tbl_def_arg->base_partition(); DBUG_ASSERT(!partition_name.empty()); search_str_part = gen_qualifier_for_table(qualifier, partition_name); diff --git a/storage/spider/CMakeLists.txt b/storage/spider/CMakeLists.txt index 14c50f35bd9..dec1cb4c6ba 100644 --- a/storage/spider/CMakeLists.txt +++ b/storage/spider/CMakeLists.txt @@ -34,6 +34,8 @@ IF(EXISTS ${PROJECT_SOURCE_DIR}/storage/mysql_storage_engine.cmake) ${CMAKE_SOURCE_DIR}/regex) MYSQL_STORAGE_ENGINE(SPIDER) +ELSEIF(PLUGIN_PARTITION MATCHES "^NO$") + MESSAGE(STATUS "Spider is skipped because partitioning is disabled") ELSE() INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/storage/spider/hs_client) From 504beb132523511e93e91cc422bcd413ca556828 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 22 Feb 2018 12:20:46 +0200 Subject: [PATCH 30/31] Add supressions for possible warning. --- mysql-test/suite/galera/r/galera_var_node_address.result | 1 + mysql-test/suite/galera/t/galera_var_node_address.test | 1 + 2 files changed, 2 insertions(+) diff --git a/mysql-test/suite/galera/r/galera_var_node_address.result b/mysql-test/suite/galera/r/galera_var_node_address.result index cf36e964f93..a9c900d8bc8 100644 --- a/mysql-test/suite/galera/r/galera_var_node_address.result +++ b/mysql-test/suite/galera/r/galera_var_node_address.result @@ -1,5 +1,6 @@ call mtr.add_suppression("WSREP: Stray state UUID msg: .* current group state WAIT_STATE_UUID .*"); call mtr.add_suppression("WSREP: Protocol violation. JOIN message sender .* is not in state transfer (.*). Message ignored."); +call mtr.add_suppression("WSREP: Sending JOIN failed: -[0-9]+ (Transport endpoint is not connected). Will retry in new primary component."); SELECT VARIABLE_VALUE = 4 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; VARIABLE_VALUE = 4 1 diff --git a/mysql-test/suite/galera/t/galera_var_node_address.test b/mysql-test/suite/galera/t/galera_var_node_address.test index 3353652d8b9..22e98e3aa82 100644 --- a/mysql-test/suite/galera/t/galera_var_node_address.test +++ b/mysql-test/suite/galera/t/galera_var_node_address.test @@ -8,6 +8,7 @@ call mtr.add_suppression("WSREP: Stray state UUID msg: .* current group state WAIT_STATE_UUID .*"); call mtr.add_suppression("WSREP: Protocol violation. JOIN message sender .* is not in state transfer (.*). Message ignored."); +call mtr.add_suppression("WSREP: Sending JOIN failed: -[0-9]+ (Transport endpoint is not connected). Will retry in new primary component."); SELECT VARIABLE_VALUE = 4 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; From e92cc097658405fdb52c80254b933424f8c1f99e Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 22 Feb 2018 15:58:07 +0100 Subject: [PATCH 31/31] MDEV-15345 Compilation fails to build my_addr_resolve.c fix the compilation error. no support for plugins yet. --- mysys/my_addr_resolve.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mysys/my_addr_resolve.c b/mysys/my_addr_resolve.c index 10f8552f226..84bff47d1a9 100644 --- a/mysys/my_addr_resolve.c +++ b/mysys/my_addr_resolve.c @@ -49,6 +49,13 @@ static const char *strip_path(const char *s) static bfd *bfdh= 0; static asymbol **symtable= 0; +#if defined(HAVE_LINK_H) && defined(HAVE_DLOPEN) +#include +static ElfW(Addr) offset= 0; +#else +#define offset 0 +#endif + /** finds a file name, a line number, and a function name corresponding to addr.