From 7c6cf7fefe68a1a3f68e7d6436da4689ec302bca Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 25 Jan 2018 14:25:48 +0100 Subject: [PATCH 01/37] bug: ha_heap was unilaterally increasing reclength proper fix replacing the hack from b80fa4000d6 don't confuse length of the data area (reclength) with the offset to the "deleted" mark. --- include/heap.h | 1 + storage/heap/_check.c | 2 +- storage/heap/ha_heap.cc | 11 +---------- storage/heap/hp_create.c | 8 +++++--- storage/heap/hp_delete.c | 2 +- storage/heap/hp_rrnd.c | 4 ++-- storage/heap/hp_rsame.c | 2 +- storage/heap/hp_scan.c | 2 +- storage/heap/hp_write.c | 4 ++-- 9 files changed, 15 insertions(+), 21 deletions(-) diff --git a/include/heap.h b/include/heap.h index 2b7fd28699d..eecb7084e66 100644 --- a/include/heap.h +++ b/include/heap.h @@ -144,6 +144,7 @@ typedef struct st_heap_share uint key_version; /* Updated on key change */ uint file_version; /* Update on clear */ uint reclength; /* Length of one record */ + uint visible; /* Offset to the visible/deleted mark */ uint changed; uint keys,max_key_length; uint currently_disabled_keys; /* saved value from "keys" when disabled */ diff --git a/storage/heap/_check.c b/storage/heap/_check.c index b64c9ab1831..f2fd8ab94be 100644 --- a/storage/heap/_check.c +++ b/storage/heap/_check.c @@ -79,7 +79,7 @@ int heap_check_heap(HP_INFO *info, my_bool print_status) } hp_find_record(info,pos); - if (!info->current_ptr[share->reclength]) + if (!info->current_ptr[share->visible]) deleted++; else records++; diff --git a/storage/heap/ha_heap.cc b/storage/heap/ha_heap.cc index 259e54bfc59..ec76d08bf97 100644 --- a/storage/heap/ha_heap.cc +++ b/storage/heap/ha_heap.cc @@ -100,15 +100,6 @@ const char **ha_heap::bas_ext() const int ha_heap::open(const char *name, int mode, uint test_if_locked) { - if (table->s->reclength < sizeof (char*)) - { - MEM_UNDEFINED(table->s->default_values + table->s->reclength, - sizeof(char*) - table->s->reclength); - table->s->reclength= sizeof(char*); - MEM_UNDEFINED(table->record[0], table->s->reclength); - MEM_UNDEFINED(table->record[1], table->s->reclength); - } - internal_table= test(test_if_locked & HA_OPEN_INTERNAL_TABLE); if (internal_table || (!(file= heap_open(name, mode)) && my_errno == ENOENT)) { @@ -736,7 +727,7 @@ heap_prepare_hp_create_info(TABLE *table_arg, bool internal_table, } } } - mem_per_row+= MY_ALIGN(share->reclength + 1, sizeof(char*)); + mem_per_row+= MY_ALIGN(max(share->reclength, sizeof(char*)) + 1, sizeof(char*)); if (table_arg->found_next_number_field) { keydef[share->next_number_index].flag|= HA_AUTO_KEY; diff --git a/storage/heap/hp_create.c b/storage/heap/hp_create.c index 0b5dc841ada..1daca0beeb5 100644 --- a/storage/heap/hp_create.c +++ b/storage/heap/hp_create.c @@ -33,6 +33,7 @@ int heap_create(const char *name, HP_CREATE_INFO *create_info, uint keys= create_info->keys; ulong min_records= create_info->min_records; ulong max_records= create_info->max_records; + uint visible_offset; DBUG_ENTER("heap_create"); if (!create_info->internal_table) @@ -58,9 +59,9 @@ int heap_create(const char *name, HP_CREATE_INFO *create_info, /* We have to store sometimes uchar* del_link in records, - so the record length should be at least sizeof(uchar*) + so the visible_offset must be least at sizeof(uchar*) */ - set_if_bigger(reclength, sizeof (uchar*)); + visible_offset= max(reclength, sizeof (char*)); for (i= key_segs= max_length= 0, keyinfo= keydef; i < keys; i++, keyinfo++) { @@ -152,7 +153,7 @@ int heap_create(const char *name, HP_CREATE_INFO *create_info, share->keydef= (HP_KEYDEF*) (share + 1); share->key_stat_version= 1; keyseg= (HA_KEYSEG*) (share->keydef + keys); - init_block(&share->block, reclength + 1, min_records, max_records); + init_block(&share->block, visible_offset + 1, min_records, max_records); /* Fix keys */ memcpy(share->keydef, keydef, (size_t) (sizeof(keydef[0]) * keys)); for (i= 0, keyinfo= share->keydef; i < keys; i++, keyinfo++) @@ -192,6 +193,7 @@ int heap_create(const char *name, HP_CREATE_INFO *create_info, share->max_table_size= create_info->max_table_size; share->data_length= share->index_length= 0; share->reclength= reclength; + share->visible= visible_offset; share->blength= 1; share->keys= keys; share->max_key_length= max_length; diff --git a/storage/heap/hp_delete.c b/storage/heap/hp_delete.c index 1cbfe7408d4..eb9749601aa 100644 --- a/storage/heap/hp_delete.c +++ b/storage/heap/hp_delete.c @@ -45,7 +45,7 @@ int heap_delete(HP_INFO *info, const uchar *record) info->update=HA_STATE_DELETED; *((uchar**) pos)=share->del_link; share->del_link=pos; - pos[share->reclength]=0; /* Record deleted */ + pos[share->visible]=0; /* Record deleted */ share->deleted++; share->key_version++; #if !defined(DBUG_OFF) && defined(EXTRA_HEAP_DEBUG) diff --git a/storage/heap/hp_rrnd.c b/storage/heap/hp_rrnd.c index 8e0d51a78ca..d105c5c2e13 100644 --- a/storage/heap/hp_rrnd.c +++ b/storage/heap/hp_rrnd.c @@ -37,7 +37,7 @@ int heap_rrnd(register HP_INFO *info, uchar *record, uchar *pos) info->update= 0; DBUG_RETURN(my_errno= HA_ERR_END_OF_FILE); } - if (!info->current_ptr[share->reclength]) + if (!info->current_ptr[share->visible]) { info->update= HA_STATE_PREV_FOUND | HA_STATE_NEXT_FOUND; DBUG_RETURN(my_errno=HA_ERR_RECORD_DELETED); @@ -91,7 +91,7 @@ int heap_rrnd_old(register HP_INFO *info, uchar *record, ulong pos) hp_find_record(info, pos); end: - if (!info->current_ptr[share->reclength]) + if (!info->current_ptr[share->visible]) { info->update= HA_STATE_PREV_FOUND | HA_STATE_NEXT_FOUND; DBUG_RETURN(my_errno=HA_ERR_RECORD_DELETED); diff --git a/storage/heap/hp_rsame.c b/storage/heap/hp_rsame.c index 40c5a18f974..19767fd752b 100644 --- a/storage/heap/hp_rsame.c +++ b/storage/heap/hp_rsame.c @@ -32,7 +32,7 @@ int heap_rsame(register HP_INFO *info, uchar *record, int inx) DBUG_ENTER("heap_rsame"); test_active(info); - if (info->current_ptr[share->reclength]) + if (info->current_ptr[share->visible]) { if (inx < -1 || inx >= (int) share->keys) { diff --git a/storage/heap/hp_scan.c b/storage/heap/hp_scan.c index 39a6f2082a3..65726c37e1f 100644 --- a/storage/heap/hp_scan.c +++ b/storage/heap/hp_scan.c @@ -62,7 +62,7 @@ int heap_scan(register HP_INFO *info, uchar *record) } hp_find_record(info, pos); } - if (!info->current_ptr[share->reclength]) + if (!info->current_ptr[share->visible]) { DBUG_PRINT("warning",("Found deleted record")); info->update= HA_STATE_PREV_FOUND | HA_STATE_NEXT_FOUND; diff --git a/storage/heap/hp_write.c b/storage/heap/hp_write.c index e45c78b75f1..4a9c1ba59c0 100644 --- a/storage/heap/hp_write.c +++ b/storage/heap/hp_write.c @@ -54,7 +54,7 @@ int heap_write(HP_INFO *info, const uchar *record) } memcpy(pos,record,(size_t) share->reclength); - pos[share->reclength]=1; /* Mark record as not deleted */ + pos[share->visible]= 1; /* Mark record as not deleted */ if (++share->records == share->blength) share->blength+= share->blength; info->s->key_version++; @@ -92,7 +92,7 @@ err: share->deleted++; *((uchar**) pos)=share->del_link; share->del_link=pos; - pos[share->reclength]=0; /* Record deleted */ + pos[share->visible]= 0; /* Record deleted */ DBUG_RETURN(my_errno); } /* heap_write */ From c8afe7daaced3339f24b88d63f6bcf8477dfaf8c Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 5 Feb 2018 14:13:26 +0100 Subject: [PATCH 02/37] cleanup: remove a duplicated test case --- mysql-test/r/view.result | 108 ------------------------------------- mysql-test/t/view.test | 112 --------------------------------------- 2 files changed, 220 deletions(-) diff --git a/mysql-test/r/view.result b/mysql-test/r/view.result index 8bc33a8860b..1d12c50954f 100644 --- a/mysql-test/r/view.result +++ b/mysql-test/r/view.result @@ -5236,114 +5236,6 @@ execute stmt1; deallocate prepare stmt1; drop view v1,v2; drop table t1,t2; -# -# MDEV-6251: SIGSEGV in query optimizer (in set_check_materialized -# with MERGE view) -# -CREATE TABLE t1 (a1 INT(11) NOT NULL DEFAULT NULL AUTO_INCREMENT PRIMARY KEY); -CREATE TABLE t2 (b1 INT(11) NOT NULL DEFAULT NULL AUTO_INCREMENT PRIMARY KEY); -CREATE TABLE t3 (c1 INT(11) NOT NULL DEFAULT NULL AUTO_INCREMENT PRIMARY KEY); -CREATE TABLE t4 (d1 INT(11) NOT NULL DEFAULT NULL AUTO_INCREMENT PRIMARY KEY); -CREATE TABLE t5 (e1 INT(11) NOT NULL DEFAULT NULL AUTO_INCREMENT PRIMARY KEY); -CREATE TABLE t6 (f1 INT(11) NOT NULL DEFAULT NULL AUTO_INCREMENT PRIMARY KEY); -CREATE OR REPLACE view v1 AS -SELECT 1 -FROM t1 a_alias_1 -LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 -LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 -LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 -LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -; -SELECT 1 -FROM (( SELECT 1 -FROM t1 a_alias_1 -LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 -LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 -LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 -LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -) t1) -LEFT OUTER JOIN (( SELECT 1 -FROM t1 a_alias_1 -LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 -LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 -LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 -LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -) t2) ON 1=1 -LEFT OUTER JOIN (( SELECT 1 -FROM t1 a_alias_1 -LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 -LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 -LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 -LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -) t3) ON 1=1 -LEFT OUTER JOIN (( SELECT 1 -FROM t1 a_alias_1 -LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 -LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 -LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 -LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -) t4) ON 1=1 -LEFT OUTER JOIN (( SELECT 1 -FROM t1 a_alias_1 -LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 -LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 -LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 -LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -) t5) ON 1=1 -LEFT OUTER JOIN (( SELECT 1 -FROM t1 a_alias_1 -LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 -LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 -LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 -LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -) t6) ON 1=1 -LEFT OUTER JOIN (( SELECT 1 -FROM t1 a_alias_1 -LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 -LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 -LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 -LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -) t7) ON 1=1 -LEFT OUTER JOIN (( SELECT 1 -FROM t1 a_alias_1 -LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 -LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 -LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 -LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 -LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -) t8) ON 1=1 -; -1 -SELECT 1 -FROM (v1 t1) -LEFT OUTER JOIN (v1 t2) ON 1=1 -LEFT OUTER JOIN (v1 t3) ON 1=1 -LEFT OUTER JOIN (v1 t4) ON 1=1 -LEFT OUTER JOIN (v1 t5) ON 1=1 -LEFT OUTER JOIN (v1 t6) ON 1=1 -LEFT OUTER JOIN (v1 t7) ON 1=1 -LEFT OUTER JOIN (v1 t8) ON 1=1 -; -1 -drop view v1; -drop table t1,t2,t3,t4,t5,t6; # ----------------------------------------------------------------- # -- End of 5.3 tests. # ----------------------------------------------------------------- diff --git a/mysql-test/t/view.test b/mysql-test/t/view.test index 0fc7cb6adf7..4b3537df2de 100644 --- a/mysql-test/t/view.test +++ b/mysql-test/t/view.test @@ -5169,118 +5169,6 @@ deallocate prepare stmt1; drop view v1,v2; drop table t1,t2; ---echo # ---echo # MDEV-6251: SIGSEGV in query optimizer (in set_check_materialized ---echo # with MERGE view) ---echo # - -CREATE TABLE t1 (a1 INT(11) NOT NULL DEFAULT NULL AUTO_INCREMENT PRIMARY KEY); -CREATE TABLE t2 (b1 INT(11) NOT NULL DEFAULT NULL AUTO_INCREMENT PRIMARY KEY); -CREATE TABLE t3 (c1 INT(11) NOT NULL DEFAULT NULL AUTO_INCREMENT PRIMARY KEY); -CREATE TABLE t4 (d1 INT(11) NOT NULL DEFAULT NULL AUTO_INCREMENT PRIMARY KEY); -CREATE TABLE t5 (e1 INT(11) NOT NULL DEFAULT NULL AUTO_INCREMENT PRIMARY KEY); -CREATE TABLE t6 (f1 INT(11) NOT NULL DEFAULT NULL AUTO_INCREMENT PRIMARY KEY); - -CREATE OR REPLACE view v1 AS - SELECT 1 - FROM t1 a_alias_1 - LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 - LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 - LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 - LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -; - -SELECT 1 -FROM (( SELECT 1 - FROM t1 a_alias_1 - LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 - LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 - LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 - LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -) t1) -LEFT OUTER JOIN (( SELECT 1 - FROM t1 a_alias_1 - LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 - LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 - LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 - LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -) t2) ON 1=1 -LEFT OUTER JOIN (( SELECT 1 - FROM t1 a_alias_1 - LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 - LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 - LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 - LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -) t3) ON 1=1 -LEFT OUTER JOIN (( SELECT 1 - FROM t1 a_alias_1 - LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 - LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 - LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 - LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -) t4) ON 1=1 -LEFT OUTER JOIN (( SELECT 1 - FROM t1 a_alias_1 - LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 - LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 - LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 - LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -) t5) ON 1=1 -LEFT OUTER JOIN (( SELECT 1 - FROM t1 a_alias_1 - LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 - LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 - LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 - LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -) t6) ON 1=1 -LEFT OUTER JOIN (( SELECT 1 - FROM t1 a_alias_1 - LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 - LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 - LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 - LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -) t7) ON 1=1 -LEFT OUTER JOIN (( SELECT 1 - FROM t1 a_alias_1 - LEFT JOIN (t2 b_alias_1 JOIN t1 a_alias_2) ON b_alias_1.b1 = a_alias_1.a1 AND a_alias_2.a1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_1 ON c_alias_1.c1 = a_alias_1.a1 - LEFT JOIN t4 d_alias_1 ON d_alias_1.d1 = a_alias_1.a1 - LEFT JOIN t3 c_alias_2 ON c_alias_2.c1 = a_alias_1.a1 - LEFT JOIN t5 e_alias_1 ON e_alias_1.e1 = a_alias_1.a1 - LEFT JOIN t6 f_alias_1 ON f_alias_1.f1 = a_alias_1.a1 -) t8) ON 1=1 -; - -SELECT 1 -FROM (v1 t1) -LEFT OUTER JOIN (v1 t2) ON 1=1 -LEFT OUTER JOIN (v1 t3) ON 1=1 -LEFT OUTER JOIN (v1 t4) ON 1=1 -LEFT OUTER JOIN (v1 t5) ON 1=1 -LEFT OUTER JOIN (v1 t6) ON 1=1 -LEFT OUTER JOIN (v1 t7) ON 1=1 -LEFT OUTER JOIN (v1 t8) ON 1=1 -; - -drop view v1; -drop table t1,t2,t3,t4,t5,t6; - --echo # ----------------------------------------------------------------- --echo # -- End of 5.3 tests. --echo # ----------------------------------------------------------------- From 2709380587bbcbc7abda77f11ee0abd207c65027 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 14 Feb 2018 18:14:24 +0100 Subject: [PATCH 03/37] MDEV-13748 Assertion `status_var.local_memory_used == 0 || !debug_assert_on_not_freed_memory' failed in virtual THD::~THD after query with INTERSECT my_safe_alloca()/my_safe_afree() work as alloca() or malloc()/free() depending on the memory size to allocate, that is, depending on reclength here. They only work correctly if reclength doesn't change in the middle. --- mysql-test/suite/maria/dynamic.result | 4 ++++ mysql-test/suite/maria/dynamic.test | 7 +++++++ storage/maria/ma_dynrec.c | 10 +++++----- 3 files changed, 16 insertions(+), 5 deletions(-) create mode 100644 mysql-test/suite/maria/dynamic.result create mode 100644 mysql-test/suite/maria/dynamic.test diff --git a/mysql-test/suite/maria/dynamic.result b/mysql-test/suite/maria/dynamic.result new file mode 100644 index 00000000000..1e87010e9ca --- /dev/null +++ b/mysql-test/suite/maria/dynamic.result @@ -0,0 +1,4 @@ +create table t1 (a blob, b varchar(20000)) engine=aria row_format=dynamic; +insert t1 (b) values (repeat('a', 20000)); +update t1 set b='b'; +drop table t1; diff --git a/mysql-test/suite/maria/dynamic.test b/mysql-test/suite/maria/dynamic.test new file mode 100644 index 00000000000..f8a1e98cd41 --- /dev/null +++ b/mysql-test/suite/maria/dynamic.test @@ -0,0 +1,7 @@ +# +# MDEV-13748 Assertion `status_var.local_memory_used == 0 || !debug_assert_on_not_freed_memory' failed in virtual THD::~THD after query with INTERSECT +# +create table t1 (a blob, b varchar(20000)) engine=aria row_format=dynamic; +insert t1 (b) values (repeat('a', 20000)); +update t1 set b='b'; +drop table t1; diff --git a/storage/maria/ma_dynrec.c b/storage/maria/ma_dynrec.c index c47da42b555..35a5040f09d 100644 --- a/storage/maria/ma_dynrec.c +++ b/storage/maria/ma_dynrec.c @@ -276,7 +276,7 @@ my_bool _ma_update_blob_record(MARIA_HA *info, MARIA_RECORD_POS pos, { uchar *rec_buff; int error; - ulong reclength,extra; + ulong reclength,reclength2,extra; extra= (ALIGN_SIZE(MARIA_MAX_DYN_BLOCK_HEADER)+MARIA_SPLIT_LENGTH+ MARIA_DYN_DELETE_BLOCK_HEADER); @@ -289,17 +289,17 @@ my_bool _ma_update_blob_record(MARIA_HA *info, MARIA_RECORD_POS pos, return 1; } #endif - if (!(rec_buff=(uchar*) my_safe_alloca(reclength, - MARIA_MAX_RECORD_ON_STACK))) + if (!(rec_buff=(uchar*) my_safe_alloca(reclength, MARIA_MAX_RECORD_ON_STACK))) { my_errno= HA_ERR_OUT_OF_MEM; /* purecov: inspected */ return(1); } - reclength= _ma_rec_pack(info,rec_buff+ALIGN_SIZE(MARIA_MAX_DYN_BLOCK_HEADER), + reclength2= _ma_rec_pack(info,rec_buff+ALIGN_SIZE(MARIA_MAX_DYN_BLOCK_HEADER), record); + DBUG_ASSERT(reclength2 <= reclength); error=update_dynamic_record(info,pos, rec_buff+ALIGN_SIZE(MARIA_MAX_DYN_BLOCK_HEADER), - reclength); + reclength2); my_safe_afree(rec_buff, reclength, MARIA_MAX_RECORD_ON_STACK); return(error != 0); } From 03de234baf423eaefe3d0b768dccb719b1298ff6 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 14 Feb 2018 19:12:23 +0100 Subject: [PATCH 04/37] MDEV-13982 Server crashes in in ha_partition::engine_name use the correct handlerton when looking for TRANSACTIONAL=1 support --- mysql-test/suite/parts/r/partition_alter_maria.result | 9 +++++++++ mysql-test/suite/parts/t/partition_alter_maria.test | 7 +++++++ sql/sql_table.cc | 2 +- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/mysql-test/suite/parts/r/partition_alter_maria.result b/mysql-test/suite/parts/r/partition_alter_maria.result index 460d20b9255..d79bc0a41fe 100644 --- a/mysql-test/suite/parts/r/partition_alter_maria.result +++ b/mysql-test/suite/parts/r/partition_alter_maria.result @@ -16,6 +16,15 @@ select * from t1; pk dt 1 2017-09-28 15:12:00 drop table t1; +create table t1 (a int) engine=Aria transactional=1 partition by hash(a) partitions 2; +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL +) ENGINE=Aria DEFAULT CHARSET=latin1 TRANSACTIONAL=1 +/*!50100 PARTITION BY HASH (a) +PARTITIONS 2 */ +drop table t1; # # MDEV-13788 Server crash when issuing bad SQL partition syntax # diff --git a/mysql-test/suite/parts/t/partition_alter_maria.test b/mysql-test/suite/parts/t/partition_alter_maria.test index e21f0dfab82..e0b9256391d 100644 --- a/mysql-test/suite/parts/t/partition_alter_maria.test +++ b/mysql-test/suite/parts/t/partition_alter_maria.test @@ -17,5 +17,12 @@ alter table t1 drop partition p20181231; select * from t1; drop table t1; +# +# MDEV-13982 Server crashes in in ha_partition::engine_name +# +create table t1 (a int) engine=Aria transactional=1 partition by hash(a) partitions 2; +show create table t1; +drop table t1; + --let $engine=Aria --source inc/part_alter_values.inc diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 19093d9b2ca..f56781faf39 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -4397,7 +4397,7 @@ bool mysql_create_table_no_lock(THD *thd, /* Give warnings for not supported table options */ #if defined(WITH_ARIA_STORAGE_ENGINE) extern handlerton *maria_hton; - if (file->ht != maria_hton) + if (file->partition_ht() != maria_hton) #endif if (create_info->transactional) push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_WARN, From 7bd258c44c00f232ecf4ec448179f114b878227b Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 15 Feb 2018 10:06:14 +0100 Subject: [PATCH 05/37] fix plugins.server_audit test for --ps --- mysql-test/suite/plugins/t/server_audit.test | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mysql-test/suite/plugins/t/server_audit.test b/mysql-test/suite/plugins/t/server_audit.test index 9be0d5556f0..6c5eaffd9a2 100644 --- a/mysql-test/suite/plugins/t/server_audit.test +++ b/mysql-test/suite/plugins/t/server_audit.test @@ -42,8 +42,10 @@ select 1, 3; insert into t2 values (1), (2); select * from t2; +--disable_ps_protocol --error ER_NO_SUCH_TABLE select * from t_doesnt_exist; +--enable_ps_protocol --error 1064 syntax_error_query; drop table renamed_t1, t2; From f853b8ed26623200405abbc943e82295ad9f6ab3 Mon Sep 17 00:00:00 2001 From: Monty Date: Sat, 17 Feb 2018 17:47:18 +0200 Subject: [PATCH 06/37] partition_alter_myisam produces warning if no symlink support --- mysql-test/suite/parts/t/partition_alter_myisam.test | 1 + 1 file changed, 1 insertion(+) diff --git a/mysql-test/suite/parts/t/partition_alter_myisam.test b/mysql-test/suite/parts/t/partition_alter_myisam.test index b2bd0e72e4c..7e984921c1c 100644 --- a/mysql-test/suite/parts/t/partition_alter_myisam.test +++ b/mysql-test/suite/parts/t/partition_alter_myisam.test @@ -1,4 +1,5 @@ --source include/have_partition.inc +--source include/have_symlink.inc --let $engine=MyISAM --source inc/part_alter_values.inc From 965e16376c251010f6c4d7182159acacfe198ff8 Mon Sep 17 00:00:00 2001 From: Monty Date: Sat, 17 Feb 2018 17:48:23 +0200 Subject: [PATCH 07/37] TokuDB didn't compile with valgrind TokuDB uses USE_VALGRIND while MariaDB uses HAVE_valgrind Fixed by defining USE_VALGRIND in TokuDB if HAVE_valgrind is used --- storage/tokudb/PerconaFT/portability/toku_race_tools.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/storage/tokudb/PerconaFT/portability/toku_race_tools.h b/storage/tokudb/PerconaFT/portability/toku_race_tools.h index 8482a164fb8..9ed46ec909d 100644 --- a/storage/tokudb/PerconaFT/portability/toku_race_tools.h +++ b/storage/tokudb/PerconaFT/portability/toku_race_tools.h @@ -40,6 +40,11 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #include +#ifdef HAVE_valgrind +#undef USE_VALGRIND +#define USE_VALGRIND 1 +#endif + #if defined(__linux__) && USE_VALGRIND # include From 55bc3f1dd9daf50fb3bda53cd09dbfdc5a60f977 Mon Sep 17 00:00:00 2001 From: Monty Date: Sat, 17 Feb 2018 18:04:06 +0200 Subject: [PATCH 08/37] Fixed performance problem with Aria in find_head() For some simple benchmarks, a majority of time was spend in find_head() which tries to find the best place to put the record. The result of this patch is a 2x or more speedup for inserts without keys for format PAGE. All changes are only related to how rows are stored Should fix some of the problems mentioned in: MDEV-8132 Temporary tables using Aria with very poor performance MDEV-9079 Aria very slow for internal temporary tables MDEV-5841 Mariadb very poor temporary performance The following changes where done: - For rows with a small row length that fits into a page (818 bytes with 8192 pages), stop as soon as we hit a match. - Added markers full_head_size and full_tail_size that tells us where to start searching on the bitmap page - Ensure that page->used_size is correctly updated when bitmap grows. This allows us to stop searching at used_size - Added code to check that the bitmap variables are correct. - Fixed a wrong test where we set "first_bitmap_with_space". This shouldn't have caused any notable problems. --- storage/maria/ma_bitmap.c | 241 +++++++++++++++++++++++++++++++++----- storage/maria/maria_def.h | 3 + 2 files changed, 212 insertions(+), 32 deletions(-) diff --git a/storage/maria/ma_bitmap.c b/storage/maria/ma_bitmap.c index 9ab5533fdbd..e8e2bc56b25 100644 --- a/storage/maria/ma_bitmap.c +++ b/storage/maria/ma_bitmap.c @@ -145,6 +145,11 @@ static my_bool _ma_bitmap_create_missing(MARIA_HA *info, MARIA_FILE_BITMAP *bitmap, pgcache_page_no_t page); static void _ma_bitmap_unpin_all(MARIA_SHARE *share); +#ifndef DBUG_OFF +static void _ma_check_bitmap(MARIA_FILE_BITMAP *bitmap); +#else +#define _ma_check_bitmap(A) do { } while(0) +#endif /* Write bitmap page to key cache */ @@ -267,6 +272,13 @@ my_bool _ma_bitmap_init(MARIA_SHARE *share, File file, bitmap->sizes[6]= max_page_size - max_page_size * 80 / 100; bitmap->sizes[7]= 0; + /* + If a record size will fit into the smallest empty page, return first + found page in find_head() + */ + if (bitmap->sizes[3] >= share->base.max_pack_length) + bitmap->return_first_match= 1; + mysql_mutex_init(key_SHARE_BITMAP_lock, &share->bitmap.bitmap_lock, MY_MUTEX_INIT_SLOW); mysql_cond_init(key_SHARE_BITMAP_cond, @@ -677,7 +689,8 @@ void _ma_bitmap_delete_all(MARIA_SHARE *share) bzero(bitmap->map, bitmap->block_size); bitmap->changed= 1; bitmap->page= 0; - bitmap->used_size= bitmap->total_size= bitmap->max_total_size; + bitmap->used_size= bitmap->full_tail_size= bitmap->full_head_size= 0; + bitmap->total_size= bitmap->max_total_size; } DBUG_VOID_RETURN; } @@ -715,6 +728,7 @@ void _ma_bitmap_reset_cache(MARIA_SHARE *share) */ bitmap->page= ((pgcache_page_no_t) 0) - bitmap->pages_covered; bitmap->used_size= bitmap->total_size= bitmap->max_total_size; + bitmap->full_head_size= bitmap->full_tail_size= bitmap->max_total_size; bfill(bitmap->map, share->block_size, 255); #ifndef DBUG_OFF memcpy(bitmap->map + bitmap->block_size, bitmap->map, bitmap->block_size); @@ -1016,9 +1030,6 @@ static void adjust_total_size(MARIA_HA *info, pgcache_page_no_t page) bitmap Bitmap handler page Page to read - TODO - Update 'bitmap->used_size' to real size of used bitmap - NOTE We don't always have share->bitmap.bitmap_lock here (when called from_ma_check_bitmap_data() for example). @@ -1035,6 +1046,9 @@ static my_bool _ma_read_bitmap_page(MARIA_HA *info, MARIA_SHARE *share= info->s; my_bool res; DBUG_ENTER("_ma_read_bitmap_page"); + DBUG_PRINT("enter", ("page: %lld data_file_length: %lld", + (longlong) page, + (longlong) share->state.state.data_file_length)); DBUG_ASSERT(page % bitmap->pages_covered == 0); DBUG_ASSERT(!bitmap->changed); @@ -1049,13 +1063,22 @@ static my_bool _ma_read_bitmap_page(MARIA_HA *info, } adjust_total_size(info, page); - bitmap->used_size= bitmap->total_size; + bitmap->full_head_size= bitmap->full_tail_size= 0; DBUG_ASSERT(share->pagecache->block_size == bitmap->block_size); res= pagecache_read(share->pagecache, &bitmap->file, page, 0, bitmap->map, PAGECACHE_PLAIN_PAGE, PAGECACHE_LOCK_LEFT_UNLOCKED, 0) == NULL; + if (!res) + { + /* Calculate used_size */ + const uchar *data, *end= bitmap->map; + for (data= bitmap->map + bitmap->total_size; --data >= end && *data == 0; ) + {} + bitmap->used_size= (uint) ((data + 1) - end); + DBUG_ASSERT(bitmap->used_size <= bitmap->total_size); + } /* We can't check maria_bitmap_marker here as if the bitmap page previously had a true checksum and the user switched mode to not checksum @@ -1067,7 +1090,10 @@ static my_bool _ma_read_bitmap_page(MARIA_HA *info, #ifndef DBUG_OFF if (!res) + { memcpy(bitmap->map + bitmap->block_size, bitmap->map, bitmap->block_size); + _ma_check_bitmap(bitmap); + } #endif DBUG_RETURN(res); } @@ -1097,6 +1123,8 @@ static my_bool _ma_change_bitmap_page(MARIA_HA *info, { DBUG_ENTER("_ma_change_bitmap_page"); + _ma_check_bitmap(bitmap); + /* We have to mark the file changed here, as otherwise the following read/write to pagecache may force a page out from this file, which would @@ -1228,6 +1256,9 @@ static void fill_block(MARIA_FILE_BITMAP *bitmap, This is defined as the first page of the set of pages with the smallest free space that can hold 'size'. + NOTES + Updates bitmap->full_head_size while scanning data + RETURN 0 ok (block is updated) 1 error (no space in bitmap; block is not touched) @@ -1238,10 +1269,11 @@ static my_bool allocate_head(MARIA_FILE_BITMAP *bitmap, uint size, MARIA_BITMAP_BLOCK *block) { uint min_bits= size_to_head_pattern(bitmap, size); - uchar *data= bitmap->map, *end= data + bitmap->used_size; + uchar *data, *end; uchar *best_data= 0; uint best_bits= (uint) -1, UNINIT_VAR(best_pos); - uint first_pattern= 0; /* if doing insert_order */ + my_bool first_pattern= 0; /* if doing insert_order */ + my_bool first_found= 1; MARIA_SHARE *share= bitmap->share; my_bool insert_order= MY_TEST(share->base.extra_options & MA_EXTRA_OPTIONS_INSERT_ORDER); @@ -1249,16 +1281,19 @@ static my_bool allocate_head(MARIA_FILE_BITMAP *bitmap, uint size, DBUG_ASSERT(size <= FULL_PAGE_SIZE(share)); + end= bitmap->map + bitmap->used_size; if (insert_order && bitmap->page == share->last_insert_bitmap) { uint last_insert_page= share->last_insert_page; uint byte= 6 * (last_insert_page / 16); first_pattern= last_insert_page % 16; - DBUG_ASSERT(data + byte < end); - data+= byte; + data= bitmap->map+byte; + DBUG_ASSERT(data <= end); } + else + data= bitmap->map + (bitmap->full_head_size/6)*6; - for (; data < end; data+= 6) + for (; data < end; data+= 6, first_pattern= 0) { ulonglong bits= uint6korr(data); /* 6 bytes = 6*8/3= 16 patterns */ uint i; @@ -1271,17 +1306,24 @@ static my_bool allocate_head(MARIA_FILE_BITMAP *bitmap, uint size, */ if ((!bits && best_data) || ((bits & 04444444444444444LL) == 04444444444444444LL)) - { - first_pattern= 0; // always restart from 0 when moving to new 6-byte continue; - } + for (i= first_pattern, bits >>= (3 * first_pattern); i < 16 ; i++, bits >>= 3) { uint pattern= (uint) (bits & 7); + + if (pattern <= 3) /* Room for more data */ + { + if (first_found) + { + first_found= 0; + bitmap->full_head_size= (data - bitmap->map); + } + } if (pattern <= min_bits) { - /* There is enough space here */ + /* There is enough space here, check if we have found better */ if ((int) pattern > (int) best_bits) { /* @@ -1292,23 +1334,32 @@ static my_bool allocate_head(MARIA_FILE_BITMAP *bitmap, uint size, best_bits= pattern; best_data= data; best_pos= i; - if (pattern == min_bits) + if (pattern == min_bits || bitmap->return_first_match) goto found; /* Best possible match */ } } } - first_pattern= 0; // always restart from 0 when moving to new 6-byte } if (!best_data) /* Found no place */ { if (data >= bitmap->map + bitmap->total_size) DBUG_RETURN(1); /* No space in bitmap */ + DBUG_ASSERT(uint6korr(data) == 0); /* Allocate data at end of bitmap */ - bitmap->used_size+= 6; - set_if_smaller(bitmap->used_size, bitmap->total_size); + bitmap->used_size= (uint) (data - bitmap->map) + 6; best_data= data; best_pos= best_bits= 0; } + else + { + /* + This is not stricly needed as used_size should be alligned on 6, + but for easier debugging lets try to keep it more accurate + */ + uint position= (uint) (best_data - bitmap->map) + 6; + set_if_bigger(bitmap->used_size, position); + } + DBUG_ASSERT(bitmap->used_size <= bitmap->total_size); found: if (insert_order) @@ -1341,12 +1392,15 @@ static my_bool allocate_tail(MARIA_FILE_BITMAP *bitmap, uint size, MARIA_BITMAP_BLOCK *block) { uint min_bits= size_to_tail_pattern(bitmap, size); - uchar *data= bitmap->map, *end= data + bitmap->used_size; - uchar *best_data= 0; + uchar *data, *end, *best_data= 0; + my_bool first_found= 1; uint best_bits= (uint) -1, UNINIT_VAR(best_pos); DBUG_ENTER("allocate_tail"); DBUG_PRINT("enter", ("size: %u", size)); + data= bitmap->map + (bitmap->full_tail_size/6)*6; + end= bitmap->map + bitmap->used_size; + /* We have to add DIR_ENTRY_SIZE here as this is not part of the data size See call to allocate_tail() in find_tail(). @@ -1375,7 +1429,19 @@ static my_bool allocate_tail(MARIA_FILE_BITMAP *bitmap, uint size, for (i= 0; i < 16; i++, bits >>= 3) { uint pattern= (uint) (bits & 7); - if (pattern <= min_bits && (!pattern || pattern >= 5)) + + if (pattern == 0 || + (pattern > FULL_HEAD_PAGE && pattern < FULL_TAIL_PAGE)) + { + /* There is room for tail data */ + if (first_found) + { + first_found= 0; + bitmap->full_tail_size= (data - bitmap->map); + } + } + + if (pattern <= min_bits && (!pattern || pattern > FULL_HEAD_PAGE)) { if ((int) pattern > (int) best_bits) { @@ -1392,10 +1458,11 @@ static my_bool allocate_tail(MARIA_FILE_BITMAP *bitmap, uint size, { if (data >= bitmap->map + bitmap->total_size) DBUG_RETURN(1); + DBUG_ASSERT(uint6korr(data) == 0); /* Allocate data at end of bitmap */ best_data= data; - bitmap->used_size+= 6; - set_if_smaller(bitmap->used_size, bitmap->total_size); + bitmap->used_size= (uint) (data - bitmap->map) + 6; + DBUG_ASSERT(bitmap->used_size <= bitmap->total_size); best_pos= best_bits= 0; } @@ -1434,8 +1501,7 @@ static ulong allocate_full_pages(MARIA_FILE_BITMAP *bitmap, ulong pages_needed, MARIA_BITMAP_BLOCK *block, my_bool full_page) { - uchar *data= bitmap->map, *data_end= data + bitmap->used_size; - uchar *page_end= data + bitmap->total_size; + uchar *data, *data_end, *page_end; uchar *best_data= 0; uint min_size; uint best_area_size, UNINIT_VAR(best_prefix_area_size); @@ -1449,6 +1515,10 @@ static ulong allocate_full_pages(MARIA_FILE_BITMAP *bitmap, min_size= BLOB_SEGMENT_MIN_SIZE; best_area_size= ~(uint) 0; + data= bitmap->map + (bitmap->full_head_size/6)*6; + data_end= bitmap->map + bitmap->used_size; + page_end= bitmap->map + bitmap->total_size; + for (; data < page_end; data+= 6) { ulonglong bits= uint6korr(data); /* 6 bytes = 6*8/3= 16 patterns */ @@ -1466,6 +1536,12 @@ static ulong allocate_full_pages(MARIA_FILE_BITMAP *bitmap, if ((bits= uint6korr(data))) break; } + /* + Check if we are end of bitmap. In this case we know that + the rest of the bitmap is usable + */ + if (data >= data_end) + data= page_end; area_size= (uint) (data - data_start) / 6 * 16; if (area_size >= best_area_size) continue; @@ -1823,7 +1899,7 @@ static my_bool allocate_blobs(MARIA_HA *info, MARIA_ROW *row) /* - Store in the bitmap the new size for a head page + Reserve the current head page SYNOPSIS use_head() @@ -2225,7 +2301,7 @@ static my_bool set_page_bits(MARIA_HA *info, MARIA_FILE_BITMAP *bitmap, pgcache_page_no_t page, uint fill_pattern) { pgcache_page_no_t bitmap_page; - uint offset_page, offset, tmp, org_tmp; + uint offset_page, offset, tmp, org_tmp, used_offset; uchar *data; DBUG_ENTER("set_page_bits"); DBUG_ASSERT(fill_pattern <= 7); @@ -2237,6 +2313,7 @@ static my_bool set_page_bits(MARIA_HA *info, MARIA_FILE_BITMAP *bitmap, /* Find page number from start of bitmap */ offset_page= (uint) (page - bitmap->page - 1); + /* Mark place used by reading/writing 2 bytes at a time to handle bitmaps in overlapping bytes @@ -2248,11 +2325,37 @@ static my_bool set_page_bits(MARIA_HA *info, MARIA_FILE_BITMAP *bitmap, tmp= (tmp & ~(7 << offset)) | (fill_pattern << offset); if (tmp == org_tmp) DBUG_RETURN(0); /* No changes */ - int2store(data, tmp); + /* + Take care to not write bytes outside of bitmap. + fill_pattern is 3 bits, so we need to write two bytes + if bit position we write to is > (8-3) + */ + if (offset > 5) + int2store(data, tmp); + else + data[0]= tmp; + + /* + Reset full_head_size or full_tail_size if we are releasing data before + it. Increase used_size if we are allocating data. + */ + used_offset= (uint) (data - bitmap->map); + if (fill_pattern < 4) + set_if_smaller(bitmap->full_head_size, used_offset); + if (fill_pattern == 0 || (fill_pattern > 4 && fill_pattern < 7)) + set_if_smaller(bitmap->full_tail_size, used_offset); + if (fill_pattern != 0) + { + /* Calulcate which was the last changed byte */ + used_offset+= offset > 5 ? 2 : 1; + set_if_bigger(bitmap->used_size, used_offset); + } + + _ma_check_bitmap(bitmap); bitmap->changed= 1; DBUG_EXECUTE("bitmap", _ma_print_bitmap_changes(bitmap);); - if (fill_pattern != 3 && fill_pattern != 7) + if (fill_pattern != FULL_HEAD_PAGE && fill_pattern != FULL_TAIL_PAGE) set_if_smaller(info->s->state.first_bitmap_with_space, bitmap_page); /* Note that if the condition above is false (page is full), and all pages of @@ -2345,7 +2448,7 @@ my_bool _ma_bitmap_reset_full_page_bits(MARIA_HA *info, uint page_count) { ulonglong bitmap_page; - uint offset, bit_start, bit_count, tmp; + uint offset, bit_start, bit_count, tmp, byte_offset; uchar *data; DBUG_ENTER("_ma_bitmap_reset_full_page_bits"); DBUG_PRINT("enter", ("page: %lu page_count: %u", (ulong) page, page_count)); @@ -2365,7 +2468,8 @@ my_bool _ma_bitmap_reset_full_page_bits(MARIA_HA *info, bit_start= offset * 3; bit_count= page_count * 3; - data= bitmap->map + bit_start / 8; + byte_offset= bit_start/8; + data= bitmap->map + byte_offset; offset= bit_start & 7; tmp= (255 << offset); /* Bits to keep */ @@ -2376,6 +2480,9 @@ my_bool _ma_bitmap_reset_full_page_bits(MARIA_HA *info, } *data&= ~tmp; + set_if_smaller(bitmap->full_head_size, byte_offset); + set_if_smaller(bitmap->full_tail_size, byte_offset); + if ((int) (bit_count-= (8 - offset)) > 0) { uint fill; @@ -2477,6 +2584,8 @@ my_bool _ma_bitmap_set_full_page_bits(MARIA_HA *info, tmp= (1 << bit_count) - 1; *data|= tmp; } + set_if_bigger(bitmap->used_size, (uint) (data - bitmap->map) + 1); + _ma_check_bitmap(bitmap); bitmap->changed= 1; DBUG_EXECUTE("bitmap", _ma_print_bitmap_changes(bitmap);); DBUG_RETURN(0); @@ -2835,6 +2944,72 @@ my_bool _ma_check_bitmap_data(MARIA_HA *info, enum en_page_type page_type, return (bitmap_pattern != bits); } +/** + Check that bitmap looks correct + + - All data before full_head_size and full_tail_size are allocated + - There is no allocated data after used_size + All of the above need to be correct only according to 6 byte + alignment as all loops reads 6 bytes at a time and we check both + start and end position according to the current 6 byte position. +*/ + +#ifndef DBUG_OFF +static void _ma_check_bitmap(MARIA_FILE_BITMAP *bitmap) +{ + uchar *data= bitmap->map; + uchar *end= bitmap->map + bitmap->total_size; + uchar *full_head_end=0, *full_tail_end=0, *first_empty= bitmap->map; + + for (; data < end; data+= 6) + { + ulonglong bits= uint6korr(data); /* 6 bytes = 6*8/3= 16 patterns */ + uint i; + + if (bits == 04444444444444444LL || bits == 0xffffffffffffLL) + { + first_empty= data + 6; + continue; /* block fully used */ + } + if (bits == 0) + { + if (!full_head_end) + full_head_end= data; + if (!full_tail_end) + full_tail_end= data; + continue; + } + + first_empty= data + 6; + if (!full_head_end || !full_tail_end) + { + for (i= 0, bits >>= 0; i < 16 ; i++, bits >>= 3) + { + uint pattern= (uint) (bits & 7); + if (pattern == FULL_HEAD_PAGE || pattern == FULL_TAIL_PAGE) + continue; + + if (pattern < 4 && !full_head_end) + full_head_end= data; + if ((pattern == 0 || (pattern > 4 && pattern < 7)) && !full_tail_end) + full_tail_end= data; + } + } + } + if (!full_head_end) + full_head_end= data; + if (!full_tail_end) + full_tail_end= data; + + /* used_size must point after the last byte that had some data) */ + DBUG_ASSERT(bitmap->used_size <= bitmap->total_size); + DBUG_ASSERT((bitmap->map + (bitmap->used_size+5)/6*6) >= first_empty); + /* full_xxxx_size can't point after the first block that has free data */ + DBUG_ASSERT((bitmap->map + (bitmap->full_head_size/6*6)) <= full_head_end); + DBUG_ASSERT((bitmap->map + (bitmap->full_tail_size/6*6)) <= full_tail_end); +} +#endif + /* Check if the page type matches the one that we have in the bitmap @@ -3072,6 +3247,7 @@ static my_bool _ma_bitmap_create_missing(MARIA_HA *info, pgcache_page_no_t from, to; my_off_t data_file_length= share->state.state.data_file_length; DBUG_ENTER("_ma_bitmap_create_missing"); + DBUG_PRINT("enter", ("page: %lld", (longlong) page)); /* First (in offset order) bitmap page to create */ if (data_file_length < block_size) @@ -3124,7 +3300,8 @@ static my_bool _ma_bitmap_create_missing(MARIA_HA *info, only later as we are going to modify it very soon. */ bzero(bitmap->map, bitmap->block_size); - bitmap->used_size= 0; + bitmap->used_size= bitmap->full_head_size= bitmap->full_tail_size= 0; + bitmap->changed=1; #ifndef DBUG_OFF /* Make a copy of the page to be able to print out bitmap changes during diff --git a/storage/maria/maria_def.h b/storage/maria/maria_def.h index d57a429365a..6cd6cda4b2f 100644 --- a/storage/maria/maria_def.h +++ b/storage/maria/maria_def.h @@ -331,7 +331,10 @@ typedef struct st_maria_file_bitmap pgcache_page_no_t last_bitmap_page; /* Last possible bitmap page */ my_bool changed; /* 1 if page needs to be written */ my_bool changed_not_flushed; /* 1 if some bitmap is not flushed */ + my_bool return_first_match; /* Shortcut find_head() */ uint used_size; /* Size of bitmap head that is not 0 */ + uint full_head_size; /* Where to start search for head */ + uint full_tail_size; /* Where to start search for tail */ uint flush_all_requested; /**< If _ma_bitmap_flush_all waiting */ uint waiting_for_flush_all_requested; /* If someone is waiting for above */ uint non_flushable; /**< 0 if bitmap and log are in sync */ From 0e8cb572f11a767849d97776c0fb4a0a0eb9c490 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Sat, 17 Feb 2018 15:18:14 +0200 Subject: [PATCH 09/37] Fix innodb_encryption-page-compression test by force flushing dirty pages. --- .../innodb_encryption-page-compression.result | 12 ++++++++++++ .../t/innodb_encryption-page-compression.test | 19 ++++++++++++++++--- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/mysql-test/suite/encryption/r/innodb_encryption-page-compression.result b/mysql-test/suite/encryption/r/innodb_encryption-page-compression.result index 88437408c32..584f8293f5d 100644 --- a/mysql-test/suite/encryption/r/innodb_encryption-page-compression.result +++ b/mysql-test/suite/encryption/r/innodb_encryption-page-compression.result @@ -135,6 +135,12 @@ count(*) select count(*) from innodb_page_compressed9 where c1 < 500000; count(*) 2000 +flush tables innodb_page_compressed1, innodb_page_compressed2, +innodb_page_compressed3, innodb_page_compressed4, +innodb_page_compressed5, innodb_page_compressed6, +innodb_page_compressed7, innodb_page_compressed8, +innodb_page_compressed9 for export; +unlock tables; SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; variable_value > 0 1 @@ -152,6 +158,12 @@ update innodb_page_compressed6 set c1 = c1 + 1; update innodb_page_compressed7 set c1 = c1 + 1; update innodb_page_compressed8 set c1 = c1 + 1; update innodb_page_compressed9 set c1 = c1 + 1; +flush tables innodb_page_compressed1, innodb_page_compressed2, +innodb_page_compressed3, innodb_page_compressed4, +innodb_page_compressed5, innodb_page_compressed6, +innodb_page_compressed7, innodb_page_compressed8, +innodb_page_compressed9 for export; +unlock tables; SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; variable_value > 0 1 diff --git a/mysql-test/suite/encryption/t/innodb_encryption-page-compression.test b/mysql-test/suite/encryption/t/innodb_encryption-page-compression.test index 0c18b97bfd6..ef9e3d6e7f4 100644 --- a/mysql-test/suite/encryption/t/innodb_encryption-page-compression.test +++ b/mysql-test/suite/encryption/t/innodb_encryption-page-compression.test @@ -81,6 +81,14 @@ select count(*) from innodb_page_compressed7 where c1 < 500000; select count(*) from innodb_page_compressed8 where c1 < 500000; select count(*) from innodb_page_compressed9 where c1 < 500000; +flush tables innodb_page_compressed1, innodb_page_compressed2, +innodb_page_compressed3, innodb_page_compressed4, +innodb_page_compressed5, innodb_page_compressed6, +innodb_page_compressed7, innodb_page_compressed8, +innodb_page_compressed9 for export; + +unlock tables; + let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_PAGE_COMPRESSED'; --source include/wait_condition.inc let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_ENCRYPTED'; @@ -104,9 +112,14 @@ update innodb_page_compressed7 set c1 = c1 + 1; update innodb_page_compressed8 set c1 = c1 + 1; update innodb_page_compressed9 set c1 = c1 + 1; -let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_ENCRYPTED'; ---source include/wait_condition.inc -let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_PAGE_COMPRESSED'; +flush tables innodb_page_compressed1, innodb_page_compressed2, +innodb_page_compressed3, innodb_page_compressed4, +innodb_page_compressed5, innodb_page_compressed6, +innodb_page_compressed7, innodb_page_compressed8, +innodb_page_compressed9 for export; + +unlock tables; + --source include/wait_condition.inc SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; From 9ea3ad6d7541035f514a6212d9b72ee165f52fb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Sun, 18 Feb 2018 07:32:19 +0200 Subject: [PATCH 10/37] Disable failing test. --- mysql-test/suite/encryption/disabled.def | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql-test/suite/encryption/disabled.def b/mysql-test/suite/encryption/disabled.def index abbb82d51f6..acb936fa6f7 100644 --- a/mysql-test/suite/encryption/disabled.def +++ b/mysql-test/suite/encryption/disabled.def @@ -12,4 +12,4 @@ innodb_scrub : MDEV-8139 scrubbing does not work reliably innodb_scrub_background : MDEV-8139 scrubbing does not work reliably - +innodb_encryption-page-compression : MDEV-14814 encryption.innodb_encryption-page-compression failed in buildbot with timeout on wait condition From 7a84688e2c30a95297246b5dc3005854bcbd5bf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Sun, 18 Feb 2018 07:47:47 +0200 Subject: [PATCH 11/37] MDEV-14814: encryption.innodb_encryption-page-compression failed in buildbot with timeout on wait condition Test changes only. --- mysql-test/suite/encryption/disabled.def | 2 +- .../r/innodb_encryption-page-compression.result | 6 +++--- .../encryption/t/innodb_encryption-page-compression.test | 8 ++++++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/mysql-test/suite/encryption/disabled.def b/mysql-test/suite/encryption/disabled.def index acb936fa6f7..abbb82d51f6 100644 --- a/mysql-test/suite/encryption/disabled.def +++ b/mysql-test/suite/encryption/disabled.def @@ -12,4 +12,4 @@ innodb_scrub : MDEV-8139 scrubbing does not work reliably innodb_scrub_background : MDEV-8139 scrubbing does not work reliably -innodb_encryption-page-compression : MDEV-14814 encryption.innodb_encryption-page-compression failed in buildbot with timeout on wait condition + diff --git a/mysql-test/suite/encryption/r/innodb_encryption-page-compression.result b/mysql-test/suite/encryption/r/innodb_encryption-page-compression.result index 584f8293f5d..14933e526c4 100644 --- a/mysql-test/suite/encryption/r/innodb_encryption-page-compression.result +++ b/mysql-test/suite/encryption/r/innodb_encryption-page-compression.result @@ -141,6 +141,7 @@ innodb_page_compressed5, innodb_page_compressed6, innodb_page_compressed7, innodb_page_compressed8, innodb_page_compressed9 for export; unlock tables; +# Wait until dirty pages are compressed and encrypted SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; variable_value > 0 1 @@ -164,9 +165,8 @@ innodb_page_compressed5, innodb_page_compressed6, innodb_page_compressed7, innodb_page_compressed8, innodb_page_compressed9 for export; unlock tables; -SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; -variable_value > 0 -1 +# Wait until dirty pages are compressed and encrypted 2 +unlock tables; SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted'; variable_value > 0 1 diff --git a/mysql-test/suite/encryption/t/innodb_encryption-page-compression.test b/mysql-test/suite/encryption/t/innodb_encryption-page-compression.test index ef9e3d6e7f4..fe132c0c59b 100644 --- a/mysql-test/suite/encryption/t/innodb_encryption-page-compression.test +++ b/mysql-test/suite/encryption/t/innodb_encryption-page-compression.test @@ -89,6 +89,7 @@ innodb_page_compressed9 for export; unlock tables; +--echo # Wait until dirty pages are compressed and encrypted let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_PAGE_COMPRESSED'; --source include/wait_condition.inc let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_ENCRYPTED'; @@ -117,12 +118,15 @@ innodb_page_compressed3, innodb_page_compressed4, innodb_page_compressed5, innodb_page_compressed6, innodb_page_compressed7, innodb_page_compressed8, innodb_page_compressed9 for export; - unlock tables; +--echo # Wait until dirty pages are compressed and encrypted 2 +let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_PAGE_COMPRESSED'; +--source include/wait_condition.inc +unlock tables; +let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_DECRYPTED'; --source include/wait_condition.inc -SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted'; SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_compressed'; SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_decompressed'; From b697f213a7bb90737e41ae09a5bfb08c6f47c0f6 Mon Sep 17 00:00:00 2001 From: Sergey Vojtovich Date: Mon, 19 Feb 2018 13:53:35 +0400 Subject: [PATCH 12/37] MDEV-14541 - Workaround GCC ICE on ARM64 Added missing include. --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index b4de70a283d..8d2efac89cb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -162,6 +162,7 @@ INCLUDE(plugin) INCLUDE(install_macros) INCLUDE(systemd) INCLUDE(mysql_add_executable) +INCLUDE(compile_flags) # Handle options OPTION(DISABLE_SHARED From 83b471348d6fc4e9bb4bd2e5e23b304979708a8e Mon Sep 17 00:00:00 2001 From: Sergey Vojtovich Date: Mon, 19 Feb 2018 16:51:15 +0400 Subject: [PATCH 13/37] MDEV-14318 - cmake updates to build on arm64 Added more files affected by GCC ICE: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=67143 --- storage/innobase/CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index 0033cdbae35..c5f9abdaa86 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -175,7 +175,13 @@ IF(CMAKE_COMPILER_IS_GNUCXX AND CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64") btr/btr0btr.cc btr/btr0cur.cc buf/buf0buf.cc + fts/fts0fts.cc gis/gis0sea.cc + handler/handler0alter.cc + mtr/mtr0mtr.cc + row/row0merge.cc + row/row0mysql.cc + srv/srv0srv.cc COMPILE_FLAGS "-O0" ) ENDIF() From 852c35f571b1b7454aab5768899e2faee8f941d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 19 Feb 2018 14:58:05 +0200 Subject: [PATCH 14/37] MDEV-11581 follow-up fix: Correct a condition fsp_fill_free_list(): Correctly determine whether the temporary tablespace file should be extended in order to respond to a page allocation request. The inverted condition was noticed by Thiru when he analyzed MDEV-13013. --- storage/innobase/fsp/fsp0fsp.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc index 2531a9fb5aa..dd0148a3275 100644 --- a/storage/innobase/fsp/fsp0fsp.cc +++ b/storage/innobase/fsp/fsp0fsp.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1108,7 +1108,7 @@ fsp_fill_free_list( skip_resize = !srv_sys_space.can_auto_extend_last_file(); break; case SRV_TMP_SPACE_ID: - skip_resize = srv_tmp_space.can_auto_extend_last_file(); + skip_resize = !srv_tmp_space.can_auto_extend_last_file(); break; } From 0ea45725d85c8426c62f4d6707ae2029b49ceb3d Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Mon, 19 Feb 2018 15:21:54 +0000 Subject: [PATCH 15/37] Fix 2 more VS2015 warnings --- sql/slave.cc | 19 +++++++++---------- win/upgrade_wizard/CMakeLists.txt | 2 +- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/sql/slave.cc b/sql/slave.cc index 9c6e8ebc200..5534ddb1b49 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -2626,7 +2626,6 @@ bool show_master_info(THD *thd, Master_info *mi, bool full) void show_master_info_get_fields(THD *thd, List *field_list, bool full, size_t gtid_pos_length) { - Master_info *mi; MEM_ROOT *mem_root= thd->mem_root; DBUG_ENTER("show_master_info_get_fields"); @@ -2645,10 +2644,10 @@ void show_master_info_get_fields(THD *thd, List *field_list, Item_empty_string(thd, "Slave_IO_State", 30), mem_root); field_list->push_back(new (mem_root) - Item_empty_string(thd, "Master_Host", sizeof(mi->host)), + Item_empty_string(thd, "Master_Host", sizeof(Master_info::host)), mem_root); field_list->push_back(new (mem_root) - Item_empty_string(thd, "Master_User", sizeof(mi->user)), + Item_empty_string(thd, "Master_User", sizeof(Master_info::user)), mem_root); field_list->push_back(new (mem_root) Item_return_int(thd, "Master_Port", 7, MYSQL_TYPE_LONG), @@ -2733,23 +2732,23 @@ void show_master_info_get_fields(THD *thd, List *field_list, mem_root); field_list->push_back(new (mem_root) Item_empty_string(thd, "Master_SSL_CA_File", - sizeof(mi->ssl_ca)), + sizeof(Master_info::ssl_ca)), mem_root); field_list->push_back(new (mem_root) Item_empty_string(thd, "Master_SSL_CA_Path", - sizeof(mi->ssl_capath)), + sizeof(Master_info::ssl_capath)), mem_root); field_list->push_back(new (mem_root) Item_empty_string(thd, "Master_SSL_Cert", - sizeof(mi->ssl_cert)), + sizeof(Master_info::ssl_cert)), mem_root); field_list->push_back(new (mem_root) Item_empty_string(thd, "Master_SSL_Cipher", - sizeof(mi->ssl_cipher)), + sizeof(Master_info::ssl_cipher)), mem_root); field_list->push_back(new (mem_root) Item_empty_string(thd, "Master_SSL_Key", - sizeof(mi->ssl_key)), + sizeof(Master_info::ssl_key)), mem_root); field_list->push_back(new (mem_root) Item_return_int(thd, "Seconds_Behind_Master", 10, @@ -2783,11 +2782,11 @@ void show_master_info_get_fields(THD *thd, List *field_list, mem_root); field_list->push_back(new (mem_root) Item_empty_string(thd, "Master_SSL_Crl", - sizeof(mi->ssl_crl)), + sizeof(Master_info::ssl_crl)), mem_root); field_list->push_back(new (mem_root) Item_empty_string(thd, "Master_SSL_Crlpath", - sizeof(mi->ssl_crlpath)), + sizeof(Master_info::ssl_crlpath)), mem_root); field_list->push_back(new (mem_root) Item_empty_string(thd, "Using_Gtid", diff --git a/win/upgrade_wizard/CMakeLists.txt b/win/upgrade_wizard/CMakeLists.txt index dc4ef67387d..5f415e4960d 100644 --- a/win/upgrade_wizard/CMakeLists.txt +++ b/win/upgrade_wizard/CMakeLists.txt @@ -24,7 +24,7 @@ ELSE() SET(CMAKE_MFC_FLAG 1) ENDIF() # Enable exception handling (avoids warnings) -SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc") +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc -DNO_WARN_MBCS_MFC_DEPRECATION") INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql) MYSQL_ADD_EXECUTABLE(mysql_upgrade_wizard From 84e8e07e8e185fe98ad2932bd32e94424fb9383a Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Mon, 19 Feb 2018 16:37:08 +0000 Subject: [PATCH 16/37] Revert "Fix 2 more VS2015 warnings" This reverts commit 0ea45725d85c8426c62f4d6707ae2029b49ceb3d, since it breaks clang --- sql/slave.cc | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/sql/slave.cc b/sql/slave.cc index 5534ddb1b49..9c6e8ebc200 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -2626,6 +2626,7 @@ bool show_master_info(THD *thd, Master_info *mi, bool full) void show_master_info_get_fields(THD *thd, List *field_list, bool full, size_t gtid_pos_length) { + Master_info *mi; MEM_ROOT *mem_root= thd->mem_root; DBUG_ENTER("show_master_info_get_fields"); @@ -2644,10 +2645,10 @@ void show_master_info_get_fields(THD *thd, List *field_list, Item_empty_string(thd, "Slave_IO_State", 30), mem_root); field_list->push_back(new (mem_root) - Item_empty_string(thd, "Master_Host", sizeof(Master_info::host)), + Item_empty_string(thd, "Master_Host", sizeof(mi->host)), mem_root); field_list->push_back(new (mem_root) - Item_empty_string(thd, "Master_User", sizeof(Master_info::user)), + Item_empty_string(thd, "Master_User", sizeof(mi->user)), mem_root); field_list->push_back(new (mem_root) Item_return_int(thd, "Master_Port", 7, MYSQL_TYPE_LONG), @@ -2732,23 +2733,23 @@ void show_master_info_get_fields(THD *thd, List *field_list, mem_root); field_list->push_back(new (mem_root) Item_empty_string(thd, "Master_SSL_CA_File", - sizeof(Master_info::ssl_ca)), + sizeof(mi->ssl_ca)), mem_root); field_list->push_back(new (mem_root) Item_empty_string(thd, "Master_SSL_CA_Path", - sizeof(Master_info::ssl_capath)), + sizeof(mi->ssl_capath)), mem_root); field_list->push_back(new (mem_root) Item_empty_string(thd, "Master_SSL_Cert", - sizeof(Master_info::ssl_cert)), + sizeof(mi->ssl_cert)), mem_root); field_list->push_back(new (mem_root) Item_empty_string(thd, "Master_SSL_Cipher", - sizeof(Master_info::ssl_cipher)), + sizeof(mi->ssl_cipher)), mem_root); field_list->push_back(new (mem_root) Item_empty_string(thd, "Master_SSL_Key", - sizeof(Master_info::ssl_key)), + sizeof(mi->ssl_key)), mem_root); field_list->push_back(new (mem_root) Item_return_int(thd, "Seconds_Behind_Master", 10, @@ -2782,11 +2783,11 @@ void show_master_info_get_fields(THD *thd, List *field_list, mem_root); field_list->push_back(new (mem_root) Item_empty_string(thd, "Master_SSL_Crl", - sizeof(Master_info::ssl_crl)), + sizeof(mi->ssl_crl)), mem_root); field_list->push_back(new (mem_root) Item_empty_string(thd, "Master_SSL_Crlpath", - sizeof(Master_info::ssl_crlpath)), + sizeof(mi->ssl_crlpath)), mem_root); field_list->push_back(new (mem_root) Item_empty_string(thd, "Using_Gtid", From 5c3d0c6badfa76e3b71bf60d3bcdd06bcd1b96c1 Mon Sep 17 00:00:00 2001 From: Hartmut Holzgraefe Date: Thu, 15 Feb 2018 10:19:05 +0100 Subject: [PATCH 17/37] apply XA RECOVER FORMAT=... from MDEV-14593 to Oracle parser variant, too --- sql/sql_yacc_ora.yy | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/sql/sql_yacc_ora.yy b/sql/sql_yacc_ora.yy index 6bc9b80c074..397ad572416 100644 --- a/sql/sql_yacc_ora.yy +++ b/sql/sql_yacc_ora.yy @@ -1097,7 +1097,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize); opt_default_time_precision case_stmt_body opt_bin_mod opt_if_exists_table_element opt_if_not_exists_table_element - opt_recursive + opt_recursive opt_format_xid %type create_or_replace @@ -16924,12 +16924,30 @@ xa: { Lex->sql_command = SQLCOM_XA_ROLLBACK; } - | XA_SYM RECOVER_SYM + | XA_SYM RECOVER_SYM opt_format_xid { Lex->sql_command = SQLCOM_XA_RECOVER; + Lex->verbose= $3; } ; +opt_format_xid: + /* empty */ { $$= false; } + | FORMAT_SYM '=' ident_or_text + { + if (!my_strcasecmp(system_charset_info, $3.str, "SQL")) + $$= true; + else if (!my_strcasecmp(system_charset_info, $3.str, "RAW")) + $$= false; + else + { + my_yyabort_error((ER_UNKNOWN_EXPLAIN_FORMAT, MYF(0), "XA RECOVER", $3.str)); + $$= false; + } + } + ; + + xid: text_string { From aef530bb6955d8c13a1ff9c5624c74fefa68943c Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Mon, 19 Feb 2018 23:41:01 +0400 Subject: [PATCH 18/37] MDEV-15340 Wrong result HOUR(case_expression_with_time_and_datetime) The problem was that Item_func_hybrid_field_type::get_date() did not convert the result to the correct data type, so MYSQL_TIME::time_type of the get_date() result could be not in sync with field_type(). Changes: 1. Adding two new classes Datetime and Date to store MYSQL_TIMESTAMP_DATETIME and MYSQL_TIMESTAMP_DATE values respectively (in addition to earlier added class Time, for MYSQL_TIMESTAMP_TIME values). 2. Adding Item_func_hybrid_field_type::time_op(). It performs the operation using TIME representation, and always returns a MYSQL_TIME value with time_type=MYSQL_TIMESTAMP_TIME. Implementing time_op() for all affected children classes. 3. Fixing all implementations of date_op() to perform the operation using strictly DATETIME representation. Now they always return a MYSQL_TIME value with time_type=MYSQL_TIMESTAMP_{DATE|DATETIME}, according to the result data type. 4. Removing assignment of ltime.time_type to mysql_timestamp_type() from all val_xxx_from_date_op(), because now date_op() makes sure to return a proper MYSQL_TIME value with a good time_type (and other member) 5. Adding Item_func_hybrid_field_type::val_xxx_from_time_op(). 6. Overriding Type_handler_time_common::Item_func_hybrid_field_type_val_xxx() to call val_xxx_from_time_op() instead of val_xxx_from_date_op(). 7. Modified Item_func::get_arg0_date() to return strictly a TIME value if TIME_TIME_ONLY is passed, or return strictly a DATETIME value otherwise. If args[0] returned a value of a different temporal type, (for example a TIME value when TIME_TIME_ONLY was not passed, or a DATETIME value when TIME_TIME_ONLY was passed), the conversion is automatically applied. Earlier, get_arg0_date() did not guarantee a result in accordance to TIME_TIME_ONLY flag. --- mysql-test/r/func_in.result | 13 +++ mysql-test/r/func_time.result | 32 ++++++ mysql-test/t/func_in.test | 14 +++ mysql-test/t/func_time.test | 30 ++++++ sql/field.cc | 50 ++++----- sql/item.cc | 32 ------ sql/item.h | 29 ++---- sql/item_cmpfunc.cc | 64 ++++++++++-- sql/item_cmpfunc.h | 11 +- sql/item_func.cc | 38 ++++++- sql/item_func.h | 47 ++++++--- sql/item_timefunc.cc | 15 ++- sql/sql_type.cc | 80 +++++++++++++- sql/sql_type.h | 191 +++++++++++++++++++++++++++++++++- 14 files changed, 527 insertions(+), 119 deletions(-) diff --git a/mysql-test/r/func_in.result b/mysql-test/r/func_in.result index 692806c7cac..65313148bf8 100644 --- a/mysql-test/r/func_in.result +++ b/mysql-test/r/func_in.result @@ -896,3 +896,16 @@ EXECUTE stmt; a b DEALLOCATE PREPARE stmt; DROP TABLE t1; +# +# MDEV-15340 Wrong result HOUR(case_expression_with_time_and_datetime) +# +SELECT +TIME'00:00:00'='' AS c1_true, +TIME'00:00:00' IN ('', TIME'10:20:30') AS c2_true, +TIME'00:00:00' NOT IN ('', TIME'10:20:30') AS c3_false; +c1_true c2_true c3_false +1 1 0 +Warnings: +Warning 1292 Truncated incorrect time value: '' +Warning 1292 Truncated incorrect time value: '' +Warning 1292 Truncated incorrect time value: '' diff --git a/mysql-test/r/func_time.result b/mysql-test/r/func_time.result index e710f3ac438..53f61f6644f 100644 --- a/mysql-test/r/func_time.result +++ b/mysql-test/r/func_time.result @@ -3351,3 +3351,35 @@ SELECT CONVERT_TZ(1, ROW(1,1), 1); ERROR HY000: Illegal parameter data type row for operation 'convert_tz' SELECT CONVERT_TZ(1, 1, ROW(1,1)); ERROR HY000: Illegal parameter data type row for operation 'convert_tz' +# +# MDEV-15340 Wrong result HOUR(case_expression_with_time_and_datetime) +# +SET TIMESTAMP=UNIX_TIMESTAMP('2018-02-17 01:02:03'); +SELECT +COALESCE(TIME'800:00:00', NOW()) AS c, +HOUR(COALESCE(TIME'800:00:00',NOW())) AS hc; +c hc +2018-03-22 08:00:00 8 +SELECT +CASE WHEN TRUE THEN TIME'800:00:00' ELSE NOW() END AS c, +HOUR(CASE WHEN TRUE THEN TIME'800:00:00' ELSE NOW() END) AS hc; +c hc +2018-03-22 08:00:00 8 +SELECT +IFNULL(TIME'800:00:00', NOW()) AS c, +HOUR(IFNULL(TIME'800:00:00', NOW())) AS hc; +c hc +2018-03-22 08:00:00 8 +SELECT +IF(TRUE,TIME'800:00:00', NOW()) AS c, +HOUR(IF(TRUE,TIME'800:00:00', NOW())) AS hc; +c hc +2018-03-22 08:00:00 8 +SELECT +ADDTIME(TIME'10:20:30', TIMESTAMP'2001-01-01 00:00:00') AS c1, +ADDTIME(TIME'10:20:30', COALESCE(TIMESTAMP'2001-01-01 00:00:00',TIMESTAMP'2001-01-01 00:00:00')) AS c2, +ADDTIME(TIME'10:20:30', DATE'2001-01-01') AS c3, +ADDTIME(TIME'10:20:30', COALESCE(DATE'2001-01-01',TIMESTAMP'2001-01-01 00:00:00')) AS c4; +c1 c2 c3 c4 +NULL NULL NULL NULL +SET TIMESTAMP=DEFAULT; diff --git a/mysql-test/t/func_in.test b/mysql-test/t/func_in.test index 8f6062fd2c2..b99fad159c2 100644 --- a/mysql-test/t/func_in.test +++ b/mysql-test/t/func_in.test @@ -676,3 +676,17 @@ EXECUTE stmt; EXECUTE stmt; DEALLOCATE PREPARE stmt; DROP TABLE t1; + +--echo # +--echo # MDEV-15340 Wrong result HOUR(case_expression_with_time_and_datetime) +--echo # + +# This is to make sure that TIME_FUZZY_DATE is always passed to str_to_time(), +# so empty strings are compared as TIME'00:00:00' all around the code: +# when using Arg_comparator (e.g. in binary comparison operators), and +# when not using it (e.g. in IN predicate). + +SELECT + TIME'00:00:00'='' AS c1_true, + TIME'00:00:00' IN ('', TIME'10:20:30') AS c2_true, + TIME'00:00:00' NOT IN ('', TIME'10:20:30') AS c3_false; diff --git a/mysql-test/t/func_time.test b/mysql-test/t/func_time.test index 09c13598cfd..0066d4a434b 100644 --- a/mysql-test/t/func_time.test +++ b/mysql-test/t/func_time.test @@ -1945,3 +1945,33 @@ SELECT CONVERT_TZ(ROW(1,1),1,1); SELECT CONVERT_TZ(1, ROW(1,1), 1); --error ER_ILLEGAL_PARAMETER_DATA_TYPE_FOR_OPERATION SELECT CONVERT_TZ(1, 1, ROW(1,1)); + + +--echo # +--echo # MDEV-15340 Wrong result HOUR(case_expression_with_time_and_datetime) +--echo # + +SET TIMESTAMP=UNIX_TIMESTAMP('2018-02-17 01:02:03'); +SELECT + COALESCE(TIME'800:00:00', NOW()) AS c, + HOUR(COALESCE(TIME'800:00:00',NOW())) AS hc; + +SELECT + CASE WHEN TRUE THEN TIME'800:00:00' ELSE NOW() END AS c, + HOUR(CASE WHEN TRUE THEN TIME'800:00:00' ELSE NOW() END) AS hc; + +SELECT + IFNULL(TIME'800:00:00', NOW()) AS c, + HOUR(IFNULL(TIME'800:00:00', NOW())) AS hc; + +SELECT + IF(TRUE,TIME'800:00:00', NOW()) AS c, + HOUR(IF(TRUE,TIME'800:00:00', NOW())) AS hc; + +SELECT + ADDTIME(TIME'10:20:30', TIMESTAMP'2001-01-01 00:00:00') AS c1, + ADDTIME(TIME'10:20:30', COALESCE(TIMESTAMP'2001-01-01 00:00:00',TIMESTAMP'2001-01-01 00:00:00')) AS c2, + ADDTIME(TIME'10:20:30', DATE'2001-01-01') AS c3, + ADDTIME(TIME'10:20:30', COALESCE(DATE'2001-01-01',TIMESTAMP'2001-01-01 00:00:00')) AS c4; + +SET TIMESTAMP=DEFAULT; diff --git a/sql/field.cc b/sql/field.cc index 9b6465988f6..38978fbb727 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -5655,30 +5655,28 @@ Item *Field_temporal::get_equal_const_item_datetime(THD *thd, const_item->field_type() != MYSQL_TYPE_TIMESTAMP) || const_item->decimals != decimals()) { - MYSQL_TIME ltime; - if (const_item->field_type() == MYSQL_TYPE_TIME ? - const_item->get_date_with_conversion(<ime, 0) : - const_item->get_date(<ime, 0)) + Datetime dt(thd, const_item, 0); + if (!dt.is_valid_datetime()) return NULL; /* See comments about truncation in the same place in Field_time::get_equal_const_item(). */ - return new (thd->mem_root) Item_datetime_literal(thd, <ime, + return new (thd->mem_root) Item_datetime_literal(thd, + dt.get_mysql_time(), decimals()); } break; case ANY_SUBST: if (!is_temporal_type_with_date(const_item->field_type())) { - MYSQL_TIME ltime; - if (const_item->get_date_with_conversion(<ime, - TIME_FUZZY_DATES | - TIME_INVALID_DATES)) + Datetime dt(thd, const_item, TIME_FUZZY_DATES | TIME_INVALID_DATES); + if (!dt.is_valid_datetime()) return NULL; return new (thd->mem_root) - Item_datetime_literal_for_invalid_dates(thd, <ime, - ltime.second_part ? + Item_datetime_literal_for_invalid_dates(thd, dt.get_mysql_time(), + dt.get_mysql_time()-> + second_part ? TIME_SECOND_PART_DIGITS : 0); } break; @@ -6030,10 +6028,8 @@ Item *Field_time::get_equal_const_item(THD *thd, const Context &ctx, { MYSQL_TIME ltime; // Get the value of const_item with conversion from DATETIME to TIME - if (const_item->get_time_with_conversion(thd, <ime, - TIME_TIME_ONLY | - TIME_FUZZY_DATES | - TIME_INVALID_DATES)) + ulonglong fuzzydate= Time::comparison_flags_for_get_date(); + if (const_item->get_time_with_conversion(thd, <ime, fuzzydate)) return NULL; /* Replace a DATE/DATETIME constant to a TIME constant: @@ -6506,10 +6502,9 @@ Item *Field_newdate::get_equal_const_item(THD *thd, const Context &ctx, case ANY_SUBST: if (!is_temporal_type_with_date(const_item->field_type())) { - MYSQL_TIME ltime; // Get the value of const_item with conversion from TIME to DATETIME - if (const_item->get_date_with_conversion(<ime, - TIME_FUZZY_DATES | TIME_INVALID_DATES)) + Datetime dt(thd, const_item, TIME_FUZZY_DATES | TIME_INVALID_DATES); + if (!dt.is_valid_datetime()) return NULL; /* Replace the constant to a DATE or DATETIME constant. @@ -6522,26 +6517,23 @@ Item *Field_newdate::get_equal_const_item(THD *thd, const Context &ctx, (assuming CURRENT_DATE is '2015-08-30' */ - if (non_zero_hhmmssuu(<ime)) + if (!dt.hhmmssff_is_zero()) return new (thd->mem_root) - Item_datetime_literal_for_invalid_dates(thd, <ime, - ltime.second_part ? + Item_datetime_literal_for_invalid_dates(thd, dt.get_mysql_time(), + dt.get_mysql_time()-> + second_part ? TIME_SECOND_PART_DIGITS : 0); - datetime_to_date(<ime); return new (thd->mem_root) - Item_date_literal_for_invalid_dates(thd, <ime); + Item_date_literal_for_invalid_dates(thd, Date(&dt).get_mysql_time()); } break; case IDENTITY_SUBST: if (const_item->field_type() != MYSQL_TYPE_DATE) { - MYSQL_TIME ltime; - if (const_item->field_type() == MYSQL_TYPE_TIME ? - const_item->get_date_with_conversion(<ime, 0) : - const_item->get_date(<ime, 0)) + Date d(thd, const_item, 0); + if (!d.is_valid_date()) return NULL; - datetime_to_date(<ime); - return new (thd->mem_root) Item_date_literal(thd, <ime); + return new (thd->mem_root) Item_date_literal(thd, d.get_mysql_time()); } break; } diff --git a/sql/item.cc b/sql/item.cc index 6ca9a2e1812..95ff0a65fb9 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -122,38 +122,6 @@ longlong Item::val_datetime_packed_result() } -/** - Get date/time/datetime. - Optionally extend TIME result to DATETIME. -*/ -bool Item::get_date_with_conversion(MYSQL_TIME *ltime, ulonglong fuzzydate) -{ - THD *thd= current_thd; - - /* - Some TIME type items return error when trying to do get_date() - without TIME_TIME_ONLY set (e.g. Item_field for Field_time). - In the SQL standard time->datetime conversion mode we add TIME_TIME_ONLY. - In the legacy time->datetime conversion mode we do not add TIME_TIME_ONLY - and leave it to get_date() to check date. - */ - ulonglong time_flag= (field_type() == MYSQL_TYPE_TIME && - !(thd->variables.old_behavior & OLD_MODE_ZERO_DATE_TIME_CAST)) ? - TIME_TIME_ONLY : 0; - if (get_date(ltime, fuzzydate | time_flag)) - return true; - if (ltime->time_type == MYSQL_TIMESTAMP_TIME && - !(fuzzydate & TIME_TIME_ONLY)) - { - MYSQL_TIME tmp; - if (time_to_datetime_with_warn(thd, ltime, &tmp, fuzzydate)) - return null_value= true; - *ltime= tmp; - } - return false; -} - - /** Get date/time/datetime. If DATETIME or DATE result is returned, it's converted to TIME. diff --git a/sql/item.h b/sql/item.h index e045a2a30bf..84cb08142dc 100644 --- a/sql/item.h +++ b/sql/item.h @@ -665,13 +665,6 @@ protected: value= NULL; return value; } - bool get_date_with_conversion_from_item(Item *item, - MYSQL_TIME *ltime, - ulonglong fuzzydate) - { - DBUG_ASSERT(fixed == 1); - return (null_value= item->get_date_with_conversion(ltime, fuzzydate)); - } /* This method is used if the item was not null but convertion to TIME/DATE/DATETIME failed. We return a zero date if allowed, @@ -1363,17 +1356,16 @@ public: bool get_date_from_decimal(MYSQL_TIME *ltime, ulonglong fuzzydate); bool get_date_from_string(MYSQL_TIME *ltime, ulonglong fuzzydate); bool get_time(MYSQL_TIME *ltime) - { return get_date(ltime, TIME_TIME_ONLY | TIME_INVALID_DATES); } - // Get date with automatic TIME->DATETIME conversion - bool get_date_with_conversion(MYSQL_TIME *ltime, ulonglong fuzzydate); + { return get_date(ltime, Time::flags_for_get_date()); } /* - Get time with automatic DATE/DATETIME to TIME conversion. + Get time with automatic DATE/DATETIME to TIME conversion, + by subtracting CURRENT_DATE. - Performce a reserve operation to get_date_with_conversion(). + Performce a reverse operation to CAST(time AS DATETIME) Suppose: - we have a set of items (typically with the native MYSQL_TYPE_TIME type) whose item->get_date() return TIME1 value, and - - item->get_date_with_conversion() for the same Items return DATETIME1, + - CAST(AS DATETIME) for the same Items return DATETIME1, after applying time-to-datetime conversion to TIME1. then all items (typically of the native MYSQL_TYPE_{DATE|DATETIME} types) @@ -1402,22 +1394,21 @@ public: // Get a DATE or DATETIME value in numeric packed format for comparison virtual longlong val_datetime_packed() { - MYSQL_TIME ltime; ulonglong fuzzydate= TIME_FUZZY_DATES | TIME_INVALID_DATES; - return get_date_with_conversion(<ime, fuzzydate) ? 0 : pack_time(<ime); + Datetime dt(current_thd, this, fuzzydate); + return dt.is_valid_datetime() ? pack_time(dt.get_mysql_time()) : 0; } // Get a TIME value in numeric packed format for comparison virtual longlong val_time_packed() { - MYSQL_TIME ltime; - ulonglong fuzzydate= TIME_FUZZY_DATES | TIME_INVALID_DATES | TIME_TIME_ONLY; - return get_date(<ime, fuzzydate) ? 0 : pack_time(<ime); + Time tm(this, Time::comparison_flags_for_get_date()); + return tm.is_valid_time() ? pack_time(tm.get_mysql_time()) : 0; } longlong val_datetime_packed_result(); longlong val_time_packed_result() { MYSQL_TIME ltime; - ulonglong fuzzydate= TIME_FUZZY_DATES | TIME_INVALID_DATES | TIME_TIME_ONLY; + ulonglong fuzzydate= Time::comparison_flags_for_get_date(); return get_date_result(<ime, fuzzydate) ? 0 : pack_time(<ime); } diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc index 5846ef094d9..9f30b238660 100644 --- a/sql/item_cmpfunc.cc +++ b/sql/item_cmpfunc.cc @@ -2368,9 +2368,25 @@ Item_func_ifnull::str_op(String *str) bool Item_func_ifnull::date_op(MYSQL_TIME *ltime, ulonglong fuzzydate) { DBUG_ASSERT(fixed == 1); - if (!args[0]->get_date_with_conversion(ltime, fuzzydate & ~TIME_FUZZY_DATES)) - return (null_value= false); - return (null_value= args[1]->get_date_with_conversion(ltime, fuzzydate & ~TIME_FUZZY_DATES)); + for (uint i= 0; i < 2; i++) + { + Datetime dt(current_thd, args[i], fuzzydate & ~TIME_FUZZY_DATES); + if (!(dt.copy_to_mysql_time(ltime, mysql_timestamp_type()))) + return (null_value= false); + } + return (null_value= true); +} + + +bool Item_func_ifnull::time_op(MYSQL_TIME *ltime) +{ + DBUG_ASSERT(fixed == 1); + for (uint i= 0; i < 2; i++) + { + if (!Time(args[i]).copy_to_mysql_time(ltime)) + return (null_value= false); + } + return (null_value= true); } @@ -2850,7 +2866,19 @@ Item_func_nullif::date_op(MYSQL_TIME *ltime, ulonglong fuzzydate) DBUG_ASSERT(fixed == 1); if (!compare()) return (null_value= true); - return (null_value= args[2]->get_date(ltime, fuzzydate)); + Datetime dt(current_thd, args[2], fuzzydate); + return (null_value= dt.copy_to_mysql_time(ltime, mysql_timestamp_type())); +} + + +bool +Item_func_nullif::time_op(MYSQL_TIME *ltime) +{ + DBUG_ASSERT(fixed == 1); + if (!compare()) + return (null_value= true); + return (null_value= Time(args[2]).copy_to_mysql_time(ltime)); + } @@ -2991,7 +3019,18 @@ bool Item_func_case::date_op(MYSQL_TIME *ltime, ulonglong fuzzydate) Item *item= find_item(); if (!item) return (null_value= true); - return (null_value= item->get_date_with_conversion(ltime, fuzzydate)); + Datetime dt(current_thd, item, fuzzydate); + return (null_value= dt.copy_to_mysql_time(ltime, mysql_timestamp_type())); +} + + +bool Item_func_case::time_op(MYSQL_TIME *ltime) +{ + DBUG_ASSERT(fixed == 1); + Item *item= find_item(); + if (!item) + return (null_value= true); + return (null_value= Time(item).copy_to_mysql_time(ltime)); } @@ -3401,7 +3440,20 @@ bool Item_func_coalesce::date_op(MYSQL_TIME *ltime, ulonglong fuzzydate) DBUG_ASSERT(fixed == 1); for (uint i= 0; i < arg_count; i++) { - if (!args[i]->get_date_with_conversion(ltime, fuzzydate & ~TIME_FUZZY_DATES)) + Datetime dt(current_thd, args[i], fuzzydate & ~TIME_FUZZY_DATES); + if (!dt.copy_to_mysql_time(ltime, mysql_timestamp_type())) + return (null_value= false); + } + return (null_value= true); +} + + +bool Item_func_coalesce::time_op(MYSQL_TIME *ltime) +{ + DBUG_ASSERT(fixed == 1); + for (uint i= 0; i < arg_count; i++) + { + if (!Time(args[i]).copy_to_mysql_time(ltime)) return (null_value= false); } return (null_value= true); diff --git a/sql/item_cmpfunc.h b/sql/item_cmpfunc.h index 900a979ce97..9d790b75ebb 100644 --- a/sql/item_cmpfunc.h +++ b/sql/item_cmpfunc.h @@ -1005,6 +1005,7 @@ public: String *str_op(String *); my_decimal *decimal_op(my_decimal *); bool date_op(MYSQL_TIME *ltime, ulonglong fuzzydate); + bool time_op(MYSQL_TIME *ltime); void fix_length_and_dec() { if (!aggregate_for_result(func_name(), args, arg_count, true)) @@ -1077,6 +1078,7 @@ public: String *str_op(String *str); my_decimal *decimal_op(my_decimal *); bool date_op(MYSQL_TIME *ltime, ulonglong fuzzydate); + bool time_op(MYSQL_TIME *ltime); void fix_length_and_dec() { Item_func_case_abbreviation2::fix_length_and_dec2(args); @@ -1110,7 +1112,12 @@ public: bool date_op(MYSQL_TIME *ltime, ulonglong fuzzydate) { - return get_date_with_conversion_from_item(find_item(), ltime, fuzzydate); + Datetime dt(current_thd, find_item(), fuzzydate); + return (null_value= dt.copy_to_mysql_time(ltime, mysql_timestamp_type())); + } + bool time_op(MYSQL_TIME *ltime) + { + return (null_value= Time(find_item()).copy_to_mysql_time(ltime)); } longlong int_op() { @@ -1222,6 +1229,7 @@ public: arg_count= 2; // See the comment to the constructor } bool date_op(MYSQL_TIME *ltime, ulonglong fuzzydate); + bool time_op(MYSQL_TIME *ltime); double real_op(); longlong int_op(); String *str_op(String *str); @@ -2111,6 +2119,7 @@ public: String *str_op(String *); my_decimal *decimal_op(my_decimal *); bool date_op(MYSQL_TIME *ltime, ulonglong fuzzydate); + bool time_op(MYSQL_TIME *ltime); bool fix_fields(THD *thd, Item **ref); table_map not_null_tables() const { return 0; } const char *func_name() const { return "case"; } diff --git a/sql/item_func.cc b/sql/item_func.cc index 4903e552c9d..71078185586 100644 --- a/sql/item_func.cc +++ b/sql/item_func.cc @@ -929,7 +929,6 @@ String *Item_func_hybrid_field_type::val_str_from_date_op(String *str) if (date_op_with_null_check(<ime) || (null_value= str->alloc(MAX_DATE_STRING_REP_LENGTH))) return (String *) 0; - ltime.time_type= mysql_timestamp_type(); str->length(my_TIME_to_str(<ime, const_cast(str->ptr()), decimals)); str->set_charset(&my_charset_bin); DBUG_ASSERT(!null_value); @@ -941,7 +940,6 @@ double Item_func_hybrid_field_type::val_real_from_date_op() MYSQL_TIME ltime; if (date_op_with_null_check(<ime)) return 0; - ltime.time_type= mysql_timestamp_type(); return TIME_to_double(<ime); } longlong Item_func_hybrid_field_type::val_int_from_date_op() @@ -949,7 +947,6 @@ longlong Item_func_hybrid_field_type::val_int_from_date_op() MYSQL_TIME ltime; if (date_op_with_null_check(<ime)) return 0; - ltime.time_type= mysql_timestamp_type(); return TIME_to_ulonglong(<ime); } @@ -962,7 +959,40 @@ Item_func_hybrid_field_type::val_decimal_from_date_op(my_decimal *dec) my_decimal_set_zero(dec); return 0; } - ltime.time_type= mysql_timestamp_type(); + return date2my_decimal(<ime, dec); +} + + +String *Item_func_hybrid_field_type::val_str_from_time_op(String *str) +{ + MYSQL_TIME ltime; + if (time_op_with_null_check(<ime) || + (null_value= my_TIME_to_str(<ime, str, decimals))) + return NULL; + return str; +} + +double Item_func_hybrid_field_type::val_real_from_time_op() +{ + MYSQL_TIME ltime; + return time_op_with_null_check(<ime) ? 0 : TIME_to_double(<ime); +} + +longlong Item_func_hybrid_field_type::val_int_from_time_op() +{ + MYSQL_TIME ltime; + return time_op_with_null_check(<ime) ? 0 : TIME_to_ulonglong(<ime); +} + +my_decimal * +Item_func_hybrid_field_type::val_decimal_from_time_op(my_decimal *dec) +{ + MYSQL_TIME ltime; + if (time_op_with_null_check(<ime)) + { + my_decimal_set_zero(dec); + return 0; + } return date2my_decimal(<ime, dec); } diff --git a/sql/item_func.h b/sql/item_func.h index adc5c238abd..063a80de737 100644 --- a/sql/item_func.h +++ b/sql/item_func.h @@ -160,14 +160,17 @@ public: void print_args(String *str, uint from, enum_query_type query_type); inline bool get_arg0_date(MYSQL_TIME *ltime, ulonglong fuzzy_date) { - return (null_value=args[0]->get_date_with_conversion(ltime, fuzzy_date)); + return fuzzy_date & TIME_TIME_ONLY ? get_arg0_time(ltime) : + get_arg0_datetime(ltime, fuzzy_date); + } + inline bool get_arg0_datetime(MYSQL_TIME *ltime, ulonglong fuzzy_date) + { + Datetime dt(current_thd, args[0], fuzzy_date); + return (null_value= dt.copy_to_mysql_time(ltime)); } inline bool get_arg0_time(MYSQL_TIME *ltime) { - null_value= args[0]->get_time(ltime); - DBUG_ASSERT(null_value || - ltime->time_type != MYSQL_TIMESTAMP_TIME || ltime->day == 0); - return null_value; + return (null_value= Time(args[0]).copy_to_mysql_time(ltime)); } bool is_null() { update_null_value(); @@ -447,11 +450,17 @@ class Item_func_hybrid_field_type: public Item_hybrid_func */ bool date_op_with_null_check(MYSQL_TIME *ltime) { - bool rc= date_op(ltime, - field_type() == MYSQL_TYPE_TIME ? TIME_TIME_ONLY : 0); + bool rc= date_op(ltime, 0); DBUG_ASSERT(!rc ^ null_value); return rc; } + bool time_op_with_null_check(MYSQL_TIME *ltime) + { + bool rc= time_op(ltime); + DBUG_ASSERT(!rc ^ null_value); + DBUG_ASSERT(rc || ltime->time_type == MYSQL_TIMESTAMP_TIME); + return rc; + } String *str_op_with_null_check(String *str) { String *res= str_op(str); @@ -488,32 +497,30 @@ public: { return real_op(); } - bool get_date_from_date_op(MYSQL_TIME *ltime, ulonglong fuzzydate) - { - return date_op(ltime, - (fuzzydate | - (field_type() == MYSQL_TYPE_TIME ? TIME_TIME_ONLY : 0))); - } // Value methods that involve conversion String *val_str_from_decimal_op(String *str); String *val_str_from_real_op(String *str); String *val_str_from_int_op(String *str); String *val_str_from_date_op(String *str); + String *val_str_from_time_op(String *str); my_decimal *val_decimal_from_str_op(my_decimal *dec); my_decimal *val_decimal_from_real_op(my_decimal *dec); my_decimal *val_decimal_from_int_op(my_decimal *dec); my_decimal *val_decimal_from_date_op(my_decimal *dec); + my_decimal *val_decimal_from_time_op(my_decimal *dec); longlong val_int_from_str_op(); longlong val_int_from_real_op(); longlong val_int_from_decimal_op(); longlong val_int_from_date_op(); + longlong val_int_from_time_op(); double val_real_from_str_op(); double val_real_from_decimal_op(); double val_real_from_date_op(); + double val_real_from_time_op(); double val_real_from_int_op(); bool get_date_from_str_op(MYSQL_TIME *ltime, ulonglong fuzzydate); @@ -609,11 +616,18 @@ public: /** @brief Performs the operation that this functions implements when - field type is a temporal type. + field type is DATETIME or DATE. @return The result of the operation. */ virtual bool date_op(MYSQL_TIME *res, ulonglong fuzzy_date)= 0; + /** + @brief Performs the operation that this functions implements when + field type is TIME. + @return The result of the operation. + */ + virtual bool time_op(MYSQL_TIME *res)= 0; + }; @@ -676,6 +690,11 @@ public: DBUG_ASSERT(0); return true; } + bool time_op(MYSQL_TIME *ltime) + { + DBUG_ASSERT(0); + return true; + } }; diff --git a/sql/item_timefunc.cc b/sql/item_timefunc.cc index a0fdd4bb8fc..816514dcc35 100644 --- a/sql/item_timefunc.cc +++ b/sql/item_timefunc.cc @@ -2771,7 +2771,7 @@ bool Item_func_add_time::get_date(MYSQL_TIME *ltime, ulonglong fuzzy_date) // ADDTIME function AND the first argument is TIME if (args[0]->get_time(&l_time1) || args[1]->get_time(&l_time2) || - l_time2.time_type == MYSQL_TIMESTAMP_DATETIME) + l_time2.time_type != MYSQL_TIMESTAMP_TIME) return (null_value= 1); is_time= (l_time1.time_type == MYSQL_TIMESTAMP_TIME); } @@ -2950,14 +2950,13 @@ longlong Item_func_timestamp_diff::val_int() long microseconds; long months= 0; int neg= 1; + THD *thd= current_thd; + ulonglong fuzzydate= TIME_NO_ZERO_DATE | TIME_NO_ZERO_IN_DATE; - null_value= 0; - if (args[0]->get_date_with_conversion(<ime1, - TIME_NO_ZERO_DATE | - TIME_NO_ZERO_IN_DATE) || - args[1]->get_date_with_conversion(<ime2, - TIME_NO_ZERO_DATE | - TIME_NO_ZERO_IN_DATE)) + null_value= 0; + + if (Datetime(thd, args[0], fuzzydate).copy_to_mysql_time(<ime1) || + Datetime(thd, args[1], fuzzydate).copy_to_mysql_time(<ime2)) goto null_date; if (calc_time_diff(<ime2,<ime1, 1, diff --git a/sql/sql_type.cc b/sql/sql_type.cc index ac9d19594fd..d2d05ae0ff4 100644 --- a/sql/sql_type.cc +++ b/sql/sql_type.cc @@ -121,15 +121,41 @@ bool Type_handler_data::init() Type_handler_data *type_handler_data= NULL; -void Time::make_from_item(Item *item) +void Time::make_from_item(Item *item, sql_mode_t flags) { - if (item->get_time(this)) + if (item->get_date(this, flags)) time_type= MYSQL_TIMESTAMP_NONE; else valid_MYSQL_TIME_to_valid_value(); } +void Temporal_with_date::make_from_item(THD *thd, Item *item, sql_mode_t flags) +{ + flags&= ~TIME_TIME_ONLY; + /* + Some TIME type items return error when trying to do get_date() + without TIME_TIME_ONLY set (e.g. Item_field for Field_time). + In the SQL standard time->datetime conversion mode we add TIME_TIME_ONLY. + In the legacy time->datetime conversion mode we do not add TIME_TIME_ONLY + and leave it to get_date() to check date. + */ + ulonglong time_flag= (item->field_type() == MYSQL_TYPE_TIME && + !(thd->variables.old_behavior & OLD_MODE_ZERO_DATE_TIME_CAST)) ? + TIME_TIME_ONLY : 0; + if (item->get_date(this, flags | time_flag)) + time_type= MYSQL_TIMESTAMP_NONE; + else if (time_type == MYSQL_TIMESTAMP_TIME) + { + MYSQL_TIME tmp; + if (time_to_datetime_with_warn(thd, this, &tmp, flags)) + time_type= MYSQL_TIMESTAMP_NONE; + else + *(static_cast(this))= tmp; + } +} + + void Type_std_attributes::set(const Field *field) { decimals= field->decimals(); @@ -3523,7 +3549,55 @@ Type_handler_temporal_result::Item_func_hybrid_field_type_get_date( MYSQL_TIME *ltime, ulonglong fuzzydate) const { - return item->get_date_from_date_op(ltime, fuzzydate); + return item->date_op(ltime, fuzzydate); +} + + +/***************************************************************************/ + +String * +Type_handler_time_common::Item_func_hybrid_field_type_val_str( + Item_func_hybrid_field_type *item, + String *str) const +{ + return item->val_str_from_time_op(str); +} + + +double +Type_handler_time_common::Item_func_hybrid_field_type_val_real( + Item_func_hybrid_field_type *item) + const +{ + return item->val_real_from_time_op(); +} + + +longlong +Type_handler_time_common::Item_func_hybrid_field_type_val_int( + Item_func_hybrid_field_type *item) + const +{ + return item->val_int_from_time_op(); +} + + +my_decimal * +Type_handler_time_common::Item_func_hybrid_field_type_val_decimal( + Item_func_hybrid_field_type *item, + my_decimal *dec) const +{ + return item->val_decimal_from_time_op(dec); +} + + +bool +Type_handler_time_common::Item_func_hybrid_field_type_get_date( + Item_func_hybrid_field_type *item, + MYSQL_TIME *ltime, + ulonglong fuzzydate) const +{ + return item->time_op(ltime); } diff --git a/sql/sql_type.h b/sql/sql_type.h index 9afdf1a915f..2e2029f43f1 100644 --- a/sql/sql_type.h +++ b/sql/sql_type.h @@ -165,19 +165,35 @@ class Time: private MYSQL_TIME break; } } - void make_from_item(class Item *item); + void make_from_item(class Item *item, sql_mode_t flags); public: Time() { time_type= MYSQL_TIMESTAMP_NONE; } - Time(Item *item) { make_from_item(item); } + Time(Item *item) { make_from_item(item, flags_for_get_date()); } + Time(Item *item, sql_mode_t flags) { make_from_item(item, flags); } + static sql_mode_t flags_for_get_date() + { return TIME_TIME_ONLY | TIME_INVALID_DATES; } + static sql_mode_t comparison_flags_for_get_date() + { return TIME_TIME_ONLY | TIME_INVALID_DATES | TIME_FUZZY_DATES; } bool is_valid_time() const { DBUG_ASSERT(is_valid_value_slow()); return time_type == MYSQL_TIMESTAMP_TIME; } - void copy_to_mysql_time(MYSQL_TIME *ltime) const + const MYSQL_TIME *get_mysql_time() const { + DBUG_ASSERT(is_valid_time_slow()); + return this; + } + bool copy_to_mysql_time(MYSQL_TIME *ltime) const + { + if (time_type == MYSQL_TIMESTAMP_NONE) + { + ltime->time_type= MYSQL_TIMESTAMP_NONE; + return true; + } DBUG_ASSERT(is_valid_time_slow()); *ltime= *this; + return false; } int cmp(const Time *other) const { @@ -194,6 +210,163 @@ public: }; +/** + Class Temporal_with_date is designed to store valid DATE or DATETIME values. + See also class Time. + + 1. Valid value: + a. MYSQL_TIMESTAMP_{DATE|DATETIME} - a valid DATE or DATETIME value + b. MYSQL_TIMESTAMP_NONE - an undefined value + + 2. Invalid value (internally only): + a. MYSQL_TIMESTAMP_{DATE|DATETIME} - a DATE or DATETIME value, but with + MYSQL_TIME members outside of the + valid/supported range + b. MYSQL_TIMESTAMP_TIME - a TIME value + c. MYSQL_TIMESTAMP_ERROR - error + + Temporarily is allowed to have an invalid value, but only internally, + during initialization time. All constructors and modification methods must + leave the value as described above (see "Valid value"). + + Derives from MYSQL_TIME using "protected" inheritance to make sure + it is accessed externally only in the valid state. +*/ + +class Temporal_with_date: protected MYSQL_TIME +{ +protected: + void make_from_item(THD *thd, Item *item, sql_mode_t flags); + Temporal_with_date(THD *thd, Item *item, sql_mode_t flags) + { + make_from_item(thd, item, flags); + } +}; + + +/** + Class Date is designed to store valid DATE values. + All constructors and modification methods leave instances + of this class in one of the following valid states: + a. MYSQL_TIMESTAMP_DATE - a DATE with all MYSQL_TIME members properly set + b. MYSQL_TIMESTAMP_NONE - an undefined value. + Other MYSQL_TIMESTAMP_XXX are not possible. + MYSQL_TIMESTAMP_DATE with MYSQL_TIME members improperly set is not possible. +*/ +class Date: public Temporal_with_date +{ + bool is_valid_value_slow() const + { + return time_type == MYSQL_TIMESTAMP_NONE || is_valid_date_slow(); + } + bool is_valid_date_slow() const + { + DBUG_ASSERT(time_type == MYSQL_TIMESTAMP_DATE); + return !check_datetime_range(this); + } +public: + Date(THD *thd, Item *item, sql_mode_t flags) + :Temporal_with_date(thd, item, flags) + { + if (time_type == MYSQL_TIMESTAMP_DATETIME) + datetime_to_date(this); + DBUG_ASSERT(is_valid_value_slow()); + } + Date(const Temporal_with_date *d) + :Temporal_with_date(*d) + { + datetime_to_date(this); + DBUG_ASSERT(is_valid_date_slow()); + } + bool is_valid_date() const + { + DBUG_ASSERT(is_valid_value_slow()); + return time_type == MYSQL_TIMESTAMP_DATE; + } + const MYSQL_TIME *get_mysql_time() const + { + DBUG_ASSERT(is_valid_date_slow()); + return this; + } +}; + + +/** + Class Datetime is designed to store valid DATETIME values. + All constructors and modification methods leave instances + of this class in one of the following valid states: + a. MYSQL_TIMESTAMP_DATETIME - a DATETIME with all members properly set + b. MYSQL_TIMESTAMP_NONE - an undefined value. + Other MYSQL_TIMESTAMP_XXX are not possible. + MYSQL_TIMESTAMP_DATETIME with MYSQL_TIME members + improperly set is not possible. +*/ +class Datetime: public Temporal_with_date +{ + bool is_valid_value_slow() const + { + return time_type == MYSQL_TIMESTAMP_NONE || is_valid_datetime_slow(); + } + bool is_valid_datetime_slow() const + { + DBUG_ASSERT(time_type == MYSQL_TIMESTAMP_DATETIME); + return !check_datetime_range(this); + } +public: + Datetime(THD *thd, Item *item, sql_mode_t flags) + :Temporal_with_date(thd, item, flags) + { + if (time_type == MYSQL_TIMESTAMP_DATE) + date_to_datetime(this); + DBUG_ASSERT(is_valid_value_slow()); + } + bool is_valid_datetime() const + { + /* + Here we quickly check for the type only. + If the type is valid, the rest of value must also be valid. + */ + DBUG_ASSERT(is_valid_value_slow()); + return time_type == MYSQL_TIMESTAMP_DATETIME; + } + bool hhmmssff_is_zero() const + { + DBUG_ASSERT(is_valid_datetime_slow()); + return hour == 0 && minute == 0 && second == 0 && second_part == 0; + } + const MYSQL_TIME *get_mysql_time() const + { + DBUG_ASSERT(is_valid_datetime_slow()); + return this; + } + bool copy_to_mysql_time(MYSQL_TIME *ltime) const + { + if (time_type == MYSQL_TIMESTAMP_NONE) + { + ltime->time_type= MYSQL_TIMESTAMP_NONE; + return true; + } + DBUG_ASSERT(is_valid_datetime_slow()); + *ltime= *this; + return false; + } + /** + Copy without data loss, with an optional DATETIME to DATE conversion. + If the value of the "type" argument is MYSQL_TIMESTAMP_DATE, + then "this" must be a datetime with a zero hhmmssff part. + */ + bool copy_to_mysql_time(MYSQL_TIME *ltime, timestamp_type type) + { + DBUG_ASSERT(type == MYSQL_TIMESTAMP_DATE || + type == MYSQL_TIMESTAMP_DATETIME); + if (copy_to_mysql_time(ltime)) + return true; + DBUG_ASSERT(type != MYSQL_TIMESTAMP_DATE || hhmmssff_is_zero()); + ltime->time_type= type; + return false; + } +}; + /* Flags for collation aggregation modes, used in TDCollation::agg(): @@ -2220,6 +2393,18 @@ public: Type_handler_hybrid_field_type *, Type_all_attributes *atrr, Item **items, uint nitems) const; + String *Item_func_hybrid_field_type_val_str(Item_func_hybrid_field_type *, + String *) const; + double Item_func_hybrid_field_type_val_real(Item_func_hybrid_field_type *) + const; + longlong Item_func_hybrid_field_type_val_int(Item_func_hybrid_field_type *) + const; + my_decimal *Item_func_hybrid_field_type_val_decimal( + Item_func_hybrid_field_type *, + my_decimal *) const; + bool Item_func_hybrid_field_type_get_date(Item_func_hybrid_field_type *, + MYSQL_TIME *, + ulonglong fuzzydate) const; bool Item_func_min_max_get_date(Item_func_min_max*, MYSQL_TIME *, ulonglong fuzzydate) const; Item *make_const_item_for_comparison(THD *, Item *src, const Item *cmp) const; From a128fe4346a466aa8e059baa620a3b689fa21241 Mon Sep 17 00:00:00 2001 From: Galina Shalygina Date: Tue, 20 Feb 2018 01:16:50 +0200 Subject: [PATCH 19/37] MDEV-14297: Lost name of a explicitly named CTE column used in the non-recursive CTE via prepared statement The problem appears as the column names of the CTE were allocated on the wrong MEMROOT and after the preparation of the statement they disappear. To fix it in the procedure With_element::rename_columns_of_derived_unit the CTE column names are now allocated in the permanent MEMROOT for the prepared statements and stored procedures. --- mysql-test/r/cte_nonrecursive.result | 27 +++++++++++++++++++++++++++ mysql-test/t/cte_nonrecursive.test | 25 +++++++++++++++++++++++++ sql/sql_cte.cc | 7 +++++++ 3 files changed, 59 insertions(+) diff --git a/mysql-test/r/cte_nonrecursive.result b/mysql-test/r/cte_nonrecursive.result index 33e14af900a..f2bfce9f6ba 100644 --- a/mysql-test/r/cte_nonrecursive.result +++ b/mysql-test/r/cte_nonrecursive.result @@ -1398,3 +1398,30 @@ i 2 3 DROP TABLE t1; +# +# MDEV-14297: Lost name of a explicitly named CTE column used in +# the non-recursive CTE via prepared statement +# +CREATE TABLE t1 (i int); +INSERT INTO t1 VALUES (1),(2),(3); +PREPARE stmt FROM "WITH cte(a) AS (SELECT 1) SELECT * FROM cte"; +EXECUTE stmt; +a +1 +DEALLOCATE PREPARE stmt; +PREPARE stmt FROM "CREATE VIEW v1 AS WITH cte(a) AS (SELECT 1) SELECT * FROM cte"; +EXECUTE stmt; +SELECT * FROM v1; +a +1 +DEALLOCATE PREPARE stmt; +PREPARE stmt FROM "CREATE VIEW v2 AS WITH cte(a) AS (SELECT * FROM t1) SELECT * FROM cte"; +EXECUTE stmt; +SELECT * FROM v2; +a +1 +2 +3 +DEALLOCATE PREPARE stmt; +DROP TABLE t1; +DROP VIEW v1,v2; diff --git a/mysql-test/t/cte_nonrecursive.test b/mysql-test/t/cte_nonrecursive.test index 05de03fc7af..3d073183877 100644 --- a/mysql-test/t/cte_nonrecursive.test +++ b/mysql-test/t/cte_nonrecursive.test @@ -964,3 +964,28 @@ WITH RECURSIVE c1 AS (WITH c3 AS (SELECT * FROM c2) SELECT * FROM c3), SELECT * FROM c1; DROP TABLE t1; + +--echo # +--echo # MDEV-14297: Lost name of a explicitly named CTE column used in +--echo # the non-recursive CTE via prepared statement +--echo # + +CREATE TABLE t1 (i int); +INSERT INTO t1 VALUES (1),(2),(3); + +PREPARE stmt FROM "WITH cte(a) AS (SELECT 1) SELECT * FROM cte"; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +PREPARE stmt FROM "CREATE VIEW v1 AS WITH cte(a) AS (SELECT 1) SELECT * FROM cte"; +EXECUTE stmt; +SELECT * FROM v1; +DEALLOCATE PREPARE stmt; + +PREPARE stmt FROM "CREATE VIEW v2 AS WITH cte(a) AS (SELECT * FROM t1) SELECT * FROM cte"; +EXECUTE stmt; +SELECT * FROM v2; +DEALLOCATE PREPARE stmt; + +DROP TABLE t1; +DROP VIEW v1,v2; diff --git a/sql/sql_cte.cc b/sql/sql_cte.cc index ac27f1713f9..dd46295d799 100644 --- a/sql/sql_cte.cc +++ b/sql/sql_cte.cc @@ -910,12 +910,19 @@ With_element::rename_columns_of_derived_unit(THD *thd, my_error(ER_WITH_COL_WRONG_LIST, MYF(0)); return true; } + + Query_arena *arena, backup; + arena= thd->activate_stmt_arena_if_needed(&backup); + /* Rename the columns of the first select in the unit */ while ((item= it++, name= nm++)) { item->set_name(thd, name->str, (uint) name->length, system_charset_info); item->is_autogenerated_name= false; } + + if (arena) + thd->restore_active_arena(arena, &backup); } else make_valid_column_names(thd, select->item_list); From 3a7126cbb7ade5c736ec4e124704faaecd5f80f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Tue, 20 Feb 2018 09:17:29 +0200 Subject: [PATCH 20/37] Disable galera_gtid until 1047: WSREP has not yet prepared node for application use is fixed. --- mysql-test/suite/galera/disabled.def | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def index b67dea0f2f4..742917dc122 100644 --- a/mysql-test/suite/galera/disabled.def +++ b/mysql-test/suite/galera/disabled.def @@ -58,4 +58,5 @@ MW-328C: MDEV-13549 Galera test failures 10.1 MW-328A: MDEV-13549 Galera test failures 10.1 MW-328B: MDEV-13549 Galera test failures 10.1 MW-328: MDEV-13549 Galera test failures 10.1 -galera_suspend_slave: MDEV-13549 Galera test failures 10.1 \ No newline at end of file +galera_suspend_slave: MDEV-13549 Galera test failures 10.1 +galera_gtid: MDEV-13549 Galera test failures 10.1 \ No newline at end of file From c3a3b77f9a1c4bc320e071a187e454a62fe70366 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Tue, 20 Feb 2018 09:27:31 +0200 Subject: [PATCH 21/37] Disable failing Galera tests: * galera_gtid_slave * galera_unicode_identifiers --- mysql-test/suite/galera/disabled.def | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def index 742917dc122..d82d5dd2023 100644 --- a/mysql-test/suite/galera/disabled.def +++ b/mysql-test/suite/galera/disabled.def @@ -54,9 +54,11 @@ galera_pc_ignore_sb : MDEV-13549 Galera test failures 10.1 galera_lock_table : MDEV-13549 Galera test failures 10.1 MW-284 : MDEV-13549 Galera test failures 10.1 galera_as_slave : MDEV-13549 Galera test failures 10.1 -MW-328C: MDEV-13549 Galera test failures 10.1 -MW-328A: MDEV-13549 Galera test failures 10.1 -MW-328B: MDEV-13549 Galera test failures 10.1 -MW-328: MDEV-13549 Galera test failures 10.1 -galera_suspend_slave: MDEV-13549 Galera test failures 10.1 -galera_gtid: MDEV-13549 Galera test failures 10.1 \ No newline at end of file +MW-328C : MDEV-13549 Galera test failures 10.1 +MW-328A : MDEV-13549 Galera test failures 10.1 +MW-328B : MDEV-13549 Galera test failures 10.1 +MW-328 : MDEV-13549 Galera test failures 10.1 +galera_suspend_slave : MDEV-13549 Galera test failures 10.1 +galera_gtid : MDEV-13549 Galera test failures 10.1 +galera_gtid_slave : MDEV-13549 Galera test failures 10.1 +galera_unicode_identifiers : MDEV-13549 Galera test failures 10.1 From 56d6776524909b5bd413996cb703e099024105a5 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 20 Feb 2018 20:03:23 +0000 Subject: [PATCH 22/37] Windows : remove freopen(), "to keep fd = 0 busy". a) We do not need this on Windows, and it is not clear what it does,inside service. b) It hinders debugging. mysql-test-run.pl --debugger=vsjitdebugger more often than , would stop here throwing "invalid handle" exception. --- sql/mysqld.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 0de9d7a9f0b..3cb8e41f94e 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -5871,12 +5871,14 @@ int mysqld_main(int argc, char **argv) mysqld_port, MYSQL_COMPILATION_COMMENT); +#ifndef _WIN32 // try to keep fd=0 busy - if (!freopen(IF_WIN("NUL","/dev/null"), "r", stdin)) + if (!freopen("/dev/null", "r", stdin)) { // fall back on failure fclose(stdin); } +#endif #if defined(_WIN32) && !defined(EMBEDDED_LIBRARY) Service.SetRunning(); From 5417002dae7a9f15ad77c58fae93eada80699b4b Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Wed, 21 Feb 2018 08:18:44 +0400 Subject: [PATCH 23/37] A cleanup for MDEV-15340 + fix MDEV-15363 Wrong result for CAST(LAST_DAY(TIME'00:00:00') AS TIME) The change N7 in MDEV-15340 (see the commit message) introduced a regression in how CAST(AS TIME), HOUR(), TIME_TO_SEC() treat datetimes '0000-00-DD mm:hh:ss' (i.e. with zero YYYYMM part and a non-zero day). These functions historically do not mix days to hours on datetime-to-time conversion. Implementations of the underlying methods used get_arg0_time() to fetch MYSQL_TIME. After MDEV-15340, get_arg0_time() went through the Time() constructor, which always adds '0000-00-DD' to hours automatically (as in all other places in the code we do mix days to hours). Changes: 1. Extending Time() to make it possible to choose a desired way of treating '0000-00-DD' (ignore or mix to hours) on datetime-to-time conversion. Adding a helper class Time::Options for this, which now describes two aspects of Time() creation: 1. Flags for get_date() 2. Days/hours mixing behavior. 2. Removing Item_func::get_arg0_time(). Using Time() directly in all affected classes. Forcing Time() to ignore (rather than mix) '0000-00-DD' in these affected classes by passing a suitable Options value. 3. Adding Time::to_seconds(), to reuse the code between Item_func_time_to_sec::decimal_op() and Item_func_time_to_sec::int_op(). 4. Item_func::get_arg0_date() now returns only a datetime value, with automatic time-to-datetime conversion if needed. An assert was added to catch attempts to pass TIME_TIME_ONLY to get_arg0_date(). All callers were checked not to pass TIME_TIME_ONLY, this revealed a bug MDEV-15363. 5. Changing Item_func_last_day::get_date() to remove the TIME_TIME_ONLY flag before calling get_arg0_date(). This fixes MDEV-15363. --- mysql-test/r/func_time.result | 19 ++++++++++ mysql-test/t/func_time.test | 23 ++++++++++++ sql/item_func.h | 10 +----- sql/item_timefunc.cc | 54 ++++++++++++---------------- sql/sql_type.cc | 6 ++-- sql/sql_type.h | 66 +++++++++++++++++++++++++++++++---- 6 files changed, 127 insertions(+), 51 deletions(-) diff --git a/mysql-test/r/func_time.result b/mysql-test/r/func_time.result index 53f61f6644f..e03de2ca582 100644 --- a/mysql-test/r/func_time.result +++ b/mysql-test/r/func_time.result @@ -3382,4 +3382,23 @@ ADDTIME(TIME'10:20:30', DATE'2001-01-01') AS c3, ADDTIME(TIME'10:20:30', COALESCE(DATE'2001-01-01',TIMESTAMP'2001-01-01 00:00:00')) AS c4; c1 c2 c3 c4 NULL NULL NULL NULL +SELECT +HOUR(TIMESTAMP'0000-00-01 10:00:00') AS h0, +TIME_TO_SEC(TIMESTAMP'0000-00-01 10:00:00') AS tts0, +TIME_TO_SEC(TIMESTAMP'0000-00-01 10:00:00.1') AS tts1, +CAST(TIMESTAMP'0000-00-01 10:00:00' AS TIME) AS c0, +CAST(TIMESTAMP'0000-00-01 10:00:00.1' AS TIME(1)) AS c2; +h0 tts0 tts1 c0 c2 +10 36000 36000.1 10:00:00 10:00:00.1 +SET TIMESTAMP=DEFAULT; +# +# MDEV-15363 Wrong result for CAST(LAST_DAY(TIME'00:00:00') AS TIME) +# +SET TIMESTAMP=UNIX_TIMESTAMP('2018-02-17 01:02:03'); +SELECT +LAST_DAY(TIME'00:00:00') AS c1, +CAST(CAST(LAST_DAY(TIME'00:00:00') AS DATE) AS TIME) AS c2, +CAST(LAST_DAY(TIME'00:00:00') AS TIME) AS c3; +c1 c2 c3 +2018-02-28 00:00:00 00:00:00 SET TIMESTAMP=DEFAULT; diff --git a/mysql-test/t/func_time.test b/mysql-test/t/func_time.test index 0066d4a434b..a475c33b925 100644 --- a/mysql-test/t/func_time.test +++ b/mysql-test/t/func_time.test @@ -1974,4 +1974,27 @@ SELECT ADDTIME(TIME'10:20:30', DATE'2001-01-01') AS c3, ADDTIME(TIME'10:20:30', COALESCE(DATE'2001-01-01',TIMESTAMP'2001-01-01 00:00:00')) AS c4; +# +# Make sure that time functions that in 10.2 used get_arg0_time() +# do not mix days to hours for dates with zero YYYYMM and non-zero days. +# + +SELECT + HOUR(TIMESTAMP'0000-00-01 10:00:00') AS h0, + TIME_TO_SEC(TIMESTAMP'0000-00-01 10:00:00') AS tts0, + TIME_TO_SEC(TIMESTAMP'0000-00-01 10:00:00.1') AS tts1, + CAST(TIMESTAMP'0000-00-01 10:00:00' AS TIME) AS c0, + CAST(TIMESTAMP'0000-00-01 10:00:00.1' AS TIME(1)) AS c2; + +SET TIMESTAMP=DEFAULT; + +--echo # +--echo # MDEV-15363 Wrong result for CAST(LAST_DAY(TIME'00:00:00') AS TIME) +--echo # + +SET TIMESTAMP=UNIX_TIMESTAMP('2018-02-17 01:02:03'); +SELECT + LAST_DAY(TIME'00:00:00') AS c1, + CAST(CAST(LAST_DAY(TIME'00:00:00') AS DATE) AS TIME) AS c2, + CAST(LAST_DAY(TIME'00:00:00') AS TIME) AS c3; SET TIMESTAMP=DEFAULT; diff --git a/sql/item_func.h b/sql/item_func.h index 063a80de737..7d7cd61f474 100644 --- a/sql/item_func.h +++ b/sql/item_func.h @@ -160,18 +160,10 @@ public: void print_args(String *str, uint from, enum_query_type query_type); inline bool get_arg0_date(MYSQL_TIME *ltime, ulonglong fuzzy_date) { - return fuzzy_date & TIME_TIME_ONLY ? get_arg0_time(ltime) : - get_arg0_datetime(ltime, fuzzy_date); - } - inline bool get_arg0_datetime(MYSQL_TIME *ltime, ulonglong fuzzy_date) - { + DBUG_ASSERT(!(fuzzy_date & TIME_TIME_ONLY)); Datetime dt(current_thd, args[0], fuzzy_date); return (null_value= dt.copy_to_mysql_time(ltime)); } - inline bool get_arg0_time(MYSQL_TIME *ltime) - { - return (null_value= Time(args[0]).copy_to_mysql_time(ltime)); - } bool is_null() { update_null_value(); return null_value; diff --git a/sql/item_timefunc.cc b/sql/item_timefunc.cc index 816514dcc35..12b6d4cb9f3 100644 --- a/sql/item_timefunc.cc +++ b/sql/item_timefunc.cc @@ -993,15 +993,15 @@ longlong Item_func_quarter::val_int() longlong Item_func_hour::val_int() { DBUG_ASSERT(fixed == 1); - MYSQL_TIME ltime; - return get_arg0_time(<ime) ? 0 : ltime.hour; + Time tm(args[0], Time::Options_for_cast()); + return (null_value= !tm.is_valid_time()) ? 0 : tm.get_mysql_time()->hour; } longlong Item_func_minute::val_int() { DBUG_ASSERT(fixed == 1); - MYSQL_TIME ltime; - return get_arg0_time(<ime) ? 0 : ltime.minute; + Time tm(args[0], Time::Options_for_cast()); + return (null_value= !tm.is_valid_time()) ? 0 : tm.get_mysql_time()->minute; } /** @@ -1010,8 +1010,8 @@ longlong Item_func_minute::val_int() longlong Item_func_second::val_int() { DBUG_ASSERT(fixed == 1); - MYSQL_TIME ltime; - return get_arg0_time(<ime) ? 0 : ltime.second; + Time tm(args[0], Time::Options_for_cast()); + return (null_value= !tm.is_valid_time()) ? 0 : tm.get_mysql_time()->second; } @@ -1267,24 +1267,20 @@ longlong Item_func_unix_timestamp::val_int_endpoint(bool left_endp, bool *incl_e longlong Item_func_time_to_sec::int_op() { DBUG_ASSERT(fixed == 1); - MYSQL_TIME ltime; - if (get_arg0_time(<ime)) - return 0; - - longlong seconds=ltime.hour*3600L+ltime.minute*60+ltime.second; - return ltime.neg ? -seconds : seconds; + Time tm(args[0], Time::Options_for_cast()); + return ((null_value= !tm.is_valid_time())) ? 0 : tm.to_seconds(); } my_decimal *Item_func_time_to_sec::decimal_op(my_decimal* buf) { DBUG_ASSERT(fixed == 1); - MYSQL_TIME ltime; - if (get_arg0_time(<ime)) + Time tm(args[0], Time::Options_for_cast()); + if ((null_value= !tm.is_valid_time())) return 0; - - longlong seconds= ltime.hour*3600L+ltime.minute*60+ltime.second; - return seconds2my_decimal(ltime.neg, seconds, ltime.second_part, buf); + const MYSQL_TIME *ltime= tm.get_mysql_time(); + longlong seconds= tm.to_seconds_abs(); + return seconds2my_decimal(ltime->neg, seconds, ltime->second_part, buf); } @@ -1991,7 +1987,7 @@ String *Item_func_date_format::val_str(String *str) const MY_LOCALE *lc= 0; DBUG_ASSERT(fixed == 1); - if (get_arg0_date(&l_time, is_time_format ? TIME_TIME_ONLY : 0)) + if ((null_value= args[0]->get_date(&l_time, is_time_format ? TIME_TIME_ONLY : 0))) return 0; if (!(format = args[1]->val_str(str)) || !format->length()) @@ -2607,17 +2603,12 @@ void Item_char_typecast::fix_length_and_dec_internal(CHARSET_INFO *from_cs) bool Item_time_typecast::get_date(MYSQL_TIME *ltime, ulonglong fuzzy_date) { - if (get_arg0_time(ltime)) - return 1; + Time tm(args[0], Time::Options_for_cast()); + if ((null_value= !tm.is_valid_time())) + return true; + tm.copy_to_mysql_time(ltime); if (decimals < TIME_SECOND_PART_DIGITS) my_time_trunc(ltime, decimals); - /* - MYSQL_TIMESTAMP_TIME value can have non-zero day part, - which we should not lose. - */ - if (ltime->time_type != MYSQL_TIMESTAMP_TIME) - ltime->year= ltime->month= ltime->day= 0; - ltime->time_type= MYSQL_TIMESTAMP_TIME; return (fuzzy_date & TIME_TIME_ONLY) ? 0 : (null_value= check_date_with_warn(ltime, fuzzy_date, MYSQL_TIMESTAMP_ERROR)); @@ -2936,10 +2927,9 @@ bool Item_func_maketime::get_date(MYSQL_TIME *ltime, ulonglong fuzzy_date) longlong Item_func_microsecond::val_int() { DBUG_ASSERT(fixed == 1); - MYSQL_TIME ltime; - if (!get_arg0_date(<ime, TIME_TIME_ONLY)) - return ltime.second_part; - return 0; + Time tm(args[0], Time::Options_for_cast()); + return ((null_value= !tm.is_valid_time())) ? + 0 : tm.get_mysql_time()->second_part; } @@ -3322,7 +3312,7 @@ bool Item_func_str_to_date::get_date(MYSQL_TIME *ltime, ulonglong fuzzy_date) bool Item_func_last_day::get_date(MYSQL_TIME *ltime, ulonglong fuzzy_date) { - if (get_arg0_date(ltime, fuzzy_date) || + if (get_arg0_date(ltime, fuzzy_date & ~TIME_TIME_ONLY) || (ltime->month == 0)) return (null_value=1); uint month_idx= ltime->month-1; diff --git a/sql/sql_type.cc b/sql/sql_type.cc index d2d05ae0ff4..5dfe13c9ad3 100644 --- a/sql/sql_type.cc +++ b/sql/sql_type.cc @@ -121,12 +121,12 @@ bool Type_handler_data::init() Type_handler_data *type_handler_data= NULL; -void Time::make_from_item(Item *item, sql_mode_t flags) +void Time::make_from_item(Item *item, const Options opt) { - if (item->get_date(this, flags)) + if (item->get_date(this, opt.get_date_flags())) time_type= MYSQL_TIMESTAMP_NONE; else - valid_MYSQL_TIME_to_valid_value(); + valid_MYSQL_TIME_to_valid_value(opt); } diff --git a/sql/sql_type.h b/sql/sql_type.h index 2e2029f43f1..348a6cd4fb6 100644 --- a/sql/sql_type.h +++ b/sql/sql_type.h @@ -94,6 +94,47 @@ struct SORT_FIELD_ATTR; */ class Time: private MYSQL_TIME { +public: + enum datetime_to_time_mode_t + { + DATETIME_TO_TIME_YYYYMMDD_000000DD_MIX_TO_HOURS, + DATETIME_TO_TIME_YYYYMMDD_TRUNCATE + }; + class Options + { + sql_mode_t m_get_date_flags; + datetime_to_time_mode_t m_datetime_to_time_mode; + public: + Options() + :m_get_date_flags(flags_for_get_date()), + m_datetime_to_time_mode(DATETIME_TO_TIME_YYYYMMDD_000000DD_MIX_TO_HOURS) + { } + Options(sql_mode_t flags) + :m_get_date_flags(flags), + m_datetime_to_time_mode(DATETIME_TO_TIME_YYYYMMDD_000000DD_MIX_TO_HOURS) + { } + Options(sql_mode_t flags, datetime_to_time_mode_t dtmode) + :m_get_date_flags(flags), + m_datetime_to_time_mode(dtmode) + { } + sql_mode_t get_date_flags() const + { return m_get_date_flags; } + datetime_to_time_mode_t datetime_to_time_mode() const + { return m_datetime_to_time_mode; } + }; + /* + CAST(AS TIME) historically does not mix days to hours. + This is different comparing to how implicit conversion + in Field::store_time_dec() works (e.g. on INSERT). + */ + class Options_for_cast: public Options + { + public: + Options_for_cast() + :Options(flags_for_get_date(), DATETIME_TO_TIME_YYYYMMDD_TRUNCATE) + { } + }; +private: bool is_valid_value_slow() const { return time_type == MYSQL_TIMESTAMP_NONE || is_valid_time_slow(); @@ -113,7 +154,7 @@ class Time: private MYSQL_TIME e.g. returned from Item::get_date(). After this call, "this" is a valid TIME value. */ - void valid_datetime_to_valid_time() + void valid_datetime_to_valid_time(const Options opt) { DBUG_ASSERT(time_type == MYSQL_TIMESTAMP_DATE || time_type == MYSQL_TIMESTAMP_DATETIME); @@ -123,7 +164,9 @@ class Time: private MYSQL_TIME */ DBUG_ASSERT(day < 32); DBUG_ASSERT(hour < 24); - if (year == 0 && month == 0) + if (year == 0 && month == 0 && + opt.datetime_to_time_mode() == + DATETIME_TO_TIME_YYYYMMDD_000000DD_MIX_TO_HOURS) { /* The maximum hour value after mixing days will be 31*24+23=767, @@ -148,12 +191,12 @@ class Time: private MYSQL_TIME - either a valid TIME (within the supported TIME range), - or MYSQL_TIMESTAMP_NONE */ - void valid_MYSQL_TIME_to_valid_value() + void valid_MYSQL_TIME_to_valid_value(const Options opt) { switch (time_type) { case MYSQL_TIMESTAMP_DATE: case MYSQL_TIMESTAMP_DATETIME: - valid_datetime_to_valid_time(); + valid_datetime_to_valid_time(opt); break; case MYSQL_TIMESTAMP_NONE: break; @@ -165,11 +208,11 @@ class Time: private MYSQL_TIME break; } } - void make_from_item(class Item *item, sql_mode_t flags); + void make_from_item(class Item *item, const Options opt); public: Time() { time_type= MYSQL_TIMESTAMP_NONE; } - Time(Item *item) { make_from_item(item, flags_for_get_date()); } - Time(Item *item, sql_mode_t flags) { make_from_item(item, flags); } + Time(Item *item) { make_from_item(item, Options()); } + Time(Item *item, const Options opt) { make_from_item(item, opt); } static sql_mode_t flags_for_get_date() { return TIME_TIME_ONLY | TIME_INVALID_DATES; } static sql_mode_t comparison_flags_for_get_date() @@ -207,6 +250,15 @@ public: return 1; return 0; } + longlong to_seconds_abs() const + { + DBUG_ASSERT(is_valid_time_slow()); + return hour * 3600L + minute * 60 + second; + } + longlong to_seconds() const + { + return neg ? -to_seconds_abs() : to_seconds_abs(); + } }; From 144c7f8b6e98e8d1edace59c95cff4a315e1336a Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Wed, 21 Feb 2018 13:11:04 +0400 Subject: [PATCH 24/37] Adding Field_timestamp::sql_mode_for_timestamp() to reuse duplicate code --- sql/field.cc | 33 +++++++++++++++------------------ sql/field.h | 1 + 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/sql/field.cc b/sql/field.cc index 38978fbb727..70cdea47a50 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -5016,6 +5016,13 @@ copy_or_convert_to_datetime(THD *thd, const MYSQL_TIME *from, MYSQL_TIME *to) } +sql_mode_t Field_timestamp::sql_mode_for_timestamp(THD *thd) const +{ + // We don't want to store invalid or fuzzy datetime values in TIMESTAMP + return (thd->variables.sql_mode & MODE_NO_ZERO_DATE) | MODE_NO_ZERO_IN_DATE; +} + + int Field_timestamp::store_time_dec(const MYSQL_TIME *ltime, uint dec) { int unused; @@ -5024,9 +5031,7 @@ int Field_timestamp::store_time_dec(const MYSQL_TIME *ltime, uint dec) MYSQL_TIME l_time; bool valid= !copy_or_convert_to_datetime(thd, ltime, &l_time) && !check_date(&l_time, pack_time(&l_time) != 0, - (thd->variables.sql_mode & MODE_NO_ZERO_DATE) | - MODE_NO_ZERO_IN_DATE, &unused); - + sql_mode_for_timestamp(thd), &unused); return store_TIME_with_warning(thd, &l_time, &str, false, valid); } @@ -5039,11 +5044,8 @@ int Field_timestamp::store(const char *from,uint len,CHARSET_INFO *cs) ErrConvString str(from, len, cs); THD *thd= get_thd(); - /* We don't want to store invalid or fuzzy datetime values in TIMESTAMP */ have_smth_to_conv= !str_to_datetime(cs, from, len, &l_time, - (thd->variables.sql_mode & - MODE_NO_ZERO_DATE) | - MODE_NO_ZERO_IN_DATE, &status); + sql_mode_for_timestamp(thd), &status); return store_TIME_with_warning(thd, &l_time, &str, status.warnings, have_smth_to_conv); } @@ -5056,9 +5058,8 @@ int Field_timestamp::store(double nr) ErrConvDouble str(nr); THD *thd= get_thd(); - longlong tmp= double_to_datetime(nr, &l_time, (thd->variables.sql_mode & - MODE_NO_ZERO_DATE) | - MODE_NO_ZERO_IN_DATE, &error); + longlong tmp= double_to_datetime(nr, &l_time, sql_mode_for_timestamp(thd), + &error); return store_TIME_with_warning(thd, &l_time, &str, error, tmp != -1); } @@ -5070,10 +5071,8 @@ int Field_timestamp::store(longlong nr, bool unsigned_val) ErrConvInteger str(nr, unsigned_val); THD *thd= get_thd(); - /* We don't want to store invalid or fuzzy datetime values in TIMESTAMP */ - longlong tmp= number_to_datetime(nr, 0, &l_time, (thd->variables.sql_mode & - MODE_NO_ZERO_DATE) | - MODE_NO_ZERO_IN_DATE, &error); + longlong tmp= number_to_datetime(nr, 0, &l_time, sql_mode_for_timestamp(thd), + &error); return store_TIME_with_warning(thd, &l_time, &str, error, tmp != -1); } @@ -5398,10 +5397,8 @@ int Field_timestamp::store_decimal(const my_decimal *d) error= 2; } else - tmp= number_to_datetime(nr, sec_part, <ime, TIME_NO_ZERO_IN_DATE | - (thd->variables.sql_mode & - MODE_NO_ZERO_DATE), &error); - + tmp= number_to_datetime(nr, sec_part, <ime, sql_mode_for_timestamp(thd), + &error); return store_TIME_with_warning(thd, <ime, &str, error, tmp != -1); } diff --git a/sql/field.h b/sql/field.h index bc56ba4db7b..74d2e2301c4 100644 --- a/sql/field.h +++ b/sql/field.h @@ -2359,6 +2359,7 @@ public: class Field_timestamp :public Field_temporal { protected: + sql_mode_t sql_mode_for_timestamp(THD *thd) const; int store_TIME_with_warning(THD *, MYSQL_TIME *, const ErrConv *, int warnings, bool have_smth_to_conv); public: From 01e89d6a86ddc26da29ad530e74c20cee0529fa5 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Wed, 21 Feb 2018 15:42:34 +0300 Subject: [PATCH 25/37] MDEV-15372: Parallel slave speedup very limited when log_slave_updates=OFF Make MyRocks' non-XA commit path to first do the commit without syncing and then sync. --- storage/rocksdb/ha_rocksdb.cc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index 4a0a3eddb1a..0231dce7cf5 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -3154,10 +3154,20 @@ static int rocksdb_commit(handlerton* hton, THD* thd, bool commit_tx) - For a COMMIT statement that finishes a multi-statement transaction - For a statement that has its own transaction */ + + // First, commit without syncing. This establishes the commit order + tx->set_sync(false); if (tx->commit()) { DBUG_RETURN(HA_ERR_ROCKSDB_COMMIT_FAILED); } thd_wakeup_subsequent_commits(thd, 0); + + if (rocksdb_flush_log_at_trx_commit == FLUSH_LOG_SYNC) + { + rocksdb::Status s= rdb->FlushWAL(true); + if (!s.ok()) + DBUG_RETURN(HA_ERR_INTERNAL_ERROR); + } } else { /* We get here when committing a statement within a transaction. From 88d1c1c5514dc922699e69d64877b7788c74ffcc Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 12 Feb 2018 15:12:49 +0100 Subject: [PATCH 26/37] MDEV-15288 Configure errors when building without INNOBASE --- storage/innobase/CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index 048a1286961..18efa58b2a3 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -489,4 +489,6 @@ MYSQL_ADD_PLUGIN(innobase ${INNOBASE_SOURCES} STORAGE_ENGINE MODULE_OUTPUT_NAME ha_innodb LINK_LIBRARIES ${ZLIB_LIBRARY} ${LINKER_SCRIPT}) -ADD_DEPENDENCIES(innobase GenError) +IF(TARGET innobase) + ADD_DEPENDENCIES(innobase GenError) +ENDIF() From 00a556c0c20f58f06c95b7976aef0b174abb30ff Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Wed, 21 Feb 2018 17:00:03 +0300 Subject: [PATCH 27/37] MDEV-15372: Parallel slave speedup very limited when log_slave_updates=OFF Part #2: some transactions have m_rocksdb_tx==NULL (and most functions of Rdb_transction_impl handle this case. Do like they do) --- storage/rocksdb/ha_rocksdb.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index 0231dce7cf5..bc15e948d41 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -2240,7 +2240,8 @@ public: } void set_sync(bool sync) override { - m_rocksdb_tx->GetWriteOptions()->sync = sync; + if (m_rocksdb_tx) + m_rocksdb_tx->GetWriteOptions()->sync = sync; } void release_lock(rocksdb::ColumnFamilyHandle *const column_family, From 4e6dab94d0931eafba502f5a91da29a54e75bb33 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 21 Feb 2018 19:38:57 +0530 Subject: [PATCH 28/37] MDEV-10.1.31 does not join an existing cluster with SST xtrabackup-v2 Analysis:- The problem is the change in the implementation of wait_for_listen in wsrep_sst_xtrabackup-v2.sh. The new script uses lsof which will always exit with an error code if it can't find all the items, and because the script has the -e option set in the hashbang line (#!/bin/bash -ue), the script will abort right after running lsof if lsof can't find even a single item among all the items listed in its arguments. This will happen even if socat is running and listening, because it can't find nc. The loop in wait_for_listen will therefore always quit after one iteration without writing the "ready" line to signal the parent. Solution:- We will or the lsof with true. Patch Credit :Daniel Black and David Wang --- scripts/wsrep_sst_xtrabackup-v2.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh index 64dd182e2f2..9104daf19bc 100644 --- a/scripts/wsrep_sst_xtrabackup-v2.sh +++ b/scripts/wsrep_sst_xtrabackup-v2.sh @@ -644,7 +644,7 @@ wait_for_listen() for i in {1..300} do - LSOF_OUT=$(lsof -sTCP:LISTEN -i TCP:${PORT} -a -c nc -c socat -F c) + LSOF_OUT=$(lsof -sTCP:LISTEN -i TCP:${PORT} -a -c nc -c socat -F c 2> /dev/null || :) [ -n "${LSOF_OUT}" ] && break sleep 0.2 done From db0484f355dae8d3a4e4aab731522521e8ff976e Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Wed, 21 Feb 2018 17:27:46 +0300 Subject: [PATCH 29/37] Change MyRocks Maturity Level from Beta to Gamma (RC) --- storage/rocksdb/ha_rocksdb.h | 2 +- .../rocksdb/r/mariadb_port_fixes.result | 24 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/storage/rocksdb/ha_rocksdb.h b/storage/rocksdb/ha_rocksdb.h index 4432a4de8d1..1769072722a 100644 --- a/storage/rocksdb/ha_rocksdb.h +++ b/storage/rocksdb/ha_rocksdb.h @@ -1412,6 +1412,6 @@ private: Rdb_inplace_alter_ctx &operator=(const Rdb_inplace_alter_ctx &); }; -const int MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL= MariaDB_PLUGIN_MATURITY_BETA; +const int MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL= MariaDB_PLUGIN_MATURITY_GAMMA; } // namespace myrocks diff --git a/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result b/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result index 021373be02a..27b1fd1e9af 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result @@ -69,15 +69,15 @@ set global rocksdb_strict_collation_check=@tmp_rscc; # select plugin_name, plugin_maturity from information_schema.plugins where plugin_name like '%rocksdb%'; plugin_name plugin_maturity -ROCKSDB Beta -ROCKSDB_CFSTATS Beta -ROCKSDB_DBSTATS Beta -ROCKSDB_PERF_CONTEXT Beta -ROCKSDB_PERF_CONTEXT_GLOBAL Beta -ROCKSDB_CF_OPTIONS Beta -ROCKSDB_COMPACTION_STATS Beta -ROCKSDB_GLOBAL_INFO Beta -ROCKSDB_DDL Beta -ROCKSDB_INDEX_FILE_MAP Beta -ROCKSDB_LOCKS Beta -ROCKSDB_TRX Beta +ROCKSDB Gamma +ROCKSDB_CFSTATS Gamma +ROCKSDB_DBSTATS Gamma +ROCKSDB_PERF_CONTEXT Gamma +ROCKSDB_PERF_CONTEXT_GLOBAL Gamma +ROCKSDB_CF_OPTIONS Gamma +ROCKSDB_COMPACTION_STATS Gamma +ROCKSDB_GLOBAL_INFO Gamma +ROCKSDB_DDL Gamma +ROCKSDB_INDEX_FILE_MAP Gamma +ROCKSDB_LOCKS Gamma +ROCKSDB_TRX Gamma From 18455ec3f1a9c22977f0ed87233852813b53eb49 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 14 Feb 2018 12:35:12 +0100 Subject: [PATCH 30/37] fix compilation wih -DPLUGIN_PARTITION=NO --- sql/partition_info.cc | 33 ++++++++++++++++----------------- storage/connect/ha_connect.cc | 16 +++++----------- 2 files changed, 21 insertions(+), 28 deletions(-) diff --git a/sql/partition_info.cc b/sql/partition_info.cc index 14f301e2d85..df0a733f121 100644 --- a/sql/partition_info.cc +++ b/sql/partition_info.cc @@ -2744,23 +2744,6 @@ end: } -bool partition_info::error_if_requires_values() const -{ - switch (part_type) { - case NOT_A_PARTITION: - case HASH_PARTITION: - break; - case RANGE_PARTITION: - my_error(ER_PARTITION_REQUIRES_VALUES_ERROR, MYF(0), "RANGE", "LESS THAN"); - return true; - case LIST_PARTITION: - my_error(ER_PARTITION_REQUIRES_VALUES_ERROR, MYF(0), "LIST", "IN"); - return true; - } - return false; -} - - /** Fix partition data from parser. @@ -3211,3 +3194,19 @@ bool check_partition_dirs(partition_info *part_info) } #endif /* WITH_PARTITION_STORAGE_ENGINE */ + +bool partition_info::error_if_requires_values() const +{ + switch (part_type) { + case NOT_A_PARTITION: + case HASH_PARTITION: + break; + case RANGE_PARTITION: + my_error(ER_PARTITION_REQUIRES_VALUES_ERROR, MYF(0), "RANGE", "LESS THAN"); + return true; + case LIST_PARTITION: + my_error(ER_PARTITION_REQUIRES_VALUES_ERROR, MYF(0), "LIST", "IN"); + return true; + } + return false; +} diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index 5964b1d049f..08a61a75048 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -1754,11 +1754,13 @@ bool ha_connect::CheckVirtualIndex(TABLE_SHARE *s) bool ha_connect::IsPartitioned(void) { +#ifdef WITH_PARTITION_STORAGE_ENGINE if (tshp) return tshp->partition_info_str_len > 0; else if (table && table->part_info) return true; else +#endif return false; } // end of IsPartitioned @@ -6165,8 +6167,10 @@ int ha_connect::create(const char *name, TABLE *table_arg, TABLE *st= table; // Probably unuseful THD *thd= ha_thd(); LEX_STRING cnc = table_arg->s->connect_string; -#if defined(WITH_PARTITION_STORAGE_ENGINE) +#ifdef WITH_PARTITION_STORAGE_ENGINE partition_info *part_info= table_arg->part_info; +#else +#define part_info 0 #endif // WITH_PARTITION_STORAGE_ENGINE xp= GetUser(thd, xp); PGLOBAL g= xp->g; @@ -6268,9 +6272,7 @@ int ha_connect::create(const char *name, TABLE *table_arg, // fall through case TAB_MYSQL: -#if defined(WITH_PARTITION_STORAGE_ENGINE) if (!part_info) -#endif // WITH_PARTITION_STORAGE_ENGINE {const char *src= options->srcdef; PCSZ host, db, tab= options->tabname; int port; @@ -6534,7 +6536,6 @@ int ha_connect::create(const char *name, TABLE *table_arg, } else lwt[i]= tolower(options->type[i]); -#if defined(WITH_PARTITION_STORAGE_ENGINE) if (part_info) { char *p; @@ -6544,7 +6545,6 @@ int ha_connect::create(const char *name, TABLE *table_arg, strcat(strcat(strcpy(buf, p), "."), lwt); *p= 0; } else { -#endif // WITH_PARTITION_STORAGE_ENGINE strcat(strcat(strcpy(buf, GetTableName()), "."), lwt); sprintf(g->Message, "No file name. Table will use %s", buf); @@ -6552,9 +6552,7 @@ int ha_connect::create(const char *name, TABLE *table_arg, push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 0, g->Message); strcat(strcat(strcpy(dbpath, "./"), table->s->db.str), "/"); -#if defined(WITH_PARTITION_STORAGE_ENGINE) } // endif part_info -#endif // WITH_PARTITION_STORAGE_ENGINE PlugSetPath(fn, buf, dbpath); @@ -6619,11 +6617,9 @@ int ha_connect::create(const char *name, TABLE *table_arg, push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 0, "Unexpected command in create, please contact CONNECT team"); -#if defined(WITH_PARTITION_STORAGE_ENGINE) if (part_info && !inward) strncpy(partname, decode(g, strrchr(name, '#') + 1), sizeof(partname) - 1); // strcpy(partname, part_info->curr_part_elem->partition_name); -#endif // WITH_PARTITION_STORAGE_ENGINE if (g->Alchecked == 0 && (!IsFileType(type) || FileExists(options->filename, false))) { @@ -6659,12 +6655,10 @@ int ha_connect::create(const char *name, TABLE *table_arg, my_message(ER_UNKNOWN_ERROR, g->Message, MYF(0)); rc = HA_ERR_INTERNAL_ERROR; } else if (cat) { -#if defined(WITH_PARTITION_STORAGE_ENGINE) if (part_info) strncpy(partname, decode(g, strrchr(name, (inward ? slash : '#')) + 1), sizeof(partname) - 1); -#endif // WITH_PARTITION_STORAGE_ENGINE if ((rc= optimize(table->in_use, NULL))) { htrc("Create rc=%d %s\n", rc, g->Message); From 50359719f0dd1f8f6d3d256a0db107654f8d39f1 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 14 Feb 2018 12:35:47 +0100 Subject: [PATCH 31/37] fix compilation -DWITH_PERFSCHEMA=NO tokudb apparently requires perfschema now --- storage/tokudb/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/storage/tokudb/CMakeLists.txt b/storage/tokudb/CMakeLists.txt index 7ea98728034..1cd86eb5808 100644 --- a/storage/tokudb/CMakeLists.txt +++ b/storage/tokudb/CMakeLists.txt @@ -4,6 +4,8 @@ IF(CMAKE_VERSION VERSION_LESS "2.8.9") MESSAGE(STATUS "CMake 2.8.9 or higher is required by TokuDB") ELSEIF(NOT HAVE_DLOPEN) MESSAGE(STATUS "dlopen is required by TokuDB") +ELSEIF(NOT TARGET perfschema) + MESSAGE(STATUS "Performance Schema is required by TokuDB") ELSEIF(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64") # tokudb requires F_NOCACHE or O_DIRECT, and designated initializers From 5d8ac1ece1687278cbc860e97eae9c7cc163ee73 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 14 Feb 2018 13:30:52 +0100 Subject: [PATCH 32/37] log all mtr output in vardir/log/stdout.log despite the name, it logs both stdout and stderr --- mysql-test/lib/My/SafeProcess/Base.pm | 12 ++++++++---- mysql-test/lib/My/Tee.pm | 23 +++++++++++++++++++++++ mysql-test/mysql-test-run.pl | 9 ++++++++- 3 files changed, 39 insertions(+), 5 deletions(-) create mode 100644 mysql-test/lib/My/Tee.pm diff --git a/mysql-test/lib/My/SafeProcess/Base.pm b/mysql-test/lib/My/SafeProcess/Base.pm index 1ac0120a735..c5ac2ab51c2 100644 --- a/mysql-test/lib/My/SafeProcess/Base.pm +++ b/mysql-test/lib/My/SafeProcess/Base.pm @@ -186,8 +186,10 @@ sub create_process { # it and any childs(that hasn't changed group themself) setpgrp(0,0) if $opts{setpgrp}; - if ( $output and !open(STDOUT, $open_mode, $output) ) { - croak("can't redirect STDOUT to '$output': $!"); + if ( $output ) { + close STDOUT; + open(STDOUT, $open_mode, $output) + or croak "can't redirect STDOUT to '$output': $!"; } if ( $error ) { @@ -196,8 +198,10 @@ sub create_process { croak("can't dup STDOUT: $!"); } } - elsif ( ! open(STDERR, $open_mode, $error) ) { - croak("can't redirect STDERR to '$error': $!"); + else { + close STDERR; + open(STDERR, $open_mode, $error) + or croak "can't redirect STDERR to '$error': $!"; } } diff --git a/mysql-test/lib/My/Tee.pm b/mysql-test/lib/My/Tee.pm new file mode 100644 index 00000000000..ee82e6f45ae --- /dev/null +++ b/mysql-test/lib/My/Tee.pm @@ -0,0 +1,23 @@ +package My::Tee; + +# see PerlIO::via + +our $copyfh; + +sub PUSHED +{ + open($copyfh, '>', "$::opt_vardir/log/stdout.log") + or die "open(>$::opt_vardir/log/stdout.log): $!" + unless $copyfh; + bless { }, shift; +} + +sub WRITE +{ + my ($obj, $buf, $fh) = @_; + print $fh $buf; + print $copyfh $buf; + return length($buf); +} + +1; diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index eaec51b82b4..2581eb7b503 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -91,6 +91,7 @@ use My::Platform; use My::SafeProcess; use My::ConfigFactory; use My::Options; +use My::Tee; use My::Find; use My::SysInfo; use My::CoreDump; @@ -392,6 +393,11 @@ sub main { initialize_servers(); init_timers(); + unless (IS_WINDOWS) { + binmode(STDOUT,":via(My::Tee)") or die "binmode(STDOUT, :via(My::Tee)):$!"; + binmode(STDERR,":via(My::Tee)") or die "binmode(STDERR, :via(My::Tee)):$!"; + } + mtr_report("Checking supported features..."); executable_setup(); @@ -6280,7 +6286,8 @@ sub xterm_stat { my $done = $num_tests - $left; my $spent = time - $^T; - printf "\e];mtr: spent %s on %d tests. %s (%d tests) left\a", + syswrite STDOUT, sprintf + "\e];mtr: spent %s on %d tests. %s (%d tests) left\a", time_format($spent), $done, time_format($spent/$done * $left), $left; } From 23d7b77358c5f36c9c4f6bb559c374725342984a Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 21 Feb 2018 18:22:43 +0000 Subject: [PATCH 33/37] Fix truncation warning on Windows. --- include/my_time.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/my_time.h b/include/my_time.h index b17bb5fe2b7..54aef17fc09 100644 --- a/include/my_time.h +++ b/include/my_time.h @@ -118,7 +118,7 @@ longlong number_to_datetime(longlong nr, ulong sec_part, MYSQL_TIME *time_res, ulonglong flags, int *was_cut); static inline -longlong double_to_datetime(double nr, MYSQL_TIME *ltime, uint flags, int *cut) +longlong double_to_datetime(double nr, MYSQL_TIME *ltime, ulonglong flags, int *cut) { if (nr < 0 || nr > LONGLONG_MAX) nr= (double)LONGLONG_MAX; From 868bca5c4fca7ca3fcddf9195800bc1ef5988885 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 20 Feb 2018 10:19:55 +1100 Subject: [PATCH 34/37] MDEV-15356: tp_timeout_handler needs to call set_killed_no_mutex as it has the mutex Regression introducted in c2118a08b144 where LOCK_thd_data was moveed to LOCK_thd_kill --- sql/threadpool_common.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/threadpool_common.cc b/sql/threadpool_common.cc index 98b2dcd1fcd..0a1f2220e2a 100644 --- a/sql/threadpool_common.cc +++ b/sql/threadpool_common.cc @@ -478,7 +478,7 @@ void tp_timeout_handler(TP_connection *c) return; THD *thd=c->thd; mysql_mutex_lock(&thd->LOCK_thd_kill); - thd->set_killed(KILL_WAIT_TIMEOUT); + thd->set_killed_no_mutex(KILL_WAIT_TIMEOUT); c->priority= TP_PRIORITY_HIGH; post_kill_notification(thd); mysql_mutex_unlock(&thd->LOCK_thd_kill); From e119799a920e8096886a51ded305bf06609438a4 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 22 Feb 2018 08:40:54 +0100 Subject: [PATCH 35/37] fix compilation wih -DPLUGIN_PARTITION=NO rocksdb and spider --- storage/rocksdb/ha_rocksdb.cc | 4 ++-- storage/rocksdb/rdb_datadic.cc | 2 +- storage/spider/CMakeLists.txt | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index bc15e948d41..5e4152f7dcf 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -6156,7 +6156,7 @@ int ha_rocksdb::create_cfs( // Internal consistency check to make sure that data in TABLE and // Rdb_tbl_def structures matches. Either both are missing or both are // specified. Yes, this is critical enough to make it into SHIP_ASSERT. - SHIP_ASSERT(!table_arg->part_info == tbl_def_arg->base_partition().empty()); + SHIP_ASSERT(IF_PARTITIONING(!table_arg->part_info,true) == tbl_def_arg->base_partition().empty()); // Generate the name for the column family to use. bool per_part_match_found = false; @@ -8395,7 +8395,7 @@ const std::string ha_rocksdb::generate_cf_name(const uint index, key_comment, table_arg, tbl_def_arg, per_part_match_found, RDB_CF_NAME_QUALIFIER); - if (table_arg->part_info != nullptr && !*per_part_match_found) { + if (IF_PARTITIONING(table_arg->part_info,nullptr) != nullptr && !*per_part_match_found) { // At this point we tried to search for a custom CF name for a partition, // but none was specified. Therefore default one will be used. return ""; diff --git a/storage/rocksdb/rdb_datadic.cc b/storage/rocksdb/rdb_datadic.cc index b2f5af705a3..68ad8e091b2 100644 --- a/storage/rocksdb/rdb_datadic.cc +++ b/storage/rocksdb/rdb_datadic.cc @@ -538,7 +538,7 @@ const std::string Rdb_key_def::parse_comment_for_qualifier( // NOTE: this means if you specify a qualifier for a specific partition it // will take precedence the 'table level' qualifier if one exists. std::string search_str_part; - if (table_arg->part_info != nullptr) { + if (IF_PARTITIONING(table_arg->part_info,nullptr) != nullptr) { std::string partition_name = tbl_def_arg->base_partition(); DBUG_ASSERT(!partition_name.empty()); search_str_part = gen_qualifier_for_table(qualifier, partition_name); diff --git a/storage/spider/CMakeLists.txt b/storage/spider/CMakeLists.txt index 14c50f35bd9..dec1cb4c6ba 100644 --- a/storage/spider/CMakeLists.txt +++ b/storage/spider/CMakeLists.txt @@ -34,6 +34,8 @@ IF(EXISTS ${PROJECT_SOURCE_DIR}/storage/mysql_storage_engine.cmake) ${CMAKE_SOURCE_DIR}/regex) MYSQL_STORAGE_ENGINE(SPIDER) +ELSEIF(PLUGIN_PARTITION MATCHES "^NO$") + MESSAGE(STATUS "Spider is skipped because partitioning is disabled") ELSE() INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/storage/spider/hs_client) From 504beb132523511e93e91cc422bcd413ca556828 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 22 Feb 2018 12:20:46 +0200 Subject: [PATCH 36/37] Add supressions for possible warning. --- mysql-test/suite/galera/r/galera_var_node_address.result | 1 + mysql-test/suite/galera/t/galera_var_node_address.test | 1 + 2 files changed, 2 insertions(+) diff --git a/mysql-test/suite/galera/r/galera_var_node_address.result b/mysql-test/suite/galera/r/galera_var_node_address.result index cf36e964f93..a9c900d8bc8 100644 --- a/mysql-test/suite/galera/r/galera_var_node_address.result +++ b/mysql-test/suite/galera/r/galera_var_node_address.result @@ -1,5 +1,6 @@ call mtr.add_suppression("WSREP: Stray state UUID msg: .* current group state WAIT_STATE_UUID .*"); call mtr.add_suppression("WSREP: Protocol violation. JOIN message sender .* is not in state transfer (.*). Message ignored."); +call mtr.add_suppression("WSREP: Sending JOIN failed: -[0-9]+ (Transport endpoint is not connected). Will retry in new primary component."); SELECT VARIABLE_VALUE = 4 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; VARIABLE_VALUE = 4 1 diff --git a/mysql-test/suite/galera/t/galera_var_node_address.test b/mysql-test/suite/galera/t/galera_var_node_address.test index 3353652d8b9..22e98e3aa82 100644 --- a/mysql-test/suite/galera/t/galera_var_node_address.test +++ b/mysql-test/suite/galera/t/galera_var_node_address.test @@ -8,6 +8,7 @@ call mtr.add_suppression("WSREP: Stray state UUID msg: .* current group state WAIT_STATE_UUID .*"); call mtr.add_suppression("WSREP: Protocol violation. JOIN message sender .* is not in state transfer (.*). Message ignored."); +call mtr.add_suppression("WSREP: Sending JOIN failed: -[0-9]+ (Transport endpoint is not connected). Will retry in new primary component."); SELECT VARIABLE_VALUE = 4 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; From e92cc097658405fdb52c80254b933424f8c1f99e Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 22 Feb 2018 15:58:07 +0100 Subject: [PATCH 37/37] MDEV-15345 Compilation fails to build my_addr_resolve.c fix the compilation error. no support for plugins yet. --- mysys/my_addr_resolve.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mysys/my_addr_resolve.c b/mysys/my_addr_resolve.c index 10f8552f226..84bff47d1a9 100644 --- a/mysys/my_addr_resolve.c +++ b/mysys/my_addr_resolve.c @@ -49,6 +49,13 @@ static const char *strip_path(const char *s) static bfd *bfdh= 0; static asymbol **symtable= 0; +#if defined(HAVE_LINK_H) && defined(HAVE_DLOPEN) +#include +static ElfW(Addr) offset= 0; +#else +#define offset 0 +#endif + /** finds a file name, a line number, and a function name corresponding to addr.