From 2965c480ec23564ec126d89a62c1c09292171831 Mon Sep 17 00:00:00 2001 From: "jonas@perch.ndb.mysql.com" <> Date: Fri, 26 Oct 2007 08:42:33 +0200 Subject: [PATCH 01/15] ndb - bug#31635 (5.0) 0 pad varsize keys in ndbapi --- mysql-test/r/ndb_basic.result | 24 +++++++++++++++++++ mysql-test/t/ndb_basic.test | 22 ++++++++++++++++-- ndb/src/ndbapi/NdbOperationDefine.cpp | 33 ++++++++++++++++++++------- ndb/src/ndbapi/NdbOperationSearch.cpp | 30 ++++++++++++++++++------ 4 files changed, 92 insertions(+), 17 deletions(-) diff --git a/mysql-test/r/ndb_basic.result b/mysql-test/r/ndb_basic.result index 346b1d5741b..08bc6c9a3aa 100644 --- a/mysql-test/r/ndb_basic.result +++ b/mysql-test/r/ndb_basic.result @@ -841,4 +841,28 @@ a b 3 30 4 1 drop table t1,t2; +create table t1 (a varchar(100) primary key, b varchar(100)) engine = NDB; +insert into t1 values +('a', 'a'),('b','b'),('c', 'c'),('aa', 'aa'),('bb', 'bb'),('cc', 'cc'); +replace into t1 values ('a', '-a'); +replace into t1 values ('b', '-b'); +replace into t1 values ('c', '-c'); +replace into t1 values ('aa', '-aa'); +replace into t1 values ('bb', '-bb'); +replace into t1 values ('cc', '-cc'); +replace into t1 values ('aaa', '-aaa'); +replace into t1 values ('bbb', '-bbb'); +replace into t1 values ('ccc', '-ccc'); +select * from t1 order by 1,2; +a b +a -a +aa -aa +aaa -aaa +b -b +bb -bb +bbb -bbb +c -c +cc -cc +ccc -ccc +drop table t1; End of 5.0 tests diff --git a/mysql-test/t/ndb_basic.test b/mysql-test/t/ndb_basic.test index 80c8942348c..e9401ff3894 100644 --- a/mysql-test/t/ndb_basic.test +++ b/mysql-test/t/ndb_basic.test @@ -780,7 +780,25 @@ update ignore t1,t2 set a = 1, c = 1 where a = 3 and c = 3; select * from t1 order by a; drop table t1,t2; +# +# Bug#31635 +# +create table t1 (a varchar(100) primary key, b varchar(100)) engine = NDB; +insert into t1 values + ('a', 'a'),('b','b'),('c', 'c'),('aa', 'aa'),('bb', 'bb'),('cc', 'cc'); +replace into t1 values ('a', '-a'); +replace into t1 values ('b', '-b'); +replace into t1 values ('c', '-c'); + +replace into t1 values ('aa', '-aa'); +replace into t1 values ('bb', '-bb'); +replace into t1 values ('cc', '-cc'); + +replace into t1 values ('aaa', '-aaa'); +replace into t1 values ('bbb', '-bbb'); +replace into t1 values ('ccc', '-ccc'); +select * from t1 order by 1,2; +drop table t1; + # End of 5.0 tests --echo End of 5.0 tests - - diff --git a/ndb/src/ndbapi/NdbOperationDefine.cpp b/ndb/src/ndbapi/NdbOperationDefine.cpp index 95e90609f9b..50f64d4dda4 100644 --- a/ndb/src/ndbapi/NdbOperationDefine.cpp +++ b/ndb/src/ndbapi/NdbOperationDefine.cpp @@ -572,15 +572,32 @@ NdbOperation::setValue( const NdbColumnImpl* tAttrInfo, * If it is not aligned then we start by copying the value to tempData and * use this as aValue instead. *************************************************************************/ - const int attributeSize = sizeInBytes; - const int slack = sizeInBytes & 3; + int attributeSize = sizeInBytes; + int slack = (sizeInBytes & 3) ? 4 - (sizeInBytes & 3) : 0; + switch(tAttrInfo->m_type){ + case NdbDictionary::Column::Varchar: + case NdbDictionary::Column::Varbinary: + attributeSize = 1 + *(Uint8*)aValue; + slack = 4 * totalSizeInWords - attributeSize; + break; + case NdbDictionary::Column::Longvarchar: + case NdbDictionary::Column::Longvarbinary: + { + const Uint8* ptr = (const Uint8*)aValue; + attributeSize = 2 + ptr[0] + 256 * ptr[1]; + slack = 4 * totalSizeInWords - attributeSize; + break; + } + } - if (((UintPtr)aValue & 3) != 0 || (slack != 0)){ - memcpy(&tempData[0], aValue, attributeSize); - aValue = (char*)&tempData[0]; - if(slack != 0) { - char * tmp = (char*)&tempData[0]; - memset(&tmp[attributeSize], 0, (4 - slack)); + if (((UintPtr)aValue & 3) != 0 || (slack != 0)) + { + char * tmp = (char*)tempData; + memcpy(tmp, aValue, attributeSize); + aValue = tmp; + if(slack != 0) + { + bzero(tmp + attributeSize, slack); }//if }//if diff --git a/ndb/src/ndbapi/NdbOperationSearch.cpp b/ndb/src/ndbapi/NdbOperationSearch.cpp index a3e3f7a7a91..5639c5dfc09 100644 --- a/ndb/src/ndbapi/NdbOperationSearch.cpp +++ b/ndb/src/ndbapi/NdbOperationSearch.cpp @@ -129,6 +129,7 @@ NdbOperation::equal_impl(const NdbColumnImpl* tAttrInfo, OperationType tOpType = theOperationType; Uint32 sizeInBytes = tAttrInfo->m_attrSize * tAttrInfo->m_arraySize; + const Uint32 totalSizeInWords = (sizeInBytes + 3) / 4; Uint32 real_len; if (! tAttrInfo->get_var_length(aValue, real_len)) { @@ -150,20 +151,35 @@ NdbOperation::equal_impl(const NdbColumnImpl* tAttrInfo, * aValue. If it is not aligned then we start by copying the value to * tempData and use this as aValue instead. ***********************************************************************/ - const int attributeSize = sizeInBytes; - const int slack = sizeInBytes & 3; + int attributeSize = sizeInBytes; + int slack = (sizeInBytes & 3) ? 4 - (sizeInBytes & 3) : 0; const int align = UintPtr(aValue) & 7; + switch(tAttrInfo->m_type){ + case NdbDictionary::Column::Varchar: + case NdbDictionary::Column::Varbinary: + attributeSize = 1 + *(Uint8*)aValue; + slack = 4 * totalSizeInWords - attributeSize; + break; + case NdbDictionary::Column::Longvarchar: + case NdbDictionary::Column::Longvarbinary: + { + const Uint8* ptr = (const Uint8*)aValue; + attributeSize = 2 + ptr[0] + 256 * ptr[1]; + slack = 4*totalSizeInWords - attributeSize; + break; + } + } + if (((align & 3) != 0) || (slack != 0) || (tDistrKey && (align != 0))) { - ((Uint32*)tempData)[attributeSize >> 2] = 0; - memcpy(&tempData[0], aValue, attributeSize); - aValue = (char*)&tempData[0]; + char * tmp = (char*)tempData; + memcpy(tmp, aValue, attributeSize); + aValue = tmp; + bzero(tmp + attributeSize, slack); }//if } - Uint32 totalSizeInWords = (sizeInBytes + 3)/4; // Inc. bits in last word - if (true){ //tArraySize != 0) { Uint32 tTupKeyLen = theTupKeyLen; From 85a9b350a66bde43937593a586143f067cb0af75 Mon Sep 17 00:00:00 2001 From: "jonas@perch.ndb.mysql.com" <> Date: Fri, 26 Oct 2007 09:06:18 +0200 Subject: [PATCH 02/15] post merge weirdness --- mysql-test/suite/ndb/r/ndb_basic.result | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/mysql-test/suite/ndb/r/ndb_basic.result b/mysql-test/suite/ndb/r/ndb_basic.result index 38273230f19..9f4f8c0755c 100644 --- a/mysql-test/suite/ndb/r/ndb_basic.result +++ b/mysql-test/suite/ndb/r/ndb_basic.result @@ -882,16 +882,16 @@ replace into t1 values ('aaa', '-aaa'); replace into t1 values ('bbb', '-bbb'); replace into t1 values ('ccc', '-ccc'); select * from t1 order by 1,2; -a b -a -a -aa -aa -aaa -aaa -b -b -bb -bb -bbb -bbb -c -c -cc -cc -ccc -ccc +a b +a -a +aa -aa +aaa -aaa +b -b +bb -bb +bbb -bbb +c -c +cc -cc +ccc -ccc drop table t1; End of 5.0 tests CREATE TABLE t1 (a VARCHAR(255) NOT NULL, From 9320ec8d16f3335b422daf73b78452dda6fb2f2d Mon Sep 17 00:00:00 2001 From: "tomas@whalegate.ndb.mysql.com" <> Date: Thu, 1 Nov 2007 15:08:00 +0100 Subject: [PATCH 03/15] Bug #31484 Cluster LOST_EVENTS entry not added to binlog on mysqld restart. --- mysql-test/include/have_multi_ndb.inc | 12 ++----- mysql-test/suite/ndb/r/ndb_multi_row.result | 1 + mysql-test/suite/ndb/t/ndb_multi_row.test | 3 +- sql/ha_ndbcluster.cc | 2 +- sql/ha_ndbcluster_binlog.cc | 35 ++++++++++++++++++--- 5 files changed, 37 insertions(+), 16 deletions(-) diff --git a/mysql-test/include/have_multi_ndb.inc b/mysql-test/include/have_multi_ndb.inc index deda22b64c0..dd6e609f130 100644 --- a/mysql-test/include/have_multi_ndb.inc +++ b/mysql-test/include/have_multi_ndb.inc @@ -5,10 +5,6 @@ connect (server2,127.0.0.1,root,,test,$MASTER_MYPORT1,); # Check that server1 has NDB support connection server1; disable_query_log; ---disable_warnings -drop table if exists t1, t2; ---enable_warnings -flush tables; --require r/true.require select (support = 'YES' or support = 'DEFAULT') as `TRUE` from information_schema.engines where engine = 'ndbcluster'; --source include/ndb_not_readonly.inc @@ -17,14 +13,10 @@ enable_query_log; # Check that server2 has NDB support connection server2; disable_query_log; ---disable_warnings -drop table if exists t1, t2; ---enable_warnings -flush tables; --require r/true.require select (support = 'YES' or support = 'DEFAULT') as `TRUE` from information_schema.engines where engine = 'ndbcluster'; --source include/ndb_not_readonly.inc enable_query_log; -# Set the default connection to 'server1' -connection server1; +# Set the default connection +connection default; diff --git a/mysql-test/suite/ndb/r/ndb_multi_row.result b/mysql-test/suite/ndb/r/ndb_multi_row.result index cf5a76d6f01..3d34b16a1a8 100644 --- a/mysql-test/suite/ndb/r/ndb_multi_row.result +++ b/mysql-test/suite/ndb/r/ndb_multi_row.result @@ -1,4 +1,5 @@ drop table if exists t1, t2, t3, t4; +flush status; drop table if exists t1, t2, t3, t4; flush status; create table t1 (a int) engine=ndbcluster; diff --git a/mysql-test/suite/ndb/t/ndb_multi_row.test b/mysql-test/suite/ndb/t/ndb_multi_row.test index c82307839f4..26953093ed0 100644 --- a/mysql-test/suite/ndb/t/ndb_multi_row.test +++ b/mysql-test/suite/ndb/t/ndb_multi_row.test @@ -6,11 +6,12 @@ --disable_warnings connection server2; drop table if exists t1, t2, t3, t4; +flush status; connection server1; drop table if exists t1, t2, t3, t4; +flush status; --enable_warnings -flush status; # Create test tables on server1 create table t1 (a int) engine=ndbcluster; diff --git a/sql/ha_ndbcluster.cc b/sql/ha_ndbcluster.cc index a799dd4841b..2294d836854 100644 --- a/sql/ha_ndbcluster.cc +++ b/sql/ha_ndbcluster.cc @@ -618,7 +618,7 @@ bool ha_ndbcluster::get_error_message(int error, DBUG_ENTER("ha_ndbcluster::get_error_message"); DBUG_PRINT("enter", ("error: %d", error)); - Ndb *ndb= get_ndb(); + Ndb *ndb= check_ndb_in_thd(current_thd); if (!ndb) DBUG_RETURN(FALSE); diff --git a/sql/ha_ndbcluster_binlog.cc b/sql/ha_ndbcluster_binlog.cc index 5d5c8a26447..75e9937ecd6 100644 --- a/sql/ha_ndbcluster_binlog.cc +++ b/sql/ha_ndbcluster_binlog.cc @@ -3587,6 +3587,7 @@ pthread_handler_t ndb_binlog_thread_func(void *arg) Thd_ndb *thd_ndb=0; int ndb_update_ndb_binlog_index= 1; injector *inj= injector::instance(); + uint incident_id= 0; #ifdef RUN_NDB_BINLOG_TIMER Timer main_timer; @@ -3692,17 +3693,43 @@ pthread_handler_t ndb_binlog_thread_func(void *arg) pthread_mutex_unlock(&injector_mutex); pthread_cond_signal(&injector_cond); + /* + wait for mysql server to start (so that the binlog is started + and thus can receive the first GAP event) + */ + pthread_mutex_lock(&LOCK_server_started); + while (!mysqld_server_started) + { + struct timespec abstime; + set_timespec(abstime, 1); + pthread_cond_timedwait(&COND_server_started, &LOCK_server_started, + &abstime); + if (ndbcluster_terminating) + { + pthread_mutex_unlock(&LOCK_server_started); + pthread_mutex_lock(&LOCK_ndb_util_thread); + goto err; + } + } + pthread_mutex_unlock(&LOCK_server_started); restart: /* Main NDB Injector loop */ { /* - Always insert a GAP event as we cannot know what has happened in the cluster - while not being connected. + Always insert a GAP event as we cannot know what has happened + in the cluster while not being connected. */ - LEX_STRING const msg= { C_STRING_WITH_LEN("Cluster connect") }; - inj->record_incident(thd, INCIDENT_LOST_EVENTS, msg); + LEX_STRING const msg[2]= + { + { C_STRING_WITH_LEN("mysqld startup") }, + { C_STRING_WITH_LEN("cluster disconnect")} + }; + IF_DBUG(int error=) + inj->record_incident(thd, INCIDENT_LOST_EVENTS, msg[incident_id]); + DBUG_ASSERT(!error); + incident_id= 1; } { thd->proc_info= "Waiting for ndbcluster to start"; From 9952ee4f913ad476243499a08f4e2e4059135091 Mon Sep 17 00:00:00 2001 From: "tomas@whalegate.ndb.mysql.com" <> Date: Thu, 1 Nov 2007 20:00:44 +0100 Subject: [PATCH 04/15] Bug #31484 Cluster LOST_EVENTS entry not added to binlog on mysqld restart - correction --- mysql-test/suite/ndb/r/ndb_multi.result | 5 +++-- mysql-test/suite/ndb/t/ndb_multi.test | 6 ++++-- sql/ha_ndbcluster_binlog.cc | 1 + 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/mysql-test/suite/ndb/r/ndb_multi.result b/mysql-test/suite/ndb/r/ndb_multi.result index 98c4265b833..40483887919 100644 --- a/mysql-test/suite/ndb/r/ndb_multi.result +++ b/mysql-test/suite/ndb/r/ndb_multi.result @@ -1,4 +1,5 @@ drop table if exists t1, t2, t3, t4; +flush status; drop table if exists t1, t2, t3, t4; flush status; create table t1 (a int) engine=ndbcluster; @@ -132,11 +133,11 @@ master_epoch, count)) engine ndb; show tables like '%$%'; Tables_in_test (%$%) -t1$EX +t1$ex use test; show tables like '%$%'; Tables_in_test (%$%) -t1$EX +t1$ex drop table `test`.`t1$EX`; show tables like '%$%'; Tables_in_test (%$%) diff --git a/mysql-test/suite/ndb/t/ndb_multi.test b/mysql-test/suite/ndb/t/ndb_multi.test index ce7e22b3b7f..e033ad1e479 100644 --- a/mysql-test/suite/ndb/t/ndb_multi.test +++ b/mysql-test/suite/ndb/t/ndb_multi.test @@ -4,11 +4,11 @@ --disable_warnings connection server2; drop table if exists t1, t2, t3, t4; +flush status; connection server1; drop table if exists t1, t2, t3, t4; ---enable_warnings - flush status; +--enable_warnings # Create test tables on server1 create table t1 (a int) engine=ndbcluster; @@ -139,9 +139,11 @@ create table `test`.`t1$EX` # check that table shows up ok on both servers # before bugfix table would not show up on server2 +--replace_regex /EX/ex/ show tables like '%$%'; connection server2; use test; +--replace_regex /EX/ex/ show tables like '%$%'; # check cleanup diff --git a/sql/ha_ndbcluster_binlog.cc b/sql/ha_ndbcluster_binlog.cc index 75e9937ecd6..48973f3e27e 100644 --- a/sql/ha_ndbcluster_binlog.cc +++ b/sql/ha_ndbcluster_binlog.cc @@ -3716,6 +3716,7 @@ restart: /* Main NDB Injector loop */ + if (ndb_binlog_running) { /* Always insert a GAP event as we cannot know what has happened From 205910571e45784f114c0ed7c0a76a1b953eec74 Mon Sep 17 00:00:00 2001 From: "tomas@whalegate.ndb.mysql.com" <> Date: Fri, 2 Nov 2007 07:14:56 +0100 Subject: [PATCH 05/15] make sure everything is clean before starting the test --- mysql-test/include/have_multi_ndb.inc | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/mysql-test/include/have_multi_ndb.inc b/mysql-test/include/have_multi_ndb.inc index dd6e609f130..c1965203492 100644 --- a/mysql-test/include/have_multi_ndb.inc +++ b/mysql-test/include/have_multi_ndb.inc @@ -18,5 +18,25 @@ select (support = 'YES' or support = 'DEFAULT') as `TRUE` from information_schem --source include/ndb_not_readonly.inc enable_query_log; +# cleanup + +connection server1; +disable_query_log; +disable_warnings; +drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9,t10; +flush tables; +flush status; +enable_warnings; +enable_query_log; + +connection server2; +disable_query_log; +disable_warnings; +drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9,t10; +flush tables; +flush status; +enable_warnings; +enable_query_log; + # Set the default connection connection default; From 18c5cc407482a467a37b020e18193a5337d0c2c7 Mon Sep 17 00:00:00 2001 From: "tomas@whalegate.ndb.mysql.com" <> Date: Fri, 2 Nov 2007 17:34:06 +0100 Subject: [PATCH 06/15] workaround for some bug to investigate --- mysql-test/include/have_multi_ndb.inc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mysql-test/include/have_multi_ndb.inc b/mysql-test/include/have_multi_ndb.inc index c1965203492..e8f66afe5bf 100644 --- a/mysql-test/include/have_multi_ndb.inc +++ b/mysql-test/include/have_multi_ndb.inc @@ -23,6 +23,7 @@ enable_query_log; connection server1; disable_query_log; disable_warnings; +--error 0,1051 drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9,t10; flush tables; flush status; @@ -32,6 +33,7 @@ enable_query_log; connection server2; disable_query_log; disable_warnings; +--error 0,1051 drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9,t10; flush tables; flush status; From ed1408270390d65558b801082bc2e4a1f8fce5a7 Mon Sep 17 00:00:00 2001 From: "tomas@whalegate.ndb.mysql.com" <> Date: Fri, 2 Nov 2007 23:28:29 +0100 Subject: [PATCH 07/15] reverting to old setting --- mysql-test/include/have_multi_ndb.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql-test/include/have_multi_ndb.inc b/mysql-test/include/have_multi_ndb.inc index e8f66afe5bf..9779f181191 100644 --- a/mysql-test/include/have_multi_ndb.inc +++ b/mysql-test/include/have_multi_ndb.inc @@ -41,4 +41,4 @@ enable_warnings; enable_query_log; # Set the default connection -connection default; +connection server1; From 96dfac7cb13f3aec89932545160c4866360f57d2 Mon Sep 17 00:00:00 2001 From: "tomas@whalegate.ndb.mysql.com" <> Date: Fri, 2 Nov 2007 23:44:17 +0100 Subject: [PATCH 08/15] this error should not be printed --- sql/ha_ndbcluster_binlog.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sql/ha_ndbcluster_binlog.cc b/sql/ha_ndbcluster_binlog.cc index 48973f3e27e..5fc8781edfe 100644 --- a/sql/ha_ndbcluster_binlog.cc +++ b/sql/ha_ndbcluster_binlog.cc @@ -779,6 +779,7 @@ static int ndbcluster_create_ndb_apply_status_table(THD *thd) const int no_print_error[4]= {ER_TABLE_EXISTS_ERROR, 701, + 702, 4009, 0}; // do not print error 701 etc run_query(thd, buf, end, no_print_error, TRUE); @@ -839,6 +840,7 @@ static int ndbcluster_create_schema_table(THD *thd) const int no_print_error[4]= {ER_TABLE_EXISTS_ERROR, 701, + 702, 4009, 0}; // do not print error 701 etc run_query(thd, buf, end, no_print_error, TRUE); From 2635cb0a93e92dd4149836679d1371cb3fe361e7 Mon Sep 17 00:00:00 2001 From: "tomas@whalegate.ndb.mysql.com" <> Date: Sat, 3 Nov 2007 13:46:49 +0100 Subject: [PATCH 09/15] compile error --- sql/ha_ndbcluster_binlog.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/ha_ndbcluster_binlog.cc b/sql/ha_ndbcluster_binlog.cc index 5fc8781edfe..9465c5dbb3c 100644 --- a/sql/ha_ndbcluster_binlog.cc +++ b/sql/ha_ndbcluster_binlog.cc @@ -777,7 +777,7 @@ static int ndbcluster_create_ndb_apply_status_table(THD *thd) " end_pos BIGINT UNSIGNED NOT NULL, " " PRIMARY KEY USING HASH (server_id) ) ENGINE=NDB"); - const int no_print_error[4]= {ER_TABLE_EXISTS_ERROR, + const int no_print_error[5]= {ER_TABLE_EXISTS_ERROR, 701, 702, 4009, @@ -838,7 +838,7 @@ static int ndbcluster_create_schema_table(THD *thd) " type INT UNSIGNED NOT NULL," " PRIMARY KEY USING HASH (db,name) ) ENGINE=NDB"); - const int no_print_error[4]= {ER_TABLE_EXISTS_ERROR, + const int no_print_error[5]= {ER_TABLE_EXISTS_ERROR, 701, 702, 4009, From bfc80edec513d50d87e0b58972196315a36eb58d Mon Sep 17 00:00:00 2001 From: "tomas@whalegate.ndb.mysql.com" <> Date: Mon, 5 Nov 2007 23:23:26 +0100 Subject: [PATCH 10/15] preserve more of the state --- sql/ha_ndbcluster_binlog.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sql/ha_ndbcluster_binlog.cc b/sql/ha_ndbcluster_binlog.cc index 9465c5dbb3c..93f7027d788 100644 --- a/sql/ha_ndbcluster_binlog.cc +++ b/sql/ha_ndbcluster_binlog.cc @@ -243,7 +243,8 @@ static void run_query(THD *thd, char *buf, char *end, { ulong save_query_length= thd->query_length; char *save_query= thd->query; - ulong save_thread_id= thd->variables.pseudo_thread_id; + struct system_variables save_variables= thd->variables; + struct system_status_var save_status_var= thd->status_var; ulonglong save_thd_options= thd->options; DBUG_ASSERT(sizeof(save_thd_options) == sizeof(thd->options)); NET save_net= thd->net; @@ -277,7 +278,8 @@ static void run_query(THD *thd, char *buf, char *end, thd->options= save_thd_options; thd->query_length= save_query_length; thd->query= save_query; - thd->variables.pseudo_thread_id= save_thread_id; + thd->variables= save_variables; + thd->status_var= save_status_var; thd->net= save_net; if (thd == injector_thd) From 47a245559c55f649b1ba15fe6f9ecc2a668557c0 Mon Sep 17 00:00:00 2001 From: "tomas@whalegate.ndb.mysql.com" <> Date: Tue, 6 Nov 2007 10:27:56 +0100 Subject: [PATCH 11/15] break out tuple data read --- sql/ha_ndbcluster_binlog.cc | 25 +++-- storage/ndb/tools/restore/Restore.cpp | 156 ++++++++++++++------------ storage/ndb/tools/restore/Restore.hpp | 4 + 3 files changed, 103 insertions(+), 82 deletions(-) diff --git a/sql/ha_ndbcluster_binlog.cc b/sql/ha_ndbcluster_binlog.cc index 93f7027d788..4bb17dffa4a 100644 --- a/sql/ha_ndbcluster_binlog.cc +++ b/sql/ha_ndbcluster_binlog.cc @@ -241,19 +241,22 @@ static void dbug_print_table(const char *info, TABLE *table) static void run_query(THD *thd, char *buf, char *end, const int *no_print_error, my_bool disable_binlog) { - ulong save_query_length= thd->query_length; - char *save_query= thd->query; - struct system_variables save_variables= thd->variables; - struct system_status_var save_status_var= thd->status_var; + ulong save_thd_query_length= thd->query_length; + char *save_thd_query= thd->query; + struct system_variables save_thd_variables= thd->variables; + struct system_status_var save_thd_status_var= thd->status_var; + THD_TRANS save_thd_transaction_all= thd->transaction.all; + THD_TRANS save_thd_transaction_stmt= thd->transaction.stmt; ulonglong save_thd_options= thd->options; DBUG_ASSERT(sizeof(save_thd_options) == sizeof(thd->options)); - NET save_net= thd->net; + NET save_thd_net= thd->net; const char* found_semicolon= NULL; bzero((char*) &thd->net, sizeof(NET)); thd->query_length= end - buf; thd->query= buf; thd->variables.pseudo_thread_id= thread_id; + thd->transaction.stmt.modified_non_trans_table= FALSE; if (disable_binlog) thd->options&= ~OPTION_BIN_LOG; @@ -276,11 +279,13 @@ static void run_query(THD *thd, char *buf, char *end, } thd->options= save_thd_options; - thd->query_length= save_query_length; - thd->query= save_query; - thd->variables= save_variables; - thd->status_var= save_status_var; - thd->net= save_net; + thd->query_length= save_thd_query_length; + thd->query= save_thd_query; + thd->variables= save_thd_variables; + thd->status_var= save_thd_status_var; + thd->transaction.all= save_thd_transaction_all; + thd->transaction.stmt= save_thd_transaction_stmt; + thd->net= save_thd_net; if (thd == injector_thd) { diff --git a/storage/ndb/tools/restore/Restore.cpp b/storage/ndb/tools/restore/Restore.cpp index a7d8a9d10d9..f599bb21978 100644 --- a/storage/ndb/tools/restore/Restore.cpp +++ b/storage/ndb/tools/restore/Restore.cpp @@ -534,6 +534,88 @@ TupleS::prepareRecord(TableS & tab){ return true; } +int +RestoreDataIterator::readTupleData(Uint32 *buf_ptr, Uint32 *ptr, + Uint32 dataLength) +{ + while (ptr + 2 < buf_ptr + dataLength) + { + typedef BackupFormat::DataFile::VariableData VarData; + VarData * data = (VarData *)ptr; + Uint32 sz = ntohl(data->Sz); + Uint32 attrId = ntohl(data->Id); // column_no + + AttributeData * attr_data = m_tuple.getData(attrId); + const AttributeDesc * attr_desc = m_tuple.getDesc(attrId); + + // just a reminder - remove when backwards compat implemented + if (m_currentTable->backupVersion < MAKE_VERSION(5,1,3) && + attr_desc->m_column->getNullable()) + { + const Uint32 ind = attr_desc->m_nullBitIndex; + if(BitmaskImpl::get(m_currentTable->m_nullBitmaskSize, + buf_ptr,ind)) + { + attr_data->null = true; + attr_data->void_value = NULL; + continue; + } + } + + if (m_currentTable->backupVersion < MAKE_VERSION(5,1,3)) + { + sz *= 4; + } + + attr_data->null = false; + attr_data->void_value = &data->Data[0]; + attr_data->size = sz; + + //if (m_currentTable->getTableId() >= 2) { ndbout << "var off=" << ptr-buf_ptr << " attrId=" << attrId << endl; } + + /** + * Compute array size + */ + const Uint32 arraySize = sz / (attr_desc->size / 8); + assert(arraySize <= attr_desc->arraySize); + + //convert the length of blob(v1) and text(v1) + if(!m_hostByteOrder + && (attr_desc->m_column->getType() == NdbDictionary::Column::Blob + || attr_desc->m_column->getType() == NdbDictionary::Column::Text) + && attr_desc->m_column->getArrayType() == NdbDictionary::Column::ArrayTypeFixed) + { + char* p = (char*)&attr_data->u_int64_value[0]; + Uint64 x; + memcpy(&x, p, sizeof(Uint64)); + x = Twiddle64(x); + memcpy(p, &x, sizeof(Uint64)); + } + + //convert datetime type + if(!m_hostByteOrder + && attr_desc->m_column->getType() == NdbDictionary::Column::Datetime) + { + char* p = (char*)&attr_data->u_int64_value[0]; + Uint64 x; + memcpy(&x, p, sizeof(Uint64)); + x = Twiddle64(x); + memcpy(p, &x, sizeof(Uint64)); + } + + if(!Twiddle(attr_desc, attr_data, attr_desc->arraySize)) + { + return -1; + } + + ptr += ((sz + 3) >> 2) + 2; + } + + assert(ptr == buf_ptr + dataLength); + + return 0; +} + const TupleS * RestoreDataIterator::getNextTuple(int & res) { @@ -630,78 +712,8 @@ RestoreDataIterator::getNextTuple(int & res) attr_data->void_value = NULL; } - while (ptr + 2 < buf_ptr + dataLength) { - typedef BackupFormat::DataFile::VariableData VarData; - VarData * data = (VarData *)ptr; - Uint32 sz = ntohl(data->Sz); - Uint32 attrId = ntohl(data->Id); // column_no - - AttributeData * attr_data = m_tuple.getData(attrId); - const AttributeDesc * attr_desc = m_tuple.getDesc(attrId); - - // just a reminder - remove when backwards compat implemented - if(m_currentTable->backupVersion < MAKE_VERSION(5,1,3) && - attr_desc->m_column->getNullable()){ - const Uint32 ind = attr_desc->m_nullBitIndex; - if(BitmaskImpl::get(m_currentTable->m_nullBitmaskSize, - buf_ptr,ind)){ - attr_data->null = true; - attr_data->void_value = NULL; - continue; - } - } - - if (m_currentTable->backupVersion < MAKE_VERSION(5,1,3)) - { - sz *= 4; - } - - attr_data->null = false; - attr_data->void_value = &data->Data[0]; - attr_data->size = sz; - - //if (m_currentTable->getTableId() >= 2) { ndbout << "var off=" << ptr-buf_ptr << " attrId=" << attrId << endl; } - - /** - * Compute array size - */ - const Uint32 arraySize = sz / (attr_desc->size / 8); - assert(arraySize <= attr_desc->arraySize); - - //convert the length of blob(v1) and text(v1) - if(!m_hostByteOrder - && (attr_desc->m_column->getType() == NdbDictionary::Column::Blob - || attr_desc->m_column->getType() == NdbDictionary::Column::Text) - && attr_desc->m_column->getArrayType() == NdbDictionary::Column::ArrayTypeFixed) - { - char* p = (char*)&attr_data->u_int64_value[0]; - Uint64 x; - memcpy(&x, p, sizeof(Uint64)); - x = Twiddle64(x); - memcpy(p, &x, sizeof(Uint64)); - } - - //convert datetime type - if(!m_hostByteOrder - && attr_desc->m_column->getType() == NdbDictionary::Column::Datetime) - { - char* p = (char*)&attr_data->u_int64_value[0]; - Uint64 x; - memcpy(&x, p, sizeof(Uint64)); - x = Twiddle64(x); - memcpy(p, &x, sizeof(Uint64)); - } - - if(!Twiddle(attr_desc, attr_data, attr_desc->arraySize)) - { - res = -1; - return NULL; - } - - ptr += ((sz + 3) >> 2) + 2; - } - - assert(ptr == buf_ptr + dataLength); + if ((res = readTupleData(buf_ptr, ptr, dataLength))) + return NULL; m_count ++; res = 0; diff --git a/storage/ndb/tools/restore/Restore.hpp b/storage/ndb/tools/restore/Restore.hpp index 5455fa17aa0..f6de9245509 100644 --- a/storage/ndb/tools/restore/Restore.hpp +++ b/storage/ndb/tools/restore/Restore.hpp @@ -355,6 +355,10 @@ public: bool validateFragmentFooter(); const TupleS *getNextTuple(int & res); + +private: + + int readTupleData(Uint32 *buf_ptr, Uint32 *ptr, Uint32 dataLength); }; class LogEntry { From 312aec9e27688bd6b797c87f612d164139b756f2 Mon Sep 17 00:00:00 2001 From: "tomas@whalegate.ndb.mysql.com" <> Date: Tue, 6 Nov 2007 15:12:27 +0100 Subject: [PATCH 12/15] Bug #31484 Cluster LOST_EVENTS entry not added to binlog on mysqld restart - correction, do not insert GAP on first startup --- sql/ha_ndbcluster_binlog.cc | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/sql/ha_ndbcluster_binlog.cc b/sql/ha_ndbcluster_binlog.cc index 4bb17dffa4a..baa8b1ada66 100644 --- a/sql/ha_ndbcluster_binlog.cc +++ b/sql/ha_ndbcluster_binlog.cc @@ -3725,8 +3725,25 @@ restart: /* Main NDB Injector loop */ - if (ndb_binlog_running) + while (ndb_binlog_running) { + /* + check if it is the first log, if so we do not insert a GAP event + as there is really no log to have a GAP in + */ + { + LOG_INFO log_info; + mysql_bin_log.get_current_log(&log_info); + int len= strlen(log_info.log_file_name); + uint no= 0; + if ((sscanf(log_info.log_file_name + len - 6, "%u", &no) == 1) && + no == 1) + { + /* this is the fist log, so skip GAP event */ + break; + } + } + /* Always insert a GAP event as we cannot know what has happened in the cluster while not being connected. @@ -3739,8 +3756,9 @@ restart: IF_DBUG(int error=) inj->record_incident(thd, INCIDENT_LOST_EVENTS, msg[incident_id]); DBUG_ASSERT(!error); - incident_id= 1; + break; } + incident_id= 1; { thd->proc_info= "Waiting for ndbcluster to start"; From 0feaff83b7daddca8384a38e1fe5b24fbfccb114 Mon Sep 17 00:00:00 2001 From: "tomas@whalegate.ndb.mysql.com" <> Date: Tue, 6 Nov 2007 22:28:44 +0100 Subject: [PATCH 13/15] only potentially skip GAP event on startup, not on cluster restart --- sql/ha_ndbcluster_binlog.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/ha_ndbcluster_binlog.cc b/sql/ha_ndbcluster_binlog.cc index baa8b1ada66..d9380b50f1e 100644 --- a/sql/ha_ndbcluster_binlog.cc +++ b/sql/ha_ndbcluster_binlog.cc @@ -3731,6 +3731,7 @@ restart: check if it is the first log, if so we do not insert a GAP event as there is really no log to have a GAP in */ + if (incident_id == 0) { LOG_INFO log_info; mysql_bin_log.get_current_log(&log_info); From f8c413a5102f07b3b8ffa7d912fbbe148bc96a18 Mon Sep 17 00:00:00 2001 From: "jonas@perch.ndb.mysql.com" <> Date: Wed, 7 Nov 2007 20:57:21 +0100 Subject: [PATCH 14/15] ndb - bug#32160 (recommit to 5.0) fix lcp master take over bug --- ndb/src/kernel/blocks/ERROR_codes.txt | 7 ++- ndb/src/kernel/blocks/dbdih/Dbdih.hpp | 10 ++++ ndb/src/kernel/blocks/dbdih/DbdihMain.cpp | 60 ++++++++++++++++++++--- ndb/test/ndbapi/testNodeRestart.cpp | 48 ++++++++++++++++++ ndb/test/run-test/daily-basic-tests.txt | 4 ++ 5 files changed, 122 insertions(+), 7 deletions(-) diff --git a/ndb/src/kernel/blocks/ERROR_codes.txt b/ndb/src/kernel/blocks/ERROR_codes.txt index e45c608b601..2599bf40988 100644 --- a/ndb/src/kernel/blocks/ERROR_codes.txt +++ b/ndb/src/kernel/blocks/ERROR_codes.txt @@ -5,7 +5,7 @@ Next DBACC 3002 Next DBTUP 4014 Next DBLQH 5043 Next DBDICT 6007 -Next DBDIH 7183 +Next DBDIH 7195 Next DBTC 8052 Next CMVMI 9000 Next BACKUP 10022 @@ -73,6 +73,11 @@ Delay GCP_SAVEREQ by 10 secs 7180: Crash master during master-take-over in execMASTER_LCPCONF +7193: Dont send LCP_FRAG_ORD to self, and crash when sending first + LCP_FRAG_ORD(last) + +7194: Force removeNodeFromStored to complete in the middle of MASTER_LCPCONF + ERROR CODES FOR TESTING NODE FAILURE, LOCAL CHECKPOINT HANDLING: ----------------------------------------------------------------- diff --git a/ndb/src/kernel/blocks/dbdih/Dbdih.hpp b/ndb/src/kernel/blocks/dbdih/Dbdih.hpp index ca91f56909d..e471a953391 100644 --- a/ndb/src/kernel/blocks/dbdih/Dbdih.hpp +++ b/ndb/src/kernel/blocks/dbdih/Dbdih.hpp @@ -1291,7 +1291,17 @@ private: LcpStatus lcpStatus; Uint32 lcpStatusUpdatedPlace; + struct Save { + LcpStatus m_status; + Uint32 m_place; + } m_saveState[10]; + void setLcpStatus(LcpStatus status, Uint32 line){ + for (Uint32 i = 9; i > 0; i--) + m_saveState[i] = m_saveState[i-1]; + m_saveState[0].m_status = lcpStatus; + m_saveState[0].m_place = lcpStatusUpdatedPlace; + lcpStatus = status; lcpStatusUpdatedPlace = line; } diff --git a/ndb/src/kernel/blocks/dbdih/DbdihMain.cpp b/ndb/src/kernel/blocks/dbdih/DbdihMain.cpp index 9191bb3fb9b..88d167f0985 100644 --- a/ndb/src/kernel/blocks/dbdih/DbdihMain.cpp +++ b/ndb/src/kernel/blocks/dbdih/DbdihMain.cpp @@ -4764,11 +4764,19 @@ void Dbdih::startRemoveFailedNode(Signal* signal, NodeRecordPtr failedNodePtr) } jam(); - signal->theData[0] = DihContinueB::ZREMOVE_NODE_FROM_TABLE; - signal->theData[1] = failedNodePtr.i; - signal->theData[2] = 0; // Tab id - sendSignal(reference(), GSN_CONTINUEB, signal, 3, JBB); - + + if (!ERROR_INSERTED(7194)) + { + signal->theData[0] = DihContinueB::ZREMOVE_NODE_FROM_TABLE; + signal->theData[1] = failedNodePtr.i; + signal->theData[2] = 0; // Tab id + sendSignal(reference(), GSN_CONTINUEB, signal, 3, JBB); + } + else + { + ndbout_c("7194 Not starting ZREMOVE_NODE_FROM_TABLE"); + } + setLocalNodefailHandling(signal, failedNodePtr.i, NF_REMOVE_NODE_FROM_TABLE); }//Dbdih::startRemoveFailedNode() @@ -5676,12 +5684,22 @@ Dbdih::checkEmptyLcpComplete(Signal *signal){ signal->theData[0] = 7012; execDUMP_STATE_ORD(signal); + + if (ERROR_INSERTED(7194)) + { + ndbout_c("7194 starting ZREMOVE_NODE_FROM_TABLE"); + signal->theData[0] = DihContinueB::ZREMOVE_NODE_FROM_TABLE; + signal->theData[1] = c_lcpMasterTakeOverState.failedNodeId; + signal->theData[2] = 0; // Tab id + sendSignal(reference(), GSN_CONTINUEB, signal, 3, JBB); + } c_lcpMasterTakeOverState.set(LMTOS_INITIAL, __LINE__); MasterLCPReq * const req = (MasterLCPReq *)&signal->theData[0]; req->masterRef = reference(); req->failedNodeId = c_lcpMasterTakeOverState.failedNodeId; sendLoopMacro(MASTER_LCPREQ, sendMASTER_LCPREQ); + } else { sendMASTER_LCPCONF(signal); } @@ -5998,6 +6016,15 @@ void Dbdih::execMASTER_LCPCONF(Signal* signal) { const MasterLCPConf * const conf = (MasterLCPConf *)&signal->theData[0]; jamEntry(); + + if (ERROR_INSERTED(7194)) + { + ndbout_c("delaying MASTER_LCPCONF due to error 7194"); + sendSignalWithDelay(reference(), GSN_MASTER_LCPCONF, signal, + 300, signal->getLength()); + return; + } + Uint32 senderNodeId = conf->senderNodeId; MasterLCPConf::State lcpState = (MasterLCPConf::State)conf->lcpState; const Uint32 failedNodeId = conf->failedNodeId; @@ -6132,7 +6159,6 @@ void Dbdih::MASTER_LCPhandling(Signal* signal, Uint32 failedNodeId) #endif c_lcpState.keepGci = SYSFILE->keepGCI; - c_lcpState.setLcpStatus(LCP_START_LCP_ROUND, __LINE__); startLcpRoundLoopLab(signal, 0, 0); break; } @@ -9924,6 +9950,8 @@ void Dbdih::sendLastLCP_FRAG_ORD(Signal* signal) if(ERROR_INSERTED(7075)){ continue; } + + CRASH_INSERTION(7193); BlockReference ref = calcLqhBlockRef(nodePtr.i); sendSignal(ref, GSN_LCP_FRAG_ORD, signal,LcpFragOrd::SignalLength, JBB); } @@ -10121,6 +10149,13 @@ Dbdih::checkLcpAllTablesDoneInLqh(){ CRASH_INSERTION2(7017, !isMaster()); c_lcpState.setLcpStatus(LCP_TAB_COMPLETED, __LINE__); + + if (ERROR_INSERTED(7194)) + { + ndbout_c("CLEARING 7194"); + CLEAR_ERROR_INSERT_VALUE; + } + return true; } @@ -10276,6 +10311,11 @@ Dbdih::sendLCP_FRAG_ORD(Signal* signal, BlockReference ref = calcLqhBlockRef(replicaPtr.p->procNode); + if (ERROR_INSERTED(7193) && replicaPtr.p->procNode == getOwnNodeId()) + { + return; + } + LcpFragOrd * const lcpFragOrd = (LcpFragOrd *)&signal->theData[0]; lcpFragOrd->tableId = info.tableId; lcpFragOrd->fragmentId = info.fragId; @@ -13686,6 +13726,14 @@ Dbdih::execDUMP_STATE_ORD(Signal* signal) ("immediateLcpStart = %d masterLcpNodeId = %d", c_lcpState.immediateLcpStart, refToNode(c_lcpState.m_masterLcpDihRef)); + + for (Uint32 i = 0; i<10; i++) + { + infoEvent("%u : status: %u place: %u", i, + c_lcpState.m_saveState[i].m_status, + c_lcpState.m_saveState[i].m_place); + } + infoEvent("-- Node %d LCP STATE --", getOwnNodeId()); } diff --git a/ndb/test/ndbapi/testNodeRestart.cpp b/ndb/test/ndbapi/testNodeRestart.cpp index 03a60b1b525..12b0187b71f 100644 --- a/ndb/test/ndbapi/testNodeRestart.cpp +++ b/ndb/test/ndbapi/testNodeRestart.cpp @@ -1347,6 +1347,51 @@ runBug28717(NDBT_Context* ctx, NDBT_Step* step) return NDBT_OK; } +int +runBug32160(NDBT_Context* ctx, NDBT_Step* step) +{ + int result = NDBT_OK; + int loops = ctx->getNumLoops(); + int records = ctx->getNumRecords(); + Ndb* pNdb = GETNDB(step); + NdbRestarter res; + + if (res.getNumDbNodes() < 2) + { + return NDBT_OK; + } + + int master = res.getMasterNodeId(); + int next = res.getNextMasterNodeId(master); + + if (res.insertErrorInNode(next, 7194)) + { + return NDBT_FAILED; + } + + int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 }; + if (res.dumpStateOneNode(master, val2, 2)) + return NDBT_FAILED; + + if (res.insertErrorInNode(master, 7193)) + return NDBT_FAILED; + + int val3[] = { 7099 }; + if (res.dumpStateOneNode(master, val3, 1)) + return NDBT_FAILED; + + if (res.waitNodesNoStart(&master, 1)) + return NDBT_FAILED; + + if (res.startNodes(&master, 1)) + return NDBT_FAILED; + + if (res.waitClusterStarted()) + return NDBT_FAILED; + + return NDBT_OK; +} + NDBT_TESTSUITE(testNodeRestart); TESTCASE("NoLoad", "Test that one node at a time can be stopped and then restarted "\ @@ -1686,6 +1731,9 @@ TESTCASE("Bug28717", ""){ TESTCASE("Bug29364", ""){ INITIALIZER(runBug29364); } +TESTCASE("Bug32160", ""){ + INITIALIZER(runBug32160); +} NDBT_TESTSUITE_END(testNodeRestart); int main(int argc, const char** argv){ diff --git a/ndb/test/run-test/daily-basic-tests.txt b/ndb/test/run-test/daily-basic-tests.txt index 4f7ba26bf27..7b4a4ca0e2d 100644 --- a/ndb/test/run-test/daily-basic-tests.txt +++ b/ndb/test/run-test/daily-basic-tests.txt @@ -497,6 +497,10 @@ max-time: 1000 cmd: testNodeRestart args: -n Bug26481 T1 +max-time: 300 +cmd: testNodeRestart +args: -n Bug32160 T1 + # OLD FLEX max-time: 500 cmd: flexBench From 7276679bc907ca4ae54762d4472c09e58882f085 Mon Sep 17 00:00:00 2001 From: "jmiller/ndbdev@mysql.com/ndb13.mysql.com" <> Date: Thu, 8 Nov 2007 15:05:38 +0100 Subject: [PATCH 15/15] testDict.cpp: Updated DropDDObjects in testDict to ignore the user tables created in the MySQL database --- storage/ndb/test/ndbapi/testDict.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/storage/ndb/test/ndbapi/testDict.cpp b/storage/ndb/test/ndbapi/testDict.cpp index 16b6e129605..e1b8f2b3c7f 100644 --- a/storage/ndb/test/ndbapi/testDict.cpp +++ b/storage/ndb/test/ndbapi/testDict.cpp @@ -2776,9 +2776,13 @@ runDropDDObjects(NDBT_Context* ctx, NDBT_Step* step){ case NdbDictionary::Object::UserTable: tableFound = list.elements[i].name; if(tableFound != 0){ - if(pDict->dropTable(tableFound) != 0){ - g_err << "Failed to drop table: " << pDict->getNdbError() << endl; - return NDBT_FAILED; + if(strcmp(tableFound, "ndb_apply_status") != 0 && + strcmp(tableFound, "NDB$BLOB_2_3") != 0 && + strcmp(tableFound, "ndb_schema") != 0){ + if(pDict->dropTable(tableFound) != 0){ + g_err << "Failed to drop table: " << tableFound << pDict->getNdbError() << endl; + return NDBT_FAILED; + } } } tableFound = 0;