diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index 9abf402edef..17ea7b74565 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -1791,8 +1791,11 @@ protected: public: const char *m_mysql_log_file_name; my_off_t m_mysql_log_offset; +#ifdef MARIAROCKS_NOT_YET + // TODO: MariaDB probably doesn't need these at all: const char *m_mysql_gtid; const char *m_mysql_max_gtid; +#endif String m_detailed_error; int64_t m_snapshot_timestamp = 0; bool m_ddl_transaction; @@ -1986,13 +1989,10 @@ public: rollback(); return true; } else { -#ifdef MARIAROCKS_NOT_YET - my_core::thd_binlog_pos(m_thd, &m_mysql_log_file_name, - &m_mysql_log_offset, &m_mysql_gtid, - &m_mysql_max_gtid); + mysql_bin_log_commit_pos(m_thd, &m_mysql_log_offset, + &m_mysql_log_file_name); binlog_manager.update(m_mysql_log_file_name, m_mysql_log_offset, - m_mysql_max_gtid, get_write_batch()); -#endif + get_write_batch()); return commit_no_binlog(); } } @@ -2831,26 +2831,25 @@ static bool rocksdb_flush_wal(handlerton* hton __attribute__((__unused__))) */ static int rocksdb_prepare(handlerton* hton, THD* thd, bool prepare_tx) { -#ifdef MARIAROCKS_NOT_YET // This is "ASYNC_COMMIT" feature which is only in webscalesql bool async=false; -#endif Rdb_transaction *&tx = get_tx_from_thd(thd); if (!tx->can_prepare()) { return HA_EXIT_FAILURE; } -#ifdef MARIAROCKS_NOT_YET // disable prepare/commit if (prepare_tx || (!my_core::thd_test_options(thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))) { /* We were instructed to prepare the whole transaction, or this is an SQL statement end and autocommit is on */ +#ifdef MARIAROCKS_NOT_YET // disable prepare/commit std::vector slave_gtid_info; my_core::thd_slave_gtid_info(thd, &slave_gtid_info); for (const auto &it : slave_gtid_info) { rocksdb::WriteBatchBase *const write_batch = tx->get_blind_write_batch(); binlog_manager.update_slave_gtid_info(it.id, it.db, it.gtid, write_batch); } +#endif if (tx->is_two_phase()) { if (thd->durability_property == HA_IGNORE_DURABILITY || async) { @@ -2865,7 +2864,7 @@ static int rocksdb_prepare(handlerton* hton, THD* thd, bool prepare_tx) #ifdef MARIAROCKS_NOT_YET (rocksdb_flush_log_at_trx_commit != FLUSH_LOG_NEVER)) { && - THDVAR(thd, flush_log_at_trx_commit)) { + THDVAR(thd, flush_log_at_trx_commit)) #endif { #ifdef MARIAROCKS_NOT_YET @@ -2882,7 +2881,6 @@ static int rocksdb_prepare(handlerton* hton, THD* thd, bool prepare_tx) DEBUG_SYNC(thd, "rocksdb.prepared"); } -#endif return HA_EXIT_SUCCESS; } @@ -3379,11 +3377,7 @@ public: const auto state_it = state_map.find(rdb_trx->GetState()); DBUG_ASSERT(state_it != state_map.end()); -#ifdef MARIAROCKS_NOT_YET - const int is_replication = (thd->rli_slave != nullptr); -#else - const int is_replication= false; -#endif + const int is_replication = (thd->rgi_slave != nullptr); uint32_t waiting_cf_id; std::string waiting_key; rdb_trx->GetWaitingTxns(&waiting_cf_id, &waiting_key), @@ -4094,8 +4088,6 @@ static int rocksdb_init_func(void *const p) { rocksdb::Options main_opts(*rocksdb_db_options, cf_options_map->get_defaults()); -#ifdef MARIAROCKS_NOT_YET -#endif rocksdb::TransactionDBOptions tx_db_options; tx_db_options.transaction_lock_timeout = 2; // 2 seconds tx_db_options.custom_mutex_factory = std::make_shared(); diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/include/have_rocksdb.inc b/storage/rocksdb/mysql-test/rocksdb_rpl/include/have_rocksdb.inc new file mode 100644 index 00000000000..1f762d38c64 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/include/have_rocksdb.inc @@ -0,0 +1,10 @@ +if (`SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'rocksdb' AND support IN ('YES', 'DEFAULT', 'ENABLED')`) +{ + --skip Test requires engine RocksDB. +} + +--disable_query_log +# Table statistics can vary depending on when the memtables are flushed, so +# flush them at the beginning of the test to ensure the test runs consistently. +set global rocksdb_force_flush_memtable_now = true; +--enable_query_log diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/include/have_rocksdb.opt b/storage/rocksdb/mysql-test/rocksdb_rpl/include/have_rocksdb.opt new file mode 100644 index 00000000000..36d7dda1609 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/include/have_rocksdb.opt @@ -0,0 +1,12 @@ +--loose-enable-rocksdb +--loose-enable-rocksdb_global_info +--loose-enable-rocksdb_ddl +--loose-enable-rocksdb_cf_options +--loose-enable_rocksdb_perf_context +--loose-enable_rocksdb_perf_context_global +--loose-enable-rocksdb_index_file_map +--loose-enable-rocksdb_dbstats +--loose-enable-rocksdb_cfstats +--loose-enable-rocksdb_lock_info +--loose-enable-rocksdb_trx +--loose-enable-rocksdb_locks diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/my.cnf b/storage/rocksdb/mysql-test/rocksdb_rpl/my.cnf new file mode 100644 index 00000000000..2beaf514cee --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/my.cnf @@ -0,0 +1,17 @@ +!include include/default_my.cnf + +[server] +skip-innodb +default-storage-engine=rocksdb + + +sql-mode=NO_ENGINE_SUBSTITUTION +explicit-defaults-for-timestamp=1 +loose-rocksdb_lock_wait_timeout=1 +loose-rocksdb_strict_collation_check=0 + +loose-rocksdb-flush-log-at-trx-commit=0 + +# The following is to get rid of the harmless +# "Deadlock found when trying to get lock" errors, see MDEV-12285. +log-warnings=1 diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/r/rpl_rocksdb_2pc_crash_recover.result b/storage/rocksdb/mysql-test/rocksdb_rpl/r/rpl_rocksdb_2pc_crash_recover.result index 59d1a231327..cf3fe03a305 100644 --- a/storage/rocksdb/mysql-test/rocksdb_rpl/r/rpl_rocksdb_2pc_crash_recover.result +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/r/rpl_rocksdb_2pc_crash_recover.result @@ -1,19 +1,19 @@ DROP TABLE IF EXISTS t1; create table t1 (a int primary key, msg varchar(255)) engine=rocksdb; SET GLOBAL ROCKSDB_ENABLE_2PC = ON; -SET SESSION debug="d,crash_commit_after_prepare"; +SET SESSION debug_dbug="d,crash_commit_after_prepare"; insert into t1 values (1, 'dogz'); select * from t1; a msg SET GLOBAL ROCKSDB_ENABLE_2PC = ON; -SET SESSION debug="d,crash_commit_after_log"; +SET SESSION debug_dbug="d,crash_commit_after_log"; insert into t1 values (2, 'catz'), (3, 'men'); select * from t1; a msg 2 catz 3 men SET GLOBAL ROCKSDB_ENABLE_2PC = ON; -SET SESSION debug="d,crash_commit_after"; +SET SESSION debug_dbug="d,crash_commit_after"; insert into t1 values (4, 'cars'), (5, 'foo'); select * from t1; a msg @@ -22,7 +22,7 @@ a msg 4 cars 5 foo SET GLOBAL ROCKSDB_ENABLE_2PC = OFF; -SET SESSION debug="d,crash_commit_after_log"; +SET SESSION debug_dbug="d,crash_commit_after_log"; insert into t1 values (6, 'shipz'), (7, 'tankz'); select * from t1; a msg @@ -31,7 +31,7 @@ a msg 4 cars 5 foo SET GLOBAL ROCKSDB_ENABLE_2PC = OFF; -SET SESSION debug="d,crash_commit_after"; +SET SESSION debug_dbug="d,crash_commit_after"; insert into t1 values (8, 'space'), (9, 'time'); select * from t1; a msg diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/suite.opt b/storage/rocksdb/mysql-test/rocksdb_rpl/suite.opt new file mode 100644 index 00000000000..f5dc0ce891c --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/suite.opt @@ -0,0 +1,2 @@ +--ignore-db-dirs=.rocksdb --plugin-load=$HA_ROCKSDB_SO --default-storage-engine=rocksdb + diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/suite.pm b/storage/rocksdb/mysql-test/rocksdb_rpl/suite.pm new file mode 100644 index 00000000000..79c630f87f1 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/suite.pm @@ -0,0 +1,25 @@ +package My::Suite::Rocksdb; + +# +# Note: ../rocksdb_sys_vars/suite.pm file has a similar +# function. If you modify this file, consider modifying that one, too. +# +@ISA = qw(My::Suite); +use My::Find; +use File::Basename; +use strict; + +sub is_default { not $::opt_embedded_server } + +my $sst_dump= +::mtr_exe_maybe_exists( + "$::bindir/storage/rocksdb$::opt_vs_config/sst_dump", + "$::path_client_bindir/sst_dump"); +return "RocksDB is not compiled, no sst_dump" unless $sst_dump; +$ENV{MARIAROCKS_SST_DUMP}="$sst_dump"; + +# Temporarily disable testing under valgrind, due to MDEV-12439 +return "RocksDB tests disabled under valgrind" if ($::opt_valgrind); + +bless { }; + diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/t/disabled.def b/storage/rocksdb/mysql-test/rocksdb_rpl/t/disabled.def new file mode 100644 index 00000000000..c45236fbd37 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/t/disabled.def @@ -0,0 +1,18 @@ + +# rpl_rocksdb_2pc_crash_recover + +consistent_snapshot_mixed_engines : Didn't try with MariaDB, yet +multiclient_2pc : Didn't try with MariaDB, yet +rpl_crash_safe_wal_corrupt : Didn't try with MariaDB, yet +rpl_gtid_crash_safe : Didn't try with MariaDB, yet +rpl_gtid_crash_safe_wal_corrupt : Didn't try with MariaDB, yet +rpl_gtid_rocksdb_sys_header : Didn't try with MariaDB, yet +rpl_no_unique_check_on_lag : Didn't try with MariaDB, yet +rpl_no_unique_check_on_lag_mts : Didn't try with MariaDB, yet +rpl_rocksdb_snapshot : Didn't try with MariaDB, yet +rpl_rocksdb_snapshot_without_gtid : Didn't try with MariaDB, yet +rpl_rocksdb_stress_crash : Didn't try with MariaDB, yet +rpl_skip_trx_api_binlog_format : Didn't try with MariaDB, yet +singledelete_idempotent_recovery : Didn't try with MariaDB, yet +singledelete_idempotent_table : Didn't try with MariaDB, yet + diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_rocksdb_2pc_crash_recover-master.opt b/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_rocksdb_2pc_crash_recover-master.opt index 74c2de37100..f8f297c567c 100644 --- a/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_rocksdb_2pc_crash_recover-master.opt +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_rocksdb_2pc_crash_recover-master.opt @@ -1 +1 @@ ---gtid_mode=ON --enforce_gtid_consistency --log_bin --log_slave_updates --rocksdb_flush_log_at_trx_commit=1 --rocksdb_write_disable_wal=OFF +--log_bin --log_slave_updates --rocksdb_flush_log_at_trx_commit=1 diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_rocksdb_2pc_crash_recover-slave.opt b/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_rocksdb_2pc_crash_recover-slave.opt index c747adc94d5..7a3f630fda2 100644 --- a/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_rocksdb_2pc_crash_recover-slave.opt +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_rocksdb_2pc_crash_recover-slave.opt @@ -1 +1 @@ ---gtid_mode=ON --enforce_gtid_consistency --log_bin --log_slave_updates +--log_bin --log_slave_updates diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_rocksdb_2pc_crash_recover.test b/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_rocksdb_2pc_crash_recover.test index ea1fe3e34d6..3aa006c84be 100644 --- a/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_rocksdb_2pc_crash_recover.test +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_rocksdb_2pc_crash_recover.test @@ -10,7 +10,7 @@ create table t1 (a int primary key, msg varchar(255)) engine=rocksdb; SET GLOBAL ROCKSDB_ENABLE_2PC = ON; --exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect -SET SESSION debug="d,crash_commit_after_prepare"; +SET SESSION debug_dbug="d,crash_commit_after_prepare"; --error 0,2013 insert into t1 values (1, 'dogz'); --enable_reconnect @@ -19,7 +19,7 @@ select * from t1; SET GLOBAL ROCKSDB_ENABLE_2PC = ON; --exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect -SET SESSION debug="d,crash_commit_after_log"; +SET SESSION debug_dbug="d,crash_commit_after_log"; --error 0,2013 insert into t1 values (2, 'catz'), (3, 'men'); --enable_reconnect @@ -28,7 +28,7 @@ select * from t1; SET GLOBAL ROCKSDB_ENABLE_2PC = ON; --exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect -SET SESSION debug="d,crash_commit_after"; +SET SESSION debug_dbug="d,crash_commit_after"; --error 0,2013 insert into t1 values (4, 'cars'), (5, 'foo'); --enable_reconnect @@ -37,7 +37,7 @@ select * from t1; SET GLOBAL ROCKSDB_ENABLE_2PC = OFF; --exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect -SET SESSION debug="d,crash_commit_after_log"; +SET SESSION debug_dbug="d,crash_commit_after_log"; --error 0,2013 insert into t1 values (6, 'shipz'), (7, 'tankz'); --enable_reconnect @@ -46,7 +46,7 @@ select * from t1; SET GLOBAL ROCKSDB_ENABLE_2PC = OFF; --exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect -SET SESSION debug="d,crash_commit_after"; +SET SESSION debug_dbug="d,crash_commit_after"; --error 0,2013 insert into t1 values (8, 'space'), (9, 'time'); --enable_reconnect diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_flush_memtable_on_analyze_basic.result b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_flush_memtable_on_analyze_basic.result index 165f3811f84..905feec9b1a 100644 --- a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_flush_memtable_on_analyze_basic.result +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_flush_memtable_on_analyze_basic.result @@ -48,11 +48,11 @@ a b 3 3 SHOW TABLE STATUS LIKE 't1'; Name Engine Version Row_format Rows Avg_row_length Data_length Max_data_length Index_length Data_free Auto_increment Create_time Update_time Check_time Collation Checksum Create_options Comment -t1 ROCKSDB 10 Fixed # # 69 0 0 0 4 NULL NULL NULL latin1_swedish_ci NULL +t1 ROCKSDB 10 Fixed # # # 0 0 0 4 NULL NULL NULL latin1_swedish_ci NULL ANALYZE TABLE t1; Table Op Msg_type Msg_text test.t1 analyze status OK SHOW TABLE STATUS LIKE 't1'; Name Engine Version Row_format Rows Avg_row_length Data_length Max_data_length Index_length Data_free Auto_increment Create_time Update_time Check_time Collation Checksum Create_options Comment -t1 ROCKSDB 10 Fixed # # 24 0 0 0 4 NULL NULL NULL latin1_swedish_ci NULL +t1 ROCKSDB 10 Fixed # # # 0 0 0 4 NULL NULL NULL latin1_swedish_ci NULL DROP TABLE t1; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_flush_memtable_on_analyze_basic.test b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_flush_memtable_on_analyze_basic.test index c7e04f89498..574375cd1ea 100644 --- a/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_flush_memtable_on_analyze_basic.test +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_flush_memtable_on_analyze_basic.test @@ -37,10 +37,10 @@ INSERT INTO t1 (b) VALUES (3); --sorted_result SELECT * FROM t1; ---replace_column 5 # 6 # +--replace_column 5 # 6 # 7 # SHOW TABLE STATUS LIKE 't1'; ANALYZE TABLE t1; ---replace_column 5 # 6 # +--replace_column 5 # 6 # 7 # SHOW TABLE STATUS LIKE 't1'; DROP TABLE t1; diff --git a/storage/rocksdb/rdb_datadic.cc b/storage/rocksdb/rdb_datadic.cc index e9c13a84374..3efd3ae6433 100644 --- a/storage/rocksdb/rdb_datadic.cc +++ b/storage/rocksdb/rdb_datadic.cc @@ -4228,12 +4228,10 @@ void Rdb_binlog_manager::cleanup() {} write succeeded or not is not possible here. @param binlog_name Binlog name @param binlog_pos Binlog pos - @param binlog_gtid Binlog max GTID @param batch WriteBatch */ void Rdb_binlog_manager::update(const char *const binlog_name, const my_off_t binlog_pos, - const char *const binlog_max_gtid, rocksdb::WriteBatchBase *const batch) { if (binlog_name && binlog_pos) { // max binlog length (512) + binlog pos (4) + binlog gtid (57) < 1024 @@ -4241,7 +4239,7 @@ void Rdb_binlog_manager::update(const char *const binlog_name, uchar value_buf[RDB_MAX_BINLOG_INFO_LEN]; m_dict->put_key( batch, m_key_slice, - pack_value(value_buf, binlog_name, binlog_pos, binlog_max_gtid)); + pack_value(value_buf, binlog_name, binlog_pos, NULL)); } } @@ -4276,7 +4274,6 @@ bool Rdb_binlog_manager::read(char *const binlog_name, @param buf Preallocated buffer to set binlog info. @param binlog_name Binlog name @param binlog_pos Binlog pos - @param binlog_gtid Binlog GTID @return rocksdb::Slice converted from buf and its length */ rocksdb::Slice @@ -4305,15 +4302,21 @@ Rdb_binlog_manager::pack_value(uchar *const buf, const char *const binlog_name, // store binlog gtid length. // If gtid was not set, store 0 instead +#ifdef MARIAROCKS_NOT_YET const uint16_t binlog_gtid_len = binlog_gtid ? (uint16_t)strlen(binlog_gtid) : 0; rdb_netbuf_store_uint16(buf + pack_len, binlog_gtid_len); +#endif pack_len += sizeof(uint16); + // MariaDB: + rdb_netbuf_store_uint16(buf + pack_len, 0); +#ifdef MARIAROCKS_NOT_YET if (binlog_gtid_len > 0) { // store binlog gtid memcpy(buf + pack_len, binlog_gtid, binlog_gtid_len); pack_len += binlog_gtid_len; } +#endif return rocksdb::Slice((char *)buf, pack_len); } diff --git a/storage/rocksdb/rdb_datadic.h b/storage/rocksdb/rdb_datadic.h index 41747eb885c..b1ecef045d1 100644 --- a/storage/rocksdb/rdb_datadic.h +++ b/storage/rocksdb/rdb_datadic.h @@ -1139,7 +1139,6 @@ public: bool init(Rdb_dict_manager *const dict); void cleanup(); void update(const char *const binlog_name, const my_off_t binlog_pos, - const char *const binlog_max_gtid, rocksdb::WriteBatchBase *const batch); bool read(char *const binlog_name, my_off_t *const binlog_pos, char *const binlog_gtid) const; diff --git a/storage/rocksdb/rdb_mariadb_port.h b/storage/rocksdb/rdb_mariadb_port.h index 1661338030f..627674905cc 100644 --- a/storage/rocksdb/rdb_mariadb_port.h +++ b/storage/rocksdb/rdb_mariadb_port.h @@ -49,4 +49,7 @@ typedef struct my_io_perf_struct my_io_perf_t; std::vector split_into_vector(const std::string& input, char delimiter); +void +mysql_bin_log_commit_pos(THD *thd, ulonglong *out_pos, const char **out_file); + #endif