From 485c9b1fb320958b13253a49d4480ee9decf92f1 Mon Sep 17 00:00:00 2001 From: Junqi Xie Date: Mon, 10 Jul 2023 15:00:00 +0800 Subject: [PATCH] MDEV-30610 Update RocksDB to v8.1.1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MariaRocks is currently lagging behind the main branch of the RocksDB engine. This commit brings MariaRocks up to date with the latest release of RocksDB by backporting changes from Facebook’s MyRocks. These changes include API updates, bug fixes, and improvements for compatibility with RocksDB v8.1.1. Some system variables and metadata tables are modified to reflect the internal changes in RocksDB. Additionally, this commit backports improved and more stable test cases from Facebook’s MyRocks, including tests for the write_unprepared isolation level of RocksDB. It also reverts workarounds for MDEV-29875 and MDEV-31057 and adds support for the latest compilation options. The default value of the following system variables are changed: * rocksdb_stats_level: 1 (kExceptHistogramOrTimers) * rocksdb_wal_recovery_mode: 2 (kPointInTimeRecovery) The following system variables are added: * rocksdb_cancel_manual_compactions * rocksdb_enable_iterate_bounds * rocksdb_enable_pipelined_write * rocksdb_enable_remove_orphaned_dropped_cfs * rocksdb_manual_compaction_bottommost_level * rocksdb_max_background_compactions * rocksdb_max_background_flushes * rocksdb_max_bottom_pri_background_compactions * rocksdb_skip_locks_if_skip_unique_check * rocksdb_track_and_verify_wals_in_manifest * rocksdb_write_batch_flush_threshold The following system variables are deprecated: * rocksdb_hash_index_allow_collision * rocksdb_new_table_reader_for_compaction_inputs The following dynamic metadata table is added: * INFORMATION_SCHEMA.ROCKSDB_LIVE_FILES_METADATA The following status variables are added: * rocksdb_manual_compactions_cancelled * rocksdb_manual_compactions_pending The following status variables are removed: * rocksdb_block_cache_filter_bytes_evict * rocksdb_block_cache_index_bytes_evict * rocksdb_block_cachecompressed_hit * rocksdb_block_cachecompressed_miss * rocksdb_no_file_closes * rocksdb_num_iterators * rocksdb_number_deletes_filtered * rocksdb_write_timedout --- .gitignore | 5 +- cmake/symlinks.cmake | 1 + debian/autobake-deb.sh | 4 - debian/mariadb-plugin-rocksdb.install | 4 + mysql-test/valgrind.supp | 27 + sql/field.cc | 2 +- sql/field.h | 2 +- sql/share/errmsg-utf8.txt | 15 +- storage/rocksdb/CMakeLists.txt | 88 +- storage/rocksdb/build_rocksdb.cmake | 184 +- storage/rocksdb/get_rocksdb_files.sh | 27 - storage/rocksdb/ha_rocksdb.cc | 1834 ++++++++++---- storage/rocksdb/ha_rocksdb.h | 46 +- storage/rocksdb/ha_rocksdb_proto.h | 10 + storage/rocksdb/mariadb-ldb.1 | 4 +- storage/rocksdb/mariadb-sst-dump.1 | 16 + .../rocksdb/mysql-test/rocksdb/combinations | 4 + .../mysql-test/rocksdb/r/bloomfilter2.result | 6 +- .../mysql-test/rocksdb/r/cancel_mc-master.opt | 1 + .../mysql-test/rocksdb/r/cancel_mc.result | 89 + .../rocksdb/r/cancel_mc_timeout.result | 19 + .../mysql-test/rocksdb/r/drop_table3.result | 10 +- .../rocksdb/mysql-test/rocksdb/r/i_s.result | 1 + .../rocksdb/r/information_schema.result | 3 + .../r/issue243_transactionStatus.result | 50 +- .../mysql-test/rocksdb/r/issue896.result | 17 + .../manual_compaction_bottommost_level.result | 37 + .../rocksdb/r/mariadb_plugin.result | 1 + .../rocksdb/r/mariadb_port_fixes.result | 1 + .../mysql-test/rocksdb/r/max_row_locks.result | 112 + .../mysql-test/rocksdb/r/perf_context.result | 4 - .../mysql-test/rocksdb/r/rocksdb.result | 48 +- .../r/rocksdb_bottom_pri_compaction.result | 15 + .../r/rocksdb_bottom_pri_compaction2.result | 16 + .../rocksdb/r/rocksdb_datadir.result | 2 +- .../r/rocksdb_deadlock_detect_rc.result | 1 - .../r/rocksdb_deadlock_detect_rr.result | 1 - ...l_compaction_bottommost_level_basic.result | 114 + .../mysql-test/rocksdb/r/show_engine.result | 84 +- .../rocksdb/r/truncate_failures.result | 95 + .../rocksdb/r/truncate_table3.result | 10 +- .../mysql-test/rocksdb/r/ttl_primary.result | 1477 +++++++++++- .../r/ttl_primary_with_partitions.result | 768 ++++++ .../mysql-test/rocksdb/r/ttl_secondary.result | 2137 ++++++++++++++++- .../r/ttl_secondary_read_filtering.result | 51 + .../r/ttl_secondary_with_partitions.result | 61 + .../mysql-test/rocksdb/r/unique_sec.result | 31 + storage/rocksdb/mysql-test/rocksdb/suite.pm | 4 +- .../mysql-test/rocksdb/t/bloomfilter2.test | 8 +- .../mysql-test/rocksdb/t/cancel_mc.test | 201 ++ .../rocksdb/t/cancel_mc_timeout.test | 49 + .../rocksdb/mysql-test/rocksdb/t/disabled.def | 1 + .../mysql-test/rocksdb/t/drop_table3.inc | 15 +- .../rocksdb/t/information_schema.test | 3 + .../rocksdb/t/issue243_transactionStatus.test | 29 + .../mysql-test/rocksdb/t/issue896.test | 15 + ...ual_compaction_bottommost_level-master.opt | 2 + .../t/manual_compaction_bottommost_level.test | 52 + .../mysql-test/rocksdb/t/max_row_locks.test | 59 + .../mysql-test/rocksdb/t/perf_context.test | 22 +- .../rocksdb/mysql-test/rocksdb/t/rocksdb.test | 3 +- .../rocksdb_bottom_pri_compaction-master.opt | 1 + .../t/rocksdb_bottom_pri_compaction.test | 30 + .../rocksdb_bottom_pri_compaction2-master.opt | 1 + .../t/rocksdb_bottom_pri_compaction2.test | 27 + .../t/rocksdb_bottom_pri_compaction_check.inc | 40 + .../rocksdb/t/rocksdb_deadlock_detect.inc | 16 +- .../rocksdb/t/truncate_failures-master.opt | 1 + .../rocksdb/t/truncate_failures.test | 115 + .../mysql-test/rocksdb/t/ttl_primary.inc | 547 +++++ .../mysql-test/rocksdb/t/ttl_primary.test | 551 +---- .../rocksdb/t/ttl_primary_with_partitions.inc | 255 ++ .../t/ttl_primary_with_partitions.test | 259 +- .../mysql-test/rocksdb/t/ttl_secondary.inc | 783 ++++++ .../mysql-test/rocksdb/t/ttl_secondary.test | 796 +----- .../t/ttl_secondary_read_filtering.test | 39 + .../t/ttl_secondary_with_partitions.test | 56 + .../mysql-test/rocksdb/t/unique_sec.test | 33 + .../mysql-test/rocksdb_rpl/combinations | 5 + ...sdb_skip_locks_if_skip_unique_check.result | 37 + .../rocksdb_rpl/r/tx_system_failure.result | 240 ++ .../rocksdb/mysql-test/rocksdb_rpl/suite.pm | 4 +- ...cksdb_skip_locks_if_skip_unique_check.test | 48 + .../t/rpl_gtid_crash_safe_wal_corrupt.inc | 33 +- .../rocksdb_rpl/t/tx_system_failure.test | 38 + .../t/tx_system_failure_commit.inc | 52 + .../t/tx_system_failure_prepare.inc | 17 + .../mysql-test/rocksdb_stress/suite.pm | 4 +- ...sdb_cancel_manual_compactions_basic.result | 50 + .../r/rocksdb_delete_cf_basic.result | 32 +- ...rocksdb_enable_iterate_bounds_basic.result | 35 + ...ocksdb_enable_pipelined_write_basic.result | 7 + ...e_remove_orphaned_dropped_cfs_basic.result | 97 + ...l_compaction_bottommost_level_basic.result | 114 + ...db_max_background_compactions_basic.result | 46 + ...ocksdb_max_background_flushes_basic.result | 14 + ...om_pri_background_compactions_basic.result | 46 + .../r/rocksdb_max_row_locks_basic.result | 42 +- ...ip_locks_if_skip_unique_check_basic.result | 100 + .../r/rocksdb_stats_level_basic.result | 20 +- ...k_and_verify_wals_in_manifest_basic.result | 14 + .../r/rocksdb_wal_recovery_mode_basic.result | 10 +- ...b_write_batch_flush_threshold_basic.result | 100 + .../mysql-test/rocksdb_sys_vars/suite.pm | 4 +- ...cksdb_cancel_manual_compactions_basic.test | 17 + .../t/rocksdb_delete_cf_basic.test | 44 +- .../rocksdb_enable_iterate_bounds_basic.test | 43 + .../rocksdb_enable_pipelined_write_basic.test | 6 + ...ble_remove_orphaned_dropped_cfs_basic.test | 21 + ...ual_compaction_bottommost_level_basic.test | 18 + ...ksdb_max_background_compactions_basic.test | 16 + .../rocksdb_max_background_flushes_basic.test | 16 + ...ttom_pri_background_compactions_basic.test | 16 + .../t/rocksdb_max_row_locks_basic.test | 2 +- ...skip_locks_if_skip_unique_check_basic.test | 18 + ...ack_and_verify_wals_in_manifest_basic.test | 16 + ...sdb_write_batch_flush_threshold_basic.test | 18 + storage/rocksdb/nosql_access.cc | 53 - storage/rocksdb/nosql_access.h | 36 - storage/rocksdb/rdb_cf_manager.cc | 299 ++- storage/rocksdb/rdb_cf_manager.h | 36 +- storage/rocksdb/rdb_cf_options.cc | 26 +- storage/rocksdb/rdb_converter.cc | 185 +- storage/rocksdb/rdb_converter.h | 31 +- storage/rocksdb/rdb_datadic.cc | 638 +++-- storage/rocksdb/rdb_datadic.h | 165 +- storage/rocksdb/rdb_i_s.cc | 317 ++- storage/rocksdb/rdb_i_s.h | 1 + storage/rocksdb/rdb_psi.cc | 5 +- storage/rocksdb/rdb_psi.h | 3 +- storage/rocksdb/rdb_threads.h | 41 +- storage/rocksdb/rocksdb | 2 +- storage/rocksdb/tools/mysql_ldb.cc | 35 +- storage/rocksdb/tools/mysql_sst_dump.cc | 44 + 134 files changed, 11955 insertions(+), 2970 deletions(-) delete mode 100755 storage/rocksdb/get_rocksdb_files.sh create mode 100644 storage/rocksdb/mariadb-sst-dump.1 create mode 100644 storage/rocksdb/mysql-test/rocksdb/r/cancel_mc-master.opt create mode 100644 storage/rocksdb/mysql-test/rocksdb/r/cancel_mc.result create mode 100644 storage/rocksdb/mysql-test/rocksdb/r/cancel_mc_timeout.result create mode 100644 storage/rocksdb/mysql-test/rocksdb/r/manual_compaction_bottommost_level.result create mode 100644 storage/rocksdb/mysql-test/rocksdb/r/max_row_locks.result create mode 100644 storage/rocksdb/mysql-test/rocksdb/r/rocksdb_bottom_pri_compaction.result create mode 100644 storage/rocksdb/mysql-test/rocksdb/r/rocksdb_bottom_pri_compaction2.result create mode 100644 storage/rocksdb/mysql-test/rocksdb/r/rocksdb_manual_compaction_bottommost_level_basic.result create mode 100644 storage/rocksdb/mysql-test/rocksdb/r/truncate_failures.result create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/cancel_mc.test create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/cancel_mc_timeout.test create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/manual_compaction_bottommost_level-master.opt create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/manual_compaction_bottommost_level.test create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/max_row_locks.test create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/rocksdb_bottom_pri_compaction-master.opt create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/rocksdb_bottom_pri_compaction.test create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/rocksdb_bottom_pri_compaction2-master.opt create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/rocksdb_bottom_pri_compaction2.test create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/rocksdb_bottom_pri_compaction_check.inc create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/truncate_failures-master.opt create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/truncate_failures.test create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/ttl_primary.inc create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/ttl_primary_with_partitions.inc create mode 100644 storage/rocksdb/mysql-test/rocksdb/t/ttl_secondary.inc create mode 100644 storage/rocksdb/mysql-test/rocksdb_rpl/r/rocksdb_skip_locks_if_skip_unique_check.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_rpl/r/tx_system_failure.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_rpl/t/rocksdb_skip_locks_if_skip_unique_check.test create mode 100644 storage/rocksdb/mysql-test/rocksdb_rpl/t/tx_system_failure.test create mode 100644 storage/rocksdb/mysql-test/rocksdb_rpl/t/tx_system_failure_commit.inc create mode 100644 storage/rocksdb/mysql-test/rocksdb_rpl/t/tx_system_failure_prepare.inc create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_cancel_manual_compactions_basic.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_enable_iterate_bounds_basic.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_enable_pipelined_write_basic.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_enable_remove_orphaned_dropped_cfs_basic.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_manual_compaction_bottommost_level_basic.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_max_background_compactions_basic.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_max_background_flushes_basic.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_max_bottom_pri_background_compactions_basic.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_skip_locks_if_skip_unique_check_basic.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_track_and_verify_wals_in_manifest_basic.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_write_batch_flush_threshold_basic.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_cancel_manual_compactions_basic.test create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_enable_iterate_bounds_basic.test create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_enable_pipelined_write_basic.test create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_enable_remove_orphaned_dropped_cfs_basic.test create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_manual_compaction_bottommost_level_basic.test create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_max_background_compactions_basic.test create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_max_background_flushes_basic.test create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_max_bottom_pri_background_compactions_basic.test create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_skip_locks_if_skip_unique_check_basic.test create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_track_and_verify_wals_in_manifest_basic.test create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_write_batch_flush_threshold_basic.test delete mode 100644 storage/rocksdb/nosql_access.cc delete mode 100644 storage/rocksdb/nosql_access.h create mode 100644 storage/rocksdb/tools/mysql_sst_dump.cc diff --git a/.gitignore b/.gitignore index ac67886c51c..7b2b241d41d 100644 --- a/.gitignore +++ b/.gitignore @@ -240,8 +240,12 @@ storage/rocksdb/ldb storage/rocksdb/myrocks_hotbackup storage/rocksdb/mysql_ldb storage/rocksdb/mysql_ldb.1 +storage/rocksdb/mariadb-ldb storage/rocksdb/rdb_source_revision.h storage/rocksdb/sst_dump +storage/rocksdb/mysql_sst_dump +storage/rocksdb/mysql_sst_dump.1 +storage/rocksdb/mariadb-sst-dump strings/conf_to_src support-files/MySQL-shared-compat.spec support-files/binary-configure @@ -615,7 +619,6 @@ scripts/mariadb-secure-installation scripts/mariadb-setpermission sql/mariadbd sql/mariadb-tzinfo-to-sql -storage/rocksdb/mariadb-ldb strings/ctype-uca1400data.h strings/uca-dump tests/mariadb-client-test diff --git a/cmake/symlinks.cmake b/cmake/symlinks.cmake index 3f3b4e4a9b5..0a722f87cb4 100644 --- a/cmake/symlinks.cmake +++ b/cmake/symlinks.cmake @@ -33,6 +33,7 @@ REGISTER_SYMLINK("mariadb-secure-installation" "mysql_secure_installation") REGISTER_SYMLINK("mariadb-setpermission" "mysql_setpermission") REGISTER_SYMLINK("mariadb-show" "mysqlshow") REGISTER_SYMLINK("mariadb-slap" "mysqlslap") +REGISTER_SYMLINK("mariadb-sst-dump" "mysql_sst_dump") REGISTER_SYMLINK("mariadb-test" "mysqltest") REGISTER_SYMLINK("mariadb-test-embedded" "mysqltest_embedded") REGISTER_SYMLINK("mariadb-tzinfo-to-sql" "mysql_tzinfo_to_sql") diff --git a/debian/autobake-deb.sh b/debian/autobake-deb.sh index 6bca826e60d..23f29b1d3ac 100755 --- a/debian/autobake-deb.sh +++ b/debian/autobake-deb.sh @@ -29,10 +29,6 @@ remove_rocksdb_tools() { sed '/rocksdb-tools/d' -i debian/control sed '/sst_dump/d' -i debian/not-installed - if ! grep -q sst_dump debian/mariadb-plugin-rocksdb.install - then - echo "usr/bin/sst_dump" >> debian/mariadb-plugin-rocksdb.install - fi } add_lsb_base_depends() diff --git a/debian/mariadb-plugin-rocksdb.install b/debian/mariadb-plugin-rocksdb.install index 67b8026d697..bf4ad8e072e 100644 --- a/debian/mariadb-plugin-rocksdb.install +++ b/debian/mariadb-plugin-rocksdb.install @@ -1,8 +1,12 @@ etc/mysql/mariadb.conf.d/rocksdb.cnf usr/bin/mariadb-ldb +usr/bin/mariadb-sst-dump usr/bin/myrocks_hotbackup usr/bin/mysql_ldb +usr/bin/mysql_sst_dump usr/lib/mysql/plugin/ha_rocksdb.so usr/share/man/man1/mariadb-ldb.1 +usr/share/man/man1/mariadb-sst-dump.1 usr/share/man/man1/myrocks_hotbackup.1 usr/share/man/man1/mysql_ldb.1 +usr/share/man/man1/mysql_sst_dump.1 diff --git a/mysql-test/valgrind.supp b/mysql-test/valgrind.supp index 9127bb1f962..e578a1e8fd0 100644 --- a/mysql-test/valgrind.supp +++ b/mysql-test/valgrind.supp @@ -1504,6 +1504,33 @@ fun:_ZN7myrocksL27rdb_init_rocksdb_db_optionsEv } +{ + Still reachable for once-per-process initializations + Memcheck:Leak + match-leak-kinds: reachable + ... + fun:_ZN10my_rocksdb24CacheEntryStatsCollectorINS_13InternalStats19CacheEntryRoleStatsEE9GetSharedEPNS_5CacheEPNS_11SystemClockEPSt10shared_ptrIS3_E + ... +} + +{ + Still reachable for once-per-process initializations + Memcheck:Leak + match-leak-kinds: reachable + ... + fun:_ZN10my_rocksdb12_GLOBAL__N_18Registry8RegisterEPFvRKNS_5SliceEPvENS_14CacheEntryRoleE + ... +} + +{ + Still reachable for once-per-process initializations + Memcheck:Leak + match-leak-kinds: reachable + ... + fun:_ZN10my_rocksdb12_GLOBAL__N_111GetRegistryEv + ... +} + { Still reachable for thread local storage initialization (SetHandle) Memcheck:Leak diff --git a/sql/field.cc b/sql/field.cc index 5acebecd5c4..e95f3a6ae9c 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -8593,7 +8593,7 @@ void Field_blob::store_length(uchar *i_ptr, uint i_packlength, uint32 i_number) } -uint32 Field_blob::get_length(const uchar *pos, uint packlength_arg) const +uint32 Field_blob::get_length(const uchar *pos, uint packlength_arg) { return (uint32)read_lowendian(pos, packlength_arg); } diff --git a/sql/field.h b/sql/field.h index 477dc0fe282..edf6cc3400f 100644 --- a/sql/field.h +++ b/sql/field.h @@ -4540,7 +4540,7 @@ public: } inline uint32 get_length(my_ptrdiff_t row_offset= 0) const { return get_length(ptr+row_offset, this->packlength); } - uint32 get_length(const uchar *ptr, uint packlength) const; + static uint32 get_length(const uchar *ptr, uint packlength); uint32 get_length(const uchar *ptr_arg) const { return get_length(ptr_arg, this->packlength); } inline uchar *get_ptr() const { return get_ptr(ptr); } diff --git a/sql/share/errmsg-utf8.txt b/sql/share/errmsg-utf8.txt index 2d025f8b80e..57dbfb3ae53 100644 --- a/sql/share/errmsg-utf8.txt +++ b/sql/share/errmsg-utf8.txt @@ -11675,10 +11675,10 @@ ER_RDB_TTL_UNSUPPORTED sw "Uauniaji wa TTL umezimwa kwa sasa wakati jedwali lina PK iliyofichwa." ER_RDB_TTL_COL_FORMAT - chi "Myrocks中的TTL列(%s)必须是一个无符号的非空64位整数，存在于表内，并具有伴随的TTL持续时间。" - eng "TTL column (%s) in MyRocks must be an unsigned non-null 64-bit integer, exist inside the table, and have an accompanying ttl duration." - spa "La columna TTL (%s) en MyRocks debe de ser un entero sin signo no-null de 64-bit, debe de existir dentro de la tabla y debe de tener una duración ttl acompañante." - sw "Safu wima ya TTL (%s) katika MyRocks lazima iwe nambari kamili ya biti 64 ambayo haijatiwa saini, iwe ndani ya jedwali, na iwe na muda wa ttl unaoandamana." + chi "Myrocks中的TTL列(%s)必须是一个无符号的非空64位整数或非空时间戳，存在于表内，并具有伴随的TTL持续时间。" + eng "TTL column (%s) in MyRocks must be an unsigned non-null 64-bit integer or non-null timestamp, exist inside the table, and have an accompanying ttl duration." + spa "La columna TTL (%s) en MyRocks debe de ser un entero sin signo no-null de 64-bit o una marca de tiempo no-null, debe de existir dentro de la tabla y debe de tener una duración ttl acompañante." + sw "Safu wima ya TTL (%s) katika MyRocks lazima iwe nambari kamili ya biti 64 ambayo haijatiwa saini au muhuri wa muda usio batili, iwe ndani ya jedwali, na iwe na muda wa ttl unaoandamana." ER_PER_INDEX_CF_DEPRECATED chi "已弃用每个索引列族选项" @@ -11697,6 +11697,7 @@ ER_SK_POPULATE_DURING_ALTER eng "MyRocks failed populating secondary key during alter." spa "MyRocks falló al poblar clave secundaria duante el 'alter'." sw "MyRocks imeshindwa kujaza ufunguo wa sekondari wakati wa kubadilisha." + # MyRocks messages end ER_SUM_FUNC_WITH_WINDOW_FUNC_AS_ARG @@ -12300,3 +12301,9 @@ ER_JSON_INVALID_VALUE_FOR_KEYWORD ER_JSON_SCHEMA_KEYWORD_UNSUPPORTED eng "%s keyword is not supported" sw "%s neno kuu halitumiki" +ER_CF_DROPPED + chi "列族 ('%s') 正在被删除。" + eng "Column family ('%s') is being dropped." +ER_CANT_DROP_CF + chi "不能删除列族 ('%s')，因为它正在使用或不存在。" + eng "Cannot drop Column family ('%s') because it is in use or does not exist." diff --git a/storage/rocksdb/CMakeLists.txt b/storage/rocksdb/CMakeLists.txt index 76e9a4ab5a1..bfaea6e1dc0 100644 --- a/storage/rocksdb/CMakeLists.txt +++ b/storage/rocksdb/CMakeLists.txt @@ -4,8 +4,6 @@ SET(CPACK_RPM_rocksdb-engine_PACKAGE_SUMMARY "RocksDB storage engine for MariaDB SET(CPACK_RPM_rocksdb-engine_PACKAGE_DESCRIPTION "The RocksDB storage engine is a high performance storage engine, aimed at maximising storage efficiency while maintaining InnoDB-like performance." PARENT_SCOPE) -MY_CHECK_AND_SET_COMPILER_FLAG(-Wno-range-loop-construct) - MACRO(SKIP_ROCKSDB_PLUGIN msg) MESSAGE_ONCE(SKIP_ROCKSDB_PLUGIN "Can't build rocksdb engine - ${msg}") ADD_FEATURE_INFO(ROCKSDB "OFF" "Storage Engine") @@ -58,41 +56,29 @@ IF(MSVC_ARM64) SKIP_ROCKSDB_PLUGIN("Windows ARM64 not supported") ENDIF() -# -# Also, disable on ARM64 when not Linux -# Requires submodule update to v6.16.3 -# containing commit https://github.com/facebook/rocksdb/commit/ee4bd4780b321ddb5f92a0f4eb956f2a2ebd60dc -# -IF(CMAKE_SYSTEM_PROCESSOR MATCHES "(arm64|aarch64)" AND NOT CMAKE_SYSTEM_NAME STREQUAL "Linux") - SKIP_ROCKSDB_PLUGIN("ARM64 disabled on all except Linux") -ENDIF() - -# This plugin needs recent C++ compilers (it is using C++11 features) +# This plugin needs recent C++ compilers (it is using C++17 features) # Skip build for the old compilers -SET(CXX11_FLAGS) -SET(OLD_COMPILER_MSG "requires c++11 -capable compiler (minimal supported versions are g++ 4.8, clang 3.3, VS2015)") +SET(CXX17_FLAGS) +SET(OLD_COMPILER_MSG "requires c++17 -capable compiler (minimal supported versions are g++ 7, clang 5, VS2017)") IF(CMAKE_CXX_COMPILER_ID MATCHES "GNU") EXECUTE_PROCESS(COMMAND ${CMAKE_CXX_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION) - IF (GCC_VERSION VERSION_LESS 4.8) + IF (GCC_VERSION VERSION_LESS 7) SKIP_ROCKSDB_PLUGIN("${OLD_COMPILER_MSG}") ENDIF() - SET(CXX11_FLAGS "-std=c++11") - IF (GCC_VERSION VERSION_LESS 5.0) - SET(CXX11_FLAGS "-std=c++11 -Wno-missing-field-initializers") - ENDIF() + SET(CXX17_FLAGS "-std=c++17") ELSEIF (CMAKE_CXX_COMPILER_ID MATCHES "Clang") - IF ((CMAKE_CXX_COMPILER_VERSION AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 3.3) OR - (CLANG_VERSION_STRING AND CLANG_VERSION_STRING VERSION_LESS 3.3)) + IF ((CMAKE_CXX_COMPILER_VERSION AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5) OR + (CLANG_VERSION_STRING AND CLANG_VERSION_STRING VERSION_LESS 5)) SKIP_ROCKSDB_PLUGIN("${OLD_COMPILER_MSG}") ENDIF() - SET(CXX11_FLAGS "-std=c++11 -stdlib=libstdc++") + SET(CXX17_FLAGS "-std=c++17 -stdlib=libstdc++") IF(MSVC) # clang-cl does not work yet SKIP_ROCKSDB_PLUGIN("Clang-cl is not supported") ENDIF() ELSEIF(MSVC) - IF (MSVC_VERSION LESS 1900) + IF (MSVC_VERSION LESS 1914) SKIP_ROCKSDB_PLUGIN("${OLD_COMPILER_MSG}") ENDIF() ELSE() @@ -100,9 +86,9 @@ ELSE() ENDIF() IF(CMAKE_VERSION GREATER 3.0) - SET(CMAKE_CXX_STANDARD 11) -ELSEIF(CXX11_FLAGS) - ADD_DEFINITIONS(${CXX11_FLAGS}) + SET(CMAKE_CXX_STANDARD 17) +ELSEIF(CXX17_FLAGS) + ADD_DEFINITIONS(${CXX17_FLAGS}) ENDIF() SET(ROCKSDB_SE_SOURCES @@ -135,11 +121,6 @@ SET(ROCKSDB_SE_SOURCES rdb_converter.h ) -# MariaDB: the following is added in build_rocksdb.cmake, when appropriate: -# This is a strong requirement coming from RocksDB. No conditional checks here. -#ADD_DEFINITIONS(-DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -#) - if (HAVE_GCC_C11_ATOMICS_WITH_LIBATOMIC) SET(ATOMIC_EXTRA_LIBS -latomic) else() @@ -160,7 +141,7 @@ IF(NOT TARGET rocksdb) RETURN() ENDIF() -INSTALL_MANPAGES(rocksdb-engine mariadb-ldb.1 myrocks_hotbackup.1) +INSTALL_MANPAGES(rocksdb-engine mariadb-ldb.1 mariadb-sst-dump.1 myrocks_hotbackup.1) CHECK_CXX_SOURCE_COMPILES(" #if defined(_MSC_VER) && !defined(__thread) @@ -193,7 +174,6 @@ ADD_CONVENIENCE_LIBRARY(rocksdb_aux_lib rdb_perf_context.h rdb_buff.h rdb_mariadb_port.h - nosql_access.cc nosql_access.h ) ADD_DEPENDENCIES(rocksdb_aux_lib GenError) @@ -204,37 +184,8 @@ if (UNIX AND NOT APPLE AND NOT CMAKE_SYSTEM_NAME STREQUAL "OpenBSD") TARGET_LINK_LIBRARIES(rocksdb_aux_lib -lrt) endif() TARGET_LINK_LIBRARIES(rocksdb_aux_lib ${ATOMIC_EXTRA_LIBS}) - -# IF (WITH_JEMALLOC) -# FIND_LIBRARY(JEMALLOC_LIBRARY -# NAMES libjemalloc${PIC_EXT}.a jemalloc -# HINTS ${WITH_JEMALLOC}/lib) -# SET(rocksdb_static_libs ${rocksdb_static_libs} -# ${JEMALLOC_LIBRARY}) -# ADD_DEFINITIONS(-DROCKSDB_JEMALLOC) -# ADD_DEFINITIONS(-DROCKSDB_MALLOC_USABLE_SIZE) -# ENDIF() - -# MariaDB: Q: why does the upstream add libunwind for a particular -# storage engine? -#IF (WITH_UNWIND) -# FIND_LIBRARY(UNWIND_LIBRARY -# NAMES libunwind${PIC_EXT}.a unwind -# HINTS ${WITH_UNWIND}/lib) -# SET(rocksdb_static_libs ${rocksdb_static_libs} -# ${UNWIND_LIBRARY}) -#ENDIF() - - TARGET_LINK_LIBRARIES(rocksdb rocksdb_aux_lib) -CHECK_FUNCTION_EXISTS(sched_getcpu HAVE_SCHED_GETCPU) -IF(HAVE_SCHED_GETCPU) - ADD_DEFINITIONS(-DHAVE_SCHED_GETCPU=1) -# MariaDB: don't do this: -# ADD_DEFINITIONS(-DZSTD_STATIC_LINKING_ONLY) -ENDIF() - IF (WITH_TBB) FIND_LIBRARY(TBB_LIBRARY NAMES libtbb${PIC_EXT}.a tbb @@ -244,13 +195,6 @@ IF (WITH_TBB) ADD_DEFINITIONS(-DTBB) ENDIF() -# -# MariaDB: Dynamic plugin build is not suitable with unittest ATM -# -#IF(WITH_UNIT_TESTS AND WITH_EMBEDDED_SERVER) -# ADD_SUBDIRECTORY(unittest) -#ENDIF() - if (UNIX AND NOT APPLE AND NOT CMAKE_SYSTEM_NAME STREQUAL "OpenBSD") SET(rocksdb_static_libs ${rocksdb_static_libs} "-lrt") endif() @@ -262,8 +206,8 @@ ADD_LIBRARY(rocksdb_tools STATIC rocksdb/tools/sst_dump_tool.cc ) -MYSQL_ADD_EXECUTABLE(sst_dump rocksdb/tools/sst_dump.cc COMPONENT rocksdb-engine) -TARGET_LINK_LIBRARIES(sst_dump rocksdblib) +MYSQL_ADD_EXECUTABLE(mariadb-sst-dump tools/mysql_sst_dump.cc COMPONENT rocksdb-engine) +TARGET_LINK_LIBRARIES(mariadb-sst-dump rocksdblib) MYSQL_ADD_EXECUTABLE(mariadb-ldb tools/mysql_ldb.cc COMPONENT rocksdb-engine) TARGET_LINK_LIBRARIES(mariadb-ldb rocksdb_tools rocksdb_aux_lib dbug) @@ -285,7 +229,7 @@ IF(MSVC) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4267") ENDIF() ELSEIF(CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") - SET_TARGET_PROPERTIES(rocksdb_tools sst_dump mariadb-ldb PROPERTIES COMPILE_FLAGS "-Wno-error") + SET_TARGET_PROPERTIES(rocksdb_tools mariadb-sst-dump mariadb-ldb PROPERTIES COMPILE_FLAGS "-Wno-error") ENDIF() IF(GIT_EXECUTABLE AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/.git) diff --git a/storage/rocksdb/build_rocksdb.cmake b/storage/rocksdb/build_rocksdb.cmake index 27a8616cfda..a35bd66f93f 100644 --- a/storage/rocksdb/build_rocksdb.cmake +++ b/storage/rocksdb/build_rocksdb.cmake @@ -9,7 +9,6 @@ INCLUDE_DIRECTORIES( ${CMAKE_CURRENT_BINARY_DIR} ${ROCKSDB_SOURCE_DIR} ${ROCKSDB_SOURCE_DIR}/include - ${ROCKSDB_SOURCE_DIR}/third-party/gtest-1.7.0/fused-src ) list(APPEND CMAKE_MODULE_PATH "${ROCKSDB_SOURCE_DIR}/cmake/modules/") @@ -90,6 +89,8 @@ elseif(CMAKE_SYSTEM_NAME MATCHES "Linux") add_definitions(-DOS_LINUX) elseif(CMAKE_SYSTEM_NAME MATCHES "SunOS") add_definitions(-DOS_SOLARIS) +elseif(CMAKE_SYSTEM_NAME MATCHES "kFreeBSD") + add_definitions(-DOS_GNU_KFREEBSD) elseif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") add_definitions(-DOS_FREEBSD) elseif(CMAKE_SYSTEM_NAME MATCHES "NetBSD") @@ -101,7 +102,7 @@ elseif(CMAKE_SYSTEM_NAME MATCHES "DragonFly") elseif(CMAKE_SYSTEM_NAME MATCHES "Android") add_definitions(-DOS_ANDROID) elseif(CMAKE_SYSTEM_NAME MATCHES "Windows") - add_definitions(-DOS_WIN) + add_definitions(-DWIN32 -DOS_WIN -D_MBCS -DWIN64 -DNOMINMAX) endif() IF(MSVC) @@ -113,7 +114,7 @@ endif() include(CheckCCompilerFlag) # ppc64 or ppc64le or powerpc64 (BSD) -if(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64|powerpc64") +if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64") CHECK_C_COMPILER_FLAG("-maltivec" HAS_ALTIVEC) if(HAS_ALTIVEC) message(STATUS " HAS_ALTIVEC yes") @@ -127,7 +128,7 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64|powerpc64") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=power8") endif() ADD_DEFINITIONS(-DHAVE_POWER8 -DHAS_ALTIVEC) -endif(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64|powerpc64") +endif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64") option(WITH_FALLOCATE "build with fallocate" ON) @@ -146,14 +147,35 @@ int main() { endif() endif() +CHECK_CXX_SOURCE_COMPILES(" +#include +int main() { + (void) PTHREAD_MUTEX_ADAPTIVE_NP; +} +" HAVE_PTHREAD_MUTEX_ADAPTIVE_NP) +if(HAVE_PTHREAD_MUTEX_ADAPTIVE_NP) + add_definitions(-DROCKSDB_PTHREAD_ADAPTIVE_MUTEX) +endif() + +CHECK_SYMBOL_EXISTS(SYNC_FILE_RANGE_WRITE "fcntl.h" HAVE_SYNC_FILE_RANGE_WRITE) +if(HAVE_SYNC_FILE_RANGE_WRITE) + add_definitions(-DROCKSDB_RANGESYNC_PRESENT) +endif() + CHECK_FUNCTION_EXISTS(malloc_usable_size HAVE_MALLOC_USABLE_SIZE) if(HAVE_MALLOC_USABLE_SIZE) add_definitions(-DROCKSDB_MALLOC_USABLE_SIZE) endif() -include_directories(${ROCKSDB_SOURCE_DIR}) -include_directories(${ROCKSDB_SOURCE_DIR}/include) -include_directories(SYSTEM ${ROCKSDB_SOURCE_DIR}/third-party/gtest-1.7.0/fused-src) +CHECK_FUNCTION_EXISTS(getauxval HAVE_AUXV_GETAUXVAL) +if(HAVE_AUXV_GETAUXVAL) + add_definitions(-DROCKSDB_AUXV_GETAUXVAL_PRESENT) +endif() + +CHECK_FUNCTION_EXISTS(F_FULLFSYNC HAVE_FULLFSYNC) +if(HAVE_FULLFSYNC) + add_definitions(-DHAVE_FULLFSYNC) +endif() find_package(Threads REQUIRED) if(WIN32) @@ -162,11 +184,9 @@ else() set(SYSTEM_LIBS ${SYSTEM_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${LIBRT} ${CMAKE_DL_LIBS} ${ATOMIC_EXTRA_LIBS}) endif() -set(ROCKSDB_LIBS rocksdblib}) +set(ROCKSDB_LIBS rocksdblib) set(LIBS ${ROCKSDB_LIBS} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS}) -#add_subdirectory(${ROCKSDB_SOURCE_DIR}/tools) - # Main library source code # Note : RocksDB has a lot of unittests. We should not include these files # in the build, because 1. they are not needed and 2. gtest causes warnings @@ -176,14 +196,36 @@ set(LIBS ${ROCKSDB_LIBS} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS}) # - *_test.cc # - *_bench.cc set(ROCKSDB_SOURCES + cache/cache.cc + cache/cache_entry_roles.cc + cache/cache_key.cc + cache/cache_helpers.cc + cache/cache_reservation_manager.cc + cache/charged_cache.cc cache/clock_cache.cc + cache/compressed_secondary_cache.cc cache/lru_cache.cc + cache/secondary_cache.cc + cache/secondary_cache_adapter.cc cache/sharded_cache.cc db/arena_wrapped_db_iter.cc + db/blob/blob_contents.cc + db/blob/blob_fetcher.cc + db/blob/blob_file_addition.cc + db/blob/blob_file_builder.cc + db/blob/blob_file_cache.cc + db/blob/blob_file_garbage.cc + db/blob/blob_file_meta.cc + db/blob/blob_file_reader.cc + db/blob/blob_garbage_meter.cc + db/blob/blob_log_format.cc + db/blob/blob_log_sequential_reader.cc + db/blob/blob_log_writer.cc + db/blob/blob_source.cc + db/blob/prefetch_buffer_collection.cc db/builder.cc db/c.cc db/column_family.cc - db/compacted_db_impl.cc db/compaction/compaction.cc db/compaction/compaction_iterator.cc db/compaction/compaction_picker.cc @@ -191,8 +233,14 @@ set(ROCKSDB_SOURCES db/compaction/compaction_picker_fifo.cc db/compaction/compaction_picker_level.cc db/compaction/compaction_picker_universal.cc + db/compaction/compaction_service_job.cc + db/compaction/compaction_state.cc + db/compaction/compaction_outputs.cc + db/compaction/sst_partitioner.cc + db/compaction/subcompaction_state.cc db/convenience.cc db/db_filesnapshot.cc + db/db_impl/compacted_db_impl.cc db/db_impl/db_impl.cc db/db_impl/db_impl_write.cc db/db_impl/db_impl_compaction_flush.cc @@ -223,9 +271,12 @@ set(ROCKSDB_SOURCES db/memtable_list.cc db/merge_helper.cc db/merge_operator.cc + db/output_validator.cc + db/periodic_task_scheduler.cc db/range_del_aggregator.cc db/range_tombstone_fragmenter.cc db/repair.cc + db/seqno_to_time_mapping.cc db/snapshot_impl.cc db/table_cache.cc db/table_properties_collector.cc @@ -233,22 +284,31 @@ set(ROCKSDB_SOURCES db/trim_history_scheduler.cc db/version_builder.cc db/version_edit.cc + db/version_edit_handler.cc db/version_set.cc + db/wal_edit.cc db/wal_manager.cc + db/wide/wide_column_serialization.cc + db/wide/wide_columns.cc db/write_batch.cc db/write_batch_base.cc db/write_controller.cc + db/write_stall_stats.cc db/write_thread.cc + env/composite_env.cc env/env.cc env/env_chroot.cc env/env_encryption.cc - env/env_hdfs.cc env/file_system.cc + env/file_system_tracer.cc + env/fs_remap.cc env/mock_env.cc + env/unique_id_gen.cc file/delete_scheduler.cc file/file_prefetch_buffer.cc file/file_util.cc file/filename.cc + file/line_file_reader.cc file/random_access_file_reader.cc file/read_write_util.cc file/readahead_raf.cc @@ -261,6 +321,8 @@ set(ROCKSDB_SOURCES memory/arena.cc memory/concurrent_arena.cc memory/jemalloc_nodump_allocator.cc + memory/memkind_kmem_allocator.cc + memory/memory_allocator.cc memtable/alloc_tracker.cc memtable/hash_linklist_rep.cc memtable/hash_skiplist_rep.cc @@ -281,19 +343,24 @@ set(ROCKSDB_SOURCES monitoring/thread_status_util.cc monitoring/thread_status_util_debug.cc options/cf_options.cc + options/configurable.cc + options/customizable.cc options/db_options.cc options/options.cc options/options_helper.cc options/options_parser.cc - options/options_sanity_check.cc + port/mmap.cc port/stack_trace.cc table/adaptive/adaptive_table_factory.cc + table/block_based/binary_search_index_reader.cc table/block_based/block.cc - table/block_based/block_based_filter_block.cc table/block_based/block_based_table_builder.cc table/block_based/block_based_table_factory.cc + table/block_based/block_based_table_iterator.cc table/block_based/block_based_table_reader.cc table/block_based/block_builder.cc + table/block_based/block_cache.cc + table/block_based/block_prefetcher.cc table/block_based/block_prefix_index.cc table/block_based/data_block_hash_index.cc table/block_based/data_block_footer.cc @@ -301,9 +368,14 @@ set(ROCKSDB_SOURCES table/block_based/filter_policy.cc table/block_based/flush_block_policy.cc table/block_based/full_filter_block.cc + table/block_based/hash_index_reader.cc table/block_based/index_builder.cc + table/block_based/index_reader_common.cc table/block_based/parsed_full_filter_block.cc table/block_based/partitioned_filter_block.cc + table/block_based/partitioned_index_iterator.cc + table/block_based/partitioned_index_reader.cc + table/block_based/reader_common.cc table/block_based/uncompression_dict_reader.cc table/block_fetcher.cc table/cuckoo/cuckoo_table_builder.cc @@ -313,6 +385,7 @@ set(ROCKSDB_SOURCES table/get_context.cc table/iterator.cc table/merging_iterator.cc + table/compaction_merging_iterator.cc table/meta_blocks.cc table/persistent_cache_helper.cc table/plain/plain_table_bloom.cc @@ -321,57 +394,80 @@ set(ROCKSDB_SOURCES table/plain/plain_table_index.cc table/plain/plain_table_key_coding.cc table/plain/plain_table_reader.cc + table/sst_file_dumper.cc table/sst_file_reader.cc table/sst_file_writer.cc + table/table_factory.cc table/table_properties.cc table/two_level_iterator.cc + table/unique_id.cc test_util/sync_point.cc test_util/sync_point_impl.cc test_util/testutil.cc test_util/transaction_test_util.cc tools/block_cache_analyzer/block_cache_trace_analyzer.cc tools/dump/db_dump_tool.cc + tools/io_tracer_parser_tool.cc tools/ldb_cmd.cc tools/ldb_tool.cc tools/sst_dump_tool.cc tools/trace_analyzer_tool.cc - trace_replay/trace_replay.cc trace_replay/block_cache_tracer.cc + trace_replay/io_tracer.cc + trace_replay/trace_record_handler.cc + trace_replay/trace_record_result.cc + trace_replay/trace_record.cc + trace_replay/trace_replay.cc + util/async_file_reader.cc + util/cleanable.cc util/coding.cc util/compaction_job_stats_impl.cc util/comparator.cc + util/compression.cc util/compression_context_cache.cc util/concurrent_task_limiter_impl.cc util/crc32c.cc + util/data_structure.cc util/dynamic_bloom.cc util/hash.cc util/murmurhash.cc util/random.cc util/rate_limiter.cc + util/ribbon_config.cc util/slice.cc util/file_checksum_helper.cc util/status.cc + util/stderr_logger.cc util/string_util.cc util/thread_local.cc util/threadpool_imp.cc util/xxhash.cc - utilities/backupable/backupable_db.cc + utilities/agg_merge/agg_merge.cc + utilities/backup/backup_engine.cc utilities/blob_db/blob_compaction_filter.cc utilities/blob_db/blob_db.cc utilities/blob_db/blob_db_impl.cc utilities/blob_db/blob_db_impl_filesnapshot.cc utilities/blob_db/blob_dump_tool.cc utilities/blob_db/blob_file.cc - utilities/blob_db/blob_log_reader.cc - utilities/blob_db/blob_log_writer.cc - utilities/blob_db/blob_log_format.cc + utilities/cache_dump_load.cc + utilities/cache_dump_load_impl.cc + utilities/cassandra/cassandra_compaction_filter.cc + utilities/cassandra/format.cc + utilities/cassandra/merge_operator.cc utilities/checkpoint/checkpoint_impl.cc + utilities/compaction_filters.cc utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc + utilities/counted_fs.cc utilities/debug.cc utilities/env_mirror.cc utilities/env_timed.cc + utilities/fault_injection_env.cc + utilities/fault_injection_fs.cc + utilities/fault_injection_secondary_cache.cc utilities/leveldb_options/leveldb_options.cc utilities/memory/memory_util.cc + utilities/merge_operators.cc utilities/merge_operators/bytesxor.cc utilities/merge_operators/max.cc utilities/merge_operators/put.cc @@ -391,6 +487,12 @@ set(ROCKSDB_SOURCES utilities/simulator_cache/sim_cache.cc utilities/table_properties_collectors/compact_on_deletion_collector.cc utilities/trace/file_trace_reader_writer.cc + utilities/trace/replayer_impl.cc + utilities/transactions/lock/lock_manager.cc + utilities/transactions/lock/point/point_lock_tracker.cc + utilities/transactions/lock/point/point_lock_manager.cc + utilities/transactions/lock/range/range_tree/range_tree_lock_manager.cc + utilities/transactions/lock/range/range_tree/range_tree_lock_tracker.cc utilities/transactions/optimistic_transaction_db_impl.cc utilities/transactions/optimistic_transaction.cc utilities/transactions/pessimistic_transaction.cc @@ -398,15 +500,27 @@ set(ROCKSDB_SOURCES utilities/transactions/snapshot_checker.cc utilities/transactions/transaction_base.cc utilities/transactions/transaction_db_mutex_impl.cc - utilities/transactions/transaction_lock_mgr.cc utilities/transactions/transaction_util.cc utilities/transactions/write_prepared_txn.cc utilities/transactions/write_prepared_txn_db.cc utilities/transactions/write_unprepared_txn.cc utilities/transactions/write_unprepared_txn_db.cc utilities/ttl/db_ttl_impl.cc + utilities/wal_filter.cc utilities/write_batch_with_index/write_batch_with_index.cc utilities/write_batch_with_index/write_batch_with_index_internal.cc + utilities/transactions/lock/range/range_tree/lib/locktree/concurrent_tree.cc + utilities/transactions/lock/range/range_tree/lib/locktree/keyrange.cc + utilities/transactions/lock/range/range_tree/lib/locktree/lock_request.cc + utilities/transactions/lock/range/range_tree/lib/locktree/locktree.cc + utilities/transactions/lock/range/range_tree/lib/locktree/manager.cc + utilities/transactions/lock/range/range_tree/lib/locktree/range_buffer.cc + utilities/transactions/lock/range/range_tree/lib/locktree/treenode.cc + utilities/transactions/lock/range/range_tree/lib/locktree/txnid_set.cc + utilities/transactions/lock/range/range_tree/lib/locktree/wfg.cc + utilities/transactions/lock/range/range_tree/lib/standalone_port.cc + utilities/transactions/lock/range/range_tree/lib/util/dbt.cc + utilities/transactions/lock/range/range_tree/lib/util/memarena.cc ) @@ -426,14 +540,14 @@ else() env/io_posix.cc env/fs_posix.cc) # ppc64 or ppc64le - if(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64") + if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64") enable_language(ASM) list(APPEND ROCKSDB_SOURCES util/crc32c_ppc.c util/crc32c_ppc_asm.S) - endif(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64") + endif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64") # aarch - if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64") + if(CMAKE_SYSTEM_PROCESSOR MATCHES "arm64|aarch64|AARCH64") INCLUDE(CheckCXXCompilerFlag) CHECK_CXX_COMPILER_FLAG("-march=armv8-a+crc+crypto" HAS_ARMV8_CRC) if(HAS_ARMV8_CRC) @@ -442,7 +556,7 @@ else() list(APPEND ROCKSDB_SOURCES util/crc32c_arm64.cc) endif(HAS_ARMV8_CRC) - endif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64") + endif(CMAKE_SYSTEM_PROCESSOR MATCHES "arm64|aarch64|AARCH64") endif() SET(SOURCES) FOREACH(s ${ROCKSDB_SOURCES}) @@ -459,7 +573,7 @@ if(MSVC) # Workaround Win8.1 SDK bug, that breaks /permissive- string(REPLACE "/permissive-" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") else() - set(CMAKE_REQUIRED_FLAGS "-msse4.2 -mpclmul ${CXX11_FLAGS}") + set(CMAKE_REQUIRED_FLAGS "-msse4.2 -mpclmul ${CXX17_FLAGS}") CHECK_CXX_SOURCE_COMPILES(" #include @@ -480,16 +594,26 @@ int main() { unset(CMAKE_REQUIRED_FLAGS) endif() -IF(CMAKE_VERSION VERSION_GREATER "2.8.10") - STRING(TIMESTAMP GIT_DATE_TIME "%Y-%m-%d %H:%M:%S") -ENDIF() +if(GIT_EXECUTABLE AND EXISTS "${ROCKSDB_SOURCE_DIR}/.git") + execute_process(WORKING_DIRECTORY "${ROCKSDB_SOURCE_DIR}" OUTPUT_VARIABLE GIT_SHA COMMAND "${GIT_EXECUTABLE}" rev-parse HEAD ) + execute_process(WORKING_DIRECTORY "${ROCKSDB_SOURCE_DIR}" RESULT_VARIABLE GIT_MOD COMMAND "${GIT_EXECUTABLE}" diff-index HEAD --quiet) + execute_process(WORKING_DIRECTORY "${ROCKSDB_SOURCE_DIR}" OUTPUT_VARIABLE GIT_DATE COMMAND "${GIT_EXECUTABLE}" log -1 --date=format:"%Y-%m-%d %T" --format="%ad") + execute_process(WORKING_DIRECTORY "${ROCKSDB_SOURCE_DIR}" OUTPUT_VARIABLE GIT_TAG RESULT_VARIABLE rv COMMAND "${GIT_EXECUTABLE}" symbolic-ref -q --short HEAD OUTPUT_STRIP_TRAILING_WHITESPACE) + if (rv AND NOT rv EQUAL 0) + execute_process(WORKING_DIRECTORY "${ROCKSDB_SOURCE_DIR}" OUTPUT_VARIABLE GIT_TAG COMMAND "${GIT_EXECUTABLE}" describe --tags --exact-match OUTPUT_STRIP_TRAILING_WHITESPACE) + endif() +else() + set(GIT_SHA 0) + set(GIT_MOD 1) +endif() +string(REGEX REPLACE "[^0-9a-fA-F]+" "" GIT_SHA "${GIT_SHA}") +string(REGEX REPLACE "[^0-9: /-]+" "" GIT_DATE "${GIT_DATE}") CONFIGURE_FILE(${ROCKSDB_SOURCE_DIR}/util/build_version.cc.in build_version.cc @ONLY) -INCLUDE_DIRECTORIES(${ROCKSDB_SOURCE_DIR}/util) list(APPEND SOURCES ${CMAKE_CURRENT_BINARY_DIR}/build_version.cc) ADD_CONVENIENCE_LIBRARY(rocksdblib ${SOURCES}) target_link_libraries(rocksdblib ${THIRDPARTY_LIBS} ${SYSTEM_LIBS}) IF(CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") - set_target_properties(rocksdblib PROPERTIES COMPILE_FLAGS "-fPIC -fno-builtin-memcmp -Wno-error") + set_target_properties(rocksdblib PROPERTIES COMPILE_FLAGS "-fPIC -fno-builtin-memcmp -Wno-error -Wno-missing-braces -Wno-strict-aliasing -Wno-invalid-offsetof") endif() diff --git a/storage/rocksdb/get_rocksdb_files.sh b/storage/rocksdb/get_rocksdb_files.sh deleted file mode 100755 index bd5128a8609..00000000000 --- a/storage/rocksdb/get_rocksdb_files.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -MKFILE=`mktemp` -# create and run a simple makefile -# include rocksdb make file relative to the path of this script -echo "include ./storage/rocksdb/rocksdb/src.mk -all: - @echo \$(LIB_SOURCES)" > $MKFILE -for f in `make --makefile $MKFILE` -do - echo ./rocksdb/$f -done -rm $MKFILE - -# create build_version.cc file. Only create one if it doesn't exists or if it is different -# this is so that we don't rebuild mysqld every time -bv=storage/rocksdb/rocksdb/util/build_version.cc -date=$(date +%F) -git_sha=$(pushd storage/rocksdb/rocksdb >/dev/null && git rev-parse HEAD 2>/dev/null && popd >/dev/null) -if [ ! -f $bv ] || [ -z $git_sha ] || [ ! `grep $git_sha $bv` ] -then -echo "#include \"build_version.h\" -const char* rocksdb_build_git_sha = -\"rocksdb_build_git_sha:$git_sha\"; -const char* rocksdb_build_git_date = -\"rocksdb_build_git_date:$date\"; -const char* rocksdb_build_compile_date = __DATE__;" > $bv -fi diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index 86300f1cf71..87f9f3aa9a3 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -78,7 +78,6 @@ #include "./event_listener.h" #include "./ha_rocksdb_proto.h" #include "./logger.h" -#include "./nosql_access.h" #include "./rdb_cf_manager.h" #include "./rdb_cf_options.h" #include "./rdb_converter.h" @@ -130,6 +129,7 @@ static st_io_stall_stats io_stall_stats; const std::string DEFAULT_CF_NAME("default"); const std::string DEFAULT_SYSTEM_CF_NAME("__system__"); const std::string PER_INDEX_CF_NAME("$per_index_cf"); +const std::string TRUNCATE_TABLE_PREFIX("#truncate_tmp#"); static std::vector rdb_indexes_to_recalc; @@ -231,10 +231,12 @@ void delete_table_version(rocksdb::WriteBatch *wb, const char *path); static rocksdb::CompactRangeOptions getCompactRangeOptions( - int concurrency = 0) { + int concurrency = 0, + rocksdb::BottommostLevelCompaction bottommost_level_compaction = + rocksdb::BottommostLevelCompaction::kForceOptimized) { rocksdb::CompactRangeOptions compact_range_options; compact_range_options.bottommost_level_compaction = - rocksdb::BottommostLevelCompaction::kForce; + bottommost_level_compaction; compact_range_options.exclusive_manual_compaction = false; if (concurrency > 0) { compact_range_options.max_subcompactions = concurrency; @@ -279,6 +281,8 @@ static Rdb_background_thread rdb_bg_thread; static Rdb_manual_compaction_thread rdb_mc_thread; +static Rdb_drop_index_thread rdb_drop_idx_thread; + // List of table names (using regex) that are exceptions to the strict // collation check requirement. Regex_list_handler *rdb_collation_exceptions; @@ -287,8 +291,11 @@ static const char **rdb_get_error_messages(int nr); static void rocksdb_flush_all_memtables() { const Rdb_cf_manager &cf_manager = rdb_get_cf_manager(); + + // RocksDB will fail the flush if the CF is deleted, + // but here we don't handle return status for (const auto &cf_handle : cf_manager.get_all_cf()) { - rdb->Flush(rocksdb::FlushOptions(), cf_handle); + rdb->Flush(rocksdb::FlushOptions(), cf_handle.get()); } } @@ -299,31 +306,35 @@ static void rocksdb_delete_column_family_stub( static int rocksdb_delete_column_family( THD *const /* thd */, struct st_mysql_sys_var *const /* var */, void *const /* var_ptr */, struct st_mysql_value *const value) { - // Return failure for now until the race condition between creating - // CF and deleting CF is resolved - return HA_EXIT_FAILURE; + DBUG_ASSERT(value != nullptr); char buff[STRING_BUFFER_USUAL_SIZE]; int len = sizeof(buff); - DBUG_ASSERT(value != nullptr); + const char *const cf = value->val_str(value, buff, &len); + if (cf == nullptr) return HA_EXIT_SUCCESS; - if (const char *const cf = value->val_str(value, buff, &len)) { - auto &cf_manager = rdb_get_cf_manager(); - auto ret = cf_manager.drop_cf(cf); - if (ret == HA_EXIT_SUCCESS) { - // NO_LINT_DEBUG - sql_print_information("RocksDB: Dropped column family: %s\n", cf); - } else { - // NO_LINT_DEBUG - sql_print_error("RocksDB: Failed to drop column family: %s, error: %d\n", - cf, ret); - } - - return ret; + std::string cf_name = std::string(cf); + // Forbid to remove these built-in CFs + if (cf_name == DEFAULT_SYSTEM_CF_NAME || cf_name == DEFAULT_CF_NAME || + cf_name.empty()) { + my_error(ER_CANT_DROP_CF, MYF(0), cf); + return HA_EXIT_FAILURE; } - return HA_EXIT_SUCCESS; + auto &cf_manager = rdb_get_cf_manager(); + int ret = 0; + + std::lock_guard dm_lock(dict_manager); + ret = cf_manager.drop_cf(&ddl_manager, &dict_manager, cf_name); + + if (ret == HA_EXIT_SUCCESS) { + rdb_drop_idx_thread.signal(); + } else { + my_error(ER_CANT_DROP_CF, MYF(0), cf); + } + + return ret; } /////////////////////////////////////////////////////////// @@ -452,7 +463,7 @@ static int rocksdb_force_flush_memtable_and_lzero_now( for (const auto &cf_handle : cf_manager.get_all_cf()) { for (i = 0; i < max_attempts; i++) { - rdb->GetColumnFamilyMetaData(cf_handle, &metadata); + rdb->GetColumnFamilyMetaData(cf_handle.get(), &metadata); cf_handle->GetDescriptor(&cf_descr); c_options.output_file_size_limit = cf_descr.options.target_file_size_base; @@ -467,20 +478,34 @@ static int rocksdb_force_flush_memtable_and_lzero_now( } rocksdb::Status s; - s = rdb->CompactFiles(c_options, cf_handle, file_names, 1); + s = rdb->CompactFiles(c_options, cf_handle.get(), file_names, 1); - // Due to a race, it's possible for CompactFiles to collide - // with auto compaction, causing an error to return - // regarding file not found. In that case, retry. - if (s.IsInvalidArgument()) { - continue; - } + if (!s.ok()) { + std::shared_ptr cfh = + cf_manager.get_cf(cf_handle->GetID()); - if (!s.ok() && !s.IsAborted()) { - rdb_handle_io_error(s, RDB_IO_ERROR_GENERAL); - return HA_EXIT_FAILURE; + // If the CF handle has been removed from cf_manager, it is not an + // error. We are done with this CF and proceed to the next CF. + if (!cfh) { + // NO_LINT_DEBUG + sql_print_information("cf %s has been dropped during CompactFiles.", + cf_handle->GetName().c_str()); + break; + } + + // Due to a race, it's possible for CompactFiles to collide + // with auto compaction, causing an error to return + // regarding file not found. In that case, retry. + if (s.IsInvalidArgument()) { + continue; + } + + if (!s.ok() && !s.IsAborted()) { + rdb_handle_io_error(s, RDB_IO_ERROR_GENERAL); + return HA_EXIT_FAILURE; + } + break; } - break; } if (i == max_attempts) { num_errors++; @@ -490,6 +515,23 @@ static int rocksdb_force_flush_memtable_and_lzero_now( return num_errors == 0 ? HA_EXIT_SUCCESS : HA_EXIT_FAILURE; } +static void rocksdb_cancel_manual_compactions_stub( + THD *const thd, struct st_mysql_sys_var *const var, void *const var_ptr, + const void *const save) {} + +static int rocksdb_cancel_manual_compactions( + THD *const thd, struct st_mysql_sys_var *const var, void *const var_ptr, + struct st_mysql_value *const value) { + rdb_mc_thread.cancel_all_pending_manual_compaction_requests(); + // NO_LINT_DEBUG + sql_print_information("RocksDB: Stopping all Manual Compactions."); + rdb->GetBaseDB()->DisableManualCompaction(); + // NO_LINT_DEBUG + sql_print_information("RocksDB: Enabling Manual Compactions."); + rdb->GetBaseDB()->EnableManualCompaction(); + return HA_EXIT_SUCCESS; +} + static void rocksdb_drop_index_wakeup_thread( my_core::THD *const thd MY_ATTRIBUTE((__unused__)), struct st_mysql_sys_var *const var MY_ATTRIBUTE((__unused__)), @@ -498,6 +540,7 @@ static void rocksdb_drop_index_wakeup_thread( static my_bool rocksdb_pause_background_work = 0; static mysql_mutex_t rdb_sysvars_mutex; static mysql_mutex_t rdb_block_cache_resize_mutex; +static mysql_mutex_t rdb_bottom_pri_background_compactions_resize_mutex; static void rocksdb_set_pause_background_work( my_core::THD *const, @@ -570,6 +613,10 @@ static void rocksdb_set_max_background_jobs(THD *thd, struct st_mysql_sys_var *const var, void *const var_ptr, const void *const save); +static void rocksdb_set_max_background_compactions(THD *thd, + struct st_mysql_sys_var *const var, + void *const var_ptr, + const void *const save); static void rocksdb_set_bytes_per_sync(THD *thd, struct st_mysql_sys_var *const var, void *const var_ptr, @@ -581,6 +628,10 @@ static void rocksdb_set_wal_bytes_per_sync(THD *thd, static int rocksdb_validate_set_block_cache_size( THD *thd, struct st_mysql_sys_var *const var, void *var_ptr, struct st_mysql_value *value); +static int rocksdb_validate_max_bottom_pri_background_compactions( + THD *thd MY_ATTRIBUTE((__unused__)), + struct st_mysql_sys_var *const var MY_ATTRIBUTE((__unused__)), + void *var_ptr, struct st_mysql_value *value); ////////////////////////////////////////////////////////////////////////////// // Options definitions ////////////////////////////////////////////////////////////////////////////// @@ -609,6 +660,7 @@ static my_bool rocksdb_force_compute_memtable_stats; static uint32_t rocksdb_force_compute_memtable_stats_cachetime; static my_bool rocksdb_debug_optimizer_no_zero_cardinality; static uint32_t rocksdb_wal_recovery_mode; +static my_bool rocksdb_track_and_verify_wals_in_manifest; static uint32_t rocksdb_stats_level; static uint32_t rocksdb_access_hint_on_compaction_start; static char *rocksdb_compact_cf_name; @@ -623,6 +675,7 @@ static char *rocksdb_strict_collation_exceptions; static my_bool rocksdb_collect_sst_properties = 1; static my_bool rocksdb_force_flush_memtable_now_var = 0; static my_bool rocksdb_force_flush_memtable_and_lzero_now_var = 0; +static my_bool rocksdb_cancel_manual_compactions_var = 0; static my_bool rocksdb_enable_ttl = 1; static my_bool rocksdb_enable_ttl_read_filtering = 1; static int rocksdb_debug_ttl_rec_ts = 0; @@ -637,8 +690,10 @@ static long long rocksdb_compaction_sequential_deletes_window = 0l; static long long rocksdb_compaction_sequential_deletes_file_size = 0l; static uint32_t rocksdb_validate_tables = 1; static char *rocksdb_datadir; +static uint32_t rocksdb_max_bottom_pri_background_compactions = 0; static uint32_t rocksdb_table_stats_sampling_pct; static my_bool rocksdb_enable_bulk_load_api = 1; +static my_bool rocksdb_enable_remove_orphaned_dropped_cfs = 1; static my_bool rocksdb_print_snapshot_conflict_queries = 0; static my_bool rocksdb_large_prefix = 0; static my_bool rocksdb_allow_to_start_after_corruption = 0; @@ -650,6 +705,7 @@ char *compression_types_val= const_cast(get_rocksdb_supported_compression_types()); static unsigned long rocksdb_write_policy = rocksdb::TxnDBWritePolicy::WRITE_COMMITTED; +ulong rocksdb_max_row_locks; #if 0 // MARIAROCKS_NOT_YET : read-free replication is not supported char *rocksdb_read_free_rpl_tables; @@ -669,13 +725,15 @@ static uint32_t rocksdb_debug_manual_compaction_delay = 0; static uint32_t rocksdb_max_manual_compactions = 0; static my_bool rocksdb_rollback_on_timeout = FALSE; static my_bool rocksdb_enable_insert_with_update_caching = TRUE; - +static my_bool rocksdb_skip_locks_if_skip_unique_check = FALSE; std::atomic rocksdb_row_lock_deadlocks(0); std::atomic rocksdb_row_lock_wait_timeouts(0); std::atomic rocksdb_snapshot_conflict_errors(0); std::atomic rocksdb_wal_group_syncs(0); std::atomic rocksdb_manual_compactions_processed(0); +std::atomic rocksdb_manual_compactions_cancelled(0); std::atomic rocksdb_manual_compactions_running(0); +std::atomic rocksdb_manual_compactions_pending(0); #ifndef DBUG_OFF std::atomic rocksdb_num_get_for_update_calls(0); #endif @@ -752,6 +810,7 @@ static std::unique_ptr rdb_init_rocksdb_db_options(void) { o->two_write_queues = true; o->manual_wal_flush = true; + o->enforce_single_del_contracts = false; return o; } @@ -790,6 +849,16 @@ static TYPELIB info_log_level_typelib = { array_elements(info_log_level_names) - 1, "info_log_level_typelib", info_log_level_names, nullptr}; +/* This enum needs to be kept up to date with rocksdb::BottommostLevelCompaction + */ +static const char *bottommost_level_compaction_names[] = { + "kSkip", "kIfHaveCompactionFilter", "kForce", "kForceOptimized", NullS}; + +static TYPELIB bottommost_level_compaction_typelib = { + array_elements(bottommost_level_compaction_names) - 1, + "bottommost_level_compaction_typelib", bottommost_level_compaction_names, + nullptr}; + static void rocksdb_set_rocksdb_info_log_level( THD *const thd, struct st_mysql_sys_var *const var, void *const var_ptr, const void *const save) { @@ -922,6 +991,7 @@ const int64 RDB_DEFAULT_BLOCK_CACHE_SIZE = 512 * 1024 * 1024; const int64 RDB_MIN_BLOCK_CACHE_SIZE = 1024; const int RDB_MAX_CHECKSUMS_PCT = 100; const ulong RDB_DEADLOCK_DETECT_DEPTH = 50; +const uint ROCKSDB_MAX_BOTTOM_PRI_BACKGROUND_COMPACTIONS = 64; // TODO: 0 means don't wait at all, and we don't support it yet? static MYSQL_THDVAR_ULONG(lock_wait_timeout, PLUGIN_VAR_RQCMDARG, @@ -974,6 +1044,21 @@ static MYSQL_SYSVAR_BOOL(enable_bulk_load_api, rocksdb_enable_bulk_load_api, "Enables using SstFileWriter for bulk loading", nullptr, nullptr, rocksdb_enable_bulk_load_api); +static MYSQL_SYSVAR_BOOL(enable_remove_orphaned_dropped_cfs, + rocksdb_enable_remove_orphaned_dropped_cfs, + PLUGIN_VAR_RQCMDARG, + "Enables removing dropped cfs from metadata if it " + "doesn't exist in cf manager", + nullptr, nullptr, + rocksdb_enable_remove_orphaned_dropped_cfs); + +static MYSQL_SYSVAR_BOOL( + enable_pipelined_write, + *reinterpret_cast(&rocksdb_db_options->enable_pipelined_write), + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + "DBOptions::enable_pipelined_write for RocksDB", nullptr, nullptr, + rocksdb_db_options->enable_pipelined_write); + static MYSQL_SYSVAR_STR(git_hash, rocksdb_git_hash, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, "Git revision of the RocksDB library used by MyRocks", @@ -1001,6 +1086,11 @@ static MYSQL_THDVAR_BOOL( " Blind delete is disabled if the table has secondary key", nullptr, nullptr, FALSE); +static MYSQL_THDVAR_BOOL( + enable_iterate_bounds, PLUGIN_VAR_OPCMDARG, + "Enable rocksdb iterator upper/lower bounds in read options.", nullptr, + nullptr, TRUE); + #if 0 // MARIAROCKS_NOT_YET : read-free replication is not supported static const char *DEFAULT_READ_FREE_RPL_TABLES = ".*"; @@ -1079,11 +1169,28 @@ static MYSQL_SYSVAR_ENUM( nullptr, nullptr, read_free_rpl_type::OFF, &read_free_rpl_typelib); #endif +static void rocksdb_set_max_bottom_pri_background_compactions_internal( + uint val) { + // Set lower priority for compactions + if (val > 0) { + // This creates background threads in rocksdb with BOTTOM priority pool. + // Compactions for bottommost level use threads in the BOTTOM pool, and + // the threads in the BOTTOM pool run with lower OS priority (19 in Linux). + rdb->GetEnv()->SetBackgroundThreads(val, rocksdb::Env::Priority::BOTTOM); + rdb->GetEnv()->LowerThreadPoolCPUPriority(rocksdb::Env::Priority::BOTTOM); + sql_print_information( + "Set %d compaction thread(s) with " + "lower scheduling priority.", + val); + } +} + static MYSQL_THDVAR_BOOL(skip_bloom_filter_on_read, PLUGIN_VAR_RQCMDARG, "Skip using bloom filter for reads", nullptr, nullptr, FALSE); -static MYSQL_THDVAR_ULONG(max_row_locks, PLUGIN_VAR_RQCMDARG, +static MYSQL_SYSVAR_ULONG(max_row_locks, rocksdb_max_row_locks, + PLUGIN_VAR_RQCMDARG, "Maximum number of locks a transaction can have", nullptr, nullptr, /*default*/ RDB_DEFAULT_MAX_ROW_LOCKS, @@ -1095,6 +1202,12 @@ static MYSQL_THDVAR_ULONGLONG( "Maximum size of write batch in bytes. 0 means no limit.", nullptr, nullptr, /* default */ 0, /* min */ 0, /* max */ SIZE_T_MAX, 1); +static MYSQL_THDVAR_ULONGLONG( + write_batch_flush_threshold, PLUGIN_VAR_RQCMDARG, + "Maximum size of write batch in bytes before flushing. Only valid if " + "rocksdb_write_policy is WRITE_UNPREPARED. 0 means no limit.", nullptr, + nullptr, /* default */ 0, /* min */ 0, /* max */ SIZE_T_MAX, 1); + static MYSQL_THDVAR_BOOL( lock_scanned_rows, PLUGIN_VAR_RQCMDARG, "Take and hold locks on rows that are scanned but not updated", nullptr, @@ -1144,6 +1257,15 @@ static MYSQL_THDVAR_INT( /* default rocksdb.dboption max_subcompactions */ 0, /* min */ 0, /* max */ 128, 0); +static MYSQL_THDVAR_ENUM( + manual_compaction_bottommost_level, PLUGIN_VAR_RQCMDARG, + "Option for bottommost level compaction during manual " + "compaction", + nullptr, nullptr, + /* default */ + (ulong)rocksdb::BottommostLevelCompaction::kForceOptimized, + &bottommost_level_compaction_typelib); + static MYSQL_SYSVAR_BOOL( create_if_missing, *reinterpret_cast(&rocksdb_db_options->create_if_missing), @@ -1239,18 +1361,25 @@ static MYSQL_THDVAR_INT( static MYSQL_SYSVAR_UINT( wal_recovery_mode, rocksdb_wal_recovery_mode, PLUGIN_VAR_RQCMDARG, - "DBOptions::wal_recovery_mode for RocksDB. Default is kAbsoluteConsistency", + "DBOptions::wal_recovery_mode for RocksDB. Default is kPointInTimeRecovery", nullptr, nullptr, - /* default */ (uint)rocksdb::WALRecoveryMode::kAbsoluteConsistency, + /* default */ (uint)rocksdb::WALRecoveryMode::kPointInTimeRecovery, /* min */ (uint)rocksdb::WALRecoveryMode::kTolerateCorruptedTailRecords, /* max */ (uint)rocksdb::WALRecoveryMode::kSkipAnyCorruptedRecords, 0); +static MYSQL_SYSVAR_BOOL( + track_and_verify_wals_in_manifest, + *reinterpret_cast(&rocksdb_track_and_verify_wals_in_manifest), + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + "DBOptions::track_and_verify_wals_in_manifest for RocksDB", nullptr, + nullptr, true); + static MYSQL_SYSVAR_UINT( stats_level, rocksdb_stats_level, PLUGIN_VAR_RQCMDARG, - "Statistics Level for RocksDB. Default is 0 (kExceptHistogramOrTimers)", + "Statistics Level for RocksDB. Default is 1 (kExceptHistogramOrTimers)", nullptr, rocksdb_set_rocksdb_stats_level, /* default */ (uint)rocksdb::StatsLevel::kExceptHistogramOrTimers, - /* min */ (uint)rocksdb::StatsLevel::kExceptHistogramOrTimers, + /* min */ (uint)rocksdb::StatsLevel::kExceptTickers, /* max */ (uint)rocksdb::StatsLevel::kAll, 0); static MYSQL_SYSVAR_SIZE_T(compaction_readahead_size, @@ -1261,13 +1390,16 @@ static MYSQL_SYSVAR_SIZE_T(compaction_readahead_size, rocksdb_db_options->compaction_readahead_size, /* min */ 0L, /* max */ SIZE_T_MAX, 0); +// Placeholder for deprecated system variable +bool rocksdb_new_table_reader_for_compaction_inputs = false; static MYSQL_SYSVAR_BOOL( new_table_reader_for_compaction_inputs, *reinterpret_cast( - &rocksdb_db_options->new_table_reader_for_compaction_inputs), + &rocksdb_new_table_reader_for_compaction_inputs), PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "DBOptions::new_table_reader_for_compaction_inputs for RocksDB", nullptr, - nullptr, rocksdb_db_options->new_table_reader_for_compaction_inputs); + "DBOptions::new_table_reader_for_compaction_inputs for RocksDB. " + "This variable is deprecated and removed from RocksDB.", + nullptr, nullptr, rocksdb_new_table_reader_for_compaction_inputs); // since 11.2.0 static MYSQL_SYSVAR_UINT( access_hint_on_compaction_start, rocksdb_access_hint_on_compaction_start, @@ -1352,6 +1484,42 @@ static MYSQL_SYSVAR_INT(max_background_jobs, rocksdb_db_options->max_background_jobs, /* min */ -1, /* max */ MAX_BACKGROUND_JOBS, 0); +static MYSQL_SYSVAR_INT(max_background_flushes, + rocksdb_db_options->max_background_flushes, + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + "DBOptions::max_background_flushes for RocksDB", nullptr, + nullptr, + rocksdb_db_options->max_background_flushes, + /* min */ -1, /* max */ 64, 0); + +static MYSQL_SYSVAR_INT(max_background_compactions, + rocksdb_db_options->max_background_compactions, + PLUGIN_VAR_RQCMDARG, + "DBOptions::max_background_compactions for RocksDB", nullptr, + rocksdb_set_max_background_compactions, + rocksdb_db_options->max_background_compactions, + /* min */ -1, /* max */ 64, 0); + +static MYSQL_SYSVAR_UINT( + max_bottom_pri_background_compactions, + rocksdb_max_bottom_pri_background_compactions, PLUGIN_VAR_RQCMDARG, + "Creating specified number of threads, setting lower " + "CPU priority, and letting Lmax compactions use them. " + "Maximum total compaction concurrency continues to be capped to " + "rocksdb_max_background_compactions or " + "rocksdb_max_background_jobs. In addition to that, Lmax " + "compaction concurrency is capped to " + "rocksdb_max_bottom_pri_background_compactions. Default value is 0, " + "which means all compactions are under concurrency of " + "rocksdb_max_background_compactions|jobs. If you set very low " + "rocksdb_max_bottom_pri_background_compactions (e.g. 1 or 2), compactions " + "may not be able to keep up. Since Lmax normally has " + "90 percent of data, it is recommended to set closer number to " + "rocksdb_max_background_compactions|jobs. This option is helpful to " + "give more CPU resources to other threads (e.g. query processing).", + rocksdb_validate_max_bottom_pri_background_compactions, nullptr, 0, + /* min */ 0, /* max */ ROCKSDB_MAX_BOTTOM_PRI_BACKGROUND_COMPACTIONS, 0); + static MYSQL_SYSVAR_UINT(max_subcompactions, rocksdb_db_options->max_subcompactions, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, @@ -1581,13 +1749,16 @@ static MYSQL_SYSVAR_ENUM(index_type, rocksdb_index_type, (ulong)rocksdb_tbl_options->index_type, &index_type_typelib); +// Placeholder for deprecated system variable +bool rocksdb_hash_index_allow_collision = true; static MYSQL_SYSVAR_BOOL( hash_index_allow_collision, *reinterpret_cast( - &rocksdb_tbl_options->hash_index_allow_collision), + &rocksdb_hash_index_allow_collision), PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "BlockBasedTableOptions::hash_index_allow_collision for RocksDB", nullptr, - nullptr, rocksdb_tbl_options->hash_index_allow_collision); + "BlockBasedTableOptions::hash_index_allow_collision for RocksDB. " + "This variable is deprecated and will be removed in a future release.", + nullptr, nullptr, rocksdb_hash_index_allow_collision); // since 11.2.0 static MYSQL_SYSVAR_BOOL( no_block_cache, @@ -1596,11 +1767,12 @@ static MYSQL_SYSVAR_BOOL( "BlockBasedTableOptions::no_block_cache for RocksDB", nullptr, nullptr, rocksdb_tbl_options->no_block_cache); -static MYSQL_SYSVAR_SIZE_T(block_size, rocksdb_tbl_options->block_size, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "BlockBasedTableOptions::block_size for RocksDB", - nullptr, nullptr, rocksdb_tbl_options->block_size, - /* min */ 1L, /* max */ SIZE_T_MAX, 0); +static MYSQL_SYSVAR_UINT64_T( + block_size, rocksdb_tbl_options->block_size, + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + "BlockBasedTableOptions::block_size for RocksDB", + nullptr, nullptr, rocksdb_tbl_options->block_size, + /* min */ 1L, /* max */ UINT64_MAX, 0); static MYSQL_SYSVAR_INT( block_size_deviation, rocksdb_tbl_options->block_size_deviation, @@ -1849,6 +2021,13 @@ static MYSQL_SYSVAR_BOOL( rocksdb_force_flush_memtable_and_lzero_now, rocksdb_force_flush_memtable_and_lzero_now_stub, FALSE); +static MYSQL_SYSVAR_BOOL(cancel_manual_compactions, + rocksdb_cancel_manual_compactions_var, + PLUGIN_VAR_RQCMDARG, + "Cancelling all ongoing manual compactions.", + rocksdb_cancel_manual_compactions, + rocksdb_cancel_manual_compactions_stub, FALSE); + static MYSQL_SYSVAR_UINT( seconds_between_stat_computes, rocksdb_seconds_between_stat_computes, PLUGIN_VAR_RQCMDARG, @@ -2002,6 +2181,12 @@ static MYSQL_SYSVAR_BOOL( "insertion attempt in INSERT ON DUPLICATE KEY UPDATE", nullptr, nullptr, TRUE); +static MYSQL_SYSVAR_BOOL(skip_locks_if_skip_unique_check, + rocksdb_skip_locks_if_skip_unique_check, + PLUGIN_VAR_RQCMDARG, + "Skip row locking when unique checks are disabled.", + nullptr, nullptr, FALSE); + static const int ROCKSDB_ASSUMED_KEY_VALUE_DISK_SIZE = 100; static struct st_mysql_sys_var *rocksdb_system_variables[] = { @@ -2011,6 +2196,7 @@ static struct st_mysql_sys_var *rocksdb_system_variables[] = { MYSQL_SYSVAR(commit_time_batch_for_recovery), MYSQL_SYSVAR(max_row_locks), MYSQL_SYSVAR(write_batch_max_bytes), + MYSQL_SYSVAR(write_batch_flush_threshold), MYSQL_SYSVAR(lock_scanned_rows), MYSQL_SYSVAR(bulk_load), MYSQL_SYSVAR(bulk_load_allow_sk), @@ -2019,6 +2205,7 @@ static struct st_mysql_sys_var *rocksdb_system_variables[] = { MYSQL_SYSVAR(trace_sst_api), MYSQL_SYSVAR(commit_in_the_middle), MYSQL_SYSVAR(blind_delete_primary_key), + MYSQL_SYSVAR(enable_iterate_bounds), #if 0 // MARIAROCKS_NOT_YET : read-free replication is not supported MYSQL_SYSVAR(read_free_rpl_tables), MYSQL_SYSVAR(read_free_rpl), @@ -2026,6 +2213,8 @@ static struct st_mysql_sys_var *rocksdb_system_variables[] = { MYSQL_SYSVAR(bulk_load_size), MYSQL_SYSVAR(merge_buf_size), MYSQL_SYSVAR(enable_bulk_load_api), + MYSQL_SYSVAR(enable_pipelined_write), + MYSQL_SYSVAR(enable_remove_orphaned_dropped_cfs), MYSQL_SYSVAR(tmpdir), MYSQL_SYSVAR(merge_combine_read_size), MYSQL_SYSVAR(merge_tmp_file_removal_delay_ms), @@ -2052,6 +2241,9 @@ static struct st_mysql_sys_var *rocksdb_system_variables[] = { MYSQL_SYSVAR(persistent_cache_size_mb), MYSQL_SYSVAR(delete_obsolete_files_period_micros), MYSQL_SYSVAR(max_background_jobs), + MYSQL_SYSVAR(max_background_flushes), + MYSQL_SYSVAR(max_background_compactions), + MYSQL_SYSVAR(max_bottom_pri_background_compactions), MYSQL_SYSVAR(max_log_file_size), MYSQL_SYSVAR(max_subcompactions), MYSQL_SYSVAR(log_file_time_to_roll), @@ -2075,6 +2267,7 @@ static struct st_mysql_sys_var *rocksdb_system_variables[] = { MYSQL_SYSVAR(enable_thread_tracking), MYSQL_SYSVAR(perf_context_level), MYSQL_SYSVAR(wal_recovery_mode), + MYSQL_SYSVAR(track_and_verify_wals_in_manifest), MYSQL_SYSVAR(stats_level), MYSQL_SYSVAR(access_hint_on_compaction_start), MYSQL_SYSVAR(new_table_reader_for_compaction_inputs), @@ -2127,6 +2320,7 @@ static struct st_mysql_sys_var *rocksdb_system_variables[] = { MYSQL_SYSVAR(collect_sst_properties), MYSQL_SYSVAR(force_flush_memtable_now), MYSQL_SYSVAR(force_flush_memtable_and_lzero_now), + MYSQL_SYSVAR(cancel_manual_compactions), MYSQL_SYSVAR(enable_ttl), MYSQL_SYSVAR(enable_ttl_read_filtering), MYSQL_SYSVAR(debug_ttl_rec_ts), @@ -2163,11 +2357,12 @@ static struct st_mysql_sys_var *rocksdb_system_variables[] = { MYSQL_SYSVAR(debug_manual_compaction_delay), MYSQL_SYSVAR(max_manual_compactions), MYSQL_SYSVAR(manual_compaction_threads), + MYSQL_SYSVAR(manual_compaction_bottommost_level), MYSQL_SYSVAR(rollback_on_timeout), MYSQL_SYSVAR(enable_insert_with_update_caching), - MYSQL_SYSVAR(ignore_datadic_errors), + MYSQL_SYSVAR(skip_locks_if_skip_unique_check), nullptr}; static rocksdb::WriteOptions rdb_get_rocksdb_write_options( @@ -2192,10 +2387,26 @@ static int rocksdb_compact_column_family(THD *const thd, DBUG_ASSERT(value != nullptr); if (const char *const cf = value->val_str(value, buff, &len)) { - auto cfh = cf_manager.get_cf(cf); + DBUG_EXECUTE_IF("rocksdb_compact_column_family", { + const char act[] = + "now signal ready_to_mark_cf_dropped_in_compact_column_family " + "wait_for mark_cf_dropped_done_in_compact_column_family"; + DBUG_ASSERT(!debug_sync_set_action(thd, STRING_WITH_LEN(act))); + }); + + std::string cf_name = std::string(cf); + // use rocksdb_compact_cf="" or "default" to compact default CF + if (cf_name.empty()) cf_name = DEFAULT_CF_NAME; + + auto cfh = cf_manager.get_cf(cf_name); if (cfh != nullptr && rdb != nullptr) { + rocksdb::BottommostLevelCompaction bottommost_level_compaction = + (rocksdb::BottommostLevelCompaction)THDVAR( + thd, manual_compaction_bottommost_level); + int mc_id = rdb_mc_thread.request_manual_compaction( - cfh, nullptr, nullptr, THDVAR(thd, manual_compaction_threads)); + cfh, nullptr, nullptr, THDVAR(thd, manual_compaction_threads), + bottommost_level_compaction); if (mc_id == -1) { my_error(ER_INTERNAL_ERROR, MYF(0), "Can't schedule more manual compactions. " @@ -2210,30 +2421,82 @@ static int rocksdb_compact_column_family(THD *const thd, cf); // Checking thd state every short cycle (100ms). This is for allowing to // exiting this function without waiting for CompactRange to finish. + Rdb_manual_compaction_thread::Manual_compaction_request::mc_state + mc_status; do { my_sleep(100000); + mc_status = rdb_mc_thread.manual_compaction_state(mc_id); } while (!thd->killed && - !rdb_mc_thread.is_manual_compaction_finished(mc_id)); + (mc_status == Rdb_manual_compaction_thread:: + Manual_compaction_request::PENDING || + mc_status == Rdb_manual_compaction_thread:: + Manual_compaction_request::RUNNING)); + bool mc_timeout = false; if (thd->killed) { - // This cancels if requested compaction state is INITED. - // TODO(yoshinorim): Cancel running compaction as well once - // it is supported in RocksDB. - rdb_mc_thread.clear_manual_compaction_request(mc_id, true); + // Cancelling pending or running manual compaction with 60s timeout + mc_timeout = rdb_mc_thread.cancel_manual_compaction_request(mc_id, 600); } + + mc_status = rdb_mc_thread.manual_compaction_state(mc_id); + if (mc_status != + Rdb_manual_compaction_thread::Manual_compaction_request::SUCCESS) { + std::string msg = "Manual Compaction Failed. Reason: "; + if (thd->killed) { + msg += "Cancelled by client."; + } else if (mc_status == Rdb_manual_compaction_thread:: + Manual_compaction_request::CANCEL) { + msg += "Cancelled by server."; + } else { + msg += "General failures."; + } + if (mc_timeout) { + msg += " (timeout)"; + } + my_error(ER_INTERNAL_ERROR, MYF(0), msg.c_str()); + rdb_mc_thread.set_client_done(mc_id); + return HA_EXIT_FAILURE; + } + // manual compaction succeeded. + rdb_mc_thread.set_client_done(mc_id); } } return HA_EXIT_SUCCESS; } +/* + * Serializes an xid to a string so that it can + * be used as a rocksdb transaction name + */ +static std::string rdb_xid_to_string(const XID &src) { + DBUG_ASSERT(src.gtrid_length >= 0 && src.gtrid_length <= MAXGTRIDSIZE); + DBUG_ASSERT(src.bqual_length >= 0 && src.bqual_length <= MAXBQUALSIZE); + + std::string buf; + buf.reserve(RDB_XIDHDR_LEN + src.gtrid_length + src.bqual_length); + + /* + * expand formatID to fill 8 bytes if it doesn't already + * then reinterpret bit pattern as unsigned and store in network order + */ + uchar fidbuf[RDB_FORMATID_SZ]; + int64 signed_fid8 = src.formatID; + const uint64 raw_fid8 = *reinterpret_cast(&signed_fid8); + rdb_netbuf_store_uint64(fidbuf, raw_fid8); + buf.append(reinterpret_cast(fidbuf), RDB_FORMATID_SZ); + + buf.push_back(src.gtrid_length); + buf.push_back(src.bqual_length); + buf.append(src.data, (src.gtrid_length) + (src.bqual_length)); + return buf; +} + /////////////////////////////////////////////////////////////////////////////////////////// /* Drop index thread's control */ -static Rdb_drop_index_thread rdb_drop_idx_thread; - static void rocksdb_drop_index_wakeup_thread( my_core::THD *const thd MY_ATTRIBUTE((__unused__)), struct st_mysql_sys_var *const var MY_ATTRIBUTE((__unused__)), @@ -2325,7 +2588,8 @@ class Rdb_transaction { ulonglong m_insert_count = 0; ulonglong m_update_count = 0; ulonglong m_delete_count = 0; - ulonglong m_lock_count = 0; + // per row data + ulonglong m_row_lock_count = 0; std::unordered_map m_auto_incr_map; bool m_is_delayed_snapshot = false; @@ -2376,6 +2640,7 @@ protected: They should provide RocksDB's savepoint semantics. */ virtual void do_set_savepoint() = 0; + virtual rocksdb::Status do_pop_savepoint() = 0; virtual void do_rollback_to_savepoint() = 0; /* @@ -2573,22 +2838,26 @@ protected: ulonglong get_delete_count() const { return m_delete_count; } + ulonglong get_row_lock_count() const { return m_row_lock_count; } + void incr_insert_count() { ++m_insert_count; } void incr_update_count() { ++m_update_count; } void incr_delete_count() { ++m_delete_count; } - int get_timeout_sec() const { return m_timeout_sec; } + void incr_row_lock_count() { ++m_row_lock_count; } - ulonglong get_lock_count() const { return m_lock_count; } + ulonglong get_max_row_lock_count() const { return m_max_row_locks; } + + int get_timeout_sec() const { return m_timeout_sec; } virtual void set_sync(bool sync) = 0; - virtual void release_lock(rocksdb::ColumnFamilyHandle *const column_family, + virtual void release_lock(const Rdb_key_def &key_descr, const std::string &rowkey) = 0; - virtual bool prepare(const rocksdb::TransactionName &name) = 0; + virtual bool prepare() = 0; bool commit_or_rollback() { bool res; @@ -2988,10 +3257,11 @@ protected: virtual rocksdb::Status get(rocksdb::ColumnFamilyHandle *const column_family, const rocksdb::Slice &key, rocksdb::PinnableSlice *const value) const = 0; - virtual rocksdb::Status get_for_update( - rocksdb::ColumnFamilyHandle *const column_family, - const rocksdb::Slice &key, rocksdb::PinnableSlice *const value, - bool exclusive, const bool do_validate) = 0; + virtual rocksdb::Status get_for_update(const Rdb_key_def &key_descr, + const rocksdb::Slice &key, + rocksdb::PinnableSlice *const value, + bool exclusive, + const bool do_validate) = 0; rocksdb::Iterator *get_iterator( rocksdb::ColumnFamilyHandle *const column_family, bool skip_bloom_filter, @@ -3009,9 +3279,12 @@ protected: rocksdb::ReadOptions options = m_read_opts; if (skip_bloom_filter) { + const bool enable_iterate_bounds = THDVAR(get_thd(), enable_iterate_bounds); options.total_order_seek = true; - options.iterate_lower_bound = &eq_cond_lower_bound; - options.iterate_upper_bound = &eq_cond_upper_bound; + options.iterate_lower_bound = + enable_iterate_bounds ? &eq_cond_lower_bound : nullptr; + options.iterate_upper_bound = + enable_iterate_bounds ? &eq_cond_upper_bound : nullptr; } else { // With this option, Iterator::Valid() returns false if key // is outside of the prefix bloom filter range set at Seek(). @@ -3028,6 +3301,7 @@ protected: virtual bool is_tx_started() const = 0; virtual void start_tx() = 0; virtual void start_stmt() = 0; + virtual void set_name() = 0; protected: // Non-virtual functions with actions to be done on transaction start and @@ -3068,9 +3342,8 @@ protected: // one. This is very important for long transactions doing lots of // SELECTs. if (m_writes_at_last_savepoint != m_write_count) { - rocksdb::WriteBatchBase *batch = get_write_batch(); rocksdb::Status status = rocksdb::Status::NotFound(); - while ((status = batch->PopSavePoint()) == rocksdb::Status::OK()) { + while ((status = do_pop_savepoint()) == rocksdb::Status::OK()) { } if (status != rocksdb::Status::NotFound()) { @@ -3135,22 +3408,50 @@ protected: void set_tx_read_only(bool val) { m_tx_read_only = val; } - explicit Rdb_transaction(THD *const thd) - : m_thd(thd), m_tbl_io_perf(nullptr) { + /* + Add or remove from the global list of active transactions + needed by information_schema queries. + */ + void add_to_global_trx_list() { RDB_MUTEX_LOCK_CHECK(s_tx_list_mutex); s_tx_list.insert(this); RDB_MUTEX_UNLOCK_CHECK(s_tx_list_mutex); } - virtual ~Rdb_transaction() { + void remove_from_global_trx_list() { RDB_MUTEX_LOCK_CHECK(s_tx_list_mutex); s_tx_list.erase(this); RDB_MUTEX_UNLOCK_CHECK(s_tx_list_mutex); } + + explicit Rdb_transaction(THD *const thd) + : m_thd(thd), m_tbl_io_perf(nullptr) {} + + virtual ~Rdb_transaction() { +#ifndef DEBUG_OFF + RDB_MUTEX_LOCK_CHECK(s_tx_list_mutex); + DBUG_ASSERT(s_tx_list.find(this) == s_tx_list.end()); + RDB_MUTEX_UNLOCK_CHECK(s_tx_list_mutex); +#endif + } + virtual bool is_prepared() { return false; }; virtual void detach_prepared_tx() {}; }; +#ifndef DBUG_OFF +// simulate that RocksDB has reported corrupted data +static void dbug_change_status_to_corrupted(rocksdb::Status *status) { + *status = rocksdb::Status::Corruption(); +} +static void dbug_change_status_to_io_error(rocksdb::Status *status) { + *status = rocksdb::Status::IOError(); +} +static void dbug_change_status_to_incomplete(rocksdb::Status *status) { + *status = rocksdb::Status::Incomplete(); +} +#endif + /* This is a rocksdb transaction. Its members represent the current transaction, which consists of: @@ -3176,10 +3477,18 @@ class Rdb_transaction_impl : public Rdb_transaction { m_rocksdb_tx->GetWriteOptions()->sync = sync; } - void release_lock(rocksdb::ColumnFamilyHandle *const column_family, + void release_lock(const Rdb_key_def &key_descr, const std::string &rowkey) override { if (!THDVAR(m_thd, lock_scanned_rows)) { - m_rocksdb_tx->UndoGetForUpdate(column_family, rocksdb::Slice(rowkey)); + m_rocksdb_tx->UndoGetForUpdate(key_descr.get_cf(), + rocksdb::Slice(rowkey)); + // row_lock_count track row(pk) + DBUG_ASSERT(!key_descr.is_primary_key() || + (key_descr.is_primary_key() && m_row_lock_count > 0)); + // m_row_lock_count tracks per row data instead of per key data + if (key_descr.is_primary_key() && m_row_lock_count > 0) { + m_row_lock_count--; + } } } @@ -3203,23 +3512,27 @@ private: m_rocksdb_tx = nullptr; } - bool prepare(const rocksdb::TransactionName &name) override { + bool prepare() override { rocksdb::Status s; - s = m_rocksdb_tx->SetName(name); - if (!s.ok()) { - rdb_handle_io_error(s, RDB_IO_ERROR_TX_COMMIT); - return false; - } s = merge_auto_incr_map(m_rocksdb_tx->GetWriteBatch()->GetWriteBatch()); +#ifndef DBUG_OFF + DBUG_EXECUTE_IF("myrocks_prepare_io_error", + dbug_change_status_to_io_error(&s);); + DBUG_EXECUTE_IF("myrocks_prepare_incomplete", + dbug_change_status_to_incomplete(&s);); +#endif if (!s.ok()) { - rdb_handle_io_error(s, RDB_IO_ERROR_TX_COMMIT); + std::string msg = + "RocksDB error on COMMIT (Prepare/merge): " + s.ToString(); + my_error(ER_INTERNAL_ERROR, MYF(0), msg.c_str()); return false; } s = m_rocksdb_tx->Prepare(); if (!s.ok()) { - rdb_handle_io_error(s, RDB_IO_ERROR_TX_COMMIT); + std::string msg = "RocksDB error on COMMIT (Prepare): " + s.ToString(); + my_error(ER_INTERNAL_ERROR, MYF(0), msg.c_str()); return false; } return true; @@ -3230,6 +3543,12 @@ private: rocksdb::Status s; s = merge_auto_incr_map(m_rocksdb_tx->GetWriteBatch()->GetWriteBatch()); +#ifndef DBUG_OFF + DBUG_EXECUTE_IF("myrocks_commit_merge_io_error", + dbug_change_status_to_io_error(&s);); + DBUG_EXECUTE_IF("myrocks_commit_merge_incomplete", + dbug_change_status_to_incomplete(&s);); +#endif if (!s.ok()) { rdb_handle_io_error(s, RDB_IO_ERROR_TX_COMMIT); res = true; @@ -3238,6 +3557,12 @@ private: release_snapshot(); s = m_rocksdb_tx->Commit(); +#ifndef DBUG_OFF + DBUG_EXECUTE_IF("myrocks_commit_io_error", + dbug_change_status_to_io_error(&s);); + DBUG_EXECUTE_IF("myrocks_commit_incomplete", + dbug_change_status_to_incomplete(&s);); +#endif if (!s.ok()) { rdb_handle_io_error(s, RDB_IO_ERROR_TX_COMMIT); res = true; @@ -3254,7 +3579,7 @@ private: m_insert_count = 0; m_update_count = 0; m_delete_count = 0; - m_lock_count = 0; + m_row_lock_count = 0; set_tx_read_only(false); m_rollback_only = false; return res; @@ -3267,7 +3592,7 @@ private: m_insert_count = 0; m_update_count = 0; m_delete_count = 0; - m_lock_count = 0; + m_row_lock_count = 0; m_auto_incr_map.clear(); m_ddl_transaction = false; if (m_rocksdb_tx) { @@ -3337,10 +3662,6 @@ private: const rocksdb::Slice &key, const rocksdb::Slice &value, const bool assume_tracked) override { ++m_write_count; - ++m_lock_count; - if (m_write_count > m_max_row_locks || m_lock_count > m_max_row_locks) { - return rocksdb::Status::Aborted(rocksdb::Status::kLockLimit); - } return m_rocksdb_tx->Put(column_family, key, value, assume_tracked); } @@ -3348,10 +3669,6 @@ private: const rocksdb::Slice &key, const bool assume_tracked) override { ++m_write_count; - ++m_lock_count; - if (m_write_count > m_max_row_locks || m_lock_count > m_max_row_locks) { - return rocksdb::Status::Aborted(rocksdb::Status::kLockLimit); - } return m_rocksdb_tx->Delete(column_family, key, assume_tracked); } @@ -3359,10 +3676,6 @@ private: rocksdb::ColumnFamilyHandle *const column_family, const rocksdb::Slice &key, const bool assume_tracked) override { ++m_write_count; - ++m_lock_count; - if (m_write_count > m_max_row_locks || m_lock_count > m_max_row_locks) { - return rocksdb::Status::Aborted(rocksdb::Status::kLockLimit); - } return m_rocksdb_tx->SingleDelete(column_family, key, assume_tracked); } @@ -3399,11 +3712,14 @@ private: return m_rocksdb_tx->Get(m_read_opts, column_family, key, value); } - rocksdb::Status get_for_update( - rocksdb::ColumnFamilyHandle *const column_family, - const rocksdb::Slice &key, rocksdb::PinnableSlice *const value, - bool exclusive, const bool do_validate) override { - if (++m_lock_count > m_max_row_locks) { + rocksdb::Status get_for_update(const Rdb_key_def &key_descr, + const rocksdb::Slice &key, + rocksdb::PinnableSlice *const value, + bool exclusive, + const bool do_validate) override { + rocksdb::ColumnFamilyHandle *const column_family = key_descr.get_cf(); + /* check row lock limit in a trx */ + if (get_row_lock_count() >= get_max_row_lock_count()) { return rocksdb::Status::Aborted(rocksdb::Status::kLockLimit); } @@ -3426,6 +3742,8 @@ private: exclusive, false); m_read_opts.snapshot = saved_snapshot; } + // row_lock_count is to track per row instead of per key + if (key_descr.is_primary_key()) incr_row_lock_count(); return s; } @@ -3452,6 +3770,8 @@ private: tx_opts.use_only_the_last_commit_time_batch_for_recovery = THDVAR(m_thd, commit_time_batch_for_recovery); tx_opts.max_write_batch_size = THDVAR(m_thd, write_batch_max_bytes); + tx_opts.write_batch_flush_threshold = + THDVAR(m_thd, write_batch_flush_threshold); write_opts.sync = (rocksdb_flush_log_at_trx_commit == FLUSH_LOG_SYNC); write_opts.disableWAL = THDVAR(m_thd, write_disable_wal); @@ -3476,10 +3796,27 @@ private: m_ddl_transaction = false; } + void set_name() override { + XID xid; + thd_get_xid(m_thd, reinterpret_cast(&xid)); + auto name = m_rocksdb_tx->GetName(); + if (!name.empty()) { + DBUG_ASSERT(name == rdb_xid_to_string(xid)); + return; + } + rocksdb::Status s = m_rocksdb_tx->SetName(rdb_xid_to_string(xid)); + DBUG_ASSERT(s.ok()); + if (!s.ok()) { + rdb_handle_io_error(s, RDB_IO_ERROR_TX_COMMIT); + } + } + /* Implementations of do_*savepoint based on rocksdB::Transaction savepoints */ void do_set_savepoint() override { m_rocksdb_tx->SetSavePoint(); } + rocksdb::Status do_pop_savepoint() override { return m_rocksdb_tx->PopSavePoint(); } + void do_rollback_to_savepoint() override { m_rocksdb_tx->RollbackToSavePoint(); } @@ -3529,6 +3866,10 @@ private: } virtual ~Rdb_transaction_impl() override { + // Remove from the global list before all other processing is started. + // Otherwise, information_schema.rocksdb_trx can crash on this object. + Rdb_transaction::remove_from_global_trx_list(); + rollback(); // Theoretically the notifier could outlive the Rdb_transaction_impl @@ -3561,7 +3902,7 @@ class Rdb_writebatch_impl : public Rdb_transaction { } private: - bool prepare(const rocksdb::TransactionName &name) override { return true; } + bool prepare() override { return true; } bool commit_no_binlog() override { bool res = false; @@ -3601,6 +3942,8 @@ class Rdb_writebatch_impl : public Rdb_transaction { /* Implementations of do_*savepoint based on rocksdB::WriteBatch savepoints */ void do_set_savepoint() override { m_batch->SetSavePoint(); } + rocksdb::Status do_pop_savepoint() override { return m_batch->PopSavePoint(); } + void do_rollback_to_savepoint() override { m_batch->RollbackToSavePoint(); } @@ -3613,7 +3956,7 @@ class Rdb_writebatch_impl : public Rdb_transaction { void set_sync(bool sync) override { write_opts.sync = sync; } - void release_lock(rocksdb::ColumnFamilyHandle *const column_family, + void release_lock(const Rdb_key_def &key_descr, const std::string &rowkey) override { // Nothing to do here since we don't hold any row locks. } @@ -3624,7 +3967,7 @@ class Rdb_writebatch_impl : public Rdb_transaction { m_insert_count = 0; m_update_count = 0; m_delete_count = 0; - m_lock_count = 0; + m_row_lock_count = 0; release_snapshot(); reset(); @@ -3688,10 +4031,12 @@ class Rdb_writebatch_impl : public Rdb_transaction { value); } - rocksdb::Status get_for_update( - rocksdb::ColumnFamilyHandle *const column_family, - const rocksdb::Slice &key, rocksdb::PinnableSlice *const value, - bool /* exclusive */, const bool /* do_validate */) override { + rocksdb::Status get_for_update(const Rdb_key_def &key_descr, + const rocksdb::Slice &key, + rocksdb::PinnableSlice *const value, + bool /* exclusive */, + const bool /* do_validate */) override { + rocksdb::ColumnFamilyHandle *const column_family = key_descr.get_cf(); if (value == nullptr) { rocksdb::PinnableSlice pin_val; rocksdb::Status s = get(column_family, key, &pin_val); @@ -3722,6 +4067,8 @@ class Rdb_writebatch_impl : public Rdb_transaction { set_initial_savepoint(); } + void set_name() override {} + void start_stmt() override {} void rollback_stmt() override { @@ -3735,6 +4082,10 @@ class Rdb_writebatch_impl : public Rdb_transaction { } virtual ~Rdb_writebatch_impl() override { + // Remove from the global list before all other processing is started. + // Otherwise, information_schema.rocksdb_trx can crash on this object. + Rdb_transaction::remove_from_global_trx_list(); + rollback(); delete m_batch; } @@ -3810,11 +4161,14 @@ static Rdb_transaction *get_or_create_tx(THD *const thd) { } else { tx = new Rdb_transaction_impl(thd); } - tx->set_params(THDVAR(thd, lock_wait_timeout), THDVAR(thd, max_row_locks)); + tx->set_params(THDVAR(thd, lock_wait_timeout), rocksdb_max_row_locks); tx->start_tx(); + // Add the transaction to the global list of transactions + // once it is fully constructed. + tx->add_to_global_trx_list(); my_core::thd_set_ha_data(thd, rocksdb_hton, tx); } else { - tx->set_params(THDVAR(thd, lock_wait_timeout), THDVAR(thd, max_row_locks)); + tx->set_params(THDVAR(thd, lock_wait_timeout), rocksdb_max_row_locks); if (!tx->is_tx_started()) { tx->start_tx(); } @@ -3842,33 +4196,6 @@ static int rocksdb_close_connection(handlerton *const hton, THD *const thd) { return HA_EXIT_SUCCESS; } -/* - * Serializes an xid to a string so that it can - * be used as a rocksdb transaction name - */ -static std::string rdb_xid_to_string(const XID &src) { - DBUG_ASSERT(src.gtrid_length >= 0 && src.gtrid_length <= MAXGTRIDSIZE); - DBUG_ASSERT(src.bqual_length >= 0 && src.bqual_length <= MAXBQUALSIZE); - - std::string buf; - buf.reserve(RDB_XIDHDR_LEN + src.gtrid_length + src.bqual_length); - - /* - * expand formatID to fill 8 bytes if it doesn't already - * then reinterpret bit pattern as unsigned and store in network order - */ - uchar fidbuf[RDB_FORMATID_SZ]; - int64 signed_fid8 = src.formatID; - const uint64 raw_fid8 = *reinterpret_cast(&signed_fid8); - rdb_netbuf_store_uint64(fidbuf, raw_fid8); - buf.append(reinterpret_cast(fidbuf), RDB_FORMATID_SZ); - - buf.push_back(src.gtrid_length); - buf.push_back(src.bqual_length); - buf.append(src.data, (src.gtrid_length) + (src.bqual_length)); - return buf; -} - #if 0 // MARIAROCKS: MariaDB doesn't have flush_wal method /** @@ -3944,9 +4271,11 @@ static int rocksdb_prepare(handlerton* hton, THD* thd, bool prepare_tx) if (!rocksdb_flush_log_at_trx_commit) tx->set_sync(false); - XID xid; - thd_get_xid(thd, reinterpret_cast(&xid)); - if (!tx->prepare(rdb_xid_to_string(xid))) { +#ifndef DBUG_OFF + // Call set_name to check that transaction name still matches. + tx->set_name(); +#endif + if (!tx->prepare()) { return HA_EXIT_FAILURE; } @@ -3992,7 +4321,7 @@ static int rocksdb_commit_by_xid(handlerton *const hton, XID *const xid) { DBUG_ASSERT(xid != nullptr); DBUG_ASSERT(commit_latency_stats != nullptr); - rocksdb::StopWatchNano timer(rocksdb::Env::Default(), true); + rocksdb::StopWatchNano timer(rocksdb::Env::Default()->GetSystemClock().get(), true); const auto name = rdb_xid_to_string(*xid); DBUG_ASSERT(!name.empty()); @@ -4187,7 +4516,7 @@ static int rocksdb_commit(handlerton* hton, THD* thd, bool commit_tx) DBUG_ASSERT(thd != nullptr); DBUG_ASSERT(commit_latency_stats != nullptr); - rocksdb::StopWatchNano timer(rocksdb::Env::Default(), true); + rocksdb::StopWatchNano timer(rocksdb::Env::Default()->GetSystemClock().get(), true); /* note: h->external_lock(F_UNLCK) is called after this function is called) */ Rdb_transaction *tx = get_tx_from_thd(thd); @@ -4404,8 +4733,14 @@ class Rdb_snapshot_status : public Rdb_tx_list_walker { (kd) ? kd->get_name() : "NOT FOUND; INDEX_ID: " + std::to_string(gl_index_id.index_id); - rocksdb::ColumnFamilyHandle *cfh = cf_manager.get_cf(txn.m_cf_id); - txn_data.cf_name = cfh->GetName(); + std::shared_ptr cfh = + cf_manager.get_cf(txn.m_cf_id); + + // Retrieve CF name from CF handle object, and it is safe if the CF is + // removed from cf_manager at this point. + txn_data.cf_name = (cfh) + ? cfh->GetName() + : "NOT FOUND; CF_ID: " + std::to_string(txn.m_cf_id); txn_data.waiting_key = rdb_hexdump(txn.m_waiting_key.c_str(), txn.m_waiting_key.length()); @@ -4461,7 +4796,7 @@ class Rdb_snapshot_status : public Rdb_tx_list_walker { "%s\n" "lock count %llu, write count %llu\n" "insert count %llu, update count %llu, delete count %llu\n", - (longlong)(curr_time - snapshot_timestamp), buffer, tx->get_lock_count(), + (longlong)(curr_time - snapshot_timestamp), buffer, tx->get_row_lock_count(), tx->get_write_count(), tx->get_insert_count(), tx->get_update_count(), tx->get_delete_count()); } @@ -4586,7 +4921,7 @@ class Rdb_trx_info_aggregator : public Rdb_tx_list_walker { m_trx_info->push_back( {rdb_trx->GetName(), rdb_trx->GetID(), tx_impl->get_write_count(), - tx_impl->get_lock_count(), tx_impl->get_timeout_sec(), + tx_impl->get_row_lock_count(), tx_impl->get_timeout_sec(), state_it->second, waiting_key, waiting_cf_id, is_replication, 0, /* skip_trx_api */ tx_impl->is_tx_read_only(), rdb_trx->IsDeadlockDetect(), @@ -4701,12 +5036,16 @@ static bool rocksdb_show_status(handlerton *const hton, THD *const thd, /* Per column family stats */ for (const auto &cf_name : cf_manager.get_cf_names()) { - rocksdb::ColumnFamilyHandle *cfh = cf_manager.get_cf(cf_name); - if (cfh == nullptr) { + std::shared_ptr cfh = + cf_manager.get_cf(cf_name); + if (!cfh) { continue; } - if (!rdb->GetProperty(cfh, "rocksdb.cfstats", &str)) { + // Retrieve information from CF handle object. + // Even if the CF is removed from CF_manager, the handle object + // is valid. + if (!rdb->GetProperty(cfh.get(), "rocksdb.cfstats", &str)) { continue; } @@ -4723,6 +5062,8 @@ static bool rocksdb_show_status(handlerton *const hton, THD *const thd, cache_set.insert(rocksdb_tbl_options->block_cache.get()); for (const auto &cf_handle : cf_manager.get_all_cf()) { + // It is safe if the CF handle is removed from cf_manager + // at this point. rocksdb::ColumnFamilyDescriptor cf_desc; cf_handle->GetDescriptor(&cf_desc); auto *const table_factory = cf_desc.options.table_factory.get(); @@ -4731,9 +5072,8 @@ static bool rocksdb_show_status(handlerton *const hton, THD *const thd, std::string tf_name = table_factory->Name(); if (tf_name.find("BlockBasedTable") != std::string::npos) { - const rocksdb::BlockBasedTableOptions *const bbt_opt = - reinterpret_cast( - table_factory->GetOptions()); + const auto bbt_opt = + table_factory->GetOptions(); if (bbt_opt != nullptr) { if (bbt_opt->block_cache.get() != nullptr) { @@ -4741,7 +5081,6 @@ static bool rocksdb_show_status(handlerton *const hton, THD *const thd, } else { internal_cache_count++; } - cache_set.insert(bbt_opt->block_cache_compressed.get()); } } } @@ -4826,6 +5165,7 @@ static inline void rocksdb_register_tx(handlerton *const hton, THD *const thd, DBUG_ASSERT(tx != nullptr); trans_register_ha(thd, FALSE, rocksdb_hton, 0); + tx->set_name(); if (my_core::thd_test_options(thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN)) { tx->start_stmt(); trans_register_ha(thd, TRUE, rocksdb_hton, 0); @@ -5175,9 +5515,12 @@ static rocksdb::Status check_rocksdb_options_compatibility( rocksdb::DBOptions loaded_db_opt; std::vector loaded_cf_descs; - rocksdb::Status status = - LoadLatestOptions(dbpath, rocksdb::Env::Default(), &loaded_db_opt, - &loaded_cf_descs, rocksdb_ignore_unknown_options); + rocksdb::ConfigOptions config_options; + config_options.ignore_unknown_options = rocksdb_ignore_unknown_options; + config_options.input_strings_escaped = true; + config_options.env = rocksdb::Env::Default(); + rocksdb::Status status = LoadLatestOptions(config_options, dbpath, + &loaded_db_opt, &loaded_cf_descs); // If we're starting from scratch and there are no options saved yet then this // is a valid case. Therefore we can't compare the current set of options to @@ -5216,9 +5559,15 @@ static rocksdb::Status check_rocksdb_options_compatibility( // This is the essence of the function - determine if it's safe to open the // database or not. - status = CheckOptionsCompatibility(dbpath, rocksdb::Env::Default(), main_opts, - loaded_cf_descs, - rocksdb_ignore_unknown_options); + rocksdb::ConfigOptions config_options_for_check(main_opts); + config_options_for_check.sanity_level = + rocksdb::ConfigOptions::kSanityLevelLooselyCompatible; + config_options_for_check.ignore_unknown_options = + rocksdb_ignore_unknown_options; + config_options_for_check.input_strings_escaped = true; + config_options_for_check.env = rocksdb::Env::Default(); + status = CheckOptionsCompatibility(config_options_for_check, dbpath, + main_opts, loaded_cf_descs); return status; } @@ -5252,6 +5601,37 @@ static void rocksdb_update_optimizer_costs(OPTIMIZER_COSTS *costs) costs->row_copy_cost= 0.00006087; } +/* Clean up tables leftover from truncation */ +void rocksdb_truncation_table_cleanup(void) { + /* Scan for tables that have the truncation prefix */ + struct Rdb_truncate_tbls : public Rdb_tables_scanner { + public: + std::vector m_tbl_list; + int add_table(Rdb_tbl_def *tdef) override { + DBUG_ASSERT(tdef != nullptr); + if (tdef->base_tablename().find(TRUNCATE_TABLE_PREFIX) != + std::string::npos) { + m_tbl_list.push_back(tdef); + } + return HA_EXIT_SUCCESS; + } + } collector; + ddl_manager.scan_for_tables(&collector); + + /* + For now, delete any table found. It's possible to rename them back, + but there's a risk the rename can potentially lead to other inconsistencies. + Removing the old table (which is being truncated anyway) seems to be the + safest solution. + */ + ha_rocksdb table(rocksdb_hton, nullptr); + for (Rdb_tbl_def *tbl_def : collector.m_tbl_list) { + // NO_LINT_DEBUG + sql_print_warning("MyRocks: Removing truncated leftover table %s", + tbl_def->full_tablename().c_str()); + table.delete_table(tbl_def); + } +} /* Storage Engine initialization function, invoked when plugin is loaded. @@ -5330,6 +5710,9 @@ static int rocksdb_init_func(void *const p) { MY_MUTEX_INIT_FAST); mysql_mutex_init(rdb_block_cache_resize_mutex_key, &rdb_block_cache_resize_mutex, MY_MUTEX_INIT_FAST); + mysql_mutex_init(rdb_bottom_pri_background_compactions_resize_mutex_key, + &rdb_bottom_pri_background_compactions_resize_mutex, + MY_MUTEX_INIT_FAST); Rdb_transaction::init_mutex(); rocksdb_hton->create = rocksdb_create_handler; @@ -5366,12 +5749,6 @@ static int rocksdb_init_func(void *const p) { rocksdb_hton->update_table_stats = rocksdb_update_table_stats; #endif // MARIAROCKS_NOT_YET - /* - Not needed in MariaDB: - rocksdb_hton->flush_logs = rocksdb_flush_wal; - rocksdb_hton->handle_single_table_select = rocksdb_handle_single_table_select; - - */ rocksdb_hton->check_version = rocksdb_check_version; rocksdb_hton->flags = HTON_TEMPORARY_NOT_SUPPORTED | @@ -5425,6 +5802,9 @@ static int rocksdb_init_func(void *const p) { rocksdb_db_options->wal_recovery_mode = static_cast(rocksdb_wal_recovery_mode); + rocksdb_db_options->track_and_verify_wals_in_manifest = + rocksdb_track_and_verify_wals_in_manifest; + rocksdb_db_options->access_hint_on_compaction_start = static_cast( rocksdb_access_hint_on_compaction_start); @@ -5536,6 +5916,7 @@ static int rocksdb_init_func(void *const p) { if (!rocksdb_tbl_options->no_block_cache) { std::shared_ptr memory_allocator; if (!rocksdb_cache_dump) { +#ifdef HAVE_JEMALLOC size_t block_size = rocksdb_tbl_options->block_size; rocksdb::JemallocAllocatorOptions alloc_opt; // Limit jemalloc tcache memory usage. The range @@ -5553,6 +5934,11 @@ static int rocksdb_init_func(void *const p) { memory_allocator = nullptr; DBUG_RETURN(HA_EXIT_FAILURE); } +#else + // NO_LINT_DEBUG + sql_print_warning( + "Ignoring rocksdb_cache_dump because jemalloc is missing."); +#endif // HAVE_JEMALLOC } std::shared_ptr block_cache = rocksdb_use_clock_cache @@ -5571,11 +5957,6 @@ static int rocksdb_init_func(void *const p) { rocksdb_tbl_options->block_cache = block_cache; } } - // Using newer BlockBasedTable format version for better compression - // and better memory allocation. - // See: - // https://github.com/facebook/rocksdb/commit/9ab5adfc59a621d12357580c94451d9f7320c2dd - rocksdb_tbl_options->format_version = 2; if (rocksdb_collect_sst_properties) { properties_collector_factory = @@ -5685,7 +6066,8 @@ static int rocksdb_init_func(void *const p) { } cf_manager.init(std::move(cf_options_map), &cf_handles); - if (dict_manager.init(rdb, &cf_manager)) { + if (dict_manager.init(rdb, &cf_manager, + rocksdb_enable_remove_orphaned_dropped_cfs)) { // NO_LINT_DEBUG sql_print_error("RocksDB: Failed to initialize data dictionary."); DBUG_RETURN(HA_EXIT_FAILURE); @@ -5818,6 +6200,9 @@ static int rocksdb_init_func(void *const p) { io_watchdog->reset_timeout(rocksdb_io_write_timeout_secs); #endif + // Remove tables that may have been leftover during truncation + rocksdb_truncation_table_cleanup(); + // NO_LINT_DEBUG sql_print_information( "MyRocks storage engine plugin has been successfully " @@ -5826,6 +6211,9 @@ static int rocksdb_init_func(void *const p) { // Skip cleaning up rdb_open_tables as we've succeeded rdb_open_tables_cleanup.skip(); + rocksdb_set_max_bottom_pri_background_compactions_internal( + rocksdb_max_bottom_pri_background_compactions); + DBUG_RETURN(HA_EXIT_SUCCESS); } @@ -5887,7 +6275,7 @@ static int rocksdb_done_func(void *const p) { // MARIADB_MERGE_2019: rdb_open_tables.m_hash.~Rdb_table_set(); mysql_mutex_destroy(&rdb_sysvars_mutex); mysql_mutex_destroy(&rdb_block_cache_resize_mutex); - + mysql_mutex_destroy(&rdb_bottom_pri_background_compactions_resize_mutex); delete rdb_collation_exceptions; @@ -5972,13 +6360,6 @@ static inline void rocksdb_smart_next(bool seek_backward, } } -#ifndef DBUG_OFF -// simulate that RocksDB has reported corrupted data -static void dbug_change_status_to_corrupted(rocksdb::Status *status) { - *status = rocksdb::Status::Corruption(); -} -#endif - // If the iterator is not valid it might be because of EOF but might be due // to IOError or corruption. The good practice is always check it. // https://github.com/facebook/rocksdb/wiki/Iterator#error-handling @@ -6384,10 +6765,11 @@ ha_rocksdb::ha_rocksdb(my_core::handlerton *const hton, m_lock_rows(RDB_LOCK_NONE), m_keyread_only(false), m_insert_with_update(false), - m_dup_pk_found(false), + m_dup_key_found(false), m_in_rpl_delete_rows(false), m_in_rpl_update_rows(false), - m_force_skip_unique_check(false) {} + m_force_skip_unique_check(false), + m_need_build_decoder(false) {} const std::string &ha_rocksdb::get_table_basename() const { @@ -7142,6 +7524,13 @@ int ha_rocksdb::create_key_defs( DBUG_ASSERT(table_arg->s != nullptr); + DBUG_EXECUTE_IF("rocksdb_truncate_failure", { + my_error(ER_INTERNAL_ERROR, MYF(0), "Simulated truncation failure."); + DBUG_RETURN(HA_EXIT_FAILURE); + }); + + DBUG_EXECUTE_IF("rocksdb_truncate_failure_crash", DBUG_SUICIDE();); + /* These need to be one greater than MAX_INDEXES since the user can create MAX_INDEXES secondary keys and no primary key which would cause us @@ -7248,10 +7637,17 @@ int ha_rocksdb::create_cfs( The first loop checks the index parameters and creates column families if necessary. */ + THD *const thd = my_core::current_thd; for (uint i = 0; i < tbl_def_arg->m_key_count; i++) { - rocksdb::ColumnFamilyHandle *cf_handle; + std::shared_ptr cf_handle; - if (!is_hidden_pk(i, table_arg, tbl_def_arg) && + /* + Skip collation checks on truncation since we might be recreating the + table that had unsupported collations and we don't want to fail the + truncation. + */ + if (thd->lex->sql_command != SQLCOM_TRUNCATE && + !is_hidden_pk(i, table_arg, tbl_def_arg) && tbl_def_arg->base_tablename().find(tmp_file_prefix) != 0) { if (!tsys_set) { @@ -7306,15 +7702,57 @@ int ha_rocksdb::create_cfs( DBUG_RETURN(HA_EXIT_FAILURE); } + DBUG_EXECUTE_IF("rocksdb_create_primary_cf", { + if (cf_name == "cf_primary_key") { + THD *const thd = my_core::current_thd; + const char act[] = + "now signal ready_to_mark_cf_dropped_in_create_cfs " + "wait_for mark_cf_dropped_done_in_create_cfs"; + DBUG_ASSERT(!debug_sync_set_action(thd, STRING_WITH_LEN(act))); + } + }); + + DBUG_EXECUTE_IF("rocksdb_create_secondary_cf", { + if (cf_name == "cf_secondary_key") { + THD *const thd = my_core::current_thd; + const char act[] = + "now signal ready_to_mark_cf_dropped_in_create_cfs " + "wait_for mark_cf_dropped_done_in_create_cfs"; + DBUG_ASSERT(!debug_sync_set_action(thd, STRING_WITH_LEN(act))); + } + }); + + // if not specified, use default CF name + if (cf_name.empty()) cf_name = DEFAULT_CF_NAME; + // Here's how `get_or_create_cf` will use the input parameters: // // `cf_name` - will be used as a CF name. + std::lock_guard dm_lock(dict_manager); cf_handle = cf_manager.get_or_create_cf(rdb, cf_name); - if (!cf_handle) { DBUG_RETURN(HA_EXIT_FAILURE); } + uint32 cf_id = cf_handle->GetID(); + + // If the cf is marked as dropped, we fail it here. + // The cf can be dropped after this point, we will + // check again when committing metadata changes. + if (dict_manager.get_dropped_cf(cf_id)) { + my_error(ER_CF_DROPPED, MYF(0), cf_name.c_str()); + DBUG_RETURN(HA_EXIT_FAILURE); + } + + if (cf_manager.create_cf_flags_if_needed(&dict_manager, + cf_handle->GetID(), cf_name, + per_part_match_found)) { + DBUG_RETURN(HA_EXIT_FAILURE); + } + + // The CF can be dropped from cf_manager at this point. This is part of + // create table or alter table. If the drop happens before metadata are + // written, create table or alter table will fail. auto &cf = (*cfs)[i]; cf.cf_handle = cf_handle; @@ -7390,7 +7828,7 @@ int ha_rocksdb::create_inplace_key_defs( itself. */ new_key_descr[i] = std::make_shared( - okd.get_index_number(), i, okd.get_cf(), + okd.get_index_number(), i, okd.get_shared_cf(), index_info.m_index_dict_version, index_info.m_index_type, index_info.m_kv_version, okd.m_is_reverse_cf, okd.m_is_per_partition_cf, okd.m_name.c_str(), @@ -7609,6 +8047,10 @@ int ha_rocksdb::create_key_def(const TABLE *const table_arg, const uint i, DBUG_RETURN((*new_key_def)->setup(table_arg, tbl_def_arg)); } +bool rdb_is_tablename_normalized(const std::string &tablename) { + return tablename.size() < 2 || (tablename[0] != '.' && tablename[1] != '/'); +} + int rdb_normalize_tablename(const std::string &tablename, std::string *const strbuf) { if (tablename.size() < 2 || tablename[0] != '.' || @@ -7737,6 +8179,35 @@ int rdb_split_normalized_tablename(const std::string &fullname, return HA_EXIT_SUCCESS; } +/* + Generates the normalized tablename using as many of the given arguments as + possible. Any of the three arguments to .#P# can be + null/empty, but return string will only ever be of the form + + .
+ .
#P# +
+
#P# +*/ +void rdb_gen_normalized_tablename(const std::string *db, + const std::string *table, + const std::string *partition, + std::string *fullname) { + if (!fullname) return; + fullname->clear(); + if (db && db->length() > 0) *fullname += *db; + + /* If table was not passed in, the partition will be ignored too */ + if (!table || table->length() == 0) return; + + if (fullname->length() > 0) *fullname += "."; + *fullname += *table; + + if (partition && partition->length() > 0) { + *fullname += std::string(RDB_PARTITION_STR) + *partition; + } +} + /* Create a table's Rdb_tbl_def and its Rdb_key_defs and store table information into MyRocks Data Dictionary @@ -7804,21 +8275,27 @@ int ha_rocksdb::create_table(const std::string &table_name, } } - dict_manager.lock(); - err = ddl_manager.put_and_write(m_tbl_def, batch); - if (err != HA_EXIT_SUCCESS) { - dict_manager.unlock(); - goto error; - } + DBUG_EXECUTE_IF("rocksdb_create_table", { + THD *const thd = my_core::current_thd; + const char act[] = + "now signal ready_to_mark_cf_dropped_in_create_table " + "wait_for mark_cf_dropped_done_in_create_table"; + DBUG_ASSERT(!debug_sync_set_action(thd, STRING_WITH_LEN(act))); + }); - save_table_version(batch, table_arg->s->path.str, 0); - err = dict_manager.commit(batch); - if (err != HA_EXIT_SUCCESS) { - dict_manager.unlock(); - goto error; - } + { + std::lock_guard dm_lock(dict_manager); + err = ddl_manager.put_and_write(m_tbl_def, batch); + if (err != HA_EXIT_SUCCESS) { + goto error; + } - dict_manager.unlock(); + save_table_version(batch, table_arg->s->path.str, 0); + err = dict_manager.commit(batch); + if (err != HA_EXIT_SUCCESS) { + goto error; + } + } DBUG_RETURN(HA_EXIT_SUCCESS); @@ -7895,13 +8372,11 @@ int ha_rocksdb::create(const char *const name, TABLE *const table_arg, } // Check whether Data Dictionary contain information - Rdb_tbl_def *tbl = ddl_manager.find(str); - if (tbl != nullptr) { + Rdb_tbl_def *old_tbl = ddl_manager.find(str); + if (old_tbl != nullptr) { if (thd->lex->sql_command == SQLCOM_TRUNCATE) { - err = delete_table(tbl); - if (err != HA_EXIT_SUCCESS) { - DBUG_RETURN(err); - } + DBUG_RETURN(truncate_table(old_tbl, table_arg, + create_info->auto_increment_value)); } else { my_error(ER_METADATA_INCONSISTENCY, MYF(0), str.c_str(), name); DBUG_RETURN(HA_ERR_ROCKSDB_CORRUPT_DATA); @@ -7933,6 +8408,108 @@ int ha_rocksdb::create(const char *const name, TABLE *const table_arg, DBUG_RETURN(create_table(str, table_arg, create_info->auto_increment_value)); } +/* + Fast truncates a table by renaming the old table, creating a new one and + restoring or deleting the old table based on the results from creation. + +@param tbl_def IN MyRocks's table structure + @param table_arg IN sql table + @param auto_increment_value IN specified table's auto increment value + + @return + HA_EXIT_SUCCESS OK + other HA_ERR error code (can be SE-specific) + */ + int ha_rocksdb::truncate_table(Rdb_tbl_def *tbl_def_arg, TABLE *table_arg, + ulonglong auto_increment_value) { + DBUG_ENTER_FUNC(); + + /* + Fast table truncation involves deleting the table and then recreating + it. However, it is possible recreating the table fails. In this case, a + table inconsistency might result between SQL and MyRocks where MyRocks is + missing a table. Since table creation involves modifying keys with the + original table name, renaming the original table first, and then renaming + it back in case of creation failure can help restore the pre-truncation + state. + + If the server were to crash during truncation, the system will end up with + an inconsistency. Future changes for atomic ddl will resolve this. For now, + if there are any truncation renamed tables found during startup, MyRocks + will automatically remove them. + */ + std::string orig_tablename = tbl_def_arg->full_tablename(); + std::string dbname, tblname, partition; + + /* + Rename the table in the data dictionary. Since this thread should be + holding the MDL for this tablename, it is safe to perform these renames + should be locked via MDL, no other process thread be able to access this + table. + */ + int err = rdb_split_normalized_tablename(orig_tablename, &dbname, &tblname, + &partition); + DBUG_ASSERT(err == 0); + if (err != HA_EXIT_SUCCESS) DBUG_RETURN(err); + tblname = std::string(TRUNCATE_TABLE_PREFIX) + tblname; + + std::string tmp_tablename; + rdb_gen_normalized_tablename(&dbname, &tblname, &partition, &tmp_tablename); + + err = rename_table(orig_tablename.c_str(), tmp_tablename.c_str()); + if (err != HA_EXIT_SUCCESS) DBUG_RETURN(err); + + /* + Attempt to create the table. If this succeeds, then drop the old table. + Otherwise, try to restore it. + */ + err = create_table(orig_tablename, table_arg, auto_increment_value); + bool should_remove_old_table = true; + + /* Restore the old table being truncated if creating the new table failed */ + if (err != HA_EXIT_SUCCESS) { + int rename_err = + rename_table(tmp_tablename.c_str(), orig_tablename.c_str()); + + /* + If the rename also fails, we are out of options, but at least try to drop + the old table contents. + */ + if (rename_err == HA_EXIT_SUCCESS) { + should_remove_old_table = false; + } else { + // NO_LINT_DEBUG + sql_print_error( + "MyRocks: Failure during truncation of table %s " + "being renamed from %s", + orig_tablename.c_str(), tmp_tablename.c_str()); + err = rename_err; + } + } + + /* + Since the table was successfully truncated or the name restore failed, no + error should be returned at this point from trying to delete the old + table. If the delete_table fails, log it instead. + */ + Rdb_tbl_def *old_tbl_def = ddl_manager.find(tmp_tablename); + if (should_remove_old_table && old_tbl_def) { + m_tbl_def = old_tbl_def; + if (delete_table(old_tbl_def) != HA_EXIT_SUCCESS) { + // NO_LINT_DEBUG + sql_print_error( + "Failure when trying to drop table %s during " + "truncation of table %s", + tmp_tablename.c_str(), orig_tablename.c_str()); + } + } + + /* Update the local m_tbl_def reference */ + m_tbl_def = ddl_manager.find(orig_tablename); + m_converter.reset(new Rdb_converter(ha_thd(), m_tbl_def, table_arg)); + DBUG_RETURN(err); +} + /** @note This function is used only when the table has not yet been opened, and @@ -8234,8 +8811,9 @@ int ha_rocksdb::read_row_from_secondary_key(uchar *const buf, #endif DBUG_EXECUTE_IF("dbug.rocksdb.HA_EXTRA_KEYREAD", { m_keyread_only = true; }); - bool covered_lookup = (m_keyread_only && kd.can_cover_lookup()) || - kd.covers_lookup(&value, &m_lookup_bitmap); + bool covered_lookup = + (m_keyread_only && kd.can_cover_lookup()) || + kd.covers_lookup(&value, m_converter->get_lookup_bitmap()); #ifndef DBUG_OFF m_keyread_only = save_keyread_only; @@ -8361,7 +8939,8 @@ int ha_rocksdb::secondary_index_read(const int keyno, uchar *const buf) { rocksdb::Slice value = m_scan_it->value(); bool covered_lookup = (m_keyread_only && m_key_descr_arr[keyno]->can_cover_lookup()) || - m_key_descr_arr[keyno]->covers_lookup(&value, &m_lookup_bitmap); + m_key_descr_arr[keyno]->covers_lookup( + &value, m_converter->get_lookup_bitmap()); if (covered_lookup && m_lock_rows == RDB_LOCK_NONE) { rc = m_key_descr_arr[keyno]->unpack_record( table, buf, &key, &value, @@ -8409,6 +8988,8 @@ int ha_rocksdb::read_range_first(const key_range *const start_key, bool eq_range_arg, bool sorted) { DBUG_ENTER_FUNC(); + check_build_decoder(); + int result; eq_range = eq_range_arg; @@ -8488,6 +9069,8 @@ int ha_rocksdb::index_read_map(uchar *const buf, const uchar *const key, enum ha_rkey_function find_flag) { DBUG_ENTER_FUNC(); + check_build_decoder(); + DBUG_RETURN(index_read_map_impl(buf, key, keypart_map, find_flag, nullptr)); } @@ -8575,6 +9158,22 @@ int ha_rocksdb::index_read_map_impl(uchar *const buf, const uchar *const key, kd.get_key_parts()) { using_full_key = false; } + + if (m_insert_with_update && m_dup_key_found && + active_index == m_dupp_errkey) { + /* + We are in INSERT ... ON DUPLICATE KEY UPDATE, and this is a read + that SQL layer does to read the duplicate key. + Its rowid is saved in m_last_rowkey. Get the full record and return it. + */ + + DBUG_ASSERT(m_dup_key_retrieved_record.length() >= packed_size); + DBUG_ASSERT(memcmp(m_dup_key_retrieved_record.ptr(), m_sk_packed_tuple, + packed_size) == 0); + + rc = get_row_by_rowid(buf, m_last_rowkey.ptr(), m_last_rowkey.length()); + DBUG_RETURN(rc); + } } else { packed_size = kd.pack_index_tuple(table, m_pack_buffer, m_sk_packed_tuple, m_record_buffer, key, keypart_map); @@ -8998,15 +9597,14 @@ void dbug_dump_database(rocksdb::DB *const db) { } rocksdb::Status ha_rocksdb::get_for_update( - Rdb_transaction *const tx, rocksdb::ColumnFamilyHandle *const column_family, + Rdb_transaction *const tx, const Rdb_key_def &key_descr, const rocksdb::Slice &key, rocksdb::PinnableSlice *const value) const { DBUG_ASSERT(m_lock_rows != RDB_LOCK_NONE); - const bool exclusive = m_lock_rows != RDB_LOCK_READ; - const bool do_validate = - my_core::thd_tx_isolation(ha_thd()) > ISO_READ_COMMITTED; + bool exclusive = m_lock_rows != RDB_LOCK_READ; + bool do_validate = my_core::thd_tx_isolation(ha_thd()) > ISO_READ_COMMITTED; rocksdb::Status s = - tx->get_for_update(column_family, key, value, exclusive, do_validate); + tx->get_for_update(key_descr, key, value, exclusive, do_validate); #ifndef DBUG_OFF ++rocksdb_num_get_for_update_calls; @@ -9040,6 +9638,8 @@ int ha_rocksdb::get_row_by_rowid(uchar *const buf, const char *const rowid, const bool skip_ttl_check) { DBUG_ENTER_FUNC(); + check_build_decoder(); + DBUG_ASSERT(table != nullptr); int rc; @@ -9078,11 +9678,11 @@ int ha_rocksdb::get_row_by_rowid(uchar *const buf, const char *const rowid, if (m_lock_rows == RDB_LOCK_NONE) { tx->acquire_snapshot(true); s = tx->get(m_pk_descr->get_cf(), key_slice, &m_retrieved_record); - } else if (m_insert_with_update && m_dup_pk_found) { - DBUG_ASSERT(m_pk_descr->get_keyno() == m_dupp_errkey); - DBUG_ASSERT(m_dup_pk_retrieved_record.length() == + } else if (m_insert_with_update && m_dup_key_found && + m_pk_descr->get_keyno() == m_dupp_errkey) { + DBUG_ASSERT(m_dup_key_retrieved_record.length() == m_retrieved_record.size()); - DBUG_ASSERT(memcmp(m_dup_pk_retrieved_record.ptr(), + DBUG_ASSERT(memcmp(m_dup_key_retrieved_record.ptr(), m_retrieved_record.data(), m_retrieved_record.size()) == 0); @@ -9090,8 +9690,7 @@ int ha_rocksdb::get_row_by_rowid(uchar *const buf, const char *const rowid, // already taken the lock s = rocksdb::Status::OK(); } else { - s = get_for_update(tx, m_pk_descr->get_cf(), key_slice, - &m_retrieved_record); + s = get_for_update(tx, *m_pk_descr, key_slice, &m_retrieved_record); } DBUG_EXECUTE_IF("rocksdb_return_status_corrupted", @@ -9137,6 +9736,8 @@ int ha_rocksdb::get_row_by_rowid(uchar *const buf, const char *const rowid, int ha_rocksdb::index_next(uchar *const buf) { DBUG_ENTER_FUNC(); + check_build_decoder(); + bool moves_forward = true; if (m_key_descr_arr[active_index]->m_is_reverse_cf) { moves_forward = false; @@ -9156,6 +9757,8 @@ int ha_rocksdb::index_next(uchar *const buf) { int ha_rocksdb::index_prev(uchar *const buf) { DBUG_ENTER_FUNC(); + check_build_decoder(); + bool moves_forward = false; if (m_key_descr_arr[active_index]->m_is_reverse_cf) { moves_forward = true; @@ -9215,6 +9818,8 @@ int ha_rocksdb::index_next_with_direction(uchar *const buf, bool move_forward) { int ha_rocksdb::index_first(uchar *const buf) { DBUG_ENTER_FUNC(); + check_build_decoder(); + m_sk_match_prefix = nullptr; int rc = m_key_descr_arr[active_index]->m_is_reverse_cf ? index_last_intern(buf) @@ -9232,6 +9837,8 @@ int ha_rocksdb::index_first(uchar *const buf) { int ha_rocksdb::index_last(uchar *const buf) { DBUG_ENTER_FUNC(); + check_build_decoder(); + m_sk_match_prefix = nullptr; int rc = m_key_descr_arr[active_index]->m_is_reverse_cf ? index_first_intern(buf) @@ -9431,7 +10038,7 @@ void ha_rocksdb::unlock_row() { if (m_lock_rows != RDB_LOCK_NONE) { Rdb_transaction *const tx = get_or_create_tx(table->in_use); - tx->release_lock(m_pk_descr->get_cf(), + tx->release_lock(*m_pk_descr, std::string(m_last_rowkey.ptr(), m_last_rowkey.length())); } @@ -9646,7 +10253,7 @@ int ha_rocksdb::write_row(const uchar *const buf) { // clear cache at beginning of write for INSERT ON DUPLICATE // we may get multiple write->fail->read->update if there are multiple // values from INSERT - m_dup_pk_found = false; + m_dup_key_found = false; const int rv = update_write_row(nullptr, buf, skip_unique_check()); @@ -9741,7 +10348,8 @@ int ha_rocksdb::get_pk_for_update(struct update_row_info *const row_info) { */ int ha_rocksdb::check_and_lock_unique_pk(const uint key_id, const struct update_row_info &row_info, - bool *const found) { + bool *const found, + const bool skip_unique_check) { DBUG_ASSERT(found != nullptr); DBUG_ASSERT(row_info.old_pk_slice.size() == 0 || @@ -9749,9 +10357,10 @@ int ha_rocksdb::check_and_lock_unique_pk(const uint key_id, /* Ignore PK violations if this is a optimized 'replace into' */ #ifdef MARIAROCKS_NOT_YET - const bool ignore_pk_unique_check = ha_thd()->lex->blind_replace_into; + const bool ignore_pk_unique_check = + ha_thd()->lex->blind_replace_into || skip_unique_check; #else - const bool ignore_pk_unique_check= false; + const bool ignore_pk_unique_check= skip_unique_check; #endif /* @@ -9774,7 +10383,7 @@ int ha_rocksdb::check_and_lock_unique_pk(const uint key_id, -> T2 Put(insert, blocked) -> T1 commit -> T2 commit(overwrite) */ const rocksdb::Status s = - get_for_update(row_info.tx, m_pk_descr->get_cf(), row_info.new_pk_slice, + get_for_update(row_info.tx, *m_pk_descr, row_info.new_pk_slice, ignore_pk_unique_check ? nullptr : &m_retrieved_record); if (!s.ok() && !s.IsNotFound()) { return row_info.tx->set_status_error( @@ -9799,12 +10408,12 @@ int ha_rocksdb::check_and_lock_unique_pk(const uint key_id, // In INSERT ON DUPLICATE KEY UPDATE ... case, if the insert failed // due to a duplicate key, remember the last key and skip the check // next time - m_dup_pk_found = true; + m_dup_key_found = true; #ifndef DBUG_OFF // save it for sanity checking later - m_dup_pk_retrieved_record.copy(m_retrieved_record.data(), - m_retrieved_record.size(), &my_charset_bin); + m_dup_key_retrieved_record.copy(m_retrieved_record.data(), + m_retrieved_record.size(), &my_charset_bin); #endif } @@ -9826,7 +10435,8 @@ int ha_rocksdb::check_and_lock_unique_pk(const uint key_id, */ int ha_rocksdb::check_and_lock_sk(const uint key_id, const struct update_row_info &row_info, - bool *const found) { + bool *const found, + const bool skip_unique_check) { DBUG_ASSERT(found != nullptr); *found = false; @@ -9884,7 +10494,7 @@ int ha_rocksdb::check_and_lock_sk(const uint key_id, rocksdb::Slice((const char *)m_sk_packed_tuple_old, size); const rocksdb::Status s = - get_for_update(row_info.tx, kd.get_cf(), old_slice, nullptr); + get_for_update(row_info.tx, kd, old_slice, nullptr); if (!s.ok()) { return row_info.tx->set_status_error(table->in_use, s, kd, m_tbl_def, m_table_handler); @@ -9930,8 +10540,7 @@ int ha_rocksdb::check_and_lock_sk(const uint key_id, lower_bound_buf, upper_bound_buf, &lower_bound_slice, &upper_bound_slice); const bool fill_cache = !THDVAR(ha_thd(), skip_fill_cache); - const rocksdb::Status s = - get_for_update(row_info.tx, kd.get_cf(), new_slice, nullptr); + const rocksdb::Status s = get_for_update(row_info.tx, kd, new_slice, nullptr); if (!s.ok() && !s.IsNotFound()) { return row_info.tx->set_status_error(table->in_use, s, kd, m_tbl_def, m_table_handler); @@ -9946,12 +10555,31 @@ int ha_rocksdb::check_and_lock_sk(const uint key_id, Also need to scan RocksDB and verify the key has not been deleted in the transaction. */ - iter->Seek(new_slice); *found = !read_key_exact(kd, iter, all_parts_used, new_slice, row_info.tx->m_snapshot_timestamp); - delete iter; - return HA_EXIT_SUCCESS; + int rc = HA_EXIT_SUCCESS; + + if (*found && m_insert_with_update) { + const rocksdb::Slice &rkey = iter->key(); + uint pk_size = kd.get_primary_key_tuple(table, *m_pk_descr, &rkey, + m_pk_packed_tuple); + if (pk_size == RDB_INVALID_KEY_LEN) { + rc = HA_ERR_ROCKSDB_CORRUPT_DATA; + } else { + m_dup_key_found = true; + m_last_rowkey.copy((const char *)m_pk_packed_tuple, pk_size, + &my_charset_bin); +#ifndef DBUG_OFF + // save it for sanity checking later + m_dup_key_retrieved_record.copy(rkey.data(), rkey.size(), + &my_charset_bin); +#endif + } + } + + delete iter; + return rc; } /** @@ -9965,7 +10593,8 @@ int ha_rocksdb::check_and_lock_sk(const uint key_id, other HA_ERR error code (can be SE-specific) */ int ha_rocksdb::check_uniqueness_and_lock( - const struct update_row_info &row_info, bool pk_changed) { + const struct update_row_info &row_info, bool pk_changed, + bool skip_unique_check) { /* Go through each index and determine if the index has uniqueness requirements. If it does, then try to obtain a row lock on the new values. @@ -9981,10 +10610,11 @@ int ha_rocksdb::check_uniqueness_and_lock( found = false; rc = HA_EXIT_SUCCESS; } else { - rc = check_and_lock_unique_pk(key_id, row_info, &found); + rc = check_and_lock_unique_pk(key_id, row_info, &found, + skip_unique_check); } } else { - rc = check_and_lock_sk(key_id, row_info, &found); + rc = check_and_lock_sk(key_id, row_info, &found, skip_unique_check); } if (rc != HA_EXIT_SUCCESS) { @@ -10420,12 +11050,15 @@ int ha_rocksdb::update_write_row(const uchar *const old_data, pk_changed = row_info.new_pk_slice.compare(row_info.old_pk_slice) != 0; } - if (!skip_unique_check) { + // Case: We skip both unique checks and rows locks only when bulk load is + // enabled or if rocksdb_skip_locks_if_skip_unique_check is ON + if (!THDVAR(table->in_use, bulk_load) && + (!rocksdb_skip_locks_if_skip_unique_check || !skip_unique_check)) { /* Check to see if we are going to have failures because of unique keys. Also lock the appropriate key values. */ - rc = check_uniqueness_and_lock(row_info, pk_changed); + rc = check_uniqueness_and_lock(row_info, pk_changed, skip_unique_check); if (rc != HA_EXIT_SUCCESS) { DBUG_RETURN(rc); } @@ -10615,6 +11248,9 @@ void ha_rocksdb::setup_iterator_for_rnd_scan() { int ha_rocksdb::rnd_init(bool scan) { DBUG_ENTER_FUNC(); + m_need_build_decoder = true; + active_index = table->s->primary_key; + THD *thd = ha_thd(); if (thd && thd->killed) { DBUG_RETURN(HA_ERR_QUERY_INTERRUPTED); @@ -10622,10 +11258,6 @@ int ha_rocksdb::rnd_init(bool scan) { Rdb_transaction *const tx = get_or_create_tx(table->in_use); - // when this table is being updated, decode all fields - m_converter->setup_field_decoders(table->read_set, - m_lock_rows == RDB_LOCK_WRITE); - if (scan) { m_rnd_scan_is_new_snapshot = !tx->has_snapshot(); setup_iterator_for_rnd_scan(); @@ -10649,6 +11281,8 @@ int ha_rocksdb::rnd_init(bool scan) { int ha_rocksdb::rnd_next(uchar *const buf) { DBUG_ENTER_FUNC(); + check_build_decoder(); + int rc; for (;;) { rc = rnd_next_with_direction(buf, true); @@ -10740,7 +11374,7 @@ int ha_rocksdb::rnd_next_with_direction(uchar *const buf, bool move_forward) { } const rocksdb::Status s = - get_for_update(tx, m_pk_descr->get_cf(), key, &m_retrieved_record); + get_for_update(tx, *m_pk_descr, key, &m_retrieved_record); if (s.IsNotFound() && should_skip_invalidated_record(HA_ERR_KEY_NOT_FOUND)) { continue; @@ -10789,11 +11423,26 @@ int ha_rocksdb::rnd_next_with_direction(uchar *const buf, bool move_forward) { int ha_rocksdb::rnd_end() { DBUG_ENTER_FUNC(); + m_need_build_decoder = false; + release_scan_iterator(); DBUG_RETURN(HA_EXIT_SUCCESS); } +void ha_rocksdb::build_decoder() { + m_converter->setup_field_decoders(table->read_set, active_index, + m_keyread_only, + m_lock_rows == RDB_LOCK_WRITE); +} + +void ha_rocksdb::check_build_decoder() { + if (m_need_build_decoder) { + build_decoder(); + m_need_build_decoder = false; + } +} + /** @return HA_EXIT_SUCCESS OK @@ -10802,6 +11451,9 @@ int ha_rocksdb::rnd_end() { int ha_rocksdb::index_init(uint idx, bool sorted) { DBUG_ENTER_FUNC(); + m_need_build_decoder = true; + active_index = idx; + THD *thd = ha_thd(); if (thd && thd->killed) { DBUG_RETURN(HA_ERR_QUERY_INTERRUPTED); @@ -10810,22 +11462,13 @@ int ha_rocksdb::index_init(uint idx, bool sorted) { Rdb_transaction *const tx = get_or_create_tx(table->in_use); DBUG_ASSERT(tx != nullptr); - m_start_range= NULL; - // when this table is being updated, decode all fields - m_converter->setup_field_decoders(table->read_set, - m_lock_rows == RDB_LOCK_WRITE); - - if (!m_keyread_only) { - m_key_descr_arr[idx]->get_lookup_bitmap(table, &m_lookup_bitmap); - } + m_start_range = NULL; // If m_lock_rows is not RDB_LOCK_NONE then we will be doing a get_for_update // when accessing the index, so don't acquire the snapshot right away. // Otherwise acquire the snapshot immediately. tx->acquire_snapshot(m_lock_rows == RDB_LOCK_NONE); - active_index = idx; - DBUG_RETURN(HA_EXIT_SUCCESS); } @@ -10836,9 +11479,9 @@ int ha_rocksdb::index_init(uint idx, bool sorted) { int ha_rocksdb::index_end() { DBUG_ENTER_FUNC(); - release_scan_iterator(); + m_need_build_decoder = false; - my_bitmap_free(&m_lookup_bitmap); + release_scan_iterator(); active_index = MAX_KEY; in_range_check_pushed_down = FALSE; @@ -10848,6 +11491,8 @@ int ha_rocksdb::index_end() { } /** + Called by the partition manager for truncating tables. + @return HA_EXIT_SUCCESS OK other HA_ERR error code (can be SE-specific) @@ -10857,19 +11502,10 @@ int ha_rocksdb::truncate() { DBUG_ASSERT(m_tbl_def != nullptr); - // Save table name to use later - std::string table_name = m_tbl_def->full_tablename(); - - // Delete current table - int err = delete_table(m_tbl_def); - if (err != HA_EXIT_SUCCESS) { - DBUG_RETURN(err); - } - // Reset auto_increment_value to 1 if auto-increment feature is enabled // By default, the starting valid value for auto_increment_value is 1 - DBUG_RETURN(create_table( - table_name, table, + DBUG_RETURN(truncate_table( + m_tbl_def, table, table->found_next_number_field ? 1 : 0 /* auto_increment_value */)); } @@ -10919,11 +11555,35 @@ int ha_rocksdb::delete_row(const uchar *const buf) { if (!is_pk(i, table, m_tbl_def)) { int packed_size; const Rdb_key_def &kd = *m_key_descr_arr[i]; + + // The unique key should be locked so that behavior is + // similar to InnoDB and reduce conflicts. The key + // used for locking does not include the extended fields. + const KEY *key_info = &table->key_info[i]; + if (key_info->flags & HA_NOSAME) { + uint user_defined_key_parts = key_info->user_defined_key_parts; + uint n_null_fields = 0; + + packed_size = kd.pack_record(table, m_pack_buffer, buf, + m_sk_packed_tuple, nullptr, false, 0, + user_defined_key_parts, &n_null_fields); + + // NULL fields are considered unique, so no lock is needed + if (n_null_fields == 0) { + rocksdb::Slice sk_slice( + reinterpret_cast(m_sk_packed_tuple), packed_size); + const rocksdb::Status s = get_for_update(tx, kd, sk_slice, nullptr); + if (!s.ok()) { + DBUG_RETURN(tx->set_status_error(table->in_use, s, kd, m_tbl_def, + m_table_handler)); + } + } + } + packed_size = kd.pack_record(table, m_pack_buffer, buf, m_sk_packed_tuple, nullptr, false, hidden_pk_id); rocksdb::Slice secondary_key_slice( reinterpret_cast(m_sk_packed_tuple), packed_size); - /* Deleting on secondary key doesn't need any locks: */ tx->get_indexed_write_batch()->SingleDelete(kd.get_cf(), secondary_key_slice); bytes_written += secondary_key_slice.size(); @@ -11020,7 +11680,8 @@ int ha_rocksdb::info(uint flag) { uchar buf[Rdb_key_def::INDEX_NUMBER_SIZE * 2]; auto r = get_range(pk_index(table, m_tbl_def), buf); uint64_t sz = 0; - uint8_t include_flags = rocksdb::DB::INCLUDE_FILES; + rocksdb::DB::SizeApproximationFlags include_flags = + rocksdb::DB::SizeApproximationFlags::INCLUDE_FILES; // recompute SST files stats only if records count is 0 if (stats.records == 0) { rdb->GetApproximateSizes(m_pk_descr->get_cf(), &r, 1, &sz, @@ -11197,6 +11858,8 @@ void ha_rocksdb::position(const uchar *const record) { int ha_rocksdb::rnd_pos(uchar *const buf, uchar *const pos) { DBUG_ENTER_FUNC(); + check_build_decoder(); + int rc; size_t len; @@ -11697,30 +12360,43 @@ void Rdb_drop_index_thread::run() { } abort(); } - rocksdb::ColumnFamilyHandle *cfh = cf_manager.get_cf(d.cf_id); + + std::shared_ptr cfh = + cf_manager.get_cf(d.cf_id); DBUG_ASSERT(cfh); + + if (dict_manager.get_dropped_cf(d.cf_id)) { + finished.insert(d); + continue; + } + const bool is_reverse_cf = cf_flags & Rdb_key_def::REVERSE_CF_FLAG; uchar buf[Rdb_key_def::INDEX_NUMBER_SIZE * 2]; rocksdb::Range range = get_range(d.index_id, buf, is_reverse_cf ? 1 : 0, is_reverse_cf ? 0 : 1); - rocksdb::Status status = DeleteFilesInRange(rdb->GetBaseDB(), cfh, + rocksdb::Status status = DeleteFilesInRange(rdb->GetBaseDB(), cfh.get(), &range.start, &range.limit); if (!status.ok()) { - if (status.IsShutdownInProgress()) { + if (status.IsIncomplete()) { + continue; + } else if (status.IsShutdownInProgress()) { break; } rdb_handle_io_error(status, RDB_IO_ERROR_BG_THREAD); } - status = rdb->CompactRange(getCompactRangeOptions(), cfh, &range.start, + + status = rdb->CompactRange(getCompactRangeOptions(), cfh.get(), &range.start, &range.limit); if (!status.ok()) { - if (status.IsShutdownInProgress()) { + if (status.IsIncomplete()) { + continue; + } else if (status.IsShutdownInProgress()) { break; } rdb_handle_io_error(status, RDB_IO_ERROR_BG_THREAD); } - if (is_myrocks_index_empty(cfh, is_reverse_cf, read_opts, d.index_id)) { + if (is_myrocks_index_empty(cfh.get(), is_reverse_cf, read_opts, d.index_id)) { finished.insert(d); } } @@ -11729,6 +12405,35 @@ void Rdb_drop_index_thread::run() { dict_manager.finish_drop_indexes(finished); } } + + // Remove dropped column family + // 1. Get all cf ids from ongoing_index_drop. + // 2. Get all cf ids for cfs marked as dropped. + // 3. If a cf id is in the list of ongoing_index_drop + // , skip removing this cf. It will be removed later. + // 4. If it is not, proceed to remove the cf. + // + // This should be under dict_manager lock + std::lock_guard dm_lock(dict_manager); + std::unordered_set dropped_cf_ids; + dict_manager.get_all_dropped_cfs(&dropped_cf_ids); + + if (!dropped_cf_ids.empty()) { + std::unordered_set ongoing_drop_indices; + dict_manager.get_ongoing_drop_indexes(&ongoing_drop_indices); + + std::unordered_set ongoing_drop_cf_ids; + for (const auto index : ongoing_drop_indices) { + ongoing_drop_cf_ids.insert(index.cf_id); + } + + for (const auto cf_id : dropped_cf_ids) { + if (ongoing_drop_cf_ids.find(cf_id) == ongoing_drop_cf_ids.end()) { + cf_manager.remove_dropped_cf(&dict_manager, rdb, cf_id); + } + } + } + RDB_MUTEX_LOCK_CHECK(m_signal_mutex); } @@ -11765,10 +12470,19 @@ int ha_rocksdb::delete_table(Rdb_tbl_def *const tbl) { const std::unique_ptr wb = dict_manager.begin(); rocksdb::WriteBatch *const batch = wb.get(); - dict_manager.add_drop_table(tbl->m_key_descr_arr, tbl->m_key_count, batch); + DBUG_EXECUTE_IF("rocksdb_before_delete_table", { + const char act[] = + "now signal ready_to_mark_cf_dropped_before_delete_table wait_for " + "mark_cf_dropped_done_before_delete_table"; + DBUG_ASSERT(!debug_sync_set_action(ha_thd(), STRING_WITH_LEN(act))); + }); std::string path = std::string("./") + tbl->base_dbname() + "/" + tbl->base_tablename(); + + std::lock_guard dm_lock(dict_manager); + dict_manager.add_drop_table(tbl->m_key_descr_arr, tbl->m_key_count, batch); + /* Remove the table entry in data dictionary (this will also remove it from the persistent data dictionary). @@ -11781,6 +12495,13 @@ int ha_rocksdb::delete_table(Rdb_tbl_def *const tbl) { DBUG_RETURN(err); } + DBUG_EXECUTE_IF("rocksdb_after_delete_table", { + const char act[] = + "now signal ready_to_mark_cf_dropped_after_delete_table " + "wait_for mark_cf_dropped_done_after_delete_table"; + DBUG_ASSERT(!debug_sync_set_action(ha_thd(), STRING_WITH_LEN(act))); + }); + rdb_drop_idx_thread.signal(); // avoid dangling pointer m_tbl_def = nullptr; @@ -11839,12 +12560,17 @@ int ha_rocksdb::remove_rows(Rdb_tbl_def *const tbl) { kd.get_infimum_key(reinterpret_cast(key_buf), &key_len); rocksdb::ColumnFamilyHandle *cf = kd.get_cf(); const rocksdb::Slice table_key(key_buf, key_len); - setup_iterator_bounds(kd, table_key, Rdb_key_def::INDEX_NUMBER_SIZE, - lower_bound_buf, upper_bound_buf, &lower_bound_slice, - &upper_bound_slice); DBUG_ASSERT(key_len == Rdb_key_def::INDEX_NUMBER_SIZE); - opts.iterate_lower_bound = &lower_bound_slice; - opts.iterate_upper_bound = &upper_bound_slice; + if (THDVAR(ha_thd(), enable_iterate_bounds)) { + setup_iterator_bounds(kd, table_key, Rdb_key_def::INDEX_NUMBER_SIZE, + lower_bound_buf, upper_bound_buf, + &lower_bound_slice, &upper_bound_slice); + opts.iterate_lower_bound = &lower_bound_slice; + opts.iterate_upper_bound = &upper_bound_slice; + } else { + opts.iterate_lower_bound = nullptr; + opts.iterate_upper_bound = nullptr; + } std::unique_ptr it(rdb->NewIterator(opts, cf)); it->Seek(table_key); @@ -11887,10 +12613,15 @@ int ha_rocksdb::rename_table(const char *const from, const char *const to) { std::string to_str; std::string from_db; std::string to_db; + int rc = HA_EXIT_SUCCESS; - int rc = rdb_normalize_tablename(from, &from_str); - if (rc != HA_EXIT_SUCCESS) { - DBUG_RETURN(rc); + if (rdb_is_tablename_normalized(from)) { + from_str = from; + } else { + rc = rdb_normalize_tablename(from, &from_str); + if (rc != HA_EXIT_SUCCESS) { + DBUG_RETURN(rc); + } } rc = rdb_split_normalized_tablename(from_str, &from_db); @@ -11898,9 +12629,13 @@ int ha_rocksdb::rename_table(const char *const from, const char *const to) { DBUG_RETURN(rc); } - rc = rdb_normalize_tablename(to, &to_str); - if (rc != HA_EXIT_SUCCESS) { - DBUG_RETURN(rc); + if (rdb_is_tablename_normalized(to)) { + to_str = to; + } else { + rc = rdb_normalize_tablename(to, &to_str); + if (rc != HA_EXIT_SUCCESS) { + DBUG_RETURN(rc); + } } rc = rdb_split_normalized_tablename(to_str, &to_db); @@ -11921,6 +12656,9 @@ int ha_rocksdb::rename_table(const char *const from, const char *const to) { const std::unique_ptr wb = dict_manager.begin(); rocksdb::WriteBatch *const batch = wb.get(); + + // rename table is under dict_manager lock, and the cfs used + // by indices of this table cannot be dropped during the process. dict_manager.lock(); if (ddl_manager.rename(from_str, to_str, batch)) { @@ -12070,9 +12808,10 @@ ha_rows ha_rocksdb::records_in_range(uint inx, const key_range *const min_key, } // Getting statistics, including from Memtables - uint8_t include_flags = rocksdb::DB::INCLUDE_FILES; + rocksdb::DB::SizeApproximationFlags include_flags = + rocksdb::DB::SizeApproximationFlags::INCLUDE_FILES; rdb->GetApproximateSizes(kd.get_cf(), &r, 1, &sz, include_flags); - ret = rows * sz / disk_size; + ret = rows * ((double)sz / (double)disk_size); uint64_t memTableCount; rdb->GetApproximateMemTableStats(kd.get_cf(), r, &memTableCount, &sz); ret += memTableCount; @@ -12243,6 +12982,7 @@ static int calculate_stats( rocksdb::Slice first_index_key((const char *)r_buf, Rdb_key_def::INDEX_NUMBER_SIZE); + // Reset m_last_key for new index cardinality_collector.Reset(); for (it->Seek(first_index_key); is_valid(it.get()); it->Next()) { const rocksdb::Slice key = it->key(); @@ -12767,6 +13507,31 @@ int ha_rocksdb::inplace_populate_sk( const std::unique_ptr wb = dict_manager.begin(); rocksdb::WriteBatch *const batch = wb.get(); + DBUG_EXECUTE_IF("rocksdb_inplace_populate_sk", { + const char act[] = + "now signal ready_to_mark_cf_dropped_in_populate_sk " + "wait_for mark_cf_dropped_done_in_populate_sk"; + DBUG_ASSERT(!debug_sync_set_action(ha_thd(), STRING_WITH_LEN(act))); + }); + + std::lock_guard dm_lock(dict_manager); + for (const auto &kd : indexes) { + const std::string cf_name = kd->get_cf()->GetName(); + std::shared_ptr cfh = + cf_manager.get_cf(cf_name); + + if (!cfh || cfh != kd->get_shared_cf()) { + // The CF has been dropped, i.e., cf_manager.remove_dropped_cf() has + // been called. + DBUG_RETURN(HA_EXIT_FAILURE); + } + + uint32 cf_id = cfh->GetID(); + if (dict_manager.get_dropped_cf(cf_id)) { + DBUG_RETURN(HA_EXIT_FAILURE); + } + } + /* Update the data dictionary */ std::unordered_set create_index_ids; for (const auto &index : indexes) { @@ -12851,7 +13616,8 @@ int ha_rocksdb::inplace_populate_sk( keys. */ const uint pk = pk_index(table, m_tbl_def); - ha_index_init(pk, true); + res = ha_index_init(pk, true); + if (res) DBUG_RETURN(res); /* Scan each record in the primary key in order */ for (res = index_first(table->record[0]); res == 0; @@ -13047,11 +13813,25 @@ bool ha_rocksdb::commit_inplace_alter_table( delete ctx0->m_new_tdef; } + std::lock_guard dm_lock(dict_manager); /* Remove uncommitted key definitons from ddl_manager */ ddl_manager.remove_uncommitted_keydefs(ctx0->m_added_indexes); - /* Rollback any partially created indexes */ - dict_manager.rollback_ongoing_index_creation(); + std::unordered_set all_gl_index_ids; + dict_manager.get_ongoing_create_indexes(&all_gl_index_ids); + + std::unordered_set gl_index_ids; + for (auto index : ctx0->m_added_indexes) { + auto gl_index_id = index->get_gl_index_id(); + if (all_gl_index_ids.find(gl_index_id) != all_gl_index_ids.end()) { + gl_index_ids.insert(gl_index_id); + } + } + + if (!gl_index_ids.empty()) { + /* Rollback any partially created indexes of this table */ + dict_manager.rollback_ongoing_index_creation(gl_index_ids); + } DBUG_RETURN(HA_EXIT_SUCCESS); } @@ -13090,7 +13870,14 @@ bool ha_rocksdb::commit_inplace_alter_table( m_key_descr_arr = m_tbl_def->m_key_descr_arr; m_pk_descr = m_key_descr_arr[pk_index(altered_table, m_tbl_def)]; - dict_manager.lock(); + DBUG_EXECUTE_IF("rocksdb_commit_alter_table", { + const char act[] = + "now signal ready_to_mark_cf_dropped_before_commit_alter_table " + "wait_for mark_cf_dropped_done_before_commit_alter_table"; + DBUG_ASSERT(!debug_sync_set_action(ha_thd(), STRING_WITH_LEN(act))); + }); + + std::lock_guard dm_lock(dict_manager); for (inplace_alter_handler_ctx **pctx = ctx_array; *pctx; pctx++) { Rdb_inplace_alter_ctx *const ctx = static_cast(*pctx); @@ -13115,27 +13902,26 @@ bool ha_rocksdb::commit_inplace_alter_table( committed into the data dictionary. */ ddl_manager.remove_uncommitted_keydefs(ctx->m_added_indexes); + + if (dict_manager.commit(batch)) { + /* + Should never reach here. We assume MyRocks will abort if commit + fails. + */ + DBUG_ASSERT(0); + } + + /* Mark ongoing create indexes as finished/remove from data dictionary */ + dict_manager.finish_indexes_operation( + create_index_ids, Rdb_key_def::DDL_CREATE_INDEX_ONGOING); } - /* - Increment the table version. - */ - ulonglong table_ver = get_table_version(table->s->path.str); - table_ver++; - save_table_version(batch, table->s->path.str, table_ver); - - if (dict_manager.commit(batch)) { - /* - Should never reach here. We assume MyRocks will abort if commit fails. - */ - DBUG_ASSERT(0); - } - - dict_manager.unlock(); - - /* Mark ongoing create indexes as finished/remove from data dictionary */ - dict_manager.finish_indexes_operation( - create_index_ids, Rdb_key_def::DDL_CREATE_INDEX_ONGOING); + DBUG_EXECUTE_IF("rocksdb_delete_index", { + const char act[] = + "now signal ready_to_mark_cf_dropped_after_commit_alter_table " + "wait_for mark_cf_dropped_done_after_commit_alter_table"; + DBUG_ASSERT(!debug_sync_set_action(ha_thd(), STRING_WITH_LEN(act))); + }); rdb_drop_idx_thread.signal(); } @@ -13196,12 +13982,10 @@ struct rocksdb_status_counters_t { uint64_t block_cache_index_hit; uint64_t block_cache_index_add; uint64_t block_cache_index_bytes_insert; - uint64_t block_cache_index_bytes_evict; uint64_t block_cache_filter_miss; uint64_t block_cache_filter_hit; uint64_t block_cache_filter_add; uint64_t block_cache_filter_bytes_insert; - uint64_t block_cache_filter_bytes_evict; uint64_t block_cache_bytes_read; uint64_t block_cache_bytes_write; uint64_t block_cache_data_bytes_insert; @@ -13231,27 +14015,21 @@ struct rocksdb_status_counters_t { uint64_t number_db_prev; uint64_t number_db_prev_found; uint64_t iter_bytes_read; - uint64_t no_file_closes; uint64_t no_file_opens; uint64_t no_file_errors; uint64_t stall_micros; - uint64_t num_iterators; uint64_t number_multiget_get; uint64_t number_multiget_keys_read; uint64_t number_multiget_bytes_read; - uint64_t number_deletes_filtered; uint64_t number_merge_failures; uint64_t bloom_filter_prefix_checked; uint64_t bloom_filter_prefix_useful; uint64_t number_reseeks_iteration; uint64_t getupdatessince_calls; - uint64_t block_cachecompressed_miss; - uint64_t block_cachecompressed_hit; uint64_t wal_synced; uint64_t wal_bytes; uint64_t write_self; uint64_t write_other; - uint64_t write_timedout; uint64_t write_wal; uint64_t flush_write_bytes; uint64_t compact_read_bytes; @@ -13272,12 +14050,10 @@ DEF_SHOW_FUNC(block_cache_index_miss, BLOCK_CACHE_INDEX_MISS) DEF_SHOW_FUNC(block_cache_index_hit, BLOCK_CACHE_INDEX_HIT) DEF_SHOW_FUNC(block_cache_index_add, BLOCK_CACHE_INDEX_ADD) DEF_SHOW_FUNC(block_cache_index_bytes_insert, BLOCK_CACHE_INDEX_BYTES_INSERT) -DEF_SHOW_FUNC(block_cache_index_bytes_evict, BLOCK_CACHE_INDEX_BYTES_EVICT) DEF_SHOW_FUNC(block_cache_filter_miss, BLOCK_CACHE_FILTER_MISS) DEF_SHOW_FUNC(block_cache_filter_hit, BLOCK_CACHE_FILTER_HIT) DEF_SHOW_FUNC(block_cache_filter_add, BLOCK_CACHE_FILTER_ADD) DEF_SHOW_FUNC(block_cache_filter_bytes_insert, BLOCK_CACHE_FILTER_BYTES_INSERT) -DEF_SHOW_FUNC(block_cache_filter_bytes_evict, BLOCK_CACHE_FILTER_BYTES_EVICT) DEF_SHOW_FUNC(block_cache_bytes_read, BLOCK_CACHE_BYTES_READ) DEF_SHOW_FUNC(block_cache_bytes_write, BLOCK_CACHE_BYTES_WRITE) DEF_SHOW_FUNC(block_cache_data_bytes_insert, BLOCK_CACHE_DATA_BYTES_INSERT) @@ -13307,27 +14083,21 @@ DEF_SHOW_FUNC(number_db_next_found, NUMBER_DB_NEXT_FOUND) DEF_SHOW_FUNC(number_db_prev, NUMBER_DB_PREV) DEF_SHOW_FUNC(number_db_prev_found, NUMBER_DB_PREV_FOUND) DEF_SHOW_FUNC(iter_bytes_read, ITER_BYTES_READ) -DEF_SHOW_FUNC(no_file_closes, NO_FILE_CLOSES) DEF_SHOW_FUNC(no_file_opens, NO_FILE_OPENS) DEF_SHOW_FUNC(no_file_errors, NO_FILE_ERRORS) DEF_SHOW_FUNC(stall_micros, STALL_MICROS) -DEF_SHOW_FUNC(num_iterators, NO_ITERATORS) DEF_SHOW_FUNC(number_multiget_get, NUMBER_MULTIGET_CALLS) DEF_SHOW_FUNC(number_multiget_keys_read, NUMBER_MULTIGET_KEYS_READ) DEF_SHOW_FUNC(number_multiget_bytes_read, NUMBER_MULTIGET_BYTES_READ) -DEF_SHOW_FUNC(number_deletes_filtered, NUMBER_FILTERED_DELETES) DEF_SHOW_FUNC(number_merge_failures, NUMBER_MERGE_FAILURES) DEF_SHOW_FUNC(bloom_filter_prefix_checked, BLOOM_FILTER_PREFIX_CHECKED) DEF_SHOW_FUNC(bloom_filter_prefix_useful, BLOOM_FILTER_PREFIX_USEFUL) DEF_SHOW_FUNC(number_reseeks_iteration, NUMBER_OF_RESEEKS_IN_ITERATION) DEF_SHOW_FUNC(getupdatessince_calls, GET_UPDATES_SINCE_CALLS) -DEF_SHOW_FUNC(block_cachecompressed_miss, BLOCK_CACHE_COMPRESSED_MISS) -DEF_SHOW_FUNC(block_cachecompressed_hit, BLOCK_CACHE_COMPRESSED_HIT) DEF_SHOW_FUNC(wal_synced, WAL_FILE_SYNCED) DEF_SHOW_FUNC(wal_bytes, WAL_FILE_BYTES) DEF_SHOW_FUNC(write_self, WRITE_DONE_BY_SELF) DEF_SHOW_FUNC(write_other, WRITE_DONE_BY_OTHER) -DEF_SHOW_FUNC(write_timedout, WRITE_TIMEDOUT) DEF_SHOW_FUNC(write_wal, WRITE_WITH_WAL) DEF_SHOW_FUNC(flush_write_bytes, FLUSH_WRITE_BYTES) DEF_SHOW_FUNC(compact_read_bytes, COMPACT_READ_BYTES) @@ -13417,10 +14187,9 @@ static int show_myrocks_vars(THD *thd, SHOW_VAR *var, void *buff, return 0; } -static ulonglong io_stall_prop_value( +static ulonglong get_prop_value_as_ulong( const std::map &props, const std::string &key) { - std::map::const_iterator iter = - props.find("io_stalls." + key); + std::map::const_iterator iter = props.find(key); if (iter != props.end()) { return std::stoull(iter->second); } else { @@ -13434,35 +14203,60 @@ static ulonglong io_stall_prop_value( static void update_rocksdb_stall_status() { st_io_stall_stats local_io_stall_stats; for (const auto &cf_name : cf_manager.get_cf_names()) { - rocksdb::ColumnFamilyHandle *cfh = cf_manager.get_cf(cf_name); - if (cfh == nullptr) { + std::shared_ptr cfh = + cf_manager.get_cf(cf_name); + if (!cfh) { continue; } + // Retrieve information from valid CF handle object. It is safe + // even if the CF is removed from cf_manager at this point. std::map props; - if (!rdb->GetMapProperty(cfh, "rocksdb.cfstats", &props)) { + if (!rdb->GetMapProperty(cfh.get(), "rocksdb.cfstats", &props)) { continue; } - local_io_stall_stats.level0_slowdown += - io_stall_prop_value(props, "level0_slowdown"); + using rocksdb::WriteStallCause; + using rocksdb::WriteStallCondition; + using rocksdb::WriteStallStatsMapKeys; + local_io_stall_stats.level0_slowdown += get_prop_value_as_ulong( + props, + WriteStallStatsMapKeys::CauseConditionCount( + WriteStallCause::kL0FileCountLimit, WriteStallCondition::kDelayed)); local_io_stall_stats.level0_slowdown_with_compaction += - io_stall_prop_value(props, "level0_slowdown_with_compaction"); - local_io_stall_stats.level0_numfiles += - io_stall_prop_value(props, "level0_numfiles"); - local_io_stall_stats.level0_numfiles_with_compaction += - io_stall_prop_value(props, "level0_numfiles_with_compaction"); + get_prop_value_as_ulong( + props, WriteStallStatsMapKeys:: + CFL0FileCountLimitDelaysWithOngoingCompaction()); + local_io_stall_stats.level0_numfiles += get_prop_value_as_ulong( + props, + WriteStallStatsMapKeys::CauseConditionCount( + WriteStallCause::kL0FileCountLimit, WriteStallCondition::kStopped)); + local_io_stall_stats + .level0_numfiles_with_compaction += get_prop_value_as_ulong( + props, + WriteStallStatsMapKeys::CFL0FileCountLimitStopsWithOngoingCompaction()); local_io_stall_stats.stop_for_pending_compaction_bytes += - io_stall_prop_value(props, "stop_for_pending_compaction_bytes"); + get_prop_value_as_ulong(props, + WriteStallStatsMapKeys::CauseConditionCount( + WriteStallCause::kPendingCompactionBytes, + WriteStallCondition::kStopped)); local_io_stall_stats.slowdown_for_pending_compaction_bytes += - io_stall_prop_value(props, "slowdown_for_pending_compaction_bytes"); - local_io_stall_stats.memtable_compaction += - io_stall_prop_value(props, "memtable_compaction"); - local_io_stall_stats.memtable_slowdown += - io_stall_prop_value(props, "memtable_slowdown"); - local_io_stall_stats.total_stop += io_stall_prop_value(props, "total_stop"); + get_prop_value_as_ulong(props, + WriteStallStatsMapKeys::CauseConditionCount( + WriteStallCause::kPendingCompactionBytes, + WriteStallCondition::kDelayed)); + local_io_stall_stats.memtable_compaction += get_prop_value_as_ulong( + props, + WriteStallStatsMapKeys::CauseConditionCount( + WriteStallCause::kMemtableLimit, WriteStallCondition::kStopped)); + local_io_stall_stats.memtable_slowdown += get_prop_value_as_ulong( + props, + WriteStallStatsMapKeys::CauseConditionCount( + WriteStallCause::kMemtableLimit, WriteStallCondition::kDelayed)); + local_io_stall_stats.total_stop += + get_prop_value_as_ulong(props, WriteStallStatsMapKeys::TotalStops()); local_io_stall_stats.total_slowdown += - io_stall_prop_value(props, "total_slowdown"); + get_prop_value_as_ulong(props, WriteStallStatsMapKeys::TotalDelays()); } io_stall_stats = local_io_stall_stats; } @@ -13513,12 +14307,10 @@ static SHOW_VAR rocksdb_status_vars[] = { DEF_STATUS_VAR(block_cache_index_hit), DEF_STATUS_VAR(block_cache_index_add), DEF_STATUS_VAR(block_cache_index_bytes_insert), - DEF_STATUS_VAR(block_cache_index_bytes_evict), DEF_STATUS_VAR(block_cache_filter_miss), DEF_STATUS_VAR(block_cache_filter_hit), DEF_STATUS_VAR(block_cache_filter_add), DEF_STATUS_VAR(block_cache_filter_bytes_insert), - DEF_STATUS_VAR(block_cache_filter_bytes_evict), DEF_STATUS_VAR(block_cache_bytes_read), DEF_STATUS_VAR(block_cache_bytes_write), DEF_STATUS_VAR(block_cache_data_bytes_insert), @@ -13548,27 +14340,21 @@ static SHOW_VAR rocksdb_status_vars[] = { DEF_STATUS_VAR(number_db_prev), DEF_STATUS_VAR(number_db_prev_found), DEF_STATUS_VAR(iter_bytes_read), - DEF_STATUS_VAR(no_file_closes), DEF_STATUS_VAR(no_file_opens), DEF_STATUS_VAR(no_file_errors), DEF_STATUS_VAR(stall_micros), - DEF_STATUS_VAR(num_iterators), DEF_STATUS_VAR(number_multiget_get), DEF_STATUS_VAR(number_multiget_keys_read), DEF_STATUS_VAR(number_multiget_bytes_read), - DEF_STATUS_VAR(number_deletes_filtered), DEF_STATUS_VAR(number_merge_failures), DEF_STATUS_VAR(bloom_filter_prefix_checked), DEF_STATUS_VAR(bloom_filter_prefix_useful), DEF_STATUS_VAR(number_reseeks_iteration), DEF_STATUS_VAR(getupdatessince_calls), - DEF_STATUS_VAR(block_cachecompressed_miss), - DEF_STATUS_VAR(block_cachecompressed_hit), DEF_STATUS_VAR(wal_synced), DEF_STATUS_VAR(wal_bytes), DEF_STATUS_VAR(write_self), DEF_STATUS_VAR(write_other), - DEF_STATUS_VAR(write_timedout), DEF_STATUS_VAR(write_wal), DEF_STATUS_VAR(flush_write_bytes), DEF_STATUS_VAR(compact_read_bytes), @@ -13587,8 +14373,12 @@ static SHOW_VAR rocksdb_status_vars[] = { SHOW_LONGLONG), DEF_STATUS_VAR_PTR("manual_compactions_processed", &rocksdb_manual_compactions_processed, SHOW_LONGLONG), + DEF_STATUS_VAR_PTR("manual_compactions_cancelled", + &rocksdb_manual_compactions_cancelled, SHOW_LONGLONG), DEF_STATUS_VAR_PTR("manual_compactions_running", &rocksdb_manual_compactions_running, SHOW_LONGLONG), + DEF_STATUS_VAR_PTR("manual_compactions_pending", + &rocksdb_manual_compactions_pending, SHOW_LONGLONG), DEF_STATUS_VAR_PTR("number_sst_entry_put", &rocksdb_num_sst_entry_put, SHOW_LONGLONG), DEF_STATUS_VAR_PTR("number_sst_entry_delete", &rocksdb_num_sst_entry_delete, @@ -13732,48 +14522,92 @@ void Rdb_manual_compaction_thread::run() { RDB_MUTEX_UNLOCK_CHECK(m_signal_mutex); RDB_MUTEX_LOCK_CHECK(m_mc_mutex); - // Grab the first item and proceed, if not empty. + // Grab the first PENDING state item and proceed, if not empty. if (m_requests.empty()) { RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); RDB_MUTEX_LOCK_CHECK(m_signal_mutex); continue; } - Manual_compaction_request &mcr = m_requests.begin()->second; - DBUG_ASSERT(mcr.cf != nullptr); - DBUG_ASSERT(mcr.state == Manual_compaction_request::INITED); + + auto it = m_requests.begin(); + auto pending_it = m_requests.end(); + // Remove all items with client_done. client_done means + // caller no longer uses the mcr object so it is safe to erase. + // Pick first PENDING state item + it = m_requests.begin(); + while (it != m_requests.end()) { + if (it->second.client_done) { + m_requests.erase(it++); + } else if (it->second.state == Manual_compaction_request::PENDING && + pending_it == m_requests.end()) { + // found + pending_it = it; + it++; + } else { + it++; + } + } + if (pending_it == m_requests.end()) { + RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); + RDB_MUTEX_LOCK_CHECK(m_signal_mutex); + continue; + } + + Manual_compaction_request &mcr = pending_it->second; + DBUG_ASSERT(mcr.cf); + DBUG_ASSERT(mcr.state == Manual_compaction_request::PENDING); mcr.state = Manual_compaction_request::RUNNING; + rocksdb_manual_compactions_running++; + rocksdb_manual_compactions_pending--; RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); DBUG_ASSERT(mcr.state == Manual_compaction_request::RUNNING); // NO_LINT_DEBUG sql_print_information("Manual Compaction id %d cf %s started.", mcr.mc_id, mcr.cf->GetName().c_str()); - rocksdb_manual_compactions_running++; if (rocksdb_debug_manual_compaction_delay > 0) { my_sleep(rocksdb_debug_manual_compaction_delay * 1000000); } // CompactRange may take a very long time. On clean shutdown, // it is cancelled by CancelAllBackgroundWork, then status is // set to shutdownInProgress. - const rocksdb::Status s = rdb->CompactRange( - getCompactRangeOptions(mcr.concurrency), mcr.cf, mcr.start, mcr.limit); + const rocksdb::Status s = + rdb->CompactRange(mcr.option, mcr.cf.get(), mcr.start, mcr.limit); + rocksdb_manual_compactions_running--; if (s.ok()) { + rocksdb_manual_compactions_processed++; // NO_LINT_DEBUG sql_print_information("Manual Compaction id %d cf %s ended.", mcr.mc_id, mcr.cf->GetName().c_str()); + set_state(mcr, Manual_compaction_request::SUCCESS); } else { - // NO_LINT_DEBUG - sql_print_information("Manual Compaction id %d cf %s aborted. %s", - mcr.mc_id, mcr.cf->GetName().c_str(), s.getState()); - if (!s.IsShutdownInProgress()) { - rdb_handle_io_error(s, RDB_IO_ERROR_BG_THREAD); + if (!cf_manager.get_cf(mcr.cf->GetID())) { + // NO_LINT_DEBUG + sql_print_information("cf %s has been dropped", + mcr.cf->GetName().c_str()); + set_state(mcr, Manual_compaction_request::SUCCESS); + } else if (s.IsIncomplete()) { + // NO_LINT_DEBUG + sql_print_information( + "Manual Compaction id %d cf %s cancelled. (%d:%d, %s)", mcr.mc_id, + mcr.cf->GetName().c_str(), s.code(), s.subcode(), s.getState()); + // Cancelled + set_state(mcr, Manual_compaction_request::CANCEL); + rocksdb_manual_compactions_cancelled++; } else { - DBUG_ASSERT(m_requests.size() == 1); + // NO_LINT_DEBUG + sql_print_information( + "Manual Compaction id %d cf %s aborted. (%d:%d, %s)", mcr.mc_id, + mcr.cf->GetName().c_str(), s.code(), s.subcode(), s.getState()); + set_state(mcr, Manual_compaction_request::FAILURE); + if (!s.IsShutdownInProgress()) { + rdb_handle_io_error(s, RDB_IO_ERROR_BG_THREAD); + } else { + DBUG_ASSERT(m_requests.size() == 1); + } } } - rocksdb_manual_compactions_processed++; - clear_manual_compaction_request(mcr.mc_id, false); RDB_MUTEX_LOCK_CHECK(m_signal_mutex); } clear_all_manual_compaction_requests(); @@ -13784,37 +14618,84 @@ void Rdb_manual_compaction_thread::run() { void Rdb_manual_compaction_thread::clear_all_manual_compaction_requests() { RDB_MUTEX_LOCK_CHECK(m_mc_mutex); + DBUG_ASSERT(rocksdb_manual_compactions_pending == 0); m_requests.clear(); RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); } -void Rdb_manual_compaction_thread::clear_manual_compaction_request( - int mc_id, bool init_only) { - bool erase = true; +void Rdb_manual_compaction_thread:: + cancel_all_pending_manual_compaction_requests() { RDB_MUTEX_LOCK_CHECK(m_mc_mutex); - auto it = m_requests.find(mc_id); - if (it != m_requests.end()) { - if (init_only) { - Manual_compaction_request mcr = it->second; - if (mcr.state != Manual_compaction_request::INITED) { - erase = false; - } + auto it = m_requests.begin(); + while (it != m_requests.end()) { + Manual_compaction_request &mcr = it->second; + if (mcr.state == Manual_compaction_request::PENDING) { + mcr.state = Manual_compaction_request::CANCEL; + rocksdb_manual_compactions_cancelled++; + rocksdb_manual_compactions_pending--; } - if (erase) { - m_requests.erase(it); - } - } else { - // Current code path guarantees that erasing by the same mc_id happens - // at most once. INITED state may be erased by a thread that requested - // the compaction. RUNNING state is erased by mc thread only. - DBUG_ASSERT(0); + it++; } + DBUG_ASSERT(rocksdb_manual_compactions_pending == 0); RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); } +/** + * Requesting to cancel a Manual Compaction job with mc_id. + * Only PENDING or RUNNING states need cancellation. + * This function may take a while if state is RUNNING. + * Returning true if hitting timeout and state is RUNNING. + */ +bool Rdb_manual_compaction_thread::cancel_manual_compaction_request( + const int mc_id, const int timeout_100ms) { + Manual_compaction_request::mc_state state = + Manual_compaction_request::PENDING; + + RDB_MUTEX_LOCK_CHECK(m_mc_mutex); + auto it = m_requests.find(mc_id); + if (it != m_requests.end()) { + Manual_compaction_request &mcr = it->second; + if (mcr.state == Manual_compaction_request::PENDING) { + mcr.state = Manual_compaction_request::CANCEL; + rocksdb_manual_compactions_cancelled++; + rocksdb_manual_compactions_pending--; + RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); + return false; + } else if (mcr.state == Manual_compaction_request::RUNNING) { + // explicitly requesting to cancel compaction (cancellation happens in + // background may take time) + mcr.option.canceled->store(true, std::memory_order_release); + state = mcr.state; + } + } + RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); + + // Waiting to get manual compaction to get cancelled. + // Even if returning timeouts to clients, manual compaction + // is still running so further compactions can remain + // in pending state until the compaction completes cancellation. + uint64_t retry = timeout_100ms; + while (retry > 0 && state == Manual_compaction_request::RUNNING) { + my_sleep(100000); + retry--; + state = manual_compaction_state(mc_id); + } + + return retry <= 0 && state == Manual_compaction_request::RUNNING; +} + +/** + * This function is for clients to request for Manual Compaction. + * This function adds mcr (Manual Compaction Request) in a queue + * as PENDING state then returns. Worker Thread then later picks it up + * and processes compaction. + * Clients should call set_client_done() when the clients are done with + * the status of the requests. + */ int Rdb_manual_compaction_thread::request_manual_compaction( - rocksdb::ColumnFamilyHandle *cf, rocksdb::Slice *start, - rocksdb::Slice *limit, int concurrency) { + std::shared_ptr cf, rocksdb::Slice *start, + rocksdb::Slice *limit, const uint manual_compaction_threads, + const rocksdb::BottommostLevelCompaction bottommost_level_compaction) { int mc_id = -1; RDB_MUTEX_LOCK_CHECK(m_mc_mutex); if (m_requests.size() >= rocksdb_max_manual_compactions) { @@ -13823,24 +14704,55 @@ int Rdb_manual_compaction_thread::request_manual_compaction( } Manual_compaction_request mcr; mc_id = mcr.mc_id = ++m_latest_mc_id; - mcr.state = Manual_compaction_request::INITED; + mcr.state = Manual_compaction_request::PENDING; mcr.cf = cf; mcr.start = start; mcr.limit = limit; - mcr.concurrency = concurrency; + mcr.option = getCompactRangeOptions(manual_compaction_threads, + bottommost_level_compaction); + mcr.canceled = + std::shared_ptr>(new std::atomic(false)); + mcr.option.canceled = mcr.canceled.get(); + mcr.client_done = false; + rocksdb_manual_compactions_pending++; m_requests.insert(std::make_pair(mcr.mc_id, mcr)); RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); return mc_id; } -bool Rdb_manual_compaction_thread::is_manual_compaction_finished(int mc_id) { - bool finished = false; +Rdb_manual_compaction_thread::Manual_compaction_request::mc_state +Rdb_manual_compaction_thread::manual_compaction_state(const int mc_id) { + Manual_compaction_request::mc_state state = + Manual_compaction_request::SUCCESS; RDB_MUTEX_LOCK_CHECK(m_mc_mutex); - if (m_requests.count(mc_id) == 0) { - finished = true; + auto it = m_requests.find(mc_id); + if (it != m_requests.end()) { + Manual_compaction_request &mcr = it->second; + state = mcr.state; } RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); - return finished; + return state; +} + +void Rdb_manual_compaction_thread::set_state( + Manual_compaction_request & mcr, + const Manual_compaction_request::mc_state new_state) { + RDB_MUTEX_LOCK_CHECK(m_mc_mutex); + mcr.state = new_state; + RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); +} + +bool Rdb_manual_compaction_thread::set_client_done(const int mc_id) { + bool rc = false; + RDB_MUTEX_LOCK_CHECK(m_mc_mutex); + auto it = m_requests.find(mc_id); + if (it != m_requests.end()) { + Manual_compaction_request &mcr = it->second; + mcr.client_done = true; + rc = true; + } + RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); + return rc; } /** @@ -13892,7 +14804,7 @@ bool ha_rocksdb::check_bloom_and_set_bounds( uchar *const upper_bound, rocksdb::Slice *lower_bound_slice, rocksdb::Slice *upper_bound_slice) { bool can_use_bloom = can_use_bloom_filter(thd, kd, eq_cond, use_all_keys); - if (!can_use_bloom) { + if (!can_use_bloom && (THDVAR(thd, enable_iterate_bounds))) { setup_iterator_bounds(kd, eq_cond, bound_len, lower_bound, upper_bound, lower_bound_slice, upper_bound_slice); } @@ -14090,8 +15002,9 @@ void rdb_handle_io_error(const rocksdb::Status status, abort(); } else if (!status.ok()) { switch (err_type) { + case RDB_IO_ERROR_TX_COMMIT: case RDB_IO_ERROR_DICT_COMMIT: { - rdb_log_status_error(status, "Failed to write to WAL (dictionary)"); + rdb_log_status_error(status, "Failed to write to WAL (non kIOError)"); /* NO_LINT_DEBUG */ sql_print_error("MyRocks: aborting on WAL write error."); abort(); @@ -14352,6 +15265,76 @@ static void rocksdb_set_max_background_jobs(THD *thd, RDB_MUTEX_UNLOCK_CHECK(rdb_sysvars_mutex); } +static void rocksdb_set_max_background_compactions(THD *thd, + struct st_mysql_sys_var *const var, + void *const var_ptr, + const void *const save) { + DBUG_ASSERT(save != nullptr); + DBUG_ASSERT(rocksdb_db_options != nullptr); + DBUG_ASSERT(rocksdb_db_options->env != nullptr); + + RDB_MUTEX_LOCK_CHECK(rdb_sysvars_mutex); + + const int new_val = *static_cast(save); + + if (rocksdb_db_options->max_background_compactions != new_val) { + rocksdb_db_options->max_background_compactions = new_val; + rocksdb::Status s = + rdb->SetDBOptions({{"max_background_compactions", std::to_string(new_val)}}); + + if (!s.ok()) { + /* NO_LINT_DEBUG */ + sql_print_warning( + "MyRocks: failed to update max_background_compactions. " + "Status code = %d, status = %s.", + s.code(), s.ToString().c_str()); + } + } + + RDB_MUTEX_UNLOCK_CHECK(rdb_sysvars_mutex); +} + +/** + rocksdb_set_max_bottom_pri_background_compactions_internal() changes + the number of rocksdb background threads. + Creating new threads may take up to a few seconds, so instead of + calling the function at sys_var::update path where global mutex is held, + doing at sys_var::check path so that other queries are not blocked. + Same optimization is done for rocksdb_block_cache_size too. +*/ +static int rocksdb_validate_max_bottom_pri_background_compactions( + THD *thd MY_ATTRIBUTE((__unused__)), + struct st_mysql_sys_var *const var MY_ATTRIBUTE((__unused__)), + void *var_ptr, struct st_mysql_value *value) { + DBUG_ASSERT(value != nullptr); + + long long new_value; + + /* value is NULL */ + if (value->val_int(value, &new_value)) { + return HA_EXIT_FAILURE; + } + if (new_value < 0 || + new_value > ROCKSDB_MAX_BOTTOM_PRI_BACKGROUND_COMPACTIONS) { + return HA_EXIT_FAILURE; + } + RDB_MUTEX_LOCK_CHECK(rdb_bottom_pri_background_compactions_resize_mutex); + if (rocksdb_max_bottom_pri_background_compactions != new_value) { + if (new_value == 0) { + my_error(ER_ERROR_WHEN_EXECUTING_COMMAND, MYF(0), "SET", + "max_bottom_pri_background_compactions can't be changed to 0 " + "online."); + RDB_MUTEX_UNLOCK_CHECK( + rdb_bottom_pri_background_compactions_resize_mutex); + return HA_EXIT_FAILURE; + } + rocksdb_set_max_bottom_pri_background_compactions_internal(new_value); + } + *static_cast(var_ptr) = static_cast(new_value); + RDB_MUTEX_UNLOCK_CHECK(rdb_bottom_pri_background_compactions_resize_mutex); + return HA_EXIT_SUCCESS; +} + static void rocksdb_set_bytes_per_sync( THD *thd MY_ATTRIBUTE((__unused__)), struct st_mysql_sys_var *const var MY_ATTRIBUTE((__unused__)), @@ -14477,7 +15460,23 @@ static int rocksdb_validate_update_cf_options( // Loop through option_map and create missing column families for (Rdb_cf_options::Name_to_config_t::iterator it = option_map.begin(); it != option_map.end(); ++it) { - cf_manager.get_or_create_cf(rdb, it->first); + // If the CF is removed at this point, i.e., cf_manager.drop_cf() has + // been called, it is OK to create a new CF. + + const auto &cf_name = it->first; + { + std::lock_guard dm_lock(dict_manager); + auto cfh = cf_manager.get_or_create_cf(rdb, cf_name); + + if (!cfh) { + return HA_EXIT_FAILURE; + } + + if (cf_manager.create_cf_flags_if_needed(&dict_manager, cfh->GetID(), + cf_name)) { + return HA_EXIT_FAILURE; + } + } } return HA_EXIT_SUCCESS; } @@ -14518,8 +15517,16 @@ static void rocksdb_set_update_cf_options( for (const auto &cf_name : cf_manager.get_cf_names()) { DBUG_ASSERT(!cf_name.empty()); - rocksdb::ColumnFamilyHandle *cfh = cf_manager.get_cf(cf_name); - DBUG_ASSERT(cfh != nullptr); + std::shared_ptr cfh = + cf_manager.get_cf(cf_name); + + if (!cfh) { + // NO_LINT_DEBUG + sql_print_information( + "Skip updating options for cf %s because the cf has been dropped.", + cf_name.c_str()); + continue; + } const auto it = option_map.find(cf_name); std::string per_cf_options = (it != option_map.end()) ? it->second : ""; @@ -14538,7 +15545,10 @@ static void rocksdb_set_update_cf_options( DBUG_ASSERT(rdb != nullptr); // Finally we can apply the options. - s = rdb->SetOptions(cfh, opt_map); + // If cf_manager.drop_cf() has been called at this point, SetOptions() + // will still succeed. The options data will only be cleared when + // the CF handle object is destroyed. + s = rdb->SetOptions(cfh.get(), opt_map); if (s != rocksdb::Status::OK()) { // NO_LINT_DEBUG @@ -14557,7 +15567,7 @@ static void rocksdb_set_update_cf_options( // the CF options. This is necessary also to make sure that the CF // options will be correctly reflected in the relevant table: // ROCKSDB_CF_OPTIONS in INFORMATION_SCHEMA. - rocksdb::ColumnFamilyOptions cf_options = rdb->GetOptions(cfh); + rocksdb::ColumnFamilyOptions cf_options = rdb->GetOptions(cfh.get()); std::string updated_options; s = rocksdb::GetStringFromColumnFamilyOptions(&updated_options, @@ -14809,5 +15819,5 @@ maria_declare_plugin(rocksdb_se){ myrocks::rdb_i_s_global_info, myrocks::rdb_i_s_ddl, myrocks::rdb_i_s_sst_props, myrocks::rdb_i_s_index_file_map, myrocks::rdb_i_s_lock_info, myrocks::rdb_i_s_trx_info, - myrocks::rdb_i_s_deadlock_info + myrocks::rdb_i_s_deadlock_info, myrocks::rdb_i_s_live_files_metadata maria_declare_plugin_end; diff --git a/storage/rocksdb/ha_rocksdb.h b/storage/rocksdb/ha_rocksdb.h index d03c183873e..8efdbc44b8e 100644 --- a/storage/rocksdb/ha_rocksdb.h +++ b/storage/rocksdb/ha_rocksdb.h @@ -93,6 +93,7 @@ enum collations_used { #if 0 // MARIAROCKS_NOT_YET : read-free replication is not supported extern char *rocksdb_read_free_rpl_tables; +extern ulong rocksdb_max_row_locks; #if defined(HAVE_PSI_INTERFACE) extern PSI_rwlock_key key_rwlock_read_free_rpl_tables; #endif @@ -290,12 +291,18 @@ class ha_rocksdb : public my_core::handler { */ bool m_insert_with_update; - /* TRUE if last time the insertion failed due to duplicated PK */ - bool m_dup_pk_found; + /* + TRUE if last time the insertion failed due to duplicate key error. + (m_dupp_errkey holds the key# that we've had error for) + */ + bool m_dup_key_found; #ifndef DBUG_OFF - /* Last retreived record for sanity checking */ - String m_dup_pk_retrieved_record; + /* + Last retrieved record (for duplicate PK) or index tuple (for duplicate + unique SK). Used for sanity checking. + */ + String m_dup_key_retrieved_record; #endif /** @@ -347,10 +354,10 @@ class ha_rocksdb : public my_core::handler { MY_ATTRIBUTE((__nonnull__)); void release_scan_iterator(void); - rocksdb::Status get_for_update( - Rdb_transaction *const tx, - rocksdb::ColumnFamilyHandle *const column_family, - const rocksdb::Slice &key, rocksdb::PinnableSlice *value) const; + rocksdb::Status get_for_update(Rdb_transaction *const tx, + const Rdb_key_def &kd, + const rocksdb::Slice &key, + rocksdb::PinnableSlice *value) const; int get_row_by_rowid(uchar *const buf, const char *const rowid, const uint rowid_size, const bool skip_lookup = false, @@ -392,13 +399,6 @@ class ha_rocksdb : public my_core::handler { void set_last_rowkey(const uchar *const old_data); - /* - For the active index, indicates which columns must be covered for the - current lookup to be covered. If the bitmap field is null, that means this - index does not cover the current lookup for any record. - */ - MY_BITMAP m_lookup_bitmap = {nullptr, nullptr, 0, 0}; - int alloc_key_buffers(const TABLE *const table_arg, const Rdb_tbl_def *const tbl_def_arg, bool alloc_alter_buffers = false) @@ -670,7 +670,7 @@ public: */ private: struct key_def_cf_info { - rocksdb::ColumnFamilyHandle *cf_handle; + std::shared_ptr cf_handle; bool is_reverse_cf; bool is_per_partition_cf; }; @@ -771,14 +771,14 @@ public: int get_pk_for_update(struct update_row_info *const row_info); int check_and_lock_unique_pk(const uint key_id, const struct update_row_info &row_info, - bool *const found) + bool *const found, const bool skip_unique_check) MY_ATTRIBUTE((__warn_unused_result__)); int check_and_lock_sk(const uint key_id, const struct update_row_info &row_info, - bool *const found) + bool *const found, const bool skip_unique_check) MY_ATTRIBUTE((__warn_unused_result__)); int check_uniqueness_and_lock(const struct update_row_info &row_info, - bool pk_changed) + bool pk_changed, const bool skip_unique_check) MY_ATTRIBUTE((__warn_unused_result__)); bool over_bulk_load_threshold(int *err) MY_ATTRIBUTE((__warn_unused_result__)); @@ -923,6 +923,8 @@ public: MY_ATTRIBUTE((__warn_unused_result__)); int create_table(const std::string &table_name, const TABLE *table_arg, ulonglong auto_increment_value); + int truncate_table(Rdb_tbl_def *tbl_def, TABLE *table_arg, + ulonglong auto_increment_value); bool check_if_incompatible_data(HA_CREATE_INFO *const info, uint table_changes) override MY_ATTRIBUTE((__warn_unused_result__)); @@ -982,6 +984,9 @@ public: bool is_read_free_rpl_table() const; #endif + void build_decoder(); + void check_build_decoder(); + #ifdef MARIAROCKS_NOT_YET // MDEV-10976 public: virtual void rpl_before_delete_rows() override; @@ -997,6 +1002,9 @@ public: bool m_in_rpl_update_rows; bool m_force_skip_unique_check; + + /* Need to build decoder on next read operation */ + bool m_need_build_decoder; }; /* diff --git a/storage/rocksdb/ha_rocksdb_proto.h b/storage/rocksdb/ha_rocksdb_proto.h index 03d24957a23..97ce207e6f8 100644 --- a/storage/rocksdb/ha_rocksdb_proto.h +++ b/storage/rocksdb/ha_rocksdb_proto.h @@ -46,6 +46,9 @@ void rdb_handle_io_error(const rocksdb::Status status, MY_ATTRIBUTE((noinline,noclone)); #endif +bool rdb_is_tablename_normalized(const std::string &tablename) + MY_ATTRIBUTE((__warn_unused_result__)); + int rdb_normalize_tablename(const std::string &tablename, std::string *str) MY_ATTRIBUTE((__nonnull__, __warn_unused_result__)); @@ -54,6 +57,11 @@ int rdb_split_normalized_tablename(const std::string &fullname, std::string *db, std::string *partition = nullptr) MY_ATTRIBUTE((__warn_unused_result__)); +void rdb_gen_normalized_tablename(const std::string *db, + const std::string *table, + const std::string *partition, + std::string *fullname); + std::vector rdb_get_open_table_names(void); class Rdb_perf_counters; @@ -66,6 +74,8 @@ void rdb_get_global_perf_counters(Rdb_perf_counters *counters) void rdb_queue_save_stats_request(); +extern const std::string TRUNCATE_TABLE_PREFIX; + /* Access to singleton objects. */ diff --git a/storage/rocksdb/mariadb-ldb.1 b/storage/rocksdb/mariadb-ldb.1 index 5d58bd0b886..b8b425d6ef9 100644 --- a/storage/rocksdb/mariadb-ldb.1 +++ b/storage/rocksdb/mariadb-ldb.1 @@ -1,6 +1,6 @@ '\" t .\" -.TH "\fBMARIADB-LDB\fR" "1" "15 May 2020" "MariaDB 10.11" "MariaDB Database System" +.TH "\FBMARIADB-LDB\FR" "1" "15 May 2023" "MariaDB 11.2" "MariaDB Database System" .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- @@ -11,6 +11,6 @@ .SH NAME mariadb-ldb \- RocksDB tool (mysql_ldb is now a symlink to mariadb-ldb) .SH DESCRIPTION -Use \fBmysql_ldb \-\-help\fR for details on usage\. +Use \fBmariadb-ldb \-\-help\fR for details on usage\. .PP For more information, please refer to the MariaDB Knowledge Base, available online at https://mariadb.com/kb/ diff --git a/storage/rocksdb/mariadb-sst-dump.1 b/storage/rocksdb/mariadb-sst-dump.1 new file mode 100644 index 00000000000..173852422fb --- /dev/null +++ b/storage/rocksdb/mariadb-sst-dump.1 @@ -0,0 +1,16 @@ +'\" t +.\" +.TH "\FBMARIADB-SST-DUMP\FR" "1" "15 May 2023" "MariaDB 11.2" "MariaDB Database System" +.\" ----------------------------------------------------------------- +.\" * set default formatting +.\" ----------------------------------------------------------------- +.\" disable hyphenation +.nh +.\" disable justification (adjust text to left margin only) +.ad l +.SH NAME +mariadb-sst-dump \- RocksDB tool (mysql_sst_dump is now a symlink to mariadb-sst-dump) +.SH DESCRIPTION +Use \fBmariadb-sst-dump \-\-help\fR for details on usage\. +.PP +For more information, please refer to the MariaDB Knowledge Base, available online at https://mariadb.com/kb/ diff --git a/storage/rocksdb/mysql-test/rocksdb/combinations b/storage/rocksdb/mysql-test/rocksdb/combinations index be8080d4b9b..7a2a3eaa6e0 100644 --- a/storage/rocksdb/mysql-test/rocksdb/combinations +++ b/storage/rocksdb/mysql-test/rocksdb/combinations @@ -3,3 +3,7 @@ loose-rocksdb_write_policy=write_committed [write_prepared] loose-rocksdb_write_policy=write_prepared + +[write_unprepared] +loose-rocksdb_write_policy=write_unprepared +loose-rocksdb_write_batch_flush_threshold=1 diff --git a/storage/rocksdb/mysql-test/rocksdb/r/bloomfilter2.result b/storage/rocksdb/mysql-test/rocksdb/r/bloomfilter2.result index d5369e2dbed..9adbe17f64b 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/bloomfilter2.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/bloomfilter2.result @@ -2,7 +2,7 @@ CREATE TABLE t0 (id1 VARCHAR(30), id2 INT, value INT, PRIMARY KEY (id1, id2)) EN select variable_value into @u from information_schema.global_status where variable_name='rocksdb_bloom_filter_prefix_useful'; SELECT COUNT(*) FROM t0 WHERE id1='X' AND id2>=1; COUNT(*) -10000 +100000 select case when variable_value-@u = 0 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_bloom_filter_prefix_useful'; case when variable_value-@u = 0 then 'true' else 'false' end true @@ -11,14 +11,14 @@ CREATE TABLE t1 (id1 BIGINT, id2 INT, id3 BIGINT, value INT, PRIMARY KEY (id1, i select variable_value into @u from information_schema.global_status where variable_name='rocksdb_bloom_filter_prefix_useful'; SELECT COUNT(*) FROM t1 WHERE id1=1 AND id2=1 AND id3>=2; COUNT(*) -9999 +99999 select case when variable_value-@u = 0 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_bloom_filter_prefix_useful'; case when variable_value-@u = 0 then 'true' else 'false' end true select variable_value into @u from information_schema.global_status where variable_name='rocksdb_bloom_filter_prefix_useful'; SELECT COUNT(*) FROM t1 WHERE id1=1 AND id2>=1 AND id3>=2; COUNT(*) -9999 +99999 select case when variable_value-@u = 0 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_bloom_filter_prefix_useful'; case when variable_value-@u = 0 then 'true' else 'false' end true diff --git a/storage/rocksdb/mysql-test/rocksdb/r/cancel_mc-master.opt b/storage/rocksdb/mysql-test/rocksdb/r/cancel_mc-master.opt new file mode 100644 index 00000000000..9fbcbc9cc88 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/r/cancel_mc-master.opt @@ -0,0 +1 @@ +--rocksdb_rate_limiter_bytes_per_sec=256k --rocksdb_default_cf_options=write_buffer_size=64k;target_file_size_base=64k;max_bytes_for_level_base=1m;compression_per_level=kNoCompression;max_compaction_bytes=256k \ No newline at end of file diff --git a/storage/rocksdb/mysql-test/rocksdb/r/cancel_mc.result b/storage/rocksdb/mysql-test/rocksdb/r/cancel_mc.result new file mode 100644 index 00000000000..e3ac4f2aa2c --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/r/cancel_mc.result @@ -0,0 +1,89 @@ +CREATE FUNCTION is_compaction_stopped() RETURNS INT +BEGIN +SELECT variable_value INTO @cwb_1 FROM information_schema.global_status WHERE variable_name="rocksdb_compact_write_bytes"; +SELECT SLEEP(3) INTO @a; +SELECT variable_value INTO @cwb_2 FROM information_schema.global_status WHERE variable_name="rocksdb_compact_write_bytes"; +IF @cwb_2 > @cwb_1 THEN +RETURN 0; +END IF; +RETURN 1; +END// +DROP TABLE IF EXISTS t1; +create table t1 (id int auto_increment, value int, value2 varchar(500), primary key (id), index(value2) comment 'cf_i') engine=rocksdb; +SET SESSION rocksdb_bulk_load_allow_sk=1; +SET SESSION rocksdb_bulk_load=1; +LOAD DATA INFILE '/se_loaddata.dat' INTO TABLE t1 +FIELDS TERMINATED BY ',' (id, value, value2); +SET SESSION rocksdb_bulk_load=0; +SET SESSION rocksdb_bulk_load_allow_sk=0; +connect con1,localhost,root,,; +connect con2,localhost,root,,; +# +# Cancelling MC, including pending MCs, by KILL statements +# +connection default; +SELECT variable_value into @executed_start FROM information_schema.global_status WHERE variable_name="rocksdb_manual_compactions_cancelled"; +connection con1; +SET GLOBAL rocksdb_compact_cf='default'; +connection con2; +set session rocksdb_manual_compaction_threads=2; +SET GLOBAL rocksdb_compact_cf='cf_i'; +connection default; +connection con1; +ERROR HY000: Internal error: Manual Compaction Failed. Reason: Cancelled by client. +connection con2; +ERROR HY000: Internal error: Manual Compaction Failed. Reason: Cancelled by client. +connection default; +SELECT variable_value into @executed_end FROM information_schema.global_status WHERE variable_name="rocksdb_manual_compactions_cancelled"; +SELECT @executed_end-@executed_start AS CANCELLED_MC; +CANCELLED_MC +2 +# +# Cancelling MC by global opt +# +connection default; +SELECT variable_value into @executed_start FROM information_schema.global_status WHERE variable_name="rocksdb_manual_compactions_cancelled"; +connection con1; +set session rocksdb_manual_compaction_threads=2; +SET GLOBAL rocksdb_compact_cf='default'; +connection default; +connection con2; +SET GLOBAL rocksdb_compact_cf='cf_i'; +connection default; +set global rocksdb_cancel_manual_compactions=ON; +connection con1; +ERROR HY000: Internal error: Manual Compaction Failed. Reason: Cancelled by server. +connection con2; +ERROR HY000: Internal error: Manual Compaction Failed. Reason: Cancelled by server. +connection default; +SELECT variable_value into @executed_end FROM information_schema.global_status WHERE variable_name="rocksdb_manual_compactions_cancelled"; +SELECT @executed_end-@executed_start AS CANCELLED_MC; +CANCELLED_MC +2 +# +# Cancelling OPTIMIZE TABLE by global opt +# +connection con1; +OPTIMIZE TABLE t1; +connection default; +SELECT variable_value into @current_cwb FROM information_schema.global_status WHERE variable_name="rocksdb_compact_write_bytes"; +set global rocksdb_cancel_manual_compactions=ON; +connection con1; +Table Op Msg_type Msg_text +test.t1 optimize Error Got error 7 'Result incomplete: Manual compaction paused' from ROCKSDB +test.t1 optimize error Unknown - internal error 215 during operation +connection default; +select 'PRIMARY' as idx, COUNT(*) as cnt FROM t1 FORCE INDEX (PRIMARY) +UNION ALL SELECT 'value2' as idx, COUNT(*) as cnt FROM t1 FORCE INDEX (value2); +idx cnt +PRIMARY 500000 +value2 500000 +# +# Dropping Indexes complete even after cancelling MC +# +INSERT INTO t1 (id) VALUES (null); +set global rocksdb_force_flush_memtable_now=ON; +SELECT variable_value into @current_cwb FROM information_schema.global_status WHERE variable_name="rocksdb_compact_write_bytes"; +DROP TABLE t1; +set global rocksdb_cancel_manual_compactions=ON; +DROP FUNCTION is_compaction_stopped; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/cancel_mc_timeout.result b/storage/rocksdb/mysql-test/rocksdb/r/cancel_mc_timeout.result new file mode 100644 index 00000000000..09161e9a9f6 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/r/cancel_mc_timeout.result @@ -0,0 +1,19 @@ +connect con1,localhost,root,,; +connect con2,localhost,root,,; +connection default; +set global rocksdb_debug_manual_compaction_delay=80; +connection con1; +SET GLOBAL rocksdb_compact_cf='default'; +connection default; +connection con2; +set session rocksdb_manual_compaction_threads=2; +SET GLOBAL rocksdb_compact_cf='default'; +connection default; +connection con2; +ERROR HY000: Internal error: Manual Compaction Failed. Reason: Cancelled by client. +connection con1; +ERROR HY000: Internal error: Manual Compaction Failed. Reason: Cancelled by client. (timeout) +connection default; +set global rocksdb_debug_manual_compaction_delay=0; +connection con1; +SET GLOBAL rocksdb_compact_cf='default'; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/drop_table3.result b/storage/rocksdb/mysql-test/rocksdb/r/drop_table3.result index 954e6079bba..de8cbcbcd02 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/drop_table3.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/drop_table3.result @@ -1,12 +1,6 @@ -call mtr.add_suppression("Column family 'cf1' not found"); -call mtr.add_suppression("Column family 'rev:cf2' not found"); DROP TABLE IF EXISTS t1; call mtr.add_suppression("Column family 'cf1' not found"); call mtr.add_suppression("Column family 'rev:cf2' not found"); -set global rocksdb_compact_cf = 'cf1'; -set global rocksdb_compact_cf = 'rev:cf2'; -set global rocksdb_signal_drop_index_thread = 1; -# restart CREATE TABLE t1 ( a int not null, b int not null, @@ -15,6 +9,10 @@ primary key (a,b) comment 'cf1', key (b) comment 'rev:cf2' ) ENGINE=RocksDB; DELETE FROM t1; +set global rocksdb_compact_cf = 'cf1'; +set global rocksdb_compact_cf = 'rev:cf2'; +set global rocksdb_signal_drop_index_thread = 1; +# restart select variable_value into @a from information_schema.global_status where variable_name='rocksdb_compact_read_bytes'; drop table t1; select case when variable_value-@a < 500000 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_compact_read_bytes'; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/i_s.result b/storage/rocksdb/mysql-test/rocksdb/r/i_s.result index a3f57610bfa..331221a5e62 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/i_s.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/i_s.result @@ -11,6 +11,7 @@ ROCKSDB_DDL ROCKSDB_DEADLOCK ROCKSDB_GLOBAL_INFO ROCKSDB_INDEX_FILE_MAP +ROCKSDB_LIVE_FILES_METADATA ROCKSDB_LOCKS ROCKSDB_PERF_CONTEXT ROCKSDB_PERF_CONTEXT_GLOBAL diff --git a/storage/rocksdb/mysql-test/rocksdb/r/information_schema.result b/storage/rocksdb/mysql-test/rocksdb/r/information_schema.result index e22a85fd5c6..046f08be199 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/information_schema.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/information_schema.result @@ -55,6 +55,9 @@ CF_FLAGS 4 cf_c [0] CF_FLAGS 5 rev:cf_d [1] CREATE TABLE t3 (a INT, PRIMARY KEY (a)) ENGINE=ROCKSDB; insert into t3 (a) values (1), (2), (3); +SELECT IF(count(*) > 0, "TRUE", "FALSE") as metadata_exist FROM INFORMATION_SCHEMA.ROCKSDB_LIVE_FILES_METADATA where CF_NAME = 'default'; +metadata_exist +TRUE SET @ORIG_ROCKSDB_PAUSE_BACKGROUND_WORK = @@GLOBAL.ROCKSDB_PAUSE_BACKGROUND_WORK; SHOW GLOBAL VARIABLES LIKE 'ROCKSDB_PAUSE_BACKGROUND_WORK'; Variable_name Value diff --git a/storage/rocksdb/mysql-test/rocksdb/r/issue243_transactionStatus.result b/storage/rocksdb/mysql-test/rocksdb/r/issue243_transactionStatus.result index d4d211b9288..4d0f4a96a73 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/issue243_transactionStatus.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/issue243_transactionStatus.result @@ -60,7 +60,7 @@ LIST OF SNAPSHOTS FOR EACH SESSION: ---SNAPSHOT, ACTIVE NUM sec MySQL thread id TID, OS thread handle PTR, query id QID localhost root ACTION SHOW ENGINE rocksdb TRANSACTION STATUS -lock count 8, write count 4 +lock count 4, write count 4 insert count 2, update count 1, delete count 1 ----------LATEST DETECTED DEADLOCKS---------- ----------------------------------------- @@ -70,7 +70,7 @@ END OF ROCKSDB TRANSACTION MONITOR OUTPUT ROLLBACK; SHOW ENGINE rocksdb TRANSACTION STATUS; Type Name Status -rocksdb +rocksdb ============================================================ TIMESTAMP ROCKSDB TRANSACTION MONITOR OUTPUT ============================================================ @@ -87,7 +87,7 @@ START TRANSACTION; INSERT INTO t1 VALUES(40,40,40); SHOW ENGINE rocksdb TRANSACTION STATUS; Type Name Status -rocksdb +rocksdb ============================================================ TIMESTAMP ROCKSDB TRANSACTION MONITOR OUTPUT ============================================================ @@ -98,7 +98,7 @@ LIST OF SNAPSHOTS FOR EACH SESSION: ---SNAPSHOT, ACTIVE NUM sec MySQL thread id TID, OS thread handle PTR, query id QID localhost root ACTION SHOW ENGINE rocksdb TRANSACTION STATUS -lock count 2, write count 1 +lock count 1, write count 1 insert count 1, update count 0, delete count 0 ----------LATEST DETECTED DEADLOCKS---------- ----------------------------------------- @@ -108,7 +108,7 @@ END OF ROCKSDB TRANSACTION MONITOR OUTPUT COMMIT; SHOW ENGINE rocksdb TRANSACTION STATUS; Type Name Status -rocksdb +rocksdb ============================================================ TIMESTAMP ROCKSDB TRANSACTION MONITOR OUTPUT ============================================================ @@ -138,7 +138,7 @@ UPDATE t2 SET value=3 WHERE id2=2; DELETE FROM t2 WHERE id1=10; SHOW ENGINE rocksdb TRANSACTION STATUS; Type Name Status -rocksdb +rocksdb ============================================================ TIMESTAMP ROCKSDB TRANSACTION MONITOR OUTPUT ============================================================ @@ -149,7 +149,43 @@ LIST OF SNAPSHOTS FOR EACH SESSION: ---SNAPSHOT, ACTIVE NUM sec MySQL thread id TID, OS thread handle PTR, query id QID localhost root ACTION SHOW ENGINE rocksdb TRANSACTION STATUS -lock count 9, write count 7 +lock count 5, write count 7 +insert count 2, update count 1, delete count 1 +----------LATEST DETECTED DEADLOCKS---------- +----------------------------------------- +END OF ROCKSDB TRANSACTION MONITOR OUTPUT +========================================= + +ROLLBACK; +SET AUTOCOMMIT=1; +DROP TABLE t2; +DROP TABLE IF EXISTS t2; +CREATE TABLE t2 ( +id1 INT, +id2 INT, +value INT, +PRIMARY KEY (id1), +UNIQUE KEY (id2) +) ENGINE=rocksdb; +SET AUTOCOMMIT=0; +START TRANSACTION; +INSERT INTO t2 VALUES(1,2,0),(10,20,30); +UPDATE t2 SET value=3 WHERE id2=2; +DELETE FROM t2 WHERE id1=10; +SHOW ENGINE rocksdb TRANSACTION STATUS; +Type Name Status +rocksdb +============================================================ +TIMESTAMP ROCKSDB TRANSACTION MONITOR OUTPUT +============================================================ +--------- +SNAPSHOTS +--------- +LIST OF SNAPSHOTS FOR EACH SESSION: +---SNAPSHOT, ACTIVE NUM sec +MySQL thread id TID, OS thread handle PTR, query id QID localhost root ACTION +SHOW ENGINE rocksdb TRANSACTION STATUS +lock count 4, write count 7 insert count 2, update count 1, delete count 1 ----------LATEST DETECTED DEADLOCKS---------- ----------------------------------------- diff --git a/storage/rocksdb/mysql-test/rocksdb/r/issue896.result b/storage/rocksdb/mysql-test/rocksdb/r/issue896.result index 6b742ebaf0c..e97febbf4b0 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/issue896.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/issue896.result @@ -15,3 +15,20 @@ SELECT COUNT(*) FROM t1 FORCE INDEX(d); COUNT(*) 1 DROP TABLE t1; +CREATE TABLE `t1` ( +`a` bigint(20) NOT NULL, +`b` varchar(10) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, +`u` timestamp NOT NULL, +`d` bigint(20) DEFAULT NULL, +PRIMARY KEY (`a`,`b`), +KEY `d` (`d`) +) ENGINE=ROCKSDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='ttl_duration=1000;ttl_col=u'; +INSERT INTO t1 VALUES (100, 'aaabbb', NOW(), 200); +EXPLAIN SELECT COUNT(*) FROM t1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 index NULL d 9 NULL # Using index +# segfault here without the fix +SELECT COUNT(*) FROM t1; +COUNT(*) +1 +DROP TABLE t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/manual_compaction_bottommost_level.result b/storage/rocksdb/mysql-test/rocksdb/r/manual_compaction_bottommost_level.result new file mode 100644 index 00000000000..8a12ae8c6a4 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/r/manual_compaction_bottommost_level.result @@ -0,0 +1,37 @@ +CREATE PROCEDURE compact_start() +BEGIN +select variable_value into @c from information_schema.global_status where variable_name='rocksdb_compact_write_bytes'; +END// +CREATE PROCEDURE compact_end() +BEGIN +select case when variable_value-@c > 0 then 'true' else 'false' end as checked from information_schema.global_status where variable_name='rocksdb_compact_write_bytes'; +END// +DROP TABLE IF EXISTS t1; +CREATE TABLE t1 ( +a int not null, +b int not null, +primary key (a,b), +key (b) +) ENGINE=RocksDB; +DELETE FROM t1; +set @@session.rocksdb_manual_compaction_bottommost_level='kSkip'; +call compact_start(); +set @@global.rocksdb_compact_cf = 'default'; +call compact_end(); +checked +true +set @@session.rocksdb_manual_compaction_bottommost_level='kSkip'; +call compact_start(); +set @@global.rocksdb_compact_cf = 'default'; +call compact_end(); +checked +false +set @@session.rocksdb_manual_compaction_bottommost_level='kForceOptimized'; +call compact_start(); +set @@global.rocksdb_compact_cf = 'default'; +call compact_end(); +checked +true +DROP PROCEDURE compact_start; +DROP PROCEDURE compact_end; +drop table t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/mariadb_plugin.result b/storage/rocksdb/mysql-test/rocksdb/r/mariadb_plugin.result index a8e42a4e2b2..c67cdb13a45 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/mariadb_plugin.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/mariadb_plugin.result @@ -23,6 +23,7 @@ Warning 1620 Plugin is busy and will be uninstalled on shutdown Warning 1620 Plugin is busy and will be uninstalled on shutdown Warning 1620 Plugin is busy and will be uninstalled on shutdown Warning 1620 Plugin is busy and will be uninstalled on shutdown +Warning 1620 Plugin is busy and will be uninstalled on shutdown SELECT ENGINE, SUPPORT FROM INFORMATION_SCHEMA.ENGINES WHERE ENGINE='ROCKSDB'; ENGINE SUPPORT ROCKSDB NO diff --git a/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result b/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result index 6645a33e356..392cd997c01 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result @@ -83,6 +83,7 @@ ROCKSDB_INDEX_FILE_MAP Stable ROCKSDB_LOCKS Stable ROCKSDB_TRX Stable ROCKSDB_DEADLOCK Stable +ROCKSDB_LIVE_FILES_METADATA Stable # # MDEV-12466 : Assertion `thd->transaction.stmt.is_empty() || thd->in_sub_stmt || ... # diff --git a/storage/rocksdb/mysql-test/rocksdb/r/max_row_locks.result b/storage/rocksdb/mysql-test/rocksdb/r/max_row_locks.result new file mode 100644 index 00000000000..b18c16e2139 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/r/max_row_locks.result @@ -0,0 +1,112 @@ +create table t1 (id1 bigint, id2 bigint, c1 bigint, c2 bigint, c3 bigint, c4 bigint, c5 bigint, c6 bigint, c7 bigint, primary key (id1, id2), index i(c1, c2)); +begin; +select * from t1 where c3=1 for update; +id1 id2 c1 c2 c3 c4 c5 c6 c7 +SHOW ENGINE rocksdb TRANSACTION STATUS; +Type Name Status +rocksdb +============================================================ +TIMESTAMP ROCKSDB TRANSACTION MONITOR OUTPUT +============================================================ +--------- +SNAPSHOTS +--------- +LIST OF SNAPSHOTS FOR EACH SESSION: +---SNAPSHOT, ACTIVE NUM sec +MySQL thread id TID, OS thread handle PTR, query id QID localhost root ACTION +SHOW ENGINE rocksdb TRANSACTION STATUS +lock count 0, write count 0 +insert count 0, update count 0, delete count 0 +----------LATEST DETECTED DEADLOCKS---------- +----------------------------------------- +END OF ROCKSDB TRANSACTION MONITOR OUTPUT +========================================= + +select * from t1 where c7 between 101 and 110 for update; +id1 id2 c1 c2 c3 c4 c5 c6 c7 +101 0 101 0 0 0 0 0 101 +102 0 102 0 0 0 0 0 102 +103 0 103 0 0 0 0 0 103 +104 0 104 0 0 0 0 0 104 +105 0 105 0 0 0 0 0 105 +106 0 106 0 0 0 0 0 106 +107 0 107 0 0 0 0 0 107 +108 0 108 0 0 0 0 0 108 +109 0 109 0 0 0 0 0 109 +110 0 110 0 0 0 0 0 110 +SHOW ENGINE rocksdb TRANSACTION STATUS; +Type Name Status +rocksdb +============================================================ +TIMESTAMP ROCKSDB TRANSACTION MONITOR OUTPUT +============================================================ +--------- +SNAPSHOTS +--------- +LIST OF SNAPSHOTS FOR EACH SESSION: +---SNAPSHOT, ACTIVE NUM sec +MySQL thread id TID, OS thread handle PTR, query id QID localhost root ACTION +SHOW ENGINE rocksdb TRANSACTION STATUS +lock count 10, write count 0 +insert count 0, update count 0, delete count 0 +----------LATEST DETECTED DEADLOCKS---------- +----------------------------------------- +END OF ROCKSDB TRANSACTION MONITOR OUTPUT +========================================= + +rollback; +set session rocksdb_lock_scanned_rows=on; +begin; +select * from t1 where c3=1 for update; +id1 id2 c1 c2 c3 c4 c5 c6 c7 +SHOW ENGINE rocksdb TRANSACTION STATUS; +Type Name Status +rocksdb +============================================================ +TIMESTAMP ROCKSDB TRANSACTION MONITOR OUTPUT +============================================================ +--------- +SNAPSHOTS +--------- +LIST OF SNAPSHOTS FOR EACH SESSION: +---SNAPSHOT, ACTIVE NUM sec +MySQL thread id TID, OS thread handle PTR, query id QID localhost root ACTION +SHOW ENGINE rocksdb TRANSACTION STATUS +lock count 1000, write count 0 +insert count 0, update count 0, delete count 0 +----------LATEST DETECTED DEADLOCKS---------- +----------------------------------------- +END OF ROCKSDB TRANSACTION MONITOR OUTPUT +========================================= + +rollback; +set session rocksdb_lock_scanned_rows=off; +SET @start_rocksdb_max_row_locks = @@global.rocksdb_max_row_locks; +set global rocksdb_max_row_locks = 20; +select * from t1 where c3=1 for update; +id1 id2 c1 c2 c3 c4 c5 c6 c7 +select * from t1 where c7 between 101 and 110 for update; +id1 id2 c1 c2 c3 c4 c5 c6 c7 +101 0 101 0 0 0 0 0 101 +102 0 102 0 0 0 0 0 102 +103 0 103 0 0 0 0 0 103 +104 0 104 0 0 0 0 0 104 +105 0 105 0 0 0 0 0 105 +106 0 106 0 0 0 0 0 106 +107 0 107 0 0 0 0 0 107 +108 0 108 0 0 0 0 0 108 +109 0 109 0 0 0 0 0 109 +110 0 110 0 0 0 0 0 110 +set global rocksdb_max_row_locks = 5; +select * from t1 where c3=1 for update; +id1 id2 c1 c2 c3 c4 c5 c6 c7 +select * from t1 where c7 between 101 and 110 for update; +ERROR HY000: Got error 10 'Operation aborted: Failed to acquire lock due to rocksdb_max_row_locks limit' from ROCKSDB +set session rocksdb_lock_scanned_rows=on; +select * from t1 where c3=1 for update; +ERROR HY000: Got error 10 'Operation aborted: Failed to acquire lock due to rocksdb_max_row_locks limit' from ROCKSDB +select * from t1 where c7 between 101 and 110 for update; +ERROR HY000: Got error 10 'Operation aborted: Failed to acquire lock due to rocksdb_max_row_locks limit' from ROCKSDB +set session rocksdb_lock_scanned_rows=off; +set @@global.rocksdb_max_row_locks = @start_rocksdb_max_row_locks; +drop table t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/perf_context.result b/storage/rocksdb/mysql-test/rocksdb/r/perf_context.result index 28f965843aa..37f3acc9f50 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/perf_context.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/perf_context.result @@ -161,10 +161,6 @@ BEGIN; INSERT INTO t2 VALUES (1), (2); INSERT INTO t2 VALUES (3), (4); COMMIT; -SELECT COUNT(*) from INFORMATION_SCHEMA.ROCKSDB_PERF_CONTEXT -WHERE TABLE_NAME = 't2' -AND STAT_TYPE = 'IO_WRITE_NANOS' -AND VALUE > 0; COUNT(*) 0 SELECT COUNT(*) from INFORMATION_SCHEMA.ROCKSDB_PERF_CONTEXT_GLOBAL diff --git a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb.result b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb.result index 0c9d29efa28..7b13cf33bff 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb.result @@ -888,6 +888,7 @@ rocksdb_cache_dump ON rocksdb_cache_high_pri_pool_ratio 0.000000 rocksdb_cache_index_and_filter_blocks ON rocksdb_cache_index_and_filter_with_high_priority ON +rocksdb_cancel_manual_compactions OFF rocksdb_checksums_pct 100 rocksdb_collect_sst_properties ON rocksdb_commit_in_the_middle OFF @@ -918,6 +919,9 @@ rocksdb_delete_obsolete_files_period_micros 21600000000 rocksdb_enable_2pc ON rocksdb_enable_bulk_load_api ON rocksdb_enable_insert_with_update_caching ON +rocksdb_enable_iterate_bounds ON +rocksdb_enable_pipelined_write OFF +rocksdb_enable_remove_orphaned_dropped_cfs ON rocksdb_enable_thread_tracking ON rocksdb_enable_ttl ON rocksdb_enable_ttl_read_filtering ON @@ -945,10 +949,14 @@ rocksdb_lock_wait_timeout 1 rocksdb_log_dir rocksdb_log_file_time_to_roll 0 rocksdb_manifest_preallocation_size 4194304 +rocksdb_manual_compaction_bottommost_level kForceOptimized rocksdb_manual_compaction_threads 0 rocksdb_manual_wal_flush ON rocksdb_master_skip_tx_api OFF +rocksdb_max_background_compactions -1 +rocksdb_max_background_flushes -1 rocksdb_max_background_jobs 2 +rocksdb_max_bottom_pri_background_compactions 0 rocksdb_max_latest_deadlocks 5 rocksdb_max_log_file_size 0 rocksdb_max_manifest_file_size 1073741824 @@ -979,10 +987,11 @@ rocksdb_signal_drop_index_thread OFF rocksdb_sim_cache_size 0 rocksdb_skip_bloom_filter_on_read OFF rocksdb_skip_fill_cache OFF +rocksdb_skip_locks_if_skip_unique_check OFF rocksdb_skip_unique_check_tables .* rocksdb_sst_mgr_rate_bytes_per_sec 0 rocksdb_stats_dump_period_sec 600 -rocksdb_stats_level 0 +rocksdb_stats_level 1 rocksdb_stats_recalc_rate 0 rocksdb_store_row_debug_checksums OFF rocksdb_strict_collation_check OFF @@ -991,6 +1000,7 @@ rocksdb_table_cache_numshardbits 6 rocksdb_table_stats_sampling_pct 10 rocksdb_tmpdir rocksdb_trace_sst_api OFF +rocksdb_track_and_verify_wals_in_manifest ON rocksdb_two_write_queues ON rocksdb_unsafe_for_binlog OFF rocksdb_update_cf_options @@ -1003,10 +1013,11 @@ rocksdb_validate_tables 1 rocksdb_verify_row_debug_checksums OFF rocksdb_wal_bytes_per_sync 0 rocksdb_wal_dir -rocksdb_wal_recovery_mode 1 +rocksdb_wal_recovery_mode 2 rocksdb_wal_size_limit_mb 0 rocksdb_wal_ttl_seconds 0 rocksdb_whole_key_filtering ON +rocksdb_write_batch_flush_threshold 0 rocksdb_write_batch_max_bytes 0 rocksdb_write_disable_wal OFF rocksdb_write_ignore_missing_column_families OFF @@ -1365,9 +1376,10 @@ CREATE TABLE t1 (pk INT PRIMARY KEY, a int) ENGINE=RocksDB; set @a=-1; insert into t1 select (@a:=@a+1), 1234 from information_schema.session_variables limit 100; set @tmp1= @@rocksdb_max_row_locks; -set rocksdb_max_row_locks= 20; +set GLOBAL rocksdb_max_row_locks= 20; update t1 set a=a+10; ERROR HY000: Got error 10 'Operation aborted: Failed to acquire lock due to rocksdb_max_row_locks limit' from ROCKSDB +set @@global.rocksdb_max_row_locks = @tmp1; DROP TABLE t1; # # Test AUTO_INCREMENT behavior problem, @@ -1500,19 +1512,15 @@ Rocksdb_block_cache_data_bytes_insert # Rocksdb_block_cache_data_hit # Rocksdb_block_cache_data_miss # Rocksdb_block_cache_filter_add # -Rocksdb_block_cache_filter_bytes_evict # Rocksdb_block_cache_filter_bytes_insert # Rocksdb_block_cache_filter_hit # Rocksdb_block_cache_filter_miss # Rocksdb_block_cache_hit # Rocksdb_block_cache_index_add # -Rocksdb_block_cache_index_bytes_evict # Rocksdb_block_cache_index_bytes_insert # Rocksdb_block_cache_index_hit # Rocksdb_block_cache_index_miss # Rocksdb_block_cache_miss # -Rocksdb_block_cachecompressed_hit # -Rocksdb_block_cachecompressed_miss # Rocksdb_bloom_filter_full_positive # Rocksdb_bloom_filter_full_true_positive # Rocksdb_bloom_filter_prefix_checked # @@ -1531,14 +1539,14 @@ Rocksdb_get_hit_l1 # Rocksdb_get_hit_l2_and_up # Rocksdb_getupdatessince_calls # Rocksdb_iter_bytes_read # +Rocksdb_manual_compactions_cancelled # +Rocksdb_manual_compactions_pending # Rocksdb_manual_compactions_processed # Rocksdb_manual_compactions_running # Rocksdb_memtable_hit # Rocksdb_memtable_miss # -Rocksdb_no_file_closes # Rocksdb_no_file_errors # Rocksdb_no_file_opens # -Rocksdb_num_iterators # Rocksdb_number_block_not_compressed # Rocksdb_number_db_next # Rocksdb_number_db_next_found # @@ -1546,7 +1554,6 @@ Rocksdb_number_db_prev # Rocksdb_number_db_prev_found # Rocksdb_number_db_seek # Rocksdb_number_db_seek_found # -Rocksdb_number_deletes_filtered # Rocksdb_number_keys_read # Rocksdb_number_keys_updated # Rocksdb_number_keys_written # @@ -1582,7 +1589,6 @@ Rocksdb_wal_group_syncs # Rocksdb_wal_synced # Rocksdb_write_other # Rocksdb_write_self # -Rocksdb_write_timedout # Rocksdb_write_wal # select VARIABLE_NAME from INFORMATION_SCHEMA.global_status where VARIABLE_NAME LIKE 'rocksdb%' and VARIABLE_NAME NOT LIKE '%num_get_for_update%'; VARIABLE_NAME @@ -1611,19 +1617,15 @@ ROCKSDB_BLOCK_CACHE_DATA_BYTES_INSERT ROCKSDB_BLOCK_CACHE_DATA_HIT ROCKSDB_BLOCK_CACHE_DATA_MISS ROCKSDB_BLOCK_CACHE_FILTER_ADD -ROCKSDB_BLOCK_CACHE_FILTER_BYTES_EVICT ROCKSDB_BLOCK_CACHE_FILTER_BYTES_INSERT ROCKSDB_BLOCK_CACHE_FILTER_HIT ROCKSDB_BLOCK_CACHE_FILTER_MISS ROCKSDB_BLOCK_CACHE_HIT ROCKSDB_BLOCK_CACHE_INDEX_ADD -ROCKSDB_BLOCK_CACHE_INDEX_BYTES_EVICT ROCKSDB_BLOCK_CACHE_INDEX_BYTES_INSERT ROCKSDB_BLOCK_CACHE_INDEX_HIT ROCKSDB_BLOCK_CACHE_INDEX_MISS ROCKSDB_BLOCK_CACHE_MISS -ROCKSDB_BLOCK_CACHECOMPRESSED_HIT -ROCKSDB_BLOCK_CACHECOMPRESSED_MISS ROCKSDB_BLOOM_FILTER_FULL_POSITIVE ROCKSDB_BLOOM_FILTER_FULL_TRUE_POSITIVE ROCKSDB_BLOOM_FILTER_PREFIX_CHECKED @@ -1642,14 +1644,14 @@ ROCKSDB_GET_HIT_L1 ROCKSDB_GET_HIT_L2_AND_UP ROCKSDB_GETUPDATESSINCE_CALLS ROCKSDB_ITER_BYTES_READ +ROCKSDB_MANUAL_COMPACTIONS_CANCELLED +ROCKSDB_MANUAL_COMPACTIONS_PENDING ROCKSDB_MANUAL_COMPACTIONS_PROCESSED ROCKSDB_MANUAL_COMPACTIONS_RUNNING ROCKSDB_MEMTABLE_HIT ROCKSDB_MEMTABLE_MISS -ROCKSDB_NO_FILE_CLOSES ROCKSDB_NO_FILE_ERRORS ROCKSDB_NO_FILE_OPENS -ROCKSDB_NUM_ITERATORS ROCKSDB_NUMBER_BLOCK_NOT_COMPRESSED ROCKSDB_NUMBER_DB_NEXT ROCKSDB_NUMBER_DB_NEXT_FOUND @@ -1657,7 +1659,6 @@ ROCKSDB_NUMBER_DB_PREV ROCKSDB_NUMBER_DB_PREV_FOUND ROCKSDB_NUMBER_DB_SEEK ROCKSDB_NUMBER_DB_SEEK_FOUND -ROCKSDB_NUMBER_DELETES_FILTERED ROCKSDB_NUMBER_KEYS_READ ROCKSDB_NUMBER_KEYS_UPDATED ROCKSDB_NUMBER_KEYS_WRITTEN @@ -1693,7 +1694,6 @@ ROCKSDB_WAL_GROUP_SYNCS ROCKSDB_WAL_SYNCED ROCKSDB_WRITE_OTHER ROCKSDB_WRITE_SELF -ROCKSDB_WRITE_TIMEDOUT ROCKSDB_WRITE_WAL # RocksDB-SE's status variables are global internally # but they are shown as both session and global, like InnoDB's status vars. @@ -1724,19 +1724,15 @@ ROCKSDB_BLOCK_CACHE_DATA_BYTES_INSERT ROCKSDB_BLOCK_CACHE_DATA_HIT ROCKSDB_BLOCK_CACHE_DATA_MISS ROCKSDB_BLOCK_CACHE_FILTER_ADD -ROCKSDB_BLOCK_CACHE_FILTER_BYTES_EVICT ROCKSDB_BLOCK_CACHE_FILTER_BYTES_INSERT ROCKSDB_BLOCK_CACHE_FILTER_HIT ROCKSDB_BLOCK_CACHE_FILTER_MISS ROCKSDB_BLOCK_CACHE_HIT ROCKSDB_BLOCK_CACHE_INDEX_ADD -ROCKSDB_BLOCK_CACHE_INDEX_BYTES_EVICT ROCKSDB_BLOCK_CACHE_INDEX_BYTES_INSERT ROCKSDB_BLOCK_CACHE_INDEX_HIT ROCKSDB_BLOCK_CACHE_INDEX_MISS ROCKSDB_BLOCK_CACHE_MISS -ROCKSDB_BLOCK_CACHECOMPRESSED_HIT -ROCKSDB_BLOCK_CACHECOMPRESSED_MISS ROCKSDB_BLOOM_FILTER_FULL_POSITIVE ROCKSDB_BLOOM_FILTER_FULL_TRUE_POSITIVE ROCKSDB_BLOOM_FILTER_PREFIX_CHECKED @@ -1755,14 +1751,14 @@ ROCKSDB_GET_HIT_L1 ROCKSDB_GET_HIT_L2_AND_UP ROCKSDB_GETUPDATESSINCE_CALLS ROCKSDB_ITER_BYTES_READ +ROCKSDB_MANUAL_COMPACTIONS_CANCELLED +ROCKSDB_MANUAL_COMPACTIONS_PENDING ROCKSDB_MANUAL_COMPACTIONS_PROCESSED ROCKSDB_MANUAL_COMPACTIONS_RUNNING ROCKSDB_MEMTABLE_HIT ROCKSDB_MEMTABLE_MISS -ROCKSDB_NO_FILE_CLOSES ROCKSDB_NO_FILE_ERRORS ROCKSDB_NO_FILE_OPENS -ROCKSDB_NUM_ITERATORS ROCKSDB_NUMBER_BLOCK_NOT_COMPRESSED ROCKSDB_NUMBER_DB_NEXT ROCKSDB_NUMBER_DB_NEXT_FOUND @@ -1770,7 +1766,6 @@ ROCKSDB_NUMBER_DB_PREV ROCKSDB_NUMBER_DB_PREV_FOUND ROCKSDB_NUMBER_DB_SEEK ROCKSDB_NUMBER_DB_SEEK_FOUND -ROCKSDB_NUMBER_DELETES_FILTERED ROCKSDB_NUMBER_KEYS_READ ROCKSDB_NUMBER_KEYS_UPDATED ROCKSDB_NUMBER_KEYS_WRITTEN @@ -1806,7 +1801,6 @@ ROCKSDB_WAL_GROUP_SYNCS ROCKSDB_WAL_SYNCED ROCKSDB_WRITE_OTHER ROCKSDB_WRITE_SELF -ROCKSDB_WRITE_TIMEDOUT ROCKSDB_WRITE_WAL # # Fix issue #9: HA_ERR_INTERNAL_ERROR when running linkbench diff --git a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_bottom_pri_compaction.result b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_bottom_pri_compaction.result new file mode 100644 index 00000000000..7e41cd09707 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_bottom_pri_compaction.result @@ -0,0 +1,15 @@ +1 +DROP TABLE IF EXISTS t1; +CREATE TABLE t1 ( +a int not null, +b int not null, +primary key (a,b), +key (b) +) ENGINE=RocksDB; +DELETE FROM t1; +set @@global.rocksdb_compact_cf = 'default'; +Bottom thread priority: +19 +Bottom thread counts: +1 +drop table t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_bottom_pri_compaction2.result b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_bottom_pri_compaction2.result new file mode 100644 index 00000000000..5555cfa8673 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_bottom_pri_compaction2.result @@ -0,0 +1,16 @@ +1 +Bottom thread counts: +5 +"Increasing bottom pri compaction threads online." +SET GLOBAL rocksdb_max_bottom_pri_background_compactions=10; +1 +Bottom thread counts: +10 +"Decreasing bottom pri compaction threads online." +SET GLOBAL rocksdb_max_bottom_pri_background_compactions=2; +1 +Bottom thread counts: +2 +SET GLOBAL rocksdb_max_bottom_pri_background_compactions=0; +ERROR HY000: Error when executing command SET: max_bottom_pri_background_compactions can't be changed to 0 online. +SET GLOBAL rocksdb_max_bottom_pri_background_compactions=5; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_datadir.result b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_datadir.result index 40c53f6fd8a..f7e58fb6aac 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_datadir.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_datadir.result @@ -1,2 +1,2 @@ Check for MANIFEST files -MANIFEST-000006 +MANIFEST-000005 diff --git a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rc.result b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rc.result index ea2506941b2..0310c422217 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rc.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rc.result @@ -55,7 +55,6 @@ select * from t; i rollback; connection con2; -i rollback; connection con1; i diff --git a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rr.result b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rr.result index ea2506941b2..0310c422217 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rr.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rr.result @@ -55,7 +55,6 @@ select * from t; i rollback; connection con2; -i rollback; connection con1; i diff --git a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_manual_compaction_bottommost_level_basic.result b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_manual_compaction_bottommost_level_basic.result new file mode 100644 index 00000000000..dd666bd1cbe --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_manual_compaction_bottommost_level_basic.result @@ -0,0 +1,114 @@ +CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO valid_values VALUES('kSkip'); +INSERT INTO valid_values VALUES('kIfHaveCompactionFilter'); +INSERT INTO valid_values VALUES('kForce'); +INSERT INTO valid_values VALUES('kForceOptimized'); +CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO invalid_values VALUES('\'aaa\''); +SET @start_global_value = @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +SELECT @start_global_value; +@start_global_value +kForceOptimized +SET @start_session_value = @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +SELECT @start_session_value; +@start_session_value +kForceOptimized +'# Setting to valid values in global scope#' +"Trying to set variable @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL to kSkip" +SET @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = kSkip; +SELECT @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kSkip +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = DEFAULT; +SELECT @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +"Trying to set variable @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL to kIfHaveCompactionFilter" +SET @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = kIfHaveCompactionFilter; +SELECT @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kIfHaveCompactionFilter +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = DEFAULT; +SELECT @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +"Trying to set variable @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL to kForce" +SET @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = kForce; +SELECT @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForce +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = DEFAULT; +SELECT @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +"Trying to set variable @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL to kForceOptimized" +SET @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = kForceOptimized; +SELECT @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = DEFAULT; +SELECT @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +'# Setting to valid values in session scope#' +"Trying to set variable @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL to kSkip" +SET @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = kSkip; +SELECT @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kSkip +"Setting the session scope variable back to default" +SET @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = DEFAULT; +SELECT @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +"Trying to set variable @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL to kIfHaveCompactionFilter" +SET @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = kIfHaveCompactionFilter; +SELECT @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kIfHaveCompactionFilter +"Setting the session scope variable back to default" +SET @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = DEFAULT; +SELECT @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +"Trying to set variable @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL to kForce" +SET @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = kForce; +SELECT @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForce +"Setting the session scope variable back to default" +SET @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = DEFAULT; +SELECT @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +"Trying to set variable @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL to kForceOptimized" +SET @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = kForceOptimized; +SELECT @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +"Setting the session scope variable back to default" +SET @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = DEFAULT; +SELECT @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +'# Testing with invalid values in global scope #' +"Trying to set variable @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL to 'aaa'" +SET @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = 'aaa'; +Got one of the listed errors +SELECT @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +SET @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = @start_global_value; +SELECT @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +SET @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = @start_session_value; +SELECT @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +DROP TABLE valid_values; +DROP TABLE invalid_values; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/show_engine.result b/storage/rocksdb/mysql-test/rocksdb/r/show_engine.result index eac329a24e7..160ff119f07 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/show_engine.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/show_engine.result @@ -45,6 +45,15 @@ __system__ NUM_ENTRIES_ACTIVE_MEM_TABLE # __system__ NUM_ENTRIES_IMM_MEM_TABLES # __system__ NON_BLOCK_CACHE_SST_MEM_USAGE # __system__ NUM_LIVE_VERSIONS # +__system__ NUM_IMMUTABLE_MEM_TABLE_FLUSHED # +__system__ NUM_RUNNING_FLUSHES # +__system__ NUM_RUNNING_COMPACTIONS # +__system__ SIZE_ALL_MEM_TABLES # +__system__ NUM_DELETES_ACTIVE_MEM_TABLE # +__system__ NUM_DELETES_IMM_MEM_TABLES # +__system__ ESTIMATE_NUM_KEYS # +__system__ ESTIMATE_LIVE_DATA_SIZE # +__system__ ESTIMATE_PENDING_COMPACTION_BYTES # cf_t1 NUM_IMMUTABLE_MEM_TABLE # cf_t1 MEM_TABLE_FLUSH_PENDING # cf_t1 COMPACTION_PENDING # @@ -54,6 +63,15 @@ cf_t1 NUM_ENTRIES_ACTIVE_MEM_TABLE # cf_t1 NUM_ENTRIES_IMM_MEM_TABLES # cf_t1 NON_BLOCK_CACHE_SST_MEM_USAGE # cf_t1 NUM_LIVE_VERSIONS # +cf_t1 NUM_IMMUTABLE_MEM_TABLE_FLUSHED # +cf_t1 NUM_RUNNING_FLUSHES # +cf_t1 NUM_RUNNING_COMPACTIONS # +cf_t1 SIZE_ALL_MEM_TABLES # +cf_t1 NUM_DELETES_ACTIVE_MEM_TABLE # +cf_t1 NUM_DELETES_IMM_MEM_TABLES # +cf_t1 ESTIMATE_NUM_KEYS # +cf_t1 ESTIMATE_LIVE_DATA_SIZE # +cf_t1 ESTIMATE_PENDING_COMPACTION_BYTES # default NUM_IMMUTABLE_MEM_TABLE # default MEM_TABLE_FLUSH_PENDING # default COMPACTION_PENDING # @@ -63,6 +81,15 @@ default NUM_ENTRIES_ACTIVE_MEM_TABLE # default NUM_ENTRIES_IMM_MEM_TABLES # default NON_BLOCK_CACHE_SST_MEM_USAGE # default NUM_LIVE_VERSIONS # +default NUM_IMMUTABLE_MEM_TABLE_FLUSHED # +default NUM_RUNNING_FLUSHES # +default NUM_RUNNING_COMPACTIONS # +default SIZE_ALL_MEM_TABLES # +default NUM_DELETES_ACTIVE_MEM_TABLE # +default NUM_DELETES_IMM_MEM_TABLES # +default ESTIMATE_NUM_KEYS # +default ESTIMATE_LIVE_DATA_SIZE # +default ESTIMATE_PENDING_COMPACTION_BYTES # rev:cf_t2 NUM_IMMUTABLE_MEM_TABLE # rev:cf_t2 MEM_TABLE_FLUSH_PENDING # rev:cf_t2 COMPACTION_PENDING # @@ -72,6 +99,15 @@ rev:cf_t2 NUM_ENTRIES_ACTIVE_MEM_TABLE # rev:cf_t2 NUM_ENTRIES_IMM_MEM_TABLES # rev:cf_t2 NON_BLOCK_CACHE_SST_MEM_USAGE # rev:cf_t2 NUM_LIVE_VERSIONS # +rev:cf_t2 NUM_IMMUTABLE_MEM_TABLE_FLUSHED # +rev:cf_t2 NUM_RUNNING_FLUSHES # +rev:cf_t2 NUM_RUNNING_COMPACTIONS # +rev:cf_t2 SIZE_ALL_MEM_TABLES # +rev:cf_t2 NUM_DELETES_ACTIVE_MEM_TABLE # +rev:cf_t2 NUM_DELETES_IMM_MEM_TABLES # +rev:cf_t2 ESTIMATE_NUM_KEYS # +rev:cf_t2 ESTIMATE_LIVE_DATA_SIZE # +rev:cf_t2 ESTIMATE_PENDING_COMPACTION_BYTES # SELECT * FROM INFORMATION_SCHEMA.ROCKSDB_DBSTATS; STAT_TYPE VALUE DB_BACKGROUND_ERRORS # @@ -102,18 +138,13 @@ __system__ NUM_LEVELS # __system__ LEVEL0_FILE_NUM_COMPACTION_TRIGGER # __system__ LEVEL0_SLOWDOWN_WRITES_TRIGGER # __system__ LEVEL0_STOP_WRITES_TRIGGER # -__system__ MAX_MEM_COMPACTION_LEVEL # __system__ TARGET_FILE_SIZE_BASE # __system__ TARGET_FILE_SIZE_MULTIPLIER # __system__ MAX_BYTES_FOR_LEVEL_BASE # __system__ LEVEL_COMPACTION_DYNAMIC_LEVEL_BYTES # __system__ MAX_BYTES_FOR_LEVEL_MULTIPLIER # -__system__ SOFT_RATE_LIMIT # -__system__ HARD_RATE_LIMIT # -__system__ RATE_LIMIT_DELAY_MAX_MILLISECONDS # __system__ ARENA_BLOCK_SIZE # __system__ DISABLE_AUTO_COMPACTIONS # -__system__ PURGE_REDUNDANT_KVS_WHILE_FLUSH # __system__ MAX_SEQUENTIAL_SKIP_IN_ITERATIONS # __system__ MEMTABLE_FACTORY # __system__ INPLACE_UPDATE_SUPPORT # @@ -141,7 +172,6 @@ __system__ TABLE_FACTORY::INDEX_TYPE # __system__ TABLE_FACTORY::DATA_BLOCK_INDEX_TYPE # __system__ TABLE_FACTORY::INDEX_SHORTENING # __system__ TABLE_FACTORY::DATA_BLOCK_HASH_TABLE_UTIL_RATIO # -__system__ TABLE_FACTORY::HASH_INDEX_ALLOW_COLLISION # __system__ TABLE_FACTORY::CHECKSUM # __system__ TABLE_FACTORY::NO_BLOCK_CACHE # __system__ TABLE_FACTORY::BLOCK_CACHE # @@ -152,7 +182,7 @@ __system__ TABLE_FACTORY::NUM_SHARD_BITS # __system__ TABLE_FACTORY::STRICT_CAPACITY_LIMIT # __system__ TABLE_FACTORY::MEMORY_ALLOCATOR # __system__ TABLE_FACTORY::HIGH_PRI_POOL_RATIO # -__system__ TABLE_FACTORY::BLOCK_CACHE_COMPRESSED # +__system__ TABLE_FACTORY::LOW_PRI_POOL_RATIO # __system__ TABLE_FACTORY::PERSISTENT_CACHE # __system__ TABLE_FACTORY::BLOCK_SIZE # __system__ TABLE_FACTORY::BLOCK_SIZE_DEVIATION # @@ -168,6 +198,10 @@ __system__ TABLE_FACTORY::READ_AMP_BYTES_PER_BIT # __system__ TABLE_FACTORY::FORMAT_VERSION # __system__ TABLE_FACTORY::ENABLE_INDEX_COMPRESSION # __system__ TABLE_FACTORY::BLOCK_ALIGN # +__system__ TABLE_FACTORY::MAX_AUTO_READAHEAD_SIZE # +__system__ TABLE_FACTORY::PREPOPULATE_BLOCK_CACHE # +__system__ TABLE_FACTORY::INITIAL_AUTO_READAHEAD_SIZE # +__system__ TABLE_FACTORY::NUM_FILE_READS_FOR_AUTO_READAHEAD # cf_t1 COMPARATOR # cf_t1 MERGE_OPERATOR # cf_t1 COMPACTION_FILTER # @@ -179,18 +213,13 @@ cf_t1 NUM_LEVELS # cf_t1 LEVEL0_FILE_NUM_COMPACTION_TRIGGER # cf_t1 LEVEL0_SLOWDOWN_WRITES_TRIGGER # cf_t1 LEVEL0_STOP_WRITES_TRIGGER # -cf_t1 MAX_MEM_COMPACTION_LEVEL # cf_t1 TARGET_FILE_SIZE_BASE # cf_t1 TARGET_FILE_SIZE_MULTIPLIER # cf_t1 MAX_BYTES_FOR_LEVEL_BASE # cf_t1 LEVEL_COMPACTION_DYNAMIC_LEVEL_BYTES # cf_t1 MAX_BYTES_FOR_LEVEL_MULTIPLIER # -cf_t1 SOFT_RATE_LIMIT # -cf_t1 HARD_RATE_LIMIT # -cf_t1 RATE_LIMIT_DELAY_MAX_MILLISECONDS # cf_t1 ARENA_BLOCK_SIZE # cf_t1 DISABLE_AUTO_COMPACTIONS # -cf_t1 PURGE_REDUNDANT_KVS_WHILE_FLUSH # cf_t1 MAX_SEQUENTIAL_SKIP_IN_ITERATIONS # cf_t1 MEMTABLE_FACTORY # cf_t1 INPLACE_UPDATE_SUPPORT # @@ -218,7 +247,6 @@ cf_t1 TABLE_FACTORY::INDEX_TYPE # cf_t1 TABLE_FACTORY::DATA_BLOCK_INDEX_TYPE # cf_t1 TABLE_FACTORY::INDEX_SHORTENING # cf_t1 TABLE_FACTORY::DATA_BLOCK_HASH_TABLE_UTIL_RATIO # -cf_t1 TABLE_FACTORY::HASH_INDEX_ALLOW_COLLISION # cf_t1 TABLE_FACTORY::CHECKSUM # cf_t1 TABLE_FACTORY::NO_BLOCK_CACHE # cf_t1 TABLE_FACTORY::BLOCK_CACHE # @@ -229,7 +257,7 @@ cf_t1 TABLE_FACTORY::NUM_SHARD_BITS # cf_t1 TABLE_FACTORY::STRICT_CAPACITY_LIMIT # cf_t1 TABLE_FACTORY::MEMORY_ALLOCATOR # cf_t1 TABLE_FACTORY::HIGH_PRI_POOL_RATIO # -cf_t1 TABLE_FACTORY::BLOCK_CACHE_COMPRESSED # +cf_t1 TABLE_FACTORY::LOW_PRI_POOL_RATIO # cf_t1 TABLE_FACTORY::PERSISTENT_CACHE # cf_t1 TABLE_FACTORY::BLOCK_SIZE # cf_t1 TABLE_FACTORY::BLOCK_SIZE_DEVIATION # @@ -245,6 +273,10 @@ cf_t1 TABLE_FACTORY::READ_AMP_BYTES_PER_BIT # cf_t1 TABLE_FACTORY::FORMAT_VERSION # cf_t1 TABLE_FACTORY::ENABLE_INDEX_COMPRESSION # cf_t1 TABLE_FACTORY::BLOCK_ALIGN # +cf_t1 TABLE_FACTORY::MAX_AUTO_READAHEAD_SIZE # +cf_t1 TABLE_FACTORY::PREPOPULATE_BLOCK_CACHE # +cf_t1 TABLE_FACTORY::INITIAL_AUTO_READAHEAD_SIZE # +cf_t1 TABLE_FACTORY::NUM_FILE_READS_FOR_AUTO_READAHEAD # default COMPARATOR # default MERGE_OPERATOR # default COMPACTION_FILTER # @@ -256,18 +288,13 @@ default NUM_LEVELS # default LEVEL0_FILE_NUM_COMPACTION_TRIGGER # default LEVEL0_SLOWDOWN_WRITES_TRIGGER # default LEVEL0_STOP_WRITES_TRIGGER # -default MAX_MEM_COMPACTION_LEVEL # default TARGET_FILE_SIZE_BASE # default TARGET_FILE_SIZE_MULTIPLIER # default MAX_BYTES_FOR_LEVEL_BASE # default LEVEL_COMPACTION_DYNAMIC_LEVEL_BYTES # default MAX_BYTES_FOR_LEVEL_MULTIPLIER # -default SOFT_RATE_LIMIT # -default HARD_RATE_LIMIT # -default RATE_LIMIT_DELAY_MAX_MILLISECONDS # default ARENA_BLOCK_SIZE # default DISABLE_AUTO_COMPACTIONS # -default PURGE_REDUNDANT_KVS_WHILE_FLUSH # default MAX_SEQUENTIAL_SKIP_IN_ITERATIONS # default MEMTABLE_FACTORY # default INPLACE_UPDATE_SUPPORT # @@ -295,7 +322,6 @@ default TABLE_FACTORY::INDEX_TYPE # default TABLE_FACTORY::DATA_BLOCK_INDEX_TYPE # default TABLE_FACTORY::INDEX_SHORTENING # default TABLE_FACTORY::DATA_BLOCK_HASH_TABLE_UTIL_RATIO # -default TABLE_FACTORY::HASH_INDEX_ALLOW_COLLISION # default TABLE_FACTORY::CHECKSUM # default TABLE_FACTORY::NO_BLOCK_CACHE # default TABLE_FACTORY::BLOCK_CACHE # @@ -306,7 +332,7 @@ default TABLE_FACTORY::NUM_SHARD_BITS # default TABLE_FACTORY::STRICT_CAPACITY_LIMIT # default TABLE_FACTORY::MEMORY_ALLOCATOR # default TABLE_FACTORY::HIGH_PRI_POOL_RATIO # -default TABLE_FACTORY::BLOCK_CACHE_COMPRESSED # +default TABLE_FACTORY::LOW_PRI_POOL_RATIO # default TABLE_FACTORY::PERSISTENT_CACHE # default TABLE_FACTORY::BLOCK_SIZE # default TABLE_FACTORY::BLOCK_SIZE_DEVIATION # @@ -322,6 +348,10 @@ default TABLE_FACTORY::READ_AMP_BYTES_PER_BIT # default TABLE_FACTORY::FORMAT_VERSION # default TABLE_FACTORY::ENABLE_INDEX_COMPRESSION # default TABLE_FACTORY::BLOCK_ALIGN # +default TABLE_FACTORY::MAX_AUTO_READAHEAD_SIZE # +default TABLE_FACTORY::PREPOPULATE_BLOCK_CACHE # +default TABLE_FACTORY::INITIAL_AUTO_READAHEAD_SIZE # +default TABLE_FACTORY::NUM_FILE_READS_FOR_AUTO_READAHEAD # rev:cf_t2 COMPARATOR # rev:cf_t2 MERGE_OPERATOR # rev:cf_t2 COMPACTION_FILTER # @@ -333,18 +363,13 @@ rev:cf_t2 NUM_LEVELS # rev:cf_t2 LEVEL0_FILE_NUM_COMPACTION_TRIGGER # rev:cf_t2 LEVEL0_SLOWDOWN_WRITES_TRIGGER # rev:cf_t2 LEVEL0_STOP_WRITES_TRIGGER # -rev:cf_t2 MAX_MEM_COMPACTION_LEVEL # rev:cf_t2 TARGET_FILE_SIZE_BASE # rev:cf_t2 TARGET_FILE_SIZE_MULTIPLIER # rev:cf_t2 MAX_BYTES_FOR_LEVEL_BASE # rev:cf_t2 LEVEL_COMPACTION_DYNAMIC_LEVEL_BYTES # rev:cf_t2 MAX_BYTES_FOR_LEVEL_MULTIPLIER # -rev:cf_t2 SOFT_RATE_LIMIT # -rev:cf_t2 HARD_RATE_LIMIT # -rev:cf_t2 RATE_LIMIT_DELAY_MAX_MILLISECONDS # rev:cf_t2 ARENA_BLOCK_SIZE # rev:cf_t2 DISABLE_AUTO_COMPACTIONS # -rev:cf_t2 PURGE_REDUNDANT_KVS_WHILE_FLUSH # rev:cf_t2 MAX_SEQUENTIAL_SKIP_IN_ITERATIONS # rev:cf_t2 MEMTABLE_FACTORY # rev:cf_t2 INPLACE_UPDATE_SUPPORT # @@ -372,7 +397,6 @@ rev:cf_t2 TABLE_FACTORY::INDEX_TYPE # rev:cf_t2 TABLE_FACTORY::DATA_BLOCK_INDEX_TYPE # rev:cf_t2 TABLE_FACTORY::INDEX_SHORTENING # rev:cf_t2 TABLE_FACTORY::DATA_BLOCK_HASH_TABLE_UTIL_RATIO # -rev:cf_t2 TABLE_FACTORY::HASH_INDEX_ALLOW_COLLISION # rev:cf_t2 TABLE_FACTORY::CHECKSUM # rev:cf_t2 TABLE_FACTORY::NO_BLOCK_CACHE # rev:cf_t2 TABLE_FACTORY::BLOCK_CACHE # @@ -383,7 +407,7 @@ rev:cf_t2 TABLE_FACTORY::NUM_SHARD_BITS # rev:cf_t2 TABLE_FACTORY::STRICT_CAPACITY_LIMIT # rev:cf_t2 TABLE_FACTORY::MEMORY_ALLOCATOR # rev:cf_t2 TABLE_FACTORY::HIGH_PRI_POOL_RATIO # -rev:cf_t2 TABLE_FACTORY::BLOCK_CACHE_COMPRESSED # +rev:cf_t2 TABLE_FACTORY::LOW_PRI_POOL_RATIO # rev:cf_t2 TABLE_FACTORY::PERSISTENT_CACHE # rev:cf_t2 TABLE_FACTORY::BLOCK_SIZE # rev:cf_t2 TABLE_FACTORY::BLOCK_SIZE_DEVIATION # @@ -399,6 +423,10 @@ rev:cf_t2 TABLE_FACTORY::READ_AMP_BYTES_PER_BIT # rev:cf_t2 TABLE_FACTORY::FORMAT_VERSION # rev:cf_t2 TABLE_FACTORY::ENABLE_INDEX_COMPRESSION # rev:cf_t2 TABLE_FACTORY::BLOCK_ALIGN # +rev:cf_t2 TABLE_FACTORY::MAX_AUTO_READAHEAD_SIZE # +rev:cf_t2 TABLE_FACTORY::PREPOPULATE_BLOCK_CACHE # +rev:cf_t2 TABLE_FACTORY::INITIAL_AUTO_READAHEAD_SIZE # +rev:cf_t2 TABLE_FACTORY::NUM_FILE_READS_FOR_AUTO_READAHEAD # DROP TABLE t1; DROP TABLE t2; DROP TABLE t3; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/truncate_failures.result b/storage/rocksdb/mysql-test/rocksdb/r/truncate_failures.result new file mode 100644 index 00000000000..f21a5e46212 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/r/truncate_failures.result @@ -0,0 +1,95 @@ +set global rocksdb_strict_collation_exceptions = 't1,t2'; +CREATE TABLE t1 ( +pk INT, +a VARCHAR(64), +PRIMARY KEY (pk), +KEY (a) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 COLLATE=latin1_general_ci; +INSERT INTO t1 VALUES (1, 'one'); +RENAME TABLE t1 TO t1_trunc; +SELECT * FROM t1_trunc ORDER BY pk; +pk a +1 one +TRUNCATE TABLE t1_trunc; +SELECT * FROM t1_trunc; +pk a +CREATE TABLE t2 ( +pk INT, +a VARCHAR(64), +PRIMARY KEY (pk), +KEY (a) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 COLLATE=latin1_general_ci +PARTITION BY RANGE (pk) ( +PARTITION p0 VALUES LESS THAN (3), +PARTITION p1 VALUES LESS THAN (6), +PARTITION p2 VALUES LESS THAN MAXVALUE +); +INSERT INTO t2 VALUES (1, 'one'), (4, 'four'), (7, 'seven'); +RENAME TABLE t2 to t2_trunc; +SELECT * FROM t2_trunc ORDER BY pk; +pk a +1 one +4 four +7 seven +ALTER TABLE t2_trunc TRUNCATE PARTITION p0; +SELECT * FROM t2_trunc ORDER BY pk; +pk a +4 four +7 seven +TRUNCATE TABLE t2_trunc; +SELECT * FROM t2_trunc ORDER BY pk; +pk a +set global rocksdb_strict_collation_exceptions = default; +DROP TABLE t1_trunc, t2_trunc; +CREATE TABLE t1 ( +pk INT PRIMARY KEY +) ENGINE=ROCKSDB; +INSERT INTO t1 VALUES (1), (2); +SELECT * FROM t1; +pk +1 +2 +SET DEBUG_DBUG = "+d,rocksdb_truncate_failure"; +TRUNCATE TABLE t1; +ERROR HY000: Internal error: Simulated truncation failure. +SET DEBUG_DBUG = "-d,rocksdb_truncate_failure"; +SELECT * FROM t1; +pk +1 +2 +CREATE TABLE t2 ( +a INT +) ENGINE=ROCKSDB +PARTITION BY RANGE (a) ( +PARTITION p0 VALUES LESS THAN (3), +PARTITION p1 VALUES LESS THAN (6), +PARTITION p2 VALUES LESS THAN MAXVALUE +); +INSERT INTO t2 VALUES (1), (4), (7); +SET DEBUG_DBUG = "+d,rocksdb_truncate_failure"; +TRUNCATE TABLE t2; +ERROR HY000: Internal error: Simulated truncation failure. +ALTER TABLE t2 TRUNCATE PARTITION p1; +ERROR HY000: Internal error: Simulated truncation failure. +SET DEBUG_DBUG = "-d,rocksdb_truncate_failure"; +SELECT * FROM t2; +a +1 +4 +7 +DROP TABLE t1, t2; +CREATE TABLE t1_crash ( +pk INT PRIMARY KEY +) ENGINE=ROCKSDB; +INSERT INTO t1_crash VALUES (100), (1000); +SELECT * FROM t1_crash; +pk +100 +1000 +SET DEBUG_DBUG = "+d,rocksdb_truncate_failure_crash"; +TRUNCATE TABLE t1_crash; +ERROR HY000: Lost connection to server during query + MyRocks: Removing truncated leftover table test.#truncate_tmp#t1_crash +DROP TABLE t1_crash; +Warnings: +Warning 1932 Table 'test.t1_crash' doesn't exist in engine diff --git a/storage/rocksdb/mysql-test/rocksdb/r/truncate_table3.result b/storage/rocksdb/mysql-test/rocksdb/r/truncate_table3.result index 7c5631a2c97..359124e83ac 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/truncate_table3.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/truncate_table3.result @@ -1,12 +1,6 @@ -call mtr.add_suppression("Column family 'cf1' not found"); -call mtr.add_suppression("Column family 'rev:cf2' not found"); DROP TABLE IF EXISTS t1; call mtr.add_suppression("Column family 'cf1' not found"); call mtr.add_suppression("Column family 'rev:cf2' not found"); -set global rocksdb_compact_cf = 'cf1'; -set global rocksdb_compact_cf = 'rev:cf2'; -set global rocksdb_signal_drop_index_thread = 1; -# restart CREATE TABLE t1 ( a int not null, b int not null, @@ -15,6 +9,10 @@ primary key (a,b) comment 'cf1', key (b) comment 'rev:cf2' ) ENGINE=RocksDB; DELETE FROM t1; +set global rocksdb_compact_cf = 'cf1'; +set global rocksdb_compact_cf = 'rev:cf2'; +set global rocksdb_signal_drop_index_thread = 1; +# restart select variable_value into @a from information_schema.global_status where variable_name='rocksdb_compact_read_bytes'; truncate table t1; select case when variable_value-@a < 500000 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_compact_read_bytes'; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/ttl_primary.result b/storage/rocksdb/mysql-test/rocksdb/r/ttl_primary.result index 97c0e0c952b..b5e130c5b52 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/ttl_primary.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/ttl_primary.result @@ -255,11 +255,11 @@ CREATE TABLE t1 ( a bigint(20) UNSIGNED NOT NULL, b int NOT NULL, c int NOT NULL, -ts bigint(20), +ts bigint(20) UNSIGNED, PRIMARY KEY (a,c) ) ENGINE=rocksdb COMMENT='ttl_duration=1;ttl_col=ts;'; -ERROR HY000: TTL column (ts) in MyRocks must be an unsigned non-null 64-bit integer, exist inside the table, and have an accompanying ttl duration. +ERROR HY000: TTL column (ts) in MyRocks must be an unsigned non-null 64-bit integer or non-null timestamp, exist inside the table, and have an accompanying ttl duration. CREATE TABLE t1 ( a bigint(20) UNSIGNED NOT NULL, b int NOT NULL, @@ -268,7 +268,7 @@ ts int, PRIMARY KEY (a,c) ) ENGINE=rocksdb COMMENT='ttl_duration=1;ttl_col=ts;'; -ERROR HY000: TTL column (ts) in MyRocks must be an unsigned non-null 64-bit integer, exist inside the table, and have an accompanying ttl duration. +ERROR HY000: TTL column (ts) in MyRocks must be an unsigned non-null 64-bit integer or non-null timestamp, exist inside the table, and have an accompanying ttl duration. CREATE TABLE t1 ( a bigint(20) UNSIGNED NOT NULL, b int NOT NULL, @@ -284,7 +284,7 @@ c int NOT NULL, PRIMARY KEY (a,c) ) ENGINE=rocksdb COMMENT='ttl_duration=1;ttl_col=abc;'; -ERROR HY000: TTL column (abc) in MyRocks must be an unsigned non-null 64-bit integer, exist inside the table, and have an accompanying ttl duration. +ERROR HY000: TTL column (abc) in MyRocks must be an unsigned non-null 64-bit integer or non-null timestamp, exist inside the table, and have an accompanying ttl duration. CREATE TABLE t1 ( a bigint(20) UNSIGNED NOT NULL, b int NOT NULL, @@ -292,7 +292,7 @@ c int NOT NULL, PRIMARY KEY (a,c) ) ENGINE=rocksdb COMMENT='ttl_col=abc;'; -ERROR HY000: TTL column (abc) in MyRocks must be an unsigned non-null 64-bit integer, exist inside the table, and have an accompanying ttl duration. +ERROR HY000: TTL column (abc) in MyRocks must be an unsigned non-null 64-bit integer or non-null timestamp, exist inside the table, and have an accompanying ttl duration. CREATE TABLE t1 ( a bigint(20) NOT NULL, PRIMARY KEY (a) @@ -487,3 +487,1470 @@ select variable_value-@c from information_schema.global_status where variable_na variable_value-@c 0 DROP TABLE t1; +CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64) NOT NULL, +`c` varbinary(256) NOT NULL, +`ts` TIMESTAMP NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`b`,`a`,`c`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values ('a', 'b', 'c', NOW(), 'd'); +INSERT INTO t1 values ('d', 'e', 'f', NOW(), 'g'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +ts TIMESTAMP NOT NULL, +c int NOT NULL, +PRIMARY KEY (a) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, 3, NOW(), 5); +INSERT INTO t1 values (2, 4, NOW(), 6); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +c int NOT NULL, +ts TIMESTAMP NOT NULL, +PRIMARY KEY (a,c) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, 3, 5, NOW()); +INSERT INTO t1 values (2, 4, 6, NOW()); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +c int NOT NULL, +ts TIMESTAMP NOT NULL, +PRIMARY KEY (a,c) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, 3, 5, NOW()); +INSERT INTO t1 values (2, 4, 6, NOW()); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int, +c int, +ts TIMESTAMP NOT NULL, +PRIMARY KEY (a) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, NULL, NULL, NOW()); +INSERT INTO t1 values (2, NULL, NULL, NOW()); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64), +`c` varbinary(256), +`ts` TIMESTAMP NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`a`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values ('a', NULL, 'bc', NOW(), 'd'); +INSERT INTO t1 values ('d', 'efghijk', NULL, NOW(), 'l'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +c int NOT NULL, +PRIMARY KEY (a) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, 3, 5); +INSERT INTO t1 values (2, 4, 6); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a int, +ts TIMESTAMP NOT NULL, +PRIMARY KEY (a, ts) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;ttl_col=ts;'; +INSERT INTO t1 values (1, NOW()); +INSERT INTO t1 values (2, NOW()); +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_debug_ttl_snapshot_ts = -10; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_debug_ttl_snapshot_ts = 10; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +ts TIMESTAMP NOT NULL, +c int NOT NULL, +PRIMARY KEY (a, ts) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, 3, NOW(), 5); +INSERT INTO t1 values (2, 4, NOW(), 6); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64), +`c` varbinary(256), +`ts` TIMESTAMP NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`a`, `ts`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values ('a', NULL, 'bc', NOW(), 'd'); +INSERT INTO t1 values ('de', 'fghijk', NULL, NOW(), 'l'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a INT NOT NULL, +b varbinary(64) NOT NULL, +c varbinary(256) NOT NULL, +ts TIMESTAMP NOT NULL, +value mediumblob NOT NULL, +PRIMARY KEY (b,a,c) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=10;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -300; +INSERT INTO t1 values (1, 'b', 'c', NOW(), 'd'); +INSERT INTO t1 values (2, 'e', 'f', NOW(), 'g'); +set global rocksdb_debug_ttl_rec_ts = 300; +INSERT INTO t1 values (3, 'i', 'j', NOW(), 'k'); +INSERT INTO t1 values (4, 'm', 'n', NOW(), 'o'); +set global rocksdb_debug_ttl_rec_ts = 0; +set global rocksdb_debug_ttl_snapshot_ts = -3600; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT a FROM t1; +a +1 +2 +3 +4 +set global rocksdb_compact_cf='default'; +SELECT a FROM t1; +a +3 +4 +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT a FROM t1; +a +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) UNSIGNED NOT NULL, +b int NOT NULL, +c int NOT NULL, +ts TIMESTAMP, +PRIMARY KEY (a,c) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +ERROR HY000: TTL column (ts) in MyRocks must be an unsigned non-null 64-bit integer or non-null timestamp, exist inside the table, and have an accompanying ttl duration. +CREATE TABLE t1 ( +a bigint(20) UNSIGNED NOT NULL, +b int NOT NULL, +c int NOT NULL, +ts int, +PRIMARY KEY (a,c) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +ERROR HY000: TTL column (ts) in MyRocks must be an unsigned non-null 64-bit integer or non-null timestamp, exist inside the table, and have an accompanying ttl duration. +CREATE TABLE t1 ( +a bigint(20) UNSIGNED NOT NULL, +b int NOT NULL, +c int NOT NULL, +PRIMARY KEY (a,c) +) ENGINE=rocksdb +COMMENT='ttl_duration=abc;'; +ERROR HY000: TTL duration (abc) in MyRocks must be an unsigned non-null 64-bit integer. +CREATE TABLE t1 ( +a bigint(20) UNSIGNED NOT NULL, +b int NOT NULL, +c int NOT NULL, +PRIMARY KEY (a,c) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=abc;'; +ERROR HY000: TTL column (abc) in MyRocks must be an unsigned non-null 64-bit integer or non-null timestamp, exist inside the table, and have an accompanying ttl duration. +CREATE TABLE t1 ( +a bigint(20) UNSIGNED NOT NULL, +b int NOT NULL, +c int NOT NULL, +PRIMARY KEY (a,c) +) ENGINE=rocksdb +COMMENT='ttl_col=abc;'; +ERROR HY000: TTL column (abc) in MyRocks must be an unsigned non-null 64-bit integer or non-null timestamp, exist inside the table, and have an accompanying ttl duration. +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +PRIMARY KEY (a) +) ENGINE=rocksdb +COMMENT='ttl_duration=500;'; +INSERT INTO t1 values (1); +SELECT COUNT(*) FROM t1; +COUNT(*) +1 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +1 +DROP TABLE t1; +CREATE TABLE t1 ( +a INT PRIMARY KEY +) ENGINE=rocksdb +COMMENT='ttl_duration=100;'; +INSERT INTO t1 values (1); +SELECT * FROM t1; +a +1 +set global rocksdb_debug_ttl_rec_ts = -300; +ALTER TABLE t1 COMMENT = 'ttl_duration=1'; +set global rocksdb_debug_ttl_rec_ts = 0; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) NOT NULL, + PRIMARY KEY (`a`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='ttl_duration=1' +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a INT PRIMARY KEY, +b INT +) ENGINE=rocksdb +COMMENT='ttl_duration=100;'; +ALTER TABLE t1 DROP PRIMARY KEY; +ERROR HY000: TTL support is currently disabled when table has a hidden PK. +DROP TABLE t1; +CREATE TABLE t1 ( +a INT PRIMARY KEY, +b INT +) ENGINE=rocksdb +COMMENT='ttl_duration=5;'; +INSERT INTO t1 VALUES (1,1); +INSERT INTO t1 VALUES (2,2); +ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY(b); +set global rocksdb_debug_ttl_snapshot_ts = -3600; +set global rocksdb_force_flush_memtable_now=1; +set @@global.rocksdb_compact_cf = 'default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set @@global.rocksdb_compact_cf = 'default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a TIMESTAMP NOT NULL, +b int, +PRIMARY KEY (a,b) +) ENGINE=rocksdb +COMMENT='asdadfasdfsadfadf ;ttl_duration=1; asfasdfasdfadfa'; +INSERT INTO t1 values (NOW(), 1); +SELECT COUNT(*) FROM t1; +COUNT(*) +1 +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +ALTER TABLE t1 COMMENT = 'adsf;;ttl_duration=5;asfasdfa;ttl_col=a;asdfasdf;'; +set global rocksdb_debug_ttl_rec_ts = 300; +INSERT INTO t1 values (NOW(), 2); +set global rocksdb_debug_ttl_rec_ts = 0; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +1 +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +PRIMARY KEY (a) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;'; +set global rocksdb_debug_ttl_rec_ts = -300; +INSERT INTO t1 values (1); +INSERT INTO t1 values (3); +INSERT INTO t1 values (5); +set global rocksdb_debug_ttl_rec_ts = 300; +INSERT INTO t1 values (7); +INSERT INTO t1 values (9); +set global rocksdb_debug_ttl_rec_ts = 0; +UPDATE t1 SET a=a+1; +SELECT * FROM t1; +a +10 +2 +4 +6 +8 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT * FROM t1; +a +10 +8 +DROP TABLE t1; +CREATE TABLE t1 ( +a INT, +b TIMESTAMP NOT NULL, +PRIMARY KEY (a) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;ttl_col=b;'; +set global rocksdb_debug_ttl_rec_ts = -300; +INSERT INTO t1 values (1, NOW()); +INSERT INTO t1 values (3, NOW()); +INSERT INTO t1 values (5, NOW()); +INSERT INTO t1 values (7, NOW()); +set global rocksdb_debug_ttl_rec_ts = 300; +UPDATE t1 SET b=NOW() WHERE a < 4; +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT a FROM t1; +a +1 +3 +5 +7 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT a FROM t1; +a +1 +3 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +PRIMARY KEY (a) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1); +INSERT INTO t1 values (2); +INSERT INTO t1 values (3); +set global rocksdb_debug_ttl_rec_ts = 0; +set global rocksdb_enable_ttl=0; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +select variable_value into @c from information_schema.global_status where variable_name='rocksdb_rows_expired'; +set global rocksdb_enable_ttl=1; +set global rocksdb_compact_cf='default'; +select variable_value-@c from information_schema.global_status where variable_name='rocksdb_rows_expired'; +variable_value-@c +3 +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +PRIMARY KEY (a) +) ENGINE=rocksdb +COMMENT='ttl_duration=100;'; +INSERT INTO t1 values (1); +INSERT INTO t1 values (2); +INSERT INTO t1 values (3); +select variable_value into @c from information_schema.global_status where variable_name='rocksdb_rows_expired'; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +select variable_value-@c from information_schema.global_status where variable_name='rocksdb_rows_expired'; +variable_value-@c +0 +DROP TABLE t1; +CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64) NOT NULL, +`c` varbinary(256) NOT NULL, +`ts` TIMESTAMP DEFAULT NOW() NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`b`,`a`,`c`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values ('a', 'b', 'c', NOW(), 'd'); +INSERT INTO t1 values ('d', 'e', 'f', NOW(), 'g'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +ts TIMESTAMP DEFAULT NOW() NOT NULL, +c int NOT NULL, +PRIMARY KEY (a) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, 3, NOW(), 5); +INSERT INTO t1 values (2, 4, NOW(), 6); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +c int NOT NULL, +ts TIMESTAMP DEFAULT NOW() NOT NULL, +PRIMARY KEY (a,c) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, 3, 5, NOW()); +INSERT INTO t1 values (2, 4, 6, NOW()); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +c int NOT NULL, +ts TIMESTAMP DEFAULT NOW() NOT NULL, +PRIMARY KEY (a,c) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, 3, 5, NOW()); +INSERT INTO t1 values (2, 4, 6, NOW()); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int, +c int, +ts TIMESTAMP DEFAULT NOW() NOT NULL, +PRIMARY KEY (a) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, NULL, NULL, NOW()); +INSERT INTO t1 values (2, NULL, NULL, NOW()); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64), +`c` varbinary(256), +`ts` TIMESTAMP DEFAULT NOW() NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`a`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values ('a', NULL, 'bc', NOW(), 'd'); +INSERT INTO t1 values ('d', 'efghijk', NULL, NOW(), 'l'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +c int NOT NULL, +PRIMARY KEY (a) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, 3, 5); +INSERT INTO t1 values (2, 4, 6); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a int, +ts TIMESTAMP DEFAULT NOW() NOT NULL, +PRIMARY KEY (a, ts) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;ttl_col=ts;'; +INSERT INTO t1 values (1, NOW()); +INSERT INTO t1 values (2, NOW()); +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_debug_ttl_snapshot_ts = -10; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_debug_ttl_snapshot_ts = 10; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +ts TIMESTAMP DEFAULT NOW() NOT NULL, +c int NOT NULL, +PRIMARY KEY (a, ts) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, 3, NOW(), 5); +INSERT INTO t1 values (2, 4, NOW(), 6); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64), +`c` varbinary(256), +`ts` TIMESTAMP DEFAULT NOW() NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`a`, `ts`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values ('a', NULL, 'bc', NOW(), 'd'); +INSERT INTO t1 values ('de', 'fghijk', NULL, NOW(), 'l'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a INT NOT NULL, +b varbinary(64) NOT NULL, +c varbinary(256) NOT NULL, +ts TIMESTAMP DEFAULT NOW() NOT NULL, +value mediumblob NOT NULL, +PRIMARY KEY (b,a,c) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=10;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -300; +INSERT INTO t1 values (1, 'b', 'c', NOW(), 'd'); +INSERT INTO t1 values (2, 'e', 'f', NOW(), 'g'); +set global rocksdb_debug_ttl_rec_ts = 300; +INSERT INTO t1 values (3, 'i', 'j', NOW(), 'k'); +INSERT INTO t1 values (4, 'm', 'n', NOW(), 'o'); +set global rocksdb_debug_ttl_rec_ts = 0; +set global rocksdb_debug_ttl_snapshot_ts = -3600; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT a FROM t1; +a +1 +2 +3 +4 +set global rocksdb_compact_cf='default'; +SELECT a FROM t1; +a +3 +4 +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT a FROM t1; +a +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) UNSIGNED NOT NULL, +b int NOT NULL, +c int NOT NULL, +ts TIMESTAMP DEFAULT NOW(), +PRIMARY KEY (a,c) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +ERROR HY000: TTL column (ts) in MyRocks must be an unsigned non-null 64-bit integer or non-null timestamp, exist inside the table, and have an accompanying ttl duration. +CREATE TABLE t1 ( +a bigint(20) UNSIGNED NOT NULL, +b int NOT NULL, +c int NOT NULL, +ts int, +PRIMARY KEY (a,c) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +ERROR HY000: TTL column (ts) in MyRocks must be an unsigned non-null 64-bit integer or non-null timestamp, exist inside the table, and have an accompanying ttl duration. +CREATE TABLE t1 ( +a bigint(20) UNSIGNED NOT NULL, +b int NOT NULL, +c int NOT NULL, +PRIMARY KEY (a,c) +) ENGINE=rocksdb +COMMENT='ttl_duration=abc;'; +ERROR HY000: TTL duration (abc) in MyRocks must be an unsigned non-null 64-bit integer. +CREATE TABLE t1 ( +a bigint(20) UNSIGNED NOT NULL, +b int NOT NULL, +c int NOT NULL, +PRIMARY KEY (a,c) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=abc;'; +ERROR HY000: TTL column (abc) in MyRocks must be an unsigned non-null 64-bit integer or non-null timestamp, exist inside the table, and have an accompanying ttl duration. +CREATE TABLE t1 ( +a bigint(20) UNSIGNED NOT NULL, +b int NOT NULL, +c int NOT NULL, +PRIMARY KEY (a,c) +) ENGINE=rocksdb +COMMENT='ttl_col=abc;'; +ERROR HY000: TTL column (abc) in MyRocks must be an unsigned non-null 64-bit integer or non-null timestamp, exist inside the table, and have an accompanying ttl duration. +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +PRIMARY KEY (a) +) ENGINE=rocksdb +COMMENT='ttl_duration=500;'; +INSERT INTO t1 values (1); +SELECT COUNT(*) FROM t1; +COUNT(*) +1 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +1 +DROP TABLE t1; +CREATE TABLE t1 ( +a INT PRIMARY KEY +) ENGINE=rocksdb +COMMENT='ttl_duration=100;'; +INSERT INTO t1 values (1); +SELECT * FROM t1; +a +1 +set global rocksdb_debug_ttl_rec_ts = -300; +ALTER TABLE t1 COMMENT = 'ttl_duration=1'; +set global rocksdb_debug_ttl_rec_ts = 0; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) NOT NULL, + PRIMARY KEY (`a`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='ttl_duration=1' +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a INT PRIMARY KEY, +b INT +) ENGINE=rocksdb +COMMENT='ttl_duration=100;'; +ALTER TABLE t1 DROP PRIMARY KEY; +ERROR HY000: TTL support is currently disabled when table has a hidden PK. +DROP TABLE t1; +CREATE TABLE t1 ( +a INT PRIMARY KEY, +b INT +) ENGINE=rocksdb +COMMENT='ttl_duration=5;'; +INSERT INTO t1 VALUES (1,1); +INSERT INTO t1 VALUES (2,2); +ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY(b); +set global rocksdb_debug_ttl_snapshot_ts = -3600; +set global rocksdb_force_flush_memtable_now=1; +set @@global.rocksdb_compact_cf = 'default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set @@global.rocksdb_compact_cf = 'default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a TIMESTAMP DEFAULT NOW() NOT NULL, +b int, +PRIMARY KEY (a,b) +) ENGINE=rocksdb +COMMENT='asdadfasdfsadfadf ;ttl_duration=1; asfasdfasdfadfa'; +INSERT INTO t1 values (NOW(), 1); +SELECT COUNT(*) FROM t1; +COUNT(*) +1 +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +ALTER TABLE t1 COMMENT = 'adsf;;ttl_duration=5;asfasdfa;ttl_col=a;asdfasdf;'; +set global rocksdb_debug_ttl_rec_ts = 300; +INSERT INTO t1 values (NOW(), 2); +set global rocksdb_debug_ttl_rec_ts = 0; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +1 +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +PRIMARY KEY (a) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;'; +set global rocksdb_debug_ttl_rec_ts = -300; +INSERT INTO t1 values (1); +INSERT INTO t1 values (3); +INSERT INTO t1 values (5); +set global rocksdb_debug_ttl_rec_ts = 300; +INSERT INTO t1 values (7); +INSERT INTO t1 values (9); +set global rocksdb_debug_ttl_rec_ts = 0; +UPDATE t1 SET a=a+1; +SELECT * FROM t1; +a +10 +2 +4 +6 +8 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT * FROM t1; +a +10 +8 +DROP TABLE t1; +CREATE TABLE t1 ( +a INT, +b TIMESTAMP DEFAULT NOW() NOT NULL, +PRIMARY KEY (a) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;ttl_col=b;'; +set global rocksdb_debug_ttl_rec_ts = -300; +INSERT INTO t1 values (1, NOW()); +INSERT INTO t1 values (3, NOW()); +INSERT INTO t1 values (5, NOW()); +INSERT INTO t1 values (7, NOW()); +set global rocksdb_debug_ttl_rec_ts = 300; +UPDATE t1 SET b=NOW() WHERE a < 4; +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT a FROM t1; +a +1 +3 +5 +7 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT a FROM t1; +a +1 +3 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +PRIMARY KEY (a) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1); +INSERT INTO t1 values (2); +INSERT INTO t1 values (3); +set global rocksdb_debug_ttl_rec_ts = 0; +set global rocksdb_enable_ttl=0; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +select variable_value into @c from information_schema.global_status where variable_name='rocksdb_rows_expired'; +set global rocksdb_enable_ttl=1; +set global rocksdb_compact_cf='default'; +select variable_value-@c from information_schema.global_status where variable_name='rocksdb_rows_expired'; +variable_value-@c +3 +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +PRIMARY KEY (a) +) ENGINE=rocksdb +COMMENT='ttl_duration=100;'; +INSERT INTO t1 values (1); +INSERT INTO t1 values (2); +INSERT INTO t1 values (3); +select variable_value into @c from information_schema.global_status where variable_name='rocksdb_rows_expired'; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +select variable_value-@c from information_schema.global_status where variable_name='rocksdb_rows_expired'; +variable_value-@c +0 +DROP TABLE t1; +CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64) NOT NULL, +`c` varbinary(256) NOT NULL, +`ts` TIMESTAMP(6) NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`b`,`a`,`c`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values ('a', 'b', 'c', NOW(), 'd'); +INSERT INTO t1 values ('d', 'e', 'f', NOW(), 'g'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +ts TIMESTAMP(6) NOT NULL, +c int NOT NULL, +PRIMARY KEY (a) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, 3, NOW(), 5); +INSERT INTO t1 values (2, 4, NOW(), 6); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +c int NOT NULL, +ts TIMESTAMP(6) NOT NULL, +PRIMARY KEY (a,c) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, 3, 5, NOW()); +INSERT INTO t1 values (2, 4, 6, NOW()); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +c int NOT NULL, +ts TIMESTAMP(6) NOT NULL, +PRIMARY KEY (a,c) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, 3, 5, NOW()); +INSERT INTO t1 values (2, 4, 6, NOW()); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int, +c int, +ts TIMESTAMP(6) NOT NULL, +PRIMARY KEY (a) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, NULL, NULL, NOW()); +INSERT INTO t1 values (2, NULL, NULL, NOW()); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64), +`c` varbinary(256), +`ts` TIMESTAMP(6) NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`a`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values ('a', NULL, 'bc', NOW(), 'd'); +INSERT INTO t1 values ('d', 'efghijk', NULL, NOW(), 'l'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +c int NOT NULL, +PRIMARY KEY (a) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, 3, 5); +INSERT INTO t1 values (2, 4, 6); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a int, +ts TIMESTAMP(6) NOT NULL, +PRIMARY KEY (a, ts) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;ttl_col=ts;'; +INSERT INTO t1 values (1, NOW()); +INSERT INTO t1 values (2, NOW()); +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_debug_ttl_snapshot_ts = -10; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_debug_ttl_snapshot_ts = 10; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +ts TIMESTAMP(6) NOT NULL, +c int NOT NULL, +PRIMARY KEY (a, ts) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, 3, NOW(), 5); +INSERT INTO t1 values (2, 4, NOW(), 6); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64), +`c` varbinary(256), +`ts` TIMESTAMP(6) NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`a`, `ts`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values ('a', NULL, 'bc', NOW(), 'd'); +INSERT INTO t1 values ('de', 'fghijk', NULL, NOW(), 'l'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a INT NOT NULL, +b varbinary(64) NOT NULL, +c varbinary(256) NOT NULL, +ts TIMESTAMP(6) NOT NULL, +value mediumblob NOT NULL, +PRIMARY KEY (b,a,c) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=10;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -300; +INSERT INTO t1 values (1, 'b', 'c', NOW(), 'd'); +INSERT INTO t1 values (2, 'e', 'f', NOW(), 'g'); +set global rocksdb_debug_ttl_rec_ts = 300; +INSERT INTO t1 values (3, 'i', 'j', NOW(), 'k'); +INSERT INTO t1 values (4, 'm', 'n', NOW(), 'o'); +set global rocksdb_debug_ttl_rec_ts = 0; +set global rocksdb_debug_ttl_snapshot_ts = -3600; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT a FROM t1; +a +1 +2 +3 +4 +set global rocksdb_compact_cf='default'; +SELECT a FROM t1; +a +3 +4 +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT a FROM t1; +a +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) UNSIGNED NOT NULL, +b int NOT NULL, +c int NOT NULL, +ts TIMESTAMP(6), +PRIMARY KEY (a,c) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +ERROR HY000: TTL column (ts) in MyRocks must be an unsigned non-null 64-bit integer or non-null timestamp, exist inside the table, and have an accompanying ttl duration. +CREATE TABLE t1 ( +a bigint(20) UNSIGNED NOT NULL, +b int NOT NULL, +c int NOT NULL, +ts int, +PRIMARY KEY (a,c) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +ERROR HY000: TTL column (ts) in MyRocks must be an unsigned non-null 64-bit integer or non-null timestamp, exist inside the table, and have an accompanying ttl duration. +CREATE TABLE t1 ( +a bigint(20) UNSIGNED NOT NULL, +b int NOT NULL, +c int NOT NULL, +PRIMARY KEY (a,c) +) ENGINE=rocksdb +COMMENT='ttl_duration=abc;'; +ERROR HY000: TTL duration (abc) in MyRocks must be an unsigned non-null 64-bit integer. +CREATE TABLE t1 ( +a bigint(20) UNSIGNED NOT NULL, +b int NOT NULL, +c int NOT NULL, +PRIMARY KEY (a,c) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=abc;'; +ERROR HY000: TTL column (abc) in MyRocks must be an unsigned non-null 64-bit integer or non-null timestamp, exist inside the table, and have an accompanying ttl duration. +CREATE TABLE t1 ( +a bigint(20) UNSIGNED NOT NULL, +b int NOT NULL, +c int NOT NULL, +PRIMARY KEY (a,c) +) ENGINE=rocksdb +COMMENT='ttl_col=abc;'; +ERROR HY000: TTL column (abc) in MyRocks must be an unsigned non-null 64-bit integer or non-null timestamp, exist inside the table, and have an accompanying ttl duration. +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +PRIMARY KEY (a) +) ENGINE=rocksdb +COMMENT='ttl_duration=500;'; +INSERT INTO t1 values (1); +SELECT COUNT(*) FROM t1; +COUNT(*) +1 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +1 +DROP TABLE t1; +CREATE TABLE t1 ( +a INT PRIMARY KEY +) ENGINE=rocksdb +COMMENT='ttl_duration=100;'; +INSERT INTO t1 values (1); +SELECT * FROM t1; +a +1 +set global rocksdb_debug_ttl_rec_ts = -300; +ALTER TABLE t1 COMMENT = 'ttl_duration=1'; +set global rocksdb_debug_ttl_rec_ts = 0; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) NOT NULL, + PRIMARY KEY (`a`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='ttl_duration=1' +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a INT PRIMARY KEY, +b INT +) ENGINE=rocksdb +COMMENT='ttl_duration=100;'; +ALTER TABLE t1 DROP PRIMARY KEY; +ERROR HY000: TTL support is currently disabled when table has a hidden PK. +DROP TABLE t1; +CREATE TABLE t1 ( +a INT PRIMARY KEY, +b INT +) ENGINE=rocksdb +COMMENT='ttl_duration=5;'; +INSERT INTO t1 VALUES (1,1); +INSERT INTO t1 VALUES (2,2); +ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY(b); +set global rocksdb_debug_ttl_snapshot_ts = -3600; +set global rocksdb_force_flush_memtable_now=1; +set @@global.rocksdb_compact_cf = 'default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set @@global.rocksdb_compact_cf = 'default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a TIMESTAMP(6) NOT NULL, +b int, +PRIMARY KEY (a,b) +) ENGINE=rocksdb +COMMENT='asdadfasdfsadfadf ;ttl_duration=1; asfasdfasdfadfa'; +INSERT INTO t1 values (NOW(), 1); +SELECT COUNT(*) FROM t1; +COUNT(*) +1 +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +ALTER TABLE t1 COMMENT = 'adsf;;ttl_duration=5;asfasdfa;ttl_col=a;asdfasdf;'; +set global rocksdb_debug_ttl_rec_ts = 300; +INSERT INTO t1 values (NOW(), 2); +set global rocksdb_debug_ttl_rec_ts = 0; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; +COUNT(*) +1 +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +PRIMARY KEY (a) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;'; +set global rocksdb_debug_ttl_rec_ts = -300; +INSERT INTO t1 values (1); +INSERT INTO t1 values (3); +INSERT INTO t1 values (5); +set global rocksdb_debug_ttl_rec_ts = 300; +INSERT INTO t1 values (7); +INSERT INTO t1 values (9); +set global rocksdb_debug_ttl_rec_ts = 0; +UPDATE t1 SET a=a+1; +SELECT * FROM t1; +a +10 +2 +4 +6 +8 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT * FROM t1; +a +10 +8 +DROP TABLE t1; +CREATE TABLE t1 ( +a INT, +b TIMESTAMP(6) NOT NULL, +PRIMARY KEY (a) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;ttl_col=b;'; +set global rocksdb_debug_ttl_rec_ts = -300; +INSERT INTO t1 values (1, NOW()); +INSERT INTO t1 values (3, NOW()); +INSERT INTO t1 values (5, NOW()); +INSERT INTO t1 values (7, NOW()); +set global rocksdb_debug_ttl_rec_ts = 300; +UPDATE t1 SET b=NOW() WHERE a < 4; +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT a FROM t1; +a +1 +3 +5 +7 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT a FROM t1; +a +1 +3 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +PRIMARY KEY (a) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1); +INSERT INTO t1 values (2); +INSERT INTO t1 values (3); +set global rocksdb_debug_ttl_rec_ts = 0; +set global rocksdb_enable_ttl=0; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +select variable_value into @c from information_schema.global_status where variable_name='rocksdb_rows_expired'; +set global rocksdb_enable_ttl=1; +set global rocksdb_compact_cf='default'; +select variable_value-@c from information_schema.global_status where variable_name='rocksdb_rows_expired'; +variable_value-@c +3 +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +PRIMARY KEY (a) +) ENGINE=rocksdb +COMMENT='ttl_duration=100;'; +INSERT INTO t1 values (1); +INSERT INTO t1 values (2); +INSERT INTO t1 values (3); +select variable_value into @c from information_schema.global_status where variable_name='rocksdb_rows_expired'; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +select variable_value-@c from information_schema.global_status where variable_name='rocksdb_rows_expired'; +variable_value-@c +0 +DROP TABLE t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/ttl_primary_with_partitions.result b/storage/rocksdb/mysql-test/rocksdb/r/ttl_primary_with_partitions.result index 49c42a8601d..dfb0d111c48 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/ttl_primary_with_partitions.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/ttl_primary_with_partitions.result @@ -254,3 +254,771 @@ set global rocksdb_debug_ttl_snapshot_ts = 0; SELECT c1 FROM t1; c1 DROP TABLE t1; +CREATE TABLE t1 ( +c1 INT, +PRIMARY KEY (`c1`) +) ENGINE=ROCKSDB +COMMENT="custom_p0_ttl_duration=1;" +PARTITION BY LIST(c1) ( +PARTITION custom_p0 VALUES IN (1, 4, 7), +PARTITION custom_p1 VALUES IN (2, 5, 8), +PARTITION custom_p2 VALUES IN (3, 6, 9) +); +set global rocksdb_debug_ttl_rec_ts = -3600; +INSERT INTO t1 values (1); +INSERT INTO t1 values (2); +INSERT INTO t1 values (3); +INSERT INTO t1 values (4); +INSERT INTO t1 values (5); +INSERT INTO t1 values (6); +INSERT INTO t1 values (7); +INSERT INTO t1 values (8); +INSERT INTO t1 values (9); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT * FROM t1; +c1 +1 +2 +3 +4 +5 +6 +7 +8 +9 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT * FROM t1; +c1 +2 +3 +5 +6 +8 +9 +DROP TABLE t1; +CREATE TABLE t1 ( +c1 INT, +c2 INT, +name VARCHAR(25) NOT NULL, +PRIMARY KEY (`c1`, `c2`) COMMENT 'custom_p0_cfname=foo;custom_p1_cfname=my_custom_cf;custom_p2_cfname=baz' +) ENGINE=ROCKSDB +COMMENT="custom_p0_ttl_duration=1;custom_p1_ttl_duration=7;" +PARTITION BY LIST(c1) ( +PARTITION custom_p0 VALUES IN (1, 4, 7), +PARTITION custom_p1 VALUES IN (2, 5, 8), +PARTITION custom_p2 VALUES IN (3, 6, 9) +); +set global rocksdb_debug_ttl_rec_ts = -1200; +INSERT INTO t1 values (1,1,'a'); +INSERT INTO t1 values (4,4,'aaaa'); +INSERT INTO t1 values (7,7,'aaaaaaa'); +set global rocksdb_debug_ttl_rec_ts = 1200; +INSERT INTO t1 values (2,2,'aa'); +INSERT INTO t1 values (3,3,'aaa'); +INSERT INTO t1 values (5,5,'aaaaa'); +INSERT INTO t1 values (6,6,'aaaaaa'); +INSERT INTO t1 values (8,8,'aaaaaaaa'); +INSERT INTO t1 values (9,9,'aaaaaaaaa'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT * FROM t1; +c1 c2 name +1 1 a +2 2 aa +3 3 aaa +4 4 aaaa +5 5 aaaaa +6 6 aaaaaa +7 7 aaaaaaa +8 8 aaaaaaaa +9 9 aaaaaaaaa +set global rocksdb_force_flush_memtable_now=1; +set @@global.rocksdb_compact_cf = 'foo'; +set @@global.rocksdb_compact_cf = 'my_custom_cf'; +SELECT * FROM t1; +c1 c2 name +2 2 aa +3 3 aaa +5 5 aaaaa +6 6 aaaaaa +8 8 aaaaaaaa +9 9 aaaaaaaaa +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set @@global.rocksdb_compact_cf = 'foo'; +SELECT * FROM t1; +c1 c2 name +2 2 aa +3 3 aaa +5 5 aaaaa +6 6 aaaaaa +8 8 aaaaaaaa +9 9 aaaaaaaaa +set @@global.rocksdb_compact_cf = 'my_custom_cf'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT * FROM t1; +c1 c2 name +3 3 aaa +6 6 aaaaaa +9 9 aaaaaaaaa +DROP TABLE t1; +CREATE TABLE t1 ( +c1 INT, +c2 INT, +name VARCHAR(25) NOT NULL, +event DATE, +PRIMARY KEY (`c1`, `c2`) COMMENT 'custom_p0_cfname=foo;custom_p1_cfname=bar;custom_p2_cfname=baz;' +) ENGINE=ROCKSDB +COMMENT="custom_p0_ttl_duration=9999;custom_p2_ttl_duration=5;" +PARTITION BY LIST(c1) ( +PARTITION custom_p0 VALUES IN (1, 2, 3), +PARTITION custom_p1 VALUES IN (4, 5, 6), +PARTITION custom_p2 VALUES IN (7, 8, 9) +); +INSERT INTO t1 VALUES (1, 1, "one", null); +INSERT INTO t1 VALUES (2, 2, "two", null); +INSERT INTO t1 VALUES (3, 3, "three", null); +INSERT INTO t1 VALUES (4, 4, "four", null); +INSERT INTO t1 VALUES (5, 5, "five", null); +INSERT INTO t1 VALUES (6, 6, "six", null); +INSERT INTO t1 VALUES (7, 7, "seven", null); +INSERT INTO t1 VALUES (8, 8, "eight", null); +INSERT INTO t1 VALUES (9, 9, "nine", null); +SELECT * FROM t1; +c1 c2 name event +1 1 one NULL +2 2 two NULL +3 3 three NULL +4 4 four NULL +5 5 five NULL +6 6 six NULL +7 7 seven NULL +8 8 eight NULL +9 9 nine NULL +set global rocksdb_debug_ttl_rec_ts = 600; +ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY(`c2`,`c1`) COMMENT 'custom_p0_cfname=foo;custom_p1_cfname=bar;custom_p2_cfname=baz;'; +set global rocksdb_debug_ttl_rec_ts = 0; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c1` int(11) NOT NULL, + `c2` int(11) NOT NULL, + `name` varchar(25) NOT NULL, + `event` date DEFAULT NULL, + PRIMARY KEY (`c2`,`c1`) COMMENT 'custom_p0_cfname=foo;custom_p1_cfname=bar;custom_p2_cfname=baz;' +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='custom_p0_ttl_duration=9999;custom_p2_ttl_duration=5;' + PARTITION BY LIST (`c1`) +(PARTITION `custom_p0` VALUES IN (1,2,3) ENGINE = ROCKSDB, + PARTITION `custom_p1` VALUES IN (4,5,6) ENGINE = ROCKSDB, + PARTITION `custom_p2` VALUES IN (7,8,9) ENGINE = ROCKSDB) +set global rocksdb_debug_ttl_snapshot_ts = 100; +set global rocksdb_force_flush_memtable_now=1; +set @@global.rocksdb_compact_cf = 'baz'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT * FROM t1; +c1 c2 name event +1 1 one NULL +2 2 two NULL +3 3 three NULL +4 4 four NULL +5 5 five NULL +6 6 six NULL +7 7 seven NULL +8 8 eight NULL +9 9 nine NULL +set global rocksdb_debug_ttl_snapshot_ts = 1200; +set @@global.rocksdb_compact_cf = 'foo'; +set @@global.rocksdb_compact_cf = 'baz'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT * FROM t1; +c1 c2 name event +1 1 one NULL +2 2 two NULL +3 3 three NULL +4 4 four NULL +5 5 five NULL +6 6 six NULL +DROP TABLE t1; +CREATE TABLE t1 ( +c1 BIGINT, +c2 TIMESTAMP NOT NULL, +name VARCHAR(25) NOT NULL, +event DATE, +PRIMARY KEY (`c1`) COMMENT 'custom_p0_cfname=foo;custom_p1_cfname=bar;custom_p2_cfname=baz;' +) ENGINE=ROCKSDB +COMMENT="ttl_duration=1;custom_p1_ttl_duration=100;custom_p1_ttl_col=c2;custom_p2_ttl_duration=5000;" +PARTITION BY LIST(c1) ( +PARTITION custom_p0 VALUES IN (1, 2, 3), +PARTITION custom_p1 VALUES IN (4, 5, 6), +PARTITION custom_p2 VALUES IN (7, 8, 9) +); +set global rocksdb_debug_ttl_rec_ts = -300; +INSERT INTO t1 VALUES (1, NOW(), "one", null); +INSERT INTO t1 VALUES (2, NOW(), "two", null); +INSERT INTO t1 VALUES (3, NOW(), "three", null); +set global rocksdb_debug_ttl_rec_ts = 0; +INSERT INTO t1 VALUES (4, NOW(), "four", null); +INSERT INTO t1 VALUES (5, NOW(), "five", null); +INSERT INTO t1 VALUES (6, NOW(), "six", null); +INSERT INTO t1 VALUES (7, NOW(), "seven", null); +INSERT INTO t1 VALUES (8, NOW(), "eight", null); +INSERT INTO t1 VALUES (9, NOW(), "nine", null); +set global rocksdb_force_flush_memtable_now=1; +set @@global.rocksdb_compact_cf = 'foo'; +set @@global.rocksdb_compact_cf = 'baz'; +set @@global.rocksdb_compact_cf = 'bar'; +SELECT c1 FROM t1; +c1 +4 +5 +6 +7 +8 +9 +set global rocksdb_debug_ttl_snapshot_ts = 600; +set @@global.rocksdb_compact_cf = 'bar'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT c1 FROM t1; +c1 +7 +8 +9 +DROP TABLE t1; +CREATE TABLE t1 ( +c1 BIGINT, +c2 TIMESTAMP NOT NULL, +PRIMARY KEY (`c1`, `c2`) +) ENGINE=ROCKSDB +COMMENT="ttl_duration=100;ttl_col=c2;" +PARTITION BY LIST(c1) ( +PARTITION custom_p0 VALUES IN (1), +PARTITION custom_p1 VALUES IN (2), +PARTITION custom_p2 VALUES IN (3) +); +INSERT INTO t1 values (1, NOW()); +INSERT INTO t1 values (2, NOW()); +INSERT INTO t1 values (3, NOW()); +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT c1 FROM t1; +c1 +1 +2 +3 +set global rocksdb_debug_ttl_snapshot_ts = 300; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT c1 FROM t1; +c1 +DROP TABLE t1; +CREATE TABLE t1 ( +c1 INT, +PRIMARY KEY (`c1`) +) ENGINE=ROCKSDB +COMMENT="custom_p0_ttl_duration=1;" +PARTITION BY LIST(c1) ( +PARTITION custom_p0 VALUES IN (1, 4, 7), +PARTITION custom_p1 VALUES IN (2, 5, 8), +PARTITION custom_p2 VALUES IN (3, 6, 9) +); +set global rocksdb_debug_ttl_rec_ts = -3600; +INSERT INTO t1 values (1); +INSERT INTO t1 values (2); +INSERT INTO t1 values (3); +INSERT INTO t1 values (4); +INSERT INTO t1 values (5); +INSERT INTO t1 values (6); +INSERT INTO t1 values (7); +INSERT INTO t1 values (8); +INSERT INTO t1 values (9); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT * FROM t1; +c1 +1 +2 +3 +4 +5 +6 +7 +8 +9 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT * FROM t1; +c1 +2 +3 +5 +6 +8 +9 +DROP TABLE t1; +CREATE TABLE t1 ( +c1 INT, +c2 INT, +name VARCHAR(25) NOT NULL, +PRIMARY KEY (`c1`, `c2`) COMMENT 'custom_p0_cfname=foo;custom_p1_cfname=my_custom_cf;custom_p2_cfname=baz' +) ENGINE=ROCKSDB +COMMENT="custom_p0_ttl_duration=1;custom_p1_ttl_duration=7;" +PARTITION BY LIST(c1) ( +PARTITION custom_p0 VALUES IN (1, 4, 7), +PARTITION custom_p1 VALUES IN (2, 5, 8), +PARTITION custom_p2 VALUES IN (3, 6, 9) +); +set global rocksdb_debug_ttl_rec_ts = -1200; +INSERT INTO t1 values (1,1,'a'); +INSERT INTO t1 values (4,4,'aaaa'); +INSERT INTO t1 values (7,7,'aaaaaaa'); +set global rocksdb_debug_ttl_rec_ts = 1200; +INSERT INTO t1 values (2,2,'aa'); +INSERT INTO t1 values (3,3,'aaa'); +INSERT INTO t1 values (5,5,'aaaaa'); +INSERT INTO t1 values (6,6,'aaaaaa'); +INSERT INTO t1 values (8,8,'aaaaaaaa'); +INSERT INTO t1 values (9,9,'aaaaaaaaa'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT * FROM t1; +c1 c2 name +1 1 a +2 2 aa +3 3 aaa +4 4 aaaa +5 5 aaaaa +6 6 aaaaaa +7 7 aaaaaaa +8 8 aaaaaaaa +9 9 aaaaaaaaa +set global rocksdb_force_flush_memtable_now=1; +set @@global.rocksdb_compact_cf = 'foo'; +set @@global.rocksdb_compact_cf = 'my_custom_cf'; +SELECT * FROM t1; +c1 c2 name +2 2 aa +3 3 aaa +5 5 aaaaa +6 6 aaaaaa +8 8 aaaaaaaa +9 9 aaaaaaaaa +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set @@global.rocksdb_compact_cf = 'foo'; +SELECT * FROM t1; +c1 c2 name +2 2 aa +3 3 aaa +5 5 aaaaa +6 6 aaaaaa +8 8 aaaaaaaa +9 9 aaaaaaaaa +set @@global.rocksdb_compact_cf = 'my_custom_cf'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT * FROM t1; +c1 c2 name +3 3 aaa +6 6 aaaaaa +9 9 aaaaaaaaa +DROP TABLE t1; +CREATE TABLE t1 ( +c1 INT, +c2 INT, +name VARCHAR(25) NOT NULL, +event DATE, +PRIMARY KEY (`c1`, `c2`) COMMENT 'custom_p0_cfname=foo;custom_p1_cfname=bar;custom_p2_cfname=baz;' +) ENGINE=ROCKSDB +COMMENT="custom_p0_ttl_duration=9999;custom_p2_ttl_duration=5;" +PARTITION BY LIST(c1) ( +PARTITION custom_p0 VALUES IN (1, 2, 3), +PARTITION custom_p1 VALUES IN (4, 5, 6), +PARTITION custom_p2 VALUES IN (7, 8, 9) +); +INSERT INTO t1 VALUES (1, 1, "one", null); +INSERT INTO t1 VALUES (2, 2, "two", null); +INSERT INTO t1 VALUES (3, 3, "three", null); +INSERT INTO t1 VALUES (4, 4, "four", null); +INSERT INTO t1 VALUES (5, 5, "five", null); +INSERT INTO t1 VALUES (6, 6, "six", null); +INSERT INTO t1 VALUES (7, 7, "seven", null); +INSERT INTO t1 VALUES (8, 8, "eight", null); +INSERT INTO t1 VALUES (9, 9, "nine", null); +SELECT * FROM t1; +c1 c2 name event +1 1 one NULL +2 2 two NULL +3 3 three NULL +4 4 four NULL +5 5 five NULL +6 6 six NULL +7 7 seven NULL +8 8 eight NULL +9 9 nine NULL +set global rocksdb_debug_ttl_rec_ts = 600; +ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY(`c2`,`c1`) COMMENT 'custom_p0_cfname=foo;custom_p1_cfname=bar;custom_p2_cfname=baz;'; +set global rocksdb_debug_ttl_rec_ts = 0; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c1` int(11) NOT NULL, + `c2` int(11) NOT NULL, + `name` varchar(25) NOT NULL, + `event` date DEFAULT NULL, + PRIMARY KEY (`c2`,`c1`) COMMENT 'custom_p0_cfname=foo;custom_p1_cfname=bar;custom_p2_cfname=baz;' +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='custom_p0_ttl_duration=9999;custom_p2_ttl_duration=5;' + PARTITION BY LIST (`c1`) +(PARTITION `custom_p0` VALUES IN (1,2,3) ENGINE = ROCKSDB, + PARTITION `custom_p1` VALUES IN (4,5,6) ENGINE = ROCKSDB, + PARTITION `custom_p2` VALUES IN (7,8,9) ENGINE = ROCKSDB) +set global rocksdb_debug_ttl_snapshot_ts = 100; +set global rocksdb_force_flush_memtable_now=1; +set @@global.rocksdb_compact_cf = 'baz'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT * FROM t1; +c1 c2 name event +1 1 one NULL +2 2 two NULL +3 3 three NULL +4 4 four NULL +5 5 five NULL +6 6 six NULL +7 7 seven NULL +8 8 eight NULL +9 9 nine NULL +set global rocksdb_debug_ttl_snapshot_ts = 1200; +set @@global.rocksdb_compact_cf = 'foo'; +set @@global.rocksdb_compact_cf = 'baz'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT * FROM t1; +c1 c2 name event +1 1 one NULL +2 2 two NULL +3 3 three NULL +4 4 four NULL +5 5 five NULL +6 6 six NULL +DROP TABLE t1; +CREATE TABLE t1 ( +c1 BIGINT, +c2 TIMESTAMP DEFAULT NOW() NOT NULL, +name VARCHAR(25) NOT NULL, +event DATE, +PRIMARY KEY (`c1`) COMMENT 'custom_p0_cfname=foo;custom_p1_cfname=bar;custom_p2_cfname=baz;' +) ENGINE=ROCKSDB +COMMENT="ttl_duration=1;custom_p1_ttl_duration=100;custom_p1_ttl_col=c2;custom_p2_ttl_duration=5000;" +PARTITION BY LIST(c1) ( +PARTITION custom_p0 VALUES IN (1, 2, 3), +PARTITION custom_p1 VALUES IN (4, 5, 6), +PARTITION custom_p2 VALUES IN (7, 8, 9) +); +set global rocksdb_debug_ttl_rec_ts = -300; +INSERT INTO t1 VALUES (1, NOW(), "one", null); +INSERT INTO t1 VALUES (2, NOW(), "two", null); +INSERT INTO t1 VALUES (3, NOW(), "three", null); +set global rocksdb_debug_ttl_rec_ts = 0; +INSERT INTO t1 VALUES (4, NOW(), "four", null); +INSERT INTO t1 VALUES (5, NOW(), "five", null); +INSERT INTO t1 VALUES (6, NOW(), "six", null); +INSERT INTO t1 VALUES (7, NOW(), "seven", null); +INSERT INTO t1 VALUES (8, NOW(), "eight", null); +INSERT INTO t1 VALUES (9, NOW(), "nine", null); +set global rocksdb_force_flush_memtable_now=1; +set @@global.rocksdb_compact_cf = 'foo'; +set @@global.rocksdb_compact_cf = 'baz'; +set @@global.rocksdb_compact_cf = 'bar'; +SELECT c1 FROM t1; +c1 +4 +5 +6 +7 +8 +9 +set global rocksdb_debug_ttl_snapshot_ts = 600; +set @@global.rocksdb_compact_cf = 'bar'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT c1 FROM t1; +c1 +7 +8 +9 +DROP TABLE t1; +CREATE TABLE t1 ( +c1 BIGINT, +c2 TIMESTAMP DEFAULT NOW() NOT NULL, +PRIMARY KEY (`c1`, `c2`) +) ENGINE=ROCKSDB +COMMENT="ttl_duration=100;ttl_col=c2;" +PARTITION BY LIST(c1) ( +PARTITION custom_p0 VALUES IN (1), +PARTITION custom_p1 VALUES IN (2), +PARTITION custom_p2 VALUES IN (3) +); +INSERT INTO t1 values (1, NOW()); +INSERT INTO t1 values (2, NOW()); +INSERT INTO t1 values (3, NOW()); +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT c1 FROM t1; +c1 +1 +2 +3 +set global rocksdb_debug_ttl_snapshot_ts = 300; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT c1 FROM t1; +c1 +DROP TABLE t1; +CREATE TABLE t1 ( +c1 INT, +PRIMARY KEY (`c1`) +) ENGINE=ROCKSDB +COMMENT="custom_p0_ttl_duration=1;" +PARTITION BY LIST(c1) ( +PARTITION custom_p0 VALUES IN (1, 4, 7), +PARTITION custom_p1 VALUES IN (2, 5, 8), +PARTITION custom_p2 VALUES IN (3, 6, 9) +); +set global rocksdb_debug_ttl_rec_ts = -3600; +INSERT INTO t1 values (1); +INSERT INTO t1 values (2); +INSERT INTO t1 values (3); +INSERT INTO t1 values (4); +INSERT INTO t1 values (5); +INSERT INTO t1 values (6); +INSERT INTO t1 values (7); +INSERT INTO t1 values (8); +INSERT INTO t1 values (9); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT * FROM t1; +c1 +1 +2 +3 +4 +5 +6 +7 +8 +9 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT * FROM t1; +c1 +2 +3 +5 +6 +8 +9 +DROP TABLE t1; +CREATE TABLE t1 ( +c1 INT, +c2 INT, +name VARCHAR(25) NOT NULL, +PRIMARY KEY (`c1`, `c2`) COMMENT 'custom_p0_cfname=foo;custom_p1_cfname=my_custom_cf;custom_p2_cfname=baz' +) ENGINE=ROCKSDB +COMMENT="custom_p0_ttl_duration=1;custom_p1_ttl_duration=7;" +PARTITION BY LIST(c1) ( +PARTITION custom_p0 VALUES IN (1, 4, 7), +PARTITION custom_p1 VALUES IN (2, 5, 8), +PARTITION custom_p2 VALUES IN (3, 6, 9) +); +set global rocksdb_debug_ttl_rec_ts = -1200; +INSERT INTO t1 values (1,1,'a'); +INSERT INTO t1 values (4,4,'aaaa'); +INSERT INTO t1 values (7,7,'aaaaaaa'); +set global rocksdb_debug_ttl_rec_ts = 1200; +INSERT INTO t1 values (2,2,'aa'); +INSERT INTO t1 values (3,3,'aaa'); +INSERT INTO t1 values (5,5,'aaaaa'); +INSERT INTO t1 values (6,6,'aaaaaa'); +INSERT INTO t1 values (8,8,'aaaaaaaa'); +INSERT INTO t1 values (9,9,'aaaaaaaaa'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT * FROM t1; +c1 c2 name +1 1 a +2 2 aa +3 3 aaa +4 4 aaaa +5 5 aaaaa +6 6 aaaaaa +7 7 aaaaaaa +8 8 aaaaaaaa +9 9 aaaaaaaaa +set global rocksdb_force_flush_memtable_now=1; +set @@global.rocksdb_compact_cf = 'foo'; +set @@global.rocksdb_compact_cf = 'my_custom_cf'; +SELECT * FROM t1; +c1 c2 name +2 2 aa +3 3 aaa +5 5 aaaaa +6 6 aaaaaa +8 8 aaaaaaaa +9 9 aaaaaaaaa +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set @@global.rocksdb_compact_cf = 'foo'; +SELECT * FROM t1; +c1 c2 name +2 2 aa +3 3 aaa +5 5 aaaaa +6 6 aaaaaa +8 8 aaaaaaaa +9 9 aaaaaaaaa +set @@global.rocksdb_compact_cf = 'my_custom_cf'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT * FROM t1; +c1 c2 name +3 3 aaa +6 6 aaaaaa +9 9 aaaaaaaaa +DROP TABLE t1; +CREATE TABLE t1 ( +c1 INT, +c2 INT, +name VARCHAR(25) NOT NULL, +event DATE, +PRIMARY KEY (`c1`, `c2`) COMMENT 'custom_p0_cfname=foo;custom_p1_cfname=bar;custom_p2_cfname=baz;' +) ENGINE=ROCKSDB +COMMENT="custom_p0_ttl_duration=9999;custom_p2_ttl_duration=5;" +PARTITION BY LIST(c1) ( +PARTITION custom_p0 VALUES IN (1, 2, 3), +PARTITION custom_p1 VALUES IN (4, 5, 6), +PARTITION custom_p2 VALUES IN (7, 8, 9) +); +INSERT INTO t1 VALUES (1, 1, "one", null); +INSERT INTO t1 VALUES (2, 2, "two", null); +INSERT INTO t1 VALUES (3, 3, "three", null); +INSERT INTO t1 VALUES (4, 4, "four", null); +INSERT INTO t1 VALUES (5, 5, "five", null); +INSERT INTO t1 VALUES (6, 6, "six", null); +INSERT INTO t1 VALUES (7, 7, "seven", null); +INSERT INTO t1 VALUES (8, 8, "eight", null); +INSERT INTO t1 VALUES (9, 9, "nine", null); +SELECT * FROM t1; +c1 c2 name event +1 1 one NULL +2 2 two NULL +3 3 three NULL +4 4 four NULL +5 5 five NULL +6 6 six NULL +7 7 seven NULL +8 8 eight NULL +9 9 nine NULL +set global rocksdb_debug_ttl_rec_ts = 600; +ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY(`c2`,`c1`) COMMENT 'custom_p0_cfname=foo;custom_p1_cfname=bar;custom_p2_cfname=baz;'; +set global rocksdb_debug_ttl_rec_ts = 0; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c1` int(11) NOT NULL, + `c2` int(11) NOT NULL, + `name` varchar(25) NOT NULL, + `event` date DEFAULT NULL, + PRIMARY KEY (`c2`,`c1`) COMMENT 'custom_p0_cfname=foo;custom_p1_cfname=bar;custom_p2_cfname=baz;' +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='custom_p0_ttl_duration=9999;custom_p2_ttl_duration=5;' + PARTITION BY LIST (`c1`) +(PARTITION `custom_p0` VALUES IN (1,2,3) ENGINE = ROCKSDB, + PARTITION `custom_p1` VALUES IN (4,5,6) ENGINE = ROCKSDB, + PARTITION `custom_p2` VALUES IN (7,8,9) ENGINE = ROCKSDB) +set global rocksdb_debug_ttl_snapshot_ts = 100; +set global rocksdb_force_flush_memtable_now=1; +set @@global.rocksdb_compact_cf = 'baz'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT * FROM t1; +c1 c2 name event +1 1 one NULL +2 2 two NULL +3 3 three NULL +4 4 four NULL +5 5 five NULL +6 6 six NULL +7 7 seven NULL +8 8 eight NULL +9 9 nine NULL +set global rocksdb_debug_ttl_snapshot_ts = 1200; +set @@global.rocksdb_compact_cf = 'foo'; +set @@global.rocksdb_compact_cf = 'baz'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT * FROM t1; +c1 c2 name event +1 1 one NULL +2 2 two NULL +3 3 three NULL +4 4 four NULL +5 5 five NULL +6 6 six NULL +DROP TABLE t1; +CREATE TABLE t1 ( +c1 BIGINT, +c2 TIMESTAMP(6) NOT NULL, +name VARCHAR(25) NOT NULL, +event DATE, +PRIMARY KEY (`c1`) COMMENT 'custom_p0_cfname=foo;custom_p1_cfname=bar;custom_p2_cfname=baz;' +) ENGINE=ROCKSDB +COMMENT="ttl_duration=1;custom_p1_ttl_duration=100;custom_p1_ttl_col=c2;custom_p2_ttl_duration=5000;" +PARTITION BY LIST(c1) ( +PARTITION custom_p0 VALUES IN (1, 2, 3), +PARTITION custom_p1 VALUES IN (4, 5, 6), +PARTITION custom_p2 VALUES IN (7, 8, 9) +); +set global rocksdb_debug_ttl_rec_ts = -300; +INSERT INTO t1 VALUES (1, NOW(), "one", null); +INSERT INTO t1 VALUES (2, NOW(), "two", null); +INSERT INTO t1 VALUES (3, NOW(), "three", null); +set global rocksdb_debug_ttl_rec_ts = 0; +INSERT INTO t1 VALUES (4, NOW(), "four", null); +INSERT INTO t1 VALUES (5, NOW(), "five", null); +INSERT INTO t1 VALUES (6, NOW(), "six", null); +INSERT INTO t1 VALUES (7, NOW(), "seven", null); +INSERT INTO t1 VALUES (8, NOW(), "eight", null); +INSERT INTO t1 VALUES (9, NOW(), "nine", null); +set global rocksdb_force_flush_memtable_now=1; +set @@global.rocksdb_compact_cf = 'foo'; +set @@global.rocksdb_compact_cf = 'baz'; +set @@global.rocksdb_compact_cf = 'bar'; +SELECT c1 FROM t1; +c1 +4 +5 +6 +7 +8 +9 +set global rocksdb_debug_ttl_snapshot_ts = 600; +set @@global.rocksdb_compact_cf = 'bar'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT c1 FROM t1; +c1 +7 +8 +9 +DROP TABLE t1; +CREATE TABLE t1 ( +c1 BIGINT, +c2 TIMESTAMP(6) NOT NULL, +PRIMARY KEY (`c1`, `c2`) +) ENGINE=ROCKSDB +COMMENT="ttl_duration=100;ttl_col=c2;" +PARTITION BY LIST(c1) ( +PARTITION custom_p0 VALUES IN (1), +PARTITION custom_p1 VALUES IN (2), +PARTITION custom_p2 VALUES IN (3) +); +INSERT INTO t1 values (1, NOW()); +INSERT INTO t1 values (2, NOW()); +INSERT INTO t1 values (3, NOW()); +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT c1 FROM t1; +c1 +1 +2 +3 +set global rocksdb_debug_ttl_snapshot_ts = 300; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT c1 FROM t1; +c1 +DROP TABLE t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/ttl_secondary.result b/storage/rocksdb/mysql-test/rocksdb/r/ttl_secondary.result index 1b47004d1f3..8dc199bacca 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/ttl_secondary.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/ttl_secondary.result @@ -290,12 +290,12 @@ CREATE TABLE t1 ( a bigint(20) UNSIGNED NOT NULL, b int NOT NULL, c int NOT NULL, -ts bigint(20), +ts bigint(20) UNSIGNED, PRIMARY KEY (a,c), KEY (b) ) ENGINE=rocksdb COMMENT='ttl_duration=1;ttl_col=ts;'; -ERROR HY000: TTL column (ts) in MyRocks must be an unsigned non-null 64-bit integer, exist inside the table, and have an accompanying ttl duration. +ERROR HY000: TTL column (ts) in MyRocks must be an unsigned non-null 64-bit integer or non-null timestamp, exist inside the table, and have an accompanying ttl duration. CREATE TABLE t1 ( a bigint(20) UNSIGNED NOT NULL, b int NOT NULL, @@ -305,7 +305,7 @@ PRIMARY KEY (a,c), KEY (b) ) ENGINE=rocksdb COMMENT='ttl_duration=1;ttl_col=ts;'; -ERROR HY000: TTL column (ts) in MyRocks must be an unsigned non-null 64-bit integer, exist inside the table, and have an accompanying ttl duration. +ERROR HY000: TTL column (ts) in MyRocks must be an unsigned non-null 64-bit integer or non-null timestamp, exist inside the table, and have an accompanying ttl duration. CREATE TABLE t1 ( a bigint(20) UNSIGNED NOT NULL, b int NOT NULL, @@ -323,7 +323,7 @@ PRIMARY KEY (a,c), KEY (b) ) ENGINE=rocksdb COMMENT='ttl_duration=1;ttl_col=abc;'; -ERROR HY000: TTL column (abc) in MyRocks must be an unsigned non-null 64-bit integer, exist inside the table, and have an accompanying ttl duration. +ERROR HY000: TTL column (abc) in MyRocks must be an unsigned non-null 64-bit integer or non-null timestamp, exist inside the table, and have an accompanying ttl duration. CREATE TABLE t1 ( a bigint(20) UNSIGNED NOT NULL, b int NOT NULL, @@ -332,7 +332,7 @@ PRIMARY KEY (a,c), KEY (b) ) ENGINE=rocksdb COMMENT='ttl_col=abc;'; -ERROR HY000: TTL column (abc) in MyRocks must be an unsigned non-null 64-bit integer, exist inside the table, and have an accompanying ttl duration. +ERROR HY000: TTL column (abc) in MyRocks must be an unsigned non-null 64-bit integer or non-null timestamp, exist inside the table, and have an accompanying ttl duration. CREATE TABLE t1 ( a bigint(20) NOT NULL, b int NOT NULL, @@ -707,3 +707,2130 @@ SELECT COUNT(*) FROM t1 FORCE INDEX (kb); COUNT(*) 0 DROP TABLE t1; +CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64) NOT NULL, +`c` varbinary(256) NOT NULL, +`ts` TIMESTAMP NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`b`,`a`,`c`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values ('a', 'b', 'c', NOW(), 'd'); +INSERT INTO t1 values ('d', 'e', 'f', NOW(), 'g'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +DROP TABLE t1; +CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64) NOT NULL, +`c` varbinary(256) NOT NULL, +`ts` TIMESTAMP NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`b`,`a`,`c`), +KEY kb (`b`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values ('a', 'b', 'c', NOW(), 'd'); +INSERT INTO t1 values ('d', 'e', 'f', NOW(), 'g'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +2 +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +ts TIMESTAMP NOT NULL, +c int NOT NULL, +PRIMARY KEY (a), +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, 3, NOW(), 5); +INSERT INTO t1 values (2, 4, NOW(), 6); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +2 +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +c int NOT NULL, +ts TIMESTAMP NOT NULL, +PRIMARY KEY (a,c), +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, 3, 5, NOW()); +INSERT INTO t1 values (2, 4, 6, NOW()); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +2 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int, +c int, +ts TIMESTAMP NOT NULL, +PRIMARY KEY (a), +KEY kbc (b, c) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, NULL, NULL, NOW()); +INSERT INTO t1 values (2, NULL, NULL, NOW()); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +2 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64), +`c` varbinary(256), +`ts` TIMESTAMP NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`a`), +KEY kbc (`b`, `c`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values ('a', NULL, 'bc', NOW(), 'd'); +INSERT INTO t1 values ('d', 'efghijk', NULL, NOW(), 'l'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +2 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +c int NOT NULL, +PRIMARY KEY (a), +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, 3, 5); +INSERT INTO t1 values (2, 4, 6); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +2 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a int, +ts TIMESTAMP NOT NULL, +PRIMARY KEY (a, ts), +KEY kt (ts) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;ttl_col=ts;'; +INSERT INTO t1 values (1, NOW()); +INSERT INTO t1 values (2, NOW()); +SELECT COUNT(*) FROM t1 FORCE INDEX(kt); +COUNT(*) +2 +set global rocksdb_debug_ttl_snapshot_ts = -10; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX(kt); +COUNT(*) +2 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_debug_ttl_snapshot_ts = 10; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT COUNT(*) FROM t1 FORCE INDEX(kt); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +ts TIMESTAMP NOT NULL, +c int NOT NULL, +PRIMARY KEY (a, ts), +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, 3, NOW(), 5); +INSERT INTO t1 values (2, 4, NOW(), 6); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +2 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64), +`c` varbinary(256), +`ts` TIMESTAMP NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`a`, `ts`), +KEY kb (`b`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values ('a', NULL, 'bc', NOW(), 'd'); +INSERT INTO t1 values ('de', 'fghijk', NULL, NOW(), 'l'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a INT NOT NULL, +b varbinary(64) NOT NULL, +c varbinary(256) NOT NULL, +ts TIMESTAMP NOT NULL, +value mediumblob NOT NULL, +PRIMARY KEY (b,a,c), +KEY kb (b) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=10;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -300; +INSERT INTO t1 values (1, 'b', 'c', NOW(), 'd'); +INSERT INTO t1 values (2, 'e', 'f', NOW(), 'g'); +set global rocksdb_debug_ttl_rec_ts = 300; +INSERT INTO t1 values (3, 'i', 'j', NOW(), 'k'); +INSERT INTO t1 values (4, 'm', 'n', NOW(), 'o'); +set global rocksdb_debug_ttl_rec_ts = 0; +set global rocksdb_debug_ttl_snapshot_ts = -3600; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT a FROM t1 FORCE INDEX (kb); +a +1 +2 +3 +4 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT a FROM t1 FORCE INDEX (kb); +a +3 +4 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT a FROM t1 FORCE INDEX (kb); +a +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) UNSIGNED NOT NULL, +b int NOT NULL, +c int NOT NULL, +ts TIMESTAMP, +PRIMARY KEY (a,c), +KEY (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +ERROR HY000: TTL column (ts) in MyRocks must be an unsigned non-null 64-bit integer or non-null timestamp, exist inside the table, and have an accompanying ttl duration. +CREATE TABLE t1 ( +a bigint(20) UNSIGNED NOT NULL, +b int NOT NULL, +c int NOT NULL, +ts int, +PRIMARY KEY (a,c), +KEY (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +ERROR HY000: TTL column (ts) in MyRocks must be an unsigned non-null 64-bit integer or non-null timestamp, exist inside the table, and have an accompanying ttl duration. +CREATE TABLE t1 ( +a bigint(20) UNSIGNED NOT NULL, +b int NOT NULL, +c int NOT NULL, +PRIMARY KEY (a,c), +KEY (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=abc;'; +ERROR HY000: TTL duration (abc) in MyRocks must be an unsigned non-null 64-bit integer. +CREATE TABLE t1 ( +a bigint(20) UNSIGNED NOT NULL, +b int NOT NULL, +c int NOT NULL, +PRIMARY KEY (a,c), +KEY (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=abc;'; +ERROR HY000: TTL column (abc) in MyRocks must be an unsigned non-null 64-bit integer or non-null timestamp, exist inside the table, and have an accompanying ttl duration. +CREATE TABLE t1 ( +a bigint(20) UNSIGNED NOT NULL, +b int NOT NULL, +c int NOT NULL, +PRIMARY KEY (a,c), +KEY (b) +) ENGINE=rocksdb +COMMENT='ttl_col=abc;'; +ERROR HY000: TTL column (abc) in MyRocks must be an unsigned non-null 64-bit integer or non-null timestamp, exist inside the table, and have an accompanying ttl duration. +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +PRIMARY KEY (a), +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=500;'; +INSERT INTO t1 values (1, 1); +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +1 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +1 +DROP TABLE t1; +CREATE TABLE t1 ( +a INT PRIMARY KEY, +b INT NOT NULL, +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=100;'; +INSERT INTO t1 values (1, 1); +SELECT * FROM t1 FORCE INDEX (kb); +a b +1 1 +set global rocksdb_debug_ttl_rec_ts = -300; +ALTER TABLE t1 COMMENT = 'ttl_duration=1'; +set global rocksdb_debug_ttl_rec_ts = 0; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) NOT NULL, + `b` int(11) NOT NULL, + PRIMARY KEY (`a`), + KEY `kb` (`b`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='ttl_duration=1' +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a INT PRIMARY KEY, +b INT, +KEY (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=100;'; +ALTER TABLE t1 DROP PRIMARY KEY; +ERROR HY000: TTL support is currently disabled when table has a hidden PK. +DROP TABLE t1; +CREATE TABLE t1 ( +a INT PRIMARY KEY, +b INT, +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;'; +INSERT INTO t1 VALUES (1,1); +INSERT INTO t1 VALUES (2,2); +ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY(b); +set global rocksdb_debug_ttl_snapshot_ts = -3600; +set global rocksdb_force_flush_memtable_now=1; +set @@global.rocksdb_compact_cf = 'default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +2 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set @@global.rocksdb_compact_cf = 'default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a TIMESTAMP NOT NULL, +b int, +PRIMARY KEY (a,b), +KEY kb (b) +) ENGINE=rocksdb +COMMENT='asdadfasdfsadfadf ;ttl_duration=1; asfasdfasdfadfa'; +INSERT INTO t1 values (NOW(), 1); +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +1 +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +ALTER TABLE t1 COMMENT = 'adsf;;ttl_duration=5;asfasdfa;ttl_col=a;asdfasdf;'; +set global rocksdb_debug_ttl_rec_ts = 300; +INSERT INTO t1 values (NOW(), 2); +set global rocksdb_debug_ttl_rec_ts = 0; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +1 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +PRIMARY KEY (a), +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;'; +set global rocksdb_debug_ttl_rec_ts = -300; +INSERT INTO t1 values (1, 0); +INSERT INTO t1 values (3, 0); +INSERT INTO t1 values (5, 0); +set global rocksdb_debug_ttl_rec_ts = 300; +INSERT INTO t1 values (7, 0); +INSERT INTO t1 values (9, 0); +set global rocksdb_debug_ttl_rec_ts = 0; +UPDATE t1 SET a=a+1; +SELECT * FROM t1 FORCE INDEX (kb); +a b +10 0 +2 0 +4 0 +6 0 +8 0 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT * FROM t1; +a b +10 0 +8 0 +DROP TABLE t1; +CREATE TABLE t1 ( +a INT, +b TIMESTAMP NOT NULL, +PRIMARY KEY (a), +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;ttl_col=b;'; +set global rocksdb_debug_ttl_rec_ts = -300; +INSERT INTO t1 values (1, NOW()); +INSERT INTO t1 values (3, NOW()); +INSERT INTO t1 values (5, NOW()); +INSERT INTO t1 values (7, NOW()); +set global rocksdb_debug_ttl_rec_ts = 300; +UPDATE t1 SET b=(ADDTIME(NOW(), '00:00:01')) WHERE a < 4; +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT a FROM t1 FORCE INDEX (kb); +a +1 +3 +5 +7 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT a FROM t1 FORCE INDEX (kb); +a +1 +3 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +PRIMARY KEY (a), +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, 1); +INSERT INTO t1 values (2, 1); +INSERT INTO t1 values (3, 1); +set global rocksdb_debug_ttl_rec_ts = 0; +set global rocksdb_enable_ttl=0; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +select variable_value into @c from information_schema.global_status where variable_name='rocksdb_rows_expired'; +set global rocksdb_enable_ttl=1; +set global rocksdb_compact_cf='default'; +select variable_value-@c from information_schema.global_status where variable_name='rocksdb_rows_expired'; +variable_value-@c +6 +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +PRIMARY KEY (a), +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=100;'; +INSERT INTO t1 values (1, 1); +INSERT INTO t1 values (2, 2); +INSERT INTO t1 values (3, 3); +select variable_value into @c from information_schema.global_status where variable_name='rocksdb_rows_expired'; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +select variable_value-@c from information_schema.global_status where variable_name='rocksdb_rows_expired'; +variable_value-@c +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a INT, +b TIMESTAMP NOT NULL, +PRIMARY KEY (a, b), +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;ttl_col=b;'; +set global rocksdb_debug_ttl_rec_ts = -300; +INSERT INTO t1 values (1, NOW()); +INSERT INTO t1 values (3, NOW()); +INSERT INTO t1 values (5, NOW()); +INSERT INTO t1 values (7, NOW()); +set global rocksdb_debug_ttl_rec_ts = 300; +UPDATE t1 SET b=(ADDTIME(NOW(), '00:00:01')) WHERE a < 4; +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT a FROM t1 FORCE INDEX (kb); +a +1 +3 +5 +7 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT a FROM t1 FORCE INDEX (kb); +a +1 +3 +DROP TABLE t1; +CREATE TABLE t1 ( +a INT, +b TIMESTAMP NOT NULL, +PRIMARY KEY (a, b) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;ttl_col=b;'; +set global rocksdb_debug_ttl_rec_ts = -300; +INSERT INTO t1 values (1, NOW()); +INSERT INTO t1 values (3, NOW()); +INSERT INTO t1 values (5, NOW()); +INSERT INTO t1 values (7, NOW()); +set global rocksdb_debug_ttl_rec_ts = 300; +UPDATE t1 SET b=(ADDTIME(NOW(), '00:00:01')) WHERE a < 4; +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT a FROM t1; +a +1 +3 +5 +7 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT a FROM t1; +a +1 +3 +DROP TABLE t1; +CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64) NOT NULL, +`c` varbinary(256) NOT NULL, +`ts` TIMESTAMP NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`b`,`a`,`c`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values ('a', 'b', 'c', NOW(), 'd'); +INSERT INTO t1 values ('d', 'e', 'f', NOW(), 'g'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*); +COUNT(*) +1 +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; +SELECT COUNT(*); +COUNT(*) +1 +CREATE INDEX kb on t1 (b); +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64) NOT NULL, +`c` varbinary(256) NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`b`,`a`,`c`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values ('a', 'b', 'c', 'd'); +INSERT INTO t1 values ('d', 'e', 'f', 'g'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*); +COUNT(*) +1 +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; +SELECT COUNT(*); +COUNT(*) +1 +CREATE INDEX kb on t1 (b); +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64) NOT NULL, +`c` varbinary(256) NOT NULL, +`ts` TIMESTAMP NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`b`,`a`,`c`, `ts`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values ('a', 'b', 'c', NOW(), 'd'); +INSERT INTO t1 values ('d', 'e', 'f', NOW(), 'g'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*); +COUNT(*) +1 +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; +SELECT COUNT(*); +COUNT(*) +1 +CREATE INDEX kb on t1 (b); +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64) NOT NULL, +`c` varbinary(256) NOT NULL, +`ts` TIMESTAMP DEFAULT NOW() NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`b`,`a`,`c`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values ('a', 'b', 'c', NOW(), 'd'); +INSERT INTO t1 values ('d', 'e', 'f', NOW(), 'g'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +DROP TABLE t1; +CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64) NOT NULL, +`c` varbinary(256) NOT NULL, +`ts` TIMESTAMP DEFAULT NOW() NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`b`,`a`,`c`), +KEY kb (`b`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values ('a', 'b', 'c', NOW(), 'd'); +INSERT INTO t1 values ('d', 'e', 'f', NOW(), 'g'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +2 +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +ts TIMESTAMP DEFAULT NOW() NOT NULL, +c int NOT NULL, +PRIMARY KEY (a), +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, 3, NOW(), 5); +INSERT INTO t1 values (2, 4, NOW(), 6); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +2 +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +c int NOT NULL, +ts TIMESTAMP DEFAULT NOW() NOT NULL, +PRIMARY KEY (a,c), +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, 3, 5, NOW()); +INSERT INTO t1 values (2, 4, 6, NOW()); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +2 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int, +c int, +ts TIMESTAMP DEFAULT NOW() NOT NULL, +PRIMARY KEY (a), +KEY kbc (b, c) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, NULL, NULL, NOW()); +INSERT INTO t1 values (2, NULL, NULL, NOW()); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +2 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64), +`c` varbinary(256), +`ts` TIMESTAMP DEFAULT NOW() NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`a`), +KEY kbc (`b`, `c`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values ('a', NULL, 'bc', NOW(), 'd'); +INSERT INTO t1 values ('d', 'efghijk', NULL, NOW(), 'l'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +2 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +c int NOT NULL, +PRIMARY KEY (a), +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, 3, 5); +INSERT INTO t1 values (2, 4, 6); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +2 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a int, +ts TIMESTAMP DEFAULT NOW() NOT NULL, +PRIMARY KEY (a, ts), +KEY kt (ts) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;ttl_col=ts;'; +INSERT INTO t1 values (1, NOW()); +INSERT INTO t1 values (2, NOW()); +SELECT COUNT(*) FROM t1 FORCE INDEX(kt); +COUNT(*) +2 +set global rocksdb_debug_ttl_snapshot_ts = -10; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX(kt); +COUNT(*) +2 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_debug_ttl_snapshot_ts = 10; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT COUNT(*) FROM t1 FORCE INDEX(kt); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +ts TIMESTAMP DEFAULT NOW() NOT NULL, +c int NOT NULL, +PRIMARY KEY (a, ts), +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, 3, NOW(), 5); +INSERT INTO t1 values (2, 4, NOW(), 6); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +2 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64), +`c` varbinary(256), +`ts` TIMESTAMP DEFAULT NOW() NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`a`, `ts`), +KEY kb (`b`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values ('a', NULL, 'bc', NOW(), 'd'); +INSERT INTO t1 values ('de', 'fghijk', NULL, NOW(), 'l'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a INT NOT NULL, +b varbinary(64) NOT NULL, +c varbinary(256) NOT NULL, +ts TIMESTAMP DEFAULT NOW() NOT NULL, +value mediumblob NOT NULL, +PRIMARY KEY (b,a,c), +KEY kb (b) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=10;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -300; +INSERT INTO t1 values (1, 'b', 'c', NOW(), 'd'); +INSERT INTO t1 values (2, 'e', 'f', NOW(), 'g'); +set global rocksdb_debug_ttl_rec_ts = 300; +INSERT INTO t1 values (3, 'i', 'j', NOW(), 'k'); +INSERT INTO t1 values (4, 'm', 'n', NOW(), 'o'); +set global rocksdb_debug_ttl_rec_ts = 0; +set global rocksdb_debug_ttl_snapshot_ts = -3600; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT a FROM t1 FORCE INDEX (kb); +a +1 +2 +3 +4 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT a FROM t1 FORCE INDEX (kb); +a +3 +4 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT a FROM t1 FORCE INDEX (kb); +a +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) UNSIGNED NOT NULL, +b int NOT NULL, +c int NOT NULL, +ts TIMESTAMP DEFAULT NOW(), +PRIMARY KEY (a,c), +KEY (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +ERROR HY000: TTL column (ts) in MyRocks must be an unsigned non-null 64-bit integer or non-null timestamp, exist inside the table, and have an accompanying ttl duration. +CREATE TABLE t1 ( +a bigint(20) UNSIGNED NOT NULL, +b int NOT NULL, +c int NOT NULL, +ts int, +PRIMARY KEY (a,c), +KEY (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +ERROR HY000: TTL column (ts) in MyRocks must be an unsigned non-null 64-bit integer or non-null timestamp, exist inside the table, and have an accompanying ttl duration. +CREATE TABLE t1 ( +a bigint(20) UNSIGNED NOT NULL, +b int NOT NULL, +c int NOT NULL, +PRIMARY KEY (a,c), +KEY (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=abc;'; +ERROR HY000: TTL duration (abc) in MyRocks must be an unsigned non-null 64-bit integer. +CREATE TABLE t1 ( +a bigint(20) UNSIGNED NOT NULL, +b int NOT NULL, +c int NOT NULL, +PRIMARY KEY (a,c), +KEY (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=abc;'; +ERROR HY000: TTL column (abc) in MyRocks must be an unsigned non-null 64-bit integer or non-null timestamp, exist inside the table, and have an accompanying ttl duration. +CREATE TABLE t1 ( +a bigint(20) UNSIGNED NOT NULL, +b int NOT NULL, +c int NOT NULL, +PRIMARY KEY (a,c), +KEY (b) +) ENGINE=rocksdb +COMMENT='ttl_col=abc;'; +ERROR HY000: TTL column (abc) in MyRocks must be an unsigned non-null 64-bit integer or non-null timestamp, exist inside the table, and have an accompanying ttl duration. +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +PRIMARY KEY (a), +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=500;'; +INSERT INTO t1 values (1, 1); +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +1 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +1 +DROP TABLE t1; +CREATE TABLE t1 ( +a INT PRIMARY KEY, +b INT NOT NULL, +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=100;'; +INSERT INTO t1 values (1, 1); +SELECT * FROM t1 FORCE INDEX (kb); +a b +1 1 +set global rocksdb_debug_ttl_rec_ts = -300; +ALTER TABLE t1 COMMENT = 'ttl_duration=1'; +set global rocksdb_debug_ttl_rec_ts = 0; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) NOT NULL, + `b` int(11) NOT NULL, + PRIMARY KEY (`a`), + KEY `kb` (`b`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='ttl_duration=1' +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a INT PRIMARY KEY, +b INT, +KEY (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=100;'; +ALTER TABLE t1 DROP PRIMARY KEY; +ERROR HY000: TTL support is currently disabled when table has a hidden PK. +DROP TABLE t1; +CREATE TABLE t1 ( +a INT PRIMARY KEY, +b INT, +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;'; +INSERT INTO t1 VALUES (1,1); +INSERT INTO t1 VALUES (2,2); +ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY(b); +set global rocksdb_debug_ttl_snapshot_ts = -3600; +set global rocksdb_force_flush_memtable_now=1; +set @@global.rocksdb_compact_cf = 'default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +2 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set @@global.rocksdb_compact_cf = 'default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a TIMESTAMP DEFAULT NOW() NOT NULL, +b int, +PRIMARY KEY (a,b), +KEY kb (b) +) ENGINE=rocksdb +COMMENT='asdadfasdfsadfadf ;ttl_duration=1; asfasdfasdfadfa'; +INSERT INTO t1 values (NOW(), 1); +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +1 +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +ALTER TABLE t1 COMMENT = 'adsf;;ttl_duration=5;asfasdfa;ttl_col=a;asdfasdf;'; +set global rocksdb_debug_ttl_rec_ts = 300; +INSERT INTO t1 values (NOW(), 2); +set global rocksdb_debug_ttl_rec_ts = 0; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +1 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +PRIMARY KEY (a), +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;'; +set global rocksdb_debug_ttl_rec_ts = -300; +INSERT INTO t1 values (1, 0); +INSERT INTO t1 values (3, 0); +INSERT INTO t1 values (5, 0); +set global rocksdb_debug_ttl_rec_ts = 300; +INSERT INTO t1 values (7, 0); +INSERT INTO t1 values (9, 0); +set global rocksdb_debug_ttl_rec_ts = 0; +UPDATE t1 SET a=a+1; +SELECT * FROM t1 FORCE INDEX (kb); +a b +10 0 +2 0 +4 0 +6 0 +8 0 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT * FROM t1; +a b +10 0 +8 0 +DROP TABLE t1; +CREATE TABLE t1 ( +a INT, +b TIMESTAMP DEFAULT NOW() NOT NULL, +PRIMARY KEY (a), +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;ttl_col=b;'; +set global rocksdb_debug_ttl_rec_ts = -300; +INSERT INTO t1 values (1, NOW()); +INSERT INTO t1 values (3, NOW()); +INSERT INTO t1 values (5, NOW()); +INSERT INTO t1 values (7, NOW()); +set global rocksdb_debug_ttl_rec_ts = 300; +UPDATE t1 SET b=(ADDTIME(NOW(), '00:00:01')) WHERE a < 4; +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT a FROM t1 FORCE INDEX (kb); +a +1 +3 +5 +7 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT a FROM t1 FORCE INDEX (kb); +a +1 +3 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +PRIMARY KEY (a), +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, 1); +INSERT INTO t1 values (2, 1); +INSERT INTO t1 values (3, 1); +set global rocksdb_debug_ttl_rec_ts = 0; +set global rocksdb_enable_ttl=0; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +select variable_value into @c from information_schema.global_status where variable_name='rocksdb_rows_expired'; +set global rocksdb_enable_ttl=1; +set global rocksdb_compact_cf='default'; +select variable_value-@c from information_schema.global_status where variable_name='rocksdb_rows_expired'; +variable_value-@c +6 +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +PRIMARY KEY (a), +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=100;'; +INSERT INTO t1 values (1, 1); +INSERT INTO t1 values (2, 2); +INSERT INTO t1 values (3, 3); +select variable_value into @c from information_schema.global_status where variable_name='rocksdb_rows_expired'; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +select variable_value-@c from information_schema.global_status where variable_name='rocksdb_rows_expired'; +variable_value-@c +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a INT, +b TIMESTAMP DEFAULT NOW() NOT NULL, +PRIMARY KEY (a, b), +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;ttl_col=b;'; +set global rocksdb_debug_ttl_rec_ts = -300; +INSERT INTO t1 values (1, NOW()); +INSERT INTO t1 values (3, NOW()); +INSERT INTO t1 values (5, NOW()); +INSERT INTO t1 values (7, NOW()); +set global rocksdb_debug_ttl_rec_ts = 300; +UPDATE t1 SET b=(ADDTIME(NOW(), '00:00:01')) WHERE a < 4; +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT a FROM t1 FORCE INDEX (kb); +a +1 +3 +5 +7 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT a FROM t1 FORCE INDEX (kb); +a +1 +3 +DROP TABLE t1; +CREATE TABLE t1 ( +a INT, +b TIMESTAMP DEFAULT NOW() NOT NULL, +PRIMARY KEY (a, b) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;ttl_col=b;'; +set global rocksdb_debug_ttl_rec_ts = -300; +INSERT INTO t1 values (1, NOW()); +INSERT INTO t1 values (3, NOW()); +INSERT INTO t1 values (5, NOW()); +INSERT INTO t1 values (7, NOW()); +set global rocksdb_debug_ttl_rec_ts = 300; +UPDATE t1 SET b=(ADDTIME(NOW(), '00:00:01')) WHERE a < 4; +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT a FROM t1; +a +1 +3 +5 +7 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT a FROM t1; +a +1 +3 +DROP TABLE t1; +CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64) NOT NULL, +`c` varbinary(256) NOT NULL, +`ts` TIMESTAMP DEFAULT NOW() NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`b`,`a`,`c`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values ('a', 'b', 'c', NOW(), 'd'); +INSERT INTO t1 values ('d', 'e', 'f', NOW(), 'g'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*); +COUNT(*) +1 +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; +SELECT COUNT(*); +COUNT(*) +1 +CREATE INDEX kb on t1 (b); +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64) NOT NULL, +`c` varbinary(256) NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`b`,`a`,`c`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values ('a', 'b', 'c', 'd'); +INSERT INTO t1 values ('d', 'e', 'f', 'g'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*); +COUNT(*) +1 +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; +SELECT COUNT(*); +COUNT(*) +1 +CREATE INDEX kb on t1 (b); +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64) NOT NULL, +`c` varbinary(256) NOT NULL, +`ts` TIMESTAMP DEFAULT NOW() NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`b`,`a`,`c`, `ts`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values ('a', 'b', 'c', NOW(), 'd'); +INSERT INTO t1 values ('d', 'e', 'f', NOW(), 'g'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*); +COUNT(*) +1 +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; +SELECT COUNT(*); +COUNT(*) +1 +CREATE INDEX kb on t1 (b); +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64) NOT NULL, +`c` varbinary(256) NOT NULL, +`ts` TIMESTAMP(6) NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`b`,`a`,`c`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values ('a', 'b', 'c', NOW(), 'd'); +INSERT INTO t1 values ('d', 'e', 'f', NOW(), 'g'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +DROP TABLE t1; +CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64) NOT NULL, +`c` varbinary(256) NOT NULL, +`ts` TIMESTAMP(6) NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`b`,`a`,`c`), +KEY kb (`b`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values ('a', 'b', 'c', NOW(), 'd'); +INSERT INTO t1 values ('d', 'e', 'f', NOW(), 'g'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +2 +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +ts TIMESTAMP(6) NOT NULL, +c int NOT NULL, +PRIMARY KEY (a), +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, 3, NOW(), 5); +INSERT INTO t1 values (2, 4, NOW(), 6); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +2 +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +c int NOT NULL, +ts TIMESTAMP(6) NOT NULL, +PRIMARY KEY (a,c), +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, 3, 5, NOW()); +INSERT INTO t1 values (2, 4, 6, NOW()); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +2 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int, +c int, +ts TIMESTAMP(6) NOT NULL, +PRIMARY KEY (a), +KEY kbc (b, c) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, NULL, NULL, NOW()); +INSERT INTO t1 values (2, NULL, NULL, NOW()); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +2 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64), +`c` varbinary(256), +`ts` TIMESTAMP(6) NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`a`), +KEY kbc (`b`, `c`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values ('a', NULL, 'bc', NOW(), 'd'); +INSERT INTO t1 values ('d', 'efghijk', NULL, NOW(), 'l'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +2 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +c int NOT NULL, +PRIMARY KEY (a), +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, 3, 5); +INSERT INTO t1 values (2, 4, 6); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +2 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a int, +ts TIMESTAMP(6) NOT NULL, +PRIMARY KEY (a, ts), +KEY kt (ts) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;ttl_col=ts;'; +INSERT INTO t1 values (1, NOW()); +INSERT INTO t1 values (2, NOW()); +SELECT COUNT(*) FROM t1 FORCE INDEX(kt); +COUNT(*) +2 +set global rocksdb_debug_ttl_snapshot_ts = -10; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX(kt); +COUNT(*) +2 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_debug_ttl_snapshot_ts = 10; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT COUNT(*) FROM t1 FORCE INDEX(kt); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +ts TIMESTAMP(6) NOT NULL, +c int NOT NULL, +PRIMARY KEY (a, ts), +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, 3, NOW(), 5); +INSERT INTO t1 values (2, 4, NOW(), 6); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +2 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64), +`c` varbinary(256), +`ts` TIMESTAMP(6) NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`a`, `ts`), +KEY kb (`b`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values ('a', NULL, 'bc', NOW(), 'd'); +INSERT INTO t1 values ('de', 'fghijk', NULL, NOW(), 'l'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a INT NOT NULL, +b varbinary(64) NOT NULL, +c varbinary(256) NOT NULL, +ts TIMESTAMP(6) NOT NULL, +value mediumblob NOT NULL, +PRIMARY KEY (b,a,c), +KEY kb (b) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=10;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -300; +INSERT INTO t1 values (1, 'b', 'c', NOW(), 'd'); +INSERT INTO t1 values (2, 'e', 'f', NOW(), 'g'); +set global rocksdb_debug_ttl_rec_ts = 300; +INSERT INTO t1 values (3, 'i', 'j', NOW(), 'k'); +INSERT INTO t1 values (4, 'm', 'n', NOW(), 'o'); +set global rocksdb_debug_ttl_rec_ts = 0; +set global rocksdb_debug_ttl_snapshot_ts = -3600; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT a FROM t1 FORCE INDEX (kb); +a +1 +2 +3 +4 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT a FROM t1 FORCE INDEX (kb); +a +3 +4 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT a FROM t1 FORCE INDEX (kb); +a +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) UNSIGNED NOT NULL, +b int NOT NULL, +c int NOT NULL, +ts TIMESTAMP(6), +PRIMARY KEY (a,c), +KEY (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +ERROR HY000: TTL column (ts) in MyRocks must be an unsigned non-null 64-bit integer or non-null timestamp, exist inside the table, and have an accompanying ttl duration. +CREATE TABLE t1 ( +a bigint(20) UNSIGNED NOT NULL, +b int NOT NULL, +c int NOT NULL, +ts int, +PRIMARY KEY (a,c), +KEY (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +ERROR HY000: TTL column (ts) in MyRocks must be an unsigned non-null 64-bit integer or non-null timestamp, exist inside the table, and have an accompanying ttl duration. +CREATE TABLE t1 ( +a bigint(20) UNSIGNED NOT NULL, +b int NOT NULL, +c int NOT NULL, +PRIMARY KEY (a,c), +KEY (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=abc;'; +ERROR HY000: TTL duration (abc) in MyRocks must be an unsigned non-null 64-bit integer. +CREATE TABLE t1 ( +a bigint(20) UNSIGNED NOT NULL, +b int NOT NULL, +c int NOT NULL, +PRIMARY KEY (a,c), +KEY (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=abc;'; +ERROR HY000: TTL column (abc) in MyRocks must be an unsigned non-null 64-bit integer or non-null timestamp, exist inside the table, and have an accompanying ttl duration. +CREATE TABLE t1 ( +a bigint(20) UNSIGNED NOT NULL, +b int NOT NULL, +c int NOT NULL, +PRIMARY KEY (a,c), +KEY (b) +) ENGINE=rocksdb +COMMENT='ttl_col=abc;'; +ERROR HY000: TTL column (abc) in MyRocks must be an unsigned non-null 64-bit integer or non-null timestamp, exist inside the table, and have an accompanying ttl duration. +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +PRIMARY KEY (a), +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=500;'; +INSERT INTO t1 values (1, 1); +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +1 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +1 +DROP TABLE t1; +CREATE TABLE t1 ( +a INT PRIMARY KEY, +b INT NOT NULL, +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=100;'; +INSERT INTO t1 values (1, 1); +SELECT * FROM t1 FORCE INDEX (kb); +a b +1 1 +set global rocksdb_debug_ttl_rec_ts = -300; +ALTER TABLE t1 COMMENT = 'ttl_duration=1'; +set global rocksdb_debug_ttl_rec_ts = 0; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) NOT NULL, + `b` int(11) NOT NULL, + PRIMARY KEY (`a`), + KEY `kb` (`b`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='ttl_duration=1' +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a INT PRIMARY KEY, +b INT, +KEY (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=100;'; +ALTER TABLE t1 DROP PRIMARY KEY; +ERROR HY000: TTL support is currently disabled when table has a hidden PK. +DROP TABLE t1; +CREATE TABLE t1 ( +a INT PRIMARY KEY, +b INT, +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;'; +INSERT INTO t1 VALUES (1,1); +INSERT INTO t1 VALUES (2,2); +ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY(b); +set global rocksdb_debug_ttl_snapshot_ts = -3600; +set global rocksdb_force_flush_memtable_now=1; +set @@global.rocksdb_compact_cf = 'default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +2 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set @@global.rocksdb_compact_cf = 'default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a TIMESTAMP(6) NOT NULL, +b int, +PRIMARY KEY (a,b), +KEY kb (b) +) ENGINE=rocksdb +COMMENT='asdadfasdfsadfadf ;ttl_duration=1; asfasdfasdfadfa'; +INSERT INTO t1 values (NOW(), 1); +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +1 +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +ALTER TABLE t1 COMMENT = 'adsf;;ttl_duration=5;asfasdfa;ttl_col=a;asdfasdf;'; +set global rocksdb_debug_ttl_rec_ts = 300; +INSERT INTO t1 values (NOW(), 2); +set global rocksdb_debug_ttl_rec_ts = 0; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +1 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +PRIMARY KEY (a), +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;'; +set global rocksdb_debug_ttl_rec_ts = -300; +INSERT INTO t1 values (1, 0); +INSERT INTO t1 values (3, 0); +INSERT INTO t1 values (5, 0); +set global rocksdb_debug_ttl_rec_ts = 300; +INSERT INTO t1 values (7, 0); +INSERT INTO t1 values (9, 0); +set global rocksdb_debug_ttl_rec_ts = 0; +UPDATE t1 SET a=a+1; +SELECT * FROM t1 FORCE INDEX (kb); +a b +10 0 +2 0 +4 0 +6 0 +8 0 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT * FROM t1; +a b +10 0 +8 0 +DROP TABLE t1; +CREATE TABLE t1 ( +a INT, +b TIMESTAMP(6) NOT NULL, +PRIMARY KEY (a), +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;ttl_col=b;'; +set global rocksdb_debug_ttl_rec_ts = -300; +INSERT INTO t1 values (1, NOW()); +INSERT INTO t1 values (3, NOW()); +INSERT INTO t1 values (5, NOW()); +INSERT INTO t1 values (7, NOW()); +set global rocksdb_debug_ttl_rec_ts = 300; +UPDATE t1 SET b=(ADDTIME(NOW(), '00:00:01')) WHERE a < 4; +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT a FROM t1 FORCE INDEX (kb); +a +1 +3 +5 +7 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT a FROM t1 FORCE INDEX (kb); +a +1 +3 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +PRIMARY KEY (a), +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, 1); +INSERT INTO t1 values (2, 1); +INSERT INTO t1 values (3, 1); +set global rocksdb_debug_ttl_rec_ts = 0; +set global rocksdb_enable_ttl=0; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +select variable_value into @c from information_schema.global_status where variable_name='rocksdb_rows_expired'; +set global rocksdb_enable_ttl=1; +set global rocksdb_compact_cf='default'; +select variable_value-@c from information_schema.global_status where variable_name='rocksdb_rows_expired'; +variable_value-@c +6 +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a bigint(20) NOT NULL, +b int NOT NULL, +PRIMARY KEY (a), +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=100;'; +INSERT INTO t1 values (1, 1); +INSERT INTO t1 values (2, 2); +INSERT INTO t1 values (3, 3); +select variable_value into @c from information_schema.global_status where variable_name='rocksdb_rows_expired'; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +select variable_value-@c from information_schema.global_status where variable_name='rocksdb_rows_expired'; +variable_value-@c +0 +DROP TABLE t1; +CREATE TABLE t1 ( +a INT, +b TIMESTAMP(6) NOT NULL, +PRIMARY KEY (a, b), +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;ttl_col=b;'; +set global rocksdb_debug_ttl_rec_ts = -300; +INSERT INTO t1 values (1, NOW()); +INSERT INTO t1 values (3, NOW()); +INSERT INTO t1 values (5, NOW()); +INSERT INTO t1 values (7, NOW()); +set global rocksdb_debug_ttl_rec_ts = 300; +UPDATE t1 SET b=(ADDTIME(NOW(), '00:00:01')) WHERE a < 4; +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT a FROM t1 FORCE INDEX (kb); +a +1 +3 +5 +7 +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT a FROM t1 FORCE INDEX (kb); +a +1 +3 +DROP TABLE t1; +CREATE TABLE t1 ( +a INT, +b TIMESTAMP(6) NOT NULL, +PRIMARY KEY (a, b) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;ttl_col=b;'; +set global rocksdb_debug_ttl_rec_ts = -300; +INSERT INTO t1 values (1, NOW()); +INSERT INTO t1 values (3, NOW()); +INSERT INTO t1 values (5, NOW()); +INSERT INTO t1 values (7, NOW()); +set global rocksdb_debug_ttl_rec_ts = 300; +UPDATE t1 SET b=(ADDTIME(NOW(), '00:00:01')) WHERE a < 4; +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT a FROM t1; +a +1 +3 +5 +7 +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +SELECT a FROM t1; +a +1 +3 +DROP TABLE t1; +CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64) NOT NULL, +`c` varbinary(256) NOT NULL, +`ts` TIMESTAMP(6) NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`b`,`a`,`c`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values ('a', 'b', 'c', NOW(), 'd'); +INSERT INTO t1 values ('d', 'e', 'f', NOW(), 'g'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*); +COUNT(*) +1 +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; +SELECT COUNT(*); +COUNT(*) +1 +CREATE INDEX kb on t1 (b); +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64) NOT NULL, +`c` varbinary(256) NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`b`,`a`,`c`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values ('a', 'b', 'c', 'd'); +INSERT INTO t1 values ('d', 'e', 'f', 'g'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*); +COUNT(*) +1 +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; +SELECT COUNT(*); +COUNT(*) +1 +CREATE INDEX kb on t1 (b); +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; +CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64) NOT NULL, +`c` varbinary(256) NOT NULL, +`ts` TIMESTAMP(6) NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`b`,`a`,`c`, `ts`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values ('a', 'b', 'c', NOW(), 'd'); +INSERT INTO t1 values ('d', 'e', 'f', NOW(), 'g'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*); +COUNT(*) +1 +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; +SELECT COUNT(*); +COUNT(*) +1 +CREATE INDEX kb on t1 (b); +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +COUNT(*) +0 +DROP TABLE t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/ttl_secondary_read_filtering.result b/storage/rocksdb/mysql-test/rocksdb/r/ttl_secondary_read_filtering.result index 395c84edfe9..e093647014a 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/ttl_secondary_read_filtering.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/ttl_secondary_read_filtering.result @@ -368,6 +368,57 @@ a b 10 10 DROP TABLE t1; CREATE TABLE t1 ( +a int, +b int, +ts TIMESTAMP NOT NULL, +PRIMARY KEY (a), +KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; +set global rocksdb_debug_ttl_rec_ts = 100; +INSERT INTO t1 VALUES (1, 1, NOW()); +INSERT INTO t1 VALUES (2, 2, NOW()); +INSERT INTO t1 VALUES (3, 3, NOW()); +INSERT INTO t1 VALUES (4, 4, NOW()); +INSERT INTO t1 VALUES (5, 5, NOW()); +INSERT INTO t1 VALUES (6, 6, NOW()); +INSERT INTO t1 VALUES (7, 7, NOW()); +INSERT INTO t1 VALUES (8, 8, NOW()); +INSERT INTO t1 VALUES (9, 9, NOW()); +INSERT INTO t1 VALUES (10, 10, NOW()); +set global rocksdb_debug_ttl_rec_ts = 0; +set global rocksdb_force_flush_memtable_now=1; +# None are expired +SELECT a, b FROM t1 FORCE INDEX (kb); +a b +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 +10 10 +set global rocksdb_debug_ttl_rec_ts = -100; +UPDATE t1 SET ts=(ADDTIME(NOW(), 1)) WHERE a IN (4, 7); +set global rocksdb_debug_ttl_rec_ts = 0; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +# 4 and 7 should be gone +SELECT a, b FROM t1 FORCE INDEX (kb); +a b +1 1 +2 2 +3 3 +5 5 +6 6 +8 8 +9 9 +10 10 +DROP TABLE t1; +CREATE TABLE t1 ( c1 INT, c2 INT, name VARCHAR(25) NOT NULL, diff --git a/storage/rocksdb/mysql-test/rocksdb/r/ttl_secondary_with_partitions.result b/storage/rocksdb/mysql-test/rocksdb/r/ttl_secondary_with_partitions.result index bbe53353d41..6e4c108fe03 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/ttl_secondary_with_partitions.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/ttl_secondary_with_partitions.result @@ -334,6 +334,67 @@ c1 9 DROP TABLE t1; CREATE TABLE t1 ( +c1 BIGINT, +c2 TIMESTAMP DEFAULT NOW() NOT NULL, +name VARCHAR(25) NOT NULL, +event DATE, +PRIMARY KEY (`c1`) COMMENT 'custom_p0_cfname=foo;custom_p1_cfname=bar;custom_p2_cfname=baz;', +KEY kc2 (`c2`) +) ENGINE=ROCKSDB +COMMENT="ttl_duration=1;custom_p1_ttl_duration=100;custom_p1_ttl_col=c2;custom_p2_ttl_duration=5000;" +PARTITION BY LIST(c1) ( +PARTITION custom_p0 VALUES IN (1, 2, 3), +PARTITION custom_p1 VALUES IN (4, 5, 6), +PARTITION custom_p2 VALUES IN (7, 8, 9) +); +set global rocksdb_debug_ttl_rec_ts = -300; +INSERT INTO t1 VALUES (1, NOW(), "one", null); +INSERT INTO t1 VALUES (2, NOW(), "two", null); +INSERT INTO t1 VALUES (3, NOW(), "three", null); +set global rocksdb_debug_ttl_rec_ts = 0; +INSERT INTO t1 VALUES (4, NOW(), "four", null); +INSERT INTO t1 VALUES (5, NOW(), "five", null); +INSERT INTO t1 VALUES (6, NOW(), "six", null); +INSERT INTO t1 VALUES (7, NOW(), "seven", null); +INSERT INTO t1 VALUES (8, NOW(), "eight", null); +INSERT INTO t1 VALUES (9, NOW(), "nine", null); +set global rocksdb_force_flush_memtable_now=1; +set @@global.rocksdb_compact_cf = 'foo'; +set @@global.rocksdb_compact_cf = 'baz'; +set @@global.rocksdb_compact_cf = 'bar'; +set @@global.rocksdb_compact_cf = 'default'; +SELECT c1 FROM t1 FORCE INDEX (PRIMARY); +c1 +4 +5 +6 +7 +8 +9 +SELECT c1 FROM t1 FORCE INDEX (kc2); +c1 +4 +5 +6 +7 +8 +9 +set global rocksdb_debug_ttl_snapshot_ts = 600; +set @@global.rocksdb_compact_cf = 'bar'; +set @@global.rocksdb_compact_cf = 'default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT c1 FROM t1 FORCE INDEX (PRIMARY); +c1 +7 +8 +9 +SELECT c1 FROM t1 FORCE INDEX (kc2); +c1 +7 +8 +9 +DROP TABLE t1; +CREATE TABLE t1 ( c1 INT, c2 INT, PRIMARY KEY (`c1`) COMMENT 'custom_p0_cfname=foo;' diff --git a/storage/rocksdb/mysql-test/rocksdb/r/unique_sec.result b/storage/rocksdb/mysql-test/rocksdb/r/unique_sec.result index a37e7f1cb31..691ea8e2260 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/unique_sec.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/unique_sec.result @@ -219,3 +219,34 @@ CHECK TABLE t EXTENDED; Table Op Msg_type Msg_text test.t check status OK DROP TABLE t; +# +# Validate lock acquired on SK during delete +# +CREATE TABLE t3 (pk INT PRIMARY KEY, sk INT, UNIQUE KEY (sk)) ENGINE=ROCKSDB; +INSERT INTO t3 VALUES (1, 100); +BEGIN; +# This should acquire a lock on sk = 100 +DELETE FROM t3 WHERE pk = 1; +connect con1, localhost, root,,; +connection con1; +set session rocksdb_lock_wait_timeout=1; +set session transaction_isolation='REPEATABLE-READ'; +SELECT * FROM t3; +pk sk +1 100 +# RR: This should fail with lock wait timeout due to sk = 100 +INSERT INTO t3 VALUES (2, 100); +ERROR HY000: Lock wait timeout exceeded; try restarting transaction +set session transaction_isolation='READ-COMMITTED'; +# RC: This should fail with lock wait timeout due to sk = 100 +INSERT INTO t3 VALUES (2, 100); +ERROR HY000: Lock wait timeout exceeded; try restarting transaction +connection default; +# The original transaction should be able to perform an insert +INSERT INTO t3 VALUES (3, 100); +COMMIT; +SELECT * FROM t3; +pk sk +3 100 +disconnect con1; +DROP TABLE t3; diff --git a/storage/rocksdb/mysql-test/rocksdb/suite.pm b/storage/rocksdb/mysql-test/rocksdb/suite.pm index b4feb20a451..c9361a32926 100644 --- a/storage/rocksdb/mysql-test/rocksdb/suite.pm +++ b/storage/rocksdb/mysql-test/rocksdb/suite.pm @@ -15,8 +15,8 @@ use strict; my $sst_dump= ::mtr_exe_maybe_exists( - "$::bindir/storage/rocksdb$::multiconfig/sst_dump", - "$::path_client_bindir/sst_dump"); + "$::bindir/storage/rocksdb$::multiconfig/mariadb-sst-dump", + "$::path_client_bindir/mariadb-sst-dump"); return "RocksDB is not compiled, no sst_dump" unless $sst_dump; $ENV{MARIAROCKS_SST_DUMP}="$sst_dump"; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/bloomfilter2.test b/storage/rocksdb/mysql-test/rocksdb/t/bloomfilter2.test index c4f1570ec41..66fa0599c21 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/bloomfilter2.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/bloomfilter2.test @@ -4,7 +4,7 @@ CREATE TABLE t0 (id1 VARCHAR(30), id2 INT, value INT, PRIMARY KEY (id1, id2)) ENGINE=rocksdb collate latin1_bin; --disable_query_log let $i = 1; -while ($i <= 10000) { +while ($i <= 100000) { let $insert = INSERT INTO t0 VALUES('X', $i, $i); inc $i; eval $insert; @@ -24,7 +24,7 @@ CREATE TABLE t1 (id1 BIGINT, id2 INT, id3 BIGINT, value INT, PRIMARY KEY (id1, i --disable_query_log let $i = 1; -while ($i <= 10000) { +while ($i <= 100000) { let $insert = INSERT INTO t1 VALUES(1, 1, $i, $i); eval $insert; inc $i; @@ -48,7 +48,7 @@ DROP TABLE t1; CREATE TABLE t2 (id1 INT, id2 VARCHAR(100), id3 BIGINT, value INT, PRIMARY KEY (id1, id2, id3)) ENGINE=rocksdb collate latin1_bin; --disable_query_log let $i = 1; -while ($i <= 10000) { +while ($i <= 100000) { let $insert = INSERT INTO t2 VALUES($i, $i, $i, $i); inc $i; eval $insert; @@ -76,7 +76,7 @@ DROP TABLE t2; CREATE TABLE t3 (id1 BIGINT, id2 BIGINT, id3 BIGINT, id4 BIGINT, PRIMARY KEY (id1, id2, id3, id4)) ENGINE=rocksdb collate latin1_bin; --disable_query_log let $i = 1; -while ($i <= 10000) { +while ($i <= 100000) { if ($i != 5000) { let $insert = INSERT INTO t3 VALUES(1, $i, $i, $i); eval $insert; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/cancel_mc.test b/storage/rocksdb/mysql-test/rocksdb/t/cancel_mc.test new file mode 100644 index 00000000000..f181bb75b94 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/cancel_mc.test @@ -0,0 +1,201 @@ +--source include/have_rocksdb.inc + +DELIMITER //; +CREATE FUNCTION is_compaction_stopped() RETURNS INT +BEGIN + SELECT variable_value INTO @cwb_1 FROM information_schema.global_status WHERE variable_name="rocksdb_compact_write_bytes"; + SELECT SLEEP(3) INTO @a; + SELECT variable_value INTO @cwb_2 FROM information_schema.global_status WHERE variable_name="rocksdb_compact_write_bytes"; + IF @cwb_2 > @cwb_1 THEN + RETURN 0; + END IF; + RETURN 1; +END// +DELIMITER ;// + + +let $datadir = `SELECT @@datadir`; + +--disable_warnings +DROP TABLE IF EXISTS t1; +--enable_warnings +create table t1 (id int auto_increment, value int, value2 varchar(500), primary key (id), index(value2) comment 'cf_i') engine=rocksdb; + +let $datadir = `SELECT @@datadir`; + +# generating test data +--let MYSQLD_DATADIR=$datadir +perl; +my $row_count = 500000; +my $datadir = $ENV{'MYSQLD_DATADIR'}; +open(DATA, ">", $datadir . "/se_loaddata.dat") || die "Can't open file: $!\n"; +my $value = 'x' x 250; +for (my $i= 1; $i <= $row_count; $i++) { + print DATA "$i,$i,$value\n"; +} +close(DATA); +EOF + +SET SESSION rocksdb_bulk_load_allow_sk=1; +SET SESSION rocksdb_bulk_load=1; + +--replace_result $datadir +eval +LOAD DATA INFILE '$datadir/se_loaddata.dat' INTO TABLE t1 + FIELDS TERMINATED BY ',' (id, value, value2); + +SET SESSION rocksdb_bulk_load=0; +SET SESSION rocksdb_bulk_load_allow_sk=0; + +--remove_file $datadir/se_loaddata.dat + + +--connect (con1,localhost,root,,) +--connect (con2,localhost,root,,) + +# Note that finishing cancelling MC may take a few seconds after instruction, +# so testing with very large datasets or slowing down compaction is needed +# to make tests reliable. rocksdb_rate_limiter_bytes_per_sec=256k +# (in -master.opt) is an option to do the latter so that RocksDB has enough +# time to cancel compactions. + +--echo # +--echo # Cancelling MC, including pending MCs, by KILL statements +--echo # + +connection default; +SELECT variable_value into @executed_start FROM information_schema.global_status WHERE variable_name="rocksdb_manual_compactions_cancelled"; + +connection con1; +--let $con1_id = `SELECT CONNECTION_ID()` +send SET GLOBAL rocksdb_compact_cf='default'; + +connection con2; +--let $con2_id = `SELECT CONNECTION_ID()` +set session rocksdb_manual_compaction_threads=2; +send SET GLOBAL rocksdb_compact_cf='cf_i'; + +connection default; +let $wait_condition= + select variable_value = 1 from information_schema.global_status WHERE variable_name="rocksdb_manual_compactions_running"; +--source include/wait_condition.inc +let $wait_condition= + select variable_value = 1 from information_schema.global_status WHERE variable_name="rocksdb_manual_compactions_pending"; +--source include/wait_condition.inc +--disable_query_log +eval kill query $con1_id; +eval kill query $con2_id; +--enable_query_log + +connection con1; +--error ER_INTERNAL_ERROR +reap; + +connection con2; +--error ER_INTERNAL_ERROR +reap; + +connection default; +let $wait_condition= + select variable_value = 0 from information_schema.global_status WHERE variable_name="rocksdb_manual_compactions_pending"; +--source include/wait_condition.inc + +SELECT variable_value into @executed_end FROM information_schema.global_status WHERE variable_name="rocksdb_manual_compactions_cancelled"; +SELECT @executed_end-@executed_start AS CANCELLED_MC; + +let $wait_condition= + select is_compaction_stopped() = 1 from dual; +--source include/wait_condition.inc + +--echo # +--echo # Cancelling MC by global opt +--echo # + +connection default; +SELECT variable_value into @executed_start FROM information_schema.global_status WHERE variable_name="rocksdb_manual_compactions_cancelled"; + +connection con1; +set session rocksdb_manual_compaction_threads=2; +send SET GLOBAL rocksdb_compact_cf='default'; + +# waiting for compaction to actually start +connection default; +let $wait_condition= + select variable_value = 1 from information_schema.global_status WHERE variable_name="rocksdb_manual_compactions_running"; +--source include/wait_condition.inc + +# Issuing second MC (pending) +connection con2; +send SET GLOBAL rocksdb_compact_cf='cf_i'; + +connection default; +let $wait_condition= + select variable_value = 1 from information_schema.global_status WHERE variable_name="rocksdb_manual_compactions_pending"; +--source include/wait_condition.inc +set global rocksdb_cancel_manual_compactions=ON; + +# first MC (running state) is cancelled +connection con1; +--error ER_INTERNAL_ERROR +reap; + +# second MC (pending state) is also cancelled +connection con2; +--error ER_INTERNAL_ERROR +reap; + +connection default; +SELECT variable_value into @executed_end FROM information_schema.global_status WHERE variable_name="rocksdb_manual_compactions_cancelled"; +SELECT @executed_end-@executed_start AS CANCELLED_MC; + +let $wait_condition= + select is_compaction_stopped() = 1 from dual; +--source include/wait_condition.inc + +--echo # +--echo # Cancelling OPTIMIZE TABLE by global opt +--echo # +connection con1; +send OPTIMIZE TABLE t1; + +connection default; +SELECT variable_value into @current_cwb FROM information_schema.global_status WHERE variable_name="rocksdb_compact_write_bytes"; + +let $wait_condition= + select count(*) = 1 from information_schema.processlist + where info IN ("OPTIMIZE TABLE t1"); +--source include/wait_condition.inc + +# compacting more than 100kb (at least a few percent) +let $wait_condition= + select variable_value > (@current_cwb + 100000) FROM information_schema.global_status WHERE variable_name="rocksdb_compact_write_bytes"; +--source include/wait_condition.inc + +set global rocksdb_cancel_manual_compactions=ON; + +connection con1; +reap; + +connection default; +select 'PRIMARY' as idx, COUNT(*) as cnt FROM t1 FORCE INDEX (PRIMARY) +UNION ALL SELECT 'value2' as idx, COUNT(*) as cnt FROM t1 FORCE INDEX (value2); + +--echo # +--echo # Dropping Indexes complete even after cancelling MC +--echo # +INSERT INTO t1 (id) VALUES (null); +set global rocksdb_force_flush_memtable_now=ON; + +SELECT variable_value into @current_cwb FROM information_schema.global_status WHERE variable_name="rocksdb_compact_write_bytes"; +DROP TABLE t1; + +let $wait_condition= + select variable_value > @current_cwb FROM information_schema.global_status WHERE variable_name="rocksdb_compact_write_bytes"; +set global rocksdb_cancel_manual_compactions=ON; + +let $wait_condition = select count(*) = 0 + as c from information_schema.rocksdb_global_info + where TYPE = 'DDL_DROP_INDEX_ONGOING'; +--source include/wait_condition.inc + +DROP FUNCTION is_compaction_stopped; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/cancel_mc_timeout.test b/storage/rocksdb/mysql-test/rocksdb/t/cancel_mc_timeout.test new file mode 100644 index 00000000000..a843f0ec975 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/cancel_mc_timeout.test @@ -0,0 +1,49 @@ +--source include/have_rocksdb.inc + +--connect (con1,localhost,root,,) +--connect (con2,localhost,root,,) + +connection default; +# longer than compaction wait timeout (60 seconds) +set global rocksdb_debug_manual_compaction_delay=80; + +connection con1; +--let $con1_id = `SELECT CONNECTION_ID()` +send SET GLOBAL rocksdb_compact_cf='default'; + + +connection default; +let $wait_condition= + select variable_value = 1 from information_schema.global_status WHERE variable_name="rocksdb_manual_compactions_running"; +--source include/wait_condition.inc + +connection con2; +--let $con2_id = `SELECT CONNECTION_ID()` +set session rocksdb_manual_compaction_threads=2; +send SET GLOBAL rocksdb_compact_cf='default'; + +connection default; +let $wait_condition= + select variable_value = 1 from information_schema.global_status WHERE variable_name="rocksdb_manual_compactions_pending"; +--source include/wait_condition.inc +--disable_query_log +eval kill query $con1_id; +eval kill query $con2_id; +--enable_query_log + +# pending compaction can be cancelled instantly +connection con2; +--error ER_INTERNAL_ERROR +reap; + +# running compaction hits timeout +connection con1; +--error ER_INTERNAL_ERROR +reap; + +connection default; +set global rocksdb_debug_manual_compaction_delay=0; + +# next compaction starts after previous (running) compaction finishes +connection con1; +SET GLOBAL rocksdb_compact_cf='default'; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/disabled.def b/storage/rocksdb/mysql-test/rocksdb/t/disabled.def index 627d7da4171..09df5403f9f 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/disabled.def +++ b/storage/rocksdb/mysql-test/rocksdb/t/disabled.def @@ -69,6 +69,7 @@ gap_lock_issue254: MDEV-11735: MyRocks: Gap Lock detector support gap_lock_raise_error: MDEV-11735: MyRocks: Gap Lock detector support show_engine : MariaRocks: MariaDB doesnt support SHOW ENGINE rocksdb TRANSACTION STATUS issue243_transactionStatus: MariaDB doesnt support SHOW ENGINE rocksdb TRANSACTION STATUS +max_row_locks: MariaDB doesnt support SHOW ENGINE rocksdb TRANSACTION STATUS rpl_row_not_found : MariaDB doesnt support slave_exec_mode='SEMI_STRICT' rpl_row_not_found_rc : MariaDB doesnt support slave_exec_mode='SEMI_STRICT' diff --git a/storage/rocksdb/mysql-test/rocksdb/t/drop_table3.inc b/storage/rocksdb/mysql-test/rocksdb/t/drop_table3.inc index 1a044384a45..5ee0f1d134d 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/drop_table3.inc +++ b/storage/rocksdb/mysql-test/rocksdb/t/drop_table3.inc @@ -1,8 +1,5 @@ --source include/have_rocksdb.inc -call mtr.add_suppression("Column family 'cf1' not found"); -call mtr.add_suppression("Column family 'rev:cf2' not found"); - --disable_warnings DROP TABLE IF EXISTS t1; --enable_warnings @@ -10,12 +7,6 @@ DROP TABLE IF EXISTS t1; call mtr.add_suppression("Column family 'cf1' not found"); call mtr.add_suppression("Column family 'rev:cf2' not found"); -# Start from clean slate -set global rocksdb_compact_cf = 'cf1'; -set global rocksdb_compact_cf = 'rev:cf2'; -set global rocksdb_signal_drop_index_thread = 1; ---source include/restart_mysqld.inc - CREATE TABLE t1 ( a int not null, b int not null, @@ -29,6 +20,12 @@ let $max = 50000; let $table = t1; --source drop_table3_repopulate_table.inc +# Start from clean slate +set global rocksdb_compact_cf = 'cf1'; +set global rocksdb_compact_cf = 'rev:cf2'; +set global rocksdb_signal_drop_index_thread = 1; +--source include/restart_mysqld.inc + select variable_value into @a from information_schema.global_status where variable_name='rocksdb_compact_read_bytes'; if ($truncate_table) { diff --git a/storage/rocksdb/mysql-test/rocksdb/t/information_schema.test b/storage/rocksdb/mysql-test/rocksdb/t/information_schema.test index 09998b9ae5f..cb5c6019c5c 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/information_schema.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/information_schema.test @@ -66,6 +66,9 @@ select * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO where TYPE = 'CF_FLAGS'; CREATE TABLE t3 (a INT, PRIMARY KEY (a)) ENGINE=ROCKSDB; insert into t3 (a) values (1), (2), (3); + +SELECT IF(count(*) > 0, "TRUE", "FALSE") as metadata_exist FROM INFORMATION_SCHEMA.ROCKSDB_LIVE_FILES_METADATA where CF_NAME = 'default'; + SET @ORIG_ROCKSDB_PAUSE_BACKGROUND_WORK = @@GLOBAL.ROCKSDB_PAUSE_BACKGROUND_WORK; --let $t3_index_id = query_get_value(SELECT * FROM INFORMATION_SCHEMA.ROCKSDB_DDL WHERE TABLE_NAME = 't3', INDEX_NUMBER, 1) --let $t3_cf_id = query_get_value(SELECT * FROM INFORMATION_SCHEMA.ROCKSDB_DDL WHERE TABLE_NAME = 't3', COLUMN_FAMILY, 1) diff --git a/storage/rocksdb/mysql-test/rocksdb/t/issue243_transactionStatus.test b/storage/rocksdb/mysql-test/rocksdb/t/issue243_transactionStatus.test index 0997bde3f49..6e910d2a426 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/issue243_transactionStatus.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/issue243_transactionStatus.test @@ -76,5 +76,34 @@ DELETE FROM t2 WHERE id1=10; --replace_regex /[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}/TIMESTAMP/ /(ACTIVE) [0-9]+ /\1 NUM / /(thread id) [0-9]+/\1 TID/ /0x[0-9a-f]+/PTR/ /(query id) [0-9]+/\1 QID/ /(root) [a-z ]+/\1 ACTION/ SHOW ENGINE rocksdb TRANSACTION STATUS; ROLLBACK; + +SET AUTOCOMMIT=1; +DROP TABLE t2; + +# +# Secondary Key with uniq key Tests +# +--disable_warnings + DROP TABLE IF EXISTS t2; +--enable_warnings + + CREATE TABLE t2 ( + id1 INT, + id2 INT, + value INT, + PRIMARY KEY (id1), + UNIQUE KEY (id2) + ) ENGINE=rocksdb; + +SET AUTOCOMMIT=0; +START TRANSACTION; +INSERT INTO t2 VALUES(1,2,0),(10,20,30); +UPDATE t2 SET value=3 WHERE id2=2; +DELETE FROM t2 WHERE id1=10; + +--replace_regex /[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}/TIMESTAMP/ /(ACTIVE) [0-9]+ /\1 NUM / /(thread id) [0-9]+/\1 TID/ /0x[0-9a-f]+/PTR/ /(query id) [0-9]+/\1 QID/ /(root) [a-z ]+/\1 ACTION/ + SHOW ENGINE rocksdb TRANSACTION STATUS; +ROLLBACK; + SET AUTOCOMMIT=1; DROP TABLE t2; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/issue896.test b/storage/rocksdb/mysql-test/rocksdb/t/issue896.test index ba57fb99832..af75a0c6434 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/issue896.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/issue896.test @@ -15,3 +15,18 @@ EXPLAIN SELECT COUNT(*) FROM t1 FORCE INDEX(d); --echo # segfault here without the fix SELECT COUNT(*) FROM t1 FORCE INDEX(d); DROP TABLE t1; + +CREATE TABLE `t1` ( +`a` bigint(20) NOT NULL, +`b` varchar(10) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, +`u` timestamp NOT NULL, +`d` bigint(20) DEFAULT NULL, +PRIMARY KEY (`a`,`b`), +KEY `d` (`d`) +) ENGINE=ROCKSDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='ttl_duration=1000;ttl_col=u'; +INSERT INTO t1 VALUES (100, 'aaabbb', NOW(), 200); +--replace_column 9 # +EXPLAIN SELECT COUNT(*) FROM t1; +--echo # segfault here without the fix +SELECT COUNT(*) FROM t1; +DROP TABLE t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/manual_compaction_bottommost_level-master.opt b/storage/rocksdb/mysql-test/rocksdb/t/manual_compaction_bottommost_level-master.opt new file mode 100644 index 00000000000..831d66bc409 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/manual_compaction_bottommost_level-master.opt @@ -0,0 +1,2 @@ +--rocksdb_max_subcompactions=1 +--rocksdb_default_cf_options=target_file_size_base=100k;max_bytes_for_level_multiplier=1;max_bytes_for_level_base=1m;target_file_size_multiplier=1 diff --git a/storage/rocksdb/mysql-test/rocksdb/t/manual_compaction_bottommost_level.test b/storage/rocksdb/mysql-test/rocksdb/t/manual_compaction_bottommost_level.test new file mode 100644 index 00000000000..a3f5c88217d --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/manual_compaction_bottommost_level.test @@ -0,0 +1,52 @@ +--source include/have_rocksdb.inc + +DELIMITER //; +CREATE PROCEDURE compact_start() +BEGIN + select variable_value into @c from information_schema.global_status where variable_name='rocksdb_compact_write_bytes'; +END// +CREATE PROCEDURE compact_end() +BEGIN + select case when variable_value-@c > 0 then 'true' else 'false' end as checked from information_schema.global_status where variable_name='rocksdb_compact_write_bytes'; +END// +DELIMITER ;// + +--disable_warnings +DROP TABLE IF EXISTS t1; +--enable_warnings + +CREATE TABLE t1 ( + a int not null, + b int not null, + primary key (a,b), + key (b) +) ENGINE=RocksDB; + +# Populate tables +let $max = 1000; +let $table = t1; +--source drop_table_repopulate_table.inc + +# compact to move all data in files in Lmax level +set @@session.rocksdb_manual_compaction_bottommost_level='kSkip'; +call compact_start(); +set @@global.rocksdb_compact_cf = 'default'; +call compact_end(); # should return true as compaction of lower levels is performed + +# skip compaction of files in bottommost level i.e. Lmax->Lmax compaction and ensure rocksdb_compact_write_byte +# does not increase after compaction +set @@session.rocksdb_manual_compaction_bottommost_level='kSkip'; +call compact_start(); +set @@global.rocksdb_compact_cf = 'default'; +call compact_end(); # should return false as files only in bottommost layer + +# restore 'kForceOptimized' default setting and check that value of rocksdb_compact_write_byte increses after compaction +set @@session.rocksdb_manual_compaction_bottommost_level='kForceOptimized'; +call compact_start(); +set @@global.rocksdb_compact_cf = 'default'; +call compact_end(); # should return true + +# cleanup +DROP PROCEDURE compact_start; +DROP PROCEDURE compact_end; +drop table t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/max_row_locks.test b/storage/rocksdb/mysql-test/rocksdb/t/max_row_locks.test new file mode 100644 index 00000000000..6f3c5adeb66 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/max_row_locks.test @@ -0,0 +1,59 @@ +--source include/have_rocksdb.inc + +create table t1 (id1 bigint, id2 bigint, c1 bigint, c2 bigint, c3 bigint, c4 bigint, c5 bigint, c6 bigint, c7 bigint, primary key (id1, id2), index i(c1, c2)); +--disable_query_log +let $i=0; +while ($i<1000) +{ + inc $i; + eval insert t1(id1, id2, c1, c2, c3, c4, c5, c6, c7) + values($i, 0, $i, 0, 0, 0, 0, 0, $i); +} +--enable_query_log + +# by default, no matching row should not have any row lock. +begin; +select * from t1 where c3=1 for update; + +# This should report 0 row lock +--replace_regex /[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}/TIMESTAMP/ /(ACTIVE) [0-9]+ /\1 NUM / /(thread id) [0-9]+/\1 TID/ /0x[0-9a-f]+/PTR/ /(query id) [0-9]+/\1 QID/ /(root) [a-z ]+/\1 ACTION/ +SHOW ENGINE rocksdb TRANSACTION STATUS; + +select * from t1 where c7 between 101 and 110 for update; + +# This should report 10 row locks +--replace_regex /[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}/TIMESTAMP/ /(ACTIVE) [0-9]+ /\1 NUM / /(thread id) [0-9]+/\1 TID/ /0x[0-9a-f]+/PTR/ /(query id) [0-9]+/\1 QID/ /(root) [a-z ]+/\1 ACTION/ +SHOW ENGINE rocksdb TRANSACTION STATUS; + +rollback; + +set session rocksdb_lock_scanned_rows=on; + +begin; +select * from t1 where c3=1 for update; +# This should report 1000 row locks +--replace_regex /[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}/TIMESTAMP/ /(ACTIVE) [0-9]+ /\1 NUM / /(thread id) [0-9]+/\1 TID/ /0x[0-9a-f]+/PTR/ /(query id) [0-9]+/\1 QID/ /(root) [a-z ]+/\1 ACTION/ +SHOW ENGINE rocksdb TRANSACTION STATUS; + +rollback; +set session rocksdb_lock_scanned_rows=off; + +SET @start_rocksdb_max_row_locks = @@global.rocksdb_max_row_locks; +# small max row locks should not hit errors if number of locked rows is smaller +set global rocksdb_max_row_locks = 20; +select * from t1 where c3=1 for update; +select * from t1 where c7 between 101 and 110 for update; + +set global rocksdb_max_row_locks = 5; +select * from t1 where c3=1 for update; +--error 1296 +select * from t1 where c7 between 101 and 110 for update; + +set session rocksdb_lock_scanned_rows=on; +--error 1296 +select * from t1 where c3=1 for update; +--error 1296 +select * from t1 where c7 between 101 and 110 for update; +set session rocksdb_lock_scanned_rows=off; +set @@global.rocksdb_max_row_locks = @start_rocksdb_max_row_locks; +drop table t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/perf_context.test b/storage/rocksdb/mysql-test/rocksdb/t/perf_context.test index ee41324a34d..ee30e29a6de 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/perf_context.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/perf_context.test @@ -64,10 +64,24 @@ INSERT INTO t2 VALUES (1), (2); INSERT INTO t2 VALUES (3), (4); COMMIT; -SELECT COUNT(*) from INFORMATION_SCHEMA.ROCKSDB_PERF_CONTEXT -WHERE TABLE_NAME = 't2' -AND STAT_TYPE = 'IO_WRITE_NANOS' -AND VALUE > 0; +# For write unprepared, writes can happen before commit, meaning some IOs can +# be attributed to specific statements/tables. +let $wup = `SELECT @@global.rocksdb_write_policy = 'write_unprepared'`; +--disable_query_log +if ($wup) { + SELECT COUNT(*) from INFORMATION_SCHEMA.ROCKSDB_PERF_CONTEXT + WHERE TABLE_NAME = 't2' + AND STAT_TYPE = 'IO_WRITE_NANOS' + AND VALUE = 0; +} + +if (!$wup) { + SELECT COUNT(*) from INFORMATION_SCHEMA.ROCKSDB_PERF_CONTEXT + WHERE TABLE_NAME = 't2' + AND STAT_TYPE = 'IO_WRITE_NANOS' + AND VALUE > 0; +} +--enable_query_log SELECT COUNT(*) from INFORMATION_SCHEMA.ROCKSDB_PERF_CONTEXT_GLOBAL WHERE STAT_TYPE = 'IO_WRITE_NANOS' AND VALUE > 0; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/rocksdb.test b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb.test index 9b24ad952d7..7637dbe7d53 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/rocksdb.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb.test @@ -1150,9 +1150,10 @@ CREATE TABLE t1 (pk INT PRIMARY KEY, a int) ENGINE=RocksDB; set @a=-1; insert into t1 select (@a:=@a+1), 1234 from information_schema.session_variables limit 100; set @tmp1= @@rocksdb_max_row_locks; -set rocksdb_max_row_locks= 20; +set GLOBAL rocksdb_max_row_locks= 20; --error ER_GET_ERRMSG update t1 set a=a+10; +set @@global.rocksdb_max_row_locks = @tmp1; DROP TABLE t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_bottom_pri_compaction-master.opt b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_bottom_pri_compaction-master.opt new file mode 100644 index 00000000000..3948a18a68a --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_bottom_pri_compaction-master.opt @@ -0,0 +1 @@ +--rocksdb_max_bottom_pri_background_compactions=1 diff --git a/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_bottom_pri_compaction.test b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_bottom_pri_compaction.test new file mode 100644 index 00000000000..fd6ebec7717 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_bottom_pri_compaction.test @@ -0,0 +1,30 @@ +--source include/have_rocksdb.inc +--source include/linux.inc + +--exec grep "Set 1 compaction thread(s) with lower scheduling priority." $MYSQLTEST_VARDIR/log/mysqld.1.err | tail -n 1 | wc -l + +# Needs at least one compaction to happen to change priority +--disable_warnings +DROP TABLE IF EXISTS t1; +--enable_warnings + +CREATE TABLE t1 ( + a int not null, + b int not null, + primary key (a,b), + key (b) +) ENGINE=RocksDB; + +# Populate tables +let $max = 1000; +let $table = t1; +--source drop_table_repopulate_table.inc + +set @@global.rocksdb_compact_cf = 'default'; + +--let pid_file = query_get_value(SELECT @@pid_file, @@pid_file, 1) +--let print_bottom_priority = 1 +--let expected_thread_counts = 1 +--source rocksdb_bottom_pri_compaction_check.inc + +drop table t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_bottom_pri_compaction2-master.opt b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_bottom_pri_compaction2-master.opt new file mode 100644 index 00000000000..40d0f9ef896 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_bottom_pri_compaction2-master.opt @@ -0,0 +1 @@ +--rocksdb_max_bottom_pri_background_compactions=5 diff --git a/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_bottom_pri_compaction2.test b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_bottom_pri_compaction2.test new file mode 100644 index 00000000000..934d1e121ca --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_bottom_pri_compaction2.test @@ -0,0 +1,27 @@ +--source include/have_rocksdb.inc +--source include/linux.inc + +--exec grep "Set 5 compaction thread(s) with lower scheduling priority." $MYSQLTEST_VARDIR/log/mysqld.1.err | tail -n 1 | wc -l + +--let pid_file = query_get_value(SELECT @@pid_file, @@pid_file, 1) +--let print_bottom_priority = 0 +--let expected_thread_counts = 5 +--source rocksdb_bottom_pri_compaction_check.inc + +--echo "Increasing bottom pri compaction threads online." +SET GLOBAL rocksdb_max_bottom_pri_background_compactions=10; +--exec grep "Set 10 compaction thread(s) with lower scheduling priority." $MYSQLTEST_VARDIR/log/mysqld.1.err | tail -n 1 | wc -l +--let expected_thread_counts = 10 +--source rocksdb_bottom_pri_compaction_check.inc + + +--echo "Decreasing bottom pri compaction threads online." +SET GLOBAL rocksdb_max_bottom_pri_background_compactions=2; +--exec grep "Set 2 compaction thread(s) with lower scheduling priority." $MYSQLTEST_VARDIR/log/mysqld.1.err | tail -n 1 | wc -l +--let expected_thread_counts = 2 +--source rocksdb_bottom_pri_compaction_check.inc + +--error ER_ERROR_WHEN_EXECUTING_COMMAND +SET GLOBAL rocksdb_max_bottom_pri_background_compactions=0; + +SET GLOBAL rocksdb_max_bottom_pri_background_compactions=5; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_bottom_pri_compaction_check.inc b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_bottom_pri_compaction_check.inc new file mode 100644 index 00000000000..693af294c35 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_bottom_pri_compaction_check.inc @@ -0,0 +1,40 @@ +perl; + $pid_file = $ENV{pid_file}; + $print_bottom_priority = $ENV{print_bottom_priority}; + $expected_thread_counts = $ENV{expected_thread_counts}; + + if(!defined($print_bottom_priority)) { + $print_bottom_priority = 0; + } + if(!defined($expected_thread_counts)) { + $expected_thread_counts = 0; + } + + # Get PID of mysqld + open(my $fh, '<', $pid_file) || die "Cannot open pid file $pid_file\n"; + my $pid = <$fh>; + $pid =~ s/\s//g; + close($fh); + + if ($pid eq "") { + die "Couldn't retrieve PID from PID file.\n"; + } + + if ($print_bottom_priority) { + print "Bottom thread priority:\n"; + system("cat /proc/$pid/task/*/stat | grep bottom | awk '{print \$19}'"); + } + + print "Bottom thread counts:\n"; + my $retries = 10; + for(my $i= 0; $i <= $retries; $i++) { + $count=`cat /proc/$pid/task/*/stat | grep bottom | wc -l`; + chomp($count); + if ($count != $expected_thread_counts) { + sleep(1); + } else { + print "$count\n"; + last; + } + } +EOF diff --git a/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_deadlock_detect.inc b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_deadlock_detect.inc index 082c61308f3..2fdd5a3e200 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_deadlock_detect.inc +++ b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_deadlock_detect.inc @@ -72,8 +72,22 @@ insert into t values (4), (1); select * from t; rollback; +# In repeatable read, rollbacks in write unprepared confuses the snapshot +# validation logic into thinking that a write has happened, because it sees +# the rollback writes in the DB. +let $snapshot_conflict = `SELECT @@global.rocksdb_write_policy = 'write_unprepared' and @@session.tx_isolation = 'repeatable-read'`; connection con2; ---reap +--disable_result_log +if ($snapshot_conflict) +{ + --error ER_LOCK_DEADLOCK + --reap +} +if (!$snapshot_conflict) +{ + --reap +} +--enable_result_log rollback; connection con1; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/truncate_failures-master.opt b/storage/rocksdb/mysql-test/rocksdb/t/truncate_failures-master.opt new file mode 100644 index 00000000000..9c511d27dce --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/truncate_failures-master.opt @@ -0,0 +1 @@ +--loose-enable-partition diff --git a/storage/rocksdb/mysql-test/rocksdb/t/truncate_failures.test b/storage/rocksdb/mysql-test/rocksdb/t/truncate_failures.test new file mode 100644 index 00000000000..10b5337c97e --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/truncate_failures.test @@ -0,0 +1,115 @@ +--source include/have_rocksdb.inc +--source include/have_debug_sync.inc +--source include/not_valgrind.inc + +# +# Test truncation of tables with potential collation issues +# + +set global rocksdb_strict_collation_exceptions = 't1,t2'; + +CREATE TABLE t1 ( + pk INT, + a VARCHAR(64), + PRIMARY KEY (pk), + KEY (a) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 COLLATE=latin1_general_ci; + +INSERT INTO t1 VALUES (1, 'one'); +RENAME TABLE t1 TO t1_trunc; +SELECT * FROM t1_trunc ORDER BY pk; +TRUNCATE TABLE t1_trunc; +SELECT * FROM t1_trunc; + +CREATE TABLE t2 ( + pk INT, + a VARCHAR(64), + PRIMARY KEY (pk), + KEY (a) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 COLLATE=latin1_general_ci +PARTITION BY RANGE (pk) ( + PARTITION p0 VALUES LESS THAN (3), + PARTITION p1 VALUES LESS THAN (6), + PARTITION p2 VALUES LESS THAN MAXVALUE +); + +INSERT INTO t2 VALUES (1, 'one'), (4, 'four'), (7, 'seven'); +RENAME TABLE t2 to t2_trunc; +SELECT * FROM t2_trunc ORDER BY pk; +ALTER TABLE t2_trunc TRUNCATE PARTITION p0; +SELECT * FROM t2_trunc ORDER BY pk; +TRUNCATE TABLE t2_trunc; +SELECT * FROM t2_trunc ORDER BY pk; + +set global rocksdb_strict_collation_exceptions = default; + +DROP TABLE t1_trunc, t2_trunc; + +# +# Test truncation of tables with explicit failures during create +# + +CREATE TABLE t1 ( + pk INT PRIMARY KEY +) ENGINE=ROCKSDB; + +INSERT INTO t1 VALUES (1), (2); +SELECT * FROM t1; + +SET DEBUG_DBUG = "+d,rocksdb_truncate_failure"; +--error 1815 +TRUNCATE TABLE t1; +SET DEBUG_DBUG = "-d,rocksdb_truncate_failure"; +SELECT * FROM t1; + +CREATE TABLE t2 ( + a INT +) ENGINE=ROCKSDB +PARTITION BY RANGE (a) ( + PARTITION p0 VALUES LESS THAN (3), + PARTITION p1 VALUES LESS THAN (6), + PARTITION p2 VALUES LESS THAN MAXVALUE +); + +INSERT INTO t2 VALUES (1), (4), (7); + +SET DEBUG_DBUG = "+d,rocksdb_truncate_failure"; +--error 1815 +TRUNCATE TABLE t2; +--error 1815 +ALTER TABLE t2 TRUNCATE PARTITION p1; +SET DEBUG_DBUG = "-d,rocksdb_truncate_failure"; +SELECT * FROM t2; + +DROP TABLE t1, t2; + +# +# Crash the server during server truncation and verify the table is removed +# + +CREATE TABLE t1_crash ( + pk INT PRIMARY KEY +) ENGINE=ROCKSDB; + +INSERT INTO t1_crash VALUES (100), (1000); +SELECT * FROM t1_crash; + +--let LOG=$MYSQLTEST_VARDIR/tmp/rocksdb_truncate_failure.err +--exec echo "restart:--rocksdb_validate_tables=2 --log-error=$LOG" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect + +SET DEBUG_DBUG = "+d,rocksdb_truncate_failure_crash"; +--error 2013 +TRUNCATE TABLE t1_crash; +--source include/wait_until_disconnected.inc +--enable_reconnect +--source include/wait_until_connected_again.inc +--disable_reconnect +--exec grep 'Removing truncated leftover' $LOG | cut -d] -f2 +DROP TABLE t1_crash; + +--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +shutdown_server; +--source include/wait_until_disconnected.inc +--enable_reconnect +--source include/wait_until_connected_again.inc +--disable_reconnect diff --git a/storage/rocksdb/mysql-test/rocksdb/t/ttl_primary.inc b/storage/rocksdb/mysql-test/rocksdb/t/ttl_primary.inc new file mode 100644 index 00000000000..b1af014870b --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/ttl_primary.inc @@ -0,0 +1,547 @@ +# +# Specify the timestamp column type using $coltype +# and the timestamp generator using $timegen +# + +# Basic TTL test +eval CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64) NOT NULL, +`c` varbinary(256) NOT NULL, +`ts` $coltype NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`b`,`a`,`c`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; + +set global rocksdb_debug_ttl_rec_ts = -100; +eval INSERT INTO t1 values ('a', 'b', 'c', $timegen, 'd'); +eval INSERT INTO t1 values ('d', 'e', 'f', $timegen, 'g'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; + +# should have filtered the rows out since ttl is passed in compaction filter +SELECT COUNT(*) FROM t1; +DROP TABLE t1; + +# column before TTL in value +eval CREATE TABLE t1 ( + a bigint(20) NOT NULL, + b int NOT NULL, + ts $coltype NOT NULL, + c int NOT NULL, + PRIMARY KEY (a) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; + +set global rocksdb_debug_ttl_rec_ts = -100; +eval INSERT INTO t1 values (1, 3, $timegen, 5); +eval INSERT INTO t1 values (2, 4, $timegen, 6); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; + +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; + +# should have filtered the rows out since ttl is passed in compaction filter +SELECT COUNT(*) FROM t1; +DROP TABLE t1; + +# multi-part PK w/ TTL +eval CREATE TABLE t1 ( + a bigint(20) NOT NULL, + b int NOT NULL, + c int NOT NULL, + ts $coltype NOT NULL, + PRIMARY KEY (a,c) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; + +set global rocksdb_debug_ttl_rec_ts = -100; +eval INSERT INTO t1 values (1, 3, 5, $timegen); +eval INSERT INTO t1 values (2, 4, 6, $timegen); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; + +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; + +# should have filtered the rows out since ttl is passed in compaction filter +SELECT COUNT(*) FROM t1; +DROP TABLE t1; + +# multi-part PK w/ TTL +eval CREATE TABLE t1 ( + a bigint(20) NOT NULL, + b int NOT NULL, + c int NOT NULL, + ts $coltype NOT NULL, + PRIMARY KEY (a,c) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; + +set global rocksdb_debug_ttl_rec_ts = -100; +eval INSERT INTO t1 values (1, 3, 5, $timegen); +eval INSERT INTO t1 values (2, 4, 6, $timegen); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; + +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; + +# should have filtered the rows out since ttl is passed in compaction filter +SELECT COUNT(*) FROM t1; +DROP TABLE t1; + +# nullable column(s) before TTL +eval CREATE TABLE t1 ( + a bigint(20) NOT NULL, + b int, + c int, + ts $coltype NOT NULL, + PRIMARY KEY (a) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; + +set global rocksdb_debug_ttl_rec_ts = -100; +eval INSERT INTO t1 values (1, NULL, NULL, $timegen); +eval INSERT INTO t1 values (2, NULL, NULL, $timegen); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; + +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; + +# should have filtered the rows out since ttl is passed in compaction filter +SELECT COUNT(*) FROM t1; +DROP TABLE t1; + +# variable len columns + null column(s) before TTL +eval CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64), +`c` varbinary(256), +`ts` $coltype NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`a`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; + +set global rocksdb_debug_ttl_rec_ts = -100; +eval INSERT INTO t1 values ('a', NULL, 'bc', $timegen, 'd'); +eval INSERT INTO t1 values ('d', 'efghijk', NULL, $timegen, 'l'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; + +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; + +# should have filtered the rows out since ttl is passed in compaction filter +SELECT COUNT(*) FROM t1; +DROP TABLE t1; + +# TTL implicitly generated (no ttl column) +CREATE TABLE t1 ( + a bigint(20) NOT NULL, + b int NOT NULL, + c int NOT NULL, + PRIMARY KEY (a) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;'; + +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, 3, 5); +INSERT INTO t1 values (2, 4, 6); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; + +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; + +# should have filtered the rows out since ttl is passed in compaction filter +SELECT COUNT(*) FROM t1; +DROP TABLE t1; + +# TTL field as the PK +eval CREATE TABLE t1 ( + a int, + ts $coltype NOT NULL, + PRIMARY KEY (a, ts) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;ttl_col=ts;'; + +eval INSERT INTO t1 values (1, $timegen); +eval INSERT INTO t1 values (2, $timegen); +SELECT COUNT(*) FROM t1; + +set global rocksdb_debug_ttl_snapshot_ts = -10; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +# should all still be there.. +SELECT COUNT(*) FROM t1; + +set global rocksdb_debug_ttl_snapshot_ts = 10; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +# should have filtered the rows out since ttl is passed in compaction filter +SELECT COUNT(*) FROM t1; +DROP TABLE t1; + + +# TTL field inside multi-part pk +eval CREATE TABLE t1 ( + a bigint(20) NOT NULL, + b int NOT NULL, + ts $coltype NOT NULL, + c int NOT NULL, + PRIMARY KEY (a, ts) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; + +set global rocksdb_debug_ttl_rec_ts = -100; +eval INSERT INTO t1 values (1, 3, $timegen, 5); +eval INSERT INTO t1 values (2, 4, $timegen, 6); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; + +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; + +# should have filtered the rows out since ttl is passed in compaction filter +SELECT COUNT(*) FROM t1; +DROP TABLE t1; + +# TTL field inside key with variable length things.. +eval CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64), +`c` varbinary(256), +`ts` $coltype NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`a`, `ts`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; + +set global rocksdb_debug_ttl_rec_ts = -100; +eval INSERT INTO t1 values ('a', NULL, 'bc', $timegen, 'd'); +eval INSERT INTO t1 values ('de', 'fghijk', NULL, $timegen, 'l'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; + +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; + +# should have filtered the rows out since ttl is passed in compaction filter +SELECT COUNT(*) FROM t1; +DROP TABLE t1; + +# TTL test where you compact (values still exist), real_sleep, then compact again, +# values should now be gone. +eval CREATE TABLE t1 ( +a INT NOT NULL, +b varbinary(64) NOT NULL, +c varbinary(256) NOT NULL, +ts $coltype NOT NULL, +value mediumblob NOT NULL, +PRIMARY KEY (b,a,c) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=10;ttl_col=ts;'; + +set global rocksdb_debug_ttl_rec_ts = -300; +eval INSERT INTO t1 values (1, 'b', 'c', $timegen, 'd'); +eval INSERT INTO t1 values (2, 'e', 'f', $timegen, 'g'); +set global rocksdb_debug_ttl_rec_ts = 300; +eval INSERT INTO t1 values (3, 'i', 'j', $timegen, 'k'); +eval INSERT INTO t1 values (4, 'm', 'n', $timegen, 'o'); +set global rocksdb_debug_ttl_rec_ts = 0; + +# Nothing should get removed here. +set global rocksdb_debug_ttl_snapshot_ts = -3600; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +--sorted_result +SELECT a FROM t1; + +# 1 and 2 should get removed here. +set global rocksdb_compact_cf='default'; +--sorted_result +SELECT a FROM t1; + +# 3 and 4 should get removed here. +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +--sorted_result +SELECT a FROM t1; + +DROP TABLE t1; + +# TTL field with nullable ttl column (should fail) +--error ER_RDB_TTL_COL_FORMAT +eval CREATE TABLE t1 ( + a bigint(20) UNSIGNED NOT NULL, + b int NOT NULL, + c int NOT NULL, + ts $coltype, + PRIMARY KEY (a,c) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; + +# TTL field with non 8-bit integer column (should fail) +--error ER_RDB_TTL_COL_FORMAT +CREATE TABLE t1 ( + a bigint(20) UNSIGNED NOT NULL, + b int NOT NULL, + c int NOT NULL, + ts int, + PRIMARY KEY (a,c) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; + +# TTL duration as some random garbage value +--error ER_RDB_TTL_DURATION_FORMAT +CREATE TABLE t1 ( + a bigint(20) UNSIGNED NOT NULL, + b int NOT NULL, + c int NOT NULL, + PRIMARY KEY (a,c) +) ENGINE=rocksdb +COMMENT='ttl_duration=abc;'; + +# TTL col is some column outside of the table +--error ER_RDB_TTL_COL_FORMAT +CREATE TABLE t1 ( + a bigint(20) UNSIGNED NOT NULL, + b int NOT NULL, + c int NOT NULL, + PRIMARY KEY (a,c) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=abc;'; + +# TTL col must have accompanying duration +--error ER_RDB_TTL_COL_FORMAT +CREATE TABLE t1 ( + a bigint(20) UNSIGNED NOT NULL, + b int NOT NULL, + c int NOT NULL, + PRIMARY KEY (a,c) +) ENGINE=rocksdb +COMMENT='ttl_col=abc;'; + +# Make sure it doesn't filter out things early +CREATE TABLE t1 ( + a bigint(20) NOT NULL, + PRIMARY KEY (a) +) ENGINE=rocksdb +COMMENT='ttl_duration=500;'; + +INSERT INTO t1 values (1); +SELECT COUNT(*) FROM t1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; + +SELECT COUNT(*) FROM t1; +DROP TABLE t1; + +# Testing altering table comment with updated TTL duration +# This should trigger a rebuild of the table +CREATE TABLE t1 ( + a INT PRIMARY KEY +) ENGINE=rocksdb +COMMENT='ttl_duration=100;'; + +INSERT INTO t1 values (1); +SELECT * FROM t1; + +set global rocksdb_debug_ttl_rec_ts = -300; +ALTER TABLE t1 COMMENT = 'ttl_duration=1'; +set global rocksdb_debug_ttl_rec_ts = 0; +SHOW CREATE TABLE t1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; + +SELECT COUNT(*) FROM t1; +DROP TABLE t1; + +# Tables with hidden PK and SK disabled +CREATE TABLE t1 ( + a INT PRIMARY KEY, + b INT +) ENGINE=rocksdb +COMMENT='ttl_duration=100;'; + +--error ER_RDB_TTL_UNSUPPORTED +ALTER TABLE t1 DROP PRIMARY KEY; + +DROP TABLE t1; + +# Test replacing PK, ttl should still work after +CREATE TABLE t1 ( + a INT PRIMARY KEY, + b INT +) ENGINE=rocksdb +COMMENT='ttl_duration=5;'; + +INSERT INTO t1 VALUES (1,1); +INSERT INTO t1 VALUES (2,2); + +ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY(b); +set global rocksdb_debug_ttl_snapshot_ts = -3600; +set global rocksdb_force_flush_memtable_now=1; +set @@global.rocksdb_compact_cf = 'default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; + +--sorted_result +SELECT COUNT(*) FROM t1; + +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set @@global.rocksdb_compact_cf = 'default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; + +--sorted_result +SELECT COUNT(*) FROM t1; + +DROP TABLE t1; + +# Make sure table comment filled with other text before/after will work +# (basically, it needs semicolon before and after) +eval CREATE TABLE t1 ( + a $coltype NOT NULL, + b int, + PRIMARY KEY (a,b) +) ENGINE=rocksdb +COMMENT='asdadfasdfsadfadf ;ttl_duration=1; asfasdfasdfadfa'; +eval INSERT INTO t1 values ($timegen, 1); +SELECT COUNT(*) FROM t1; + +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; + +SELECT COUNT(*) FROM t1; + +ALTER TABLE t1 COMMENT = 'adsf;;ttl_duration=5;asfasdfa;ttl_col=a;asdfasdf;'; + +set global rocksdb_debug_ttl_rec_ts = 300; +eval INSERT INTO t1 values ($timegen, 2); +set global rocksdb_debug_ttl_rec_ts = 0; +set global rocksdb_force_flush_memtable_now=1; + +# nothing removed here +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1; + +# all removed here +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +SELECT COUNT(*) FROM t1; + +DROP TABLE t1; + +# Test to make sure that TTL retains original timestamp during update +CREATE TABLE t1 ( + a bigint(20) NOT NULL, + PRIMARY KEY (a) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;'; + +set global rocksdb_debug_ttl_rec_ts = -300; +INSERT INTO t1 values (1); +INSERT INTO t1 values (3); +INSERT INTO t1 values (5); +set global rocksdb_debug_ttl_rec_ts = 300; +INSERT INTO t1 values (7); +INSERT INTO t1 values (9); +set global rocksdb_debug_ttl_rec_ts = 0; + +UPDATE t1 SET a=a+1; +--sorted_result +SELECT * FROM t1; + +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; + +# 1,3,5 should be dropped +--sorted_result +SELECT * FROM t1; +DROP TABLE t1; + +# test behaviour on update with TTL column, TTL time can be updated here. +eval CREATE TABLE t1 ( + a INT, + b $coltype NOT NULL, + PRIMARY KEY (a) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;ttl_col=b;'; + +set global rocksdb_debug_ttl_rec_ts = -300; +eval INSERT INTO t1 values (1, $timegen); +eval INSERT INTO t1 values (3, $timegen); +eval INSERT INTO t1 values (5, $timegen); +eval INSERT INTO t1 values (7, $timegen); + +set global rocksdb_debug_ttl_rec_ts = 300; +eval UPDATE t1 SET b=$timegen WHERE a < 4; +set global rocksdb_debug_ttl_rec_ts = 0; + +--sorted_result +SELECT a FROM t1; + +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; + +# 5 and 7 should be gone here +--sorted_result +SELECT a FROM t1; +DROP TABLE t1; + +# Test rows expired stat variable and disable ttl variable +CREATE TABLE t1 ( + a bigint(20) NOT NULL, + PRIMARY KEY (a) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;'; + +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1); +INSERT INTO t1 values (2); +INSERT INTO t1 values (3); +set global rocksdb_debug_ttl_rec_ts = 0; + +set global rocksdb_enable_ttl=0; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; + +select variable_value into @c from information_schema.global_status where variable_name='rocksdb_rows_expired'; +set global rocksdb_enable_ttl=1; +set global rocksdb_compact_cf='default'; + +select variable_value-@c from information_schema.global_status where variable_name='rocksdb_rows_expired'; +SELECT COUNT(*) FROM t1; +DROP TABLE t1; + + +# Table with TTL won't increment rows expired when no records have been +# compacted +CREATE TABLE t1 ( + a bigint(20) NOT NULL, + PRIMARY KEY (a) +) ENGINE=rocksdb +COMMENT='ttl_duration=100;'; + +INSERT INTO t1 values (1); +INSERT INTO t1 values (2); +INSERT INTO t1 values (3); + +select variable_value into @c from information_schema.global_status where variable_name='rocksdb_rows_expired'; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +select variable_value-@c from information_schema.global_status where variable_name='rocksdb_rows_expired'; + +DROP TABLE t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/ttl_primary.test b/storage/rocksdb/mysql-test/rocksdb/t/ttl_primary.test index 38bfb2eef8f..a8c9db5c188 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/ttl_primary.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/ttl_primary.test @@ -1,545 +1,18 @@ --source include/have_debug.inc --source include/have_rocksdb.inc -# Basic TTL test -CREATE TABLE t1 ( -`a` binary(8) NOT NULL, -`b` varbinary(64) NOT NULL, -`c` varbinary(256) NOT NULL, -`ts` bigint(20) UNSIGNED NOT NULL, -`value` mediumblob NOT NULL, -PRIMARY KEY (`b`,`a`,`c`) -) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 -COMMENT='ttl_duration=1;ttl_col=ts;'; +--let $coltype=bigint(20) UNSIGNED +--let $timegen=UNIX_TIMESTAMP() +--source ttl_primary.inc -set global rocksdb_debug_ttl_rec_ts = -100; -INSERT INTO t1 values ('a', 'b', 'c', UNIX_TIMESTAMP(), 'd'); -INSERT INTO t1 values ('d', 'e', 'f', UNIX_TIMESTAMP(), 'g'); -set global rocksdb_debug_ttl_rec_ts = 0; -SELECT COUNT(*) FROM t1; -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; +--let $coltype=TIMESTAMP +--let $timegen=NOW() +--source ttl_primary.inc -# should have filtered the rows out since ttl is passed in compaction filter -SELECT COUNT(*) FROM t1; -DROP TABLE t1; +--let $coltype=TIMESTAMP DEFAULT NOW() +--let $timegen=NOW() +--source ttl_primary.inc -# column before TTL in value -CREATE TABLE t1 ( - a bigint(20) NOT NULL, - b int NOT NULL, - ts bigint(20) UNSIGNED NOT NULL, - c int NOT NULL, - PRIMARY KEY (a) -) ENGINE=rocksdb -COMMENT='ttl_duration=1;ttl_col=ts;'; - -set global rocksdb_debug_ttl_rec_ts = -100; -INSERT INTO t1 values (1, 3, UNIX_TIMESTAMP(), 5); -INSERT INTO t1 values (2, 4, UNIX_TIMESTAMP(), 6); -set global rocksdb_debug_ttl_rec_ts = 0; -SELECT COUNT(*) FROM t1; - -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; - -# should have filtered the rows out since ttl is passed in compaction filter -SELECT COUNT(*) FROM t1; -DROP TABLE t1; - -# multi-part PK w/ TTL -CREATE TABLE t1 ( - a bigint(20) NOT NULL, - b int NOT NULL, - c int NOT NULL, - ts bigint(20) UNSIGNED NOT NULL, - PRIMARY KEY (a,c) -) ENGINE=rocksdb -COMMENT='ttl_duration=1;ttl_col=ts;'; - -set global rocksdb_debug_ttl_rec_ts = -100; -INSERT INTO t1 values (1, 3, 5, UNIX_TIMESTAMP()); -INSERT INTO t1 values (2, 4, 6, UNIX_TIMESTAMP()); -set global rocksdb_debug_ttl_rec_ts = 0; -SELECT COUNT(*) FROM t1; - -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; - -# should have filtered the rows out since ttl is passed in compaction filter -SELECT COUNT(*) FROM t1; -DROP TABLE t1; - -# multi-part PK w/ TTL -CREATE TABLE t1 ( - a bigint(20) NOT NULL, - b int NOT NULL, - c int NOT NULL, - ts bigint(20) UNSIGNED NOT NULL, - PRIMARY KEY (a,c) -) ENGINE=rocksdb -COMMENT='ttl_duration=1;ttl_col=ts;'; - -set global rocksdb_debug_ttl_rec_ts = -100; -INSERT INTO t1 values (1, 3, 5, UNIX_TIMESTAMP()); -INSERT INTO t1 values (2, 4, 6, UNIX_TIMESTAMP()); -set global rocksdb_debug_ttl_rec_ts = 0; -SELECT COUNT(*) FROM t1; - -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; - -# should have filtered the rows out since ttl is passed in compaction filter -SELECT COUNT(*) FROM t1; -DROP TABLE t1; - -# nullable column(s) before TTL -CREATE TABLE t1 ( - a bigint(20) NOT NULL, - b int, - c int, - ts bigint(20) UNSIGNED NOT NULL, - PRIMARY KEY (a) -) ENGINE=rocksdb -COMMENT='ttl_duration=1;ttl_col=ts;'; - -set global rocksdb_debug_ttl_rec_ts = -100; -INSERT INTO t1 values (1, NULL, NULL, UNIX_TIMESTAMP()); -INSERT INTO t1 values (2, NULL, NULL, UNIX_TIMESTAMP()); -set global rocksdb_debug_ttl_rec_ts = 0; -SELECT COUNT(*) FROM t1; - -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; - -# should have filtered the rows out since ttl is passed in compaction filter -SELECT COUNT(*) FROM t1; -DROP TABLE t1; - -# variable len columns + null column(s) before TTL -CREATE TABLE t1 ( -`a` binary(8) NOT NULL, -`b` varbinary(64), -`c` varbinary(256), -`ts` bigint(20) UNSIGNED NOT NULL, -`value` mediumblob NOT NULL, -PRIMARY KEY (`a`) -) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 -COMMENT='ttl_duration=1;ttl_col=ts;'; - -set global rocksdb_debug_ttl_rec_ts = -100; -INSERT INTO t1 values ('a', NULL, 'bc', UNIX_TIMESTAMP(), 'd'); -INSERT INTO t1 values ('d', 'efghijk', NULL, UNIX_TIMESTAMP(), 'l'); -set global rocksdb_debug_ttl_rec_ts = 0; -SELECT COUNT(*) FROM t1; - -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; - -# should have filtered the rows out since ttl is passed in compaction filter -SELECT COUNT(*) FROM t1; -DROP TABLE t1; - -# TTL implicitly generated (no ttl column) -CREATE TABLE t1 ( - a bigint(20) NOT NULL, - b int NOT NULL, - c int NOT NULL, - PRIMARY KEY (a) -) ENGINE=rocksdb -COMMENT='ttl_duration=1;'; - -set global rocksdb_debug_ttl_rec_ts = -100; -INSERT INTO t1 values (1, 3, 5); -INSERT INTO t1 values (2, 4, 6); -set global rocksdb_debug_ttl_rec_ts = 0; -SELECT COUNT(*) FROM t1; - -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; - -# should have filtered the rows out since ttl is passed in compaction filter -SELECT COUNT(*) FROM t1; -DROP TABLE t1; - -# TTL field as the PK -CREATE TABLE t1 ( - a int, - ts bigint(20) UNSIGNED NOT NULL, - PRIMARY KEY (a, ts) -) ENGINE=rocksdb -COMMENT='ttl_duration=5;ttl_col=ts;'; - -INSERT INTO t1 values (1, UNIX_TIMESTAMP()); -INSERT INTO t1 values (2, UNIX_TIMESTAMP()); -SELECT COUNT(*) FROM t1; - -set global rocksdb_debug_ttl_snapshot_ts = -10; -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; -set global rocksdb_debug_ttl_snapshot_ts = 0; -# should all still be there.. -SELECT COUNT(*) FROM t1; - -set global rocksdb_debug_ttl_snapshot_ts = 10; -set global rocksdb_compact_cf='default'; -set global rocksdb_debug_ttl_snapshot_ts = 0; -# should have filtered the rows out since ttl is passed in compaction filter -SELECT COUNT(*) FROM t1; -DROP TABLE t1; - - -# TTL field inside multi-part pk -CREATE TABLE t1 ( - a bigint(20) NOT NULL, - b int NOT NULL, - ts bigint(20) UNSIGNED NOT NULL, - c int NOT NULL, - PRIMARY KEY (a, ts) -) ENGINE=rocksdb -COMMENT='ttl_duration=1;ttl_col=ts;'; - -set global rocksdb_debug_ttl_rec_ts = -100; -INSERT INTO t1 values (1, 3, UNIX_TIMESTAMP(), 5); -INSERT INTO t1 values (2, 4, UNIX_TIMESTAMP(), 6); -set global rocksdb_debug_ttl_rec_ts = 0; -SELECT COUNT(*) FROM t1; - -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; - -# should have filtered the rows out since ttl is passed in compaction filter -SELECT COUNT(*) FROM t1; -DROP TABLE t1; - -# TTL field inside key with variable length things.. -CREATE TABLE t1 ( -`a` binary(8) NOT NULL, -`b` varbinary(64), -`c` varbinary(256), -`ts` bigint(20) UNSIGNED NOT NULL, -`value` mediumblob NOT NULL, -PRIMARY KEY (`a`, `ts`) -) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 -COMMENT='ttl_duration=1;ttl_col=ts;'; - -set global rocksdb_debug_ttl_rec_ts = -100; -INSERT INTO t1 values ('a', NULL, 'bc', UNIX_TIMESTAMP(), 'd'); -INSERT INTO t1 values ('de', 'fghijk', NULL, UNIX_TIMESTAMP(), 'l'); -set global rocksdb_debug_ttl_rec_ts = 0; -SELECT COUNT(*) FROM t1; - -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; - -# should have filtered the rows out since ttl is passed in compaction filter -SELECT COUNT(*) FROM t1; -DROP TABLE t1; - -# TTL test where you compact (values still exist), real_sleep, then compact again, -# values should now be gone. -CREATE TABLE t1 ( -a INT NOT NULL, -b varbinary(64) NOT NULL, -c varbinary(256) NOT NULL, -ts bigint(20) UNSIGNED NOT NULL, -value mediumblob NOT NULL, -PRIMARY KEY (b,a,c) -) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 -COMMENT='ttl_duration=10;ttl_col=ts;'; - -set global rocksdb_debug_ttl_rec_ts = -300; -INSERT INTO t1 values (1, 'b', 'c', UNIX_TIMESTAMP(), 'd'); -INSERT INTO t1 values (2, 'e', 'f', UNIX_TIMESTAMP(), 'g'); -set global rocksdb_debug_ttl_rec_ts = 300; -INSERT INTO t1 values (3, 'i', 'j', UNIX_TIMESTAMP(), 'k'); -INSERT INTO t1 values (4, 'm', 'n', UNIX_TIMESTAMP(), 'o'); -set global rocksdb_debug_ttl_rec_ts = 0; - -# Nothing should get removed here. -set global rocksdb_debug_ttl_snapshot_ts = -3600; -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; -set global rocksdb_debug_ttl_snapshot_ts = 0; ---sorted_result -SELECT a FROM t1; - -# 1 and 2 should get removed here. -set global rocksdb_compact_cf='default'; ---sorted_result -SELECT a FROM t1; - -# 3 and 4 should get removed here. -set global rocksdb_debug_ttl_snapshot_ts = 3600; -set global rocksdb_compact_cf='default'; -set global rocksdb_debug_ttl_snapshot_ts = 0; ---sorted_result -SELECT a FROM t1; - -DROP TABLE t1; - -# TTL field with nullable ttl column (should fail) ---error ER_RDB_TTL_COL_FORMAT -CREATE TABLE t1 ( - a bigint(20) UNSIGNED NOT NULL, - b int NOT NULL, - c int NOT NULL, - ts bigint(20), - PRIMARY KEY (a,c) -) ENGINE=rocksdb -COMMENT='ttl_duration=1;ttl_col=ts;'; - -# TTL field with non 8-bit integer column (should fail) ---error ER_RDB_TTL_COL_FORMAT -CREATE TABLE t1 ( - a bigint(20) UNSIGNED NOT NULL, - b int NOT NULL, - c int NOT NULL, - ts int, - PRIMARY KEY (a,c) -) ENGINE=rocksdb -COMMENT='ttl_duration=1;ttl_col=ts;'; - -# TTL duration as some random garbage value ---error ER_RDB_TTL_DURATION_FORMAT -CREATE TABLE t1 ( - a bigint(20) UNSIGNED NOT NULL, - b int NOT NULL, - c int NOT NULL, - PRIMARY KEY (a,c) -) ENGINE=rocksdb -COMMENT='ttl_duration=abc;'; - -# TTL col is some column outside of the table ---error ER_RDB_TTL_COL_FORMAT -CREATE TABLE t1 ( - a bigint(20) UNSIGNED NOT NULL, - b int NOT NULL, - c int NOT NULL, - PRIMARY KEY (a,c) -) ENGINE=rocksdb -COMMENT='ttl_duration=1;ttl_col=abc;'; - -# TTL col must have accompanying duration ---error ER_RDB_TTL_COL_FORMAT -CREATE TABLE t1 ( - a bigint(20) UNSIGNED NOT NULL, - b int NOT NULL, - c int NOT NULL, - PRIMARY KEY (a,c) -) ENGINE=rocksdb -COMMENT='ttl_col=abc;'; - -# Make sure it doesn't filter out things early -CREATE TABLE t1 ( - a bigint(20) NOT NULL, - PRIMARY KEY (a) -) ENGINE=rocksdb -COMMENT='ttl_duration=500;'; - -INSERT INTO t1 values (1); -SELECT COUNT(*) FROM t1; -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; - -SELECT COUNT(*) FROM t1; -DROP TABLE t1; - -# Testing altering table comment with updated TTL duration -# This should trigger a rebuild of the table -CREATE TABLE t1 ( - a INT PRIMARY KEY -) ENGINE=rocksdb -COMMENT='ttl_duration=100;'; - -INSERT INTO t1 values (1); -SELECT * FROM t1; - -set global rocksdb_debug_ttl_rec_ts = -300; -ALTER TABLE t1 COMMENT = 'ttl_duration=1'; -set global rocksdb_debug_ttl_rec_ts = 0; -SHOW CREATE TABLE t1; -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; - -SELECT COUNT(*) FROM t1; -DROP TABLE t1; - -# Tables with hidden PK and SK disabled -CREATE TABLE t1 ( - a INT PRIMARY KEY, - b INT -) ENGINE=rocksdb -COMMENT='ttl_duration=100;'; - ---error ER_RDB_TTL_UNSUPPORTED -ALTER TABLE t1 DROP PRIMARY KEY; - -DROP TABLE t1; - -# Test replacing PK, ttl should still work after -CREATE TABLE t1 ( - a INT PRIMARY KEY, - b INT -) ENGINE=rocksdb -COMMENT='ttl_duration=5;'; - -INSERT INTO t1 VALUES (1,1); -INSERT INTO t1 VALUES (2,2); - -ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY(b); -set global rocksdb_debug_ttl_snapshot_ts = -3600; -set global rocksdb_force_flush_memtable_now=1; -set @@global.rocksdb_compact_cf = 'default'; -set global rocksdb_debug_ttl_snapshot_ts = 0; - ---sorted_result -SELECT COUNT(*) FROM t1; - -set global rocksdb_debug_ttl_snapshot_ts = 3600; -set @@global.rocksdb_compact_cf = 'default'; -set global rocksdb_debug_ttl_snapshot_ts = 0; - ---sorted_result -SELECT COUNT(*) FROM t1; - -DROP TABLE t1; - -# Make sure table comment filled with other text before/after will work -# (basically, it needs semicolon before and after) -CREATE TABLE t1 ( - a bigint(20) UNSIGNED NOT NULL, - b int, - PRIMARY KEY (a,b) -) ENGINE=rocksdb -COMMENT='asdadfasdfsadfadf ;ttl_duration=1; asfasdfasdfadfa'; -INSERT INTO t1 values (UNIX_TIMESTAMP(), 1); -SELECT COUNT(*) FROM t1; - -set global rocksdb_debug_ttl_snapshot_ts = 3600; -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; -set global rocksdb_debug_ttl_snapshot_ts = 0; - -SELECT COUNT(*) FROM t1; - -ALTER TABLE t1 COMMENT = 'adsf;;ttl_duration=5;asfasdfa;ttl_col=a;asdfasdf;'; - -set global rocksdb_debug_ttl_rec_ts = 300; -INSERT INTO t1 values (UNIX_TIMESTAMP(), 2); -set global rocksdb_debug_ttl_rec_ts = 0; -set global rocksdb_force_flush_memtable_now=1; - -# nothing removed here -set global rocksdb_compact_cf='default'; -SELECT COUNT(*) FROM t1; - -# all removed here -set global rocksdb_debug_ttl_snapshot_ts = 3600; -set global rocksdb_compact_cf='default'; -set global rocksdb_debug_ttl_snapshot_ts = 0; -SELECT COUNT(*) FROM t1; - -DROP TABLE t1; - -# Test to make sure that TTL retains original timestamp during update -CREATE TABLE t1 ( - a bigint(20) NOT NULL, - PRIMARY KEY (a) -) ENGINE=rocksdb -COMMENT='ttl_duration=5;'; - -set global rocksdb_debug_ttl_rec_ts = -300; -INSERT INTO t1 values (1); -INSERT INTO t1 values (3); -INSERT INTO t1 values (5); -set global rocksdb_debug_ttl_rec_ts = 300; -INSERT INTO t1 values (7); -INSERT INTO t1 values (9); -set global rocksdb_debug_ttl_rec_ts = 0; - -UPDATE t1 SET a=a+1; ---sorted_result -SELECT * FROM t1; - -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; - -# 1,3,5 should be dropped ---sorted_result -SELECT * FROM t1; -DROP TABLE t1; - -# test behaviour on update with TTL column, TTL time can be updated here. -CREATE TABLE t1 ( - a INT, - b bigint(20) UNSIGNED NOT NULL, - PRIMARY KEY (a) -) ENGINE=rocksdb -COMMENT='ttl_duration=5;ttl_col=b;'; - -set global rocksdb_debug_ttl_rec_ts = -300; -INSERT INTO t1 values (1, UNIX_TIMESTAMP()); -INSERT INTO t1 values (3, UNIX_TIMESTAMP()); -INSERT INTO t1 values (5, UNIX_TIMESTAMP()); -INSERT INTO t1 values (7, UNIX_TIMESTAMP()); - -set global rocksdb_debug_ttl_rec_ts = 300; -UPDATE t1 SET b=UNIX_TIMESTAMP() WHERE a < 4; -set global rocksdb_debug_ttl_rec_ts = 0; - ---sorted_result -SELECT a FROM t1; - -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; - -# 5 and 7 should be gone here ---sorted_result -SELECT a FROM t1; -DROP TABLE t1; - -# Test rows expired stat variable and disable ttl variable -CREATE TABLE t1 ( - a bigint(20) NOT NULL, - PRIMARY KEY (a) -) ENGINE=rocksdb -COMMENT='ttl_duration=1;'; - -set global rocksdb_debug_ttl_rec_ts = -100; -INSERT INTO t1 values (1); -INSERT INTO t1 values (2); -INSERT INTO t1 values (3); -set global rocksdb_debug_ttl_rec_ts = 0; - -set global rocksdb_enable_ttl=0; -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; - -select variable_value into @c from information_schema.global_status where variable_name='rocksdb_rows_expired'; -set global rocksdb_enable_ttl=1; -set global rocksdb_compact_cf='default'; - -select variable_value-@c from information_schema.global_status where variable_name='rocksdb_rows_expired'; -SELECT COUNT(*) FROM t1; -DROP TABLE t1; - - -# Table with TTL won't increment rows expired when no records have been -# compacted -CREATE TABLE t1 ( - a bigint(20) NOT NULL, - PRIMARY KEY (a) -) ENGINE=rocksdb -COMMENT='ttl_duration=100;'; - -INSERT INTO t1 values (1); -INSERT INTO t1 values (2); -INSERT INTO t1 values (3); - -select variable_value into @c from information_schema.global_status where variable_name='rocksdb_rows_expired'; -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; -select variable_value-@c from information_schema.global_status where variable_name='rocksdb_rows_expired'; - -DROP TABLE t1; +--let $coltype=TIMESTAMP(6) +--let $timegen=NOW() +--source ttl_primary.inc diff --git a/storage/rocksdb/mysql-test/rocksdb/t/ttl_primary_with_partitions.inc b/storage/rocksdb/mysql-test/rocksdb/t/ttl_primary_with_partitions.inc new file mode 100644 index 00000000000..9ebcb1092e5 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/ttl_primary_with_partitions.inc @@ -0,0 +1,255 @@ +# +# Specify the timestamp column type using $coltype +# and the timestamp generator using $timegen +# + +# +# Create a table with multiple partitions, but in the comment don't specify +# that per-partition based column families (CF) should be created. Expect that +# default CF will be used and new one won't be created. +# +# In addition, specify TTL on one of the partitions. Insert a few things +# inside all the partitions, verify after compaction that the rows inside the +# partition with TTL has disappeared. +# +CREATE TABLE t1 ( + c1 INT, + PRIMARY KEY (`c1`) +) ENGINE=ROCKSDB +COMMENT="custom_p0_ttl_duration=1;" +PARTITION BY LIST(c1) ( + PARTITION custom_p0 VALUES IN (1, 4, 7), + PARTITION custom_p1 VALUES IN (2, 5, 8), + PARTITION custom_p2 VALUES IN (3, 6, 9) +); + +set global rocksdb_debug_ttl_rec_ts = -3600; +INSERT INTO t1 values (1); +INSERT INTO t1 values (2); +INSERT INTO t1 values (3); +INSERT INTO t1 values (4); +INSERT INTO t1 values (5); +INSERT INTO t1 values (6); +INSERT INTO t1 values (7); +INSERT INTO t1 values (8); +INSERT INTO t1 values (9); +set global rocksdb_debug_ttl_rec_ts = 0; + +--sorted_result +SELECT * FROM t1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; + +# 1,4, and 7 should be gone +--sorted_result +SELECT * FROM t1; +DROP TABLE t1; + +# +# Create a table with multiple partitions and request for separate CF to be +# created per every partition. As a result we expect three different CF-s to be +# created. +# +# In addition, specify TTL on some of the partitions. Insert a few things +# inside all the partitions, verify after compaction that the rows inside the +# partition with TTL has disappeared. +# +CREATE TABLE t1 ( + c1 INT, + c2 INT, + name VARCHAR(25) NOT NULL, + PRIMARY KEY (`c1`, `c2`) COMMENT 'custom_p0_cfname=foo;custom_p1_cfname=my_custom_cf;custom_p2_cfname=baz' +) ENGINE=ROCKSDB +COMMENT="custom_p0_ttl_duration=1;custom_p1_ttl_duration=7;" +PARTITION BY LIST(c1) ( + PARTITION custom_p0 VALUES IN (1, 4, 7), + PARTITION custom_p1 VALUES IN (2, 5, 8), + PARTITION custom_p2 VALUES IN (3, 6, 9) +); + +set global rocksdb_debug_ttl_rec_ts = -1200; +INSERT INTO t1 values (1,1,'a'); +INSERT INTO t1 values (4,4,'aaaa'); +INSERT INTO t1 values (7,7,'aaaaaaa'); + +set global rocksdb_debug_ttl_rec_ts = 1200; +INSERT INTO t1 values (2,2,'aa'); +INSERT INTO t1 values (3,3,'aaa'); +INSERT INTO t1 values (5,5,'aaaaa'); +INSERT INTO t1 values (6,6,'aaaaaa'); +INSERT INTO t1 values (8,8,'aaaaaaaa'); +INSERT INTO t1 values (9,9,'aaaaaaaaa'); +set global rocksdb_debug_ttl_rec_ts = 0; + +--sorted_result +SELECT * FROM t1; + +set global rocksdb_force_flush_memtable_now=1; +set @@global.rocksdb_compact_cf = 'foo'; +set @@global.rocksdb_compact_cf = 'my_custom_cf'; +--sorted_result +SELECT * FROM t1; + +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set @@global.rocksdb_compact_cf = 'foo'; +--sorted_result +SELECT * FROM t1; + +# Now 2,5,8 should be removed (this verifies that TTL is only operating on the +# particular CF. +set @@global.rocksdb_compact_cf = 'my_custom_cf'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +--sorted_result +SELECT * FROM t1; + +DROP TABLE t1; + +# +# Create a table with CF-s/TTL per partition and verify that ALTER TABLE + DROP +# PRIMARY, ADD PRIMARY work for that scenario and data is persisted/filtered as +# expected. +# +CREATE TABLE t1 ( + c1 INT, + c2 INT, + name VARCHAR(25) NOT NULL, + event DATE, + PRIMARY KEY (`c1`, `c2`) COMMENT 'custom_p0_cfname=foo;custom_p1_cfname=bar;custom_p2_cfname=baz;' +) ENGINE=ROCKSDB +COMMENT="custom_p0_ttl_duration=9999;custom_p2_ttl_duration=5;" +PARTITION BY LIST(c1) ( + PARTITION custom_p0 VALUES IN (1, 2, 3), + PARTITION custom_p1 VALUES IN (4, 5, 6), + PARTITION custom_p2 VALUES IN (7, 8, 9) +); + +INSERT INTO t1 VALUES (1, 1, "one", null); +INSERT INTO t1 VALUES (2, 2, "two", null); +INSERT INTO t1 VALUES (3, 3, "three", null); + +INSERT INTO t1 VALUES (4, 4, "four", null); +INSERT INTO t1 VALUES (5, 5, "five", null); +INSERT INTO t1 VALUES (6, 6, "six", null); + +INSERT INTO t1 VALUES (7, 7, "seven", null); +INSERT INTO t1 VALUES (8, 8, "eight", null); +INSERT INTO t1 VALUES (9, 9, "nine", null); + +--sorted_result +SELECT * FROM t1; + +# TTL should be reset after alter table +set global rocksdb_debug_ttl_rec_ts = 600; +ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY(`c2`,`c1`) COMMENT 'custom_p0_cfname=foo;custom_p1_cfname=bar;custom_p2_cfname=baz;'; +set global rocksdb_debug_ttl_rec_ts = 0; +SHOW CREATE TABLE t1; + +# ...so nothing should be gone here +set global rocksdb_debug_ttl_snapshot_ts = 100; +set global rocksdb_force_flush_memtable_now=1; +set @@global.rocksdb_compact_cf = 'baz'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +--sorted_result +SELECT * FROM t1; + +set global rocksdb_debug_ttl_snapshot_ts = 1200; +set @@global.rocksdb_compact_cf = 'foo'; +set @@global.rocksdb_compact_cf = 'baz'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +--sorted_result +SELECT * FROM t1; + +DROP TABLE t1; + +# +# Create a table with non-partitioned TTL duration, with partitioned TTL +# columns +# +# In this case the same TTL duration will be applied across different TTL +# columns in different partitions, except for in p2 where we override the ttl +# duration. +# +eval CREATE TABLE t1 ( + c1 BIGINT, + c2 $coltype NOT NULL, + name VARCHAR(25) NOT NULL, + event DATE, + PRIMARY KEY (`c1`) COMMENT 'custom_p0_cfname=foo;custom_p1_cfname=bar;custom_p2_cfname=baz;' +) ENGINE=ROCKSDB +COMMENT="ttl_duration=1;custom_p1_ttl_duration=100;custom_p1_ttl_col=c2;custom_p2_ttl_duration=5000;" +PARTITION BY LIST(c1) ( + PARTITION custom_p0 VALUES IN (1, 2, 3), + PARTITION custom_p1 VALUES IN (4, 5, 6), + PARTITION custom_p2 VALUES IN (7, 8, 9) +); + +set global rocksdb_debug_ttl_rec_ts = -300; +eval INSERT INTO t1 VALUES (1, $timegen, "one", null); +eval INSERT INTO t1 VALUES (2, $timegen, "two", null); +eval INSERT INTO t1 VALUES (3, $timegen, "three", null); +set global rocksdb_debug_ttl_rec_ts = 0; + +eval INSERT INTO t1 VALUES (4, $timegen, "four", null); +eval INSERT INTO t1 VALUES (5, $timegen, "five", null); +eval INSERT INTO t1 VALUES (6, $timegen, "six", null); + +eval INSERT INTO t1 VALUES (7, $timegen, "seven", null); +eval INSERT INTO t1 VALUES (8, $timegen, "eight", null); +eval INSERT INTO t1 VALUES (9, $timegen, "nine", null); + +set global rocksdb_force_flush_memtable_now=1; +set @@global.rocksdb_compact_cf = 'foo'; +set @@global.rocksdb_compact_cf = 'baz'; +set @@global.rocksdb_compact_cf = 'bar'; + +# here we expect only 1,2,3 to be gone, ttl implicit. +--sorted_result +SELECT c1 FROM t1; + +# here we expect only 4,5,6 to be gone, ttl based on column c2. +set global rocksdb_debug_ttl_snapshot_ts = 600; +set @@global.rocksdb_compact_cf = 'bar'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +--sorted_result +SELECT c1 FROM t1; + +# at this point only 7,8,9 should be left.. +DROP TABLE t1; + +# +# Make sure non-partitioned TTL duration/col still works on table with +# partitions. +# +# Simultaneously tests when TTL col is part of the key in partitioned table +# +eval CREATE TABLE t1 ( + c1 BIGINT, + c2 $coltype NOT NULL, + PRIMARY KEY (`c1`, `c2`) +) ENGINE=ROCKSDB +COMMENT="ttl_duration=100;ttl_col=c2;" +PARTITION BY LIST(c1) ( + PARTITION custom_p0 VALUES IN (1), + PARTITION custom_p1 VALUES IN (2), + PARTITION custom_p2 VALUES IN (3) +); + +eval INSERT INTO t1 values (1, $timegen); +eval INSERT INTO t1 values (2, $timegen); +eval INSERT INTO t1 values (3, $timegen); + +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; + +# everything should still be here +--sorted_result +SELECT c1 FROM t1; + +set global rocksdb_debug_ttl_snapshot_ts = 300; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +# everything should now be gone +--sorted_result +SELECT c1 FROM t1; + +DROP TABLE t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/ttl_primary_with_partitions.test b/storage/rocksdb/mysql-test/rocksdb/t/ttl_primary_with_partitions.test index aba2b594db5..7b693dc6081 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/ttl_primary_with_partitions.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/ttl_primary_with_partitions.test @@ -2,253 +2,18 @@ --source include/have_rocksdb.inc --source include/have_partition.inc -# -# Create a table with multiple partitions, but in the comment don't specify -# that per-partition based column families (CF) should be created. Expect that -# default CF will be used and new one won't be created. -# -# In addition, specify TTL on one of the partitions. Insert a few things -# inside all the partitions, verify after compaction that the rows inside the -# partition with TTL has disappeared. -# -CREATE TABLE t1 ( - c1 INT, - PRIMARY KEY (`c1`) -) ENGINE=ROCKSDB -COMMENT="custom_p0_ttl_duration=1;" -PARTITION BY LIST(c1) ( - PARTITION custom_p0 VALUES IN (1, 4, 7), - PARTITION custom_p1 VALUES IN (2, 5, 8), - PARTITION custom_p2 VALUES IN (3, 6, 9) -); +--let $coltype=BIGINT UNSIGNED +--let $timegen=UNIX_TIMESTAMP() +--source ttl_primary_with_partitions.inc -set global rocksdb_debug_ttl_rec_ts = -3600; -INSERT INTO t1 values (1); -INSERT INTO t1 values (2); -INSERT INTO t1 values (3); -INSERT INTO t1 values (4); -INSERT INTO t1 values (5); -INSERT INTO t1 values (6); -INSERT INTO t1 values (7); -INSERT INTO t1 values (8); -INSERT INTO t1 values (9); -set global rocksdb_debug_ttl_rec_ts = 0; +--let $coltype=TIMESTAMP +--let $timegen=NOW() +--source ttl_primary_with_partitions.inc ---sorted_result -SELECT * FROM t1; -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; +--let $coltype=TIMESTAMP DEFAULT NOW() +--let $timegen=NOW() +--source ttl_primary_with_partitions.inc -# 1,4, and 7 should be gone ---sorted_result -SELECT * FROM t1; -DROP TABLE t1; - -# -# Create a table with multiple partitions and request for separate CF to be -# created per every partition. As a result we expect three different CF-s to be -# created. -# -# In addition, specify TTL on some of the partitions. Insert a few things -# inside all the partitions, verify after compaction that the rows inside the -# partition with TTL has disappeared. -# -CREATE TABLE t1 ( - c1 INT, - c2 INT, - name VARCHAR(25) NOT NULL, - PRIMARY KEY (`c1`, `c2`) COMMENT 'custom_p0_cfname=foo;custom_p1_cfname=my_custom_cf;custom_p2_cfname=baz' -) ENGINE=ROCKSDB -COMMENT="custom_p0_ttl_duration=1;custom_p1_ttl_duration=7;" -PARTITION BY LIST(c1) ( - PARTITION custom_p0 VALUES IN (1, 4, 7), - PARTITION custom_p1 VALUES IN (2, 5, 8), - PARTITION custom_p2 VALUES IN (3, 6, 9) -); - -set global rocksdb_debug_ttl_rec_ts = -1200; -INSERT INTO t1 values (1,1,'a'); -INSERT INTO t1 values (4,4,'aaaa'); -INSERT INTO t1 values (7,7,'aaaaaaa'); - -set global rocksdb_debug_ttl_rec_ts = 1200; -INSERT INTO t1 values (2,2,'aa'); -INSERT INTO t1 values (3,3,'aaa'); -INSERT INTO t1 values (5,5,'aaaaa'); -INSERT INTO t1 values (6,6,'aaaaaa'); -INSERT INTO t1 values (8,8,'aaaaaaaa'); -INSERT INTO t1 values (9,9,'aaaaaaaaa'); -set global rocksdb_debug_ttl_rec_ts = 0; - ---sorted_result -SELECT * FROM t1; - -set global rocksdb_force_flush_memtable_now=1; -set @@global.rocksdb_compact_cf = 'foo'; -set @@global.rocksdb_compact_cf = 'my_custom_cf'; ---sorted_result -SELECT * FROM t1; - -set global rocksdb_debug_ttl_snapshot_ts = 3600; -set @@global.rocksdb_compact_cf = 'foo'; ---sorted_result -SELECT * FROM t1; - -# Now 2,5,8 should be removed (this verifies that TTL is only operating on the -# particular CF. -set @@global.rocksdb_compact_cf = 'my_custom_cf'; -set global rocksdb_debug_ttl_snapshot_ts = 0; ---sorted_result -SELECT * FROM t1; - -DROP TABLE t1; - -# -# Create a table with CF-s/TTL per partition and verify that ALTER TABLE + DROP -# PRIMARY, ADD PRIMARY work for that scenario and data is persisted/filtered as -# expected. -# -CREATE TABLE t1 ( - c1 INT, - c2 INT, - name VARCHAR(25) NOT NULL, - event DATE, - PRIMARY KEY (`c1`, `c2`) COMMENT 'custom_p0_cfname=foo;custom_p1_cfname=bar;custom_p2_cfname=baz;' -) ENGINE=ROCKSDB -COMMENT="custom_p0_ttl_duration=9999;custom_p2_ttl_duration=5;" -PARTITION BY LIST(c1) ( - PARTITION custom_p0 VALUES IN (1, 2, 3), - PARTITION custom_p1 VALUES IN (4, 5, 6), - PARTITION custom_p2 VALUES IN (7, 8, 9) -); - -INSERT INTO t1 VALUES (1, 1, "one", null); -INSERT INTO t1 VALUES (2, 2, "two", null); -INSERT INTO t1 VALUES (3, 3, "three", null); - -INSERT INTO t1 VALUES (4, 4, "four", null); -INSERT INTO t1 VALUES (5, 5, "five", null); -INSERT INTO t1 VALUES (6, 6, "six", null); - -INSERT INTO t1 VALUES (7, 7, "seven", null); -INSERT INTO t1 VALUES (8, 8, "eight", null); -INSERT INTO t1 VALUES (9, 9, "nine", null); - ---sorted_result -SELECT * FROM t1; - -# TTL should be reset after alter table -set global rocksdb_debug_ttl_rec_ts = 600; -ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY(`c2`,`c1`) COMMENT 'custom_p0_cfname=foo;custom_p1_cfname=bar;custom_p2_cfname=baz;'; -set global rocksdb_debug_ttl_rec_ts = 0; -SHOW CREATE TABLE t1; - -# ...so nothing should be gone here -set global rocksdb_debug_ttl_snapshot_ts = 100; -set global rocksdb_force_flush_memtable_now=1; -set @@global.rocksdb_compact_cf = 'baz'; -set global rocksdb_debug_ttl_snapshot_ts = 0; ---sorted_result -SELECT * FROM t1; - -set global rocksdb_debug_ttl_snapshot_ts = 1200; -set @@global.rocksdb_compact_cf = 'foo'; -set @@global.rocksdb_compact_cf = 'baz'; -set global rocksdb_debug_ttl_snapshot_ts = 0; ---sorted_result -SELECT * FROM t1; - -DROP TABLE t1; - -# -# Create a table with non-partitioned TTL duration, with partitioned TTL -# columns -# -# In this case the same TTL duration will be applied across different TTL -# columns in different partitions, except for in p2 where we override the ttl -# duration. -# -CREATE TABLE t1 ( - c1 BIGINT, - c2 BIGINT UNSIGNED NOT NULL, - name VARCHAR(25) NOT NULL, - event DATE, - PRIMARY KEY (`c1`) COMMENT 'custom_p0_cfname=foo;custom_p1_cfname=bar;custom_p2_cfname=baz;' -) ENGINE=ROCKSDB -COMMENT="ttl_duration=1;custom_p1_ttl_duration=100;custom_p1_ttl_col=c2;custom_p2_ttl_duration=5000;" -PARTITION BY LIST(c1) ( - PARTITION custom_p0 VALUES IN (1, 2, 3), - PARTITION custom_p1 VALUES IN (4, 5, 6), - PARTITION custom_p2 VALUES IN (7, 8, 9) -); - -set global rocksdb_debug_ttl_rec_ts = -300; -INSERT INTO t1 VALUES (1, UNIX_TIMESTAMP(), "one", null); -INSERT INTO t1 VALUES (2, UNIX_TIMESTAMP(), "two", null); -INSERT INTO t1 VALUES (3, UNIX_TIMESTAMP(), "three", null); -set global rocksdb_debug_ttl_rec_ts = 0; - -INSERT INTO t1 VALUES (4, UNIX_TIMESTAMP(), "four", null); -INSERT INTO t1 VALUES (5, UNIX_TIMESTAMP(), "five", null); -INSERT INTO t1 VALUES (6, UNIX_TIMESTAMP(), "six", null); - -INSERT INTO t1 VALUES (7, UNIX_TIMESTAMP(), "seven", null); -INSERT INTO t1 VALUES (8, UNIX_TIMESTAMP(), "eight", null); -INSERT INTO t1 VALUES (9, UNIX_TIMESTAMP(), "nine", null); - -set global rocksdb_force_flush_memtable_now=1; -set @@global.rocksdb_compact_cf = 'foo'; -set @@global.rocksdb_compact_cf = 'baz'; -set @@global.rocksdb_compact_cf = 'bar'; - -# here we expect only 1,2,3 to be gone, ttl implicit. ---sorted_result -SELECT c1 FROM t1; - -# here we expect only 4,5,6 to be gone, ttl based on column c2. -set global rocksdb_debug_ttl_snapshot_ts = 600; -set @@global.rocksdb_compact_cf = 'bar'; -set global rocksdb_debug_ttl_snapshot_ts = 0; ---sorted_result -SELECT c1 FROM t1; - -# at this point only 7,8,9 should be left.. -DROP TABLE t1; - -# -# Make sure non-partitioned TTL duration/col still works on table with -# partitions. -# -# Simultaneously tests when TTL col is part of the key in partitioned table -# -CREATE TABLE t1 ( - c1 BIGINT, - c2 BIGINT UNSIGNED NOT NULL, - PRIMARY KEY (`c1`, `c2`) -) ENGINE=ROCKSDB -COMMENT="ttl_duration=100;ttl_col=c2;" -PARTITION BY LIST(c1) ( - PARTITION custom_p0 VALUES IN (1), - PARTITION custom_p1 VALUES IN (2), - PARTITION custom_p2 VALUES IN (3) -); - -INSERT INTO t1 values (1, UNIX_TIMESTAMP()); -INSERT INTO t1 values (2, UNIX_TIMESTAMP()); -INSERT INTO t1 values (3, UNIX_TIMESTAMP()); - -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; - -# everything should still be here ---sorted_result -SELECT c1 FROM t1; - -set global rocksdb_debug_ttl_snapshot_ts = 300; -set global rocksdb_compact_cf='default'; -set global rocksdb_debug_ttl_snapshot_ts = 0; -# everything should now be gone ---sorted_result -SELECT c1 FROM t1; - -DROP TABLE t1; +--let $coltype=TIMESTAMP(6) +--let $timegen=NOW() +--source ttl_primary_with_partitions.inc diff --git a/storage/rocksdb/mysql-test/rocksdb/t/ttl_secondary.inc b/storage/rocksdb/mysql-test/rocksdb/t/ttl_secondary.inc new file mode 100644 index 00000000000..9fb10a08c3c --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/ttl_secondary.inc @@ -0,0 +1,783 @@ +# +# Specify the timestamp column type using $coltype, +# the timestamp generator using $timegen, +# and the timestamp generater + 1 sec using $timegen_1s +# + +# Basic TTL test, pk ignored, no sk +eval CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64) NOT NULL, +`c` varbinary(256) NOT NULL, +`ts` $coltype NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`b`,`a`,`c`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; + +set global rocksdb_debug_ttl_rec_ts = -100; +eval INSERT INTO t1 values ('a', 'b', 'c', $timegen, 'd'); +eval INSERT INTO t1 values ('d', 'e', 'f', $timegen, 'g'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; + +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; + +# no rows should be filtered +SELECT COUNT(*) FROM t1; +DROP TABLE t1; + +# Basic TTL test +eval CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64) NOT NULL, +`c` varbinary(256) NOT NULL, +`ts` $coltype NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`b`,`a`,`c`), +KEY kb (`b`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; + +set global rocksdb_debug_ttl_rec_ts = -100; +eval INSERT INTO t1 values ('a', 'b', 'c', $timegen, 'd'); +eval INSERT INTO t1 values ('d', 'e', 'f', $timegen, 'g'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); + +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; + +# should have filtered the rows out since ttl is passed in compaction filter +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +DROP TABLE t1; + +# column before TTL in value +eval CREATE TABLE t1 ( + a bigint(20) NOT NULL, + b int NOT NULL, + ts $coltype NOT NULL, + c int NOT NULL, + PRIMARY KEY (a), + KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; + +set global rocksdb_debug_ttl_rec_ts = -100; +eval INSERT INTO t1 values (1, 3, $timegen, 5); +eval INSERT INTO t1 values (2, 4, $timegen, 6); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); + +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; + +# should have filtered the rows out since ttl is passed in compaction filter +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +DROP TABLE t1; + +# multi-part PK w/ TTL +eval CREATE TABLE t1 ( + a bigint(20) NOT NULL, + b int NOT NULL, + c int NOT NULL, + ts $coltype NOT NULL, + PRIMARY KEY (a,c), + KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; + +set global rocksdb_debug_ttl_rec_ts = -100; +eval INSERT INTO t1 values (1, 3, 5, $timegen); +eval INSERT INTO t1 values (2, 4, 6, $timegen); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); + +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; + +# should have filtered the rows out since ttl is passed in compaction filter +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +DROP TABLE t1; + +# nullable column(s) before TTL +eval CREATE TABLE t1 ( + a bigint(20) NOT NULL, + b int, + c int, + ts $coltype NOT NULL, + PRIMARY KEY (a), + KEY kbc (b, c) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; + +set global rocksdb_debug_ttl_rec_ts = -100; +eval INSERT INTO t1 values (1, NULL, NULL, $timegen); +eval INSERT INTO t1 values (2, NULL, NULL, $timegen); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); + +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; + +# should have filtered the rows out since ttl is passed in compaction filter +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +DROP TABLE t1; + +# variable len columns + null column(s) before TTL +eval CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64), +`c` varbinary(256), +`ts` $coltype NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`a`), +KEY kbc (`b`, `c`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; + +set global rocksdb_debug_ttl_rec_ts = -100; +eval INSERT INTO t1 values ('a', NULL, 'bc', $timegen, 'd'); +eval INSERT INTO t1 values ('d', 'efghijk', NULL, $timegen, 'l'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); + +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; + +# should have filtered the rows out since ttl is passed in compaction filter +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +DROP TABLE t1; + +# TTL implicitly generated (no ttl column) +CREATE TABLE t1 ( + a bigint(20) NOT NULL, + b int NOT NULL, + c int NOT NULL, + PRIMARY KEY (a), + KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;'; + +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, 3, 5); +INSERT INTO t1 values (2, 4, 6); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); + +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; + +# should have filtered the rows out since ttl is passed in compaction filter +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +DROP TABLE t1; + +# TTL field as the PK +eval CREATE TABLE t1 ( + a int, + ts $coltype NOT NULL, + PRIMARY KEY (a, ts), + KEY kt (ts) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;ttl_col=ts;'; + +eval INSERT INTO t1 values (1, $timegen); +eval INSERT INTO t1 values (2, $timegen); +SELECT COUNT(*) FROM t1 FORCE INDEX(kt); + +set global rocksdb_debug_ttl_snapshot_ts = -10; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +# should all still be there.. +SELECT COUNT(*) FROM t1 FORCE INDEX(kt); + +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_debug_ttl_snapshot_ts = 10; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +set global rocksdb_debug_ttl_ignore_pk=0; + +# should have filtered the rows out since ttl is passed in compaction filter +SELECT COUNT(*) FROM t1 FORCE INDEX(kt); +DROP TABLE t1; + +# TTL field inside multi-part pk +eval CREATE TABLE t1 ( + a bigint(20) NOT NULL, + b int NOT NULL, + ts $coltype NOT NULL, + c int NOT NULL, + PRIMARY KEY (a, ts), + KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; + +set global rocksdb_debug_ttl_rec_ts = -100; +eval INSERT INTO t1 values (1, 3, $timegen, 5); +eval INSERT INTO t1 values (2, 4, $timegen, 6); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); + +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; + +# should have filtered the rows out since ttl is passed in compaction filter +SELECT COUNT(*) FROM t1; +DROP TABLE t1; + +# TTL field inside key with variable length things.. +eval CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64), +`c` varbinary(256), +`ts` $coltype NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`a`, `ts`), +KEY kb (`b`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; + +set global rocksdb_debug_ttl_rec_ts = -100; +eval INSERT INTO t1 values ('a', NULL, 'bc', $timegen, 'd'); +eval INSERT INTO t1 values ('de', 'fghijk', NULL, $timegen, 'l'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*) FROM t1; + +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; + +# should have filtered the rows out since ttl is passed in compaction filter +SELECT COUNT(*) FROM t1; +DROP TABLE t1; + +# TTL test where you compact (values still exist), real_sleep, then compact again, +# values should now be gone. +eval CREATE TABLE t1 ( +a INT NOT NULL, +b varbinary(64) NOT NULL, +c varbinary(256) NOT NULL, +ts $coltype NOT NULL, +value mediumblob NOT NULL, +PRIMARY KEY (b,a,c), +KEY kb (b) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=10;ttl_col=ts;'; + +set global rocksdb_debug_ttl_rec_ts = -300; +eval INSERT INTO t1 values (1, 'b', 'c', $timegen, 'd'); +eval INSERT INTO t1 values (2, 'e', 'f', $timegen, 'g'); +set global rocksdb_debug_ttl_rec_ts = 300; +eval INSERT INTO t1 values (3, 'i', 'j', $timegen, 'k'); +eval INSERT INTO t1 values (4, 'm', 'n', $timegen, 'o'); +set global rocksdb_debug_ttl_rec_ts = 0; + +# Nothing should get removed here. +set global rocksdb_debug_ttl_snapshot_ts = -3600; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +--sorted_result +SELECT a FROM t1 FORCE INDEX (kb); + +# 1 and 2 should get removed here. +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; +--sorted_result +SELECT a FROM t1 FORCE INDEX (kb); + +# 3 and 4 should get removed here. +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +set global rocksdb_debug_ttl_ignore_pk=0; +--sorted_result +SELECT a FROM t1 FORCE INDEX (kb); + +DROP TABLE t1; + +# TTL field with nullable ttl column (should fail) +--error ER_RDB_TTL_COL_FORMAT +eval CREATE TABLE t1 ( + a bigint(20) UNSIGNED NOT NULL, + b int NOT NULL, + c int NOT NULL, + ts $coltype, + PRIMARY KEY (a,c), + KEY (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; + +# TTL field with non 8-bit integer column (should fail) +--error ER_RDB_TTL_COL_FORMAT +eval CREATE TABLE t1 ( + a bigint(20) UNSIGNED NOT NULL, + b int NOT NULL, + c int NOT NULL, + ts int, + PRIMARY KEY (a,c), + KEY (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; + +# TTL duration as some random garbage value +--error ER_RDB_TTL_DURATION_FORMAT +CREATE TABLE t1 ( + a bigint(20) UNSIGNED NOT NULL, + b int NOT NULL, + c int NOT NULL, + PRIMARY KEY (a,c), + KEY (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=abc;'; + +# TTL col is some column outside of the table +--error ER_RDB_TTL_COL_FORMAT +eval CREATE TABLE t1 ( + a bigint(20) UNSIGNED NOT NULL, + b int NOT NULL, + c int NOT NULL, + PRIMARY KEY (a,c), + KEY (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=abc;'; + +# TTL col must have accompanying duration +--error ER_RDB_TTL_COL_FORMAT +eval CREATE TABLE t1 ( + a bigint(20) UNSIGNED NOT NULL, + b int NOT NULL, + c int NOT NULL, + PRIMARY KEY (a,c), + KEY (b) +) ENGINE=rocksdb +COMMENT='ttl_col=abc;'; + +# Make sure it doesn't filter out things early +CREATE TABLE t1 ( + a bigint(20) NOT NULL, + b int NOT NULL, + PRIMARY KEY (a), + KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=500;'; + +INSERT INTO t1 values (1, 1); +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); + +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; + +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +DROP TABLE t1; + +# Testing altering table comment with updated TTL duration +# This should trigger a rebuild of the table +CREATE TABLE t1 ( + a INT PRIMARY KEY, + b INT NOT NULL, + KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=100;'; + +INSERT INTO t1 values (1, 1); +SELECT * FROM t1 FORCE INDEX (kb); + +set global rocksdb_debug_ttl_rec_ts = -300; +ALTER TABLE t1 COMMENT = 'ttl_duration=1'; +set global rocksdb_debug_ttl_rec_ts = 0; +SHOW CREATE TABLE t1; + +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; + +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +DROP TABLE t1; + +# Tables with hidden PK disabled +CREATE TABLE t1 ( + a INT PRIMARY KEY, + b INT, + KEY (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=100;'; + +--error ER_RDB_TTL_UNSUPPORTED +ALTER TABLE t1 DROP PRIMARY KEY; + +DROP TABLE t1; + +# Test replacing PK, ttl should still work after +CREATE TABLE t1 ( + a INT PRIMARY KEY, + b INT, + KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;'; + +INSERT INTO t1 VALUES (1,1); +INSERT INTO t1 VALUES (2,2); + +ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY(b); +set global rocksdb_debug_ttl_snapshot_ts = -3600; +set global rocksdb_force_flush_memtable_now=1; +set @@global.rocksdb_compact_cf = 'default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; + +--sorted_result +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); + +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set @@global.rocksdb_compact_cf = 'default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +set global rocksdb_debug_ttl_ignore_pk=0; + +--sorted_result +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); + +DROP TABLE t1; + +# Make sure table comment filled with other text before/after will work +# (basically, it needs semicolon before and after) +eval CREATE TABLE t1 ( + a $coltype NOT NULL, + b int, + PRIMARY KEY (a,b), + KEY kb (b) +) ENGINE=rocksdb +COMMENT='asdadfasdfsadfadf ;ttl_duration=1; asfasdfasdfadfa'; +eval INSERT INTO t1 values ($timegen, 1); +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); + +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; + +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); + +ALTER TABLE t1 COMMENT = 'adsf;;ttl_duration=5;asfasdfa;ttl_col=a;asdfasdf;'; +set global rocksdb_debug_ttl_rec_ts = 300; +eval INSERT INTO t1 values ($timegen, 2); +set global rocksdb_debug_ttl_rec_ts = 0; +set global rocksdb_force_flush_memtable_now=1; + +# nothing removed here +set global rocksdb_compact_cf='default'; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); + +# all removed here +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_debug_ttl_snapshot_ts = 3600; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +set global rocksdb_debug_ttl_ignore_pk=0; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); + +DROP TABLE t1; + +# Test to make sure that TTL retains original timestamp during update +CREATE TABLE t1 ( + a bigint(20) NOT NULL, + b int NOT NULL, + PRIMARY KEY (a), + KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;'; + +set global rocksdb_debug_ttl_rec_ts = -300; +INSERT INTO t1 values (1, 0); +INSERT INTO t1 values (3, 0); +INSERT INTO t1 values (5, 0); +set global rocksdb_debug_ttl_rec_ts = 300; +INSERT INTO t1 values (7, 0); +INSERT INTO t1 values (9, 0); +set global rocksdb_debug_ttl_rec_ts = 0; + +UPDATE t1 SET a=a+1; +--sorted_result +SELECT * FROM t1 FORCE INDEX (kb); + +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; + +# 1,3,5 should be dropped +--sorted_result +SELECT * FROM t1; +DROP TABLE t1; + +# test behaviour on update with TTL column, TTL time can be updated here. +eval CREATE TABLE t1 ( + a INT, + b $coltype NOT NULL, + PRIMARY KEY (a), + KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;ttl_col=b;'; + +set global rocksdb_debug_ttl_rec_ts = -300; +eval INSERT INTO t1 values (1, $timegen); +eval INSERT INTO t1 values (3, $timegen); +eval INSERT INTO t1 values (5, $timegen); +eval INSERT INTO t1 values (7, $timegen); + +set global rocksdb_debug_ttl_rec_ts = 300; +eval UPDATE t1 SET b=($timegen_1s) WHERE a < 4; +set global rocksdb_debug_ttl_rec_ts = 0; + +--sorted_result +SELECT a FROM t1 FORCE INDEX (kb); + +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; + +# 5 and 7 should be gone here +--sorted_result +SELECT a FROM t1 FORCE INDEX (kb); +DROP TABLE t1; + +# Test rows expired stat variable and disable ttl variable +CREATE TABLE t1 ( + a bigint(20) NOT NULL, + b int NOT NULL, + PRIMARY KEY (a), + KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;'; + +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values (1, 1); +INSERT INTO t1 values (2, 1); +INSERT INTO t1 values (3, 1); +set global rocksdb_debug_ttl_rec_ts = 0; + +set global rocksdb_enable_ttl=0; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; + +select variable_value into @c from information_schema.global_status where variable_name='rocksdb_rows_expired'; +set global rocksdb_enable_ttl=1; +set global rocksdb_compact_cf='default'; + +select variable_value-@c from information_schema.global_status where variable_name='rocksdb_rows_expired'; +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +DROP TABLE t1; + +# Table with TTL won't increment rows expired when no records have been +# compacted +CREATE TABLE t1 ( + a bigint(20) NOT NULL, + b int NOT NULL, + PRIMARY KEY (a), + KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=100;'; + +INSERT INTO t1 values (1, 1); +INSERT INTO t1 values (2, 2); +INSERT INTO t1 values (3, 3); + +select variable_value into @c from information_schema.global_status where variable_name='rocksdb_rows_expired'; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +select variable_value-@c from information_schema.global_status where variable_name='rocksdb_rows_expired'; + +DROP TABLE t1; + +# Test update on TTL column in pk +eval CREATE TABLE t1 ( + a INT, + b $coltype NOT NULL, + PRIMARY KEY (a, b), + KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;ttl_col=b;'; + +set global rocksdb_debug_ttl_rec_ts = -300; +eval INSERT INTO t1 values (1, $timegen); +eval INSERT INTO t1 values (3, $timegen); +eval INSERT INTO t1 values (5, $timegen); +eval INSERT INTO t1 values (7, $timegen); + +set global rocksdb_debug_ttl_rec_ts = 300; +eval UPDATE t1 SET b=($timegen_1s) WHERE a < 4; +set global rocksdb_debug_ttl_rec_ts = 0; + +--sorted_result +SELECT a FROM t1 FORCE INDEX (kb); + +set global rocksdb_debug_ttl_ignore_pk=1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk=0; + +# 5 and 7 should be gone here +--sorted_result +SELECT a FROM t1 FORCE INDEX (kb); +DROP TABLE t1; + +# test behaviour on update with TTL column, TTL time can be updated here. +eval CREATE TABLE t1 ( + a INT, + b $coltype NOT NULL, + PRIMARY KEY (a, b) +) ENGINE=rocksdb +COMMENT='ttl_duration=5;ttl_col=b;'; + +set global rocksdb_debug_ttl_rec_ts = -300; +eval INSERT INTO t1 values (1, $timegen); +eval INSERT INTO t1 values (3, $timegen); +eval INSERT INTO t1 values (5, $timegen); +eval INSERT INTO t1 values (7, $timegen); + +set global rocksdb_debug_ttl_rec_ts = 300; +eval UPDATE t1 SET b=($timegen_1s) WHERE a < 4; +set global rocksdb_debug_ttl_rec_ts = 0; + +--sorted_result +SELECT a FROM t1; + +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; + +# 7 should be gone here +--sorted_result +SELECT a FROM t1; +DROP TABLE t1; + +# Add index inplace +eval CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64) NOT NULL, +`c` varbinary(256) NOT NULL, +`ts` $coltype NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`b`,`a`,`c`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; + +set global rocksdb_debug_ttl_rec_ts = -100; +eval INSERT INTO t1 values ('a', 'b', 'c', $timegen, 'd'); +eval INSERT INTO t1 values ('d', 'e', 'f', $timegen, 'g'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*); + +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; + +# nothing filtered out +SELECT COUNT(*); + +CREATE INDEX kb on t1 (b); + +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; + +# should have filtered the rows out since ttl is passed in compaction filter +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +DROP TABLE t1; + +# Add index inplace, implicit TTL +CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64) NOT NULL, +`c` varbinary(256) NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`b`,`a`,`c`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1'; + +set global rocksdb_debug_ttl_rec_ts = -100; +INSERT INTO t1 values ('a', 'b', 'c', 'd'); +INSERT INTO t1 values ('d', 'e', 'f', 'g'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*); + +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; + +# nothing filtered out +SELECT COUNT(*); + +CREATE INDEX kb on t1 (b); + +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; + +# should have filtered the rows out since ttl is passed in compaction filter +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +DROP TABLE t1; + +# Add index inplace, TTL column in PK +eval CREATE TABLE t1 ( +`a` binary(8) NOT NULL, +`b` varbinary(64) NOT NULL, +`c` varbinary(256) NOT NULL, +`ts` $coltype NOT NULL, +`value` mediumblob NOT NULL, +PRIMARY KEY (`b`,`a`,`c`, `ts`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +COMMENT='ttl_duration=1;ttl_col=ts;'; + +set global rocksdb_debug_ttl_rec_ts = -100; +eval INSERT INTO t1 values ('a', 'b', 'c', $timegen, 'd'); +eval INSERT INTO t1 values ('d', 'e', 'f', $timegen, 'g'); +set global rocksdb_debug_ttl_rec_ts = 0; +SELECT COUNT(*); + +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; + +# nothing filtered out +SELECT COUNT(*); + +CREATE INDEX kb on t1 (b); + +set global rocksdb_debug_ttl_ignore_pk = 1; +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; +set global rocksdb_debug_ttl_ignore_pk = 0; + +# should have filtered the rows out since ttl is passed in compaction filter +SELECT COUNT(*) FROM t1 FORCE INDEX (kb); +DROP TABLE t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/ttl_secondary.test b/storage/rocksdb/mysql-test/rocksdb/t/ttl_secondary.test index fb439e109e7..b465175e98a 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/ttl_secondary.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/ttl_secondary.test @@ -1,780 +1,22 @@ --source include/have_debug.inc --source include/have_rocksdb.inc -# Basic TTL test, pk ignored, no sk -CREATE TABLE t1 ( -`a` binary(8) NOT NULL, -`b` varbinary(64) NOT NULL, -`c` varbinary(256) NOT NULL, -`ts` bigint(20) UNSIGNED NOT NULL, -`value` mediumblob NOT NULL, -PRIMARY KEY (`b`,`a`,`c`) -) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 -COMMENT='ttl_duration=1;ttl_col=ts;'; - -set global rocksdb_debug_ttl_rec_ts = -100; -INSERT INTO t1 values ('a', 'b', 'c', UNIX_TIMESTAMP(), 'd'); -INSERT INTO t1 values ('d', 'e', 'f', UNIX_TIMESTAMP(), 'g'); -set global rocksdb_debug_ttl_rec_ts = 0; -SELECT COUNT(*) FROM t1; - -set global rocksdb_debug_ttl_ignore_pk = 1; -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; -set global rocksdb_debug_ttl_ignore_pk = 0; - -# no rows should be filtered -SELECT COUNT(*) FROM t1; -DROP TABLE t1; - -# Basic TTL test -CREATE TABLE t1 ( -`a` binary(8) NOT NULL, -`b` varbinary(64) NOT NULL, -`c` varbinary(256) NOT NULL, -`ts` bigint(20) UNSIGNED NOT NULL, -`value` mediumblob NOT NULL, -PRIMARY KEY (`b`,`a`,`c`), -KEY kb (`b`) -) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 -COMMENT='ttl_duration=1;ttl_col=ts;'; - -set global rocksdb_debug_ttl_rec_ts = -100; -INSERT INTO t1 values ('a', 'b', 'c', UNIX_TIMESTAMP(), 'd'); -INSERT INTO t1 values ('d', 'e', 'f', UNIX_TIMESTAMP(), 'g'); -set global rocksdb_debug_ttl_rec_ts = 0; -SELECT COUNT(*) FROM t1 FORCE INDEX (kb); - -set global rocksdb_debug_ttl_ignore_pk = 1; -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; -set global rocksdb_debug_ttl_ignore_pk = 0; - -# should have filtered the rows out since ttl is passed in compaction filter -SELECT COUNT(*) FROM t1 FORCE INDEX (kb); -DROP TABLE t1; - -# column before TTL in value -CREATE TABLE t1 ( - a bigint(20) NOT NULL, - b int NOT NULL, - ts bigint(20) UNSIGNED NOT NULL, - c int NOT NULL, - PRIMARY KEY (a), - KEY kb (b) -) ENGINE=rocksdb -COMMENT='ttl_duration=1;ttl_col=ts;'; - -set global rocksdb_debug_ttl_rec_ts = -100; -INSERT INTO t1 values (1, 3, UNIX_TIMESTAMP(), 5); -INSERT INTO t1 values (2, 4, UNIX_TIMESTAMP(), 6); -set global rocksdb_debug_ttl_rec_ts = 0; -SELECT COUNT(*) FROM t1 FORCE INDEX (kb); - -set global rocksdb_debug_ttl_ignore_pk = 1; -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; -set global rocksdb_debug_ttl_ignore_pk = 0; - -# should have filtered the rows out since ttl is passed in compaction filter -SELECT COUNT(*) FROM t1 FORCE INDEX (kb); -DROP TABLE t1; - -# multi-part PK w/ TTL -CREATE TABLE t1 ( - a bigint(20) NOT NULL, - b int NOT NULL, - c int NOT NULL, - ts bigint(20) UNSIGNED NOT NULL, - PRIMARY KEY (a,c), - KEY kb (b) -) ENGINE=rocksdb -COMMENT='ttl_duration=1;ttl_col=ts;'; - -set global rocksdb_debug_ttl_rec_ts = -100; -INSERT INTO t1 values (1, 3, 5, UNIX_TIMESTAMP()); -INSERT INTO t1 values (2, 4, 6, UNIX_TIMESTAMP()); -set global rocksdb_debug_ttl_rec_ts = 0; -SELECT COUNT(*) FROM t1 FORCE INDEX (kb); - -set global rocksdb_debug_ttl_ignore_pk=1; -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; -set global rocksdb_debug_ttl_ignore_pk=0; - -# should have filtered the rows out since ttl is passed in compaction filter -SELECT COUNT(*) FROM t1 FORCE INDEX (kb); -DROP TABLE t1; - -# nullable column(s) before TTL -CREATE TABLE t1 ( - a bigint(20) NOT NULL, - b int, - c int, - ts bigint(20) UNSIGNED NOT NULL, - PRIMARY KEY (a), - KEY kbc (b, c) -) ENGINE=rocksdb -COMMENT='ttl_duration=1;ttl_col=ts;'; - -set global rocksdb_debug_ttl_rec_ts = -100; -INSERT INTO t1 values (1, NULL, NULL, UNIX_TIMESTAMP()); -INSERT INTO t1 values (2, NULL, NULL, UNIX_TIMESTAMP()); -set global rocksdb_debug_ttl_rec_ts = 0; -SELECT COUNT(*) FROM t1 FORCE INDEX (kb); - -set global rocksdb_debug_ttl_ignore_pk=1; -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; -set global rocksdb_debug_ttl_ignore_pk=0; - -# should have filtered the rows out since ttl is passed in compaction filter -SELECT COUNT(*) FROM t1 FORCE INDEX (kb); -DROP TABLE t1; - -# variable len columns + null column(s) before TTL -CREATE TABLE t1 ( -`a` binary(8) NOT NULL, -`b` varbinary(64), -`c` varbinary(256), -`ts` bigint(20) UNSIGNED NOT NULL, -`value` mediumblob NOT NULL, -PRIMARY KEY (`a`), -KEY kbc (`b`, `c`) -) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 -COMMENT='ttl_duration=1;ttl_col=ts;'; - -set global rocksdb_debug_ttl_rec_ts = -100; -INSERT INTO t1 values ('a', NULL, 'bc', UNIX_TIMESTAMP(), 'd'); -INSERT INTO t1 values ('d', 'efghijk', NULL, UNIX_TIMESTAMP(), 'l'); -set global rocksdb_debug_ttl_rec_ts = 0; -SELECT COUNT(*) FROM t1 FORCE INDEX (kb); - -set global rocksdb_debug_ttl_ignore_pk=1; -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; -set global rocksdb_debug_ttl_ignore_pk=0; - -# should have filtered the rows out since ttl is passed in compaction filter -SELECT COUNT(*) FROM t1 FORCE INDEX (kb); -DROP TABLE t1; - -# TTL implicitly generated (no ttl column) -CREATE TABLE t1 ( - a bigint(20) NOT NULL, - b int NOT NULL, - c int NOT NULL, - PRIMARY KEY (a), - KEY kb (b) -) ENGINE=rocksdb -COMMENT='ttl_duration=1;'; - -set global rocksdb_debug_ttl_rec_ts = -100; -INSERT INTO t1 values (1, 3, 5); -INSERT INTO t1 values (2, 4, 6); -set global rocksdb_debug_ttl_rec_ts = 0; -SELECT COUNT(*) FROM t1 FORCE INDEX (kb); - -set global rocksdb_debug_ttl_ignore_pk=1; -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; -set global rocksdb_debug_ttl_ignore_pk=0; - -# should have filtered the rows out since ttl is passed in compaction filter -SELECT COUNT(*) FROM t1 FORCE INDEX (kb); -DROP TABLE t1; - -# TTL field as the PK -CREATE TABLE t1 ( - a int, - ts bigint(20) UNSIGNED NOT NULL, - PRIMARY KEY (a, ts), - KEY kt (ts) -) ENGINE=rocksdb -COMMENT='ttl_duration=5;ttl_col=ts;'; - -INSERT INTO t1 values (1, UNIX_TIMESTAMP()); -INSERT INTO t1 values (2, UNIX_TIMESTAMP()); -SELECT COUNT(*) FROM t1 FORCE INDEX(kt); - -set global rocksdb_debug_ttl_snapshot_ts = -10; -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; -set global rocksdb_debug_ttl_snapshot_ts = 0; -# should all still be there.. -SELECT COUNT(*) FROM t1 FORCE INDEX(kt); - -set global rocksdb_debug_ttl_ignore_pk=1; -set global rocksdb_debug_ttl_snapshot_ts = 10; -set global rocksdb_compact_cf='default'; -set global rocksdb_debug_ttl_snapshot_ts = 0; -set global rocksdb_debug_ttl_ignore_pk=0; - -# should have filtered the rows out since ttl is passed in compaction filter -SELECT COUNT(*) FROM t1 FORCE INDEX(kt); -DROP TABLE t1; - -# TTL field inside multi-part pk -CREATE TABLE t1 ( - a bigint(20) NOT NULL, - b int NOT NULL, - ts bigint(20) UNSIGNED NOT NULL, - c int NOT NULL, - PRIMARY KEY (a, ts), - KEY kb (b) -) ENGINE=rocksdb -COMMENT='ttl_duration=1;ttl_col=ts;'; - -set global rocksdb_debug_ttl_rec_ts = -100; -INSERT INTO t1 values (1, 3, UNIX_TIMESTAMP(), 5); -INSERT INTO t1 values (2, 4, UNIX_TIMESTAMP(), 6); -set global rocksdb_debug_ttl_rec_ts = 0; -SELECT COUNT(*) FROM t1 FORCE INDEX (kb); - -set global rocksdb_debug_ttl_ignore_pk=1; -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; -set global rocksdb_debug_ttl_ignore_pk=0; - -# should have filtered the rows out since ttl is passed in compaction filter -SELECT COUNT(*) FROM t1; -DROP TABLE t1; - -# TTL field inside key with variable length things.. -CREATE TABLE t1 ( -`a` binary(8) NOT NULL, -`b` varbinary(64), -`c` varbinary(256), -`ts` bigint(20) UNSIGNED NOT NULL, -`value` mediumblob NOT NULL, -PRIMARY KEY (`a`, `ts`), -KEY kb (`b`) -) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 -COMMENT='ttl_duration=1;ttl_col=ts;'; - -set global rocksdb_debug_ttl_rec_ts = -100; -INSERT INTO t1 values ('a', NULL, 'bc', UNIX_TIMESTAMP(), 'd'); -INSERT INTO t1 values ('de', 'fghijk', NULL, UNIX_TIMESTAMP(), 'l'); -set global rocksdb_debug_ttl_rec_ts = 0; -SELECT COUNT(*) FROM t1; - -set global rocksdb_debug_ttl_ignore_pk=1; -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; -set global rocksdb_debug_ttl_ignore_pk=0; - -# should have filtered the rows out since ttl is passed in compaction filter -SELECT COUNT(*) FROM t1; -DROP TABLE t1; - -# TTL test where you compact (values still exist), real_sleep, then compact again, -# values should now be gone. -CREATE TABLE t1 ( -a INT NOT NULL, -b varbinary(64) NOT NULL, -c varbinary(256) NOT NULL, -ts bigint(20) UNSIGNED NOT NULL, -value mediumblob NOT NULL, -PRIMARY KEY (b,a,c), -KEY kb (b) -) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 -COMMENT='ttl_duration=10;ttl_col=ts;'; - -set global rocksdb_debug_ttl_rec_ts = -300; -INSERT INTO t1 values (1, 'b', 'c', UNIX_TIMESTAMP(), 'd'); -INSERT INTO t1 values (2, 'e', 'f', UNIX_TIMESTAMP(), 'g'); -set global rocksdb_debug_ttl_rec_ts = 300; -INSERT INTO t1 values (3, 'i', 'j', UNIX_TIMESTAMP(), 'k'); -INSERT INTO t1 values (4, 'm', 'n', UNIX_TIMESTAMP(), 'o'); -set global rocksdb_debug_ttl_rec_ts = 0; - -# Nothing should get removed here. -set global rocksdb_debug_ttl_snapshot_ts = -3600; -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; -set global rocksdb_debug_ttl_snapshot_ts = 0; ---sorted_result -SELECT a FROM t1 FORCE INDEX (kb); - -# 1 and 2 should get removed here. -set global rocksdb_debug_ttl_ignore_pk=1; -set global rocksdb_compact_cf='default'; -set global rocksdb_debug_ttl_ignore_pk=0; ---sorted_result -SELECT a FROM t1 FORCE INDEX (kb); - -# 3 and 4 should get removed here. -set global rocksdb_debug_ttl_ignore_pk=1; -set global rocksdb_debug_ttl_snapshot_ts = 3600; -set global rocksdb_compact_cf='default'; -set global rocksdb_debug_ttl_snapshot_ts = 0; -set global rocksdb_debug_ttl_ignore_pk=0; ---sorted_result -SELECT a FROM t1 FORCE INDEX (kb); - -DROP TABLE t1; - -# TTL field with nullable ttl column (should fail) ---error ER_RDB_TTL_COL_FORMAT -CREATE TABLE t1 ( - a bigint(20) UNSIGNED NOT NULL, - b int NOT NULL, - c int NOT NULL, - ts bigint(20), - PRIMARY KEY (a,c), - KEY (b) -) ENGINE=rocksdb -COMMENT='ttl_duration=1;ttl_col=ts;'; - -# TTL field with non 8-bit integer column (should fail) ---error ER_RDB_TTL_COL_FORMAT -CREATE TABLE t1 ( - a bigint(20) UNSIGNED NOT NULL, - b int NOT NULL, - c int NOT NULL, - ts int, - PRIMARY KEY (a,c), - KEY (b) -) ENGINE=rocksdb -COMMENT='ttl_duration=1;ttl_col=ts;'; - -# TTL duration as some random garbage value ---error ER_RDB_TTL_DURATION_FORMAT -CREATE TABLE t1 ( - a bigint(20) UNSIGNED NOT NULL, - b int NOT NULL, - c int NOT NULL, - PRIMARY KEY (a,c), - KEY (b) -) ENGINE=rocksdb -COMMENT='ttl_duration=abc;'; - -# TTL col is some column outside of the table ---error ER_RDB_TTL_COL_FORMAT -CREATE TABLE t1 ( - a bigint(20) UNSIGNED NOT NULL, - b int NOT NULL, - c int NOT NULL, - PRIMARY KEY (a,c), - KEY (b) -) ENGINE=rocksdb -COMMENT='ttl_duration=1;ttl_col=abc;'; - -# TTL col must have accompanying duration ---error ER_RDB_TTL_COL_FORMAT -CREATE TABLE t1 ( - a bigint(20) UNSIGNED NOT NULL, - b int NOT NULL, - c int NOT NULL, - PRIMARY KEY (a,c), - KEY (b) -) ENGINE=rocksdb -COMMENT='ttl_col=abc;'; - -# Make sure it doesn't filter out things early -CREATE TABLE t1 ( - a bigint(20) NOT NULL, - b int NOT NULL, - PRIMARY KEY (a), - KEY kb (b) -) ENGINE=rocksdb -COMMENT='ttl_duration=500;'; - -INSERT INTO t1 values (1, 1); -SELECT COUNT(*) FROM t1 FORCE INDEX (kb); - -set global rocksdb_debug_ttl_ignore_pk=1; -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; -set global rocksdb_debug_ttl_ignore_pk=0; - -SELECT COUNT(*) FROM t1 FORCE INDEX (kb); -DROP TABLE t1; - -# Testing altering table comment with updated TTL duration -# This should trigger a rebuild of the table -CREATE TABLE t1 ( - a INT PRIMARY KEY, - b INT NOT NULL, - KEY kb (b) -) ENGINE=rocksdb -COMMENT='ttl_duration=100;'; - -INSERT INTO t1 values (1, 1); -SELECT * FROM t1 FORCE INDEX (kb); - -set global rocksdb_debug_ttl_rec_ts = -300; -ALTER TABLE t1 COMMENT = 'ttl_duration=1'; -set global rocksdb_debug_ttl_rec_ts = 0; -SHOW CREATE TABLE t1; - -set global rocksdb_debug_ttl_ignore_pk=1; -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; -set global rocksdb_debug_ttl_ignore_pk=0; - -SELECT COUNT(*) FROM t1 FORCE INDEX (kb); -DROP TABLE t1; - -# Tables with hidden PK disabled -CREATE TABLE t1 ( - a INT PRIMARY KEY, - b INT, - KEY (b) -) ENGINE=rocksdb -COMMENT='ttl_duration=100;'; - ---error ER_RDB_TTL_UNSUPPORTED -ALTER TABLE t1 DROP PRIMARY KEY; - -DROP TABLE t1; - -# Test replacing PK, ttl should still work after -CREATE TABLE t1 ( - a INT PRIMARY KEY, - b INT, - KEY kb (b) -) ENGINE=rocksdb -COMMENT='ttl_duration=5;'; - -INSERT INTO t1 VALUES (1,1); -INSERT INTO t1 VALUES (2,2); - -ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY(b); -set global rocksdb_debug_ttl_snapshot_ts = -3600; -set global rocksdb_force_flush_memtable_now=1; -set @@global.rocksdb_compact_cf = 'default'; -set global rocksdb_debug_ttl_snapshot_ts = 0; - ---sorted_result -SELECT COUNT(*) FROM t1 FORCE INDEX (kb); - -set global rocksdb_debug_ttl_ignore_pk=1; -set global rocksdb_debug_ttl_snapshot_ts = 3600; -set @@global.rocksdb_compact_cf = 'default'; -set global rocksdb_debug_ttl_snapshot_ts = 0; -set global rocksdb_debug_ttl_ignore_pk=0; - ---sorted_result -SELECT COUNT(*) FROM t1 FORCE INDEX (kb); - -DROP TABLE t1; - -# Make sure table comment filled with other text before/after will work -# (basically, it needs semicolon before and after) -CREATE TABLE t1 ( - a bigint(20) UNSIGNED NOT NULL, - b int, - PRIMARY KEY (a,b), - KEY kb (b) -) ENGINE=rocksdb -COMMENT='asdadfasdfsadfadf ;ttl_duration=1; asfasdfasdfadfa'; -INSERT INTO t1 values (UNIX_TIMESTAMP(), 1); -SELECT COUNT(*) FROM t1 FORCE INDEX (kb); - -set global rocksdb_debug_ttl_snapshot_ts = 3600; -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; -set global rocksdb_debug_ttl_snapshot_ts = 0; - -SELECT COUNT(*) FROM t1 FORCE INDEX (kb); - -ALTER TABLE t1 COMMENT = 'adsf;;ttl_duration=5;asfasdfa;ttl_col=a;asdfasdf;'; -set global rocksdb_debug_ttl_rec_ts = 300; -INSERT INTO t1 values (UNIX_TIMESTAMP(), 2); -set global rocksdb_debug_ttl_rec_ts = 0; -set global rocksdb_force_flush_memtable_now=1; - -# nothing removed here -set global rocksdb_compact_cf='default'; -SELECT COUNT(*) FROM t1 FORCE INDEX (kb); - -# all removed here -set global rocksdb_debug_ttl_ignore_pk=1; -set global rocksdb_debug_ttl_snapshot_ts = 3600; -set global rocksdb_compact_cf='default'; -set global rocksdb_debug_ttl_snapshot_ts = 0; -set global rocksdb_debug_ttl_ignore_pk=0; -SELECT COUNT(*) FROM t1 FORCE INDEX (kb); - -DROP TABLE t1; - -# Test to make sure that TTL retains original timestamp during update -CREATE TABLE t1 ( - a bigint(20) NOT NULL, - b int NOT NULL, - PRIMARY KEY (a), - KEY kb (b) -) ENGINE=rocksdb -COMMENT='ttl_duration=5;'; - -set global rocksdb_debug_ttl_rec_ts = -300; -INSERT INTO t1 values (1, 0); -INSERT INTO t1 values (3, 0); -INSERT INTO t1 values (5, 0); -set global rocksdb_debug_ttl_rec_ts = 300; -INSERT INTO t1 values (7, 0); -INSERT INTO t1 values (9, 0); -set global rocksdb_debug_ttl_rec_ts = 0; - -UPDATE t1 SET a=a+1; ---sorted_result -SELECT * FROM t1 FORCE INDEX (kb); - -set global rocksdb_debug_ttl_ignore_pk=1; -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; -set global rocksdb_debug_ttl_ignore_pk=0; - -# 1,3,5 should be dropped ---sorted_result -SELECT * FROM t1; -DROP TABLE t1; - -# test behaviour on update with TTL column, TTL time can be updated here. -CREATE TABLE t1 ( - a INT, - b bigint(20) UNSIGNED NOT NULL, - PRIMARY KEY (a), - KEY kb (b) -) ENGINE=rocksdb -COMMENT='ttl_duration=5;ttl_col=b;'; - -set global rocksdb_debug_ttl_rec_ts = -300; -INSERT INTO t1 values (1, UNIX_TIMESTAMP()); -INSERT INTO t1 values (3, UNIX_TIMESTAMP()); -INSERT INTO t1 values (5, UNIX_TIMESTAMP()); -INSERT INTO t1 values (7, UNIX_TIMESTAMP()); - -set global rocksdb_debug_ttl_rec_ts = 300; -UPDATE t1 SET b=(UNIX_TIMESTAMP()+1) WHERE a < 4; -set global rocksdb_debug_ttl_rec_ts = 0; - ---sorted_result -SELECT a FROM t1 FORCE INDEX (kb); - -set global rocksdb_debug_ttl_ignore_pk=1; -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; -set global rocksdb_debug_ttl_ignore_pk=0; - -# 5 and 7 should be gone here ---sorted_result -SELECT a FROM t1 FORCE INDEX (kb); -DROP TABLE t1; - -# Test rows expired stat variable and disable ttl variable -CREATE TABLE t1 ( - a bigint(20) NOT NULL, - b int NOT NULL, - PRIMARY KEY (a), - KEY kb (b) -) ENGINE=rocksdb -COMMENT='ttl_duration=1;'; - -set global rocksdb_debug_ttl_rec_ts = -100; -INSERT INTO t1 values (1, 1); -INSERT INTO t1 values (2, 1); -INSERT INTO t1 values (3, 1); -set global rocksdb_debug_ttl_rec_ts = 0; - -set global rocksdb_enable_ttl=0; -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; - -select variable_value into @c from information_schema.global_status where variable_name='rocksdb_rows_expired'; -set global rocksdb_enable_ttl=1; -set global rocksdb_compact_cf='default'; - -select variable_value-@c from information_schema.global_status where variable_name='rocksdb_rows_expired'; -SELECT COUNT(*) FROM t1 FORCE INDEX (kb); -DROP TABLE t1; - -# Table with TTL won't increment rows expired when no records have been -# compacted -CREATE TABLE t1 ( - a bigint(20) NOT NULL, - b int NOT NULL, - PRIMARY KEY (a), - KEY kb (b) -) ENGINE=rocksdb -COMMENT='ttl_duration=100;'; - -INSERT INTO t1 values (1, 1); -INSERT INTO t1 values (2, 2); -INSERT INTO t1 values (3, 3); - -select variable_value into @c from information_schema.global_status where variable_name='rocksdb_rows_expired'; -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; -select variable_value-@c from information_schema.global_status where variable_name='rocksdb_rows_expired'; - -DROP TABLE t1; - -# Test update on TTL column in pk -CREATE TABLE t1 ( - a INT, - b bigint(20) UNSIGNED NOT NULL, - PRIMARY KEY (a, b), - KEY kb (b) -) ENGINE=rocksdb -COMMENT='ttl_duration=5;ttl_col=b;'; - -set global rocksdb_debug_ttl_rec_ts = -300; -INSERT INTO t1 values (1, UNIX_TIMESTAMP()); -INSERT INTO t1 values (3, UNIX_TIMESTAMP()); -INSERT INTO t1 values (5, UNIX_TIMESTAMP()); -INSERT INTO t1 values (7, UNIX_TIMESTAMP()); - -set global rocksdb_debug_ttl_rec_ts = 300; -UPDATE t1 SET b=(UNIX_TIMESTAMP()+1) WHERE a < 4; -set global rocksdb_debug_ttl_rec_ts = 0; - ---sorted_result -SELECT a FROM t1 FORCE INDEX (kb); - -set global rocksdb_debug_ttl_ignore_pk=1; -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; -set global rocksdb_debug_ttl_ignore_pk=0; - -# 5 and 7 should be gone here ---sorted_result -SELECT a FROM t1 FORCE INDEX (kb); -DROP TABLE t1; - -# test behaviour on update with TTL column, TTL time can be updated here. -CREATE TABLE t1 ( - a INT, - b bigint(20) UNSIGNED NOT NULL, - PRIMARY KEY (a, b) -) ENGINE=rocksdb -COMMENT='ttl_duration=5;ttl_col=b;'; - -set global rocksdb_debug_ttl_rec_ts = -300; -INSERT INTO t1 values (1, UNIX_TIMESTAMP()); -INSERT INTO t1 values (3, UNIX_TIMESTAMP()); -INSERT INTO t1 values (5, UNIX_TIMESTAMP()); -INSERT INTO t1 values (7, UNIX_TIMESTAMP()); - -set global rocksdb_debug_ttl_rec_ts = 300; -UPDATE t1 SET b=(UNIX_TIMESTAMP()+1) WHERE a < 4; -set global rocksdb_debug_ttl_rec_ts = 0; - ---sorted_result -SELECT a FROM t1; - -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; - -# 7 should be gone here ---sorted_result -SELECT a FROM t1; -DROP TABLE t1; - -# Add index inplace -CREATE TABLE t1 ( -`a` binary(8) NOT NULL, -`b` varbinary(64) NOT NULL, -`c` varbinary(256) NOT NULL, -`ts` bigint(20) UNSIGNED NOT NULL, -`value` mediumblob NOT NULL, -PRIMARY KEY (`b`,`a`,`c`) -) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 -COMMENT='ttl_duration=1;ttl_col=ts;'; - -set global rocksdb_debug_ttl_rec_ts = -100; -INSERT INTO t1 values ('a', 'b', 'c', UNIX_TIMESTAMP(), 'd'); -INSERT INTO t1 values ('d', 'e', 'f', UNIX_TIMESTAMP(), 'g'); -set global rocksdb_debug_ttl_rec_ts = 0; -SELECT COUNT(*); - -set global rocksdb_debug_ttl_ignore_pk = 1; -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; -set global rocksdb_debug_ttl_ignore_pk = 0; - -# nothing filtered out -SELECT COUNT(*); - -CREATE INDEX kb on t1 (b); - -set global rocksdb_debug_ttl_ignore_pk = 1; -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; -set global rocksdb_debug_ttl_ignore_pk = 0; - -# should have filtered the rows out since ttl is passed in compaction filter -SELECT COUNT(*) FROM t1 FORCE INDEX (kb); -DROP TABLE t1; - -# Add index inplace, implicit TTL -CREATE TABLE t1 ( -`a` binary(8) NOT NULL, -`b` varbinary(64) NOT NULL, -`c` varbinary(256) NOT NULL, -`value` mediumblob NOT NULL, -PRIMARY KEY (`b`,`a`,`c`) -) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 -COMMENT='ttl_duration=1'; - -set global rocksdb_debug_ttl_rec_ts = -100; -INSERT INTO t1 values ('a', 'b', 'c', 'd'); -INSERT INTO t1 values ('d', 'e', 'f', 'g'); -set global rocksdb_debug_ttl_rec_ts = 0; -SELECT COUNT(*); - -set global rocksdb_debug_ttl_ignore_pk = 1; -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; -set global rocksdb_debug_ttl_ignore_pk = 0; - -# nothing filtered out -SELECT COUNT(*); - -CREATE INDEX kb on t1 (b); - -set global rocksdb_debug_ttl_ignore_pk = 1; -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; -set global rocksdb_debug_ttl_ignore_pk = 0; - -# should have filtered the rows out since ttl is passed in compaction filter -SELECT COUNT(*) FROM t1 FORCE INDEX (kb); -DROP TABLE t1; - -# Add index inplace, TTL column in PK -CREATE TABLE t1 ( -`a` binary(8) NOT NULL, -`b` varbinary(64) NOT NULL, -`c` varbinary(256) NOT NULL, -`ts` bigint(20) UNSIGNED NOT NULL, -`value` mediumblob NOT NULL, -PRIMARY KEY (`b`,`a`,`c`, `ts`) -) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 -COMMENT='ttl_duration=1;ttl_col=ts;'; - -set global rocksdb_debug_ttl_rec_ts = -100; -INSERT INTO t1 values ('a', 'b', 'c', UNIX_TIMESTAMP(), 'd'); -INSERT INTO t1 values ('d', 'e', 'f', UNIX_TIMESTAMP(), 'g'); -set global rocksdb_debug_ttl_rec_ts = 0; -SELECT COUNT(*); - -set global rocksdb_debug_ttl_ignore_pk = 1; -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; -set global rocksdb_debug_ttl_ignore_pk = 0; - -# nothing filtered out -SELECT COUNT(*); - -CREATE INDEX kb on t1 (b); - -set global rocksdb_debug_ttl_ignore_pk = 1; -set global rocksdb_force_flush_memtable_now=1; -set global rocksdb_compact_cf='default'; -set global rocksdb_debug_ttl_ignore_pk = 0; - -# should have filtered the rows out since ttl is passed in compaction filter -SELECT COUNT(*) FROM t1 FORCE INDEX (kb); -DROP TABLE t1; +--let $coltype=bigint(20) UNSIGNED +--let $timegen=UNIX_TIMESTAMP() +--let $timegen_1s=UNIX_TIMESTAMP()+1 +--source ttl_secondary.inc + +--let $coltype=TIMESTAMP +--let $timegen=NOW() +--let $timegen_1s=ADDTIME(NOW(), '00:00:01') +--source ttl_secondary.inc + +--let $coltype=TIMESTAMP DEFAULT NOW() +--let $timegen=NOW() +--let $timegen_1s=ADDTIME(NOW(), '00:00:01') +--source ttl_secondary.inc + +--let $coltype=TIMESTAMP(6) +--let $timegen=NOW() +--let $timegen_1s=ADDTIME(NOW(), '00:00:01') +--source ttl_secondary.inc diff --git a/storage/rocksdb/mysql-test/rocksdb/t/ttl_secondary_read_filtering.test b/storage/rocksdb/mysql-test/rocksdb/t/ttl_secondary_read_filtering.test index f6042cc517e..96a413cbea8 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/ttl_secondary_read_filtering.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/ttl_secondary_read_filtering.test @@ -409,6 +409,45 @@ SELECT a, b FROM t1 FORCE INDEX (kb); DROP TABLE t1; +# Test that index_next_with_direction skips records properly for timestamps +CREATE TABLE t1 ( + a int, + b int, + ts TIMESTAMP NOT NULL, + PRIMARY KEY (a), + KEY kb (b) +) ENGINE=rocksdb +COMMENT='ttl_duration=1;ttl_col=ts;'; + +set global rocksdb_debug_ttl_rec_ts = 100; +INSERT INTO t1 VALUES (1, 1, NOW()); +INSERT INTO t1 VALUES (2, 2, NOW()); +INSERT INTO t1 VALUES (3, 3, NOW()); +INSERT INTO t1 VALUES (4, 4, NOW()); +INSERT INTO t1 VALUES (5, 5, NOW()); +INSERT INTO t1 VALUES (6, 6, NOW()); +INSERT INTO t1 VALUES (7, 7, NOW()); +INSERT INTO t1 VALUES (8, 8, NOW()); +INSERT INTO t1 VALUES (9, 9, NOW()); +INSERT INTO t1 VALUES (10, 10, NOW()); +set global rocksdb_debug_ttl_rec_ts = 0; + +set global rocksdb_force_flush_memtable_now=1; +--echo # None are expired +SELECT a, b FROM t1 FORCE INDEX (kb); + +set global rocksdb_debug_ttl_rec_ts = -100; +UPDATE t1 SET ts=(ADDTIME(NOW(), 1)) WHERE a IN (4, 7); +set global rocksdb_debug_ttl_rec_ts = 0; + +set global rocksdb_force_flush_memtable_now=1; +set global rocksdb_compact_cf='default'; + +--echo # 4 and 7 should be gone +SELECT a, b FROM t1 FORCE INDEX (kb); + +DROP TABLE t1; + # Test range scans with various conditionals CREATE TABLE t1 ( c1 INT, diff --git a/storage/rocksdb/mysql-test/rocksdb/t/ttl_secondary_with_partitions.test b/storage/rocksdb/mysql-test/rocksdb/t/ttl_secondary_with_partitions.test index c10c83f9f9d..21f5fb6bafb 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/ttl_secondary_with_partitions.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/ttl_secondary_with_partitions.test @@ -255,6 +255,62 @@ SELECT c1 FROM t1 FORCE INDEX (kc2); # at this point only 7,8,9 should be left.. DROP TABLE t1; +CREATE TABLE t1 ( + c1 BIGINT, + c2 TIMESTAMP DEFAULT NOW() NOT NULL, + name VARCHAR(25) NOT NULL, + event DATE, + PRIMARY KEY (`c1`) COMMENT 'custom_p0_cfname=foo;custom_p1_cfname=bar;custom_p2_cfname=baz;', + KEY kc2 (`c2`) +) ENGINE=ROCKSDB +COMMENT="ttl_duration=1;custom_p1_ttl_duration=100;custom_p1_ttl_col=c2;custom_p2_ttl_duration=5000;" +PARTITION BY LIST(c1) ( + PARTITION custom_p0 VALUES IN (1, 2, 3), + PARTITION custom_p1 VALUES IN (4, 5, 6), + PARTITION custom_p2 VALUES IN (7, 8, 9) +); + +set global rocksdb_debug_ttl_rec_ts = -300; +INSERT INTO t1 VALUES (1, NOW(), "one", null); +INSERT INTO t1 VALUES (2, NOW(), "two", null); +INSERT INTO t1 VALUES (3, NOW(), "three", null); +set global rocksdb_debug_ttl_rec_ts = 0; + +INSERT INTO t1 VALUES (4, NOW(), "four", null); +INSERT INTO t1 VALUES (5, NOW(), "five", null); +INSERT INTO t1 VALUES (6, NOW(), "six", null); + +INSERT INTO t1 VALUES (7, NOW(), "seven", null); +INSERT INTO t1 VALUES (8, NOW(), "eight", null); +INSERT INTO t1 VALUES (9, NOW(), "nine", null); + +set global rocksdb_force_flush_memtable_now=1; +set @@global.rocksdb_compact_cf = 'foo'; +set @@global.rocksdb_compact_cf = 'baz'; +set @@global.rocksdb_compact_cf = 'bar'; +# Filter out expired secondary keys too +set @@global.rocksdb_compact_cf = 'default'; + +# here we expect only 1,2,3 to be gone, ttl implicit. +--sorted_result +SELECT c1 FROM t1 FORCE INDEX (PRIMARY); +--sorted_result +SELECT c1 FROM t1 FORCE INDEX (kc2); + +# here we expect only 4,5,6 to be gone, ttl based on column c2. +set global rocksdb_debug_ttl_snapshot_ts = 600; +set @@global.rocksdb_compact_cf = 'bar'; +# Filter out expired secondary keys too +set @@global.rocksdb_compact_cf = 'default'; +set global rocksdb_debug_ttl_snapshot_ts = 0; +--sorted_result +SELECT c1 FROM t1 FORCE INDEX (PRIMARY); +--sorted_result +SELECT c1 FROM t1 FORCE INDEX (kc2); + +# at this point only 7,8,9 should be left.. +DROP TABLE t1; + # # Add index inplace # diff --git a/storage/rocksdb/mysql-test/rocksdb/t/unique_sec.test b/storage/rocksdb/mysql-test/rocksdb/t/unique_sec.test index 1dedd75f561..03f230fa4c0 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/unique_sec.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/unique_sec.test @@ -49,3 +49,36 @@ CREATE TABLE t (a VARCHAR(255), PRIMARY KEY(a(2)), UNIQUE KEY (a(1))) engine=roc INSERT INTO t VALUES('a'); CHECK TABLE t EXTENDED; DROP TABLE t; + +--echo # +--echo # Validate lock acquired on SK during delete +--echo # +CREATE TABLE t3 (pk INT PRIMARY KEY, sk INT, UNIQUE KEY (sk)) ENGINE=ROCKSDB; +INSERT INTO t3 VALUES (1, 100); + +BEGIN; +--echo # This should acquire a lock on sk = 100 +DELETE FROM t3 WHERE pk = 1; + +connect (con1, localhost, root,,); +connection con1; +set session rocksdb_lock_wait_timeout=1; +set session transaction_isolation='REPEATABLE-READ'; +SELECT * FROM t3; +--echo # RR: This should fail with lock wait timeout due to sk = 100 +--error ER_LOCK_WAIT_TIMEOUT +INSERT INTO t3 VALUES (2, 100); + +set session transaction_isolation='READ-COMMITTED'; +--echo # RC: This should fail with lock wait timeout due to sk = 100 +--error ER_LOCK_WAIT_TIMEOUT +INSERT INTO t3 VALUES (2, 100); + +connection default; +--echo # The original transaction should be able to perform an insert +INSERT INTO t3 VALUES (3, 100); +COMMIT; +SELECT * FROM t3; + +disconnect con1; +DROP TABLE t3; diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/combinations b/storage/rocksdb/mysql-test/rocksdb_rpl/combinations index eae7431662b..8d7ce2e4f90 100644 --- a/storage/rocksdb/mysql-test/rocksdb_rpl/combinations +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/combinations @@ -5,3 +5,8 @@ rocksdb_write_policy=write_committed [row-write-prepared] binlog-format=row rocksdb_write_policy=write_prepared + +[row-write-unprepared] +binlog-format=row +rocksdb_write_policy=write_unprepared +rocksdb_write_batch_flush_threshold=1 diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/r/rocksdb_skip_locks_if_skip_unique_check.result b/storage/rocksdb/mysql-test/rocksdb_rpl/r/rocksdb_skip_locks_if_skip_unique_check.result new file mode 100644 index 00000000000..5098a53ba23 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/r/rocksdb_skip_locks_if_skip_unique_check.result @@ -0,0 +1,37 @@ +include/master-slave.inc +[connection master] +connection master; +create table t1 (a int primary key, b int) engine = rocksdb; +set @@unique_checks = 0; +insert into t1 values(1, 1); +insert into t1 values(2, 2); +include/sync_slave_sql_with_master.inc +connection slave; +begin; +update t1 set b = 20 where a = 2; +connection master; +set @@unique_checks = 0; +insert into t1 values(2, 200); +connection slave; +rollback; +connection slave; +set @@global.rocksdb_skip_locks_if_skip_unique_check = 1; +stop slave; +start slave; +begin; +update t1 set b = 10 where a = 1; +connection master; +set @@unique_checks = 0; +insert into t1 values(1, 100); +include/sync_slave_sql_with_master.inc +connection slave; +rollback; +select * from t1; +a b +1 100 +2 200 +set @@global.rocksdb_skip_locks_if_skip_unique_check = 0; +connection master; +drop table t1; +include/sync_slave_sql_with_master.inc +include/rpl_end.inc diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/r/tx_system_failure.result b/storage/rocksdb/mysql-test/rocksdb_rpl/r/tx_system_failure.result new file mode 100644 index 00000000000..58d72d57f9e --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/r/tx_system_failure.result @@ -0,0 +1,240 @@ +include/master-slave.inc +[connection master] +call mtr.add_suppression("Attempting backtrace"); +call mtr.add_suppression("failed to write to WAL"); +call mtr.add_suppression("aborting on WAL write error"); +create table t1 (id int primary key, value int) engine = rocksdb; +insert into t1 values (1,1),(2,2),(3,3); +set session debug= "+d,myrocks_prepare_io_error"; +insert into t1 values (4,4); +ERROR HY000: Internal error: RocksDB error on COMMIT (Prepare/merge): IO error: +begin; +insert into t1 values (5,5); +insert into t1 values (6,6); +commit; +ERROR HY000: Internal error: RocksDB error on COMMIT (Prepare/merge): IO error: +set session debug= "-d,myrocks_prepare_io_error"; +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +drop table t1; +create table t1 (id int primary key, value int) engine = rocksdb; +insert into t1 values (1,1),(2,2),(3,3); +set session debug= "+d,myrocks_prepare_incomplete"; +insert into t1 values (4,4); +ERROR HY000: Internal error: RocksDB error on COMMIT (Prepare/merge): Result incomplete: +begin; +insert into t1 values (5,5); +insert into t1 values (6,6); +commit; +ERROR HY000: Internal error: RocksDB error on COMMIT (Prepare/merge): Result incomplete: +set session debug= "-d,myrocks_prepare_incomplete"; +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +drop table t1; +connection master; +create table t1 (id int primary key, value int) engine = rocksdb; +insert into t1 values (1,1),(2,2),(3,3); +set session debug= "+d,myrocks_commit_io_error"; +insert into t1 values (4,4); +ERROR HY000: Lost connection to server during query +include/rpl_reconnect.inc +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +4 4 +include/sync_slave_sql_with_master.inc +connection slave; +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +4 4 +connection master; +set session debug= "+d,myrocks_commit_io_error"; +begin; +insert into t1 values (5,5); +insert into t1 values (6,6); +commit; +ERROR HY000: Lost connection to server during query +include/rpl_reconnect.inc +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +include/sync_slave_sql_with_master.inc +connection slave; +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +connection master; +drop table t1; +include/sync_slave_sql_with_master.inc +connection master; +create table t1 (id int primary key, value int) engine = rocksdb; +insert into t1 values (1,1),(2,2),(3,3); +set session debug= "+d,myrocks_commit_incomplete"; +insert into t1 values (4,4); +ERROR HY000: Lost connection to server during query +include/rpl_reconnect.inc +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +4 4 +include/sync_slave_sql_with_master.inc +connection slave; +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +4 4 +connection master; +set session debug= "+d,myrocks_commit_incomplete"; +begin; +insert into t1 values (5,5); +insert into t1 values (6,6); +commit; +ERROR HY000: Lost connection to server during query +include/rpl_reconnect.inc +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +include/sync_slave_sql_with_master.inc +connection slave; +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +connection master; +drop table t1; +include/sync_slave_sql_with_master.inc +connection master; +create table t1 (id int primary key, value int) engine = rocksdb; +insert into t1 values (1,1),(2,2),(3,3); +set session debug= "+d,myrocks_commit_merge_io_error"; +insert into t1 values (4,4); +ERROR HY000: Lost connection to server during query +include/rpl_reconnect.inc +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +4 4 +include/sync_slave_sql_with_master.inc +connection slave; +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +4 4 +connection master; +set session debug= "+d,myrocks_commit_merge_io_error"; +begin; +insert into t1 values (5,5); +insert into t1 values (6,6); +commit; +ERROR HY000: Lost connection to server during query +include/rpl_reconnect.inc +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +include/sync_slave_sql_with_master.inc +connection slave; +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +connection master; +drop table t1; +include/sync_slave_sql_with_master.inc +connection master; +create table t1 (id int primary key, value int) engine = rocksdb; +insert into t1 values (1,1),(2,2),(3,3); +set session debug= "+d,myrocks_commit_merge_incomplete"; +insert into t1 values (4,4); +ERROR HY000: Lost connection to server during query +include/rpl_reconnect.inc +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +4 4 +include/sync_slave_sql_with_master.inc +connection slave; +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +4 4 +connection master; +set session debug= "+d,myrocks_commit_merge_incomplete"; +begin; +insert into t1 values (5,5); +insert into t1 values (6,6); +commit; +ERROR HY000: Lost connection to server during query +include/rpl_reconnect.inc +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +include/sync_slave_sql_with_master.inc +connection slave; +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +connection master; +drop table t1; +include/sync_slave_sql_with_master.inc +include/rpl_end.inc diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/suite.pm b/storage/rocksdb/mysql-test/rocksdb_rpl/suite.pm index 7e9fba4a59c..b5267c2d005 100644 --- a/storage/rocksdb/mysql-test/rocksdb_rpl/suite.pm +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/suite.pm @@ -13,8 +13,8 @@ use strict; my $sst_dump= ::mtr_exe_maybe_exists( - "$::bindir/storage/rocksdb$::multiconfig/sst_dump", - "$::path_client_bindir/sst_dump"); + "$::bindir/storage/rocksdb$::multiconfig/mariadb-sst-dump", + "$::path_client_bindir/mariadb-sst-dump"); return "RocksDB is not compiled, no sst_dump" unless $sst_dump; $ENV{MARIAROCKS_SST_DUMP}="$sst_dump"; diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/t/rocksdb_skip_locks_if_skip_unique_check.test b/storage/rocksdb/mysql-test/rocksdb_rpl/t/rocksdb_skip_locks_if_skip_unique_check.test new file mode 100644 index 00000000000..e8665930bbe --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/t/rocksdb_skip_locks_if_skip_unique_check.test @@ -0,0 +1,48 @@ +source include/master-slave.inc; +source include/have_rocksdb.inc; + +connection master; +create table t1 (a int primary key, b int) engine = rocksdb; +set @@unique_checks = 0; +insert into t1 values(1, 1); +insert into t1 values(2, 2); +source include/sync_slave_sql_with_master.inc; + +connection slave; +begin; +update t1 set b = 20 where a = 2; + +connection master; +set @@unique_checks = 0; +insert into t1 values(2, 200); + +connection slave; +let $wait_condition= + select count(*)= 1 from information_schema.processlist + where state = 'Waiting for row lock'; +source include/wait_condition.inc; +rollback; + + +# Now let's check if locks are not taken when # rocksdb_skip_locks_if_skip_unique_check is enabled +connection slave; +set @@global.rocksdb_skip_locks_if_skip_unique_check = 1; +stop slave; start slave; +begin; +update t1 set b = 10 where a = 1; + +connection master; +set @@unique_checks = 0; +insert into t1 values(1, 100); +source include/sync_slave_sql_with_master.inc; + +connection slave; +rollback; +select * from t1; +set @@global.rocksdb_skip_locks_if_skip_unique_check = 0; + +connection master; +drop table t1; +source include/sync_slave_sql_with_master.inc; + +source include/rpl_end.inc; diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_gtid_crash_safe_wal_corrupt.inc b/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_gtid_crash_safe_wal_corrupt.inc index 16ad535ff9e..935e8362f9b 100644 --- a/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_gtid_crash_safe_wal_corrupt.inc +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_gtid_crash_safe_wal_corrupt.inc @@ -3,6 +3,7 @@ source include/master-slave.inc; -- let $uuid = `select @@server_uuid;` --exec echo > $MYSQLTEST_VARDIR/log/mysqld.1.err +--exec echo > $MYSQLTEST_VARDIR/log/mysqld.2.err connection master; --disable_warnings @@ -26,6 +27,7 @@ sync_slave_with_master; connection slave; --let slave_data_dir= query_get_value(SELECT @@DATADIR, @@DATADIR, 1) --let slave_pid_file= query_get_value(SELECT @@pid_file, @@pid_file, 1) +--let write_policy= query_get_value(SELECT @@rocksdb_write_policy, @@rocksdb_write_policy, 1) --disable_query_log select "--- slave state before crash ---" as ""; --enable_query_log @@ -42,7 +44,19 @@ select * from mysql.slave_gtid_info; F=`ls -t $slave_data_dir/\#rocksdb/*.log | head -n 1` SIZE=`stat -c %s $F` -NEW_SIZE=`expr $SIZE - 30` + +case $1 in + write_committed) + NEW_SIZE=`expr $SIZE - 30` + ;; + write_prepared) + NEW_SIZE=`expr $SIZE - 30` + ;; + write_unprepared) + NEW_SIZE=`expr $SIZE - 200` + ;; +esac + truncate -s $NEW_SIZE $F rc=$? if [[ $rc != 0 ]]; then @@ -54,7 +68,7 @@ kill -9 `head -1 $slave_pid_file` exit 0 EOF --chmod 0755 $MYSQL_TMP_DIR/truncate_tail_wal.sh ---exec $MYSQL_TMP_DIR/truncate_tail_wal.sh +--exec $MYSQL_TMP_DIR/truncate_tail_wal.sh $write_policy --let $rpl_skip_start_slave= 1 --source include/rpl_start_server.inc @@ -118,7 +132,18 @@ connection slave; # expected to be around 950 bytes F=`ls -t $slave_data_dir/\#rocksdb/*.log | head -n 1` SIZE=`stat -c %s $F` -OFFSET=$(( $SIZE-500 )) +case $1 in + write_committed) + OFFSET=$(( $SIZE-500 )) + ;; + write_prepared) + OFFSET=$(( $SIZE-500 )) + ;; + write_unprepared) + OFFSET=$(( $SIZE-1000 )) + ;; +esac + dd bs=1 if=/dev/zero of=$F count=100 seek=$OFFSET conv=notrunc kill -9 `head -1 $slave_pid_file` @@ -126,7 +151,7 @@ kill -9 `head -1 $slave_pid_file` exit 0 EOF --chmod 0755 $MYSQL_TMP_DIR/corrupt_wal.sh ---exec $MYSQL_TMP_DIR/corrupt_wal.sh +--exec $MYSQL_TMP_DIR/corrupt_wal.sh $write_policy --let $rpl_skip_start_slave= 1 --source include/rpl_start_server.inc diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/t/tx_system_failure.test b/storage/rocksdb/mysql-test/rocksdb_rpl/t/tx_system_failure.test new file mode 100644 index 00000000000..4bf2e955176 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/t/tx_system_failure.test @@ -0,0 +1,38 @@ +--disable_warnings +--source include/have_rocksdb.inc +--source include/have_binlog_format_row.inc +--source include/master-slave.inc +--source include/not_asan.inc +--source include/have_debug.inc +--source include/not_valgrind.inc +--source include/not_windows.inc + + +call mtr.add_suppression("Attempting backtrace"); +call mtr.add_suppression("failed to write to WAL"); +call mtr.add_suppression("aborting on WAL write error"); + +# systems errors on prepare should just roll back +let $myrocks_debug_set_code = "+d,myrocks_prepare_io_error"; +let $myrocks_debug_unset_code = "-d,myrocks_prepare_io_error"; +--source tx_system_failure_prepare.inc + +let $myrocks_debug_set_code = "+d,myrocks_prepare_incomplete"; +let $myrocks_debug_unset_code = "-d,myrocks_prepare_incomplete"; +--source tx_system_failure_prepare.inc + +# systems errors on commit should abort, then roll forward, +# replication should consistent +let $myrocks_debug_set_code = "+d,myrocks_commit_io_error"; +--source tx_system_failure_commit.inc + +let $myrocks_debug_set_code = "+d,myrocks_commit_incomplete"; +--source tx_system_failure_commit.inc + +let $myrocks_debug_set_code = "+d,myrocks_commit_merge_io_error"; +--source tx_system_failure_commit.inc + +let $myrocks_debug_set_code = "+d,myrocks_commit_merge_incomplete"; +--source tx_system_failure_commit.inc + +--source include/rpl_end.inc diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/t/tx_system_failure_commit.inc b/storage/rocksdb/mysql-test/rocksdb_rpl/t/tx_system_failure_commit.inc new file mode 100644 index 00000000000..7962b2cc3b3 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/t/tx_system_failure_commit.inc @@ -0,0 +1,52 @@ +connection master; +create table t1 (id int primary key, value int) engine = rocksdb; + +insert into t1 values (1,1),(2,2),(3,3); + +# auto commit failure at engine commit, crash and restart (roll forward with binlog) +--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +eval set session debug= $myrocks_debug_set_code; +--error 2013 +insert into t1 values (4,4); + +--enable_reconnect +--source include/wait_until_connected_again.inc + +# Primary instance restart needs this, otherwise rpl_end fails. +--let $rpl_server_number = 1 +--source include/rpl_reconnect.inc + +select * from t1 order by id; +source include/sync_slave_sql_with_master.inc; + +connection slave; +select * from t1 order by id; + + +connection master; +# tx failure at engine commit, crash and restart (roll forward with binlog) +--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +eval set session debug= $myrocks_debug_set_code; +begin; +insert into t1 values (5,5); +insert into t1 values (6,6); +--error 2013 +commit; + +--enable_reconnect +--source include/wait_until_connected_again.inc + +# Primary instance restart needs this, otherwise rpl_end fails. +--let $rpl_server_number = 1 +--source include/rpl_reconnect.inc + +select * from t1 order by id; +source include/sync_slave_sql_with_master.inc; + +connection slave; +select * from t1 order by id; + + +connection master; +drop table t1; +source include/sync_slave_sql_with_master.inc; diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/t/tx_system_failure_prepare.inc b/storage/rocksdb/mysql-test/rocksdb_rpl/t/tx_system_failure_prepare.inc new file mode 100644 index 00000000000..bb72dcb5f30 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/t/tx_system_failure_prepare.inc @@ -0,0 +1,17 @@ +create table t1 (id int primary key, value int) engine = rocksdb; + +insert into t1 values (1,1),(2,2),(3,3); + +eval set session debug= $myrocks_debug_set_code; +--error ER_INTERNAL_ERROR +insert into t1 values (4,4); + +begin; +insert into t1 values (5,5); +insert into t1 values (6,6); +--error ER_INTERNAL_ERROR +commit; +eval set session debug= $myrocks_debug_unset_code; +select * from t1 order by id; + +drop table t1; diff --git a/storage/rocksdb/mysql-test/rocksdb_stress/suite.pm b/storage/rocksdb/mysql-test/rocksdb_stress/suite.pm index 66a46291a8c..c599d8c4e36 100644 --- a/storage/rocksdb/mysql-test/rocksdb_stress/suite.pm +++ b/storage/rocksdb/mysql-test/rocksdb_stress/suite.pm @@ -15,8 +15,8 @@ use strict; my $sst_dump= ::mtr_exe_maybe_exists( - "$::bindir/storage/rocksdb$::multiconfig/sst_dump", - "$::path_client_bindir/sst_dump"); + "$::bindir/storage/rocksdb$::multiconfig/mariadb-sst-dump", + "$::path_client_bindir/mariadb-sst-dump"); return "RocksDB is not compiled, no sst_dump" unless $sst_dump; $ENV{MARIAROCKS_SST_DUMP}="$sst_dump"; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_cancel_manual_compactions_basic.result b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_cancel_manual_compactions_basic.result new file mode 100644 index 00000000000..f3ae1da7d68 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_cancel_manual_compactions_basic.result @@ -0,0 +1,50 @@ +CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO valid_values VALUES(1); +INSERT INTO valid_values VALUES(0); +INSERT INTO valid_values VALUES('on'); +CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; +SET @start_global_value = @@global.ROCKSDB_CANCEL_MANUAL_COMPACTIONS; +SELECT @start_global_value; +@start_global_value +0 +'# Setting to valid values in global scope#' +"Trying to set variable @@global.ROCKSDB_CANCEL_MANUAL_COMPACTIONS to 1" +SET @@global.ROCKSDB_CANCEL_MANUAL_COMPACTIONS = 1; +SELECT @@global.ROCKSDB_CANCEL_MANUAL_COMPACTIONS; +@@global.ROCKSDB_CANCEL_MANUAL_COMPACTIONS +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_CANCEL_MANUAL_COMPACTIONS = DEFAULT; +SELECT @@global.ROCKSDB_CANCEL_MANUAL_COMPACTIONS; +@@global.ROCKSDB_CANCEL_MANUAL_COMPACTIONS +0 +"Trying to set variable @@global.ROCKSDB_CANCEL_MANUAL_COMPACTIONS to 0" +SET @@global.ROCKSDB_CANCEL_MANUAL_COMPACTIONS = 0; +SELECT @@global.ROCKSDB_CANCEL_MANUAL_COMPACTIONS; +@@global.ROCKSDB_CANCEL_MANUAL_COMPACTIONS +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_CANCEL_MANUAL_COMPACTIONS = DEFAULT; +SELECT @@global.ROCKSDB_CANCEL_MANUAL_COMPACTIONS; +@@global.ROCKSDB_CANCEL_MANUAL_COMPACTIONS +0 +"Trying to set variable @@global.ROCKSDB_CANCEL_MANUAL_COMPACTIONS to on" +SET @@global.ROCKSDB_CANCEL_MANUAL_COMPACTIONS = on; +SELECT @@global.ROCKSDB_CANCEL_MANUAL_COMPACTIONS; +@@global.ROCKSDB_CANCEL_MANUAL_COMPACTIONS +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_CANCEL_MANUAL_COMPACTIONS = DEFAULT; +SELECT @@global.ROCKSDB_CANCEL_MANUAL_COMPACTIONS; +@@global.ROCKSDB_CANCEL_MANUAL_COMPACTIONS +0 +"Trying to set variable @@session.ROCKSDB_CANCEL_MANUAL_COMPACTIONS to 444. It should fail because it is not session." +SET @@session.ROCKSDB_CANCEL_MANUAL_COMPACTIONS = 444; +ERROR HY000: Variable 'rocksdb_cancel_manual_compactions' is a GLOBAL variable and should be set with SET GLOBAL +'# Testing with invalid values in global scope #' +SET @@global.ROCKSDB_CANCEL_MANUAL_COMPACTIONS = @start_global_value; +SELECT @@global.ROCKSDB_CANCEL_MANUAL_COMPACTIONS; +@@global.ROCKSDB_CANCEL_MANUAL_COMPACTIONS +0 +DROP TABLE valid_values; +DROP TABLE invalid_values; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_delete_cf_basic.result b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_delete_cf_basic.result index b840baf29f8..2987f378fb9 100644 --- a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_delete_cf_basic.result +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_delete_cf_basic.result @@ -1,6 +1,32 @@ SET @@global.ROCKSDB_DELETE_CF = 'nonexistent_cf'; -ERROR 42000: Variable 'rocksdb_delete_cf' can't be set to the value of 'nonexistent_cf' +ERROR HY000: Cannot drop Column family ('nonexistent_cf') because it is in use or does not exist. SET @@global.ROCKSDB_DELETE_CF = '__system__'; -ERROR 42000: Variable 'rocksdb_delete_cf' can't be set to the value of '__system__' +ERROR HY000: Cannot drop Column family ('__system__') because it is in use or does not exist. +select count(*) from information_schema.rocksdb_ddl where cf = 'default'; +count(*) +0 SET @@global.ROCKSDB_DELETE_CF = 'default'; -ERROR 42000: Variable 'rocksdb_delete_cf' can't be set to the value of 'default' +ERROR HY000: Cannot drop Column family ('default') because it is in use or does not exist. +SET @@global.ROCKSDB_DELETE_CF = ''; +ERROR HY000: Cannot drop Column family ('') because it is in use or does not exist. +SET @@global.ROCKSDB_DELETE_CF = NULL; +CREATE TABLE cf_deletion_test_table1 ( +id1 int(10) unsigned NOT NULL DEFAULT '0', +id2 int(10) unsigned NOT NULL DEFAULT '0', +PRIMARY KEY (id1) COMMENT 'cf_primary_key', +KEY `sec_key` (id2) COMMENT 'cf_secondary_key' +) ENGINE=ROCKSDB; +SET @@global.ROCKSDB_DELETE_CF = 'cf_primary_key'; +ERROR HY000: Cannot drop Column family ('cf_primary_key') because it is in use or does not exist. +SET @@global.ROCKSDB_DELETE_CF = 'cf_secondary_key'; +ERROR HY000: Cannot drop Column family ('cf_secondary_key') because it is in use or does not exist. +CREATE TABLE cf_deletion_test_table2 ( +id1 int(10) unsigned NOT NULL DEFAULT '0', +PRIMARY KEY (id1) COMMENT 'cf_primary_key' +) ENGINE=ROCKSDB; +DROP TABLE cf_deletion_test_table1; +SET @@global.ROCKSDB_DELETE_CF = 'cf_primary_key'; +ERROR HY000: Cannot drop Column family ('cf_primary_key') because it is in use or does not exist. +SET @@global.ROCKSDB_DELETE_CF = 'cf_secondary_key'; +DROP TABLE cf_deletion_test_table2; +SET @@global.ROCKSDB_DELETE_CF = 'cf_primary_key'; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_enable_iterate_bounds_basic.result b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_enable_iterate_bounds_basic.result new file mode 100644 index 00000000000..d8534cc8cf4 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_enable_iterate_bounds_basic.result @@ -0,0 +1,35 @@ +connect con1, localhost, root,,; +connect con2, localhost, root,,; +connection con2; +SET SESSION rocksdb_enable_iterate_bounds=1; +connection con1; +SET SESSION rocksdb_enable_iterate_bounds=0; +SELECT @@rocksdb_enable_iterate_bounds; +@@rocksdb_enable_iterate_bounds +0 +DROP TABLE IF EXISTS t; +Warnings: +Note 1051 Unknown table 'test.t' +SELECT COUNT(*) FROM t WHERE a = 1; +COUNT(*) +1 +SELECT COUNT(*) FROM t WHERE a >= 1 AND a <= 32; +COUNT(*) +32 +SELECT COUNT(*) FROM t; +COUNT(*) +64 +connection con2; +SELECT @@rocksdb_enable_iterate_bounds; +@@rocksdb_enable_iterate_bounds +1 +SELECT COUNT(*) FROM t WHERE a = 1; +COUNT(*) +1 +SELECT COUNT(*) FROM t WHERE a >= 1 AND a <= 32; +COUNT(*) +32 +SELECT COUNT(*) FROM t; +COUNT(*) +64 +DROP TABLE t; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_enable_pipelined_write_basic.result b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_enable_pipelined_write_basic.result new file mode 100644 index 00000000000..ccf4c597967 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_enable_pipelined_write_basic.result @@ -0,0 +1,7 @@ +SET @start_global_value = @@global.ROCKSDB_ENABLE_PIPELINED_WRITE; +SELECT @start_global_value; +@start_global_value +0 +"Trying to set variable @@global.ROCKSDB_ENABLE_PIPELINED_WRITE to 444. It should fail because it is readonly." +SET @@global.ROCKSDB_ENABLE_PIPELINED_WRITE = 444; +ERROR HY000: Variable 'rocksdb_enable_pipelined_write' is a read only variable diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_enable_remove_orphaned_dropped_cfs_basic.result b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_enable_remove_orphaned_dropped_cfs_basic.result new file mode 100644 index 00000000000..ff36fbb6fe8 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_enable_remove_orphaned_dropped_cfs_basic.result @@ -0,0 +1,97 @@ +CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO valid_values VALUES(1); +INSERT INTO valid_values VALUES(0); +INSERT INTO valid_values VALUES('on'); +INSERT INTO valid_values VALUES('off'); +INSERT INTO valid_values VALUES('true'); +INSERT INTO valid_values VALUES('false'); +CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO invalid_values VALUES('\'aaa\''); +INSERT INTO invalid_values VALUES('\'bbb\''); +SET @start_global_value = @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS; +SELECT @start_global_value; +@start_global_value +1 +'# Setting to valid values in global scope#' +"Trying to set variable @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS to 1" +SET @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS = 1; +SELECT @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS; +@@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS +1 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS = DEFAULT; +SELECT @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS; +@@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS +1 +"Trying to set variable @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS to 0" +SET @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS = 0; +SELECT @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS; +@@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS = DEFAULT; +SELECT @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS; +@@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS +1 +"Trying to set variable @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS to on" +SET @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS = on; +SELECT @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS; +@@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS +1 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS = DEFAULT; +SELECT @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS; +@@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS +1 +"Trying to set variable @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS to off" +SET @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS = off; +SELECT @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS; +@@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS = DEFAULT; +SELECT @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS; +@@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS +1 +"Trying to set variable @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS to true" +SET @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS = true; +SELECT @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS; +@@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS +1 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS = DEFAULT; +SELECT @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS; +@@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS +1 +"Trying to set variable @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS to false" +SET @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS = false; +SELECT @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS; +@@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS = DEFAULT; +SELECT @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS; +@@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS +1 +"Trying to set variable @@session.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS to 444. It should fail because it is not session." +SET @@session.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS = 444; +ERROR HY000: Variable 'rocksdb_enable_remove_orphaned_dropped_cfs' is a GLOBAL variable and should be set with SET GLOBAL +'# Testing with invalid values in global scope #' +"Trying to set variable @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS to 'aaa'" +SET @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS = 'aaa'; +Got one of the listed errors +SELECT @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS; +@@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS +1 +"Trying to set variable @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS to 'bbb'" +SET @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS = 'bbb'; +Got one of the listed errors +SELECT @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS; +@@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS +1 +SET @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS = @start_global_value; +SELECT @@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS; +@@global.ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS +1 +DROP TABLE valid_values; +DROP TABLE invalid_values; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_manual_compaction_bottommost_level_basic.result b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_manual_compaction_bottommost_level_basic.result new file mode 100644 index 00000000000..dd666bd1cbe --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_manual_compaction_bottommost_level_basic.result @@ -0,0 +1,114 @@ +CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO valid_values VALUES('kSkip'); +INSERT INTO valid_values VALUES('kIfHaveCompactionFilter'); +INSERT INTO valid_values VALUES('kForce'); +INSERT INTO valid_values VALUES('kForceOptimized'); +CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO invalid_values VALUES('\'aaa\''); +SET @start_global_value = @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +SELECT @start_global_value; +@start_global_value +kForceOptimized +SET @start_session_value = @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +SELECT @start_session_value; +@start_session_value +kForceOptimized +'# Setting to valid values in global scope#' +"Trying to set variable @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL to kSkip" +SET @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = kSkip; +SELECT @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kSkip +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = DEFAULT; +SELECT @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +"Trying to set variable @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL to kIfHaveCompactionFilter" +SET @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = kIfHaveCompactionFilter; +SELECT @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kIfHaveCompactionFilter +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = DEFAULT; +SELECT @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +"Trying to set variable @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL to kForce" +SET @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = kForce; +SELECT @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForce +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = DEFAULT; +SELECT @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +"Trying to set variable @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL to kForceOptimized" +SET @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = kForceOptimized; +SELECT @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = DEFAULT; +SELECT @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +'# Setting to valid values in session scope#' +"Trying to set variable @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL to kSkip" +SET @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = kSkip; +SELECT @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kSkip +"Setting the session scope variable back to default" +SET @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = DEFAULT; +SELECT @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +"Trying to set variable @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL to kIfHaveCompactionFilter" +SET @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = kIfHaveCompactionFilter; +SELECT @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kIfHaveCompactionFilter +"Setting the session scope variable back to default" +SET @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = DEFAULT; +SELECT @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +"Trying to set variable @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL to kForce" +SET @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = kForce; +SELECT @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForce +"Setting the session scope variable back to default" +SET @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = DEFAULT; +SELECT @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +"Trying to set variable @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL to kForceOptimized" +SET @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = kForceOptimized; +SELECT @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +"Setting the session scope variable back to default" +SET @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = DEFAULT; +SELECT @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +'# Testing with invalid values in global scope #' +"Trying to set variable @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL to 'aaa'" +SET @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = 'aaa'; +Got one of the listed errors +SELECT @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +SET @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = @start_global_value; +SELECT @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +SET @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = @start_session_value; +SELECT @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +DROP TABLE valid_values; +DROP TABLE invalid_values; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_max_background_compactions_basic.result b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_max_background_compactions_basic.result new file mode 100644 index 00000000000..60911d82d8c --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_max_background_compactions_basic.result @@ -0,0 +1,46 @@ +CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO valid_values VALUES(1); +INSERT INTO valid_values VALUES(64); +CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO invalid_values VALUES('\'abc\''); +SET @start_global_value = @@global.ROCKSDB_MAX_BACKGROUND_COMPACTIONS; +SELECT @start_global_value; +@start_global_value +-1 +'# Setting to valid values in global scope#' +"Trying to set variable @@global.ROCKSDB_MAX_BACKGROUND_COMPACTIONS to 1" +SET @@global.ROCKSDB_MAX_BACKGROUND_COMPACTIONS = 1; +SELECT @@global.ROCKSDB_MAX_BACKGROUND_COMPACTIONS; +@@global.ROCKSDB_MAX_BACKGROUND_COMPACTIONS +1 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_MAX_BACKGROUND_COMPACTIONS = DEFAULT; +SELECT @@global.ROCKSDB_MAX_BACKGROUND_COMPACTIONS; +@@global.ROCKSDB_MAX_BACKGROUND_COMPACTIONS +-1 +"Trying to set variable @@global.ROCKSDB_MAX_BACKGROUND_COMPACTIONS to 64" +SET @@global.ROCKSDB_MAX_BACKGROUND_COMPACTIONS = 64; +SELECT @@global.ROCKSDB_MAX_BACKGROUND_COMPACTIONS; +@@global.ROCKSDB_MAX_BACKGROUND_COMPACTIONS +64 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_MAX_BACKGROUND_COMPACTIONS = DEFAULT; +SELECT @@global.ROCKSDB_MAX_BACKGROUND_COMPACTIONS; +@@global.ROCKSDB_MAX_BACKGROUND_COMPACTIONS +-1 +"Trying to set variable @@session.ROCKSDB_MAX_BACKGROUND_COMPACTIONS to 444. It should fail because it is not session." +SET @@session.ROCKSDB_MAX_BACKGROUND_COMPACTIONS = 444; +ERROR HY000: Variable 'rocksdb_max_background_compactions' is a GLOBAL variable and should be set with SET GLOBAL +'# Testing with invalid values in global scope #' +"Trying to set variable @@global.ROCKSDB_MAX_BACKGROUND_COMPACTIONS to 'abc'" +SET @@global.ROCKSDB_MAX_BACKGROUND_COMPACTIONS = 'abc'; +Got one of the listed errors +SELECT @@global.ROCKSDB_MAX_BACKGROUND_COMPACTIONS; +@@global.ROCKSDB_MAX_BACKGROUND_COMPACTIONS +-1 +SET @@global.ROCKSDB_MAX_BACKGROUND_COMPACTIONS = @start_global_value; +SELECT @@global.ROCKSDB_MAX_BACKGROUND_COMPACTIONS; +@@global.ROCKSDB_MAX_BACKGROUND_COMPACTIONS +-1 +DROP TABLE valid_values; +DROP TABLE invalid_values; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_max_background_flushes_basic.result b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_max_background_flushes_basic.result new file mode 100644 index 00000000000..7d607ad2742 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_max_background_flushes_basic.result @@ -0,0 +1,14 @@ +CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO valid_values VALUES(1); +INSERT INTO valid_values VALUES(64); +CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO invalid_values VALUES('\'abc\''); +SET @start_global_value = @@global.ROCKSDB_MAX_BACKGROUND_FLUSHES; +SELECT @start_global_value; +@start_global_value +-1 +"Trying to set variable @@global.ROCKSDB_MAX_BACKGROUND_FLUSHES to 444. It should fail because it is readonly." +SET @@global.ROCKSDB_MAX_BACKGROUND_FLUSHES = 444; +ERROR HY000: Variable 'rocksdb_max_background_flushes' is a read only variable +DROP TABLE valid_values; +DROP TABLE invalid_values; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_max_bottom_pri_background_compactions_basic.result b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_max_bottom_pri_background_compactions_basic.result new file mode 100644 index 00000000000..2ad5a6808a6 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_max_bottom_pri_background_compactions_basic.result @@ -0,0 +1,46 @@ +CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO valid_values VALUES(1); +INSERT INTO valid_values VALUES(64); +CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO invalid_values VALUES('\'abc\''); +SET @start_global_value = @@global.ROCKSDB_MAX_BOTTOM_PRI_BACKGROUND_COMPACTIONS; +SELECT @start_global_value; +@start_global_value +0 +'# Setting to valid values in global scope#' +"Trying to set variable @@global.ROCKSDB_MAX_BOTTOM_PRI_BACKGROUND_COMPACTIONS to 1" +SET @@global.ROCKSDB_MAX_BOTTOM_PRI_BACKGROUND_COMPACTIONS = 1; +SELECT @@global.ROCKSDB_MAX_BOTTOM_PRI_BACKGROUND_COMPACTIONS; +@@global.ROCKSDB_MAX_BOTTOM_PRI_BACKGROUND_COMPACTIONS +1 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_MAX_BOTTOM_PRI_BACKGROUND_COMPACTIONS = DEFAULT; +SELECT @@global.ROCKSDB_MAX_BOTTOM_PRI_BACKGROUND_COMPACTIONS; +@@global.ROCKSDB_MAX_BOTTOM_PRI_BACKGROUND_COMPACTIONS +0 +"Trying to set variable @@global.ROCKSDB_MAX_BOTTOM_PRI_BACKGROUND_COMPACTIONS to 64" +SET @@global.ROCKSDB_MAX_BOTTOM_PRI_BACKGROUND_COMPACTIONS = 64; +SELECT @@global.ROCKSDB_MAX_BOTTOM_PRI_BACKGROUND_COMPACTIONS; +@@global.ROCKSDB_MAX_BOTTOM_PRI_BACKGROUND_COMPACTIONS +64 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_MAX_BOTTOM_PRI_BACKGROUND_COMPACTIONS = DEFAULT; +SELECT @@global.ROCKSDB_MAX_BOTTOM_PRI_BACKGROUND_COMPACTIONS; +@@global.ROCKSDB_MAX_BOTTOM_PRI_BACKGROUND_COMPACTIONS +0 +"Trying to set variable @@session.ROCKSDB_MAX_BOTTOM_PRI_BACKGROUND_COMPACTIONS to 444. It should fail because it is not session." +SET @@session.ROCKSDB_MAX_BOTTOM_PRI_BACKGROUND_COMPACTIONS = 444; +ERROR HY000: Variable 'rocksdb_max_bottom_pri_background_compactions' is a GLOBAL variable and should be set with SET GLOBAL +'# Testing with invalid values in global scope #' +"Trying to set variable @@global.ROCKSDB_MAX_BOTTOM_PRI_BACKGROUND_COMPACTIONS to 'abc'" +SET @@global.ROCKSDB_MAX_BOTTOM_PRI_BACKGROUND_COMPACTIONS = 'abc'; +Got one of the listed errors +SELECT @@global.ROCKSDB_MAX_BOTTOM_PRI_BACKGROUND_COMPACTIONS; +@@global.ROCKSDB_MAX_BOTTOM_PRI_BACKGROUND_COMPACTIONS +0 +SET @@global.ROCKSDB_MAX_BOTTOM_PRI_BACKGROUND_COMPACTIONS = @start_global_value; +SELECT @@global.ROCKSDB_MAX_BOTTOM_PRI_BACKGROUND_COMPACTIONS; +@@global.ROCKSDB_MAX_BOTTOM_PRI_BACKGROUND_COMPACTIONS +0 +DROP TABLE valid_values; +DROP TABLE invalid_values; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_max_row_locks_basic.result b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_max_row_locks_basic.result index b195df092dc..331a8f313cf 100644 --- a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_max_row_locks_basic.result +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_max_row_locks_basic.result @@ -8,10 +8,6 @@ SET @start_global_value = @@global.ROCKSDB_MAX_ROW_LOCKS; SELECT @start_global_value; @start_global_value 1048576 -SET @start_session_value = @@session.ROCKSDB_MAX_ROW_LOCKS; -SELECT @start_session_value; -@start_session_value -1048576 '# Setting to valid values in global scope#' "Trying to set variable @@global.ROCKSDB_MAX_ROW_LOCKS to 1" SET @@global.ROCKSDB_MAX_ROW_LOCKS = 1; @@ -43,37 +39,9 @@ SET @@global.ROCKSDB_MAX_ROW_LOCKS = DEFAULT; SELECT @@global.ROCKSDB_MAX_ROW_LOCKS; @@global.ROCKSDB_MAX_ROW_LOCKS 1048576 -'# Setting to valid values in session scope#' -"Trying to set variable @@session.ROCKSDB_MAX_ROW_LOCKS to 1" -SET @@session.ROCKSDB_MAX_ROW_LOCKS = 1; -SELECT @@session.ROCKSDB_MAX_ROW_LOCKS; -@@session.ROCKSDB_MAX_ROW_LOCKS -1 -"Setting the session scope variable back to default" -SET @@session.ROCKSDB_MAX_ROW_LOCKS = DEFAULT; -SELECT @@session.ROCKSDB_MAX_ROW_LOCKS; -@@session.ROCKSDB_MAX_ROW_LOCKS -1048576 -"Trying to set variable @@session.ROCKSDB_MAX_ROW_LOCKS to 1024" -SET @@session.ROCKSDB_MAX_ROW_LOCKS = 1024; -SELECT @@session.ROCKSDB_MAX_ROW_LOCKS; -@@session.ROCKSDB_MAX_ROW_LOCKS -1024 -"Setting the session scope variable back to default" -SET @@session.ROCKSDB_MAX_ROW_LOCKS = DEFAULT; -SELECT @@session.ROCKSDB_MAX_ROW_LOCKS; -@@session.ROCKSDB_MAX_ROW_LOCKS -1048576 -"Trying to set variable @@session.ROCKSDB_MAX_ROW_LOCKS to 536870912" -SET @@session.ROCKSDB_MAX_ROW_LOCKS = 536870912; -SELECT @@session.ROCKSDB_MAX_ROW_LOCKS; -@@session.ROCKSDB_MAX_ROW_LOCKS -536870912 -"Setting the session scope variable back to default" -SET @@session.ROCKSDB_MAX_ROW_LOCKS = DEFAULT; -SELECT @@session.ROCKSDB_MAX_ROW_LOCKS; -@@session.ROCKSDB_MAX_ROW_LOCKS -1048576 +"Trying to set variable @@session.ROCKSDB_MAX_ROW_LOCKS to 444. It should fail because it is not session." +SET @@session.ROCKSDB_MAX_ROW_LOCKS = 444; +ERROR HY000: Variable 'rocksdb_max_row_locks' is a GLOBAL variable and should be set with SET GLOBAL '# Testing with invalid values in global scope #' "Trying to set variable @@global.ROCKSDB_MAX_ROW_LOCKS to 'aaa'" SET @@global.ROCKSDB_MAX_ROW_LOCKS = 'aaa'; @@ -85,9 +53,5 @@ SET @@global.ROCKSDB_MAX_ROW_LOCKS = @start_global_value; SELECT @@global.ROCKSDB_MAX_ROW_LOCKS; @@global.ROCKSDB_MAX_ROW_LOCKS 1048576 -SET @@session.ROCKSDB_MAX_ROW_LOCKS = @start_session_value; -SELECT @@session.ROCKSDB_MAX_ROW_LOCKS; -@@session.ROCKSDB_MAX_ROW_LOCKS -1048576 DROP TABLE valid_values; DROP TABLE invalid_values; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_skip_locks_if_skip_unique_check_basic.result b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_skip_locks_if_skip_unique_check_basic.result new file mode 100644 index 00000000000..96b78cf669e --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_skip_locks_if_skip_unique_check_basic.result @@ -0,0 +1,100 @@ +CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO valid_values VALUES(1); +INSERT INTO valid_values VALUES(0); +INSERT INTO valid_values VALUES('on'); +CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO invalid_values VALUES('\'aaa\''); +INSERT INTO invalid_values VALUES('\'bbb\''); +SET @start_global_value = @@global.ROCKSDB_BULK_LOAD; +SELECT @start_global_value; +@start_global_value +0 +SET @start_session_value = @@session.ROCKSDB_BULK_LOAD; +SELECT @start_session_value; +@start_session_value +0 +'# Setting to valid values in global scope#' +"Trying to set variable @@global.ROCKSDB_BULK_LOAD to 1" +SET @@global.ROCKSDB_BULK_LOAD = 1; +SELECT @@global.ROCKSDB_BULK_LOAD; +@@global.ROCKSDB_BULK_LOAD +1 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_BULK_LOAD = DEFAULT; +SELECT @@global.ROCKSDB_BULK_LOAD; +@@global.ROCKSDB_BULK_LOAD +0 +"Trying to set variable @@global.ROCKSDB_BULK_LOAD to 0" +SET @@global.ROCKSDB_BULK_LOAD = 0; +SELECT @@global.ROCKSDB_BULK_LOAD; +@@global.ROCKSDB_BULK_LOAD +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_BULK_LOAD = DEFAULT; +SELECT @@global.ROCKSDB_BULK_LOAD; +@@global.ROCKSDB_BULK_LOAD +0 +"Trying to set variable @@global.ROCKSDB_BULK_LOAD to on" +SET @@global.ROCKSDB_BULK_LOAD = on; +SELECT @@global.ROCKSDB_BULK_LOAD; +@@global.ROCKSDB_BULK_LOAD +1 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_BULK_LOAD = DEFAULT; +SELECT @@global.ROCKSDB_BULK_LOAD; +@@global.ROCKSDB_BULK_LOAD +0 +'# Setting to valid values in session scope#' +"Trying to set variable @@session.ROCKSDB_BULK_LOAD to 1" +SET @@session.ROCKSDB_BULK_LOAD = 1; +SELECT @@session.ROCKSDB_BULK_LOAD; +@@session.ROCKSDB_BULK_LOAD +1 +"Setting the session scope variable back to default" +SET @@session.ROCKSDB_BULK_LOAD = DEFAULT; +SELECT @@session.ROCKSDB_BULK_LOAD; +@@session.ROCKSDB_BULK_LOAD +0 +"Trying to set variable @@session.ROCKSDB_BULK_LOAD to 0" +SET @@session.ROCKSDB_BULK_LOAD = 0; +SELECT @@session.ROCKSDB_BULK_LOAD; +@@session.ROCKSDB_BULK_LOAD +0 +"Setting the session scope variable back to default" +SET @@session.ROCKSDB_BULK_LOAD = DEFAULT; +SELECT @@session.ROCKSDB_BULK_LOAD; +@@session.ROCKSDB_BULK_LOAD +0 +"Trying to set variable @@session.ROCKSDB_BULK_LOAD to on" +SET @@session.ROCKSDB_BULK_LOAD = on; +SELECT @@session.ROCKSDB_BULK_LOAD; +@@session.ROCKSDB_BULK_LOAD +1 +"Setting the session scope variable back to default" +SET @@session.ROCKSDB_BULK_LOAD = DEFAULT; +SELECT @@session.ROCKSDB_BULK_LOAD; +@@session.ROCKSDB_BULK_LOAD +0 +'# Testing with invalid values in global scope #' +"Trying to set variable @@global.ROCKSDB_BULK_LOAD to 'aaa'" +SET @@global.ROCKSDB_BULK_LOAD = 'aaa'; +Got one of the listed errors +SELECT @@global.ROCKSDB_BULK_LOAD; +@@global.ROCKSDB_BULK_LOAD +0 +"Trying to set variable @@global.ROCKSDB_BULK_LOAD to 'bbb'" +SET @@global.ROCKSDB_BULK_LOAD = 'bbb'; +Got one of the listed errors +SELECT @@global.ROCKSDB_BULK_LOAD; +@@global.ROCKSDB_BULK_LOAD +0 +SET @@global.ROCKSDB_BULK_LOAD = @start_global_value; +SELECT @@global.ROCKSDB_BULK_LOAD; +@@global.ROCKSDB_BULK_LOAD +0 +SET @@session.ROCKSDB_BULK_LOAD = @start_session_value; +SELECT @@session.ROCKSDB_BULK_LOAD; +@@session.ROCKSDB_BULK_LOAD +0 +DROP TABLE valid_values; +DROP TABLE invalid_values; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_stats_level_basic.result b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_stats_level_basic.result index d8d218fe3e8..154cdc31850 100644 --- a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_stats_level_basic.result +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_stats_level_basic.result @@ -11,7 +11,7 @@ INSERT INTO invalid_values VALUES('\'484436\''); SET @start_global_value = @@global.ROCKSDB_STATS_LEVEL; SELECT @start_global_value; @start_global_value -0 +1 '# Setting to valid values in global scope#' "Trying to set variable @@global.ROCKSDB_STATS_LEVEL to 0" SET @@global.ROCKSDB_STATS_LEVEL = 0; @@ -22,7 +22,7 @@ SELECT @@global.ROCKSDB_STATS_LEVEL; SET @@global.ROCKSDB_STATS_LEVEL = DEFAULT; SELECT @@global.ROCKSDB_STATS_LEVEL; @@global.ROCKSDB_STATS_LEVEL -0 +1 "Trying to set variable @@global.ROCKSDB_STATS_LEVEL to 4" SET @@global.ROCKSDB_STATS_LEVEL = 4; SELECT @@global.ROCKSDB_STATS_LEVEL; @@ -32,7 +32,7 @@ SELECT @@global.ROCKSDB_STATS_LEVEL; SET @@global.ROCKSDB_STATS_LEVEL = DEFAULT; SELECT @@global.ROCKSDB_STATS_LEVEL; @@global.ROCKSDB_STATS_LEVEL -0 +1 "Trying to set variable @@global.ROCKSDB_STATS_LEVEL to 2" SET @@global.ROCKSDB_STATS_LEVEL = 2; SELECT @@global.ROCKSDB_STATS_LEVEL; @@ -42,7 +42,7 @@ SELECT @@global.ROCKSDB_STATS_LEVEL; SET @@global.ROCKSDB_STATS_LEVEL = DEFAULT; SELECT @@global.ROCKSDB_STATS_LEVEL; @@global.ROCKSDB_STATS_LEVEL -0 +1 "Trying to set variable @@session.ROCKSDB_STATS_LEVEL to 444. It should fail because it is not session." SET @@session.ROCKSDB_STATS_LEVEL = 444; ERROR HY000: Variable 'rocksdb_stats_level' is a GLOBAL variable and should be set with SET GLOBAL @@ -52,34 +52,34 @@ SET @@global.ROCKSDB_STATS_LEVEL = 'aaa'; Got one of the listed errors SELECT @@global.ROCKSDB_STATS_LEVEL; @@global.ROCKSDB_STATS_LEVEL -0 +1 "Trying to set variable @@global.ROCKSDB_STATS_LEVEL to 'bbb'" SET @@global.ROCKSDB_STATS_LEVEL = 'bbb'; Got one of the listed errors SELECT @@global.ROCKSDB_STATS_LEVEL; @@global.ROCKSDB_STATS_LEVEL -0 +1 "Trying to set variable @@global.ROCKSDB_STATS_LEVEL to '-1'" SET @@global.ROCKSDB_STATS_LEVEL = '-1'; Got one of the listed errors SELECT @@global.ROCKSDB_STATS_LEVEL; @@global.ROCKSDB_STATS_LEVEL -0 +1 "Trying to set variable @@global.ROCKSDB_STATS_LEVEL to '101'" SET @@global.ROCKSDB_STATS_LEVEL = '101'; Got one of the listed errors SELECT @@global.ROCKSDB_STATS_LEVEL; @@global.ROCKSDB_STATS_LEVEL -0 +1 "Trying to set variable @@global.ROCKSDB_STATS_LEVEL to '484436'" SET @@global.ROCKSDB_STATS_LEVEL = '484436'; Got one of the listed errors SELECT @@global.ROCKSDB_STATS_LEVEL; @@global.ROCKSDB_STATS_LEVEL -0 +1 SET @@global.ROCKSDB_STATS_LEVEL = @start_global_value; SELECT @@global.ROCKSDB_STATS_LEVEL; @@global.ROCKSDB_STATS_LEVEL -0 +1 DROP TABLE valid_values; DROP TABLE invalid_values; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_track_and_verify_wals_in_manifest_basic.result b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_track_and_verify_wals_in_manifest_basic.result new file mode 100644 index 00000000000..7471dd82eb2 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_track_and_verify_wals_in_manifest_basic.result @@ -0,0 +1,14 @@ +CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO valid_values VALUES(1); +INSERT INTO valid_values VALUES(1024); +CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO invalid_values VALUES('\'aaa\''); +SET @start_global_value = @@global.ROCKSDB_TRACK_AND_VERIFY_WALS_IN_MANIFEST; +SELECT @start_global_value; +@start_global_value +1 +"Trying to set variable @@global.ROCKSDB_TRACK_AND_VERIFY_WALS_IN_MANIFEST to 444. It should fail because it is readonly." +SET @@global.ROCKSDB_TRACK_AND_VERIFY_WALS_IN_MANIFEST = 444; +ERROR HY000: Variable 'rocksdb_track_and_verify_wals_in_manifest' is a read only variable +DROP TABLE valid_values; +DROP TABLE invalid_values; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_wal_recovery_mode_basic.result b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_wal_recovery_mode_basic.result index 9fec4a24bd8..cf11f295c29 100644 --- a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_wal_recovery_mode_basic.result +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_wal_recovery_mode_basic.result @@ -6,7 +6,7 @@ INSERT INTO invalid_values VALUES('\'aaa\''); SET @start_global_value = @@global.ROCKSDB_WAL_RECOVERY_MODE; SELECT @start_global_value; @start_global_value -1 +2 '# Setting to valid values in global scope#' "Trying to set variable @@global.ROCKSDB_WAL_RECOVERY_MODE to 1" SET @@global.ROCKSDB_WAL_RECOVERY_MODE = 1; @@ -17,7 +17,7 @@ SELECT @@global.ROCKSDB_WAL_RECOVERY_MODE; SET @@global.ROCKSDB_WAL_RECOVERY_MODE = DEFAULT; SELECT @@global.ROCKSDB_WAL_RECOVERY_MODE; @@global.ROCKSDB_WAL_RECOVERY_MODE -1 +2 "Trying to set variable @@global.ROCKSDB_WAL_RECOVERY_MODE to 0" SET @@global.ROCKSDB_WAL_RECOVERY_MODE = 0; SELECT @@global.ROCKSDB_WAL_RECOVERY_MODE; @@ -27,7 +27,7 @@ SELECT @@global.ROCKSDB_WAL_RECOVERY_MODE; SET @@global.ROCKSDB_WAL_RECOVERY_MODE = DEFAULT; SELECT @@global.ROCKSDB_WAL_RECOVERY_MODE; @@global.ROCKSDB_WAL_RECOVERY_MODE -1 +2 "Trying to set variable @@session.ROCKSDB_WAL_RECOVERY_MODE to 444. It should fail because it is not session." SET @@session.ROCKSDB_WAL_RECOVERY_MODE = 444; ERROR HY000: Variable 'rocksdb_wal_recovery_mode' is a GLOBAL variable and should be set with SET GLOBAL @@ -37,10 +37,10 @@ SET @@global.ROCKSDB_WAL_RECOVERY_MODE = 'aaa'; Got one of the listed errors SELECT @@global.ROCKSDB_WAL_RECOVERY_MODE; @@global.ROCKSDB_WAL_RECOVERY_MODE -1 +2 SET @@global.ROCKSDB_WAL_RECOVERY_MODE = @start_global_value; SELECT @@global.ROCKSDB_WAL_RECOVERY_MODE; @@global.ROCKSDB_WAL_RECOVERY_MODE -1 +2 DROP TABLE valid_values; DROP TABLE invalid_values; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_write_batch_flush_threshold_basic.result b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_write_batch_flush_threshold_basic.result new file mode 100644 index 00000000000..8fd5ba9a3a7 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_write_batch_flush_threshold_basic.result @@ -0,0 +1,100 @@ +CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO valid_values VALUES(1); +INSERT INTO valid_values VALUES(1024); +INSERT INTO valid_values VALUES(2000000); +CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO invalid_values VALUES('\'aaa\''); +INSERT INTO invalid_values VALUES('\'2000001\''); +SET @start_global_value = @@global.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD; +SELECT @start_global_value; +@start_global_value +0 +SET @start_session_value = @@session.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD; +SELECT @start_session_value; +@start_session_value +0 +'# Setting to valid values in global scope#' +"Trying to set variable @@global.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD to 1" +SET @@global.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD = 1; +SELECT @@global.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD; +@@global.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD +1 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD = DEFAULT; +SELECT @@global.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD; +@@global.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD +0 +"Trying to set variable @@global.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD to 1024" +SET @@global.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD = 1024; +SELECT @@global.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD; +@@global.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD +1024 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD = DEFAULT; +SELECT @@global.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD; +@@global.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD +0 +"Trying to set variable @@global.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD to 2000000" +SET @@global.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD = 2000000; +SELECT @@global.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD; +@@global.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD +2000000 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD = DEFAULT; +SELECT @@global.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD; +@@global.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD +0 +'# Setting to valid values in session scope#' +"Trying to set variable @@session.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD to 1" +SET @@session.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD = 1; +SELECT @@session.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD; +@@session.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD +1 +"Setting the session scope variable back to default" +SET @@session.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD = DEFAULT; +SELECT @@session.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD; +@@session.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD +0 +"Trying to set variable @@session.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD to 1024" +SET @@session.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD = 1024; +SELECT @@session.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD; +@@session.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD +1024 +"Setting the session scope variable back to default" +SET @@session.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD = DEFAULT; +SELECT @@session.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD; +@@session.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD +0 +"Trying to set variable @@session.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD to 2000000" +SET @@session.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD = 2000000; +SELECT @@session.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD; +@@session.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD +2000000 +"Setting the session scope variable back to default" +SET @@session.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD = DEFAULT; +SELECT @@session.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD; +@@session.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD +0 +'# Testing with invalid values in global scope #' +"Trying to set variable @@global.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD to 'aaa'" +SET @@global.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD = 'aaa'; +Got one of the listed errors +SELECT @@global.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD; +@@global.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD +0 +"Trying to set variable @@global.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD to '2000001'" +SET @@global.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD = '2000001'; +Got one of the listed errors +SELECT @@global.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD; +@@global.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD +0 +SET @@global.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD = @start_global_value; +SELECT @@global.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD; +@@global.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD +0 +SET @@session.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD = @start_session_value; +SELECT @@session.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD; +@@session.ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD +0 +DROP TABLE valid_values; +DROP TABLE invalid_values; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/suite.pm b/storage/rocksdb/mysql-test/rocksdb_sys_vars/suite.pm index d50f4d2422e..8c6d33357c7 100644 --- a/storage/rocksdb/mysql-test/rocksdb_sys_vars/suite.pm +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/suite.pm @@ -12,8 +12,8 @@ use strict; my $sst_dump= ::mtr_exe_maybe_exists( - "$::bindir/storage/rocksdb$::multiconfig/sst_dump", - "$::path_client_bindir/sst_dump"); + "$::bindir/storage/rocksdb$::multiconfig/mariadb-sst-dump", + "$::path_client_bindir/mariadb-sst-dump"); return "RocksDB is not compiled, no sst_dump" unless $sst_dump; $ENV{MARIAROCKS_SST_DUMP}="$sst_dump"; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_cancel_manual_compactions_basic.test b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_cancel_manual_compactions_basic.test new file mode 100644 index 00000000000..5b82c225147 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_cancel_manual_compactions_basic.test @@ -0,0 +1,17 @@ +--source include/have_rocksdb.inc + +CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO valid_values VALUES(1); +INSERT INTO valid_values VALUES(0); +INSERT INTO valid_values VALUES('on'); + +CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; + +--let $sys_var=ROCKSDB_CANCEL_MANUAL_COMPACTIONS +--let $read_only=0 +--let $session=0 +--let $sticky=1 +--source ../include/rocksdb_sys_var.inc + +DROP TABLE valid_values; +DROP TABLE invalid_values; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_delete_cf_basic.test b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_delete_cf_basic.test index 0875e492b2c..6c12b0c2482 100644 --- a/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_delete_cf_basic.test +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_delete_cf_basic.test @@ -1,41 +1,31 @@ --disable_query_log -call mtr.add_suppression("Failed to drop column family"); +call mtr.add_suppression("Cannot mark Column family.*to be dropped"); call mtr.add_suppression("Column family '[a-z_]+' not found"); --enable_query_log --source include/have_rocksdb.inc # should fail for not existing CF ---error ER_WRONG_VALUE_FOR_VAR +--error ER_CANT_DROP_CF --eval SET @@global.ROCKSDB_DELETE_CF = 'nonexistent_cf' # should fail for default system cf ---error ER_WRONG_VALUE_FOR_VAR +--error ER_CANT_DROP_CF --eval SET @@global.ROCKSDB_DELETE_CF = '__system__' -# should fail for default cf ---error ER_WRONG_VALUE_FOR_VAR ---eval SET @@global.ROCKSDB_DELETE_CF = 'default' - ---disable_parsing -# should succeed for not existing CF ---eval SET @@global.ROCKSDB_DELETE_CF = 'nonexistent_cf' - -# should fail for default system cf ---error ER_WRONG_VALUE_FOR_VAR ---eval SET @@global.ROCKSDB_DELETE_CF = '__system__' - -alter table mysql.slave_worker_info engine = MyISAM; -alter table mysql.slave_relay_log_info engine = MyISAM; -alter table mysql.slave_gtid_info engine = MyISAM; -alter table mysql.slave_master_info engine = MyISAM; - select count(*) from information_schema.rocksdb_ddl where cf = 'default'; # should fail for default cf ---error ER_GET_ERRMSG +--error ER_CANT_DROP_CF --eval SET @@global.ROCKSDB_DELETE_CF = 'default' +# should fail for empty string, which is same to default cf +--error ER_CANT_DROP_CF +--eval SET @@global.ROCKSDB_DELETE_CF = '' + +# should succeed for NULL +--eval SET @@global.ROCKSDB_DELETE_CF = NULL + CREATE TABLE cf_deletion_test_table1 ( id1 int(10) unsigned NOT NULL DEFAULT '0', id2 int(10) unsigned NOT NULL DEFAULT '0', @@ -44,9 +34,9 @@ CREATE TABLE cf_deletion_test_table1 ( ) ENGINE=ROCKSDB; # should fail, CFs are still in use ---error ER_WRONG_VALUE_FOR_VAR +--error ER_CANT_DROP_CF --eval SET @@global.ROCKSDB_DELETE_CF = 'cf_primary_key' ---error ER_WRONG_VALUE_FOR_VAR +--error ER_CANT_DROP_CF --eval SET @@global.ROCKSDB_DELETE_CF = 'cf_secondary_key' CREATE TABLE cf_deletion_test_table2 ( @@ -57,7 +47,7 @@ CREATE TABLE cf_deletion_test_table2 ( DROP TABLE cf_deletion_test_table1; # should fail, still used by second table ---error ER_WRONG_VALUE_FOR_VAR +--error ER_CANT_DROP_CF --eval SET @@global.ROCKSDB_DELETE_CF = 'cf_primary_key' # should succeed, no one is using it anymore @@ -67,9 +57,3 @@ DROP TABLE cf_deletion_test_table2; # should succeed now --eval SET @@global.ROCKSDB_DELETE_CF = 'cf_primary_key' - -alter table mysql.slave_worker_info engine = ROCKSDB; -alter table mysql.slave_relay_log_info engine = ROCKSDB; -alter table mysql.slave_gtid_info engine = ROCKSDB; -alter table mysql.slave_master_info engine = ROCKSDB; ---enable_parsing diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_enable_iterate_bounds_basic.test b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_enable_iterate_bounds_basic.test new file mode 100644 index 00000000000..e7c66642f00 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_enable_iterate_bounds_basic.test @@ -0,0 +1,43 @@ +# +# test rocksdb_enable_iterate_bounds +# +connect (con1, localhost, root,,); +connect (con2, localhost, root,,); + +connection con2; +# explicitly enable the iterate bounds +SET SESSION rocksdb_enable_iterate_bounds=1; + +connection con1; +SET SESSION rocksdb_enable_iterate_bounds=0; +SELECT @@rocksdb_enable_iterate_bounds; # should be disabled + +DROP TABLE IF EXISTS t; + +--disable_query_log +CREATE TABLE t (a INT AUTO_INCREMENT PRIMARY KEY, b CHAR(8)) ENGINE=rocksdb; +INSERT INTO t (a, b) VALUES (NULL, 'foo'); +INSERT INTO t SELECT 0, b FROM t; +INSERT INTO t SELECT 0, b FROM t; +INSERT INTO t SELECT 0, b FROM t; +INSERT INTO t SELECT 0, b FROM t; +INSERT INTO t SELECT 0, b FROM t; +INSERT INTO t SELECT 0, b FROM t; +--enable_query_log + +# point select +SELECT COUNT(*) FROM t WHERE a = 1; +# range/scan +SELECT COUNT(*) FROM t WHERE a >= 1 AND a <= 32; +SELECT COUNT(*) FROM t; + + +# In another session, it should still be enabled +connection con2; +SELECT @@rocksdb_enable_iterate_bounds; + +SELECT COUNT(*) FROM t WHERE a = 1; +SELECT COUNT(*) FROM t WHERE a >= 1 AND a <= 32; +SELECT COUNT(*) FROM t; + +DROP TABLE t; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_enable_pipelined_write_basic.test b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_enable_pipelined_write_basic.test new file mode 100644 index 00000000000..17ab8a05c41 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_enable_pipelined_write_basic.test @@ -0,0 +1,6 @@ +--source include/have_rocksdb.inc + +--let $sys_var=ROCKSDB_ENABLE_PIPELINED_WRITE +--let $read_only=1 +--let $session=0 +--source ../include/rocksdb_sys_var.inc diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_enable_remove_orphaned_dropped_cfs_basic.test b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_enable_remove_orphaned_dropped_cfs_basic.test new file mode 100644 index 00000000000..d3b1469a658 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_enable_remove_orphaned_dropped_cfs_basic.test @@ -0,0 +1,21 @@ +--source include/have_rocksdb.inc + +CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO valid_values VALUES(1); +INSERT INTO valid_values VALUES(0); +INSERT INTO valid_values VALUES('on'); +INSERT INTO valid_values VALUES('off'); +INSERT INTO valid_values VALUES('true'); +INSERT INTO valid_values VALUES('false'); + +CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO invalid_values VALUES('\'aaa\''); +INSERT INTO invalid_values VALUES('\'bbb\''); + +--let $sys_var=ROCKSDB_ENABLE_REMOVE_ORPHANED_DROPPED_CFS +--let $session=0 +--source ../include/rocksdb_sys_var.inc + +DROP TABLE valid_values; +DROP TABLE invalid_values; + diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_manual_compaction_bottommost_level_basic.test b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_manual_compaction_bottommost_level_basic.test new file mode 100644 index 00000000000..bd716ec0f32 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_manual_compaction_bottommost_level_basic.test @@ -0,0 +1,18 @@ +--source include/have_rocksdb.inc + +CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO valid_values VALUES('kSkip'); +INSERT INTO valid_values VALUES('kIfHaveCompactionFilter'); +INSERT INTO valid_values VALUES('kForce'); +INSERT INTO valid_values VALUES('kForceOptimized'); + +CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO invalid_values VALUES('\'aaa\''); + +--let $sys_var=ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +--let $read_only=0 +--let $session=1 +--source ../include/rocksdb_sys_var.inc + +DROP TABLE valid_values; +DROP TABLE invalid_values; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_max_background_compactions_basic.test b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_max_background_compactions_basic.test new file mode 100644 index 00000000000..0ba185a2d0b --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_max_background_compactions_basic.test @@ -0,0 +1,16 @@ +--source include/have_rocksdb.inc + +CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO valid_values VALUES(1); +INSERT INTO valid_values VALUES(64); + +CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO invalid_values VALUES('\'abc\''); + +--let $sys_var=ROCKSDB_MAX_BACKGROUND_COMPACTIONS +--let $read_only=0 +--let $session=0 +--source ../include/rocksdb_sys_var.inc + +DROP TABLE valid_values; +DROP TABLE invalid_values; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_max_background_flushes_basic.test b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_max_background_flushes_basic.test new file mode 100644 index 00000000000..5800004518b --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_max_background_flushes_basic.test @@ -0,0 +1,16 @@ +--source include/have_rocksdb.inc + +CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO valid_values VALUES(1); +INSERT INTO valid_values VALUES(64); + +CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO invalid_values VALUES('\'abc\''); + +--let $sys_var=ROCKSDB_MAX_BACKGROUND_FLUSHES +--let $read_only=1 +--let $session=0 +--source ../include/rocksdb_sys_var.inc + +DROP TABLE valid_values; +DROP TABLE invalid_values; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_max_bottom_pri_background_compactions_basic.test b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_max_bottom_pri_background_compactions_basic.test new file mode 100644 index 00000000000..a0daf4c8eb1 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_max_bottom_pri_background_compactions_basic.test @@ -0,0 +1,16 @@ +--source include/have_rocksdb.inc + +CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO valid_values VALUES(1); +INSERT INTO valid_values VALUES(64); + +CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO invalid_values VALUES('\'abc\''); + +--let $sys_var=ROCKSDB_MAX_BOTTOM_PRI_BACKGROUND_COMPACTIONS +--let $read_only=0 +--let $session=0 +--source ../include/rocksdb_sys_var.inc + +DROP TABLE valid_values; +DROP TABLE invalid_values; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_max_row_locks_basic.test b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_max_row_locks_basic.test index 8a26ae91411..4b4c72be245 100644 --- a/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_max_row_locks_basic.test +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_max_row_locks_basic.test @@ -10,7 +10,7 @@ INSERT INTO invalid_values VALUES('\'aaa\''); --let $sys_var=ROCKSDB_MAX_ROW_LOCKS --let $read_only=0 ---let $session=1 +--let $session=0 --source include/rocksdb_sys_var.inc DROP TABLE valid_values; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_skip_locks_if_skip_unique_check_basic.test b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_skip_locks_if_skip_unique_check_basic.test new file mode 100644 index 00000000000..641a3871fd5 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_skip_locks_if_skip_unique_check_basic.test @@ -0,0 +1,18 @@ +--source include/have_rocksdb.inc + +CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO valid_values VALUES(1); +INSERT INTO valid_values VALUES(0); +INSERT INTO valid_values VALUES('on'); + +CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO invalid_values VALUES('\'aaa\''); +INSERT INTO invalid_values VALUES('\'bbb\''); + +--let $sys_var=ROCKSDB_BULK_LOAD +--let $read_only=0 +--let $session=1 +--source ../include/rocksdb_sys_var.inc + +DROP TABLE valid_values; +DROP TABLE invalid_values; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_track_and_verify_wals_in_manifest_basic.test b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_track_and_verify_wals_in_manifest_basic.test new file mode 100644 index 00000000000..f52b898ee50 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_track_and_verify_wals_in_manifest_basic.test @@ -0,0 +1,16 @@ +--source include/have_rocksdb.inc + +CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO valid_values VALUES(1); +INSERT INTO valid_values VALUES(1024); + +CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO invalid_values VALUES('\'aaa\''); + +--let $sys_var=ROCKSDB_TRACK_AND_VERIFY_WALS_IN_MANIFEST +--let $read_only=1 +--let $session=0 +--source ../include/rocksdb_sys_var.inc + +DROP TABLE valid_values; +DROP TABLE invalid_values; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_write_batch_flush_threshold_basic.test b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_write_batch_flush_threshold_basic.test new file mode 100644 index 00000000000..33760491ad2 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_write_batch_flush_threshold_basic.test @@ -0,0 +1,18 @@ +--source include/have_rocksdb.inc + +CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO valid_values VALUES(1); +INSERT INTO valid_values VALUES(1024); +INSERT INTO valid_values VALUES(2000000); + +CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO invalid_values VALUES('\'aaa\''); +INSERT INTO invalid_values VALUES('\'2000001\''); + +--let $sys_var=ROCKSDB_WRITE_BATCH_FLUSH_THRESHOLD +--let $read_only=0 +--let $session=1 +--source ../include/rocksdb_sys_var.inc + +DROP TABLE valid_values; +DROP TABLE invalid_values; diff --git a/storage/rocksdb/nosql_access.cc b/storage/rocksdb/nosql_access.cc deleted file mode 100644 index 080977f3722..00000000000 --- a/storage/rocksdb/nosql_access.cc +++ /dev/null @@ -1,53 +0,0 @@ -/* - Copyright (c) 2019, Facebook, Inc. - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ - -#define MYSQL_SERVER 1 - -#include -/* This C++ file's header */ -#include "./nosql_access.h" - -/* C++ standard header files */ -#include -#include -#include -#include -#include -#include - -/* C standard header files */ -#include - -/* MySQL header files */ -#include "../../sql/item.h" -#include "../../sql/sql_base.h" -#include "../../sql/sql_class.h" -#include "../../sql/strfunc.h" - -/* MyRocks header files */ -#include "./ha_rocksdb.h" -#include "./ha_rocksdb_proto.h" -#include "./rdb_buff.h" -#include "./rdb_datadic.h" - -namespace myrocks { - -bool rocksdb_handle_single_table_select(THD * /* unused */, - st_select_lex * /* unused */) { - return false; -} - -} // namespace myrocks diff --git a/storage/rocksdb/nosql_access.h b/storage/rocksdb/nosql_access.h deleted file mode 100644 index c6e75ec01c3..00000000000 --- a/storage/rocksdb/nosql_access.h +++ /dev/null @@ -1,36 +0,0 @@ -/* - Copyright (c) 2019, Facebook, Inc. - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ - -/* C++ standard header files */ -#include -#include -#include - -/* C standard header files */ -#include - -/* MySQL header files */ -#include "../../sql/protocol.h" -#include "./sql_string.h" - -#pragma once - -namespace myrocks { - -// Not needed in MyRocks: -//bool rocksdb_handle_single_table_select(THD *thd, st_select_lex *select_lex); - -} // namespace myrocks diff --git a/storage/rocksdb/rdb_cf_manager.cc b/storage/rocksdb/rdb_cf_manager.cc index 789481c5f00..1c95a3831c3 100644 --- a/storage/rocksdb/rdb_cf_manager.cc +++ b/storage/rocksdb/rdb_cf_manager.cc @@ -24,6 +24,10 @@ #include #include +#include +#include + +#include "./debug_sync.h" /* This C++ files header file */ #include "./rdb_cf_manager.h" @@ -33,8 +37,7 @@ #include "./ha_rocksdb_proto.h" #include "./rdb_datadic.h" #include "./rdb_psi.h" - -#include +#include "./rdb_threads.h" namespace myrocks { @@ -56,17 +59,18 @@ void Rdb_cf_manager::init( m_cf_options = std::move(cf_options); - for (auto cfh : *handles) { - DBUG_ASSERT(cfh != nullptr); - m_cf_name_map[cfh->GetName()] = cfh; - m_cf_id_map[cfh->GetID()] = cfh; + for (auto cfh_ptr : *handles) { + DBUG_ASSERT(cfh_ptr != nullptr); + + std::shared_ptr cfh(cfh_ptr); + m_cf_name_map[cfh_ptr->GetName()] = cfh; + m_cf_id_map[cfh_ptr->GetID()] = cfh; } } void Rdb_cf_manager::cleanup() { - for (auto it : m_cf_name_map) { - delete it.second; - } + m_cf_name_map.clear(); + m_cf_id_map.clear(); mysql_mutex_destroy(&m_mutex); m_cf_options = nullptr; } @@ -78,21 +82,18 @@ void Rdb_cf_manager::cleanup() { @detail See Rdb_cf_manager::get_cf */ -rocksdb::ColumnFamilyHandle *Rdb_cf_manager::get_or_create_cf( - rocksdb::DB *const rdb, const std::string &cf_name_arg) { +std::shared_ptr Rdb_cf_manager::get_or_create_cf( + rocksdb::DB *const rdb, const std::string &cf_name) { DBUG_ASSERT(rdb != nullptr); + DBUG_ASSERT(!cf_name.empty()); + std::shared_ptr cf_handle; - rocksdb::ColumnFamilyHandle *cf_handle = nullptr; - - if (cf_name_arg == PER_INDEX_CF_NAME) { + if (cf_name == PER_INDEX_CF_NAME) { // per-index column families is no longer supported. my_error(ER_PER_INDEX_CF_DEPRECATED, MYF(0)); - return nullptr; + return cf_handle; } - const std::string &cf_name = - cf_name_arg.empty() ? DEFAULT_CF_NAME : cf_name_arg; - RDB_MUTEX_LOCK_CHECK(m_mutex); const auto it = m_cf_name_map.find(cf_name); @@ -114,14 +115,15 @@ rocksdb::ColumnFamilyHandle *Rdb_cf_manager::get_or_create_cf( sql_print_information(" target_file_size_base=%" PRIu64, opts.target_file_size_base); + rocksdb::ColumnFamilyHandle *cf_handle_ptr = nullptr; const rocksdb::Status s = - rdb->CreateColumnFamily(opts, cf_name, &cf_handle); + rdb->CreateColumnFamily(opts, cf_name, &cf_handle_ptr); if (s.ok()) { - m_cf_name_map[cf_handle->GetName()] = cf_handle; - m_cf_id_map[cf_handle->GetID()] = cf_handle; - } else { - cf_handle = nullptr; + DBUG_ASSERT(cf_handle_ptr != nullptr); + cf_handle.reset(cf_handle_ptr); + m_cf_name_map[cf_handle_ptr->GetName()] = cf_handle; + m_cf_id_map[cf_handle_ptr->GetID()] = cf_handle; } } @@ -133,18 +135,25 @@ rocksdb::ColumnFamilyHandle *Rdb_cf_manager::get_or_create_cf( /* Find column family by its cf_name. */ +std::shared_ptr Rdb_cf_manager::get_cf( + const std::string &cf_name) const { + return get_cf(cf_name, false /*lock_held_by_caller*/); +} -rocksdb::ColumnFamilyHandle *Rdb_cf_manager::get_cf( - const std::string &cf_name_arg, const bool lock_held_by_caller) const { - rocksdb::ColumnFamilyHandle *cf_handle; +std::shared_ptr Rdb_cf_manager::get_cf( + const std::string &cf_name, const bool lock_held_by_caller) const { + DBUG_ASSERT(!cf_name.empty()); + std::shared_ptr cf_handle; if (!lock_held_by_caller) { RDB_MUTEX_LOCK_CHECK(m_mutex); } - std::string cf_name = cf_name_arg.empty() ? DEFAULT_CF_NAME : cf_name_arg; const auto it = m_cf_name_map.find(cf_name); - cf_handle = (it != m_cf_name_map.end()) ? it->second : nullptr; + + if (it != m_cf_name_map.end()) { + cf_handle = it->second; + } if (!cf_handle) { // NO_LINT_DEBUG @@ -158,8 +167,9 @@ rocksdb::ColumnFamilyHandle *Rdb_cf_manager::get_cf( return cf_handle; } -rocksdb::ColumnFamilyHandle *Rdb_cf_manager::get_cf(const uint32_t id) const { - rocksdb::ColumnFamilyHandle *cf_handle = nullptr; +std::shared_ptr Rdb_cf_manager::get_cf( + const uint32_t id) const { + std::shared_ptr cf_handle; RDB_MUTEX_LOCK_CHECK(m_mutex); const auto it = m_cf_id_map.find(id); @@ -181,9 +191,9 @@ std::vector Rdb_cf_manager::get_cf_names(void) const { return names; } -std::vector Rdb_cf_manager::get_all_cf( - void) const { - std::vector list; +std::vector> +Rdb_cf_manager::get_all_cf(void) const { + std::vector> list; RDB_MUTEX_LOCK_CHECK(m_mutex); @@ -197,67 +207,57 @@ std::vector Rdb_cf_manager::get_all_cf( return list; } -struct Rdb_cf_scanner : public Rdb_tables_scanner { - uint32_t m_cf_id; - int m_is_cf_used; - - explicit Rdb_cf_scanner(uint32_t cf_id) - : m_cf_id(cf_id), m_is_cf_used(false) {} - - int add_table(Rdb_tbl_def *tdef) override { - DBUG_ASSERT(tdef != nullptr); - - for (uint i = 0; i < tdef->m_key_count; i++) { - const Rdb_key_def &kd = *tdef->m_key_descr_arr[i]; - - if (kd.get_cf()->GetID() == m_cf_id) { - m_is_cf_used = true; - return HA_EXIT_SUCCESS; - } - } - return HA_EXIT_SUCCESS; - } -}; - -int Rdb_cf_manager::drop_cf(const std::string &cf_name) { - auto ddl_manager = rdb_get_ddl_manager(); - uint32_t cf_id = 0; - - if (cf_name == DEFAULT_SYSTEM_CF_NAME) { - return HA_EXIT_FAILURE; - } - +int Rdb_cf_manager::remove_dropped_cf(Rdb_dict_manager *const dict_manager, + rocksdb::TransactionDB *const rdb, + const uint32 &cf_id) { + dict_manager->assert_lock_held(); RDB_MUTEX_LOCK_CHECK(m_mutex); - auto cf_handle = get_cf(cf_name, true /* lock_held_by_caller */); - if (cf_handle == nullptr) { - RDB_MUTEX_UNLOCK_CHECK(m_mutex); - return HA_EXIT_SUCCESS; - } + const std::unique_ptr wb = dict_manager->begin(); + rocksdb::WriteBatch *const batch = wb.get(); - cf_id = cf_handle->GetID(); - Rdb_cf_scanner scanner(cf_id); - - auto ret = ddl_manager->scan_for_tables(&scanner); - if (ret) { + const auto it = m_cf_id_map.find(cf_id); + if (it == m_cf_id_map.end()) { + dict_manager->delete_dropped_cf_and_flags(batch, cf_id); + dict_manager->commit(batch); RDB_MUTEX_UNLOCK_CHECK(m_mutex); - return ret; - } - if (scanner.m_is_cf_used) { - // column family is used by existing key - RDB_MUTEX_UNLOCK_CHECK(m_mutex); + // NO_LINT_DEBUG + sql_print_warning( + "RocksDB: Column family with id %u is marked as dropped, " + "but doesn't exist in cf manager", + cf_id); + return HA_EXIT_FAILURE; } - auto rdb = rdb_get_rocksdb_db(); - auto status = rdb->DropColumnFamily(cf_handle); - if (!status.ok()) { + auto cf_handle = it->second.get(); + const std::string cf_name = cf_handle->GetName(); + + if (!dict_manager->get_dropped_cf(cf_id)) { RDB_MUTEX_UNLOCK_CHECK(m_mutex); + // NO_LINT_DEBUG + sql_print_warning( + "RocksDB: Column family %s with id %u is not in " + "the list of cf ids to be dropped", + cf_name.c_str(), cf_id); + return HA_EXIT_FAILURE; + } + + auto status = rdb->DropColumnFamily(cf_handle); + + if (!status.ok()) { + dict_manager->delete_dropped_cf(batch, cf_id); + dict_manager->commit(batch); + RDB_MUTEX_UNLOCK_CHECK(m_mutex); + + // NO_LINT_DEBUG + sql_print_error( + "RocksDB: Dropping column family %s with id %u on RocksDB failed", + cf_name.c_str(), cf_id); + return ha_rocksdb::rdb_error_to_mysql(status); } - delete cf_handle; - auto id_iter = m_cf_id_map.find(cf_id); DBUG_ASSERT(id_iter != m_cf_id_map.end()); m_cf_id_map.erase(id_iter); @@ -266,8 +266,141 @@ int Rdb_cf_manager::drop_cf(const std::string &cf_name) { DBUG_ASSERT(name_iter != m_cf_name_map.end()); m_cf_name_map.erase(name_iter); + dict_manager->delete_dropped_cf_and_flags(batch, cf_id); + + dict_manager->commit(batch); RDB_MUTEX_UNLOCK_CHECK(m_mutex); + // NO_LINT_DEBUG + sql_print_information( + "RocksDB: Column family %s with id %u has been dropped successfully", + cf_name.c_str(), cf_id); + return HA_EXIT_SUCCESS; } + +struct Rdb_cf_scanner : public Rdb_tables_scanner { + uint32_t m_cf_id; + + explicit Rdb_cf_scanner(uint32_t cf_id) : m_cf_id(cf_id) {} + + int add_table(Rdb_tbl_def *tdef) override { + DBUG_ASSERT(tdef != nullptr); + + for (uint i = 0; i < tdef->m_key_count; i++) { + const Rdb_key_def &kd = *tdef->m_key_descr_arr[i]; + + if (kd.get_cf()->GetID() == m_cf_id) { + return HA_EXIT_FAILURE; + } + } + return HA_EXIT_SUCCESS; + } +}; + +int Rdb_cf_manager::drop_cf(Rdb_ddl_manager *const ddl_manager, + Rdb_dict_manager *const dict_manager, + const std::string &cf_name) { + DBUG_ASSERT(!cf_name.empty()); + dict_manager->assert_lock_held(); + uint32_t cf_id = 0; + + if (cf_name == DEFAULT_SYSTEM_CF_NAME || cf_name == DEFAULT_CF_NAME) { + return HA_EXIT_FAILURE; + } + + // We should have already acquired dict manager lock. The order + // of lock acquisition here is: + // dict_manager -> cf_manager -> ddl_manager + RDB_MUTEX_LOCK_CHECK(m_mutex); + auto cf_handle = get_cf(cf_name, true /* lock_held_by_caller */).get(); + if (cf_handle == nullptr) { + RDB_MUTEX_UNLOCK_CHECK(m_mutex); + // NO_LINT_DEBUG + sql_print_warning( + "RocksDB: Cannot mark Column family %s to be dropped, " + "because it doesn't exist in cf manager", + cf_name.c_str()); + + return HA_EXIT_FAILURE; + } + + cf_id = cf_handle->GetID(); + Rdb_cf_scanner scanner(cf_id); + + auto ret = ddl_manager->scan_for_tables(&scanner); + if (ret) { + RDB_MUTEX_UNLOCK_CHECK(m_mutex); + // NO_LINT_DEBUG + sql_print_warning( + "RocksDB: Cannot mark Column family %s with id %u to be dropped, " + "because it is in use", + cf_name.c_str(), cf_id); + return ret; + } + + ret = ddl_manager->find_in_uncommitted_keydef(cf_id); + if (ret) { + RDB_MUTEX_UNLOCK_CHECK(m_mutex); + // NO_LINT_DEBUG + sql_print_warning( + "RocksDB: Cannot mark Column family %s with id %u to be dropped, " + "because it is used by an ongoing add index command", + cf_name.c_str(), cf_id); + return ret; + } + + // We don't call DropColumnFamily on RocksDB here and + // we don't delete handle object. Here we mark the column family + // as dropped. + + const std::unique_ptr wb = dict_manager->begin(); + rocksdb::WriteBatch *const batch = wb.get(); + + dict_manager->add_dropped_cf(batch, cf_id); + dict_manager->commit(batch); + + RDB_MUTEX_UNLOCK_CHECK(m_mutex); + + // NO_LINT_DEBUG + sql_print_information( + "RocksDB: Column family %s with id %u has been marked to be dropped", + cf_name.c_str(), cf_id); + + return HA_EXIT_SUCCESS; +} + +int Rdb_cf_manager::create_cf_flags_if_needed( + const Rdb_dict_manager *const dict_manager, const uint32 &cf_id, + const std::string &cf_name, const bool is_per_partition_cf) { + DBUG_ASSERT(!cf_name.empty()); + uchar flags = + (is_cf_name_reverse(cf_name.c_str()) ? Rdb_key_def::REVERSE_CF_FLAG : 0) | + (is_per_partition_cf ? Rdb_key_def::PER_PARTITION_CF_FLAG : 0); + + uint existing_cf_flags; + if (dict_manager->get_cf_flags(cf_id, &existing_cf_flags)) { + // For the purposes of comparison we'll clear the partitioning bit. The + // intent here is to make sure that both partitioned and non-partitioned + // tables can refer to the same CF. + existing_cf_flags &= ~Rdb_key_def::CF_FLAGS_TO_IGNORE; + flags &= ~Rdb_key_def::CF_FLAGS_TO_IGNORE; + + if (existing_cf_flags != flags) { + my_error(ER_CF_DIFFERENT, MYF(0), cf_name.c_str(), flags, + existing_cf_flags); + + return HA_EXIT_FAILURE; + } + } else { + const std::unique_ptr wb = dict_manager->begin(); + rocksdb::WriteBatch *const batch = wb.get(); + + dict_manager->add_cf_flags(batch, cf_id, flags); + dict_manager->commit(batch); + } + + return HA_EXIT_SUCCESS; +} + } // namespace myrocks diff --git a/storage/rocksdb/rdb_cf_manager.h b/storage/rocksdb/rdb_cf_manager.h index cf7b3d6cfb8..815e0dd599f 100644 --- a/storage/rocksdb/rdb_cf_manager.h +++ b/storage/rocksdb/rdb_cf_manager.h @@ -29,6 +29,7 @@ /* MyRocks header files */ #include "./rdb_cf_options.h" +#include "./rdb_datadic.h" namespace myrocks { @@ -47,8 +48,9 @@ namespace myrocks { */ class Rdb_cf_manager { - std::map m_cf_name_map; - std::map m_cf_id_map; + std::map> + m_cf_name_map; + std::map> m_cf_id_map; mutable mysql_mutex_t m_mutex; @@ -72,26 +74,36 @@ class Rdb_cf_manager { /* Used by CREATE TABLE. - - cf_name=nullptr means use default column family + cf_name requires non-empty string */ - rocksdb::ColumnFamilyHandle *get_or_create_cf(rocksdb::DB *const rdb, - const std::string &cf_name); + std::shared_ptr get_or_create_cf( + rocksdb::DB *const rdb, const std::string &cf_name); /* Used by table open */ - rocksdb::ColumnFamilyHandle *get_cf( - const std::string &cf_name, const bool lock_held_by_caller = false) const; + std::shared_ptr get_cf( + const std::string &cf_name) const; /* Look up cf by id; used by datadic */ - rocksdb::ColumnFamilyHandle *get_cf(const uint32_t id) const; + std::shared_ptr get_cf(const uint32_t id) const; /* Used to iterate over column families for show status */ std::vector get_cf_names(void) const; /* Used to iterate over column families */ - std::vector get_all_cf(void) const; + std::vector> get_all_cf( + void) const; + + int remove_dropped_cf(Rdb_dict_manager *const dict_manager, + rocksdb::TransactionDB *const rdb, const uint32 &cf_id); /* Used to delete cf by name */ - int drop_cf(const std::string &cf_name); + int drop_cf(Rdb_ddl_manager *const ddl_manager, + Rdb_dict_manager *const dict_manager, const std::string &cf_name); + + /* Create cf flags if it does not exist */ + int create_cf_flags_if_needed(const Rdb_dict_manager *const dict_manager, + const uint32 &cf_id, const std::string &cf_name, + const bool is_per_partition_cf = false); void get_cf_options(const std::string &cf_name, rocksdb::ColumnFamilyOptions *const opts) @@ -103,6 +115,10 @@ class Rdb_cf_manager { const std::string &updated_options) { m_cf_options->update(cf_name, updated_options); } + + private: + std::shared_ptr get_cf( + const std::string &cf_name, const bool lock_held_by_caller) const; }; } // namespace myrocks diff --git a/storage/rocksdb/rdb_cf_options.cc b/storage/rocksdb/rdb_cf_options.cc index c0d16893cda..75bb837b003 100644 --- a/storage/rocksdb/rdb_cf_options.cc +++ b/storage/rocksdb/rdb_cf_options.cc @@ -74,14 +74,20 @@ void Rdb_cf_options::get(const std::string &cf_name, rocksdb::ColumnFamilyOptions *const opts) { DBUG_ASSERT(opts != nullptr); + rocksdb::ConfigOptions config_options; + config_options.input_strings_escaped = false; + config_options.ignore_unknown_options = false; + // Get defaults. - rocksdb::GetColumnFamilyOptionsFromString(*opts, m_default_config, opts); + rocksdb::GetColumnFamilyOptionsFromString(config_options, *opts, + m_default_config, opts); // Get a custom confguration if we have one. Name_to_config_t::iterator it = m_name_map.find(cf_name); if (it != m_name_map.end()) { - rocksdb::GetColumnFamilyOptionsFromString(*opts, it->second, opts); + rocksdb::GetColumnFamilyOptionsFromString(config_options, *opts, it->second, + opts); } } @@ -98,10 +104,14 @@ void Rdb_cf_options::update(const std::string &cf_name, bool Rdb_cf_options::set_default(const std::string &default_config) { rocksdb::ColumnFamilyOptions options; + rocksdb::ConfigOptions config_options; + config_options.input_strings_escaped = false; + config_options.ignore_unknown_options = false; - if (!default_config.empty() && !rocksdb::GetColumnFamilyOptionsFromString( - options, default_config, &options) - .ok()) { + if (!default_config.empty() && + !rocksdb::GetColumnFamilyOptionsFromString(config_options, options, + default_config, &options) + .ok()) { // NO_LINT_DEBUG fprintf(stderr, "Invalid default column family config: %s\n", default_config.c_str()); @@ -260,6 +270,9 @@ bool Rdb_cf_options::parse_cf_options(const std::string &cf_options, std::string cf; std::string opt_str; rocksdb::ColumnFamilyOptions options; + rocksdb::ConfigOptions config_options; + config_options.input_strings_escaped = false; + config_options.ignore_unknown_options = false; DBUG_ASSERT(option_map != nullptr); DBUG_ASSERT(option_map->empty()); @@ -283,7 +296,8 @@ bool Rdb_cf_options::parse_cf_options(const std::string &cf_options, } // Generate an error if the is not valid according to RocksDB. - if (!rocksdb::GetColumnFamilyOptionsFromString(options, opt_str, &options) + if (!rocksdb::GetColumnFamilyOptionsFromString(config_options, options, + opt_str, &options) .ok()) { // NO_LINT_DEBUG sql_print_warning( diff --git a/storage/rocksdb/rdb_converter.cc b/storage/rocksdb/rdb_converter.cc index 6f8aa306668..9761d816027 100644 --- a/storage/rocksdb/rdb_converter.cc +++ b/storage/rocksdb/rdb_converter.cc @@ -68,52 +68,38 @@ void dbug_modify_key_varchar8(String *on_disk_rec) { 0 OK other HA_ERR error code (can be SE-specific) */ -int Rdb_convert_to_record_value_decoder::decode(uchar *const buf, uint *offset, - TABLE *table, - my_core::Field *field, +int Rdb_convert_to_record_value_decoder::decode(uchar *const buf, TABLE *table, Rdb_field_encoder *field_dec, Rdb_string_reader *reader, bool decode, bool is_null) { int err = HA_EXIT_SUCCESS; - - uint field_offset = field->ptr - table->record[0]; - *offset = field_offset; - uint null_offset = field->null_offset(); - bool maybe_null = field->real_maybe_null(); - field->move_field(buf + field_offset, - maybe_null ? buf + null_offset : nullptr, field->null_bit); - + auto ptr = buf + field_dec->m_field_offset; if (is_null) { - if (decode) { + if (decode && field_dec->maybe_null()) { // This sets the NULL-bit of this record - field->set_null(); + buf[field_dec->m_field_null_offset] |= field_dec->m_field_null_mask; /* Besides that, set the field value to default value. CHECKSUM TABLE depends on this. */ - memcpy(field->ptr, table->s->default_values + field_offset, - field->pack_length()); + memcpy(ptr, table->s->default_values + field_dec->m_field_offset, + field_dec->m_field_pack_length); } } else { - if (decode) { + if (decode && field_dec->maybe_null()) { // sets non-null bits for this record - field->set_notnull(); + buf[field_dec->m_field_null_offset] &= ~(field_dec->m_field_null_mask); } if (field_dec->m_field_type == MYSQL_TYPE_BLOB) { - err = decode_blob(table, field, reader, decode); + err = decode_blob(table, ptr, field_dec, reader, decode); } else if (field_dec->m_field_type == MYSQL_TYPE_VARCHAR) { - err = decode_varchar(field, reader, decode); + err = decode_varchar(ptr, field_dec, reader, decode); } else { - err = decode_fixed_length_field(field, field_dec, reader, decode); + err = decode_fixed_length_field(ptr, field_dec, reader, decode); } } - // Restore field->ptr and field->null_ptr - field->move_field(table->record[0] + field_offset, - maybe_null ? table->record[0] + null_offset : nullptr, - field->null_bit); - return err; } @@ -127,23 +113,22 @@ int Rdb_convert_to_record_value_decoder::decode(uchar *const buf, uint *offset, 0 OK other HA_ERR error code (can be SE-specific) */ -int Rdb_convert_to_record_value_decoder::decode_blob(TABLE *table, Field *field, - Rdb_string_reader *reader, - bool decode) { - my_core::Field_blob *blob = (my_core::Field_blob *)field; - +int Rdb_convert_to_record_value_decoder::decode_blob( + TABLE *table, uchar *const buf, Rdb_field_encoder *field_dec, + Rdb_string_reader *reader, bool decode) { // Get the number of bytes needed to store length - const uint length_bytes = blob->pack_length() - portable_sizeof_char_ptr; + const uint length_bytes = + field_dec->m_field_pack_length - portable_sizeof_char_ptr; const char *data_len_str; if (!(data_len_str = reader->read(length_bytes))) { return HA_ERR_ROCKSDB_CORRUPT_DATA; } - memcpy(blob->ptr, data_len_str, length_bytes); + memcpy(buf, data_len_str, length_bytes); uint32 data_len = - blob->get_length(reinterpret_cast(data_len_str), - length_bytes); + Field_blob::get_length(reinterpret_cast(data_len_str), + length_bytes); const char *blob_ptr; if (!(blob_ptr = reader->read(data_len))) { return HA_ERR_ROCKSDB_CORRUPT_DATA; @@ -152,8 +137,8 @@ int Rdb_convert_to_record_value_decoder::decode_blob(TABLE *table, Field *field, if (decode) { // set 8-byte pointer to 0, like innodb does (relevant for 32-bit // platforms) - memset(blob->ptr + length_bytes, 0, 8); - memcpy(blob->ptr + length_bytes, &blob_ptr, sizeof(uchar **)); + memset(buf + length_bytes, 0, 8); + memcpy(buf + length_bytes, &blob_ptr, sizeof(uchar **)); } return HA_EXIT_SUCCESS; @@ -171,9 +156,9 @@ int Rdb_convert_to_record_value_decoder::decode_blob(TABLE *table, Field *field, other HA_ERR error code (can be SE-specific) */ int Rdb_convert_to_record_value_decoder::decode_fixed_length_field( - my_core::Field *const field, Rdb_field_encoder *field_dec, + uchar *const buf, Rdb_field_encoder *field_dec, Rdb_string_reader *const reader, bool decode) { - uint len = field_dec->m_pack_length_in_rec; + uint len = field_dec->m_field_pack_length; if (len > 0) { const char *data_bytes; if ((data_bytes = reader->read(len)) == nullptr) { @@ -181,7 +166,7 @@ int Rdb_convert_to_record_value_decoder::decode_fixed_length_field( } if (decode) { - memcpy(field->ptr, data_bytes, len); + memcpy(buf, data_bytes, len); } } @@ -199,24 +184,23 @@ int Rdb_convert_to_record_value_decoder::decode_fixed_length_field( other HA_ERR error code (can be SE-specific) */ int Rdb_convert_to_record_value_decoder::decode_varchar( - Field *field, Rdb_string_reader *const reader, bool decode) { - my_core::Field_varstring *const field_var = (my_core::Field_varstring *)field; - + uchar *const buf, Rdb_field_encoder *field_dec, + Rdb_string_reader *const reader, bool decode) { const char *data_len_str; - if (!(data_len_str = reader->read(field_var->length_bytes))) { + if (!(data_len_str = reader->read(field_dec->m_field_length_bytes))) { return HA_ERR_ROCKSDB_CORRUPT_DATA; } uint data_len; - // field_var->length_bytes is 1 or 2 - if (field_var->length_bytes == 1) { + // field_dec->length_bytes is 1 or 2 + if (field_dec->m_field_length_bytes == 1) { data_len = (uchar)data_len_str[0]; } else { - DBUG_ASSERT(field_var->length_bytes == 2); + DBUG_ASSERT(field_dec->m_field_length_bytes == 2); data_len = uint2korr(data_len_str); } - if (data_len > field_var->field_length) { + if (data_len > field_dec->m_field_length) { // The data on disk is longer than table DDL allows? return HA_ERR_ROCKSDB_CORRUPT_DATA; } @@ -226,7 +210,7 @@ int Rdb_convert_to_record_value_decoder::decode_varchar( } if (decode) { - memcpy(field_var->ptr, data_len_str, field_var->length_bytes + data_len); + memcpy(buf, data_len_str, field_dec->m_field_length_bytes + data_len); } return HA_EXIT_SUCCESS; @@ -246,7 +230,6 @@ Rdb_value_field_iterator::Rdb_value_field_iterator( m_field_iter = fields->begin(); m_field_end = fields->end(); m_null_bytes = rdb_converter->get_null_bytes(); - m_offset = 0; } // Iterate each requested field and decode one by one @@ -267,15 +250,14 @@ int Rdb_value_field_iterator::next() { return HA_ERR_ROCKSDB_CORRUPT_DATA; } - m_field = m_table->field[m_field_dec->m_field_index]; // Decode each field - err = value_field_decoder::decode(m_buf, &m_offset, m_table, m_field, - m_field_dec, m_value_slice_reader, decode, - m_is_null); + err = value_field_decoder::decode(m_buf, m_table, m_field_dec, + m_value_slice_reader, decode, m_is_null); if (err != HA_EXIT_SUCCESS) { return err; } m_field_iter++; + // Only break for the field that are actually decoding rather than skipping if (decode) { break; @@ -289,12 +271,6 @@ bool Rdb_value_field_iterator::end_of_fields() const { return m_field_iter == m_field_end; } -template -Field *Rdb_value_field_iterator::get_field() const { - DBUG_ASSERT(m_field != nullptr); - return m_field; -} - template void *Rdb_value_field_iterator::get_dst() const { DBUG_ASSERT(m_buf != nullptr); @@ -316,7 +292,6 @@ enum_field_types Rdb_value_field_iterator::get_field_type() template bool Rdb_value_field_iterator::is_null() const { - DBUG_ASSERT(m_field != nullptr); return m_is_null; } @@ -339,6 +314,7 @@ Rdb_converter::Rdb_converter(const THD *thd, const Rdb_tbl_def *tbl_def, m_row_checksums_checked = 0; m_null_bytes = nullptr; setup_field_encoders(); + m_lookup_bitmap = {nullptr, nullptr, 0, 0}; } Rdb_converter::~Rdb_converter() { @@ -346,6 +322,7 @@ Rdb_converter::~Rdb_converter() { m_encoder_arr = nullptr; // These are needed to suppress valgrind errors in rocksdb.partition m_storage_record.free(); + my_bitmap_free(&m_lookup_bitmap); } /* @@ -370,30 +347,29 @@ void Rdb_converter::get_storage_type(Rdb_field_encoder *const encoder, Setup which fields will be unpacked when reading rows @detail - Three special cases when we still unpack all fields: + Two special cases when we still unpack all fields: - When client requires decode_all_fields, such as this table is being updated (m_lock_rows==RDB_LOCK_WRITE). - When @@rocksdb_verify_row_debug_checksums is ON (In this mode, we need to read all fields to find whether there is a row checksum at the end. We could skip the fields instead of decoding them, but currently we do decoding.) - - On index merge as bitmap is cleared during that operation @seealso Rdb_converter::setup_field_encoders() Rdb_converter::convert_record_from_storage_format() */ void Rdb_converter::setup_field_decoders(const MY_BITMAP *field_map, + uint active_index, bool keyread_only, bool decode_all_fields) { m_key_requested = false; m_decoders_vect.clear(); + my_bitmap_free(&m_lookup_bitmap); int last_useful = 0; int skip_size = 0; for (uint i = 0; i < m_table->s->fields; i++) { - // bitmap is cleared on index merge, but it still needs to decode columns bool field_requested = decode_all_fields || m_verify_row_debug_checksums || - bitmap_is_clear_all(field_map) || bitmap_is_set(field_map, m_table->field[i]->field_index); // We only need the decoder if the whole record is stored. @@ -420,7 +396,7 @@ void Rdb_converter::setup_field_decoders(const MY_BITMAP *field_map, } else { // Fixed-width field can be skipped without looking at it. // Add appropriate skip_size to the next field. - skip_size += m_encoder_arr[i].m_pack_length_in_rec; + skip_size += m_encoder_arr[i].m_field_pack_length; } } } @@ -429,6 +405,11 @@ void Rdb_converter::setup_field_decoders(const MY_BITMAP *field_map, // skipping. Remove them. m_decoders_vect.erase(m_decoders_vect.begin() + last_useful, m_decoders_vect.end()); + + if (!keyread_only && active_index != m_table->s->primary_key) { + m_tbl_def->m_key_descr_arr[active_index]->get_lookup_bitmap( + m_table, &m_lookup_bitmap); + } } void Rdb_converter::setup_field_encoders() { @@ -466,13 +447,39 @@ void Rdb_converter::setup_field_encoders() { } } - m_encoder_arr[i].m_field_type = field->real_type(); - m_encoder_arr[i].m_field_index = i; - m_encoder_arr[i].m_pack_length_in_rec = field->pack_length_in_rec(); + /* + The difference between pack_length and pack_length_in_rec is fairly + subtle. The only difference is in Field_bit case where it borrows some + bits in null bytes in memory to store the 'uneven' high bits, therefore + the pack_length is the length of remaining bits while the + pack_length_in_rec is the full length of all bits when you store it on + disk. Only MyIsam and archive supports it, indicating by + HA_CAN_BIT_FIELD. We don't handle this case today at all (nor do we need + to), and we use pack_length everywhere, so just assert it and move on. + */ + DBUG_ASSERT(field->pack_length() == field->pack_length_in_rec()); - if (field->real_maybe_null()) { + auto field_type = field->real_type(); + m_encoder_arr[i].m_field_type = field_type; + m_encoder_arr[i].m_field_index = i; + m_encoder_arr[i].m_field_pack_length = field->pack_length(); + m_encoder_arr[i].m_field_offset = field->ptr - m_table->record[0]; + + if (field_type == MYSQL_TYPE_VARCHAR) { + auto varchar = reinterpret_cast(field); + m_encoder_arr[i].m_field_length = varchar->field_length; + m_encoder_arr[i].m_field_length_bytes = varchar->length_bytes; + } else { + m_encoder_arr[i].m_field_length = -1; + m_encoder_arr[i].m_field_length_bytes = -1; + } + + auto maybe_null = field->real_maybe_null(); + if (maybe_null) { m_encoder_arr[i].m_null_mask = cur_null_mask; m_encoder_arr[i].m_null_offset = null_bytes_length; + m_encoder_arr[i].m_field_null_offset = field->null_offset(); + m_encoder_arr[i].m_field_null_mask = field->null_bit; if (cur_null_mask == 0x80) { cur_null_mask = 0x1; null_bytes_length++; @@ -480,6 +487,7 @@ void Rdb_converter::setup_field_encoders() { cur_null_mask = cur_null_mask << 1; } } else { + m_encoder_arr[i].m_null_offset = 0; m_encoder_arr[i].m_null_mask = 0; } } @@ -585,6 +593,11 @@ int Rdb_converter::convert_record_from_storage_format( const std::shared_ptr &pk_def, const rocksdb::Slice *const key_slice, const rocksdb::Slice *const value_slice, uchar *const dst) { + bool skip_value = get_decode_fields()->size() == 0; + if (!m_key_requested && skip_value) { + return HA_EXIT_SUCCESS; + } + int err = HA_EXIT_SUCCESS; Rdb_string_reader value_slice_reader(value_slice); @@ -601,9 +614,9 @@ int Rdb_converter::convert_record_from_storage_format( err = pk_def->unpack_record(m_table, dst, key_slice, !unpack_slice.empty() ? &unpack_slice : nullptr, false /* verify_checksum */); - } - if (err != HA_EXIT_SUCCESS) { - return err; + if (err != HA_EXIT_SUCCESS) { + return err; + } } Rdb_value_field_iterator @@ -714,10 +727,30 @@ int Rdb_converter::encode_value_slice( if (has_ttl_column) { DBUG_ASSERT(pk_def->get_ttl_field_index() != UINT_MAX); Field *const field = m_table->field[pk_def->get_ttl_field_index()]; - DBUG_ASSERT(field->pack_length_in_rec() == ROCKSDB_SIZEOF_TTL_RECORD); - DBUG_ASSERT(field->real_type() == MYSQL_TYPE_LONGLONG); + DBUG_ASSERT(field->real_type() == MYSQL_TYPE_LONGLONG || + field->type() == MYSQL_TYPE_TIMESTAMP); + + uint64 ts; + + if (field->type() == MYSQL_TYPE_TIMESTAMP) { + /* + Timestamp packed length can be 4 to 7 bytes depending on precision. + The first four bytes is the unix time network byte order and is used + for MYSQL_TYPE_TIMESTAMP and MYSQL_TYPE_TIMESTAMP2. The optional + remaining bytes are for fractional seconds. Only the first four bytes + are needed. + */ + DBUG_ASSERT(field->real_type() == MYSQL_TYPE_TIMESTAMP || + field->real_type() == MYSQL_TYPE_TIMESTAMP2); + DBUG_ASSERT(field->pack_length_in_rec() >= 4); + DBUG_ASSERT(field->key_type() == HA_KEYTYPE_BINARY); + ts = (uint64)rdb_netbuf_to_uint32(field->ptr); + } else { + DBUG_ASSERT(field->key_type() == HA_KEYTYPE_ULONGLONG); + DBUG_ASSERT(field->pack_length_in_rec() == ROCKSDB_SIZEOF_TTL_RECORD); + ts = uint8korr(field->ptr); + } - uint64 ts = uint8korr(field->ptr); #ifndef DBUG_OFF ts += rdb_dbug_set_ttl_rec_ts(); #endif @@ -810,7 +843,7 @@ int Rdb_converter::encode_value_slice( field_var->length_bytes + data_len); } else { /* Copy the field data */ - const uint len = field->pack_length_in_rec(); + const uint len = field->pack_length(); m_storage_record.append(reinterpret_cast(field->ptr), len); } } diff --git a/storage/rocksdb/rdb_converter.h b/storage/rocksdb/rdb_converter.h index 6ace89b3366..204e17c8b3a 100644 --- a/storage/rocksdb/rdb_converter.h +++ b/storage/rocksdb/rdb_converter.h @@ -57,20 +57,21 @@ class Rdb_convert_to_record_value_decoder { Rdb_convert_to_record_value_decoder &operator=( const Rdb_convert_to_record_value_decoder &decoder) = delete; - static int decode(uchar *const buf, uint *offset, TABLE *table, - my_core::Field *field, Rdb_field_encoder *field_dec, - Rdb_string_reader *reader, bool decode, bool is_null); + static int decode(uchar *const buf, TABLE *table, + Rdb_field_encoder *field_dec, Rdb_string_reader *reader, + bool decode, bool is_null); private: - static int decode_blob(TABLE *table, Field *field, Rdb_string_reader *reader, - bool decode); - static int decode_fixed_length_field(Field *const field, + static int decode_blob(TABLE *table, uchar *const buf, + Rdb_field_encoder *field_dec, + Rdb_string_reader *reader, bool decode); + static int decode_fixed_length_field(uchar *const buf, Rdb_field_encoder *field_dec, Rdb_string_reader *const reader, bool decode); - static int decode_varchar(Field *const field, Rdb_string_reader *const reader, - bool decode); + static int decode_varchar(uchar *const buf, Rdb_field_encoder *field_dec, + Rdb_string_reader *const reader, bool decode); }; /** @@ -119,8 +120,6 @@ class Rdb_value_field_iterator { int get_field_index() const; // get current field type enum_field_types get_field_type() const; - // get current field - Field *get_field() const; }; /** @@ -136,8 +135,8 @@ class Rdb_converter { Rdb_converter &operator=(const Rdb_converter &decoder) = delete; ~Rdb_converter(); - void setup_field_decoders(const MY_BITMAP *field_map, - bool decode_all_fields = false); + void setup_field_decoders(const MY_BITMAP *field_map, uint active_index, + bool keyread_only, bool decode_all_fields = false); int decode(const std::shared_ptr &key_def, uchar *dst, const rocksdb::Slice *key_slice, @@ -174,6 +173,8 @@ class Rdb_converter { return &m_decoders_vect; } + const MY_BITMAP *get_lookup_bitmap() { return &m_lookup_bitmap; } + private: int decode_value_header(Rdb_string_reader *reader, const std::shared_ptr &pk_def, @@ -243,5 +244,11 @@ class Rdb_converter { my_core::ha_rows m_row_checksums_checked; // buffer to hold data during encode_value_slice String m_storage_record; + /* + For the active index, indicates which columns must be covered for the + current lookup to be covered. If the bitmap field is null, that means this + index does not cover the current lookup for any record. + */ + MY_BITMAP m_lookup_bitmap; }; } // namespace myrocks diff --git a/storage/rocksdb/rdb_datadic.cc b/storage/rocksdb/rdb_datadic.cc index 9afc2100069..a1cbee55c1d 100644 --- a/storage/rocksdb/rdb_datadic.cc +++ b/storage/rocksdb/rdb_datadic.cc @@ -81,28 +81,33 @@ inline bool field_check_field_name_match(Field *field, const char *name) other HA_ERR error code */ int Rdb_convert_to_record_key_decoder::decode_field( - Rdb_field_packing *fpi, Field *field, Rdb_string_reader *reader, - const uchar *const default_value, Rdb_string_reader *unpack_reader) { - if (fpi->m_maybe_null) { + Rdb_field_packing *fpi, TABLE *table, uchar *buf, Rdb_string_reader *reader, + Rdb_string_reader *unpack_reader) { + if (fpi->m_field_maybe_null) { const char *nullp; if (!(nullp = reader->read(1))) { return HA_EXIT_FAILURE; } - if (*nullp == 0) { + if (likely(*nullp == 1)) { + /* Clear the NULL-bit of this field */ + buf[fpi->m_field_null_offset] &= (uchar) ~(fpi->m_field_null_bit_mask); + } else if (*nullp == 0) { /* Set the NULL-bit of this field */ - field->set_null(); + buf[fpi->m_field_null_offset] |= fpi->m_field_null_bit_mask; + /* Also set the field to its default value */ - memcpy(field->ptr, default_value, field->pack_length()); + auto default_value = table->s->default_values + fpi->m_field_offset; + memcpy(buf + fpi->m_field_offset, default_value, + fpi->m_field_pack_length); return HA_EXIT_SUCCESS; - } else if (*nullp == 1) { - field->set_notnull(); } else { return HA_EXIT_FAILURE; } } - return (fpi->m_unpack_func)(fpi, field, field->ptr, reader, unpack_reader); + return (fpi->m_unpack_func)(fpi, buf + fpi->m_field_offset, reader, + unpack_reader); } /* @@ -121,33 +126,19 @@ int Rdb_convert_to_record_key_decoder::decode_field( other HA_ERR error code */ int Rdb_convert_to_record_key_decoder::decode( - uchar *const buf, uint *offset, Rdb_field_packing *fpi, TABLE *table, - Field *field, bool has_unpack_info, Rdb_string_reader *reader, + uchar *const buf, Rdb_field_packing *fpi, TABLE *table, + bool has_unpack_info, Rdb_string_reader *reader, Rdb_string_reader *unpack_reader) { DBUG_ASSERT(buf != nullptr); - DBUG_ASSERT(offset != nullptr); - - uint field_offset = field->ptr - table->record[0]; - *offset = field_offset; - uint null_offset = field->null_offset(); - bool maybe_null = field->real_maybe_null(); - - field->move_field(buf + field_offset, - maybe_null ? buf + null_offset : nullptr, field->null_bit); // If we need unpack info, but there is none, tell the unpack function // this by passing unp_reader as nullptr. If we never read unpack_info // during unpacking anyway, then there won't an error. bool maybe_missing_unpack = !has_unpack_info && fpi->uses_unpack_info(); - int res = - decode_field(fpi, field, reader, table->s->default_values + field_offset, - maybe_missing_unpack ? nullptr : unpack_reader); + int res = decode_field(fpi, table, buf, reader, + maybe_missing_unpack ? nullptr : unpack_reader); - // Restore field->ptr and field->null_ptr - field->move_field(table->record[0] + field_offset, - maybe_null ? table->record[0] + null_offset : nullptr, - field->null_bit); if (res != UNPACK_SUCCESS) { return HA_ERR_ROCKSDB_CORRUPT_DATA; } @@ -170,7 +161,7 @@ int Rdb_convert_to_record_key_decoder::skip(const Rdb_field_packing *fpi, Rdb_string_reader *reader, Rdb_string_reader *unp_reader) { /* It is impossible to unpack the column. Skip it. */ - if (fpi->m_maybe_null) { + if (fpi->m_field_maybe_null) { const char *nullp; if (!(nullp = reader->read(1))) { return HA_ERR_ROCKSDB_CORRUPT_DATA; @@ -184,7 +175,7 @@ int Rdb_convert_to_record_key_decoder::skip(const Rdb_field_packing *fpi, return HA_ERR_ROCKSDB_CORRUPT_DATA; } } - if ((fpi->m_skip_func)(fpi, field, reader)) { + if ((fpi->m_skip_func)(fpi, reader)) { return HA_ERR_ROCKSDB_CORRUPT_DATA; } // If this is a space padded varchar, we need to skip the indicator @@ -206,9 +197,8 @@ Rdb_key_field_iterator::Rdb_key_field_iterator( Rdb_string_reader *reader, Rdb_string_reader *unp_reader, TABLE *table, bool has_unpack_info, const MY_BITMAP *covered_bitmap, uchar *const buf) { m_key_def = key_def; - m_pack_info = pack_info; - m_iter_index = 0; - m_iter_end = key_def->get_key_parts(); + m_fpi = pack_info; + m_fpi_end = pack_info + key_def->get_key_parts(); m_reader = reader; m_unp_reader = unp_reader; m_table = table; @@ -221,68 +211,50 @@ Rdb_key_field_iterator::Rdb_key_field_iterator( m_is_hidden_pk = (key_def->m_index_type == Rdb_key_def::INDEX_TYPE_HIDDEN_PRIMARY); m_curr_bitmap_pos = 0; - m_offset = 0; } -void *Rdb_key_field_iterator::get_dst() const { return m_buf + m_offset; } - -int Rdb_key_field_iterator::get_field_index() const { - DBUG_ASSERT(m_field != nullptr); - return m_field->field_index; -} - -bool Rdb_key_field_iterator::get_is_null() const { return m_is_null; } -Field *Rdb_key_field_iterator::get_field() const { - DBUG_ASSERT(m_field != nullptr); - return m_field; -} - -bool Rdb_key_field_iterator::has_next() { return m_iter_index < m_iter_end; } +bool Rdb_key_field_iterator::has_next() { return m_fpi < m_fpi_end; } /** Iterate each field in the key and decode/skip one by one */ int Rdb_key_field_iterator::next() { int status = HA_EXIT_SUCCESS; - while (m_iter_index < m_iter_end) { - int curr_index = m_iter_index++; + while (m_fpi < m_fpi_end) { + auto fpi = m_fpi++; - m_fpi = &m_pack_info[curr_index]; /* Hidden pk field is packed at the end of the secondary keys, but the SQL layer does not know about it. Skip retrieving field if hidden pk. */ - if ((m_secondary_key && m_hidden_pk_exists && - curr_index + 1 == m_iter_end) || + if ((m_secondary_key && m_hidden_pk_exists && fpi + 1 == m_fpi_end) || m_is_hidden_pk) { - DBUG_ASSERT(m_fpi->m_unpack_func); - if ((m_fpi->m_skip_func)(m_fpi, nullptr, m_reader)) { + DBUG_ASSERT(fpi->m_unpack_func); + if ((fpi->m_skip_func)(fpi, m_reader)) { return HA_ERR_ROCKSDB_CORRUPT_DATA; } return HA_EXIT_SUCCESS; } - m_field = m_fpi->get_field_in_table(m_table); - bool covered_column = true; if (m_covered_bitmap != nullptr && - m_field->real_type() == MYSQL_TYPE_VARCHAR && !m_fpi->m_covered) { + fpi->m_field_real_type == MYSQL_TYPE_VARCHAR && !fpi->m_covered) { uint tmp= m_curr_bitmap_pos++; covered_column = m_curr_bitmap_pos < MAX_REF_PARTS && bitmap_is_set(m_covered_bitmap, tmp); } - if (m_fpi->m_unpack_func && covered_column) { + if (fpi->m_unpack_func && covered_column) { /* It is possible to unpack this column. Do it. */ status = Rdb_convert_to_record_key_decoder::decode( - m_buf, &m_offset, m_fpi, m_table, m_field, m_has_unpack_info, - m_reader, m_unp_reader); + m_buf, fpi, m_table, m_has_unpack_info, m_reader, m_unp_reader); if (status) { return status; } break; } else { - status = Rdb_convert_to_record_key_decoder::skip(m_fpi, m_field, m_reader, + auto field = fpi->get_field_in_table(m_table); + status = Rdb_convert_to_record_key_decoder::skip(fpi, field, m_reader, m_unp_reader); if (status) { return status; @@ -295,13 +267,13 @@ int Rdb_key_field_iterator::next() { /* Rdb_key_def class implementation */ -Rdb_key_def::Rdb_key_def(uint indexnr_arg, uint keyno_arg, - rocksdb::ColumnFamilyHandle *cf_handle_arg, - uint16_t index_dict_version_arg, uchar index_type_arg, - uint16_t kv_format_version_arg, bool is_reverse_cf_arg, - bool is_per_partition_cf_arg, const char *_name, - Rdb_index_stats _stats, uint32 index_flags_bitmap, - uint32 ttl_rec_offset, uint64 ttl_duration) +Rdb_key_def::Rdb_key_def( + uint indexnr_arg, uint keyno_arg, + std::shared_ptr cf_handle_arg, + uint16_t index_dict_version_arg, uchar index_type_arg, + uint16_t kv_format_version_arg, bool is_reverse_cf_arg, + bool is_per_partition_cf_arg, const char *_name, Rdb_index_stats _stats, + uint32 index_flags_bitmap, uint32 ttl_rec_offset, uint64 ttl_duration) : m_index_number(indexnr_arg), m_cf_handle(cf_handle_arg), m_index_dict_version(index_dict_version_arg), @@ -334,7 +306,7 @@ Rdb_key_def::Rdb_key_def(uint indexnr_arg, uint keyno_arg, DBUG_ASSERT_IMP(m_index_type == INDEX_TYPE_PRIMARY && m_kv_format_version <= PRIMARY_FORMAT_VERSION_UPDATE2, m_total_index_flags_length == 0); - DBUG_ASSERT(m_cf_handle != nullptr); + DBUG_ASSERT(m_cf_handle); } Rdb_key_def::Rdb_key_def(const Rdb_key_def &k) @@ -549,8 +521,9 @@ uint Rdb_key_def::setup(const TABLE *const tbl, */ if (!m_ttl_column.empty() && field_check_field_name_match(field, m_ttl_column.c_str())) { - DBUG_ASSERT(field->real_type() == MYSQL_TYPE_LONGLONG); - DBUG_ASSERT(field->key_type() == HA_KEYTYPE_ULONGLONG); + DBUG_ASSERT((field->real_type() == MYSQL_TYPE_LONGLONG && + field->key_type() == HA_KEYTYPE_ULONGLONG) || + field->type() == MYSQL_TYPE_TIMESTAMP); DBUG_ASSERT(!field->real_maybe_null()); m_ttl_pk_key_part_offset = dst_i; } @@ -683,8 +656,9 @@ uint Rdb_key_def::extract_ttl_col(const TABLE *const table_arg, for (uint i = 0; i < table_arg->s->fields; i++) { Field *const field = table_arg->field[i]; if (field_check_field_name_match(field, ttl_col_str.c_str()) && - field->real_type() == MYSQL_TYPE_LONGLONG && - field->key_type() == HA_KEYTYPE_ULONGLONG && + (field->type() == MYSQL_TYPE_TIMESTAMP || + (field->real_type() == MYSQL_TYPE_LONGLONG && + field->key_type() == HA_KEYTYPE_ULONGLONG)) && !field->real_maybe_null()) { *ttl_column = ttl_col_str; *ttl_field_index = i; @@ -849,7 +823,7 @@ int Rdb_key_def::read_memcmp_key_part(const TABLE *table_arg, Rdb_string_reader *reader, const uint part_num) const { /* It is impossible to unpack the column. Skip it. */ - if (m_pack_info[part_num].m_maybe_null) { + if (m_pack_info[part_num].m_field_maybe_null) { const char *nullp; if (!(nullp = reader->read(1))) return 1; if (*nullp == 0) { @@ -864,13 +838,7 @@ int Rdb_key_def::read_memcmp_key_part(const TABLE *table_arg, Rdb_field_packing *fpi = &m_pack_info[part_num]; DBUG_ASSERT(table_arg->s != nullptr); - bool is_hidden_pk_part = (part_num + 1 == m_key_parts) && - (table_arg->s->primary_key == MAX_INDEXES); - Field *field = nullptr; - if (!is_hidden_pk_part) { - field = fpi->get_field_in_table(table_arg); - } - if ((fpi->m_skip_func)(fpi, field, reader)) { + if ((fpi->m_skip_func)(fpi, reader)) { return 1; } return 0; @@ -1540,7 +1508,7 @@ int Rdb_key_def::compare_keys(const rocksdb::Slice *key1, for (uint i = 0; i < m_key_parts; i++) { const Rdb_field_packing *const fpi = &m_pack_info[i]; - if (fpi->m_maybe_null) { + if (fpi->m_field_maybe_null) { const auto nullp1 = reader1.read(1); const auto nullp2 = reader2.read(1); @@ -1562,10 +1530,10 @@ int Rdb_key_def::compare_keys(const rocksdb::Slice *key1, const auto before_skip1 = reader1.get_current_ptr(); const auto before_skip2 = reader2.get_current_ptr(); DBUG_ASSERT(fpi->m_skip_func); - if ((fpi->m_skip_func)(fpi, nullptr, &reader1)) { + if ((fpi->m_skip_func)(fpi, &reader1)) { return HA_EXIT_FAILURE; } - if ((fpi->m_skip_func)(fpi, nullptr, &reader2)) { + if ((fpi->m_skip_func)(fpi, &reader2)) { return HA_EXIT_FAILURE; } const auto size1 = reader1.get_current_ptr() - before_skip1; @@ -1604,17 +1572,45 @@ size_t Rdb_key_def::key_length(const TABLE *const table, } for (uint i = 0; i < m_key_parts; i++) { const Rdb_field_packing *fpi = &m_pack_info[i]; - const Field *field = nullptr; - if (m_index_type != INDEX_TYPE_HIDDEN_PRIMARY) { - field = fpi->get_field_in_table(table); - } - if ((fpi->m_skip_func)(fpi, field, &reader)) { + if ((fpi->m_skip_func)(fpi, &reader)) { return size_t(-1); } } return key.size() - reader.remaining_bytes(); } +/* + Decode unpack info in value + @return + HA_EXIT_SUCCESS OK + other HA_ERR error code + */ +int Rdb_key_def::decode_unpack_info(Rdb_string_reader *unp_reader, + bool *has_unpack_info, + const char **unpack_header) const { + // For secondary keys, we expect the value field to contain index flags, + // unpack data, and checksum data in that order. One or all can be missing, + // but they cannot be reordered. + if (unp_reader->remaining_bytes()) { + if (m_index_type == INDEX_TYPE_SECONDARY && + m_total_index_flags_length > 0 && + !unp_reader->read(m_total_index_flags_length)) { + return HA_ERR_ROCKSDB_CORRUPT_DATA; + } + } + + *unpack_header = unp_reader->get_current_ptr(); + *has_unpack_info = + unp_reader->remaining_bytes() && is_unpack_data_tag(*unpack_header[0]); + if (*has_unpack_info) { + if (!unp_reader->read(get_unpack_header_size(*unpack_header[0]))) { + return HA_ERR_ROCKSDB_CORRUPT_DATA; + } + } + + return HA_EXIT_SUCCESS; +} + /* Take mem-comparable form and unpack_info and unpack it to Table->record @@ -1640,28 +1636,16 @@ int Rdb_key_def::unpack_record(TABLE *const table, uchar *const buf, !verify_row_debug_checksums); // Skip the index number - if ((!reader.read(INDEX_NUMBER_SIZE))) { + if ((unlikely(!reader.read(INDEX_NUMBER_SIZE)))) { return HA_ERR_ROCKSDB_CORRUPT_DATA; } - // For secondary keys, we expect the value field to contain index flags, - // unpack data, and checksum data in that order. One or all can be missing, - // but they cannot be reordered. - if (unp_reader.remaining_bytes()) { - if (m_index_type == INDEX_TYPE_SECONDARY && - m_total_index_flags_length > 0 && - !unp_reader.read(m_total_index_flags_length)) { - return HA_ERR_ROCKSDB_CORRUPT_DATA; - } - } - - const char *unpack_header = unp_reader.get_current_ptr(); - bool has_unpack_info = - unp_reader.remaining_bytes() && is_unpack_data_tag(unpack_header[0]); - if (has_unpack_info) { - if (!unp_reader.read(get_unpack_header_size(unpack_header[0]))) { - return HA_ERR_ROCKSDB_CORRUPT_DATA; - } + const char *unpack_header; + bool has_unpack_info; + int err = HA_EXIT_SUCCESS; + err = decode_unpack_info(&unp_reader, &has_unpack_info, &unpack_header); + if (unlikely(err)) { + return err; } // Read the covered bitmap @@ -1676,15 +1660,12 @@ int Rdb_key_def::unpack_record(TABLE *const table, uchar *const buf, RDB_UNPACK_COVERED_DATA_LEN_SIZE); } - int err = HA_EXIT_SUCCESS; - - Rdb_key_field_iterator iter( this, m_pack_info, &reader, &unp_reader, table, has_unpack_info, has_covered_bitmap ? &covered_bitmap : nullptr, buf); while (iter.has_next()) { err = iter.next(); - if (err) { + if (unlikely(err)) { return err; } } @@ -1693,7 +1674,7 @@ int Rdb_key_def::unpack_record(TABLE *const table, uchar *const buf, Check checksum values if present */ const char *ptr; - if ((ptr = unp_reader.read(1)) && *ptr == RDB_CHECKSUM_DATA_TAG) { + if (unlikely((ptr = unp_reader.read(1)) && *ptr == RDB_CHECKSUM_DATA_TAG)) { if (verify_row_debug_checksums) { uint32_t stored_key_chksum = rdb_netbuf_to_uint32( (const uchar *)unp_reader.read(RDB_CHECKSUM_SIZE)); @@ -1724,7 +1705,7 @@ int Rdb_key_def::unpack_record(TABLE *const table, uchar *const buf, } } - if (reader.remaining_bytes()) return HA_ERR_ROCKSDB_CORRUPT_DATA; + if (unlikely(reader.remaining_bytes())) return HA_ERR_ROCKSDB_CORRUPT_DATA; return HA_EXIT_SUCCESS; } @@ -1771,8 +1752,6 @@ bool Rdb_key_def::index_format_min_check(const int pk_min, */ int Rdb_key_def::skip_max_length(const Rdb_field_packing *const fpi, - const Field *const field - MY_ATTRIBUTE((__unused__)), Rdb_string_reader *const reader) { if (!reader->read(fpi->m_max_image_len)) return HA_EXIT_FAILURE; return HA_EXIT_SUCCESS; @@ -1801,19 +1780,12 @@ static_assert((RDB_ESCAPE_LENGTH - 1) % 2 == 0, */ int Rdb_key_def::skip_variable_length(const Rdb_field_packing *const fpi, - const Field *const field, Rdb_string_reader *const reader) { const uchar *ptr; bool finished = false; - size_t dst_len; /* How much data can be there */ - if (field) { - const Field_varstring *const field_var = - static_cast(field); - dst_len = field_var->pack_length() - field_var->length_bytes; - } else { - dst_len = UINT_MAX; - } + /* How much data can be there */ + size_t dst_len = fpi->m_field_pack_length - fpi->m_varchar_length_bytes; bool use_legacy_format = fpi->m_use_legacy_varbinary_format; @@ -1857,18 +1829,12 @@ const int VARCHAR_CMP_GREATER_THAN_SPACES = 3; */ int Rdb_key_def::skip_variable_space_pad(const Rdb_field_packing *const fpi, - const Field *const field, Rdb_string_reader *const reader) { const uchar *ptr; bool finished = false; - size_t dst_len = UINT_MAX; /* How much data can be there */ - - if (field) { - const Field_varstring *const field_var = - static_cast(field); - dst_len = field_var->pack_length() - field_var->length_bytes; - } + /* How much data can be there */ + size_t dst_len = fpi->m_field_pack_length - fpi->m_varchar_length_bytes; /* Decode the length-emitted encoding here */ while ((ptr = (const uchar *)reader->read(fpi->m_segment_size))) { @@ -1899,12 +1865,12 @@ int Rdb_key_def::skip_variable_space_pad(const Rdb_field_packing *const fpi, /* Function of type rdb_index_field_unpack_t */ - -int Rdb_key_def::unpack_integer( - Rdb_field_packing *const fpi, Field *const field, uchar *const to, - Rdb_string_reader *const reader, - Rdb_string_reader *const unp_reader MY_ATTRIBUTE((__unused__))) { - const int length = fpi->m_max_image_len; +template +int Rdb_key_def::unpack_integer(Rdb_field_packing *const fpi, uchar *const to, + Rdb_string_reader *const reader, + Rdb_string_reader *const unp_reader + MY_ATTRIBUTE((__unused__))) { + DBUG_ASSERT(length == fpi->m_max_image_len); const uchar *from; if (!(from = (const uchar *)reader->read(length))) { @@ -1913,22 +1879,25 @@ int Rdb_key_def::unpack_integer( #ifdef WORDS_BIGENDIAN { - if (static_cast(field)->unsigned_flag) { + if (fpi->m_field_unsigned_flag) { to[0] = from[0]; } else { to[0] = static_cast(from[0] ^ 128); // Reverse the sign bit. } - memcpy(to + 1, from + 1, length - 1); + /* Parameterized length should enable loop unrolling */ + for (int i = 1; i < length; i++) to[i] = from[i]; } #else { const int sign_byte = from[0]; - if (static_cast(field)->unsigned_flag) { + if (fpi->m_field_unsigned_flag) { to[length - 1] = sign_byte; } else { to[length - 1] = static_cast(sign_byte ^ 128); // Reverse the sign bit. } + + /* Parameterized length should enable loop unrolling */ for (int i = 0, j = length - 1; i < length - 1; ++i, --j) to[i] = from[j]; } #endif @@ -2034,8 +2003,7 @@ int Rdb_key_def::unpack_floating_point( */ int Rdb_key_def::unpack_double( Rdb_field_packing *const fpi MY_ATTRIBUTE((__unused__)), - Field *const field MY_ATTRIBUTE((__unused__)), uchar *const field_ptr, - Rdb_string_reader *const reader, + uchar *const field_ptr, Rdb_string_reader *const reader, Rdb_string_reader *const unp_reader MY_ATTRIBUTE((__unused__))) { static double zero_val = 0.0; static const uchar zero_pattern[8] = {128, 0, 0, 0, 0, 0, 0, 0}; @@ -2058,8 +2026,8 @@ int Rdb_key_def::unpack_double( allowed in the database. */ int Rdb_key_def::unpack_float( - Rdb_field_packing *const fpi, Field *const field MY_ATTRIBUTE((__unused__)), - uchar *const field_ptr, Rdb_string_reader *const reader, + Rdb_field_packing *const fpi, uchar *const field_ptr, + Rdb_string_reader *const reader, Rdb_string_reader *const unp_reader MY_ATTRIBUTE((__unused__))) { static float zero_val = 0.0; static const uchar zero_pattern[4] = {128, 0, 0, 0}; @@ -2075,8 +2043,8 @@ int Rdb_key_def::unpack_float( */ int Rdb_key_def::unpack_newdate( - Rdb_field_packing *const fpi, Field *const field MY_ATTRIBUTE((__unused__)), - uchar *const field_ptr, Rdb_string_reader *const reader, + Rdb_field_packing *const fpi, uchar *const field_ptr, + Rdb_string_reader *const reader, Rdb_string_reader *const unp_reader MY_ATTRIBUTE((__unused__))) { const char *from; DBUG_ASSERT(fpi->m_max_image_len == 3); @@ -2099,7 +2067,7 @@ int Rdb_key_def::unpack_newdate( */ int Rdb_key_def::unpack_binary_str( - Rdb_field_packing *const fpi, Field *const field, uchar *const to, + Rdb_field_packing *const fpi, uchar *const to, Rdb_string_reader *const reader, Rdb_string_reader *const unp_reader MY_ATTRIBUTE((__unused__))) { const char *from; @@ -2119,10 +2087,10 @@ int Rdb_key_def::unpack_binary_str( */ int Rdb_key_def::unpack_utf8_str( - Rdb_field_packing *const fpi, Field *const field, uchar *dst, - Rdb_string_reader *const reader, + Rdb_field_packing *const fpi, uchar *dst, Rdb_string_reader *const reader, Rdb_string_reader *const unp_reader MY_ATTRIBUTE((__unused__))) { - my_core::CHARSET_INFO *const cset = (my_core::CHARSET_INFO *)field->charset(); + my_core::CHARSET_INFO *const cset = + (my_core::CHARSET_INFO *)fpi->m_field_charset; const uchar *src; if (!(src = (const uchar *)reader->read(fpi->m_max_image_len))) { /* Mem-comparable image doesn't have enough bytes */ @@ -2130,7 +2098,7 @@ int Rdb_key_def::unpack_utf8_str( } const uchar *const src_end = src + fpi->m_max_image_len; - uchar *const dst_end = dst + field->pack_length(); + uchar *const dst_end = dst + fpi->m_field_pack_length; while (src < src_end) { my_wc_t wc = (src[0] << 8) | src[1]; @@ -2262,12 +2230,12 @@ void Rdb_key_def::pack_with_varchar_encoding( const CHARSET_INFO *const charset = field->charset(); Field_varstring *const field_var = (Field_varstring *)field; - const size_t value_length = (field_var->length_bytes == 1) + const size_t value_length = (fpi->m_varchar_length_bytes == 1) ? (uint)*field->ptr : uint2korr(field->ptr); size_t xfrm_len = charset->strnxfrm( buf, fpi->m_max_image_len, field_var->char_length(), - field_var->ptr + field_var->length_bytes, value_length, 0); + field_var->ptr + fpi->m_varchar_length_bytes, value_length, 0); /* Got a mem-comparable image in 'buf'. Now, produce varlength encoding */ if (fpi->m_use_legacy_varbinary_format) { @@ -2373,16 +2341,16 @@ void Rdb_key_def::pack_with_varchar_space_pad( const CHARSET_INFO *const charset = field->charset(); const auto field_var = static_cast(field); - const size_t value_length = (field_var->length_bytes == 1) + const size_t value_length = (fpi->m_varchar_length_bytes == 1) ? (uint)*field->ptr : uint2korr(field->ptr); const size_t trimmed_len = charset->lengthsp( - (const char *)field_var->ptr + field_var->length_bytes, + (const char *)field_var->ptr + fpi->m_varchar_length_bytes, value_length); const size_t xfrm_len = charset->strnxfrm( buf, fpi->m_max_image_len, field_var->char_length(), - field_var->ptr + field_var->length_bytes, trimmed_len, 0); + field_var->ptr + fpi->m_varchar_length_bytes, trimmed_len, 0); /* Got a mem-comparable image in 'buf'. Now, produce varlength encoding */ uchar *const buf_end = buf + xfrm_len; @@ -2531,17 +2499,15 @@ static int unpack_charset( */ int Rdb_key_def::unpack_binary_or_utf8_varchar( - Rdb_field_packing *const fpi, Field *const field, uchar *dst, - Rdb_string_reader *const reader, + Rdb_field_packing *const fpi, uchar *dst, Rdb_string_reader *const reader, Rdb_string_reader *const unp_reader MY_ATTRIBUTE((__unused__))) { const uchar *ptr; size_t len = 0; bool finished = false; uchar *d0 = dst; - Field_varstring *const field_var = (Field_varstring *)field; - dst += field_var->length_bytes; + dst += fpi->m_varchar_length_bytes; // How much we can unpack - size_t dst_len = field_var->pack_length() - field_var->length_bytes; + size_t dst_len = fpi->m_field_pack_length - fpi->m_varchar_length_bytes; bool use_legacy_format = fpi->m_use_legacy_varbinary_format; @@ -2565,8 +2531,8 @@ int Rdb_key_def::unpack_binary_or_utf8_varchar( /* Now, we need to decode used_bytes of data and append them to the value. */ - if (fpi->m_varchar_charset->number == COLLATION_UTF8_BIN) { - int err = unpack_charset(fpi->m_varchar_charset, ptr, used_bytes, dst, + if (fpi->m_field_charset->number == COLLATION_UTF8_BIN) { + int err = unpack_charset(fpi->m_field_charset, ptr, used_bytes, dst, dst_len, &used_bytes); if (err != UNPACK_SUCCESS) { return err; @@ -2589,10 +2555,10 @@ int Rdb_key_def::unpack_binary_or_utf8_varchar( } /* Save the length */ - if (field_var->length_bytes == 1) { + if (fpi->m_varchar_length_bytes == 1) { d0[0] = (uchar)len; } else { - DBUG_ASSERT(field_var->length_bytes == 2); + DBUG_ASSERT(fpi->m_varchar_length_bytes == 2); int2store(d0, len); } return UNPACK_SUCCESS; @@ -2606,15 +2572,14 @@ int Rdb_key_def::unpack_binary_or_utf8_varchar( skip_variable_space_pad - skip function */ int Rdb_key_def::unpack_binary_or_utf8_varchar_space_pad( - Rdb_field_packing *const fpi, Field *const field, uchar *dst, - Rdb_string_reader *const reader, Rdb_string_reader *const unp_reader) { + Rdb_field_packing *const fpi, uchar *dst, Rdb_string_reader *const reader, + Rdb_string_reader *const unp_reader) { const uchar *ptr; size_t len = 0; bool finished = false; - Field_varstring *const field_var = static_cast(field); uchar *d0 = dst; - uchar *dst_end = dst + field_var->pack_length(); - dst += field_var->length_bytes; + uchar *dst_end = dst + fpi->m_field_pack_length; + dst += fpi->m_varchar_length_bytes; uint space_padding_bytes = 0; uint extra_spaces; @@ -2654,7 +2619,7 @@ int Rdb_key_def::unpack_binary_or_utf8_varchar_space_pad( } // Now, need to decode used_bytes of data and append them to the value. - if (fpi->m_varchar_charset->number == COLLATION_UTF8_BIN) { + if (fpi->m_field_charset->number == COLLATION_UTF8_BIN) { if (used_bytes & 1) { /* UTF-8 characters are encoded into two-byte entities. There is no way @@ -2668,7 +2633,7 @@ int Rdb_key_def::unpack_binary_or_utf8_varchar_space_pad( while (src < src_end) { my_wc_t wc = (src[0] << 8) | src[1]; src += 2; - const CHARSET_INFO *cset = fpi->m_varchar_charset; + const CHARSET_INFO *cset = fpi->m_field_charset; int res = cset->wc_mb(wc, dst, dst_end); DBUG_ASSERT(res <= 3); if (res <= 0) return UNPACK_FAILURE; @@ -2687,7 +2652,7 @@ int Rdb_key_def::unpack_binary_or_utf8_varchar_space_pad( // Both binary and UTF-8 charset store space as ' ', // so the following is ok: if (dst + extra_spaces > dst_end) return UNPACK_FAILURE; - memset(dst, fpi->m_varchar_charset->pad_char, extra_spaces); + memset(dst, fpi->m_field_charset->pad_char, extra_spaces); len += extra_spaces; } break; @@ -2697,10 +2662,10 @@ int Rdb_key_def::unpack_binary_or_utf8_varchar_space_pad( if (!finished) return UNPACK_FAILURE; /* Save the length */ - if (field_var->length_bytes == 1) { + if (fpi->m_varchar_length_bytes == 1) { d0[0] = (uchar)len; } else { - DBUG_ASSERT(field_var->length_bytes == 2); + DBUG_ASSERT(fpi->m_varchar_length_bytes == 2); int2store(d0, len); } return UNPACK_SUCCESS; @@ -2737,14 +2702,13 @@ void Rdb_key_def::dummy_make_unpack_info( Function of type rdb_index_field_unpack_t */ -int Rdb_key_def::unpack_unknown(Rdb_field_packing *const fpi, - Field *const field, uchar *const dst, +int Rdb_key_def::unpack_unknown(Rdb_field_packing *const fpi, uchar *const dst, Rdb_string_reader *const reader, Rdb_string_reader *const unp_reader) { const uchar *ptr; const uint len = fpi->m_unpack_data_len; // We don't use anything from the key, so skip over it. - if (skip_max_length(fpi, field, reader)) { + if (skip_max_length(fpi, reader)) { return UNPACK_FAILURE; } @@ -2785,16 +2749,15 @@ void Rdb_key_def::make_unpack_unknown_varchar( */ int Rdb_key_def::unpack_unknown_varchar(Rdb_field_packing *const fpi, - Field *const field, uchar *dst, + uchar *dst, Rdb_string_reader *const reader, Rdb_string_reader *const unp_reader) { const uchar *ptr; uchar *const d0 = dst; - const auto f = static_cast(field); - dst += f->length_bytes; - const uint len_bytes = f->length_bytes; + dst += fpi->m_varchar_length_bytes; + const uint len_bytes = fpi->m_varchar_length_bytes; // We don't use anything from the key, so skip over it. - if ((fpi->m_skip_func)(fpi, field, reader)) { + if ((fpi->m_skip_func)(fpi, reader)) { return UNPACK_FAILURE; } @@ -2875,18 +2838,16 @@ void Rdb_key_def::make_unpack_simple_varchar( */ int Rdb_key_def::unpack_simple_varchar_space_pad( - Rdb_field_packing *const fpi, Field *const field, uchar *dst, - Rdb_string_reader *const reader, Rdb_string_reader *const unp_reader) { + Rdb_field_packing *const fpi, uchar *dst, Rdb_string_reader *const reader, + Rdb_string_reader *const unp_reader) { const uchar *ptr; size_t len = 0; bool finished = false; uchar *d0 = dst; - const Field_varstring *const field_var = - static_cast(field); // For simple collations, char_length is also number of bytes. - DBUG_ASSERT((size_t)fpi->m_max_image_len >= field_var->char_length()); - uchar *dst_end = dst + field_var->pack_length(); - dst += field_var->length_bytes; + DBUG_ASSERT((size_t)fpi->m_max_image_len >= fpi->m_varchar_char_length); + uchar *dst_end = dst + fpi->m_field_pack_length; + dst += fpi->m_varchar_length_bytes; Rdb_bit_reader bit_reader(unp_reader); uint space_padding_bytes = 0; @@ -2947,7 +2908,7 @@ int Rdb_key_def::unpack_simple_varchar_space_pad( if (dst + extra_spaces > dst_end) return UNPACK_FAILURE; // pad_char has a 1-byte form in all charsets that // are handled by rdb_init_collation_mapping. - memset(dst, field_var->charset()->pad_char, extra_spaces); + memset(dst, fpi->m_field_charset->pad_char, extra_spaces); len += extra_spaces; } break; @@ -2957,10 +2918,10 @@ int Rdb_key_def::unpack_simple_varchar_space_pad( if (!finished) return UNPACK_FAILURE; /* Save the length */ - if (field_var->length_bytes == 1) { + if (fpi->m_varchar_length_bytes == 1) { d0[0] = (uchar)len; } else { - DBUG_ASSERT(field_var->length_bytes == 2); + DBUG_ASSERT(fpi->m_varchar_length_bytes == 2); int2store(d0, len); } return UNPACK_SUCCESS; @@ -2989,9 +2950,7 @@ void Rdb_key_def::make_unpack_simple(const Rdb_collation_codec *const codec, Function of type rdb_index_field_unpack_t */ -int Rdb_key_def::unpack_simple(Rdb_field_packing *const fpi, - Field *const field MY_ATTRIBUTE((__unused__)), - uchar *const dst, +int Rdb_key_def::unpack_simple(Rdb_field_packing *const fpi, uchar *const dst, Rdb_string_reader *const reader, Rdb_string_reader *const unp_reader) { const uchar *ptr; @@ -3218,10 +3177,21 @@ bool Rdb_field_packing::setup(const Rdb_key_def *const key_descr, int res = false; enum_field_types type = field ? field->real_type() : MYSQL_TYPE_LONGLONG; + m_field_real_type = type; + m_field_offset = + (key_descr && field) ? (field->ptr - field->table->record[0]) : -1; + m_field_null_offset = (key_descr && field) ? field->null_offset() : -1; + m_field_null_bit_mask = (key_descr && field) ? field->null_bit : 0; + m_field_pack_length = (key_descr && field) ? field->pack_length() : -1; + m_field_charset = (key_descr && field) ? field->charset() : nullptr; + m_field_unsigned_flag = false; + m_field_maybe_null = field ? field->real_maybe_null() : false; + m_varchar_length_bytes = -1; + m_varchar_char_length = -1; + m_keynr = keynr_arg; m_key_part = key_part_arg; - m_maybe_null = field ? field->real_maybe_null() : false; m_unpack_func = nullptr; m_make_unpack_info_func = nullptr; m_unpack_data_len = 0; @@ -3245,11 +3215,37 @@ bool Rdb_field_packing::setup(const Rdb_key_def *const key_descr, switch (type) { case MYSQL_TYPE_LONGLONG: + m_field_unsigned_flag = + field ? static_cast(field)->unsigned_flag : false; + m_unpack_func = Rdb_key_def::unpack_integer<8>; + m_covered = true; + return true; + case MYSQL_TYPE_LONG: + m_field_unsigned_flag = + field ? static_cast(field)->unsigned_flag : false; + m_unpack_func = Rdb_key_def::unpack_integer<4>; + m_covered = true; + return true; + case MYSQL_TYPE_INT24: + m_field_unsigned_flag = + field ? static_cast(field)->unsigned_flag : false; + m_unpack_func = Rdb_key_def::unpack_integer<3>; + m_covered = true; + return true; + case MYSQL_TYPE_SHORT: + m_field_unsigned_flag = + field ? static_cast(field)->unsigned_flag : false; + m_unpack_func = Rdb_key_def::unpack_integer<2>; + m_covered = true; + return true; + case MYSQL_TYPE_TINY: - m_unpack_func = Rdb_key_def::unpack_integer; + m_field_unsigned_flag = + field ? static_cast(field)->unsigned_flag : false; + m_unpack_func = Rdb_key_def::unpack_integer<1>; m_covered = true; return true; @@ -3338,7 +3334,9 @@ bool Rdb_field_packing::setup(const Rdb_key_def *const key_descr, if (is_varchar) { // The default for varchar is variable-length, without space-padding for // comparisons - m_varchar_charset = cs; + const auto field_var = static_cast(field); + m_varchar_length_bytes = field_var->length_bytes; + m_varchar_char_length = field_var->char_length(); m_skip_func = Rdb_key_def::skip_variable_length; m_pack_func = Rdb_key_def::pack_with_varchar_encoding; if (!key_descr || key_descr->use_legacy_varbinary_format()) { @@ -3349,7 +3347,6 @@ bool Rdb_field_packing::setup(const Rdb_key_def *const key_descr, m_max_image_len = RDB_ENCODED_SIZE(m_max_image_len); } - const auto field_var = static_cast(field); m_unpack_info_uses_two_bytes = (field_var->field_length + 8 >= 0x100); } @@ -3544,6 +3541,7 @@ Rdb_tbl_def::~Rdb_tbl_def() { */ bool Rdb_tbl_def::put_dict(Rdb_dict_manager *const dict, + Rdb_cf_manager *cf_manager, rocksdb::WriteBatch *const batch, const rocksdb::Slice &key) { StringBuffer<8 * Rdb_key_def::PACKED_SIZE> indexes; @@ -3554,10 +3552,6 @@ bool Rdb_tbl_def::put_dict(Rdb_dict_manager *const dict, for (uint i = 0; i < m_key_count; i++) { const Rdb_key_def &kd = *m_key_descr_arr[i]; - uchar flags = - (kd.m_is_reverse_cf ? Rdb_key_def::REVERSE_CF_FLAG : 0) | - (kd.m_is_per_partition_cf ? Rdb_key_def::PER_PARTITION_CF_FLAG : 0); - const uint cf_id = kd.get_cf()->GetID(); /* If cf_id already exists, cf_flags must be the same. @@ -3566,23 +3560,17 @@ bool Rdb_tbl_def::put_dict(Rdb_dict_manager *const dict, When RocksDB supports transaction with pessimistic concurrency control, we can switch to use it and removing mutex. */ - uint existing_cf_flags; const std::string cf_name = kd.get_cf()->GetName(); - if (dict->get_cf_flags(cf_id, &existing_cf_flags)) { - // For the purposes of comparison we'll clear the partitioning bit. The - // intent here is to make sure that both partitioned and non-partitioned - // tables can refer to the same CF. - existing_cf_flags &= ~Rdb_key_def::CF_FLAGS_TO_IGNORE; - flags &= ~Rdb_key_def::CF_FLAGS_TO_IGNORE; + std::shared_ptr cfh = + cf_manager->get_cf(cf_name); - if (existing_cf_flags != flags) { - my_error(ER_CF_DIFFERENT, MYF(0), cf_name.c_str(), flags, - existing_cf_flags); - return true; - } - } else { - dict->add_cf_flags(batch, cf_id, flags); + if (!cfh || cfh != kd.get_shared_cf() || dict->get_dropped_cf(cf_id)) { + // The CF has been dropped, i.e., cf_manager.remove_dropped_cf() has been + // called; or the CF is being dropped, i.e., cf_manager.drop_cf() has + // been called. + my_error(ER_CF_DROPPED, MYF(0), cf_name.c_str()); + return true; } rdb_netstr_append_uint32(&indexes, cf_id); @@ -3616,7 +3604,7 @@ time_t Rdb_tbl_def::get_create_time() { char path[FN_REFLEN]; snprintf(path, sizeof(path), "%s/%s/%s%s", mysql_data_home, m_dbname.c_str(), m_tablename.c_str(), reg_ext); - unpack_filename(path,path); + unpack_filename(path, path); MY_STAT f_stat; if (my_stat(path, &f_stat, MYF(0))) create_time = f_stat.st_ctime; @@ -3743,6 +3731,21 @@ void Rdb_ddl_manager::remove_uncommitted_keydefs( mysql_rwlock_unlock(&m_rwlock); } +int Rdb_ddl_manager::find_in_uncommitted_keydef(const uint32_t &cf_id) { + mysql_rwlock_rdlock(&m_rwlock); + for (const auto &pr : m_index_num_to_uncommitted_keydef) { + const auto &kd = pr.second; + + if (kd->get_cf()->GetID() == cf_id) { + mysql_rwlock_unlock(&m_rwlock); + return HA_EXIT_FAILURE; + } + } + + mysql_rwlock_unlock(&m_rwlock); + return HA_EXIT_SUCCESS; +} + namespace // anonymous namespace = not visible outside this source file { struct Rdb_validate_tbls : public Rdb_tables_scanner { @@ -3770,8 +3773,12 @@ struct Rdb_validate_tbls : public Rdb_tables_scanner { int Rdb_validate_tbls::add_table(Rdb_tbl_def *tdef) { DBUG_ASSERT(tdef != nullptr); - /* Add the database/table into the list that are not temp table */ - if (tdef->base_tablename().find(tmp_file_prefix) == std::string::npos) { + /* + Add the database/table into the list that are not temp table. + Also skip over truncate temp table. + */ + if (tdef->base_tablename().find(tmp_file_prefix) == std::string::npos && + tdef->base_tablename().find(TRUNCATE_TABLE_PREFIX) == std::string::npos) { bool is_partition = tdef->base_partition().size() != 0; m_list[tdef->base_dbname()].insert( tbl_info_t(tdef->base_tablename(), is_partition)); @@ -4033,6 +4040,7 @@ bool Rdb_ddl_manager::init(Rdb_dict_manager *const dict_arg, Rdb_cf_manager *const cf_manager, const uint32_t validate_tables) { m_dict = dict_arg; + m_cf_manager = cf_manager; mysql_rwlock_init(0, &m_rwlock); /* Read the data dictionary and populate the hash */ @@ -4133,9 +4141,9 @@ bool Rdb_ddl_manager::init(Rdb_dict_manager *const dict_arg, gl_index_id.cf_id, tdef->full_tablename().c_str()); } - rocksdb::ColumnFamilyHandle *const cfh = + std::shared_ptr cfh = cf_manager->get_cf(gl_index_id.cf_id); - DBUG_ASSERT(cfh != nullptr); + DBUG_ASSERT(cfh); uint32 ttl_rec_offset = Rdb_key_def::has_index_flag(index_info.m_index_flags, @@ -4363,7 +4371,8 @@ int Rdb_ddl_manager::put_and_write(Rdb_tbl_def *const tbl, buf_writer.write(dbname_tablename.c_str(), dbname_tablename.size()); int res; - if ((res = tbl->put_dict(m_dict, batch, buf_writer.to_slice()))) { + if ((res = + tbl->put_dict(m_dict, m_cf_manager, batch, buf_writer.to_slice()))) { return res; } if ((res = put(tbl))) { @@ -4461,7 +4470,8 @@ bool Rdb_ddl_manager::rename(const std::string &from, const std::string &to, new_buf_writer.write(dbname_tablename.c_str(), dbname_tablename.size()); // Create a key to add - if (!new_rec->put_dict(m_dict, batch, new_buf_writer.to_slice())) { + if (!new_rec->put_dict(m_dict, m_cf_manager, batch, + new_buf_writer.to_slice())) { remove(rec, batch, false); put(new_rec, false); res = false; // ok @@ -4487,6 +4497,8 @@ int Rdb_ddl_manager::scan_for_tables(Rdb_tables_scanner *const tables_scanner) { DBUG_ASSERT(tables_scanner != nullptr); + // This method should NOT accquire dict_manager lock and + // cf_manager lock in order to prevent deadlocks. mysql_rwlock_rdlock(&m_rwlock); ret = 0; @@ -4722,7 +4734,8 @@ void Rdb_binlog_manager::update_slave_gtid_info( } bool Rdb_dict_manager::init(rocksdb::TransactionDB *const rdb_dict, - Rdb_cf_manager *const cf_manager) { + Rdb_cf_manager *const cf_manager, + const my_bool enable_remove_orphaned_dropped_cfs) { DBUG_ASSERT(rdb_dict != nullptr); DBUG_ASSERT(cf_manager != nullptr); @@ -4730,9 +4743,13 @@ bool Rdb_dict_manager::init(rocksdb::TransactionDB *const rdb_dict, m_db = rdb_dict; - m_system_cfh = cf_manager->get_or_create_cf(m_db, DEFAULT_SYSTEM_CF_NAME); + // It is safe to get raw pointers here since: + // 1. System CF and default CF cannot be dropped + // 2. cf_manager outlives dict_manager + m_system_cfh = + cf_manager->get_or_create_cf(m_db, DEFAULT_SYSTEM_CF_NAME).get(); rocksdb::ColumnFamilyHandle *default_cfh = - cf_manager->get_cf(DEFAULT_CF_NAME); + cf_manager->get_cf(DEFAULT_CF_NAME).get(); // System CF and default CF should be initialized if (m_system_cfh == nullptr || default_cfh == nullptr) { @@ -4756,6 +4773,15 @@ bool Rdb_dict_manager::init(rocksdb::TransactionDB *const rdb_dict, add_cf_flags(batch, default_cfh->GetID(), 0); commit(batch); + if (add_missing_cf_flags(cf_manager)) { + return HA_EXIT_FAILURE; + } + + if (remove_orphaned_dropped_cfs(cf_manager, + enable_remove_orphaned_dropped_cfs)) { + return HA_EXIT_FAILURE; + } + return HA_EXIT_SUCCESS; } @@ -4858,6 +4884,20 @@ void Rdb_dict_manager::add_cf_flags(rocksdb::WriteBatch *const batch, batch->Put(m_system_cfh, key_writer.to_slice(), value_writer.to_slice()); } +void Rdb_dict_manager::delete_cf_flags(rocksdb::WriteBatch *const batch, + const uint &cf_id) const { + DBUG_ASSERT(batch != nullptr); + + uchar key_buf[Rdb_key_def::INDEX_NUMBER_SIZE * 2] = {0}; + + rdb_netbuf_store_uint32(key_buf, Rdb_key_def::CF_DEFINITION); + rdb_netbuf_store_uint32(key_buf + Rdb_key_def::INDEX_NUMBER_SIZE, cf_id); + const rocksdb::Slice key = + rocksdb::Slice(reinterpret_cast(key_buf), sizeof(key_buf)); + + delete_key(batch, key); +} + void Rdb_dict_manager::delete_index_info(rocksdb::WriteBatch *batch, const GL_INDEX_ID &gl_index_id) const { delete_with_prefix(batch, Rdb_key_def::INDEX_INFO, gl_index_id); @@ -5002,6 +5042,82 @@ bool Rdb_dict_manager::get_cf_flags(const uint32_t cf_id, return found; } +void Rdb_dict_manager::add_dropped_cf(rocksdb::WriteBatch *const batch, + const uint &cf_id) const { + DBUG_ASSERT(batch != nullptr); + + uchar key_buf[Rdb_key_def::INDEX_NUMBER_SIZE * 2] = {0}; + uchar value_buf[Rdb_key_def::VERSION_SIZE] = {0}; + rdb_netbuf_store_uint32(key_buf, Rdb_key_def::DROPPED_CF); + rdb_netbuf_store_uint32(key_buf + Rdb_key_def::INDEX_NUMBER_SIZE, cf_id); + const rocksdb::Slice key = + rocksdb::Slice(reinterpret_cast(key_buf), sizeof(key_buf)); + + rdb_netbuf_store_uint16(value_buf, Rdb_key_def::DROPPED_CF_VERSION); + const rocksdb::Slice value = + rocksdb::Slice(reinterpret_cast(value_buf), sizeof(value_buf)); + batch->Put(m_system_cfh, key, value); +} + +bool Rdb_dict_manager::get_dropped_cf(const uint &cf_id) const { + std::string value; + uchar key_buf[Rdb_key_def::INDEX_NUMBER_SIZE * 2] = {0}; + + rdb_netbuf_store_uint32(key_buf, Rdb_key_def::DROPPED_CF); + rdb_netbuf_store_uint32(key_buf + Rdb_key_def::INDEX_NUMBER_SIZE, cf_id); + + const rocksdb::Slice key = + rocksdb::Slice(reinterpret_cast(key_buf), sizeof(key_buf)); + const rocksdb::Status status = get_value(key, &value); + + return status.ok(); +} + +void Rdb_dict_manager::delete_dropped_cf_and_flags( + rocksdb::WriteBatch *const batch, const uint &cf_id) const { + DBUG_ASSERT(batch != nullptr); + delete_dropped_cf(batch, cf_id); + delete_cf_flags(batch, cf_id); +} + +void Rdb_dict_manager::delete_dropped_cf(rocksdb::WriteBatch *const batch, + const uint &cf_id) const { + DBUG_ASSERT(batch != nullptr); + + uchar key_buf[Rdb_key_def::INDEX_NUMBER_SIZE * 2] = {0}; + + rdb_netbuf_store_uint32(key_buf, Rdb_key_def::DROPPED_CF); + rdb_netbuf_store_uint32(key_buf + Rdb_key_def::INDEX_NUMBER_SIZE, cf_id); + const rocksdb::Slice key = + rocksdb::Slice(reinterpret_cast(key_buf), sizeof(key_buf)); + + delete_key(batch, key); +} + +void Rdb_dict_manager::get_all_dropped_cfs( + std::unordered_set *dropped_cf_ids) const { + uchar dropped_cf_buf[Rdb_key_def::INDEX_NUMBER_SIZE]; + rdb_netbuf_store_uint32(dropped_cf_buf, Rdb_key_def::DROPPED_CF); + const rocksdb::Slice dropped_cf_slice( + reinterpret_cast(dropped_cf_buf), Rdb_key_def::INDEX_NUMBER_SIZE); + + rocksdb::Iterator *it = new_iterator(); + for (it->Seek(dropped_cf_slice); it->Valid(); it->Next()) { + rocksdb::Slice key = it->key(); + const uchar *const ptr = (const uchar *)key.data(); + + if (key.size() != Rdb_key_def::INDEX_NUMBER_SIZE * 2 || + rdb_netbuf_to_uint32(ptr) != Rdb_key_def::DROPPED_CF) { + break; + } + + uint32 cf_id = rdb_netbuf_to_uint32(ptr + Rdb_key_def::INDEX_NUMBER_SIZE); + dropped_cf_ids->insert(cf_id); + } + + delete it; +} + /* Returning index ids that were marked as deleted (via DROP TABLE) but still not removed by drop_index_thread yet, or indexes that are marked as @@ -5047,6 +5163,58 @@ void Rdb_dict_manager::get_ongoing_index_operation( delete it; } +/* + If mysqld reboots during create table, a column family can be + created without cf flags. This method adds missing cf flags. It + only should be called during mysqld startup. + */ +int Rdb_dict_manager::add_missing_cf_flags( + Rdb_cf_manager *const cf_manager) const { + for (const auto &cf_name : cf_manager->get_cf_names()) { + std::shared_ptr cfh = + cf_manager->get_cf(cf_name); + + if (cf_manager->create_cf_flags_if_needed(this, cfh->GetID(), cf_name)) { + return HA_EXIT_FAILURE; + } + } + + return HA_EXIT_SUCCESS; +} + +/* + If mysqld reboots during dropping a column family, it can happen + that the column family is deleted from RocksDB, but its id is + in the list of cf ids that are to be dropped. + This method cleans up these orphaned cf ids. It only should be + called during mysqld startup. + */ +int Rdb_dict_manager::remove_orphaned_dropped_cfs( + Rdb_cf_manager *const cf_manager, + const my_bool &enable_remove_orphaned_dropped_cfs) const { + const std::unique_ptr wb = begin(); + rocksdb::WriteBatch *const batch = wb.get(); + + std::unordered_set dropped_cf_ids; + get_all_dropped_cfs(&dropped_cf_ids); + for (const auto cf_id : dropped_cf_ids) { + if (!cf_manager->get_cf(cf_id)) { + // NO_LINT_DEBUG + sql_print_warning( + "RocksDB: Column family with id %u doesn't exist in " + "cf manager, but it is listed to be dropped", + cf_id); + + if (enable_remove_orphaned_dropped_cfs) { + delete_dropped_cf_and_flags(batch, cf_id); + } + } + } + + commit(batch); + return HA_EXIT_SUCCESS; +} + /* Returning true if index_id is create/delete ongoing (undergoing creation or marked as deleted via DROP TABLE but drop_index_thread has not wiped yet) @@ -5230,11 +5398,15 @@ void Rdb_dict_manager::resume_drop_indexes() const { } void Rdb_dict_manager::rollback_ongoing_index_creation() const { - const std::unique_ptr wb = begin(); - rocksdb::WriteBatch *const batch = wb.get(); - std::unordered_set gl_index_ids; get_ongoing_create_indexes(&gl_index_ids); + rollback_ongoing_index_creation(gl_index_ids); +} + +void Rdb_dict_manager::rollback_ongoing_index_creation( + const std::unordered_set &gl_index_ids) const { + const std::unique_ptr wb = begin(); + rocksdb::WriteBatch *const batch = wb.get(); for (const auto &gl_index_id : gl_index_ids) { // NO_LINT_DEBUG diff --git a/storage/rocksdb/rdb_datadic.h b/storage/rocksdb/rdb_datadic.h index e9fc1da728e..33e74c4f9f5 100644 --- a/storage/rocksdb/rdb_datadic.h +++ b/storage/rocksdb/rdb_datadic.h @@ -56,17 +56,15 @@ class Rdb_convert_to_record_key_decoder { const Rdb_convert_to_record_key_decoder &decoder) = delete; Rdb_convert_to_record_key_decoder &operator=( const Rdb_convert_to_record_key_decoder &decoder) = delete; - static int decode(uchar *const buf, uint *offset, Rdb_field_packing *fpi, - TABLE *table, Field *field, bool has_unpack_info, - Rdb_string_reader *reader, + static int decode(uchar *const buf, Rdb_field_packing *fpi, TABLE *table, + bool has_unpack_info, Rdb_string_reader *reader, Rdb_string_reader *unpack_reader); static int skip(const Rdb_field_packing *fpi, const Field *field, Rdb_string_reader *reader, Rdb_string_reader *unpack_reader); private: - static int decode_field(Rdb_field_packing *fpi, Field *field, + static int decode_field(Rdb_field_packing *fpi, TABLE *table, uchar *buf, Rdb_string_reader *reader, - const uchar *const default_value, Rdb_string_reader *unpack_reader); }; @@ -98,9 +96,6 @@ class Rdb_pack_field_context { class Rdb_key_field_iterator { private: - Rdb_field_packing *m_pack_info; - int m_iter_index; - int m_iter_end; TABLE *m_table; Rdb_string_reader *m_reader; Rdb_string_reader *m_unp_reader; @@ -113,9 +108,8 @@ class Rdb_key_field_iterator { bool m_hidden_pk_exists; bool m_is_hidden_pk; bool m_is_null; - Field *m_field; - uint m_offset; Rdb_field_packing *m_fpi; + Rdb_field_packing *m_fpi_end; public: Rdb_key_field_iterator(const Rdb_key_field_iterator &) = delete; @@ -129,10 +123,6 @@ class Rdb_key_field_iterator { int next(); bool has_next(); - bool get_is_null() const; - Field *get_field() const; - int get_field_index() const; - void *get_dst() const; }; struct Rdb_collation_codec; @@ -145,12 +135,11 @@ struct Rdb_index_info; using rdb_make_unpack_info_t = void (*)(const Rdb_collation_codec *codec, const Field *field, Rdb_pack_field_context *pack_ctx); -using rdb_index_field_unpack_t = int (*)(Rdb_field_packing *fpi, Field *field, +using rdb_index_field_unpack_t = int (*)(Rdb_field_packing *fpi, uchar *field_ptr, Rdb_string_reader *reader, Rdb_string_reader *unpack_reader); using rdb_index_field_skip_t = int (*)(const Rdb_field_packing *fpi, - const Field *field, Rdb_string_reader *reader); using rdb_index_field_pack_t = void (*)(Rdb_field_packing *fpi, Field *field, uchar *buf, uchar **dst, @@ -277,6 +266,9 @@ class Rdb_key_def { const rocksdb::Slice *const unpack_info, const bool verify_row_debug_checksums) const; + int decode_unpack_info(Rdb_string_reader *unp_reader, bool *has_unpack_info, + const char **unpack_header) const; + static bool unpack_info_has_checksum(const rocksdb::Slice &unpack_info); int compare_keys(const rocksdb::Slice *key1, const rocksdb::Slice *key2, std::size_t *const column_index) const; @@ -392,6 +384,11 @@ class Rdb_key_def { m_kv_format_version >= SECONDARY_FORMAT_VERSION_UPDATE3; } + inline bool is_primary_key() const { + return m_index_type == INDEX_TYPE_PRIMARY || + m_index_type == INDEX_TYPE_HIDDEN_PRIMARY; + } + /* Indicates that all key parts can be unpacked to cover a secondary lookup */ bool can_cover_lookup() const; @@ -457,7 +454,7 @@ class Rdb_key_def { Rdb_key_def &operator=(const Rdb_key_def &) = delete; Rdb_key_def(const Rdb_key_def &k); Rdb_key_def(uint indexnr_arg, uint keyno_arg, - rocksdb::ColumnFamilyHandle *cf_handle_arg, + std::shared_ptr cf_handle_arg, uint16_t index_dict_version_arg, uchar index_type_arg, uint16_t kv_format_version_arg, bool is_reverse_cf_arg, bool is_per_partition_cf, const char *name, @@ -505,7 +502,7 @@ class Rdb_key_def { MAX_INDEX_ID = 7, DDL_CREATE_INDEX_ONGOING = 8, AUTO_INC = 9, - // MariaDB: 10 through 12 are already taken in upstream + DROPPED_CF = 10, TABLE_VERSION = 20, // MariaDB: table version record END_DICT_INDEX_ID = 255 }; @@ -520,6 +517,7 @@ class Rdb_key_def { MAX_INDEX_ID_VERSION = 1, DDL_CREATE_INDEX_ONGOING_VERSION = 1, AUTO_INCREMENT_VERSION = 1, + DROPPED_CF_VERSION = 1, // Version for index stats is stored in IndexStats struct }; @@ -626,7 +624,10 @@ class Rdb_key_def { const Rdb_tbl_def *const tbl_def_arg, bool *per_part_match_found, const char *const qualifier); - rocksdb::ColumnFamilyHandle *get_cf() const { return m_cf_handle; } + rocksdb::ColumnFamilyHandle *get_cf() const { return m_cf_handle.get(); } + std::shared_ptr get_shared_cf() const { + return m_cf_handle; + } /* Check if keypart #kp can be unpacked from index tuple */ inline bool can_unpack(const uint kp) const; @@ -656,64 +657,58 @@ class Rdb_key_def { Rdb_field_packing *const fpi, Field *const field, uchar *buf, uchar **dst, Rdb_pack_field_context *const pack_ctx); - static int unpack_integer(Rdb_field_packing *const fpi, Field *const field, - uchar *const to, Rdb_string_reader *const reader, + template + static int unpack_integer(Rdb_field_packing *const fpi, uchar *const to, + Rdb_string_reader *const reader, Rdb_string_reader *const unp_reader MY_ATTRIBUTE((__unused__))); static int unpack_double( Rdb_field_packing *const fpi MY_ATTRIBUTE((__unused__)), - Field *const field MY_ATTRIBUTE((__unused__)), uchar *const field_ptr, - Rdb_string_reader *const reader, + uchar *const field_ptr, Rdb_string_reader *const reader, Rdb_string_reader *const unp_reader MY_ATTRIBUTE((__unused__))); - static int unpack_float( - Rdb_field_packing *const fpi, - Field *const field MY_ATTRIBUTE((__unused__)), uchar *const field_ptr, - Rdb_string_reader *const reader, - Rdb_string_reader *const unp_reader MY_ATTRIBUTE((__unused__))); + static int unpack_float(Rdb_field_packing *const fpi, uchar *const field_ptr, + Rdb_string_reader *const reader, + Rdb_string_reader *const unp_reader + MY_ATTRIBUTE((__unused__))); - static int unpack_binary_str(Rdb_field_packing *const fpi, Field *const field, - uchar *const to, Rdb_string_reader *const reader, + static int unpack_binary_str(Rdb_field_packing *const fpi, uchar *const to, + Rdb_string_reader *const reader, Rdb_string_reader *const unp_reader MY_ATTRIBUTE((__unused__))); static int unpack_binary_or_utf8_varchar( - Rdb_field_packing *const fpi, Field *const field, uchar *dst, - Rdb_string_reader *const reader, + Rdb_field_packing *const fpi, uchar *dst, Rdb_string_reader *const reader, Rdb_string_reader *const unp_reader MY_ATTRIBUTE((__unused__))); static int unpack_binary_or_utf8_varchar_space_pad( - Rdb_field_packing *const fpi, Field *const field, uchar *dst, - Rdb_string_reader *const reader, Rdb_string_reader *const unp_reader); + Rdb_field_packing *const fpi, uchar *dst, Rdb_string_reader *const reader, + Rdb_string_reader *const unp_reader); static int unpack_newdate( - Rdb_field_packing *const fpi, - Field *const field MY_ATTRIBUTE((__unused__)), uchar *const field_ptr, + Rdb_field_packing *const fpi, uchar *const field_ptr, Rdb_string_reader *const reader, Rdb_string_reader *const unp_reader MY_ATTRIBUTE((__unused__))); - static int unpack_utf8_str(Rdb_field_packing *const fpi, Field *const field, - uchar *dst, Rdb_string_reader *const reader, - Rdb_string_reader *const unp_reader - MY_ATTRIBUTE((__unused__))); + static int unpack_utf8_str( + Rdb_field_packing *const fpi, uchar *dst, Rdb_string_reader *const reader, + Rdb_string_reader *const unp_reader MY_ATTRIBUTE((__unused__))); - static int unpack_unknown_varchar(Rdb_field_packing *const fpi, - Field *const field, uchar *dst, + static int unpack_unknown_varchar(Rdb_field_packing *const fpi, uchar *dst, Rdb_string_reader *const reader, Rdb_string_reader *const unp_reader); static int unpack_simple_varchar_space_pad( - Rdb_field_packing *const fpi, Field *const field, uchar *dst, - Rdb_string_reader *const reader, Rdb_string_reader *const unp_reader); + Rdb_field_packing *const fpi, uchar *dst, Rdb_string_reader *const reader, + Rdb_string_reader *const unp_reader); - static int unpack_simple(Rdb_field_packing *const fpi, - Field *const field MY_ATTRIBUTE((__unused__)), - uchar *const dst, Rdb_string_reader *const reader, + static int unpack_simple(Rdb_field_packing *const fpi, uchar *const dst, + Rdb_string_reader *const reader, Rdb_string_reader *const unp_reader); - static int unpack_unknown(Rdb_field_packing *const fpi, Field *const field, - uchar *const dst, Rdb_string_reader *const reader, + static int unpack_unknown(Rdb_field_packing *const fpi, uchar *const dst, + Rdb_string_reader *const reader, Rdb_string_reader *const unp_reader); static int unpack_floating_point(uchar *const dst, @@ -745,16 +740,12 @@ class Rdb_key_def { Rdb_pack_field_context *pack_ctx MY_ATTRIBUTE((__unused__))); static int skip_max_length(const Rdb_field_packing *const fpi, - const Field *const field - MY_ATTRIBUTE((__unused__)), Rdb_string_reader *const reader); static int skip_variable_length(const Rdb_field_packing *const fpi, - const Field *const field, Rdb_string_reader *const reader); static int skip_variable_space_pad(const Rdb_field_packing *const fpi, - const Field *const field, Rdb_string_reader *const reader); inline bool use_legacy_varbinary_format() const { @@ -783,7 +774,7 @@ class Rdb_key_def { uchar m_index_number_storage_form[INDEX_NUMBER_SIZE]; - rocksdb::ColumnFamilyHandle *m_cf_handle; + std::shared_ptr m_cf_handle; static void pack_legacy_variable_format(const uchar *src, size_t src_len, uchar **dst); @@ -922,12 +913,23 @@ class Rdb_field_packing { int m_unpack_data_len; int m_unpack_data_offset; - bool m_maybe_null; /* TRUE <=> NULL-byte is stored */ + /* + Cached field information for faster access + */ + bool m_field_maybe_null; /* TRUE <=> NULL-byte is stored */ + bool m_field_unsigned_flag; + enum_field_types m_field_real_type; + uchar m_field_null_bit_mask; + uint m_field_pack_length; + uint m_field_null_offset; + my_ptrdiff_t m_field_offset; + const CHARSET_INFO *m_field_charset; /* Valid only for VARCHAR fields. */ - const CHARSET_INFO *m_varchar_charset; + uint m_varchar_length_bytes; + uint m_varchar_char_length; bool m_use_legacy_varbinary_format; // (Valid when Variable Length Space Padded Encoding is used): @@ -1035,13 +1037,20 @@ class Rdb_field_encoder { STORAGE_TYPE m_storage_type; uint m_null_offset; - uint16 m_field_index; - uchar m_null_mask; // 0 means the field cannot be null + /* + Cached field information + */ my_core::enum_field_types m_field_type; - uint m_pack_length_in_rec; + uchar m_field_null_mask; + uint16 m_field_index; + uint m_field_pack_length; + uint m_field_length_bytes; + uint m_field_length; + my_ptrdiff_t m_field_null_offset; + my_ptrdiff_t m_field_offset; bool maybe_null() const { return m_null_mask != 0; } @@ -1131,8 +1140,8 @@ class Rdb_tbl_def { /* Is this table read free repl enabled */ std::atomic_bool m_is_read_free_rpl_table{false}; - bool put_dict(Rdb_dict_manager *const dict, rocksdb::WriteBatch *const batch, - const rocksdb::Slice &key); + bool put_dict(Rdb_dict_manager *const dict, Rdb_cf_manager *const cf_manager, + rocksdb::WriteBatch *const batch, const rocksdb::Slice &key); const std::string &full_tablename() const { return m_dbname_tablename; } const std::string &base_dbname() const { return m_dbname; } @@ -1190,6 +1199,7 @@ interface Rdb_tables_scanner { class Rdb_ddl_manager { Rdb_dict_manager *m_dict = nullptr; + Rdb_cf_manager *m_cf_manager = nullptr; // Contains Rdb_tbl_def elements std::unordered_map m_ddl_map; @@ -1252,6 +1262,7 @@ class Rdb_ddl_manager { const std::unordered_set> &indexes); void remove_uncommitted_keydefs( const std::unordered_set> &indexes); + int find_in_uncommitted_keydef(const uint32_t &cf_id); private: /* Put the data into in-memory table (only) */ @@ -1362,6 +1373,10 @@ class Rdb_binlog_manager { value: version, {max auto_increment so far} max auto_increment is 8 bytes + 10. dropped cfs + key: Rdb_key_def::DROPPED_CF(0xa) + cf_id + value: version + Data dictionary operations are atomic inside RocksDB. For example, when creating a table with two indexes, it is necessary to call Put three times. They have to be atomic. Rdb_dict_manager has a wrapper function @@ -1407,7 +1422,8 @@ class Rdb_dict_manager { Rdb_dict_manager() = default; bool init(rocksdb::TransactionDB *const rdb_dict, - Rdb_cf_manager *const cf_manager); + Rdb_cf_manager *const cf_manager, + const my_bool enable_remove_orphaned_cf_flags); inline void cleanup() { mysql_mutex_destroy(&m_mutex); } @@ -1415,6 +1431,8 @@ class Rdb_dict_manager { inline void unlock() { RDB_MUTEX_UNLOCK_CHECK(m_mutex); } + inline void assert_lock_held() { mysql_mutex_assert_owner(&m_mutex); } + inline rocksdb::ColumnFamilyHandle *get_system_cf() const { return m_system_cfh; } @@ -1444,6 +1462,22 @@ class Rdb_dict_manager { const uint cf_flags) const; bool get_cf_flags(const uint cf_id, uint *const cf_flags) const; + void add_dropped_cf(rocksdb::WriteBatch *const batch, + const uint &cf_id) const; + void delete_dropped_cf(rocksdb::WriteBatch *const batch, + const uint &cf_id) const; + bool get_dropped_cf(const uint &cf_id) const; + void get_all_dropped_cfs(std::unordered_set *dropped_cf_ids) const; + + int add_missing_cf_flags(Rdb_cf_manager *const cf_manager) const; + + int remove_orphaned_dropped_cfs( + Rdb_cf_manager *const cf_manager, + const my_bool &enable_remove_orphaned_dropped_cfs) const; + + void delete_dropped_cf_and_flags(rocksdb::WriteBatch *const batch, + const uint &cf_id) const; + /* Functions for fast CREATE/DROP TABLE/INDEX */ void get_ongoing_index_operation( std::unordered_set *gl_index_ids, @@ -1468,6 +1502,8 @@ class Rdb_dict_manager { const std::unordered_set &gl_index_ids, Rdb_key_def::DATA_DICT_TYPE dd_type) const; void rollback_ongoing_index_creation() const; + void rollback_ongoing_index_creation( + const std::unordered_set &gl_index_ids) const; inline void get_ongoing_drop_indexes( std::unordered_set *gl_index_ids) const { @@ -1520,6 +1556,11 @@ class Rdb_dict_manager { bool overwrite = false) const; bool get_auto_incr_val(const GL_INDEX_ID &gl_index_id, ulonglong *new_val) const; + + private: + /* dropped cf flags */ + void delete_cf_flags(rocksdb::WriteBatch *const batch, + const uint &cf_id) const; }; struct Rdb_index_info { diff --git a/storage/rocksdb/rdb_i_s.cc b/storage/rocksdb/rdb_i_s.cc index a3d284dfa64..3a8a24ce8ad 100644 --- a/storage/rocksdb/rdb_i_s.cc +++ b/storage/rocksdb/rdb_i_s.cc @@ -27,6 +27,7 @@ #include /* RocksDB header files */ +#include "rocksdb/advanced_cache.h" #include "rocksdb/compaction_filter.h" #include "rocksdb/convenience.h" #include "rocksdb/filter_policy.h" @@ -36,6 +37,7 @@ #include "rocksdb/utilities/transaction_db.h" /* MyRocks header files */ +#include "./debug_sync.h" #include "./ha_rocksdb.h" #include "./ha_rocksdb_proto.h" #include "./rdb_cf_manager.h" @@ -102,7 +104,22 @@ static int rdb_i_s_cfstats_fill_table( "NUM_ENTRIES_IMM_MEM_TABLES"}, {rocksdb::DB::Properties::kEstimateTableReadersMem, "NON_BLOCK_CACHE_SST_MEM_USAGE"}, - {rocksdb::DB::Properties::kNumLiveVersions, "NUM_LIVE_VERSIONS"}}; + {rocksdb::DB::Properties::kNumLiveVersions, "NUM_LIVE_VERSIONS"}, + {rocksdb::DB::Properties::kNumImmutableMemTableFlushed, + "NUM_IMMUTABLE_MEM_TABLE_FLUSHED"}, + {rocksdb::DB::Properties::kNumRunningFlushes, "NUM_RUNNING_FLUSHES"}, + {rocksdb::DB::Properties::kNumRunningCompactions, + "NUM_RUNNING_COMPACTIONS"}, + {rocksdb::DB::Properties::kSizeAllMemTables, "SIZE_ALL_MEM_TABLES"}, + {rocksdb::DB::Properties::kNumDeletesActiveMemTable, + "NUM_DELETES_ACTIVE_MEM_TABLE"}, + {rocksdb::DB::Properties::kNumDeletesImmMemTables, + "NUM_DELETES_IMM_MEM_TABLES"}, + {rocksdb::DB::Properties::kEstimateNumKeys, "ESTIMATE_NUM_KEYS"}, + {rocksdb::DB::Properties::kEstimateLiveDataSize, + "ESTIMATE_LIVE_DATA_SIZE"}, + {rocksdb::DB::Properties::kEstimatePendingCompactionBytes, + "ESTIMATE_PENDING_COMPACTION_BYTES"}}; rocksdb::DB *const rdb = rdb_get_rocksdb_db(); @@ -114,13 +131,16 @@ static int rdb_i_s_cfstats_fill_table( for (const auto &cf_name : cf_manager.get_cf_names()) { DBUG_ASSERT(!cf_name.empty()); - rocksdb::ColumnFamilyHandle *cfh = cf_manager.get_cf(cf_name); - if (cfh == nullptr) { + std::shared_ptr cfh = + cf_manager.get_cf(cf_name); + if (!cfh) { continue; } + // It is safe if the CF is removed from cf_manager at + // this point. The CF handle object is valid and sufficient here. for (const auto &property : cf_properties) { - if (!rdb->GetIntProperty(cfh, property.first, &val)) { + if (!rdb->GetIntProperty(cfh.get(), property.first, &val)) { continue; } @@ -482,8 +502,6 @@ static int rdb_i_s_cfoptions_fill_table( std::to_string(opts.level0_slowdown_writes_trigger)}, {"LEVEL0_STOP_WRITES_TRIGGER", std::to_string(opts.level0_stop_writes_trigger)}, - {"MAX_MEM_COMPACTION_LEVEL", - std::to_string(opts.max_mem_compaction_level)}, {"TARGET_FILE_SIZE_BASE", std::to_string(opts.target_file_size_base)}, {"TARGET_FILE_SIZE_MULTIPLIER", std::to_string(opts.target_file_size_multiplier)}, @@ -493,15 +511,9 @@ static int rdb_i_s_cfoptions_fill_table( opts.level_compaction_dynamic_level_bytes ? "ON" : "OFF"}, {"MAX_BYTES_FOR_LEVEL_MULTIPLIER", std::to_string(opts.max_bytes_for_level_multiplier)}, - {"SOFT_RATE_LIMIT", std::to_string(opts.soft_rate_limit)}, - {"HARD_RATE_LIMIT", std::to_string(opts.hard_rate_limit)}, - {"RATE_LIMIT_DELAY_MAX_MILLISECONDS", - std::to_string(opts.rate_limit_delay_max_milliseconds)}, {"ARENA_BLOCK_SIZE", std::to_string(opts.arena_block_size)}, {"DISABLE_AUTO_COMPACTIONS", opts.disable_auto_compactions ? "ON" : "OFF"}, - {"PURGE_REDUNDANT_KVS_WHILE_FLUSH", - opts.purge_redundant_kvs_while_flush ? "ON" : "OFF"}, {"MAX_SEQUENTIAL_SKIP_IN_ITERATIONS", std::to_string(opts.max_sequential_skip_in_iterations)}, {"MEMTABLE_FACTORY", opts.memtable_factory == nullptr @@ -576,9 +588,10 @@ static int rdb_i_s_cfoptions_fill_table( // get PREFIX_EXTRACTOR option cf_option_types.push_back( - {"PREFIX_EXTRACTOR", opts.prefix_extractor == nullptr - ? "NULL" - : std::string(opts.prefix_extractor->Name())}); + {"PREFIX_EXTRACTOR", + opts.prefix_extractor == nullptr + ? "NULL" + : std::string(opts.prefix_extractor->GetId())}); // get COMPACTION_STYLE option switch (opts.compaction_style) { @@ -637,7 +650,7 @@ static int rdb_i_s_cfoptions_fill_table( // get table related options std::vector table_options = - split_into_vector(opts.table_factory->GetPrintableTableOptions(), '\n'); + split_into_vector(opts.table_factory->GetPrintableOptions(), '\n'); for (auto option : table_options) { option.erase(std::remove(option.begin(), option.end(), ' '), @@ -759,7 +772,7 @@ static int rdb_i_s_global_info_fill_table( } /* max index info */ - const Rdb_dict_manager *const dict_manager = rdb_get_dict_manager(); + Rdb_dict_manager *const dict_manager = rdb_get_dict_manager(); DBUG_ASSERT(dict_manager != nullptr); uint32_t max_index_id; @@ -780,16 +793,22 @@ static int rdb_i_s_global_info_fill_table( for (const auto &cf_handle : cf_manager.get_all_cf()) { DBUG_ASSERT(cf_handle != nullptr); + DBUG_EXECUTE_IF("information_schema_global_info", { + if (cf_handle->GetName() == "cf_primary_key") { + const char act[] = + "now signal ready_to_mark_cf_dropped_in_global_info " + "wait_for mark_cf_dropped_done_in_global_info"; + DBUG_ASSERT(!debug_sync_set_action(thd, STRING_WITH_LEN(act))); + } + }); + uint flags; if (!dict_manager->get_cf_flags(cf_handle->GetID(), &flags)) { - // NO_LINT_DEBUG - sql_print_error( - "RocksDB: Failed to get column family flags " - "from CF with id = %u. MyRocks data dictionary may " - "be corrupted.", - cf_handle->GetID()); - abort(); + // If cf flags cannot be retrieved, set flags to 0. It can happen + // if the CF is dropped. flags is only used to print information + // here and so it doesn't affect functional correctness. + flags = 0; } snprintf(cf_id_buf, INT_BUF_LEN, "%u", cf_handle->GetID()); @@ -846,15 +865,18 @@ static int rdb_i_s_compact_stats_fill_table( Rdb_cf_manager &cf_manager = rdb_get_cf_manager(); for (auto cf_name : cf_manager.get_cf_names()) { - rocksdb::ColumnFamilyHandle *cfh = cf_manager.get_cf(cf_name); + std::shared_ptr cfh = + cf_manager.get_cf(cf_name); - if (cfh == nullptr) { + if (!cfh) { continue; } + // It is safe if the CF is removed from cf_manager at + // this point. The CF handle object is valid and sufficient here. std::map props; bool bool_ret MY_ATTRIBUTE((__unused__)); - bool_ret = rdb->GetMapProperty(cfh, "rocksdb.cfstats", &props); + bool_ret = rdb->GetMapProperty(cfh.get(), "rocksdb.cfstats", &props); DBUG_ASSERT(bool_ret); const std::string prop_name_prefix = "compaction."; @@ -890,6 +912,208 @@ static int rdb_i_s_compact_stats_fill_table( DBUG_RETURN(ret); } +/* + Support for INFORMATION_SCHEMA.ROCKSDB_LIVE_FILES_METADATA dynamic table + */ +namespace RDB_LIVE_FILES_METADATA_FIELD { +enum { + CF_NAME = 0, + LEVEL, + NAME, + DB_PATH, + FILE_NUMBER, + FILE_TYPE, + SIZE, + RELATIVE_FILENAME, + DIRECTORY, + TEMPERATURE, + FILE_CHECKSUM, + FILE_CHECKSUM_FUNC_NAME, + SMALLEST_SEQNO, + LARGEST_SEQNO, + SMALLEST_KEY, + LARGEST_KEY, + NUM_READS_SAMPLED, + BEING_COMPACTED, + NUM_ENTRIES, + NUM_DELETIONS, + OLDEST_BLOB_FILE_NUMBER, + OLDEST_ANCESTER_TIME, + FILE_CREATION_TIME, +}; +} // namespace RDB_LIVE_FILES_METADATA_FIELD + +static ST_FIELD_INFO rdb_i_s_live_files_metadata_fields_info[] = { + Column("CF_NAME", Varchar(NAME_LEN + 1), NOT_NULL), + Column("LEVEL", Varchar(FN_REFLEN + 1), NOT_NULL), + Column("NAME", Varchar(FN_REFLEN + 1), NOT_NULL), + Column("DB_PATH", Varchar(FN_REFLEN + 1), NOT_NULL), + Column("FILE_NUMBER", SLonglong(), NOT_NULL), + Column("FILE_TYPE", Varchar(NAME_LEN + 1), NOT_NULL), + Column("SIZE", SLonglong(), NOT_NULL), + Column("RELATIVE_FILENAME", Varchar(NAME_LEN + 1), NOT_NULL), + Column("DIRECTORY", Varchar(FN_REFLEN + 1), NOT_NULL), + Column("TEMPERATURE", Varchar(NAME_LEN + 1), NOT_NULL), + Column("FILE_CHECKSUM", Varchar(FN_REFLEN + 1), NOT_NULL), + Column("FILE_CHECKSUM_FUNC_NAME", Varchar(NAME_LEN + 1), NOT_NULL), + Column("SMALLEST_SEQNO", SLonglong(), NOT_NULL), + Column("LARGEST_SEQNO", SLonglong(), NOT_NULL), + Column("SMALLEST_KEY", Varchar(FN_REFLEN + 1), NOT_NULL), + Column("LARGEST_KEY", Varchar(FN_REFLEN + 1), NOT_NULL), + Column("NUM_READS_SAMPLED", SLonglong(), NOT_NULL), + Column("BEING_COMPACTED", STiny(1), NOT_NULL), + Column("NUM_ENTRIES", SLonglong(), NOT_NULL), + Column("NUM_DELETIONS", SLonglong(), NOT_NULL), + Column("OLDEST_BLOB_FILE_NUMBER", SLonglong(), NOT_NULL), + Column("OLDEST_ANCESTER_TIME", SLonglong(), NOT_NULL), + Column("FILE_CREATION_TIME", SLonglong(), NOT_NULL), + CEnd()}; + +namespace { + +using rocksdb::FileType; + +std::string GetFileTypeString(FileType file_type) { + switch (file_type) { + case FileType::kWalFile: + return "WalFile"; + case FileType::kDBLockFile: + return "DBLockFile"; + case FileType::kTableFile: + return "TableFile"; + case FileType::kDescriptorFile: + return "DescriptorFile"; + case FileType::kCurrentFile: + return "CurrentFile"; + case FileType::kTempFile: + return "TempFile"; + case FileType::kInfoLogFile: + return "InfoLogFile"; + case FileType::kMetaDatabase: + return "MetaDatabase"; + case FileType::kIdentityFile: + return "IdentityFile"; + case FileType::kOptionsFile: + return "OptionsFile"; + case FileType::kBlobFile: + return "BlobFile"; + default: + return std::to_string(static_cast(file_type)); + } +} + +using rocksdb::Temperature; + +std::string GetTemperatureString(Temperature temperature) { + switch (temperature) { + case Temperature::kUnknown: + return "Unknown"; + case Temperature::kHot: + return "Hot"; + case Temperature::kWarm: + return "Warm"; + case Temperature::kCold: + return "Cold"; + default: + return std::to_string(static_cast(temperature)); + } +} + +} // anonymous namespace + +static int rdb_i_s_live_files_metadata_fill_table( + my_core::THD *thd, my_core::TABLE_LIST *tables, + my_core::Item *cond MY_ATTRIBUTE((__unused__))) { + DBUG_ASSERT(thd != nullptr); + DBUG_ASSERT(tables != nullptr); + + DBUG_ENTER_FUNC(); + + int ret = 0; + rocksdb::DB *rdb = rdb_get_rocksdb_db(); + + if (!rdb) { + DBUG_RETURN(ret); + } + + std::vector metadata; + rdb->GetLiveFilesMetaData(&metadata); + + for (const auto &file : metadata) { + Field **field = tables->table->field; + DBUG_ASSERT(field != nullptr); + + field[RDB_LIVE_FILES_METADATA_FIELD::CF_NAME]->store( + file.column_family_name.c_str(), file.column_family_name.size(), + system_charset_info); + field[RDB_LIVE_FILES_METADATA_FIELD::LEVEL]->store(file.level, true); + field[RDB_LIVE_FILES_METADATA_FIELD::NAME]->store( + file.name.c_str(),file.name.size(), system_charset_info); + field[RDB_LIVE_FILES_METADATA_FIELD::DB_PATH]->store( + file.db_path.c_str(), file.db_path.size(), system_charset_info); + field[RDB_LIVE_FILES_METADATA_FIELD::FILE_NUMBER]->store(file.file_number, + true); + std::string file_type = GetFileTypeString(file.file_type); + field[RDB_LIVE_FILES_METADATA_FIELD::FILE_TYPE]->store( + file_type.c_str(), file_type.size(), system_charset_info); + field[RDB_LIVE_FILES_METADATA_FIELD::SIZE]->store(file.size, true); + field[RDB_LIVE_FILES_METADATA_FIELD::RELATIVE_FILENAME]->store( + file.relative_filename.c_str(), file.relative_filename.size(), + system_charset_info); + field[RDB_LIVE_FILES_METADATA_FIELD::DIRECTORY]->store( + file.directory.c_str(), file.directory.size(), system_charset_info); + std::string temperature = GetTemperatureString(file.temperature); + field[RDB_LIVE_FILES_METADATA_FIELD::TEMPERATURE]->store( + temperature.c_str(), temperature.size(), system_charset_info); + rocksdb::Slice file_checksum_slice(file.file_checksum); + auto file_checksum = "0x" + file_checksum_slice.ToString(true); + field[RDB_LIVE_FILES_METADATA_FIELD::FILE_CHECKSUM]->store( + file_checksum.c_str(), file_checksum.size(), system_charset_info); + field[RDB_LIVE_FILES_METADATA_FIELD::FILE_CHECKSUM_FUNC_NAME]->store( + file.file_checksum_func_name.c_str(), + file.file_checksum_func_name.size(), system_charset_info); + field[RDB_LIVE_FILES_METADATA_FIELD::SMALLEST_SEQNO]->store( + file.smallest_seqno, true); + field[RDB_LIVE_FILES_METADATA_FIELD::LARGEST_SEQNO]->store( + file.largest_seqno, true); + rocksdb::Slice smallest_key_slice(file.smallestkey); + auto smallest_key = "0x" + smallest_key_slice.ToString(true); + field[RDB_LIVE_FILES_METADATA_FIELD::SMALLEST_KEY]->store( + smallest_key.c_str(), smallest_key.size(), system_charset_info); + rocksdb::Slice largest_key_slice(file.largestkey); + auto largest_key = "0x" + largest_key_slice.ToString(true); + field[RDB_LIVE_FILES_METADATA_FIELD::LARGEST_KEY]->store( + largest_key.c_str(), largest_key.size(), system_charset_info); + field[RDB_LIVE_FILES_METADATA_FIELD::NUM_READS_SAMPLED]->store( + file.num_reads_sampled, true); + field[RDB_LIVE_FILES_METADATA_FIELD::BEING_COMPACTED]->store( + file.being_compacted); + field[RDB_LIVE_FILES_METADATA_FIELD::NUM_ENTRIES]->store(file.num_entries, + true); + field[RDB_LIVE_FILES_METADATA_FIELD::NUM_DELETIONS]->store( + file.num_deletions, true); + field[RDB_LIVE_FILES_METADATA_FIELD::OLDEST_BLOB_FILE_NUMBER]->store( + file.oldest_blob_file_number, true); + field[RDB_LIVE_FILES_METADATA_FIELD::OLDEST_ANCESTER_TIME]->store( + file.oldest_ancester_time, true); + field[RDB_LIVE_FILES_METADATA_FIELD::FILE_CREATION_TIME]->store( + file.file_creation_time, true); + + ret |= static_cast( + my_core::schema_table_store_record(thd, tables->table)); + + if (ret != 0) { + DBUG_RETURN(ret); + } + } + + if (!rdb) { + DBUG_RETURN(ret); + } + + DBUG_RETURN(ret); +} + static ST_FIELD_INFO rdb_i_s_compact_stats_fields_info[] = { Column("CF_NAME", Varchar(NAME_LEN + 1), NOT_NULL), Column("LEVEL", Varchar(FN_REFLEN + 1), NOT_NULL), @@ -1091,6 +1315,20 @@ static int rdb_i_s_compact_stats_init(void *p) { DBUG_RETURN(0); } +static int rdb_i_s_live_files_metadata_init(void *p) { + my_core::ST_SCHEMA_TABLE *schema; + + DBUG_ENTER_FUNC(); + DBUG_ASSERT(p != nullptr); + + schema = reinterpret_cast(p); + + schema->fields_info = rdb_i_s_live_files_metadata_fields_info; + schema->fill_table = rdb_i_s_live_files_metadata_fill_table; + + DBUG_RETURN(0); +} + /* Given a path to a file return just the filename portion. */ static std::string rdb_filename_without_path(const std::string &path) { /* Find last slash in path */ @@ -1176,12 +1414,14 @@ static int rdb_i_s_sst_props_fill_table( /* Grab the the properties of all the tables in the column family */ rocksdb::TablePropertiesCollection table_props_collection; const rocksdb::Status s = - rdb->GetPropertiesOfAllTables(cf_handle, &table_props_collection); + rdb->GetPropertiesOfAllTables(cf_handle.get(), &table_props_collection); if (!s.ok()) { continue; } + // It is safe if the CF is removed from cf_manager at + // this point. The CF handle object is valid and sufficient here. /* Iterate over all the items in the collection, each of which contains a * name and the actual properties */ for (const auto &props : table_props_collection) { @@ -1332,8 +1572,11 @@ static int rdb_i_s_index_file_map_fill_table( for (const auto &cf_handle : cf_manager.get_all_cf()) { /* Grab the the properties of all the tables in the column family */ rocksdb::TablePropertiesCollection table_props_collection; + + // It is safe if the CF is removed from cf_manager at + // this point. The CF handle object is valid and sufficient here. const rocksdb::Status s = - rdb->GetPropertiesOfAllTables(cf_handle, &table_props_collection); + rdb->GetPropertiesOfAllTables(cf_handle.get(), &table_props_collection); if (!s.ok()) { continue; @@ -1868,6 +2111,22 @@ struct st_maria_plugin rdb_i_s_compact_stats = { MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL }; +struct st_maria_plugin rdb_i_s_live_files_metadata = { + MYSQL_INFORMATION_SCHEMA_PLUGIN, + &rdb_i_s_info, + "ROCKSDB_LIVE_FILES_METADATA", + "Facebook", + "RocksDB live files metadata", + PLUGIN_LICENSE_GPL, + rdb_i_s_live_files_metadata_init, + rdb_i_s_deinit, + 0x0001, /* version number (0.1) */ + nullptr, /* status variables */ + nullptr, /* system variables */ + nullptr, /* config options */ + MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL +}; + struct st_maria_plugin rdb_i_s_ddl = { MYSQL_INFORMATION_SCHEMA_PLUGIN, &rdb_i_s_info, diff --git a/storage/rocksdb/rdb_i_s.h b/storage/rocksdb/rdb_i_s.h index 6001742d984..c5e887248cb 100644 --- a/storage/rocksdb/rdb_i_s.h +++ b/storage/rocksdb/rdb_i_s.h @@ -34,4 +34,5 @@ extern struct st_maria_plugin rdb_i_s_index_file_map; extern struct st_maria_plugin rdb_i_s_lock_info; extern struct st_maria_plugin rdb_i_s_trx_info; extern struct st_maria_plugin rdb_i_s_deadlock_info; +extern struct st_maria_plugin rdb_i_s_live_files_metadata; } // namespace myrocks diff --git a/storage/rocksdb/rdb_psi.cc b/storage/rocksdb/rdb_psi.cc index 77003b1bb48..ad7410147d3 100644 --- a/storage/rocksdb/rdb_psi.cc +++ b/storage/rocksdb/rdb_psi.cc @@ -47,7 +47,8 @@ my_core::PSI_mutex_key rdb_psi_open_tbls_mutex_key, rdb_signal_bg_psi_mutex_key, rdb_signal_drop_idx_psi_mutex_key, rdb_signal_mc_psi_mutex_key, rdb_collation_data_mutex_key, rdb_mem_cmp_space_mutex_key, key_mutex_tx_list, rdb_sysvars_psi_mutex_key, rdb_cfm_mutex_key, - rdb_sst_commit_key, rdb_block_cache_resize_mutex_key; + rdb_sst_commit_key, rdb_block_cache_resize_mutex_key, + rdb_bottom_pri_background_compactions_resize_mutex_key; my_core::PSI_mutex_info all_rocksdb_mutexes[] = { {&rdb_psi_open_tbls_mutex_key, "open tables", PSI_FLAG_GLOBAL}, @@ -63,6 +64,8 @@ my_core::PSI_mutex_info all_rocksdb_mutexes[] = { {&rdb_sst_commit_key, "sst commit", PSI_FLAG_GLOBAL}, {&rdb_block_cache_resize_mutex_key, "resizing block cache", PSI_FLAG_GLOBAL}, + {&rdb_bottom_pri_background_compactions_resize_mutex_key, + "resizing bottom pri compaction threads", PSI_FLAG_GLOBAL} }; my_core::PSI_rwlock_key key_rwlock_collation_exception_list, diff --git a/storage/rocksdb/rdb_psi.h b/storage/rocksdb/rdb_psi.h index 2703837a156..3b2ad915817 100644 --- a/storage/rocksdb/rdb_psi.h +++ b/storage/rocksdb/rdb_psi.h @@ -42,7 +42,8 @@ extern my_core::PSI_mutex_key rdb_psi_open_tbls_mutex_key, rdb_signal_bg_psi_mutex_key, rdb_signal_drop_idx_psi_mutex_key, rdb_signal_mc_psi_mutex_key, rdb_collation_data_mutex_key, rdb_mem_cmp_space_mutex_key, key_mutex_tx_list, rdb_sysvars_psi_mutex_key, - rdb_cfm_mutex_key, rdb_sst_commit_key, rdb_block_cache_resize_mutex_key; + rdb_cfm_mutex_key, rdb_sst_commit_key, rdb_block_cache_resize_mutex_key, + rdb_bottom_pri_background_compactions_resize_mutex_key; extern my_core::PSI_rwlock_key key_rwlock_collation_exception_list, key_rwlock_read_free_rpl_tables, key_rwlock_skip_unique_check_tables; diff --git a/storage/rocksdb/rdb_threads.h b/storage/rocksdb/rdb_threads.h index d23419df3b9..b334bb53a51 100644 --- a/storage/rocksdb/rdb_threads.h +++ b/storage/rocksdb/rdb_threads.h @@ -160,28 +160,43 @@ class Rdb_background_thread : public Rdb_thread { }; class Rdb_manual_compaction_thread : public Rdb_thread { - private: + public: struct Manual_compaction_request { int mc_id; - enum mc_state { INITED = 0, RUNNING } state; - rocksdb::ColumnFamilyHandle *cf; + enum mc_state { + PENDING = 0, + RUNNING = 1, + SUCCESS = 2, + FAILURE = 3, + CANCEL = 4, + } state; + std::shared_ptr cf; rocksdb::Slice *start; rocksdb::Slice *limit; - int concurrency = 0; + rocksdb::CompactRangeOptions option; + std::shared_ptr> canceled; + // state is modified by user threads + bool client_done; }; + virtual void run() override; + int request_manual_compaction( + std::shared_ptr cf, rocksdb::Slice *start, + rocksdb::Slice *limit, const uint manual_compaction_threads, + const rocksdb::BottommostLevelCompaction bottommost_level_compaction); + Manual_compaction_request::mc_state manual_compaction_state(const int mc_id); + void set_state(Manual_compaction_request &mcr, + const Manual_compaction_request::mc_state state); + bool set_client_done(const int mc_id); + void clear_all_manual_compaction_requests(); + void cancel_all_pending_manual_compaction_requests(); + bool cancel_manual_compaction_request(const int mc_id, + const int timeout_100ms); + + private: int m_latest_mc_id; mysql_mutex_t m_mc_mutex; std::map m_requests; - - public: - virtual void run() override; - int request_manual_compaction(rocksdb::ColumnFamilyHandle *cf, - rocksdb::Slice *start, rocksdb::Slice *limit, - int concurrency = 0); - bool is_manual_compaction_finished(int mc_id); - void clear_manual_compaction_request(int mc_id, bool init_only = false); - void clear_all_manual_compaction_requests(); }; /* diff --git a/storage/rocksdb/rocksdb b/storage/rocksdb/rocksdb index bba5e7bc210..6a436150417 160000 --- a/storage/rocksdb/rocksdb +++ b/storage/rocksdb/rocksdb @@ -1 +1 @@ -Subproject commit bba5e7bc21093d7cfa765e1280a7c4fdcd284288 +Subproject commit 6a436150417120a3f9732d65a2a5c2b8d19b60fc diff --git a/storage/rocksdb/tools/mysql_ldb.cc b/storage/rocksdb/tools/mysql_ldb.cc index 454b7a63c73..c898a356a5f 100644 --- a/storage/rocksdb/tools/mysql_ldb.cc +++ b/storage/rocksdb/tools/mysql_ldb.cc @@ -6,14 +6,39 @@ #include #include "../rdb_comparator.h" #include "rocksdb/ldb_tool.h" +#include "rocksdb/utilities/object_registry.h" int main(int argc, char **argv) { - MY_INIT(argv[0]); - rocksdb::Options db_options; - myrocks::Rdb_pk_comparator pk_comparator; - db_options.comparator = &pk_comparator; + // Register the comparators so they can be loaded from OPTIONS file when + // `--try_load_options` is provided. +#if ROCKSDB_MAJOR > 6 || (ROCKSDB_MAJOR == 6 && ROCKSDB_MINOR >= 29) + rocksdb::ObjectLibrary::Default()->AddFactory( +#else + rocksdb::ObjectLibrary::Default()->Register( +#endif + myrocks::Rdb_pk_comparator().Name(), + rocksdb::FactoryFunc( + [](const std::string & /* uri */, + std::unique_ptr * /* res_guard */, + std::string * /* err_msg */) { + static myrocks::Rdb_pk_comparator cmp; + return &cmp; + })); +#if ROCKSDB_MAJOR > 6 || (ROCKSDB_MAJOR == 6 && ROCKSDB_MINOR >= 29) + rocksdb::ObjectLibrary::Default()->AddFactory( +#else + rocksdb::ObjectLibrary::Default()->Register( +#endif + myrocks::Rdb_rev_comparator().Name(), + rocksdb::FactoryFunc( + [](const std::string & /* uri */, + std::unique_ptr * /* res_guard */, + std::string * /* err_msg */) { + static myrocks::Rdb_rev_comparator cmp; + return &cmp; + })); rocksdb::LDBTool tool; - tool.Run(argc, argv, db_options); + tool.Run(argc, argv); return 0; } diff --git a/storage/rocksdb/tools/mysql_sst_dump.cc b/storage/rocksdb/tools/mysql_sst_dump.cc new file mode 100644 index 00000000000..e6875150964 --- /dev/null +++ b/storage/rocksdb/tools/mysql_sst_dump.cc @@ -0,0 +1,44 @@ +// Copyright (c) 2013, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. +// +#include +#include "../rdb_comparator.h" +#include "rocksdb/sst_dump_tool.h" +#include "rocksdb/utilities/object_registry.h" + +int main(int argc, char **argv) { + // Register the comparators so they can be loaded from OPTIONS file when + // `--try_load_options` is provided. +#if ROCKSDB_MAJOR > 6 || (ROCKSDB_MAJOR == 6 && ROCKSDB_MINOR >= 29) + rocksdb::ObjectLibrary::Default()->AddFactory( +#else + rocksdb::ObjectLibrary::Default()->Register( +#endif + myrocks::Rdb_pk_comparator().Name(), + rocksdb::FactoryFunc( + [](const std::string & /* uri */, + std::unique_ptr * /* res_guard */, + std::string * /* err_msg */) { + static myrocks::Rdb_pk_comparator cmp; + return &cmp; + })); +#if ROCKSDB_MAJOR > 6 || (ROCKSDB_MAJOR == 6 && ROCKSDB_MINOR >= 29) + rocksdb::ObjectLibrary::Default()->AddFactory( +#else + rocksdb::ObjectLibrary::Default()->Register( +#endif + myrocks::Rdb_rev_comparator().Name(), + rocksdb::FactoryFunc( + [](const std::string & /* uri */, + std::unique_ptr * /* res_guard */, + std::string * /* err_msg */) { + static myrocks::Rdb_rev_comparator cmp; + return &cmp; + })); + + rocksdb::SSTDumpTool tool; + tool.Run(argc, argv); + return 0; +}