diff --git a/db/db_impl/db_impl_open.cc b/db/db_impl/db_impl_open.cc index 48bfef0111..e3a8cf371b 100644 --- a/db/db_impl/db_impl_open.cc +++ b/db/db_impl/db_impl_open.cc @@ -1585,6 +1585,30 @@ Status DBImpl::InsertLogRecordToMemtable(WriteBatch* batch_to_use, &trim_history_scheduler_, true, wal_number, this, false /* concurrent_memtable_writes */, next_sequence, has_valid_writes, seq_per_batch_, batch_per_txn_); + + // Check WriteBufferManager global limit during recovery. + // When multiple RocksDB instances share a WriteBufferManager, a recovering + // instance could exceed the global memory limit. Schedule flushes when needed + // to prevent OOM during WAL recovery. + // + // TODO: Currently we schedule all CFs with non-empty memtables for flush + // (similar to the atomic_flush=false path in the normal write flow). This + // may produce more, smaller L0 files in some CFs. A future improvement + // could flush only the oldest CF or pick CFs more selectively to reduce + // unnecessary L0 file creation. + if (status.ok() && *has_valid_writes && + immutable_db_options_.enforce_write_buffer_manager_during_recovery && + write_buffer_manager_ != nullptr && + write_buffer_manager_->ShouldFlush()) { + for (auto cfd : *versions_->GetColumnFamilySet()) { + if (cfd->mem() != nullptr && cfd->mem()->GetFirstSequenceNumber() != 0 && + !cfd->mem()->HasFlushScheduled()) { + cfd->mem()->MarkFlushScheduled(); + flush_scheduler_.ScheduleWork(cfd); + } + } + } + return status; } diff --git a/db/db_write_buffer_manager_test.cc b/db/db_write_buffer_manager_test.cc index 2eff1d397f..f053eda0ab 100644 --- a/db/db_write_buffer_manager_test.cc +++ b/db/db_write_buffer_manager_test.cc @@ -913,6 +913,228 @@ TEST_F(DBWriteBufferManagerTest, RuntimeChangeableAllowStall) { sleeping_task->WakeUp(); } +// Test that enforce_write_buffer_manager_during_recovery option controls +// whether WriteBufferManager limits are respected during WAL recovery. +// When enabled, flushes are triggered to keep memory bounded. +// When disabled (default), memory can grow beyond the configured limit. +TEST_F(DBWriteBufferManagerTest, + WriteBufferManagerLimitDuringWALRecoverySingleDB) { + const size_t kWbmLimit = 1 * 1024 * 1024; // 1 MB + const size_t kWbmLimitForWrites = 100 * 1024 * 1024; // 100 MB (no flush) + + Options options = CurrentOptions(); + options.arena_block_size = 4096; + options.write_buffer_size = 10 * 1024 * 1024; // 10MB per CF, never hit + options.max_write_buffer_number = 10; // Allow many memtables + options.disable_auto_compactions = true; + + // Use avoid_flush_during_recovery = true to prevent any flushes triggered + // during the recovery + options.avoid_flush_during_recovery = true; + + const int kNumKeys = 50; + const int kValueSize = 50 * 1024; // 50 KB each, total ~2.5 MB > 1MB limit + + // ========== Part 1: Test with enforcement DISABLED (default behavior) ===== + // WBM limits are not enforced during recovery + options.enforce_write_buffer_manager_during_recovery = false; + + // Use large WBM limit during writes to avoid triggering flushes + options.write_buffer_manager = + std::make_shared(kWbmLimitForWrites, nullptr, true); + DestroyAndReopen(options); + + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(Put(Key(i), DummyString(kValueSize))); + } + + // Check to make sure there's no L0 file + ASSERT_EQ(0, TotalTableFiles()); + Close(); + + // Use smaller WBM limit for recovery + options.write_buffer_manager = + std::make_shared(kWbmLimit, nullptr, true); + + // Recovery without enforcement - memory should exceed the limit + Reopen(options); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + size_t memory_without_enforcement = + options.write_buffer_manager->mutable_memtable_memory_usage(); + + ASSERT_GT(memory_without_enforcement, kWbmLimit) + << "Without enforcement, memory (" << memory_without_enforcement + << ") should exceed the WBM limit (" << kWbmLimit << ")"; + + // Still no L0 file since avoid_flush_during_recovery is true + ASSERT_EQ(0, TotalTableFiles()); + + // ========== Part 2: Test with enforcement ENABLED ========================== + options.enforce_write_buffer_manager_during_recovery = true; + + // Use large WBM limit during writes to avoid triggering flushes + options.write_buffer_manager = + std::make_shared(kWbmLimitForWrites, nullptr, true); + DestroyAndReopen(options); + + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(Put(Key(i), DummyString(kValueSize))); + } + // Check to make sure there's no L0 file + ASSERT_EQ(0, TotalTableFiles()); + Close(); + + // Use smaller WBM limit for recovery + options.write_buffer_manager = + std::make_shared(kWbmLimit, nullptr, true); + + // Recovery with enforcement - memory should be bounded + Reopen(options); + + // Wait for flush to finish + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + + // WBM's ShouldFlush() compares active memtable mem usage against + // mutable_limit_ which is 7/8 of buffer_size. + size_t expected_num_l0_files = + memory_without_enforcement / (kWbmLimit * 7 / 8) + 1; + ASSERT_EQ(expected_num_l0_files, TotalTableFiles()); + + size_t memory_with_enforcement = + options.write_buffer_manager->mutable_memtable_memory_usage(); + + ASSERT_LT(memory_with_enforcement, kWbmLimit) + << "With enforcement, memory (" << memory_with_enforcement + << ") should be less than the limit " << kWbmLimit << ")"; +} + +TEST_F(DBWriteBufferManagerTest, + WriteBufferManagerLimitDuringWALRecoveryMultipleDBs) { + // Two DBs with 4MB WBM limit. + // First DB writes 2.5MB and closes, no flush (mem usage goes back to 0) + // Second DB writes 2.5MB then first DB reopens. + const size_t kWbmLimitForTwoDbs = 4 * 1024 * 1024; + + Options options = CurrentOptions(); + options.arena_block_size = 2048; + options.write_buffer_size = 10 * 1024 * 1024; // 10MB per CF, never hit + options.max_write_buffer_number = 10; // Allow many memtables + options.disable_auto_compactions = true; + + // Use avoid_flush_during_recovery = true to prevent any flushes triggered + // during the recovery + options.avoid_flush_during_recovery = true; + + const int kNumKeys = 50; + const int kValueSize = 50 * 1024; + + // ========== Part 1: Test with enforcement DISABLED (default behavior) ===== + // WBM limits are not enforced during recovery + options.enforce_write_buffer_manager_during_recovery = false; + + options.write_buffer_manager = + std::make_shared(kWbmLimitForTwoDbs, nullptr, true); + DestroyAndReopen(options); + + // Use of 2.5MB shouldn't trigger flush + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(Put(Key(i), DummyString(kValueSize))); + } + ASSERT_EQ(0, TotalTableFiles()); + + size_t mem_usage_first_db_only = + options.write_buffer_manager->mutable_memtable_memory_usage(); + + ASSERT_LT(mem_usage_first_db_only, kWbmLimitForTwoDbs) + << "Memory (" << mem_usage_first_db_only + << ") should be less than the limit " << kWbmLimitForTwoDbs << ")"; + + Close(); + ASSERT_EQ(0, options.write_buffer_manager->mutable_memtable_memory_usage()); + + // Open a second DB sharing the same WBM, write data to consume memory + std::string second_dbname = test::PerThreadDBPath("db_shared_wbm_recovery"); + std::unique_ptr second_db; + ASSERT_OK(DestroyDB(second_dbname, options)); + ASSERT_OK(DB::Open(options, second_dbname, &second_db)); + + WriteOptions wo; + for (int i = 0; i < kNumKeys; i++) { + ASSERT_OK(second_db->Put(wo, Key(i), DummyString(kValueSize))); + } + + // First DB reopens without enforcement + Reopen(options); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + + // No flush + ASSERT_EQ(0, TotalTableFiles()); + + size_t memory_usage_for_both = + options.write_buffer_manager->mutable_memtable_memory_usage(); + ASSERT_GT(memory_usage_for_both, kWbmLimitForTwoDbs) + << "Without enforcement + shared WBM, memory (" << memory_usage_for_both + << ") should be greater than the limit (" << kWbmLimitForTwoDbs << ")"; + + // Close the first DB + Close(); + + // ========== Part 2: Test with enforcement ENABLED ===== + // WBM limits enforced during recovery + options.enforce_write_buffer_manager_during_recovery = true; + + // Reopen the first DB with enforcement option enabled. + Reopen(options); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + + // With enforcement enabled, there were flushes + ASSERT_GT(TotalTableFiles(), 0); + + memory_usage_for_both = + options.write_buffer_manager->mutable_memtable_memory_usage(); + ASSERT_LT(memory_usage_for_both, kWbmLimitForTwoDbs) + << "With enforcement + shared WBM, memory (" << memory_usage_for_both + << ") should be less than the limit (" << kWbmLimitForTwoDbs << ")"; + + Close(); + + // Clean up second DB + ASSERT_OK(second_db->Close()); + ASSERT_OK(DestroyDB(second_dbname, options)); + second_db.reset(); +} + +// Regression test: a WriteBatch that exceeds both per-CF memtable limit and +// WBM global limit during WAL recovery should not double-schedule a CF on +// flush_scheduler_ (which crashes debug builds via assert). +TEST_F(DBWriteBufferManagerTest, DoubleSchedulingBugDuringWALRecovery) { + Options options = CurrentOptions(); + options.arena_block_size = 4096; + // Small per-CF limit so memtable triggers CheckMemtableFull during recovery + options.write_buffer_size = 64 * 1024; // 64KB + options.max_write_buffer_number = 10; + options.disable_auto_compactions = true; + options.avoid_flush_during_recovery = true; + options.enforce_write_buffer_manager_during_recovery = true; + + // WBM limit also small so the WBM loop in InsertLogRecordToMemtable fires + options.write_buffer_manager = + std::make_shared(128 * 1024, nullptr, true); + + DestroyAndReopen(options); + + // Write enough data to exceed both limits during recovery replay + for (int i = 0; i < 50; i++) { + ASSERT_OK(Put(Key(i), DummyString(4096))); // ~200KB total > both limits + } + + Close(); + + // Reopen triggers WAL recovery. Without the fix, this crashes in debug + // builds with assert(checking_set_.count(cfd) == 0) in ScheduleWork(). + ASSERT_OK(TryReopen(options)); +} + INSTANTIATE_TEST_CASE_P(DBWriteBufferManagerTest, DBWriteBufferManagerTest, testing::Bool()); diff --git a/db/memtable.h b/db/memtable.h index fb3d232315..3ce4f9a416 100644 --- a/db/memtable.h +++ b/db/memtable.h @@ -599,6 +599,11 @@ class MemTable final : public ReadOnlyMemTable { std::memory_order_relaxed); } + // Returns true if a flush has already been scheduled for this memtable + bool HasFlushScheduled() const { + return flush_state_.load(std::memory_order_relaxed) == FLUSH_SCHEDULED; + } + InternalIterator* NewIterator( const ReadOptions& read_options, UnownedPtr seqno_to_time_mapping, Arena* arena, diff --git a/db_stress_tool/db_stress_common.h b/db_stress_tool/db_stress_common.h index ece60788c4..d0cf0c91ae 100644 --- a/db_stress_tool/db_stress_common.h +++ b/db_stress_tool/db_stress_common.h @@ -272,6 +272,7 @@ DECLARE_bool(avoid_unnecessary_blocking_io); DECLARE_bool(write_dbid_to_manifest); DECLARE_bool(write_identity_file); DECLARE_bool(avoid_flush_during_recovery); +DECLARE_bool(enforce_write_buffer_manager_during_recovery); DECLARE_uint64(max_write_batch_group_size_bytes); DECLARE_bool(level_compaction_dynamic_level_bytes); DECLARE_int32(verify_checksum_one_in); diff --git a/db_stress_tool/db_stress_gflags.cc b/db_stress_tool/db_stress_gflags.cc index b4dce9c573..12bf8993f5 100644 --- a/db_stress_tool/db_stress_gflags.cc +++ b/db_stress_tool/db_stress_gflags.cc @@ -1073,6 +1073,11 @@ DEFINE_bool(avoid_flush_during_recovery, ROCKSDB_NAMESPACE::Options().avoid_flush_during_recovery, "Avoid flush during recovery"); +DEFINE_bool( + enforce_write_buffer_manager_during_recovery, + ROCKSDB_NAMESPACE::Options().enforce_write_buffer_manager_during_recovery, + "Enforce write buffer manager memory limit during WAL recovery"); + DEFINE_uint64(max_write_batch_group_size_bytes, ROCKSDB_NAMESPACE::Options().max_write_batch_group_size_bytes, "Max write batch group size"); diff --git a/db_stress_tool/db_stress_test_base.cc b/db_stress_tool/db_stress_test_base.cc index 32e4ff9247..38276caeb6 100644 --- a/db_stress_tool/db_stress_test_base.cc +++ b/db_stress_tool/db_stress_test_base.cc @@ -4518,6 +4518,8 @@ void InitializeOptionsFromFlags( options.write_dbid_to_manifest = FLAGS_write_dbid_to_manifest; options.write_identity_file = FLAGS_write_identity_file; options.avoid_flush_during_recovery = FLAGS_avoid_flush_during_recovery; + options.enforce_write_buffer_manager_during_recovery = + FLAGS_enforce_write_buffer_manager_during_recovery; options.max_write_batch_group_size_bytes = FLAGS_max_write_batch_group_size_bytes; options.level_compaction_dynamic_level_bytes = @@ -4760,10 +4762,18 @@ void InitializeOptionsGeneral( "`preserve_unverified_changes` to keep all files\n"); options.avoid_flush_during_recovery = true; } - // Together with `avoid_flush_during_recovery == true`, this will prevent - // live files from becoming obsolete and deleted between `DB::Open()` and - // `DisableFileDeletions()` due to flush or compaction. We do not need to - // warn the user since we will reenable compaction soon. + if (options.enforce_write_buffer_manager_during_recovery) { + fprintf( + stderr, + "WARNING: flipping `enforce_write_buffer_manager_during_recovery` " + "to false for `preserve_unverified_changes` to keep all files\n"); + options.enforce_write_buffer_manager_during_recovery = false; + } + // Together with `avoid_flush_during_recovery == true` and + // `enforce_write_buffer_manager_during_recovery == false`, this will + // prevent live files from becoming obsolete and deleted between + // `DB::Open()` and `DisableFileDeletions()` due to flush or compaction. + // We do not need to warn the user since we will reenable compaction soon. options.disable_auto_compactions = true; } diff --git a/include/rocksdb/options.h b/include/rocksdb/options.h index c38c4f2d0a..cca97226a5 100644 --- a/include/rocksdb/options.h +++ b/include/rocksdb/options.h @@ -1411,9 +1411,30 @@ struct DBOptions { // WAL logs will be kept, so that if crash happened before flush, we still // have logs to recover from. // + // Note: when `enforce_write_buffer_manager_during_recovery` is also enabled, + // flushes may still occur during recovery to respect the + // WriteBufferManager's global memory limit, even if this option is true. + // Once any such WBM-triggered flush happens, all remaining memtables will + // also be flushed at the end of recovery (similar to the behavior when this + // option is false). + // // DEFAULT: false bool avoid_flush_during_recovery = false; + // If true and a WriteBufferManager is configured, RocksDB will check + // WriteBufferManager::ShouldFlush() during WAL recovery and schedule + // flushes when needed. This prevents OOM when multiple RocksDB instances + // share a WriteBufferManager and one instance is recovering from WAL. + // + // When triggered, all column families with non-empty memtables are scheduled + // for flush, which may produce smaller L0 files in some column families. + // This also overrides `avoid_flush_during_recovery`: once a WBM-triggered + // flush occurs mid-recovery, all remaining non-empty memtables will be + // flushed at the end of recovery as well. + // + // DEFAULT: true + bool enforce_write_buffer_manager_during_recovery = true; + // By default RocksDB will flush all memtables on DB close if there are // unpersisted data (i.e. with WAL disabled) The flush can be skip to speedup // DB close. Unpersisted data WILL BE LOST. diff --git a/options/db_options.cc b/options/db_options.cc index 0cefcb7be9..d0b1a38d05 100644 --- a/options/db_options.cc +++ b/options/db_options.cc @@ -375,6 +375,11 @@ static std::unordered_map {offsetof(struct ImmutableDBOptions, avoid_flush_during_recovery), OptionType::kBoolean, OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"enforce_write_buffer_manager_during_recovery", + {offsetof(struct ImmutableDBOptions, + enforce_write_buffer_manager_during_recovery), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, {"allow_ingest_behind", {offsetof(struct ImmutableDBOptions, allow_ingest_behind), OptionType::kBoolean, OptionVerificationType::kNormal, @@ -776,6 +781,8 @@ ImmutableDBOptions::ImmutableDBOptions(const DBOptions& options) wal_filter(options.wal_filter), dump_malloc_stats(options.dump_malloc_stats), avoid_flush_during_recovery(options.avoid_flush_during_recovery), + enforce_write_buffer_manager_during_recovery( + options.enforce_write_buffer_manager_during_recovery), allow_ingest_behind(options.allow_ingest_behind), two_write_queues(options.two_write_queues), manual_wal_flush(options.manual_wal_flush), @@ -942,6 +949,9 @@ void ImmutableDBOptions::Dump(Logger* log) const { ROCKS_LOG_HEADER(log, " Options.avoid_flush_during_recovery: %d", avoid_flush_during_recovery); + ROCKS_LOG_HEADER( + log, " Options.enforce_write_buffer_manager_during_recovery: %d", + enforce_write_buffer_manager_during_recovery); ROCKS_LOG_HEADER(log, " Options.allow_ingest_behind: %d", allow_ingest_behind); ROCKS_LOG_HEADER(log, " Options.two_write_queues: %d", diff --git a/options/db_options.h b/options/db_options.h index 2192e099c3..b814788f5e 100644 --- a/options/db_options.h +++ b/options/db_options.h @@ -77,6 +77,7 @@ struct ImmutableDBOptions { WalFilter* wal_filter; bool dump_malloc_stats; bool avoid_flush_during_recovery; + bool enforce_write_buffer_manager_during_recovery; bool allow_ingest_behind; bool two_write_queues; bool manual_wal_flush; diff --git a/options/options_helper.cc b/options/options_helper.cc index 50fe1da95a..42d9bd7e29 100644 --- a/options/options_helper.cc +++ b/options/options_helper.cc @@ -157,6 +157,8 @@ void BuildDBOptions(const ImmutableDBOptions& immutable_db_options, options.dump_malloc_stats = immutable_db_options.dump_malloc_stats; options.avoid_flush_during_recovery = immutable_db_options.avoid_flush_during_recovery; + options.enforce_write_buffer_manager_during_recovery = + immutable_db_options.enforce_write_buffer_manager_during_recovery; options.avoid_flush_during_shutdown = mutable_db_options.avoid_flush_during_shutdown; options.allow_ingest_behind = immutable_db_options.allow_ingest_behind; diff --git a/options/options_settable_test.cc b/options/options_settable_test.cc index d63746f337..fa9b045ce5 100644 --- a/options/options_settable_test.cc +++ b/options/options_settable_test.cc @@ -388,102 +388,103 @@ TEST_F(OptionsSettableTest, DBOptionsAllFieldsSettable) { ConfigOptions config_options(*options); config_options.input_strings_escaped = false; config_options.ignore_unknown_options = false; - ASSERT_OK( - GetDBOptionsFromString(config_options, *options, - "wal_bytes_per_sync=4295048118;" - "delete_obsolete_files_period_micros=4294967758;" - "WAL_ttl_seconds=4295008036;" - "WAL_size_limit_MB=4295036161;" - "max_write_batch_group_size_bytes=1048576;" - "wal_dir=path/to/wal_dir;" - "db_write_buffer_size=2587;" - "max_subcompactions=64330;" - "table_cache_numshardbits=28;" - "max_open_files=72;" - "max_file_opening_threads=35;" - "max_background_jobs=8;" - "max_background_compactions=33;" - "use_fsync=true;" - "use_adaptive_mutex=false;" - "max_total_wal_size=4295005604;" - "compaction_readahead_size=0;" - "keep_log_file_num=4890;" - "skip_stats_update_on_db_open=false;" - "max_manifest_file_size=4295009941;" - "max_manifest_space_amp_pct=321;" - "db_log_dir=path/to/db_log_dir;" - "writable_file_max_buffer_size=1048576;" - "paranoid_checks=true;" - "open_files_async=true;" - "flush_verify_memtable_count=true;" - "compaction_verify_record_count=true;" - "track_and_verify_wals_in_manifest=true;" - "track_and_verify_wals=true;" - "verify_sst_unique_id_in_manifest=true;" - "is_fd_close_on_exec=false;" - "bytes_per_sync=4295013613;" - "strict_bytes_per_sync=true;" - "enable_thread_tracking=false;" - "recycle_log_file_num=0;" - "create_missing_column_families=true;" - "log_file_time_to_roll=3097;" - "max_background_flushes=35;" - "create_if_missing=false;" - "error_if_exists=true;" - "delayed_write_rate=4294976214;" - "manifest_preallocation_size=1222;" - "allow_mmap_writes=false;" - "stats_dump_period_sec=70127;" - "stats_persist_period_sec=54321;" - "persist_stats_to_disk=true;" - "stats_history_buffer_size=14159;" - "allow_fallocate=true;" - "allow_mmap_reads=false;" - "use_direct_reads=false;" - "use_direct_io_for_flush_and_compaction=false;" - "max_log_file_size=4607;" - "advise_random_on_open=true;" - "enable_pipelined_write=false;" - "unordered_write=false;" - "allow_concurrent_memtable_write=true;" - "wal_recovery_mode=kPointInTimeRecovery;" - "enable_write_thread_adaptive_yield=true;" - "write_thread_slow_yield_usec=5;" - "write_thread_max_yield_usec=1000;" - "info_log_level=DEBUG_LEVEL;" - "dump_malloc_stats=false;" - "allow_2pc=false;" - "avoid_flush_during_recovery=false;" - "avoid_flush_during_shutdown=false;" - "allow_ingest_behind=false;" - "concurrent_prepare=false;" - "two_write_queues=false;" - "manual_wal_flush=false;" - "wal_compression=kZSTD;" - "background_close_inactive_wals=true;" - "seq_per_batch=false;" - "atomic_flush=false;" - "avoid_unnecessary_blocking_io=false;" - "log_readahead_size=0;" - "write_dbid_to_manifest=false;" - "best_efforts_recovery=false;" - "max_bgerror_resume_count=2;" - "bgerror_resume_retry_interval=1000000;" - "db_host_id=hostname;" - "lowest_used_cache_tier=kNonVolatileBlockTier;" - "allow_data_in_errors=false;" - "enforce_single_del_contracts=false;" - "daily_offpeak_time_utc=08:30-19:00;" - "follower_refresh_catchup_period_ms=123;" - "follower_catchup_retry_count=456;" - "follower_catchup_retry_wait_ms=789;" - "metadata_write_temperature=kCold;" - "wal_write_temperature=kHot;" - "background_close_inactive_wals=true;" - "write_dbid_to_manifest=true;" - "write_identity_file=true;" - "prefix_seek_opt_in_only=true;", - new_options)); + ASSERT_OK(GetDBOptionsFromString( + config_options, *options, + "wal_bytes_per_sync=4295048118;" + "delete_obsolete_files_period_micros=4294967758;" + "WAL_ttl_seconds=4295008036;" + "WAL_size_limit_MB=4295036161;" + "max_write_batch_group_size_bytes=1048576;" + "wal_dir=path/to/wal_dir;" + "db_write_buffer_size=2587;" + "max_subcompactions=64330;" + "table_cache_numshardbits=28;" + "max_open_files=72;" + "max_file_opening_threads=35;" + "max_background_jobs=8;" + "max_background_compactions=33;" + "use_fsync=true;" + "use_adaptive_mutex=false;" + "max_total_wal_size=4295005604;" + "compaction_readahead_size=0;" + "keep_log_file_num=4890;" + "skip_stats_update_on_db_open=false;" + "max_manifest_file_size=4295009941;" + "max_manifest_space_amp_pct=321;" + "db_log_dir=path/to/db_log_dir;" + "writable_file_max_buffer_size=1048576;" + "paranoid_checks=true;" + "open_files_async=true;" + "flush_verify_memtable_count=true;" + "compaction_verify_record_count=true;" + "track_and_verify_wals_in_manifest=true;" + "track_and_verify_wals=true;" + "verify_sst_unique_id_in_manifest=true;" + "is_fd_close_on_exec=false;" + "bytes_per_sync=4295013613;" + "strict_bytes_per_sync=true;" + "enable_thread_tracking=false;" + "recycle_log_file_num=0;" + "create_missing_column_families=true;" + "log_file_time_to_roll=3097;" + "max_background_flushes=35;" + "create_if_missing=false;" + "error_if_exists=true;" + "delayed_write_rate=4294976214;" + "manifest_preallocation_size=1222;" + "allow_mmap_writes=false;" + "stats_dump_period_sec=70127;" + "stats_persist_period_sec=54321;" + "persist_stats_to_disk=true;" + "stats_history_buffer_size=14159;" + "allow_fallocate=true;" + "allow_mmap_reads=false;" + "use_direct_reads=false;" + "use_direct_io_for_flush_and_compaction=false;" + "max_log_file_size=4607;" + "advise_random_on_open=true;" + "enable_pipelined_write=false;" + "unordered_write=false;" + "allow_concurrent_memtable_write=true;" + "wal_recovery_mode=kPointInTimeRecovery;" + "enable_write_thread_adaptive_yield=true;" + "write_thread_slow_yield_usec=5;" + "write_thread_max_yield_usec=1000;" + "info_log_level=DEBUG_LEVEL;" + "dump_malloc_stats=false;" + "allow_2pc=false;" + "avoid_flush_during_recovery=false;" + "enforce_write_buffer_manager_during_recovery=true;" + "avoid_flush_during_shutdown=false;" + "allow_ingest_behind=false;" + "concurrent_prepare=false;" + "two_write_queues=false;" + "manual_wal_flush=false;" + "wal_compression=kZSTD;" + "background_close_inactive_wals=true;" + "seq_per_batch=false;" + "atomic_flush=false;" + "avoid_unnecessary_blocking_io=false;" + "log_readahead_size=0;" + "write_dbid_to_manifest=false;" + "best_efforts_recovery=false;" + "max_bgerror_resume_count=2;" + "bgerror_resume_retry_interval=1000000;" + "db_host_id=hostname;" + "lowest_used_cache_tier=kNonVolatileBlockTier;" + "allow_data_in_errors=false;" + "enforce_single_del_contracts=false;" + "daily_offpeak_time_utc=08:30-19:00;" + "follower_refresh_catchup_period_ms=123;" + "follower_catchup_retry_count=456;" + "follower_catchup_retry_wait_ms=789;" + "metadata_write_temperature=kCold;" + "wal_write_temperature=kHot;" + "background_close_inactive_wals=true;" + "write_dbid_to_manifest=true;" + "write_identity_file=true;" + "prefix_seek_opt_in_only=true;", + new_options)); ASSERT_EQ(unset_bytes_base, NumUnsetBytes(new_options_ptr, sizeof(DBOptions), kDBOptionsExcluded)); diff --git a/tools/db_crashtest.py b/tools/db_crashtest.py index ebda3245fd..1f5fdba28b 100644 --- a/tools/db_crashtest.py +++ b/tools/db_crashtest.py @@ -295,6 +295,7 @@ default_params = { "avoid_flush_during_recovery": lambda: random.choice( [1 if t == 0 else 0 for t in range(0, 8)] ), + "enforce_write_buffer_manager_during_recovery": random.randint(0, 1), "max_write_batch_group_size_bytes": lambda: random.choice( [16, 64, 1024 * 1024, 16 * 1024 * 1024] ), diff --git a/unreleased_history/new_features/enforce_wbm_during_recovery.md b/unreleased_history/new_features/enforce_wbm_during_recovery.md new file mode 100644 index 0000000000..198c6aa0da --- /dev/null +++ b/unreleased_history/new_features/enforce_wbm_during_recovery.md @@ -0,0 +1 @@ +Introduced enforce_write_buffer_manager_during_recovery option to allow WriteBufferManager to be enforced during WAL recovery. (Default: true)