1
0
mirror of https://github.com/facebook/rocksdb.git synced 2026-10-23 08:09:02 +03:00

Add test for deletion-triggered compaction with min file size (#13825)

Summary:
**Summary:**
This test verifies that compaction respects the min_file_size parameter when triggered by deletions, preventing the compaction of files with deletions smaller than the threshold. The test logic includes two scenarios:
1. Verify that a large L0 file with deletions exceeding the minimum file size threshold triggers deletion-triggered compaction (DTC) and compacts to L1.
2. Verify that a small L0 file with deletions, but below the minimum file size threshold, does not trigger DTC and remains at L0.

Added the DeletionTriggeredCompactionWithMinFileSizeTestListener, which verifies that files selected for compaction based on deletion triggers meet the minimum file size threshold. The listener validates in OnCompactionBegin that all input files have sizes greater than or equal to the configured min_file_size parameter.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/13825

Test Plan:
Tested this feature on our devserver using the following commands:
```
DEBUG_LEVEL=2 make -j64 db_compaction_test && KEEP_DB=1 ./db_compaction_test --gtest_filter="*DBCompactionTest.CompactionWith*"
```

Test output confirms the expected behavior:
```
2025/07/31-11:24:49.473181 1431671 [/compaction/compaction_job.cc:2291] [default] [JOB 6] Compacting 2@0 files to L1, score 0.04
2025/07/31-11:24:49.473240 1431671 [/compaction/compaction_job.cc:2297] [default]: Compaction start summary: Base version 6 Base level 0, inputs: [15(52KB) 9(103KB)]
2025/07/31-11:24:49.473304 1431671 EVENT_LOG_v1 {"time_micros": 1753986289473273, "job": 6, "event": "compaction_started", "cf_name": "default", "compaction_reason": "FilesMarkedForCompaction", "files_L0": [15, 9], "score": 0.04, "input_data_size": 159848, "oldest_snapshot_seqno": -1}

```

**Tasks:**
T228156639

Reviewed By: cbi42

Differential Revision: D79395851

Pulled By: nmk70

fbshipit-source-id: 4c2a80a95521b40543981dd81b347f3984cd2a8b
This commit is contained in:
ngina
2025-08-06 11:40:09 -07:00
committed by Facebook GitHub Bot
parent 9c0a0c0058
commit dfb4efaae3

View File

@@ -74,6 +74,43 @@ class CompactionStatsCollector : public EventListener {
std::vector<std::atomic<int>> compaction_completed_;
};
class DeletionTriggeredCompactionWithMinFileSizeTestListener
: public EventListener {
public:
explicit DeletionTriggeredCompactionWithMinFileSizeTestListener(
uint64_t min_file_size)
: min_file_size_(min_file_size) {}
void OnCompactionBegin(DB* db, const CompactionJobInfo& ci) override {
ASSERT_EQ(ci.compaction_reason,
CompactionReason::kFilesMarkedForCompaction);
auto env = db->GetEnv();
const std::vector<DbPath>& db_paths = db->GetOptions().db_paths;
for (const auto& file : ci.input_file_infos) {
uint64_t file_size = GetSstFileSize(env, db_paths, file.file_number);
// Assert that the file size respects the minimum threshold
ASSERT_GE(file_size, min_file_size_);
}
}
private:
static uint64_t GetSstFileSize(Env* env, const std::vector<DbPath>& db_paths,
uint64_t file_number) {
uint32_t path_id = 0; // since only one path
std::string sst_file_name = TableFileName(db_paths, file_number, path_id);
uint64_t file_size = 0;
Status s = env->GetFileSize(sst_file_name, &file_size);
if (!s.ok()) {
return 0;
}
return file_size;
}
uint64_t min_file_size_;
};
class DBCompactionTest : public DBTestBase {
public:
DBCompactionTest()
@@ -1371,6 +1408,89 @@ TEST_F(DBCompactionTest, RecoverDuringMemtableCompaction) {
} while (ChangeOptions());
}
TEST_F(DBCompactionTest, CompactionWithDeletionsAndMinFileSize) {
const uint64_t kMinFileSize = 32 * 1024; // 32KB
const int kDeletionTriggerCount = 50;
const int kInitialKeyCount = 100;
const int kAdditionalKeyCount = 50;
const int kValueSize = 1024;
const int kSmallValueSize = 512;
const int kSeed = 301;
Options options = CurrentOptions();
options.compaction_style = kCompactionStyleLevel;
options.write_buffer_size = 1024 * 1024; // 1MB
options.level0_file_num_compaction_trigger = 100;
options.table_properties_collector_factories = {
NewCompactOnDeletionCollectorFactory(
kInitialKeyCount /* sliding window size */, kDeletionTriggerCount,
0.5 /* deletion ratio */, kMinFileSize)};
auto listener =
new DeletionTriggeredCompactionWithMinFileSizeTestListener(kMinFileSize);
options.listeners.emplace_back(listener);
DestroyAndReopen(options);
Random rnd(kSeed);
// Create a large file that will be subject to DTC later
for (int i = 0; i < kInitialKeyCount; i++) {
ASSERT_OK(Put(Key(i), rnd.RandomString(kValueSize)));
}
ASSERT_OK(Flush());
std::vector<LiveFileMetaData> initial_metadata;
db_->GetLiveFilesMetaData(&initial_metadata);
ASSERT_EQ(initial_metadata.size(), 1);
// Create small files that should not trigger compaction
ASSERT_OK(Put("small_file_key1", rnd.RandomString(kSmallValueSize)));
ASSERT_OK(Put("small_file_key2", rnd.RandomString(kSmallValueSize)));
ASSERT_OK(Flush());
ASSERT_OK(Delete("small_file_key1"));
ASSERT_OK(Flush());
// Create a file with enough deletions and size to trigger DTC
// Delete keys from the large file to reach deletion threshold
for (int i = 0; i < kDeletionTriggerCount; i++) {
ASSERT_OK(Delete(Key(i)));
}
// Add new keys to ensure the deletion file meets the min_file_size threshold
for (int i = kInitialKeyCount; i < kInitialKeyCount + kAdditionalKeyCount;
i++) {
ASSERT_OK(Put(Key(i), rnd.RandomString(kValueSize)));
}
ASSERT_OK(Flush());
ASSERT_OK(dbfull()->TEST_WaitForCompact());
// Verify file count after compaction
ASSERT_EQ(NumTableFilesAtLevel(0), 2); // Small file and deletion file
ASSERT_EQ(NumTableFilesAtLevel(1), 1); // Compacted large file
// Verify deleted keys are gone
for (int i = 0; i < kDeletionTriggerCount; i++) {
std::string value;
ASSERT_TRUE(db_->Get(ReadOptions(), Key(i), &value).IsNotFound());
}
// Verify non-deleted keys from large file are still accessible
for (int i = kDeletionTriggerCount; i < kInitialKeyCount; i++) {
std::string value;
ASSERT_OK(db_->Get(ReadOptions(), Key(i), &value));
ASSERT_EQ(value.size(), kValueSize);
}
// Verify new keys are accessible
for (int i = kInitialKeyCount; i < kInitialKeyCount + kAdditionalKeyCount;
i++) {
std::string value;
ASSERT_OK(db_->Get(ReadOptions(), Key(i), &value));
ASSERT_EQ(value.size(), kValueSize);
}
}
TEST_P(DBCompactionTestWithParam, TrivialMoveOneFile) {
int32_t trivial_move = 0;
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(