From a0ff5b143f84fc5e1cf288cf40925d5460d54fe2 Mon Sep 17 00:00:00 2001 From: Jialiang Tan Date: Fri, 3 Apr 2026 16:48:09 -0700 Subject: [PATCH] Add num_data_blocks_compression_rejected/bypassed to TableProperties (#14551) Summary: Pull Request resolved: https://github.com/facebook/rocksdb/pull/14551 Add two new per-SST TableProperties fields that track how many data blocks were stored uncompressed: - `num_data_blocks_compression_rejected`: blocks where compression was attempted but the compressed output exceeded the ratio limit set by `CompressionOptions::max_compressed_bytes_per_kb`. - `num_data_blocks_compression_bypassed`: blocks where compression was never attempted (e.g., kNoCompression type, no compressor available). Together with `num_data_blocks`, these give a full decomposition: num_data_blocks = compressed + rejected + bypassed Previously this information was only available via global Statistics tickers (NUMBER_BLOCK_COMPRESSION_REJECTED / BYPASSED) which aggregate across all SSTs. Now it is persisted per-SST in the properties block and visible via sst_dump --show_properties. Reviewed By: anand1976 Differential Revision: D99229339 fbshipit-source-id: 8333f51acff30a759d725e5e94bd80f4fcb18b57 --- db/db_test_util.h | 23 +++++++++------- include/rocksdb/table_properties.h | 9 +++++++ options/options_settable_test.cc | 4 ++- .../block_based/block_based_table_builder.cc | 14 ++++++++++ table/meta_blocks.cc | 12 +++++++++ table/table_properties.cc | 26 +++++++++++++++++++ 6 files changed, 78 insertions(+), 10 deletions(-) diff --git a/db/db_test_util.h b/db/db_test_util.h index aa76c4d724..d9e538f973 100644 --- a/db/db_test_util.h +++ b/db/db_test_util.h @@ -1433,6 +1433,8 @@ class DBTestBase : public testing::Test { tp->raw_key_size = 0; tp->raw_value_size = 0; tp->num_data_blocks = 0; + tp->num_data_blocks_compression_rejected = 0; + tp->num_data_blocks_compression_bypassed = 0; tp->num_entries = 0; tp->num_deletions = 0; tp->num_merge_operands = 0; @@ -1446,22 +1448,25 @@ class DBTestBase : public testing::Test { ResetTableProperties(tp); int count = sscanf( tp_string.c_str(), - "# data blocks %" SCNu64 " # uniform blocks %" SCNu64 - " # entries %" SCNu64 " # deletions %" SCNu64 - " # merge operands %" SCNu64 " # range deletions %" SCNu64 - " raw key size %" SCNu64 + "# data blocks %" SCNu64 " # data blocks compression rejected %" SCNu64 + " # data blocks compression bypassed %" SCNu64 + " # uniform blocks %" SCNu64 " # entries %" SCNu64 + " # deletions %" SCNu64 " # merge operands %" SCNu64 + " # range deletions %" SCNu64 " raw key size %" SCNu64 " raw average key size %lf " " raw value size %" SCNu64 " raw average value size %lf " " data block size %" SCNu64 " data uncompressed size %" SCNu64 " index block size (user-key? %" SCNu64 ", delta-value? %" SCNu64 ") %" SCNu64 " filter block size %" SCNu64, - &tp->num_data_blocks, &tp->num_uniform_blocks, &tp->num_entries, - &tp->num_deletions, &tp->num_merge_operands, &tp->num_range_deletions, - &tp->raw_key_size, &dummy_double, &tp->raw_value_size, &dummy_double, - &tp->data_size, &tp->uncompressed_data_size, &tp->index_key_is_user_key, + &tp->num_data_blocks, &tp->num_data_blocks_compression_rejected, + &tp->num_data_blocks_compression_bypassed, &tp->num_uniform_blocks, + &tp->num_entries, &tp->num_deletions, &tp->num_merge_operands, + &tp->num_range_deletions, &tp->raw_key_size, &dummy_double, + &tp->raw_value_size, &dummy_double, &tp->data_size, + &tp->uncompressed_data_size, &tp->index_key_is_user_key, &tp->index_value_is_delta_encoded, &tp->index_size, &tp->filter_size); - ASSERT_EQ(count, 16); + ASSERT_EQ(count, 18); } private: // Prone to error on direct use diff --git a/include/rocksdb/table_properties.h b/include/rocksdb/table_properties.h index e4d2f3503b..f2526a7652 100644 --- a/include/rocksdb/table_properties.h +++ b/include/rocksdb/table_properties.h @@ -50,6 +50,8 @@ struct TablePropertiesNames { static const std::string kRawKeySize; static const std::string kRawValueSize; static const std::string kNumDataBlocks; + static const std::string kNumDataBlocksCompressionRejected; + static const std::string kNumDataBlocksCompressionBypassed; static const std::string kNumUniformBlocks; static const std::string kNumEntries; static const std::string kNumFilterEntries; @@ -246,6 +248,13 @@ struct TableProperties { uint64_t raw_value_size = 0; // the number of blocks in this table uint64_t num_data_blocks = 0; + // Number of data blocks stored uncompressed because compression was + // attempted but the compressed output exceeded the ratio limit set by + // CompressionOptions::max_compressed_bytes_per_kb. + uint64_t num_data_blocks_compression_rejected = 0; + // Number of data blocks stored uncompressed because compression was + // never attempted (e.g., kNoCompression, no compressor available). + uint64_t num_data_blocks_compression_bypassed = 0; // the number of uniform blocks in this table uint64_t num_uniform_blocks = 0; // the number of entries in this table diff --git a/options/options_settable_test.cc b/options/options_settable_test.cc index 39f9337524..34f1372b38 100644 --- a/options/options_settable_test.cc +++ b/options/options_settable_test.cc @@ -279,7 +279,9 @@ TEST_F(OptionsSettableTest, TablePropertiesAllFieldsSettable) { "7265616461626C655F76616C7565;};compression_options=;compression_name=;" "property_collectors_names=;prefix_extractor_name=;db_host_id=" "64625F686F73745F6964;db_session_id=64625F73657373696F6E5F6964;creation_" - "time=0;num_data_blocks=123;index_value_is_delta_encoded=0;top_level_" + "time=0;num_data_blocks=123;num_data_blocks_compression_rejected=42;" + "num_data_blocks_compression_bypassed=7;" + "index_value_is_delta_encoded=0;top_level_" "index_size=0;data_size=100;uncompressed_data_size=1234;" "merge_operator_name=;index_partitions=0;file_" "creation_time=0;raw_value_size=0;index_size=200;user_collected_" diff --git a/table/block_based/block_based_table_builder.cc b/table/block_based/block_based_table_builder.cc index fe6a2a0b5a..e162ba23a5 100644 --- a/table/block_based/block_based_table_builder.cc +++ b/table/block_based/block_based_table_builder.cc @@ -853,6 +853,8 @@ struct BlockBasedTableBuilder::Rep { uint64_t sample_for_compression; RelaxedAtomic compressible_input_data_bytes{0}; RelaxedAtomic uncompressible_input_data_bytes{0}; + RelaxedAtomic num_data_blocks_compression_rejected{0}; + RelaxedAtomic num_data_blocks_compression_bypassed{0}; RelaxedAtomic sampled_input_data_bytes{0}; RelaxedAtomic sampled_output_slow_data_bytes{0}; RelaxedAtomic sampled_output_fast_data_bytes{0}; @@ -2021,6 +2023,13 @@ Status BlockBasedTableBuilder::CompressAndVerifyBlock( compression_attempted ? BYTES_COMPRESSION_REJECTED : BYTES_COMPRESSION_BYPASSED, uncompressed_block_data.size()); + if (is_data_block) { + if (compression_attempted) { + r->num_data_blocks_compression_rejected.FetchAddRelaxed(1); + } else { + r->num_data_blocks_compression_bypassed.FetchAddRelaxed(1); + } + } } else { RecordTick(r->ioptions.stats, NUMBER_BLOCK_COMPRESSED); RecordTick(r->ioptions.stats, BYTES_COMPRESSED_FROM, @@ -2514,6 +2523,11 @@ void BlockBasedTableBuilder::WritePropertiesBlock( rep_->props.user_defined_timestamps_persisted = rep_->persist_user_defined_timestamps; + rep_->props.num_data_blocks_compression_rejected = + rep_->num_data_blocks_compression_rejected.LoadRelaxed(); + rep_->props.num_data_blocks_compression_bypassed = + rep_->num_data_blocks_compression_bypassed.LoadRelaxed(); + assert(IsEmpty() || rep_->props.key_largest_seqno != UINT64_MAX); // Add basic properties property_block_builder.AddTableProperty(rep_->props); diff --git a/table/meta_blocks.cc b/table/meta_blocks.cc index 491e77c026..4e342fa797 100644 --- a/table/meta_blocks.cc +++ b/table/meta_blocks.cc @@ -97,6 +97,14 @@ void PropertyBlockBuilder::AddTableProperty(const TableProperties& props) { Add(TablePropertiesNames::kMergeOperands, props.num_merge_operands); Add(TablePropertiesNames::kNumRangeDeletions, props.num_range_deletions); Add(TablePropertiesNames::kNumDataBlocks, props.num_data_blocks); + if (props.num_data_blocks_compression_rejected > 0) { + Add(TablePropertiesNames::kNumDataBlocksCompressionRejected, + props.num_data_blocks_compression_rejected); + } + if (props.num_data_blocks_compression_bypassed > 0) { + Add(TablePropertiesNames::kNumDataBlocksCompressionBypassed, + props.num_data_blocks_compression_bypassed); + } Add(TablePropertiesNames::kNumUniformBlocks, props.num_uniform_blocks); Add(TablePropertiesNames::kFilterSize, props.filter_size); Add(TablePropertiesNames::kFormatVersion, props.format_version); @@ -292,6 +300,10 @@ Status ParsePropertiesBlock( &new_table_properties->raw_value_size}, {TablePropertiesNames::kNumDataBlocks, &new_table_properties->num_data_blocks}, + {TablePropertiesNames::kNumDataBlocksCompressionRejected, + &new_table_properties->num_data_blocks_compression_rejected}, + {TablePropertiesNames::kNumDataBlocksCompressionBypassed, + &new_table_properties->num_data_blocks_compression_bypassed}, {TablePropertiesNames::kNumUniformBlocks, &new_table_properties->num_uniform_blocks}, {TablePropertiesNames::kNumEntries, &new_table_properties->num_entries}, diff --git a/table/table_properties.cc b/table/table_properties.cc index 2b703498b5..d49302cce2 100644 --- a/table/table_properties.cc +++ b/table/table_properties.cc @@ -47,6 +47,10 @@ std::string TableProperties::ToString(const std::string& prop_delim, // Basic Info AppendProperty(result, "# data blocks", num_data_blocks, prop_delim, kv_delim); + AppendProperty(result, "# data blocks compression rejected", + num_data_blocks_compression_rejected, prop_delim, kv_delim); + AppendProperty(result, "# data blocks compression bypassed", + num_data_blocks_compression_bypassed, prop_delim, kv_delim); AppendProperty(result, "# uniform blocks", num_uniform_blocks, prop_delim, kv_delim); AppendProperty(result, "# entries", num_entries, prop_delim, kv_delim); @@ -194,6 +198,10 @@ void TableProperties::Add(const TableProperties& tp) { raw_key_size += tp.raw_key_size; raw_value_size += tp.raw_value_size; num_data_blocks += tp.num_data_blocks; + num_data_blocks_compression_rejected += + tp.num_data_blocks_compression_rejected; + num_data_blocks_compression_bypassed += + tp.num_data_blocks_compression_bypassed; num_uniform_blocks += tp.num_uniform_blocks; num_entries += tp.num_entries; num_filter_entries += tp.num_filter_entries; @@ -218,6 +226,10 @@ TableProperties::GetAggregatablePropertiesAsMap() const { rv["raw_key_size"] = raw_key_size; rv["raw_value_size"] = raw_value_size; rv["num_data_blocks"] = num_data_blocks; + rv["num_data_blocks_compression_rejected"] = + num_data_blocks_compression_rejected; + rv["num_data_blocks_compression_bypassed"] = + num_data_blocks_compression_bypassed; rv["num_uniform_blocks"] = num_uniform_blocks; rv["num_entries"] = num_entries; rv["num_filter_entries"] = num_filter_entries; @@ -284,6 +296,10 @@ const std::string TablePropertiesNames::kRawValueSize = "rocksdb.raw.value.size"; const std::string TablePropertiesNames::kNumDataBlocks = "rocksdb.num.data.blocks"; +const std::string TablePropertiesNames::kNumDataBlocksCompressionRejected = + "rocksdb.num.data.blocks.compression.rejected"; +const std::string TablePropertiesNames::kNumDataBlocksCompressionBypassed = + "rocksdb.num.data.blocks.compression.bypassed"; const std::string TablePropertiesNames::kNumUniformBlocks = "rocksdb.num.uniform.blocks"; const std::string TablePropertiesNames::kNumEntries = "rocksdb.num.entries"; @@ -387,6 +403,16 @@ static std::unordered_map {offsetof(struct TableProperties, num_data_blocks), OptionType::kUInt64T, OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"num_data_blocks_compression_rejected", + {offsetof(struct TableProperties, + num_data_blocks_compression_rejected), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"num_data_blocks_compression_bypassed", + {offsetof(struct TableProperties, + num_data_blocks_compression_bypassed), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, {"num_uniform_blocks", {offsetof(struct TableProperties, num_uniform_blocks), OptionType::kUInt64T, OptionVerificationType::kNormal,