database/rocksdb
1
0
Fork 0
mirror of https://github.com/facebook/rocksdb.git synced 2026-10-23 08:09:02 +03:00
Code Activity

Add num_data_blocks_compression_rejected/bypassed to TableProperties (#14551)

Browse Source 
Summary:
Pull Request resolved: https://github.com/facebook/rocksdb/pull/14551

Add two new per-SST TableProperties fields that track how many data blocks
were stored uncompressed:

- `num_data_blocks_compression_rejected`: blocks where compression was
  attempted but the compressed output exceeded the ratio limit set by
  `CompressionOptions::max_compressed_bytes_per_kb`.
- `num_data_blocks_compression_bypassed`: blocks where compression was
  never attempted (e.g., kNoCompression type, no compressor available).

Together with `num_data_blocks`, these give a full decomposition:
  num_data_blocks = compressed + rejected + bypassed

Previously this information was only available via global Statistics tickers
(NUMBER_BLOCK_COMPRESSION_REJECTED / BYPASSED) which aggregate across all
SSTs. Now it is persisted per-SST in the properties block and visible via
sst_dump --show_properties.

Reviewed By: anand1976

Differential Revision: D99229339

fbshipit-source-id: 8333f51acff30a759d725e5e94bd80f4fcb18b57
This commit is contained in:
Jialiang Tan
2026-04-03 16:48:09 -07:00
committed by meta-codesync[bot]
parent 64f8d5765b
commit a0ff5b143f
6 changed files with 78 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
db/db_test_util.h
View File

@@ -1433,6 +1433,8 @@ class DBTestBase : public testing::Test {
tp->raw_key_size = 0;
tp->raw_value_size = 0;
tp->num_data_blocks = 0;
tp->num_data_blocks_compression_rejected = 0;
tp->num_data_blocks_compression_bypassed = 0;
tp->num_entries = 0;
tp->num_deletions = 0;
tp->num_merge_operands = 0;
@@ -1446,22 +1448,25 @@ class DBTestBase : public testing::Test {
ResetTableProperties(tp);
int count = sscanf(
tp_string.c_str(),
"# data blocks %" SCNu64 " # uniform blocks %" SCNu64
" # entries %" SCNu64 " # deletions %" SCNu64
" # merge operands %" SCNu64 " # range deletions %" SCNu64
" raw key size %" SCNu64
"# data blocks %" SCNu64 " # data blocks compression rejected %" SCNu64
" # data blocks compression bypassed %" SCNu64
" # uniform blocks %" SCNu64 " # entries %" SCNu64
" # deletions %" SCNu64 " # merge operands %" SCNu64
" # range deletions %" SCNu64 " raw key size %" SCNu64
" raw average key size %lf "
" raw value size %" SCNu64
" raw average value size %lf "
" data block size %" SCNu64 " data uncompressed size %" SCNu64
" index block size (user-key? %" SCNu64 ", delta-value? %" SCNu64
") %" SCNu64 " filter block size %" SCNu64,
&tp->num_data_blocks, &tp->num_uniform_blocks, &tp->num_entries,
&tp->num_deletions, &tp->num_merge_operands, &tp->num_range_deletions,
&tp->raw_key_size, &dummy_double, &tp->raw_value_size, &dummy_double,
&tp->data_size, &tp->uncompressed_data_size, &tp->index_key_is_user_key,
&tp->num_data_blocks, &tp->num_data_blocks_compression_rejected,
&tp->num_data_blocks_compression_bypassed, &tp->num_uniform_blocks,
&tp->num_entries, &tp->num_deletions, &tp->num_merge_operands,
&tp->num_range_deletions, &tp->raw_key_size, &dummy_double,
&tp->raw_value_size, &dummy_double, &tp->data_size,
&tp->uncompressed_data_size, &tp->index_key_is_user_key,
&tp->index_value_is_delta_encoded, &tp->index_size, &tp->filter_size);
ASSERT_EQ(count, 16);
ASSERT_EQ(count, 18);
}
private: // Prone to error on direct use

9
include/rocksdb/table_properties.h
View File

@@ -50,6 +50,8 @@ struct TablePropertiesNames {
static const std::string kRawKeySize;
static const std::string kRawValueSize;
static const std::string kNumDataBlocks;
static const std::string kNumDataBlocksCompressionRejected;
static const std::string kNumDataBlocksCompressionBypassed;
static const std::string kNumUniformBlocks;
static const std::string kNumEntries;
static const std::string kNumFilterEntries;
@@ -246,6 +248,13 @@ struct TableProperties {
uint64_t raw_value_size = 0;
// the number of blocks in this table
uint64_t num_data_blocks = 0;
// Number of data blocks stored uncompressed because compression was
// attempted but the compressed output exceeded the ratio limit set by
// CompressionOptions::max_compressed_bytes_per_kb.
uint64_t num_data_blocks_compression_rejected = 0;
// Number of data blocks stored uncompressed because compression was
// never attempted (e.g., kNoCompression, no compressor available).
uint64_t num_data_blocks_compression_bypassed = 0;
// the number of uniform blocks in this table
uint64_t num_uniform_blocks = 0;
// the number of entries in this table

4
options/options_settable_test.cc
View File

@@ -279,7 +279,9 @@ TEST_F(OptionsSettableTest, TablePropertiesAllFieldsSettable) {
"7265616461626C655F76616C7565;};compression_options=;compression_name=;"
"property_collectors_names=;prefix_extractor_name=;db_host_id="
"64625F686F73745F6964;db_session_id=64625F73657373696F6E5F6964;creation_"
"time=0;num_data_blocks=123;index_value_is_delta_encoded=0;top_level_"
"time=0;num_data_blocks=123;num_data_blocks_compression_rejected=42;"
"num_data_blocks_compression_bypassed=7;"
"index_value_is_delta_encoded=0;top_level_"
"index_size=0;data_size=100;uncompressed_data_size=1234;"
"merge_operator_name=;index_partitions=0;file_"
"creation_time=0;raw_value_size=0;index_size=200;user_collected_"

14
table/block_based/block_based_table_builder.cc
View File

@@ -853,6 +853,8 @@ struct BlockBasedTableBuilder::Rep {
uint64_t sample_for_compression;
RelaxedAtomic<uint64_t> compressible_input_data_bytes{0};
RelaxedAtomic<uint64_t> uncompressible_input_data_bytes{0};
RelaxedAtomic<uint64_t> num_data_blocks_compression_rejected{0};
RelaxedAtomic<uint64_t> num_data_blocks_compression_bypassed{0};
RelaxedAtomic<uint64_t> sampled_input_data_bytes{0};
RelaxedAtomic<uint64_t> sampled_output_slow_data_bytes{0};
RelaxedAtomic<uint64_t> sampled_output_fast_data_bytes{0};
@@ -2021,6 +2023,13 @@ Status BlockBasedTableBuilder::CompressAndVerifyBlock(
compression_attempted ? BYTES_COMPRESSION_REJECTED
: BYTES_COMPRESSION_BYPASSED,
uncompressed_block_data.size());
if (is_data_block) {
if (compression_attempted) {
r->num_data_blocks_compression_rejected.FetchAddRelaxed(1);
} else {
r->num_data_blocks_compression_bypassed.FetchAddRelaxed(1);
}
}
} else {
RecordTick(r->ioptions.stats, NUMBER_BLOCK_COMPRESSED);
RecordTick(r->ioptions.stats, BYTES_COMPRESSED_FROM,
@@ -2514,6 +2523,11 @@ void BlockBasedTableBuilder::WritePropertiesBlock(
rep_->props.user_defined_timestamps_persisted =
rep_->persist_user_defined_timestamps;
rep_->props.num_data_blocks_compression_rejected =
rep_->num_data_blocks_compression_rejected.LoadRelaxed();
rep_->props.num_data_blocks_compression_bypassed =
rep_->num_data_blocks_compression_bypassed.LoadRelaxed();
assert(IsEmpty() || rep_->props.key_largest_seqno != UINT64_MAX);
// Add basic properties
property_block_builder.AddTableProperty(rep_->props);

12
table/meta_blocks.cc
View File

@@ -97,6 +97,14 @@ void PropertyBlockBuilder::AddTableProperty(const TableProperties& props) {
Add(TablePropertiesNames::kMergeOperands, props.num_merge_operands);
Add(TablePropertiesNames::kNumRangeDeletions, props.num_range_deletions);
Add(TablePropertiesNames::kNumDataBlocks, props.num_data_blocks);
if (props.num_data_blocks_compression_rejected > 0) {
Add(TablePropertiesNames::kNumDataBlocksCompressionRejected,
props.num_data_blocks_compression_rejected);
}
if (props.num_data_blocks_compression_bypassed > 0) {
Add(TablePropertiesNames::kNumDataBlocksCompressionBypassed,
props.num_data_blocks_compression_bypassed);
}
Add(TablePropertiesNames::kNumUniformBlocks, props.num_uniform_blocks);
Add(TablePropertiesNames::kFilterSize, props.filter_size);
Add(TablePropertiesNames::kFormatVersion, props.format_version);
@@ -292,6 +300,10 @@ Status ParsePropertiesBlock(
&new_table_properties->raw_value_size},
{TablePropertiesNames::kNumDataBlocks,
&new_table_properties->num_data_blocks},
{TablePropertiesNames::kNumDataBlocksCompressionRejected,
&new_table_properties->num_data_blocks_compression_rejected},
{TablePropertiesNames::kNumDataBlocksCompressionBypassed,
&new_table_properties->num_data_blocks_compression_bypassed},
{TablePropertiesNames::kNumUniformBlocks,
&new_table_properties->num_uniform_blocks},
{TablePropertiesNames::kNumEntries, &new_table_properties->num_entries},

26
table/table_properties.cc
View File

@@ -47,6 +47,10 @@ std::string TableProperties::ToString(const std::string& prop_delim,
// Basic Info
AppendProperty(result, "# data blocks", num_data_blocks, prop_delim,
kv_delim);
AppendProperty(result, "# data blocks compression rejected",
num_data_blocks_compression_rejected, prop_delim, kv_delim);
AppendProperty(result, "# data blocks compression bypassed",
num_data_blocks_compression_bypassed, prop_delim, kv_delim);
AppendProperty(result, "# uniform blocks", num_uniform_blocks, prop_delim,
kv_delim);
AppendProperty(result, "# entries", num_entries, prop_delim, kv_delim);
@@ -194,6 +198,10 @@ void TableProperties::Add(const TableProperties& tp) {
raw_key_size += tp.raw_key_size;
raw_value_size += tp.raw_value_size;
num_data_blocks += tp.num_data_blocks;
num_data_blocks_compression_rejected +=
tp.num_data_blocks_compression_rejected;
num_data_blocks_compression_bypassed +=
tp.num_data_blocks_compression_bypassed;
num_uniform_blocks += tp.num_uniform_blocks;
num_entries += tp.num_entries;
num_filter_entries += tp.num_filter_entries;
@@ -218,6 +226,10 @@ TableProperties::GetAggregatablePropertiesAsMap() const {
rv["raw_key_size"] = raw_key_size;
rv["raw_value_size"] = raw_value_size;
rv["num_data_blocks"] = num_data_blocks;
rv["num_data_blocks_compression_rejected"] =
num_data_blocks_compression_rejected;
rv["num_data_blocks_compression_bypassed"] =
num_data_blocks_compression_bypassed;
rv["num_uniform_blocks"] = num_uniform_blocks;
rv["num_entries"] = num_entries;
rv["num_filter_entries"] = num_filter_entries;
@@ -284,6 +296,10 @@ const std::string TablePropertiesNames::kRawValueSize =
"rocksdb.raw.value.size";
const std::string TablePropertiesNames::kNumDataBlocks =
"rocksdb.num.data.blocks";
const std::string TablePropertiesNames::kNumDataBlocksCompressionRejected =
"rocksdb.num.data.blocks.compression.rejected";
const std::string TablePropertiesNames::kNumDataBlocksCompressionBypassed =
"rocksdb.num.data.blocks.compression.bypassed";
const std::string TablePropertiesNames::kNumUniformBlocks =
"rocksdb.num.uniform.blocks";
const std::string TablePropertiesNames::kNumEntries = "rocksdb.num.entries";
@@ -387,6 +403,16 @@ static std::unordered_map<std::string, OptionTypeInfo>
{offsetof(struct TableProperties, num_data_blocks),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"num_data_blocks_compression_rejected",
{offsetof(struct TableProperties,
num_data_blocks_compression_rejected),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"num_data_blocks_compression_bypassed",
{offsetof(struct TableProperties,
num_data_blocks_compression_bypassed),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"num_uniform_blocks",
{offsetof(struct TableProperties, num_uniform_blocks),
OptionType::kUInt64T, OptionVerificationType::kNormal,