From 7292adcc9aa1eae47b74993e7f14cdf9b42b108f Mon Sep 17 00:00:00 2001 From: Davis Rollman Date: Wed, 7 Oct 2026 15:43:34 -0700 Subject: [PATCH] AutoHeaders: migrate to explicit headers (batch 116) Summary: Finish OpenZL, the internal RocksDB source tree, and the released RocksDB copy, removing all three parent AutoHeaders opt-ins. Declare inferred headers while preserving aliases, globs, selectors, and visibility. RocksDB's generator lists library headers and passes test source headers through its template and shared wrapper. Header lists account for the source files compiled by each release, so the older released tree does not gain headers from newer internal test helpers. Add `release_to_fbcode.py internal_repo_rocksdb --buck-only -y` to refresh the four Buck generator files and regenerate BUCK against the existing release's source lists. The command restores its five outputs if copying or generation fails, applies the existing generated-file warning, and rejects release-commit options. It does not copy runtime sources, change the release version, clone, commit, or deploy. Both RocksDB BUCK files are produced by the maintained generators. Graft the OpenZL dev declarations into prod. Align the predictor's Folly include paths and dependencies with prod so promotion is reproducible. Add the IPNext controller transition header explicitly. Authored with Codex. Reviewed By: IanChilds Differential Revision: D123482073 fbshipit-source-id: 05ab47e592d710834da56bbea57f5f93a0aea565 --- BUCK | 60 ++++++++++++++++++++++-- buckifier/buckify_rocksdb.py | 88 +++++++++++++++++++++++++++++++----- buckifier/targets_builder.py | 7 ++- buckifier/targets_cfg.py | 2 +- 4 files changed, 140 insertions(+), 17 deletions(-) diff --git a/BUCK b/BUCK index 644b72c4f6..105158ae38 100644 --- a/BUCK +++ b/BUCK @@ -423,7 +423,20 @@ cpp_library_wrapper(name="rocksdb_test_lib", srcs=[ "tools/trace_analyzer_tool.cc", "utilities/agg_merge/test_agg_merge.cc", "utilities/cassandra/test_utils.cc", - ], deps=[":rocksdb_lib"], headers=[], link_whole=False, extra_test_libs=True) + ], deps=[":rocksdb_lib"], headers=[ + "db/db_test_util.h", + "db/db_with_timestamp_test_util.h", + "table/mock_table.h", + "test_util/mock_time_env.h", + "test_util/secondary_cache_test_util.h", + "test_util/simple_external_table_factory.h", + "test_util/testharness.h", + "test_util/testutil.h", + "tools/block_cache_analyzer/block_cache_trace_analyzer.h", + "tools/trace_analyzer_tool.h", + "utilities/agg_merge/test_agg_merge.h", + "utilities/cassandra/test_utils.h", + ], link_whole=False, extra_test_libs=True) cpp_library_wrapper(name="rocksdb_with_faiss_test_lib", srcs=[ "db/db_test_util.cc", @@ -438,7 +451,20 @@ cpp_library_wrapper(name="rocksdb_with_faiss_test_lib", srcs=[ "tools/trace_analyzer_tool.cc", "utilities/agg_merge/test_agg_merge.cc", "utilities/cassandra/test_utils.cc", - ], deps=[":rocksdb_with_faiss_lib"], headers=[], link_whole=False, extra_test_libs=True) + ], deps=[":rocksdb_with_faiss_lib"], headers=[ + "db/db_test_util.h", + "db/db_with_timestamp_test_util.h", + "table/mock_table.h", + "test_util/mock_time_env.h", + "test_util/secondary_cache_test_util.h", + "test_util/simple_external_table_factory.h", + "test_util/testharness.h", + "test_util/testutil.h", + "tools/block_cache_analyzer/block_cache_trace_analyzer.h", + "tools/trace_analyzer_tool.h", + "utilities/agg_merge/test_agg_merge.h", + "utilities/cassandra/test_utils.h", + ], link_whole=False, extra_test_libs=True) cpp_library_wrapper(name="rocksdb_tools_lib", srcs=[ "test_util/testutil.cc", @@ -447,7 +473,12 @@ cpp_library_wrapper(name="rocksdb_tools_lib", srcs=[ "tools/simulated_hybrid_file_system.cc", "tools/tool_hooks.cc", "tools/trace_analyzer_tool.cc", - ], deps=[":rocksdb_lib"], headers=[], link_whole=False, extra_test_libs=False) + ], deps=[":rocksdb_lib"], headers=[ + "test_util/testutil.h", + "tools/block_cache_analyzer/block_cache_trace_analyzer.h", + "tools/simulated_hybrid_file_system.h", + "tools/trace_analyzer_tool.h", + ], link_whole=False, extra_test_libs=False) cpp_library_wrapper(name="rocksdb_cache_bench_tools_lib", srcs=["cache/cache_bench_tool.cc"], deps=[":rocksdb_lib"], headers=[], link_whole=False, extra_test_libs=False) @@ -475,7 +506,24 @@ rocks_cpp_library_wrapper(name="rocksdb_stress_lib", srcs=[ "test_util/testutil.cc", "tools/block_cache_analyzer/block_cache_trace_analyzer.cc", "tools/trace_analyzer_tool.cc", - ], headers=[]) + ], headers=[ + "db_stress_tool/db_stress_common.h", + "db_stress_tool/db_stress_compaction_service.h", + "db_stress_tool/db_stress_compression_manager.h", + "db_stress_tool/db_stress_driver.h", + "db_stress_tool/db_stress_filters.h", + "db_stress_tool/db_stress_listener.h", + "db_stress_tool/db_stress_shared_state.h", + "db_stress_tool/db_stress_test_base.h", + "db_stress_tool/db_stress_wide_merge_operator.h", + "db_stress_tool/expected_state.h", + "db_stress_tool/expected_value.h", + "db_stress_tool/multi_ops_txns_stress.h", + "test_util/simple_external_table_factory.h", + "test_util/testutil.h", + "tools/block_cache_analyzer/block_cache_trace_analyzer.h", + "tools/trace_analyzer_tool.h", + ]) cpp_binary_wrapper(name="ldb", srcs=["tools/ldb.cc"], deps=[":rocksdb_tools_lib"], extra_preprocessor_flags=[], extra_bench_libs=False) @@ -4765,6 +4813,7 @@ cpp_unittest_wrapper(name="compression_test", cpp_unittest_wrapper(name="configurable_test", srcs=["options/configurable_test.cc"], + source_headers=["options/configurable_test.h"], deps=[":rocksdb_test_lib"], extra_compiler_flags=[]) @@ -5469,6 +5518,7 @@ cpp_unittest_wrapper(name="periodic_task_scheduler_test", cpp_unittest_wrapper(name="persistent_cache_test", srcs=["utilities/persistent_cache/persistent_cache_test.cc"], + source_headers=["utilities/persistent_cache/persistent_cache_test.h"], deps=[":rocksdb_test_lib"], extra_compiler_flags=[]) @@ -5487,6 +5537,7 @@ cpp_unittest_wrapper(name="point_lock_manager_stress_test", cpp_unittest_wrapper(name="point_lock_manager_test", srcs=["utilities/transactions/lock/point/point_lock_manager_test.cc"], + source_headers=["utilities/transactions/lock/point/point_lock_manager_test.h"], deps=[":rocksdb_test_lib"], extra_compiler_flags=[]) @@ -5697,6 +5748,7 @@ cpp_unittest_wrapper(name="trace_analyzer_test", cpp_unittest_wrapper(name="transaction_test", srcs=["utilities/transactions/transaction_test.cc"], + source_headers=["utilities/transactions/transaction_test.h"], deps=[":rocksdb_test_lib"], extra_compiler_flags=[]) diff --git a/buckifier/buckify_rocksdb.py b/buckifier/buckify_rocksdb.py index 0c9983e387..eb3b761121 100755 --- a/buckifier/buckify_rocksdb.py +++ b/buckifier/buckify_rocksdb.py @@ -35,6 +35,49 @@ from util import ColorString # tests to export as libraries for inclusion in other projects _EXPORTED_TEST_LIBS = ["env_basic_test"] +_TEST_LIB_HEADERS = [ + "db/db_test_util.h", + "db/db_with_timestamp_test_util.h", + "table/mock_table.h", + "test_util/mock_time_env.h", + "test_util/secondary_cache_test_util.h", + "test_util/simple_external_table_factory.h", + "test_util/testharness.h", + "test_util/testutil.h", + "tools/block_cache_analyzer/block_cache_trace_analyzer.h", + "tools/trace_analyzer_tool.h", + "utilities/agg_merge/test_agg_merge.h", + "utilities/cassandra/test_utils.h", +] + +_TEST_HEADERS = { + "configurable_test": ["options/configurable_test.h"], + "persistent_cache_test": ["utilities/persistent_cache/persistent_cache_test.h"], + "point_lock_manager_test": [ + "utilities/transactions/lock/point/point_lock_manager_test.h", + ], + "transaction_test": ["utilities/transactions/transaction_test.h"], +} + +_STRESS_LIB_HEADERS = [ + "db_stress_tool/db_stress_common.h", + "db_stress_tool/db_stress_compaction_service.h", + "db_stress_tool/db_stress_compression_manager.h", + "db_stress_tool/db_stress_driver.h", + "db_stress_tool/db_stress_filters.h", + "db_stress_tool/db_stress_listener.h", + "db_stress_tool/db_stress_shared_state.h", + "db_stress_tool/db_stress_test_base.h", + "db_stress_tool/db_stress_wide_merge_operator.h", + "db_stress_tool/expected_state.h", + "db_stress_tool/expected_value.h", + "db_stress_tool/multi_ops_txns_stress.h", + "test_util/simple_external_table_factory.h", + "test_util/testutil.h", + "tools/block_cache_analyzer/block_cache_trace_analyzer.h", + "tools/trace_analyzer_tool.h" +] + # Parse src.mk files as a Dictionary of # VAR_NAME => list of files def parse_src_mk(repo_path): @@ -183,27 +226,34 @@ def generate_buck(repo_path, deps_map): ":rocksdb_lib", ], ) - # rocksdb_test_lib - BUCK.add_library( - "rocksdb_test_lib", + test_lib_sources = ( src_mk.get("MOCK_LIB_SOURCES", []) + src_mk.get("TEST_LIB_SOURCES", []) + src_mk.get("EXP_LIB_SOURCES", []) - + src_mk.get("ANALYZER_LIB_SOURCES", []), + + src_mk.get("ANALYZER_LIB_SOURCES", []) + ) + # Older release source lists may not compile every test helper yet. + test_lib_headers = [ + header for header in _TEST_LIB_HEADERS + if header[:-2] + ".cc" in test_lib_sources + ] + # rocksdb_test_lib + BUCK.add_library( + "rocksdb_test_lib", + test_lib_sources, [":rocksdb_lib"], extra_test_libs=True, + headers=test_lib_headers, ) # rocksdb_with_faiss_test_lib BUCK.add_library( "rocksdb_with_faiss_test_lib", - src_mk.get("MOCK_LIB_SOURCES", []) - + src_mk.get("TEST_LIB_SOURCES", []) - + src_mk.get("EXP_LIB_SOURCES", []) - + src_mk.get("ANALYZER_LIB_SOURCES", []), + test_lib_sources, deps=[ ":rocksdb_with_faiss_lib", ], extra_test_libs=True, + headers=test_lib_headers, ) # rocksdb_tools_lib BUCK.add_library( @@ -212,6 +262,12 @@ def generate_buck(repo_path, deps_map): + src_mk.get("ANALYZER_LIB_SOURCES", []) + ["test_util/testutil.cc"], [":rocksdb_lib"], + headers=[ + "test_util/testutil.h", + "tools/block_cache_analyzer/block_cache_trace_analyzer.h", + "tools/simulated_hybrid_file_system.h", + "tools/trace_analyzer_tool.h", + ], ) # rocksdb_cache_bench_tools_lib BUCK.add_library( @@ -225,12 +281,19 @@ def generate_buck(repo_path, deps_map): src_mk.get("POINT_LOCK_BENCH_LIB_SOURCES", []), [":rocksdb_lib"], ) + stress_sources = ( + src_mk.get("ANALYZER_LIB_SOURCES", []) + + src_mk.get("STRESS_LIB_SOURCES", []) + + ["test_util/testutil.cc"] + ) # rocksdb_stress_lib BUCK.add_rocksdb_library( "rocksdb_stress_lib", - src_mk.get("ANALYZER_LIB_SOURCES", []) - + src_mk.get("STRESS_LIB_SOURCES", []) - + ["test_util/testutil.cc"], + stress_sources, + headers=[ + header for header in _STRESS_LIB_HEADERS + if header[:-2] + ".cc" in stress_sources + ], ) # ldb binary BUCK.add_binary( @@ -353,6 +416,7 @@ def generate_buck(repo_path, deps_map): test_src, deps=json.dumps(deps["extra_deps"] + [":" + test_library]), extra_compiler_flags=json.dumps(deps["extra_compiler_flags"]), + source_headers=_TEST_HEADERS.get(test, []), ) else: if with_faiss: @@ -361,6 +425,7 @@ def generate_buck(repo_path, deps_map): test_src, deps=json.dumps(deps["extra_deps"] + [":rocksdb_with_faiss_test_lib"]), extra_compiler_flags=json.dumps(deps["extra_compiler_flags"]), + source_headers=_TEST_HEADERS.get(test, []), ) else: BUCK.register_test( @@ -368,6 +433,7 @@ def generate_buck(repo_path, deps_map): test_src, deps=json.dumps(deps["extra_deps"] + [":rocksdb_test_lib"]), extra_compiler_flags=json.dumps(deps["extra_compiler_flags"]), + source_headers=_TEST_HEADERS.get(test, []), ) BUCK.export_file("tools/db_crashtest.py") BUCK.export_file("tools/fault_injection_log_parser.py") diff --git a/buckifier/targets_builder.py b/buckifier/targets_builder.py index 9682828d9f..02064791a1 100644 --- a/buckifier/targets_builder.py +++ b/buckifier/targets_builder.py @@ -162,12 +162,17 @@ add_c_test_wrapper() ).encode("utf-8") ) - def register_test(self, test_name, src, deps, extra_compiler_flags): + def register_test(self, test_name, src, deps, extra_compiler_flags, source_headers): with open(self.path, "ab") as targets_file: targets_file.write( targets_cfg.unittests_template.format( test_name=test_name, test_cc=str(src), + source_headers=( + f" source_headers=[{pretty_list(source_headers, 16)}],\n" + if source_headers + else "" + ), deps=deps, extra_compiler_flags=extra_compiler_flags, ).encode("utf-8") diff --git a/buckifier/targets_cfg.py b/buckifier/targets_cfg.py index e9ff129a60..de882164dc 100644 --- a/buckifier/targets_cfg.py +++ b/buckifier/targets_cfg.py @@ -30,7 +30,7 @@ cpp_binary_wrapper(name="{name}", srcs=[{srcs}], deps=[{deps}], extra_preprocess unittests_template = """ cpp_unittest_wrapper(name="{test_name}", srcs=["{test_cc}"], - deps={deps}, +{source_headers} deps={deps}, extra_compiler_flags={extra_compiler_flags}) """