AutoHeaders: migrate to explicit headers (batch 116)

Summary:
Finish OpenZL, the internal RocksDB source tree, and the released RocksDB copy, removing all three parent AutoHeaders opt-ins. Declare inferred headers while preserving aliases, globs, selectors, and visibility. RocksDB's generator lists library headers and passes test source headers through its template and shared wrapper. Header lists account for the source files compiled by each release, so the older released tree does not gain headers from newer internal test helpers.

Add `release_to_fbcode.py internal_repo_rocksdb --buck-only -y` to refresh the four Buck generator files and regenerate BUCK against the existing release's source lists. The command restores its five outputs if copying or generation fails, applies the existing generated-file warning, and rejects release-commit options. It does not copy runtime sources, change the release version, clone, commit, or deploy. Both RocksDB BUCK files are produced by the maintained generators.

Graft the OpenZL dev declarations into prod. Align the predictor's Folly include paths and dependencies with prod so promotion is reproducible. Add the IPNext controller transition header explicitly.

Authored with Codex.

Reviewed By: IanChilds

Differential Revision: D123482073

fbshipit-source-id: 05ab47e592d710834da56bbea57f5f93a0aea565

BUCK

@@ -423,7 +423,20 @@ cpp_library_wrapper(name="rocksdb_test_lib", srcs=[
         "tools/trace_analyzer_tool.cc",
         "utilities/agg_merge/test_agg_merge.cc",
         "utilities/cassandra/test_utils.cc",
-    ], deps=[":rocksdb_lib"], headers=[], link_whole=False, extra_test_libs=True)
+    ], deps=[":rocksdb_lib"], headers=[
+        "db/db_test_util.h",
+        "db/db_with_timestamp_test_util.h",
+        "table/mock_table.h",
+        "test_util/mock_time_env.h",
+        "test_util/secondary_cache_test_util.h",
+        "test_util/simple_external_table_factory.h",
+        "test_util/testharness.h",
+        "test_util/testutil.h",
+        "tools/block_cache_analyzer/block_cache_trace_analyzer.h",
+        "tools/trace_analyzer_tool.h",
+        "utilities/agg_merge/test_agg_merge.h",
+        "utilities/cassandra/test_utils.h",
+    ], link_whole=False, extra_test_libs=True)
 
 cpp_library_wrapper(name="rocksdb_with_faiss_test_lib", srcs=[
         "db/db_test_util.cc",
@@ -438,7 +451,20 @@ cpp_library_wrapper(name="rocksdb_with_faiss_test_lib", srcs=[
         "tools/trace_analyzer_tool.cc",
         "utilities/agg_merge/test_agg_merge.cc",
         "utilities/cassandra/test_utils.cc",
-    ], deps=[":rocksdb_with_faiss_lib"], headers=[], link_whole=False, extra_test_libs=True)
+    ], deps=[":rocksdb_with_faiss_lib"], headers=[
+        "db/db_test_util.h",
+        "db/db_with_timestamp_test_util.h",
+        "table/mock_table.h",
+        "test_util/mock_time_env.h",
+        "test_util/secondary_cache_test_util.h",
+        "test_util/simple_external_table_factory.h",
+        "test_util/testharness.h",
+        "test_util/testutil.h",
+        "tools/block_cache_analyzer/block_cache_trace_analyzer.h",
+        "tools/trace_analyzer_tool.h",
+        "utilities/agg_merge/test_agg_merge.h",
+        "utilities/cassandra/test_utils.h",
+    ], link_whole=False, extra_test_libs=True)
 
 cpp_library_wrapper(name="rocksdb_tools_lib", srcs=[
         "test_util/testutil.cc",
@@ -447,7 +473,12 @@ cpp_library_wrapper(name="rocksdb_tools_lib", srcs=[
         "tools/simulated_hybrid_file_system.cc",
         "tools/tool_hooks.cc",
         "tools/trace_analyzer_tool.cc",
-    ], deps=[":rocksdb_lib"], headers=[], link_whole=False, extra_test_libs=False)
+    ], deps=[":rocksdb_lib"], headers=[
+        "test_util/testutil.h",
+        "tools/block_cache_analyzer/block_cache_trace_analyzer.h",
+        "tools/simulated_hybrid_file_system.h",
+        "tools/trace_analyzer_tool.h",
+    ], link_whole=False, extra_test_libs=False)
 
 cpp_library_wrapper(name="rocksdb_cache_bench_tools_lib", srcs=["cache/cache_bench_tool.cc"], deps=[":rocksdb_lib"], headers=[], link_whole=False, extra_test_libs=False)
 
@@ -475,7 +506,24 @@ rocks_cpp_library_wrapper(name="rocksdb_stress_lib", srcs=[
         "test_util/testutil.cc",
         "tools/block_cache_analyzer/block_cache_trace_analyzer.cc",
         "tools/trace_analyzer_tool.cc",
-    ], headers=[])
+    ], headers=[
+        "db_stress_tool/db_stress_common.h",
+        "db_stress_tool/db_stress_compaction_service.h",
+        "db_stress_tool/db_stress_compression_manager.h",
+        "db_stress_tool/db_stress_driver.h",
+        "db_stress_tool/db_stress_filters.h",
+        "db_stress_tool/db_stress_listener.h",
+        "db_stress_tool/db_stress_shared_state.h",
+        "db_stress_tool/db_stress_test_base.h",
+        "db_stress_tool/db_stress_wide_merge_operator.h",
+        "db_stress_tool/expected_state.h",
+        "db_stress_tool/expected_value.h",
+        "db_stress_tool/multi_ops_txns_stress.h",
+        "test_util/simple_external_table_factory.h",
+        "test_util/testutil.h",
+        "tools/block_cache_analyzer/block_cache_trace_analyzer.h",
+        "tools/trace_analyzer_tool.h",
+    ])
 
 
 cpp_binary_wrapper(name="ldb", srcs=["tools/ldb.cc"], deps=[":rocksdb_tools_lib"], extra_preprocessor_flags=[], extra_bench_libs=False)
@@ -4765,6 +4813,7 @@ cpp_unittest_wrapper(name="compression_test",
 
 cpp_unittest_wrapper(name="configurable_test",
             srcs=["options/configurable_test.cc"],
+            source_headers=["options/configurable_test.h"],
             deps=[":rocksdb_test_lib"],
             extra_compiler_flags=[])
 
@@ -5469,6 +5518,7 @@ cpp_unittest_wrapper(name="periodic_task_scheduler_test",
 
 cpp_unittest_wrapper(name="persistent_cache_test",
             srcs=["utilities/persistent_cache/persistent_cache_test.cc"],
+            source_headers=["utilities/persistent_cache/persistent_cache_test.h"],
             deps=[":rocksdb_test_lib"],
             extra_compiler_flags=[])
 
@@ -5487,6 +5537,7 @@ cpp_unittest_wrapper(name="point_lock_manager_stress_test",
 
 cpp_unittest_wrapper(name="point_lock_manager_test",
             srcs=["utilities/transactions/lock/point/point_lock_manager_test.cc"],
+            source_headers=["utilities/transactions/lock/point/point_lock_manager_test.h"],
             deps=[":rocksdb_test_lib"],
             extra_compiler_flags=[])
 
@@ -5697,6 +5748,7 @@ cpp_unittest_wrapper(name="trace_analyzer_test",
 
 cpp_unittest_wrapper(name="transaction_test",
             srcs=["utilities/transactions/transaction_test.cc"],
+            source_headers=["utilities/transactions/transaction_test.h"],
             deps=[":rocksdb_test_lib"],
             extra_compiler_flags=[])
 

buckifier/buckify_rocksdb.py

@@ -35,6 +35,49 @@ from util import ColorString
 # tests to export as libraries for inclusion in other projects
 _EXPORTED_TEST_LIBS = ["env_basic_test"]
 
+_TEST_LIB_HEADERS = [
+    "db/db_test_util.h",
+    "db/db_with_timestamp_test_util.h",
+    "table/mock_table.h",
+    "test_util/mock_time_env.h",
+    "test_util/secondary_cache_test_util.h",
+    "test_util/simple_external_table_factory.h",
+    "test_util/testharness.h",
+    "test_util/testutil.h",
+    "tools/block_cache_analyzer/block_cache_trace_analyzer.h",
+    "tools/trace_analyzer_tool.h",
+    "utilities/agg_merge/test_agg_merge.h",
+    "utilities/cassandra/test_utils.h",
+]
+
+_TEST_HEADERS = {
+    "configurable_test": ["options/configurable_test.h"],
+    "persistent_cache_test": ["utilities/persistent_cache/persistent_cache_test.h"],
+    "point_lock_manager_test": [
+        "utilities/transactions/lock/point/point_lock_manager_test.h",
+    ],
+    "transaction_test": ["utilities/transactions/transaction_test.h"],
+}
+
+_STRESS_LIB_HEADERS = [
+    "db_stress_tool/db_stress_common.h",
+    "db_stress_tool/db_stress_compaction_service.h",
+    "db_stress_tool/db_stress_compression_manager.h",
+    "db_stress_tool/db_stress_driver.h",
+    "db_stress_tool/db_stress_filters.h",
+    "db_stress_tool/db_stress_listener.h",
+    "db_stress_tool/db_stress_shared_state.h",
+    "db_stress_tool/db_stress_test_base.h",
+    "db_stress_tool/db_stress_wide_merge_operator.h",
+    "db_stress_tool/expected_state.h",
+    "db_stress_tool/expected_value.h",
+    "db_stress_tool/multi_ops_txns_stress.h",
+    "test_util/simple_external_table_factory.h",
+    "test_util/testutil.h",
+    "tools/block_cache_analyzer/block_cache_trace_analyzer.h",
+    "tools/trace_analyzer_tool.h"
+]
+
 # Parse src.mk files as a Dictionary of
 # VAR_NAME => list of files
 def parse_src_mk(repo_path):
@@ -183,27 +226,34 @@ def generate_buck(repo_path, deps_map):
             ":rocksdb_lib",
         ],
     )
-    # rocksdb_test_lib
-    BUCK.add_library(
-        "rocksdb_test_lib",
+    test_lib_sources = (
         src_mk.get("MOCK_LIB_SOURCES", [])
         + src_mk.get("TEST_LIB_SOURCES", [])
         + src_mk.get("EXP_LIB_SOURCES", [])
-        + src_mk.get("ANALYZER_LIB_SOURCES", []),
+        + src_mk.get("ANALYZER_LIB_SOURCES", [])
+    )
+    # Older release source lists may not compile every test helper yet.
+    test_lib_headers = [
+        header for header in _TEST_LIB_HEADERS
+        if header[:-2] + ".cc" in test_lib_sources
+    ]
+    # rocksdb_test_lib
+    BUCK.add_library(
+        "rocksdb_test_lib",
+        test_lib_sources,
         [":rocksdb_lib"],
         extra_test_libs=True,
+        headers=test_lib_headers,
     )
     # rocksdb_with_faiss_test_lib
     BUCK.add_library(
         "rocksdb_with_faiss_test_lib",
-        src_mk.get("MOCK_LIB_SOURCES", [])
-        + src_mk.get("TEST_LIB_SOURCES", [])
-        + src_mk.get("EXP_LIB_SOURCES", [])
-        + src_mk.get("ANALYZER_LIB_SOURCES", []),
+        test_lib_sources,
         deps=[
             ":rocksdb_with_faiss_lib",
         ],
         extra_test_libs=True,
+        headers=test_lib_headers,
     )
     # rocksdb_tools_lib
     BUCK.add_library(
@@ -212,6 +262,12 @@ def generate_buck(repo_path, deps_map):
         + src_mk.get("ANALYZER_LIB_SOURCES", [])
         + ["test_util/testutil.cc"],
         [":rocksdb_lib"],
+        headers=[
+            "test_util/testutil.h",
+            "tools/block_cache_analyzer/block_cache_trace_analyzer.h",
+            "tools/simulated_hybrid_file_system.h",
+            "tools/trace_analyzer_tool.h",
+        ],
     )
     # rocksdb_cache_bench_tools_lib
     BUCK.add_library(
@@ -225,12 +281,19 @@ def generate_buck(repo_path, deps_map):
         src_mk.get("POINT_LOCK_BENCH_LIB_SOURCES", []),
         [":rocksdb_lib"],
     )
+    stress_sources = (
+        src_mk.get("ANALYZER_LIB_SOURCES", [])
+        + src_mk.get("STRESS_LIB_SOURCES", [])
+        + ["test_util/testutil.cc"]
+    )
     # rocksdb_stress_lib
     BUCK.add_rocksdb_library(
         "rocksdb_stress_lib",
-        src_mk.get("ANALYZER_LIB_SOURCES", [])
-        + src_mk.get("STRESS_LIB_SOURCES", [])
-        + ["test_util/testutil.cc"],
+        stress_sources,
+        headers=[
+            header for header in _STRESS_LIB_HEADERS
+            if header[:-2] + ".cc" in stress_sources
+        ],
     )
     # ldb binary
     BUCK.add_binary(
@@ -353,6 +416,7 @@ def generate_buck(repo_path, deps_map):
                     test_src,
                     deps=json.dumps(deps["extra_deps"] + [":" + test_library]),
                     extra_compiler_flags=json.dumps(deps["extra_compiler_flags"]),
+                    source_headers=_TEST_HEADERS.get(test, []),
                 )
             else:
                 if with_faiss:
@@ -361,6 +425,7 @@ def generate_buck(repo_path, deps_map):
                         test_src,
                         deps=json.dumps(deps["extra_deps"] + [":rocksdb_with_faiss_test_lib"]),
                         extra_compiler_flags=json.dumps(deps["extra_compiler_flags"]),
+                        source_headers=_TEST_HEADERS.get(test, []),
                     )
                 else:
                     BUCK.register_test(
@@ -368,6 +433,7 @@ def generate_buck(repo_path, deps_map):
                         test_src,
                         deps=json.dumps(deps["extra_deps"] + [":rocksdb_test_lib"]),
                         extra_compiler_flags=json.dumps(deps["extra_compiler_flags"]),
+                        source_headers=_TEST_HEADERS.get(test, []),
                     )
     BUCK.export_file("tools/db_crashtest.py")
     BUCK.export_file("tools/fault_injection_log_parser.py")

buckifier/targets_builder.py

@@ -162,12 +162,17 @@ add_c_test_wrapper()
                 ).encode("utf-8")
             )
 
-    def register_test(self, test_name, src, deps, extra_compiler_flags):
+    def register_test(self, test_name, src, deps, extra_compiler_flags, source_headers):
         with open(self.path, "ab") as targets_file:
             targets_file.write(
                 targets_cfg.unittests_template.format(
                     test_name=test_name,
                     test_cc=str(src),
+                    source_headers=(
+                        f"            source_headers=[{pretty_list(source_headers, 16)}],\n"
+                        if source_headers
+                        else ""
+                    ),
                     deps=deps,
                     extra_compiler_flags=extra_compiler_flags,
                 ).encode("utf-8")

buckifier/targets_cfg.py

@@ -30,7 +30,7 @@ cpp_binary_wrapper(name="{name}", srcs=[{srcs}], deps=[{deps}], extra_preprocess
 unittests_template = """
 cpp_unittest_wrapper(name="{test_name}",
             srcs=["{test_cc}"],
-            deps={deps},
+{source_headers}            deps={deps},
             extra_compiler_flags={extra_compiler_flags})
 
 """