lib/proxygen
1
0
Fork 0
mirror of https://github.com/facebook/proxygen.git synced 2025-08-08 18:02:05 +03:00
Code Activity

Refactoring

Browse Source 
Summary:
X-link: https://github.com/facebookincubator/zstrong/pull/897

Builder refactoring: instead of providing `install_dirs` to `build()`, `test()` etc., provide `loader` and `dep_manifests` when creating the builder. This is a cleaner API because we were computing `install_dirs` in multiple places before.

Furthermore this lets us do things that need to see the manifests of the dependencies, not just the list of `install_dirs`, such as treating direct dependencies differently from indirect dependencies (see D58244928).

Reviewed By: chadaustin

Differential Revision: D58200528

fbshipit-source-id: e52d35e84161b83ab49ab43099c3e3b9bb03f36e
This commit is contained in:
Simon Marlow
2024-06-26 17:02:17 -07:00
committed by Facebook GitHub Bot
parent 1c567ba69a
commit 6790282a52
6 changed files with 307 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
build/fbcode_builder/getdeps.py
View File

@@ -210,15 +210,6 @@ class ProjectCmdBase(SubCmd):
def setup_project_cmd_parser(self, parser): def setup_project_cmd_parser(self, parser):
pass pass
# For commands that don't build but need the full list of install_dirs from
# dependencies (test, debug).
def get_install_dirs(self, loader, manifest):
install_dirs = []
for m in loader.manifests_in_dependency_order():
if m != manifest:
install_dirs.append(loader.get_project_install_dir(m))
return install_dirs
def create_builder(self, loader, manifest): def create_builder(self, loader, manifest):
fetcher = loader.create_fetcher(manifest) fetcher = loader.create_fetcher(manifest)
src_dir = fetcher.get_src_dir() src_dir = fetcher.get_src_dir()
@@ -226,7 +217,13 @@ class ProjectCmdBase(SubCmd):
build_dir = loader.get_project_build_dir(manifest) build_dir = loader.get_project_build_dir(manifest)
inst_dir = loader.get_project_install_dir(manifest) inst_dir = loader.get_project_install_dir(manifest)
return manifest.create_builder( return manifest.create_builder(
loader.build_opts, src_dir, build_dir, inst_dir, ctx, loader loader.build_opts,
src_dir,
build_dir,
inst_dir,
ctx,
loader,
loader.dependencies_of(manifest),
) )
def check_built(self, loader, manifest): def check_built(self, loader, manifest):
@@ -579,11 +576,11 @@ class BuildCmd(ProjectCmdBase):
cache = cache_module.create_cache() if args.use_build_cache else None cache = cache_module.create_cache() if args.use_build_cache else None
# Accumulate the install directories so that the build steps dep_manifests = []
# can find their dep installation
install_dirs = []
for m in projects: for m in projects:
dep_manifests.append(m)
fetcher = loader.create_fetcher(m) fetcher = loader.create_fetcher(m)
if args.build_skip_lfs_download and hasattr(fetcher, "skip_lfs_download"): if args.build_skip_lfs_download and hasattr(fetcher, "skip_lfs_download"):
@@ -650,9 +647,10 @@ class BuildCmd(ProjectCmdBase):
build_dir, build_dir,
inst_dir, inst_dir,
loader, loader,
dep_manifests,
) )
for preparer in prepare_builders: for preparer in prepare_builders:
preparer.prepare(install_dirs, reconfigure=reconfigure) preparer.prepare(reconfigure=reconfigure)
builder = m.create_builder( builder = m.create_builder(
loader.build_opts, loader.build_opts,
@@ -661,12 +659,13 @@ class BuildCmd(ProjectCmdBase):
inst_dir, inst_dir,
ctx, ctx,
loader, loader,
dep_manifests,
final_install_prefix=loader.get_project_install_prefix(m), final_install_prefix=loader.get_project_install_prefix(m),
extra_cmake_defines=extra_cmake_defines, extra_cmake_defines=extra_cmake_defines,
cmake_target=args.cmake_target if m == manifest else "install", cmake_target=args.cmake_target if m == manifest else "install",
extra_b2_args=extra_b2_args, extra_b2_args=extra_b2_args,
) )
builder.build(install_dirs, reconfigure=reconfigure) builder.build(reconfigure=reconfigure)
# If we are building the project (not dependency) and a specific # If we are building the project (not dependency) and a specific
# cmake_target (not 'install') has been requested, then we don't # cmake_target (not 'install') has been requested, then we don't
@@ -690,11 +689,6 @@ class BuildCmd(ProjectCmdBase):
elif args.verbose: elif args.verbose:
print("found good %s" % built_marker) print("found good %s" % built_marker)
# Paths are resolved from front. We prepend rather than append as
# the last project in topo order is the project itself, which
# should be first in the path, then its deps and so on.
install_dirs.insert(0, inst_dir)
def compute_dep_change_status(self, m, built_marker, loader): def compute_dep_change_status(self, m, built_marker, loader):
reconfigure = False reconfigure = False
sources_changed = False sources_changed = False
@@ -883,11 +877,7 @@ class TestCmd(ProjectCmdBase):
if not self.check_built(loader, manifest): if not self.check_built(loader, manifest):
print("project %s has not been built" % manifest.name) print("project %s has not been built" % manifest.name)
return 1 return 1
builder = self.create_builder(loader, manifest) self.create_builder(loader, manifest).run_tests(
install_dirs = self.get_install_dirs(loader, manifest)
builder.run_tests(
install_dirs,
schedule_type=args.schedule_type, schedule_type=args.schedule_type,
owner=args.test_owner, owner=args.test_owner,
test_filter=args.filter, test_filter=args.filter,
@@ -921,9 +911,7 @@ class TestCmd(ProjectCmdBase):
) )
class DebugCmd(ProjectCmdBase): class DebugCmd(ProjectCmdBase):
def run_project_cmd(self, args, loader, manifest): def run_project_cmd(self, args, loader, manifest):
install_dirs = self.get_install_dirs(loader, manifest) self.create_builder(loader, manifest).debug(reconfigure=False)
builder = self.create_builder(loader, manifest)
builder.debug(install_dirs, reconfigure=False)
@cmd("generate-github-actions", "generate a GitHub actions configuration") @cmd("generate-github-actions", "generate a GitHub actions configuration")

229
build/fbcode_builder/getdeps/builder.py
View File

@@ -29,6 +29,8 @@ if typing.TYPE_CHECKING:
class BuilderBase(object): class BuilderBase(object):
def __init__( def __init__(
self, self,
loader,
dep_manifests, # manifests of dependencies
build_opts: "BuildOptions", build_opts: "BuildOptions",
ctx, ctx,
manifest, manifest,
@@ -55,6 +57,9 @@ class BuilderBase(object):
self.build_opts = build_opts self.build_opts = build_opts
self.manifest = manifest self.manifest = manifest
self.final_install_prefix = final_install_prefix self.final_install_prefix = final_install_prefix
self.loader = loader
self.dep_manifests = dep_manifests
self.install_dirs = [loader.get_project_install_dir(m) for m in dep_manifests]
def _get_cmd_prefix(self): def _get_cmd_prefix(self):
if self.build_opts.is_windows(): if self.build_opts.is_windows():
@@ -136,28 +141,28 @@ class BuilderBase(object):
os.chdir(old_wd) os.chdir(old_wd)
patched_sentinel_file.touch() patched_sentinel_file.touch()
def prepare(self, install_dirs, reconfigure: bool) -> None: def prepare(self, reconfigure: bool) -> None:
print("Preparing %s..." % self.manifest.name) print("Preparing %s..." % self.manifest.name)
reconfigure = self._reconfigure(reconfigure) reconfigure = self._reconfigure(reconfigure)
self._apply_patchfile() self._apply_patchfile()
self._prepare(install_dirs=install_dirs, reconfigure=reconfigure) self._prepare(reconfigure=reconfigure)
def debug(self, install_dirs, reconfigure: bool) -> None: def debug(self, reconfigure: bool) -> None:
reconfigure = self._reconfigure(reconfigure) reconfigure = self._reconfigure(reconfigure)
self._apply_patchfile() self._apply_patchfile()
self._prepare(install_dirs=install_dirs, reconfigure=reconfigure) self._prepare(reconfigure=reconfigure)
env = self._compute_env(install_dirs) env = self._compute_env()
print("Starting a shell in %s, ^D to exit..." % self.build_dir) print("Starting a shell in %s, ^D to exit..." % self.build_dir)
# TODO: print the command to run the build # TODO: print the command to run the build
shell = ["powershell.exe"] if sys.platform == "win32" else ["/bin/sh", "-i"] shell = ["powershell.exe"] if sys.platform == "win32" else ["/bin/sh", "-i"]
self._run_cmd(shell, cwd=self.build_dir, env=env) self._run_cmd(shell, cwd=self.build_dir, env=env)
def build(self, install_dirs, reconfigure: bool) -> None: def build(self, reconfigure: bool) -> None:
print("Building %s..." % self.manifest.name) print("Building %s..." % self.manifest.name)
reconfigure = self._reconfigure(reconfigure) reconfigure = self._reconfigure(reconfigure)
self._apply_patchfile() self._apply_patchfile()
self._prepare(install_dirs=install_dirs, reconfigure=reconfigure) self._prepare(reconfigure=reconfigure)
self._build(install_dirs=install_dirs, reconfigure=reconfigure) self._build(reconfigure=reconfigure)
if self.build_opts.free_up_disk: if self.build_opts.free_up_disk:
# don't clean --src-dir=. case as user may want to build again or run tests on the build # don't clean --src-dir=. case as user may want to build again or run tests on the build
@@ -174,8 +179,8 @@ class BuilderBase(object):
# needs to be updated to include all of the directories containing the runtime # needs to be updated to include all of the directories containing the runtime
# library dependencies in order to run the binaries. # library dependencies in order to run the binaries.
script_path = self.get_dev_run_script_path() script_path = self.get_dev_run_script_path()
dep_munger = create_dyn_dep_munger(self.build_opts, install_dirs) dep_munger = create_dyn_dep_munger(self.build_opts, self.install_dirs)
dep_dirs = self.get_dev_run_extra_path_dirs(install_dirs, dep_munger) dep_dirs = self.get_dev_run_extra_path_dirs(dep_munger)
# pyre-fixme[16]: Optional type has no attribute `emit_dev_run_script`. # pyre-fixme[16]: Optional type has no attribute `emit_dev_run_script`.
dep_munger.emit_dev_run_script(script_path, dep_dirs) dep_munger.emit_dev_run_script(script_path, dep_dirs)
@@ -200,49 +205,47 @@ class BuilderBase(object):
) )
) )
def run_tests( def run_tests(self, schedule_type, owner, test_filter, retry, no_testpilot) -> None:
self, install_dirs, schedule_type, owner, test_filter, retry, no_testpilot
) -> None:
"""Execute any tests that we know how to run. If they fail, """Execute any tests that we know how to run. If they fail,
raise an exception.""" raise an exception."""
pass pass
def _prepare(self, install_dirs, reconfigure) -> None: def _prepare(self, reconfigure) -> None:
"""Prepare the build. Useful when need to generate config, """Prepare the build. Useful when need to generate config,
but builder is not the primary build system. but builder is not the primary build system.
e.g. cargo when called from cmake""" e.g. cargo when called from cmake"""
pass pass
def _build(self, install_dirs, reconfigure) -> None: def _build(self, reconfigure) -> None:
"""Perform the build. """Perform the build.
install_dirs contains the list of installation directories for
the dependencies of this project.
reconfigure will be set to true if the fetcher determined reconfigure will be set to true if the fetcher determined
that the sources have changed in such a way that the build that the sources have changed in such a way that the build
system needs to regenerate its rules.""" system needs to regenerate its rules."""
pass pass
def _compute_env(self, install_dirs): def _compute_env(self):
# CMAKE_PREFIX_PATH is only respected when passed through the # CMAKE_PREFIX_PATH is only respected when passed through the
# environment, so we construct an appropriate path to pass down # environment, so we construct an appropriate path to pass down
return self.build_opts.compute_env_for_install_dirs( return self.build_opts.compute_env_for_install_dirs(
install_dirs, env=self.env, manifest=self.manifest self.install_dirs, env=self.env, manifest=self.manifest
) )
def get_dev_run_script_path(self): def get_dev_run_script_path(self):
assert self.build_opts.is_windows() assert self.build_opts.is_windows()
return os.path.join(self.build_dir, "run.ps1") return os.path.join(self.build_dir, "run.ps1")
def get_dev_run_extra_path_dirs(self, install_dirs, dep_munger=None): def get_dev_run_extra_path_dirs(self, dep_munger=None):
assert self.build_opts.is_windows() assert self.build_opts.is_windows()
if dep_munger is None: if dep_munger is None:
dep_munger = create_dyn_dep_munger(self.build_opts, install_dirs) dep_munger = create_dyn_dep_munger(self.build_opts, self.install_dirs)
return dep_munger.compute_dependency_paths(self.build_dir) return dep_munger.compute_dependency_paths(self.build_dir)
class MakeBuilder(BuilderBase): class MakeBuilder(BuilderBase):
def __init__( def __init__(
self, self,
loader,
dep_manifests,
build_opts, build_opts,
ctx, ctx,
manifest, manifest,
@@ -254,7 +257,14 @@ class MakeBuilder(BuilderBase):
test_args, test_args,
) -> None: ) -> None:
super(MakeBuilder, self).__init__( super(MakeBuilder, self).__init__(
build_opts, ctx, manifest, src_dir, build_dir, inst_dir loader,
dep_manifests,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
) )
self.build_args = build_args or [] self.build_args = build_args or []
self.install_args = install_args or [] self.install_args = install_args or []
@@ -267,9 +277,9 @@ class MakeBuilder(BuilderBase):
def _get_prefix(self): def _get_prefix(self):
return ["PREFIX=" + self.inst_dir, "prefix=" + self.inst_dir] return ["PREFIX=" + self.inst_dir, "prefix=" + self.inst_dir]
def _build(self, install_dirs, reconfigure) -> None: def _build(self, reconfigure) -> None:
env = self._compute_env(install_dirs) env = self._compute_env()
# Need to ensure that PREFIX is set prior to install because # Need to ensure that PREFIX is set prior to install because
# libbpf uses it when generating its pkg-config file. # libbpf uses it when generating its pkg-config file.
@@ -292,20 +302,18 @@ class MakeBuilder(BuilderBase):
for file in glob.glob(srcpattern): for file in glob.glob(srcpattern):
shutil.copy(file, libdir) shutil.copy(file, libdir)
def run_tests( def run_tests(self, schedule_type, owner, test_filter, retry, no_testpilot) -> None:
self, install_dirs, schedule_type, owner, test_filter, retry, no_testpilot
) -> None:
if not self.test_args: if not self.test_args:
return return
env = self._compute_env(install_dirs) env = self._compute_env()
cmd = [self._make_binary] + self.test_args + self._get_prefix() cmd = [self._make_binary] + self.test_args + self._get_prefix()
self._run_cmd(cmd, env=env) self._run_cmd(cmd, env=env)
class CMakeBootStrapBuilder(MakeBuilder): class CMakeBootStrapBuilder(MakeBuilder):
def _build(self, install_dirs, reconfigure) -> None: def _build(self, reconfigure) -> None:
self._run_cmd( self._run_cmd(
[ [
"./bootstrap", "./bootstrap",
@@ -313,12 +321,14 @@ class CMakeBootStrapBuilder(MakeBuilder):
f"--parallel={self.num_jobs}", f"--parallel={self.num_jobs}",
] ]
) )
super(CMakeBootStrapBuilder, self)._build(install_dirs, reconfigure) super(CMakeBootStrapBuilder, self)._build(reconfigure)
class AutoconfBuilder(BuilderBase): class AutoconfBuilder(BuilderBase):
def __init__( def __init__(
self, self,
loader,
dep_manifests,
build_opts, build_opts,
ctx, ctx,
manifest, manifest,
@@ -329,7 +339,14 @@ class AutoconfBuilder(BuilderBase):
conf_env_args, conf_env_args,
) -> None: ) -> None:
super(AutoconfBuilder, self).__init__( super(AutoconfBuilder, self).__init__(
build_opts, ctx, manifest, src_dir, build_dir, inst_dir loader,
dep_manifests,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
) )
self.args = args or [] self.args = args or []
self.conf_env_args = conf_env_args or {} self.conf_env_args = conf_env_args or {}
@@ -338,11 +355,11 @@ class AutoconfBuilder(BuilderBase):
def _make_binary(self): def _make_binary(self):
return self.manifest.get("build", "make_binary", "make", ctx=self.ctx) return self.manifest.get("build", "make_binary", "make", ctx=self.ctx)
def _build(self, install_dirs, reconfigure) -> None: def _build(self, reconfigure) -> None:
configure_path = os.path.join(self.src_dir, "configure") configure_path = os.path.join(self.src_dir, "configure")
autogen_path = os.path.join(self.src_dir, "autogen.sh") autogen_path = os.path.join(self.src_dir, "autogen.sh")
env = self._compute_env(install_dirs) env = self._compute_env()
# Some configure scripts need additional env values passed derived from cmds # Some configure scripts need additional env values passed derived from cmds
for k, cmd_args in self.conf_env_args.items(): for k, cmd_args in self.conf_env_args.items():
@@ -383,12 +400,29 @@ class Iproute2Builder(BuilderBase):
# Thus, explicitly copy sources from src_dir to build_dir, build, # Thus, explicitly copy sources from src_dir to build_dir, build,
# and then install to inst_dir using DESTDIR # and then install to inst_dir using DESTDIR
# lastly, also copy include from build_dir to inst_dir # lastly, also copy include from build_dir to inst_dir
def __init__(self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir) -> None: def __init__(
self,
loader,
dep_manifests,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
) -> None:
super(Iproute2Builder, self).__init__( super(Iproute2Builder, self).__init__(
build_opts, ctx, manifest, src_dir, build_dir, inst_dir loader,
dep_manifests,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
) )
def _build(self, install_dirs, reconfigure) -> None: def _build(self, reconfigure) -> None:
configure_path = os.path.join(self.src_dir, "configure") configure_path = os.path.join(self.src_dir, "configure")
env = self.env.copy() env = self.env.copy()
self._run_cmd([configure_path], env=env) self._run_cmd([configure_path], env=env)
@@ -521,6 +555,8 @@ if __name__ == "__main__":
def __init__( def __init__(
self, self,
loader,
dep_manifests,
build_opts, build_opts,
ctx, ctx,
manifest, manifest,
@@ -528,12 +564,13 @@ if __name__ == "__main__":
build_dir, build_dir,
inst_dir, inst_dir,
defines, defines,
loader=None,
final_install_prefix=None, final_install_prefix=None,
extra_cmake_defines=None, extra_cmake_defines=None,
cmake_target="install", cmake_target="install",
) -> None: ) -> None:
super(CMakeBuilder, self).__init__( super(CMakeBuilder, self).__init__(
loader,
dep_manifests,
build_opts, build_opts,
ctx, ctx,
manifest, manifest,
@@ -694,10 +731,10 @@ if __name__ == "__main__":
return define_args return define_args
def _build(self, install_dirs, reconfigure: bool) -> None: def _build(self, reconfigure: bool) -> None:
reconfigure = reconfigure or self._needs_reconfigure() reconfigure = reconfigure or self._needs_reconfigure()
env = self._compute_env(install_dirs) env = self._compute_env()
if not self.build_opts.is_windows() and self.final_install_prefix: if not self.build_opts.is_windows() and self.final_install_prefix:
env["DESTDIR"] = self.inst_dir env["DESTDIR"] = self.inst_dir
@@ -740,9 +777,9 @@ if __name__ == "__main__":
) )
def run_tests( def run_tests(
self, install_dirs, schedule_type, owner, test_filter, retry: int, no_testpilot self, schedule_type, owner, test_filter, retry: int, no_testpilot
) -> None: ) -> None:
env = self._compute_env(install_dirs) env = self._compute_env()
ctest = path_search(env, "ctest") ctest = path_search(env, "ctest")
cmake = path_search(env, "cmake") cmake = path_search(env, "cmake")
@@ -756,7 +793,7 @@ if __name__ == "__main__":
# since CMake will emit RPATH properly in the binary so they can find these # since CMake will emit RPATH properly in the binary so they can find these
# dependencies. # dependencies.
if self.build_opts.is_windows(): if self.build_opts.is_windows():
path_entries = self.get_dev_run_extra_path_dirs(install_dirs) path_entries = self.get_dev_run_extra_path_dirs()
path = env.get("PATH") path = env.get("PATH")
if path: if path:
path_entries.insert(0, path) path_entries.insert(0, path)
@@ -853,7 +890,6 @@ if __name__ == "__main__":
env.set("http_proxy", "") env.set("http_proxy", "")
env.set("https_proxy", "") env.set("https_proxy", "")
runs = [] runs = []
from sys import platform
with start_run(env["FBSOURCE_HASH"]) as run_id: with start_run(env["FBSOURCE_HASH"]) as run_id:
testpilot_args = [ testpilot_args = [
@@ -957,12 +993,29 @@ if __name__ == "__main__":
class NinjaBootstrap(BuilderBase): class NinjaBootstrap(BuilderBase):
def __init__(self, build_opts, ctx, manifest, build_dir, src_dir, inst_dir) -> None: def __init__(
self,
loader,
dep_manifests,
build_opts,
ctx,
manifest,
build_dir,
src_dir,
inst_dir,
) -> None:
super(NinjaBootstrap, self).__init__( super(NinjaBootstrap, self).__init__(
build_opts, ctx, manifest, src_dir, build_dir, inst_dir loader,
dep_manifests,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
) )
def _build(self, install_dirs, reconfigure) -> None: def _build(self, reconfigure) -> None:
self._run_cmd([sys.executable, "configure.py", "--bootstrap"], cwd=self.src_dir) self._run_cmd([sys.executable, "configure.py", "--bootstrap"], cwd=self.src_dir)
src_ninja = os.path.join(self.src_dir, "ninja") src_ninja = os.path.join(self.src_dir, "ninja")
dest_ninja = os.path.join(self.inst_dir, "bin/ninja") dest_ninja = os.path.join(self.inst_dir, "bin/ninja")
@@ -974,20 +1027,37 @@ class NinjaBootstrap(BuilderBase):
class OpenSSLBuilder(BuilderBase): class OpenSSLBuilder(BuilderBase):
def __init__(self, build_opts, ctx, manifest, build_dir, src_dir, inst_dir) -> None: def __init__(
self,
loader,
dep_manifests,
build_opts,
ctx,
manifest,
build_dir,
src_dir,
inst_dir,
) -> None:
super(OpenSSLBuilder, self).__init__( super(OpenSSLBuilder, self).__init__(
build_opts, ctx, manifest, src_dir, build_dir, inst_dir loader,
dep_manifests,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
) )
def _build(self, install_dirs, reconfigure) -> None: def _build(self, reconfigure) -> None:
configure = os.path.join(self.src_dir, "Configure") configure = os.path.join(self.src_dir, "Configure")
# prefer to resolve the perl that we installed from # prefer to resolve the perl that we installed from
# our manifest on windows, but fall back to the system # our manifest on windows, but fall back to the system
# path on eg: darwin # path on eg: darwin
env = self.env.copy() env = self.env.copy()
for d in install_dirs: for m in self.dep_manifests:
bindir = os.path.join(d, "bin") bindir = os.path.join(self.loader.get_project_install_dir(m), "bin")
add_path_entry(env, "PATH", bindir, append=False) add_path_entry(env, "PATH", bindir, append=False)
perl = typing.cast(str, path_search(env, "perl", "perl")) perl = typing.cast(str, path_search(env, "perl", "perl"))
@@ -1037,7 +1107,16 @@ class OpenSSLBuilder(BuilderBase):
class Boost(BuilderBase): class Boost(BuilderBase):
def __init__( def __init__(
self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir, b2_args self,
loader,
dep_manifests,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
b2_args,
) -> None: ) -> None:
children = os.listdir(src_dir) children = os.listdir(src_dir)
assert len(children) == 1, "expected a single directory entry: %r" % (children,) assert len(children) == 1, "expected a single directory entry: %r" % (children,)
@@ -1045,12 +1124,19 @@ class Boost(BuilderBase):
assert boost_src.startswith("boost") assert boost_src.startswith("boost")
src_dir = os.path.join(src_dir, children[0]) src_dir = os.path.join(src_dir, children[0])
super(Boost, self).__init__( super(Boost, self).__init__(
build_opts, ctx, manifest, src_dir, build_dir, inst_dir loader,
dep_manifests,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
) )
self.b2_args = b2_args self.b2_args = b2_args
def _build(self, install_dirs, reconfigure) -> None: def _build(self, reconfigure) -> None:
env = self._compute_env(install_dirs) env = self._compute_env()
linkage = ["static"] linkage = ["static"]
if self.build_opts.is_windows() or self.build_opts.shared_libs: if self.build_opts.is_windows() or self.build_opts.shared_libs:
linkage.append("shared") linkage.append("shared")
@@ -1105,12 +1191,14 @@ class Boost(BuilderBase):
class NopBuilder(BuilderBase): class NopBuilder(BuilderBase):
def __init__(self, build_opts, ctx, manifest, src_dir, inst_dir) -> None: def __init__(
self, loader, dep_manifests, build_opts, ctx, manifest, src_dir, inst_dir
) -> None:
super(NopBuilder, self).__init__( super(NopBuilder, self).__init__(
build_opts, ctx, manifest, src_dir, None, inst_dir loader, dep_manifests, build_opts, ctx, manifest, src_dir, None, inst_dir
) )
def build(self, install_dirs, reconfigure: bool) -> None: def build(self, reconfigure: bool) -> None:
print("Installing %s -> %s" % (self.src_dir, self.inst_dir)) print("Installing %s -> %s" % (self.src_dir, self.inst_dir))
parent = os.path.dirname(self.inst_dir) parent = os.path.dirname(self.inst_dir)
if not os.path.exists(parent): if not os.path.exists(parent):
@@ -1147,12 +1235,29 @@ class NopBuilder(BuilderBase):
class SqliteBuilder(BuilderBase): class SqliteBuilder(BuilderBase):
def __init__(self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir) -> None: def __init__(
self,
loader,
dep_manifests,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
) -> None:
super(SqliteBuilder, self).__init__( super(SqliteBuilder, self).__init__(
build_opts, ctx, manifest, src_dir, build_dir, inst_dir loader,
dep_manifests,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
) )
def _build(self, install_dirs, reconfigure) -> None: def _build(self, reconfigure) -> None:
for f in ["sqlite3.c", "sqlite3.h", "sqlite3ext.h"]: for f in ["sqlite3.c", "sqlite3.h", "sqlite3ext.h"]:
src = os.path.join(self.src_dir, f) src = os.path.join(self.src_dir, f)
dest = os.path.join(self.build_dir, f) dest = os.path.join(self.build_dir, f)
@@ -1191,7 +1296,7 @@ install(FILES sqlite3.h sqlite3ext.h DESTINATION include)
define_args = ["-D%s=%s" % (k, v) for (k, v) in defines.items()] define_args = ["-D%s=%s" % (k, v) for (k, v) in defines.items()]
define_args += ["-G", "Ninja"] define_args += ["-G", "Ninja"]
env = self._compute_env(install_dirs) env = self._compute_env()
# Resolve the cmake that we installed # Resolve the cmake that we installed
cmake = path_search(env, "cmake") cmake = path_search(env, "cmake")

34
build/fbcode_builder/getdeps/cargo.py
View File

@@ -20,6 +20,8 @@ if typing.TYPE_CHECKING:
class CargoBuilder(BuilderBase): class CargoBuilder(BuilderBase):
def __init__( def __init__(
self, self,
loader,
dep_manifests, # manifests of dependencies
build_opts: "BuildOptions", build_opts: "BuildOptions",
ctx, ctx,
manifest, manifest,
@@ -29,11 +31,17 @@ class CargoBuilder(BuilderBase):
build_doc, build_doc,
workspace_dir, workspace_dir,
manifests_to_build, manifests_to_build,
loader,
cargo_config_file, cargo_config_file,
) -> None: ) -> None:
super(CargoBuilder, self).__init__( super(CargoBuilder, self).__init__(
build_opts, ctx, manifest, src_dir, build_dir, inst_dir loader,
dep_manifests,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
) )
self.build_doc = build_doc self.build_doc = build_doc
self.ws_dir = workspace_dir self.ws_dir = workspace_dir
@@ -43,7 +51,7 @@ class CargoBuilder(BuilderBase):
def run_cargo(self, install_dirs, operation, args=None) -> None: def run_cargo(self, install_dirs, operation, args=None) -> None:
args = args or [] args = args or []
env = self._compute_env(install_dirs) env = self._compute_env()
# Enable using nightly features with stable compiler # Enable using nightly features with stable compiler
env["RUSTC_BOOTSTRAP"] = "1" env["RUSTC_BOOTSTRAP"] = "1"
env["LIBZ_SYS_STATIC"] = "1" env["LIBZ_SYS_STATIC"] = "1"
@@ -138,7 +146,7 @@ incremental = false
return dep_to_git return dep_to_git
def _prepare(self, install_dirs, reconfigure) -> None: def _prepare(self, reconfigure) -> None:
build_source_dir = self.build_source_dir() build_source_dir = self.build_source_dir()
self.recreate_dir(self.src_dir, build_source_dir) self.recreate_dir(self.src_dir, build_source_dir)
@@ -147,7 +155,7 @@ incremental = false
if self.ws_dir is not None: if self.ws_dir is not None:
self._patchup_workspace(dep_to_git) self._patchup_workspace(dep_to_git)
def _build(self, install_dirs, reconfigure) -> None: def _build(self, reconfigure) -> None:
# _prepare has been run already. Actually do the build # _prepare has been run already. Actually do the build
build_source_dir = self.build_source_dir() build_source_dir = self.build_source_dir()
@@ -162,14 +170,14 @@ incremental = false
if self.manifests_to_build is None: if self.manifests_to_build is None:
self.run_cargo( self.run_cargo(
install_dirs, self.install_dirs,
"build", "build",
build_args, build_args,
) )
else: else:
for manifest in self.manifests_to_build: for manifest in self.manifests_to_build:
self.run_cargo( self.run_cargo(
install_dirs, self.install_dirs,
"build", "build",
build_args build_args
+ [ + [
@@ -180,24 +188,22 @@ incremental = false
self.recreate_dir(build_source_dir, os.path.join(self.inst_dir, "source")) self.recreate_dir(build_source_dir, os.path.join(self.inst_dir, "source"))
def run_tests( def run_tests(self, schedule_type, owner, test_filter, retry, no_testpilot) -> None:
self, install_dirs, schedule_type, owner, test_filter, retry, no_testpilot
) -> None:
if test_filter: if test_filter:
args = ["--", test_filter] args = ["--", test_filter]
else: else:
args = [] args = []
if self.manifests_to_build is None: if self.manifests_to_build is None:
self.run_cargo(install_dirs, "test", args) self.run_cargo(self.install_dirs, "test", args)
if self.build_doc: if self.build_doc:
self.run_cargo(install_dirs, "doc", ["--no-deps"]) self.run_cargo(self.install_dirs, "doc", ["--no-deps"])
else: else:
for manifest in self.manifests_to_build: for manifest in self.manifests_to_build:
margs = ["--manifest-path", self.manifest_dir(manifest)] margs = ["--manifest-path", self.manifest_dir(manifest)]
self.run_cargo(install_dirs, "test", args + margs) self.run_cargo(self.install_dirs, "test", args + margs)
if self.build_doc: if self.build_doc:
self.run_cargo(install_dirs, "doc", ["--no-deps"] + margs) self.run_cargo(self.install_dirs, "doc", ["--no-deps"] + margs)
def _patchup_workspace(self, dep_to_git) -> None: def _patchup_workspace(self, dep_to_git) -> None:
""" """

8
build/fbcode_builder/getdeps/load.py
View File

@@ -159,6 +159,14 @@ class ManifestLoader(object):
return self.manifests_by_name return self.manifests_by_name
def dependencies_of(self, manifest):
"""Returns the dependencies of the given project, not including the project itself, in topological order."""
return [
dep
for dep in self.manifests_in_dependency_order(manifest)
if dep != manifest
]
def manifests_in_dependency_order(self, manifest=None): def manifests_in_dependency_order(self, manifest=None):
"""Compute all dependencies of the specified project. Returns a list of the """Compute all dependencies of the specified project. Returns a list of the
dependencies plus the project itself, in topologically sorted order. dependencies plus the project itself, in topologically sorted order.

103
build/fbcode_builder/getdeps/manifest.py
View File

@@ -485,6 +485,7 @@ class ManifestParser(object):
inst_dir, inst_dir,
ctx, ctx,
loader, loader,
dep_manifests,
final_install_prefix=None, final_install_prefix=None,
extra_cmake_defines=None, extra_cmake_defines=None,
cmake_target=None, cmake_target=None,
@@ -508,6 +509,8 @@ class ManifestParser(object):
test_args = self.get_section_as_args("make.test_args", ctx) test_args = self.get_section_as_args("make.test_args", ctx)
if builder == "cmakebootstrap": if builder == "cmakebootstrap":
return CMakeBootStrapBuilder( return CMakeBootStrapBuilder(
loader,
dep_manifests,
build_options, build_options,
ctx, ctx,
self, self,
@@ -520,6 +523,8 @@ class ManifestParser(object):
) )
else: else:
return MakeBuilder( return MakeBuilder(
loader,
dep_manifests,
build_options, build_options,
ctx, ctx,
self, self,
@@ -538,6 +543,8 @@ class ManifestParser(object):
if ldflags_cmd: if ldflags_cmd:
conf_env_args["LDFLAGS"] = ldflags_cmd conf_env_args["LDFLAGS"] = ldflags_cmd
return AutoconfBuilder( return AutoconfBuilder(
loader,
dep_manifests,
build_options, build_options,
ctx, ctx,
self, self,
@@ -552,11 +559,23 @@ class ManifestParser(object):
args = self.get_section_as_args("b2.args", ctx) args = self.get_section_as_args("b2.args", ctx)
if extra_b2_args is not None: if extra_b2_args is not None:
args += extra_b2_args args += extra_b2_args
return Boost(build_options, ctx, self, src_dir, build_dir, inst_dir, args) return Boost(
loader,
dep_manifests,
build_options,
ctx,
self,
src_dir,
build_dir,
inst_dir,
args,
)
if builder == "cmake": if builder == "cmake":
defines = self.get_section_as_dict("cmake.defines", ctx) defines = self.get_section_as_dict("cmake.defines", ctx)
return CMakeBuilder( return CMakeBuilder(
loader,
dep_manifests,
build_options, build_options,
ctx, ctx,
self, self,
@@ -564,7 +583,6 @@ class ManifestParser(object):
build_dir, build_dir,
inst_dir, inst_dir,
defines, defines,
loader,
final_install_prefix, final_install_prefix,
extra_cmake_defines, extra_cmake_defines,
cmake_target, cmake_target,
@@ -572,39 +590,91 @@ class ManifestParser(object):
if builder == "python-wheel": if builder == "python-wheel":
return PythonWheelBuilder( return PythonWheelBuilder(
build_options, ctx, self, src_dir, build_dir, inst_dir loader,
dep_manifests,
build_options,
ctx,
self,
src_dir,
build_dir,
inst_dir,
) )
if builder == "sqlite": if builder == "sqlite":
return SqliteBuilder(build_options, ctx, self, src_dir, build_dir, inst_dir) return SqliteBuilder(
loader,
dep_manifests,
build_options,
ctx,
self,
src_dir,
build_dir,
inst_dir,
)
if builder == "ninja_bootstrap": if builder == "ninja_bootstrap":
return NinjaBootstrap( return NinjaBootstrap(
build_options, ctx, self, build_dir, src_dir, inst_dir loader,
dep_manifests,
build_options,
ctx,
self,
build_dir,
src_dir,
inst_dir,
) )
if builder == "nop": if builder == "nop":
return NopBuilder(build_options, ctx, self, src_dir, inst_dir) return NopBuilder(
loader, dep_manifests, build_options, ctx, self, src_dir, inst_dir
)
if builder == "openssl": if builder == "openssl":
return OpenSSLBuilder( return OpenSSLBuilder(
build_options, ctx, self, build_dir, src_dir, inst_dir loader,
dep_manifests,
build_options,
ctx,
self,
build_dir,
src_dir,
inst_dir,
) )
if builder == "iproute2": if builder == "iproute2":
return Iproute2Builder( return Iproute2Builder(
build_options, ctx, self, src_dir, build_dir, inst_dir loader,
dep_manifests,
build_options,
ctx,
self,
src_dir,
build_dir,
inst_dir,
) )
if builder == "cargo": if builder == "cargo":
return self.create_cargo_builder( return self.create_cargo_builder(
build_options, ctx, src_dir, build_dir, inst_dir, loader loader,
dep_manifests,
build_options,
ctx,
src_dir,
build_dir,
inst_dir,
) )
raise KeyError("project %s has no known builder" % (self.name)) raise KeyError("project %s has no known builder" % (self.name))
def create_prepare_builders( def create_prepare_builders(
self, build_options, ctx, src_dir, build_dir, inst_dir, loader self,
build_options,
ctx,
src_dir,
build_dir,
inst_dir,
loader,
dep_manifests,
): ):
"""Create builders that have a prepare step run, e.g. to write config files""" """Create builders that have a prepare step run, e.g. to write config files"""
prepare_builders = [] prepare_builders = []
@@ -612,19 +682,27 @@ class ManifestParser(object):
cargo = self.get_section_as_dict("cargo", ctx) cargo = self.get_section_as_dict("cargo", ctx)
if not builder == "cargo" and cargo: if not builder == "cargo" and cargo:
cargo_builder = self.create_cargo_builder( cargo_builder = self.create_cargo_builder(
build_options, ctx, src_dir, build_dir, inst_dir, loader loader,
dep_manifests,
build_options,
ctx,
src_dir,
build_dir,
inst_dir,
) )
prepare_builders.append(cargo_builder) prepare_builders.append(cargo_builder)
return prepare_builders return prepare_builders
def create_cargo_builder( def create_cargo_builder(
self, build_options, ctx, src_dir, build_dir, inst_dir, loader self, loader, dep_manifests, build_options, ctx, src_dir, build_dir, inst_dir
): ):
build_doc = self.get("cargo", "build_doc", False, ctx) build_doc = self.get("cargo", "build_doc", False, ctx)
workspace_dir = self.get("cargo", "workspace_dir", None, ctx) workspace_dir = self.get("cargo", "workspace_dir", None, ctx)
manifests_to_build = self.get("cargo", "manifests_to_build", None, ctx) manifests_to_build = self.get("cargo", "manifests_to_build", None, ctx)
cargo_config_file = self.get("cargo", "cargo_config_file", None, ctx) cargo_config_file = self.get("cargo", "cargo_config_file", None, ctx)
return CargoBuilder( return CargoBuilder(
loader,
dep_manifests,
build_options, build_options,
ctx, ctx,
self, self,
@@ -634,7 +712,6 @@ class ManifestParser(object):
build_doc, build_doc,
workspace_dir, workspace_dir,
manifests_to_build, manifests_to_build,
loader,
cargo_config_file, cargo_config_file,
) )

11
build/fbcode_builder/getdeps/py_wheel_builder.py
View File

@@ -104,7 +104,7 @@ class PythonWheelBuilder(BuilderBase):
dist_info_dir: str dist_info_dir: str
template_format_dict: Dict[str, str] template_format_dict: Dict[str, str]
def _build(self, install_dirs: List[str], reconfigure: bool) -> None: def _build(self, reconfigure: bool) -> None:
# When we are invoked, self.src_dir contains the unpacked wheel contents. # When we are invoked, self.src_dir contains the unpacked wheel contents.
# #
# Since a wheel file is just a zip file, the Fetcher code recognizes it as such # Since a wheel file is just a zip file, the Fetcher code recognizes it as such
@@ -171,10 +171,12 @@ class PythonWheelBuilder(BuilderBase):
self._write_cmake_config_template() self._write_cmake_config_template()
# Run the build # Run the build
self._run_cmake_build(install_dirs, reconfigure) self._run_cmake_build(reconfigure)
def _run_cmake_build(self, install_dirs: List[str], reconfigure: bool) -> None: def _run_cmake_build(self, reconfigure: bool) -> None:
cmake_builder = CMakeBuilder( cmake_builder = CMakeBuilder(
loader=self.loader,
dep_manifests=self.dep_manifests,
build_opts=self.build_opts, build_opts=self.build_opts,
ctx=self.ctx, ctx=self.ctx,
manifest=self.manifest, manifest=self.manifest,
@@ -183,11 +185,10 @@ class PythonWheelBuilder(BuilderBase):
src_dir=self.build_dir, src_dir=self.build_dir,
build_dir=self.build_dir, build_dir=self.build_dir,
inst_dir=self.inst_dir, inst_dir=self.inst_dir,
loader=None,
defines={}, defines={},
final_install_prefix=None, final_install_prefix=None,
) )
cmake_builder.build(install_dirs=install_dirs, reconfigure=reconfigure) cmake_builder.build(reconfigure=reconfigure)
def _write_cmakelists(self, path_mapping: Dict[str, str], dependencies) -> None: def _write_cmakelists(self, path_mapping: Dict[str, str], dependencies) -> None:
cmake_path = os.path.join(self.build_dir, "CMakeLists.txt") cmake_path = os.path.join(self.build_dir, "CMakeLists.txt")