lib/proxygen
1
0
Fork 0
mirror of https://github.com/facebook/proxygen.git synced 2025-08-05 19:55:47 +03:00
Code Activity

fix(fbcode_builder): fail the build when cmake build fails

Browse Source 
Summary:
This change causes fbcode_builder (getdeps.py) to fail immediately on most non-zero error codes from delegated commands.

While internal tools may be able to surface failures in a build that doesn't fail fast, the debug cycle is difficult externally -- this makes failures pop out more easily and has been instrumental in debugging OSS build issues.

X-link: https://github.com/facebook/sapling/pull/1099

Differential Revision: D77608371

Pulled By: quark-zju

fbshipit-source-id: e9fc00a574bc64fbc165a22e51406e85d015e2ae
This commit is contained in:
ben--
2025-07-03 14:28:13 -07:00
committed by Facebook GitHub Bot
parent 4f0db407cf
commit 603a3a6686
4 changed files with 51 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
build/fbcode_builder/getdeps.py
View File

@@ -28,7 +28,7 @@ from getdeps.fetcher import (
from getdeps.load import ManifestLoader from getdeps.load import ManifestLoader
from getdeps.manifest import ManifestParser from getdeps.manifest import ManifestParser
from getdeps.platform import HostType from getdeps.platform import HostType
from getdeps.runcmd import run_cmd from getdeps.runcmd import check_cmd
from getdeps.subcmd import add_subcommands, cmd, SubCmd from getdeps.subcmd import add_subcommands, cmd, SubCmd
try: try:
@@ -472,7 +472,7 @@ class InstallSysDepsCmd(ProjectCmdBase):
if args.dry_run: if args.dry_run:
print(" ".join(cmd_args)) print(" ".join(cmd_args))
else: else:
run_cmd(cmd_args) check_cmd(cmd_args)
else: else:
print("no packages to install") print("no packages to install")

73
build/fbcode_builder/getdeps/builder.py
View File

@@ -85,6 +85,12 @@ class BuilderBase(object):
return [wrapper, "&&"] return [wrapper, "&&"]
return [] return []
def _check_cmd(self, cmd, **kwargs) -> None:
"""Run the command and abort on failure"""
rc = self._run_cmd(cmd, **kwargs)
if rc != 0:
raise RuntimeError(f"Failure exit code {rc} for command {cmd}")
def _run_cmd( def _run_cmd(
self, self,
cmd, cmd,
@@ -324,10 +330,10 @@ class MakeBuilder(BuilderBase):
+ self.build_args + self.build_args
+ self._get_prefix() + self._get_prefix()
) )
self._run_cmd(cmd, env=env) self._check_cmd(cmd, env=env)
install_cmd = [self._make_binary] + self.install_args + self._get_prefix() install_cmd = [self._make_binary] + self.install_args + self._get_prefix()
self._run_cmd(install_cmd, env=env) self._check_cmd(install_cmd, env=env)
# bz2's Makefile doesn't install its .so properly # bz2's Makefile doesn't install its .so properly
if self.manifest and self.manifest.name == "bz2": if self.manifest and self.manifest.name == "bz2":
@@ -357,12 +363,12 @@ class MakeBuilder(BuilderBase):
+ self.test_args + self.test_args
+ self._get_prefix() + self._get_prefix()
) )
self._run_cmd(cmd, allow_fail=False, env=env) self._check_cmd(cmd, allow_fail=False, env=env)
class CMakeBootStrapBuilder(MakeBuilder): class CMakeBootStrapBuilder(MakeBuilder):
def _build(self, reconfigure) -> None: def _build(self, reconfigure) -> None:
self._run_cmd( self._check_cmd(
[ [
"./bootstrap", "./bootstrap",
"--prefix=" + self.inst_dir, "--prefix=" + self.inst_dir,
@@ -426,21 +432,21 @@ class AutoconfBuilder(BuilderBase):
# seem to realize that it should invoke libtoolize and then # seem to realize that it should invoke libtoolize and then
# error out when the configure script references a libtool # error out when the configure script references a libtool
# related symbol. # related symbol.
self._run_cmd(["libtoolize"], cwd=self.src_dir, env=env) self._check_cmd(["libtoolize"], cwd=self.src_dir, env=env)
# We generally prefer to call the `autogen.sh` script provided # We generally prefer to call the `autogen.sh` script provided
# by the project on the basis that it may know more than plain # by the project on the basis that it may know more than plain
# autoreconf does. # autoreconf does.
if os.path.exists(autogen_path): if os.path.exists(autogen_path):
self._run_cmd(["bash", autogen_path], cwd=self.src_dir, env=env) self._check_cmd(["bash", autogen_path], cwd=self.src_dir, env=env)
else: else:
self._run_cmd(["autoreconf", "-ivf"], cwd=self.src_dir, env=env) self._check_cmd(["autoreconf", "-ivf"], cwd=self.src_dir, env=env)
configure_cmd = [configure_path, "--prefix=" + self.inst_dir] + self.args configure_cmd = [configure_path, "--prefix=" + self.inst_dir] + self.args
self._run_cmd(configure_cmd, env=env) self._check_cmd(configure_cmd, env=env)
only_install = self.manifest.get("build", "only_install", "false", ctx=self.ctx) only_install = self.manifest.get("build", "only_install", "false", ctx=self.ctx)
if not only_install: if not only_install:
self._run_cmd([self._make_binary, "-j%s" % self.num_jobs], env=env) self._check_cmd([self._make_binary, "-j%s" % self.num_jobs], env=env)
self._run_cmd([self._make_binary, "install"], env=env) self._check_cmd([self._make_binary, "install"], env=env)
class Iproute2Builder(BuilderBase): class Iproute2Builder(BuilderBase):
@@ -473,10 +479,10 @@ class Iproute2Builder(BuilderBase):
def _build(self, reconfigure) -> None: def _build(self, reconfigure) -> None:
configure_path = os.path.join(self.src_dir, "configure") configure_path = os.path.join(self.src_dir, "configure")
env = self.env.copy() env = self.env.copy()
self._run_cmd([configure_path], env=env) self._check_cmd([configure_path], env=env)
shutil.rmtree(self.build_dir) shutil.rmtree(self.build_dir)
shutil.copytree(self.src_dir, self.build_dir) shutil.copytree(self.src_dir, self.build_dir)
self._run_cmd(["make", "-j%s" % self.num_jobs], env=env) self._check_cmd(["make", "-j%s" % self.num_jobs], env=env)
install_cmd = ["make", "install", "DESTDIR=" + self.inst_dir] install_cmd = ["make", "install", "DESTDIR=" + self.inst_dir]
for d in ["include", "lib"]: for d in ["include", "lib"]:
@@ -485,7 +491,7 @@ class Iproute2Builder(BuilderBase):
os.path.join(self.build_dir, d), os.path.join(self.inst_dir, d) os.path.join(self.build_dir, d), os.path.join(self.inst_dir, d)
) )
self._run_cmd(install_cmd, env=env) self._check_cmd(install_cmd, env=env)
class SystemdBuilder(BuilderBase): class SystemdBuilder(BuilderBase):
@@ -522,7 +528,7 @@ class SystemdBuilder(BuilderBase):
# Meson builds typically require setup, compile, and install steps. # Meson builds typically require setup, compile, and install steps.
# During this setup step we ensure that the static library is built and # During this setup step we ensure that the static library is built and
# the prefix is empty. # the prefix is empty.
self._run_cmd( self._check_cmd(
[ [
meson, meson,
"setup", "setup",
@@ -535,10 +541,10 @@ class SystemdBuilder(BuilderBase):
# Compile step needs to satisfy the build directory that was previously # Compile step needs to satisfy the build directory that was previously
# prepared during setup. # prepared during setup.
self._run_cmd([meson, "compile", "-C", self.build_dir]) self._check_cmd([meson, "compile", "-C", self.build_dir])
# Install step # Install step
self._run_cmd( self._check_cmd(
[meson, "install", "-C", self.build_dir, "--destdir", self.inst_dir] [meson, "install", "-C", self.build_dir, "--destdir", self.inst_dir]
) )
@@ -862,9 +868,9 @@ if __name__ == "__main__":
) )
self._invalidate_cache() self._invalidate_cache()
self._run_cmd([cmake, self.src_dir] + define_args, env=env) self._check_cmd([cmake, self.src_dir] + define_args, env=env)
self._run_cmd( self._check_cmd(
[ [
cmake, cmake,
"--build", "--build",
@@ -1065,6 +1071,7 @@ if __name__ == "__main__":
runs.append([]) runs.append([])
for run in runs: for run in runs:
# FIXME: What is this trying to accomplish? Should it fail on first or >=1 errors?
self._run_cmd( self._run_cmd(
testpilot_args + run, testpilot_args + run,
cwd=self.build_opts.fbcode_builder_dir, cwd=self.build_opts.fbcode_builder_dir,
@@ -1082,8 +1089,10 @@ if __name__ == "__main__":
args += ["-R", test_filter] args += ["-R", test_filter]
count = 0 count = 0
retcode = -1
while count <= retry: while count <= retry:
retcode = self._run_cmd( # FIXME: What is this trying to accomplish? Should it fail on first or >=1 errors?
retcode = self._check_cmd(
args, env=env, use_cmd_prefix=use_cmd_prefix, allow_fail=True args, env=env, use_cmd_prefix=use_cmd_prefix, allow_fail=True
) )
@@ -1093,11 +1102,9 @@ if __name__ == "__main__":
# Only add this option in the second run. # Only add this option in the second run.
args += ["--rerun-failed"] args += ["--rerun-failed"]
count += 1 count += 1
# pyre-fixme[61]: `retcode` is undefined, or not always defined. if retcode is not None and retcode != 0:
if retcode != 0:
# Allow except clause in getdeps.main to catch and exit gracefully # Allow except clause in getdeps.main to catch and exit gracefully
# This allows non-testpilot runs to fail through the same logic as failed testpilot runs, which may become handy in case if post test processing is needed in the future # This allows non-testpilot runs to fail through the same logic as failed testpilot runs, which may become handy in case if post test processing is needed in the future
# pyre-fixme[61]: `retcode` is undefined, or not always defined.
raise subprocess.CalledProcessError(retcode, args) raise subprocess.CalledProcessError(retcode, args)
@@ -1125,7 +1132,9 @@ class NinjaBootstrap(BuilderBase):
) )
def _build(self, reconfigure) -> None: def _build(self, reconfigure) -> None:
self._run_cmd([sys.executable, "configure.py", "--bootstrap"], cwd=self.src_dir) self._check_cmd(
[sys.executable, "configure.py", "--bootstrap"], cwd=self.src_dir
)
src_ninja = os.path.join(self.src_dir, "ninja") src_ninja = os.path.join(self.src_dir, "ninja")
dest_ninja = os.path.join(self.inst_dir, "bin/ninja") dest_ninja = os.path.join(self.inst_dir, "bin/ninja")
bin_dir = os.path.dirname(dest_ninja) bin_dir = os.path.dirname(dest_ninja)
@@ -1197,7 +1206,7 @@ class OpenSSLBuilder(BuilderBase):
else: else:
raise Exception("don't know how to build openssl for %r" % self.ctx) raise Exception("don't know how to build openssl for %r" % self.ctx)
self._run_cmd( self._check_cmd(
[ [
perl, perl,
configure, configure,
@@ -1215,11 +1224,11 @@ class OpenSSLBuilder(BuilderBase):
+ extra_args + extra_args
) )
# show the config produced # show the config produced
self._run_cmd([perl, "configdata.pm", "--dump"], env=env) self._check_cmd([perl, "configdata.pm", "--dump"], env=env)
make_build = [make] + make_j_args make_build = [make] + make_j_args
self._run_cmd(make_build, env=env) self._check_cmd(make_build, env=env)
make_install = [make, "install_sw", "install_ssldirs"] make_install = [make, "install_sw", "install_ssldirs"]
self._run_cmd(make_install, env=env) self._check_cmd(make_install, env=env)
class Boost(BuilderBase): class Boost(BuilderBase):
@@ -1272,18 +1281,18 @@ class Boost(BuilderBase):
) )
if self.build_opts.is_windows(): if self.build_opts.is_windows():
bootstrap = os.path.join(self.src_dir, "bootstrap.bat") bootstrap = os.path.join(self.src_dir, "bootstrap.bat")
self._run_cmd([bootstrap] + bootstrap_args, cwd=self.src_dir, env=env) self._check_cmd([bootstrap] + bootstrap_args, cwd=self.src_dir, env=env)
args += ["address-model=64"] args += ["address-model=64"]
else: else:
bootstrap = os.path.join(self.src_dir, "bootstrap.sh") bootstrap = os.path.join(self.src_dir, "bootstrap.sh")
self._run_cmd( self._check_cmd(
[bootstrap, "--prefix=%s" % self.inst_dir] + bootstrap_args, [bootstrap, "--prefix=%s" % self.inst_dir] + bootstrap_args,
cwd=self.src_dir, cwd=self.src_dir,
env=env, env=env,
) )
b2 = os.path.join(self.src_dir, "b2") b2 = os.path.join(self.src_dir, "b2")
self._run_cmd( self._check_cmd(
[ [
b2, b2,
"-j%s" % self.num_jobs, "-j%s" % self.num_jobs,
@@ -1418,8 +1427,8 @@ install(FILES sqlite3.h sqlite3ext.h DESTINATION include)
# Resolve the cmake that we installed # Resolve the cmake that we installed
cmake = path_search(env, "cmake") cmake = path_search(env, "cmake")
self._run_cmd([cmake, self.build_dir] + define_args, env=env) self._check_cmd([cmake, self.build_dir] + define_args, env=env)
self._run_cmd( self._check_cmd(
[ [
cmake, cmake,
"--build", "--build",

2
build/fbcode_builder/getdeps/cargo.py
View File

@@ -63,7 +63,7 @@ class CargoBuilder(BuilderBase):
"--workspace", "--workspace",
"-j%s" % self.num_jobs, "-j%s" % self.num_jobs,
] + args ] + args
self._run_cmd(cmd, cwd=self.workspace_dir(), env=env) self._check_cmd(cmd, cwd=self.workspace_dir(), env=env)
def build_source_dir(self): def build_source_dir(self):
return os.path.join(self.build_dir, "source") return os.path.join(self.build_dir, "source")

7
build/fbcode_builder/getdeps/runcmd.py
View File

@@ -42,6 +42,13 @@ def _print_env_diff(env, log_fn) -> None:
log_fn("+ %s=%s \\\n" % (k, shellquote(env[k]))) log_fn("+ %s=%s \\\n" % (k, shellquote(env[k])))
def check_cmd(cmd, **kwargs) -> None:
"""Run the command and abort on failure"""
rc = run_cmd(cmd, **kwargs)
if rc != 0:
raise RuntimeError(f"Failure exit code {rc} for command {cmd}")
def run_cmd(cmd, env=None, cwd=None, allow_fail: bool = False, log_file=None) -> int: def run_cmd(cmd, env=None, cwd=None, allow_fail: bool = False, log_file=None) -> int:
def log_to_stdout(msg): def log_to_stdout(msg):
sys.stdout.buffer.write(msg.encode(errors="surrogateescape")) sys.stdout.buffer.write(msg.encode(errors="surrogateescape"))