diff --git a/build/fbcode_builder/getdeps/builder.py b/build/fbcode_builder/getdeps/builder.py index 1baa497a7..75af982be 100644 --- a/build/fbcode_builder/getdeps/builder.py +++ b/build/fbcode_builder/getdeps/builder.py @@ -198,11 +198,22 @@ class CMakeBootStrapBuilder(MakeBuilder): class AutoconfBuilder(BuilderBase): - def __init__(self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir, args): + def __init__( + self, + build_opts, + ctx, + manifest, + src_dir, + build_dir, + inst_dir, + args, + conf_env_args, + ): super(AutoconfBuilder, self).__init__( build_opts, ctx, manifest, src_dir, build_dir, inst_dir ) self.args = args or [] + self.conf_env_args = conf_env_args or {} def _build(self, install_dirs, reconfigure): configure_path = os.path.join(self.src_dir, "configure") @@ -210,6 +221,16 @@ class AutoconfBuilder(BuilderBase): env = self._compute_env(install_dirs) + # Some configure scripts need additional env values passed derived from cmds + for (k, cmd_args) in self.conf_env_args.items(): + out = ( + subprocess.check_output(cmd_args, env=dict(env.items())) + .decode("utf-8") + .strip() + ) + if out: + env.set(k, out) + if not os.path.exists(configure_path): print("%s doesn't exist, so reconfiguring" % configure_path) # This libtoolize call is a bit gross; the issue is that diff --git a/build/fbcode_builder/getdeps/buildopts.py b/build/fbcode_builder/getdeps/buildopts.py index fc6e544ae..02fe8ccb2 100644 --- a/build/fbcode_builder/getdeps/buildopts.py +++ b/build/fbcode_builder/getdeps/buildopts.py @@ -233,33 +233,29 @@ class BuildOptions(object): for d in install_dirs: bindir = os.path.join(d, "bin") - if not ( - manifest and manifest.get("build", "disable_env_override_pkgconfig") - ): - pkgconfig = os.path.join(d, "lib/pkgconfig") - if os.path.exists(pkgconfig): - add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig) + pkgconfig = os.path.join(d, "lib/pkgconfig") + if os.path.exists(pkgconfig): + add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig) - pkgconfig = os.path.join(d, "lib64/pkgconfig") - if os.path.exists(pkgconfig): - add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig) + pkgconfig = os.path.join(d, "lib64/pkgconfig") + if os.path.exists(pkgconfig): + add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig) - if not (manifest and manifest.get("build", "disable_env_override_path")): - add_path_entry(env, "CMAKE_PREFIX_PATH", d) + add_path_entry(env, "CMAKE_PREFIX_PATH", d) - # Allow resolving shared objects built earlier (eg: zstd - # doesn't include the full path to the dylib in its linkage - # so we need to give it an assist) - if lib_path: - for lib in ["lib", "lib64"]: - libdir = os.path.join(d, lib) - if os.path.exists(libdir): - add_path_entry(env, lib_path, libdir) + # Allow resolving shared objects built earlier (eg: zstd + # doesn't include the full path to the dylib in its linkage + # so we need to give it an assist) + if lib_path: + for lib in ["lib", "lib64"]: + libdir = os.path.join(d, lib) + if os.path.exists(libdir): + add_path_entry(env, lib_path, libdir) - # Allow resolving binaries (eg: cmake, ninja) and dlls - # built by earlier steps - if os.path.exists(bindir): - add_path_entry(env, "PATH", bindir, append=False) + # Allow resolving binaries (eg: cmake, ninja) and dlls + # built by earlier steps + if os.path.exists(bindir): + add_path_entry(env, "PATH", bindir, append=False) # If rustc is present in the `bin` directory, set RUSTC to prevent # cargo uses the rustc installed in the system. diff --git a/build/fbcode_builder/getdeps/manifest.py b/build/fbcode_builder/getdeps/manifest.py index f69375047..06bf82c0d 100644 --- a/build/fbcode_builder/getdeps/manifest.py +++ b/build/fbcode_builder/getdeps/manifest.py @@ -69,8 +69,6 @@ SCHEMA = { "builder": REQUIRED, "subdir": OPTIONAL, "build_in_src_dir": OPTIONAL, - "disable_env_override_pkgconfig": OPTIONAL, - "disable_env_override_path": OPTIONAL, }, }, "msbuild": {"optional_section": True, "fields": {"project": REQUIRED}}, @@ -84,6 +82,7 @@ SCHEMA = { }, "cmake.defines": {"optional_section": True}, "autoconf.args": {"optional_section": True}, + "autoconf.envcmd.LDFLAGS": {"optional_section": True}, "rpms": {"optional_section": True}, "debs": {"optional_section": True}, "preinstalled.env": {"optional_section": True}, @@ -101,6 +100,7 @@ SCHEMA = { # using the expression syntax to enable/disable sections ALLOWED_EXPR_SECTIONS = [ "autoconf.args", + "autoconf.envcmd.LDFLAGS", "build", "cmake.defines", "dependencies", @@ -469,8 +469,19 @@ class ManifestParser(object): if builder == "autoconf": args = self.get_section_as_args("autoconf.args", ctx) + conf_env_args = {} + ldflags_cmd = self.get_section_as_args("autoconf.envcmd.LDFLAGS", ctx) + if ldflags_cmd: + conf_env_args["LDFLAGS"] = ldflags_cmd return AutoconfBuilder( - build_options, ctx, self, src_dir, build_dir, inst_dir, args + build_options, + ctx, + self, + src_dir, + build_dir, + inst_dir, + args, + conf_env_args, ) if builder == "boost": diff --git a/build/fbcode_builder/manifests/eden_scm b/build/fbcode_builder/manifests/eden_scm index ad5ce3049..57c5b4a66 100644 --- a/build/fbcode_builder/manifests/eden_scm +++ b/build/fbcode_builder/manifests/eden_scm @@ -10,8 +10,6 @@ repo_url = https://github.com/facebookexperimental/eden.git [build.os=linux] builder = make subdir = eden/scm -disable_env_override_pkgconfig = 1 -disable_env_override_path = 1 [build.os=windows] # For now the biggest blocker is missing "make" on windows, but there are bound @@ -53,6 +51,7 @@ fbcode/fboss/common = common [dependencies] fb303-source fbthrift-source +python rust-shed # We use the system openssl on linux diff --git a/build/fbcode_builder/manifests/fboss b/build/fbcode_builder/manifests/fboss index f29873e72..702cfe0c1 100644 --- a/build/fbcode_builder/manifests/fboss +++ b/build/fbcode_builder/manifests/fboss @@ -31,7 +31,7 @@ libnl libsai OpenNSA re2 -python +python_3_7 yaml-cpp libyaml CLI11 diff --git a/build/fbcode_builder/manifests/python b/build/fbcode_builder/manifests/python index e51c0ab51..55525c3e9 100644 --- a/build/fbcode_builder/manifests/python +++ b/build/fbcode_builder/manifests/python @@ -8,10 +8,27 @@ python3-devel [debs] python3-all-dev -[download.os=linux] -url = https://www.python.org/ftp/python/3.7.6/Python-3.7.6.tgz -sha256 = aeee681c235ad336af116f08ab6563361a0c81c537072c1b309d6e4050aa2114 +[download] +url = https://www.python.org/ftp/python/3.8.12/Python-3.8.12.tgz +sha256 = 316aa33f3b7707d041e73f246efedb297a70898c4b91f127f66dc8d80c596f1a -[build.os=linux] +[build] builder = autoconf -subdir = Python-3.7.6 +subdir = Python-3.8.12 + +[autoconf.args] +--enable-shared +--with-ensurepip=install + +# python's pkg-config libffi detection is broken +# See https://bugs.python.org/issue34823 for clearest description +# and pending PR https://github.com/python/cpython/pull/20451 +# The documented workaround requires an environment variable derived from +# pkg-config to be passed into its configure step +[autoconf.envcmd.LDFLAGS] +pkg-config +--libs-only-L +libffi + +[dependencies] +libffi diff --git a/build/fbcode_builder/manifests/python_3_7 b/build/fbcode_builder/manifests/python_3_7 new file mode 100644 index 000000000..ac8f7b3d3 --- /dev/null +++ b/build/fbcode_builder/manifests/python_3_7 @@ -0,0 +1,17 @@ +[manifest] +name = python_3_7 + +[rpms] +python3 +python3-devel + +[debs] +python3-all-dev + +[download.os=linux] +url = https://www.python.org/ftp/python/3.7.6/Python-3.7.6.tgz +sha256 = aeee681c235ad336af116f08ab6563361a0c81c537072c1b309d6e4050aa2114 + +[build.os=linux] +builder = autoconf +subdir = Python-3.7.6