From 076407407a0c56bc2573a988456764936d457273 Mon Sep 17 00:00:00 2001 From: Alex Hornby Date: Mon, 14 Feb 2022 11:55:21 -0800 Subject: [PATCH] add github actions for EdenFS on linux and fix Eden SCM Mac build (#106) Summary: Pull Request resolved: https://github.com/facebookexperimental/eden/pull/106 Pull Request resolved: https://github.com/facebookexperimental/eden/pull/107 Summary * Add EdenFS builds on external CI now EdenSCM is good * Mac builds on github actions by using brew for system dependencies To make this work had to fix some path ordering issues with install directories for Linux and Mac, and generalise the homebrew path fixups we were doing for bison to all the used homebrew packages. Previously Installed packages were being added after system paths, so our own installed thing might be ignored. On github these meant system python 3.9 was being used for hg tests rather than our specified 3.8 (this showed we have some test fails on python 3.9 with "SystemError: deallocated bytearray object has exported buffers", that are beyond the scope of this diff to fix) Also needed to include the getdeps generated python into the generated edenscmdeps3.zip archive setup.py produces otherwise EdenFS tests failed to import thrift.Thrift Eden tests are hanging when run externally about half way through, so disable them on github actions for now as this PR is already fairly large. They work when run locally on an internal devserver, so probably some bit of environment necessary is not defined in the test runner Reviewed By: chadaustin Differential Revision: D34116505 fbshipit-source-id: d0d628db5daabc28d0bd8997cd5c1bc885ed1e73 --- build/fbcode_builder/getdeps.py | 48 +++- build/fbcode_builder/getdeps/builder.py | 314 +--------------------- build/fbcode_builder/getdeps/buildopts.py | 255 +++++++++++------- build/fbcode_builder/getdeps/fetcher.py | 13 - build/fbcode_builder/getdeps/load.py | 15 ++ build/fbcode_builder/getdeps/manifest.py | 13 +- build/fbcode_builder/manifests/boost | 3 + build/fbcode_builder/manifests/bz2 | 30 +++ build/fbcode_builder/manifests/eden | 11 +- build/fbcode_builder/manifests/ncurses | 30 +++ build/fbcode_builder/manifests/openssl | 1 + build/fbcode_builder/manifests/python | 4 + 12 files changed, 311 insertions(+), 426 deletions(-) create mode 100644 build/fbcode_builder/manifests/bz2 create mode 100644 build/fbcode_builder/manifests/ncurses diff --git a/build/fbcode_builder/getdeps.py b/build/fbcode_builder/getdeps.py index 6272eec14..2da326ef3 100755 --- a/build/fbcode_builder/getdeps.py +++ b/build/fbcode_builder/getdeps.py @@ -971,20 +971,26 @@ jobs: out.write(" - name: Fix Git config\n") out.write(" run: git config --system core.longpaths true\n") - projects = loader.manifests_in_dependency_order() - allow_sys_arg = "" if ( build_opts.allow_system_packages - and build_opts.is_linux() and build_opts.host_type.get_package_manager() ): allow_sys_arg = " --allow-system-packages" out.write(" - name: Install system deps\n") + sudo_arg = "sudo " + if build_opts.is_darwin(): + # brew is installed as regular user + sudo_arg = "" out.write( - f" run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive {manifest.name}\n" + f" run: {sudo_arg}python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive {manifest.name}\n" ) + projects = loader.manifests_in_dependency_order() + + main_repo_url = manifest.get_repo_url(manifest_ctx) + has_same_repo_dep = False + for m in projects: if m != manifest: if m.name == "rust": @@ -995,19 +1001,27 @@ jobs: out.write(" default: true\n") out.write(" profile: minimal\n") else: - out.write(" - name: Fetch %s\n" % m.name) - out.write( - f" run: {getdepscmd}{allow_sys_arg} fetch --no-tests {m.name}\n" - ) + ctx = loader.ctx_gen.get_context(m.name) + if m.get_repo_url(ctx) != main_repo_url: + out.write(" - name: Fetch %s\n" % m.name) + out.write( + f" run: {getdepscmd}{allow_sys_arg} fetch --no-tests {m.name}\n" + ) for m in projects: if m != manifest: if m.name == "rust": continue else: + src_dir_arg = "" + ctx = loader.ctx_gen.get_context(m.name) + if main_repo_url and m.get_repo_url(ctx) == main_repo_url: + # Its in the same repo, so src-dir is also . + src_dir_arg = "--src-dir=. " + has_same_repo_dep = True out.write(" - name: Build %s\n" % m.name) out.write( - f" run: {getdepscmd}{allow_sys_arg} build --no-tests {m.name}\n" + f" run: {getdepscmd}{allow_sys_arg} build {src_dir_arg}--no-tests {m.name}\n" ) out.write(" - name: Build %s\n" % manifest.name) @@ -1018,8 +1032,13 @@ jobs: " --project-install-prefix %s:/usr/local" % manifest.name ) + # If we have dep from same repo, we already built it and don't want to rebuild it again + no_deps_arg = "" + if has_same_repo_dep: + no_deps_arg = "--no-deps " + out.write( - f" run: {getdepscmd}{allow_sys_arg} build --src-dir=. {manifest.name} {project_prefix}\n" + f" run: {getdepscmd}{allow_sys_arg} build {no_deps_arg}--src-dir=. {manifest.name} {project_prefix}\n" ) out.write(" - name: Copy artifacts\n") @@ -1043,10 +1062,11 @@ jobs: out.write(" name: %s\n" % manifest.name) out.write(" path: _artifacts\n") - out.write(" - name: Test %s\n" % manifest.name) - out.write( - f" run: {getdepscmd}{allow_sys_arg} test --src-dir=. {manifest.name} {project_prefix}\n" - ) + if manifest.get("github.actions", "run_tests", ctx=manifest_ctx) != "off": + out.write(" - name: Test %s\n" % manifest.name) + out.write( + f" run: {getdepscmd}{allow_sys_arg} test --src-dir=. {manifest.name} {project_prefix}\n" + ) def setup_project_cmd_parser(self, parser): parser.add_argument( diff --git a/build/fbcode_builder/getdeps/builder.py b/build/fbcode_builder/getdeps/builder.py index 43cea96de..d889eded5 100644 --- a/build/fbcode_builder/getdeps/builder.py +++ b/build/fbcode_builder/getdeps/builder.py @@ -4,6 +4,7 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. +import glob import json import os import shutil @@ -197,6 +198,14 @@ class MakeBuilder(BuilderBase): install_cmd = [self._make_binary] + self.install_args + self._get_prefix() self._run_cmd(install_cmd, env=env) + # bz2's Makefile doesn't install its .so properly + if self.manifest and self.manifest.name == "bz2": + libdir = os.path.join(self.inst_dir, "lib") + srcpattern = os.path.join(self.src_dir, "lib*.so.*") + print(f"copying to {libdir} from {srcpattern}") + for file in glob.glob(srcpattern): + shutil.copy(file, libdir) + def run_tests( self, install_dirs, schedule_type, owner, test_filter, retry, no_testpilot ): @@ -1185,308 +1194,3 @@ install(FILES sqlite3.h sqlite3ext.h DESTINATION include) ], env=env, ) - - -class CargoBuilder(BuilderBase): - def __init__( - self, - build_opts, - ctx, - manifest, - src_dir, - build_dir, - inst_dir, - build_doc, - workspace_dir, - manifests_to_build, - loader, - ): - super(CargoBuilder, self).__init__( - build_opts, ctx, manifest, src_dir, build_dir, inst_dir - ) - self.build_doc = build_doc - self.ws_dir = workspace_dir - self.manifests_to_build = manifests_to_build and manifests_to_build.split(",") - self.loader = loader - - def run_cargo(self, install_dirs, operation, args=None): - args = args or [] - env = self._compute_env(install_dirs) - # Enable using nightly features with stable compiler - env["RUSTC_BOOTSTRAP"] = "1" - env["LIBZ_SYS_STATIC"] = "1" - cmd = [ - "cargo", - operation, - "--workspace", - "-j%s" % self.num_jobs, - ] + args - self._run_cmd(cmd, cwd=self.workspace_dir(), env=env) - - def build_source_dir(self): - return os.path.join(self.build_dir, "source") - - def workspace_dir(self): - return os.path.join(self.build_source_dir(), self.ws_dir or "") - - def manifest_dir(self, manifest): - return os.path.join(self.build_source_dir(), manifest) - - def recreate_dir(self, src, dst): - if os.path.isdir(dst): - shutil.rmtree(dst) - shutil.copytree(src, dst) - - def _build(self, install_dirs, reconfigure): - build_source_dir = self.build_source_dir() - self.recreate_dir(self.src_dir, build_source_dir) - - dot_cargo_dir = os.path.join(build_source_dir, ".cargo") - if not os.path.isdir(dot_cargo_dir): - os.mkdir(dot_cargo_dir) - - with open(os.path.join(dot_cargo_dir, "config"), "w+") as f: - f.write( - """\ -[build] -target-dir = '''{}''' - -[net] -git-fetch-with-cli = true - -[profile.dev] -debug = false -incremental = false -""".format( - self.build_dir.replace("\\", "\\\\") - ) - ) - - if self.ws_dir is not None: - self._patchup_workspace() - - try: - from .facebook.rust import vendored_crates - - vendored_crates(self.build_opts, build_source_dir) - except ImportError: - # This FB internal module isn't shippped to github, - # so just rely on cargo downloading crates on it's own - pass - - if self.manifests_to_build is None: - self.run_cargo( - install_dirs, - "build", - ["--out-dir", os.path.join(self.inst_dir, "bin"), "-Zunstable-options"], - ) - else: - for manifest in self.manifests_to_build: - self.run_cargo( - install_dirs, - "build", - [ - "--out-dir", - os.path.join(self.inst_dir, "bin"), - "-Zunstable-options", - "--manifest-path", - self.manifest_dir(manifest), - ], - ) - - self.recreate_dir(build_source_dir, os.path.join(self.inst_dir, "source")) - - def run_tests( - self, install_dirs, schedule_type, owner, test_filter, retry, no_testpilot - ): - if test_filter: - args = ["--", test_filter] - else: - args = [] - - if self.manifests_to_build is None: - self.run_cargo(install_dirs, "test", args) - if self.build_doc: - self.run_cargo(install_dirs, "doc", ["--no-deps"]) - else: - for manifest in self.manifests_to_build: - margs = ["--manifest-path", self.manifest_dir(manifest)] - self.run_cargo(install_dirs, "test", args + margs) - if self.build_doc: - self.run_cargo(install_dirs, "doc", ["--no-deps"] + margs) - - def _patchup_workspace(self): - """ - This method makes some assumptions about the state of the project and - its cargo dependendies: - 1. Crates from cargo dependencies can be extracted from Cargo.toml files - using _extract_crates function. It is using a heuristic so check its - code to understand how it is done. - 2. The extracted cargo dependencies crates can be found in the - dependency's install dir using _resolve_crate_to_path function - which again is using a heuristic. - - Notice that many things might go wrong here. E.g. if someone depends - on another getdeps crate by writing in their Cargo.toml file: - - my-rename-of-crate = { package = "crate", git = "..." } - - they can count themselves lucky because the code will raise an - Exception. There migh be more cases where the code will silently pass - producing bad results. - """ - workspace_dir = self.workspace_dir() - config = self._resolve_config() - if config: - with open(os.path.join(workspace_dir, "Cargo.toml"), "r+") as f: - manifest_content = f.read() - if "[package]" not in manifest_content: - # A fake manifest has to be crated to change the virtual - # manifest into a non-virtual. The virtual manifests are limited - # in many ways and the inability to define patches on them is - # one. Check https://github.com/rust-lang/cargo/issues/4934 to - # see if it is resolved. - f.write( - """ - [package] - name = "fake_manifest_of_{}" - version = "0.0.0" - [lib] - path = "/dev/null" - """.format( - self.manifest.name - ) - ) - else: - f.write("\n") - f.write(config) - - def _resolve_config(self): - """ - Returns a configuration to be put inside root Cargo.toml file which - patches the dependencies git code with local getdeps versions. - See https://doc.rust-lang.org/cargo/reference/manifest.html#the-patch-section - """ - dep_to_git = self._resolve_dep_to_git() - dep_to_crates = CargoBuilder._resolve_dep_to_crates( - self.build_source_dir(), dep_to_git - ) - - config = [] - for name in sorted(dep_to_git.keys()): - git_conf = dep_to_git[name] - crates = sorted(dep_to_crates.get(name, [])) - if not crates: - continue # nothing to patch, move along - crates_patches = [ - '{} = {{ path = "{}" }}'.format( - crate, - CargoBuilder._resolve_crate_to_path(crate, git_conf).replace( - "\\", "\\\\" - ), - ) - for crate in crates - ] - - config.append( - '[patch."{0}"]\n'.format(git_conf["repo_url"]) - + "\n".join(crates_patches) - ) - return "\n".join(config) - - def _resolve_dep_to_git(self): - """ - For each direct dependency of the currently build manifest check if it - is also cargo-builded and if yes then extract it's git configs and - install dir - """ - dependencies = self.manifest.get_dependencies(self.ctx) - if not dependencies: - return [] - - dep_to_git = {} - for dep in dependencies: - dep_manifest = self.loader.load_manifest(dep) - dep_builder = dep_manifest.get("build", "builder", ctx=self.ctx) - if dep_builder not in ["cargo", "nop"] or dep == "rust": - # This is a direct dependency, but it is not build with cargo - # and it is not simply copying files with nop, so ignore it. - # The "rust" dependency is an exception since it contains the - # toolchain. - continue - - git_conf = dep_manifest.get_section_as_dict("git", self.ctx) - if "repo_url" not in git_conf: - raise Exception( - "A cargo dependency requires git.repo_url to be defined." - ) - source_dir = self.loader.get_project_install_dir(dep_manifest) - if dep_builder == "cargo": - source_dir = os.path.join(source_dir, "source") - git_conf["source_dir"] = source_dir - dep_to_git[dep] = git_conf - return dep_to_git - - @staticmethod - def _resolve_dep_to_crates(build_source_dir, dep_to_git): - """ - This function traverse the build_source_dir in search of Cargo.toml - files, extracts the crate names from them using _extract_crates - function and returns a merged result containing crate names per - dependency name from all Cargo.toml files in the project. - """ - if not dep_to_git: - return {} # no deps, so don't waste time traversing files - - dep_to_crates = {} - for root, _, files in os.walk(build_source_dir): - for f in files: - if f == "Cargo.toml": - more_dep_to_crates = CargoBuilder._extract_crates( - os.path.join(root, f), dep_to_git - ) - for name, crates in more_dep_to_crates.items(): - dep_to_crates.setdefault(name, set()).update(crates) - return dep_to_crates - - @staticmethod - def _extract_crates(cargo_toml_file, dep_to_git): - """ - This functions reads content of provided cargo toml file and extracts - crate names per each dependency. The extraction is done by a heuristic - so it might be incorrect. - """ - deps_to_crates = {} - with open(cargo_toml_file, "r") as f: - for line in f.readlines(): - if line.startswith("#") or "git = " not in line: - continue # filter out commented lines and ones without git deps - for name, conf in dep_to_git.items(): - if 'git = "{}"'.format(conf["repo_url"]) in line: - pkg_template = ' package = "' - if pkg_template in line: - crate_name, _, _ = line.partition(pkg_template)[ - 2 - ].partition('"') - else: - crate_name, _, _ = line.partition("=") - deps_to_crates.setdefault(name, set()).add(crate_name.strip()) - return deps_to_crates - - @staticmethod - def _resolve_crate_to_path(crate, git_conf): - """ - Tries to find in git_conf["inst_dir"] by searching a [package] - keyword followed by name = "". - """ - source_dir = git_conf["source_dir"] - search_pattern = '[package]\nname = "{}"'.format(crate) - - for root, _, files in os.walk(source_dir): - for fname in files: - if fname == "Cargo.toml": - with open(os.path.join(root, fname), "r") as f: - if search_pattern in f.read(): - return root - - raise Exception("Failed to found crate {} in path {}".format(crate, source_dir)) diff --git a/build/fbcode_builder/getdeps/buildopts.py b/build/fbcode_builder/getdeps/buildopts.py index 5c32ee147..58bd7d642 100644 --- a/build/fbcode_builder/getdeps/buildopts.py +++ b/build/fbcode_builder/getdeps/buildopts.py @@ -13,7 +13,7 @@ import tempfile from typing import Optional, Mapping from .copytree import containing_repo_type -from .envfuncs import Env, add_path_entry +from .envfuncs import Env, add_flag, add_path_entry from .fetcher import get_fbsource_repo_data, homebrew_package_prefix from .manifest import ContextGenerator from .platform import HostType, is_windows, get_available_ram @@ -103,6 +103,18 @@ class BuildOptions(object): self.allow_system_packages = allow_system_packages self.lfs_path = lfs_path self.shared_libs = shared_libs + + lib_path = None + if self.is_darwin(): + lib_path = "DYLD_LIBRARY_PATH" + elif self.is_linux(): + lib_path = "LD_LIBRARY_PATH" + elif self.is_windows(): + lib_path = "PATH" + else: + lib_path = None + self.lib_path = lib_path + if vcvars_path is None and is_windows(): try: @@ -196,7 +208,9 @@ class BuildOptions(object): } ) - def compute_env_for_install_dirs(self, install_dirs, env=None, manifest=None): + def compute_env_for_install_dirs( + self, install_dirs, env=None, manifest=None + ): # noqa: C901 if env is not None: env = env.copy() else: @@ -236,11 +250,23 @@ class BuildOptions(object): sdkroot = subprocess.check_output(["xcrun", "--show-sdk-path"]) env["SDKROOT"] = sdkroot.decode().strip() - # MacOS includes a version of bison so homebrew won't automatically add - # its own version to PATH. Find where the homebrew bison is and prepend - # it to PATH. - if self.is_darwin() and self.host_type.get_package_manager() == "homebrew": - add_homebrew_package_to_path(env, "bison") + if ( + self.is_darwin() + and self.allow_system_packages + and self.host_type.get_package_manager() == "homebrew" + and manifest + and manifest.resolved_system_packages + ): + # Homebrew packages may not be on the default PATHs + brew_packages = manifest.resolved_system_packages.get("homebrew", []) + for p in brew_packages: + found = self.add_homebrew_package_to_env(p, env) + # Try extra hard to find openssl, needed with homebrew on macOS + if found and p.startswith("openssl"): + candidate = homebrew_package_prefix("openssl@1.1") + if os.path.exists(candidate): + os.environ["OPENSSL_ROOT_DIR"] = candidate + env["OPENSSL_ROOT_DIR"] = os.environ["OPENSSL_ROOT_DIR"] if self.fbsource_dir: env["YARN_YARN_OFFLINE_MIRROR"] = os.path.join( @@ -261,91 +287,142 @@ class BuildOptions(object): env["FBSOURCE_HASH"] = hash_data.hash env["FBSOURCE_DATE"] = hash_data.date - lib_path = None - if self.is_darwin(): - lib_path = "DYLD_LIBRARY_PATH" - elif self.is_linux(): - lib_path = "LD_LIBRARY_PATH" - elif self.is_windows(): - lib_path = "PATH" - else: - lib_path = None + # reverse as we are prepending to the PATHs + for d in reversed(install_dirs): + self.add_prefix_to_env(d, env, append=False) - for d in install_dirs: - bindir = os.path.join(d, "bin") + # Linux is always system openssl + system_openssl = self.is_linux() - pkgconfig = os.path.join(d, "lib/pkgconfig") - if os.path.exists(pkgconfig): - add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig) + # For other systems lets see if package is requested + if not system_openssl and manifest and manifest.resolved_system_packages: + for _pkg_type, pkgs in manifest.resolved_system_packages.items(): + for p in pkgs: + if p.startswith("openssl") or p.startswith("libssl"): + system_openssl = True + break - pkgconfig = os.path.join(d, "lib64/pkgconfig") - if os.path.exists(pkgconfig): - add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig) - - add_path_entry(env, "CMAKE_PREFIX_PATH", d) - - # Tell the thrift compiler about includes it needs to consider - thriftdir = os.path.join(d, "include/thrift-files") - if os.path.exists(thriftdir): - add_path_entry(env, "THRIFT_INCLUDE_PATH", thriftdir) - - # Allow resolving shared objects built earlier (eg: zstd - # doesn't include the full path to the dylib in its linkage - # so we need to give it an assist) - if lib_path: - for lib in ["lib", "lib64"]: - libdir = os.path.join(d, lib) - if os.path.exists(libdir): - add_path_entry(env, lib_path, libdir) - - # Allow resolving binaries (eg: cmake, ninja) and dlls - # built by earlier steps - if os.path.exists(bindir): - add_path_entry(env, "PATH", bindir, append=False) - - # If rustc is present in the `bin` directory, set RUSTC to prevent - # cargo uses the rustc installed in the system. - if self.is_windows(): - cargo_path = os.path.join(bindir, "cargo.exe") - rustc_path = os.path.join(bindir, "rustc.exe") - rustdoc_path = os.path.join(bindir, "rustdoc.exe") - else: - cargo_path = os.path.join(bindir, "cargo") - rustc_path = os.path.join(bindir, "rustc") - rustdoc_path = os.path.join(bindir, "rustdoc") - - if os.path.isfile(rustc_path): - env["CARGO_BIN"] = cargo_path - env["RUSTC"] = rustc_path - env["RUSTDOC"] = rustdoc_path - - openssl_include = os.path.join(d, "include/openssl") - if os.path.isdir(openssl_include) and any( - os.path.isfile(os.path.join(d, "lib", libcrypto)) - for libcrypto in ("libcrypto.lib", "libcrypto.so", "libcrypto.a") - ): - # This must be the openssl library, let Rust know about it - env["OPENSSL_DIR"] = d - # And let openssl know to pick up the system certs if present - for system_ssl_cfg in ["/etc/pki/tls", "/etc/ssl"]: - if os.path.isdir(system_ssl_cfg): - cert_dir = system_ssl_cfg + "/certs" - if os.path.isdir(cert_dir): - env["SSL_CERT_DIR"] = cert_dir - cert_file = system_ssl_cfg + "/cert.pem" - if os.path.isfile(cert_file): - env["SSL_CERT_FILE"] = cert_file - - # Try extra hard to find openssl, needed with homebrew on macOS - if ( - self.is_darwin() - and "OPENSSL_DIR" not in env - and "OPENSSL_ROOT_DIR" in os.environ - ): - env["OPENSSL_ROOT_DIR"] = os.environ["OPENSSL_ROOT_DIR"] + # Let openssl know to pick up the system certs if present + if system_openssl or "OPENSSL_DIR" in env: + for system_ssl_cfg in ["/etc/pki/tls", "/etc/ssl"]: + if os.path.isdir(system_ssl_cfg): + cert_dir = system_ssl_cfg + "/certs" + if os.path.isdir(cert_dir): + env["SSL_CERT_DIR"] = cert_dir + cert_file = system_ssl_cfg + "/cert.pem" + if os.path.isfile(cert_file): + env["SSL_CERT_FILE"] = cert_file return env + def add_homebrew_package_to_env(self, package, env): + prefix = homebrew_package_prefix(package) + if prefix and os.path.exists(prefix): + return self.add_prefix_to_env( + prefix, env, append=False, add_library_path=True + ) + return False + + def add_prefix_to_env( + self, d, env, append=True, add_library_path=False + ): # noqa: C901 + bindir = os.path.join(d, "bin") + found = False + pkgconfig = os.path.join(d, "lib", "pkgconfig") + if os.path.exists(pkgconfig): + found = True + add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig, append=append) + + pkgconfig = os.path.join(d, "lib64", "pkgconfig") + if os.path.exists(pkgconfig): + found = True + add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig, append=append) + + add_path_entry(env, "CMAKE_PREFIX_PATH", d, append=append) + + # Tell the thrift compiler about includes it needs to consider + thriftdir = os.path.join(d, "include", "thrift-files") + if os.path.exists(thriftdir): + found = True + add_path_entry(env, "THRIFT_INCLUDE_PATH", thriftdir, append=append) + + # module detection for python is old fashioned and needs flags + includedir = os.path.join(d, "include") + if os.path.exists(includedir): + found = True + ncursesincludedir = os.path.join(d, "include", "ncurses") + if os.path.exists(ncursesincludedir): + add_path_entry(env, "C_INCLUDE_PATH", ncursesincludedir, append=append) + add_flag(env, "CPPFLAGS", f"-I{includedir}", append=append) + add_flag(env, "CPPFLAGS", f"-I{ncursesincludedir}", append=append) + elif "/bz2-" in d: + add_flag(env, "CPPFLAGS", f"-I{includedir}", append=append) + + # Map from FB python manifests to PYTHONPATH + pydir = os.path.join(d, "lib", "fb-py-libs") + if os.path.exists(pydir): + found = True + manifest_ext = ".manifest" + pymanifestfiles = [ + f + for f in os.listdir(pydir) + if f.endswith(manifest_ext) and os.path.isfile(os.path.join(pydir, f)) + ] + for f in pymanifestfiles: + subdir = f[: -len(manifest_ext)] + add_path_entry( + env, "PYTHONPATH", os.path.join(pydir, subdir), append=append + ) + + # Allow resolving shared objects built earlier (eg: zstd + # doesn't include the full path to the dylib in its linkage + # so we need to give it an assist) + if self.lib_path: + for lib in ["lib", "lib64"]: + libdir = os.path.join(d, lib) + if os.path.exists(libdir): + found = True + add_path_entry(env, self.lib_path, libdir, append=append) + # module detection for python is old fashioned and needs flags + if "/ncurses-" in d: + add_flag(env, "LDFLAGS", f"-L{libdir}", append=append) + elif "/bz2-" in d: + add_flag(env, "LDFLAGS", f"-L{libdir}", append=append) + if add_library_path: + add_path_entry(env, "LIBRARY_PATH", libdir, append=append) + + # Allow resolving binaries (eg: cmake, ninja) and dlls + # built by earlier steps + if os.path.exists(bindir): + found = True + add_path_entry(env, "PATH", bindir, append=append) + + # If rustc is present in the `bin` directory, set RUSTC to prevent + # cargo uses the rustc installed in the system. + if self.is_windows(): + cargo_path = os.path.join(bindir, "cargo.exe") + rustc_path = os.path.join(bindir, "rustc.exe") + rustdoc_path = os.path.join(bindir, "rustdoc.exe") + else: + cargo_path = os.path.join(bindir, "cargo") + rustc_path = os.path.join(bindir, "rustc") + rustdoc_path = os.path.join(bindir, "rustdoc") + + if os.path.isfile(rustc_path): + env["CARGO_BIN"] = cargo_path + env["RUSTC"] = rustc_path + env["RUSTDOC"] = rustdoc_path + + openssl_include = os.path.join(d, "include", "openssl") + if os.path.isdir(openssl_include) and any( + os.path.isfile(os.path.join(d, "lib", libcrypto)) + for libcrypto in ("libcrypto.lib", "libcrypto.so", "libcrypto.a") + ): + # This must be the openssl library, let Rust know about it + env["OPENSSL_DIR"] = d + + return found + def list_win32_subst_letters(): output = subprocess.check_output(["subst"]).decode("utf-8") @@ -536,9 +613,3 @@ def setup_build_options(args, host_type=None): facebook_internal=args.facebook_internal, **build_args, ) - - -def add_homebrew_package_to_path(env, package): - prefix = homebrew_package_prefix(package) - if prefix and os.path.exists(os.path.join(prefix, "bin")): - add_path_entry(env, "PATH", os.path.join(prefix, "bin"), append=False) diff --git a/build/fbcode_builder/getdeps/fetcher.py b/build/fbcode_builder/getdeps/fetcher.py index 50483b0f3..ca4eb242a 100644 --- a/build/fbcode_builder/getdeps/fetcher.py +++ b/build/fbcode_builder/getdeps/fetcher.py @@ -188,19 +188,6 @@ class SystemPackageFetcher(object): else: self.installed = False - # Hack to make openssl discovery with homebrew work. If openssl was - # built with autoconf we could use autoconf.envcmd.OPENSSL_ROOT_DIR - # from the manifest, but it isn't, so handle the special case here. - if ( - self.installed - and self.host_type.is_darwin() - and self.manager == "homebrew" - and "openssl@1.1" in self.packages - ): - candidate = homebrew_package_prefix("openssl@1.1") - if os.path.exists(candidate): - os.environ["OPENSSL_ROOT_DIR"] = candidate - return bool(self.installed) def update(self): diff --git a/build/fbcode_builder/getdeps/load.py b/build/fbcode_builder/getdeps/load.py index 481900986..1b26bbff9 100644 --- a/build/fbcode_builder/getdeps/load.py +++ b/build/fbcode_builder/getdeps/load.py @@ -4,6 +4,7 @@ # LICENSE file in the root directory of this source tree. import base64 +import copy import hashlib import os @@ -178,6 +179,7 @@ class ManifestLoader(object): deps = [manifest] # The list of manifests in dependency order dep_order = [] + system_packages = {} while len(deps) > 0: m = deps.pop(0) @@ -212,6 +214,19 @@ class ManifestLoader(object): # Its deps are done, so we can emit it seen.add(m.name) + # Capture system packages as we may need to set PATHs to then later + if ( + self.build_opts.allow_system_packages + and self.build_opts.host_type.get_package_manager() + ): + packages = m.get_required_system_packages(ctx) + for pkg_type, v in packages.items(): + merged = system_packages.get(pkg_type, []) + if v not in merged: + merged += v + system_packages[pkg_type] = merged + # A manifest depends on all system packages in it dependencies as well + m.resolved_system_packages = copy.copy(system_packages) dep_order.append(m) return dep_order diff --git a/build/fbcode_builder/getdeps/manifest.py b/build/fbcode_builder/getdeps/manifest.py index 2c43f86e1..99c47bb2f 100644 --- a/build/fbcode_builder/getdeps/manifest.py +++ b/build/fbcode_builder/getdeps/manifest.py @@ -77,6 +77,12 @@ SCHEMA = { "manifests_to_build": OPTIONAL, }, }, + "github.actions": { + "optional_section": True, + "fields": { + "run_tests": OPTIONAL, + }, + }, "cmake.defines": {"optional_section": True}, "autoconf.args": {"optional_section": True}, "autoconf.envcmd.LDFLAGS": {"optional_section": True}, @@ -115,6 +121,7 @@ ALLOWED_EXPR_SECTIONS = [ "shipit.pathmap", "shipit.strip", "homebrew", + "github.actions", ] @@ -223,6 +230,7 @@ class ManifestParser(object): self.fbsource_path = self.get("manifest", "fbsource_path") self.shipit_project = self.get("manifest", "shipit_project") self.shipit_fbcode_builder = self.get("manifest", "shipit_fbcode_builder") + self.resolved_system_packages = {} if self.name != os.path.basename(file_name): raise Exception( @@ -378,6 +386,9 @@ class ManifestParser(object): return True + def get_repo_url(self, ctx): + return self.get("git", "repo_url", ctx=ctx) + def create_fetcher(self, build_options, ctx): use_real_shipit = ( ShipitTransformerFetcher.available() and build_options.use_shipit @@ -409,7 +420,7 @@ class ManifestParser(object): if package_fetcher.packages_are_installed(): return package_fetcher - repo_url = self.get("git", "repo_url", ctx=ctx) + repo_url = self.get_repo_url(ctx) if repo_url: rev = self.get("git", "rev") depth = self.get("git", "depth") diff --git a/build/fbcode_builder/manifests/boost b/build/fbcode_builder/manifests/boost index f544114e2..82ae6e542 100644 --- a/build/fbcode_builder/manifests/boost +++ b/build/fbcode_builder/manifests/boost @@ -19,6 +19,8 @@ libboost-all-dev [homebrew] boost +# Boost cmake detection on homebrew adds this as requirement: https://github.com/Homebrew/homebrew-core/issues/67427#issuecomment-754187345 +icu4c [rpms.all(distro=centos_stream,distro_vers=8)] boost169 @@ -55,6 +57,7 @@ boost169-program-options [rpms.not(all(distro=centos_stream,distro_vers=8))] boost-devel +boost-static [build] builder = boost diff --git a/build/fbcode_builder/manifests/bz2 b/build/fbcode_builder/manifests/bz2 new file mode 100644 index 000000000..af2f357d5 --- /dev/null +++ b/build/fbcode_builder/manifests/bz2 @@ -0,0 +1,30 @@ +[manifest] +name = bz2 + +[debs] +libbz2-dev + +[homebrew] +bzip2 + +[rpms] +bzip2-devel + +[download] +url = https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz +sha256 = ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269 + +[build.not(os=windows)] +builder = make +subdir = bzip2-1.0.8 + +[make.build_args.os=linux] +# python bz2 support on linux needs dynamic library +-f +Makefile-libbz2_so + +[make.install_args] +install + +[build.os=windows] +builder = nop diff --git a/build/fbcode_builder/manifests/eden b/build/fbcode_builder/manifests/eden index dfc3e8bb1..6509270f6 100644 --- a/build/fbcode_builder/manifests/eden +++ b/build/fbcode_builder/manifests/eden @@ -7,6 +7,9 @@ shipit_fbcode_builder = true [git] repo_url = https://github.com/facebookexperimental/eden.git +[github.actions] +run_tests = off + [build] builder = cmake @@ -37,11 +40,17 @@ sqlite3 [dependencies.os=darwin] osxfuse +[dependencies.not(os=windows)] # TODO: teach getdeps to compile curl on Windows. # Enabling curl on Windows requires us to find a way to compile libcurl with # msvc. -[dependencies.not(os=windows)] libcurl +# Added so that OSS doesn't see system "python" which is python 2 on darwin and some linux +python + +[dependencies.all(fb=off)] +# Outside Meta hg is not installed, or if it is, its not the one we want to test with +eden_scm [shipit.pathmap.fb=on] # for internal builds that use getdeps diff --git a/build/fbcode_builder/manifests/ncurses b/build/fbcode_builder/manifests/ncurses new file mode 100644 index 000000000..1bb5e8a45 --- /dev/null +++ b/build/fbcode_builder/manifests/ncurses @@ -0,0 +1,30 @@ +[manifest] +name = ncurses + +[debs] +libncurses-dev + +[homebrew] +ncurses + +[rpms] +ncurses-devel + +[download] +url = https://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.3.tar.gz +sha256 = 97fc51ac2b085d4cde31ef4d2c3122c21abc217e9090a43a30fc5ec21684e059 + +[build.not(os=windows)] +builder = autoconf +subdir = ncurses-6.3 + +[autoconf.args] +--without-cxx-binding +--without-ada + +[autoconf.args.os=linux] +--enable-shared +--with-shared + +[build.os=windows] +builder = nop diff --git a/build/fbcode_builder/manifests/openssl b/build/fbcode_builder/manifests/openssl index 5dcfc3868..b806b701a 100644 --- a/build/fbcode_builder/manifests/openssl +++ b/build/fbcode_builder/manifests/openssl @@ -6,6 +6,7 @@ libssl-dev [homebrew] openssl@1.1 +# on homebrew need the matching curl and ca- [rpms] openssl diff --git a/build/fbcode_builder/manifests/python b/build/fbcode_builder/manifests/python index f9877e783..de7b6da7f 100644 --- a/build/fbcode_builder/manifests/python +++ b/build/fbcode_builder/manifests/python @@ -36,3 +36,7 @@ libffi [dependencies] libffi +# eden tests expect the python bz2 support +bz2 +# eden tests expect the python curses support +ncurses