diff --git a/build/fbcode_builder/getdeps.py b/build/fbcode_builder/getdeps.py index 6272eec14..2da326ef3 100755 --- a/build/fbcode_builder/getdeps.py +++ b/build/fbcode_builder/getdeps.py @@ -971,20 +971,26 @@ jobs: out.write(" - name: Fix Git config\n") out.write(" run: git config --system core.longpaths true\n") - projects = loader.manifests_in_dependency_order() - allow_sys_arg = "" if ( build_opts.allow_system_packages - and build_opts.is_linux() and build_opts.host_type.get_package_manager() ): allow_sys_arg = " --allow-system-packages" out.write(" - name: Install system deps\n") + sudo_arg = "sudo " + if build_opts.is_darwin(): + # brew is installed as regular user + sudo_arg = "" out.write( - f" run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive {manifest.name}\n" + f" run: {sudo_arg}python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive {manifest.name}\n" ) + projects = loader.manifests_in_dependency_order() + + main_repo_url = manifest.get_repo_url(manifest_ctx) + has_same_repo_dep = False + for m in projects: if m != manifest: if m.name == "rust": @@ -995,19 +1001,27 @@ jobs: out.write(" default: true\n") out.write(" profile: minimal\n") else: - out.write(" - name: Fetch %s\n" % m.name) - out.write( - f" run: {getdepscmd}{allow_sys_arg} fetch --no-tests {m.name}\n" - ) + ctx = loader.ctx_gen.get_context(m.name) + if m.get_repo_url(ctx) != main_repo_url: + out.write(" - name: Fetch %s\n" % m.name) + out.write( + f" run: {getdepscmd}{allow_sys_arg} fetch --no-tests {m.name}\n" + ) for m in projects: if m != manifest: if m.name == "rust": continue else: + src_dir_arg = "" + ctx = loader.ctx_gen.get_context(m.name) + if main_repo_url and m.get_repo_url(ctx) == main_repo_url: + # Its in the same repo, so src-dir is also . + src_dir_arg = "--src-dir=. " + has_same_repo_dep = True out.write(" - name: Build %s\n" % m.name) out.write( - f" run: {getdepscmd}{allow_sys_arg} build --no-tests {m.name}\n" + f" run: {getdepscmd}{allow_sys_arg} build {src_dir_arg}--no-tests {m.name}\n" ) out.write(" - name: Build %s\n" % manifest.name) @@ -1018,8 +1032,13 @@ jobs: " --project-install-prefix %s:/usr/local" % manifest.name ) + # If we have dep from same repo, we already built it and don't want to rebuild it again + no_deps_arg = "" + if has_same_repo_dep: + no_deps_arg = "--no-deps " + out.write( - f" run: {getdepscmd}{allow_sys_arg} build --src-dir=. {manifest.name} {project_prefix}\n" + f" run: {getdepscmd}{allow_sys_arg} build {no_deps_arg}--src-dir=. {manifest.name} {project_prefix}\n" ) out.write(" - name: Copy artifacts\n") @@ -1043,10 +1062,11 @@ jobs: out.write(" name: %s\n" % manifest.name) out.write(" path: _artifacts\n") - out.write(" - name: Test %s\n" % manifest.name) - out.write( - f" run: {getdepscmd}{allow_sys_arg} test --src-dir=. {manifest.name} {project_prefix}\n" - ) + if manifest.get("github.actions", "run_tests", ctx=manifest_ctx) != "off": + out.write(" - name: Test %s\n" % manifest.name) + out.write( + f" run: {getdepscmd}{allow_sys_arg} test --src-dir=. {manifest.name} {project_prefix}\n" + ) def setup_project_cmd_parser(self, parser): parser.add_argument( diff --git a/build/fbcode_builder/getdeps/builder.py b/build/fbcode_builder/getdeps/builder.py index 43cea96de..d889eded5 100644 --- a/build/fbcode_builder/getdeps/builder.py +++ b/build/fbcode_builder/getdeps/builder.py @@ -4,6 +4,7 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. +import glob import json import os import shutil @@ -197,6 +198,14 @@ class MakeBuilder(BuilderBase): install_cmd = [self._make_binary] + self.install_args + self._get_prefix() self._run_cmd(install_cmd, env=env) + # bz2's Makefile doesn't install its .so properly + if self.manifest and self.manifest.name == "bz2": + libdir = os.path.join(self.inst_dir, "lib") + srcpattern = os.path.join(self.src_dir, "lib*.so.*") + print(f"copying to {libdir} from {srcpattern}") + for file in glob.glob(srcpattern): + shutil.copy(file, libdir) + def run_tests( self, install_dirs, schedule_type, owner, test_filter, retry, no_testpilot ): @@ -1185,308 +1194,3 @@ install(FILES sqlite3.h sqlite3ext.h DESTINATION include) ], env=env, ) - - -class CargoBuilder(BuilderBase): - def __init__( - self, - build_opts, - ctx, - manifest, - src_dir, - build_dir, - inst_dir, - build_doc, - workspace_dir, - manifests_to_build, - loader, - ): - super(CargoBuilder, self).__init__( - build_opts, ctx, manifest, src_dir, build_dir, inst_dir - ) - self.build_doc = build_doc - self.ws_dir = workspace_dir - self.manifests_to_build = manifests_to_build and manifests_to_build.split(",") - self.loader = loader - - def run_cargo(self, install_dirs, operation, args=None): - args = args or [] - env = self._compute_env(install_dirs) - # Enable using nightly features with stable compiler - env["RUSTC_BOOTSTRAP"] = "1" - env["LIBZ_SYS_STATIC"] = "1" - cmd = [ - "cargo", - operation, - "--workspace", - "-j%s" % self.num_jobs, - ] + args - self._run_cmd(cmd, cwd=self.workspace_dir(), env=env) - - def build_source_dir(self): - return os.path.join(self.build_dir, "source") - - def workspace_dir(self): - return os.path.join(self.build_source_dir(), self.ws_dir or "") - - def manifest_dir(self, manifest): - return os.path.join(self.build_source_dir(), manifest) - - def recreate_dir(self, src, dst): - if os.path.isdir(dst): - shutil.rmtree(dst) - shutil.copytree(src, dst) - - def _build(self, install_dirs, reconfigure): - build_source_dir = self.build_source_dir() - self.recreate_dir(self.src_dir, build_source_dir) - - dot_cargo_dir = os.path.join(build_source_dir, ".cargo") - if not os.path.isdir(dot_cargo_dir): - os.mkdir(dot_cargo_dir) - - with open(os.path.join(dot_cargo_dir, "config"), "w+") as f: - f.write( - """\ -[build] -target-dir = '''{}''' - -[net] -git-fetch-with-cli = true - -[profile.dev] -debug = false -incremental = false -""".format( - self.build_dir.replace("\\", "\\\\") - ) - ) - - if self.ws_dir is not None: - self._patchup_workspace() - - try: - from .facebook.rust import vendored_crates - - vendored_crates(self.build_opts, build_source_dir) - except ImportError: - # This FB internal module isn't shippped to github, - # so just rely on cargo downloading crates on it's own - pass - - if self.manifests_to_build is None: - self.run_cargo( - install_dirs, - "build", - ["--out-dir", os.path.join(self.inst_dir, "bin"), "-Zunstable-options"], - ) - else: - for manifest in self.manifests_to_build: - self.run_cargo( - install_dirs, - "build", - [ - "--out-dir", - os.path.join(self.inst_dir, "bin"), - "-Zunstable-options", - "--manifest-path", - self.manifest_dir(manifest), - ], - ) - - self.recreate_dir(build_source_dir, os.path.join(self.inst_dir, "source")) - - def run_tests( - self, install_dirs, schedule_type, owner, test_filter, retry, no_testpilot - ): - if test_filter: - args = ["--", test_filter] - else: - args = [] - - if self.manifests_to_build is None: - self.run_cargo(install_dirs, "test", args) - if self.build_doc: - self.run_cargo(install_dirs, "doc", ["--no-deps"]) - else: - for manifest in self.manifests_to_build: - margs = ["--manifest-path", self.manifest_dir(manifest)] - self.run_cargo(install_dirs, "test", args + margs) - if self.build_doc: - self.run_cargo(install_dirs, "doc", ["--no-deps"] + margs) - - def _patchup_workspace(self): - """ - This method makes some assumptions about the state of the project and - its cargo dependendies: - 1. Crates from cargo dependencies can be extracted from Cargo.toml files - using _extract_crates function. It is using a heuristic so check its - code to understand how it is done. - 2. The extracted cargo dependencies crates can be found in the - dependency's install dir using _resolve_crate_to_path function - which again is using a heuristic. - - Notice that many things might go wrong here. E.g. if someone depends - on another getdeps crate by writing in their Cargo.toml file: - - my-rename-of-crate = { package = "crate", git = "..." } - - they can count themselves lucky because the code will raise an - Exception. There migh be more cases where the code will silently pass - producing bad results. - """ - workspace_dir = self.workspace_dir() - config = self._resolve_config() - if config: - with open(os.path.join(workspace_dir, "Cargo.toml"), "r+") as f: - manifest_content = f.read() - if "[package]" not in manifest_content: - # A fake manifest has to be crated to change the virtual - # manifest into a non-virtual. The virtual manifests are limited - # in many ways and the inability to define patches on them is - # one. Check https://github.com/rust-lang/cargo/issues/4934 to - # see if it is resolved. - f.write( - """ - [package] - name = "fake_manifest_of_{}" - version = "0.0.0" - [lib] - path = "/dev/null" - """.format( - self.manifest.name - ) - ) - else: - f.write("\n") - f.write(config) - - def _resolve_config(self): - """ - Returns a configuration to be put inside root Cargo.toml file which - patches the dependencies git code with local getdeps versions. - See https://doc.rust-lang.org/cargo/reference/manifest.html#the-patch-section - """ - dep_to_git = self._resolve_dep_to_git() - dep_to_crates = CargoBuilder._resolve_dep_to_crates( - self.build_source_dir(), dep_to_git - ) - - config = [] - for name in sorted(dep_to_git.keys()): - git_conf = dep_to_git[name] - crates = sorted(dep_to_crates.get(name, [])) - if not crates: - continue # nothing to patch, move along - crates_patches = [ - '{} = {{ path = "{}" }}'.format( - crate, - CargoBuilder._resolve_crate_to_path(crate, git_conf).replace( - "\\", "\\\\" - ), - ) - for crate in crates - ] - - config.append( - '[patch."{0}"]\n'.format(git_conf["repo_url"]) - + "\n".join(crates_patches) - ) - return "\n".join(config) - - def _resolve_dep_to_git(self): - """ - For each direct dependency of the currently build manifest check if it - is also cargo-builded and if yes then extract it's git configs and - install dir - """ - dependencies = self.manifest.get_dependencies(self.ctx) - if not dependencies: - return [] - - dep_to_git = {} - for dep in dependencies: - dep_manifest = self.loader.load_manifest(dep) - dep_builder = dep_manifest.get("build", "builder", ctx=self.ctx) - if dep_builder not in ["cargo", "nop"] or dep == "rust": - # This is a direct dependency, but it is not build with cargo - # and it is not simply copying files with nop, so ignore it. - # The "rust" dependency is an exception since it contains the - # toolchain. - continue - - git_conf = dep_manifest.get_section_as_dict("git", self.ctx) - if "repo_url" not in git_conf: - raise Exception( - "A cargo dependency requires git.repo_url to be defined." - ) - source_dir = self.loader.get_project_install_dir(dep_manifest) - if dep_builder == "cargo": - source_dir = os.path.join(source_dir, "source") - git_conf["source_dir"] = source_dir - dep_to_git[dep] = git_conf - return dep_to_git - - @staticmethod - def _resolve_dep_to_crates(build_source_dir, dep_to_git): - """ - This function traverse the build_source_dir in search of Cargo.toml - files, extracts the crate names from them using _extract_crates - function and returns a merged result containing crate names per - dependency name from all Cargo.toml files in the project. - """ - if not dep_to_git: - return {} # no deps, so don't waste time traversing files - - dep_to_crates = {} - for root, _, files in os.walk(build_source_dir): - for f in files: - if f == "Cargo.toml": - more_dep_to_crates = CargoBuilder._extract_crates( - os.path.join(root, f), dep_to_git - ) - for name, crates in more_dep_to_crates.items(): - dep_to_crates.setdefault(name, set()).update(crates) - return dep_to_crates - - @staticmethod - def _extract_crates(cargo_toml_file, dep_to_git): - """ - This functions reads content of provided cargo toml file and extracts - crate names per each dependency. The extraction is done by a heuristic - so it might be incorrect. - """ - deps_to_crates = {} - with open(cargo_toml_file, "r") as f: - for line in f.readlines(): - if line.startswith("#") or "git = " not in line: - continue # filter out commented lines and ones without git deps - for name, conf in dep_to_git.items(): - if 'git = "{}"'.format(conf["repo_url"]) in line: - pkg_template = ' package = "' - if pkg_template in line: - crate_name, _, _ = line.partition(pkg_template)[ - 2 - ].partition('"') - else: - crate_name, _, _ = line.partition("=") - deps_to_crates.setdefault(name, set()).add(crate_name.strip()) - return deps_to_crates - - @staticmethod - def _resolve_crate_to_path(crate, git_conf): - """ - Tries to find in git_conf["inst_dir"] by searching a [package] - keyword followed by name = "". - """ - source_dir = git_conf["source_dir"] - search_pattern = '[package]\nname = "{}"'.format(crate) - - for root, _, files in os.walk(source_dir): - for fname in files: - if fname == "Cargo.toml": - with open(os.path.join(root, fname), "r") as f: - if search_pattern in f.read(): - return root - - raise Exception("Failed to found crate {} in path {}".format(crate, source_dir)) diff --git a/build/fbcode_builder/getdeps/buildopts.py b/build/fbcode_builder/getdeps/buildopts.py index 5c32ee147..58bd7d642 100644 --- a/build/fbcode_builder/getdeps/buildopts.py +++ b/build/fbcode_builder/getdeps/buildopts.py @@ -13,7 +13,7 @@ import tempfile from typing import Optional, Mapping from .copytree import containing_repo_type -from .envfuncs import Env, add_path_entry +from .envfuncs import Env, add_flag, add_path_entry from .fetcher import get_fbsource_repo_data, homebrew_package_prefix from .manifest import ContextGenerator from .platform import HostType, is_windows, get_available_ram @@ -103,6 +103,18 @@ class BuildOptions(object): self.allow_system_packages = allow_system_packages self.lfs_path = lfs_path self.shared_libs = shared_libs + + lib_path = None + if self.is_darwin(): + lib_path = "DYLD_LIBRARY_PATH" + elif self.is_linux(): + lib_path = "LD_LIBRARY_PATH" + elif self.is_windows(): + lib_path = "PATH" + else: + lib_path = None + self.lib_path = lib_path + if vcvars_path is None and is_windows(): try: @@ -196,7 +208,9 @@ class BuildOptions(object): } ) - def compute_env_for_install_dirs(self, install_dirs, env=None, manifest=None): + def compute_env_for_install_dirs( + self, install_dirs, env=None, manifest=None + ): # noqa: C901 if env is not None: env = env.copy() else: @@ -236,11 +250,23 @@ class BuildOptions(object): sdkroot = subprocess.check_output(["xcrun", "--show-sdk-path"]) env["SDKROOT"] = sdkroot.decode().strip() - # MacOS includes a version of bison so homebrew won't automatically add - # its own version to PATH. Find where the homebrew bison is and prepend - # it to PATH. - if self.is_darwin() and self.host_type.get_package_manager() == "homebrew": - add_homebrew_package_to_path(env, "bison") + if ( + self.is_darwin() + and self.allow_system_packages + and self.host_type.get_package_manager() == "homebrew" + and manifest + and manifest.resolved_system_packages + ): + # Homebrew packages may not be on the default PATHs + brew_packages = manifest.resolved_system_packages.get("homebrew", []) + for p in brew_packages: + found = self.add_homebrew_package_to_env(p, env) + # Try extra hard to find openssl, needed with homebrew on macOS + if found and p.startswith("openssl"): + candidate = homebrew_package_prefix("openssl@1.1") + if os.path.exists(candidate): + os.environ["OPENSSL_ROOT_DIR"] = candidate + env["OPENSSL_ROOT_DIR"] = os.environ["OPENSSL_ROOT_DIR"] if self.fbsource_dir: env["YARN_YARN_OFFLINE_MIRROR"] = os.path.join( @@ -261,91 +287,142 @@ class BuildOptions(object): env["FBSOURCE_HASH"] = hash_data.hash env["FBSOURCE_DATE"] = hash_data.date - lib_path = None - if self.is_darwin(): - lib_path = "DYLD_LIBRARY_PATH" - elif self.is_linux(): - lib_path = "LD_LIBRARY_PATH" - elif self.is_windows(): - lib_path = "PATH" - else: - lib_path = None + # reverse as we are prepending to the PATHs + for d in reversed(install_dirs): + self.add_prefix_to_env(d, env, append=False) - for d in install_dirs: - bindir = os.path.join(d, "bin") + # Linux is always system openssl + system_openssl = self.is_linux() - pkgconfig = os.path.join(d, "lib/pkgconfig") - if os.path.exists(pkgconfig): - add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig) + # For other systems lets see if package is requested + if not system_openssl and manifest and manifest.resolved_system_packages: + for _pkg_type, pkgs in manifest.resolved_system_packages.items(): + for p in pkgs: + if p.startswith("openssl") or p.startswith("libssl"): + system_openssl = True + break - pkgconfig = os.path.join(d, "lib64/pkgconfig") - if os.path.exists(pkgconfig): - add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig) - - add_path_entry(env, "CMAKE_PREFIX_PATH", d) - - # Tell the thrift compiler about includes it needs to consider - thriftdir = os.path.join(d, "include/thrift-files") - if os.path.exists(thriftdir): - add_path_entry(env, "THRIFT_INCLUDE_PATH", thriftdir) - - # Allow resolving shared objects built earlier (eg: zstd - # doesn't include the full path to the dylib in its linkage - # so we need to give it an assist) - if lib_path: - for lib in ["lib", "lib64"]: - libdir = os.path.join(d, lib) - if os.path.exists(libdir): - add_path_entry(env, lib_path, libdir) - - # Allow resolving binaries (eg: cmake, ninja) and dlls - # built by earlier steps - if os.path.exists(bindir): - add_path_entry(env, "PATH", bindir, append=False) - - # If rustc is present in the `bin` directory, set RUSTC to prevent - # cargo uses the rustc installed in the system. - if self.is_windows(): - cargo_path = os.path.join(bindir, "cargo.exe") - rustc_path = os.path.join(bindir, "rustc.exe") - rustdoc_path = os.path.join(bindir, "rustdoc.exe") - else: - cargo_path = os.path.join(bindir, "cargo") - rustc_path = os.path.join(bindir, "rustc") - rustdoc_path = os.path.join(bindir, "rustdoc") - - if os.path.isfile(rustc_path): - env["CARGO_BIN"] = cargo_path - env["RUSTC"] = rustc_path - env["RUSTDOC"] = rustdoc_path - - openssl_include = os.path.join(d, "include/openssl") - if os.path.isdir(openssl_include) and any( - os.path.isfile(os.path.join(d, "lib", libcrypto)) - for libcrypto in ("libcrypto.lib", "libcrypto.so", "libcrypto.a") - ): - # This must be the openssl library, let Rust know about it - env["OPENSSL_DIR"] = d - # And let openssl know to pick up the system certs if present - for system_ssl_cfg in ["/etc/pki/tls", "/etc/ssl"]: - if os.path.isdir(system_ssl_cfg): - cert_dir = system_ssl_cfg + "/certs" - if os.path.isdir(cert_dir): - env["SSL_CERT_DIR"] = cert_dir - cert_file = system_ssl_cfg + "/cert.pem" - if os.path.isfile(cert_file): - env["SSL_CERT_FILE"] = cert_file - - # Try extra hard to find openssl, needed with homebrew on macOS - if ( - self.is_darwin() - and "OPENSSL_DIR" not in env - and "OPENSSL_ROOT_DIR" in os.environ - ): - env["OPENSSL_ROOT_DIR"] = os.environ["OPENSSL_ROOT_DIR"] + # Let openssl know to pick up the system certs if present + if system_openssl or "OPENSSL_DIR" in env: + for system_ssl_cfg in ["/etc/pki/tls", "/etc/ssl"]: + if os.path.isdir(system_ssl_cfg): + cert_dir = system_ssl_cfg + "/certs" + if os.path.isdir(cert_dir): + env["SSL_CERT_DIR"] = cert_dir + cert_file = system_ssl_cfg + "/cert.pem" + if os.path.isfile(cert_file): + env["SSL_CERT_FILE"] = cert_file return env + def add_homebrew_package_to_env(self, package, env): + prefix = homebrew_package_prefix(package) + if prefix and os.path.exists(prefix): + return self.add_prefix_to_env( + prefix, env, append=False, add_library_path=True + ) + return False + + def add_prefix_to_env( + self, d, env, append=True, add_library_path=False + ): # noqa: C901 + bindir = os.path.join(d, "bin") + found = False + pkgconfig = os.path.join(d, "lib", "pkgconfig") + if os.path.exists(pkgconfig): + found = True + add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig, append=append) + + pkgconfig = os.path.join(d, "lib64", "pkgconfig") + if os.path.exists(pkgconfig): + found = True + add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig, append=append) + + add_path_entry(env, "CMAKE_PREFIX_PATH", d, append=append) + + # Tell the thrift compiler about includes it needs to consider + thriftdir = os.path.join(d, "include", "thrift-files") + if os.path.exists(thriftdir): + found = True + add_path_entry(env, "THRIFT_INCLUDE_PATH", thriftdir, append=append) + + # module detection for python is old fashioned and needs flags + includedir = os.path.join(d, "include") + if os.path.exists(includedir): + found = True + ncursesincludedir = os.path.join(d, "include", "ncurses") + if os.path.exists(ncursesincludedir): + add_path_entry(env, "C_INCLUDE_PATH", ncursesincludedir, append=append) + add_flag(env, "CPPFLAGS", f"-I{includedir}", append=append) + add_flag(env, "CPPFLAGS", f"-I{ncursesincludedir}", append=append) + elif "/bz2-" in d: + add_flag(env, "CPPFLAGS", f"-I{includedir}", append=append) + + # Map from FB python manifests to PYTHONPATH + pydir = os.path.join(d, "lib", "fb-py-libs") + if os.path.exists(pydir): + found = True + manifest_ext = ".manifest" + pymanifestfiles = [ + f + for f in os.listdir(pydir) + if f.endswith(manifest_ext) and os.path.isfile(os.path.join(pydir, f)) + ] + for f in pymanifestfiles: + subdir = f[: -len(manifest_ext)] + add_path_entry( + env, "PYTHONPATH", os.path.join(pydir, subdir), append=append + ) + + # Allow resolving shared objects built earlier (eg: zstd + # doesn't include the full path to the dylib in its linkage + # so we need to give it an assist) + if self.lib_path: + for lib in ["lib", "lib64"]: + libdir = os.path.join(d, lib) + if os.path.exists(libdir): + found = True + add_path_entry(env, self.lib_path, libdir, append=append) + # module detection for python is old fashioned and needs flags + if "/ncurses-" in d: + add_flag(env, "LDFLAGS", f"-L{libdir}", append=append) + elif "/bz2-" in d: + add_flag(env, "LDFLAGS", f"-L{libdir}", append=append) + if add_library_path: + add_path_entry(env, "LIBRARY_PATH", libdir, append=append) + + # Allow resolving binaries (eg: cmake, ninja) and dlls + # built by earlier steps + if os.path.exists(bindir): + found = True + add_path_entry(env, "PATH", bindir, append=append) + + # If rustc is present in the `bin` directory, set RUSTC to prevent + # cargo uses the rustc installed in the system. + if self.is_windows(): + cargo_path = os.path.join(bindir, "cargo.exe") + rustc_path = os.path.join(bindir, "rustc.exe") + rustdoc_path = os.path.join(bindir, "rustdoc.exe") + else: + cargo_path = os.path.join(bindir, "cargo") + rustc_path = os.path.join(bindir, "rustc") + rustdoc_path = os.path.join(bindir, "rustdoc") + + if os.path.isfile(rustc_path): + env["CARGO_BIN"] = cargo_path + env["RUSTC"] = rustc_path + env["RUSTDOC"] = rustdoc_path + + openssl_include = os.path.join(d, "include", "openssl") + if os.path.isdir(openssl_include) and any( + os.path.isfile(os.path.join(d, "lib", libcrypto)) + for libcrypto in ("libcrypto.lib", "libcrypto.so", "libcrypto.a") + ): + # This must be the openssl library, let Rust know about it + env["OPENSSL_DIR"] = d + + return found + def list_win32_subst_letters(): output = subprocess.check_output(["subst"]).decode("utf-8") @@ -536,9 +613,3 @@ def setup_build_options(args, host_type=None): facebook_internal=args.facebook_internal, **build_args, ) - - -def add_homebrew_package_to_path(env, package): - prefix = homebrew_package_prefix(package) - if prefix and os.path.exists(os.path.join(prefix, "bin")): - add_path_entry(env, "PATH", os.path.join(prefix, "bin"), append=False) diff --git a/build/fbcode_builder/getdeps/fetcher.py b/build/fbcode_builder/getdeps/fetcher.py index 50483b0f3..ca4eb242a 100644 --- a/build/fbcode_builder/getdeps/fetcher.py +++ b/build/fbcode_builder/getdeps/fetcher.py @@ -188,19 +188,6 @@ class SystemPackageFetcher(object): else: self.installed = False - # Hack to make openssl discovery with homebrew work. If openssl was - # built with autoconf we could use autoconf.envcmd.OPENSSL_ROOT_DIR - # from the manifest, but it isn't, so handle the special case here. - if ( - self.installed - and self.host_type.is_darwin() - and self.manager == "homebrew" - and "openssl@1.1" in self.packages - ): - candidate = homebrew_package_prefix("openssl@1.1") - if os.path.exists(candidate): - os.environ["OPENSSL_ROOT_DIR"] = candidate - return bool(self.installed) def update(self): diff --git a/build/fbcode_builder/getdeps/load.py b/build/fbcode_builder/getdeps/load.py index 481900986..1b26bbff9 100644 --- a/build/fbcode_builder/getdeps/load.py +++ b/build/fbcode_builder/getdeps/load.py @@ -4,6 +4,7 @@ # LICENSE file in the root directory of this source tree. import base64 +import copy import hashlib import os @@ -178,6 +179,7 @@ class ManifestLoader(object): deps = [manifest] # The list of manifests in dependency order dep_order = [] + system_packages = {} while len(deps) > 0: m = deps.pop(0) @@ -212,6 +214,19 @@ class ManifestLoader(object): # Its deps are done, so we can emit it seen.add(m.name) + # Capture system packages as we may need to set PATHs to then later + if ( + self.build_opts.allow_system_packages + and self.build_opts.host_type.get_package_manager() + ): + packages = m.get_required_system_packages(ctx) + for pkg_type, v in packages.items(): + merged = system_packages.get(pkg_type, []) + if v not in merged: + merged += v + system_packages[pkg_type] = merged + # A manifest depends on all system packages in it dependencies as well + m.resolved_system_packages = copy.copy(system_packages) dep_order.append(m) return dep_order diff --git a/build/fbcode_builder/getdeps/manifest.py b/build/fbcode_builder/getdeps/manifest.py index 2c43f86e1..99c47bb2f 100644 --- a/build/fbcode_builder/getdeps/manifest.py +++ b/build/fbcode_builder/getdeps/manifest.py @@ -77,6 +77,12 @@ SCHEMA = { "manifests_to_build": OPTIONAL, }, }, + "github.actions": { + "optional_section": True, + "fields": { + "run_tests": OPTIONAL, + }, + }, "cmake.defines": {"optional_section": True}, "autoconf.args": {"optional_section": True}, "autoconf.envcmd.LDFLAGS": {"optional_section": True}, @@ -115,6 +121,7 @@ ALLOWED_EXPR_SECTIONS = [ "shipit.pathmap", "shipit.strip", "homebrew", + "github.actions", ] @@ -223,6 +230,7 @@ class ManifestParser(object): self.fbsource_path = self.get("manifest", "fbsource_path") self.shipit_project = self.get("manifest", "shipit_project") self.shipit_fbcode_builder = self.get("manifest", "shipit_fbcode_builder") + self.resolved_system_packages = {} if self.name != os.path.basename(file_name): raise Exception( @@ -378,6 +386,9 @@ class ManifestParser(object): return True + def get_repo_url(self, ctx): + return self.get("git", "repo_url", ctx=ctx) + def create_fetcher(self, build_options, ctx): use_real_shipit = ( ShipitTransformerFetcher.available() and build_options.use_shipit @@ -409,7 +420,7 @@ class ManifestParser(object): if package_fetcher.packages_are_installed(): return package_fetcher - repo_url = self.get("git", "repo_url", ctx=ctx) + repo_url = self.get_repo_url(ctx) if repo_url: rev = self.get("git", "rev") depth = self.get("git", "depth") diff --git a/build/fbcode_builder/manifests/boost b/build/fbcode_builder/manifests/boost index f544114e2..82ae6e542 100644 --- a/build/fbcode_builder/manifests/boost +++ b/build/fbcode_builder/manifests/boost @@ -19,6 +19,8 @@ libboost-all-dev [homebrew] boost +# Boost cmake detection on homebrew adds this as requirement: https://github.com/Homebrew/homebrew-core/issues/67427#issuecomment-754187345 +icu4c [rpms.all(distro=centos_stream,distro_vers=8)] boost169 @@ -55,6 +57,7 @@ boost169-program-options [rpms.not(all(distro=centos_stream,distro_vers=8))] boost-devel +boost-static [build] builder = boost diff --git a/build/fbcode_builder/manifests/bz2 b/build/fbcode_builder/manifests/bz2 new file mode 100644 index 000000000..af2f357d5 --- /dev/null +++ b/build/fbcode_builder/manifests/bz2 @@ -0,0 +1,30 @@ +[manifest] +name = bz2 + +[debs] +libbz2-dev + +[homebrew] +bzip2 + +[rpms] +bzip2-devel + +[download] +url = https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz +sha256 = ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269 + +[build.not(os=windows)] +builder = make +subdir = bzip2-1.0.8 + +[make.build_args.os=linux] +# python bz2 support on linux needs dynamic library +-f +Makefile-libbz2_so + +[make.install_args] +install + +[build.os=windows] +builder = nop diff --git a/build/fbcode_builder/manifests/eden b/build/fbcode_builder/manifests/eden index dfc3e8bb1..6509270f6 100644 --- a/build/fbcode_builder/manifests/eden +++ b/build/fbcode_builder/manifests/eden @@ -7,6 +7,9 @@ shipit_fbcode_builder = true [git] repo_url = https://github.com/facebookexperimental/eden.git +[github.actions] +run_tests = off + [build] builder = cmake @@ -37,11 +40,17 @@ sqlite3 [dependencies.os=darwin] osxfuse +[dependencies.not(os=windows)] # TODO: teach getdeps to compile curl on Windows. # Enabling curl on Windows requires us to find a way to compile libcurl with # msvc. -[dependencies.not(os=windows)] libcurl +# Added so that OSS doesn't see system "python" which is python 2 on darwin and some linux +python + +[dependencies.all(fb=off)] +# Outside Meta hg is not installed, or if it is, its not the one we want to test with +eden_scm [shipit.pathmap.fb=on] # for internal builds that use getdeps diff --git a/build/fbcode_builder/manifests/ncurses b/build/fbcode_builder/manifests/ncurses new file mode 100644 index 000000000..1bb5e8a45 --- /dev/null +++ b/build/fbcode_builder/manifests/ncurses @@ -0,0 +1,30 @@ +[manifest] +name = ncurses + +[debs] +libncurses-dev + +[homebrew] +ncurses + +[rpms] +ncurses-devel + +[download] +url = https://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.3.tar.gz +sha256 = 97fc51ac2b085d4cde31ef4d2c3122c21abc217e9090a43a30fc5ec21684e059 + +[build.not(os=windows)] +builder = autoconf +subdir = ncurses-6.3 + +[autoconf.args] +--without-cxx-binding +--without-ada + +[autoconf.args.os=linux] +--enable-shared +--with-shared + +[build.os=windows] +builder = nop diff --git a/build/fbcode_builder/manifests/openssl b/build/fbcode_builder/manifests/openssl index 5dcfc3868..b806b701a 100644 --- a/build/fbcode_builder/manifests/openssl +++ b/build/fbcode_builder/manifests/openssl @@ -6,6 +6,7 @@ libssl-dev [homebrew] openssl@1.1 +# on homebrew need the matching curl and ca- [rpms] openssl diff --git a/build/fbcode_builder/manifests/python b/build/fbcode_builder/manifests/python index f9877e783..de7b6da7f 100644 --- a/build/fbcode_builder/manifests/python +++ b/build/fbcode_builder/manifests/python @@ -36,3 +36,7 @@ libffi [dependencies] libffi +# eden tests expect the python bz2 support +bz2 +# eden tests expect the python curses support +ncurses