add github actions for EdenFS on linux and fix Eden SCM Mac build (#106)

Summary:
Pull Request resolved: https://github.com/facebookexperimental/eden/pull/106

Pull Request resolved: https://github.com/facebookexperimental/eden/pull/107

Summary

* Add EdenFS builds on external CI now EdenSCM is good

* Mac builds on github actions by using brew for system dependencies

To make this work had to fix some path ordering issues with install directories for Linux and Mac, and generalise the homebrew path fixups we were doing for bison to all the used homebrew packages.

Previously Installed packages were being added after system paths, so our own installed thing might be ignored. On github these meant system python 3.9 was being used for hg tests rather than our specified 3.8 (this showed we have some test fails on python 3.9 with "SystemError: deallocated bytearray object has exported buffers", that are beyond the scope of this diff to fix)

Also needed to include the getdeps generated python into the generated edenscmdeps3.zip archive setup.py produces otherwise EdenFS tests failed to import thrift.Thrift

Eden tests are hanging when run externally about half way through, so disable them on github actions for now as this PR is already fairly large. They work when run locally on an internal devserver, so probably some bit of environment necessary is not defined in the test runner

Reviewed By: chadaustin

Differential Revision: D34116505

fbshipit-source-id: d0d628db5daabc28d0bd8997cd5c1bc885ed1e73

build/fbcode_builder/getdeps.py

@@ -971,20 +971,26 @@ jobs:
                 out.write("    - name: Fix Git config\n")
                 out.write("      run: git config --system core.longpaths true\n")
 
-            projects = loader.manifests_in_dependency_order()
-
             allow_sys_arg = ""
             if (
                 build_opts.allow_system_packages
-                and build_opts.is_linux()
                 and build_opts.host_type.get_package_manager()
             ):
                 allow_sys_arg = " --allow-system-packages"
                 out.write("    - name: Install system deps\n")
+                sudo_arg = "sudo "
+                if build_opts.is_darwin():
+                    # brew is installed as regular user
+                    sudo_arg = ""
                 out.write(
-                    f"      run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive {manifest.name}\n"
+                    f"      run: {sudo_arg}python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive {manifest.name}\n"
                 )
 
+            projects = loader.manifests_in_dependency_order()
+
+            main_repo_url = manifest.get_repo_url(manifest_ctx)
+            has_same_repo_dep = False
+
             for m in projects:
                 if m != manifest:
                     if m.name == "rust":
@@ -995,19 +1001,27 @@ jobs:
                         out.write("        default: true\n")
                         out.write("        profile: minimal\n")
                     else:
-                        out.write("    - name: Fetch %s\n" % m.name)
+                        ctx = loader.ctx_gen.get_context(m.name)
-                        out.write(
+                        if m.get_repo_url(ctx) != main_repo_url:
-                            f"      run: {getdepscmd}{allow_sys_arg} fetch --no-tests {m.name}\n"
+                            out.write("    - name: Fetch %s\n" % m.name)
-                        )
+                            out.write(
+                                f"      run: {getdepscmd}{allow_sys_arg} fetch --no-tests {m.name}\n"
+                            )
 
             for m in projects:
                 if m != manifest:
                     if m.name == "rust":
                         continue
                     else:
+                        src_dir_arg = ""
+                        ctx = loader.ctx_gen.get_context(m.name)
+                        if main_repo_url and m.get_repo_url(ctx) == main_repo_url:
+                            # Its in the same repo, so src-dir is also .
+                            src_dir_arg = "--src-dir=. "
+                            has_same_repo_dep = True
                         out.write("    - name: Build %s\n" % m.name)
                         out.write(
-                            f"      run: {getdepscmd}{allow_sys_arg} build --no-tests {m.name}\n"
+                            f"      run: {getdepscmd}{allow_sys_arg} build {src_dir_arg}--no-tests {m.name}\n"
                         )
 
             out.write("    - name: Build %s\n" % manifest.name)
@@ -1018,8 +1032,13 @@ jobs:
                     " --project-install-prefix %s:/usr/local" % manifest.name
                 )
 
+            # If we have dep from same repo, we already built it and don't want to rebuild it again
+            no_deps_arg = ""
+            if has_same_repo_dep:
+                no_deps_arg = "--no-deps "
+
             out.write(
-                f"      run: {getdepscmd}{allow_sys_arg} build --src-dir=. {manifest.name} {project_prefix}\n"
+                f"      run: {getdepscmd}{allow_sys_arg} build {no_deps_arg}--src-dir=. {manifest.name} {project_prefix}\n"
             )
 
             out.write("    - name: Copy artifacts\n")
@@ -1043,10 +1062,11 @@ jobs:
             out.write("        name: %s\n" % manifest.name)
             out.write("        path: _artifacts\n")
 
-            out.write("    - name: Test %s\n" % manifest.name)
+            if manifest.get("github.actions", "run_tests", ctx=manifest_ctx) != "off":
-            out.write(
+                out.write("    - name: Test %s\n" % manifest.name)
-                f"      run: {getdepscmd}{allow_sys_arg} test --src-dir=. {manifest.name} {project_prefix}\n"
+                out.write(
-            )
+                    f"      run: {getdepscmd}{allow_sys_arg} test --src-dir=. {manifest.name} {project_prefix}\n"
+                )
 
     def setup_project_cmd_parser(self, parser):
         parser.add_argument(

build/fbcode_builder/getdeps/builder.py

@@ -4,6 +4,7 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 
+import glob
 import json
 import os
 import shutil
@@ -197,6 +198,14 @@ class MakeBuilder(BuilderBase):
         install_cmd = [self._make_binary] + self.install_args + self._get_prefix()
         self._run_cmd(install_cmd, env=env)
 
+        # bz2's Makefile doesn't install its .so properly
+        if self.manifest and self.manifest.name == "bz2":
+            libdir = os.path.join(self.inst_dir, "lib")
+            srcpattern = os.path.join(self.src_dir, "lib*.so.*")
+            print(f"copying to {libdir} from {srcpattern}")
+            for file in glob.glob(srcpattern):
+                shutil.copy(file, libdir)
+
     def run_tests(
         self, install_dirs, schedule_type, owner, test_filter, retry, no_testpilot
     ):
@@ -1185,308 +1194,3 @@ install(FILES sqlite3.h sqlite3ext.h DESTINATION include)
             ],
             env=env,
         )
-
-
-class CargoBuilder(BuilderBase):
-    def __init__(
-        self,
-        build_opts,
-        ctx,
-        manifest,
-        src_dir,
-        build_dir,
-        inst_dir,
-        build_doc,
-        workspace_dir,
-        manifests_to_build,
-        loader,
-    ):
-        super(CargoBuilder, self).__init__(
-            build_opts, ctx, manifest, src_dir, build_dir, inst_dir
-        )
-        self.build_doc = build_doc
-        self.ws_dir = workspace_dir
-        self.manifests_to_build = manifests_to_build and manifests_to_build.split(",")
-        self.loader = loader
-
-    def run_cargo(self, install_dirs, operation, args=None):
-        args = args or []
-        env = self._compute_env(install_dirs)
-        # Enable using nightly features with stable compiler
-        env["RUSTC_BOOTSTRAP"] = "1"
-        env["LIBZ_SYS_STATIC"] = "1"
-        cmd = [
-            "cargo",
-            operation,
-            "--workspace",
-            "-j%s" % self.num_jobs,
-        ] + args
-        self._run_cmd(cmd, cwd=self.workspace_dir(), env=env)
-
-    def build_source_dir(self):
-        return os.path.join(self.build_dir, "source")
-
-    def workspace_dir(self):
-        return os.path.join(self.build_source_dir(), self.ws_dir or "")
-
-    def manifest_dir(self, manifest):
-        return os.path.join(self.build_source_dir(), manifest)
-
-    def recreate_dir(self, src, dst):
-        if os.path.isdir(dst):
-            shutil.rmtree(dst)
-        shutil.copytree(src, dst)
-
-    def _build(self, install_dirs, reconfigure):
-        build_source_dir = self.build_source_dir()
-        self.recreate_dir(self.src_dir, build_source_dir)
-
-        dot_cargo_dir = os.path.join(build_source_dir, ".cargo")
-        if not os.path.isdir(dot_cargo_dir):
-            os.mkdir(dot_cargo_dir)
-
-        with open(os.path.join(dot_cargo_dir, "config"), "w+") as f:
-            f.write(
-                """\
-[build]
-target-dir = '''{}'''
-
-[net]
-git-fetch-with-cli = true
-
-[profile.dev]
-debug = false
-incremental = false
-""".format(
-                    self.build_dir.replace("\\", "\\\\")
-                )
-            )
-
-        if self.ws_dir is not None:
-            self._patchup_workspace()
-
-        try:
-            from .facebook.rust import vendored_crates
-
-            vendored_crates(self.build_opts, build_source_dir)
-        except ImportError:
-            # This FB internal module isn't shippped to github,
-            # so just rely on cargo downloading crates on it's own
-            pass
-
-        if self.manifests_to_build is None:
-            self.run_cargo(
-                install_dirs,
-                "build",
-                ["--out-dir", os.path.join(self.inst_dir, "bin"), "-Zunstable-options"],
-            )
-        else:
-            for manifest in self.manifests_to_build:
-                self.run_cargo(
-                    install_dirs,
-                    "build",
-                    [
-                        "--out-dir",
-                        os.path.join(self.inst_dir, "bin"),
-                        "-Zunstable-options",
-                        "--manifest-path",
-                        self.manifest_dir(manifest),
-                    ],
-                )
-
-        self.recreate_dir(build_source_dir, os.path.join(self.inst_dir, "source"))
-
-    def run_tests(
-        self, install_dirs, schedule_type, owner, test_filter, retry, no_testpilot
-    ):
-        if test_filter:
-            args = ["--", test_filter]
-        else:
-            args = []
-
-        if self.manifests_to_build is None:
-            self.run_cargo(install_dirs, "test", args)
-            if self.build_doc:
-                self.run_cargo(install_dirs, "doc", ["--no-deps"])
-        else:
-            for manifest in self.manifests_to_build:
-                margs = ["--manifest-path", self.manifest_dir(manifest)]
-                self.run_cargo(install_dirs, "test", args + margs)
-                if self.build_doc:
-                    self.run_cargo(install_dirs, "doc", ["--no-deps"] + margs)
-
-    def _patchup_workspace(self):
-        """
-        This method makes some assumptions about the state of the project and
-        its cargo dependendies:
-        1. Crates from cargo dependencies can be extracted from Cargo.toml files
-           using _extract_crates function. It is using a heuristic so check its
-           code to understand how it is done.
-        2. The extracted cargo dependencies crates can be found in the
-           dependency's install dir using _resolve_crate_to_path function
-           which again is using a heuristic.
-
-        Notice that many things might go wrong here. E.g. if someone depends
-        on another getdeps crate by writing in their Cargo.toml file:
-
-            my-rename-of-crate = { package = "crate", git = "..." }
-
-        they can count themselves lucky because the code will raise an
-        Exception. There migh be more cases where the code will silently pass
-        producing bad results.
-        """
-        workspace_dir = self.workspace_dir()
-        config = self._resolve_config()
-        if config:
-            with open(os.path.join(workspace_dir, "Cargo.toml"), "r+") as f:
-                manifest_content = f.read()
-                if "[package]" not in manifest_content:
-                    # A fake manifest has to be crated to change the virtual
-                    # manifest into a non-virtual. The virtual manifests are limited
-                    # in many ways and the inability to define patches on them is
-                    # one. Check https://github.com/rust-lang/cargo/issues/4934 to
-                    # see if it is resolved.
-                    f.write(
-                        """
-    [package]
-    name = "fake_manifest_of_{}"
-    version = "0.0.0"
-    [lib]
-    path = "/dev/null"
-    """.format(
-                            self.manifest.name
-                        )
-                    )
-                else:
-                    f.write("\n")
-                f.write(config)
-
-    def _resolve_config(self):
-        """
-        Returns a configuration to be put inside root Cargo.toml file which
-        patches the dependencies git code with local getdeps versions.
-        See https://doc.rust-lang.org/cargo/reference/manifest.html#the-patch-section
-        """
-        dep_to_git = self._resolve_dep_to_git()
-        dep_to_crates = CargoBuilder._resolve_dep_to_crates(
-            self.build_source_dir(), dep_to_git
-        )
-
-        config = []
-        for name in sorted(dep_to_git.keys()):
-            git_conf = dep_to_git[name]
-            crates = sorted(dep_to_crates.get(name, []))
-            if not crates:
-                continue  # nothing to patch, move along
-            crates_patches = [
-                '{} = {{ path = "{}" }}'.format(
-                    crate,
-                    CargoBuilder._resolve_crate_to_path(crate, git_conf).replace(
-                        "\\", "\\\\"
-                    ),
-                )
-                for crate in crates
-            ]
-
-            config.append(
-                '[patch."{0}"]\n'.format(git_conf["repo_url"])
-                + "\n".join(crates_patches)
-            )
-        return "\n".join(config)
-
-    def _resolve_dep_to_git(self):
-        """
-        For each direct dependency of the currently build manifest check if it
-        is also cargo-builded and if yes then extract it's git configs and
-        install dir
-        """
-        dependencies = self.manifest.get_dependencies(self.ctx)
-        if not dependencies:
-            return []
-
-        dep_to_git = {}
-        for dep in dependencies:
-            dep_manifest = self.loader.load_manifest(dep)
-            dep_builder = dep_manifest.get("build", "builder", ctx=self.ctx)
-            if dep_builder not in ["cargo", "nop"] or dep == "rust":
-                # This is a direct dependency, but it is not build with cargo
-                # and it is not simply copying files with nop, so ignore it.
-                # The "rust" dependency is an exception since it contains the
-                # toolchain.
-                continue
-
-            git_conf = dep_manifest.get_section_as_dict("git", self.ctx)
-            if "repo_url" not in git_conf:
-                raise Exception(
-                    "A cargo dependency requires git.repo_url to be defined."
-                )
-            source_dir = self.loader.get_project_install_dir(dep_manifest)
-            if dep_builder == "cargo":
-                source_dir = os.path.join(source_dir, "source")
-            git_conf["source_dir"] = source_dir
-            dep_to_git[dep] = git_conf
-        return dep_to_git
-
-    @staticmethod
-    def _resolve_dep_to_crates(build_source_dir, dep_to_git):
-        """
-        This function traverse the build_source_dir in search of Cargo.toml
-        files, extracts the crate names from them using _extract_crates
-        function and returns a merged result containing crate names per
-        dependency name from all Cargo.toml files in the project.
-        """
-        if not dep_to_git:
-            return {}  # no deps, so don't waste time traversing files
-
-        dep_to_crates = {}
-        for root, _, files in os.walk(build_source_dir):
-            for f in files:
-                if f == "Cargo.toml":
-                    more_dep_to_crates = CargoBuilder._extract_crates(
-                        os.path.join(root, f), dep_to_git
-                    )
-                    for name, crates in more_dep_to_crates.items():
-                        dep_to_crates.setdefault(name, set()).update(crates)
-        return dep_to_crates
-
-    @staticmethod
-    def _extract_crates(cargo_toml_file, dep_to_git):
-        """
-        This functions reads content of provided cargo toml file and extracts
-        crate names per each dependency. The extraction is done by a heuristic
-        so it might be incorrect.
-        """
-        deps_to_crates = {}
-        with open(cargo_toml_file, "r") as f:
-            for line in f.readlines():
-                if line.startswith("#") or "git = " not in line:
-                    continue  # filter out commented lines and ones without git deps
-                for name, conf in dep_to_git.items():
-                    if 'git = "{}"'.format(conf["repo_url"]) in line:
-                        pkg_template = ' package = "'
-                        if pkg_template in line:
-                            crate_name, _, _ = line.partition(pkg_template)[
-                                2
-                            ].partition('"')
-                        else:
-                            crate_name, _, _ = line.partition("=")
-                        deps_to_crates.setdefault(name, set()).add(crate_name.strip())
-        return deps_to_crates
-
-    @staticmethod
-    def _resolve_crate_to_path(crate, git_conf):
-        """
-        Tries to find <crate> in git_conf["inst_dir"] by searching a [package]
-        keyword followed by name = "<crate>".
-        """
-        source_dir = git_conf["source_dir"]
-        search_pattern = '[package]\nname = "{}"'.format(crate)
-
-        for root, _, files in os.walk(source_dir):
-            for fname in files:
-                if fname == "Cargo.toml":
-                    with open(os.path.join(root, fname), "r") as f:
-                        if search_pattern in f.read():
-                            return root
-
-        raise Exception("Failed to found crate {} in path {}".format(crate, source_dir))

build/fbcode_builder/getdeps/buildopts.py

@@ -13,7 +13,7 @@ import tempfile
 from typing import Optional, Mapping
 
 from .copytree import containing_repo_type
-from .envfuncs import Env, add_path_entry
+from .envfuncs import Env, add_flag, add_path_entry
 from .fetcher import get_fbsource_repo_data, homebrew_package_prefix
 from .manifest import ContextGenerator
 from .platform import HostType, is_windows, get_available_ram
@@ -103,6 +103,18 @@ class BuildOptions(object):
         self.allow_system_packages = allow_system_packages
         self.lfs_path = lfs_path
         self.shared_libs = shared_libs
+
+        lib_path = None
+        if self.is_darwin():
+            lib_path = "DYLD_LIBRARY_PATH"
+        elif self.is_linux():
+            lib_path = "LD_LIBRARY_PATH"
+        elif self.is_windows():
+            lib_path = "PATH"
+        else:
+            lib_path = None
+        self.lib_path = lib_path
+
         if vcvars_path is None and is_windows():
 
             try:
@@ -196,7 +208,9 @@ class BuildOptions(object):
             }
         )
 
-    def compute_env_for_install_dirs(self, install_dirs, env=None, manifest=None):
+    def compute_env_for_install_dirs(
+        self, install_dirs, env=None, manifest=None
+    ):  # noqa: C901
         if env is not None:
             env = env.copy()
         else:
@@ -236,11 +250,23 @@ class BuildOptions(object):
             sdkroot = subprocess.check_output(["xcrun", "--show-sdk-path"])
             env["SDKROOT"] = sdkroot.decode().strip()
 
-        # MacOS includes a version of bison so homebrew won't automatically add
+        if (
-        # its own version to PATH. Find where the homebrew bison is and prepend
+            self.is_darwin()
-        # it to PATH.
+            and self.allow_system_packages
-        if self.is_darwin() and self.host_type.get_package_manager() == "homebrew":
+            and self.host_type.get_package_manager() == "homebrew"
-            add_homebrew_package_to_path(env, "bison")
+            and manifest
+            and manifest.resolved_system_packages
+        ):
+            # Homebrew packages may not be on the default PATHs
+            brew_packages = manifest.resolved_system_packages.get("homebrew", [])
+            for p in brew_packages:
+                found = self.add_homebrew_package_to_env(p, env)
+                # Try extra hard to find openssl, needed with homebrew on macOS
+                if found and p.startswith("openssl"):
+                    candidate = homebrew_package_prefix("openssl@1.1")
+                    if os.path.exists(candidate):
+                        os.environ["OPENSSL_ROOT_DIR"] = candidate
+                        env["OPENSSL_ROOT_DIR"] = os.environ["OPENSSL_ROOT_DIR"]
 
         if self.fbsource_dir:
             env["YARN_YARN_OFFLINE_MIRROR"] = os.path.join(
@@ -261,91 +287,142 @@ class BuildOptions(object):
             env["FBSOURCE_HASH"] = hash_data.hash
             env["FBSOURCE_DATE"] = hash_data.date
 
-        lib_path = None
+        # reverse as we are prepending to the PATHs
-        if self.is_darwin():
+        for d in reversed(install_dirs):
-            lib_path = "DYLD_LIBRARY_PATH"
+            self.add_prefix_to_env(d, env, append=False)
-        elif self.is_linux():
-            lib_path = "LD_LIBRARY_PATH"
-        elif self.is_windows():
-            lib_path = "PATH"
-        else:
-            lib_path = None
 
-        for d in install_dirs:
+        # Linux is always system openssl
-            bindir = os.path.join(d, "bin")
+        system_openssl = self.is_linux()
 
-            pkgconfig = os.path.join(d, "lib/pkgconfig")
+        # For other systems lets see if package is requested
-            if os.path.exists(pkgconfig):
+        if not system_openssl and manifest and manifest.resolved_system_packages:
-                add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig)
+            for _pkg_type, pkgs in manifest.resolved_system_packages.items():
+                for p in pkgs:
+                    if p.startswith("openssl") or p.startswith("libssl"):
+                        system_openssl = True
+                        break
 
-            pkgconfig = os.path.join(d, "lib64/pkgconfig")
+        # Let openssl know to pick up the system certs if present
-            if os.path.exists(pkgconfig):
+        if system_openssl or "OPENSSL_DIR" in env:
-                add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig)
+            for system_ssl_cfg in ["/etc/pki/tls", "/etc/ssl"]:
-
+                if os.path.isdir(system_ssl_cfg):
-            add_path_entry(env, "CMAKE_PREFIX_PATH", d)
+                    cert_dir = system_ssl_cfg + "/certs"
-
+                    if os.path.isdir(cert_dir):
-            # Tell the thrift compiler about includes it needs to consider
+                        env["SSL_CERT_DIR"] = cert_dir
-            thriftdir = os.path.join(d, "include/thrift-files")
+                    cert_file = system_ssl_cfg + "/cert.pem"
-            if os.path.exists(thriftdir):
+                    if os.path.isfile(cert_file):
-                add_path_entry(env, "THRIFT_INCLUDE_PATH", thriftdir)
+                        env["SSL_CERT_FILE"] = cert_file
-
-            # Allow resolving shared objects built earlier (eg: zstd
-            # doesn't include the full path to the dylib in its linkage
-            # so we need to give it an assist)
-            if lib_path:
-                for lib in ["lib", "lib64"]:
-                    libdir = os.path.join(d, lib)
-                    if os.path.exists(libdir):
-                        add_path_entry(env, lib_path, libdir)
-
-            # Allow resolving binaries (eg: cmake, ninja) and dlls
-            # built by earlier steps
-            if os.path.exists(bindir):
-                add_path_entry(env, "PATH", bindir, append=False)
-
-            # If rustc is present in the `bin` directory, set RUSTC to prevent
-            # cargo uses the rustc installed in the system.
-            if self.is_windows():
-                cargo_path = os.path.join(bindir, "cargo.exe")
-                rustc_path = os.path.join(bindir, "rustc.exe")
-                rustdoc_path = os.path.join(bindir, "rustdoc.exe")
-            else:
-                cargo_path = os.path.join(bindir, "cargo")
-                rustc_path = os.path.join(bindir, "rustc")
-                rustdoc_path = os.path.join(bindir, "rustdoc")
-
-            if os.path.isfile(rustc_path):
-                env["CARGO_BIN"] = cargo_path
-                env["RUSTC"] = rustc_path
-                env["RUSTDOC"] = rustdoc_path
-
-            openssl_include = os.path.join(d, "include/openssl")
-            if os.path.isdir(openssl_include) and any(
-                os.path.isfile(os.path.join(d, "lib", libcrypto))
-                for libcrypto in ("libcrypto.lib", "libcrypto.so", "libcrypto.a")
-            ):
-                # This must be the openssl library, let Rust know about it
-                env["OPENSSL_DIR"] = d
-                # And let openssl know to pick up the system certs if present
-                for system_ssl_cfg in ["/etc/pki/tls", "/etc/ssl"]:
-                    if os.path.isdir(system_ssl_cfg):
-                        cert_dir = system_ssl_cfg + "/certs"
-                        if os.path.isdir(cert_dir):
-                            env["SSL_CERT_DIR"] = cert_dir
-                        cert_file = system_ssl_cfg + "/cert.pem"
-                        if os.path.isfile(cert_file):
-                            env["SSL_CERT_FILE"] = cert_file
-
-        # Try extra hard to find openssl, needed with homebrew on macOS
-        if (
-            self.is_darwin()
-            and "OPENSSL_DIR" not in env
-            and "OPENSSL_ROOT_DIR" in os.environ
-        ):
-            env["OPENSSL_ROOT_DIR"] = os.environ["OPENSSL_ROOT_DIR"]
 
         return env
 
+    def add_homebrew_package_to_env(self, package, env):
+        prefix = homebrew_package_prefix(package)
+        if prefix and os.path.exists(prefix):
+            return self.add_prefix_to_env(
+                prefix, env, append=False, add_library_path=True
+            )
+        return False
+
+    def add_prefix_to_env(
+        self, d, env, append=True, add_library_path=False
+    ):  # noqa: C901
+        bindir = os.path.join(d, "bin")
+        found = False
+        pkgconfig = os.path.join(d, "lib", "pkgconfig")
+        if os.path.exists(pkgconfig):
+            found = True
+            add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig, append=append)
+
+        pkgconfig = os.path.join(d, "lib64", "pkgconfig")
+        if os.path.exists(pkgconfig):
+            found = True
+            add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig, append=append)
+
+        add_path_entry(env, "CMAKE_PREFIX_PATH", d, append=append)
+
+        # Tell the thrift compiler about includes it needs to consider
+        thriftdir = os.path.join(d, "include", "thrift-files")
+        if os.path.exists(thriftdir):
+            found = True
+            add_path_entry(env, "THRIFT_INCLUDE_PATH", thriftdir, append=append)
+
+        # module detection for python is old fashioned and needs flags
+        includedir = os.path.join(d, "include")
+        if os.path.exists(includedir):
+            found = True
+            ncursesincludedir = os.path.join(d, "include", "ncurses")
+            if os.path.exists(ncursesincludedir):
+                add_path_entry(env, "C_INCLUDE_PATH", ncursesincludedir, append=append)
+                add_flag(env, "CPPFLAGS", f"-I{includedir}", append=append)
+                add_flag(env, "CPPFLAGS", f"-I{ncursesincludedir}", append=append)
+            elif "/bz2-" in d:
+                add_flag(env, "CPPFLAGS", f"-I{includedir}", append=append)
+
+        # Map from FB python manifests to PYTHONPATH
+        pydir = os.path.join(d, "lib", "fb-py-libs")
+        if os.path.exists(pydir):
+            found = True
+            manifest_ext = ".manifest"
+            pymanifestfiles = [
+                f
+                for f in os.listdir(pydir)
+                if f.endswith(manifest_ext) and os.path.isfile(os.path.join(pydir, f))
+            ]
+            for f in pymanifestfiles:
+                subdir = f[: -len(manifest_ext)]
+                add_path_entry(
+                    env, "PYTHONPATH", os.path.join(pydir, subdir), append=append
+                )
+
+        # Allow resolving shared objects built earlier (eg: zstd
+        # doesn't include the full path to the dylib in its linkage
+        # so we need to give it an assist)
+        if self.lib_path:
+            for lib in ["lib", "lib64"]:
+                libdir = os.path.join(d, lib)
+                if os.path.exists(libdir):
+                    found = True
+                    add_path_entry(env, self.lib_path, libdir, append=append)
+                    # module detection for python is old fashioned and needs flags
+                    if "/ncurses-" in d:
+                        add_flag(env, "LDFLAGS", f"-L{libdir}", append=append)
+                    elif "/bz2-" in d:
+                        add_flag(env, "LDFLAGS", f"-L{libdir}", append=append)
+                    if add_library_path:
+                        add_path_entry(env, "LIBRARY_PATH", libdir, append=append)
+
+        # Allow resolving binaries (eg: cmake, ninja) and dlls
+        # built by earlier steps
+        if os.path.exists(bindir):
+            found = True
+            add_path_entry(env, "PATH", bindir, append=append)
+
+        # If rustc is present in the `bin` directory, set RUSTC to prevent
+        # cargo uses the rustc installed in the system.
+        if self.is_windows():
+            cargo_path = os.path.join(bindir, "cargo.exe")
+            rustc_path = os.path.join(bindir, "rustc.exe")
+            rustdoc_path = os.path.join(bindir, "rustdoc.exe")
+        else:
+            cargo_path = os.path.join(bindir, "cargo")
+            rustc_path = os.path.join(bindir, "rustc")
+            rustdoc_path = os.path.join(bindir, "rustdoc")
+
+        if os.path.isfile(rustc_path):
+            env["CARGO_BIN"] = cargo_path
+            env["RUSTC"] = rustc_path
+            env["RUSTDOC"] = rustdoc_path
+
+        openssl_include = os.path.join(d, "include", "openssl")
+        if os.path.isdir(openssl_include) and any(
+            os.path.isfile(os.path.join(d, "lib", libcrypto))
+            for libcrypto in ("libcrypto.lib", "libcrypto.so", "libcrypto.a")
+        ):
+            # This must be the openssl library, let Rust know about it
+            env["OPENSSL_DIR"] = d
+
+        return found
+
 
 def list_win32_subst_letters():
     output = subprocess.check_output(["subst"]).decode("utf-8")
@@ -536,9 +613,3 @@ def setup_build_options(args, host_type=None):
         facebook_internal=args.facebook_internal,
         **build_args,
     )
-
-
-def add_homebrew_package_to_path(env, package):
-    prefix = homebrew_package_prefix(package)
-    if prefix and os.path.exists(os.path.join(prefix, "bin")):
-        add_path_entry(env, "PATH", os.path.join(prefix, "bin"), append=False)

build/fbcode_builder/getdeps/fetcher.py

@@ -188,19 +188,6 @@ class SystemPackageFetcher(object):
         else:
             self.installed = False
 
-        # Hack to make openssl discovery with homebrew work. If openssl was
-        # built with autoconf we could use autoconf.envcmd.OPENSSL_ROOT_DIR
-        # from the manifest, but it isn't, so handle the special case here.
-        if (
-            self.installed
-            and self.host_type.is_darwin()
-            and self.manager == "homebrew"
-            and "openssl@1.1" in self.packages
-        ):
-            candidate = homebrew_package_prefix("openssl@1.1")
-            if os.path.exists(candidate):
-                os.environ["OPENSSL_ROOT_DIR"] = candidate
-
         return bool(self.installed)
 
     def update(self):

build/fbcode_builder/getdeps/load.py

@@ -4,6 +4,7 @@
 # LICENSE file in the root directory of this source tree.
 
 import base64
+import copy
 import hashlib
 import os
 
@@ -178,6 +179,7 @@ class ManifestLoader(object):
             deps = [manifest]
         # The list of manifests in dependency order
         dep_order = []
+        system_packages = {}
 
         while len(deps) > 0:
             m = deps.pop(0)
@@ -212,6 +214,19 @@ class ManifestLoader(object):
 
             # Its deps are done, so we can emit it
             seen.add(m.name)
+            # Capture system packages as we may need to set PATHs to then later
+            if (
+                self.build_opts.allow_system_packages
+                and self.build_opts.host_type.get_package_manager()
+            ):
+                packages = m.get_required_system_packages(ctx)
+                for pkg_type, v in packages.items():
+                    merged = system_packages.get(pkg_type, [])
+                    if v not in merged:
+                        merged += v
+                    system_packages[pkg_type] = merged
+                # A manifest depends on all system packages in it dependencies as well
+                m.resolved_system_packages = copy.copy(system_packages)
             dep_order.append(m)
 
         return dep_order

build/fbcode_builder/getdeps/manifest.py

@@ -77,6 +77,12 @@ SCHEMA = {
             "manifests_to_build": OPTIONAL,
         },
     },
+    "github.actions": {
+        "optional_section": True,
+        "fields": {
+            "run_tests": OPTIONAL,
+        },
+    },
     "cmake.defines": {"optional_section": True},
     "autoconf.args": {"optional_section": True},
     "autoconf.envcmd.LDFLAGS": {"optional_section": True},
@@ -115,6 +121,7 @@ ALLOWED_EXPR_SECTIONS = [
     "shipit.pathmap",
     "shipit.strip",
     "homebrew",
+    "github.actions",
 ]
 
 
@@ -223,6 +230,7 @@ class ManifestParser(object):
         self.fbsource_path = self.get("manifest", "fbsource_path")
         self.shipit_project = self.get("manifest", "shipit_project")
         self.shipit_fbcode_builder = self.get("manifest", "shipit_fbcode_builder")
+        self.resolved_system_packages = {}
 
         if self.name != os.path.basename(file_name):
             raise Exception(
@@ -378,6 +386,9 @@ class ManifestParser(object):
 
         return True
 
+    def get_repo_url(self, ctx):
+        return self.get("git", "repo_url", ctx=ctx)
+
     def create_fetcher(self, build_options, ctx):
         use_real_shipit = (
             ShipitTransformerFetcher.available() and build_options.use_shipit
@@ -409,7 +420,7 @@ class ManifestParser(object):
             if package_fetcher.packages_are_installed():
                 return package_fetcher
 
-        repo_url = self.get("git", "repo_url", ctx=ctx)
+        repo_url = self.get_repo_url(ctx)
         if repo_url:
             rev = self.get("git", "rev")
             depth = self.get("git", "depth")

build/fbcode_builder/manifests/boost

@@ -19,6 +19,8 @@ libboost-all-dev
 
 [homebrew]
 boost
+# Boost cmake detection on homebrew adds this as requirement: https://github.com/Homebrew/homebrew-core/issues/67427#issuecomment-754187345
+icu4c
 
 [rpms.all(distro=centos_stream,distro_vers=8)]
 boost169
@@ -55,6 +57,7 @@ boost169-program-options
 
 [rpms.not(all(distro=centos_stream,distro_vers=8))]
 boost-devel
+boost-static
 
 [build]
 builder = boost

build/fbcode_builder/manifests/bz2

@@ -0,0 +1,30 @@
+[manifest]
+name = bz2
+
+[debs]
+libbz2-dev
+
+[homebrew]
+bzip2
+
+[rpms]
+bzip2-devel
+
+[download]
+url = https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz
+sha256 = ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269
+
+[build.not(os=windows)]
+builder = make
+subdir = bzip2-1.0.8
+
+[make.build_args.os=linux]
+# python bz2 support on linux needs dynamic library
+-f
+Makefile-libbz2_so
+
+[make.install_args]
+install
+
+[build.os=windows]
+builder = nop

build/fbcode_builder/manifests/eden

@@ -7,6 +7,9 @@ shipit_fbcode_builder = true
 [git]
 repo_url = https://github.com/facebookexperimental/eden.git
 
+[github.actions]
+run_tests = off
+
 [build]
 builder = cmake
 
@@ -37,11 +40,17 @@ sqlite3
 [dependencies.os=darwin]
 osxfuse
 
+[dependencies.not(os=windows)]
 # TODO: teach getdeps to compile curl on Windows.
 # Enabling curl on Windows requires us to find a way to compile libcurl with
 # msvc.
-[dependencies.not(os=windows)]
 libcurl
+# Added so that OSS doesn't see system "python" which is python 2 on darwin and some linux
+python
+
+[dependencies.all(fb=off)]
+# Outside Meta hg is not installed, or if it is, its not the one we want to test with
+eden_scm
 
 [shipit.pathmap.fb=on]
 # for internal builds that use getdeps

build/fbcode_builder/manifests/ncurses

@@ -0,0 +1,30 @@
+[manifest]
+name = ncurses
+
+[debs]
+libncurses-dev
+
+[homebrew]
+ncurses
+
+[rpms]
+ncurses-devel
+
+[download]
+url = https://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.3.tar.gz
+sha256 = 97fc51ac2b085d4cde31ef4d2c3122c21abc217e9090a43a30fc5ec21684e059
+
+[build.not(os=windows)]
+builder = autoconf
+subdir = ncurses-6.3
+
+[autoconf.args]
+--without-cxx-binding
+--without-ada
+
+[autoconf.args.os=linux]
+--enable-shared
+--with-shared
+
+[build.os=windows]
+builder = nop

build/fbcode_builder/manifests/openssl

@@ -6,6 +6,7 @@ libssl-dev
 
 [homebrew]
 openssl@1.1
+# on homebrew need the matching curl and ca-
 
 [rpms]
 openssl

build/fbcode_builder/manifests/python

@@ -36,3 +36,7 @@ libffi
 
 [dependencies]
 libffi
+# eden tests expect the python bz2 support
+bz2
+# eden tests expect the python curses support
+ncurses