diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fe1d2412c..02d4bc54f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -15,191 +15,195 @@ jobs: runs-on: ubuntu-18.04 steps: - uses: actions/checkout@v1 + - name: Install system deps + run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive proxygen - name: Fetch boost - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests boost + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests boost - name: Fetch ninja - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests ninja + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests ninja - name: Fetch cmake - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests cmake + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests cmake - name: Fetch double-conversion - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests double-conversion + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests double-conversion - name: Fetch fmt - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests fmt + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fmt - name: Fetch gflags - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests gflags + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests gflags - name: Fetch glog - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests glog + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests glog - name: Fetch googletest - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests googletest + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests googletest - name: Fetch libevent - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests libevent + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libevent - name: Fetch snappy - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests snappy + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests snappy - name: Fetch zstd - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests zstd + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests zstd - name: Fetch folly - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests folly + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests folly - name: Fetch autoconf - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests autoconf + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests autoconf - name: Fetch automake - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests automake + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests automake - name: Fetch libtool - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests libtool + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libtool - name: Fetch gperf - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests gperf + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests gperf - name: Fetch libsodium - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests libsodium + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libsodium - name: Fetch fizz - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests fizz + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fizz - name: Fetch mvfst - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests mvfst + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests mvfst - name: Fetch wangle - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests wangle + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests wangle - name: Build boost - run: python3 build/fbcode_builder/getdeps.py build --no-tests boost + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests boost - name: Build ninja - run: python3 build/fbcode_builder/getdeps.py build --no-tests ninja + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests ninja - name: Build cmake - run: python3 build/fbcode_builder/getdeps.py build --no-tests cmake + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests cmake - name: Build double-conversion - run: python3 build/fbcode_builder/getdeps.py build --no-tests double-conversion + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests double-conversion - name: Build fmt - run: python3 build/fbcode_builder/getdeps.py build --no-tests fmt + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fmt - name: Build gflags - run: python3 build/fbcode_builder/getdeps.py build --no-tests gflags + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests gflags - name: Build glog - run: python3 build/fbcode_builder/getdeps.py build --no-tests glog + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests glog - name: Build googletest - run: python3 build/fbcode_builder/getdeps.py build --no-tests googletest + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests googletest - name: Build libevent - run: python3 build/fbcode_builder/getdeps.py build --no-tests libevent + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libevent - name: Build snappy - run: python3 build/fbcode_builder/getdeps.py build --no-tests snappy + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests snappy - name: Build zstd - run: python3 build/fbcode_builder/getdeps.py build --no-tests zstd + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests zstd - name: Build folly - run: python3 build/fbcode_builder/getdeps.py build --no-tests folly + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests folly - name: Build autoconf - run: python3 build/fbcode_builder/getdeps.py build --no-tests autoconf + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests autoconf - name: Build automake - run: python3 build/fbcode_builder/getdeps.py build --no-tests automake + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests automake - name: Build libtool - run: python3 build/fbcode_builder/getdeps.py build --no-tests libtool + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libtool - name: Build gperf - run: python3 build/fbcode_builder/getdeps.py build --no-tests gperf + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests gperf - name: Build libsodium - run: python3 build/fbcode_builder/getdeps.py build --no-tests libsodium + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libsodium - name: Build fizz - run: python3 build/fbcode_builder/getdeps.py build --no-tests fizz + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fizz - name: Build mvfst - run: python3 build/fbcode_builder/getdeps.py build --no-tests mvfst + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests mvfst - name: Build wangle - run: python3 build/fbcode_builder/getdeps.py build --no-tests wangle + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests wangle - name: Build proxygen - run: python3 build/fbcode_builder/getdeps.py build --src-dir=. proxygen --project-install-prefix proxygen:/usr/local + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --src-dir=. proxygen --project-install-prefix proxygen:/usr/local - name: Copy artifacts - run: python3 build/fbcode_builder/getdeps.py fixup-dyn-deps --src-dir=. proxygen _artifacts/linux --final-install-prefix /usr/local + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fixup-dyn-deps --src-dir=. proxygen _artifacts/linux --final-install-prefix /usr/local - uses: actions/upload-artifact@master with: name: proxygen path: _artifacts - name: Test proxygen - run: python3 build/fbcode_builder/getdeps.py test --src-dir=. proxygen --project-install-prefix proxygen:/usr/local + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages test --src-dir=. proxygen --project-install-prefix proxygen:/usr/local mac: runs-on: macOS-latest steps: - uses: actions/checkout@v1 + - name: Install system deps + run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive proxygen - name: Fetch boost - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests boost + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests boost - name: Fetch openssl - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests openssl + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests openssl - name: Fetch ninja - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests ninja + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests ninja - name: Fetch cmake - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests cmake + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests cmake - name: Fetch double-conversion - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests double-conversion + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests double-conversion - name: Fetch fmt - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests fmt + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fmt - name: Fetch gflags - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests gflags + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests gflags - name: Fetch glog - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests glog + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests glog - name: Fetch googletest - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests googletest + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests googletest - name: Fetch libevent - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests libevent + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libevent - name: Fetch snappy - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests snappy + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests snappy - name: Fetch zstd - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests zstd + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests zstd - name: Fetch folly - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests folly + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests folly - name: Fetch autoconf - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests autoconf + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests autoconf - name: Fetch automake - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests automake + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests automake - name: Fetch libtool - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests libtool + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libtool - name: Fetch gperf - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests gperf + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests gperf - name: Fetch libsodium - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests libsodium + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libsodium - name: Fetch fizz - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests fizz + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fizz - name: Fetch mvfst - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests mvfst + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests mvfst - name: Fetch wangle - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests wangle + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests wangle - name: Build boost - run: python3 build/fbcode_builder/getdeps.py build --no-tests boost + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests boost - name: Build openssl - run: python3 build/fbcode_builder/getdeps.py build --no-tests openssl + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests openssl - name: Build ninja - run: python3 build/fbcode_builder/getdeps.py build --no-tests ninja + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests ninja - name: Build cmake - run: python3 build/fbcode_builder/getdeps.py build --no-tests cmake + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests cmake - name: Build double-conversion - run: python3 build/fbcode_builder/getdeps.py build --no-tests double-conversion + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests double-conversion - name: Build fmt - run: python3 build/fbcode_builder/getdeps.py build --no-tests fmt + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fmt - name: Build gflags - run: python3 build/fbcode_builder/getdeps.py build --no-tests gflags + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests gflags - name: Build glog - run: python3 build/fbcode_builder/getdeps.py build --no-tests glog + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests glog - name: Build googletest - run: python3 build/fbcode_builder/getdeps.py build --no-tests googletest + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests googletest - name: Build libevent - run: python3 build/fbcode_builder/getdeps.py build --no-tests libevent + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libevent - name: Build snappy - run: python3 build/fbcode_builder/getdeps.py build --no-tests snappy + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests snappy - name: Build zstd - run: python3 build/fbcode_builder/getdeps.py build --no-tests zstd + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests zstd - name: Build folly - run: python3 build/fbcode_builder/getdeps.py build --no-tests folly + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests folly - name: Build autoconf - run: python3 build/fbcode_builder/getdeps.py build --no-tests autoconf + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests autoconf - name: Build automake - run: python3 build/fbcode_builder/getdeps.py build --no-tests automake + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests automake - name: Build libtool - run: python3 build/fbcode_builder/getdeps.py build --no-tests libtool + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libtool - name: Build gperf - run: python3 build/fbcode_builder/getdeps.py build --no-tests gperf + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests gperf - name: Build libsodium - run: python3 build/fbcode_builder/getdeps.py build --no-tests libsodium + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libsodium - name: Build fizz - run: python3 build/fbcode_builder/getdeps.py build --no-tests fizz + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fizz - name: Build mvfst - run: python3 build/fbcode_builder/getdeps.py build --no-tests mvfst + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests mvfst - name: Build wangle - run: python3 build/fbcode_builder/getdeps.py build --no-tests wangle + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests wangle - name: Build proxygen - run: python3 build/fbcode_builder/getdeps.py build --src-dir=. proxygen --project-install-prefix proxygen:/usr/local + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --src-dir=. proxygen --project-install-prefix proxygen:/usr/local - name: Copy artifacts - run: python3 build/fbcode_builder/getdeps.py fixup-dyn-deps --src-dir=. proxygen _artifacts/mac --final-install-prefix /usr/local + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fixup-dyn-deps --src-dir=. proxygen _artifacts/mac --final-install-prefix /usr/local - uses: actions/upload-artifact@master with: name: proxygen path: _artifacts - name: Test proxygen - run: python3 build/fbcode_builder/getdeps.py test --src-dir=. proxygen --project-install-prefix proxygen:/usr/local + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages test --src-dir=. proxygen --project-install-prefix proxygen:/usr/local diff --git a/build/fbcode_builder/getdeps.py b/build/fbcode_builder/getdeps.py index d5ce2b250..ae8ccc00b 100755 --- a/build/fbcode_builder/getdeps.py +++ b/build/fbcode_builder/getdeps.py @@ -21,9 +21,11 @@ import getdeps.cache as cache_module from getdeps.buildopts import setup_build_options from getdeps.dyndeps import create_dyn_dep_munger from getdeps.errors import TransientFailure +from getdeps.fetcher import SystemPackageFetcher from getdeps.load import ManifestLoader from getdeps.manifest import ManifestParser from getdeps.platform import HostType +from getdeps.runcmd import run_cmd from getdeps.subcmd import SubCmd, add_subcommands, cmd @@ -310,6 +312,45 @@ class FetchCmd(ProjectCmdBase): fetcher.update() +@cmd("install-system-deps", "Install system packages to satisfy the deps for a project") +class InstallSysDepsCmd(ProjectCmdBase): + def setup_project_cmd_parser(self, parser): + parser.add_argument( + "--recursive", + help="install the transitive deps also", + action="store_true", + default=False, + ) + + def run_project_cmd(self, args, loader, manifest): + if args.recursive: + projects = loader.manifests_in_dependency_order() + else: + projects = [manifest] + + cache = cache_module.create_cache() + all_packages = {} + for m in projects: + ctx = loader.ctx_gen.get_context(m.name) + packages = m.get_required_system_packages(ctx) + for k, v in packages.items(): + merged = all_packages.get(k, []) + merged += v + all_packages[k] = merged + + manager = loader.build_opts.host_type.get_package_manager() + if manager == "rpm": + packages = sorted(list(set(all_packages["rpm"]))) + if packages: + run_cmd(["dnf", "install", "-y"] + packages) + elif manager == "deb": + packages = sorted(list(set(all_packages["deb"]))) + if packages: + run_cmd(["apt", "install", "-y"] + packages) + else: + print("I don't know how to install any packages on this system") + + @cmd("list-deps", "lists the transitive deps for a given project") class ListDepsCmd(ProjectCmdBase): def run_project_cmd(self, args, loader, manifest): @@ -402,6 +443,12 @@ class BuildCmd(ProjectCmdBase): for m in projects: fetcher = loader.create_fetcher(m) + if isinstance(fetcher, SystemPackageFetcher): + # We are guaranteed that if the fetcher is set to + # SystemPackageFetcher then this item is completely + # satisfied by the appropriate system packages + continue + if args.clean: fetcher.clean() @@ -650,7 +697,7 @@ jobs: job_name = "mac" runs_on = "macOS-latest" - getdeps = f"{py3} build/fbcode_builder/getdeps.py" + getdeps = f"{py3} build/fbcode_builder/getdeps.py --allow-system-packages" out.write(" %s:\n" % job_name) out.write(" runs-on: %s\n" % runs_on) @@ -662,6 +709,11 @@ jobs: # that we want it to use them! out.write(" - name: Fix Git config\n") out.write(" run: git config --system core.longpaths true\n") + else: + out.write(" - name: Install system deps\n") + out.write( + f" run: sudo {getdeps} install-system-deps --recursive {manifest.name}\n" + ) projects = loader.manifests_in_dependency_order() @@ -769,6 +821,12 @@ def parse_args(): action="store_false", dest="facebook_internal", ) + add_common_arg( + "--allow-system-packages", + help="Allow satisfying third party deps from installed system packages", + action="store_true", + default=False, + ) ap = argparse.ArgumentParser( description="Get and build dependencies and projects", parents=[common_args] diff --git a/build/fbcode_builder/getdeps/buildopts.py b/build/fbcode_builder/getdeps/buildopts.py index cfc147106..c23683f8d 100644 --- a/build/fbcode_builder/getdeps/buildopts.py +++ b/build/fbcode_builder/getdeps/buildopts.py @@ -54,6 +54,7 @@ class BuildOptions(object): num_jobs=0, use_shipit=False, vcvars_path=None, + allow_system_packages=False, ): """ fbcode_builder_dir - the path to either the in-fbsource fbcode_builder dir, or for shipit-transformed repos, the build dir that @@ -107,6 +108,7 @@ class BuildOptions(object): self.fbcode_builder_dir = fbcode_builder_dir self.host_type = host_type self.use_shipit = use_shipit + self.allow_system_packages = allow_system_packages if vcvars_path is None and is_windows(): # On Windows, the compiler is not available in the PATH by @@ -386,6 +388,13 @@ def setup_build_options(args, host_type=None): temp = tempfile.gettempdir() scratch_dir = os.path.join(temp, "fbcode_builder_getdeps-%s" % munged) + if not is_windows() and os.geteuid() == 0: + # Running as root; in the case where someone runs + # sudo getdeps.py install-system-deps + # and then runs as build without privs, we want to avoid creating + # a scratch dir that the second stage cannot write to. + # So we generate a different path if we are root. + scratch_dir += "-root" if not os.path.exists(scratch_dir): os.makedirs(scratch_dir) @@ -415,4 +424,5 @@ def setup_build_options(args, host_type=None): num_jobs=args.num_jobs, use_shipit=args.use_shipit, vcvars_path=args.vcvars_path, + allow_system_packages=args.allow_system_packages, ) diff --git a/build/fbcode_builder/getdeps/fetcher.py b/build/fbcode_builder/getdeps/fetcher.py index 0b52d5c7b..9ea306056 100644 --- a/build/fbcode_builder/getdeps/fetcher.py +++ b/build/fbcode_builder/getdeps/fetcher.py @@ -151,6 +151,46 @@ class LocalDirFetcher(object): return self.path +class SystemPackageFetcher(object): + def __init__(self, build_options, packages): + self.manager = build_options.host_type.get_package_manager() + self.packages = packages.get(self.manager) + if self.packages: + self.installed = None + else: + self.installed = False + + def packages_are_installed(self): + if self.installed is not None: + return self.installed + + if self.manager == "rpm": + result = run_cmd(["rpm", "-q"] + self.packages, allow_fail=True) + self.installed = result == 0 + elif self.manager == "deb": + result = run_cmd(["dpkg", "-s"] + self.packages, allow_fail=True) + self.installed = result == 0 + else: + self.installed = False + + return self.installed + + def update(self): + assert self.installed + return ChangeStatus(all_changed=False) + + def hash(self): + return "0" * 40 + + def get_src_dir(self): + return None + + +class PreinstalledNopFetcher(SystemPackageFetcher): + def __init__(self): + self.installed = True + + class GitFetcher(Fetcher): DEFAULT_DEPTH = 100 diff --git a/build/fbcode_builder/getdeps/manifest.py b/build/fbcode_builder/getdeps/manifest.py index 7451c7264..b6c894cc5 100644 --- a/build/fbcode_builder/getdeps/manifest.py +++ b/build/fbcode_builder/getdeps/manifest.py @@ -25,8 +25,10 @@ from .expr import parse_expr from .fetcher import ( ArchiveFetcher, GitFetcher, + PreinstalledNopFetcher, ShipitTransformerFetcher, SimpleShipitTransformerFetcher, + SystemPackageFetcher, ) from .py_wheel_builder import PythonWheelBuilder @@ -74,6 +76,9 @@ SCHEMA = { }, "cmake.defines": {"optional_section": True}, "autoconf.args": {"optional_section": True}, + "rpms": {"optional_section": True}, + "debs": {"optional_section": True}, + "preinstalled.env": {"optional_section": True}, "b2.args": {"optional_section": True}, "make.args": {"optional_section": True}, "header-only": {"optional_section": True, "fields": {"includedir": REQUIRED}}, @@ -318,6 +323,27 @@ class ManifestParser(object): """ returns true if this is an FB first-party project """ return self.shipit_project is not None + def get_required_system_packages(self, ctx): + """ Returns dictionary of packager system -> list of packages """ + return { + "rpm": self.get_section_as_args("rpms", ctx), + "deb": self.get_section_as_args("debs", ctx), + } + + def _is_satisfied_by_preinstalled_environment(self, ctx): + envs = self.get_section_as_args("preinstalled.env", ctx) + if not envs: + return False + for key in envs: + val = os.environ.get(key, None) + print(f"Testing ENV[{key}]: {repr(val)}") + if val is None: + return False + if len(val) == 0: + return False + + return True + def create_fetcher(self, build_options, ctx): use_real_shipit = ( ShipitTransformerFetcher.available() and build_options.use_shipit @@ -339,6 +365,16 @@ class ManifestParser(object): # We can use the code from fbsource return ShipitTransformerFetcher(build_options, self.shipit_project) + # Can we satisfy this dep with system packages? + if build_options.allow_system_packages: + if self._is_satisfied_by_preinstalled_environment(ctx): + return PreinstalledNopFetcher() + + packages = self.get_required_system_packages(ctx) + package_fetcher = SystemPackageFetcher(build_options, packages) + if package_fetcher.packages_are_installed(): + return package_fetcher + repo_url = self.get("git", "repo_url", ctx=ctx) if repo_url: rev = self.get("git", "rev") diff --git a/build/fbcode_builder/getdeps/platform.py b/build/fbcode_builder/getdeps/platform.py index 8c3d767e4..0f79bb139 100644 --- a/build/fbcode_builder/getdeps/platform.py +++ b/build/fbcode_builder/getdeps/platform.py @@ -87,6 +87,15 @@ class HostType(object): self.distrovers or "none", ) + def get_package_manager(self): + if not self.is_linux(): + return None + if self.distro in ("fedora", "centos"): + return "rpm" + if self.distro in ("debian", "ubuntu"): + return "deb" + return None + @staticmethod def from_tuple_string(s): ostype, distro, distrovers = s.split("-") diff --git a/build/fbcode_builder/getdeps/runcmd.py b/build/fbcode_builder/getdeps/runcmd.py index 8c88c37bc..0606a2da1 100644 --- a/build/fbcode_builder/getdeps/runcmd.py +++ b/build/fbcode_builder/getdeps/runcmd.py @@ -54,9 +54,13 @@ def run_cmd(cmd, env=None, cwd=None, allow_fail=False, log_file=None): log.write(msg) sys.stdout.write(msg) - _run_cmd(cmd, env=env, cwd=cwd, allow_fail=allow_fail, log_fn=log_function) + return _run_cmd( + cmd, env=env, cwd=cwd, allow_fail=allow_fail, log_fn=log_function + ) else: - _run_cmd(cmd, env=env, cwd=cwd, allow_fail=allow_fail, log_fn=sys.stdout.write) + return _run_cmd( + cmd, env=env, cwd=cwd, allow_fail=allow_fail, log_fn=sys.stdout.write + ) def _run_cmd(cmd, env, cwd, allow_fail, log_fn): diff --git a/build/fbcode_builder/manifests/autoconf b/build/fbcode_builder/manifests/autoconf index b898c968d..35963096c 100644 --- a/build/fbcode_builder/manifests/autoconf +++ b/build/fbcode_builder/manifests/autoconf @@ -1,6 +1,12 @@ [manifest] name = autoconf +[rpms] +autoconf + +[debs] +autoconf + [download] url = http://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz sha256 = 954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969 diff --git a/build/fbcode_builder/manifests/automake b/build/fbcode_builder/manifests/automake index 55a7677fe..71115068a 100644 --- a/build/fbcode_builder/manifests/automake +++ b/build/fbcode_builder/manifests/automake @@ -1,6 +1,12 @@ [manifest] name = automake +[rpms] +automake + +[debs] +automake + [download] url = http://ftp.gnu.org/gnu/automake/automake-1.16.1.tar.gz sha256 = 608a97523f97db32f1f5d5615c98ca69326ced2054c9f82e65bade7fc4c9dea8 diff --git a/build/fbcode_builder/manifests/bison b/build/fbcode_builder/manifests/bison index c46e1edd7..6e355d052 100644 --- a/build/fbcode_builder/manifests/bison +++ b/build/fbcode_builder/manifests/bison @@ -1,6 +1,12 @@ [manifest] name = bison +[rpms] +bison + +[debs] +bison + [download.not(os=windows)] url = https://mirrors.kernel.org/gnu/bison/bison-3.3.tar.gz sha256 = fdeafb7fffade05604a61e66b8c040af4b2b5cbb1021dcfe498ed657ac970efd diff --git a/build/fbcode_builder/manifests/boost b/build/fbcode_builder/manifests/boost index 4af5ac8f3..f051cfd1b 100644 --- a/build/fbcode_builder/manifests/boost +++ b/build/fbcode_builder/manifests/boost @@ -9,6 +9,45 @@ sha256 = 8f32d4617390d1c2d16f26a27ab60d97807b35440d45891fa340fc2648b04406 url = https://versaweb.dl.sourceforge.net/project/boost/boost/1.69.0/boost_1_69_0.zip sha256 = d074bcbcc0501c4917b965fc890e303ee70d8b01ff5712bae4a6c54f2b6b4e52 +[preinstalled.env] +BOOST_ROOT_1_69_0 + +[debs] +libboost-all-dev + +[rpms] +boost +boost-math +boost-test +boost-fiber +boost-graph +boost-log +boost-openmpi +boost-timer +boost-chrono +boost-locale +boost-thread +boost-atomic +boost-random +boost-static +boost-contract +boost-date-time +boost-iostreams +boost-container +boost-coroutine +boost-filesystem +boost-system +boost-stacktrace +boost-regex +boost-devel +boost-context +boost-python3-devel +boost-type_erasure +boost-wave +boost-python3 +boost-serialization +boost-program-options + [build] builder = boost diff --git a/build/fbcode_builder/manifests/cmake b/build/fbcode_builder/manifests/cmake index f97d409e7..e02f7091c 100644 --- a/build/fbcode_builder/manifests/cmake +++ b/build/fbcode_builder/manifests/cmake @@ -1,6 +1,13 @@ [manifest] name = cmake +[rpms] +cmake + +# All current deb based distros have a cmake that is too old +#[debs] +#cmake + [dependencies] ninja diff --git a/build/fbcode_builder/manifests/fboss b/build/fbcode_builder/manifests/fboss index 1416c25a0..497271d1b 100644 --- a/build/fbcode_builder/manifests/fboss +++ b/build/fbcode_builder/manifests/fboss @@ -32,7 +32,7 @@ libnl libsai OpenNSA re2 -Python-3.7.6 +python [shipit.pathmap] fbcode/fboss/github = . diff --git a/build/fbcode_builder/manifests/flex b/build/fbcode_builder/manifests/flex index efb3a7cc3..f266c4033 100644 --- a/build/fbcode_builder/manifests/flex +++ b/build/fbcode_builder/manifests/flex @@ -1,6 +1,12 @@ [manifest] name = flex +[rpms] +flex + +[debs] +flex + [download.not(os=windows)] url = https://github.com/westes/flex/releases/download/v2.6.4/flex-2.6.4.tar.gz sha256 = e87aae032bf07c26f85ac0ed3250998c37621d95f8bd748b31f15b33c45ee995 diff --git a/build/fbcode_builder/manifests/libcurl b/build/fbcode_builder/manifests/libcurl index fdfa3b25f..466b4497c 100644 --- a/build/fbcode_builder/manifests/libcurl +++ b/build/fbcode_builder/manifests/libcurl @@ -1,6 +1,13 @@ [manifest] name = libcurl +[rpms] +libcurl-devel +libcurl + +[debs] +libcurl4-openssl-dev + [download] url = https://curl.haxx.se/download/curl-7.65.1.tar.gz sha256 = 821aeb78421375f70e55381c9ad2474bf279fc454b791b7e95fc83562951c690 diff --git a/build/fbcode_builder/manifests/libelf b/build/fbcode_builder/manifests/libelf index 4ad23d4c1..a46aab879 100644 --- a/build/fbcode_builder/manifests/libelf +++ b/build/fbcode_builder/manifests/libelf @@ -1,6 +1,12 @@ [manifest] name = libelf +[rpms] +elfutils-libelf-devel-static + +[debs] +libelf-dev + [download] url = https://ftp.osuosl.org/pub/blfs/conglomeration/libelf/libelf-0.8.13.tar.gz sha256 = 591a9b4ec81c1f2042a97aa60564e0cb79d041c52faa7416acb38bc95bd2c76d diff --git a/build/fbcode_builder/manifests/libevent b/build/fbcode_builder/manifests/libevent index 473ebbdac..eaa39a9e6 100644 --- a/build/fbcode_builder/manifests/libevent +++ b/build/fbcode_builder/manifests/libevent @@ -1,6 +1,12 @@ [manifest] name = libevent +[rpms] +libevent-devel + +[debs] +libevent-dev + # Note that the CMakeLists.txt file is present only in # git repo and not in the release tarball, so take care # to use the github generated source tarball rather than diff --git a/build/fbcode_builder/manifests/libgit2 b/build/fbcode_builder/manifests/libgit2 index ae43c62dd..1d6a53e5e 100644 --- a/build/fbcode_builder/manifests/libgit2 +++ b/build/fbcode_builder/manifests/libgit2 @@ -1,6 +1,12 @@ [manifest] name = libgit2 +[rpms] +libgit2-devel + +[debs] +libgit2-dev + [download] url = https://github.com/libgit2/libgit2/archive/v0.28.1.tar.gz sha256 = 0ca11048795b0d6338f2e57717370208c2c97ad66c6d5eac0c97a8827d13936b diff --git a/build/fbcode_builder/manifests/libmnl b/build/fbcode_builder/manifests/libmnl index ddf46c392..3444cbe68 100644 --- a/build/fbcode_builder/manifests/libmnl +++ b/build/fbcode_builder/manifests/libmnl @@ -1,6 +1,13 @@ [manifest] name = libmnl +[rpms] +libmnl-devel +libmnl-static + +[debs] +libmnl-dev + [download] url = http://www.lg.ps.pl/mirrors/ftp.netfilter.org/libmnl/libmnl-1.0.4.tar.bz2 sha256 = 171f89699f286a5854b72b91d06e8f8e3683064c5901fb09d954a9ab6f551f81 diff --git a/build/fbcode_builder/manifests/libnl b/build/fbcode_builder/manifests/libnl index 1a08feba1..f864acb49 100644 --- a/build/fbcode_builder/manifests/libnl +++ b/build/fbcode_builder/manifests/libnl @@ -1,6 +1,13 @@ [manifest] name = libnl +[rpms] +libnl3-devel +libnl3 + +[debs] +libnl-3-dev + [download] url = https://www.infradead.org/~tgr/libnl/files/libnl-3.2.25.tar.gz sha256 = 8beb7590674957b931de6b7f81c530b85dc7c1ad8fbda015398bc1e8d1ce8ec5 diff --git a/build/fbcode_builder/manifests/libsodium b/build/fbcode_builder/manifests/libsodium index 098f68c4c..d69bfcc4b 100644 --- a/build/fbcode_builder/manifests/libsodium +++ b/build/fbcode_builder/manifests/libsodium @@ -1,6 +1,13 @@ [manifest] name = libsodium +[rpms] +libsodium-devel +libsodium-static + +[debs] +libsodium-dev + [download.not(os=windows)] url = https://github.com/jedisct1/libsodium/releases/download/1.0.17/libsodium-1.0.17.tar.gz sha256 = 0cc3dae33e642cc187b5ceb467e0ad0e1b51dcba577de1190e9ffa17766ac2b1 diff --git a/build/fbcode_builder/manifests/libtool b/build/fbcode_builder/manifests/libtool index 32740c7cb..1ec99b5f4 100644 --- a/build/fbcode_builder/manifests/libtool +++ b/build/fbcode_builder/manifests/libtool @@ -1,6 +1,12 @@ [manifest] name = libtool +[rpms] +libtool + +[debs] +libtool + [download] url = http://ftp.gnu.org/gnu/libtool/libtool-2.4.6.tar.gz sha256 = e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3 diff --git a/build/fbcode_builder/manifests/libusb b/build/fbcode_builder/manifests/libusb index 4decbecfc..74702d3f0 100644 --- a/build/fbcode_builder/manifests/libusb +++ b/build/fbcode_builder/manifests/libusb @@ -1,6 +1,13 @@ [manifest] name = libusb +[rpms] +libusb-devel +libusb + +[debs] +libusb-1.0-0-dev + [download] url = https://github.com/libusb/libusb/releases/download/v1.0.22/libusb-1.0.22.tar.bz2 sha256 = 75aeb9d59a4fdb800d329a545c2e6799f732362193b465ea198f2aa275518157 diff --git a/build/fbcode_builder/manifests/libzmq b/build/fbcode_builder/manifests/libzmq index fd7a6fb6d..af1acbd77 100644 --- a/build/fbcode_builder/manifests/libzmq +++ b/build/fbcode_builder/manifests/libzmq @@ -1,6 +1,13 @@ [manifest] name = libzmq +[rpms] +zeromq-devel +zeromq + +[debs] +libzmq-dev + [download] url = https://github.com/zeromq/libzmq/releases/download/v4.3.1/zeromq-4.3.1.tar.gz sha256 = bcbabe1e2c7d0eec4ed612e10b94b112dd5f06fcefa994a0c79a45d835cd21eb diff --git a/build/fbcode_builder/manifests/lz4 b/build/fbcode_builder/manifests/lz4 index 39cc5fe17..03dbd9de4 100644 --- a/build/fbcode_builder/manifests/lz4 +++ b/build/fbcode_builder/manifests/lz4 @@ -1,6 +1,13 @@ [manifest] name = lz4 +[rpms] +lz4-devel +lz4-static + +[debs] +liblz4-dev + [download] url = https://github.com/lz4/lz4/archive/v1.8.3.tar.gz sha256 = 33af5936ac06536805f9745e0b6d61da606a1f8b4cc5c04dd3cbaca3b9b4fc43 diff --git a/build/fbcode_builder/manifests/nghttp2 b/build/fbcode_builder/manifests/nghttp2 index 5d09b9e2f..151daf8af 100644 --- a/build/fbcode_builder/manifests/nghttp2 +++ b/build/fbcode_builder/manifests/nghttp2 @@ -1,6 +1,13 @@ [manifest] name = nghttp2 +[rpms] +libnghttp2-devel +libnghttp2 + +[debs] +libnghttp2-dev + [download] url = https://github.com/nghttp2/nghttp2/releases/download/v1.39.2/nghttp2-1.39.2.tar.gz sha256 = fc820a305e2f410fade1a3260f09229f15c0494fc089b0100312cd64a33a38c0 diff --git a/build/fbcode_builder/manifests/ninja b/build/fbcode_builder/manifests/ninja index 5fac6792c..24fa990e4 100644 --- a/build/fbcode_builder/manifests/ninja +++ b/build/fbcode_builder/manifests/ninja @@ -1,6 +1,12 @@ [manifest] name = ninja +[rpms] +ninja-build + +[debs] +ninja-build + [download.os=windows] url = https://github.com/ninja-build/ninja/releases/download/v1.9.0/ninja-win.zip sha256 = 2d70010633ddaacc3af4ffbd21e22fae90d158674a09e132e06424ba3ab036e9 diff --git a/build/fbcode_builder/manifests/openssl b/build/fbcode_builder/manifests/openssl index b9a2f2532..11e3e0fe5 100644 --- a/build/fbcode_builder/manifests/openssl +++ b/build/fbcode_builder/manifests/openssl @@ -1,6 +1,13 @@ [manifest] name = openssl +[rpms] +openssl-devel +openssl + +[debs] +libssl-dev + [download] url = https://www.openssl.org/source/openssl-1.1.1f.tar.gz sha256 = 186c6bfe6ecfba7a5b48c47f8a1673d0f3b0e5ba2e25602dd23b629975da3f35 diff --git a/build/fbcode_builder/manifests/patchelf b/build/fbcode_builder/manifests/patchelf index 2fbd16c9b..f9d050424 100644 --- a/build/fbcode_builder/manifests/patchelf +++ b/build/fbcode_builder/manifests/patchelf @@ -1,6 +1,12 @@ [manifest] name = patchelf +[rpms] +patchelf + +[debs] +patchelf + [download] url = https://github.com/NixOS/patchelf/archive/0.10.tar.gz sha256 = b3cb6bdedcef5607ce34a350cf0b182eb979f8f7bc31eae55a93a70a3f020d13 diff --git a/build/fbcode_builder/manifests/pcre b/build/fbcode_builder/manifests/pcre index 0475aee78..5353d8c27 100644 --- a/build/fbcode_builder/manifests/pcre +++ b/build/fbcode_builder/manifests/pcre @@ -1,6 +1,13 @@ [manifest] name = pcre +[rpms] +pcre-devel +pcre-static + +[debs] +libpcre3-dev + [download] url = https://ftp.pcre.org/pub/pcre/pcre-8.43.tar.gz sha256 = 0b8e7465dc5e98c757cc3650a20a7843ee4c3edf50aaf60bb33fd879690d2c73 diff --git a/build/fbcode_builder/manifests/Python-3.7.6 b/build/fbcode_builder/manifests/python similarity index 77% rename from build/fbcode_builder/manifests/Python-3.7.6 rename to build/fbcode_builder/manifests/python index daa412380..e51c0ab51 100644 --- a/build/fbcode_builder/manifests/Python-3.7.6 +++ b/build/fbcode_builder/manifests/python @@ -1,5 +1,12 @@ [manifest] -name = Python-3.7.6 +name = python + +[rpms] +python3 +python3-devel + +[debs] +python3-all-dev [download.os=linux] url = https://www.python.org/ftp/python/3.7.6/Python-3.7.6.tgz diff --git a/build/fbcode_builder/manifests/re2 b/build/fbcode_builder/manifests/re2 index 1bf58b1fb..eb4d6a92c 100644 --- a/build/fbcode_builder/manifests/re2 +++ b/build/fbcode_builder/manifests/re2 @@ -1,6 +1,13 @@ [manifest] name = re2 +[rpms] +re2 +re2-devel + +[debs] +libre2-dev + [download] url = https://github.com/google/re2/archive/2019-06-01.tar.gz sha256 = 02b7d73126bd18e9fbfe5d6375a8bb13fadaf8e99e48cbb062e4500fc18e8e2e diff --git a/build/fbcode_builder/manifests/snappy b/build/fbcode_builder/manifests/snappy index 2807f1d97..2f46a7734 100644 --- a/build/fbcode_builder/manifests/snappy +++ b/build/fbcode_builder/manifests/snappy @@ -1,6 +1,13 @@ [manifest] name = snappy +[rpms] +snappy +snappy-devel + +[debs] +libsnappy-dev + [download] url = https://github.com/google/snappy/archive/1.1.7.tar.gz sha256 = 3dfa02e873ff51a11ee02b9ca391807f0c8ea0529a4924afa645fbf97163f9d4 diff --git a/build/fbcode_builder/manifests/sqlite3 b/build/fbcode_builder/manifests/sqlite3 index ef7e1a2f8..2463f5761 100644 --- a/build/fbcode_builder/manifests/sqlite3 +++ b/build/fbcode_builder/manifests/sqlite3 @@ -1,6 +1,13 @@ [manifest] name = sqlite3 +[rpms] +sqlite-devel +sqlite-libs + +[debs] +libsqlite3-dev + [download] url = https://sqlite.org/2019/sqlite-amalgamation-3280000.zip sha256 = d02fc4e95cfef672b45052e221617a050b7f2e20103661cda88387349a9b1327 diff --git a/build/fbcode_builder/manifests/zlib b/build/fbcode_builder/manifests/zlib index 2105aea24..8df0e3e48 100644 --- a/build/fbcode_builder/manifests/zlib +++ b/build/fbcode_builder/manifests/zlib @@ -1,6 +1,13 @@ [manifest] name = zlib +[rpms] +zlib-devel +zlib-static + +[debs] +zlib1g-dev + [download] url = http://www.zlib.net/zlib-1.2.11.tar.gz sha256 = c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1 diff --git a/build/fbcode_builder/manifests/zstd b/build/fbcode_builder/manifests/zstd index ad34d9ddf..1a069b834 100644 --- a/build/fbcode_builder/manifests/zstd +++ b/build/fbcode_builder/manifests/zstd @@ -1,6 +1,13 @@ [manifest] name = zstd +[rpms] +libzstd-devel +libzstd + +[debs] +libzstd1-dev + [git] repo_url = https://github.com/facebook/zstd.git rev = v1.3.8