diff --git a/.github/workflows/getdeps_linux.yml b/.github/workflows/getdeps_linux.yml new file mode 100644 index 000000000..3f1f7c7af --- /dev/null +++ b/.github/workflows/getdeps_linux.yml @@ -0,0 +1,109 @@ +# This file was @generated by getdeps.py + +name: linux + +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + build: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v1 + - name: Install system deps + run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive proxygen + - name: Fetch boost + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests boost + - name: Fetch ninja + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests ninja + - name: Fetch cmake + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests cmake + - name: Fetch double-conversion + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests double-conversion + - name: Fetch fmt + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fmt + - name: Fetch gflags + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests gflags + - name: Fetch glog + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests glog + - name: Fetch googletest + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests googletest + - name: Fetch libevent + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libevent + - name: Fetch snappy + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests snappy + - name: Fetch zstd + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests zstd + - name: Fetch folly + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests folly + - name: Fetch autoconf + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests autoconf + - name: Fetch automake + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests automake + - name: Fetch libtool + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libtool + - name: Fetch gperf + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests gperf + - name: Fetch libsodium + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libsodium + - name: Fetch fizz + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fizz + - name: Fetch mvfst + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests mvfst + - name: Fetch wangle + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests wangle + - name: Build boost + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests boost + - name: Build ninja + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests ninja + - name: Build cmake + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests cmake + - name: Build double-conversion + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests double-conversion + - name: Build fmt + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fmt + - name: Build gflags + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests gflags + - name: Build glog + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests glog + - name: Build googletest + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests googletest + - name: Build libevent + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libevent + - name: Build snappy + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests snappy + - name: Build zstd + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests zstd + - name: Build folly + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests folly + - name: Build autoconf + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests autoconf + - name: Build automake + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests automake + - name: Build libtool + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libtool + - name: Build gperf + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests gperf + - name: Build libsodium + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libsodium + - name: Build fizz + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fizz + - name: Build mvfst + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests mvfst + - name: Build wangle + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests wangle + - name: Build proxygen + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --src-dir=. proxygen --project-install-prefix proxygen:/usr/local + - name: Copy artifacts + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fixup-dyn-deps --src-dir=. proxygen _artifacts/linux --final-install-prefix /usr/local + - uses: actions/upload-artifact@master + with: + name: proxygen + path: _artifacts + - name: Test proxygen + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages test --src-dir=. proxygen --project-install-prefix proxygen:/usr/local diff --git a/.github/workflows/main.yml b/.github/workflows/getdeps_mac.yml similarity index 51% rename from .github/workflows/main.yml rename to .github/workflows/getdeps_mac.yml index 02d4bc54f..4001ac4e1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/getdeps_mac.yml @@ -1,6 +1,6 @@ # This file was @generated by getdeps.py -name: CI +name: mac on: push: @@ -11,103 +11,7 @@ on: - master jobs: - linux: - runs-on: ubuntu-18.04 - steps: - - uses: actions/checkout@v1 - - name: Install system deps - run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive proxygen - - name: Fetch boost - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests boost - - name: Fetch ninja - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests ninja - - name: Fetch cmake - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests cmake - - name: Fetch double-conversion - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests double-conversion - - name: Fetch fmt - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fmt - - name: Fetch gflags - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests gflags - - name: Fetch glog - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests glog - - name: Fetch googletest - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests googletest - - name: Fetch libevent - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libevent - - name: Fetch snappy - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests snappy - - name: Fetch zstd - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests zstd - - name: Fetch folly - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests folly - - name: Fetch autoconf - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests autoconf - - name: Fetch automake - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests automake - - name: Fetch libtool - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libtool - - name: Fetch gperf - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests gperf - - name: Fetch libsodium - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libsodium - - name: Fetch fizz - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fizz - - name: Fetch mvfst - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests mvfst - - name: Fetch wangle - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests wangle - - name: Build boost - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests boost - - name: Build ninja - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests ninja - - name: Build cmake - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests cmake - - name: Build double-conversion - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests double-conversion - - name: Build fmt - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fmt - - name: Build gflags - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests gflags - - name: Build glog - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests glog - - name: Build googletest - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests googletest - - name: Build libevent - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libevent - - name: Build snappy - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests snappy - - name: Build zstd - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests zstd - - name: Build folly - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests folly - - name: Build autoconf - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests autoconf - - name: Build automake - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests automake - - name: Build libtool - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libtool - - name: Build gperf - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests gperf - - name: Build libsodium - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libsodium - - name: Build fizz - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fizz - - name: Build mvfst - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests mvfst - - name: Build wangle - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests wangle - - name: Build proxygen - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --src-dir=. proxygen --project-install-prefix proxygen:/usr/local - - name: Copy artifacts - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fixup-dyn-deps --src-dir=. proxygen _artifacts/linux --final-install-prefix /usr/local - - uses: actions/upload-artifact@master - with: - name: proxygen - path: _artifacts - - name: Test proxygen - run: python3 build/fbcode_builder/getdeps.py --allow-system-packages test --src-dir=. proxygen --project-install-prefix proxygen:/usr/local - mac: + build: runs-on: macOS-latest steps: - uses: actions/checkout@v1 diff --git a/build/fbcode_builder/getdeps.py b/build/fbcode_builder/getdeps.py index ce06b5660..f6508a119 100755 --- a/build/fbcode_builder/getdeps.py +++ b/build/fbcode_builder/getdeps.py @@ -633,32 +633,11 @@ class GenerateGitHubActionsCmd(ProjectCmdBase): HostType("windows", None, None), ] - with open(args.output_file, "w") as out: - # Deliberate line break here because the @ and the generated - # symbols are meaningful to our internal tooling when they - # appear in a single token - out.write("# This file was @") - out.write("generated by getdeps.py\n") - out.write( - """ -name: CI + for p in platforms: + self.write_job_for_platform(p, args) -on: - push: - branches: - - master - pull_request: - branches: - - master - -jobs: -""" - ) - for p in platforms: - build_opts = setup_build_options(args, p) - self.write_job_for_platform(out, args, build_opts) - - def write_job_for_platform(self, out, args, build_opts): + def write_job_for_platform(self, platform, args): + build_opts = setup_build_options(args, platform) ctx_gen = build_opts.get_context_generator() loader = ManifestLoader(build_opts, ctx_gen) manifest = loader.load_manifest(args.project) @@ -697,78 +676,106 @@ jobs: job_name = "mac" runs_on = "macOS-latest" - getdeps = f"{py3} build/fbcode_builder/getdeps.py --allow-system-packages" - - out.write(" %s:\n" % job_name) - out.write(" runs-on: %s\n" % runs_on) - out.write(" steps:\n") - out.write(" - uses: actions/checkout@v1\n") - - if build_opts.is_windows(): - # cmake relies on BOOST_ROOT but GH deliberately don't set it in order - # to avoid versioning issues: - # https://github.com/actions/virtual-environments/issues/319 - # Instead, set the version we think we need; this is effectively - # coupled with the boost manifest - # This is the unusual syntax for setting an env var for the rest of - # the steps in a workflow: - # https://help.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable - out.write(" - name: Export boost environment\n") + os.makedirs(args.output_dir, exist_ok=True) + output_file = os.path.join(args.output_dir, f"getdeps_{job_name}.yml") + with open(output_file, "w") as out: + # Deliberate line break here because the @ and the generated + # symbols are meaningful to our internal tooling when they + # appear in a single token + out.write("# This file was @") + out.write("generated by getdeps.py\n") out.write( - ' run: "echo ::set-env name=BOOST_ROOT::%BOOST_ROOT_1_69_0%"\n' - ) - out.write(" shell: cmd\n") + f""" +name: {job_name} - # The git installation may not like long filenames, so tell it - # that we want it to use them! - out.write(" - name: Fix Git config\n") - out.write(" run: git config --system core.longpaths true\n") - else: - out.write(" - name: Install system deps\n") - out.write( - f" run: sudo {getdeps} install-system-deps --recursive {manifest.name}\n" +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: +""" ) - projects = loader.manifests_in_dependency_order() + getdeps = f"{py3} build/fbcode_builder/getdeps.py --allow-system-packages" - for m in projects: - if m != manifest: - out.write(" - name: Fetch %s\n" % m.name) - out.write(f" run: {getdeps} fetch --no-tests {m.name}\n") + out.write(" build:\n") + out.write(" runs-on: %s\n" % runs_on) + out.write(" steps:\n") + out.write(" - uses: actions/checkout@v1\n") - for m in projects: - if m != manifest: - out.write(" - name: Build %s\n" % m.name) - out.write(f" run: {getdeps} build --no-tests {m.name}\n") + if build_opts.is_windows(): + # cmake relies on BOOST_ROOT but GH deliberately don't set it in order + # to avoid versioning issues: + # https://github.com/actions/virtual-environments/issues/319 + # Instead, set the version we think we need; this is effectively + # coupled with the boost manifest + # This is the unusual syntax for setting an env var for the rest of + # the steps in a workflow: + # https://help.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable + out.write(" - name: Export boost environment\n") + out.write( + ' run: "echo ::set-env name=BOOST_ROOT::%BOOST_ROOT_1_69_0%"\n' + ) + out.write(" shell: cmd\n") - out.write(" - name: Build %s\n" % manifest.name) + # The git installation may not like long filenames, so tell it + # that we want it to use them! + out.write(" - name: Fix Git config\n") + out.write(" run: git config --system core.longpaths true\n") + else: + out.write(" - name: Install system deps\n") + out.write( + f" run: sudo {getdeps} install-system-deps --recursive {manifest.name}\n" + ) - project_prefix = "" - if not build_opts.is_windows(): - project_prefix = " --project-install-prefix %s:/usr/local" % manifest.name + projects = loader.manifests_in_dependency_order() - out.write( - f" run: {getdeps} build --src-dir=. {manifest.name} {project_prefix}\n" - ) + for m in projects: + if m != manifest: + out.write(" - name: Fetch %s\n" % m.name) + out.write(f" run: {getdeps} fetch --no-tests {m.name}\n") - out.write(" - name: Copy artifacts\n") - out.write( - f" run: {getdeps} fixup-dyn-deps " - f"--src-dir=. {manifest.name} _artifacts/{job_name} --final-install-prefix /usr/local\n" - ) + for m in projects: + if m != manifest: + out.write(" - name: Build %s\n" % m.name) + out.write(f" run: {getdeps} build --no-tests {m.name}\n") - out.write(" - uses: actions/upload-artifact@master\n") - out.write(" with:\n") - out.write(" name: %s\n" % manifest.name) - out.write(" path: _artifacts\n") + out.write(" - name: Build %s\n" % manifest.name) - out.write(" - name: Test %s\n" % manifest.name) - out.write( - f" run: {getdeps} test --src-dir=. {manifest.name} {project_prefix}\n" - ) + project_prefix = "" + if not build_opts.is_windows(): + project_prefix = ( + " --project-install-prefix %s:/usr/local" % manifest.name + ) + + out.write( + f" run: {getdeps} build --src-dir=. {manifest.name} {project_prefix}\n" + ) + + out.write(" - name: Copy artifacts\n") + out.write( + f" run: {getdeps} fixup-dyn-deps " + f"--src-dir=. {manifest.name} _artifacts/{job_name} --final-install-prefix /usr/local\n" + ) + + out.write(" - uses: actions/upload-artifact@master\n") + out.write(" with:\n") + out.write(" name: %s\n" % manifest.name) + out.write(" path: _artifacts\n") + + out.write(" - name: Test %s\n" % manifest.name) + out.write( + f" run: {getdeps} test --src-dir=. {manifest.name} {project_prefix}\n" + ) def setup_project_cmd_parser(self, parser): - parser.add_argument("--output-file", help="The name of the yaml file") + parser.add_argument( + "--output-dir", help="The directory that will contain the yml files" + ) def get_arg_var_name(args):