diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2fd0ff32d..fe1d2412c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -96,15 +96,15 @@ jobs: - name: Build wangle run: python3 build/fbcode_builder/getdeps.py build --no-tests wangle - name: Build proxygen - run: python3 build/fbcode_builder/getdeps.py build --src-dir=. proxygen + run: python3 build/fbcode_builder/getdeps.py build --src-dir=. proxygen --project-install-prefix proxygen:/usr/local - name: Copy artifacts - run: python3 build/fbcode_builder/getdeps.py fixup-dyn-deps --src-dir=. proxygen _artifacts/linux + run: python3 build/fbcode_builder/getdeps.py fixup-dyn-deps --src-dir=. proxygen _artifacts/linux --final-install-prefix /usr/local - uses: actions/upload-artifact@master with: name: proxygen path: _artifacts - name: Test proxygen - run: python3 build/fbcode_builder/getdeps.py test --src-dir=. proxygen + run: python3 build/fbcode_builder/getdeps.py test --src-dir=. proxygen --project-install-prefix proxygen:/usr/local mac: runs-on: macOS-latest steps: @@ -194,12 +194,12 @@ jobs: - name: Build wangle run: python3 build/fbcode_builder/getdeps.py build --no-tests wangle - name: Build proxygen - run: python3 build/fbcode_builder/getdeps.py build --src-dir=. proxygen + run: python3 build/fbcode_builder/getdeps.py build --src-dir=. proxygen --project-install-prefix proxygen:/usr/local - name: Copy artifacts - run: python3 build/fbcode_builder/getdeps.py fixup-dyn-deps --src-dir=. proxygen _artifacts/mac + run: python3 build/fbcode_builder/getdeps.py fixup-dyn-deps --src-dir=. proxygen _artifacts/mac --final-install-prefix /usr/local - uses: actions/upload-artifact@master with: name: proxygen path: _artifacts - name: Test proxygen - run: python3 build/fbcode_builder/getdeps.py test --src-dir=. proxygen + run: python3 build/fbcode_builder/getdeps.py test --src-dir=. proxygen --project-install-prefix proxygen:/usr/local diff --git a/build/fbcode_builder/getdeps.py b/build/fbcode_builder/getdeps.py index 8a74647bc..d5ce2b250 100755 --- a/build/fbcode_builder/getdeps.py +++ b/build/fbcode_builder/getdeps.py @@ -133,6 +133,10 @@ class ProjectCmdBase(SubCmd): project, path = parse_project_arg(arg, "--install-dir") loader.set_project_install_dir(project, path) + for arg in args.project_install_prefix: + project, path = parse_project_arg(arg, "--install-prefix") + loader.set_project_install_prefix(project, path) + def setup_parser(self, parser): parser.add_argument( "project", @@ -183,6 +187,12 @@ class ProjectCmdBase(SubCmd): "project, instead of the default location in the scratch path. " "This only affects the project specified, and not its dependencies.", ) + parser.add_argument( + "--project-install-prefix", + default=[], + action="append", + help="Specify the final deployment installation path for a project", + ) self.setup_project_cmd_parser(parser) @@ -341,7 +351,7 @@ class ShowInstDirCmd(ProjectCmdBase): manifests = [manifest] for m in manifests: - inst_dir = loader.get_project_install_dir(m) + inst_dir = loader.get_project_install_dir_respecting_install_prefix(m) print(inst_dir) def setup_project_cmd_parser(self, parser): @@ -415,7 +425,13 @@ class BuildCmd(ProjectCmdBase): os.unlink(built_marker) src_dir = fetcher.get_src_dir() builder = m.create_builder( - loader.build_opts, src_dir, build_dir, inst_dir, ctx, loader + loader.build_opts, + src_dir, + build_dir, + inst_dir, + ctx, + loader, + final_install_prefix=loader.get_project_install_prefix(m), ) builder.build(install_dirs, reconfigure=reconfigure) @@ -505,7 +521,7 @@ class FixupDeps(ProjectCmdBase): install_dirs = [] for m in projects: - inst_dir = loader.get_project_install_dir(m) + inst_dir = loader.get_project_install_dir_respecting_install_prefix(m) install_dirs.append(inst_dir) if m == manifest: @@ -660,20 +676,30 @@ jobs: out.write(f" run: {getdeps} build --no-tests {m.name}\n") out.write(" - name: Build %s\n" % manifest.name) - out.write(f" run: {getdeps} build --src-dir=. {manifest.name}\n") + + project_prefix = "" + if not build_opts.is_windows(): + project_prefix = " --project-install-prefix %s:/usr/local" % manifest.name + + out.write( + f" run: {getdeps} build --src-dir=. {manifest.name} {project_prefix}\n" + ) out.write(" - name: Copy artifacts\n") out.write( f" run: {getdeps} fixup-dyn-deps " - f"--src-dir=. {manifest.name} _artifacts/{job_name}\n" + f"--src-dir=. {manifest.name} _artifacts/{job_name} --final-install-prefix /usr/local\n" ) + out.write(" - uses: actions/upload-artifact@master\n") out.write(" with:\n") out.write(" name: %s\n" % manifest.name) out.write(" path: _artifacts\n") out.write(" - name: Test %s\n" % manifest.name) - out.write(f" run: {getdeps} test --src-dir=. {manifest.name}\n") + out.write( + f" run: {getdeps} test --src-dir=. {manifest.name} {project_prefix}\n" + ) def setup_project_cmd_parser(self, parser): parser.add_argument("--output-file", help="The name of the yaml file") diff --git a/build/fbcode_builder/getdeps/builder.py b/build/fbcode_builder/getdeps/builder.py index 266bdfb87..d5b356c3c 100644 --- a/build/fbcode_builder/getdeps/builder.py +++ b/build/fbcode_builder/getdeps/builder.py @@ -19,7 +19,15 @@ from .runcmd import run_cmd class BuilderBase(object): def __init__( - self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir, env=None + self, + build_opts, + ctx, + manifest, + src_dir, + build_dir, + inst_dir, + env=None, + final_install_prefix=None, ): self.env = Env() if env: @@ -35,6 +43,7 @@ class BuilderBase(object): self.inst_dir = inst_dir self.build_opts = build_opts self.manifest = manifest + self.final_install_prefix = final_install_prefix def _get_cmd_prefix(self): if self.build_opts.is_windows(): @@ -286,10 +295,24 @@ if __name__ == "__main__": """ def __init__( - self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir, defines + self, + build_opts, + ctx, + manifest, + src_dir, + build_dir, + inst_dir, + defines, + final_install_prefix=None, ): super(CMakeBuilder, self).__init__( - build_opts, ctx, manifest, src_dir, build_dir, inst_dir + build_opts, + ctx, + manifest, + src_dir, + build_dir, + inst_dir, + final_install_prefix=final_install_prefix, ) self.defines = defines or {} @@ -337,7 +360,7 @@ if __name__ == "__main__": def _compute_cmake_define_args(self, env): defines = { - "CMAKE_INSTALL_PREFIX": self.inst_dir, + "CMAKE_INSTALL_PREFIX": self.final_install_prefix or self.inst_dir, "BUILD_SHARED_LIBS": "OFF", # Some of the deps (rsocket) default to UBSAN enabled if left # unspecified. Some of the deps fail to compile in release mode @@ -393,6 +416,8 @@ if __name__ == "__main__": reconfigure = reconfigure or self._needs_reconfigure() env = self._compute_env(install_dirs) + if not self.build_opts.is_windows() and self.final_install_prefix: + env["DESTDIR"] = self.inst_dir # Resolve the cmake that we installed cmake = path_search(env, "cmake") diff --git a/build/fbcode_builder/getdeps/load.py b/build/fbcode_builder/getdeps/load.py index a3d30fae6..9bd876e25 100644 --- a/build/fbcode_builder/getdeps/load.py +++ b/build/fbcode_builder/getdeps/load.py @@ -134,6 +134,7 @@ class ManifestLoader(object): self._fetcher_overrides = {} self._build_dir_overrides = {} self._install_dir_overrides = {} + self._install_prefix_overrides = {} def load_manifest(self, name): manifest = self.manifests_by_name.get(name) @@ -238,6 +239,9 @@ class ManifestLoader(object): def set_project_install_dir(self, project_name, path): self._install_dir_overrides[project_name] = path + def set_project_install_prefix(self, project_name, path): + self._install_prefix_overrides[project_name] = path + def create_fetcher(self, manifest): override = self._fetcher_overrides.get(manifest.name) if override is not None: @@ -284,7 +288,10 @@ class ManifestLoader(object): hasher.update(name.encode("utf-8")) value = env.get(name) if value is not None: - hasher.update(value.encode("utf-8")) + try: + hasher.update(value.encode("utf-8")) + except AttributeError as exc: + raise AttributeError("name=%r, value=%r: %s" % (name, value, exc)) manifest.update_hash(hasher, ctx) @@ -327,3 +334,13 @@ class ManifestLoader(object): project_dir_name = self._get_project_dir_name(manifest) return os.path.join(self.build_opts.scratch_dir, "build", project_dir_name) + + def get_project_install_prefix(self, manifest): + return self._install_prefix_overrides.get(manifest.name) + + def get_project_install_dir_respecting_install_prefix(self, manifest): + inst_dir = self.get_project_install_dir(manifest) + prefix = self.get_project_install_prefix(manifest) + if prefix: + return inst_dir + prefix + return inst_dir diff --git a/build/fbcode_builder/getdeps/manifest.py b/build/fbcode_builder/getdeps/manifest.py index 034b07966..7451c7264 100644 --- a/build/fbcode_builder/getdeps/manifest.py +++ b/build/fbcode_builder/getdeps/manifest.py @@ -366,7 +366,16 @@ class ManifestParser(object): "project %s has no fetcher configuration matching %s" % (self.name, ctx) ) - def create_builder(self, build_options, src_dir, build_dir, inst_dir, ctx, loader): + def create_builder( # noqa:C901 + self, + build_options, + src_dir, + build_dir, + inst_dir, + ctx, + loader, + final_install_prefix=None, + ): builder = self.get("build", "builder", ctx=ctx) if not builder: raise Exception("project %s has no builder for %r" % (self.name, ctx)) @@ -392,7 +401,14 @@ class ManifestParser(object): if builder == "cmake": defines = self.get_section_as_dict("cmake.defines", ctx) return CMakeBuilder( - build_options, ctx, self, src_dir, build_dir, inst_dir, defines + build_options, + ctx, + self, + src_dir, + build_dir, + inst_dir, + defines, + final_install_prefix, ) if builder == "python-wheel": diff --git a/build/fbcode_builder/getdeps/py_wheel_builder.py b/build/fbcode_builder/getdeps/py_wheel_builder.py index 151efc9aa..82ad8b807 100644 --- a/build/fbcode_builder/getdeps/py_wheel_builder.py +++ b/build/fbcode_builder/getdeps/py_wheel_builder.py @@ -183,6 +183,7 @@ class PythonWheelBuilder(BuilderBase): build_dir=self.build_dir, inst_dir=self.inst_dir, defines={}, + final_install_prefix=None, ) cmake_builder.build(install_dirs=install_dirs, reconfigure=reconfigure)