From 99dd5d742927c8f9b0676f7e3e04f74c54b5fdf1 Mon Sep 17 00:00:00 2001 From: Wez Furlong Date: Tue, 31 Mar 2020 12:07:56 -0700 Subject: [PATCH] getdeps: allow setting per-project install prefix for DESTDIR installs Summary: We have a global `--install-prefix` argument that can be used to set the prefix for all projects, but that is only suitable if you are running with sufficient privileges to install each of the deps to that location during the build. Cmake dependency resolution won't work from the build directory in that situation; it can only see the final installed location and it will error out if those files are not present, or link against the currently installed version instead of the version we just built; not great! This commit adds a project specific `--project-install-prefix` that can be used on just the leaf project in a set of deps. That sidesteps the dependency concern because only the last stage is built in that mode. This option can technically be applied to an arbitrary set of projects, but in light of the above, in practice it only makes sense to use it for the final cmake project. Only the CMakeBuilder respects this option. In the watchman repo, this commit adjusts the autogen.sh script to allow specifying the installation prefix; it defaults to `/usr/local` as you might expect. refs: https://github.com/facebook/watchman/issues/760 Reviewed By: yfeldblum Differential Revision: D20674439 fbshipit-source-id: 52799dbd47f3c295e2d6469ee2b74cedeaa20138 --- .github/workflows/main.yml | 12 +++--- build/fbcode_builder/getdeps.py | 38 ++++++++++++++++--- build/fbcode_builder/getdeps/builder.py | 33 ++++++++++++++-- build/fbcode_builder/getdeps/load.py | 19 +++++++++- build/fbcode_builder/getdeps/manifest.py | 20 +++++++++- .../getdeps/py_wheel_builder.py | 1 + 6 files changed, 104 insertions(+), 19 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2fd0ff32d..fe1d2412c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -96,15 +96,15 @@ jobs: - name: Build wangle run: python3 build/fbcode_builder/getdeps.py build --no-tests wangle - name: Build proxygen - run: python3 build/fbcode_builder/getdeps.py build --src-dir=. proxygen + run: python3 build/fbcode_builder/getdeps.py build --src-dir=. proxygen --project-install-prefix proxygen:/usr/local - name: Copy artifacts - run: python3 build/fbcode_builder/getdeps.py fixup-dyn-deps --src-dir=. proxygen _artifacts/linux + run: python3 build/fbcode_builder/getdeps.py fixup-dyn-deps --src-dir=. proxygen _artifacts/linux --final-install-prefix /usr/local - uses: actions/upload-artifact@master with: name: proxygen path: _artifacts - name: Test proxygen - run: python3 build/fbcode_builder/getdeps.py test --src-dir=. proxygen + run: python3 build/fbcode_builder/getdeps.py test --src-dir=. proxygen --project-install-prefix proxygen:/usr/local mac: runs-on: macOS-latest steps: @@ -194,12 +194,12 @@ jobs: - name: Build wangle run: python3 build/fbcode_builder/getdeps.py build --no-tests wangle - name: Build proxygen - run: python3 build/fbcode_builder/getdeps.py build --src-dir=. proxygen + run: python3 build/fbcode_builder/getdeps.py build --src-dir=. proxygen --project-install-prefix proxygen:/usr/local - name: Copy artifacts - run: python3 build/fbcode_builder/getdeps.py fixup-dyn-deps --src-dir=. proxygen _artifacts/mac + run: python3 build/fbcode_builder/getdeps.py fixup-dyn-deps --src-dir=. proxygen _artifacts/mac --final-install-prefix /usr/local - uses: actions/upload-artifact@master with: name: proxygen path: _artifacts - name: Test proxygen - run: python3 build/fbcode_builder/getdeps.py test --src-dir=. proxygen + run: python3 build/fbcode_builder/getdeps.py test --src-dir=. proxygen --project-install-prefix proxygen:/usr/local diff --git a/build/fbcode_builder/getdeps.py b/build/fbcode_builder/getdeps.py index 8a74647bc..d5ce2b250 100755 --- a/build/fbcode_builder/getdeps.py +++ b/build/fbcode_builder/getdeps.py @@ -133,6 +133,10 @@ class ProjectCmdBase(SubCmd): project, path = parse_project_arg(arg, "--install-dir") loader.set_project_install_dir(project, path) + for arg in args.project_install_prefix: + project, path = parse_project_arg(arg, "--install-prefix") + loader.set_project_install_prefix(project, path) + def setup_parser(self, parser): parser.add_argument( "project", @@ -183,6 +187,12 @@ class ProjectCmdBase(SubCmd): "project, instead of the default location in the scratch path. " "This only affects the project specified, and not its dependencies.", ) + parser.add_argument( + "--project-install-prefix", + default=[], + action="append", + help="Specify the final deployment installation path for a project", + ) self.setup_project_cmd_parser(parser) @@ -341,7 +351,7 @@ class ShowInstDirCmd(ProjectCmdBase): manifests = [manifest] for m in manifests: - inst_dir = loader.get_project_install_dir(m) + inst_dir = loader.get_project_install_dir_respecting_install_prefix(m) print(inst_dir) def setup_project_cmd_parser(self, parser): @@ -415,7 +425,13 @@ class BuildCmd(ProjectCmdBase): os.unlink(built_marker) src_dir = fetcher.get_src_dir() builder = m.create_builder( - loader.build_opts, src_dir, build_dir, inst_dir, ctx, loader + loader.build_opts, + src_dir, + build_dir, + inst_dir, + ctx, + loader, + final_install_prefix=loader.get_project_install_prefix(m), ) builder.build(install_dirs, reconfigure=reconfigure) @@ -505,7 +521,7 @@ class FixupDeps(ProjectCmdBase): install_dirs = [] for m in projects: - inst_dir = loader.get_project_install_dir(m) + inst_dir = loader.get_project_install_dir_respecting_install_prefix(m) install_dirs.append(inst_dir) if m == manifest: @@ -660,20 +676,30 @@ jobs: out.write(f" run: {getdeps} build --no-tests {m.name}\n") out.write(" - name: Build %s\n" % manifest.name) - out.write(f" run: {getdeps} build --src-dir=. {manifest.name}\n") + + project_prefix = "" + if not build_opts.is_windows(): + project_prefix = " --project-install-prefix %s:/usr/local" % manifest.name + + out.write( + f" run: {getdeps} build --src-dir=. {manifest.name} {project_prefix}\n" + ) out.write(" - name: Copy artifacts\n") out.write( f" run: {getdeps} fixup-dyn-deps " - f"--src-dir=. {manifest.name} _artifacts/{job_name}\n" + f"--src-dir=. {manifest.name} _artifacts/{job_name} --final-install-prefix /usr/local\n" ) + out.write(" - uses: actions/upload-artifact@master\n") out.write(" with:\n") out.write(" name: %s\n" % manifest.name) out.write(" path: _artifacts\n") out.write(" - name: Test %s\n" % manifest.name) - out.write(f" run: {getdeps} test --src-dir=. {manifest.name}\n") + out.write( + f" run: {getdeps} test --src-dir=. {manifest.name} {project_prefix}\n" + ) def setup_project_cmd_parser(self, parser): parser.add_argument("--output-file", help="The name of the yaml file") diff --git a/build/fbcode_builder/getdeps/builder.py b/build/fbcode_builder/getdeps/builder.py index 266bdfb87..d5b356c3c 100644 --- a/build/fbcode_builder/getdeps/builder.py +++ b/build/fbcode_builder/getdeps/builder.py @@ -19,7 +19,15 @@ from .runcmd import run_cmd class BuilderBase(object): def __init__( - self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir, env=None + self, + build_opts, + ctx, + manifest, + src_dir, + build_dir, + inst_dir, + env=None, + final_install_prefix=None, ): self.env = Env() if env: @@ -35,6 +43,7 @@ class BuilderBase(object): self.inst_dir = inst_dir self.build_opts = build_opts self.manifest = manifest + self.final_install_prefix = final_install_prefix def _get_cmd_prefix(self): if self.build_opts.is_windows(): @@ -286,10 +295,24 @@ if __name__ == "__main__": """ def __init__( - self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir, defines + self, + build_opts, + ctx, + manifest, + src_dir, + build_dir, + inst_dir, + defines, + final_install_prefix=None, ): super(CMakeBuilder, self).__init__( - build_opts, ctx, manifest, src_dir, build_dir, inst_dir + build_opts, + ctx, + manifest, + src_dir, + build_dir, + inst_dir, + final_install_prefix=final_install_prefix, ) self.defines = defines or {} @@ -337,7 +360,7 @@ if __name__ == "__main__": def _compute_cmake_define_args(self, env): defines = { - "CMAKE_INSTALL_PREFIX": self.inst_dir, + "CMAKE_INSTALL_PREFIX": self.final_install_prefix or self.inst_dir, "BUILD_SHARED_LIBS": "OFF", # Some of the deps (rsocket) default to UBSAN enabled if left # unspecified. Some of the deps fail to compile in release mode @@ -393,6 +416,8 @@ if __name__ == "__main__": reconfigure = reconfigure or self._needs_reconfigure() env = self._compute_env(install_dirs) + if not self.build_opts.is_windows() and self.final_install_prefix: + env["DESTDIR"] = self.inst_dir # Resolve the cmake that we installed cmake = path_search(env, "cmake") diff --git a/build/fbcode_builder/getdeps/load.py b/build/fbcode_builder/getdeps/load.py index a3d30fae6..9bd876e25 100644 --- a/build/fbcode_builder/getdeps/load.py +++ b/build/fbcode_builder/getdeps/load.py @@ -134,6 +134,7 @@ class ManifestLoader(object): self._fetcher_overrides = {} self._build_dir_overrides = {} self._install_dir_overrides = {} + self._install_prefix_overrides = {} def load_manifest(self, name): manifest = self.manifests_by_name.get(name) @@ -238,6 +239,9 @@ class ManifestLoader(object): def set_project_install_dir(self, project_name, path): self._install_dir_overrides[project_name] = path + def set_project_install_prefix(self, project_name, path): + self._install_prefix_overrides[project_name] = path + def create_fetcher(self, manifest): override = self._fetcher_overrides.get(manifest.name) if override is not None: @@ -284,7 +288,10 @@ class ManifestLoader(object): hasher.update(name.encode("utf-8")) value = env.get(name) if value is not None: - hasher.update(value.encode("utf-8")) + try: + hasher.update(value.encode("utf-8")) + except AttributeError as exc: + raise AttributeError("name=%r, value=%r: %s" % (name, value, exc)) manifest.update_hash(hasher, ctx) @@ -327,3 +334,13 @@ class ManifestLoader(object): project_dir_name = self._get_project_dir_name(manifest) return os.path.join(self.build_opts.scratch_dir, "build", project_dir_name) + + def get_project_install_prefix(self, manifest): + return self._install_prefix_overrides.get(manifest.name) + + def get_project_install_dir_respecting_install_prefix(self, manifest): + inst_dir = self.get_project_install_dir(manifest) + prefix = self.get_project_install_prefix(manifest) + if prefix: + return inst_dir + prefix + return inst_dir diff --git a/build/fbcode_builder/getdeps/manifest.py b/build/fbcode_builder/getdeps/manifest.py index 034b07966..7451c7264 100644 --- a/build/fbcode_builder/getdeps/manifest.py +++ b/build/fbcode_builder/getdeps/manifest.py @@ -366,7 +366,16 @@ class ManifestParser(object): "project %s has no fetcher configuration matching %s" % (self.name, ctx) ) - def create_builder(self, build_options, src_dir, build_dir, inst_dir, ctx, loader): + def create_builder( # noqa:C901 + self, + build_options, + src_dir, + build_dir, + inst_dir, + ctx, + loader, + final_install_prefix=None, + ): builder = self.get("build", "builder", ctx=ctx) if not builder: raise Exception("project %s has no builder for %r" % (self.name, ctx)) @@ -392,7 +401,14 @@ class ManifestParser(object): if builder == "cmake": defines = self.get_section_as_dict("cmake.defines", ctx) return CMakeBuilder( - build_options, ctx, self, src_dir, build_dir, inst_dir, defines + build_options, + ctx, + self, + src_dir, + build_dir, + inst_dir, + defines, + final_install_prefix, ) if builder == "python-wheel": diff --git a/build/fbcode_builder/getdeps/py_wheel_builder.py b/build/fbcode_builder/getdeps/py_wheel_builder.py index 151efc9aa..82ad8b807 100644 --- a/build/fbcode_builder/getdeps/py_wheel_builder.py +++ b/build/fbcode_builder/getdeps/py_wheel_builder.py @@ -183,6 +183,7 @@ class PythonWheelBuilder(BuilderBase): build_dir=self.build_dir, inst_dir=self.inst_dir, defines={}, + final_install_prefix=None, ) cmake_builder.build(install_dirs=install_dirs, reconfigure=reconfigure)