From 8ecccf04e3e283bd054fb98e0ee82cc8c9d4c2f3 Mon Sep 17 00:00:00 2001 From: Lukas Piatkowski Date: Fri, 18 Sep 2020 04:19:27 -0700 Subject: [PATCH] eden/edenapi and mononoke integration tests: add edenapi/tools to getdeps and use them in tests (#51) Summary: Pull Request resolved: https://github.com/facebookexperimental/eden/pull/51 This diff extends capabilities of CargoBuilder in getdeps so that individual manifests can be build even without workspaces. Thanks to that a build for edenapi/tools can be made and its artifacts can be used in mononoke integration tests. Reviewed By: StanislavGlebik Differential Revision: D23574887 fbshipit-source-id: 8a974a6b5235d36a44fe082aad55cd380d84dd09 --- build/fbcode_builder/getdeps/builder.py | 50 +++++++++++++++---- build/fbcode_builder/getdeps/manifest.py | 10 +++- .../manifests/eden_scm_lib_edenapi_tools | 36 +++++++++++++ build/fbcode_builder/manifests/rust-shed | 1 + 4 files changed, 84 insertions(+), 13 deletions(-) create mode 100644 build/fbcode_builder/manifests/eden_scm_lib_edenapi_tools diff --git a/build/fbcode_builder/getdeps/builder.py b/build/fbcode_builder/getdeps/builder.py index e74a6ef47..121b43f45 100644 --- a/build/fbcode_builder/getdeps/builder.py +++ b/build/fbcode_builder/getdeps/builder.py @@ -999,6 +999,7 @@ class CargoBuilder(BuilderBase): inst_dir, build_doc, workspace_dir, + manifests_to_build, loader, ): super(CargoBuilder, self).__init__( @@ -1006,6 +1007,7 @@ class CargoBuilder(BuilderBase): ) self.build_doc = build_doc self.ws_dir = workspace_dir + self.manifests_to_build = manifests_to_build and manifests_to_build.split(",") self.loader = loader def run_cargo(self, install_dirs, operation, args=None): @@ -1026,7 +1028,10 @@ class CargoBuilder(BuilderBase): return os.path.join(self.build_dir, "source") def workspace_dir(self): - return os.path.join(self.build_source_dir(), self.ws_dir) + return os.path.join(self.build_source_dir(), self.ws_dir or "") + + def manifest_dir(self, manifest): + return os.path.join(self.build_source_dir(), manifest) def recreate_dir(self, src, dst): if os.path.isdir(dst): @@ -1058,7 +1063,8 @@ incremental = false ) ) - self._patchup_workspace() + if self.ws_dir is not None: + self._patchup_workspace() try: from getdeps.facebook.rust import vendored_crates @@ -1069,11 +1075,26 @@ incremental = false # so just rely on cargo downloading crates on it's own pass - self.run_cargo( - install_dirs, - "build", - ["--out-dir", os.path.join(self.inst_dir, "bin"), "-Zunstable-options"], - ) + if self.manifests_to_build is None: + self.run_cargo( + install_dirs, + "build", + ["--out-dir", os.path.join(self.inst_dir, "bin"), "-Zunstable-options"], + ) + else: + for manifest in self.manifests_to_build: + self.run_cargo( + install_dirs, + "build", + [ + "--out-dir", + os.path.join(self.inst_dir, "bin"), + "-Zunstable-options", + "--manifest-path", + self.manifest_dir(manifest), + ], + ) + self.recreate_dir(build_source_dir, os.path.join(self.inst_dir, "source")) def run_tests( @@ -1082,11 +1103,18 @@ incremental = false if test_filter: args = ["--", test_filter] else: - args = None + args = [] - self.run_cargo(install_dirs, "test", args) - if self.build_doc: - self.run_cargo(install_dirs, "doc", ["--no-deps"]) + if self.manifests_to_build is None: + self.run_cargo(install_dirs, "test", args) + if self.build_doc: + self.run_cargo(install_dirs, "doc", ["--no-deps"]) + else: + for manifest in self.manifests_to_build: + margs = ["--manifest-path", self.manifest_dir(manifest)] + self.run_cargo(install_dirs, "test", args + margs) + if self.build_doc: + self.run_cargo(install_dirs, "doc", ["--no-deps"] + margs) def _patchup_workspace(self): """ diff --git a/build/fbcode_builder/getdeps/manifest.py b/build/fbcode_builder/getdeps/manifest.py index 710e15697..35b0948cf 100644 --- a/build/fbcode_builder/getdeps/manifest.py +++ b/build/fbcode_builder/getdeps/manifest.py @@ -74,7 +74,11 @@ SCHEMA = { "msbuild": {"optional_section": True, "fields": {"project": REQUIRED}}, "cargo": { "optional_section": True, - "fields": {"build_doc": OPTIONAL, "workspace_dir": OPTIONAL}, + "fields": { + "build_doc": OPTIONAL, + "workspace_dir": OPTIONAL, + "manifests_to_build": OPTIONAL, + }, }, "cmake.defines": {"optional_section": True}, "autoconf.args": {"optional_section": True}, @@ -489,7 +493,8 @@ class ManifestParser(object): if builder == "cargo": build_doc = self.get("cargo", "build_doc", False, ctx) - workspace_dir = self.get("cargo", "workspace_dir", "", ctx) + workspace_dir = self.get("cargo", "workspace_dir", None, ctx) + manifests_to_build = self.get("cargo", "manifests_to_build", None, ctx) return CargoBuilder( build_options, ctx, @@ -499,6 +504,7 @@ class ManifestParser(object): inst_dir, build_doc, workspace_dir, + manifests_to_build, loader, ) diff --git a/build/fbcode_builder/manifests/eden_scm_lib_edenapi_tools b/build/fbcode_builder/manifests/eden_scm_lib_edenapi_tools new file mode 100644 index 000000000..be29d70f8 --- /dev/null +++ b/build/fbcode_builder/manifests/eden_scm_lib_edenapi_tools @@ -0,0 +1,36 @@ +[manifest] +name = eden_scm_lib_edenapi_tools +fbsource_path = fbcode/eden +shipit_project = eden +shipit_fbcode_builder = true + +[git] +repo_url = https://github.com/facebookexperimental/eden.git + +[build] +builder = cargo + +[cargo] +build_doc = true +manifests_to_build = eden/scm/lib/edenapi/tools/make_req/Cargo.toml,eden/scm/lib/edenapi/tools/read_res/Cargo.toml + +[shipit.pathmap] +fbcode/eden/oss = . +fbcode/eden = eden +fbcode/tools/lfs = tools/lfs +fbcode/fboss/common = common + +[shipit.strip] +^fbcode/eden/fs/eden-config\.h$ +^fbcode/eden/fs/py/eden/config\.py$ +^fbcode/eden/hg/.*$ +^fbcode/eden/mononoke/(?!lfs_protocol) +^fbcode/eden/scm/build/.*$ +^fbcode/eden/scm/lib/third-party/rust/.*/Cargo.toml$ +^fbcode/eden/.*/\.cargo/.*$ +^.*/fb/.*$ +/Cargo\.lock$ +\.pyc$ + +[dependencies.fb=on] +rust diff --git a/build/fbcode_builder/manifests/rust-shed b/build/fbcode_builder/manifests/rust-shed index e5cd7d75c..c94b3fdd6 100644 --- a/build/fbcode_builder/manifests/rust-shed +++ b/build/fbcode_builder/manifests/rust-shed @@ -12,6 +12,7 @@ builder = cargo [cargo] build_doc = true +workspace_dir = [shipit.pathmap] fbcode/common/rust/shed = shed