diff --git a/build/fbcode_builder/getdeps.py b/build/fbcode_builder/getdeps.py index 92c14087f..d93e2a43b 100755 --- a/build/fbcode_builder/getdeps.py +++ b/build/fbcode_builder/getdeps.py @@ -415,7 +415,7 @@ class BuildCmd(ProjectCmdBase): os.unlink(built_marker) src_dir = fetcher.get_src_dir() builder = m.create_builder( - loader.build_opts, src_dir, build_dir, inst_dir, ctx + loader.build_opts, src_dir, build_dir, inst_dir, ctx, loader ) builder.build(install_dirs, reconfigure=reconfigure) @@ -544,7 +544,7 @@ class TestCmd(ProjectCmdBase): ctx = loader.ctx_gen.get_context(m.name) build_dir = loader.get_project_build_dir(m) builder = m.create_builder( - loader.build_opts, src_dir, build_dir, inst_dir, ctx + loader.build_opts, src_dir, build_dir, inst_dir, ctx, loader ) builder.run_tests( install_dirs, diff --git a/build/fbcode_builder/getdeps/builder.py b/build/fbcode_builder/getdeps/builder.py index 049b2537d..3bc056717 100644 --- a/build/fbcode_builder/getdeps/builder.py +++ b/build/fbcode_builder/getdeps/builder.py @@ -813,19 +813,25 @@ install(FILES sqlite3.h sqlite3ext.h DESTINATION include) class CargoBuilder(BuilderBase): def __init__( - self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir, build_doc + self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir, build_doc, loader ): super(CargoBuilder, self).__init__( build_opts, ctx, manifest, src_dir, build_dir, inst_dir ) self.build_doc = build_doc + self.loader = loader def run_cargo(self, install_dirs, operation, args=None): args = args or [] env = self._compute_env(install_dirs) # Enable using nightly features with stable compiler env["RUSTC_BOOTSTRAP"] = "1" - cmd = ["cargo", operation, "-j%s" % self.build_opts.num_jobs] + args + cmd = [ + "cargo", + operation, + "--workspace", + "-j%s" % self.build_opts.num_jobs, + ] + args self._run_cmd(cmd, cwd=self.build_source_dir(), env=env) def build_source_dir(self): @@ -856,6 +862,8 @@ git-fetch-with-cli = true ) ) + self._patchup_workspace(build_source_dir) + try: from getdeps.facebook.lfs import crates_io_download @@ -872,3 +880,163 @@ git-fetch-with-cli = true self.run_cargo(install_dirs, "test") if self.build_doc: self.run_cargo(install_dirs, "doc", ["--no-deps"]) + + def _patchup_workspace(self, build_source_dir): + """ + This method makes a lot of assumptions about the state of the project + and its cargo dependendies: + 1. There is a virtual manifest with workspace in the root of this project + 2. Crates from cargo dependencies can be extracted from Cargo.toml files + using _extract_crates function. It is using a heuristic so check its + code to understand how it is done. + 3. The extracted cargo dependencies crates can be found in the + dependency's install dir using _resolve_crate_to_path function + which again is using a heuristic. + + Notice that many things might go wrong here. E.g. if someone depends + on another getdeps crate by writing in their Cargo.toml file: + + my-rename-of-crate = { package = "crate", git = "..." } + + they can count themselves lucky because the code will raise an + Exception. There migh be more cases where the code will silently pass + producing bad results. + """ + config = self._resolve_config(build_source_dir) + if config: + with open(os.path.join(build_source_dir, "Cargo.toml"), "a") as f: + # A fake manifest has to be crated to change the virtual + # manifest into a non-virtual. The virtual manifests are limited + # in many ways and the inability to define patches on them is + # one. Check https://github.com/rust-lang/cargo/issues/4934 to + # see if it is resolved. + f.write( + """ +[package] +name = "fake_manifest_of_{}" +version = "0.0.0" +[lib] +path = "/dev/null" +""".format( + self.manifest.name + ) + ) + f.write(config) + + def _resolve_config(self, build_source_dir): + """ + Returns a configuration to be put inside root Cargo.toml file which + patches the dependencies git code with local getdeps versions. + See https://doc.rust-lang.org/cargo/reference/manifest.html#the-patch-section + """ + dep_to_git = self._resolve_dep_to_git() + dep_to_crates = CargoBuilder._resolve_dep_to_crates( + build_source_dir, dep_to_git + ) + + config = [] + for name in sorted(dep_to_git.keys()): + git_conf = dep_to_git[name] + crates = sorted(dep_to_crates.get(name, [])) + if not crates: + continue # nothing to patch, move along + crates_patches = [ + '{} = {{ path = "{}" }}'.format( + crate, + CargoBuilder._resolve_crate_to_path(crate, git_conf).replace( + "\\", "\\\\" + ), + ) + for crate in crates + ] + + config.append( + '[patch."{0}"]\n'.format(git_conf["repo_url"]) + + "\n".join(crates_patches) + ) + return "\n".join(config) + + def _resolve_dep_to_git(self): + """ + For each direct dependency of the currently build manifest check if it + is also cargo-builded and if yes then extract it's git configs and + install dir + """ + dependencies = self.manifest.get_section_as_dict("dependencies", ctx=self.ctx) + if not dependencies: + return [] + + dep_to_git = {} + for dep in dependencies.keys(): + dep_manifest = self.loader.load_manifest(dep) + if dep_manifest.get("build", "builder", ctx=self.ctx) != "cargo": + # This is a direct dependency, but it is not build with cargo + # so ignore it. + continue + + git_conf = dep_manifest.get_section_as_dict("git", ctx=self.ctx) + if "repo_url" not in git_conf: + raise Exception( + "A cargo dependency requires git.repo_url to be defined." + ) + git_conf["inst_dir"] = self.loader.get_project_install_dir(dep_manifest) + dep_to_git[dep] = git_conf + return dep_to_git + + @staticmethod + def _resolve_dep_to_crates(build_source_dir, dep_to_git): + """ + This function traverse the build_source_dir in search of Cargo.toml + files, extracts the crate names from them using _extract_crates + function and returns a merged result containing crate names per + dependency name from all Cargo.toml files in the project. + """ + if not dep_to_git: + return {} # no deps, so don't waste time traversing files + + dep_to_crates = {} + for root, _, files in os.walk(build_source_dir): + for f in files: + if f == "Cargo.toml": + more_dep_to_crates = CargoBuilder._extract_crates( + os.path.join(root, f), dep_to_git + ) + for name, crates in more_dep_to_crates.items(): + dep_to_crates.setdefault(name, set()).update(crates) + return dep_to_crates + + @staticmethod + def _extract_crates(cargo_toml_file, dep_to_git): + """ + This functions reads content of provided cargo toml file and extracts + crate names per each dependency. The extraction is done by a heuristic + so it might be incorrect. + """ + deps_to_crates = {} + with open(cargo_toml_file, "r") as f: + for line in f.readlines(): + if line.startswith("#") or "git = " not in line: + continue # filter out commented lines and ones without git deps + for name, conf in dep_to_git.items(): + if 'git = "{}"'.format(conf["repo_url"]) in line: + crate_name, _, _ = line.partition("=") + deps_to_crates.setdefault(name, set()).add(crate_name.strip()) + return deps_to_crates + + @staticmethod + def _resolve_crate_to_path(crate, git_conf): + """ + Tries to find in git_conf["inst_dir"] by searching a [package] + keyword followed by name = "". + """ + source_dir = os.path.join(git_conf["inst_dir"], "source") + search_pattern = '[package]\nname = "{}"'.format(crate) + + for root, _, files in os.walk(source_dir): + for fname in files: + if fname == "Cargo.toml": + with open(os.path.join(root, fname), "r") as f: + if search_pattern in f.read(): + return root + + raise Exception("Failed to found crate {} in path {}".format(crate, source_dir)) diff --git a/build/fbcode_builder/getdeps/manifest.py b/build/fbcode_builder/getdeps/manifest.py index a898112ca..1b26b58df 100644 --- a/build/fbcode_builder/getdeps/manifest.py +++ b/build/fbcode_builder/getdeps/manifest.py @@ -364,7 +364,7 @@ class ManifestParser(object): "project %s has no fetcher configuration matching %s" % (self.name, ctx) ) - def create_builder(self, build_options, src_dir, build_dir, inst_dir, ctx): + def create_builder(self, build_options, src_dir, build_dir, inst_dir, ctx, loader): builder = self.get("build", "builder", ctx=ctx) if not builder: raise Exception("project %s has no builder for %r" % (self.name, ctx)) @@ -422,7 +422,14 @@ class ManifestParser(object): if builder == "cargo": build_doc = self.get("cargo", "build_doc", False, ctx) return CargoBuilder( - build_options, ctx, self, src_dir, build_dir, inst_dir, build_doc + build_options, + ctx, + self, + src_dir, + build_dir, + inst_dir, + build_doc, + loader, ) if builder == "OpenNSA": diff --git a/build/fbcode_builder/manifests/mononoke b/build/fbcode_builder/manifests/mononoke new file mode 100644 index 000000000..898d1761e --- /dev/null +++ b/build/fbcode_builder/manifests/mononoke @@ -0,0 +1,33 @@ +[manifest] +name = mononoke +fbsource_path = fbcode/scm/mononoke +shipit_project = mononoke +shipit_fbcode_builder = true + +[git] +repo_url = https://github.com/facebookexperimental/mononoke.git + +[build.not(os=windows)] +builder = cargo + +[build.os=windows] +# building Mononoke on windows is not supported +builder = nop + +[cargo] +build_doc = true + +[shipit.pathmap] +fbcode/scm/mononoke = mononoke +fbcode/scm/mononoke/public_autocargo = mononoke +fbcode/scm/mononoke/public_tld = . +tools/rust/ossconfigs = . + +[shipit.strip] +^fbcode/scm/mononoke/(?!public_autocargo|public_tld).+/Cargo\.toml$ + +[dependencies] +rust-shed + +[dependencies.fb=on] +rust