From b361d8c2f967d89a1f9c5476b84bc8bdf26cbad3 Mon Sep 17 00:00:00 2001 From: Adam Simpkins Date: Wed, 31 Jul 2019 20:53:07 -0700 Subject: [PATCH] add a new ManifestLoader class Summary: Add a new ManifestLoader class to handle loading manifests and computing dependencies. For now the main thing this class does is maintain the `manifest_by_name` mapping. In subsequent diffs we should be able to move some additional logic into this class, which will help clean up the code and eliminate some redudant work. In particular, we can have this class cache project hashes, which will avoid re-computing hashes over and over again for the same projects as we do in many cases today. We should also be able to save and re-use some of the project dependency ordering computation in some cases as well. Reviewed By: strager Differential Revision: D16477400 fbshipit-source-id: f06f62f77d8443fccaa69fe4c1306e39c395b325 --- build/fbcode_builder/getdeps.py | 74 ++++++------- build/fbcode_builder/getdeps/load.py | 155 +++++++++++++++++---------- 2 files changed, 135 insertions(+), 94 deletions(-) diff --git a/build/fbcode_builder/getdeps.py b/build/fbcode_builder/getdeps.py index 80f8812b1..80afa5505 100755 --- a/build/fbcode_builder/getdeps.py +++ b/build/fbcode_builder/getdeps.py @@ -18,7 +18,7 @@ import sys from getdeps.buildopts import setup_build_options from getdeps.dyndeps import create_dyn_dep_munger from getdeps.errors import TransientFailure -from getdeps.load import load_project, manifests_in_dependency_order +from getdeps.load import ManifestLoader from getdeps.manifest import ManifestParser from getdeps.platform import HostType from getdeps.subcmd import SubCmd, add_subcommands, cmd @@ -84,14 +84,14 @@ class FetchCmd(SubCmd): def run(self, args): opts = setup_build_options(args) - ctx_gen = opts.get_context_generator() - manifest = load_project(opts, args.project) + loader = ManifestLoader(opts) + manifest = loader.load_manifest(args.project) if args.recursive: - projects = manifests_in_dependency_order(opts, manifest, ctx_gen) + projects = loader.manifests_in_dependency_order() else: projects = [manifest] for m in projects: - fetcher = m.create_fetcher(opts, ctx_gen.get_context(m.name)) + fetcher = m.create_fetcher(opts, loader.ctx_gen.get_context(m.name)) fetcher.update() @@ -99,10 +99,10 @@ class FetchCmd(SubCmd): class ListDepsCmd(SubCmd): def run(self, args): opts = setup_build_options(args) - ctx_gen = opts.get_context_generator() - ctx_gen.set_value_for_project(args.project, "test", "on") - manifest = load_project(opts, args.project) - for m in manifests_in_dependency_order(opts, manifest, ctx_gen): + loader = ManifestLoader(opts) + loader.ctx_gen.set_value_for_project(args.project, "test", "on") + loader.load_manifest(args.project) + for m in loader.manifests_in_dependency_order(): print(m.name) return 0 @@ -142,11 +142,10 @@ class CleanCmd(SubCmd): class ShowInstDirCmd(SubCmd): def run(self, args): opts = setup_build_options(args) - ctx_gen = opts.get_context_generator() - ctx_gen.set_value_for_project(args.project, "test", "on") - manifest = load_project(opts, args.project) - projects = manifests_in_dependency_order(opts, manifest, ctx_gen) - manifests_by_name = {m.name: m for m in projects} + loader = ManifestLoader(opts) + loader.ctx_gen.set_value_for_project(args.project, "test", "on") + manifest = loader.load_manifest(args.project) + projects = loader.manifests_in_dependency_order() if args.recursive: manifests = projects @@ -154,9 +153,11 @@ class ShowInstDirCmd(SubCmd): manifests = [manifest] for m in manifests: - ctx = ctx_gen.get_context(m.name) + ctx = loader.ctx_gen.get_context(m.name) fetcher = m.create_fetcher(opts, ctx) - dirs = opts.compute_dirs(m, fetcher, manifests_by_name, ctx_gen) + dirs = opts.compute_dirs( + m, fetcher, loader.manifests_by_name, loader.ctx_gen + ) inst_dir = dirs["inst_dir"] print(inst_dir) @@ -180,17 +181,17 @@ class ShowInstDirCmd(SubCmd): class ShowSourceDirCmd(SubCmd): def run(self, args): opts = setup_build_options(args) - ctx_gen = opts.get_context_generator() - ctx_gen.set_value_for_project(args.project, "test", "on") - manifest = load_project(opts, args.project) + loader = ManifestLoader(opts) + loader.ctx_gen.set_value_for_project(args.project, "test", "on") + manifest = loader.load_manifest(args.project) if args.recursive: - manifests = manifests_in_dependency_order(opts, manifest, ctx_gen) + manifests = loader.manifests_in_dependency_order() else: manifests = [manifest] for m in manifests: - fetcher = m.create_fetcher(opts, ctx_gen.get_context(m.name)) + fetcher = m.create_fetcher(opts, loader.ctx_gen.get_context(m.name)) print(fetcher.get_src_dir()) def setup_parser(self, parser): @@ -213,17 +214,18 @@ class ShowSourceDirCmd(SubCmd): class BuildCmd(SubCmd): def run(self, args): opts = setup_build_options(args) - if args.clean: - clean_dirs(opts) - ctx_gen = opts.get_context_generator(facebook_internal=args.facebook_internal) if args.enable_tests: ctx_gen.set_value_for_project(args.project, "test", "on") - manifest = load_project(opts, args.project) + loader = ManifestLoader(opts, ctx_gen) + + if args.clean: + clean_dirs(opts) + + manifest = loader.load_manifest(args.project) print("Building on %s" % ctx_gen.get_context(args.project)) - projects = manifests_in_dependency_order(opts, manifest, ctx_gen) - manifests_by_name = {m.name: m for m in projects} + projects = loader.manifests_in_dependency_order() # Accumulate the install directories so that the build steps # can find their dep installation @@ -236,7 +238,7 @@ class BuildCmd(SubCmd): if args.clean: fetcher.clean() - dirs = opts.compute_dirs(m, fetcher, manifests_by_name, ctx_gen) + dirs = opts.compute_dirs(m, fetcher, loader.manifests_by_name, ctx_gen) build_dir = dirs["build_dir"] inst_dir = dirs["inst_dir"] @@ -318,10 +320,10 @@ class FixupDeps(SubCmd): if args.enable_tests: ctx_gen.set_value_for_project(args.project, "test", "on") - manifest = load_project(opts, args.project) + loader = ManifestLoader(opts, ctx_gen) + manifest = loader.load_manifest(args.project) - projects = manifests_in_dependency_order(opts, manifest, ctx_gen) - manifests_by_name = {m.name: m for m in projects} + projects = loader.manifests_in_dependency_order() # Accumulate the install directories so that the build steps # can find their dep installation @@ -331,7 +333,7 @@ class FixupDeps(SubCmd): ctx = ctx_gen.get_context(m.name) fetcher = m.create_fetcher(opts, ctx) - dirs = opts.compute_dirs(m, fetcher, manifests_by_name, ctx_gen) + dirs = opts.compute_dirs(m, fetcher, loader.manifests_by_name, ctx_gen) inst_dir = dirs["inst_dir"] install_dirs.append(inst_dir) @@ -376,9 +378,9 @@ class TestCmd(SubCmd): else: ctx_gen.set_value_for_project(args.project, "test", "on") - manifest = load_project(opts, args.project) - projects = manifests_in_dependency_order(opts, manifest, ctx_gen) - manifests_by_name = {m.name: m for m in projects} + loader = ManifestLoader(opts, ctx_gen) + manifest = loader.load_manifest(args.project) + projects = loader.manifests_in_dependency_order() # Accumulate the install directories so that the test steps # can find their dep installation @@ -388,7 +390,7 @@ class TestCmd(SubCmd): ctx = ctx_gen.get_context(m.name) fetcher = m.create_fetcher(opts, ctx) - dirs = opts.compute_dirs(m, fetcher, manifests_by_name, ctx_gen) + dirs = opts.compute_dirs(m, fetcher, loader.manifests_by_name, ctx_gen) build_dir = dirs["build_dir"] inst_dir = dirs["inst_dir"] diff --git a/build/fbcode_builder/getdeps/load.py b/build/fbcode_builder/getdeps/load.py index 6665c8638..bd1d05c98 100644 --- a/build/fbcode_builder/getdeps/load.py +++ b/build/fbcode_builder/getdeps/load.py @@ -83,68 +83,107 @@ def load_all_manifests(build_opts): return LOADER.load_all(build_opts) -def manifests_in_dependency_order(build_opts, manifest, ctx_gen): - """ Given a manifest, expand its dependencies and return a list - of the manifest objects that would need to be built in the order - that they would need to be built. This does not evaluate whether - a build is needed; it just returns the list in the order specified - by the manifest files. """ - # A dict to save loading a project multiple times - manifests_by_name = {manifest.name: manifest} - # The list of deps that have been fully processed - seen = set() - # The list of deps which have yet to be evaluated. This - # can potentially contain duplicates. - deps = [manifest] - # The list of manifests in dependency order - dep_order = [] +class ManifestLoader(object): + """ ManifestLoader stores information about project manifest relationships for a + given set of (build options + platform) configuration. - while len(deps) > 0: - m = deps.pop(0) - if m.name in seen: - continue + The ManifestLoader class primarily serves as a location to cache project dependency + relationships and project hash values for this build configuration. + """ - # Consider its deps, if any. - # We sort them for increased determinism; we'll produce - # a correct order even if they aren't sorted, but we prefer - # to produce the same order regardless of how they are listed - # in the project manifest files. - ctx = ctx_gen.get_context(m.name) - dep_list = sorted(m.get_section_as_dict("dependencies", ctx).keys()) - builder = m.get("build", "builder", ctx=ctx) - if builder == "cmake": - dep_list.append("cmake") - elif builder == "autoconf" and m.name not in ( - "autoconf", - "libtool", - "automake", - ): - # they need libtool and its deps (automake, autoconf) so add - # those as deps (but obviously not if we're building those - # projects themselves) - dep_list.append("libtool") + def __init__(self, build_opts, ctx_gen=None): + self._loader = LOADER + self.build_opts = build_opts + if ctx_gen is None: + self.ctx_gen = self.build_opts.get_context_generator() + else: + self.ctx_gen = ctx_gen - dep_count = 0 - for dep in dep_list: - # If we're not sure whether it is done, queue it up - if dep not in seen: - if dep not in manifests_by_name: - dep = load_project(build_opts, dep) - manifests_by_name[dep.name] = dep - else: - dep = manifests_by_name[dep] + self.manifests_by_name = {} + self._loaded_all = False - deps.append(dep) - dep_count += 1 + def load_manifest(self, name): + manifest = self.manifests_by_name.get(name) + if manifest is None: + manifest = self._loader.load_project(self.build_opts, name) + self.manifests_by_name[name] = manifest + return manifest - if dep_count > 0: - # If we queued anything, re-queue this item, as it depends - # those new item(s) and their transitive deps. - deps.append(m) - continue + def load_all_manifests(self): + if not self._loaded_all: + self.manifests_by_name = self._loader.load_all(self.build_opts) + self._loaded_all = True - # Its deps are done, so we can emit it - seen.add(m.name) - dep_order.append(m) + return self.manifests_by_name - return dep_order + def manifests_in_dependency_order(self, manifest=None): + """ Compute all dependencies of the specified project. Returns a list of the + dependencies plus the project itself, in topologically sorted order. + + Each entry in the returned list only depends on projects that appear before it + in the list. + + If the input manifest is None, the dependencies for all currently loaded + projects will be computed. i.e., if you call load_all_manifests() followed by + manifests_in_dependency_order() this will return a global dependency ordering of + all projects. """ + # The list of deps that have been fully processed + seen = set() + # The list of deps which have yet to be evaluated. This + # can potentially contain duplicates. + if manifest is None: + deps = list(self.manifests_by_name.values()) + else: + assert manifest.name in self.manifests_by_name + deps = [manifest] + # The list of manifests in dependency order + dep_order = [] + + while len(deps) > 0: + m = deps.pop(0) + if m.name in seen: + continue + + # Consider its deps, if any. + # We sort them for increased determinism; we'll produce + # a correct order even if they aren't sorted, but we prefer + # to produce the same order regardless of how they are listed + # in the project manifest files. + ctx = self.ctx_gen.get_context(m.name) + dep_list = sorted(m.get_section_as_dict("dependencies", ctx).keys()) + builder = m.get("build", "builder", ctx=ctx) + if builder == "cmake": + dep_list.append("cmake") + elif builder == "autoconf" and m.name not in ( + "autoconf", + "libtool", + "automake", + ): + # they need libtool and its deps (automake, autoconf) so add + # those as deps (but obviously not if we're building those + # projects themselves) + dep_list.append("libtool") + + dep_count = 0 + for dep_name in dep_list: + # If we're not sure whether it is done, queue it up + if dep_name not in seen: + dep = self.manifests_by_name.get(dep_name) + if dep is None: + dep = self._loader.load_project(self.build_opts, dep_name) + self.manifests_by_name[dep.name] = dep + + deps.append(dep) + dep_count += 1 + + if dep_count > 0: + # If we queued anything, re-queue this item, as it depends + # those new item(s) and their transitive deps. + deps.append(m) + continue + + # Its deps are done, so we can emit it + seen.add(m.name) + dep_order.append(m) + + return dep_order