diff --git a/build/fbcode_builder/getdeps.py b/build/fbcode_builder/getdeps.py index 80f8812b1..80afa5505 100755 --- a/build/fbcode_builder/getdeps.py +++ b/build/fbcode_builder/getdeps.py @@ -18,7 +18,7 @@ import sys from getdeps.buildopts import setup_build_options from getdeps.dyndeps import create_dyn_dep_munger from getdeps.errors import TransientFailure -from getdeps.load import load_project, manifests_in_dependency_order +from getdeps.load import ManifestLoader from getdeps.manifest import ManifestParser from getdeps.platform import HostType from getdeps.subcmd import SubCmd, add_subcommands, cmd @@ -84,14 +84,14 @@ class FetchCmd(SubCmd): def run(self, args): opts = setup_build_options(args) - ctx_gen = opts.get_context_generator() - manifest = load_project(opts, args.project) + loader = ManifestLoader(opts) + manifest = loader.load_manifest(args.project) if args.recursive: - projects = manifests_in_dependency_order(opts, manifest, ctx_gen) + projects = loader.manifests_in_dependency_order() else: projects = [manifest] for m in projects: - fetcher = m.create_fetcher(opts, ctx_gen.get_context(m.name)) + fetcher = m.create_fetcher(opts, loader.ctx_gen.get_context(m.name)) fetcher.update() @@ -99,10 +99,10 @@ class FetchCmd(SubCmd): class ListDepsCmd(SubCmd): def run(self, args): opts = setup_build_options(args) - ctx_gen = opts.get_context_generator() - ctx_gen.set_value_for_project(args.project, "test", "on") - manifest = load_project(opts, args.project) - for m in manifests_in_dependency_order(opts, manifest, ctx_gen): + loader = ManifestLoader(opts) + loader.ctx_gen.set_value_for_project(args.project, "test", "on") + loader.load_manifest(args.project) + for m in loader.manifests_in_dependency_order(): print(m.name) return 0 @@ -142,11 +142,10 @@ class CleanCmd(SubCmd): class ShowInstDirCmd(SubCmd): def run(self, args): opts = setup_build_options(args) - ctx_gen = opts.get_context_generator() - ctx_gen.set_value_for_project(args.project, "test", "on") - manifest = load_project(opts, args.project) - projects = manifests_in_dependency_order(opts, manifest, ctx_gen) - manifests_by_name = {m.name: m for m in projects} + loader = ManifestLoader(opts) + loader.ctx_gen.set_value_for_project(args.project, "test", "on") + manifest = loader.load_manifest(args.project) + projects = loader.manifests_in_dependency_order() if args.recursive: manifests = projects @@ -154,9 +153,11 @@ class ShowInstDirCmd(SubCmd): manifests = [manifest] for m in manifests: - ctx = ctx_gen.get_context(m.name) + ctx = loader.ctx_gen.get_context(m.name) fetcher = m.create_fetcher(opts, ctx) - dirs = opts.compute_dirs(m, fetcher, manifests_by_name, ctx_gen) + dirs = opts.compute_dirs( + m, fetcher, loader.manifests_by_name, loader.ctx_gen + ) inst_dir = dirs["inst_dir"] print(inst_dir) @@ -180,17 +181,17 @@ class ShowInstDirCmd(SubCmd): class ShowSourceDirCmd(SubCmd): def run(self, args): opts = setup_build_options(args) - ctx_gen = opts.get_context_generator() - ctx_gen.set_value_for_project(args.project, "test", "on") - manifest = load_project(opts, args.project) + loader = ManifestLoader(opts) + loader.ctx_gen.set_value_for_project(args.project, "test", "on") + manifest = loader.load_manifest(args.project) if args.recursive: - manifests = manifests_in_dependency_order(opts, manifest, ctx_gen) + manifests = loader.manifests_in_dependency_order() else: manifests = [manifest] for m in manifests: - fetcher = m.create_fetcher(opts, ctx_gen.get_context(m.name)) + fetcher = m.create_fetcher(opts, loader.ctx_gen.get_context(m.name)) print(fetcher.get_src_dir()) def setup_parser(self, parser): @@ -213,17 +214,18 @@ class ShowSourceDirCmd(SubCmd): class BuildCmd(SubCmd): def run(self, args): opts = setup_build_options(args) - if args.clean: - clean_dirs(opts) - ctx_gen = opts.get_context_generator(facebook_internal=args.facebook_internal) if args.enable_tests: ctx_gen.set_value_for_project(args.project, "test", "on") - manifest = load_project(opts, args.project) + loader = ManifestLoader(opts, ctx_gen) + + if args.clean: + clean_dirs(opts) + + manifest = loader.load_manifest(args.project) print("Building on %s" % ctx_gen.get_context(args.project)) - projects = manifests_in_dependency_order(opts, manifest, ctx_gen) - manifests_by_name = {m.name: m for m in projects} + projects = loader.manifests_in_dependency_order() # Accumulate the install directories so that the build steps # can find their dep installation @@ -236,7 +238,7 @@ class BuildCmd(SubCmd): if args.clean: fetcher.clean() - dirs = opts.compute_dirs(m, fetcher, manifests_by_name, ctx_gen) + dirs = opts.compute_dirs(m, fetcher, loader.manifests_by_name, ctx_gen) build_dir = dirs["build_dir"] inst_dir = dirs["inst_dir"] @@ -318,10 +320,10 @@ class FixupDeps(SubCmd): if args.enable_tests: ctx_gen.set_value_for_project(args.project, "test", "on") - manifest = load_project(opts, args.project) + loader = ManifestLoader(opts, ctx_gen) + manifest = loader.load_manifest(args.project) - projects = manifests_in_dependency_order(opts, manifest, ctx_gen) - manifests_by_name = {m.name: m for m in projects} + projects = loader.manifests_in_dependency_order() # Accumulate the install directories so that the build steps # can find their dep installation @@ -331,7 +333,7 @@ class FixupDeps(SubCmd): ctx = ctx_gen.get_context(m.name) fetcher = m.create_fetcher(opts, ctx) - dirs = opts.compute_dirs(m, fetcher, manifests_by_name, ctx_gen) + dirs = opts.compute_dirs(m, fetcher, loader.manifests_by_name, ctx_gen) inst_dir = dirs["inst_dir"] install_dirs.append(inst_dir) @@ -376,9 +378,9 @@ class TestCmd(SubCmd): else: ctx_gen.set_value_for_project(args.project, "test", "on") - manifest = load_project(opts, args.project) - projects = manifests_in_dependency_order(opts, manifest, ctx_gen) - manifests_by_name = {m.name: m for m in projects} + loader = ManifestLoader(opts, ctx_gen) + manifest = loader.load_manifest(args.project) + projects = loader.manifests_in_dependency_order() # Accumulate the install directories so that the test steps # can find their dep installation @@ -388,7 +390,7 @@ class TestCmd(SubCmd): ctx = ctx_gen.get_context(m.name) fetcher = m.create_fetcher(opts, ctx) - dirs = opts.compute_dirs(m, fetcher, manifests_by_name, ctx_gen) + dirs = opts.compute_dirs(m, fetcher, loader.manifests_by_name, ctx_gen) build_dir = dirs["build_dir"] inst_dir = dirs["inst_dir"] diff --git a/build/fbcode_builder/getdeps/load.py b/build/fbcode_builder/getdeps/load.py index 6665c8638..bd1d05c98 100644 --- a/build/fbcode_builder/getdeps/load.py +++ b/build/fbcode_builder/getdeps/load.py @@ -83,68 +83,107 @@ def load_all_manifests(build_opts): return LOADER.load_all(build_opts) -def manifests_in_dependency_order(build_opts, manifest, ctx_gen): - """ Given a manifest, expand its dependencies and return a list - of the manifest objects that would need to be built in the order - that they would need to be built. This does not evaluate whether - a build is needed; it just returns the list in the order specified - by the manifest files. """ - # A dict to save loading a project multiple times - manifests_by_name = {manifest.name: manifest} - # The list of deps that have been fully processed - seen = set() - # The list of deps which have yet to be evaluated. This - # can potentially contain duplicates. - deps = [manifest] - # The list of manifests in dependency order - dep_order = [] +class ManifestLoader(object): + """ ManifestLoader stores information about project manifest relationships for a + given set of (build options + platform) configuration. - while len(deps) > 0: - m = deps.pop(0) - if m.name in seen: - continue + The ManifestLoader class primarily serves as a location to cache project dependency + relationships and project hash values for this build configuration. + """ - # Consider its deps, if any. - # We sort them for increased determinism; we'll produce - # a correct order even if they aren't sorted, but we prefer - # to produce the same order regardless of how they are listed - # in the project manifest files. - ctx = ctx_gen.get_context(m.name) - dep_list = sorted(m.get_section_as_dict("dependencies", ctx).keys()) - builder = m.get("build", "builder", ctx=ctx) - if builder == "cmake": - dep_list.append("cmake") - elif builder == "autoconf" and m.name not in ( - "autoconf", - "libtool", - "automake", - ): - # they need libtool and its deps (automake, autoconf) so add - # those as deps (but obviously not if we're building those - # projects themselves) - dep_list.append("libtool") + def __init__(self, build_opts, ctx_gen=None): + self._loader = LOADER + self.build_opts = build_opts + if ctx_gen is None: + self.ctx_gen = self.build_opts.get_context_generator() + else: + self.ctx_gen = ctx_gen - dep_count = 0 - for dep in dep_list: - # If we're not sure whether it is done, queue it up - if dep not in seen: - if dep not in manifests_by_name: - dep = load_project(build_opts, dep) - manifests_by_name[dep.name] = dep - else: - dep = manifests_by_name[dep] + self.manifests_by_name = {} + self._loaded_all = False - deps.append(dep) - dep_count += 1 + def load_manifest(self, name): + manifest = self.manifests_by_name.get(name) + if manifest is None: + manifest = self._loader.load_project(self.build_opts, name) + self.manifests_by_name[name] = manifest + return manifest - if dep_count > 0: - # If we queued anything, re-queue this item, as it depends - # those new item(s) and their transitive deps. - deps.append(m) - continue + def load_all_manifests(self): + if not self._loaded_all: + self.manifests_by_name = self._loader.load_all(self.build_opts) + self._loaded_all = True - # Its deps are done, so we can emit it - seen.add(m.name) - dep_order.append(m) + return self.manifests_by_name - return dep_order + def manifests_in_dependency_order(self, manifest=None): + """ Compute all dependencies of the specified project. Returns a list of the + dependencies plus the project itself, in topologically sorted order. + + Each entry in the returned list only depends on projects that appear before it + in the list. + + If the input manifest is None, the dependencies for all currently loaded + projects will be computed. i.e., if you call load_all_manifests() followed by + manifests_in_dependency_order() this will return a global dependency ordering of + all projects. """ + # The list of deps that have been fully processed + seen = set() + # The list of deps which have yet to be evaluated. This + # can potentially contain duplicates. + if manifest is None: + deps = list(self.manifests_by_name.values()) + else: + assert manifest.name in self.manifests_by_name + deps = [manifest] + # The list of manifests in dependency order + dep_order = [] + + while len(deps) > 0: + m = deps.pop(0) + if m.name in seen: + continue + + # Consider its deps, if any. + # We sort them for increased determinism; we'll produce + # a correct order even if they aren't sorted, but we prefer + # to produce the same order regardless of how they are listed + # in the project manifest files. + ctx = self.ctx_gen.get_context(m.name) + dep_list = sorted(m.get_section_as_dict("dependencies", ctx).keys()) + builder = m.get("build", "builder", ctx=ctx) + if builder == "cmake": + dep_list.append("cmake") + elif builder == "autoconf" and m.name not in ( + "autoconf", + "libtool", + "automake", + ): + # they need libtool and its deps (automake, autoconf) so add + # those as deps (but obviously not if we're building those + # projects themselves) + dep_list.append("libtool") + + dep_count = 0 + for dep_name in dep_list: + # If we're not sure whether it is done, queue it up + if dep_name not in seen: + dep = self.manifests_by_name.get(dep_name) + if dep is None: + dep = self._loader.load_project(self.build_opts, dep_name) + self.manifests_by_name[dep.name] = dep + + deps.append(dep) + dep_count += 1 + + if dep_count > 0: + # If we queued anything, re-queue this item, as it depends + # those new item(s) and their transitive deps. + deps.append(m) + continue + + # Its deps are done, so we can emit it + seen.add(m.name) + dep_order.append(m) + + return dep_order