From d0c4fccca92d2f267c2f12cdd89ff0280c0488a4 Mon Sep 17 00:00:00 2001 From: Wez Furlong Date: Fri, 3 May 2019 15:52:39 -0700 Subject: [PATCH] fbcode_builder: getdeps: add list-deps subcommand Summary: While the command isn't necessarily super useful on its own, it does show that the plumbing for walking the deps is functioning, and that is important when it comes to building. The output lists the projects in the order that they would be built. The `fetch` command has been augmented to add a `--recursive` flag that uses the same mechanism to recursively fetch the dependencies. Reviewed By: simpkins Differential Revision: D14691004 fbshipit-source-id: b00bf6ad4742f8bb0a70698f71a5fe03d6a1f453 --- build/fbcode_builder/getdeps.py | 57 ++++++++++++++++-- build/fbcode_builder/getdeps/load.py | 75 ++++++++++++++++++++++++ build/fbcode_builder/getdeps/platform.py | 18 ++++++ 3 files changed, 144 insertions(+), 6 deletions(-) diff --git a/build/fbcode_builder/getdeps.py b/build/fbcode_builder/getdeps.py index 0fde4c749..0d9c723be 100755 --- a/build/fbcode_builder/getdeps.py +++ b/build/fbcode_builder/getdeps.py @@ -15,9 +15,9 @@ import subprocess import sys from getdeps.buildopts import setup_build_options -from getdeps.load import resolve_manifest_path +from getdeps.load import load_project, manifests_in_dependency_order from getdeps.manifest import ManifestParser -from getdeps.platform import HostType +from getdeps.platform import HostType, context_from_host_tuple from getdeps.subcmd import SubCmd, add_subcommands, cmd @@ -57,13 +57,58 @@ class FetchCmd(SubCmd): "file describing the project" ), ) + parser.add_argument( + "--recursive", + help="fetch the transitive deps also", + action="store_true", + default=False, + ) + parser.add_argument( + "--host-type", + help=( + "When recursively fetching, fetch deps for " + "this host type rather than the current system" + ), + ) def run(self, args): opts = setup_build_options(args) - manifest_path = resolve_manifest_path(opts, args.project) - manifest = ManifestParser(manifest_path) - fetcher = manifest.create_fetcher(opts, ctx={}) - fetcher.update() + manifest = load_project(opts, args.project) + ctx = context_from_host_tuple(args.host_type) + if args.recursive: + projects = manifests_in_dependency_order(opts, manifest, ctx) + else: + projects = [manifest] + for m in projects: + fetcher = m.create_fetcher(opts, ctx) + fetcher.update() + + +@cmd("list-deps", "lists the transitive deps for a given project") +class ListDepsCmd(SubCmd): + def run(self, args): + opts = setup_build_options(args) + manifest = load_project(opts, args.project) + ctx = context_from_host_tuple(args.host_type) + for m in manifests_in_dependency_order(opts, manifest, ctx): + print(m.name) + return 0 + + def setup_parser(self, parser): + parser.add_argument( + "--host-type", + help=( + "Produce the list for the specified host type, " + "rather than that of the current system" + ), + ) + parser.add_argument( + "project", + help=( + "name of the project or path to a manifest " + "file describing the project" + ), + ) def build_argparser(): diff --git a/build/fbcode_builder/getdeps/load.py b/build/fbcode_builder/getdeps/load.py index ffbca2b92..116f64492 100644 --- a/build/fbcode_builder/getdeps/load.py +++ b/build/fbcode_builder/getdeps/load.py @@ -9,6 +9,8 @@ from __future__ import absolute_import, division, print_function, unicode_litera import os +from .manifest import ManifestParser + def resolve_manifest_path(build_opts, project_name): if "/" in project_name or "\\" in project_name: @@ -17,3 +19,76 @@ def resolve_manifest_path(build_opts, project_name): # Otherwise, resolve it relative to the manifests dir return os.path.join(build_opts.fbcode_builder_dir, "manifests", project_name) + + +def load_project(build_opts, project_name): + """ given the name of a project or a path to a manifest file, + load up the ManifestParser instance for it and return it """ + manifest_path = resolve_manifest_path(build_opts, project_name) + return ManifestParser(manifest_path) + + +def manifests_in_dependency_order(build_opts, manifest, ctx): + """ Given a manifest, expand its dependencies and return a list + of the manifest objects that would need to be built in the order + that they would need to be built. This does not evaluate whether + a build is needed; it just returns the list in the order specified + by the manifest files. """ + # A dict to save loading a project multiple times + manifests_by_name = {manifest.name: manifest} + # The list of deps that have been fully processed + seen = set() + # The list of deps which have yet to be evaluated. This + # can potentially contain duplicates. + deps = [manifest] + # The list of manifests in dependency order + dep_order = [] + + while len(deps) > 0: + m = deps.pop(0) + if m.name in seen: + continue + + # Consider its deps, if any. + # We sort them for increased determinism; we'll produce + # a correct order even if they aren't sorted, but we prefer + # to produce the same order regardless of how they are listed + # in the project manifest files. + dep_list = sorted(m.get_section_as_dict("dependencies", ctx).keys()) + builder = m.get("build", "builder", ctx=ctx) + if builder == "cmake": + dep_list.append("cmake") + elif builder == "autoconf" and m.name not in ( + "autoconf", + "libtool", + "automake", + ): + # they need libtool and its deps (automake, autoconf) so add + # those as deps (but obviously not if we're building those + # projects themselves) + dep_list.append("libtool") + + dep_count = 0 + for dep in dep_list: + # If we're not sure whether it is done, queue it up + if dep not in seen: + if dep not in manifests_by_name: + dep = load_project(build_opts, dep) + manifests_by_name[dep.name] = dep + else: + dep = manifests_by_name[dep] + + deps.append(dep) + dep_count += 1 + + if dep_count > 0: + # If we queued anything, re-queue this item, as it depends + # those new item(s) and their transitive deps. + deps.append(m) + continue + + # Its deps are done, so we can emit it + seen.add(m.name) + dep_order.append(m) + + return dep_order diff --git a/build/fbcode_builder/getdeps/platform.py b/build/fbcode_builder/getdeps/platform.py index 27270c7e6..c52bc0fc6 100644 --- a/build/fbcode_builder/getdeps/platform.py +++ b/build/fbcode_builder/getdeps/platform.py @@ -96,3 +96,21 @@ class HostType(object): and self.distro == b.distro and self.distrovers == b.distrovers ) + + +def context_from_host_tuple(host_tuple=None): + """ Given an optional host tuple, construct a context appropriate + for passing to the boolean expression evaluator so that conditional + sections in manifests can be resolved. """ + if host_tuple is None: + host_type = HostType() + elif isinstance(host_tuple, HostType): + host_type = host_tuple + else: + host_type = HostType.from_tuple_string(host_tuple) + + return { + "os": host_type.ostype, + "distro": host_type.distro, + "distro_vers": host_type.distrovers, + }