diff --git a/build/fbcode_builder/getdeps.py b/build/fbcode_builder/getdeps.py index 0fde4c749..0d9c723be 100755 --- a/build/fbcode_builder/getdeps.py +++ b/build/fbcode_builder/getdeps.py @@ -15,9 +15,9 @@ import subprocess import sys from getdeps.buildopts import setup_build_options -from getdeps.load import resolve_manifest_path +from getdeps.load import load_project, manifests_in_dependency_order from getdeps.manifest import ManifestParser -from getdeps.platform import HostType +from getdeps.platform import HostType, context_from_host_tuple from getdeps.subcmd import SubCmd, add_subcommands, cmd @@ -57,13 +57,58 @@ class FetchCmd(SubCmd): "file describing the project" ), ) + parser.add_argument( + "--recursive", + help="fetch the transitive deps also", + action="store_true", + default=False, + ) + parser.add_argument( + "--host-type", + help=( + "When recursively fetching, fetch deps for " + "this host type rather than the current system" + ), + ) def run(self, args): opts = setup_build_options(args) - manifest_path = resolve_manifest_path(opts, args.project) - manifest = ManifestParser(manifest_path) - fetcher = manifest.create_fetcher(opts, ctx={}) - fetcher.update() + manifest = load_project(opts, args.project) + ctx = context_from_host_tuple(args.host_type) + if args.recursive: + projects = manifests_in_dependency_order(opts, manifest, ctx) + else: + projects = [manifest] + for m in projects: + fetcher = m.create_fetcher(opts, ctx) + fetcher.update() + + +@cmd("list-deps", "lists the transitive deps for a given project") +class ListDepsCmd(SubCmd): + def run(self, args): + opts = setup_build_options(args) + manifest = load_project(opts, args.project) + ctx = context_from_host_tuple(args.host_type) + for m in manifests_in_dependency_order(opts, manifest, ctx): + print(m.name) + return 0 + + def setup_parser(self, parser): + parser.add_argument( + "--host-type", + help=( + "Produce the list for the specified host type, " + "rather than that of the current system" + ), + ) + parser.add_argument( + "project", + help=( + "name of the project or path to a manifest " + "file describing the project" + ), + ) def build_argparser(): diff --git a/build/fbcode_builder/getdeps/load.py b/build/fbcode_builder/getdeps/load.py index ffbca2b92..116f64492 100644 --- a/build/fbcode_builder/getdeps/load.py +++ b/build/fbcode_builder/getdeps/load.py @@ -9,6 +9,8 @@ from __future__ import absolute_import, division, print_function, unicode_litera import os +from .manifest import ManifestParser + def resolve_manifest_path(build_opts, project_name): if "/" in project_name or "\\" in project_name: @@ -17,3 +19,76 @@ def resolve_manifest_path(build_opts, project_name): # Otherwise, resolve it relative to the manifests dir return os.path.join(build_opts.fbcode_builder_dir, "manifests", project_name) + + +def load_project(build_opts, project_name): + """ given the name of a project or a path to a manifest file, + load up the ManifestParser instance for it and return it """ + manifest_path = resolve_manifest_path(build_opts, project_name) + return ManifestParser(manifest_path) + + +def manifests_in_dependency_order(build_opts, manifest, ctx): + """ Given a manifest, expand its dependencies and return a list + of the manifest objects that would need to be built in the order + that they would need to be built. This does not evaluate whether + a build is needed; it just returns the list in the order specified + by the manifest files. """ + # A dict to save loading a project multiple times + manifests_by_name = {manifest.name: manifest} + # The list of deps that have been fully processed + seen = set() + # The list of deps which have yet to be evaluated. This + # can potentially contain duplicates. + deps = [manifest] + # The list of manifests in dependency order + dep_order = [] + + while len(deps) > 0: + m = deps.pop(0) + if m.name in seen: + continue + + # Consider its deps, if any. + # We sort them for increased determinism; we'll produce + # a correct order even if they aren't sorted, but we prefer + # to produce the same order regardless of how they are listed + # in the project manifest files. + dep_list = sorted(m.get_section_as_dict("dependencies", ctx).keys()) + builder = m.get("build", "builder", ctx=ctx) + if builder == "cmake": + dep_list.append("cmake") + elif builder == "autoconf" and m.name not in ( + "autoconf", + "libtool", + "automake", + ): + # they need libtool and its deps (automake, autoconf) so add + # those as deps (but obviously not if we're building those + # projects themselves) + dep_list.append("libtool") + + dep_count = 0 + for dep in dep_list: + # If we're not sure whether it is done, queue it up + if dep not in seen: + if dep not in manifests_by_name: + dep = load_project(build_opts, dep) + manifests_by_name[dep.name] = dep + else: + dep = manifests_by_name[dep] + + deps.append(dep) + dep_count += 1 + + if dep_count > 0: + # If we queued anything, re-queue this item, as it depends + # those new item(s) and their transitive deps. + deps.append(m) + continue + + # Its deps are done, so we can emit it + seen.add(m.name) + dep_order.append(m) + + return dep_order diff --git a/build/fbcode_builder/getdeps/platform.py b/build/fbcode_builder/getdeps/platform.py index 27270c7e6..c52bc0fc6 100644 --- a/build/fbcode_builder/getdeps/platform.py +++ b/build/fbcode_builder/getdeps/platform.py @@ -96,3 +96,21 @@ class HostType(object): and self.distro == b.distro and self.distrovers == b.distrovers ) + + +def context_from_host_tuple(host_tuple=None): + """ Given an optional host tuple, construct a context appropriate + for passing to the boolean expression evaluator so that conditional + sections in manifests can be resolved. """ + if host_tuple is None: + host_type = HostType() + elif isinstance(host_tuple, HostType): + host_type = host_tuple + else: + host_type = HostType.from_tuple_string(host_tuple) + + return { + "os": host_type.ostype, + "distro": host_type.distro, + "distro_vers": host_type.distrovers, + }