fbcode_builder: getdeps: add list-deps subcommand

Summary:
While the command isn't necessarily super useful
on its own, it does show that the plumbing for walking the
deps is functioning, and that is important when it comes
to building.

The output lists the projects in the order that they
would be built.

The `fetch` command has been augmented to add a `--recursive`
flag that uses the same mechanism to recursively fetch
the dependencies.

Reviewed By: simpkins

Differential Revision: D14691004

fbshipit-source-id: b00bf6ad4742f8bb0a70698f71a5fe03d6a1f453

build/fbcode_builder/getdeps.py

@@ -15,9 +15,9 @@ import subprocess
 import sys
 
 from getdeps.buildopts import setup_build_options
-from getdeps.load import resolve_manifest_path
+from getdeps.load import load_project, manifests_in_dependency_order
 from getdeps.manifest import ManifestParser
-from getdeps.platform import HostType
+from getdeps.platform import HostType, context_from_host_tuple
 from getdeps.subcmd import SubCmd, add_subcommands, cmd
 
 
@@ -57,15 +57,60 @@ class FetchCmd(SubCmd):
                 "file describing the project"
             ),
         )
+        parser.add_argument(
+            "--recursive",
+            help="fetch the transitive deps also",
+            action="store_true",
+            default=False,
+        )
+        parser.add_argument(
+            "--host-type",
+            help=(
+                "When recursively fetching, fetch deps for "
+                "this host type rather than the current system"
+            ),
+        )
 
     def run(self, args):
         opts = setup_build_options(args)
-        manifest_path = resolve_manifest_path(opts, args.project)
+        manifest = load_project(opts, args.project)
-        manifest = ManifestParser(manifest_path)
+        ctx = context_from_host_tuple(args.host_type)
-        fetcher = manifest.create_fetcher(opts, ctx={})
+        if args.recursive:
+            projects = manifests_in_dependency_order(opts, manifest, ctx)
+        else:
+            projects = [manifest]
+        for m in projects:
+            fetcher = m.create_fetcher(opts, ctx)
             fetcher.update()
 
 
+@cmd("list-deps", "lists the transitive deps for a given project")
+class ListDepsCmd(SubCmd):
+    def run(self, args):
+        opts = setup_build_options(args)
+        manifest = load_project(opts, args.project)
+        ctx = context_from_host_tuple(args.host_type)
+        for m in manifests_in_dependency_order(opts, manifest, ctx):
+            print(m.name)
+        return 0
+
+    def setup_parser(self, parser):
+        parser.add_argument(
+            "--host-type",
+            help=(
+                "Produce the list for the specified host type, "
+                "rather than that of the current system"
+            ),
+        )
+        parser.add_argument(
+            "project",
+            help=(
+                "name of the project or path to a manifest "
+                "file describing the project"
+            ),
+        )
+
+
 def build_argparser():
     common_args = argparse.ArgumentParser(add_help=False)
     common_args.add_argument(

build/fbcode_builder/getdeps/load.py

@@ -9,6 +9,8 @@ from __future__ import absolute_import, division, print_function, unicode_litera
 
 import os
 
+from .manifest import ManifestParser
+
 
 def resolve_manifest_path(build_opts, project_name):
     if "/" in project_name or "\\" in project_name:
@@ -17,3 +19,76 @@ def resolve_manifest_path(build_opts, project_name):
 
     # Otherwise, resolve it relative to the manifests dir
     return os.path.join(build_opts.fbcode_builder_dir, "manifests", project_name)
+
+
+def load_project(build_opts, project_name):
+    """ given the name of a project or a path to a manifest file,
+    load up the ManifestParser instance for it and return it """
+    manifest_path = resolve_manifest_path(build_opts, project_name)
+    return ManifestParser(manifest_path)
+
+
+def manifests_in_dependency_order(build_opts, manifest, ctx):
+    """ Given a manifest, expand its dependencies and return a list
+    of the manifest objects that would need to be built in the order
+    that they would need to be built.  This does not evaluate whether
+    a build is needed; it just returns the list in the order specified
+    by the manifest files. """
+    # A dict to save loading a project multiple times
+    manifests_by_name = {manifest.name: manifest}
+    # The list of deps that have been fully processed
+    seen = set()
+    # The list of deps which have yet to be evaluated.  This
+    # can potentially contain duplicates.
+    deps = [manifest]
+    # The list of manifests in dependency order
+    dep_order = []
+
+    while len(deps) > 0:
+        m = deps.pop(0)
+        if m.name in seen:
+            continue
+
+        # Consider its deps, if any.
+        # We sort them for increased determinism; we'll produce
+        # a correct order even if they aren't sorted, but we prefer
+        # to produce the same order regardless of how they are listed
+        # in the project manifest files.
+        dep_list = sorted(m.get_section_as_dict("dependencies", ctx).keys())
+        builder = m.get("build", "builder", ctx=ctx)
+        if builder == "cmake":
+            dep_list.append("cmake")
+        elif builder == "autoconf" and m.name not in (
+            "autoconf",
+            "libtool",
+            "automake",
+        ):
+            # they need libtool and its deps (automake, autoconf) so add
+            # those as deps (but obviously not if we're building those
+            # projects themselves)
+            dep_list.append("libtool")
+
+        dep_count = 0
+        for dep in dep_list:
+            # If we're not sure whether it is done, queue it up
+            if dep not in seen:
+                if dep not in manifests_by_name:
+                    dep = load_project(build_opts, dep)
+                    manifests_by_name[dep.name] = dep
+                else:
+                    dep = manifests_by_name[dep]
+
+                deps.append(dep)
+                dep_count += 1
+
+        if dep_count > 0:
+            # If we queued anything, re-queue this item, as it depends
+            # those new item(s) and their transitive deps.
+            deps.append(m)
+            continue
+
+        # Its deps are done, so we can emit it
+        seen.add(m.name)
+        dep_order.append(m)
+
+    return dep_order

build/fbcode_builder/getdeps/platform.py

@@ -96,3 +96,21 @@ class HostType(object):
             and self.distro == b.distro
             and self.distrovers == b.distrovers
         )
+
+
+def context_from_host_tuple(host_tuple=None):
+    """ Given an optional host tuple, construct a context appropriate
+    for passing to the boolean expression evaluator so that conditional
+    sections in manifests can be resolved. """
+    if host_tuple is None:
+        host_type = HostType()
+    elif isinstance(host_tuple, HostType):
+        host_type = host_tuple
+    else:
+        host_type = HostType.from_tuple_string(host_tuple)
+
+    return {
+        "os": host_type.ostype,
+        "distro": host_type.distro,
+        "distro_vers": host_type.distrovers,
+    }