add a new ManifestLoader class

Summary:
Add a new ManifestLoader class to handle loading manifests and computing
dependencies.

For now the main thing this class does is maintain the `manifest_by_name`
mapping.  In subsequent diffs we should be able to move some additional logic
into this class, which will help clean up the code and eliminate some redudant
work.  In particular, we can have this class cache project hashes, which will
avoid re-computing hashes over and over again for the same projects as we do
in many cases today.  We should also be able to save and re-use some of the
project dependency ordering computation in some cases as well.

Reviewed By: strager

Differential Revision: D16477400

fbshipit-source-id: f06f62f77d8443fccaa69fe4c1306e39c395b325

build/fbcode_builder/getdeps.py

@@ -18,7 +18,7 @@ import sys
 from getdeps.buildopts import setup_build_options
 from getdeps.dyndeps import create_dyn_dep_munger
 from getdeps.errors import TransientFailure
-from getdeps.load import load_project, manifests_in_dependency_order
+from getdeps.load import ManifestLoader
 from getdeps.manifest import ManifestParser
 from getdeps.platform import HostType
 from getdeps.subcmd import SubCmd, add_subcommands, cmd
@@ -84,14 +84,14 @@ class FetchCmd(SubCmd):
 
     def run(self, args):
         opts = setup_build_options(args)
-        ctx_gen = opts.get_context_generator()
+        loader = ManifestLoader(opts)
-        manifest = load_project(opts, args.project)
+        manifest = loader.load_manifest(args.project)
         if args.recursive:
-            projects = manifests_in_dependency_order(opts, manifest, ctx_gen)
+            projects = loader.manifests_in_dependency_order()
         else:
             projects = [manifest]
         for m in projects:
-            fetcher = m.create_fetcher(opts, ctx_gen.get_context(m.name))
+            fetcher = m.create_fetcher(opts, loader.ctx_gen.get_context(m.name))
             fetcher.update()
 
 
@@ -99,10 +99,10 @@ class FetchCmd(SubCmd):
 class ListDepsCmd(SubCmd):
     def run(self, args):
         opts = setup_build_options(args)
-        ctx_gen = opts.get_context_generator()
+        loader = ManifestLoader(opts)
-        ctx_gen.set_value_for_project(args.project, "test", "on")
+        loader.ctx_gen.set_value_for_project(args.project, "test", "on")
-        manifest = load_project(opts, args.project)
+        loader.load_manifest(args.project)
-        for m in manifests_in_dependency_order(opts, manifest, ctx_gen):
+        for m in loader.manifests_in_dependency_order():
             print(m.name)
         return 0
 
@@ -142,11 +142,10 @@ class CleanCmd(SubCmd):
 class ShowInstDirCmd(SubCmd):
     def run(self, args):
         opts = setup_build_options(args)
-        ctx_gen = opts.get_context_generator()
+        loader = ManifestLoader(opts)
-        ctx_gen.set_value_for_project(args.project, "test", "on")
+        loader.ctx_gen.set_value_for_project(args.project, "test", "on")
-        manifest = load_project(opts, args.project)
+        manifest = loader.load_manifest(args.project)
-        projects = manifests_in_dependency_order(opts, manifest, ctx_gen)
+        projects = loader.manifests_in_dependency_order()
-        manifests_by_name = {m.name: m for m in projects}
 
         if args.recursive:
             manifests = projects
@@ -154,9 +153,11 @@ class ShowInstDirCmd(SubCmd):
             manifests = [manifest]
 
         for m in manifests:
-            ctx = ctx_gen.get_context(m.name)
+            ctx = loader.ctx_gen.get_context(m.name)
             fetcher = m.create_fetcher(opts, ctx)
-            dirs = opts.compute_dirs(m, fetcher, manifests_by_name, ctx_gen)
+            dirs = opts.compute_dirs(
+                m, fetcher, loader.manifests_by_name, loader.ctx_gen
+            )
             inst_dir = dirs["inst_dir"]
             print(inst_dir)
 
@@ -180,17 +181,17 @@ class ShowInstDirCmd(SubCmd):
 class ShowSourceDirCmd(SubCmd):
     def run(self, args):
         opts = setup_build_options(args)
-        ctx_gen = opts.get_context_generator()
+        loader = ManifestLoader(opts)
-        ctx_gen.set_value_for_project(args.project, "test", "on")
+        loader.ctx_gen.set_value_for_project(args.project, "test", "on")
-        manifest = load_project(opts, args.project)
+        manifest = loader.load_manifest(args.project)
 
         if args.recursive:
-            manifests = manifests_in_dependency_order(opts, manifest, ctx_gen)
+            manifests = loader.manifests_in_dependency_order()
         else:
             manifests = [manifest]
 
         for m in manifests:
-            fetcher = m.create_fetcher(opts, ctx_gen.get_context(m.name))
+            fetcher = m.create_fetcher(opts, loader.ctx_gen.get_context(m.name))
             print(fetcher.get_src_dir())
 
     def setup_parser(self, parser):
@@ -213,17 +214,18 @@ class ShowSourceDirCmd(SubCmd):
 class BuildCmd(SubCmd):
     def run(self, args):
         opts = setup_build_options(args)
-        if args.clean:
-            clean_dirs(opts)
-
         ctx_gen = opts.get_context_generator(facebook_internal=args.facebook_internal)
         if args.enable_tests:
             ctx_gen.set_value_for_project(args.project, "test", "on")
-        manifest = load_project(opts, args.project)
+        loader = ManifestLoader(opts, ctx_gen)
+
+        if args.clean:
+            clean_dirs(opts)
+
+        manifest = loader.load_manifest(args.project)
 
         print("Building on %s" % ctx_gen.get_context(args.project))
-        projects = manifests_in_dependency_order(opts, manifest, ctx_gen)
+        projects = loader.manifests_in_dependency_order()
-        manifests_by_name = {m.name: m for m in projects}
 
         # Accumulate the install directories so that the build steps
         # can find their dep installation
@@ -236,7 +238,7 @@ class BuildCmd(SubCmd):
             if args.clean:
                 fetcher.clean()
 
-            dirs = opts.compute_dirs(m, fetcher, manifests_by_name, ctx_gen)
+            dirs = opts.compute_dirs(m, fetcher, loader.manifests_by_name, ctx_gen)
             build_dir = dirs["build_dir"]
             inst_dir = dirs["inst_dir"]
 
@@ -318,10 +320,10 @@ class FixupDeps(SubCmd):
         if args.enable_tests:
             ctx_gen.set_value_for_project(args.project, "test", "on")
 
-        manifest = load_project(opts, args.project)
+        loader = ManifestLoader(opts, ctx_gen)
+        manifest = loader.load_manifest(args.project)
 
-        projects = manifests_in_dependency_order(opts, manifest, ctx_gen)
+        projects = loader.manifests_in_dependency_order()
-        manifests_by_name = {m.name: m for m in projects}
 
         # Accumulate the install directories so that the build steps
         # can find their dep installation
@@ -331,7 +333,7 @@ class FixupDeps(SubCmd):
             ctx = ctx_gen.get_context(m.name)
             fetcher = m.create_fetcher(opts, ctx)
 
-            dirs = opts.compute_dirs(m, fetcher, manifests_by_name, ctx_gen)
+            dirs = opts.compute_dirs(m, fetcher, loader.manifests_by_name, ctx_gen)
             inst_dir = dirs["inst_dir"]
 
             install_dirs.append(inst_dir)
@@ -376,9 +378,9 @@ class TestCmd(SubCmd):
         else:
             ctx_gen.set_value_for_project(args.project, "test", "on")
 
-        manifest = load_project(opts, args.project)
+        loader = ManifestLoader(opts, ctx_gen)
-        projects = manifests_in_dependency_order(opts, manifest, ctx_gen)
+        manifest = loader.load_manifest(args.project)
-        manifests_by_name = {m.name: m for m in projects}
+        projects = loader.manifests_in_dependency_order()
 
         # Accumulate the install directories so that the test steps
         # can find their dep installation
@@ -388,7 +390,7 @@ class TestCmd(SubCmd):
             ctx = ctx_gen.get_context(m.name)
             fetcher = m.create_fetcher(opts, ctx)
 
-            dirs = opts.compute_dirs(m, fetcher, manifests_by_name, ctx_gen)
+            dirs = opts.compute_dirs(m, fetcher, loader.manifests_by_name, ctx_gen)
             build_dir = dirs["build_dir"]
             inst_dir = dirs["inst_dir"]
 

build/fbcode_builder/getdeps/load.py

@@ -83,68 +83,107 @@ def load_all_manifests(build_opts):
     return LOADER.load_all(build_opts)
 
 
-def manifests_in_dependency_order(build_opts, manifest, ctx_gen):
+class ManifestLoader(object):
-    """ Given a manifest, expand its dependencies and return a list
+    """ ManifestLoader stores information about project manifest relationships for a
-    of the manifest objects that would need to be built in the order
+    given set of (build options + platform) configuration.
-    that they would need to be built.  This does not evaluate whether
-    a build is needed; it just returns the list in the order specified
-    by the manifest files. """
-    # A dict to save loading a project multiple times
-    manifests_by_name = {manifest.name: manifest}
-    # The list of deps that have been fully processed
-    seen = set()
-    # The list of deps which have yet to be evaluated.  This
-    # can potentially contain duplicates.
-    deps = [manifest]
-    # The list of manifests in dependency order
-    dep_order = []
 
-    while len(deps) > 0:
+    The ManifestLoader class primarily serves as a location to cache project dependency
-        m = deps.pop(0)
+    relationships and project hash values for this build configuration.
-        if m.name in seen:
+    """
-            continue
 
-        # Consider its deps, if any.
+    def __init__(self, build_opts, ctx_gen=None):
-        # We sort them for increased determinism; we'll produce
+        self._loader = LOADER
-        # a correct order even if they aren't sorted, but we prefer
+        self.build_opts = build_opts
-        # to produce the same order regardless of how they are listed
+        if ctx_gen is None:
-        # in the project manifest files.
+            self.ctx_gen = self.build_opts.get_context_generator()
-        ctx = ctx_gen.get_context(m.name)
+        else:
-        dep_list = sorted(m.get_section_as_dict("dependencies", ctx).keys())
+            self.ctx_gen = ctx_gen
-        builder = m.get("build", "builder", ctx=ctx)
-        if builder == "cmake":
-            dep_list.append("cmake")
-        elif builder == "autoconf" and m.name not in (
-            "autoconf",
-            "libtool",
-            "automake",
-        ):
-            # they need libtool and its deps (automake, autoconf) so add
-            # those as deps (but obviously not if we're building those
-            # projects themselves)
-            dep_list.append("libtool")
 
-        dep_count = 0
+        self.manifests_by_name = {}
-        for dep in dep_list:
+        self._loaded_all = False
-            # If we're not sure whether it is done, queue it up
-            if dep not in seen:
-                if dep not in manifests_by_name:
-                    dep = load_project(build_opts, dep)
-                    manifests_by_name[dep.name] = dep
-                else:
-                    dep = manifests_by_name[dep]
 
-                deps.append(dep)
+    def load_manifest(self, name):
-                dep_count += 1
+        manifest = self.manifests_by_name.get(name)
+        if manifest is None:
+            manifest = self._loader.load_project(self.build_opts, name)
+            self.manifests_by_name[name] = manifest
+        return manifest
 
-        if dep_count > 0:
+    def load_all_manifests(self):
-            # If we queued anything, re-queue this item, as it depends
+        if not self._loaded_all:
-            # those new item(s) and their transitive deps.
+            self.manifests_by_name = self._loader.load_all(self.build_opts)
-            deps.append(m)
+            self._loaded_all = True
-            continue
 
-        # Its deps are done, so we can emit it
+        return self.manifests_by_name
-        seen.add(m.name)
-        dep_order.append(m)
 
-    return dep_order
+    def manifests_in_dependency_order(self, manifest=None):
+        """ Compute all dependencies of the specified project.  Returns a list of the
+        dependencies plus the project itself, in topologically sorted order.
+
+        Each entry in the returned list only depends on projects that appear before it
+        in the list.
+
+        If the input manifest is None, the dependencies for all currently loaded
+        projects will be computed.  i.e., if you call load_all_manifests() followed by
+        manifests_in_dependency_order() this will return a global dependency ordering of
+        all projects.  """
+        # The list of deps that have been fully processed
+        seen = set()
+        # The list of deps which have yet to be evaluated.  This
+        # can potentially contain duplicates.
+        if manifest is None:
+            deps = list(self.manifests_by_name.values())
+        else:
+            assert manifest.name in self.manifests_by_name
+            deps = [manifest]
+        # The list of manifests in dependency order
+        dep_order = []
+
+        while len(deps) > 0:
+            m = deps.pop(0)
+            if m.name in seen:
+                continue
+
+            # Consider its deps, if any.
+            # We sort them for increased determinism; we'll produce
+            # a correct order even if they aren't sorted, but we prefer
+            # to produce the same order regardless of how they are listed
+            # in the project manifest files.
+            ctx = self.ctx_gen.get_context(m.name)
+            dep_list = sorted(m.get_section_as_dict("dependencies", ctx).keys())
+            builder = m.get("build", "builder", ctx=ctx)
+            if builder == "cmake":
+                dep_list.append("cmake")
+            elif builder == "autoconf" and m.name not in (
+                "autoconf",
+                "libtool",
+                "automake",
+            ):
+                # they need libtool and its deps (automake, autoconf) so add
+                # those as deps (but obviously not if we're building those
+                # projects themselves)
+                dep_list.append("libtool")
+
+            dep_count = 0
+            for dep_name in dep_list:
+                # If we're not sure whether it is done, queue it up
+                if dep_name not in seen:
+                    dep = self.manifests_by_name.get(dep_name)
+                    if dep is None:
+                        dep = self._loader.load_project(self.build_opts, dep_name)
+                        self.manifests_by_name[dep.name] = dep
+
+                    deps.append(dep)
+                    dep_count += 1
+
+            if dep_count > 0:
+                # If we queued anything, re-queue this item, as it depends
+                # those new item(s) and their transitive deps.
+                deps.append(m)
+                continue
+
+            # Its deps are done, so we can emit it
+            seen.add(m.name)
+            dep_order.append(m)
+
+        return dep_order