Subprojects support

Summary:
X-link: https://github.com/facebookincubator/zstrong/pull/857

Where one project should be checked out in a subdirectory of another
project. Like git submodules.

This is how the Glean build currently works: hsthrift is a separate
git repo, but Glean builds with hsthrift checked out in a
subdirectory.

Reviewed By: chadaustin

Differential Revision: D58055066

fbshipit-source-id: 1a22abaa8c5261c40b752d685a03d01625215b12

build/fbcode_builder/getdeps/fetcher.py

@@ -587,6 +587,39 @@ class SimpleShipitTransformerFetcher(Fetcher):
         return self.repo_dir
 
 
+class SubFetcher(Fetcher):
+    """Fetcher for a project with subprojects"""
+
+    def __init__(self, base, subs) -> None:
+        self.base = base
+        self.subs = subs
+
+    def update(self) -> ChangeStatus:
+        base = self.base.update()
+        changed = base.build_changed() or base.sources_changed()
+        for fetcher, dir in self.subs:
+            stat = fetcher.update()
+            if stat.build_changed() or stat.sources_changed():
+                changed = True
+            link = self.base.get_src_dir() + "/" + dir
+            if not os.path.exists(link):
+                os.symlink(fetcher.get_src_dir(), link)
+        return ChangeStatus(changed)
+
+    def clean(self) -> None:
+        self.base.clean()
+        for fetcher, _ in self.subs:
+            fetcher.clean()
+
+    def hash(self) -> None:
+        hash = self.base.hash()
+        for fetcher, _ in self.subs:
+            hash += fetcher.hash()
+
+    def get_src_dir(self):
+        return self.base.get_src_dir()
+
+
 class ShipitTransformerFetcher(Fetcher):
     @classmethod
     def _shipit_paths(cls, build_options):

build/fbcode_builder/getdeps/load.py

@@ -251,7 +251,7 @@ class ManifestLoader(object):
             return override
 
         ctx = self.ctx_gen.get_context(manifest.name)
-        return manifest.create_fetcher(self.build_opts, ctx)
+        return manifest.create_fetcher(self.build_opts, self, ctx)
 
     def get_project_hash(self, manifest):
         h = self._project_hashes.get(manifest.name)

build/fbcode_builder/getdeps/manifest.py

@@ -30,6 +30,7 @@ from .fetcher import (
     PreinstalledNopFetcher,
     ShipitTransformerFetcher,
     SimpleShipitTransformerFetcher,
+    SubFetcher,
     SystemPackageFetcher,
 )
 from .py_wheel_builder import PythonWheelBuilder
@@ -105,6 +106,7 @@ SCHEMA = {
     "shipit.pathmap": {"optional_section": True},
     "shipit.strip": {"optional_section": True},
     "install.files": {"optional_section": True},
+    "subprojects": {"optional_section": True},
     # fb-only
     "sandcastle": {"optional_section": True, "fields": {"run_tests": OPTIONAL}},
 }
@@ -396,7 +398,7 @@ class ManifestParser(object):
     def get_repo_url(self, ctx):
         return self.get("git", "repo_url", ctx=ctx)
 
-    def create_fetcher(self, build_options, ctx):
+    def _create_fetcher(self, build_options, ctx):
         real_shipit_available = ShipitTransformerFetcher.available(build_options)
         use_real_shipit = real_shipit_available and (
             build_options.use_shipit
@@ -456,6 +458,19 @@ class ManifestParser(object):
             "project %s has no fetcher configuration matching %s" % (self.name, ctx)
         )
 
+    def create_fetcher(self, build_options, loader, ctx):
+        fetcher = self._create_fetcher(build_options, ctx)
+        subprojects = self.get_section_as_ordered_pairs("subprojects", ctx)
+        if subprojects:
+            subs = []
+            for project, subdir in subprojects:
+                submanifest = loader.load_manifest(project)
+                subfetcher = submanifest.create_fetcher(build_options, loader, ctx)
+                subs.append((subfetcher, subdir))
+            return SubFetcher(fetcher, subs)
+        else:
+            return fetcher
+
     def get_builder_name(self, ctx):
         builder = self.get("build", "builder", ctx=ctx)
         if not builder: