getdeps: allow overriding project source, build, and install directories

Summary:
Add arguments to getdeps.py to allow overriding the source, build, and install
directories a per-project basis.  The arguments take the form `[PROJECT:]PATH`
If the `PROJECT` portion is omitted, it defaults to the current project being
built.

In particular this makes it possible to specify `--src-dir .` to tell
getdeps.py to find the project sources from the current directory rather than
downloading them.

Reviewed By: wez

Differential Revision: D16778011

fbshipit-source-id: f33b87213ace04abb66334f588babdf59df91964

build/fbcode_builder/getdeps.py

@@ -35,6 +35,10 @@ except ImportError:
 sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "getdeps"))
 
 
+class UsageError(Exception):
+    pass
+
+
 @cmd("validate-manifest", "parse a manifest and validate that it is correct")
 class ValidateManifest(SubCmd):
     def run(self, args):
@@ -70,10 +74,39 @@ class ProjectCmdBase(SubCmd):
             ctx_gen.set_value_for_project(args.project, "test", "off")
 
         loader = ManifestLoader(opts, ctx_gen)
+        self.process_project_dir_arguments(args, loader)
+
         manifest = loader.load_manifest(args.project)
 
         self.run_project_cmd(args, loader, manifest)
 
+    def process_project_dir_arguments(self, args, loader):
+        def parse_project_arg(arg, arg_type):
+            parts = arg.split(":")
+            if len(parts) == 2:
+                project, path = parts
+            elif len(parts) == 1:
+                project = args.project
+                path = parts[0]
+            else:
+                raise UsageError(
+                    "invalid %s argument; too many ':' characters: %s" % (arg_type, arg)
+                )
+
+            return project, os.path.abspath(path)
+
+        for arg in args.src_dir:
+            project, path = parse_project_arg(arg, "--src-dir")
+            loader.set_project_src_dir(project, path)
+
+        for arg in args.build_dir:
+            project, path = parse_project_arg(arg, "--build-dir")
+            loader.set_project_build_dir(project, path)
+
+        for arg in args.install_dir:
+            project, path = parse_project_arg(arg, "--install-dir")
+            loader.set_project_install_dir(project, path)
+
     def setup_parser(self, parser):
         parser.add_argument(
             "project",
@@ -94,6 +127,29 @@ class ProjectCmdBase(SubCmd):
             action="store_true",
             help="Enable building tests for dependencies as well.",
         )
+        parser.add_argument(
+            "--src-dir",
+            default=[],
+            action="append",
+            help="Specify a local directory to use for the project source, "
+            "rather than fetching it.",
+        )
+        parser.add_argument(
+            "--build-dir",
+            default=[],
+            action="append",
+            help="Explicitly specify the build directory to use for the "
+            "project, instead of the default location in the scratch path. "
+            "This only affects the project specified, and not its dependencies.",
+        )
+        parser.add_argument(
+            "--install-dir",
+            default=[],
+            action="append",
+            help="Explicitly specify the install directory to use for the "
+            "project, instead of the default location in the scratch path. "
+            "This only affects the project specified, and not its dependencies.",
+        )
 
         self.setup_project_cmd_parser(parser)
 
@@ -428,6 +484,9 @@ def main():
         return 0
     try:
         return args.func(args)
+    except UsageError as exc:
+        ap.error(str(exc))
+        return 1
     except TransientFailure as exc:
         print("TransientFailure: %s" % str(exc))
         # This return code is treated as a retryable transient infrastructure

build/fbcode_builder/getdeps/fetcher.py

@@ -130,6 +130,26 @@ class Fetcher(object):
         pass
 
 
+class LocalDirFetcher(object):
+    """ This class exists to override the normal fetching behavior, and
+    use an explicit user-specified directory for the project sources.
+
+    This fetcher cannot update or track changes.  It always reports that the
+    project has changed, forcing it to always be built. """
+
+    def __init__(self, path):
+        self.path = os.path.realpath(path)
+
+    def update(self):
+        return ChangeStatus(all_changed=True)
+
+    def hash(self):
+        return "0" * 40
+
+    def get_src_dir(self):
+        return self.path
+
+
 class GitFetcher(Fetcher):
     DEFAULT_DEPTH = 100
 

build/fbcode_builder/getdeps/load.py

@@ -12,6 +12,7 @@ import glob
 import hashlib
 import os
 
+from . import fetcher
 from .envfuncs import path_search
 from .manifest import ManifestParser
 
@@ -105,6 +106,9 @@ class ManifestLoader(object):
         self.manifests_by_name = {}
         self._loaded_all = False
         self._project_hashes = {}
+        self._fetcher_overrides = {}
+        self._build_dir_overrides = {}
+        self._install_dir_overrides = {}
 
     def load_manifest(self, name):
         manifest = self.manifests_by_name.get(name)
@@ -200,7 +204,20 @@ class ManifestLoader(object):
 
         return dep_order
 
+    def set_project_src_dir(self, project_name, path):
+        self._fetcher_overrides[project_name] = fetcher.LocalDirFetcher(path)
+
+    def set_project_build_dir(self, project_name, path):
+        self._build_dir_overrides[project_name] = path
+
+    def set_project_install_dir(self, project_name, path):
+        self._install_dir_overrides[project_name] = path
+
     def create_fetcher(self, manifest):
+        override = self._fetcher_overrides.get(manifest.name)
+        if override is not None:
+            return override
+
         ctx = self.ctx_gen.get_context(manifest.name)
         return manifest.create_fetcher(self.build_opts, ctx)
 
@@ -268,9 +285,17 @@ class ManifestLoader(object):
             return "%s-%s" % (manifest.name, project_hash)
 
     def get_project_install_dir(self, manifest):
+        override = self._install_dir_overrides.get(manifest.name)
+        if override:
+            return override
+
         project_dir_name = self._get_project_dir_name(manifest)
         return os.path.join(self.build_opts.install_dir, project_dir_name)
 
     def get_project_build_dir(self, manifest):
+        override = self._build_dir_overrides.get(manifest.name)
+        if override:
+            return override
+
         project_dir_name = self._get_project_dir_name(manifest)
         return os.path.join(self.build_opts.scratch_dir, "build", project_dir_name)