diff --git a/build/fbcode_builder/getdeps/fetcher.py b/build/fbcode_builder/getdeps/fetcher.py index ec3f9034c..a414747c4 100644 --- a/build/fbcode_builder/getdeps/fetcher.py +++ b/build/fbcode_builder/getdeps/fetcher.py @@ -225,6 +225,79 @@ class GitFetcher(Fetcher): return self.repo_dir +class ShipitTransformerFetcher(Fetcher): + SHIPIT = "/var/www/scripts/opensource/shipit/run_shipit.php" + + def __init__(self, build_options, project_name): + self.build_options = build_options + self.project_name = project_name + self.repo_dir = os.path.join(build_options.scratch_dir, "shipit", project_name) + + def update(self): + if os.path.exists(self.repo_dir): + return ChangeStatus() + self.run_shipit() + return ChangeStatus(True) + + def clean(self): + if os.path.exists(self.repo_dir): + shutil.rmtree(self.repo_dir) + + @classmethod + def available(cls): + return os.path.exists(cls.SHIPIT) + + def run_shipit(self): + tmp_path = self.repo_dir + ".new" + try: + if os.path.exists(tmp_path): + shutil.rmtree(tmp_path) + + # Run shipit + run_cmd( + [ + "php", + ShipitTransformerFetcher.SHIPIT, + "--project=" + self.project_name, + "--create-new-repo", + "--source-repo-dir=" + self.build_options.fbsource_dir, + "--source-branch=.", + "--skip-source-init", + "--skip-source-pull", + "--skip-source-clean", + "--skip-push", + "--skip-reset", + "--skip-project-specific", + "--destination-use-anonymous-https", + "--create-new-repo-output-path=" + tmp_path, + ] + ) + + # Remove the .git directory from the repository it generated. + # There is no need to commit this. + repo_git_dir = os.path.join(tmp_path, ".git") + shutil.rmtree(repo_git_dir) + os.rename(tmp_path, self.repo_dir) + except Exception: + # Clean up after a failed extraction + if os.path.exists(tmp_path): + shutil.rmtree(tmp_path) + self.clean() + raise + + def hash(self): + cmd = ["hg", "log", "-r.", "-T{node}"] + env = os.environ.copy() + env["HGPLAIN"] = "1" + fbsource_hash = subprocess.check_output( + cmd, cwd=self.build_options.fbsource_dir, env=env + ) + return fbsource_hash[0:6] + + def get_src_dir(self): + return self.repo_dir + + def download_url_to_file_with_progress(url, file_name): print("Download %s -> %s ..." % (url, file_name)) diff --git a/build/fbcode_builder/getdeps/manifest.py b/build/fbcode_builder/getdeps/manifest.py index 737c8dbd7..2bee58b80 100644 --- a/build/fbcode_builder/getdeps/manifest.py +++ b/build/fbcode_builder/getdeps/manifest.py @@ -11,7 +11,7 @@ from __future__ import absolute_import, division, print_function, unicode_litera import io from .expr import parse_expr -from .fetcher import ArchiveFetcher, GitFetcher +from .fetcher import ArchiveFetcher, GitFetcher, ShipitTransformerFetcher try: @@ -258,6 +258,15 @@ class ManifestParser(object): return d def create_fetcher(self, build_options, ctx): + if ( + self.fbsource_path + and build_options.fbsource_dir + and self.shipit_project + and ShipitTransformerFetcher.available() + ): + # We can use the code from fbsource + return ShipitTransformerFetcher(build_options, self.shipit_project) + repo_url = self.get("git", "repo_url", ctx=ctx) if repo_url: rev = self.get("git", "rev")