From 7919050c9adb00ae6ab0f5dc20b631d453edcf9a Mon Sep 17 00:00:00 2001
From: Wez Furlong <wez@fb.com>
Date: Fri, 3 May 2019 15:52:39 -0700
Subject: [PATCH] fbcode_builder: getdeps: add LFS caching for ArchiveFetcher

Summary:
When running in FB infra, prefer to download from our local LFS
server rather than going out to the internet.

Fall back to a normal internet download if the LFS get fails for some reason.

Upload to LFS after successfully verifying the hash for the downloaded archive.

Add a subcommand that performs a fetch for all possible platforms so that it
is easier to ensure that the lfs-pointers file is up to date.

Reviewed By: simpkins

Differential Revision: D14978660

fbshipit-source-id: 240fc32fc7003d1e06c88b80d85054dae36e2f31
---
 build/fbcode_builder/getdeps/fetcher.py  |  7 +++++--
 build/fbcode_builder/getdeps/manifest.py | 17 ++++++++++++++---
 2 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/build/fbcode_builder/getdeps/fetcher.py b/build/fbcode_builder/getdeps/fetcher.py
index 3cfd05f05..ea2cf2fd2 100644
--- a/build/fbcode_builder/getdeps/fetcher.py
+++ b/build/fbcode_builder/getdeps/fetcher.py
@@ -583,13 +583,16 @@ class ArchiveFetcher(Fetcher):
                 "%s: expected sha256 %s but got %s" % (self.url, self.sha256, digest)
             )
 
-    def _download(self):
+    def _download_dir(self):
+        """ returns the download dir, creating it if it doesn't already exist """
         download_dir = os.path.dirname(self.file_name)
         if not os.path.exists(download_dir):
             os.makedirs(download_dir)
+        return download_dir
 
+    def _download(self):
+        self._download_dir()
         download_url_to_file_with_progress(self.url, self.file_name)
-
         self._verify_hash()
 
     def clean(self):
diff --git a/build/fbcode_builder/getdeps/manifest.py b/build/fbcode_builder/getdeps/manifest.py
index 906afcb59..8247c0f3e 100644
--- a/build/fbcode_builder/getdeps/manifest.py
+++ b/build/fbcode_builder/getdeps/manifest.py
@@ -329,9 +329,20 @@ class ManifestParser(object):
 
         url = self.get("download", "url", ctx=ctx)
         if url:
-            return ArchiveFetcher(
-                build_options, self, url, self.get("download", "sha256", ctx=ctx)
-            )
+            # We need to defer this import until now to avoid triggering
+            # a cycle when the facebook/__init__.py is loaded.
+            try:
+                from getdeps.facebook.lfs import LFSCachingArchiveFetcher
+
+                return LFSCachingArchiveFetcher(
+                    build_options, self, url, self.get("download", "sha256", ctx=ctx)
+                )
+            except ImportError:
+                # This FB internal module isn't shippped to github,
+                # so just use its base class
+                return ArchiveFetcher(
+                    build_options, self, url, self.get("download", "sha256", ctx=ctx)
+                )
 
         raise KeyError(
             "project %s has no fetcher configuration matching %r" % (self.name, ctx)