fbcode_builder: getdeps: add LFS caching for ArchiveFetcher

Summary:
When running in FB infra, prefer to download from our local LFS
server rather than going out to the internet.

Fall back to a normal internet download if the LFS get fails for some reason.

Upload to LFS after successfully verifying the hash for the downloaded archive.

Add a subcommand that performs a fetch for all possible platforms so that it
is easier to ensure that the lfs-pointers file is up to date.

Reviewed By: simpkins

Differential Revision: D14978660

fbshipit-source-id: 240fc32fc7003d1e06c88b80d85054dae36e2f31

build/fbcode_builder/getdeps/fetcher.py

@@ -583,13 +583,16 @@ class ArchiveFetcher(Fetcher):
                 "%s: expected sha256 %s but got %s" % (self.url, self.sha256, digest)
             )
 
-    def _download(self):
+    def _download_dir(self):
+        """ returns the download dir, creating it if it doesn't already exist """
         download_dir = os.path.dirname(self.file_name)
         if not os.path.exists(download_dir):
             os.makedirs(download_dir)
+        return download_dir
 
+    def _download(self):
+        self._download_dir()
         download_url_to_file_with_progress(self.url, self.file_name)
-
         self._verify_hash()
 
     def clean(self):

build/fbcode_builder/getdeps/manifest.py

@@ -329,9 +329,20 @@ class ManifestParser(object):
 
         url = self.get("download", "url", ctx=ctx)
         if url:
-            return ArchiveFetcher(
-                build_options, self, url, self.get("download", "sha256", ctx=ctx)
-            )
+            # We need to defer this import until now to avoid triggering
+            # a cycle when the facebook/__init__.py is loaded.
+            try:
+                from getdeps.facebook.lfs import LFSCachingArchiveFetcher
+
+                return LFSCachingArchiveFetcher(
+                    build_options, self, url, self.get("download", "sha256", ctx=ctx)
+                )
+            except ImportError:
+                # This FB internal module isn't shippped to github,
+                # so just use its base class
+                return ArchiveFetcher(
+                    build_options, self, url, self.get("download", "sha256", ctx=ctx)
+                )
 
         raise KeyError(
             "project %s has no fetcher configuration matching %r" % (self.name, ctx)