From 7aadb3e7c0d50bfb7ad6bf9af6b14a31e40b4476 Mon Sep 17 00:00:00 2001
From: Ivan Morett <ivanmorett@meta.com>
Date: Mon, 24 Oct 2022 06:51:28 -0700
Subject: [PATCH] Adding "Accept" header to urllib type request (#180)

Summary:
X-link: https://github.com/facebookincubator/katran/pull/180

The urlretrieve method originally doesn't add this header, and in katran we noticed that the libmnl wasn't fetched anymore. After investigation, the request is blocked with error 403 because this header is missing. Adding it to the request.

Reviewed By: avasylev

Differential Revision: D40386108

fbshipit-source-id: a9b4b937ca12a04bc701d3f945681914151c0110
---
 build/fbcode_builder/getdeps/fetcher.py | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/build/fbcode_builder/getdeps/fetcher.py b/build/fbcode_builder/getdeps/fetcher.py
index eeed36dda..d9c17f755 100644
--- a/build/fbcode_builder/getdeps/fetcher.py
+++ b/build/fbcode_builder/getdeps/fetcher.py
@@ -18,7 +18,7 @@ import zipfile
 from datetime import datetime
 from typing import Dict, NamedTuple
 from urllib.parse import urlparse
-from urllib.request import urlretrieve
+from urllib.request import Request, urlopen
 
 from .copytree import prefetch_dir_if_eden
 from .envfuncs import Env
@@ -677,10 +677,6 @@ def download_url_to_file_with_progress(url: str, file_name) -> None:
         def progress_pycurl(self, total, amount, _uploadtotal, _uploadamount):
             self.write_update(total, amount)
 
-        def progress_urllib(self, count, block, total):
-            amount = count * block
-            self.write_update(total, amount)
-
     progress = Progress()
     start = time.time()
     try:
@@ -698,9 +694,20 @@ def download_url_to_file_with_progress(url: str, file_name) -> None:
                 c.close()
             headers = None
         else:
-            (_filename, headers) = urlretrieve(
-                url, file_name, reporthook=progress.progress_urllib
-            )
+            req_header = {"Accept": "application/*"}
+            res = urlopen(Request(url, None, req_header))
+            chunk_size = 8192  # urlretrieve uses this value
+            headers = res.headers
+            content_length = res.headers.get("Content-Length")
+            total = int(content_length.strip()) if content_length else -1
+            amount = 0
+            with open(file_name, "wb") as f:
+                chunk = res.read(chunk_size)
+                while chunk:
+                    f.write(chunk)
+                    amount += len(chunk)
+                    progress.write_update(total, amount)
+                    chunk = res.read(chunk_size)
     except (OSError, IOError) as exc:  # noqa: B014
         raise TransientFailure(
             "Failed to download %s to %s: %s" % (url, file_name, str(exc))