lib/proxygen
1
0
Fork 0
mirror of https://github.com/facebook/proxygen.git synced 2025-08-07 07:02:53 +03:00
Code Activity

Adding "Accept" header to urllib type request (#180)

Browse Source 
Summary:
X-link: https://github.com/facebookincubator/katran/pull/180

The urlretrieve method originally doesn't add this header, and in katran we noticed that the libmnl wasn't fetched anymore. After investigation, the request is blocked with error 403 because this header is missing. Adding it to the request.

Reviewed By: avasylev

Differential Revision: D40386108

fbshipit-source-id: a9b4b937ca12a04bc701d3f945681914151c0110
This commit is contained in:
Ivan Morett
2022-10-24 06:51:28 -07:00
committed by Facebook GitHub Bot
parent aad2a98c5e
commit 7aadb3e7c0
1 changed files with 15 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
build/fbcode_builder/getdeps/fetcher.py
View File

@@ -18,7 +18,7 @@ import zipfile
from datetime import datetime from datetime import datetime
from typing import Dict, NamedTuple from typing import Dict, NamedTuple
from urllib.parse import urlparse from urllib.parse import urlparse
from urllib.request import urlretrieve from urllib.request import Request, urlopen
from .copytree import prefetch_dir_if_eden from .copytree import prefetch_dir_if_eden
from .envfuncs import Env from .envfuncs import Env
@@ -677,10 +677,6 @@ def download_url_to_file_with_progress(url: str, file_name) -> None:
def progress_pycurl(self, total, amount, _uploadtotal, _uploadamount): def progress_pycurl(self, total, amount, _uploadtotal, _uploadamount):
self.write_update(total, amount) self.write_update(total, amount)
def progress_urllib(self, count, block, total):
amount = count * block
self.write_update(total, amount)
progress = Progress() progress = Progress()
start = time.time() start = time.time()
try: try:
@@ -698,9 +694,20 @@ def download_url_to_file_with_progress(url: str, file_name) -> None:
c.close() c.close()
headers = None headers = None
else: else:
(_filename, headers) = urlretrieve( req_header = {"Accept": "application/*"}
url, file_name, reporthook=progress.progress_urllib res = urlopen(Request(url, None, req_header))
) chunk_size = 8192 # urlretrieve uses this value
headers = res.headers
content_length = res.headers.get("Content-Length")
total = int(content_length.strip()) if content_length else -1
amount = 0
with open(file_name, "wb") as f:
chunk = res.read(chunk_size)
while chunk:
f.write(chunk)
amount += len(chunk)
progress.write_update(total, amount)
chunk = res.read(chunk_size)
except (OSError, IOError) as exc: # noqa: B014 except (OSError, IOError) as exc: # noqa: B014
raise TransientFailure( raise TransientFailure(
"Failed to download %s to %s: %s" % (url, file_name, str(exc)) "Failed to download %s to %s: %s" % (url, file_name, str(exc))