storage: add namespace filter to direct download responses (PROJQUAY-8147) (#3363)

* add namespace filter to direct download responses

config.py

@@ -901,3 +901,7 @@ class DefaultConfig(ImmutableConfig):
     # Disable pushes while allowing other registry operations.
     # Defaults to "False".
     DISABLE_PUSHES = False
+
+    # Specific namespaces that be exceptions to the s3-cloudflare optimization
+    # used for registry-proxy namespaces
+    CDN_SPECIFIC_NAMESPACES: Optional[List[str]] = []

endpoints/v2/blob.py

@@ -127,6 +127,7 @@ def download_blob(namespace_name, repo_name, digest, registry_model):
         username=username,
         namespace=namespace_name,
         repo_name=repo_name,
+        cdn_specific=_is_cdn_specific(namespace_name),
     )
     if direct_download_url:
         logger.debug("Returning direct download URL")
@@ -151,6 +152,13 @@ def download_blob(namespace_name, repo_name, digest, registry_model):
         )
 
 
+def _is_cdn_specific(namespace):
+    # Checks if blob belongs to namespace that should have cdn url returned
+    logger.debug("Checking for namespace %s", namespace)
+    namespaces = app.config.get("CDN_SPECIFIC_NAMESPACES")
+    return namespace in namespaces
+
+
 def _try_to_mount_blob(repository_ref, mount_blob_digest):
     """
     Attempts to mount a blob requested by the user from another repository.

storage/cloudflarestorage.py

@@ -49,11 +49,14 @@ class CloudFlareS3Storage(S3Storage):
         if self.cloudflare_privatekey is None or request_ip is None:
             return s3_presigned_url
 
-        logger.debug('Got direct download request for path "%s" with IP "%s"', path, request_ip)
-
         if is_in_network_request(self._context, request_ip, self.region):
-            logger.debug("Request is from within the network, returning S3 URL")
-            return s3_presigned_url
+            if kwargs.get("cdn_specific", False):
+                logger.debug(
+                    "Request came from within network but namespace is protected: %s", path
+                )
+            else:
+                logger.debug("Request is from within the network, returning S3 URL")
+                return s3_presigned_url
 
         s3_url_parsed = urllib.parse.urlparse(s3_presigned_url)
 

storage/test/test_cloudfront.py

@@ -7,7 +7,7 @@ from mock import patch
 from moto import mock_s3
 
 from app import config_provider
-from storage import CloudFrontedS3Storage, StorageContext
+from storage import CloudFlareS3Storage, CloudFrontedS3Storage, StorageContext
 from test.fixtures import *
 from util.ipresolver import IPResolver
 from util.ipresolver.test.test_ipresolver import (
@@ -222,3 +222,34 @@ def test_direct_download_with_repo_name(test_aws_ip, aws_ip_range_data, ipranges
     assert engine.exists(_TEST_PATH)
     url = engine.get_direct_download_url(_TEST_PATH, request_ip="1.2.3.4", repo_name=_TEST_REPO)
     assert f"repo_name={_TEST_REPO}" in url
+
+
+@mock_s3
+def test_direct_download_cdn_specific(ipranges_populated, test_ip_range_cache, app):
+    ipresolver = IPResolver(app)
+    if ipranges_populated:
+        ipresolver.sync_token = test_ip_range_cache["sync_token"]
+        ipresolver.amazon_ranges = test_ip_range_cache["all_amazon"]
+        context = StorageContext("nyc", None, config_provider, ipresolver)
+
+        # Create a test bucket and put some test content.
+        boto3.client("s3").create_bucket(Bucket=_TEST_BUCKET)
+
+        engine = CloudFlareS3Storage(
+            context,
+            "cloudflare-domain",
+            "test/data/test.pem",
+            "some/path",
+            _TEST_BUCKET,
+            _TEST_REGION,
+            None,
+        )
+
+        engine.put_content(_TEST_PATH, _TEST_CONTENT)
+        assert engine.exists(_TEST_PATH)
+        assert "amazonaws.com" in engine.get_direct_download_url(
+            _TEST_PATH, request_ip="4.0.0.2", cdn_specific=False
+        )
+        assert "cloudflare-domain" in engine.get_direct_download_url(
+            _TEST_PATH, request_ip="4.0.0.2", cdn_specific=True
+        )

test/testconfig.py

@@ -128,3 +128,5 @@ class TestConfig(DefaultConfig):
     AUTOPRUNE_FETCH_TAGS_PAGE_LIMIT = 2
     AUTOPRUNE_FETCH_REPOSITORIES_PAGE_LIMIT = 2
     FEATURE_IMAGE_EXPIRY_TRIGGER = True
+
+    CDN_SPECIFIC_NAMESPACES = ["redhat"]

util/config/schema.py

@@ -93,6 +93,7 @@ INTERNAL_ONLY_PROPERTIES = {
     "RESET_CHILD_MANIFEST_EXPIRATION",
     "PERMANENTLY_DELETE_TAGS",
     "FEATURE_RH_MARKETPLACE",
+    "CDN_SPECIFIC_NAMESPACES",
 }
 
 CONFIG_SCHEMA = {