Proxy Cache: Interface and UI for Proxy cache Configuration (PROJQUAY-3029) (#1204)

data/model/proxy_cache.py

@@ -43,3 +43,21 @@ def get_proxy_cache_config_for_org(org_name):
         )
     except ProxyCacheConfig.DoesNotExist as e:
         raise InvalidProxyCacheConfigException(str(e))
+
+
+def delete_proxy_cache_config(org_name):
+    """
+    Delete proxy cache configuration for the given organization name
+    """
+    org = get_organization(org_name)
+
+    try:
+        config = (ProxyCacheConfig.select().where(ProxyCacheConfig.organization == org.id)).get()
+    except ProxyCacheConfig.DoesNotExist:
+        return False
+
+    if config is not None:
+        ProxyCacheConfig.delete().where(ProxyCacheConfig.organization == org.id).execute()
+        return True
+
+    return False

data/model/test/test_proxy_cache_config.py

@@ -121,3 +121,26 @@ def test_get_proxy_cache_config_for_org_only_queries_db_once(initialized_db):
     # first call caches the result
     with assert_query_count(1):
         get_proxy_cache_config_for_org(org.username)
+
+
+def test_delete_proxy_cache_config(initialized_db):
+    org = create_org(
+        user_name="test",
+        user_email="test@example.com",
+        org_name="foobar",
+        org_email="foo@example.com",
+    )
+    create_proxy_cache_config(org.username, "docker.io")
+    result = delete_proxy_cache_config(org.username)
+    assert result is True
+
+
+def test_delete_for_nonexistant_config(initialized_db):
+    org = create_org(
+        user_name="test",
+        user_email="test@example.com",
+        org_name="foobar",
+        org_email="foo@example.com",
+    )
+    result = delete_proxy_cache_config(org.username)
+    assert result is False

endpoints/api/organization.py

@@ -44,10 +44,11 @@ from auth.permissions import (
 from auth.auth_context import get_authenticated_user
 from auth import scopes
 from data import model
+from data.database import ProxyCacheConfig
 from data.billing import get_plan
 from util.names import parse_robot_username
 from util.request import get_request_ip
-
+from proxy import Proxy, UpstreamRegistryError
 
 logger = logging.getLogger(__name__)
 
@@ -819,3 +820,156 @@ class OrganizationApplicationResetClientSecret(ApiResource):
 
             return app_view(application)
         raise Unauthorized()
+
+
+def proxy_cache_view(proxy_cache_config):
+    return {
+        "upstream_registry": proxy_cache_config.upstream_registry if proxy_cache_config else "",
+        "expiration_s": proxy_cache_config.expiration_s if proxy_cache_config else "",
+        "insecure": proxy_cache_config.insecure if proxy_cache_config else "",
+    }
+
+
+@resource("/v1/organization/<orgname>/proxycache")
+@path_param("orgname", "The name of the organization")
+@show_if(features.PROXY_CACHE)
+class OrganizationProxyCacheConfig(ApiResource):
+    """
+    Resource for managing Proxy Cache Config.
+    """
+
+    schemas = {
+        "NewProxyCacheConfig": {
+            "type": "object",
+            "description": "Proxy cache configuration for an organization",
+            "required": ["upstream_registry"],
+            "properties": {
+                "upstream_registry": {
+                    "type": "string",
+                    "description": "Name of the upstream registry that is to be cached",
+                },
+            },
+        },
+    }
+
+    @nickname("getProxyCacheConfig")
+    def get(self, orgname):
+        """
+        Retrieves the proxy cache configuration of the organization.
+        """
+        permission = OrganizationMemberPermission(orgname)
+        if not permission.can():
+            raise Unauthorized()
+
+        try:
+            config = model.proxy_cache.get_proxy_cache_config_for_org(orgname)
+        except model.InvalidProxyCacheConfigException:
+            return proxy_cache_view(None)
+
+        return proxy_cache_view(config)
+
+    @nickname("createProxyCacheConfig")
+    @validate_json_request("NewProxyCacheConfig")
+    def post(self, orgname):
+        """
+        Creates proxy cache configuration for the organization.
+        """
+        permission = AdministerOrganizationPermission(orgname)
+        if not permission.can():
+            raise Unauthorized()
+
+        try:
+            model.proxy_cache.get_proxy_cache_config_for_org(orgname)
+            raise request_error("Proxy Cache Configuration already exists")
+        except model.InvalidProxyCacheConfigException:
+            pass
+
+        data = request.get_json()
+        # filter None values
+        data = {k: v for k, v in data.items() if (v is not None or not "")}
+
+        try:
+            config = model.proxy_cache.create_proxy_cache_config(**data)
+            if config is not None:
+                return "Created", 201
+        except model.DataModelException as e:
+            logger.error("Error while creating Proxy cache configuration as: %s", str(e))
+
+        return request_error("Error while creating Proxy cache configuration")
+
+    @nickname("deleteProxyCacheConfig")
+    def delete(self, orgname):
+        """
+        Delete proxy cache configuration for the organization.
+        """
+        permission = AdministerOrganizationPermission(orgname)
+        if not permission.can():
+            raise Unauthorized()
+
+        try:
+            model.proxy_cache.get_proxy_cache_config_for_org(orgname)
+        except model.InvalidProxyCacheConfigException:
+            raise NotFound()
+
+        try:
+            success = model.proxy_cache.delete_proxy_cache_config(orgname)
+            if success:
+                return "Deleted", 201
+        except model.DataModelException as e:
+            logger.error("Error while deleting Proxy cache configuration as: %s", str(e))
+        raise request_error(message="Proxy Cache Configuration failed to delete")
+
+
+@resource("/v1/organization/<orgname>/validateproxycache")
+@show_if(features.PROXY_CACHE)
+class ProxyCacheConfigValidation(ApiResource):
+    """
+    Resource for validating Proxy Cache Config.
+    """
+
+    schemas = {
+        "NewProxyCacheConfig": {
+            "type": "object",
+            "description": "Proxy cache configuration for an organization",
+            "required": ["upstream_registry"],
+            "properties": {
+                "upstream_registry": {
+                    "type": "string",
+                    "description": "Name of the upstream registry that is to be cached",
+                },
+            },
+        },
+    }
+
+    @nickname("validateProxyCacheConfig")
+    @validate_json_request("NewProxyCacheConfig")
+    def post(self, orgname):
+        permission = AdministerOrganizationPermission(orgname)
+        if not permission.can():
+            raise Unauthorized()
+
+        try:
+            model.proxy_cache.get_proxy_cache_config_for_org(orgname)
+            request_error("Proxy Cache Configuration already exists")
+        except model.InvalidProxyCacheConfigException:
+            pass
+
+        data = request.get_json()
+
+        # filter None values
+        data = {k: v for k, v in data.items() if v is not None}
+
+        try:
+            config = ProxyCacheConfig(**data)
+            existing = model.organization.get_organization(orgname)
+            config.organization = existing
+
+            proxy = Proxy(config, "something-totally-fake", True)
+            response = proxy.get(f"{proxy.base_url}/v2/")
+            if response.status_code == 200:
+                return "Valid", 202
+        except UpstreamRegistryError as e:
+            raise request_error(
+                message="Failed login to remote registry. Please verify entered details and try again."
+            )
+        raise request_error(message="Failed to validate Proxy cache configuration")

endpoints/api/test/test_security.py

@@ -5567,6 +5567,110 @@ SECURITY_TESTS: List[
     (OrganizationQuota, "DELETE", {"namespace": "buynlarge"}, {}, None, 401),
     (OrganizationQuotaReport, "GET", {"namespace": "buynlarge"}, {}, None, 401),
     (SuperUserOrganizationQuotaReport, "GET", {"namespace": "buynlarge"}, {}, None, 401),
+    (
+        OrganizationProxyCacheConfig,
+        "GET",
+        {"orgname": "buynlarge"},
+        None,
+        "randomuser",
+        403,
+    ),
+    (
+        OrganizationProxyCacheConfig,
+        "GET",
+        {"orgname": "sellnsmall"},
+        None,
+        "devtable",
+        200,
+    ),
+    (
+        OrganizationProxyCacheConfig,
+        "POST",
+        {"orgname": "buynlarge"},
+        {"org_name": "buynlarge", "upstream_registry": "some-upstream-registry"},
+        None,
+        401,
+    ),
+    (
+        OrganizationProxyCacheConfig,
+        "POST",
+        {"orgname": "buynlarge"},
+        {"org_name": "buynlarge", "upstream_registry": "some-upstream-registry"},
+        "randomuser",
+        403,
+    ),
+    (
+        OrganizationProxyCacheConfig,
+        "POST",
+        {"orgname": "sellnsmall"},
+        {"org_name": "sellnsmall", "upstream_registry": None},
+        "devtable",
+        400,
+    ),
+    (
+        OrganizationProxyCacheConfig,
+        "POST",
+        {"orgname": "library"},
+        {"org_name": "library", "upstream_registry": "some-upstream-registry"},
+        "devtable",
+        201,
+    ),
+    (
+        OrganizationProxyCacheConfig,
+        "DELETE",
+        {"orgname": "buynlarge"},
+        None,
+        None,
+        401,
+    ),
+    (
+        OrganizationProxyCacheConfig,
+        "DELETE",
+        {"orgname": "buynlarge"},
+        None,
+        "randomuser",
+        403,
+    ),
+    (
+        OrganizationProxyCacheConfig,
+        "DELETE",
+        {"orgname": "proxyorg"},
+        None,
+        "devtable",
+        201,
+    ),
+    (
+        ProxyCacheConfigValidation,
+        "POST",
+        {"orgname": "buynlarge"},
+        {"org_name": "buynlarge", "upstream_registry": "some-upstream-registry"},
+        None,
+        401,
+    ),
+    (
+        ProxyCacheConfigValidation,
+        "POST",
+        {"orgname": "buynlarge"},
+        {"org_name": "buynlarge", "upstream_registry": "some-upstream-registry"},
+        "randomuser",
+        403,
+    ),
+    (
+        ProxyCacheConfigValidation,
+        "POST",
+        {"orgname": "sellnsmall"},
+        {"org_name": "sellnsmall", "upstream_registry": None},
+        "devtable",
+        400,
+    ),
+    (
+        ProxyCacheConfigValidation,
+        "POST",
+        {"orgname": "buynlarge"},
+        {"org_name": "buynlarge", "upstream_registry": "docker.io"},
+        "devtable",
+        202,
+    ),
 ]
 
 

initdb.py

@@ -62,6 +62,7 @@ from data.database import (
     QuotaLimits,
     UserOrganizationQuota,
     RepositorySize,
+    ProxyCacheConfig,
 )
 from data import model
 from data.decorators import is_deprecated_model
@@ -885,6 +886,12 @@ def populate_database(minimal=False):
 
     model.user.create_robot("coolrobot", org)
 
+    proxyorg = model.organization.create_organization(
+        "proxyorg", "quay+proxyorg@devtable.com", new_user_1
+    )
+    proxyorg.save()
+    model.proxy_cache.create_proxy_cache_config(proxyorg.username, "docker.io")
+
     oauth_app_1 = model.oauth.create_application(
         org,
         "Some Test App",

proxy/__init__.py

@@ -43,8 +43,9 @@ def parse_www_auth(value: str) -> dict[str, str]:
 
 
 class Proxy:
-    def __init__(self, config: ProxyCacheConfig, repository: str):
+    def __init__(self, config: ProxyCacheConfig, repository: str, validation: bool = False):
         self._config = config
+        self._validation = validation
 
         hostname = REGISTRY_URLS.get(
             config.upstream_registry_hostname,
@@ -57,7 +58,8 @@ class Proxy:
         self.base_url = url
         self._session = requests.Session()
         self._repo = repository
-        self._authorize(self._credentials())
+        self._authorize(self._credentials(), force_renewal=self._validation)
+        # flag used for validating Proxy cache config before saving to db
 
     def get_manifest(
         self, image_ref: str, media_types: list[str] | None = None
@@ -130,7 +132,11 @@ class Proxy:
         username = self._config.upstream_registry_username
         password = self._config.upstream_registry_password
         if username is not None and password is not None:
-            auth = (username.decrypt(), password.decrypt())
+            auth = (
+                (username, password)
+                if isinstance(username, str) and isinstance(password, str)
+                else (username.decrypt(), password.decrypt())
+            )
         return auth
 
     def _authorize(self, auth: tuple[str, str] | None = None, force_renewal: bool = False) -> None:
@@ -171,7 +177,7 @@ class Proxy:
         resp = self._session.get(auth_url, auth=basic_auth)
         if not resp.ok:
             raise UpstreamRegistryError(
-                f"Failed to get token from '{auth_url}', {resp.status_code}"
+                f"Failed to get token from: '{realm}', with status code: {resp.status_code}"
             )
 
         resp_json = resp.json()

static/css/directives/ui/proxy-cache-view.css

@@ -0,0 +1,27 @@
+.td-decoration {
+    text-transform: capitalize !important;
+    font-weight: 100 !important;
+    width: 250px !important;
+}
+
+.width-30 {
+    width: 30px;
+}
+
+.decorate-save {
+    width: 17%;
+}
+
+.decorate-form-btn {
+    display: flex;
+    width: 110%;
+}
+
+.margin-right-2 {
+    margin-right: 2%;
+}
+
+.disable-remove {
+    background-color: #e7a2a0;
+    border-color: #e7a2a0;
+}

static/directives/proxy-cache-view.html

@@ -0,0 +1,80 @@
+<div class="proxy-cache-view-element">
+    <form ng-submit="saveDetails()">
+        <table class="co-list-table">
+            <tbody>
+                <tr>
+                    <td class="td-decoration">
+                        Remote Registry:
+                    </td>
+                    <td>
+                        <input class="form-control" type="text" required ng-model="currentConfig['upstream_registry']"/>
+                        <div class="help-text">
+                            Remote registry that is to be cached. (Eg: For docker hub, docker.io, docker.io/library)
+                        </div>
+                    </td>
+                </tr>
+                <tr>
+                    <td class="td-decoration">
+                        Remote Registry username:
+                    </td>
+                    <td>
+                        <input autocomplete="new-username" class="form-control" type="text" ng-model="currentConfig['upstream_registry_username']"/>
+                        <div class="help-text">
+                            Username for authenticating into the entered remote registry
+                        </div>
+                    </td>
+                </tr>
+                <tr>
+                    <td class="td-decoration">
+                        Remote Registry password:
+                    </td>
+                    <td>
+                        <input autocomplete="new-password" class="form-control" type="password" ng-model="currentConfig['upstream_registry_password']"/>
+                        <div class="help-text">
+                            Password for authenticating into the entered remote registry
+                        </div>
+                    </td>
+                </tr>
+                <tr>
+                    <td class="td-decoration">
+                        Expiration:
+                        <div class="help-text">
+                            In seconds
+                        </div>
+                    </td>
+                    <td>
+                        <input class="form-control" type="number" min="1" max="2147483647" ng-model="currentConfig['expiration_s']"/>
+                        <div class="help-text">
+                            Default tag expiration for cached images, in seconds. This value is refreshed on every pull.
+                            Default is 86400 i.e, 24 hours.
+                        </div>
+                    </td>
+                </tr>
+                <tr>
+                    <td class="td-decoration">
+                        Insecure:
+                    </td>
+                    <td>
+                        <input class="form-control width-30" type="checkbox" ng-model="currentConfig['insecure']"/>
+                        <div class="help-text">
+                            If set, http (unsecure protocol) will be used. If not set, https (secure protocol) will be used
+                            to request the remote registry.
+                        </div>
+                    </td>
+                </tr>
+
+            </tbody>
+        </table>
+        <div class="co-alert co-alert-success" ng-show="alertSaveSuccess">
+            Successfully created proxy cache configuration
+        </div>
+        <div class="co-alert co-alert-success" ng-show="alertRemoveSuccess">
+            Successfully removed proxy cache configuration
+        </div>
+
+        <div class="decorate-form-btn">
+            <button type="submit" class="decorate-save form-control btn-success margin-right-2" ng-disabled="prevEnabled">Save</button>
+            <button class="decorate-save form-control btn-danger" ng-disabled="!prevEnabled" ng-click="deleteConfig()">Remove</button>
+        </div>
+    </form>
+</div>

static/js/directives/ui/proxy-cache-view.js

@@ -0,0 +1,94 @@
+/**
+ * An element which displays proxy cache configuration.
+ */
+angular.module('quay').directive('proxyCacheView', function () {
+    var directiveDefinitionObject = {
+        templateUrl: '/static/directives/proxy-cache-view.html',
+        restrict: 'AEC',
+        scope: {
+            'organization': '=organization'
+        },
+        controller: function ($scope, $timeout, $location, $element, ApiService) {
+            $scope.prevEnabled = false;
+            $scope.alertSaveSuccess = false;
+            $scope.alertRemoveSuccess = false;
+
+            $scope.initializeData = function () {
+                return {
+                    "org_name": $scope.organization.name,
+                    "expiration_s": 86400,
+                    "insecure": false,
+                    'upstream_registry': null,
+                    'upstream_registry_username': null,
+                    'upstream_registry_password': null,
+                };
+            }
+            $scope.currentConfig = $scope.initializeData();
+
+            var fetchProxyConfig = function () {
+                ApiService.getProxyCacheConfig(null, {'orgname': $scope.currentConfig.org_name})
+                    .then((resp) => {
+                        $scope.currentConfig['upstream_registry'] = resp["upstream_registry"];
+                        $scope.currentConfig['expiration_s'] = resp["expiration_s"] || 86400;
+                        $scope.currentConfig['insecure'] = resp["insecure"] || false;
+
+                        if ($scope.currentConfig['upstream_registry']) {
+                            $scope.prevEnabled = true;
+                        }
+                    });
+            }
+
+            var displayError = function(message = 'Could not update details') {
+                let errorDisplay = ApiService.errorDisplay(message, () => {});
+                return errorDisplay;
+            }
+
+            $scope.saveDetails = function () {
+                let params = {'orgname': $scope.currentConfig.org_name};
+
+                // validate payload
+                ApiService.validateProxyCacheConfig($scope.currentConfig, params).then(function(response) {
+                    if (response == "Valid") {
+                        // save payload
+                        ApiService.createProxyCacheConfig($scope.currentConfig, params).then((resp) => {
+                            fetchProxyConfig();
+                            alertSaveSuccessMessage();
+                        },  displayError());
+                    }
+                },  displayError("Validation Error"));
+
+            }
+
+            var alertSaveSuccessMessage = function() {
+                $timeout(function () {
+                    $scope.alertSaveSuccess = true;
+                }, 1);
+                $timeout(function () {
+                    $scope.alertSaveSuccess = false;
+                }, 5000);
+            };
+
+            var alertRemoveSuccessMessage = function() {
+                $timeout(function () {
+                    $scope.alertRemoveSuccess = true;
+                }, 1);
+                $timeout(function () {
+                    $scope.alertRemoveSuccess = false;
+                }, 5000);
+            };
+
+            $scope.deleteConfig = function () {
+                let params = {'orgname': $scope.currentConfig.org_name};
+                ApiService.deleteProxyCacheConfig(null, params).then((resp) => {
+                    $scope.prevEnabled = false;
+                    alertRemoveSuccessMessage();
+                }, displayError());
+                $scope.currentConfig = $scope.initializeData();
+            }
+
+            fetchProxyConfig();
+        }
+    }
+
+    return directiveDefinitionObject;
+});

static/partials/org-view.html

@@ -143,6 +143,18 @@
               <div quay-show="Features.QUOTA_MANAGEMENT">
                 <quota-management-view organization="organization" disabled="true"></quota-management-view>
               </div>
+
+              <div quay-show="Features.PROXY_CACHE">
+                <table class="co-list-table">
+                  <tr>
+                    <td>Proxy Cache:</td>
+                    <td>
+                      <proxy-cache-view organization="organization"></proxy-cache-view>
+                    </td>
+                  </tr>
+                </table>
+              </div>
+
             </div>
 
             <!-- Billing Information -->

test/test_api_usage.py

@@ -968,6 +968,9 @@ class TestDeleteNamespace(ApiTestCase):
             )
             self.deleteResponse(User, expected_code=400)  # Should still fail.
             self.deleteEmptyResponse(Organization, params=dict(orgname="titi"), expected_code=204)
+            self.deleteEmptyResponse(
+                Organization, params=dict(orgname="proxyorg"), expected_code=204
+            )
 
         # Add some queue items for the user.
         notification_queue.put([ADMIN_ACCESS_USER, "somerepo", "somename"], "{}")
@@ -1099,7 +1102,7 @@ class TestConductSearch(ApiTestCase):
 
         json = self.getJsonResponse(ConductSearch, params=dict(query="owners"))
 
-        self.assertEqual(4, len(json["results"]))
+        self.assertEqual(5, len(json["results"]))
         self.assertEqual(json["results"][0]["kind"], "team")
         self.assertEqual(json["results"][0]["name"], "owners")
 

test/testconfig.py

@@ -111,3 +111,4 @@ class TestConfig(DefaultConfig):
 
     FEATURE_QUOTA_MANAGEMENT = True
     DEFAULT_SYSTEM_REJECT_QUOTA_BYTES = 0
+    FEATURE_PROXY_CACHE = True