diff --git a/data/database.py b/data/database.py index 9a78e3a88..59f38f7f6 100644 --- a/data/database.py +++ b/data/database.py @@ -748,6 +748,7 @@ class User(BaseModel): OrganizationRhSkus, NamespaceAutoPrunePolicy, AutoPruneTaskStatus, + RepositoryAutoPrunePolicy, } | appr_classes | v22_classes @@ -968,6 +969,7 @@ class Repository(BaseModel): UploadedBlob, QuotaNamespaceSize, QuotaRepositorySize, + RepositoryAutoPrunePolicy, } | appr_classes | v22_classes @@ -2021,6 +2023,13 @@ class AutoPruneTaskStatus(BaseModel): status = TextField(null=True) +class RepositoryAutoPrunePolicy(BaseModel): + uuid = CharField(default=uuid_generator, max_length=36, index=True, null=False) + repository = ForeignKeyField(Repository, index=True, null=False) + namespace = QuayUserField(index=True, null=False) + policy = JSONField(null=False, default={}) + + # Defines a map from full-length index names to the legacy names used in our code # to meet length restrictions. LEGACY_INDEX_MAP = { diff --git a/data/migrations/versions/b4da5b09c8df_repository_autoprune_policy.py b/data/migrations/versions/b4da5b09c8df_repository_autoprune_policy.py new file mode 100644 index 000000000..002e829ff --- /dev/null +++ b/data/migrations/versions/b4da5b09c8df_repository_autoprune_policy.py @@ -0,0 +1,78 @@ +"""repository autoprune policy + +Revision ID: b4da5b09c8df +Revises: 41d15c93c299 +Create Date: 2024-02-05 10:47:32.172623 + +""" + +# revision identifiers, used by Alembic. +revision = "b4da5b09c8df" +down_revision = "41d15c93c299" + +import sqlalchemy as sa + + +def upgrade(op, tables, tester): + op.create_table( + "repositoryautoprunepolicy", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("uuid", sa.String(length=36), nullable=False), + sa.Column("repository_id", sa.Integer(), nullable=False), + sa.Column("namespace_id", sa.Integer(), nullable=False), + sa.Column("policy", sa.Text(), nullable=False), + sa.ForeignKeyConstraint( + ["repository_id"], + ["repository.id"], + name=op.f("fk_repositoryautoprunepolicy_repository_id_repository"), + ), + sa.ForeignKeyConstraint( + ["namespace_id"], + ["user.id"], + name=op.f("fk_repositoryautoprunepolicy_namespace_id_user"), + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_repositoryautoprunepolicyid")), + ) + + op.create_index( + "repositoryautoprunepolicy_repository_id", + "repositoryautoprunepolicy", + ["repository_id"], + unique=True, + ) + + op.create_index( + "repositoryautoprunepolicy_namespace_id", + "repositoryautoprunepolicy", + ["namespace_id"], + unique=True, + ) + + op.create_index( + "repositoryautoprunepolicy_uuid", + "repositoryautoprunepolicy", + ["uuid"], + unique=True, + ) + + op.bulk_insert( + tables.logentrykind, + [ + {"name": "create_repository_autoprune_policy"}, + {"name": "update_repository_autoprune_policy"}, + {"name": "delete_repository_autoprune_policy"}, + ], + ) + + +def downgrade(op, tables, tester): + op.drop_table("repositoryautoprunepolicy") + + op.execute( + tables.logentrykind.delete().where( + tables.logentrykind.c.name + == op.inline_literal("create_repository_autoprune_policy") | tables.logentrykind.c.name + == op.inline_literal("update_repository_autoprune_policy") | tables.logentrykind.c.name + == op.inline_literal("delete_repository_autoprune_policy") + ) + ) diff --git a/data/model/__init__.py b/data/model/__init__.py index ae8f0b0bb..9ca62c2d6 100644 --- a/data/model/__init__.py +++ b/data/model/__init__.py @@ -173,6 +173,26 @@ class InvalidNamespaceException(DataModelException): pass +class RepositoryAutoPrunePolicyAlreadyExists(DataModelException): + pass + + +class RepositoryAutoPrunePolicyDoesNotExist(DataModelException): + pass + + +class InvalidRepositoryAutoPrunePolicy(DataModelException): + pass + + +class InvalidRepositoryAutoPruneMethod(DataModelException): + pass + + +class InvalidRepositoryException(DataModelException): + pass + + class TooManyLoginAttemptsException(Exception): def __init__(self, message, retry_after): super(TooManyLoginAttemptsException, self).__init__(message) diff --git a/data/model/autoprune.py b/data/model/autoprune.py index 0a9c55fce..161cbf9c2 100644 --- a/data/model/autoprune.py +++ b/data/model/autoprune.py @@ -2,21 +2,21 @@ import json import logging.config from enum import Enum -from data.database import AutoPruneTaskStatus, DeletedNamespace +from data.database import AutoPruneTaskStatus from data.database import NamespaceAutoPrunePolicy as NamespaceAutoPrunePolicyTable -from data.database import ( - Repository, - RepositoryState, - User, - db_for_update, - get_epoch_timestamp_ms, -) +from data.database import Repository +from data.database import RepositoryAutoPrunePolicy as RepositoryAutoPrunePolicyTable +from data.database import RepositoryState, User, db_for_update, get_epoch_timestamp_ms from data.model import ( InvalidNamespaceAutoPruneMethod, InvalidNamespaceAutoPrunePolicy, InvalidNamespaceException, + InvalidRepositoryAutoPrunePolicy, + InvalidRepositoryException, NamespaceAutoPrunePolicyAlreadyExists, NamespaceAutoPrunePolicyDoesNotExist, + RepositoryAutoPrunePolicyAlreadyExists, + RepositoryAutoPrunePolicyDoesNotExist, db_transaction, log, modelutil, @@ -53,6 +53,22 @@ class NamespaceAutoPrunePolicy: return {"uuid": self.uuid, "method": self.method, "value": self.config.get("value")} +class RepositoryAutoPrunePolicy: + def __init__(self, db_row): + config = json.loads(db_row.policy) + self._db_row = db_row + self.uuid = db_row.uuid + self.method = config.get("method") + self.config = config + self.repository_id = db_row.repository_id + + def get_row(self): + return self._db_row + + def get_view(self): + return {"uuid": self.uuid, "method": self.method, "value": self.config.get("value")} + + def valid_value(method, value): """ Method for validating the value provided for the policy method. @@ -88,6 +104,19 @@ def assert_valid_namespace_autoprune_policy(policy_config): raise InvalidNamespaceAutoPrunePolicy("Invalid value given for method type") +def assert_valid_repository_autoprune_policy(policy_config): + """ + Asserts that the policy config is valid. + """ + try: + method = AutoPruneMethod(policy_config.get("method")) + except ValueError: + raise InvalidRepositoryAutoPrunePolicy("Invalid method provided") + + if not valid_value(method, policy_config.get("value")): + raise InvalidRepositoryAutoPrunePolicy("Invalid value given for method type") + + def get_namespace_autoprune_policies_by_orgname(orgname): """ Get the autopruning policies for the specified namespace. @@ -102,6 +131,34 @@ def get_namespace_autoprune_policies_by_orgname(orgname): return [NamespaceAutoPrunePolicy(row) for row in query] +def get_repository_autoprune_policies_by_repo_name(orgname, repo_name): + """ + Get the autopruning policies for the specified repository. + """ + query = ( + RepositoryAutoPrunePolicyTable.select(RepositoryAutoPrunePolicyTable) + .join(Repository) + .join(User) + .where( + User.username == orgname, + RepositoryAutoPrunePolicyTable.repository == Repository.id, + Repository.name == repo_name, + ) + ) + return [RepositoryAutoPrunePolicy(row) for row in query] + + +def get_repository_autoprune_policies_by_repo_id(repo_id): + """ + Get the autopruning policies for the specified repository. + """ + + query = RepositoryAutoPrunePolicyTable.select().where( + RepositoryAutoPrunePolicyTable.repository == repo_id, + ) + return [RepositoryAutoPrunePolicy(row) for row in query] + + def get_namespace_autoprune_policies_by_id(namespace_id): """ Get the autopruning policies for the namespace by id. @@ -112,6 +169,16 @@ def get_namespace_autoprune_policies_by_id(namespace_id): return [NamespaceAutoPrunePolicy(row) for row in query] +def get_repository_autoprune_policies_by_namespace_id(namespace_id): + """ + Get all repository autopruning policies for a namespace by id. + """ + query = RepositoryAutoPrunePolicyTable.select().where( + RepositoryAutoPrunePolicyTable.namespace == namespace_id, + ) + return [RepositoryAutoPrunePolicy(row) for row in query] + + def get_namespace_autoprune_policy(orgname, uuid): """ Get the specific autopruning policy for the specified namespace by uuid. @@ -128,6 +195,22 @@ def get_namespace_autoprune_policy(orgname, uuid): return None +def get_repository_autoprune_policy_by_uuid(repo_name, uuid): + """ + Get the specific autopruning policy for the specified repository by uuid. + """ + try: + row = ( + RepositoryAutoPrunePolicyTable.select(RepositoryAutoPrunePolicyTable) + .join(Repository) + .where(Repository.name == repo_name, RepositoryAutoPrunePolicyTable.uuid == uuid) + .get() + ) + return RepositoryAutoPrunePolicy(row) + except RepositoryAutoPrunePolicyTable.DoesNotExist: + return None + + def create_namespace_autoprune_policy(orgname, policy_config, create_task=False): """ Creates the namespace auto-prune policy. If create_task is True, then it will also create @@ -156,6 +239,37 @@ def create_namespace_autoprune_policy(orgname, policy_config, create_task=False) return new_policy +def create_repository_autoprune_policy(orgname, repo_name, policy_config, create_task=False): + """ + Creates the repository auto-prune policy. If create_task is True, it will check if auto-prune task is not already present, + and only then it will create the auto-prune task. Deletion of the task will be handled by the autoprune worker. + """ + + with db_transaction(): + namespace = get_active_namespace_user_by_username(orgname) + namespace_id = namespace.id + + repo = repository.get_repository(orgname, repo_name) + + if repo is None: + raise InvalidRepositoryException("Repository does not exist: %s" % repo_name) + + if repository_has_autoprune_policy(repo.id): + raise RepositoryAutoPrunePolicyAlreadyExists( + "Policy for this repository already exists, delete existing to create new policy" + ) + + assert_valid_repository_autoprune_policy(policy_config) + + new_policy = RepositoryAutoPrunePolicyTable.create( + namespace=namespace_id, repository=repo.id, policy=json.dumps(policy_config) + ) + if create_task and not namespace_has_autoprune_task(namespace_id): + create_autoprune_task(namespace_id) + + return new_policy + + def update_namespace_autoprune_policy(orgname, uuid, policy_config): """ Updates the namespace auto-prune policy with the provided policy config @@ -184,9 +298,41 @@ def update_namespace_autoprune_policy(orgname, uuid, policy_config): return True +def update_repository_autoprune_policy(orgname, repo_name, uuid, policy_config): + """ + Updates the repository auto-prune policy with the provided policy config + for the specified uuid. + """ + + namespace = get_active_namespace_user_by_username(orgname) + namespace_id = namespace.id + + repo = repository.get_repository(orgname, repo_name) + if repo is None: + raise InvalidRepositoryException("Repository does not exist: %s" % repo_name) + + policy = get_repository_autoprune_policy_by_uuid(repo_name, uuid) + if policy is None: + raise RepositoryAutoPrunePolicyDoesNotExist( + f"Policy not found for repository: {repo_name} with uuid: {uuid}" + ) + + assert_valid_repository_autoprune_policy(policy_config) + + ( + RepositoryAutoPrunePolicyTable.update(policy=json.dumps(policy_config)) + .where( + RepositoryAutoPrunePolicyTable.uuid == uuid, + RepositoryAutoPrunePolicyTable.namespace == namespace_id, + ) + .execute() + ) + return True + + def delete_namespace_autoprune_policy(orgname, uuid): """ - Deletes the policy specified by the uuid + Deletes the namespace policy specified by the uuid """ with db_transaction(): @@ -217,6 +363,43 @@ def delete_namespace_autoprune_policy(orgname, uuid): return True +def delete_repository_autoprune_policy(orgname, repo_name, uuid): + """ + Deletes the repository policy specified by the uuid + """ + + with db_transaction(): + try: + namespace_id = User.select().where(User.username == orgname).get().id + except User.DoesNotExist: + raise InvalidNamespaceException("Invalid namespace provided: %s" % (orgname)) + + repo = repository.get_repository(orgname, repo_name) + if repo is None: + raise InvalidRepositoryException("Repository does not exist: %s" % repo_name) + + policy = get_repository_autoprune_policy_by_uuid(repo_name, uuid) + if policy is None: + raise RepositoryAutoPrunePolicyDoesNotExist( + f"Policy not found for repository: {repo_name} with uuid: {uuid}" + ) + + response = ( + RepositoryAutoPrunePolicyTable.delete() + .where( + RepositoryAutoPrunePolicyTable.uuid == uuid, + RepositoryAutoPrunePolicyTable.namespace == namespace_id, + ) + .execute() + ) + + if not response: + raise RepositoryAutoPrunePolicyTable.DoesNotExist( + f"Policy not found for repository: {repo_name} with uuid: {uuid}" + ) + return True + + def namespace_has_autoprune_policy(namespace_id): return ( NamespaceAutoPrunePolicyTable.select(1) @@ -225,6 +408,14 @@ def namespace_has_autoprune_policy(namespace_id): ) +def repository_has_autoprune_policy(repository_id): + return ( + RepositoryAutoPrunePolicyTable.select(1) + .where(RepositoryAutoPrunePolicyTable.repository == repository_id) + .exists() + ) + + def namespace_has_autoprune_task(namespace_id): return ( AutoPruneTaskStatus.select(1).where(AutoPruneTaskStatus.namespace == namespace_id).exists() @@ -260,6 +451,7 @@ def fetch_autoprune_task(task_run_interval_ms=60 * 60 * 1000): AutoPruneTaskStatus.select(AutoPruneTaskStatus) .where( AutoPruneTaskStatus.namespace.not_in( + # this basically skips ns if user is not enabled User.select(User.id).where( User.enabled == False, User.id == AutoPruneTaskStatus.namespace ) @@ -411,12 +603,18 @@ def execute_policy_on_repo(policy, repo_id, namespace_id, tag_page_limit=100): policy_to_func_map[policy.method](repo, policy.config, namespace, tag_page_limit) -def execute_policies_for_repo(policies, repo, namespace_id, tag_page_limit=100): +def execute_policies_for_repo(ns_policies, repo, namespace_id, tag_page_limit=100): """ - Executes the policies for the given repository. + Executes both repository and namespace level policies for the given repository. The policies + are applied in a serial fashion and are run asynchronosly in the background. """ - for policy in policies: - execute_policy_on_repo(policy, repo, namespace_id, tag_page_limit) + for ns_policy in ns_policies: + repo_policies = get_repository_autoprune_policies_by_repo_id(repo.id) + # note: currently only one policy is configured per repo + for repo_policy in repo_policies: + execute_policy_on_repo(repo_policy, repo.id, namespace_id, tag_page_limit) + # execute associated namespace policy + execute_policy_on_repo(ns_policy, repo.id, namespace_id, tag_page_limit) def get_paginated_repositories_for_namespace(namespace_id, page_token=None, page_size=50): @@ -438,12 +636,28 @@ def get_paginated_repositories_for_namespace(namespace_id, page_token=None, page ) -def execute_namespace_polices(policies, namespace_id, repository_page_limit=50, tag_page_limit=100): +def get_repository_by_policy_repo_id(policy_repo_id): + try: + return ( + Repository.select(Repository.name) + .where( + Repository.id == policy_repo_id, + Repository.state == RepositoryState.NORMAL, + ) + .get() + ) + except Repository.DoesNotExist: + return None + + +def execute_namespace_polices( + ns_policies, namespace_id, repository_page_limit=50, tag_page_limit=100 +): """ Executes the given policies for the repositories in the provided namespace. """ - if not policies: + if not ns_policies: return page_token = None @@ -453,7 +667,7 @@ def execute_namespace_polices(policies, namespace_id, repository_page_limit=50, ) for repo in repos: - execute_policies_for_repo(policies, repo, namespace_id, tag_page_limit) + execute_policies_for_repo(ns_policies, repo, namespace_id, tag_page_limit) if page_token is None: break diff --git a/data/model/gc.py b/data/model/gc.py index c810457c8..6b82e5cdf 100644 --- a/data/model/gc.py +++ b/data/model/gc.py @@ -21,6 +21,7 @@ from data.database import ( Repository, RepositoryActionCount, RepositoryAuthorizedEmail, + RepositoryAutoPrunePolicy, RepositoryBuild, RepositoryBuildTrigger, RepositoryNotification, @@ -105,6 +106,8 @@ def purge_repository(repo, force=False): ManifestSecurityStatus.select().where(ManifestSecurityStatus.repository == repo).count() == 0 ) + # Delete auto-prune policy associated with the repository + RepositoryAutoPrunePolicy.delete().where(RepositoryAutoPrunePolicy.repository == repo).execute() # Delete any repository build triggers, builds, and any other large-ish reference tables for # the repository. diff --git a/data/secscan_model/test/test_secscan_interface.py b/data/secscan_model/test/test_secscan_interface.py index 745660daa..7bf0d63ef 100644 --- a/data/secscan_model/test/test_secscan_interface.py +++ b/data/secscan_model/test/test_secscan_interface.py @@ -52,9 +52,9 @@ def test_load_security_information(indexed_v4, expected_status, initialized_db): @pytest.mark.parametrize( "next_token, expected_next_token, expected_error", [ - (None, V4ScanToken(56), None), - (V4ScanToken(None), V4ScanToken(56), AssertionError), - (V4ScanToken(1), V4ScanToken(56), None), + (None, V4ScanToken(58), None), + (V4ScanToken(None), V4ScanToken(58), AssertionError), + (V4ScanToken(1), V4ScanToken(58), None), ], ) def test_perform_indexing(next_token, expected_next_token, expected_error, initialized_db): diff --git a/data/test/test_repository_autoprune.py b/data/test/test_repository_autoprune.py new file mode 100644 index 000000000..6a59494ad --- /dev/null +++ b/data/test/test_repository_autoprune.py @@ -0,0 +1,169 @@ +import json + +import pytest + +from data.model.autoprune import * +from data.model.organization import create_organization +from data.model.repository import create_repository, set_repository_state +from data.model.user import get_user +from test.fixtures import * + +ORG1_NAME = "org1" +ORG2_NAME = "org2" +ORG3_NAME = "org3" +REPO1_NAME = "repo1" +REPO2_NAME = "repo2" +REPO3_NAME = "repo3" + + +class TestRepositoryAutoprune: + @pytest.fixture(autouse=True) + def setup(self, initialized_db): + user = get_user("devtable") + self.org1 = create_organization(ORG1_NAME, f"{ORG1_NAME}@devtable.com", user) + self.org2 = create_organization(ORG2_NAME, f"{ORG2_NAME}@devtable.com", user) + self.org3 = create_organization(ORG3_NAME, f"{ORG3_NAME}@devtable.com", user) + self.number_of_tags_policy = {"method": "number_of_tags", "value": 10} + + self.repo1 = create_repository(ORG1_NAME, REPO1_NAME, None) + set_repository_state(self.repo1, RepositoryState.NORMAL) + + self.repo2 = create_repository(ORG2_NAME, REPO2_NAME, None) + set_repository_state(self.repo2, RepositoryState.NORMAL) + + self.repo3 = create_repository(ORG3_NAME, REPO3_NAME, None) + set_repository_state(self.repo3, RepositoryState.NORMAL) + + self.repository_policy3 = create_repository_autoprune_policy( + ORG3_NAME, REPO3_NAME, self.number_of_tags_policy, create_task=False + ) + + def test_repo_policy_creation_without_task(self): + # policy based on tag count + new_repo_policy1 = create_repository_autoprune_policy( + ORG1_NAME, REPO1_NAME, self.number_of_tags_policy, create_task=False + ) + assert new_repo_policy1.namespace.id == self.org1.id + assert new_repo_policy1.repository.id == self.repo1.id + assert json.loads(new_repo_policy1.policy) == self.number_of_tags_policy + + # policy based on tag creation date + create_date_policy = {"method": "creation_date", "value": "7d"} + new_repo_policy2 = create_repository_autoprune_policy( + ORG2_NAME, REPO2_NAME, create_date_policy, create_task=False + ) + assert new_repo_policy2.namespace.id == self.org2.id + assert new_repo_policy2.repository.id == self.repo2.id + assert json.loads(new_repo_policy2.policy) == create_date_policy + + def test_repo_policy_creation_with_task(self): + new_repo_policy = create_repository_autoprune_policy( + ORG1_NAME, REPO1_NAME, self.number_of_tags_policy, create_task=True + ) + assert new_repo_policy.namespace.id == self.org1.id + assert new_repo_policy.repository.id == self.repo1.id + assert namespace_has_autoprune_task(self.org1.id) is True + + def test_repo_policy_creation_with_incorrect_repo_name(self): + with pytest.raises(InvalidRepositoryException) as excerror: + create_repository_autoprune_policy( + ORG1_NAME, "nonexistentrepo", self.number_of_tags_policy, create_task=True + ) + assert str(excerror.value) == "Repository does not exist: nonexistentrepo" + + def test_repo_policy_creation_for_repo_with_policy(self): + with pytest.raises(RepositoryAutoPrunePolicyAlreadyExists) as excerror: + create_repository_autoprune_policy( + ORG3_NAME, REPO3_NAME, self.number_of_tags_policy, create_task=True + ) + assert ( + str(excerror.value) + == "Policy for this repository already exists, delete existing to create new policy" + ) + + def test_get_repo_policies_by_reponame(self): + repo_policies = get_repository_autoprune_policies_by_repo_name(ORG3_NAME, REPO3_NAME) + assert len(repo_policies) == 1 + assert repo_policies[0]._db_row.namespace_id == self.org3.id + assert repo_policies[0].repository_id == self.repo3.id + + repo2_policies = get_repository_autoprune_policies_by_repo_name(ORG2_NAME, REPO2_NAME) + assert len(repo2_policies) == 0 + + def test_get_repo_policies_by_namespace_id(self): + repo_policies = get_repository_autoprune_policies_by_namespace_id(self.org3.id) + assert len(repo_policies) == 1 + assert repo_policies[0]._db_row.namespace_id == self.org3.id + assert repo_policies[0].repository_id == self.repo3.id + + repo2_policies = get_repository_autoprune_policies_by_namespace_id(self.org2.id) + assert len(repo2_policies) == 0 + + def test_get_repo_policies_by_repo_id(self): + repo_policies = get_repository_autoprune_policies_by_repo_id(self.repo3.id) + assert len(repo_policies) == 1 + assert repo_policies[0]._db_row.namespace_id == self.org3.id + assert repo_policies[0].repository_id == self.repo3.id + + repo2_policies = get_repository_autoprune_policies_by_repo_id(self.repo2.id) + assert len(repo2_policies) == 0 + + def test_update_repo_policy(self): + new_policy_config = {"method": "number_of_tags", "value": 100} + updated = update_repository_autoprune_policy( + ORG3_NAME, REPO3_NAME, self.repository_policy3.uuid, new_policy_config + ) + assert updated is True + + repo_policies = get_repository_autoprune_policies_by_repo_name(ORG3_NAME, REPO3_NAME) + assert repo_policies[0].config == new_policy_config + + def test_incorrect_update_repo_policy(self): + # incorrect uuid + with pytest.raises(RepositoryAutoPrunePolicyDoesNotExist) as excerror: + update_repository_autoprune_policy(ORG3_NAME, REPO3_NAME, "random-uuid", {}) + assert ( + str(excerror.value) + == f"Policy not found for repository: {REPO3_NAME} with uuid: random-uuid" + ) + + # incorrect reponame + with pytest.raises(InvalidRepositoryException) as excerror: + update_repository_autoprune_policy( + ORG3_NAME, "nonexistentrepo", self.repository_policy3.uuid, {} + ) + assert str(excerror.value) == "Repository does not exist: nonexistentrepo" + + def test_delete_repo_policy(self): + deleted = delete_repository_autoprune_policy( + ORG3_NAME, REPO3_NAME, self.repository_policy3.uuid + ) + assert deleted is True + + def test_incorrect_delete_repo_policy(self): + # incorrect uuid + with pytest.raises(RepositoryAutoPrunePolicyDoesNotExist) as excerror: + delete_repository_autoprune_policy(ORG3_NAME, REPO3_NAME, "random-uuid") + assert ( + str(excerror.value) + == f"Policy not found for repository: {REPO3_NAME} with uuid: random-uuid" + ) + + # incorrect reponame + with pytest.raises(InvalidRepositoryException) as excerror: + delete_repository_autoprune_policy( + ORG3_NAME, "nonexistentrepo", self.repository_policy3.uuid + ) + assert str(excerror.value) == "Repository does not exist: nonexistentrepo" + + def test_repository_policy(self): + policy_exists = repository_has_autoprune_policy(self.repo3.id) + assert policy_exists is True + repository_policy = get_repository_autoprune_policy_by_uuid( + self.repo3.name, self.repository_policy3.uuid + ) + assert repository_policy.uuid == self.repository_policy3.uuid + assert repository_policy.repository_id == self.repo3.id + + resp = get_repository_autoprune_policy_by_uuid("nonexistentrepo", "randome-uuid") + assert resp is None diff --git a/endpoints/api/policy.py b/endpoints/api/policy.py index cbfecf0b8..784302a57 100644 --- a/endpoints/api/policy.py +++ b/endpoints/api/policy.py @@ -5,15 +5,21 @@ from flask import request import features from auth import scopes from auth.auth_context import get_authenticated_user -from auth.permissions import AdministerOrganizationPermission +from auth.permissions import ( + AdministerOrganizationPermission, + AdministerRepositoryPermission, +) from data import model +from data.registry_model import registry_model from endpoints.api import ( ApiResource, + RepositoryParamResource, allow_if_superuser, log_action, nickname, path_param, request_error, + require_repo_admin, require_scope, require_user_admin, resource, @@ -231,6 +237,233 @@ class OrgAutoPrunePolicy(ApiResource): return {"uuid": policy_uuid}, 200 +@resource("/v1/repository//autoprunepolicy/") +@path_param("repository", "The full path of the repository. e.g. namespace/name") +@show_if(features.AUTO_PRUNE) +class RepositoryAutoPrunePolicies(RepositoryParamResource): + """ + Resource for listing and creating repository auto-prune policies + """ + + schemas = { + "AutoPrunePolicyConfig": { + "type": "object", + "description": "The policy configuration that is to be applied to the repository", + "required": ["method", "value"], + "properties": { + "method": { + "type": "string", + "description": "The method to use for pruning tags (number_of_tags, creation_date)", + }, + "value": { + "type": ["integer", "string"], + "description": "The value to use for the pruning method (number of tags e.g. 10, time delta e.g. 7d (7 days))", + }, + }, + }, + } + + @require_repo_admin(allow_for_superuser=True) + @nickname("listRepositoryAutoPrunePolicies") + def get(self, namespace, repository): + """ + Lists the auto-prune policies for the repository + """ + permission = AdministerRepositoryPermission(namespace, repository) + if not permission.can(): + raise Unauthorized() + + if registry_model.lookup_repository(namespace, repository) is None: + raise NotFound() + + policies = model.autoprune.get_repository_autoprune_policies_by_repo_name( + namespace, repository + ) + + return {"policies": [policy.get_view() for policy in policies]} + + @require_repo_admin(allow_for_superuser=True) + @validate_json_request("AutoPrunePolicyConfig") + @nickname("createRepositoryAutoPrunePolicy") + def post(self, namespace, repository): + """ + Creates an auto-prune policy for the repository + """ + permission = AdministerRepositoryPermission(namespace, repository) + if not permission.can(): + raise Unauthorized() + + if registry_model.lookup_repository(namespace, repository) is None: + raise NotFound() + + app_data = request.get_json() + method = app_data.get("method", None) + value = app_data.get("value", None) + + if method is None or value is None: + request_error(message="Missing the following parameters: method, value") + + policy_config = { + "method": method, + "value": value, + } + + try: + policy = model.autoprune.create_repository_autoprune_policy( + namespace, repository, policy_config, create_task=True + ) + except model.InvalidNamespaceException: + raise NotFound() + except model.InvalidRepositoryException: + raise NotFound() + except model.InvalidRepositoryAutoPrunePolicy as ex: + request_error(ex) + except model.RepositoryAutoPrunePolicyAlreadyExists as ex: + request_error(ex) + + log_action( + "create_repository_autoprune_policy", + namespace, + { + "method": policy_config["method"], + "value": policy_config["value"], + "namespace": namespace, + "repo": repository, + }, + repo_name=repository, + ) + + return {"uuid": policy.uuid}, 201 + + +@resource("/v1/repository//autoprunepolicy/") +@path_param("repository", "The full path of the repository. e.g. namespace/name") +@path_param("policy_uuid", "The unique ID of the policy") +@show_if(features.AUTO_PRUNE) +class RepositoryAutoPrunePolicy(RepositoryParamResource): + """ + Resource for fetching, updating, and deleting repository specific auto-prune policies + """ + + schemas = { + "AutoPrunePolicyConfig": { + "type": "object", + "description": "The policy configuration that is to be applied to the repository", + "required": ["method", "value"], + "properties": { + "method": { + "type": "string", + "description": "The method to use for pruning tags (number_of_tags, creation_date)", + }, + "value": { + "type": ["integer", "string"], + "description": "The value to use for the pruning method (number of tags e.g. 10, time delta e.g. 7d (7 days))", + }, + }, + }, + } + + @require_repo_admin(allow_for_superuser=True) + @nickname("getRepositoryAutoPrunePolicy") + def get(self, namespace, repository, policy_uuid): + """ + Fetches the auto-prune policy for the repository + """ + permission = AdministerRepositoryPermission(namespace, repository) + if not permission.can(): + raise Unauthorized() + + policy = model.autoprune.get_repository_autoprune_policy_by_uuid(repository, policy_uuid) + if policy is None: + raise NotFound() + + return policy.get_view() + + @require_repo_admin(allow_for_superuser=True) + @validate_json_request("AutoPrunePolicyConfig") + @nickname("updateRepositoryAutoPrunePolicy") + def put(self, namespace, repository, policy_uuid): + """ + Updates the auto-prune policy for the repository + """ + permission = AdministerRepositoryPermission(namespace, repository) + if not permission.can(): + raise Unauthorized() + + app_data = request.get_json() + method = app_data.get("method", None) + value = app_data.get("value", None) + + if method is None or value is None: + request_error(message="Missing the following parameters: method, value") + + policy_config = { + "method": method, + "value": value, + } + + try: + updated = model.autoprune.update_repository_autoprune_policy( + namespace, repository, policy_uuid, policy_config + ) + if not updated: + request_error(message="could not update policy") + except model.InvalidNamespaceException: + raise NotFound() + except model.InvalidRepositoryException: + raise NotFound() + except model.InvalidRepositoryAutoPrunePolicy as ex: + request_error(ex) + except model.RepositoryAutoPrunePolicyDoesNotExist as ex: + raise NotFound() + + log_action( + "update_repository_autoprune_policy", + namespace, + { + "method": policy_config["method"], + "value": policy_config["value"], + "namespace": namespace, + "repo": repository, + }, + repo_name=repository, + ) + + return {"uuid": policy_uuid}, 204 + + @require_repo_admin(allow_for_superuser=True) + @nickname("deleteRepositoryAutoPrunePolicy") + def delete(self, namespace, repository, policy_uuid): + """ + Deletes the auto-prune policy for the repository + """ + permission = AdministerRepositoryPermission(namespace, repository) + if not permission.can(): + raise Unauthorized() + + try: + updated = model.autoprune.delete_repository_autoprune_policy( + namespace, repository, policy_uuid + ) + if not updated: + raise InvalidRequest("could not delete policy") + except model.InvalidNamespaceException: + raise NotFound() + except model.InvalidRepositoryException as ex: + raise NotFound() + except model.RepositoryAutoPrunePolicyDoesNotExist as ex: + raise NotFound() + + log_action( + "delete_repository_autoprune_policy", + namespace, + {"policy_uuid": policy_uuid, "namespace": namespace, "repo": repository}, + repo_name=repository, + ) + + return {"uuid": policy_uuid}, 200 + + @resource("/v1/user/autoprunepolicy/") @show_if(features.AUTO_PRUNE) class UserAutoPrunePolicies(ApiResource): diff --git a/endpoints/api/test/test_policy.py b/endpoints/api/test/test_policy.py index 6ac37fa02..3c298ce8f 100644 --- a/endpoints/api/test/test_policy.py +++ b/endpoints/api/test/test_policy.py @@ -7,6 +7,8 @@ from data.model.log import get_latest_logs_query, get_log_entry_kinds from endpoints.api.policy import ( OrgAutoPrunePolicies, OrgAutoPrunePolicy, + RepositoryAutoPrunePolicies, + RepositoryAutoPrunePolicy, UserAutoPrunePolicies, UserAutoPrunePolicy, ) @@ -345,3 +347,187 @@ def test_delete_user_policy_nonexistent_policy(initialized_db, app): {"orgname": "devtable", "policy_uuid": "doesnotexist"}, expected_code=404, ) + + +def test_get_repo_policies(initialized_db, app): + with client_with_identity("devtable", app) as cl: + params = {"repository": "devtable/simple"} + response = conduct_api_call(cl, RepositoryAutoPrunePolicies, "GET", params).json + assert len(response["policies"]) == 1 + assert response["policies"][0]["method"] == "number_of_tags" + assert response["policies"][0]["value"] == 10 + + +def test_create_repo_policy(initialized_db, app): + with client_with_identity("devtable", app) as cl: + params = {"repository": "testorgforautoprune/autoprunerepo"} + response = conduct_api_call( + cl, + RepositoryAutoPrunePolicies, + "POST", + params, + {"method": "creation_date", "value": "2w"}, + 201, + ).json + assert response["uuid"] is not None + assert ( + model.autoprune.get_repository_autoprune_policy_by_uuid( + "autoprunerepo", response["uuid"] + ) + is not None + ) + org = model.organization.get_organization("testorgforautoprune") + assert model.autoprune.namespace_has_autoprune_task(org.id) + + # Check audit log was created + logs = list(get_latest_logs_query(namespace="testorgforautoprune")) + log_kinds = get_log_entry_kinds() + log = None + for l in logs: + if l.kind == log_kinds["create_repository_autoprune_policy"]: + log = l + break + assert log is not None + assert json.loads(log.metadata_json)["method"] == "creation_date" + assert json.loads(log.metadata_json)["value"] == "2w" + assert json.loads(log.metadata_json)["namespace"] == "testorgforautoprune" + + +def test_create_repo_policy_already_existing(initialized_db, app): + with client_with_identity("devtable", app) as cl: + params = {"repository": "devtable/simple"} + response = conduct_api_call( + cl, + RepositoryAutoPrunePolicies, + "POST", + params, + {"method": "creation_date", "value": "2w"}, + expected_code=400, + ).json + assert ( + response["error_message"] + == "Policy for this repository already exists, delete existing to create new policy" + ) + + +def test_create_repo_policy_nonexistent_method(initialized_db, app): + with client_with_identity("devtable", app) as cl: + params = {"repository": "testorgforautoprune/autoprunerepo"} + response = conduct_api_call( + cl, + RepositoryAutoPrunePolicies, + "POST", + params, + {"method": "doesnotexist", "value": "2w"}, + expected_code=400, + ).json + assert response["error_message"] == "Invalid method provided" + + +def test_get_repo_policy(initialized_db, app): + policies = model.autoprune.get_repository_autoprune_policies_by_repo_name("devtable", "simple") + assert len(policies) == 1 + policy_uuid = policies[0].uuid + with client_with_identity("devtable", app) as cl: + params = {"repository": "devtable/simple", "policy_uuid": policy_uuid} + response = conduct_api_call(cl, RepositoryAutoPrunePolicy, "GET", params).json + assert response["method"] == "number_of_tags" + assert response["value"] == 10 + + +def test_update_repo_policy(initialized_db, app): + policies = model.autoprune.get_repository_autoprune_policies_by_repo_name("devtable", "simple") + assert len(policies) == 1 + policy_uuid = policies[0].uuid + with client_with_identity("devtable", app) as cl: + params_for_update = {"repository": "devtable/simple", "policy_uuid": policy_uuid} + conduct_api_call( + cl, + RepositoryAutoPrunePolicy, + "PUT", + params_for_update, + {"method": "creation_date", "value": "2w"}, + expected_code=204, + ) + + # Make another request asserting it was updated + params = {"repository": "devtable/simple", "policy_uuid": policy_uuid} + get_response = conduct_api_call(cl, RepositoryAutoPrunePolicy, "GET", params).json + assert get_response["method"] == "creation_date" + assert get_response["value"] == "2w" + + # Check audit log was created + logs = list(get_latest_logs_query(namespace="devtable")) + log_kinds = get_log_entry_kinds() + log = None + for l in logs: + if l.kind == log_kinds["update_repository_autoprune_policy"]: + log = l + break + assert log is not None + assert json.loads(log.metadata_json)["method"] == "creation_date" + assert json.loads(log.metadata_json)["value"] == "2w" + assert json.loads(log.metadata_json)["namespace"] == "devtable" + + +def test_update_repo_policy_nonexistent_policy(initialized_db, app): + with client_with_identity("devtable", app) as cl: + params_for_update = {"repository": "devtable/simple", "policy_uuid": "doesnotexist"} + conduct_api_call( + cl, + RepositoryAutoPrunePolicy, + "PUT", + params_for_update, + {"method": "creation_date", "value": "2w"}, + expected_code=404, + ) + + +def test_delete_repo_policy(initialized_db, app): + policies = model.autoprune.get_repository_autoprune_policies_by_repo_name("devtable", "simple") + assert len(policies) == 1 + policy_uuid = policies[0].uuid + with client_with_identity("devtable", app) as cl: + params_for_delete = {"repository": "devtable/simple", "policy_uuid": policy_uuid} + conduct_api_call( + cl, + RepositoryAutoPrunePolicy, + "DELETE", + params_for_delete, + expected_code=200, + ) + params = {"repository": "devtable/simple", "policy_uuid": policy_uuid} + conduct_api_call( + cl, + RepositoryAutoPrunePolicy, + "GET", + params, + expected_code=404, + ) + + # Check audit log was created + logs = list(get_latest_logs_query(namespace="devtable")) + log_kinds = get_log_entry_kinds() + log = None + for l in logs: + if l.kind == log_kinds["delete_repository_autoprune_policy"]: + log = l + break + assert log is not None + assert json.loads(log.metadata_json)["policy_uuid"] == policy_uuid + assert json.loads(log.metadata_json)["namespace"] == "devtable" + + +def test_delete_repo_policy_nonexistent_policy(initialized_db, app): + with client_with_identity("devtable", app) as cl: + params_for_delete = { + "repository": "testorgforautoprune/autoprunerepo", + "policy_uuid": "doesnotexist", + } + conduct_api_call( + cl, + RepositoryAutoPrunePolicy, + "DELETE", + params_for_delete, + expected_code=404, + ) diff --git a/endpoints/api/test/test_security.py b/endpoints/api/test/test_security.py index ecbb43eef..a476f3a43 100644 --- a/endpoints/api/test/test_security.py +++ b/endpoints/api/test/test_security.py @@ -6270,6 +6270,182 @@ SECURITY_TESTS: List[ (UserAutoPrunePolicy, "DELETE", {"policy_uuid": "some_uuid"}, None, "devtable", 404), (UserAutoPrunePolicy, "DELETE", {"policy_uuid": "some_uuid"}, None, "freshuser", 404), (UserAutoPrunePolicy, "DELETE", {"policy_uuid": "some_uuid"}, None, "reader", 404), + ( + RepositoryAutoPrunePolicies, + "GET", + {"repository": "testorgforautoprune/autoprunerepo"}, + None, + None, + 401, + ), + ( + RepositoryAutoPrunePolicies, + "GET", + {"repository": "testorgforautoprune/autoprunerepo"}, + None, + "devtable", + 200, + ), + ( + RepositoryAutoPrunePolicies, + "GET", + {"repository": "testorgforautoprune/unknown"}, + None, + "devtable", + 404, + ), + ( + RepositoryAutoPrunePolicies, + "GET", + {"repository": "testorgforautoprune/autoprunerepo"}, + None, + "freshuser", + 403, + ), + ( + RepositoryAutoPrunePolicies, + "GET", + {"repository": "testorgforautoprune/autoprunerepo"}, + None, + "reader", + 403, + ), + ( + RepositoryAutoPrunePolicies, + "POST", + {"repository": "testorgforautoprune/autoprunerepo"}, + {"method": "number_of_tags", "value": 10}, + None, + 401, + ), + ( + RepositoryAutoPrunePolicies, + "POST", + {"repository": "testorgforautoprune/autoprunerepo"}, + {"method": "creation_date", "value": "2w"}, + "devtable", + 201, + ), + ( + RepositoryAutoPrunePolicies, + "POST", + {"repository": "testorgforautoprune/unknown"}, + {"method": "number_of_tags", "value": 10}, + "devtable", + 404, + ), + ( + RepositoryAutoPrunePolicies, + "POST", + {"repository": "testorgforautoprune/autoprunerepo"}, + {"method": "number_of_tags", "value": 10}, + "freshuser", + 403, + ), + ( + RepositoryAutoPrunePolicies, + "POST", + {"repository": "testorgforautoprune/autoprunerepo"}, + {"method": "number_of_tags", "value": 10}, + "reader", + 403, + ), + ( + RepositoryAutoPrunePolicy, + "GET", + {"repository": "testorgforautoprune/autoprunerepo", "policy_uuid": "some_uuid"}, + None, + None, + 401, + ), + ( + RepositoryAutoPrunePolicy, + "GET", + {"repository": "testorgforautoprune/unknown", "policy_uuid": "some_uuid"}, + None, + "devtable", + 404, + ), + ( + RepositoryAutoPrunePolicy, + "GET", + {"repository": "testorgforautoprune/autoprunerepo", "policy_uuid": "some_uuid"}, + None, + "freshuser", + 403, + ), + ( + RepositoryAutoPrunePolicy, + "GET", + {"repository": "testorgforautoprune/autoprunerepo", "policy_uuid": "some_uuid"}, + None, + "reader", + 403, + ), + ( + RepositoryAutoPrunePolicy, + "PUT", + {"repository": "testorgforautoprune/autoprunerepo", "policy_uuid": "some_uuid"}, + {"method": "number_of_tags", "value": 10}, + None, + 401, + ), + ( + RepositoryAutoPrunePolicy, + "PUT", + {"repository": "testorgforautoprune/unknown", "policy_uuid": "some_uuid"}, + {"method": "number_of_tags", "value": 10}, + "devtable", + 404, + ), + ( + RepositoryAutoPrunePolicy, + "PUT", + {"repository": "testorgforautoprune/autoprunerepo", "policy_uuid": "some_uuid"}, + {"method": "number_of_tags", "value": 10}, + "freshuser", + 403, + ), + ( + RepositoryAutoPrunePolicy, + "PUT", + {"repository": "testorgforautoprune/autoprunerepo", "policy_uuid": "some_uuid"}, + {"method": "number_of_tags", "value": 10}, + "reader", + 403, + ), + ( + RepositoryAutoPrunePolicy, + "DELETE", + {"repository": "testorgforautoprune/autoprunerepo", "policy_uuid": "some_uuid"}, + None, + None, + 401, + ), + ( + RepositoryAutoPrunePolicy, + "DELETE", + {"repository": "testorgforautoprune/unknown", "policy_uuid": "some_uuid"}, + None, + "devtable", + 404, + ), + ( + RepositoryAutoPrunePolicy, + "DELETE", + {"repository": "testorgforautoprune/autoprunerepo", "policy_uuid": "some_uuid"}, + None, + "freshuser", + 403, + ), + ( + RepositoryAutoPrunePolicy, + "DELETE", + {"repository": "testorgforautoprune/autoprunerepo", "policy_uuid": "some_uuid"}, + None, + "reader", + 403, + ), ] diff --git a/endpoints/api/test/test_superuser.py b/endpoints/api/test/test_superuser.py index b4d1d0d2e..21ea8d195 100644 --- a/endpoints/api/test/test_superuser.py +++ b/endpoints/api/test/test_superuser.py @@ -31,14 +31,14 @@ def test_list_all_users(disabled, app): def test_list_all_orgs(app): with client_with_identity("devtable", app) as cl: result = conduct_api_call(cl, SuperUserOrganizationList, "GET", None, None, 200).json - assert len(result["organizations"]) == 6 + assert len(result["organizations"]) == 7 def test_paginate_orgs(app): with client_with_identity("devtable", app) as cl: - params = {"limit": 3} + params = {"limit": 4} firstResult = conduct_api_call(cl, SuperUserOrganizationList, "GET", params, None, 200).json - assert len(firstResult["organizations"]) == 3 + assert len(firstResult["organizations"]) == 4 assert firstResult["next_page"] is not None params["next_page"] = firstResult["next_page"] secondResult = conduct_api_call( @@ -50,13 +50,13 @@ def test_paginate_orgs(app): def test_paginate_test_list_all_users(app): with client_with_identity("devtable", app) as cl: - params = {"limit": 6} + params = {"limit": 7} firstResult = conduct_api_call(cl, SuperUserList, "GET", params, None, 200).json - assert len(firstResult["users"]) == 6 + assert len(firstResult["users"]) == 7 assert firstResult["next_page"] is not None params["next_page"] = firstResult["next_page"] secondResult = conduct_api_call(cl, SuperUserList, "GET", params, None, 200).json - assert len(secondResult["users"]) == 5 + assert len(secondResult["users"]) == 4 assert secondResult.get("next_page", None) is None diff --git a/initdb.py b/initdb.py index f9138a7f5..f11eb75b1 100644 --- a/initdb.py +++ b/initdb.py @@ -70,7 +70,10 @@ from data.decorators import is_deprecated_model from data.encryption import FieldEncrypter from data.fields import Credential from data.logs_model import logs_model -from data.model.autoprune import create_namespace_autoprune_policy +from data.model.autoprune import ( + create_namespace_autoprune_policy, + create_repository_autoprune_policy, +) from data.queue import WorkQueue from data.registry_model import registry_model from data.registry_model.datatypes import RepositoryReference @@ -464,6 +467,10 @@ def initialize_database(): LogEntryKind.create(name="update_namespace_autoprune_policy") LogEntryKind.create(name="delete_namespace_autoprune_policy") + LogEntryKind.create(name="create_repository_autoprune_policy") + LogEntryKind.create(name="update_repository_autoprune_policy") + LogEntryKind.create(name="delete_repository_autoprune_policy") + ImageStorageLocation.create(name="local_eu") ImageStorageLocation.create(name="local_us") @@ -922,6 +929,25 @@ def populate_database(minimal=False): "buynlarge", {"method": "creation_date", "value": "5d"}, create_task=True ) + org_for_autoprune = model.organization.create_organization( + "testorgforautoprune", "autoprune@devtable.com", new_user_1 + ) + org_repo = __generate_repository( + org_for_autoprune, + "autoprunerepo", + "Repository owned by an org.", + False, + [], + (4, [], ["latest", "prod"]), + ) + + create_repository_autoprune_policy( + "devtable", simple_repo.name, {"method": "number_of_tags", "value": 10}, create_task=True + ) + create_repository_autoprune_policy( + "public", publicrepo.name, {"method": "creation_date", "value": "5d"}, create_task=True + ) + liborg = model.organization.create_organization( "library", "quay+library@devtable.com", new_user_1 ) diff --git a/test/test_api_usage.py b/test/test_api_usage.py index 0af34b400..345270ce9 100644 --- a/test/test_api_usage.py +++ b/test/test_api_usage.py @@ -970,11 +970,11 @@ class TestDeleteNamespace(ApiTestCase): def test_deletenamespaces(self): self.login(ADMIN_ACCESS_USER) - # Try to first delete the user. Since they are the sole admin of three orgs, it should fail. + # Try to first delete the user. Since they are the sole admin of five orgs, it should fail. with check_transitive_modifications(): self.deleteResponse(User, expected_code=400) - # Delete the three orgs, checking in between. + # Delete the five orgs, checking in between. with check_transitive_modifications(): self.deleteEmptyResponse( Organization, params=dict(orgname=ORGANIZATION), expected_code=204 @@ -985,9 +985,14 @@ class TestDeleteNamespace(ApiTestCase): ) self.deleteResponse(User, expected_code=400) # Should still fail. self.deleteEmptyResponse(Organization, params=dict(orgname="titi"), expected_code=204) + self.deleteResponse(User, expected_code=400) # Should still fail. self.deleteEmptyResponse( Organization, params=dict(orgname="proxyorg"), expected_code=204 ) + self.deleteResponse(User, expected_code=400) # Should still fail. + self.deleteEmptyResponse( + Organization, params=dict(orgname="testorgforautoprune"), expected_code=204 + ) # Add some queue items for the user. notification_queue.put([ADMIN_ACCESS_USER, "somerepo", "somename"], "{}") diff --git a/web/cypress/e2e/repository-autopruning.cy.ts b/web/cypress/e2e/repository-autopruning.cy.ts new file mode 100644 index 000000000..9b845e372 --- /dev/null +++ b/web/cypress/e2e/repository-autopruning.cy.ts @@ -0,0 +1,157 @@ +/// + +describe('Repository settings - Repository autoprune policies', () => { + beforeEach(() => { + cy.exec('npm run quay:seed'); + cy.request('GET', `${Cypress.env('REACT_QUAY_APP_API_URL')}/csrf_token`) + .then((response) => response.body.csrf_token) + .then((token) => { + cy.loginByCSRF(token); + }); + cy.intercept('GET', '/config', {fixture: 'config.json'}).as('getConfig'); + }); + + const attemptCreateTagNumberRepoPolicy = (cy) => { + cy.get('[data-testid="repository-auto-prune-method"]').select( + 'By number of tags', + ); + cy.get('input[aria-label="number of tags"]').should('have.value', '20'); + cy.get('input[aria-label="number of tags"]').type('{end}{backspace}5'); + cy.contains('Save').click(); + }; + + const attemptCreateCreationDateRepoPolicy = (cy) => { + cy.get('[data-testid="repository-auto-prune-method"]').select( + 'By age of tags', + ); + cy.get('input[aria-label="tag creation date value"]').should( + 'have.value', + '7', + ); + cy.get('select[aria-label="tag creation date unit"]').contains('days'); + cy.get('input[aria-label="tag creation date value"]').type( + '2{leftArrow}{backspace}', + ); + cy.get('select[aria-label="tag creation date unit"]').select('weeks'); + cy.contains('Save').click(); + }; + + it('creates repo policy based on number of tags', () => { + cy.visit('/repository/projectquay/repo1?tab=settings'); + cy.contains('Repository Auto-Prune Policies').click(); + cy.get('[data-testid="repository-auto-prune-method"]').contains('None'); + + // Create policy + attemptCreateTagNumberRepoPolicy(cy); + cy.contains('Successfully created repository auto-prune policy'); + cy.get('input[aria-label="number of tags"]').should('have.value', '25'); + }); + + it('creates repo policy based on creation date', () => { + cy.visit('/repository/projectquay/repo1?tab=settings'); + cy.contains('Repository Auto-Prune Policies').click(); + cy.get('[data-testid="repository-auto-prune-method"]').contains('None'); + + // Create policy + attemptCreateCreationDateRepoPolicy(cy); + cy.contains('Successfully created repository auto-prune policy'); + cy.get('input[aria-label="tag creation date value"]').should( + 'have.value', + '2', + ); + cy.get('select[aria-label="tag creation date unit"]').contains('weeks'); + }); + + it('updates repo policy', () => { + cy.visit('/repository/projectquay/repo1?tab=settings'); + cy.contains('Repository Auto-Prune Policies').click(); + cy.get('[data-testid="repository-auto-prune-method"]').contains('None'); + + // Create initial policy + attemptCreateTagNumberRepoPolicy(cy); + cy.contains('Successfully created repository auto-prune policy'); + cy.get('input[aria-label="number of tags"]').should('have.value', '25'); + + // Update policy + attemptCreateCreationDateRepoPolicy(cy); + cy.contains('Successfully updated repository auto-prune policy'); + cy.get('input[aria-label="tag creation date value"]').should( + 'have.value', + '2', + ); + cy.get('select[aria-label="tag creation date unit"]').contains('weeks'); + }); + + it('deletes repo policy', () => { + cy.visit('/repository/projectquay/repo1?tab=settings'); + cy.contains('Repository Auto-Prune Policies').click(); + cy.get('[data-testid="repository-auto-prune-method"]').contains('None'); + + // Create initial policy + attemptCreateTagNumberRepoPolicy(cy); + cy.contains('Successfully created repository auto-prune policy'); + cy.get('input[aria-label="number of tags"]').should('have.value', '25'); + + // Delete policy + cy.get('[data-testid="repository-auto-prune-method"]').select('None'); + cy.contains('Save').click(); + cy.contains('Successfully deleted repository auto-prune policy'); + }); + + it('displays error when failing to load repo policy', () => { + cy.intercept('GET', '**/autoprunepolicy/**', {statusCode: 500}).as( + 'getServerFailure', + ); + cy.visit('/repository/projectquay/repo1?tab=settings'); + cy.contains('Repository Auto-Prune Policies').click(); + cy.contains('Unable to complete request'); + cy.contains('AxiosError: Request failed with status code 500'); + }); + + it('displays error when failing to create repo policy', () => { + cy.intercept('POST', '**/autoprunepolicy/**', {statusCode: 500}).as( + 'getServerFailure', + ); + cy.visit('/repository/projectquay/repo1?tab=settings'); + cy.contains('Repository Auto-Prune Policies').click(); + + attemptCreateTagNumberRepoPolicy(cy); + cy.contains('Could not create repository auto-prune policy'); + cy.get('button[aria-label="Danger alert details"]').click(); + cy.contains('AxiosError: Request failed with status code 500'); + }); + + it('displays error when failing to update repo policy', () => { + cy.intercept('PUT', '**/autoprunepolicy/**', {statusCode: 500}).as( + 'getServerFailure', + ); + cy.visit('/repository/projectquay/repo1?tab=settings'); + cy.contains('Repository Auto-Prune Policies').click(); + cy.get('[data-testid="repository-auto-prune-method"]').contains('None'); + + attemptCreateTagNumberRepoPolicy(cy); + attemptCreateCreationDateRepoPolicy(cy); + cy.contains('Could not update repository auto-prune policy'); + cy.get('button[aria-label="Danger alert details"]').click(); + cy.contains('AxiosError: Request failed with status code 500'); + }); + + it('displays error when failing to delete repo policy', () => { + cy.intercept('DELETE', '**/autoprunepolicy/**', {statusCode: 500}).as( + 'getServerFailure', + ); + cy.visit('/repository/projectquay/repo1?tab=settings'); + cy.contains('Repository Auto-Prune Policies').click(); + cy.get('[data-testid="repository-auto-prune-method"]').contains('None'); + + attemptCreateTagNumberRepoPolicy(cy); + cy.contains('Successfully created repository auto-prune policy'); + cy.get('input[aria-label="number of tags"]').should('have.value', '25'); + + cy.get('[data-testid="repository-auto-prune-method"]').select('None'); + cy.contains('Save').click(); + cy.contains('Could not delete repository auto-prune policy'); + cy.get('button[aria-label="Danger alert details"]').click(); + cy.contains('AxiosError: Request failed with status code 500'); + }); +}); diff --git a/web/cypress/test/quay-db-data.txt b/web/cypress/test/quay-db-data.txt index 13f8400d3..cac3bda4b 100644 --- a/web/cypress/test/quay-db-data.txt +++ b/web/cypress/test/quay-db-data.txt @@ -60,6 +60,8 @@ ALTER TABLE IF EXISTS ONLY public.repositorybuild DROP CONSTRAINT IF EXISTS fk_r ALTER TABLE IF EXISTS ONLY public.repositorybuild DROP CONSTRAINT IF EXISTS fk_repositorybuild_repository_id_repository; ALTER TABLE IF EXISTS ONLY public.repositorybuild DROP CONSTRAINT IF EXISTS fk_repositorybuild_pull_robot_id_user; ALTER TABLE IF EXISTS ONLY public.repositorybuild DROP CONSTRAINT IF EXISTS fk_repositorybuild_access_token_id_accesstoken; +ALTER TABLE IF EXISTS ONLY public.repositoryautoprunepolicy DROP CONSTRAINT IF EXISTS fk_repositoryautoprunepolicy_repository_id_repository; +ALTER TABLE IF EXISTS ONLY public.repositoryautoprunepolicy DROP CONSTRAINT IF EXISTS fk_repositoryautoprunepolicy_namespace_id_user; ALTER TABLE IF EXISTS ONLY public.repositoryauthorizedemail DROP CONSTRAINT IF EXISTS fk_repositoryauthorizedemail_repository_id_repository; ALTER TABLE IF EXISTS ONLY public.repositoryactioncount DROP CONSTRAINT IF EXISTS fk_repositoryactioncount_repository_id_repository; ALTER TABLE IF EXISTS ONLY public.repository DROP CONSTRAINT IF EXISTS fk_repository_visibility_id_visibility; @@ -259,6 +261,9 @@ DROP INDEX IF EXISTS public.repositorybuild_queue_id; DROP INDEX IF EXISTS public.repositorybuild_pull_robot_id; DROP INDEX IF EXISTS public.repositorybuild_logs_archived; DROP INDEX IF EXISTS public.repositorybuild_access_token_id; +DROP INDEX IF EXISTS public.repositoryautoprunepolicy_uuid; +DROP INDEX IF EXISTS public.repositoryautoprunepolicy_repository_id; +DROP INDEX IF EXISTS public.repositoryautoprunepolicy_namespace_id; DROP INDEX IF EXISTS public.repositoryauthorizedemail_repository_id; DROP INDEX IF EXISTS public.repositoryauthorizedemail_email_repository_id; DROP INDEX IF EXISTS public.repositoryauthorizedemail_code; @@ -512,6 +517,7 @@ ALTER TABLE IF EXISTS ONLY public.repositorynotification DROP CONSTRAINT IF EXIS ALTER TABLE IF EXISTS ONLY public.repositorykind DROP CONSTRAINT IF EXISTS pk_repositorykind; ALTER TABLE IF EXISTS ONLY public.repositorybuildtrigger DROP CONSTRAINT IF EXISTS pk_repositorybuildtrigger; ALTER TABLE IF EXISTS ONLY public.repositorybuild DROP CONSTRAINT IF EXISTS pk_repositorybuild; +ALTER TABLE IF EXISTS ONLY public.repositoryautoprunepolicy DROP CONSTRAINT IF EXISTS pk_repositoryautoprunepolicyid; ALTER TABLE IF EXISTS ONLY public.repositoryauthorizedemail DROP CONSTRAINT IF EXISTS pk_repositoryauthorizedemail; ALTER TABLE IF EXISTS ONLY public.repositoryactioncount DROP CONSTRAINT IF EXISTS pk_repositoryactioncount; ALTER TABLE IF EXISTS ONLY public.repository DROP CONSTRAINT IF EXISTS pk_repository; @@ -617,6 +623,7 @@ ALTER TABLE IF EXISTS public.repositorynotification ALTER COLUMN id DROP DEFAULT ALTER TABLE IF EXISTS public.repositorykind ALTER COLUMN id DROP DEFAULT; ALTER TABLE IF EXISTS public.repositorybuildtrigger ALTER COLUMN id DROP DEFAULT; ALTER TABLE IF EXISTS public.repositorybuild ALTER COLUMN id DROP DEFAULT; +ALTER TABLE IF EXISTS public.repositoryautoprunepolicy ALTER COLUMN id DROP DEFAULT; ALTER TABLE IF EXISTS public.repositoryauthorizedemail ALTER COLUMN id DROP DEFAULT; ALTER TABLE IF EXISTS public.repositoryactioncount ALTER COLUMN id DROP DEFAULT; ALTER TABLE IF EXISTS public.repository ALTER COLUMN id DROP DEFAULT; @@ -755,6 +762,8 @@ DROP SEQUENCE IF EXISTS public.repositorybuildtrigger_id_seq; DROP TABLE IF EXISTS public.repositorybuildtrigger; DROP SEQUENCE IF EXISTS public.repositorybuild_id_seq; DROP TABLE IF EXISTS public.repositorybuild; +DROP SEQUENCE IF EXISTS public.repositoryautoprunepolicy_id_seq; +DROP TABLE IF EXISTS public.repositoryautoprunepolicy; DROP SEQUENCE IF EXISTS public.repositoryauthorizedemail_id_seq; DROP TABLE IF EXISTS public.repositoryauthorizedemail; DROP SEQUENCE IF EXISTS public.repositoryactioncount_id_seq; @@ -3542,6 +3551,43 @@ ALTER TABLE public.repositoryauthorizedemail_id_seq OWNER TO quay; ALTER SEQUENCE public.repositoryauthorizedemail_id_seq OWNED BY public.repositoryauthorizedemail.id; +-- +-- Name: repositoryautoprunepolicy; Type: TABLE; Schema: public; Owner: quay +-- + +CREATE TABLE public.repositoryautoprunepolicy ( + id integer NOT NULL, + uuid character varying(36) NOT NULL, + repository_id integer NOT NULL, + namespace_id integer NOT NULL, + policy text NOT NULL +); + + +ALTER TABLE public.repositoryautoprunepolicy OWNER TO quay; + +-- +-- Name: repositoryautoprunepolicy_id_seq; Type: SEQUENCE; Schema: public; Owner: quay +-- + +CREATE SEQUENCE public.repositoryautoprunepolicy_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.repositoryautoprunepolicy_id_seq OWNER TO quay; + +-- +-- Name: repositoryautoprunepolicy_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: quay +-- + +ALTER SEQUENCE public.repositoryautoprunepolicy_id_seq OWNED BY public.repositoryautoprunepolicy.id; + + -- -- Name: repositorybuild; Type: TABLE; Schema: public; Owner: quay -- @@ -5309,6 +5355,13 @@ ALTER TABLE ONLY public.repositoryactioncount ALTER COLUMN id SET DEFAULT nextva ALTER TABLE ONLY public.repositoryauthorizedemail ALTER COLUMN id SET DEFAULT nextval('public.repositoryauthorizedemail_id_seq'::regclass); +-- +-- Name: repositoryautoprunepolicy id; Type: DEFAULT; Schema: public; Owner: quay +-- + +ALTER TABLE ONLY public.repositoryautoprunepolicy ALTER COLUMN id SET DEFAULT nextval('public.repositoryautoprunepolicy_id_seq'::regclass); + + -- -- Name: repositorybuild id; Type: DEFAULT; Schema: public; Owner: quay -- @@ -5570,7 +5623,7 @@ COPY public.accesstokenkind (id, name) FROM stdin; -- COPY public.alembic_version (version_num) FROM stdin; -41d15c93c299 +b4da5b09c8df \. @@ -6242,6 +6295,9 @@ COPY public.logentrykind (id, name) FROM stdin; 105 push_repo_failed 106 pull_repo_failed 107 delete_tag_failed +108 create_repository_autoprune_policy +109 update_repository_autoprune_policy +110 delete_repository_autoprune_policy \. @@ -6518,8 +6574,8 @@ COPY public.quayservice (id, name) FROM stdin; -- COPY public.queueitem (id, queue_name, body, available_after, available, processing_expires, retries_remaining, state_id) FROM stdin; -1 namespacegc/2/ {"marker_id": 1, "original_username": "quay"} 2023-11-07 19:57:36.102126 t 2023-11-07 22:52:36.037751 5 e90498e4-d92b-487a-90a3-1ab34388af7c -2 namespacegc/3/ {"marker_id": 2, "original_username": "clair"} 2023-11-07 19:57:37.766105 t 2023-11-07 22:52:37.740851 5 9ccfcc76-8277-4dc0-8a14-b37440753124 +2 namespacegc/3/ {"marker_id": 2, "original_username": "clair"} 2024-02-20 14:01:29.891875 t 2024-02-20 16:56:29.836048 5 ff6a0b72-d8cc-4efe-ad83-70225ac325e1 +1 namespacegc/2/ {"marker_id": 1, "original_username": "quay"} 2024-02-20 14:01:34.91615 t 2024-02-20 16:56:34.901017 5 6e996e00-e92b-45a9-962f-7e7f3cdb9e63 \. @@ -6928,6 +6984,14 @@ COPY public.repositoryauthorizedemail (id, repository_id, email, code, confirmed \. +-- +-- Data for Name: repositoryautoprunepolicy; Type: TABLE DATA; Schema: public; Owner: quay +-- + +COPY public.repositoryautoprunepolicy (id, uuid, repository_id, namespace_id, policy) FROM stdin; +\. + + -- -- Data for Name: repositorybuild; Type: TABLE DATA; Schema: public; Owner: quay -- @@ -8084,7 +8148,7 @@ SELECT pg_catalog.setval('public.logentry_id_seq', 1, false); -- Name: logentrykind_id_seq; Type: SEQUENCE SET; Schema: public; Owner: quay -- -SELECT pg_catalog.setval('public.logentrykind_id_seq', 107, true); +SELECT pg_catalog.setval('public.logentrykind_id_seq', 110, true); -- @@ -8168,7 +8232,7 @@ SELECT pg_catalog.setval('public.namespacegeorestriction_id_seq', 1, false); -- Name: notification_id_seq; Type: SEQUENCE SET; Schema: public; Owner: quay -- -SELECT pg_catalog.setval('public.notification_id_seq', 1, false); +SELECT pg_catalog.setval('public.notification_id_seq', 1, true); -- @@ -8325,6 +8389,13 @@ SELECT pg_catalog.setval('public.repositoryactioncount_id_seq', 157, true); SELECT pg_catalog.setval('public.repositoryauthorizedemail_id_seq', 1, true); +-- +-- Name: repositoryautoprunepolicy_id_seq; Type: SEQUENCE SET; Schema: public; Owner: quay +-- + +SELECT pg_catalog.setval('public.repositoryautoprunepolicy_id_seq', 1, false); + + -- -- Name: repositorybuild_id_seq; Type: SEQUENCE SET; Schema: public; Owner: quay -- @@ -8406,14 +8477,14 @@ SELECT pg_catalog.setval('public.role_id_seq', 3, true); -- Name: servicekey_id_seq; Type: SEQUENCE SET; Schema: public; Owner: quay -- -SELECT pg_catalog.setval('public.servicekey_id_seq', 1, false); +SELECT pg_catalog.setval('public.servicekey_id_seq', 1, true); -- -- Name: servicekeyapproval_id_seq; Type: SEQUENCE SET; Schema: public; Owner: quay -- -SELECT pg_catalog.setval('public.servicekeyapproval_id_seq', 1, false); +SELECT pg_catalog.setval('public.servicekeyapproval_id_seq', 1, true); -- @@ -9131,6 +9202,14 @@ ALTER TABLE ONLY public.repositoryauthorizedemail ADD CONSTRAINT pk_repositoryauthorizedemail PRIMARY KEY (id); +-- +-- Name: repositoryautoprunepolicy pk_repositoryautoprunepolicyid; Type: CONSTRAINT; Schema: public; Owner: quay +-- + +ALTER TABLE ONLY public.repositoryautoprunepolicy + ADD CONSTRAINT pk_repositoryautoprunepolicyid PRIMARY KEY (id); + + -- -- Name: repositorybuild pk_repositorybuild; Type: CONSTRAINT; Schema: public; Owner: quay -- @@ -10936,6 +11015,27 @@ CREATE UNIQUE INDEX repositoryauthorizedemail_email_repository_id ON public.repo CREATE INDEX repositoryauthorizedemail_repository_id ON public.repositoryauthorizedemail USING btree (repository_id); +-- +-- Name: repositoryautoprunepolicy_namespace_id; Type: INDEX; Schema: public; Owner: quay +-- + +CREATE UNIQUE INDEX repositoryautoprunepolicy_namespace_id ON public.repositoryautoprunepolicy USING btree (namespace_id); + + +-- +-- Name: repositoryautoprunepolicy_repository_id; Type: INDEX; Schema: public; Owner: quay +-- + +CREATE UNIQUE INDEX repositoryautoprunepolicy_repository_id ON public.repositoryautoprunepolicy USING btree (repository_id); + + +-- +-- Name: repositoryautoprunepolicy_uuid; Type: INDEX; Schema: public; Owner: quay +-- + +CREATE UNIQUE INDEX repositoryautoprunepolicy_uuid ON public.repositoryautoprunepolicy USING btree (uuid); + + -- -- Name: repositorybuild_access_token_id; Type: INDEX; Schema: public; Owner: quay -- @@ -12411,6 +12511,22 @@ ALTER TABLE ONLY public.repositoryauthorizedemail ADD CONSTRAINT fk_repositoryauthorizedemail_repository_id_repository FOREIGN KEY (repository_id) REFERENCES public.repository(id); +-- +-- Name: repositoryautoprunepolicy fk_repositoryautoprunepolicy_namespace_id_user; Type: FK CONSTRAINT; Schema: public; Owner: quay +-- + +ALTER TABLE ONLY public.repositoryautoprunepolicy + ADD CONSTRAINT fk_repositoryautoprunepolicy_namespace_id_user FOREIGN KEY (namespace_id) REFERENCES public."user"(id); + + +-- +-- Name: repositoryautoprunepolicy fk_repositoryautoprunepolicy_repository_id_repository; Type: FK CONSTRAINT; Schema: public; Owner: quay +-- + +ALTER TABLE ONLY public.repositoryautoprunepolicy + ADD CONSTRAINT fk_repositoryautoprunepolicy_repository_id_repository FOREIGN KEY (repository_id) REFERENCES public.repository(id); + + -- -- Name: repositorybuild fk_repositorybuild_access_token_id_accesstoken; Type: FK CONSTRAINT; Schema: public; Owner: quay -- diff --git a/web/src/hooks/UseNamespaceAutoPrunePolicies.ts b/web/src/hooks/UseNamespaceAutoPrunePolicies.ts index e7c9d4853..9d5046037 100644 --- a/web/src/hooks/UseNamespaceAutoPrunePolicies.ts +++ b/web/src/hooks/UseNamespaceAutoPrunePolicies.ts @@ -10,15 +10,20 @@ import {useMutation, useQuery, useQueryClient} from '@tanstack/react-query'; export function useNamespaceAutoPrunePolicies( namespace: string, isUser: boolean, + isEnabled: boolean = true, ) { const { - data: policies, + data: nsPolicies, isLoading, error, isSuccess, dataUpdatedAt, - } = useQuery(['namespace', 'autoprunepolicies', namespace], ({signal}) => - fetchNamespaceAutoPrunePolicies(namespace, isUser, signal), + } = useQuery( + ['namespace', 'autoprunepolicies', namespace], + ({signal}) => fetchNamespaceAutoPrunePolicies(namespace, isUser, signal), + { + enabled: isEnabled, + }, ); return { @@ -26,7 +31,7 @@ export function useNamespaceAutoPrunePolicies( isSuccess, isLoading, dataUpdatedAt, - policies, + nsPolicies, }; } diff --git a/web/src/hooks/UseRepositoryAutoPrunePolicies.ts b/web/src/hooks/UseRepositoryAutoPrunePolicies.ts new file mode 100644 index 000000000..d9c7addce --- /dev/null +++ b/web/src/hooks/UseRepositoryAutoPrunePolicies.ts @@ -0,0 +1,129 @@ +import {useMutation, useQuery, useQueryClient} from '@tanstack/react-query'; +import { + RepositoryAutoPrunePolicy, + createRepositoryAutoPrunePolicy, + deleteRepositoryAutoPrunePolicy, + fetchRepositoryAutoPrunePolicies, + updateRepositoryAutoPrunePolicy, +} from 'src/resources/RepositoryAutoPruneResource'; + +export function useFetchRepositoryAutoPrunePolicies( + organizationName: string, + repoName: string, +) { + const { + data: repoPolicies, + isLoading: isLoadingRepoPolicies, + error: errorFetchingRepoPolicies, + isSuccess: successFetchingRepoPolicies, + dataUpdatedAt: repoPoliciesDataUpdatedAt, + } = useQuery( + ['repositoryautoprunepolicies', organizationName, repoName], + ({signal}) => + fetchRepositoryAutoPrunePolicies(organizationName, repoName, signal), + ); + + return { + errorFetchingRepoPolicies, + successFetchingRepoPolicies, + isLoadingRepoPolicies, + repoPoliciesDataUpdatedAt, + repoPolicies, + }; +} + +export function useCreateRepositoryAutoPrunePolicy( + organizationName: string, + repoName: string, +) { + const queryClient = useQueryClient(); + const { + mutate: createRepoPolicy, + isSuccess: successRepoPolicyCreation, + isError: errorRepoPolicyCreation, + error: errorDetailsRepoPolicyCreation, + } = useMutation( + async (policy: RepositoryAutoPrunePolicy) => + createRepositoryAutoPrunePolicy(organizationName, repoName, policy), + { + onSuccess: () => { + queryClient.invalidateQueries([ + 'repositoryautoprunepolicies', + organizationName, + repoName, + ]); + }, + }, + ); + + return { + createRepoPolicy, + successRepoPolicyCreation, + errorRepoPolicyCreation, + errorDetailsRepoPolicyCreation, + }; +} + +export function useUpdateRepositoryAutoPrunePolicy( + organizationName: string, + repoName: string, +) { + const queryClient = useQueryClient(); + const { + mutate: updateRepoPolicy, + isSuccess: successRepoPolicyUpdation, + isError: errorRepoPolicyUpdation, + error: errorDetailsRepoPolicyUpdation, + } = useMutation( + async (policy: RepositoryAutoPrunePolicy) => + updateRepositoryAutoPrunePolicy(organizationName, repoName, policy), + { + onSuccess: () => { + queryClient.invalidateQueries([ + 'repositoryautoprunepolicies', + organizationName, + repoName, + ]); + }, + }, + ); + + return { + updateRepoPolicy, + successRepoPolicyUpdation, + errorRepoPolicyUpdation, + errorDetailsRepoPolicyUpdation, + }; +} + +export function useDeleteRepositoryAutoPrunePolicy( + organizationName: string, + repoName: string, +) { + const queryClient = useQueryClient(); + const { + mutate: deleteRepoPolicy, + isSuccess: successRepoPolicyDeletion, + isError: errorRepoPolicyDeletion, + error: errorDetailsRepoPolicyDeletion, + } = useMutation( + async (uuid: string) => + deleteRepositoryAutoPrunePolicy(organizationName, repoName, uuid), + { + onSuccess: () => { + queryClient.invalidateQueries([ + 'repositoryautoprunepolicies', + organizationName, + repoName, + ]); + }, + }, + ); + + return { + deleteRepoPolicy, + successRepoPolicyDeletion, + errorRepoPolicyDeletion, + errorDetailsRepoPolicyDeletion, + }; +} diff --git a/web/src/resources/RepositoryAutoPruneResource.ts b/web/src/resources/RepositoryAutoPruneResource.ts new file mode 100644 index 000000000..73c703c9a --- /dev/null +++ b/web/src/resources/RepositoryAutoPruneResource.ts @@ -0,0 +1,54 @@ +import {AxiosResponse} from 'axios'; +import axios from 'src/libs/axios'; +import {assertHttpCode} from './ErrorHandling'; +import {AutoPruneMethod} from './NamespaceAutoPruneResource'; + +export interface RepositoryAutoPrunePolicy { + method: AutoPruneMethod; + uuid?: string; + value?: string | number; +} + +export async function fetchRepositoryAutoPrunePolicies( + organizationName: string, + repoName: string, + signal: AbortSignal, +) { + const repositoryAutoPruneUrl = `/api/v1/repository/${organizationName}/${repoName}/autoprunepolicy/`; + const response: AxiosResponse = await axios.get(repositoryAutoPruneUrl, { + signal, + }); + assertHttpCode(response.status, 200); + const res = response.data.policies as RepositoryAutoPrunePolicy[]; + return res; +} + +export async function createRepositoryAutoPrunePolicy( + organizationName: string, + repoName: string, + policy: RepositoryAutoPrunePolicy, +) { + const repositoryAutoPruneUrl = `/api/v1/repository/${organizationName}/${repoName}/autoprunepolicy/`; + const response = await axios.post(repositoryAutoPruneUrl, policy); + assertHttpCode(response.status, 201); +} + +export async function updateRepositoryAutoPrunePolicy( + organizationName: string, + repoName: string, + policy: RepositoryAutoPrunePolicy, +) { + const repositoryAutoPruneUrl = `/api/v1/repository/${organizationName}/${repoName}/autoprunepolicy/${policy.uuid}`; + const response = await axios.put(repositoryAutoPruneUrl, policy); + assertHttpCode(response.status, 204); +} + +export async function deleteRepositoryAutoPrunePolicy( + organizationName: string, + repoName: string, + uuid: string, +) { + const repositoryAutoPruneUrl = `/api/v1/repository/${organizationName}/${repoName}/autoprunepolicy/${uuid}`; + const response = await axios.delete(repositoryAutoPruneUrl); + assertHttpCode(response.status, 200); +} diff --git a/web/src/routes/OrganizationsList/Organization/Organization.tsx b/web/src/routes/OrganizationsList/Organization/Organization.tsx index 8d29d1772..d9b865806 100644 --- a/web/src/routes/OrganizationsList/Organization/Organization.tsx +++ b/web/src/routes/OrganizationsList/Organization/Organization.tsx @@ -23,7 +23,6 @@ import Settings from './Tabs/Settings/Settings'; import TeamsAndMembershipList from './Tabs/TeamsAndMembership/TeamsAndMembershipList'; import AddNewTeamMemberDrawer from './Tabs/TeamsAndMembership/TeamsView/ManageMembers/AddNewTeamMemberDrawer'; import ManageMembersList from './Tabs/TeamsAndMembership/TeamsView/ManageMembers/ManageMembersList'; -import UsageLogs from 'src/routes/UsageLogs/UsageLogs'; export enum OrganizationDrawerContentType { None, diff --git a/web/src/routes/OrganizationsList/Organization/Tabs/Settings/AutoPruning.tsx b/web/src/routes/OrganizationsList/Organization/Tabs/Settings/AutoPruning.tsx index 7186743e7..895d3ed78 100644 --- a/web/src/routes/OrganizationsList/Organization/Tabs/Settings/AutoPruning.tsx +++ b/web/src/routes/OrganizationsList/Organization/Tabs/Settings/AutoPruning.tsx @@ -50,7 +50,7 @@ export default function AutoPruning(props: AutoPruning) { error, isSuccess: successFetchingPolicies, isLoading, - policies, + nsPolicies, dataUpdatedAt, } = useNamespaceAutoPrunePolicies(props.org, props.isUser); const { @@ -76,8 +76,8 @@ export default function AutoPruning(props: AutoPruning) { if (successFetchingPolicies) { // Currently we only support one policy per namespace but // this will change in the future. - if (policies.length > 0) { - const policy: NamespaceAutoPrunePolicy = policies[0]; + if (nsPolicies.length > 0) { + const policy: NamespaceAutoPrunePolicy = nsPolicies[0]; setMethod(policy.method); setUuid(policy.uuid); switch (policy.method) { diff --git a/web/src/routes/RepositoryDetails/Settings/RepositoryAutoPruning.tsx b/web/src/routes/RepositoryDetails/Settings/RepositoryAutoPruning.tsx new file mode 100644 index 000000000..565d6b1be --- /dev/null +++ b/web/src/routes/RepositoryDetails/Settings/RepositoryAutoPruning.tsx @@ -0,0 +1,426 @@ +import { + ActionGroup, + Button, + Flex, + Form, + FormGroup, + FormSelect, + FormSelectOption, + NumberInput, + Spinner, + Title, + FormHelperText, + HelperText, + HelperTextItem, + DataList, + DataListItem, + DataListItemRow, + DataListItemCells, + DataListCell, + Gallery, +} from '@patternfly/react-core'; +import {useEffect, useState} from 'react'; +import {AlertVariant} from 'src/atoms/AlertState'; +import Conditional from 'src/components/empty/Conditional'; +import RequestError from 'src/components/errors/RequestError'; +import {useAlerts} from 'src/hooks/UseAlerts'; +import {useNamespaceAutoPrunePolicies} from 'src/hooks/UseNamespaceAutoPrunePolicies'; +import {useOrganization} from 'src/hooks/UseOrganization'; +import { + useCreateRepositoryAutoPrunePolicy, + useDeleteRepositoryAutoPrunePolicy, + useFetchRepositoryAutoPrunePolicies, + useUpdateRepositoryAutoPrunePolicy, +} from 'src/hooks/UseRepositoryAutoPrunePolicies'; +import {isNullOrUndefined} from 'src/libs/utils'; +import {AutoPruneMethod} from 'src/resources/NamespaceAutoPruneResource'; +import {RepositoryAutoPrunePolicy} from 'src/resources/RepositoryAutoPruneResource'; +import {shorthandTimeUnits} from 'src/routes/OrganizationsList/Organization/Tabs/Settings/AutoPruning'; + +export default function RepositoryAutoPruning(props: RepositoryAutoPruning) { + const [uuid, setUuid] = useState(null); + const [method, setMethod] = useState(AutoPruneMethod.NONE); + const [tagCount, setTagCount] = useState(20); + const [tagCreationDateUnit, setTagCreationDateUnit] = useState('d'); + const [tagCreationDateValue, setTagCreationDateValue] = useState(7); + const {addAlert} = useAlerts(); + const {organization} = useOrganization(props.organizationName); + + const { + error, + isSuccess: successFetchingPolicies, + isLoading, + nsPolicies, + dataUpdatedAt, + } = useNamespaceAutoPrunePolicies( + props.organizationName, + props.isUser, + organization?.is_org_admin || false, + ); + + const { + errorFetchingRepoPolicies, + successFetchingRepoPolicies, + isLoadingRepoPolicies, + repoPolicies, + repoPoliciesDataUpdatedAt, + } = useFetchRepositoryAutoPrunePolicies( + props.organizationName, + props.repoName, + ); + + const { + createRepoPolicy, + successRepoPolicyCreation, + errorRepoPolicyCreation, + errorDetailsRepoPolicyCreation, + } = useCreateRepositoryAutoPrunePolicy( + props.organizationName, + props.repoName, + ); + + const { + updateRepoPolicy, + successRepoPolicyUpdation, + errorRepoPolicyUpdation, + errorDetailsRepoPolicyUpdation, + } = useUpdateRepositoryAutoPrunePolicy( + props.organizationName, + props.repoName, + ); + + const { + deleteRepoPolicy, + successRepoPolicyDeletion, + errorRepoPolicyDeletion, + errorDetailsRepoPolicyDeletion, + } = useDeleteRepositoryAutoPrunePolicy( + props.organizationName, + props.repoName, + ); + + useEffect(() => { + if (successFetchingRepoPolicies) { + // Currently we only support one policy per repository but + // this will change in the future. + if (repoPolicies.length > 0) { + const policy: RepositoryAutoPrunePolicy = repoPolicies[0]; + setMethod(policy.method); + setUuid(policy.uuid); + switch (policy.method) { + case AutoPruneMethod.TAG_NUMBER: { + setTagCount(policy.value as number); + break; + } + case AutoPruneMethod.TAG_CREATION_DATE: { + const tagAgeValue = (policy.value as string).match(/\d+/g); + const tagAgeUnit = (policy.value as string).match(/[a-zA-Z]+/g); + if (tagAgeValue.length > 0 && tagAgeUnit.length > 0) { + setTagCreationDateValue(Number(tagAgeValue[0])); + setTagCreationDateUnit(tagAgeUnit[0]); + } else { + console.error('Invalid tag age value'); + } + break; + } + } + } else { + // If no policy was returned it's possible this was + // after the deletion of the policy, in which all the state + // has to be reset + setUuid(null); + setMethod(AutoPruneMethod.NONE); + setTagCount(20); + setTagCreationDateUnit('d'); + setTagCreationDateValue(7); + } + } + }, [ + successFetchingRepoPolicies, + successFetchingPolicies, + repoPoliciesDataUpdatedAt, + ]); + + useEffect(() => { + if (successRepoPolicyCreation) { + addAlert({ + title: 'Successfully created repository auto-prune policy', + variant: AlertVariant.Success, + }); + } + }, [successRepoPolicyCreation]); + + useEffect(() => { + if (successRepoPolicyUpdation) { + addAlert({ + title: 'Successfully updated repository auto-prune policy', + variant: AlertVariant.Success, + }); + } + }, [successRepoPolicyUpdation]); + + useEffect(() => { + if (successRepoPolicyDeletion) { + addAlert({ + title: 'Successfully deleted repository auto-prune policy', + variant: AlertVariant.Success, + }); + } + }, [successRepoPolicyDeletion]); + + useEffect(() => { + if (errorRepoPolicyCreation) { + addAlert({ + title: 'Could not create repository auto-prune policy', + variant: AlertVariant.Failure, + message: errorDetailsRepoPolicyCreation.toString(), + }); + } + }, [errorRepoPolicyCreation]); + + useEffect(() => { + if (errorRepoPolicyUpdation) { + addAlert({ + title: 'Could not update repository auto-prune policy', + variant: AlertVariant.Failure, + message: errorDetailsRepoPolicyUpdation.toString(), + }); + } + }, [errorRepoPolicyUpdation]); + + useEffect(() => { + if (errorRepoPolicyDeletion) { + addAlert({ + title: 'Could not delete repository auto-prune policy', + variant: AlertVariant.Failure, + message: errorDetailsRepoPolicyDeletion.toString(), + }); + } + }, [errorRepoPolicyDeletion]); + + const onSave = (e) => { + e.preventDefault(); + let value = null; + switch (method) { + case AutoPruneMethod.TAG_NUMBER: + value = tagCount; + break; + case AutoPruneMethod.TAG_CREATION_DATE: + value = `${String(tagCreationDateValue)}${tagCreationDateUnit}`; + break; + case AutoPruneMethod.NONE: + // Delete the policy is done by setting the method to none + if (!isNullOrUndefined(uuid)) { + deleteRepoPolicy(uuid); + } + return; + default: + // Reaching here indicates programming error, component should always be aware of valid methods + return; + } + if (isNullOrUndefined(uuid)) { + createRepoPolicy({method: method, value: value}); + } else { + updateRepoPolicy({uuid: uuid, method: method, value: value}); + } + }; + + if (isLoadingRepoPolicies) { + return ; + } + + if (!isNullOrUndefined(errorFetchingRepoPolicies)) { + return ; + } + + return ( + <> + + + Namespace Auto-Pruning Policies + + + + + + + + {nsPolicies[0]?.method}: + + , + + + {nsPolicies[0]?.value} + + , + ] + : [] + } + /> + + + + + + + Repository Auto-Pruning Policies + +

+ Auto-pruning policies automatically delete tags under this repository by + a given method. +

+
+ + setMethod(val as AutoPruneMethod)} + > + + + + + + + The method used to prune tags. + + + + + + { + tagCount > 1 ? setTagCount(tagCount - 1) : setTagCount(1); + }} + onChange={(e) => { + const input = (e.target as HTMLInputElement).value; + const value = Number(input); + if (value > 0 && /^\d+$/.test(input)) { + setTagCount(value); + } + }} + onPlus={() => { + setTagCount(tagCount + 1); + }} + inputAriaLabel="number of tags" + minusBtnAriaLabel="minus" + plusBtnAriaLabel="plus" + data-testid="repository-auto-prune-tag-count" + /> + + + + All tags sorted by earliest creation date will be deleted + until the repository total falls below the threshold + + + + + + + +
+ { + tagCreationDateValue > 1 + ? setTagCreationDateValue(tagCreationDateValue - 1) + : setTagCreationDateValue(1); + }} + onChange={(e) => { + const input = (e.target as HTMLInputElement).value; + const value = Number(input); + if (value > 0 && /^\d+$/.test(input)) { + setTagCreationDateValue(value); + } + }} + onPlus={() => { + setTagCreationDateValue(tagCreationDateValue + 1); + }} + inputAriaLabel="tag creation date value" + minusBtnAriaLabel="minus" + plusBtnAriaLabel="plus" + data-testid="repository-auto-prune-tag-creation-date-value" + style={{paddingRight: '1em'}} + /> + setTagCreationDateUnit(val)} + style={{width: '10em'}} + > + {Object.keys(shorthandTimeUnits).map((key) => ( + + ))} + +
+ + + + All tags with a creation date earlier than the selected time + period will be deleted + + + + + + + + + + + +
+ + ); +} + +interface RepositoryAutoPruning { + organizationName: string; + repoName: string; + isUser: boolean; +} diff --git a/web/src/routes/RepositoryDetails/Settings/Settings.tsx b/web/src/routes/RepositoryDetails/Settings/Settings.tsx index 24e02bf61..b9487c512 100644 --- a/web/src/routes/RepositoryDetails/Settings/Settings.tsx +++ b/web/src/routes/RepositoryDetails/Settings/Settings.tsx @@ -8,10 +8,14 @@ import Visibility from './Visibility'; import {RepositoryStateForm} from './RepositoryState'; import {RepositoryDetails} from 'src/resources/RepositoryResource'; import {useQuayConfig} from 'src/hooks/UseQuayConfig'; +import RepositoryAutoPruning from 'src/routes/RepositoryDetails/Settings/RepositoryAutoPruning'; +import {useOrganization} from 'src/hooks/UseOrganization'; export default function Settings(props: SettingsProps) { const [activeTabIndex, setActiveTabIndex] = useState(0); const config = useQuayConfig(); + const {isUserOrganization} = useOrganization(props.org); + const tabs = [ { name: 'User and robot permissions', @@ -24,6 +28,21 @@ export default function Settings(props: SettingsProps) { /> ), }, + ...(config?.features?.AUTO_PRUNE + ? [ + { + name: 'Repository Auto-Prune Policies', + id: 'repositoryautoprunepolicies', + content: ( + + ), + }, + ] + : []), { name: 'Events and notifications', id: 'eventsandnotifications', diff --git a/workers/autopruneworker.py b/workers/autopruneworker.py index 1aaa35def..e8635bd5e 100644 --- a/workers/autopruneworker.py +++ b/workers/autopruneworker.py @@ -1,4 +1,6 @@ import logging.config +import os +import sys import time import features @@ -35,21 +37,42 @@ class AutoPruneWorker(Worker): autoprune_task.id, autoprune_task.namespace, ) - + repo_policies = [] try: - policies = get_namespace_autoprune_policies_by_id(autoprune_task.namespace) - if not policies: - # When implementing repo policies, fetch repo policies before deleting the task - delete_autoprune_task(autoprune_task) - continue + ns_policies = get_namespace_autoprune_policies_by_id(autoprune_task.namespace) + if not ns_policies: + repo_policies = get_repository_autoprune_policies_by_namespace_id( + autoprune_task.namespace + ) + if not repo_policies: + logger.info( + "deleting autoprune task %s for namespace %s", + autoprune_task.id, + autoprune_task.namespace, + ) + delete_autoprune_task(autoprune_task) + continue execute_namespace_polices( - policies, + ns_policies, autoprune_task.namespace, FETCH_REPOSITORIES_PAGE_LIMIT, FETCH_TAGS_PAGE_LIMIT, ) + # case: only repo policies exists & no namespace policy + for policy in repo_policies: + repo_id = policy.repository_id + repo = get_repository_by_policy_repo_id(repo_id) + logger.info( + "processing autoprune task %s for repository %s", + autoprune_task.id, + repo.name, + ) + execute_policy_on_repo( + policy, repo_id, autoprune_task.namespace, tag_page_limit=100 + ) + update_autoprune_task(autoprune_task, task_status="success") except Exception as err: update_autoprune_task(autoprune_task, task_status=f"failure: {str(err)}")