docker/quay
1
0
Fork 0
mirror of https://github.com/quay/quay.git synced 2026-01-26 06:21:37 +03:00
Code Activity

api: reducing db calls in repo list endpoints with quota enabled (PROJQUAY-6895) (#2770)

Browse Source 
Reducing the number of DB calls in the repo list endpoint with quota enabled by:
- Adding the id to RepositoryBaseElement when the repositories are initially fetched, removing the need to fetch the repository ID's again
- Fetching the repository sizes with a single DB call using the IN operator
This commit is contained in:
Brandon Caton
2024-03-26 10:33:24 -04:00
committed by GitHub
parent 131d66d13f
commit f241767005
6 changed files with 132 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
data/model/repository.py
View File

@@ -701,6 +701,41 @@ def get_repository_size(repo_id: int):
return 0
def get_repository_sizes(repo_ids: list):
"""
Returns a map from repository ID to the size of the repository in bytes.
List of repo_ids should be kept below 1000 to avoid performance issues.
"""
if not repo_ids:
return {}
if len(repo_ids) > 1000:
logger.warning(
"Fetching more than 1000 repository sizes at once, you may experience performance issues."
)
tuples = (
QuotaRepositorySize.select(
QuotaRepositorySize.repository,
QuotaRepositorySize.size_bytes,
can_use_read_replica=True,
)
.where(QuotaRepositorySize.repository << repo_ids)
.tuples()
)
size_map = {}
for record in tuples:
size_map[record[0]] = record[1] if record[1] is not None else 0
# Default to 0 for any repositories that don't have a size.
for repo_id in repo_ids:
if repo_id not in size_map:
size_map[repo_id] = 0
return size_map
def get_size_during_upload(repo_id: int):
query = (
BlobUpload.select(fn.Sum(BlobUpload.byte_count).alias("size_bytes")).where(

26
data/model/test/test_repository.py
View File

@@ -3,11 +3,13 @@ from datetime import timedelta
import pytest
from data.database import BlobUpload, Repository
from data.database import BlobUpload, QuotaRepositorySize, Repository
from data.model.repository import (
create_repository,
get_estimated_repository_count,
get_filtered_matching_repositories,
get_repository,
get_repository_sizes,
get_size_during_upload,
)
from data.model.storage import get_image_location_for_name
@@ -81,3 +83,25 @@ def test_get_size_during_upload(initialized_db):
)
size = get_size_during_upload(repo1.id)
assert size == upload_size
def test_get_repository_sizes(initialized_db):
# empty state
assert get_repository_sizes([]) == {}
assert get_repository_sizes(None) == {}
# repos with size entries
repo1 = get_repository("buynlarge", "orgrepo")
repo2 = get_repository("devtable", "simple")
assert get_repository_sizes([repo1.id, repo2.id]) == {repo1.id: 92, repo2.id: 92}
# some repos without size entries
repo3 = get_repository("devtable", "building")
assert (
QuotaRepositorySize.select().where(QuotaRepositorySize.repository == repo3.id).count() == 0
)
assert get_repository_sizes([repo1.id, repo2.id, repo3.id]) == {
repo1.id: 92,
repo2.id: 92,
repo3.id: 0,
}

6
endpoints/api/repository.py
View File

@@ -255,10 +255,10 @@ class RepositoryList(ApiResource):
popularity,
)
repositories_with_view = [repo.to_dict() for repo in repos]
if features.QUOTA_MANAGEMENT and features.EDIT_QUOTA:
model.add_quota_view(repositories_with_view)
repositories_with_view = model.add_quota_view(repos)
else:
repositories_with_view = [repo.to_dict() for repo in repos]
return {"repositories": repositories_with_view}, next_page_token

2
endpoints/api/repository_models_interface.py
View File

@@ -14,6 +14,7 @@ class RepositoryBaseElement(
namedtuple(
"RepositoryBaseElement",
[
"id",
"namespace_name",
"repository_name",
"is_starred",
@@ -35,6 +36,7 @@ class RepositoryBaseElement(
"""
Repository a single quay repository.
:type id: int
:type namespace_name: string
:type repository_name: string
:type is_starred: boolean

33
endpoints/api/repository_models_pre_oci.py
View File

@@ -151,6 +151,7 @@ class PreOCIModel(RepositoryDataInterface):
return (
[
RepositoryBaseElement(
repo.rid,
repo.namespace_user.username,
repo.name,
repo.rid in star_set,
@@ -210,6 +211,7 @@ class PreOCIModel(RepositoryDataInterface):
is_public = model.repository.is_repository_public(repo)
kind_name = RepositoryTable.kind.get_name(repo.kind_id)
base = RepositoryBaseElement(
repo.id,
namespace_name,
repository_name,
is_starred,
@@ -259,37 +261,38 @@ class PreOCIModel(RepositoryDataInterface):
def add_quota_view(self, repos):
namespace_limit_bytes = {}
repos_with_view = []
repo_sizes = model.repository.get_repository_sizes([repo.id for repo in repos])
for repo in repos:
repo_with_view = repo.to_dict()
repos_with_view.append(repo_with_view)
if repo.get("namespace", None) is None or repo.get("name", None) is None:
continue
repository_ref = model.repository.get_repository(
repo.get("namespace"), repo.get("name")
)
if not repository_ref:
if (
repo_with_view.get("namespace", None) is None
or repo_with_view.get("name", None) is None
):
continue
# Caching result in namespace_limit_bytes
if repository_ref.namespace_user.username not in namespace_limit_bytes:
if repo_with_view.get("namespace") not in namespace_limit_bytes:
quotas = model.namespacequota.get_namespace_quota_list(
repository_ref.namespace_user.username
repo_with_view.get("namespace")
)
# Currently only one quota per namespace is supported
namespace_limit_bytes[repository_ref.namespace_user.username] = (
namespace_limit_bytes[repo_with_view.get("namespace")] = (
quotas[0].limit_bytes
if quotas
else model.namespacequota.fetch_system_default(quotas)
)
repo_size = model.repository.get_repository_size(repository_ref.id)
# If FEATURE_QUOTA_MANAGEMENT is enabled & quota is not set for an org,
# we still want to report repo's storage consumption
repo["quota_report"] = {
"quota_bytes": repo_size,
"configured_quota": namespace_limit_bytes[repository_ref.namespace_user.username],
repo_with_view["quota_report"] = {
"quota_bytes": repo_sizes.get(repo.id, 0),
"configured_quota": namespace_limit_bytes[repo_with_view.get("namespace")],
}
return repos_with_view
pre_oci_model = PreOCIModel()

73
endpoints/api/test/test_repository_models_pre_oci.py
View File

@@ -1,8 +1,10 @@
import pytest
from mock import ANY, MagicMock, patch
from data import database, model
from data.database import QuotaRepositorySize
from data.model.repository import get_repository
from endpoints.api.repository import Repository, RepositoryList, RepositoryTrust
from endpoints.api.repository_models_interface import RepositoryBaseElement
from endpoints.api.repository_models_pre_oci import pre_oci_model
from endpoints.api.test.shared import conduct_api_call
from endpoints.test.shared import client_with_identity
@@ -11,33 +13,56 @@ from test.fixtures import *
def test_add_quota_view(initialized_db):
repo_with_no_size_row = get_repo_base_element("randomuser", "randomrepo")
repos = [
{
"namespace": "buynlarge",
"name": "orgrepo",
},
{
"namespace": "devtable",
"name": "simple",
},
{
"namespace": "devtable",
"name": None,
},
{
"namespace": "buynlarge",
"name": "doesnotexist",
},
get_repo_base_element("buynlarge", "orgrepo"),
get_repo_base_element("devtable", "simple"),
repo_with_no_size_row,
get_repo_base_element("devtable", "building"),
]
pre_oci_model.add_quota_view(repos)
QuotaRepositorySize.delete().where(
QuotaRepositorySize.repository == repo_with_no_size_row.id
).execute()
assert (
QuotaRepositorySize.select()
.where(QuotaRepositorySize.repository == repo_with_no_size_row.id)
.count()
== 0
)
assert repos[0].get("quota_report").get("quota_bytes") == 92
assert repos[0].get("quota_report").get("configured_quota") == 3000
repos_with_view = pre_oci_model.add_quota_view(repos)
assert repos[1].get("quota_report").get("quota_bytes") == 92
assert repos[1].get("quota_report").get("configured_quota") is None
assert repos_with_view[0].get("quota_report").get("quota_bytes") == 92
assert repos_with_view[0].get("quota_report").get("configured_quota") == 3000
assert repos[2].get("quota_report") is None
assert repos_with_view[1].get("quota_report").get("quota_bytes") == 92
assert repos_with_view[1].get("quota_report").get("configured_quota") is None
assert repos[3].get("quota_report") is None
assert repos_with_view[2].get("quota_report").get("quota_bytes") == 0
assert repos_with_view[2].get("quota_report").get("configured_quota") == 6000
assert repos_with_view[3].get("quota_report").get("quota_bytes") == 0
assert repos_with_view[3].get("quota_report").get("configured_quota") is None
def get_repo_base_element(namespace, repo):
repo = get_repository(namespace, repo)
return RepositoryBaseElement(
repo.id,
repo.namespace_user.username,
repo.name,
False,
False,
"",
repo.description,
repo.namespace_user.organization,
repo.namespace_user.removed_tag_expiration_s,
None,
None,
False,
False,
False,
repo.namespace_user.stripe_id is None,
repo.state,
)