docker/quay
1
0
Fork 0
mirror of https://github.com/quay/quay.git synced 2025-07-28 20:22:05 +03:00
Code Activity

Deprecate Image rows and move to in-memory synthesized legacy images [Python 3] (#442)

Browse Source 
* Change verbs to use a DerivedStorageForManifest table instead of DerivedStorageForImage

This allows us to deprecate the DerivedStorageForImage table.

Fixes https://issues.redhat.com/browse/PROJQUAY-519

* Change uploaded blob tracking to use its own table and deprecate
RepositoryTag

* Start recording the compressed layers size and config media type on the
manifest row in the database

NOTE: This change includes a database migration which will *lock* the
manifest table

* Change tag API to return the layers size from the manifest

* Remove unused code

* Add new config_media_type field to OCI types

* Fix secscan V2 test for us no longer writing temp images

* Remove unused uploading field

* Switch registry model to use synthetic legacy images

Legacy images are now (with exception of the V2 security model) read from the *manifest* and sythensized in memory. The legacy image IDs are generated realtime based on the hashids library. This change also further deprecates a bunch of our Image APIs, reducing them to only returning the image IDs, and emptying out the remaining metadata (to avoid the requirement of us loading the information for the manifest from storage).

This has been tested with our full clients test suite with success.

* Add a backfill worker for manifest layers compressed sizes

* Change image tracks into manifest tracks now that we no longer have
manifest-less tags

* Add back in the missing method

* Add missing joins to reduce extra queries

* Remove unnecessary join when looking up legacy images

* Remove extra hidden filter on tag queries

* Further DB improvements

* Delete all Verbs, as they were deprecated

* Add back missing parameter in manifest data type

* Fix join to return None for the robot if not defined on mirror config

* switch to using secscan_v4_model for all indexing and remove most of secscan_v2_model code

* Add a missing join

* Remove files accidentally re-added due to rebase

* Add back hashids lib

* Rebase fixes

* Fix broken test

* Remove unused GPG signer now that ACI conversion is removed

* Remove duplicated repomirrorworker

* Remove unused notification code for secscan. We'll re-add it once Clair
V4 security notifications are ready to go

* Fix formatting

* Stop writing Image rows when creating manifests

* Stop writing empty layer blobs for manifests

As these blobs are shared, we don't need to write ManifestBlob rows
for them

* Remove further unused code

* Add doc comment to _build_blob_map

* Add unit test for synthetic V1 IDs

* Remove unused import

* Add an invalid value test to synthetic ID decode tests

* Add manifest backfill worker back in

Seems to have been removed at some point

* Add a test for cached active tags

* Rename test_shared to not conflict with another same-named test file

Pytest doesn't like having two test modules with the same name

* Have manifestbackfillworker also copy over the config_media_type if present

Co-authored-by: alecmerdler <alecmerdler@gmail.com>
This commit is contained in:
Joseph Schorr
2020-07-28 13:03:10 -04:00
committed by GitHub
parent 8d57e769fa
commit 0e628b1569
127 changed files with 1881 additions and 10002 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
app.py
View File

@ -66,7 +66,6 @@ from util.metrics.prometheus import PrometheusPlugin
from util.repomirror.api import RepoMirrorAPI
from util.tufmetadata.api import TUFMetadataAPI
from util.security.instancekeys import InstanceKeys
from util.security.signing import Signer
from util.greenlet_tracing import enable_tracing
@ -244,7 +243,6 @@ build_logs = BuildLogs(app)
authentication = UserAuthentication(app, config_provider, OVERRIDE_CONFIG_DIRECTORY)
userevents = UserEventsBuilderModule(app)
superusers = SuperUserManager(app)
signer = Signer(app, config_provider)
instance_keys = InstanceKeys(app)
label_validator = LabelValidator(app)
build_canceller = BuildCanceller(app)
@ -260,9 +258,6 @@ dockerfile_build_queue = WorkQueue(
app.config["DOCKERFILE_BUILD_QUEUE_NAME"], tf, has_namespace=True
)
notification_queue = WorkQueue(app.config["NOTIFICATION_QUEUE_NAME"], tf, has_namespace=True)
secscan_notification_queue = WorkQueue(
app.config["SECSCAN_NOTIFICATION_QUEUE_NAME"], tf, has_namespace=False
)
export_action_logs_queue = WorkQueue(
app.config["EXPORT_ACTION_LOGS_QUEUE_NAME"], tf, has_namespace=True
)
@ -277,7 +272,6 @@ all_queues = [
image_replication_queue,
dockerfile_build_queue,
notification_queue,
secscan_notification_queue,
chunk_cleanup_queue,
repository_gc_queue,
namespace_gc_queue,
@ -315,10 +309,13 @@ model.config.store = storage
model.config.register_repo_cleanup_callback(tuf_metadata_api.delete_metadata)
secscan_model.configure(app, instance_keys, storage)
secscan_model.register_model_cleanup_callbacks(model.config)
logs_model.configure(app.config)
# NOTE: We re-use the page token key here as this is just to obfuscate IDs for V1, and
# does not need to actually be secure.
registry_model.set_id_hash_salt(app.config.get("PAGE_TOKEN_KEY"))
@login_manager.user_loader
def load_user(user_uuid):

1
application.py
View File

@ -13,7 +13,6 @@ from app import app as application
# Bind all of the blueprints
import web
import verbs
import registry
import secscan

3
conf/init/supervisord_conf_create.py
View File

@ -29,14 +29,12 @@ def default_services():
"notificationworker": {"autostart": "true"},
"queuecleanupworker": {"autostart": "true"},
"repositoryactioncounter": {"autostart": "true"},
"security_notification_worker": {"autostart": "true"},
"securityworker": {"autostart": "true"},
"storagereplication": {"autostart": "true"},
"teamsyncworker": {"autostart": "true"},
"dnsmasq": {"autostart": "true"},
"gunicorn-registry": {"autostart": "true"},
"gunicorn-secscan": {"autostart": "true"},
"gunicorn-verbs": {"autostart": "true"},
"gunicorn-web": {"autostart": "true"},
"ip-resolver-update-worker": {"autostart": "true"},
"jwtproxy": {"autostart": "true"},
@ -45,6 +43,7 @@ def default_services():
"pushgateway": {"autostart": "true"},
"servicekey": {"autostart": "true"},
"repomirrorworker": {"autostart": "false"},
"backfillmanifestworker": {"autostart": "false"},
}

3
conf/nginx/http-base.conf
View File

@ -49,9 +49,6 @@ upstream web_app_server {
upstream jwtproxy_secscan {
server unix:/tmp/jwtproxy_secscan.sock fail_timeout=0;
}
upstream verbs_app_server {
server unix:/tmp/gunicorn_verbs.sock fail_timeout=0;
}
upstream registry_app_server {
server unix:/tmp/gunicorn_registry.sock fail_timeout=0;
}

13
conf/nginx/server-base.conf.jnj
View File

@ -306,19 +306,6 @@ location = /v1/_ping {
return 200 'true';
}
location /c1/ {
proxy_buffering off;
proxy_request_buffering off;
proxy_pass http://verbs_app_server;
proxy_temp_path /tmp 1 2;
{% if enable_rate_limits %}
limit_req zone=staticauth burst=5 nodelay;
{% endif %}
}
location /static/ {
# checks for static file, if not found proxy to app
alias {{static_dir}}/;

16
conf/supervisord.conf.jnj
View File

@ -138,14 +138,6 @@ autostart = {{ config['repositoryactioncounter']['autostart'] }}
stdout_events_enabled = true
stderr_events_enabled = true
[program:security_notification_worker]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=python -m workers.security_notification_worker
autostart = {{ config['security_notification_worker']['autostart'] }}
stdout_events_enabled = true
stderr_events_enabled = true
[program:securityworker]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
@ -194,14 +186,6 @@ autostart = {{ config['gunicorn-secscan']['autostart'] }}
stdout_events_enabled = true
stderr_events_enabled = true
[program:gunicorn-verbs]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s
command=nice -n 10 gunicorn -c %(ENV_QUAYCONF)s/gunicorn_verbs.py verbs:application
autostart = {{ config['gunicorn-verbs']['autostart'] }}
stdout_events_enabled = true
stderr_events_enabled = true
[program:gunicorn-web]
environment=
PYTHONPATH=%(ENV_QUAYDIR)s

7
config.py
View File

@ -259,7 +259,6 @@ class DefaultConfig(ImmutableConfig):
NOTIFICATION_QUEUE_NAME = "notification"
DOCKERFILE_BUILD_QUEUE_NAME = "dockerfilebuild"
REPLICATION_QUEUE_NAME = "imagestoragereplication"
SECSCAN_NOTIFICATION_QUEUE_NAME = "security_notification"
CHUNK_CLEANUP_QUEUE_NAME = "chunk_cleanup"
NAMESPACE_GC_QUEUE_NAME = "namespacegc"
REPOSITORY_GC_QUEUE_NAME = "repositorygc"
@ -476,9 +475,6 @@ class DefaultConfig(ImmutableConfig):
# The version of the API to use for the security scanner.
SECURITY_SCANNER_API_VERSION = "v1"
# Namespace whitelist for security scanner.
SECURITY_SCANNER_V4_NAMESPACE_WHITELIST = []
# Minimum number of seconds before re-indexing a manifest with the security scanner.
SECURITY_SCANNER_V4_REINDEX_THRESHOLD = 300
@ -739,3 +735,6 @@ class DefaultConfig(ImmutableConfig):
# Feature Flag: Whether the repository action count worker is enabled.
FEATURE_REPOSITORY_ACTION_COUNTER = True
# TEMP FEATURE: Backfill the sizes of manifests.
FEATURE_MANIFEST_SIZE_BACKFILL = True

4
config_app/js/core-config-setup/core-config-setup.js
View File

@ -74,10 +74,6 @@ angular.module("quay-config")
return config.AUTHENTICATION_TYPE == 'AppToken';
}},
{'id': 'signer', 'title': 'ACI Signing', 'condition': function(config) {
return config.FEATURE_ACI_CONVERSION;
}},
{'id': 'github-login', 'title': 'Github (Enterprise) Authentication', 'condition': function(config) {
return config.FEATURE_GITHUB_LOGIN;
}},

25
data/database.py
View File

@ -685,6 +685,7 @@ class User(BaseModel):
NamespaceGeoRestriction,
ManifestSecurityStatus,
RepoMirrorConfig,
UploadedBlob,
}
| appr_classes
| v22_classes
@ -888,6 +889,7 @@ class Repository(BaseModel):
RepoMirrorRule,
DeletedRepository,
ManifestSecurityStatus,
UploadedBlob,
}
| appr_classes
| v22_classes
@ -1115,6 +1117,7 @@ class Image(BaseModel):
return list(map(int, self.ancestors.split("/")[1:-1]))
@deprecated_model
class DerivedStorageForImage(BaseModel):
source_image = ForeignKeyField(Image)
derivative = ForeignKeyField(ImageStorage)
@ -1127,6 +1130,7 @@ class DerivedStorageForImage(BaseModel):
indexes = ((("source_image", "transformation", "uniqueness_hash"), True),)
@deprecated_model
class RepositoryTag(BaseModel):
name = CharField()
image = ForeignKeyField(Image)
@ -1391,8 +1395,8 @@ class ExternalNotificationMethod(BaseModel):
class RepositoryNotification(BaseModel):
uuid = CharField(default=uuid_generator, index=True)
repository = ForeignKeyField(Repository)
event = ForeignKeyField(ExternalNotificationEvent)
method = ForeignKeyField(ExternalNotificationMethod)
event = EnumField(ExternalNotificationEvent)
method = EnumField(ExternalNotificationMethod)
title = CharField(null=True)
config_json = TextField()
event_config_json = TextField(default="{}")
@ -1414,6 +1418,19 @@ class RepositoryAuthorizedEmail(BaseModel):
)
class UploadedBlob(BaseModel):
"""
UploadedBlob tracks a recently uploaded blob and prevents it from being GCed
while within the expiration window.
"""
id = BigAutoField()
repository = ForeignKeyField(Repository)
blob = ForeignKeyField(ImageStorage)
uploaded_at = DateTimeField(default=datetime.utcnow)
expires_at = DateTimeField(index=True)
class BlobUpload(BaseModel):
repository = ForeignKeyField(Repository)
uuid = CharField(index=True, unique=True)
@ -1699,12 +1716,16 @@ class Manifest(BaseModel):
media_type = EnumField(MediaType)
manifest_bytes = TextField()
config_media_type = CharField(null=True)
layers_compressed_size = BigIntegerField(null=True)
class Meta:
database = db
read_only_config = read_only_config
indexes = (
(("repository", "digest"), True),
(("repository", "media_type"), False),
(("repository", "config_media_type"), False),
)

51
data/migrations/dba_operator/3383aad1e992-databasemigration.yaml Normal file
View File

@ -0,0 +1,51 @@
---
apiVersion: dbaoperator.app-sre.redhat.com/v1alpha1
kind: DatabaseMigration
metadata:
name: 3383aad1e992
spec:
migrationContainerSpec:
command:
- /quay-registry/quay-entrypoint.sh
- migrate
- 3383aad1e992
image: quay.io/quay/quay
name: 3383aad1e992
previous: !!python/tuple
- 04b9d2191450
schemaHints:
- columns:
- name: id
nullable: false
- name: repository_id
nullable: false
- name: blob_id
nullable: false
- name: uploaded_at
nullable: false
- name: expires_at
nullable: false
operation: createTable
table: uploadedblob
- columns:
- name: blob_id
nullable: false
indexName: uploadedblob_blob_id
indexType: index
operation: createIndex
table: uploadedblob
- columns:
- name: expires_at
nullable: false
indexName: uploadedblob_expires_at
indexType: index
operation: createIndex
table: uploadedblob
- columns:
- name: repository_id
nullable: false
indexName: uploadedblob_repository_id
indexType: index
operation: createIndex
table: uploadedblob

36
data/migrations/dba_operator/88e64904d000-databasemigration.yaml Normal file
View File

@ -0,0 +1,36 @@
---
apiVersion: dbaoperator.app-sre.redhat.com/v1alpha1
kind: DatabaseMigration
metadata:
name: 88e64904d000
spec:
migrationContainerSpec:
command:
- /quay-registry/quay-entrypoint.sh
- migrate
- 88e64904d000
image: quay.io/quay/quay
name: 88e64904d000
previous: !!python/tuple
- 3383aad1e992
schemaHints:
- columns:
- name: config_media_type
nullable: true
operation: addColumn
table: manifest
- columns:
- name: layers_compressed_size
nullable: true
operation: addColumn
table: manifest
- columns:
- name: repository_id
nullable: false
- name: config_media_type
nullable: true
indexName: manifest_repository_id_config_media_type
indexType: index
operation: createIndex
table: manifest

57
data/migrations/versions/3383aad1e992_add_uploadedblob_table.py Normal file
View File

@ -0,0 +1,57 @@
"""Add UploadedBlob table
Revision ID: 3383aad1e992
Revises: 04b9d2191450
Create Date: 2020-04-21 11:45:54.837077
"""
# revision identifiers, used by Alembic.
revision = "3383aad1e992"
down_revision = "04b9d2191450"
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
def upgrade(op, tables, tester):
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"uploadedblob",
sa.Column("id", sa.BigInteger(), nullable=False),
sa.Column("repository_id", sa.Integer(), nullable=False),
sa.Column("blob_id", sa.Integer(), nullable=False),
sa.Column("uploaded_at", sa.DateTime(), nullable=False),
sa.Column("expires_at", sa.DateTime(), nullable=False),
sa.ForeignKeyConstraint(
["blob_id"], ["imagestorage.id"], name=op.f("fk_uploadedblob_blob_id_imagestorage")
),
sa.ForeignKeyConstraint(
["repository_id"],
["repository.id"],
name=op.f("fk_uploadedblob_repository_id_repository"),
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_uploadedblob")),
)
op.create_index("uploadedblob_blob_id", "uploadedblob", ["blob_id"], unique=False)
op.create_index("uploadedblob_expires_at", "uploadedblob", ["expires_at"], unique=False)
op.create_index("uploadedblob_repository_id", "uploadedblob", ["repository_id"], unique=False)
# ### end Alembic commands ###
# ### population of test data ### #
tester.populate_table(
"uploadedblob",
[
("repository_id", tester.TestDataType.Foreign("repository")),
("blob_id", tester.TestDataType.Foreign("imagestorage")),
("uploaded_at", tester.TestDataType.DateTime),
("expires_at", tester.TestDataType.DateTime),
],
)
# ### end population of test data ### #
def downgrade(op, tables, tester):
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table("uploadedblob")
# ### end Alembic commands ###

39
data/migrations/versions/88e64904d000_add_new_metadata_columns_to_manifest_.py Normal file
View File

@ -0,0 +1,39 @@
"""Add new metadata columns to Manifest table
Revision ID: 88e64904d000
Revises: 3383aad1e992
Create Date: 2020-04-21 14:00:50.376517
"""
# revision identifiers, used by Alembic.
revision = "88e64904d000"
down_revision = "3383aad1e992"
import sqlalchemy as sa
def upgrade(op, tables, tester):
# ### commands auto generated by Alembic - please adjust! ###
op.add_column("manifest", sa.Column("config_media_type", sa.String(length=255), nullable=True))
op.add_column("manifest", sa.Column("layers_compressed_size", sa.BigInteger(), nullable=True))
op.create_index(
"manifest_repository_id_config_media_type",
"manifest",
["repository_id", "config_media_type"],
unique=False,
)
# ### end Alembic commands ###
# ### population of test data ### #
tester.populate_column("manifest", "config_media_type", tester.TestDataType.String)
tester.populate_column("manifest", "layers_compressed_size", tester.TestDataType.Integer)
# ### end population of test data ### #
def downgrade(op, tables, tester):
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index("manifest_repository_id_config_media_type", table_name="manifest")
op.drop_column("manifest", "layers_compressed_size")
op.drop_column("manifest", "config_media_type")
# ### end Alembic commands ###

84
data/model/blob.py
View File

@ -1,6 +1,6 @@
import logging
from datetime import datetime
from datetime import datetime, timedelta
from uuid import uuid4
from data.model import (
@ -14,11 +14,13 @@ from data.model import (
)
from data.database import (
Repository,
RepositoryState,
Namespace,
ImageStorage,
Image,
ImageStoragePlacement,
BlobUpload,
UploadedBlob,
ImageStorageLocation,
db_random_func,
)
@ -27,53 +29,6 @@ from data.database import (
logger = logging.getLogger(__name__)
def get_repository_blob_by_digest(repository, blob_digest):
"""
Find the content-addressable blob linked to the specified repository.
"""
assert blob_digest
try:
storage = (
ImageStorage.select(ImageStorage.uuid)
.join(Image)
.where(
Image.repository == repository,
ImageStorage.content_checksum == blob_digest,
ImageStorage.uploading == False,
)
.get()
)
return storage_model.get_storage_by_uuid(storage.uuid)
except (ImageStorage.DoesNotExist, InvalidImageException):
raise BlobDoesNotExist("Blob does not exist with digest: {0}".format(blob_digest))
def get_repo_blob_by_digest(namespace, repo_name, blob_digest):
"""
Find the content-addressable blob linked to the specified repository.
"""
assert blob_digest
try:
storage = (
ImageStorage.select(ImageStorage.uuid)
.join(Image)
.join(Repository)
.join(Namespace, on=(Namespace.id == Repository.namespace_user))
.where(
Repository.name == repo_name,
Namespace.username == namespace,
ImageStorage.content_checksum == blob_digest,
ImageStorage.uploading == False,
)
.get()
)
return storage_model.get_storage_by_uuid(storage.uuid)
except (ImageStorage.DoesNotExist, InvalidImageException):
raise BlobDoesNotExist("Blob does not exist with digest: {0}".format(blob_digest))
def store_blob_record_and_temp_link(
namespace,
repo_name,
@ -157,16 +112,26 @@ def temp_link_blob(repository_id, blob_digest, link_expiration_s):
def _temp_link_blob(repository_id, storage, link_expiration_s):
""" Note: Should *always* be called by a parent under a transaction. """
random_image_name = str(uuid4())
try:
repository = Repository.get(id=repository_id)
except Repository.DoesNotExist:
return None
# Create a temporary link into the repository, to be replaced by the v1 metadata later
# and create a temporary tag to reference it
image = Image.create(
storage=storage, docker_image_id=random_image_name, repository=repository_id
if repository.state == RepositoryState.MARKED_FOR_DELETION:
return None
return UploadedBlob.create(
repository=repository_id,
blob=storage,
expires_at=datetime.utcnow() + timedelta(seconds=link_expiration_s),
)
def lookup_expired_uploaded_blobs(repository):
""" Looks up all expired uploaded blobs in a repository. """
return UploadedBlob.select().where(
UploadedBlob.repository == repository, UploadedBlob.expires_at <= datetime.utcnow()
)
temp_tag = tag.create_temporary_hidden_tag(repository_id, image, link_expiration_s)
if temp_tag is None:
image.delete_instance()
def get_stale_blob_upload(stale_timespan):
@ -192,7 +157,12 @@ def get_blob_upload_by_uuid(upload_uuid):
Loads the upload with the given UUID, if any.
"""
try:
return BlobUpload.select().where(BlobUpload.uuid == upload_uuid).get()
return (
BlobUpload.select(BlobUpload, ImageStorageLocation)
.join(ImageStorageLocation)
.where(BlobUpload.uuid == upload_uuid)
.get()
)
except BlobUpload.DoesNotExist:
return None

58
data/model/gc.py
View File

@ -1,8 +1,9 @@
import logging
from peewee import fn, IntegrityError
from datetime import datetime
from data.model import config, db_transaction, storage, _basequery, tag as pre_oci_tag
from data.model import config, db_transaction, storage, _basequery, tag as pre_oci_tag, blob
from data.model.oci import tag as oci_tag
from data.database import Repository, db_for_update
from data.database import ApprTag
@ -28,8 +29,14 @@ from data.database import (
RepoMirrorConfig,
RepositoryPermission,
RepositoryAuthorizedEmail,
UploadedBlob,
)
from data.database import (
RepositoryTag,
TagManifest,
Image,
DerivedStorageForImage,
)
from data.database import RepositoryTag, TagManifest, Image, DerivedStorageForImage
from data.database import TagManifestToManifest, TagToRepositoryTag, TagManifestLabelMap
logger = logging.getLogger(__name__)
@ -98,6 +105,7 @@ def purge_repository(repo, force=False):
assert RepositoryTag.select().where(RepositoryTag.repository == repo).count() == 0
assert Manifest.select().where(Manifest.repository == repo).count() == 0
assert ManifestBlob.select().where(ManifestBlob.repository == repo).count() == 0
assert UploadedBlob.select().where(UploadedBlob.repository == repo).count() == 0
assert (
ManifestSecurityStatus.select().where(ManifestSecurityStatus.repository == repo).count()
== 0
@ -194,7 +202,27 @@ def _purge_repository_contents(repo):
if not found:
break
# TODO: remove this once we're fully on the OCI data model.
# Purge any uploaded blobs that have expired.
while True:
found = False
for uploaded_blobs in _chunk_iterate_for_deletion(
UploadedBlob.select().where(UploadedBlob.repository == repo)
):
logger.debug(
"Found %s uploaded blobs to GC under repository %s", len(uploaded_blobs), repo
)
found = True
context = _GarbageCollectorContext(repo)
for uploaded_blob in uploaded_blobs:
logger.debug("Deleting uploaded blob %s under repository %s", uploaded_blob, repo)
assert uploaded_blob.repository_id == repo.id
_purge_uploaded_blob(uploaded_blob, context, allow_non_expired=True)
if not found:
break
# TODO: remove this once we've removed the foreign key constraints from RepositoryTag
# and Image.
while True:
found = False
repo_tag_query = RepositoryTag.select().where(RepositoryTag.repository == repo)
@ -217,6 +245,7 @@ def _purge_repository_contents(repo):
assert RepositoryTag.select().where(RepositoryTag.repository == repo).count() == 0
assert Manifest.select().where(Manifest.repository == repo).count() == 0
assert ManifestBlob.select().where(ManifestBlob.repository == repo).count() == 0
assert UploadedBlob.select().where(UploadedBlob.repository == repo).count() == 0
# Add all remaining images to a new context. We do this here to minimize the number of images
# we need to load.
@ -259,6 +288,7 @@ def garbage_collect_repo(repo):
_run_garbage_collection(context)
had_changes = True
# TODO: Remove once we've removed the foreign key constraints from RepositoryTag and Image.
for tags in _chunk_iterate_for_deletion(pre_oci_tag.lookup_unrecoverable_tags(repo)):
logger.debug("Found %s tags to GC under repository %s", len(tags), repo)
context = _GarbageCollectorContext(repo)
@ -271,6 +301,18 @@ def garbage_collect_repo(repo):
_run_garbage_collection(context)
had_changes = True
# Purge expired uploaded blobs.
for uploaded_blobs in _chunk_iterate_for_deletion(blob.lookup_expired_uploaded_blobs(repo)):
logger.debug("Found %s uploaded blobs to GC under repository %s", len(uploaded_blobs), repo)
context = _GarbageCollectorContext(repo)
for uploaded_blob in uploaded_blobs:
logger.debug("Deleting uploaded blob %s under repository %s", uploaded_blob, repo)
assert uploaded_blob.repository_id == repo.id
_purge_uploaded_blob(uploaded_blob, context)
_run_garbage_collection(context)
had_changes = True
return had_changes
@ -376,6 +418,16 @@ def _purge_pre_oci_tag(tag, context, allow_non_expired=False):
reloaded_tag.delete_instance()
def _purge_uploaded_blob(uploaded_blob, context, allow_non_expired=False):
assert allow_non_expired or uploaded_blob.expires_at <= datetime.utcnow()
# Add the storage to be checked.
context.add_blob_id(uploaded_blob.blob_id)
# Delete the uploaded blob.
uploaded_blob.delete_instance()
def _check_manifest_used(manifest_id):
assert manifest_id is not None

62
data/model/image.py
View File

@ -23,13 +23,10 @@ from data.database import (
ImageStorage,
ImageStorageLocation,
RepositoryPermission,
DerivedStorageForImage,
ImageStorageTransformation,
User,
)
from util.canonicaljson import canonicalize
logger = logging.getLogger(__name__)
@ -554,62 +551,3 @@ def set_secscan_status(image, indexed, version):
.where((Image.security_indexed_engine != version) | (Image.security_indexed != indexed))
.execute()
) != 0
def _get_uniqueness_hash(varying_metadata):
if not varying_metadata:
return None
return hashlib.sha256(json.dumps(canonicalize(varying_metadata)).encode("utf-8")).hexdigest()
def find_or_create_derived_storage(
source_image, transformation_name, preferred_location, varying_metadata=None
):
existing = find_derived_storage_for_image(source_image, transformation_name, varying_metadata)
if existing is not None:
return existing
uniqueness_hash = _get_uniqueness_hash(varying_metadata)
trans = ImageStorageTransformation.get(name=transformation_name)
new_storage = storage.create_v1_storage(preferred_location)
try:
derived = DerivedStorageForImage.create(
source_image=source_image,
derivative=new_storage,
transformation=trans,
uniqueness_hash=uniqueness_hash,
)
except IntegrityError:
# Storage was created while this method executed. Just return the existing.
ImageStoragePlacement.delete().where(ImageStoragePlacement.storage == new_storage).execute()
new_storage.delete_instance()
return find_derived_storage_for_image(source_image, transformation_name, varying_metadata)
return derived
def find_derived_storage_for_image(source_image, transformation_name, varying_metadata=None):
uniqueness_hash = _get_uniqueness_hash(varying_metadata)
try:
found = (
DerivedStorageForImage.select(ImageStorage, DerivedStorageForImage)
.join(ImageStorage)
.switch(DerivedStorageForImage)
.join(ImageStorageTransformation)
.where(
DerivedStorageForImage.source_image == source_image,
ImageStorageTransformation.name == transformation_name,
DerivedStorageForImage.uniqueness_hash == uniqueness_hash,
)
.get()
)
return found
except DerivedStorageForImage.DoesNotExist:
return None
def delete_derived_storage(derived_storage):
derived_storage.derivative.delete_instance(recursive=True)

9
data/model/oauth.py
View File

@ -352,8 +352,13 @@ def lookup_access_token_by_uuid(token_uuid):
def lookup_access_token_for_user(user_obj, token_uuid):
try:
return OAuthAccessToken.get(
OAuthAccessToken.authorized_user == user_obj, OAuthAccessToken.uuid == token_uuid
return (
OAuthAccessToken.select(OAuthAccessToken, User)
.join(User)
.where(
OAuthAccessToken.authorized_user == user_obj, OAuthAccessToken.uuid == token_uuid
)
.get()
)
except OAuthAccessToken.DoesNotExist:
return None

42
data/model/oci/blob.py
View File

@ -1,7 +1,6 @@
from data.database import ImageStorage, ManifestBlob
from data.database import ImageStorage, ManifestBlob, UploadedBlob
from data.model import BlobDoesNotExist
from data.model.storage import get_storage_by_uuid, InvalidImageException
from data.model.blob import get_repository_blob_by_digest as legacy_get
def get_repository_blob_by_digest(repository, blob_digest):
@ -9,8 +8,34 @@ def get_repository_blob_by_digest(repository, blob_digest):
Find the content-addressable blob linked to the specified repository and returns it or None if
none.
"""
# First try looking for a recently uploaded blob. If none found that is matching,
# check the repository itself.
storage = _lookup_blob_uploaded(repository, blob_digest)
if storage is None:
storage = _lookup_blob_in_repository(repository, blob_digest)
return get_storage_by_uuid(storage.uuid) if storage is not None else None
def _lookup_blob_uploaded(repository, blob_digest):
try:
storage = (
return (
ImageStorage.select(ImageStorage.uuid)
.join(UploadedBlob)
.where(
UploadedBlob.repository == repository,
ImageStorage.content_checksum == blob_digest,
ImageStorage.uploading == False,
)
.get()
)
except ImageStorage.DoesNotExist:
return None
def _lookup_blob_in_repository(repository, blob_digest):
try:
return (
ImageStorage.select(ImageStorage.uuid)
.join(ManifestBlob)
.where(
@ -20,12 +45,5 @@ def get_repository_blob_by_digest(repository, blob_digest):
)
.get()
)
return get_storage_by_uuid(storage.uuid)
except (ImageStorage.DoesNotExist, InvalidImageException):
# TODO: Remove once we are no longer using the legacy tables.
# Try the legacy call.
try:
return legacy_get(repository, blob_digest)
except BlobDoesNotExist:
return None
except ImageStorage.DoesNotExist:
return None

238
data/model/oci/manifest.py
View File

@ -1,4 +1,6 @@
import json
import logging
import os
from collections import namedtuple
@ -10,6 +12,10 @@ from data.database import (
ManifestBlob,
ManifestLegacyImage,
ManifestChild,
ImageStorage,
ImageStoragePlacement,
ImageStorageTransformation,
ImageStorageSignature,
db_transaction,
)
from data.model import BlobDoesNotExist
@ -17,11 +23,12 @@ from data.model.blob import get_or_create_shared_blob, get_shared_blob
from data.model.oci.tag import filter_to_alive_tags, create_temporary_tag_if_necessary
from data.model.oci.label import create_manifest_label
from data.model.oci.retriever import RepositoryContentRetriever
from data.model.storage import lookup_repo_storages_by_content_checksum
from data.model.storage import lookup_repo_storages_by_content_checksum, create_v1_storage
from data.model.image import lookup_repository_images, get_image, synthesize_v1_image
from image.docker.schema2 import EMPTY_LAYER_BLOB_DIGEST, EMPTY_LAYER_BYTES
from image.docker.schema1 import ManifestException
from image.docker.schema2.list import MalformedSchema2ManifestList
from util.canonicaljson import canonicalize
from util.validation import is_json
@ -206,91 +213,17 @@ def _create_manifest(
child_manifest_rows[child_manifest_info.manifest.digest] = child_manifest_info.manifest
child_manifest_label_dicts.append(labels)
# Ensure all the blobs in the manifest exist.
digests = set(manifest_interface_instance.local_blob_digests)
blob_map = {}
# If the special empty layer is required, simply load it directly. This is much faster
# than trying to load it on a per repository basis, and that is unnecessary anyway since
# this layer is predefined.
if EMPTY_LAYER_BLOB_DIGEST in digests:
digests.remove(EMPTY_LAYER_BLOB_DIGEST)
blob_map[EMPTY_LAYER_BLOB_DIGEST] = get_shared_blob(EMPTY_LAYER_BLOB_DIGEST)
if not blob_map[EMPTY_LAYER_BLOB_DIGEST]:
if raise_on_error:
raise CreateManifestException("Unable to retrieve specialized empty blob")
logger.warning("Could not find the special empty blob in storage")
return None
if digests:
query = lookup_repo_storages_by_content_checksum(repository_id, digests)
blob_map.update({s.content_checksum: s for s in query})
for digest_str in digests:
if digest_str not in blob_map:
logger.warning(
"Unknown blob `%s` under manifest `%s` for repository `%s`",
digest_str,
manifest_interface_instance.digest,
repository_id,
)
if raise_on_error:
raise CreateManifestException("Unknown blob `%s`" % digest_str)
return None
# Special check: If the empty layer blob is needed for this manifest, add it to the
# blob map. This is necessary because Docker decided to elide sending of this special
# empty layer in schema version 2, but we need to have it referenced for GC and schema version 1.
if EMPTY_LAYER_BLOB_DIGEST not in blob_map:
try:
requires_empty_layer = manifest_interface_instance.get_requires_empty_layer_blob(
retriever
)
except ManifestException as ex:
if raise_on_error:
raise CreateManifestException(str(ex))
return None
if requires_empty_layer is None:
if raise_on_error:
raise CreateManifestException("Could not load configuration blob")
return None
if requires_empty_layer:
shared_blob = get_or_create_shared_blob(
EMPTY_LAYER_BLOB_DIGEST, EMPTY_LAYER_BYTES, storage
)
assert not shared_blob.uploading
assert shared_blob.content_checksum == EMPTY_LAYER_BLOB_DIGEST
blob_map[EMPTY_LAYER_BLOB_DIGEST] = shared_blob
# Determine and populate the legacy image if necessary. Manifest lists will not have a legacy
# image.
legacy_image = None
if manifest_interface_instance.has_legacy_image:
try:
legacy_image_id = _populate_legacy_image(
repository_id, manifest_interface_instance, blob_map, retriever, raise_on_error
)
except ManifestException as me:
logger.error("Got manifest error when populating legacy images: %s", me)
if raise_on_error:
raise CreateManifestException(
"Attempt to create an invalid manifest: %s. Please report this issue." % me
)
return None
if legacy_image_id is None:
return None
legacy_image = get_image(repository_id, legacy_image_id)
if legacy_image is None:
return None
# Build the map from required blob digests to the blob objects.
blob_map = _build_blob_map(
repository_id,
manifest_interface_instance,
retriever,
storage,
raise_on_error,
require_empty_layer=False,
)
if blob_map is None:
return None
# Create the manifest and its blobs.
media_type = Manifest.media_type.get_id(manifest_interface_instance.media_type)
@ -314,6 +247,8 @@ def _create_manifest(
digest=manifest_interface_instance.digest,
media_type=media_type,
manifest_bytes=manifest_interface_instance.bytes.as_encoded_str(),
config_media_type=manifest_interface_instance.config_media_type,
layers_compressed_size=manifest_interface_instance.layers_compressed_size,
)
except IntegrityError as ie:
try:
@ -339,12 +274,6 @@ def _create_manifest(
if blobs_to_insert:
ManifestBlob.insert_many(blobs_to_insert).execute()
# Set the legacy image (if applicable).
if legacy_image is not None:
ManifestLegacyImage.create(
repository=repository_id, image=legacy_image, manifest=manifest
)
# Insert the manifest child rows (if applicable).
if child_manifest_rows:
children_to_insert = [
@ -392,6 +321,131 @@ def _create_manifest(
return CreatedManifest(manifest=manifest, newly_created=True, labels_to_apply=labels_to_apply)
def _build_blob_map(
repository_id,
manifest_interface_instance,
retriever,
storage,
raise_on_error=False,
require_empty_layer=True,
):
""" Builds a map containing the digest of each blob referenced by the given manifest,
to its associated Blob row in the database. This method also verifies that the blob
is accessible under the given repository. Returns None on error (unless raise_on_error
is specified). If require_empty_layer is set to True, the method will check if the manifest
references the special shared empty layer blob and, if so, add it to the map. Otherwise,
the empty layer blob is only returned if it was *explicitly* referenced in the manifest.
This is necessary because Docker V2_2/OCI manifests can implicitly reference an empty blob
layer for image layers that only change metadata.
"""
# Ensure all the blobs in the manifest exist.
digests = set(manifest_interface_instance.local_blob_digests)
blob_map = {}
# If the special empty layer is required, simply load it directly. This is much faster
# than trying to load it on a per repository basis, and that is unnecessary anyway since
# this layer is predefined.
if EMPTY_LAYER_BLOB_DIGEST in digests:
digests.remove(EMPTY_LAYER_BLOB_DIGEST)
blob_map[EMPTY_LAYER_BLOB_DIGEST] = get_shared_blob(EMPTY_LAYER_BLOB_DIGEST)
if not blob_map[EMPTY_LAYER_BLOB_DIGEST]:
if raise_on_error:
raise CreateManifestException("Unable to retrieve specialized empty blob")
logger.warning("Could not find the special empty blob in storage")
return None
if digests:
query = lookup_repo_storages_by_content_checksum(repository_id, digests, with_uploads=True)
blob_map.update({s.content_checksum: s for s in query})
for digest_str in digests:
if digest_str not in blob_map:
logger.warning(
"Unknown blob `%s` under manifest `%s` for repository `%s`",
digest_str,
manifest_interface_instance.digest,
repository_id,
)
if raise_on_error:
raise CreateManifestException("Unknown blob `%s`" % digest_str)
return None
# Special check: If the empty layer blob is needed for this manifest, add it to the
# blob map. This is necessary because Docker decided to elide sending of this special
# empty layer in schema version 2, but we need to have it referenced for schema version 1.
if require_empty_layer and EMPTY_LAYER_BLOB_DIGEST not in blob_map:
try:
requires_empty_layer = manifest_interface_instance.get_requires_empty_layer_blob(
retriever
)
except ManifestException as ex:
if raise_on_error:
raise CreateManifestException(str(ex))
return None
if requires_empty_layer is None:
if raise_on_error:
raise CreateManifestException("Could not load configuration blob")
return None
if requires_empty_layer:
shared_blob = get_or_create_shared_blob(
EMPTY_LAYER_BLOB_DIGEST, EMPTY_LAYER_BYTES, storage
)
assert not shared_blob.uploading
assert shared_blob.content_checksum == EMPTY_LAYER_BLOB_DIGEST
blob_map[EMPTY_LAYER_BLOB_DIGEST] = shared_blob
return blob_map
def populate_legacy_images_for_testing(manifest, manifest_interface_instance, storage):
""" Populates the legacy image rows for the given manifest. """
# NOTE: This method is only kept around for use by legacy tests that still require
# legacy images. As a result, we make sure we're in testing mode before we run.
assert os.getenv("TEST") == "true"
repository_id = manifest.repository_id
retriever = RepositoryContentRetriever.for_repository(repository_id, storage)
blob_map = _build_blob_map(
repository_id, manifest_interface_instance, storage, True, require_empty_layer=True
)
if blob_map is None:
return None
# Determine and populate the legacy image if necessary. Manifest lists will not have a legacy
# image.
legacy_image = None
if manifest_interface_instance.has_legacy_image:
try:
legacy_image_id = _populate_legacy_image(
repository_id, manifest_interface_instance, blob_map, retriever, True
)
except ManifestException as me:
raise CreateManifestException(
"Attempt to create an invalid manifest: %s. Please report this issue." % me
)
if legacy_image_id is None:
return None
legacy_image = get_image(repository_id, legacy_image_id)
if legacy_image is None:
return None
# Set the legacy image (if applicable).
if legacy_image is not None:
ManifestLegacyImage.create(
repository=repository_id, image=legacy_image, manifest=manifest
)
def _populate_legacy_image(
repository_id, manifest_interface_instance, blob_map, retriever, raise_on_error=False
):

117
data/model/oci/tag.py
View File

@ -123,7 +123,14 @@ def list_repository_tag_history(
Note that the returned Manifest will not contain the manifest contents.
"""
query = (
Tag.select(Tag, Manifest.id, Manifest.digest, Manifest.media_type)
Tag.select(
Tag,
Manifest.id,
Manifest.digest,
Manifest.media_type,
Manifest.layers_compressed_size,
Manifest.config_media_type,
)
.join(Manifest)
.where(Tag.repository == repository_id)
.order_by(Tag.lifetime_start_ms.desc(), Tag.name)
@ -141,31 +148,14 @@ def list_repository_tag_history(
if active_tags_only:
query = filter_to_alive_tags(query)
else:
query = filter_to_visible_tags(query)
query = filter_to_visible_tags(query)
results = list(query)
return results[0:page_size], len(results) > page_size
def get_legacy_images_for_tags(tags):
"""
Returns a map from tag ID to the legacy image for the tag.
"""
if not tags:
return {}
query = (
ManifestLegacyImage.select(ManifestLegacyImage, Image, ImageStorage)
.join(Image)
.join(ImageStorage)
.where(ManifestLegacyImage.manifest << [tag.manifest_id for tag in tags])
)
by_manifest = {mli.manifest_id: mli.image for mli in query}
return {tag.id: by_manifest[tag.manifest_id] for tag in tags if tag.manifest_id in by_manifest}
def find_matching_tag(repository_id, tag_names, tag_kinds=None):
"""
Finds an alive tag in the specified repository with one of the specified tag names and returns
@ -417,7 +407,6 @@ def delete_tags_for_manifest(manifest):
"""
query = Tag.select().where(Tag.manifest == manifest)
query = filter_to_alive_tags(query)
query = filter_to_visible_tags(query)
tags = list(query)
now_ms = get_epoch_timestamp_ms()
@ -446,9 +435,8 @@ def filter_to_alive_tags(query, now_ms=None, model=Tag):
if now_ms is None:
now_ms = get_epoch_timestamp_ms()
return query.where((model.lifetime_end_ms >> None) | (model.lifetime_end_ms > now_ms)).where(
model.hidden == False
)
query = query.where((model.lifetime_end_ms >> None) | (model.lifetime_end_ms > now_ms))
return filter_to_visible_tags(query)
def set_tag_expiration_sec_for_manifest(manifest_id, expiration_seconds):
@ -578,70 +566,6 @@ def tags_containing_legacy_image(image):
return filter_to_alive_tags(tags)
def lookup_notifiable_tags_for_legacy_image(docker_image_id, storage_uuid, event_name):
"""
Yields any alive Tags found in repositories with an event with the given name registered and
whose legacy Image has the given docker image ID and storage UUID.
"""
event = ExternalNotificationEvent.get(name=event_name)
images = (
Image.select()
.join(ImageStorage)
.where(Image.docker_image_id == docker_image_id, ImageStorage.uuid == storage_uuid)
)
for image in list(images):
# Ensure the image is under a repository that supports the event.
try:
RepositoryNotification.get(repository=image.repository_id, event=event)
except RepositoryNotification.DoesNotExist:
continue
# If found in a repository with the valid event, yield the tag(s) that contains the image.
for tag in tags_containing_legacy_image(image):
yield tag
def get_tags_for_legacy_image(image_id):
""" Returns the Tag's that have the associated legacy image.
NOTE: This is for legacy support in the old security notification worker and should
be removed once that code is no longer necessary.
"""
return filter_to_alive_tags(
Tag.select()
.distinct()
.join(Manifest)
.join(ManifestLegacyImage)
.where(ManifestLegacyImage.image == image_id)
)
def _filter_has_repository_event(query, event):
""" Filters the query by ensuring the repositories returned have the given event.
NOTE: This is for legacy support in the old security notification worker and should
be removed once that code is no longer necessary.
"""
return (
query.join(Repository)
.join(RepositoryNotification)
.where(RepositoryNotification.event == event)
)
def filter_tags_have_repository_event(query, event):
""" Filters the query by ensuring the tags live in a repository that has the given
event. Also orders the results by lifetime_start_ms.
NOTE: This is for legacy support in the old security notification worker and should
be removed once that code is no longer necessary.
"""
query = _filter_has_repository_event(query, event)
query = query.switch(Tag).order_by(Tag.lifetime_start_ms.desc())
return query
def find_repository_with_garbage(limit_to_gc_policy_s):
""" Returns a repository that has garbage (defined as an expired Tag that is past
the repo's namespace's expiration window) or None if none.
@ -680,3 +604,20 @@ def find_repository_with_garbage(limit_to_gc_policy_s):
return None
except Repository.DoesNotExist:
return None
def get_legacy_images_for_tags(tags):
"""
Returns a map from tag ID to the legacy image for the tag.
"""
if not tags:
return {}
query = (
ManifestLegacyImage.select(ManifestLegacyImage, Image)
.join(Image)
.where(ManifestLegacyImage.manifest << [tag.manifest_id for tag in tags])
)
by_manifest = {mli.manifest_id: mli.image for mli in query}
return {tag.id: by_manifest[tag.manifest_id] for tag in tags if tag.manifest_id in by_manifest}

19
data/model/oci/test/test_oci_manifest.py
View File

@ -166,6 +166,8 @@ def test_get_or_create_manifest(schema_version, initialized_db):
builder.add_layer(random_digest, len(random_data.encode("utf-8")))
sample_manifest_instance = builder.build()
assert sample_manifest_instance.layers_compressed_size is not None
# Create a new manifest.
created_manifest = get_or_create_manifest(repository, sample_manifest_instance, storage)
created = created_manifest.manifest
@ -177,15 +179,18 @@ def test_get_or_create_manifest(schema_version, initialized_db):
assert created.digest == sample_manifest_instance.digest
assert created.manifest_bytes == sample_manifest_instance.bytes.as_encoded_str()
assert created_manifest.labels_to_apply == expected_labels
assert created.config_media_type == sample_manifest_instance.config_media_type
assert created.layers_compressed_size == sample_manifest_instance.layers_compressed_size
# Lookup the manifest and verify.
found = lookup_manifest(repository, created.digest, allow_dead=True)
assert found.digest == created.digest
assert found.config_media_type == created.config_media_type
assert found.layers_compressed_size == created.layers_compressed_size
# Verify it has a temporary tag pointing to it.
assert Tag.get(manifest=created, hidden=True).lifetime_end_ms
# Verify the legacy image.
legacy_image = get_legacy_image_for_manifest(created)
assert legacy_image is not None
assert legacy_image.storage.content_checksum == random_digest
# Verify the linked blobs.
blob_digests = [
mb.blob.content_checksum
@ -295,6 +300,8 @@ def test_get_or_create_manifest_list(initialized_db):
assert created_list
assert created_list.media_type.name == manifest_list.media_type
assert created_list.digest == manifest_list.digest
assert created_list.config_media_type == manifest_list.config_media_type
assert created_list.layers_compressed_size == manifest_list.layers_compressed_size
# Ensure the child manifest links exist.
child_manifests = {
@ -423,6 +430,8 @@ def test_get_or_create_manifest_with_remote_layers(initialized_db):
assert created_manifest
assert created_manifest.media_type.name == manifest.media_type
assert created_manifest.digest == manifest.digest
assert created_manifest.config_media_type == manifest.config_media_type
assert created_manifest.layers_compressed_size == manifest.layers_compressed_size
# Verify the legacy image.
legacy_image = get_legacy_image_for_manifest(created_manifest)

13
data/model/oci/test/test_oci_tag.py
View File

@ -18,7 +18,6 @@ from data.model.oci.tag import (
get_most_recent_tag,
get_most_recent_tag_lifetime_start,
list_alive_tags,
get_legacy_images_for_tags,
filter_to_alive_tags,
filter_to_visible_tags,
list_repository_tag_history,
@ -92,13 +91,6 @@ def test_list_alive_tags(initialized_db):
for tag in filter_to_visible_tags(filter_to_alive_tags(Tag.select())):
tags = list_alive_tags(tag.repository)
assert tag in tags
with assert_query_count(1):
legacy_images = get_legacy_images_for_tags(tags)
for tag in tags:
assert ManifestLegacyImage.get(manifest=tag.manifest).image == legacy_images[tag.id]
found = True
assert found
@ -154,6 +146,11 @@ def test_list_repository_tag_history(namespace_name, repo_name, initialized_db):
assert results
assert not has_more
assert results[0].manifest.id is not None
assert results[0].manifest.digest is not None
assert results[0].manifest.media_type is not None
assert results[0].manifest.layers_compressed_size is not None
def test_list_repository_tag_history_with_history(initialized_db):
repo = get_repository("devtable", "history")

12
data/model/repo_mirror.py
View File

@ -2,7 +2,7 @@ import re
from datetime import datetime, timedelta
from peewee import IntegrityError, fn
from peewee import IntegrityError, fn, JOIN
from jsonschema import ValidationError
from data.database import (
@ -14,6 +14,7 @@ from data.database import (
Repository,
uuid_generator,
db_transaction,
User,
)
from data.fields import DecryptedValue
from data.model import DataModelException
@ -362,7 +363,14 @@ def get_mirror(repository):
Return the RepoMirrorConfig associated with the given Repository, or None if it doesn't exist.
"""
try:
return RepoMirrorConfig.get(repository=repository)
return (
RepoMirrorConfig.select(RepoMirrorConfig, User, RepoMirrorRule)
.join(User, JOIN.LEFT_OUTER)
.switch(RepoMirrorConfig)
.join(RepoMirrorRule)
.where(RepoMirrorConfig.repository == repository)
.get()
)
except RepoMirrorConfig.DoesNotExist:
return None

5
data/model/repository.py
View File

@ -32,7 +32,6 @@ from data.database import (
RepositoryActionCount,
Role,
RepositoryAuthorizedEmail,
DerivedStorageForImage,
Label,
db_for_update,
get_epoch_timestamp,
@ -500,6 +499,10 @@ def lookup_repository(repo_id):
return None
def repository_visibility_name(repository):
return "public" if is_repository_public(repository) else "private"
def is_repository_public(repository):
return repository.visibility_id == _basequery.get_public_repo_visibility().id

87
data/model/storage.py
View File

@ -25,6 +25,7 @@ from data.database import (
ApprBlob,
ensure_under_transaction,
ManifestBlob,
UploadedBlob,
)
logger = logging.getLogger(__name__)
@ -86,7 +87,13 @@ def _is_storage_orphaned(candidate_id):
except Image.DoesNotExist:
pass
return True
try:
UploadedBlob.get(blob=candidate_id)
return False
except UploadedBlob.DoesNotExist:
pass
return True
def garbage_collect_storage(storage_id_whitelist):
@ -307,57 +314,65 @@ def get_layer_path_for_storage(storage_uuid, cas_path, content_checksum):
return store.blob_path(content_checksum)
def lookup_repo_storages_by_content_checksum(repo, checksums, by_manifest=False):
def lookup_repo_storages_by_content_checksum(repo, checksums, with_uploads=False):
"""
Looks up repository storages (without placements) matching the given repository and checksum.
"""
checksums = list(set(checksums))
if not checksums:
return []
# If the request is not with uploads, simply return the blobs found under the manifests
# for the repository.
if not with_uploads:
return _lookup_repo_storages_by_content_checksum(repo, checksums, ManifestBlob)
# Otherwise, first check the UploadedBlob table and, once done, then check the ManifestBlob
# table.
found_via_uploaded = list(
_lookup_repo_storages_by_content_checksum(repo, checksums, UploadedBlob)
)
if len(found_via_uploaded) == len(checksums):
return found_via_uploaded
checksums_remaining = set(checksums) - {
uploaded.content_checksum for uploaded in found_via_uploaded
}
found_via_manifest = list(
_lookup_repo_storages_by_content_checksum(repo, checksums_remaining, ManifestBlob)
)
return found_via_uploaded + found_via_manifest
def _lookup_repo_storages_by_content_checksum(repo, checksums, model_class):
assert checksums
# There may be many duplicates of the checksums, so for performance reasons we are going
# to use a union to select just one storage with each checksum
queries = []
for counter, checksum in enumerate(set(checksums)):
for counter, checksum in enumerate(checksums):
query_alias = "q{0}".format(counter)
# TODO: Remove once we have a new-style model for tracking temp uploaded blobs and
# all legacy tables have been removed.
if by_manifest:
candidate_subq = (
ImageStorage.select(
ImageStorage.id,
ImageStorage.content_checksum,
ImageStorage.image_size,
ImageStorage.uuid,
ImageStorage.cas_path,
ImageStorage.uncompressed_size,
ImageStorage.uploading,
)
.join(ManifestBlob)
.where(ManifestBlob.repository == repo, ImageStorage.content_checksum == checksum)
.limit(1)
.alias(query_alias)
)
else:
candidate_subq = (
ImageStorage.select(
ImageStorage.id,
ImageStorage.content_checksum,
ImageStorage.image_size,
ImageStorage.uuid,
ImageStorage.cas_path,
ImageStorage.uncompressed_size,
ImageStorage.uploading,
)
.join(Image)
.where(Image.repository == repo, ImageStorage.content_checksum == checksum)
.limit(1)
.alias(query_alias)
candidate_subq = (
ImageStorage.select(
ImageStorage.id,
ImageStorage.content_checksum,
ImageStorage.image_size,
ImageStorage.uuid,
ImageStorage.cas_path,
ImageStorage.uncompressed_size,
ImageStorage.uploading,
)
.join(model_class)
.where(model_class.repository == repo, ImageStorage.content_checksum == checksum)
.limit(1)
.alias(query_alias)
)
queries.append(ImageStorage.select(SQL("*")).from_(candidate_subq))
assert queries
return _basequery.reduce_as_tree(queries)

66
data/model/tag.py
View File

@ -1,75 +1,9 @@
import logging
from calendar import timegm
from datetime import datetime
from uuid import uuid4
from peewee import IntegrityError, JOIN, fn
from data.model import (
image,
storage,
db_transaction,
DataModelException,
_basequery,
InvalidManifestException,
TagAlreadyCreatedException,
StaleTagException,
config,
)
from data.database import (
RepositoryTag,
Repository,
RepositoryState,
Image,
ImageStorage,
Namespace,
TagManifest,
RepositoryNotification,
Label,
TagManifestLabel,
get_epoch_timestamp,
db_for_update,
Manifest,
ManifestLabel,
ManifestBlob,
ManifestLegacyImage,
TagManifestToManifest,
TagManifestLabelMap,
TagToRepositoryTag,
Tag,
get_epoch_timestamp_ms,
)
from util.timedeltastring import convert_to_timedelta
logger = logging.getLogger(__name__)
def create_temporary_hidden_tag(repo, image_obj, expiration_s):
"""
Create a tag with a defined timeline, that will not appear in the UI or CLI.
Returns the name of the temporary tag or None on error.
"""
now_ts = get_epoch_timestamp()
expire_ts = now_ts + expiration_s
tag_name = str(uuid4())
# Ensure the repository is not marked for deletion.
with db_transaction():
current = Repository.get(id=repo)
if current.state == RepositoryState.MARKED_FOR_DELETION:
return None
RepositoryTag.create(
repository=repo,
image=image_obj,
name=tag_name,
lifetime_start_ts=now_ts,
lifetime_end_ts=expire_ts,
hidden=True,
)
return tag_name
def lookup_unrecoverable_tags(repo):

39
data/model/test/test_gc.py
View File

@ -30,6 +30,7 @@ from data.database import (
TagToRepositoryTag,
ImageStorageLocation,
RepositoryTag,
UploadedBlob,
)
from data.model.oci.test.test_oci_manifest import create_manifest_for_testing
from digest.digest_tools import sha256_digest
@ -61,11 +62,7 @@ def default_tag_policy(initialized_db):
def _delete_temp_links(repo):
""" Deletes any temp links to blobs. """
for hidden in list(
RepositoryTag.select().where(RepositoryTag.hidden == True, RepositoryTag.repository == repo)
):
hidden.delete_instance()
hidden.image.delete_instance()
UploadedBlob.delete().where(UploadedBlob.repository == repo).execute()
def _populate_blob(repo, content):
@ -128,6 +125,10 @@ def move_tag(repository, tag, image_ids, expect_gc=True):
repo_ref, manifest, tag, storage, raise_on_error=True
)
tag_ref = registry_model.get_repo_tag(repo_ref, tag)
manifest_ref = registry_model.get_manifest_for_tag(tag_ref)
registry_model.populate_legacy_images_for_testing(manifest_ref, storage)
if expect_gc:
assert model.gc.garbage_collect_repo(repository) == expect_gc
@ -156,10 +157,17 @@ def _get_dangling_storage_count():
storage_ids = set([current.id for current in ImageStorage.select()])
referenced_by_image = set([image.storage_id for image in Image.select()])
referenced_by_manifest = set([blob.blob_id for blob in ManifestBlob.select()])
referenced_by_derived = set(
referenced_by_uploaded = set([upload.blob_id for upload in UploadedBlob.select()])
referenced_by_derived_image = set(
[derived.derivative_id for derived in DerivedStorageForImage.select()]
)
return len(storage_ids - referenced_by_image - referenced_by_derived - referenced_by_manifest)
return len(
storage_ids
- referenced_by_image
- referenced_by_derived_image
- referenced_by_manifest
- referenced_by_uploaded
)
def _get_dangling_label_count():
@ -199,7 +207,7 @@ def assert_gc_integrity(expect_storage_removed=True):
for blob_row in ApprBlob.select():
existing_digests.add(blob_row.digest)
# Store the number of dangling storages and labels.
# Store the number of dangling objects.
existing_storage_count = _get_dangling_storage_count()
existing_label_count = _get_dangling_label_count()
existing_manifest_count = _get_dangling_manifest_count()
@ -247,6 +255,13 @@ def assert_gc_integrity(expect_storage_removed=True):
.count()
)
if shared == 0:
shared = (
UploadedBlob.select()
.where(UploadedBlob.blob == removed_image_and_storage.storage_id)
.count()
)
if shared == 0:
with pytest.raises(ImageStorage.DoesNotExist):
ImageStorage.get(id=removed_image_and_storage.storage_id)
@ -672,6 +687,10 @@ def test_images_shared_cas(default_tag_policy, initialized_db):
repo_ref, manifest, "first", storage, raise_on_error=True
)
tag_ref = registry_model.get_repo_tag(repo_ref, "first")
manifest_ref = registry_model.get_manifest_for_tag(tag_ref)
registry_model.populate_legacy_images_for_testing(manifest_ref, storage)
# Store another as `second`.
builder = DockerSchema1ManifestBuilder(
repository.namespace_user.username, repository.name, "second"
@ -682,6 +701,10 @@ def test_images_shared_cas(default_tag_policy, initialized_db):
repo_ref, manifest, "second", storage, raise_on_error=True
)
tag_ref = registry_model.get_repo_tag(repo_ref, "second")
manifest_ref = registry_model.get_manifest_for_tag(tag_ref)
registry_model.populate_legacy_images_for_testing(manifest_ref, storage)
# Manually retarget the second manifest's blob to the second row.
try:
second_blob = ManifestBlob.get(manifest=created._db_id, blob=is1)

109
data/model/test/test_image.py
View File

@ -1,109 +0,0 @@
import pytest
from collections import defaultdict
from data.model import image, repository
from playhouse.test_utils import assert_query_count
from test.fixtures import *
@pytest.fixture()
def images(initialized_db):
images = image.get_repository_images("devtable", "simple")
assert len(images)
return images
def test_get_image_with_storage(images, initialized_db):
for current in images:
storage_uuid = current.storage.uuid
with assert_query_count(1):
retrieved = image.get_image_with_storage(current.docker_image_id, storage_uuid)
assert retrieved.id == current.id
assert retrieved.storage.uuid == storage_uuid
def test_get_parent_images(images, initialized_db):
for current in images:
if not len(current.ancestor_id_list()):
continue
with assert_query_count(1):
parent_images = list(image.get_parent_images("devtable", "simple", current))
assert len(parent_images) == len(current.ancestor_id_list())
assert set(current.ancestor_id_list()) == {i.id for i in parent_images}
for parent in parent_images:
with assert_query_count(0):
assert parent.storage.id
def test_get_image(images, initialized_db):
for current in images:
repo = current.repository
with assert_query_count(1):
found = image.get_image(repo, current.docker_image_id)
assert found.id == current.id
def test_placements(images, initialized_db):
with assert_query_count(1):
placements_map = image.get_placements_for_images(images)
for current in images:
assert current.storage.id in placements_map
with assert_query_count(2):
expected_image, expected_placements = image.get_image_and_placements(
"devtable", "simple", current.docker_image_id
)
assert expected_image.id == current.id
assert len(expected_placements) == len(placements_map.get(current.storage.id))
assert {p.id for p in expected_placements} == {
p.id for p in placements_map.get(current.storage.id)
}
def test_get_repo_image(images, initialized_db):
for current in images:
with assert_query_count(1):
found = image.get_repo_image("devtable", "simple", current.docker_image_id)
assert found.id == current.id
with assert_query_count(1):
assert found.storage.id
def test_get_repo_image_and_storage(images, initialized_db):
for current in images:
with assert_query_count(1):
found = image.get_repo_image_and_storage("devtable", "simple", current.docker_image_id)
assert found.id == current.id
with assert_query_count(0):
assert found.storage.id
def test_get_repository_images_without_placements(images, initialized_db):
ancestors_map = defaultdict(list)
for img in images:
current = img.parent
while current is not None:
ancestors_map[current.id].append(img.id)
current = current.parent
for current in images:
repo = current.repository
with assert_query_count(1):
found = list(
image.get_repository_images_without_placements(repo, with_ancestor=current)
)
assert len(found) == len(ancestors_map[current.id]) + 1
assert {i.id for i in found} == set(ancestors_map[current.id] + [current.id])

23
data/model/test/test_tag.py
View File

@ -1,23 +0,0 @@
import pytest
from data.database import (
RepositoryState,
Image,
)
from test.fixtures import *
def test_create_temp_tag(initialized_db):
repo = model.repository.get_repository("devtable", "simple")
image = Image.get(repository=repo)
assert model.tag.create_temporary_hidden_tag(repo, image, 10000000) is not None
def test_create_temp_tag_deleted_repo(initialized_db):
repo = model.repository.get_repository("devtable", "simple")
repo.state = RepositoryState.MARKED_FOR_DELETION
repo.save()
image = Image.get(repository=repo)
assert model.tag.create_temporary_hidden_tag(repo, image, 10000000) is None

6
data/model/user.py
View File

@ -1326,7 +1326,11 @@ def get_region_locations(user):
"""
Returns the locations defined as preferred storage for the given user.
"""
query = UserRegion.select().join(ImageStorageLocation).where(UserRegion.user == user)
query = (
UserRegion.select(UserRegion, ImageStorageLocation)
.join(ImageStorageLocation)
.where(UserRegion.user == user)
)
return set([region.location.name for region in query])

3
data/registry_model/__init__.py
View File

@ -13,6 +13,9 @@ class RegistryModelProxy(object):
def __getattr__(self, attr):
return getattr(self._model, attr)
def set_id_hash_salt(self, hash_salt):
self._model.set_id_hash_salt(hash_salt)
registry_model = RegistryModelProxy()
logger.info("===============================")

385
data/registry_model/datatypes.py
View File

@ -1,4 +1,5 @@
import hashlib
import json
from collections import namedtuple
from enum import Enum, unique
@ -172,8 +173,8 @@ class Label(datatype("Label", ["key", "value", "uuid", "source_type_name", "medi
key=label.key,
value=label.value,
uuid=label.uuid,
media_type_name=label.media_type.name,
source_type_name=label.source_type.name,
media_type_name=model.label.get_media_types()[label.media_type_id],
source_type_name=model.label.get_label_source_types()[label.source_type_id],
)
@ -189,13 +190,6 @@ class ShallowTag(datatype("ShallowTag", ["name"])):
return ShallowTag(db_id=tag.id, name=tag.name)
@classmethod
def for_repository_tag(cls, repository_tag):
if repository_tag is None:
return None
return ShallowTag(db_id=repository_tag.id, name=repository_tag.name)
@property
def id(self):
"""
@ -223,7 +217,7 @@ class Tag(
"""
@classmethod
def for_tag(cls, tag, legacy_image=None):
def for_tag(cls, tag, legacy_id_handler, manifest_row=None, legacy_image_row=None):
if tag is None:
return None
@ -235,55 +229,34 @@ class Tag(
lifetime_end_ms=tag.lifetime_end_ms,
lifetime_start_ts=tag.lifetime_start_ms // 1000,
lifetime_end_ts=tag.lifetime_end_ms // 1000 if tag.lifetime_end_ms else None,
manifest_digest=tag.manifest.digest,
manifest_digest=manifest_row.digest if manifest_row else tag.manifest.digest,
inputs=dict(
legacy_image=legacy_image,
manifest=tag.manifest,
legacy_id_handler=legacy_id_handler,
legacy_image_row=legacy_image_row,
manifest_row=manifest_row or tag.manifest,
repository=RepositoryReference.for_id(tag.repository_id),
),
)
@classmethod
def for_repository_tag(cls, repository_tag, manifest_digest=None, legacy_image=None):
if repository_tag is None:
return None
return Tag(
db_id=repository_tag.id,
name=repository_tag.name,
reversion=repository_tag.reversion,
lifetime_start_ts=repository_tag.lifetime_start_ts,
lifetime_end_ts=repository_tag.lifetime_end_ts,
lifetime_start_ms=repository_tag.lifetime_start_ts * 1000,
lifetime_end_ms=(
repository_tag.lifetime_end_ts * 1000 if repository_tag.lifetime_end_ts else None
),
manifest_digest=manifest_digest,
inputs=dict(
legacy_image=legacy_image,
repository=RepositoryReference.for_id(repository_tag.repository_id),
),
)
@property
@requiresinput("manifest_row")
def _manifest_row(self, manifest_row):
"""
Returns the database Manifest object for this tag.
"""
return manifest_row
@property
@requiresinput("manifest")
def _manifest(self, manifest):
@requiresinput("manifest_row")
@requiresinput("legacy_id_handler")
@optionalinput("legacy_image_row")
def manifest(self, manifest_row, legacy_id_handler, legacy_image_row):
"""
Returns the manifest for this tag.
Will only apply to new-style OCI tags.
"""
return manifest
@property
@optionalinput("manifest")
def manifest(self, manifest):
"""
Returns the manifest for this tag or None if none.
Will only apply to new-style OCI tags.
"""
return Manifest.for_manifest(manifest, self.legacy_image_if_present)
return Manifest.for_manifest(
manifest_row, legacy_id_handler, legacy_image_row=legacy_image_row
)
@property
@requiresinput("repository")
@ -293,28 +266,6 @@ class Tag(
"""
return repository
@property
@requiresinput("legacy_image")
def legacy_image(self, legacy_image):
"""
Returns the legacy Docker V1-style image for this tag.
Note that this will be None for tags whose manifests point to other manifests instead of
images.
"""
return legacy_image
@property
@optionalinput("legacy_image")
def legacy_image_if_present(self, legacy_image):
"""
Returns the legacy Docker V1-style image for this tag.
Note that this will be None for tags whose manifests point to other manifests instead of
images.
"""
return legacy_image
@property
def id(self):
"""
@ -322,31 +273,32 @@ class Tag(
"""
return self._db_id
@property
def manifest_layers_size(self):
""" Returns the compressed size of the layers of the manifest for the Tag or
None if none applicable or loaded.
"""
return self.manifest.layers_compressed_size
class Manifest(datatype("Manifest", ["digest", "media_type", "internal_manifest_bytes"])):
class Manifest(
datatype(
"Manifest",
[
"digest",
"media_type",
"config_media_type",
"_layers_compressed_size",
"internal_manifest_bytes",
],
)
):
"""
Manifest represents a manifest in a repository.
"""
@classmethod
def for_tag_manifest(cls, tag_manifest, legacy_image=None):
if tag_manifest is None:
return None
return Manifest(
db_id=tag_manifest.id,
digest=tag_manifest.digest,
internal_manifest_bytes=Bytes.for_string_or_unicode(tag_manifest.json_data),
media_type=DOCKER_SCHEMA1_SIGNED_MANIFEST_CONTENT_TYPE, # Always in legacy.
inputs=dict(
legacy_image=legacy_image,
tag_manifest=True,
repository=RepositoryReference.for_id(tag_manifest.repository_id),
),
)
@classmethod
def for_manifest(cls, manifest, legacy_image):
def for_manifest(cls, manifest, legacy_id_handler, legacy_image_row=None):
if manifest is None:
return None
@ -361,36 +313,15 @@ class Manifest(datatype("Manifest", ["digest", "media_type", "internal_manifest_
digest=manifest.digest,
internal_manifest_bytes=manifest_bytes,
media_type=ManifestTable.media_type.get_name(manifest.media_type_id),
_layers_compressed_size=manifest.layers_compressed_size,
config_media_type=manifest.config_media_type,
inputs=dict(
legacy_image=legacy_image,
tag_manifest=False,
legacy_id_handler=legacy_id_handler,
legacy_image_row=legacy_image_row,
repository=RepositoryReference.for_id(manifest.repository_id),
),
)
@property
@requiresinput("tag_manifest")
def _is_tag_manifest(self, tag_manifest):
return tag_manifest
@property
@requiresinput("legacy_image")
def legacy_image(self, legacy_image):
"""
Returns the legacy Docker V1-style image for this manifest.
"""
return legacy_image
@property
@optionalinput("legacy_image")
def legacy_image_if_present(self, legacy_image):
"""
Returns the legacy Docker V1-style image for this manifest.
Note that this will be None for manifests that point to other manifests instead of images.
"""
return legacy_image
def get_parsed_manifest(self, validate=True):
"""
Returns the parsed manifest for this manifest.
@ -400,17 +331,6 @@ class Manifest(datatype("Manifest", ["digest", "media_type", "internal_manifest_
self.internal_manifest_bytes, self.media_type, validate=validate
)
@property
def layers_compressed_size(self):
"""
Returns the total compressed size of the layers in the manifest or None if this could not be
computed.
"""
try:
return self.get_parsed_manifest().layers_compressed_size
except ManifestException:
return None
@property
def is_manifest_list(self):
"""
@ -426,9 +346,67 @@ class Manifest(datatype("Manifest", ["digest", "media_type", "internal_manifest_
"""
return repository
@optionalinput("legacy_image_row")
def _legacy_image_row(self, legacy_image_row):
return legacy_image_row
@property
def layers_compressed_size(self):
# TODO: Simplify once we've stopped writing Image rows and we've backfilled the
# sizes.
# First check the manifest itself, as all newly written manifests will have the
# size.
if self._layers_compressed_size is not None:
return self._layers_compressed_size
# Secondly, check for the size of the legacy Image row.
legacy_image_row = self._legacy_image_row
if legacy_image_row:
return legacy_image_row.aggregate_size
# Otherwise, return None.
return None
@property
@requiresinput("legacy_id_handler")
def legacy_image_root_id(self, legacy_id_handler):
"""
Returns the legacy Docker V1-style image ID for this manifest. Note that an ID will
be returned even if the manifest does not support a legacy image.
"""
return legacy_id_handler.encode(self._db_id)
def as_manifest(self):
""" Returns the manifest or legacy image as a manifest. """
return self
@property
@requiresinput("legacy_id_handler")
def _legacy_id_handler(self, legacy_id_handler):
return legacy_id_handler
def lookup_legacy_image(self, layer_index, retriever):
""" Looks up and returns the legacy image for index-th layer in this manifest
or None if none. The indexes here are from leaf to root, with index 0 being
the leaf.
"""
# Retrieve the schema1 manifest. If none exists, legacy images are not supported.
parsed = self.get_parsed_manifest()
if parsed is None:
return None
schema1 = parsed.get_schema1_manifest("$namespace", "$repo", "$tag", retriever)
if schema1 is None:
return None
return LegacyImage.for_schema1_manifest_layer_index(
self, schema1, layer_index, self._legacy_id_handler
)
class LegacyImage(
datatype(
namedtuple(
"LegacyImage",
[
"docker_image_id",
@ -437,8 +415,14 @@ class LegacyImage(
"command",
"image_size",
"aggregate_size",
"uploading",
"blob",
"blob_digest",
"v1_metadata_string",
# Internal fields.
"layer_index",
"manifest",
"parsed_manifest",
"id_handler",
],
)
):
@ -447,74 +431,80 @@ class LegacyImage(
"""
@classmethod
def for_image(cls, image, images_map=None, tags_map=None, blob=None):
if image is None:
def for_schema1_manifest_layer_index(
cls, manifest, parsed_manifest, layer_index, id_handler, blob=None
):
assert parsed_manifest.schema_version == 1
layers = parsed_manifest.layers
if layer_index >= len(layers):
return None
# NOTE: Schema1 keeps its layers in the order from base to leaf, so we have
# to reverse our lookup order.
leaf_to_base = list(reversed(layers))
aggregated_size = sum(
[
l.compressed_size
for index, l in enumerate(leaf_to_base)
if index >= layer_index and l.compressed_size is not None
]
)
layer = leaf_to_base[layer_index]
synthetic_layer_id = id_handler.encode(manifest._db_id, layer_index)
# Replace the image ID and parent ID with our synethetic IDs.
try:
parsed = json.loads(layer.raw_v1_metadata)
parsed["id"] = synthetic_layer_id
if layer_index < len(leaf_to_base) - 1:
parsed["parent"] = id_handler.encode(manifest._db_id, layer_index + 1)
except (ValueError, TypeError):
return None
return LegacyImage(
db_id=image.id,
inputs=dict(
images_map=images_map,
tags_map=tags_map,
ancestor_id_list=image.ancestor_id_list(),
blob=blob,
),
docker_image_id=image.docker_image_id,
created=image.created,
comment=image.comment,
command=image.command,
v1_metadata_string=image.v1_json_metadata,
image_size=image.storage.image_size,
aggregate_size=image.aggregate_size,
uploading=image.storage.uploading,
docker_image_id=synthetic_layer_id,
created=layer.v1_metadata.created,
comment=layer.v1_metadata.comment,
command=layer.v1_metadata.command,
image_size=layer.compressed_size,
aggregate_size=aggregated_size,
blob=blob,
blob_digest=layer.digest,
v1_metadata_string=json.dumps(parsed),
layer_index=layer_index,
manifest=manifest,
parsed_manifest=parsed_manifest,
id_handler=id_handler,
)
@property
def id(self):
"""
Returns the database ID of the legacy image.
"""
return self._db_id
def with_blob(self, blob):
""" Sets the blob for the legacy image. """
return self._replace(blob=blob)
@property
@requiresinput("images_map")
@requiresinput("ancestor_id_list")
def parents(self, images_map, ancestor_id_list):
"""
Returns the parent images for this image.
def parent_image_id(self):
ancestor_ids = self.ancestor_ids
if not ancestor_ids:
return None
Raises an exception if the parents have not been loaded before this property is invoked.
Parents are returned starting at the leaf image.
"""
return [
LegacyImage.for_image(images_map[ancestor_id], images_map=images_map)
for ancestor_id in reversed(ancestor_id_list)
if images_map.get(ancestor_id)
]
return ancestor_ids[-1]
@property
@requiresinput("blob")
def blob(self, blob):
"""
Returns the blob for this image.
Raises an exception if the blob has not been loaded before this property is invoked.
"""
return blob
def ancestor_ids(self):
ancestor_ids = []
for layer_index in range(self.layer_index + 1, len(self.parsed_manifest.layers)):
ancestor_ids.append(self.id_handler.encode(self.manifest._db_id, layer_index))
return ancestor_ids
@property
@requiresinput("tags_map")
def tags(self, tags_map):
"""
Returns the tags pointing to this image.
def full_image_id_chain(self):
return [self.docker_image_id] + self.ancestor_ids
Raises an exception if the tags have not been loaded before this property is invoked.
"""
tags = tags_map.get(self._db_id)
if not tags:
return []
return [Tag.for_tag(tag) for tag in tags]
def as_manifest(self):
""" Returns the parent manifest for the legacy image. """
return self.manifest
@unique
@ -579,7 +569,6 @@ class Blob(
"""
Returns the path of this blob in storage.
"""
# TODO: change this to take in the storage engine?
return storage_path
@property
@ -591,27 +580,6 @@ class Blob(
return placements
class DerivedImage(datatype("DerivedImage", ["verb", "varying_metadata", "blob"])):
"""
DerivedImage represents an image derived from a manifest via some form of verb.
"""
@classmethod
def for_derived_storage(cls, derived, verb, varying_metadata, blob):
return DerivedImage(
db_id=derived.id, verb=verb, varying_metadata=varying_metadata, blob=blob
)
@property
def unique_id(self):
"""
Returns a unique ID for this derived image.
This call will consistently produce the same unique ID across calls in the same code base.
"""
return hashlib.sha256(("%s:%s" % (self.verb, self._db_id)).encode("utf-8")).hexdigest()
class BlobUpload(
datatype(
"BlobUpload",
@ -662,13 +630,6 @@ class LikelyVulnerableTag(datatype("LikelyVulnerableTag", ["layer_id", "name"]))
db_id=tag.id, name=tag.name, layer_id=layer_id, inputs=dict(repository=repository)
)
@classmethod
def for_repository_tag(cls, tag, repository):
tag_layer_id = "%s.%s" % (tag.image.docker_image_id, tag.image.storage.uuid)
return LikelyVulnerableTag(
db_id=tag.id, name=tag.name, layer_id=tag_layer_id, inputs=dict(repository=repository)
)
@property
@requiresinput("repository")
def repository(self, repository):

104
data/registry_model/interface.py
View File

@ -14,16 +14,13 @@ class RegistryDataInterface(object):
@abstractmethod
def get_tag_legacy_image_id(self, repository_ref, tag_name, storage):
"""
Returns the legacy image ID for the tag with a legacy images in the repository.
Returns None if None.
Returns the legacy image ID for the tag in the repository or None if none.
"""
@abstractmethod
def get_legacy_tags_map(self, repository_ref, storage):
"""
Returns a map from tag name to its legacy image ID, for all tags with legacy images in the
repository.
Returns a map from tag name to its legacy image ID, for all tags in the repository.
Note that this can be a *very* heavy operation.
"""
@ -51,19 +48,14 @@ class RegistryDataInterface(object):
"""
@abstractmethod
def get_manifest_for_tag(self, tag, backfill_if_necessary=False, include_legacy_image=False):
def get_manifest_for_tag(self, tag):
"""
Returns the manifest associated with the given tag.
"""
@abstractmethod
def lookup_manifest_by_digest(
self,
repository_ref,
manifest_digest,
allow_dead=False,
include_legacy_image=False,
require_available=False,
self, repository_ref, manifest_digest, allow_dead=False, require_available=False,
):
"""
Looks up the manifest with the given digest under the given repository and returns it or
@ -92,15 +84,7 @@ class RegistryDataInterface(object):
"""
@abstractmethod
def get_legacy_images(self, repository_ref):
"""
Returns an iterator of all the LegacyImage's defined in the matching repository.
"""
@abstractmethod
def get_legacy_image(
self, repository_ref, docker_image_id, include_parents=False, include_blob=False
):
def get_legacy_image(self, repository_ref, docker_image_id, storage, include_blob=False):
"""
Returns the matching LegacyImages under the matching repository, if any.
@ -170,12 +154,12 @@ class RegistryDataInterface(object):
"""
@abstractmethod
def list_all_active_repository_tags(self, repository_ref, include_legacy_images=False):
def list_all_active_repository_tags(self, repository_ref):
"""
Returns a list of all the active tags in the repository.
Note that this is a *HEAVY* operation on repositories with a lot of tags, and should only be
used for testing or where other more specific operations are not possible.
used for testing or legacy operations.
"""
@abstractmethod
@ -204,7 +188,7 @@ class RegistryDataInterface(object):
"""
@abstractmethod
def get_repo_tag(self, repository_ref, tag_name, include_legacy_image=False):
def get_repo_tag(self, repository_ref, tag_name):
"""
Returns the latest, *active* tag found in the repository, with the matching name or None if
none.
@ -259,12 +243,6 @@ class RegistryDataInterface(object):
previous expiration timestamp in seconds (if any), and whether the operation succeeded.
"""
@abstractmethod
def get_legacy_images_owned_by_tag(self, tag):
"""
Returns all legacy images *solely owned and used* by the given tag.
"""
@abstractmethod
def get_security_status(self, manifest_or_legacy_image):
"""
@ -319,57 +297,6 @@ class RegistryDataInterface(object):
`image.docker.types.ManifestImageLayer`. Should not be called for a manifest list.
"""
@abstractmethod
def lookup_derived_image(
self, manifest, verb, storage, varying_metadata=None, include_placements=False
):
"""
Looks up the derived image for the given manifest, verb and optional varying metadata and
returns it or None if none.
"""
@abstractmethod
def lookup_or_create_derived_image(
self,
manifest,
verb,
storage_location,
storage,
varying_metadata=None,
include_placements=False,
):
"""
Looks up the derived image for the given maniest, verb and optional varying metadata and
returns it.
If none exists, a new derived image is created.
"""
@abstractmethod
def get_derived_image_signature(self, derived_image, signer_name):
"""
Returns the signature associated with the derived image and a specific signer or None if
none.
"""
@abstractmethod
def set_derived_image_signature(self, derived_image, signer_name, signature):
"""
Sets the calculated signature for the given derived image and signer to that specified.
"""
@abstractmethod
def delete_derived_image(self, derived_image):
"""
Deletes a derived image and all of its storage.
"""
@abstractmethod
def set_derived_image_size(self, derived_image, compressed_size):
"""
Sets the compressed size on the given derived image.
"""
@abstractmethod
def get_repo_blob_by_digest(self, repository_ref, blob_digest, include_placements=False):
"""
@ -474,17 +401,14 @@ class RegistryDataInterface(object):
If not possible, or an error occurs, returns None.
"""
@abstractmethod
def yield_tags_for_vulnerability_notification(self, layer_id_pairs):
"""
Yields tags that contain one (or more) of the given layer ID pairs, in repositories which
have been registered for vulnerability_found notifications.
Returns an iterator of LikelyVulnerableTag instances.
"""
@abstractmethod
def find_repository_with_garbage(self, limit_to_gc_policy_s):
""" Returns a repository reference to a repository that contains garbage for collection
or None if none.
"""
@abstractmethod
def populate_legacy_images_for_testing(self, manifest, storage):
""" Populates legacy images for the given manifest, for testing only. This call
will fail if called under non-testing code.
"""

4
data/registry_model/manifestbuilder.py
View File

@ -85,8 +85,8 @@ class _ManifestBuilder(object):
Returns the tags committed by this builder, if any.
"""
return [
registry_model.get_repo_tag(self._repository_ref, tag_name, include_legacy_image=True)
for tag_name in list(self._builder_state.tags.keys())
registry_model.get_repo_tag(self._repository_ref, tag_name)
for tag_name in self._builder_state.tags.keys()
]
def start_layer(

658
data/registry_model/registry_oci_model.py
View File

@ -25,13 +25,13 @@ from data.registry_model.datatypes import (
SecurityScanStatus,
Blob,
BlobUpload,
DerivedImage,
ShallowTag,
LikelyVulnerableTag,
RepositoryReference,
ManifestLayer,
)
from data.registry_model.label_handlers import apply_label_to_manifest
from data.registry_model.shared import SyntheticIDHandler
from image.shared import ManifestException
from image.docker.schema1 import (
DOCKER_SCHEMA1_CONTENT_TYPES,
@ -42,9 +42,6 @@ from image.docker.schema2 import EMPTY_LAYER_BLOB_DIGEST
logger = logging.getLogger(__name__)
# The maximum size for generated manifest after which we remove extra metadata.
MAXIMUM_GENERATED_MANIFEST_SIZE = 3 * 1024 * 1024 # 3 MB
class OCIModel(RegistryDataInterface):
"""
@ -52,78 +49,71 @@ class OCIModel(RegistryDataInterface):
changed to support the OCI specification.
"""
def __init__(self):
self._legacy_image_id_handler = SyntheticIDHandler()
def set_id_hash_salt(self, id_hash_salt):
self._legacy_image_id_handler = SyntheticIDHandler(id_hash_salt)
def _resolve_legacy_image_id_to_manifest_row(self, legacy_image_id):
decoded = self._legacy_image_id_handler.decode(legacy_image_id)
if len(decoded) == 0:
return (None, None)
manifest_id, layer_index = decoded
if manifest_id is None:
return (None, None)
try:
return database.Manifest.get(id=manifest_id), layer_index
except database.Manifest.DoesNotExist:
return (None, None)
def _resolve_legacy_image_id(self, legacy_image_id):
""" Decodes the given legacy image ID and returns the manifest to which it points,
as well as the layer index for the image. If invalid, or the manifest was not found,
returns (None, None).
"""
manifest, layer_index = self._resolve_legacy_image_id_to_manifest_row(legacy_image_id)
if manifest is None:
return (None, None)
return Manifest.for_manifest(manifest, self._legacy_image_id_handler), layer_index
def get_tag_legacy_image_id(self, repository_ref, tag_name, storage):
"""
Returns the legacy image ID for the tag with a legacy images in the repository.
Returns None if None.
Returns the legacy image ID for the tag in the repository. If there is no legacy image,
returns None.
"""
tag = self.get_repo_tag(repository_ref, tag_name, include_legacy_image=True)
tag = self.get_repo_tag(repository_ref, tag_name)
if tag is None:
return None
if tag.legacy_image_if_present is not None:
return tag.legacy_image_if_present.docker_image_id
retriever = RepositoryContentRetriever(repository_ref.id, storage)
legacy_image = tag.manifest.lookup_legacy_image(0, retriever)
if legacy_image is None:
return None
if tag.manifest.is_manifest_list:
# See if we can lookup a schema1 legacy image.
v1_compatible = self.get_schema1_parsed_manifest(tag.manifest, "", "", "", storage)
if v1_compatible is not None:
return v1_compatible.leaf_layer_v1_image_id
return None
return legacy_image.docker_image_id
def get_legacy_tags_map(self, repository_ref, storage):
"""
Returns a map from tag name to its legacy image ID, for all tags with legacy images in the
Returns a map from tag name to its legacy image ID, for all tags in the
repository.
Note that this can be a *very* heavy operation.
"""
tags = oci.tag.list_alive_tags(repository_ref._db_id)
legacy_images_map = oci.tag.get_legacy_images_for_tags(tags)
tags_map = {}
for tag in tags:
legacy_image = legacy_images_map.get(tag.id)
if legacy_image is not None:
tags_map[tag.name] = legacy_image.docker_image_id
else:
manifest = Manifest.for_manifest(tag.manifest, None)
if legacy_image is None and manifest.is_manifest_list:
# See if we can lookup a schema1 legacy image.
v1_compatible = self.get_schema1_parsed_manifest(manifest, "", "", "", storage)
if v1_compatible is not None:
v1_id = v1_compatible.leaf_layer_v1_image_id
if v1_id is not None:
tags_map[tag.name] = v1_id
root_id = Manifest.for_manifest(
tag.manifest, self._legacy_image_id_handler
).legacy_image_root_id
if root_id is not None:
tags_map[tag.name] = root_id
return tags_map
def _get_legacy_compatible_image_for_manifest(self, manifest, storage):
# Check for a legacy image directly on the manifest.
if not manifest.is_manifest_list:
return oci.shared.get_legacy_image_for_manifest(manifest._db_id)
# Otherwise, lookup a legacy image associated with the v1-compatible manifest
# in the list.
try:
manifest_obj = database.Manifest.get(id=manifest._db_id)
except database.Manifest.DoesNotExist:
logger.exception("Could not find manifest for manifest `%s`", manifest._db_id)
return None
# See if we can lookup a schema1 legacy image.
v1_compatible = self.get_schema1_parsed_manifest(manifest, "", "", "", storage)
if v1_compatible is None:
return None
v1_id = v1_compatible.leaf_layer_v1_image_id
if v1_id is None:
return None
return model.image.get_image(manifest_obj.repository_id, v1_id)
def find_matching_tag(self, repository_ref, tag_names):
"""
Finds an alive tag in the repository matching one of the given tag names and returns it or
@ -131,7 +121,7 @@ class OCIModel(RegistryDataInterface):
"""
found_tag = oci.tag.find_matching_tag(repository_ref._db_id, tag_names)
assert found_tag is None or not found_tag.hidden
return Tag.for_tag(found_tag)
return Tag.for_tag(found_tag, self._legacy_image_id_handler)
def get_most_recent_tag(self, repository_ref):
"""
@ -141,27 +131,17 @@ class OCIModel(RegistryDataInterface):
"""
found_tag = oci.tag.get_most_recent_tag(repository_ref._db_id)
assert found_tag is None or not found_tag.hidden
return Tag.for_tag(found_tag)
return Tag.for_tag(found_tag, self._legacy_image_id_handler)
def get_manifest_for_tag(self, tag, backfill_if_necessary=False, include_legacy_image=False):
def get_manifest_for_tag(self, tag):
"""
Returns the manifest associated with the given tag.
"""
assert tag is not None
legacy_image = None
if include_legacy_image:
legacy_image = oci.shared.get_legacy_image_for_manifest(tag._manifest)
return Manifest.for_manifest(tag._manifest, LegacyImage.for_image(legacy_image))
return tag.manifest
def lookup_manifest_by_digest(
self,
repository_ref,
manifest_digest,
allow_dead=False,
include_legacy_image=False,
require_available=False,
self, repository_ref, manifest_digest, allow_dead=False, require_available=False,
):
"""
Looks up the manifest with the given digest under the given repository and returns it or
@ -176,19 +156,7 @@ class OCIModel(RegistryDataInterface):
if manifest is None:
return None
legacy_image = None
if include_legacy_image:
try:
legacy_image_id = database.ManifestLegacyImage.get(
manifest=manifest
).image.docker_image_id
legacy_image = self.get_legacy_image(
repository_ref, legacy_image_id, include_parents=True
)
except database.ManifestLegacyImage.DoesNotExist:
pass
return Manifest.for_manifest(manifest, legacy_image)
return Manifest.for_manifest(manifest, self._legacy_image_id_handler)
def create_manifest_label(self, manifest, key, value, source_type_name, media_type_name=None):
"""
@ -276,22 +244,15 @@ class OCIModel(RegistryDataInterface):
tags = oci.tag.lookup_alive_tags_shallow(repository_ref._db_id, start_pagination_id, limit)
return [ShallowTag.for_tag(tag) for tag in tags]
def list_all_active_repository_tags(self, repository_ref, include_legacy_images=False):
def list_all_active_repository_tags(self, repository_ref):
"""
Returns a list of all the active tags in the repository.
Note that this is a *HEAVY* operation on repositories with a lot of tags, and should only be
used for testing or where other more specific operations are not possible.
used for testing or legacy operations.
"""
tags = list(oci.tag.list_alive_tags(repository_ref._db_id))
legacy_images_map = {}
if include_legacy_images:
legacy_images_map = oci.tag.get_legacy_images_for_tags(tags)
return [
Tag.for_tag(tag, legacy_image=LegacyImage.for_image(legacy_images_map.get(tag.id)))
for tag in tags
]
return [Tag.for_tag(tag, self._legacy_image_id_handler) for tag in tags]
def list_repository_tag_history(
self,
@ -312,11 +273,19 @@ class OCIModel(RegistryDataInterface):
repository_ref._db_id, page, size, specific_tag_name, active_tags_only, since_time_ms
)
# TODO: do we need legacy images here?
legacy_images_map = oci.tag.get_legacy_images_for_tags(tags)
# TODO: Remove this once the layers compressed sizes have been fully backfilled.
tags_missing_sizes = [tag for tag in tags if tag.manifest.layers_compressed_size is None]
legacy_images_map = {}
if tags_missing_sizes:
legacy_images_map = oci.tag.get_legacy_images_for_tags(tags_missing_sizes)
return (
[
Tag.for_tag(tag, LegacyImage.for_image(legacy_images_map.get(tag.id)))
Tag.for_tag(
tag,
self._legacy_image_id_handler,
legacy_image_row=legacy_images_map.get(tag.id),
)
for tag in tags
],
has_more,
@ -342,7 +311,7 @@ class OCIModel(RegistryDataInterface):
return {repo_id: toSeconds(ms) for repo_id, ms in list(last_modified.items())}
def get_repo_tag(self, repository_ref, tag_name, include_legacy_image=False):
def get_repo_tag(self, repository_ref, tag_name):
"""
Returns the latest, *active* tag found in the repository, with the matching name or None if
none.
@ -353,12 +322,7 @@ class OCIModel(RegistryDataInterface):
if tag is None:
return None
legacy_image = None
if include_legacy_image:
legacy_images = oci.tag.get_legacy_images_for_tags([tag])
legacy_image = legacy_images.get(tag.id)
return Tag.for_tag(tag, legacy_image=LegacyImage.for_image(legacy_image))
return Tag.for_tag(tag, self._legacy_image_id_handler)
def create_manifest_and_retarget_tag(
self, repository_ref, manifest_interface_instance, tag_name, storage, raise_on_error=False
@ -395,9 +359,9 @@ class OCIModel(RegistryDataInterface):
if tag is None:
return (None, None)
legacy_image = oci.shared.get_legacy_image_for_manifest(created_manifest.manifest)
li = LegacyImage.for_image(legacy_image)
wrapped_manifest = Manifest.for_manifest(created_manifest.manifest, li)
wrapped_manifest = Manifest.for_manifest(
created_manifest.manifest, self._legacy_image_id_handler
)
# Apply any labels that should modify the created tag.
if created_manifest.labels_to_apply:
@ -407,7 +371,12 @@ class OCIModel(RegistryDataInterface):
# Reload the tag in case any updates were applied.
tag = database.Tag.get(id=tag.id)
return (wrapped_manifest, Tag.for_tag(tag, li))
return (
wrapped_manifest,
Tag.for_tag(
tag, self._legacy_image_id_handler, manifest_row=created_manifest.manifest
),
)
def retarget_tag(
self,
@ -427,62 +396,37 @@ class OCIModel(RegistryDataInterface):
"""
with db_disallow_replica_use():
assert legacy_manifest_key is not None
manifest_id = manifest_or_legacy_image._db_id
if isinstance(manifest_or_legacy_image, LegacyImage):
# If a legacy image was required, build a new manifest for it and move the tag to that.
manifest = manifest_or_legacy_image.as_manifest()
manifest_id = manifest._db_id
# If the manifest is a schema 1 manifest and its tag name does not match that
# specified, then we need to create a new manifest, but with that tag name.
if manifest.media_type in DOCKER_SCHEMA1_CONTENT_TYPES:
try:
image_row = database.Image.get(id=manifest_or_legacy_image._db_id)
except database.Image.DoesNotExist:
parsed = manifest.get_parsed_manifest()
except ManifestException:
logger.exception(
"Could not parse manifest `%s` in retarget_tag", manifest._db_id,
)
return None
manifest_instance = self._build_manifest_for_legacy_image(tag_name, image_row)
if manifest_instance is None:
return None
if parsed.tag != tag_name:
logger.debug(
"Rewriting manifest `%s` for tag named `%s`", manifest._db_id, tag_name,
)
created = oci.manifest.get_or_create_manifest(
repository_ref._db_id, manifest_instance, storage
)
if created is None:
return None
repository_id = repository_ref._db_id
updated = parsed.with_tag_name(tag_name, legacy_manifest_key)
assert updated.is_signed
manifest_id = created.manifest.id
else:
# If the manifest is a schema 1 manifest and its tag name does not match that
# specified, then we need to create a new manifest, but with that tag name.
if manifest_or_legacy_image.media_type in DOCKER_SCHEMA1_CONTENT_TYPES:
try:
parsed = manifest_or_legacy_image.get_parsed_manifest()
except ManifestException:
logger.exception(
"Could not parse manifest `%s` in retarget_tag",
manifest_or_legacy_image._db_id,
)
created = oci.manifest.get_or_create_manifest(repository_id, updated, storage)
if created is None:
return None
if parsed.tag != tag_name:
logger.debug(
"Rewriting manifest `%s` for tag named `%s`",
manifest_or_legacy_image._db_id,
tag_name,
)
repository_id = repository_ref._db_id
updated = parsed.with_tag_name(tag_name, legacy_manifest_key)
assert updated.is_signed
created = oci.manifest.get_or_create_manifest(
repository_id, updated, storage
)
if created is None:
return None
manifest_id = created.manifest.id
manifest_id = created.manifest.id
tag = oci.tag.retarget_tag(tag_name, manifest_id, is_reversion=is_reversion)
legacy_image = LegacyImage.for_image(
oci.shared.get_legacy_image_for_manifest(manifest_id)
)
return Tag.for_tag(tag, legacy_image)
return Tag.for_tag(tag, self._legacy_image_id_handler)
def delete_tag(self, repository_ref, tag_name):
"""
@ -496,18 +440,18 @@ class OCIModel(RegistryDataInterface):
msg = "Invalid repository tag '%s' on repository" % tag_name
raise DataModelException(msg)
return Tag.for_tag(deleted_tag)
return Tag.for_tag(deleted_tag, self._legacy_image_id_handler)
def delete_tags_for_manifest(self, manifest):
"""
Deletes all tags pointing to the given manifest, making the manifest inaccessible for
pulling.
Returns the tags deleted, if any. Returns None on error.
Returns the tags (ShallowTag) deleted. Returns None on error.
"""
with db_disallow_replica_use():
deleted_tags = oci.tag.delete_tags_for_manifest(manifest._db_id)
return [Tag.for_tag(tag) for tag in deleted_tags]
return [ShallowTag.for_tag(tag) for tag in deleted_tags]
def change_repository_tag_expiration(self, tag, expiration_date):
"""
@ -519,75 +463,15 @@ class OCIModel(RegistryDataInterface):
with db_disallow_replica_use():
return oci.tag.change_tag_expiration(tag._db_id, expiration_date)
def get_legacy_images_owned_by_tag(self, tag):
"""
Returns all legacy images *solely owned and used* by the given tag.
"""
tag_obj = oci.tag.get_tag_by_id(tag._db_id)
if tag_obj is None:
return None
tags = oci.tag.list_alive_tags(tag_obj.repository_id)
legacy_images = oci.tag.get_legacy_images_for_tags(tags)
tag_legacy_image = legacy_images.get(tag._db_id)
if tag_legacy_image is None:
return None
assert isinstance(tag_legacy_image, Image)
# Collect the IDs of all images that the tag uses.
tag_image_ids = set()
tag_image_ids.add(tag_legacy_image.id)
tag_image_ids.update(tag_legacy_image.ancestor_id_list())
# Remove any images shared by other tags.
for current in tags:
if current == tag_obj:
continue
current_image = legacy_images.get(current.id)
if current_image is None:
continue
tag_image_ids.discard(current_image.id)
tag_image_ids = tag_image_ids.difference(current_image.ancestor_id_list())
if not tag_image_ids:
return []
if not tag_image_ids:
return []
# Load the images we need to return.
images = database.Image.select().where(database.Image.id << list(tag_image_ids))
all_image_ids = set()
for image in images:
all_image_ids.add(image.id)
all_image_ids.update(image.ancestor_id_list())
# Build a map of all the images and their parents.
images_map = {}
all_images = database.Image.select().where(database.Image.id << list(all_image_ids))
for image in all_images:
images_map[image.id] = image
return [LegacyImage.for_image(image, images_map=images_map) for image in images]
def get_security_status(self, manifest_or_legacy_image):
"""
Returns the security status for the given manifest or legacy image or None if none.
"""
image = None
if isinstance(manifest_or_legacy_image, Manifest):
image = oci.shared.get_legacy_image_for_manifest(manifest_or_legacy_image._db_id)
if image is None:
return SecurityScanStatus.UNSUPPORTED
else:
try:
image = database.Image.get(id=manifest_or_legacy_image._db_id)
except database.Image.DoesNotExist:
return None
# TODO: change from using the Image row once we've moved all security info into MSS.
manifest_id = manifest_or_legacy_image.as_manifest()._db_id
image = oci.shared.get_legacy_image_for_manifest(manifest_id)
if image is None:
return SecurityScanStatus.UNSUPPORTED
if image.security_indexed_engine is not None and image.security_indexed_engine >= 0:
return (
@ -602,22 +486,16 @@ class OCIModel(RegistryDataInterface):
re-indexed.
"""
with db_disallow_replica_use():
image = None
# TODO: change from using the Image row once we've moved all security info into MSS.
manifest_id = manifest_or_legacy_image.as_manifest()._db_id
image = oci.shared.get_legacy_image_for_manifest(manifest_id)
if image is None:
return None
if isinstance(manifest_or_legacy_image, Manifest):
image = oci.shared.get_legacy_image_for_manifest(manifest_or_legacy_image._db_id)
if image is None:
return None
else:
try:
image = database.Image.get(id=manifest_or_legacy_image._db_id)
except database.Image.DoesNotExist:
return None
assert image
image.security_indexed = False
image.security_indexed_engine = IMAGE_NOT_SCANNED_ENGINE_VERSION
image.save()
assert image
image.security_indexed = False
image.security_indexed_engine = IMAGE_NOT_SCANNED_ENGINE_VERSION
image.save()
def list_manifest_layers(self, manifest, storage, include_placements=False):
try:
@ -633,48 +511,9 @@ class OCIModel(RegistryDataInterface):
return None
return self._list_manifest_layers(
manifest_obj.repository_id, parsed, storage, include_placements, by_manifest=True
manifest_obj.repository_id, parsed, storage, include_placements
)
def lookup_derived_image(
self, manifest, verb, storage, varying_metadata=None, include_placements=False
):
"""
Looks up the derived image for the given manifest, verb and optional varying metadata and
returns it or None if none.
"""
legacy_image = self._get_legacy_compatible_image_for_manifest(manifest, storage)
if legacy_image is None:
return None
derived = model.image.find_derived_storage_for_image(legacy_image, verb, varying_metadata)
return self._build_derived(derived, verb, varying_metadata, include_placements)
def lookup_or_create_derived_image(
self,
manifest,
verb,
storage_location,
storage,
varying_metadata=None,
include_placements=False,
):
"""
Looks up the derived image for the given maniest, verb and optional varying metadata and
returns it.
If none exists, a new derived image is created.
"""
with db_disallow_replica_use():
legacy_image = self._get_legacy_compatible_image_for_manifest(manifest, storage)
if legacy_image is None:
return None
derived = model.image.find_or_create_derived_storage(
legacy_image, verb, storage_location, varying_metadata
)
return self._build_derived(derived, verb, varying_metadata, include_placements)
def set_tags_expiration_for_manifest(self, manifest, expiration_sec):
"""
Sets the expiration on all tags that point to the given manifest to that specified.
@ -737,9 +576,7 @@ class OCIModel(RegistryDataInterface):
if created_manifest is None:
return None
legacy_image = oci.shared.get_legacy_image_for_manifest(created_manifest.manifest)
li = LegacyImage.for_image(legacy_image)
return Manifest.for_manifest(created_manifest.manifest, li)
return Manifest.for_manifest(created_manifest.manifest, self._legacy_image_id_handler)
def get_repo_blob_by_digest(self, repository_ref, blob_digest, include_placements=False):
"""
@ -777,11 +614,7 @@ class OCIModel(RegistryDataInterface):
specified).
"""
return self._list_manifest_layers(
repository_ref._db_id,
parsed_manifest,
storage,
include_placements=include_placements,
by_manifest=True,
repository_ref._db_id, parsed_manifest, storage, include_placements=include_placements,
)
def get_manifest_local_blobs(self, manifest, include_placements=False):
@ -794,25 +627,9 @@ class OCIModel(RegistryDataInterface):
return None
return self._get_manifest_local_blobs(
manifest, manifest_row.repository_id, include_placements, by_manifest=True
manifest, manifest_row.repository_id, include_placements
)
def yield_tags_for_vulnerability_notification(self, layer_id_pairs):
"""
Yields tags that contain one (or more) of the given layer ID pairs, in repositories which
have been registered for vulnerability_found notifications.
Returns an iterator of LikelyVulnerableTag instances.
"""
for docker_image_id, storage_uuid in layer_id_pairs:
tags = oci.tag.lookup_notifiable_tags_for_legacy_image(
docker_image_id, storage_uuid, "vulnerability_found"
)
for tag in tags:
yield LikelyVulnerableTag.for_tag(
tag, tag.repository, docker_image_id, storage_uuid
)
def find_repository_with_garbage(self, limit_to_gc_policy_s):
repo = model.oci.tag.find_repository_with_garbage(limit_to_gc_policy_s)
if repo is None:
@ -849,66 +666,6 @@ class OCIModel(RegistryDataInterface):
namespace = model.user.get_namespace_user(namespace_name)
return namespace is not None and namespace.enabled
def get_derived_image_signature(self, derived_image, signer_name):
"""
Returns the signature associated with the derived image and a specific signer or None if
none.
"""
try:
derived_storage = database.DerivedStorageForImage.get(id=derived_image._db_id)
except database.DerivedStorageForImage.DoesNotExist:
return None
storage = derived_storage.derivative
signature_entry = model.storage.lookup_storage_signature(storage, signer_name)
if signature_entry is None:
return None
return signature_entry.signature
def set_derived_image_signature(self, derived_image, signer_name, signature):
"""
Sets the calculated signature for the given derived image and signer to that specified.
"""
with db_disallow_replica_use():
try:
derived_storage = database.DerivedStorageForImage.get(id=derived_image._db_id)
except database.DerivedStorageForImage.DoesNotExist:
return None
storage = derived_storage.derivative
signature_entry = model.storage.find_or_create_storage_signature(storage, signer_name)
signature_entry.signature = signature
signature_entry.uploading = False
signature_entry.save()
def delete_derived_image(self, derived_image):
"""
Deletes a derived image and all of its storage.
"""
with db_disallow_replica_use():
try:
derived_storage = database.DerivedStorageForImage.get(id=derived_image._db_id)
except database.DerivedStorageForImage.DoesNotExist:
return None
model.image.delete_derived_storage(derived_storage)
def set_derived_image_size(self, derived_image, compressed_size):
"""
Sets the compressed size on the given derived image.
"""
with db_disallow_replica_use():
try:
derived_storage = database.DerivedStorageForImage.get(id=derived_image._db_id)
except database.DerivedStorageForImage.DoesNotExist:
return None
storage_entry = derived_storage.derivative
storage_entry.image_size = compressed_size
storage_entry.uploading = False
storage_entry.save()
def lookup_cached_active_repository_tags(
self, model_cache, repository_ref, start_pagination_id, limit
):
@ -1098,68 +855,41 @@ class OCIModel(RegistryDataInterface):
)
return bool(storage)
def get_legacy_images(self, repository_ref):
def get_legacy_image(self, repository_ref, docker_image_id, storage, include_blob=False):
"""
Returns an iterator of all the LegacyImage's defined in the matching repository.
"""
repo = model.repository.lookup_repository(repository_ref._db_id)
if repo is None:
return None
all_images = model.image.get_repository_images_without_placements(repo)
all_images_map = {image.id: image for image in all_images}
all_tags = model.oci.tag.list_alive_tags(repo)
tags_by_image_id = defaultdict(list)
for tag in all_tags:
try:
mli = database.ManifestLegacyImage.get(manifest=tag.manifest_id)
tags_by_image_id[mli.image_id].append(tag)
except database.ManifestLegacyImage.DoesNotExist:
continue
return [
LegacyImage.for_image(image, images_map=all_images_map, tags_map=tags_by_image_id)
for image in all_images
]
def get_legacy_image(
self, repository_ref, docker_image_id, include_parents=False, include_blob=False
):
"""
Returns the matching LegacyImages under the matching repository, if any.
Returns the matching LegacyImage under the matching repository, if any.
If none, returns None.
"""
repo = model.repository.lookup_repository(repository_ref._db_id)
if repo is None:
retriever = RepositoryContentRetriever(repository_ref._db_id, storage)
# Resolves the manifest and the layer index from the synthetic ID.
manifest, layer_index = self._resolve_legacy_image_id(docker_image_id)
if manifest is None:
return None
image = model.image.get_image(repository_ref._db_id, docker_image_id)
if image is None:
return None
# Lookup the legacy image for the index.
legacy_image = manifest.lookup_legacy_image(layer_index, retriever)
if legacy_image is None or not include_blob:
return legacy_image
parent_images_map = None
if include_parents:
parent_images = model.image.get_parent_images(
repo.namespace_user.username, repo.name, image
# If a blob was requested, load it into the legacy image.
return legacy_image.with_blob(
self.get_repo_blob_by_digest(
repository_ref, legacy_image.blob_digest, include_placements=True
)
parent_images_map = {image.id: image for image in parent_images}
)
blob = None
if include_blob:
placements = list(model.storage.get_storage_locations(image.storage.uuid))
blob = Blob.for_image_storage(
image.storage,
storage_path=model.storage.get_layer_path(image.storage),
placements=placements,
)
def populate_legacy_images_for_testing(self, manifest, storage):
""" Populates legacy images for the given manifest, for testing only. This call
will fail if called under non-testing code.
"""
manifest_row = database.Manifest.get(id=manifest._db_id)
oci.manifest.populate_legacy_images_for_testing(
manifest_row, manifest.get_parsed_manifest(), storage
)
return LegacyImage.for_image(image, images_map=parent_images_map, blob=blob)
def _get_manifest_local_blobs(
self, manifest, repo_id, include_placements=False, by_manifest=False
):
def _get_manifest_local_blobs(self, manifest, repo_id, include_placements=False):
parsed = manifest.get_parsed_manifest()
if parsed is None:
return None
@ -1168,9 +898,7 @@ class OCIModel(RegistryDataInterface):
if not len(local_blob_digests):
return []
blob_query = self._lookup_repo_storages_by_content_checksum(
repo_id, local_blob_digests, by_manifest=by_manifest
)
blob_query = self._lookup_repo_storages_by_content_checksum(repo_id, local_blob_digests)
blobs = []
for image_storage in blob_query:
placements = None
@ -1186,9 +914,7 @@ class OCIModel(RegistryDataInterface):
return blobs
def _list_manifest_layers(
self, repo_id, parsed, storage, include_placements=False, by_manifest=False
):
def _list_manifest_layers(self, repo_id, parsed, storage, include_placements=False):
"""
Returns an *ordered list* of the layers found in the manifest, starting at the base and
working towards the leaf, including the associated Blob and its placements (if specified).
@ -1206,9 +932,7 @@ class OCIModel(RegistryDataInterface):
blob_digests.append(EMPTY_LAYER_BLOB_DIGEST)
if blob_digests:
blob_query = self._lookup_repo_storages_by_content_checksum(
repo_id, blob_digests, by_manifest=by_manifest
)
blob_query = self._lookup_repo_storages_by_content_checksum(repo_id, blob_digests)
storage_map = {blob.content_checksum: blob for blob in blob_query}
layers = parsed.get_layers(retriever)
@ -1246,84 +970,6 @@ class OCIModel(RegistryDataInterface):
return manifest_layers
def _build_derived(self, derived, verb, varying_metadata, include_placements):
if derived is None:
return None
derived_storage = derived.derivative
placements = None
if include_placements:
placements = list(model.storage.get_storage_locations(derived_storage.uuid))
blob = Blob.for_image_storage(
derived_storage,
storage_path=model.storage.get_layer_path(derived_storage),
placements=placements,
)
return DerivedImage.for_derived_storage(derived, verb, varying_metadata, blob)
def _build_manifest_for_legacy_image(self, tag_name, legacy_image_row):
import features
from app import app, docker_v2_signing_key
repo = legacy_image_row.repository
namespace_name = repo.namespace_user.username
repo_name = repo.name
# Find the v1 metadata for this image and its parents.
try:
parents = model.image.get_parent_images(namespace_name, repo_name, legacy_image_row)
except model.DataModelException:
logger.exception(
"Could not load parent images for legacy image %s", legacy_image_row.id
)
return None
# If the manifest is being generated under the library namespace, then we make its namespace
# empty.
manifest_namespace = namespace_name
if features.LIBRARY_SUPPORT and namespace_name == app.config["LIBRARY_NAMESPACE"]:
manifest_namespace = ""
# Create and populate the manifest builder
builder = DockerSchema1ManifestBuilder(manifest_namespace, repo_name, tag_name)
# Add the leaf layer
builder.add_layer(
legacy_image_row.storage.content_checksum, legacy_image_row.v1_json_metadata
)
if legacy_image_row.storage.uploading:
logger.error("Cannot add an uploading storage row: %s", legacy_image_row.storage.id)
return None
for parent_image in parents:
if parent_image.storage.uploading:
logger.error("Cannot add an uploading storage row: %s", legacy_image_row.storage.id)
return None
builder.add_layer(parent_image.storage.content_checksum, parent_image.v1_json_metadata)
try:
built_manifest = builder.build(docker_v2_signing_key)
# If the generated manifest is greater than the maximum size, regenerate it with
# intermediate metadata layers stripped down to their bare essentials.
if len(built_manifest.bytes.as_encoded_str()) > MAXIMUM_GENERATED_MANIFEST_SIZE:
built_manifest = builder.with_metadata_removed().build(docker_v2_signing_key)
if len(built_manifest.bytes.as_encoded_str()) > MAXIMUM_GENERATED_MANIFEST_SIZE:
logger.error("Legacy image is too large to generate manifest")
return None
return built_manifest
except ManifestException as me:
logger.exception(
"Got exception when trying to build manifest for legacy image %s", legacy_image_row
)
return None
def _get_shared_storage(self, blob_digest):
"""
Returns an ImageStorage row for the blob digest if it is a globally shared storage.
@ -1337,7 +983,7 @@ class OCIModel(RegistryDataInterface):
return None
def _lookup_repo_storages_by_content_checksum(self, repo, checksums, by_manifest=False):
def _lookup_repo_storages_by_content_checksum(self, repo, checksums):
checksums = set(checksums)
# Load any shared storages first.
@ -1350,11 +996,7 @@ class OCIModel(RegistryDataInterface):
found = []
if checksums:
found = list(
model.storage.lookup_repo_storages_by_content_checksum(
repo, checksums, by_manifest=by_manifest
)
)
found = list(model.storage.lookup_repo_storages_by_content_checksum(repo, checksums))
return found + extra_storages

17
data/registry_model/shared.py Normal file
View File

@ -0,0 +1,17 @@
import uuid
from hashids import Hashids
class SyntheticIDHandler(object):
def __init__(self, hash_salt=None):
self.hash_salt = hash_salt or str(uuid.uuid4())
self.hashids = Hashids(alphabet="0123456789abcdef", min_length=64, salt=self.hash_salt)
def encode(self, manifest_id, layer_index=0):
encoded = self.hashids.encode(manifest_id, layer_index)
assert len(encoded) == 64
return encoded
def decode(self, synthetic_v1_id):
return self.hashids.decode(synthetic_v1_id)

291
data/registry_model/test/test_interface.py
View File

@ -23,7 +23,6 @@ from data.database import (
ManifestLabel,
TagManifest,
TagManifestLabel,
DerivedStorageForImage,
Tag,
TagToRepositoryTag,
ImageStorageLocation,
@ -32,6 +31,7 @@ from data.cache.impl import InMemoryDataModelCache
from data.registry_model.registry_oci_model import OCIModel
from data.registry_model.datatypes import RepositoryReference
from data.registry_model.blobuploader import upload_blob, BlobUploadSettings
from data.model.oci.retriever import RepositoryContentRetriever
from data.model.blob import store_blob_record_and_temp_link
from image.shared.types import ManifestImageLayer
from image.docker.schema1 import (
@ -78,7 +78,6 @@ def test_find_matching_tag(names, expected, registry_model):
assert found is None
else:
assert found.name in expected
assert found.repository.namespace_name == "devtable"
assert found.repository.name == "simple"
@ -120,13 +119,9 @@ def test_lookup_manifests(repo_namespace, repo_name, registry_model):
repository_ref = RepositoryReference.for_repo_obj(repo)
found_tag = registry_model.find_matching_tag(repository_ref, ["latest"])
found_manifest = registry_model.get_manifest_for_tag(found_tag)
found = registry_model.lookup_manifest_by_digest(
repository_ref, found_manifest.digest, include_legacy_image=True
)
found = registry_model.lookup_manifest_by_digest(repository_ref, found_manifest.digest)
assert found._db_id == found_manifest._db_id
assert found.digest == found_manifest.digest
assert found.legacy_image
assert found.legacy_image.parents
schema1_parsed = registry_model.get_schema1_parsed_manifest(found, "foo", "bar", "baz", storage)
assert schema1_parsed is not None
@ -211,26 +206,24 @@ def test_batch_labels(registry_model):
)
def test_repository_tags(repo_namespace, repo_name, registry_model):
repository_ref = registry_model.lookup_repository(repo_namespace, repo_name)
tags = registry_model.list_all_active_repository_tags(
repository_ref, include_legacy_images=True
)
tags = registry_model.list_all_active_repository_tags(repository_ref)
assert len(tags)
tags_map = registry_model.get_legacy_tags_map(repository_ref, storage)
for tag in tags:
found_tag = registry_model.get_repo_tag(repository_ref, tag.name, include_legacy_image=True)
found_tag = registry_model.get_repo_tag(repository_ref, tag.name)
assert found_tag == tag
if found_tag.legacy_image is None:
continue
retriever = RepositoryContentRetriever(repository_ref.id, storage)
legacy_image = tag.manifest.lookup_legacy_image(0, retriever)
found_image = registry_model.get_legacy_image(
repository_ref, found_tag.legacy_image.docker_image_id
repository_ref, found_tag.manifest.legacy_image_root_id, storage
)
assert found_image == found_tag.legacy_image
assert tag.name in tags_map
assert tags_map[tag.name] == found_image.docker_image_id
if found_image is not None:
assert found_image.docker_image_id == legacy_image.docker_image_id
assert tags_map[tag.name] == found_image.docker_image_id
@pytest.mark.parametrize(
@ -242,12 +235,19 @@ def test_repository_tags(repo_namespace, repo_name, registry_model):
("public", "publicrepo", 1, False),
],
)
def test_repository_tag_history(namespace, name, expected_tag_count, has_expired, registry_model):
@pytest.mark.parametrize("with_size_fallback", [False, True,])
def test_repository_tag_history(
namespace, name, expected_tag_count, has_expired, registry_model, with_size_fallback
):
# Pre-cache media type loads to ensure consistent query count.
Manifest.media_type.get_name(1)
# If size fallback is requested, delete the sizes on the manifest rows.
if with_size_fallback:
Manifest.update(layers_compressed_size=None).execute()
repository_ref = registry_model.lookup_repository(namespace, name)
with assert_query_count(2):
with assert_query_count(2 if with_size_fallback else 1):
history, has_more = registry_model.list_repository_tag_history(repository_ref)
assert not has_more
assert len(history) == expected_tag_count
@ -323,9 +323,7 @@ def test_delete_tags(repo_namespace, repo_name, via_manifest, registry_model):
# Make sure the tag is no longer found.
with assert_query_count(1):
found_tag = registry_model.get_repo_tag(
repository_ref, tag.name, include_legacy_image=True
)
found_tag = registry_model.get_repo_tag(repository_ref, tag.name)
assert found_tag is None
# Ensure all tags have been deleted.
@ -347,7 +345,9 @@ def test_retarget_tag_history(use_manifest, registry_model):
repository_ref, history[0].manifest_digest, allow_dead=True
)
else:
manifest_or_legacy_image = history[0].legacy_image
manifest_or_legacy_image = registry_model.get_legacy_image(
repository_ref, history[0].manifest.legacy_image_root_id, storage
)
# Retarget the tag.
assert manifest_or_legacy_image
@ -364,7 +364,7 @@ def test_retarget_tag_history(use_manifest, registry_model):
if use_manifest:
assert updated_tag.manifest_digest == manifest_or_legacy_image.digest
else:
assert updated_tag.legacy_image == manifest_or_legacy_image
assert updated_tag.manifest.legacy_image_root_id == manifest_or_legacy_image.docker_image_id
# Ensure history has been updated.
new_history, _ = registry_model.list_repository_tag_history(repository_ref)
@ -388,15 +388,17 @@ def test_change_repository_tag_expiration(registry_model):
def test_get_security_status(registry_model):
repository_ref = registry_model.lookup_repository("devtable", "simple")
tags = registry_model.list_all_active_repository_tags(
repository_ref, include_legacy_images=True
)
tags = registry_model.list_all_active_repository_tags(repository_ref)
assert len(tags)
for tag in tags:
assert registry_model.get_security_status(tag.legacy_image)
registry_model.reset_security_status(tag.legacy_image)
assert registry_model.get_security_status(tag.legacy_image)
legacy_image = registry_model.get_legacy_image(
repository_ref, tag.manifest.legacy_image_root_id, storage
)
assert legacy_image
assert registry_model.get_security_status(legacy_image)
registry_model.reset_security_status(legacy_image)
assert registry_model.get_security_status(legacy_image)
@pytest.fixture()
@ -504,145 +506,6 @@ def test_manifest_remote_layers(oci_model):
assert layers[0].blob is None
def test_derived_image(registry_model):
# Clear all existing derived storage.
DerivedStorageForImage.delete().execute()
repository_ref = registry_model.lookup_repository("devtable", "simple")
tag = registry_model.get_repo_tag(repository_ref, "latest")
manifest = registry_model.get_manifest_for_tag(tag)
# Ensure the squashed image doesn't exist.
assert registry_model.lookup_derived_image(manifest, "squash", storage, {}) is None
# Create a new one.
squashed = registry_model.lookup_or_create_derived_image(
manifest, "squash", "local_us", storage, {}
)
assert (
registry_model.lookup_or_create_derived_image(manifest, "squash", "local_us", storage, {})
== squashed
)
assert squashed.unique_id
# Check and set the size.
assert squashed.blob.compressed_size is None
registry_model.set_derived_image_size(squashed, 1234)
found = registry_model.lookup_derived_image(manifest, "squash", storage, {})
assert found.blob.compressed_size == 1234
assert found.unique_id == squashed.unique_id
# Ensure its returned now.
assert found == squashed
# Ensure different metadata results in a different derived image.
found = registry_model.lookup_derived_image(manifest, "squash", storage, {"foo": "bar"})
assert found is None
squashed_foo = registry_model.lookup_or_create_derived_image(
manifest, "squash", "local_us", storage, {"foo": "bar"}
)
assert squashed_foo != squashed
found = registry_model.lookup_derived_image(manifest, "squash", storage, {"foo": "bar"})
assert found == squashed_foo
assert squashed.unique_id != squashed_foo.unique_id
# Lookup with placements.
squashed = registry_model.lookup_or_create_derived_image(
manifest, "squash", "local_us", storage, {}, include_placements=True
)
assert squashed.blob.placements
# Delete the derived image.
registry_model.delete_derived_image(squashed)
assert registry_model.lookup_derived_image(manifest, "squash", storage, {}) is None
def test_derived_image_signatures(registry_model):
repository_ref = registry_model.lookup_repository("devtable", "simple")
tag = registry_model.get_repo_tag(repository_ref, "latest")
manifest = registry_model.get_manifest_for_tag(tag)
derived = registry_model.lookup_or_create_derived_image(
manifest, "squash", "local_us", storage, {}
)
assert derived
registry_model.set_derived_image_signature(derived, "gpg2", "foo")
assert registry_model.get_derived_image_signature(derived, "gpg2") == "foo"
@pytest.mark.parametrize(
"manifest_builder, list_builder",
[
(DockerSchema2ManifestBuilder, DockerSchema2ManifestListBuilder),
(OCIManifestBuilder, OCIIndexBuilder),
],
)
def test_derived_image_for_manifest_list(manifest_builder, list_builder, oci_model):
# Clear all existing derived storage.
DerivedStorageForImage.delete().execute()
# Create a config blob for testing.
config_json = json.dumps(
{
"config": {},
"architecture": "amd64",
"os": "linux",
"rootfs": {"type": "layers", "diff_ids": []},
"history": [
{"created": "2018-04-03T18:37:09.284840891Z", "created_by": "do something",},
],
}
)
app_config = {"TESTING": True}
repository_ref = oci_model.lookup_repository("devtable", "simple")
with upload_blob(repository_ref, storage, BlobUploadSettings(500, 500)) as upload:
upload.upload_chunk(app_config, BytesIO(config_json.encode("utf-8")))
blob = upload.commit_to_blob(app_config)
# Create the manifest in the repo.
builder = manifest_builder()
builder.set_config_digest(blob.digest, blob.compressed_size)
builder.add_layer(blob.digest, blob.compressed_size)
amd64_manifest = builder.build()
oci_model.create_manifest_and_retarget_tag(
repository_ref, amd64_manifest, "submanifest", storage, raise_on_error=True
)
# Create a manifest list, pointing to at least one amd64+linux manifest.
builder = list_builder()
builder.add_manifest(amd64_manifest, "amd64", "linux")
manifestlist = builder.build()
oci_model.create_manifest_and_retarget_tag(
repository_ref, manifestlist, "listtag", storage, raise_on_error=True
)
manifest = oci_model.get_manifest_for_tag(oci_model.get_repo_tag(repository_ref, "listtag"))
assert manifest
assert manifest.get_parsed_manifest().is_manifest_list
# Ensure the squashed image doesn't exist.
assert oci_model.lookup_derived_image(manifest, "squash", storage, {}) is None
# Create a new one.
squashed = oci_model.lookup_or_create_derived_image(manifest, "squash", "local_us", storage, {})
assert squashed.unique_id
assert (
oci_model.lookup_or_create_derived_image(manifest, "squash", "local_us", storage, {})
== squashed
)
# Perform lookup.
assert oci_model.lookup_derived_image(manifest, "squash", storage, {}) == squashed
def test_blob_uploads(registry_model):
repository_ref = registry_model.lookup_repository("devtable", "simple")
@ -763,13 +626,11 @@ def test_get_cached_repo_blob(registry_model):
def test_create_manifest_and_retarget_tag(registry_model):
repository_ref = registry_model.lookup_repository("devtable", "simple")
latest_tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True)
latest_tag = registry_model.get_repo_tag(repository_ref, "latest")
manifest = registry_model.get_manifest_for_tag(latest_tag).get_parsed_manifest()
builder = DockerSchema1ManifestBuilder("devtable", "simple", "anothertag")
builder.add_layer(
manifest.blob_digests[0], '{"id": "%s"}' % latest_tag.legacy_image.docker_image_id
)
builder.add_layer(manifest.blob_digests[0], '{"id": "%s"}' % "someid")
sample_manifest = builder.build(docker_v2_signing_key)
assert sample_manifest is not None
@ -785,14 +646,14 @@ def test_create_manifest_and_retarget_tag(registry_model):
def test_get_schema1_parsed_manifest(registry_model):
repository_ref = registry_model.lookup_repository("devtable", "simple")
latest_tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True)
latest_tag = registry_model.get_repo_tag(repository_ref, "latest")
manifest = registry_model.get_manifest_for_tag(latest_tag)
assert registry_model.get_schema1_parsed_manifest(manifest, "", "", "", storage)
def test_convert_manifest(registry_model):
repository_ref = registry_model.lookup_repository("devtable", "simple")
latest_tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True)
latest_tag = registry_model.get_repo_tag(repository_ref, "latest")
manifest = registry_model.get_manifest_for_tag(latest_tag)
mediatypes = DOCKER_SCHEMA1_CONTENT_TYPES
@ -804,11 +665,11 @@ def test_convert_manifest(registry_model):
def test_create_manifest_and_retarget_tag_with_labels(registry_model):
repository_ref = registry_model.lookup_repository("devtable", "simple")
latest_tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True)
latest_tag = registry_model.get_repo_tag(repository_ref, "latest")
manifest = registry_model.get_manifest_for_tag(latest_tag).get_parsed_manifest()
json_metadata = {
"id": latest_tag.legacy_image.docker_image_id,
"id": "someid",
"config": {"Labels": {"quay.expires-after": "2w",},},
}
@ -903,7 +764,8 @@ def test_unicode_emoji(registry_model):
assert found.get_parsed_manifest().digest == manifest.digest
def test_lookup_active_repository_tags(oci_model):
@pytest.mark.parametrize("test_cached", [False, True,])
def test_lookup_active_repository_tags(test_cached, oci_model):
repository_ref = oci_model.lookup_repository("devtable", "simple")
latest_tag = oci_model.get_repo_tag(repository_ref, "latest")
manifest = oci_model.get_manifest_for_tag(latest_tag)
@ -924,7 +786,14 @@ def test_lookup_active_repository_tags(oci_model):
tags_found = set()
tag_id = None
while True:
tags = oci_model.lookup_active_repository_tags(repository_ref, tag_id, 11)
if test_cached:
model_cache = InMemoryDataModelCache()
tags = oci_model.lookup_cached_active_repository_tags(
model_cache, repository_ref, tag_id, 11
)
else:
tags = oci_model.lookup_active_repository_tags(repository_ref, tag_id, 11)
assert len(tags) <= 11
for tag in tags[0:10]:
assert tag.name not in tags_found
@ -942,49 +811,27 @@ def test_lookup_active_repository_tags(oci_model):
assert not tags_expected
def test_yield_tags_for_vulnerability_notification(registry_model):
repository_ref = registry_model.lookup_repository("devtable", "complex")
# Check for all legacy images under the tags and ensure not raised because
# no notification is yet registered.
for tag in registry_model.list_all_active_repository_tags(
repository_ref, include_legacy_images=True
):
image = registry_model.get_legacy_image(
repository_ref, tag.legacy_image.docker_image_id, include_blob=True
)
pairs = [(image.docker_image_id, image.blob.uuid)]
results = list(registry_model.yield_tags_for_vulnerability_notification(pairs))
assert not len(results)
# Register a notification.
model.notification.create_repo_notification(
repository_ref.id, "vulnerability_found", "email", {}, {}
def test_create_manifest_with_temp_tag(initialized_db, registry_model):
builder = DockerSchema1ManifestBuilder("devtable", "simple", "latest")
builder.add_layer(
"sha256:abcde", json.dumps({"id": "someid", "author": "some user",}, ensure_ascii=False)
)
# Check again.
for tag in registry_model.list_all_active_repository_tags(
repository_ref, include_legacy_images=True
):
image = registry_model.get_legacy_image(
repository_ref,
tag.legacy_image.docker_image_id,
include_blob=True,
include_parents=True,
)
manifest = builder.build(ensure_ascii=False)
# Check for every parent of the image.
for current in image.parents:
img = registry_model.get_legacy_image(
repository_ref, current.docker_image_id, include_blob=True
)
pairs = [(img.docker_image_id, img.blob.uuid)]
results = list(registry_model.yield_tags_for_vulnerability_notification(pairs))
assert len(results) > 0
assert tag.name in {t.name for t in results}
for blob_digest in manifest.local_blob_digests:
_populate_blob(blob_digest)
# Check for the image itself.
pairs = [(image.docker_image_id, image.blob.uuid)]
results = list(registry_model.yield_tags_for_vulnerability_notification(pairs))
assert len(results) > 0
assert tag.name in {t.name for t in results}
# Create the manifest in the database.
repository_ref = registry_model.lookup_repository("devtable", "simple")
created = registry_model.create_manifest_with_temp_tag(repository_ref, manifest, 300, storage)
assert created.digest == manifest.digest
# Ensure it cannot be found normally, since it is simply temp-tagged.
assert registry_model.lookup_manifest_by_digest(repository_ref, manifest.digest) is None
# Ensure it can be found, which means it is temp-tagged.
found = registry_model.lookup_manifest_by_digest(
repository_ref, manifest.digest, allow_dead=True
)
assert found is not None

3
data/registry_model/test/test_manifestbuilder.py
View File

@ -82,10 +82,9 @@ def test_build_manifest(layers, fake_session, registry_model):
builder.done()
# Verify the legacy image for the tag.
found = registry_model.get_repo_tag(repository_ref, "somenewtag", include_legacy_image=True)
found = registry_model.get_repo_tag(repository_ref, "somenewtag")
assert found
assert found.name == "somenewtag"
assert found.legacy_image.docker_image_id == layers[-1][0]
# Verify the blob and manifest.
manifest = registry_model.get_manifest_for_tag(found)

19
data/registry_model/test/test_model_shared.py Normal file
View File

@ -0,0 +1,19 @@
import pytest
from data.registry_model.shared import SyntheticIDHandler
@pytest.mark.parametrize("manifest_id", [1, 1000, 10000, 60000])
@pytest.mark.parametrize("hash_salt", [None, "", "testing1234", "foobarbaz",])
def test_handler(manifest_id, hash_salt):
handler = SyntheticIDHandler(hash_salt)
for index in range(0, 10):
assert handler.decode(handler.encode(manifest_id, layer_index=index)) == (
manifest_id,
index,
)
def test_invalid_value():
handler = SyntheticIDHandler("somehash")
assert handler.decode("invalidvalue") == ()

61
data/secscan_model/__init__.py
View File

@ -3,8 +3,13 @@ import logging
from collections import namedtuple
from data.secscan_model.secscan_v2_model import V2SecurityScanner, NoopV2SecurityScanner
from data.secscan_model.secscan_v4_model import V4SecurityScanner, NoopV4SecurityScanner
from data.secscan_model.secscan_v4_model import (
V4SecurityScanner,
NoopV4SecurityScanner,
ScanToken as V4ScanToken,
)
from data.secscan_model.interface import SecurityScannerInterface, InvalidConfigurationException
from data.secscan_model.datatypes import SecurityInformationLookupResult, ScanLookupStatus
from data.database import Manifest
from data.registry_model.datatypes import Manifest as ManifestDataType
@ -12,68 +17,52 @@ from data.registry_model.datatypes import Manifest as ManifestDataType
logger = logging.getLogger(__name__)
SplitScanToken = namedtuple("NextScanToken", ["version", "token"])
class SecurityScannerModelProxy(SecurityScannerInterface):
def configure(self, app, instance_keys, storage):
# TODO(alecmerdler): Just use `V4SecurityScanner` once Clair V2 is removed.
try:
self._model = V2SecurityScanner(app, instance_keys, storage)
self._model = V4SecurityScanner(app, instance_keys, storage)
except InvalidConfigurationException:
self._model = NoopV2SecurityScanner()
self._model = NoopV4SecurityScanner()
try:
self._v4_model = V4SecurityScanner(app, instance_keys, storage)
self._legacy_model = V2SecurityScanner(app, instance_keys, storage)
except InvalidConfigurationException:
self._v4_model = NoopV4SecurityScanner()
self._v4_namespace_whitelist = app.config.get("SECURITY_SCANNER_V4_NAMESPACE_WHITELIST", [])
self._legacy_model = NoopV2SecurityScanner()
logger.info("===============================")
logger.info("Using split secscan model: `%s`", [self._model, self._v4_model])
logger.info("v4 whitelist `%s`", self._v4_namespace_whitelist)
logger.info("Using split secscan model: `%s`", [self._legacy_model, self._model])
logger.info("===============================")
return self
def perform_indexing(self, next_token=None):
if next_token is None:
return SplitScanToken("v4", self._v4_model.perform_indexing(None))
if next_token is not None:
assert isinstance(next_token, V4ScanToken)
assert isinstance(next_token.min_id, int)
if next_token.version == "v4" and next_token.token is not None:
return SplitScanToken("v4", self._v4_model.perform_indexing(next_token.token))
if next_token.version == "v4" and next_token.token is None:
return SplitScanToken("v2", self._model.perform_indexing(None))
if next_token.version == "v2" and next_token.token is not None:
return SplitScanToken("v2", self._model.perform_indexing(next_token.token))
if next_token.version == "v2" and next_token.token is None:
return None
return self._model.perform_indexing(next_token)
def load_security_information(self, manifest_or_legacy_image, include_vulnerabilities):
if isinstance(manifest_or_legacy_image, ManifestDataType):
namespace = Manifest.get(
manifest_or_legacy_image._db_id
).repository.namespace_user.username
manifest = manifest_or_legacy_image.as_manifest()
if namespace in self._v4_namespace_whitelist:
return self._v4_model.load_security_information(
manifest_or_legacy_image, include_vulnerabilities
)
info = self._model.load_security_information(manifest, include_vulnerabilities)
if info.status != ScanLookupStatus.NOT_YET_INDEXED:
return info
return self._model.load_security_information(
legacy_info = self._legacy_model.load_security_information(
manifest_or_legacy_image, include_vulnerabilities
)
if legacy_info.status != ScanLookupStatus.UNSUPPORTED_FOR_INDEXING:
return legacy_info
return SecurityInformationLookupResult.with_status(ScanLookupStatus.NOT_YET_INDEXED)
def register_model_cleanup_callbacks(self, data_model_config):
return self._model.register_model_cleanup_callbacks(data_model_config)
@property
def legacy_api_handler(self):
return self._model.legacy_api_handler
return self._legacy_model.legacy_api_handler
secscan_model = SecurityScannerModelProxy()

96
data/secscan_model/secscan_v2_model.py
View File

@ -1,13 +1,10 @@
import logging
from collections import namedtuple
from math import log10
from prometheus_client import Gauge
from deprecated import deprecated
from data.database import UseThenDisconnect
from data.secscan_model.interface import SecurityScannerInterface, InvalidConfigurationException
from data.secscan_model.datatypes import (
ScanLookupStatus,
@ -21,14 +18,6 @@ from data.secscan_model.datatypes import (
from data.registry_model import registry_model
from data.registry_model.datatypes import SecurityScanStatus
from data.model.image import (
get_images_eligible_for_scan,
get_image_pk_field,
get_max_id_for_sec_scan,
get_min_id_for_sec_scan,
)
from util.migrate.allocator import yield_random_entries
from util.config import URLSchemeAndHostname
from util.secscan.api import V2SecurityConfigValidator, SecurityScannerAPI, APIRequestFailure
from util.secscan.secscan_util import get_blob_download_uri_getter
@ -111,12 +100,8 @@ class V2SecurityScanner(SecurityScannerInterface):
instance_keys=instance_keys,
)
# NOTE: This import is in here because otherwise this class would depend upon app.
# Its not great, but as this is intended to be legacy until its removed, its okay.
from util.secscan.analyzer import LayerAnalyzer
self._target_version = app.config.get("SECURITY_SCANNER_ENGINE_VERSION_TARGET", 3)
self._analyzer = LayerAnalyzer(app.config, self._legacy_secscan_api)
def register_model_cleanup_callbacks(self, data_model_config):
pass
@property
def legacy_api_handler(self):
@ -125,12 +110,6 @@ class V2SecurityScanner(SecurityScannerInterface):
"""
return self._legacy_secscan_api
def register_model_cleanup_callbacks(self, data_model_config):
if self._legacy_secscan_api is not None:
data_model_config.register_image_cleanup_callback(
self._legacy_secscan_api.cleanup_layers
)
def load_security_information(self, manifest_or_legacy_image, include_vulnerabilities=False):
status = registry_model.get_security_status(manifest_or_legacy_image)
if status is None:
@ -164,80 +143,13 @@ class V2SecurityScanner(SecurityScannerInterface):
return SecurityInformationLookupResult.for_request_error(str(arf))
if data is None:
# If no data was found but we reached this point, then it indicates we have incorrect security
# status for the manifest or legacy image. Mark the manifest or legacy image as unindexed
# so it automatically gets re-indexed.
if self.app.config.get("REGISTRY_STATE", "normal") == "normal":
registry_model.reset_security_status(manifest_or_legacy_image)
return SecurityInformationLookupResult.with_status(ScanLookupStatus.NOT_YET_INDEXED)
return SecurityInformationLookupResult.for_data(SecurityInformation.from_dict(data))
def _candidates_to_scan(self, start_token=None):
target_version = self._target_version
def batch_query():
return get_images_eligible_for_scan(target_version)
# Find the minimum ID.
min_id = None
if start_token is not None:
min_id = start_token.min_id
else:
min_id = self.app.config.get("SECURITY_SCANNER_INDEXING_MIN_ID")
if min_id is None:
min_id = get_min_id_for_sec_scan(target_version)
# Get the ID of the last image we can analyze. Will be None if there are no images in the
# database.
max_id = get_max_id_for_sec_scan()
if max_id is None:
return (None, None)
if min_id is None or min_id > max_id:
return (None, None)
# 4^log10(total) gives us a scalable batch size into the billions.
batch_size = int(4 ** log10(max(10, max_id - min_id)))
# TODO: Once we have a clean shared NamedTuple for Images, send that to the secscan analyzer
# rather than the database Image itself.
iterator = yield_random_entries(
batch_query, get_image_pk_field(), batch_size, max_id, min_id,
)
return (iterator, ScanToken(max_id + 1))
def perform_indexing(self, start_token=None):
"""
Performs indexing of the next set of unindexed manifests/images.
If start_token is given, the indexing should resume from that point. Returns a new start
index for the next iteration of indexing. The tokens returned and given are assumed to be
opaque outside of this implementation and should not be relied upon by the caller to conform
to any particular format.
NOTE: Raises `NotImplementedError` because indexing for v2 is not supported.
"""
# NOTE: This import is in here because otherwise this class would depend upon app.
# Its not great, but as this is intended to be legacy until its removed, its okay.
from util.secscan.analyzer import PreemptedException
iterator, next_token = self._candidates_to_scan(start_token)
if iterator is None:
logger.debug("Found no additional images to scan")
return None
with UseThenDisconnect(self.app.config):
for candidate, abt, num_remaining in iterator:
try:
self._analyzer.analyze_recursively(candidate)
except PreemptedException:
logger.debug("Another worker pre-empted us for layer: %s", candidate.id)
abt.set()
except APIRequestFailure:
logger.exception("Security scanner service unavailable")
return
unscanned_images.set(num_remaining)
return next_token
raise NotImplementedError("Unsupported for this security scanner version")

18
data/secscan_model/secscan_v4_model.py
View File

@ -148,19 +148,11 @@ class V4SecurityScanner(SecurityScannerInterface):
)
def perform_indexing(self, start_token=None):
whitelisted_namespaces = self.app.config.get("SECURITY_SCANNER_V4_NAMESPACE_WHITELIST", [])
try:
indexer_state = self._secscan_api.state()
except APIRequestFailure:
return None
def eligible_manifests(base_query):
return (
base_query.join(Repository)
.join(User)
.where(User.username << whitelisted_namespaces)
)
min_id = (
start_token.min_id
if start_token is not None
@ -178,16 +170,14 @@ class V4SecurityScanner(SecurityScannerInterface):
# TODO(alecmerdler): Filter out any `Manifests` that are still being uploaded
def not_indexed_query():
return (
eligible_manifests(Manifest.select())
.switch(Manifest)
Manifest.select()
.join(ManifestSecurityStatus, JOIN.LEFT_OUTER)
.where(ManifestSecurityStatus.id >> None)
)
def index_error_query():
return (
eligible_manifests(Manifest.select())
.switch(Manifest)
Manifest.select()
.join(ManifestSecurityStatus)
.where(
ManifestSecurityStatus.index_status == IndexStatus.FAILED,
@ -197,8 +187,7 @@ class V4SecurityScanner(SecurityScannerInterface):
def needs_reindexing_query(indexer_hash):
return (
eligible_manifests(Manifest.select())
.switch(Manifest)
Manifest.select()
.join(ManifestSecurityStatus)
.where(
ManifestSecurityStatus.indexer_hash != indexer_hash,
@ -209,6 +198,7 @@ class V4SecurityScanner(SecurityScannerInterface):
# 4^log10(total) gives us a scalable batch size into the billions.
batch_size = int(4 ** log10(max(10, max_id - min_id)))
# TODO(alecmerdler): We want to index newer manifests first, while backfilling older manifests...
iterator = itertools.chain(
yield_random_entries(not_indexed_query, Manifest.id, batch_size, max_id, min_id,),
yield_random_entries(index_error_query, Manifest.id, batch_size, max_id, min_id,),

129
data/secscan_model/test/test_secscan_interface.py
View File

@ -1,4 +1,5 @@
import pytest
from mock import patch, Mock
from data.secscan_model.datatypes import ScanLookupStatus, SecurityInformationLookupResult
@ -8,8 +9,10 @@ from data.secscan_model.secscan_v4_model import (
IndexReportState,
ScanToken as V4ScanToken,
)
from data.secscan_model import secscan_model, SplitScanToken
from data.secscan_model import secscan_model
from data.registry_model import registry_model
from data.model.oci import shared
from data.database import ManifestSecurityStatus, IndexerVersion, IndexStatus, ManifestLegacyImage
from test.fixtures import *
@ -17,84 +20,62 @@ from app import app, instance_keys, storage
@pytest.mark.parametrize(
"repository, v4_whitelist",
[(("devtable", "complex"), []), (("devtable", "complex"), ["devtable"]),],
"indexed_v2, indexed_v4, expected_status",
[
(False, False, ScanLookupStatus.NOT_YET_INDEXED),
(False, True, ScanLookupStatus.UNSUPPORTED_FOR_INDEXING),
(True, False, ScanLookupStatus.FAILED_TO_INDEX),
(True, True, ScanLookupStatus.UNSUPPORTED_FOR_INDEXING),
],
)
def test_load_security_information_v2_only(repository, v4_whitelist, initialized_db):
app.config["SECURITY_SCANNER_V4_NAMESPACE_WHITELIST"] = v4_whitelist
def test_load_security_information(indexed_v2, indexed_v4, expected_status, initialized_db):
secscan_model.configure(app, instance_keys, storage)
repo = registry_model.lookup_repository(*repository)
for tag in registry_model.list_all_active_repository_tags(repo):
manifest = registry_model.get_manifest_for_tag(tag)
assert manifest
repository_ref = registry_model.lookup_repository("devtable", "simple")
tag = registry_model.find_matching_tag(repository_ref, ["latest"])
manifest = registry_model.get_manifest_for_tag(tag)
assert manifest
result = secscan_model.load_security_information(manifest, True)
assert isinstance(result, SecurityInformationLookupResult)
assert result.status == ScanLookupStatus.NOT_YET_INDEXED
registry_model.populate_legacy_images_for_testing(manifest, storage)
image = shared.get_legacy_image_for_manifest(manifest._db_id)
if indexed_v2:
image.security_indexed = False
image.security_indexed_engine = 3
image.save()
else:
ManifestLegacyImage.delete().where(
ManifestLegacyImage.manifest == manifest._db_id
).execute()
if indexed_v4:
ManifestSecurityStatus.create(
manifest=manifest._db_id,
repository=repository_ref._db_id,
error_json={},
index_status=IndexStatus.MANIFEST_UNSUPPORTED,
indexer_hash="abc",
indexer_version=IndexerVersion.V4,
metadata_json={},
)
result = secscan_model.load_security_information(manifest, True)
assert isinstance(result, SecurityInformationLookupResult)
assert result.status == expected_status
@pytest.mark.parametrize(
"repository, v4_whitelist",
"next_token, expected_next_token, expected_error",
[
(("devtable", "complex"), []),
(("devtable", "complex"), ["devtable"]),
(("buynlarge", "orgrepo"), ["devtable"]),
(("buynlarge", "orgrepo"), ["devtable", "buynlarge"]),
(("buynlarge", "orgrepo"), ["devtable", "buynlarge", "sellnsmall"]),
(None, V4ScanToken(56), None),
(V4ScanToken(None), V4ScanToken(56), AssertionError),
(V4ScanToken(1), V4ScanToken(56), None),
(V2ScanToken(158), V4ScanToken(56), AssertionError),
],
)
def test_load_security_information(repository, v4_whitelist, initialized_db):
app.config["SECURITY_SCANNER_V4_NAMESPACE_WHITELIST"] = v4_whitelist
app.config["SECURITY_SCANNER_V4_ENDPOINT"] = "http://clairv4:6060"
secscan_api = Mock()
with patch("data.secscan_model.secscan_v4_model.ClairSecurityScannerAPI", secscan_api):
secscan_model.configure(app, instance_keys, storage)
repo = registry_model.lookup_repository(*repository)
for tag in registry_model.list_all_active_repository_tags(repo):
manifest = registry_model.get_manifest_for_tag(tag)
assert manifest
result = secscan_model.load_security_information(manifest, True)
assert isinstance(result, SecurityInformationLookupResult)
assert result.status == ScanLookupStatus.NOT_YET_INDEXED
@pytest.mark.parametrize(
"next_token, expected_next_token",
[
(None, SplitScanToken("v4", None)),
(SplitScanToken("v4", V4ScanToken(1)), SplitScanToken("v4", None)),
(SplitScanToken("v4", None), SplitScanToken("v2", V2ScanToken(318))),
(SplitScanToken("v2", V2ScanToken(318)), SplitScanToken("v2", None)),
(SplitScanToken("v2", None), None),
],
)
def test_perform_indexing_v2_only(next_token, expected_next_token, initialized_db):
def layer_analyzer(*args, **kwargs):
return Mock()
with patch("util.secscan.analyzer.LayerAnalyzer", layer_analyzer):
secscan_model.configure(app, instance_keys, storage)
assert secscan_model.perform_indexing(next_token) == expected_next_token
@pytest.mark.parametrize(
"next_token, expected_next_token",
[
(None, SplitScanToken("v4", V4ScanToken(56))),
(SplitScanToken("v4", V4ScanToken(1)), SplitScanToken("v4", V4ScanToken(56))),
(SplitScanToken("v4", None), SplitScanToken("v2", V2ScanToken(318))),
(SplitScanToken("v2", V2ScanToken(318)), SplitScanToken("v2", None)),
(SplitScanToken("v2", None), None),
],
)
def test_perform_indexing(next_token, expected_next_token, initialized_db):
app.config["SECURITY_SCANNER_V4_NAMESPACE_WHITELIST"] = ["devtable"]
def test_perform_indexing(next_token, expected_next_token, expected_error, initialized_db):
app.config["SECURITY_SCANNER_V4_ENDPOINT"] = "http://clairv4:6060"
def secscan_api(*args, **kwargs):
@ -104,11 +85,11 @@ def test_perform_indexing(next_token, expected_next_token, initialized_db):
return api
def layer_analyzer(*args, **kwargs):
return Mock()
with patch("data.secscan_model.secscan_v4_model.ClairSecurityScannerAPI", secscan_api):
with patch("util.secscan.analyzer.LayerAnalyzer", layer_analyzer):
secscan_model.configure(app, instance_keys, storage)
secscan_model.configure(app, instance_keys, storage)
if expected_error is not None:
with pytest.raises(expected_error):
secscan_model.perform_indexing(next_token)
else:
assert secscan_model.perform_indexing(next_token) == expected_next_token

40
data/secscan_model/test/test_secscan_v2_model.py
View File

@ -4,7 +4,7 @@ import pytest
from data.secscan_model.datatypes import ScanLookupStatus, SecurityInformation
from data.secscan_model.secscan_v2_model import V2SecurityScanner
from data.registry_model import registry_model
from data.database import Manifest, Image
from data.database import Manifest, Image, ManifestSecurityStatus, IndexStatus, IndexerVersion
from data.model.oci import shared
from data.model.image import set_secscan_status
@ -15,8 +15,10 @@ from app import app, instance_keys, storage
def test_load_security_information_unknown_manifest(initialized_db):
repository_ref = registry_model.lookup_repository("devtable", "simple")
tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True)
manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True)
tag = registry_model.get_repo_tag(repository_ref, "latest")
manifest = registry_model.get_manifest_for_tag(tag)
registry_model.populate_legacy_images_for_testing(manifest, storage)
# Delete the manifest.
Manifest.get(id=manifest._db_id).delete_instance(recursive=True)
@ -30,8 +32,10 @@ def test_load_security_information_unknown_manifest(initialized_db):
def test_load_security_information_failed_to_index(initialized_db):
repository_ref = registry_model.lookup_repository("devtable", "simple")
tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True)
manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True)
tag = registry_model.get_repo_tag(repository_ref, "latest")
manifest = registry_model.get_manifest_for_tag(tag)
registry_model.populate_legacy_images_for_testing(manifest, storage)
# Set the index status.
image = shared.get_legacy_image_for_manifest(manifest._db_id)
@ -45,8 +49,10 @@ def test_load_security_information_failed_to_index(initialized_db):
def test_load_security_information_queued(initialized_db):
repository_ref = registry_model.lookup_repository("devtable", "simple")
tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True)
manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True)
tag = registry_model.get_repo_tag(repository_ref, "latest")
manifest = registry_model.get_manifest_for_tag(tag)
registry_model.populate_legacy_images_for_testing(manifest, storage)
secscan = V2SecurityScanner(app, instance_keys, storage)
assert secscan.load_security_information(manifest).status == ScanLookupStatus.NOT_YET_INDEXED
@ -87,11 +93,14 @@ def test_load_security_information_queued(initialized_db):
)
def test_load_security_information_api_responses(secscan_api_response, initialized_db):
repository_ref = registry_model.lookup_repository("devtable", "simple")
tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True)
manifest = registry_model.get_manifest_for_tag(
tag, backfill_if_necessary=True, include_legacy_image=True
)
set_secscan_status(Image.get(id=manifest.legacy_image._db_id), True, 3)
tag = registry_model.get_repo_tag(repository_ref, "latest")
manifest = registry_model.get_manifest_for_tag(tag)
registry_model.populate_legacy_images_for_testing(manifest, storage)
legacy_image_row = shared.get_legacy_image_for_manifest(manifest._db_id)
assert legacy_image_row is not None
set_secscan_status(legacy_image_row, True, 3)
secscan = V2SecurityScanner(app, instance_keys, storage)
secscan._legacy_secscan_api = mock.Mock()
@ -110,3 +119,10 @@ def test_load_security_information_api_responses(secscan_api_response, initializ
assert len(security_information.Layer.Features) == len(
secscan_api_response["Layer"].get("Features", [])
)
def test_perform_indexing(initialized_db):
secscan = V2SecurityScanner(app, instance_keys, storage)
with pytest.raises(NotImplementedError):
secscan.perform_indexing()

100
data/secscan_model/test/test_secscan_v4_model.py
View File

@ -33,8 +33,8 @@ def set_secscan_config():
def test_load_security_information_queued(initialized_db, set_secscan_config):
repository_ref = registry_model.lookup_repository("devtable", "simple")
tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True)
manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True)
tag = registry_model.get_repo_tag(repository_ref, "latest")
manifest = registry_model.get_manifest_for_tag(tag)
secscan = V4SecurityScanner(app, instance_keys, storage)
assert secscan.load_security_information(manifest).status == ScanLookupStatus.NOT_YET_INDEXED
@ -42,8 +42,8 @@ def test_load_security_information_queued(initialized_db, set_secscan_config):
def test_load_security_information_failed_to_index(initialized_db, set_secscan_config):
repository_ref = registry_model.lookup_repository("devtable", "simple")
tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True)
manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True)
tag = registry_model.get_repo_tag(repository_ref, "latest")
manifest = registry_model.get_manifest_for_tag(tag)
ManifestSecurityStatus.create(
manifest=manifest._db_id,
@ -61,8 +61,8 @@ def test_load_security_information_failed_to_index(initialized_db, set_secscan_c
def test_load_security_information_api_returns_none(initialized_db, set_secscan_config):
repository_ref = registry_model.lookup_repository("devtable", "simple")
tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True)
manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True)
tag = registry_model.get_repo_tag(repository_ref, "latest")
manifest = registry_model.get_manifest_for_tag(tag)
ManifestSecurityStatus.create(
manifest=manifest._db_id,
@ -83,8 +83,8 @@ def test_load_security_information_api_returns_none(initialized_db, set_secscan_
def test_load_security_information_api_request_failure(initialized_db, set_secscan_config):
repository_ref = registry_model.lookup_repository("devtable", "simple")
tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True)
manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True)
tag = registry_model.get_repo_tag(repository_ref, "latest")
manifest = registry_model.get_manifest_for_tag(tag)
mss = ManifestSecurityStatus.create(
manifest=manifest._db_id,
@ -106,8 +106,8 @@ def test_load_security_information_api_request_failure(initialized_db, set_secsc
def test_load_security_information_success(initialized_db, set_secscan_config):
repository_ref = registry_model.lookup_repository("devtable", "simple")
tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True)
manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True)
tag = registry_model.get_repo_tag(repository_ref, "latest")
manifest = registry_model.get_manifest_for_tag(tag)
ManifestSecurityStatus.create(
manifest=manifest._db_id,
@ -140,11 +140,6 @@ def test_load_security_information_success(initialized_db, set_secscan_config):
def test_perform_indexing_whitelist(initialized_db, set_secscan_config):
app.config["SECURITY_SCANNER_V4_NAMESPACE_WHITELIST"] = ["devtable"]
expected_manifests = (
Manifest.select().join(Repository).join(User).where(User.username == "devtable")
)
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.state.return_value = {"state": "abc"}
@ -155,38 +150,15 @@ def test_perform_indexing_whitelist(initialized_db, set_secscan_config):
next_token = secscan.perform_indexing()
assert secscan._secscan_api.index.call_count == expected_manifests.count()
for mss in ManifestSecurityStatus.select():
assert mss.repository.namespace_user.username == "devtable"
assert ManifestSecurityStatus.select().count() == expected_manifests.count()
assert (
Manifest.get_by_id(next_token.min_id - 1).repository.namespace_user.username == "devtable"
)
def test_perform_indexing_empty_whitelist(initialized_db, set_secscan_config):
app.config["SECURITY_SCANNER_V4_NAMESPACE_WHITELIST"] = []
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.state.return_value = {"state": "abc"}
secscan._secscan_api.index.return_value = (
{"err": None, "state": IndexReportState.Index_Finished},
"abc",
)
next_token = secscan.perform_indexing()
assert secscan._secscan_api.index.call_count == 0
assert ManifestSecurityStatus.select().count() == 0
assert next_token.min_id == Manifest.select(fn.Max(Manifest.id)).scalar() + 1
assert secscan._secscan_api.index.call_count == Manifest.select().count()
assert ManifestSecurityStatus.select().count() == Manifest.select().count()
for mss in ManifestSecurityStatus.select():
assert mss.index_status == IndexStatus.COMPLETED
def test_perform_indexing_failed(initialized_db, set_secscan_config):
app.config["SECURITY_SCANNER_V4_NAMESPACE_WHITELIST"] = ["devtable"]
expected_manifests = (
Manifest.select().join(Repository).join(User).where(User.username == "devtable")
)
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.state.return_value = {"state": "abc"}
@ -195,7 +167,7 @@ def test_perform_indexing_failed(initialized_db, set_secscan_config):
"abc",
)
for manifest in expected_manifests:
for manifest in Manifest.select():
ManifestSecurityStatus.create(
manifest=manifest,
repository=manifest.repository,
@ -210,16 +182,13 @@ def test_perform_indexing_failed(initialized_db, set_secscan_config):
secscan.perform_indexing()
assert ManifestSecurityStatus.select().count() == expected_manifests.count()
assert ManifestSecurityStatus.select().count() == Manifest.select().count()
for mss in ManifestSecurityStatus.select():
assert mss.index_status == IndexStatus.COMPLETED
def test_perform_indexing_failed_within_reindex_threshold(initialized_db, set_secscan_config):
app.config["SECURITY_SCANNER_V4_REINDEX_THRESHOLD"] = 300
expected_manifests = (
Manifest.select().join(Repository).join(User).where(User.username == "devtable")
)
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
@ -229,7 +198,7 @@ def test_perform_indexing_failed_within_reindex_threshold(initialized_db, set_se
"abc",
)
for manifest in expected_manifests:
for manifest in Manifest.select():
ManifestSecurityStatus.create(
manifest=manifest,
repository=manifest.repository,
@ -242,17 +211,12 @@ def test_perform_indexing_failed_within_reindex_threshold(initialized_db, set_se
secscan.perform_indexing()
assert ManifestSecurityStatus.select().count() == expected_manifests.count()
assert ManifestSecurityStatus.select().count() == Manifest.select().count()
for mss in ManifestSecurityStatus.select():
assert mss.index_status == IndexStatus.FAILED
def test_perform_indexing_needs_reindexing(initialized_db, set_secscan_config):
app.config["SECURITY_SCANNER_V4_NAMESPACE_WHITELIST"] = ["devtable"]
expected_manifests = (
Manifest.select().join(Repository).join(User).where(User.username == "devtable")
)
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.state.return_value = {"state": "xyz"}
@ -261,7 +225,7 @@ def test_perform_indexing_needs_reindexing(initialized_db, set_secscan_config):
"xyz",
)
for manifest in expected_manifests:
for manifest in Manifest.select():
ManifestSecurityStatus.create(
manifest=manifest,
repository=manifest.repository,
@ -276,7 +240,7 @@ def test_perform_indexing_needs_reindexing(initialized_db, set_secscan_config):
secscan.perform_indexing()
assert ManifestSecurityStatus.select().count() == expected_manifests.count()
assert ManifestSecurityStatus.select().count() == Manifest.select().count()
for mss in ManifestSecurityStatus.select():
assert mss.indexer_hash == "xyz"
@ -285,10 +249,6 @@ def test_perform_indexing_needs_reindexing_within_reindex_threshold(
initialized_db, set_secscan_config
):
app.config["SECURITY_SCANNER_V4_REINDEX_THRESHOLD"] = 300
app.config["SECURITY_SCANNER_V4_NAMESPACE_WHITELIST"] = ["devtable"]
expected_manifests = (
Manifest.select().join(Repository).join(User).where(User.username == "devtable")
)
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
@ -298,7 +258,7 @@ def test_perform_indexing_needs_reindexing_within_reindex_threshold(
"xyz",
)
for manifest in expected_manifests:
for manifest in Manifest.select():
ManifestSecurityStatus.create(
manifest=manifest,
repository=manifest.repository,
@ -311,14 +271,12 @@ def test_perform_indexing_needs_reindexing_within_reindex_threshold(
secscan.perform_indexing()
assert ManifestSecurityStatus.select().count() == expected_manifests.count()
assert ManifestSecurityStatus.select().count() == Manifest.select().count()
for mss in ManifestSecurityStatus.select():
assert mss.indexer_hash == "abc"
def test_perform_indexing_api_request_failure_state(initialized_db, set_secscan_config):
app.config["SECURITY_SCANNER_V4_NAMESPACE_WHITELIST"] = ["devtable"]
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.state.side_effect = APIRequestFailure()
@ -330,14 +288,6 @@ def test_perform_indexing_api_request_failure_state(initialized_db, set_secscan_
def test_perform_indexing_api_request_failure_index(initialized_db, set_secscan_config):
app.config["SECURITY_SCANNER_V4_NAMESPACE_WHITELIST"] = ["devtable"]
expected_manifests = (
Manifest.select(fn.Max(Manifest.id))
.join(Repository)
.join(User)
.where(User.username == "devtable")
)
secscan = V4SecurityScanner(app, instance_keys, storage)
secscan._secscan_api = mock.Mock()
secscan._secscan_api.state.return_value = {"state": "abc"}
@ -357,8 +307,8 @@ def test_perform_indexing_api_request_failure_index(initialized_db, set_secscan_
next_token = secscan.perform_indexing()
assert next_token.min_id == expected_manifests.scalar() + 1
assert ManifestSecurityStatus.select().count() == expected_manifests.count()
assert next_token.min_id == Manifest.select(fn.Max(Manifest.id)).scalar() + 1
assert ManifestSecurityStatus.select().count() == Manifest.select(fn.Max(Manifest.id)).count()
def test_features_for():

55
endpoints/api/image.py
View File

@ -3,6 +3,10 @@ List and lookup repository images.
"""
import json
from collections import defaultdict
from datetime import datetime
from app import storage
from data.registry_model import registry_model
from endpoints.api import (
resource,
@ -17,7 +21,7 @@ from endpoints.api import (
from endpoints.exception import NotFound
def image_dict(image, with_history=False, with_tags=False):
def image_dict(image):
parsed_command = None
if image.command:
try:
@ -31,19 +35,11 @@ def image_dict(image, with_history=False, with_tags=False):
"comment": image.comment,
"command": parsed_command,
"size": image.image_size,
"uploading": image.uploading,
"sort_index": len(image.parents),
"uploading": False,
"sort_index": 0,
}
if with_tags:
image_data["tags"] = [tag.name for tag in image.tags]
if with_history:
image_data["history"] = [image_dict(parent) for parent in image.parents]
# Calculate the ancestors string, with the DBID's replaced with the docker IDs.
parent_docker_ids = [parent_image.docker_image_id for parent_image in image.parents]
image_data["ancestors"] = "/{0}/".format("/".join(parent_docker_ids))
image_data["ancestors"] = "/{0}/".format("/".join(image.ancestor_ids))
return image_data
@ -66,8 +62,35 @@ class RepositoryImageList(RepositoryParamResource):
if repo_ref is None:
raise NotFound()
images = registry_model.get_legacy_images(repo_ref)
return {"images": [image_dict(image, with_tags=True) for image in images]}
tags = registry_model.list_all_active_repository_tags(repo_ref)
images_with_tags = defaultdict(list)
for tag in tags:
legacy_image_id = tag.manifest.legacy_image_root_id
if legacy_image_id is not None:
images_with_tags[legacy_image_id].append(tag)
# NOTE: This is replicating our older response for this endpoint, but
# returns empty for the metadata fields. This is to ensure back-compat
# for callers still using the deprecated API, while not having to load
# all the manifests from storage.
return {
"images": [
{
"id": image_id,
"created": format_date(
datetime.utcfromtimestamp((min([tag.lifetime_start_ts for tag in tags])))
),
"comment": "",
"command": "",
"size": 0,
"uploading": False,
"sort_index": 0,
"tags": [tag.name for tag in tags],
"ancestors": "",
}
for image_id, tags in images_with_tags.items()
]
}
@resource("/v1/repository/<apirepopath:repository>/image/<image_id>")
@ -90,8 +113,8 @@ class RepositoryImage(RepositoryParamResource):
if repo_ref is None:
raise NotFound()
image = registry_model.get_legacy_image(repo_ref, image_id, include_parents=True)
image = registry_model.get_legacy_image(repo_ref, image_id, storage)
if image is None:
raise NotFound()
return image_dict(image, with_history=True)
return image_dict(image)

27
endpoints/api/manifest.py
View File

@ -4,6 +4,7 @@ Manage the manifests of a repository.
import json
import logging
from datetime import datetime
from flask import request
from app import label_validator, storage
@ -74,10 +75,6 @@ def _layer_dict(manifest_layer, index):
def _manifest_dict(manifest):
image = None
if manifest.legacy_image_if_present is not None:
image = image_dict(manifest.legacy_image, with_history=True)
layers = None
if not manifest.is_manifest_list:
layers = registry_model.list_manifest_layers(manifest, storage)
@ -85,14 +82,30 @@ def _manifest_dict(manifest):
logger.debug("Missing layers for manifest `%s`", manifest.digest)
abort(404)
image = None
if manifest.legacy_image_root_id:
# NOTE: This is replicating our older response for this endpoint, but
# returns empty for the metadata fields. This is to ensure back-compat
# for callers still using the deprecated API.
image = {
"id": manifest.legacy_image_root_id,
"created": format_date(datetime.utcnow()),
"comment": "",
"command": "",
"size": 0,
"uploading": False,
"sort_index": 0,
"ancestors": "",
}
return {
"digest": manifest.digest,
"is_manifest_list": manifest.is_manifest_list,
"manifest_data": manifest.internal_manifest_bytes.as_unicode(),
"image": image,
"layers": (
[_layer_dict(lyr.layer_info, idx) for idx, lyr in enumerate(layers)] if layers else None
),
"image": image,
}
@ -112,9 +125,7 @@ class RepositoryManifest(RepositoryParamResource):
if repo_ref is None:
raise NotFound()
manifest = registry_model.lookup_manifest_by_digest(
repo_ref, manifestref, include_legacy_image=True
)
manifest = registry_model.lookup_manifest_by_digest(repo_ref, manifestref)
if manifest is None:
raise NotFound()

6
endpoints/api/repository_models_pre_oci.py
View File

@ -161,7 +161,7 @@ class PreOCIModel(RepositoryDataInterface):
repo.namespace_user.username,
repo.name,
repo.rid in star_set,
repo.visibility_id == model.repository.get_public_repo_visibility().id,
model.repository.is_repository_public(repo),
repo_kind,
repo.description,
repo.namespace_user.organization,
@ -257,8 +257,8 @@ class PreOCIModel(RepositoryDataInterface):
tags = [
Tag(
tag.name,
tag.legacy_image.docker_image_id if tag.legacy_image_if_present else None,
tag.legacy_image.aggregate_size if tag.legacy_image_if_present else None,
tag.manifest.legacy_image_root_id,
tag.manifest_layers_size,
tag.lifetime_start_ts,
tag.manifest_digest,
tag.lifetime_end_ts,

2
endpoints/api/robot_models_pre_oci.py
View File

@ -25,7 +25,7 @@ class RobotPreOCIModel(RobotInterface):
return [
Permission(
permission.repository.name,
permission.repository.visibility.name,
model.repositoy.repository_visibility_name(permission.repository),
permission.role.name,
)
for permission in permissions

3
endpoints/api/secscan.py
View File

@ -7,6 +7,7 @@ import features
from enum import Enum, unique
from app import storage
from auth.decorators import process_basic_auth_no_pass
from data.registry_model import registry_model
from data.secscan_model import secscan_model
@ -101,7 +102,7 @@ class RepositoryImageSecurity(RepositoryParamResource):
if repo_ref is None:
raise NotFound()
legacy_image = registry_model.get_legacy_image(repo_ref, imageid)
legacy_image = registry_model.get_legacy_image(repo_ref, imageid, storage)
if legacy_image is None:
raise NotFound()

69
endpoints/api/tag.py
View File

@ -9,6 +9,7 @@ from auth.auth_context import get_authenticated_user
from data.registry_model import registry_model
from endpoints.api import (
resource,
deprecated,
nickname,
require_repo_read,
require_repo_write,
@ -40,18 +41,11 @@ def _tag_dict(tag):
if tag.lifetime_end_ts and tag.lifetime_end_ts > 0:
tag_info["end_ts"] = tag.lifetime_end_ts
# TODO: Remove this once fully on OCI data model.
if tag.legacy_image_if_present:
tag_info["docker_image_id"] = tag.legacy_image.docker_image_id
tag_info["image_id"] = tag.legacy_image.docker_image_id
tag_info["size"] = tag.legacy_image.aggregate_size
# TODO: Remove this check once fully on OCI data model.
if tag.manifest_digest:
tag_info["manifest_digest"] = tag.manifest_digest
if tag.manifest:
tag_info["is_manifest_list"] = tag.manifest.is_manifest_list
tag_info["manifest_digest"] = tag.manifest_digest
tag_info["is_manifest_list"] = tag.manifest.is_manifest_list
tag_info["size"] = tag.manifest_layers_size
tag_info["docker_image_id"] = tag.manifest.legacy_image_root_id
tag_info["image_id"] = tag.manifest.legacy_image_root_id
if tag.lifetime_start_ts and tag.lifetime_start_ts > 0:
last_modified = format_date(datetime.utcfromtimestamp(tag.lifetime_start_ts))
@ -188,7 +182,7 @@ class RepositoryTag(RepositoryParamResource):
raise InvalidRequest("Could not update tag expiration; Tag has probably changed")
if "image" in request.get_json() or "manifest_digest" in request.get_json():
existing_tag = registry_model.get_repo_tag(repo_ref, tag, include_legacy_image=True)
existing_tag = registry_model.get_repo_tag(repo_ref, tag)
manifest_or_image = None
image_id = None
@ -201,7 +195,7 @@ class RepositoryTag(RepositoryParamResource):
)
else:
image_id = request.get_json()["image"]
manifest_or_image = registry_model.get_legacy_image(repo_ref, image_id)
manifest_or_image = registry_model.get_legacy_image(repo_ref, image_id, storage)
if manifest_or_image is None:
raise NotFound()
@ -272,6 +266,7 @@ class RepositoryTagImages(RepositoryParamResource):
@nickname("listTagImages")
@disallow_for_app_repositories
@parse_args()
@deprecated()
@query_param(
"owned",
"If specified, only images wholely owned by this tag are returned.",
@ -286,30 +281,42 @@ class RepositoryTagImages(RepositoryParamResource):
if repo_ref is None:
raise NotFound()
tag_ref = registry_model.get_repo_tag(repo_ref, tag, include_legacy_image=True)
tag_ref = registry_model.get_repo_tag(repo_ref, tag)
if tag_ref is None:
raise NotFound()
if tag_ref.legacy_image_if_present is None:
if parsed_args["owned"]:
# NOTE: This is deprecated, so we just return empty now.
return {"images": []}
image_id = tag_ref.legacy_image.docker_image_id
manifest = registry_model.get_manifest_for_tag(tag_ref)
if manifest is None:
raise NotFound()
all_images = None
if parsed_args["owned"]:
# TODO: Remove the `owned` image concept once we are fully on V2_2.
all_images = registry_model.get_legacy_images_owned_by_tag(tag_ref)
else:
image_with_parents = registry_model.get_legacy_image(
repo_ref, image_id, include_parents=True
)
if image_with_parents is None:
raise NotFound()
all_images = [image_with_parents] + image_with_parents.parents
legacy_image = registry_model.get_legacy_image(
repo_ref, manifest.legacy_image_root_id, storage
)
if legacy_image is None:
raise NotFound()
# NOTE: This is replicating our older response for this endpoint, but
# returns empty for the metadata fields. This is to ensure back-compat
# for callers still using the deprecated API, while not having to load
# all the manifests from storage.
return {
"images": [image_dict(image) for image in all_images],
"images": [
{
"id": image_id,
"created": format_date(datetime.utcfromtimestamp(tag_ref.lifetime_start_ts)),
"comment": "",
"command": "",
"size": 0,
"uploading": False,
"sort_index": 0,
"ancestors": "",
}
for image_id in legacy_image.full_image_id_chain
]
}
@ -374,7 +381,7 @@ class RestoreTag(RepositoryParamResource):
repo_ref, manifest_digest, allow_dead=True, require_available=True
)
elif image_id is not None:
manifest_or_legacy_image = registry_model.get_legacy_image(repo_ref, image_id)
manifest_or_legacy_image = registry_model.get_legacy_image(repo_ref, image_id, storage)
if manifest_or_legacy_image is None:
raise NotFound()

2
endpoints/api/team.py
View File

@ -49,7 +49,7 @@ def permission_view(permission):
return {
"repository": {
"name": permission.repository.name,
"is_public": permission.repository.visibility.name == "public",
"is_public": model.repository.is_repository_public(permission.repository),
},
"role": permission.role.name,
}

7
endpoints/api/test/test_deprecated_route.py
View File

@ -11,16 +11,15 @@ from test.fixtures import *
def test_deprecated_route(client):
repository_ref = registry_model.lookup_repository("devtable", "simple")
tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True)
manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True)
image = shared.get_legacy_image_for_manifest(manifest._db_id)
tag = registry_model.get_repo_tag(repository_ref, "latest")
manifest = registry_model.get_manifest_for_tag(tag)
with client_with_identity("devtable", client) as cl:
resp = conduct_api_call(
cl,
RepositoryImageSecurity,
"get",
{"repository": "devtable/simple", "imageid": image.docker_image_id},
{"repository": "devtable/simple", "imageid": manifest.legacy_image_root_id},
expected_code=200,
)

6
endpoints/api/test/test_secscan.py
View File

@ -13,12 +13,12 @@ from test.fixtures import *
@pytest.mark.parametrize("endpoint", [RepositoryImageSecurity, RepositoryManifestSecurity,])
def test_get_security_info_with_pull_secret(endpoint, client):
repository_ref = registry_model.lookup_repository("devtable", "simple")
tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True)
manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True)
tag = registry_model.get_repo_tag(repository_ref, "latest")
manifest = registry_model.get_manifest_for_tag(tag)
params = {
"repository": "devtable/simple",
"imageid": tag.legacy_image.docker_image_id,
"imageid": tag.manifest.legacy_image_root_id,
"manifestref": manifest.digest,
}

31
endpoints/api/test/test_tag.py
View File

@ -69,10 +69,10 @@ def test_move_tag(image_exists, test_tag, expected_status, client, app):
test_image = "unknown"
if image_exists:
repo_ref = registry_model.lookup_repository("devtable", "simple")
tag_ref = registry_model.get_repo_tag(repo_ref, "latest", include_legacy_image=True)
tag_ref = registry_model.get_repo_tag(repo_ref, "latest")
assert tag_ref
test_image = tag_ref.legacy_image.docker_image_id
test_image = tag_ref.manifest.legacy_image_root_id
params = {"repository": "devtable/simple", "tag": test_tag}
request_body = {"image": test_image}
@ -86,12 +86,12 @@ def test_move_tag(image_exists, test_tag, expected_status, client, app):
@pytest.mark.parametrize(
"repo_namespace, repo_name, query_count",
[
("devtable", "simple", 5),
("devtable", "history", 5),
("devtable", "complex", 5),
("devtable", "gargantuan", 5),
("buynlarge", "orgrepo", 7), # +2 for permissions checks.
("buynlarge", "anotherorgrepo", 7), # +2 for permissions checks.
("devtable", "simple", 4),
("devtable", "history", 4),
("devtable", "complex", 4),
("devtable", "gargantuan", 4),
("buynlarge", "orgrepo", 6), # +2 for permissions checks.
("buynlarge", "anotherorgrepo", 6), # +2 for permissions checks.
],
)
def test_list_repo_tags(repo_namespace, repo_name, client, query_count, app):
@ -109,18 +109,15 @@ def test_list_repo_tags(repo_namespace, repo_name, client, query_count, app):
@pytest.mark.parametrize(
"repository, tag, owned, expect_images",
"repository, tag, expect_images",
[
("devtable/simple", "prod", False, True),
("devtable/simple", "prod", True, False),
("devtable/simple", "latest", False, True),
("devtable/simple", "latest", True, False),
("devtable/complex", "prod", False, True),
("devtable/complex", "prod", True, True),
("devtable/simple", "prod", True),
("devtable/simple", "latest", True),
("devtable/complex", "prod", True),
],
)
def test_list_tag_images(repository, tag, owned, expect_images, client, app):
def test_list_tag_images(repository, tag, expect_images, client, app):
with client_with_identity("devtable", client) as cl:
params = {"repository": repository, "tag": tag, "owned": owned}
params = {"repository": repository, "tag": tag}
result = conduct_api_call(cl, RepositoryTagImages, "get", params, None, 200).json
assert bool(result["images"]) == expect_images

2
endpoints/api/user.py
View File

@ -1087,7 +1087,7 @@ class StarredRepositoryList(ApiResource):
"namespace": repo_obj.namespace_user.username,
"name": repo_obj.name,
"description": repo_obj.description,
"is_public": repo_obj.visibility.name == "public",
"is_public": model.repository.is_repository_public(repo_obj),
}
return {"repositories": [repo_view(repo) for repo in repos]}, next_page_token

3
endpoints/appr/models_cnr.py
View File

@ -10,6 +10,7 @@ import data.model
from app import app, storage, authentication, model_cache
from data import appr_model
from data import model as data_model
from data.cache import cache_key
from data.database import Repository, MediaType, db_transaction
from data.appr_model.models import NEW_MODELS
@ -173,7 +174,7 @@ class CNRAppModel(AppRegistryDataInterface):
view = ApplicationSummaryView(
namespace=repo.namespace_user.username,
name=app_name,
visibility=repo.visibility.name,
visibility=data_model.repository.repository_visibility_name(repo),
default=available_releases[0],
channels=channels,
manifests=manifests,

25
endpoints/secscan.py
View File

@ -1,33 +1,12 @@
import logging
import json
import features
from app import secscan_notification_queue
from flask import request, make_response, Blueprint, abort
from endpoints.decorators import route_show_if, anon_allowed
from flask import make_response, Blueprint
from endpoints.decorators import anon_allowed
logger = logging.getLogger(__name__)
secscan = Blueprint("secscan", __name__)
@route_show_if(features.SECURITY_SCANNER)
@secscan.route("/notify", methods=["POST"])
def secscan_notification():
data = request.get_json()
logger.debug("Got notification from Security Scanner: %s", data)
if "Notification" not in data:
abort(400)
notification = data["Notification"]
name = ["named", notification["Name"]]
if not secscan_notification_queue.alive(name):
secscan_notification_queue.put(name, json.dumps(notification))
return make_response("Okay")
@secscan.route("/_internal_ping")
@anon_allowed
def internal_ping():

3
endpoints/test/test_anon_checked.py
View File

@ -3,10 +3,9 @@ import pytest
from app import app
from endpoints.v1 import v1_bp
from endpoints.v2 import v2_bp
from endpoints.verbs import verbs
@pytest.mark.parametrize("blueprint", [v2_bp, v1_bp, verbs,])
@pytest.mark.parametrize("blueprint", [v2_bp, v1_bp,])
def test_verify_blueprint(blueprint):
class Checker(object):
def __init__(self):

32
endpoints/v1/registry.py
View File

@ -40,18 +40,7 @@ def require_completion(f):
@wraps(f)
def wrapper(namespace, repository, *args, **kwargs):
image_id = kwargs["image_id"]
repository_ref = registry_model.lookup_repository(namespace, repository)
if repository_ref is not None:
legacy_image = registry_model.get_legacy_image(repository_ref, image_id)
if legacy_image is not None and legacy_image.uploading:
abort(
400,
"Image %(image_id)s is being uploaded, retry later",
issue="upload-in-progress",
image_id=image_id,
)
# TODO: Remove this
return f(namespace, repository, *args, **kwargs)
return wrapper
@ -102,7 +91,9 @@ def head_image_layer(namespace, repository, image_id, headers):
abort(404)
logger.debug("Looking up placement locations")
legacy_image = registry_model.get_legacy_image(repository_ref, image_id, include_blob=True)
legacy_image = registry_model.get_legacy_image(
repository_ref, image_id, store, include_blob=True
)
if legacy_image is None:
logger.debug("Could not find any blob placement locations")
abort(404, "Image %(image_id)s not found", issue="unknown-image", image_id=image_id)
@ -139,7 +130,9 @@ def get_image_layer(namespace, repository, image_id, headers):
if repository_ref is None:
abort(404)
legacy_image = registry_model.get_legacy_image(repository_ref, image_id, include_blob=True)
legacy_image = registry_model.get_legacy_image(
repository_ref, image_id, store, include_blob=True
)
if legacy_image is None:
abort(404, "Image %(image_id)s not found", issue="unknown-image", image_id=image_id)
@ -351,7 +344,9 @@ def get_image_json(namespace, repository, image_id, headers):
abort(403)
logger.debug("Looking up repo image")
legacy_image = registry_model.get_legacy_image(repository_ref, image_id, include_blob=True)
legacy_image = registry_model.get_legacy_image(
repository_ref, image_id, store, include_blob=True
)
if legacy_image is None:
flask_abort(404)
@ -381,15 +376,12 @@ def get_image_ancestry(namespace, repository, image_id, headers):
abort(403)
logger.debug("Looking up repo image")
legacy_image = registry_model.get_legacy_image(repository_ref, image_id, include_parents=True)
legacy_image = registry_model.get_legacy_image(repository_ref, image_id, store)
if legacy_image is None:
abort(404, "Image %(image_id)s not found", issue="unknown-image", image_id=image_id)
# NOTE: We can not use jsonify here because we are returning a list not an object.
ancestor_ids = [legacy_image.docker_image_id] + [
a.docker_image_id for a in legacy_image.parents
]
response = make_response(json.dumps(ancestor_ids), 200)
response = make_response(json.dumps(legacy_image.full_image_id_chain), 200)
response.headers.extend(headers)
return response

2
endpoints/v1/tag.py
View File

@ -98,7 +98,7 @@ def put_tag(namespace_name, repo_name, tag):
# Check if there is an existing image we should use (for PUT calls outside of a normal push
# operation).
legacy_image = registry_model.get_legacy_image(repository_ref, image_id)
legacy_image = registry_model.get_legacy_image(repository_ref, image_id, storage)
if legacy_image is None:
abort(400)

2
endpoints/v2/manifest.py
View File

@ -68,7 +68,7 @@ def fetch_manifest_by_tagname(namespace_name, repo_name, manifest_ref):
image_pulls.labels("v2", "tag", 404).inc()
raise ManifestUnknown()
manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True)
manifest = registry_model.get_manifest_for_tag(tag)
if manifest is None:
# Something went wrong.
image_pulls.labels("v2", "tag", 400).inc()

7
endpoints/v2/test/test_blob.py
View File

@ -129,12 +129,13 @@ def test_blob_mounting(mount_digest, source_repo, username, expect_success, clie
headers=headers,
)
repository = model.repository.get_repository("devtable", "building")
if expect_success:
# Ensure the blob now exists under the repo.
model.blob.get_repo_blob_by_digest("devtable", "building", mount_digest)
assert model.oci.blob.get_repository_blob_by_digest(repository, mount_digest)
else:
with pytest.raises(model.blob.BlobDoesNotExist):
model.blob.get_repo_blob_by_digest("devtable", "building", mount_digest)
assert model.oci.blob.get_repository_blob_by_digest(repository, mount_digest) is None
def test_blob_upload_offset(client, app):

55
endpoints/v2/test/test_manifest_cornercases.py
View File

@ -31,6 +31,23 @@ def _perform_cleanup():
model.gc.garbage_collect_repo(repo_object)
def _get_legacy_image_row_id(tag):
return (
database.ManifestLegacyImage.select(database.ManifestLegacyImage, database.Image)
.join(database.Image)
.where(database.ManifestLegacyImage.manifest == tag.manifest._db_id)
.get()
.image.docker_image_id
)
def _add_legacy_image(namespace, repo_name, tag_name):
repo_ref = registry_model.lookup_repository(namespace, repo_name)
tag_ref = registry_model.get_repo_tag(repo_ref, tag_name)
manifest_ref = registry_model.get_manifest_for_tag(tag_ref)
registry_model.populate_legacy_images_for_testing(manifest_ref, storage)
def test_missing_link(initialized_db):
"""
Tests for a corner case that could result in missing a link to a blob referenced by a manifest.
@ -54,6 +71,8 @@ def test_missing_link(initialized_db):
that of `SECOND_ID`, leaving `THIRD_ID` unlinked and therefore, after a GC, missing
`FOURTH_BLOB`.
"""
# TODO: Remove this test once we stop writing legacy image rows.
with set_tag_expiration_policy("devtable", 0):
location_name = storage.preferred_locations[0]
location = database.ImageStorageLocation.get(name=location_name)
@ -72,21 +91,19 @@ def test_missing_link(initialized_db):
)
_write_manifest(ADMIN_ACCESS_USER, REPO, FIRST_TAG, first_manifest)
_add_legacy_image(ADMIN_ACCESS_USER, REPO, FIRST_TAG)
# Delete all temp tags and perform GC.
_perform_cleanup()
# Ensure that the first blob still exists, along with the first tag.
assert (
model.blob.get_repo_blob_by_digest(ADMIN_ACCESS_USER, REPO, first_blob_sha) is not None
)
repo = model.repository.get_repository(ADMIN_ACCESS_USER, REPO)
assert model.oci.blob.get_repository_blob_by_digest(repo, first_blob_sha) is not None
repository_ref = registry_model.lookup_repository(ADMIN_ACCESS_USER, REPO)
found_tag = registry_model.get_repo_tag(
repository_ref, FIRST_TAG, include_legacy_image=True
)
found_tag = registry_model.get_repo_tag(repository_ref, FIRST_TAG)
assert found_tag is not None
assert found_tag.legacy_image.docker_image_id == "first"
assert _get_legacy_image_row_id(found_tag) == "first"
# Create the second and third blobs.
second_blob_sha = "sha256:" + hashlib.sha256(b"SECOND").hexdigest()
@ -108,6 +125,7 @@ def test_missing_link(initialized_db):
)
_write_manifest(ADMIN_ACCESS_USER, REPO, SECOND_TAG, second_manifest)
_add_legacy_image(ADMIN_ACCESS_USER, REPO, SECOND_TAG)
# Delete all temp tags and perform GC.
_perform_cleanup()
@ -117,18 +135,14 @@ def test_missing_link(initialized_db):
assert registry_model.get_repo_blob_by_digest(repository_ref, second_blob_sha) is not None
assert registry_model.get_repo_blob_by_digest(repository_ref, third_blob_sha) is not None
found_tag = registry_model.get_repo_tag(
repository_ref, FIRST_TAG, include_legacy_image=True
)
found_tag = registry_model.get_repo_tag(repository_ref, FIRST_TAG)
assert found_tag is not None
assert found_tag.legacy_image.docker_image_id == "first"
assert _get_legacy_image_row_id(found_tag) == "first"
# Ensure the IDs have changed.
found_tag = registry_model.get_repo_tag(
repository_ref, SECOND_TAG, include_legacy_image=True
)
found_tag = registry_model.get_repo_tag(repository_ref, SECOND_TAG)
assert found_tag is not None
assert found_tag.legacy_image.docker_image_id != "second"
assert _get_legacy_image_row_id(found_tag) != "second"
# Create the fourth blob.
fourth_blob_sha = "sha256:" + hashlib.sha256(b"FOURTH").hexdigest()
@ -147,6 +161,7 @@ def test_missing_link(initialized_db):
)
_write_manifest(ADMIN_ACCESS_USER, REPO, THIRD_TAG, third_manifest)
_add_legacy_image(ADMIN_ACCESS_USER, REPO, THIRD_TAG)
# Delete all temp tags and perform GC.
_perform_cleanup()
@ -158,10 +173,6 @@ def test_missing_link(initialized_db):
assert registry_model.get_repo_blob_by_digest(repository_ref, fourth_blob_sha) is not None
# Ensure new synthesized IDs were created.
second_tag = registry_model.get_repo_tag(
repository_ref, SECOND_TAG, include_legacy_image=True
)
third_tag = registry_model.get_repo_tag(
repository_ref, THIRD_TAG, include_legacy_image=True
)
assert second_tag.legacy_image.docker_image_id != third_tag.legacy_image.docker_image_id
second_tag = registry_model.get_repo_tag(repository_ref, SECOND_TAG)
third_tag = registry_model.get_repo_tag(repository_ref, THIRD_TAG)
assert _get_legacy_image_row_id(second_tag) != _get_legacy_image_row_id(third_tag)

535
endpoints/verbs/__init__.py
View File

@ -1,535 +0,0 @@
import hashlib
import json
import logging
import uuid
from functools import wraps
from flask import redirect, Blueprint, abort, send_file, make_response, request
from prometheus_client import Counter
import features
from app import app, signer, storage, config_provider, ip_resolver, instance_keys
from auth.auth_context import get_authenticated_user
from auth.decorators import process_auth
from auth.permissions import ReadRepositoryPermission
from data import database
from data import model
from data.registry_model import registry_model
from endpoints.decorators import (
anon_protect,
anon_allowed,
route_show_if,
parse_repository_name,
check_region_blacklisted,
)
from endpoints.metrics import image_pulls, image_pulled_bytes
from endpoints.v2.blob import BLOB_DIGEST_ROUTE
from image.appc import AppCImageFormatter
from image.shared import ManifestException
from image.docker.squashed import SquashedDockerImageFormatter
from storage import Storage
from util.audit import track_and_log, wrap_repository
from util.http import exact_abort
from util.metrics.prometheus import timed_blueprint
from util.registry.filelike import wrap_with_handler
from util.registry.queuefile import QueueFile
from util.registry.queueprocess import QueueProcess
from util.registry.tarlayerformat import TarLayerFormatterReporter
logger = logging.getLogger(__name__)
verbs = timed_blueprint(Blueprint("verbs", __name__))
verb_stream_passes = Counter(
"quay_verb_stream_passes_total",
"number of passes over a tar stream used by verb requests",
labelnames=["kind"],
)
LAYER_MIMETYPE = "binary/octet-stream"
QUEUE_FILE_TIMEOUT = 15 # seconds
class VerbReporter(TarLayerFormatterReporter):
def __init__(self, kind):
self.kind = kind
def report_pass(self, pass_count):
if pass_count:
verb_stream_passes.labels(self.kind).inc(pass_count)
def _open_stream(formatter, tag, schema1_manifest, derived_image_id, handlers, reporter):
"""
This method generates a stream of data which will be replicated and read from the queue files.
This method runs in a separate process.
"""
# For performance reasons, we load the full image list here, cache it, then disconnect from
# the database.
with database.UseThenDisconnect(app.config):
layers = registry_model.list_parsed_manifest_layers(
tag.repository, schema1_manifest, storage, include_placements=True
)
def image_stream_getter(store, blob):
def get_stream_for_storage():
current_image_stream = store.stream_read_file(blob.placements, blob.storage_path)
logger.debug("Returning blob %s: %s", blob.digest, blob.storage_path)
return current_image_stream
return get_stream_for_storage
def tar_stream_getter_iterator():
# Re-Initialize the storage engine because some may not respond well to forking (e.g. S3)
store = Storage(app, config_provider=config_provider, ip_resolver=ip_resolver)
# Note: We reverse because we have to start at the leaf layer and move upward,
# as per the spec for the formatters.
for layer in reversed(layers):
yield image_stream_getter(store, layer.blob)
stream = formatter.build_stream(
tag,
schema1_manifest,
derived_image_id,
layers,
tar_stream_getter_iterator,
reporter=reporter,
)
for handler_fn in handlers:
stream = wrap_with_handler(stream, handler_fn)
return stream.read
def _sign_derived_image(verb, derived_image, queue_file):
"""
Read from the queue file and sign the contents which are generated.
This method runs in a separate process.
"""
signature = None
try:
signature = signer.detached_sign(queue_file)
except Exception as e:
logger.exception(
"Exception when signing %s deriving image %s: $s", verb, derived_image, str(e)
)
return
# Setup the database (since this is a new process) and then disconnect immediately
# once the operation completes.
if not queue_file.raised_exception:
with database.UseThenDisconnect(app.config):
registry_model.set_derived_image_signature(derived_image, signer.name, signature)
def _write_derived_image_to_storage(
verb, derived_image, queue_file, namespace, repository, tag_name
):
"""
Read from the generated stream and write it back to the storage engine.
This method runs in a separate process.
"""
def handle_exception(ex):
logger.debug(
"Exception when building %s derived image %s (%s/%s:%s): %s",
verb,
derived_image,
namespace,
repository,
tag_name,
ex,
)
with database.UseThenDisconnect(app.config):
registry_model.delete_derived_image(derived_image)
queue_file.add_exception_handler(handle_exception)
# Re-Initialize the storage engine because some may not respond well to forking (e.g. S3)
store = Storage(app, config_provider=config_provider, ip_resolver=ip_resolver)
try:
store.stream_write(
derived_image.blob.placements, derived_image.blob.storage_path, queue_file
)
except IOError as ex:
logger.error(
"Exception when writing %s derived image %s (%s/%s:%s): %s",
verb,
derived_image,
namespace,
repository,
tag_name,
ex,
)
with database.UseThenDisconnect(app.config):
registry_model.delete_derived_image(derived_image)
queue_file.close()
def _verify_repo_verb(_, namespace, repo_name, tag_name, verb, checker=None):
permission = ReadRepositoryPermission(namespace, repo_name)
repo = model.repository.get_repository(namespace, repo_name)
repo_is_public = repo is not None and model.repository.is_repository_public(repo)
if not permission.can() and not repo_is_public:
logger.debug(
"No permission to read repository %s/%s for user %s with verb %s",
namespace,
repo_name,
get_authenticated_user(),
verb,
)
abort(403)
if repo is not None and repo.kind.name != "image":
logger.debug(
"Repository %s/%s for user %s is not an image repo",
namespace,
repo_name,
get_authenticated_user(),
)
abort(405)
# Make sure the repo's namespace isn't disabled.
if not registry_model.is_namespace_enabled(namespace):
abort(400)
# Lookup the requested tag.
repo_ref = registry_model.lookup_repository(namespace, repo_name)
if repo_ref is None:
abort(404)
tag = registry_model.get_repo_tag(repo_ref, tag_name)
if tag is None:
logger.debug(
"Tag %s does not exist in repository %s/%s for user %s",
tag,
namespace,
repo_name,
get_authenticated_user(),
)
abort(404)
# Get its associated manifest.
manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True)
if manifest is None:
logger.debug("Could not get manifest on %s/%s:%s::%s", namespace, repo_name, tag.name, verb)
abort(404)
# Retrieve the schema1-compatible version of the manifest.
try:
schema1_manifest = registry_model.get_schema1_parsed_manifest(
manifest, namespace, repo_name, tag.name, storage
)
except ManifestException:
logger.exception(
"Could not get manifest on %s/%s:%s::%s", namespace, repo_name, tag.name, verb
)
abort(400)
if schema1_manifest is None:
abort(404)
# If there is a data checker, call it first.
if checker is not None:
if not checker(tag, schema1_manifest):
logger.debug(
"Check mismatch on %s/%s:%s, verb %s", namespace, repo_name, tag.name, verb
)
abort(404)
# Preload the tag's repository information, so it gets cached.
assert tag.repository.namespace_name
assert tag.repository.name
return tag, manifest, schema1_manifest
def _repo_verb_signature(namespace, repository, tag_name, verb, checker=None, **kwargs):
# Verify that the tag exists and that we have access to it.
tag, manifest, _ = _verify_repo_verb(storage, namespace, repository, tag_name, verb, checker)
# Find the derived image storage for the verb.
derived_image = registry_model.lookup_derived_image(
manifest, verb, storage, varying_metadata={"tag": tag.name}
)
if derived_image is None or derived_image.blob.uploading:
return make_response("", 202)
# Check if we have a valid signer configured.
if not signer.name:
abort(404)
# Lookup the signature for the verb.
signature_value = registry_model.get_derived_image_signature(derived_image, signer.name)
if signature_value is None:
abort(404)
# Return the signature.
return make_response(signature_value)
class SimpleHasher(object):
def __init__(self):
self._current_offset = 0
def update(self, buf):
self._current_offset += len(buf)
@property
def hashed_bytes(self):
return self._current_offset
@check_region_blacklisted()
def _repo_verb(
namespace, repository, tag_name, verb, formatter, sign=False, checker=None, **kwargs
):
# Verify that the image exists and that we have access to it.
logger.debug(
"Verifying repo verb %s for repository %s/%s with user %s with mimetype %s",
verb,
namespace,
repository,
get_authenticated_user(),
request.accept_mimetypes.best,
)
tag, manifest, schema1_manifest = _verify_repo_verb(
storage, namespace, repository, tag_name, verb, checker
)
# Load the repository for later.
repo = model.repository.get_repository(namespace, repository)
if repo is None:
abort(404)
# Check for torrent, which is no longer supported.
if request.accept_mimetypes.best == "application/x-bittorrent":
abort(406)
# Log the action.
track_and_log("repo_verb", wrap_repository(repo), tag=tag.name, verb=verb, **kwargs)
is_readonly = app.config.get("REGISTRY_STATE", "normal") == "readonly"
# Lookup/create the derived image for the verb and repo image.
if is_readonly:
derived_image = registry_model.lookup_derived_image(
manifest, verb, storage, varying_metadata={"tag": tag.name}, include_placements=True
)
else:
derived_image = registry_model.lookup_or_create_derived_image(
manifest,
verb,
storage.preferred_locations[0],
storage,
varying_metadata={"tag": tag.name},
include_placements=True,
)
if derived_image is None:
logger.error("Could not create or lookup a derived image for manifest %s", manifest)
abort(400)
if derived_image is not None and not derived_image.blob.uploading:
logger.debug("Derived %s image %s exists in storage", verb, derived_image)
is_head_request = request.method == "HEAD"
if derived_image.blob.compressed_size:
image_pulled_bytes.labels("verbs").inc(derived_image.blob.compressed_size)
download_url = storage.get_direct_download_url(
derived_image.blob.placements, derived_image.blob.storage_path, head=is_head_request
)
if download_url:
logger.debug("Redirecting to download URL for derived %s image %s", verb, derived_image)
return redirect(download_url)
# Close the database handle here for this process before we send the long download.
database.close_db_filter(None)
logger.debug("Sending cached derived %s image %s", verb, derived_image)
return send_file(
storage.stream_read_file(
derived_image.blob.placements, derived_image.blob.storage_path
),
mimetype=LAYER_MIMETYPE,
)
logger.debug("Building and returning derived %s image", verb)
hasher = SimpleHasher()
# Close the database connection before any process forking occurs. This is important because
# the Postgres driver does not react kindly to forking, so we need to make sure it is closed
# so that each process will get its own unique connection.
database.close_db_filter(None)
def _cleanup():
# Close any existing DB connection once the process has exited.
database.close_db_filter(None)
def _store_metadata_and_cleanup():
if is_readonly:
return
with database.UseThenDisconnect(app.config):
registry_model.set_derived_image_size(derived_image, hasher.hashed_bytes)
# Create a queue process to generate the data. The queue files will read from the process
# and send the results to the client and storage.
unique_id = (
derived_image.unique_id
if derived_image is not None
else hashlib.sha256(("%s:%s" % (verb, uuid.uuid4())).encode("utf-8")).hexdigest()
)
handlers = [hasher.update]
reporter = VerbReporter(verb)
args = (formatter, tag, schema1_manifest, unique_id, handlers, reporter)
queue_process = QueueProcess(
_open_stream,
8 * 1024,
10 * 1024 * 1024, # 8K/10M chunk/max
args,
finished=_store_metadata_and_cleanup,
)
client_queue_file = QueueFile(
queue_process.create_queue(), "client", timeout=QUEUE_FILE_TIMEOUT
)
if not is_readonly:
storage_queue_file = QueueFile(
queue_process.create_queue(), "storage", timeout=QUEUE_FILE_TIMEOUT
)
# If signing is required, add a QueueFile for signing the image as we stream it out.
signing_queue_file = None
if sign and signer.name:
signing_queue_file = QueueFile(
queue_process.create_queue(), "signing", timeout=QUEUE_FILE_TIMEOUT
)
# Start building.
queue_process.run()
# Start the storage saving.
if not is_readonly:
storage_args = (verb, derived_image, storage_queue_file, namespace, repository, tag_name)
QueueProcess.run_process(_write_derived_image_to_storage, storage_args, finished=_cleanup)
if sign and signer.name:
signing_args = (verb, derived_image, signing_queue_file)
QueueProcess.run_process(_sign_derived_image, signing_args, finished=_cleanup)
# Close the database handle here for this process before we send the long download.
database.close_db_filter(None)
# Return the client's data.
return send_file(client_queue_file, mimetype=LAYER_MIMETYPE)
def os_arch_checker(os, arch):
def checker(tag, manifest):
try:
image_json = json.loads(manifest.leaf_layer.raw_v1_metadata)
except ValueError:
logger.exception("Could not parse leaf layer JSON for manifest %s", manifest)
return False
except TypeError:
logger.exception("Could not parse leaf layer JSON for manifest %s", manifest)
return False
# Verify the architecture and os.
operating_system = image_json.get("os", "linux")
if operating_system != os:
return False
architecture = image_json.get("architecture", "amd64")
# Note: Some older Docker images have 'x86_64' rather than 'amd64'.
# We allow the conversion here.
if architecture == "x86_64" and operating_system == "linux":
architecture = "amd64"
if architecture != arch:
return False
return True
return checker
def observe_route(protocol):
"""
Decorates verb endpoints to record the image_pulls metric into Prometheus.
"""
def decorator(func):
@wraps(func)
def wrapper(*args, **kwargs):
rv = func(*args, **kwargs)
image_pulls.labels(protocol, "tag", rv.status_code)
return rv
return wrapper
return decorator
@route_show_if(features.ACI_CONVERSION)
@anon_protect
@verbs.route("/aci/<server>/<namespace>/<repository>/<tag>/sig/<os>/<arch>/", methods=["GET"])
@verbs.route("/aci/<server>/<namespace>/<repository>/<tag>/aci.asc/<os>/<arch>/", methods=["GET"])
@observe_route("aci")
@process_auth
def get_aci_signature(server, namespace, repository, tag, os, arch):
return _repo_verb_signature(
namespace, repository, tag, "aci", checker=os_arch_checker(os, arch), os=os, arch=arch
)
@route_show_if(features.ACI_CONVERSION)
@anon_protect
@verbs.route(
"/aci/<server>/<namespace>/<repository>/<tag>/aci/<os>/<arch>/", methods=["GET", "HEAD"]
)
@observe_route("aci")
@process_auth
def get_aci_image(server, namespace, repository, tag, os, arch):
return _repo_verb(
namespace,
repository,
tag,
"aci",
AppCImageFormatter(),
sign=True,
checker=os_arch_checker(os, arch),
os=os,
arch=arch,
)
@anon_protect
@verbs.route("/squash/<namespace>/<repository>/<tag>", methods=["GET"])
@observe_route("squash")
@process_auth
def get_squashed_tag(namespace, repository, tag):
return _repo_verb(namespace, repository, tag, "squash", SquashedDockerImageFormatter())
@verbs.route("/_internal_ping")
@anon_allowed
def internal_ping():
return make_response("true", 200)

97
endpoints/verbs/test/test_security.py
View File

@ -1,97 +0,0 @@
import pytest
from flask import url_for
from endpoints.test.shared import conduct_call, gen_basic_auth
from test.fixtures import *
NO_ACCESS_USER = "freshuser"
READ_ACCESS_USER = "reader"
ADMIN_ACCESS_USER = "devtable"
CREATOR_ACCESS_USER = "creator"
PUBLIC_REPO = "public/publicrepo"
PRIVATE_REPO = "devtable/shared"
ORG_REPO = "buynlarge/orgrepo"
ANOTHER_ORG_REPO = "buynlarge/anotherorgrepo"
ACI_ARGS = {
"server": "someserver",
"tag": "fake",
"os": "linux",
"arch": "x64",
}
@pytest.mark.parametrize(
"user",
[
(0, None),
(1, NO_ACCESS_USER),
(2, READ_ACCESS_USER),
(3, CREATOR_ACCESS_USER),
(4, ADMIN_ACCESS_USER),
],
)
@pytest.mark.parametrize(
"endpoint,method,repository,single_repo_path,params,expected_statuses",
[
("get_aci_signature", "GET", PUBLIC_REPO, False, ACI_ARGS, (404, 404, 404, 404, 404)),
("get_aci_signature", "GET", PRIVATE_REPO, False, ACI_ARGS, (403, 403, 404, 403, 404)),
("get_aci_signature", "GET", ORG_REPO, False, ACI_ARGS, (403, 403, 404, 403, 404)),
("get_aci_signature", "GET", ANOTHER_ORG_REPO, False, ACI_ARGS, (403, 403, 403, 403, 404)),
# get_aci_image
("get_aci_image", "GET", PUBLIC_REPO, False, ACI_ARGS, (404, 404, 404, 404, 404)),
("get_aci_image", "GET", PRIVATE_REPO, False, ACI_ARGS, (403, 403, 404, 403, 404)),
("get_aci_image", "GET", ORG_REPO, False, ACI_ARGS, (403, 403, 404, 403, 404)),
("get_aci_image", "GET", ANOTHER_ORG_REPO, False, ACI_ARGS, (403, 403, 403, 403, 404)),
# get_squashed_tag
(
"get_squashed_tag",
"GET",
PUBLIC_REPO,
False,
dict(tag="fake"),
(404, 404, 404, 404, 404),
),
(
"get_squashed_tag",
"GET",
PRIVATE_REPO,
False,
dict(tag="fake"),
(403, 403, 404, 403, 404),
),
("get_squashed_tag", "GET", ORG_REPO, False, dict(tag="fake"), (403, 403, 404, 403, 404)),
(
"get_squashed_tag",
"GET",
ANOTHER_ORG_REPO,
False,
dict(tag="fake"),
(403, 403, 403, 403, 404),
),
],
)
def test_verbs_security(
user, endpoint, method, repository, single_repo_path, params, expected_statuses, app, client
):
headers = {}
if user[1] is not None:
headers["Authorization"] = gen_basic_auth(user[1], "password")
if single_repo_path:
params["repository"] = repository
else:
(namespace, repo_name) = repository.split("/")
params["namespace"] = namespace
params["repository"] = repo_name
conduct_call(
client,
"verbs." + endpoint,
url_for,
method,
params,
expected_code=expected_statuses[user[0]],
headers=headers,
)

12
endpoints/web.py
View File

@ -27,7 +27,6 @@ from app import (
billing as stripe,
build_logs,
avatar,
signer,
log_archive,
config_provider,
get_app_url,
@ -144,17 +143,6 @@ def user_view(path):
return index("")
@route_show_if(features.ACI_CONVERSION)
@web.route("/aci-signing-key")
@no_cache
@anon_protect
def aci_signing_key():
if not signer.name:
abort(404)
return send_file(signer.open_public_key_file(), mimetype=PGP_KEY_MIMETYPE)
@web.route("/plans/")
@no_cache
@route_show_if(features.BILLING)

1
health/services.py
View File

@ -178,7 +178,6 @@ def _check_disk_space(for_warning):
_INSTANCE_SERVICES = {
"registry_gunicorn": _check_gunicorn("v1/_internal_ping"),
"web_gunicorn": _check_gunicorn("_internal_ping"),
"verbs_gunicorn": _check_gunicorn("c1/_internal_ping"),
"service_key": _check_service_key,
"disk_space": _check_disk_space(for_warning=False),
"jwtproxy": _check_jwt_proxy,

227
image/appc/__init__.py
View File

@ -1,227 +0,0 @@
import json
import re
import calendar
from uuid import uuid4
from app import app
from util.registry.streamlayerformat import StreamLayerMerger
from util.dict_wrappers import JSONPathDict
from image.common import TarImageFormatter
ACNAME_REGEX = re.compile(r"[^a-z-]+")
class AppCImageFormatter(TarImageFormatter):
"""
Image formatter which produces an tarball according to the AppC specification.
"""
def stream_generator(
self,
tag,
parsed_manifest,
synthetic_image_id,
layer_iterator,
tar_stream_getter_iterator,
reporter=None,
):
image_mtime = 0
created = parsed_manifest.created_datetime
if created is not None:
image_mtime = calendar.timegm(created.utctimetuple())
# ACI Format (.tar):
# manifest - The JSON manifest
# rootfs - The root file system
# Yield the manifest.
aci_manifest = json.dumps(
DockerV1ToACIManifestTranslator.build_manifest(tag, parsed_manifest, synthetic_image_id)
)
yield self.tar_file("manifest", aci_manifest.encode("utf-8"), mtime=image_mtime)
# Yield the merged layer dtaa.
yield self.tar_folder("rootfs", mtime=image_mtime)
layer_merger = StreamLayerMerger(
tar_stream_getter_iterator, path_prefix="rootfs/", reporter=reporter
)
for entry in layer_merger.get_generator():
yield entry
class DockerV1ToACIManifestTranslator(object):
@staticmethod
def _build_isolators(docker_config):
"""
Builds ACI isolator config from the docker config.
"""
def _isolate_memory(memory):
return {"name": "memory/limit", "value": {"request": str(memory) + "B",}}
def _isolate_swap(memory):
return {"name": "memory/swap", "value": {"request": str(memory) + "B",}}
def _isolate_cpu(cpu):
return {"name": "cpu/shares", "value": {"request": str(cpu),}}
def _isolate_capabilities(capabilities_set_value):
capabilities_set = re.split(r"[\s,]", capabilities_set_value)
return {"name": "os/linux/capabilities-retain-set", "value": {"set": capabilities_set,}}
mappers = {
"Memory": _isolate_memory,
"MemorySwap": _isolate_swap,
"CpuShares": _isolate_cpu,
"Cpuset": _isolate_capabilities,
}
isolators = []
for config_key in mappers:
value = docker_config.get(config_key)
if value:
isolators.append(mappers[config_key](value))
return isolators
@staticmethod
def _build_ports(docker_config):
"""
Builds the ports definitions for the ACI.
Formats:
port/tcp
port/udp
port
"""
ports = []
exposed_ports = docker_config["ExposedPorts"]
if exposed_ports is not None:
port_list = list(exposed_ports.keys())
else:
port_list = docker_config["Ports"] or docker_config["ports"] or []
for docker_port in port_list:
protocol = "tcp"
port_number = -1
if "/" in docker_port:
(port_number, protocol) = docker_port.split("/")
else:
port_number = docker_port
try:
port_number = int(port_number)
ports.append(
{"name": "port-%s" % port_number, "port": port_number, "protocol": protocol,}
)
except ValueError:
pass
return ports
@staticmethod
def _ac_name(value):
sanitized = ACNAME_REGEX.sub("-", value.lower()).strip("-")
if sanitized == "":
return str(uuid4())
return sanitized
@staticmethod
def _build_volumes(docker_config):
"""
Builds the volumes definitions for the ACI.
"""
volumes = []
def get_name(docker_volume_path):
volume_name = DockerV1ToACIManifestTranslator._ac_name(docker_volume_path)
return "volume-%s" % volume_name
volume_list = docker_config["Volumes"] or docker_config["volumes"] or {}
for docker_volume_path in volume_list.keys():
if not docker_volume_path:
continue
volumes.append(
{
"name": get_name(docker_volume_path),
"path": docker_volume_path,
"readOnly": False,
}
)
return volumes
@staticmethod
def build_manifest(tag, manifest, synthetic_image_id):
"""
Builds an ACI manifest of an existing repository image.
"""
docker_layer_data = JSONPathDict(json.loads(manifest.leaf_layer.raw_v1_metadata))
config = docker_layer_data["config"] or JSONPathDict({})
namespace = tag.repository.namespace_name
repo_name = tag.repository.name
source_url = "%s://%s/%s/%s:%s" % (
app.config["PREFERRED_URL_SCHEME"],
app.config["SERVER_HOSTNAME"],
namespace,
repo_name,
tag.name,
)
# ACI requires that the execution command be absolutely referenced. Therefore, if we find
# a relative command, we give it as an argument to /bin/sh to resolve and execute for us.
entrypoint = config["Entrypoint"] or []
exec_path = entrypoint + (config["Cmd"] or [])
if exec_path and not exec_path[0].startswith("/"):
exec_path = ["/bin/sh", "-c", '""%s""' % " ".join(exec_path)]
# TODO: ACI doesn't support : in the name, so remove any ports.
hostname = app.config["SERVER_HOSTNAME"]
hostname = hostname.split(":", 1)[0]
# Calculate the environment variables.
docker_env_vars = config.get("Env") or []
env_vars = []
for var in docker_env_vars:
pieces = var.split("=")
if len(pieces) != 2:
continue
env_vars.append(pieces)
manifest = {
"acKind": "ImageManifest",
"acVersion": "0.6.1",
"name": "%s/%s/%s" % (hostname.lower(), namespace.lower(), repo_name.lower()),
"labels": [
{"name": "version", "value": tag.name,},
{"name": "arch", "value": docker_layer_data.get("architecture") or "amd64"},
{"name": "os", "value": docker_layer_data.get("os") or "linux"},
],
"app": {
"exec": exec_path,
# Below, `or 'root'` is required to replace empty string from Dockerfiles.
"user": config.get("User") or "root",
"group": config.get("Group") or "root",
"eventHandlers": [],
"workingDirectory": config.get("WorkingDir") or "/",
"environment": [{"name": key, "value": value} for (key, value) in env_vars],
"isolators": DockerV1ToACIManifestTranslator._build_isolators(config),
"mountPoints": DockerV1ToACIManifestTranslator._build_volumes(config),
"ports": DockerV1ToACIManifestTranslator._build_ports(config),
"annotations": [
{"name": "created", "value": docker_layer_data.get("created") or ""},
{"name": "homepage", "value": source_url},
{"name": "quay.io/derived-image", "value": synthetic_image_id},
],
},
}
return manifest

74
image/appc/test/test_appc.py
View File

@ -1,74 +0,0 @@
import pytest
from image.appc import DockerV1ToACIManifestTranslator
from util.dict_wrappers import JSONPathDict
EXAMPLE_MANIFEST_OBJ = {
"architecture": "amd64",
"config": {
"Hostname": "1d811a9194c4",
"Domainname": "",
"User": "",
"AttachStdin": False,
"AttachStdout": False,
"AttachStderr": False,
"ExposedPorts": {"2379/tcp": {}, "2380/tcp": {}},
"Tty": False,
"OpenStdin": False,
"StdinOnce": False,
"Env": ["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],
"Cmd": ["/usr/local/bin/etcd"],
"ArgsEscaped": True,
"Image": "sha256:4c86d1f362d42420c137846fae31667ee85ce6f2cab406cdff26a8ff8a2c31c4",
"Volumes": None,
"WorkingDir": "",
"Entrypoint": None,
"OnBuild": [],
"Labels": {},
},
"container": "5a3565ce9b808a0eb0bcbc966dad624f76ad308ad24e11525b5da1201a1df135",
"container_config": {
"Hostname": "1d811a9194c4",
"Domainname": "",
"User": "",
"AttachStdin": False,
"AttachStdout": False,
"AttachStderr": False,
"ExposedPorts": {"2379/tcp": {}, "2380/tcp": {}},
"Tty": False,
"OpenStdin": False,
"StdinOnce": False,
"Env": ["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],
"Cmd": ["/bin/sh", "-c", '#(nop) CMD ["/usr/local/bin/etcd"]'],
"ArgsEscaped": True,
"Image": "sha256:4c86d1f362d42420c137846fae31667ee85ce6f2cab406cdff26a8ff8a2c31c4",
"Volumes": None,
"WorkingDir": "",
"Entrypoint": None,
"OnBuild": [],
"Labels": {},
},
"created": "2016-11-11T19:03:55.137387628Z",
"docker_version": "1.11.1",
"id": "3314a3781a526fe728e2e96cfcfb3cc0de901b5c102e6204e8b0155c8f7d5fd2",
"os": "linux",
"parent": "625342ec4d0f3d7a96fd3bb1ef0b4b0b6bc65ebb3d252fd33af0691f7984440e",
"throwaway": True,
}
@pytest.mark.parametrize(
"vcfg,expected",
[
({"Volumes": None}, []),
({"Volumes": {}}, []),
({"Volumes": {"/bin": {}}}, [{"name": "volume-bin", "path": "/bin", "readOnly": False}]),
({"volumes": None}, []),
({"volumes": {}}, []),
({"volumes": {"/bin": {}}}, [{"name": "volume-bin", "path": "/bin", "readOnly": False}]),
],
)
def test_volume_version_easy(vcfg, expected):
output = DockerV1ToACIManifestTranslator._build_volumes(JSONPathDict(vcfg))
assert output == expected

89
image/common.py
View File

@ -1,89 +0,0 @@
import tarfile
from util.registry.gzipwrap import GzipWrap
class TarImageFormatter(object):
"""
Base class for classes which produce a tar containing image and layer data.
"""
def build_stream(
self,
tag,
manifest,
synthetic_image_id,
layer_iterator,
tar_stream_getter_iterator,
reporter=None,
):
"""
Builds and streams a synthetic .tar.gz that represents the formatted tar created by this
class's implementation.
"""
return GzipWrap(
self.stream_generator(
tag,
manifest,
synthetic_image_id,
layer_iterator,
tar_stream_getter_iterator,
reporter=reporter,
)
)
def stream_generator(
self,
tag,
manifest,
synthetic_image_id,
layer_iterator,
tar_stream_getter_iterator,
reporter=None,
):
raise NotImplementedError
def tar_file(self, name, contents, mtime=None):
"""
Returns the tar binary representation for a file with the given name and file contents.
"""
assert isinstance(contents, bytes)
length = len(contents)
tar_data = self.tar_file_header(name, length, mtime=mtime)
tar_data += contents
tar_data += self.tar_file_padding(length)
return tar_data
def tar_file_padding(self, length):
"""
Returns tar file padding for file data of the given length.
"""
if length % 512 != 0:
return b"\0" * (512 - (length % 512))
return b""
def tar_file_header(self, name, file_size, mtime=None):
"""
Returns tar file header data for a file with the given name and size.
"""
info = tarfile.TarInfo(name=name)
info.type = tarfile.REGTYPE
info.size = file_size
if mtime is not None:
info.mtime = mtime
return info.tobuf()
def tar_folder(self, name, mtime=None):
"""
Returns tar file header data for a folder with the given name.
"""
info = tarfile.TarInfo(name=name)
info.type = tarfile.DIRTYPE
if mtime is not None:
info.mtime = mtime
# allow the directory to be readable by non-root users
info.mode = 0o755
return info.tobuf()

16
image/docker/schema1.py
View File

@ -220,7 +220,17 @@ class DockerSchema1Manifest(ManifestInterface):
Raises a ManifestException on failure.
"""
# Already validated.
# Validate the parent image IDs.
encountered_ids = set()
for layer in self.layers:
if layer.v1_metadata.parent_image_id:
if layer.v1_metadata.parent_image_id not in encountered_ids:
raise ManifestException(
"Unknown parent image %s" % layer.v1_metadata.parent_image_id
)
if layer.v1_metadata.image_id:
encountered_ids.add(layer.v1_metadata.image_id)
@property
def is_signed(self):
@ -283,6 +293,10 @@ class DockerSchema1Manifest(ManifestInterface):
@property
def layers_compressed_size(self):
return sum(l.compressed_size for l in self.layers if l.compressed_size is not None)
@property
def config_media_type(self):
return None
@property

22
image/docker/schema2/list.py
View File

@ -6,7 +6,7 @@ from jsonschema import validate as validate_schema, ValidationError
from digest import digest_tools
from image.shared import ManifestException
from image.shared.interfaces import ManifestInterface
from image.shared.interfaces import ManifestListInterface
from image.shared.schemautil import LazyManifestLoader
from image.docker.schema1 import DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE
from image.docker.schema1 import DockerSchema1Manifest
@ -53,7 +53,7 @@ class MismatchManifestException(MalformedSchema2ManifestList):
pass
class DockerSchema2ManifestList(ManifestInterface):
class DockerSchema2ManifestList(ManifestListInterface):
METASCHEMA = {
"type": "object",
"properties": {
@ -228,6 +228,10 @@ class DockerSchema2ManifestList(ManifestInterface):
def layers_compressed_size(self):
return None
@property
def config_media_type(self):
return None
@lru_cache(maxsize=1)
def manifests(self, content_retriever):
"""
@ -249,6 +253,20 @@ class DockerSchema2ManifestList(ManifestInterface):
for m in manifests
]
@property
def amd64_linux_manifest_digest(self):
""" Returns the digest of the AMD64+Linux manifest in this list, if any, or None
if none.
"""
for manifest_ref in self._parsed[DOCKER_SCHEMA2_MANIFESTLIST_MANIFESTS_KEY]:
platform = manifest_ref[DOCKER_SCHEMA2_MANIFESTLIST_PLATFORM_KEY]
architecture = platform[DOCKER_SCHEMA2_MANIFESTLIST_ARCHITECTURE_KEY]
os = platform[DOCKER_SCHEMA2_MANIFESTLIST_OS_KEY]
if architecture == "amd64" and os == "linux":
return manifest_ref[DOCKER_SCHEMA2_MANIFESTLIST_DIGEST_KEY]
return None
def validate(self, content_retriever):
"""
Performs validation of required assertions about the manifest.

8
image/docker/schema2/manifest.py
View File

@ -172,7 +172,7 @@ class DockerSchema2Manifest(ManifestInterface):
Raises a ManifestException on failure.
"""
# Nothing to validate.
self._get_built_config(content_retriever)
@property
def is_manifest_list(self):
@ -222,6 +222,12 @@ class DockerSchema2Manifest(ManifestInterface):
def layers_compressed_size(self):
return sum(layer.compressed_size for layer in self.filesystem_layers)
@property
def config_media_type(self):
return self._parsed[DOCKER_SCHEMA2_MANIFEST_CONFIG_KEY][
DOCKER_SCHEMA2_MANIFEST_MEDIATYPE_KEY
]
@property
def has_remote_layer(self):
for layer in self.filesystem_layers:

21
image/docker/schema2/test/test_list.py
View File

@ -50,7 +50,7 @@ MANIFESTLIST_BYTES = json.dumps(
},
{
"mediaType": "application/vnd.docker.distribution.manifest.v1+json",
"size": 878,
"size": 1051,
"digest": "sha256:5b",
"platform": {"architecture": "amd64", "os": "linux", "features": ["sse4"]},
},
@ -84,6 +84,8 @@ def test_valid_manifestlist():
assert manifestlist.bytes.as_encoded_str() == MANIFESTLIST_BYTES
assert manifestlist.manifest_dict == json.loads(MANIFESTLIST_BYTES)
assert manifestlist.get_layers(retriever) is None
assert manifestlist.config_media_type is None
assert manifestlist.layers_compressed_size is None
assert not manifestlist.blob_digests
for index, manifest in enumerate(manifestlist.manifests(retriever)):
@ -114,6 +116,8 @@ def test_valid_manifestlist():
# Ensure it validates.
manifestlist.validate(retriever)
assert manifestlist.amd64_linux_manifest_digest == "sha256:5b"
def test_get_schema1_manifest_no_matching_list():
manifestlist = DockerSchema2ManifestList(Bytes.for_string_or_unicode(NO_AMD_MANIFESTLIST_BYTES))
@ -121,6 +125,7 @@ def test_get_schema1_manifest_no_matching_list():
assert manifestlist.media_type == "application/vnd.docker.distribution.manifest.list.v2+json"
assert manifestlist.bytes.as_encoded_str() == NO_AMD_MANIFESTLIST_BYTES
assert manifestlist.amd64_linux_manifest_digest is None
compatible_manifest = manifestlist.get_schema1_manifest("foo", "bar", "baz", retriever)
assert compatible_manifest is None
@ -130,10 +135,22 @@ def test_builder():
existing = DockerSchema2ManifestList(Bytes.for_string_or_unicode(MANIFESTLIST_BYTES))
builder = DockerSchema2ManifestListBuilder()
for index, manifest in enumerate(existing.manifests(retriever)):
builder.add_manifest(manifest.manifest_obj, "amd64", "os")
builder.add_manifest(manifest.manifest_obj, "amd64", "linux")
built = builder.build()
assert len(built.manifests(retriever)) == 2
assert built.amd64_linux_manifest_digest is not None
def test_builder_no_amd():
existing = DockerSchema2ManifestList(Bytes.for_string_or_unicode(MANIFESTLIST_BYTES))
builder = DockerSchema2ManifestListBuilder()
for index, manifest in enumerate(existing.manifests(retriever)):
builder.add_manifest(manifest.manifest_obj, "intel386", "os")
built = builder.build()
assert len(built.manifests(retriever)) == 2
assert built.amd64_linux_manifest_digest is None
def test_invalid_manifestlist():

4
image/docker/schema2/test/test_manifest.py
View File

@ -119,6 +119,8 @@ def test_valid_manifest():
assert manifest.media_type == "application/vnd.docker.distribution.manifest.v2+json"
assert not manifest.has_remote_layer
assert manifest.has_legacy_image
assert manifest.config_media_type == "application/vnd.docker.container.image.v1+json"
assert manifest.layers_compressed_size == 123721
retriever = ContentRetrieverForTesting.for_config(
{
@ -171,6 +173,8 @@ def test_valid_remote_manifest():
)
assert manifest.media_type == "application/vnd.docker.distribution.manifest.v2+json"
assert manifest.has_remote_layer
assert manifest.config_media_type == "application/vnd.docker.container.image.v1+json"
assert manifest.layers_compressed_size == 123721
assert len(manifest.filesystem_layers) == 4
assert manifest.filesystem_layers[0].compressed_size == 1234

149
image/docker/squashed.py
View File

@ -1,149 +0,0 @@
import copy
import json
import math
import calendar
from app import app
from image.common import TarImageFormatter
from util.registry.gzipwrap import GZIP_BUFFER_SIZE
from util.registry.streamlayerformat import StreamLayerMerger
class FileEstimationException(Exception):
"""
Exception raised by build_docker_load_stream if the estimated size of the layer tar was lower
than the actual size.
This means the sent tar header is wrong, and we have to fail.
"""
pass
class SquashedDockerImageFormatter(TarImageFormatter):
"""
Image formatter which produces a squashed image compatible with the `docker load` command.
"""
# Multiplier against the image size reported by Docker to account for the tar metadata.
# Note: This multiplier was not formally calculated in anyway and should be adjusted overtime
# if/when we encounter issues with it. Unfortunately, we cannot make it too large or the Docker
# daemon dies when trying to load the entire tar into memory.
SIZE_MULTIPLIER = 1.2
def stream_generator(
self,
tag,
parsed_manifest,
synthetic_image_id,
layer_iterator,
tar_stream_getter_iterator,
reporter=None,
):
image_mtime = 0
created = parsed_manifest.created_datetime
if created is not None:
image_mtime = calendar.timegm(created.utctimetuple())
# Docker import V1 Format (.tar):
# repositories - JSON file containing a repo -> tag -> image map
# {image ID folder}:
# json - The layer JSON
# layer.tar - The tarballed contents of the layer
# VERSION - The docker import version: '1.0'
layer_merger = StreamLayerMerger(tar_stream_getter_iterator, reporter=reporter)
# Yield the repositories file:
synthetic_layer_info = {}
synthetic_layer_info[tag.name + ".squash"] = synthetic_image_id
hostname = app.config["SERVER_HOSTNAME"]
repositories = {}
namespace = tag.repository.namespace_name
repository = tag.repository.name
repositories[hostname + "/" + namespace + "/" + repository] = synthetic_layer_info
yield self.tar_file(
"repositories", json.dumps(repositories).encode("utf-8"), mtime=image_mtime
)
# Yield the image ID folder.
yield self.tar_folder(synthetic_image_id, mtime=image_mtime)
# Yield the JSON layer data.
layer_json = SquashedDockerImageFormatter._build_layer_json(
parsed_manifest, synthetic_image_id
)
yield self.tar_file(
synthetic_image_id + "/json", json.dumps(layer_json).encode("utf-8"), mtime=image_mtime
)
# Yield the VERSION file.
yield self.tar_file(synthetic_image_id + "/VERSION", b"1.0", mtime=image_mtime)
# Yield the merged layer data's header.
estimated_file_size = 0
for layer in layer_iterator:
estimated_file_size += layer.estimated_size(
SquashedDockerImageFormatter.SIZE_MULTIPLIER
)
# Make sure the estimated file size is an integer number of bytes.
estimated_file_size = int(math.ceil(estimated_file_size))
yield self.tar_file_header(
synthetic_image_id + "/layer.tar", estimated_file_size, mtime=image_mtime
)
# Yield the contents of the merged layer.
yielded_size = 0
for entry in layer_merger.get_generator():
yield entry
yielded_size += len(entry)
# If the yielded size is more than the estimated size (which is unlikely but possible), then
# raise an exception since the tar header will be wrong.
if yielded_size > estimated_file_size:
leaf_image_id = parsed_manifest.leaf_layer_v1_image_id
message = "For %s/%s:%s (%s:%s): Expected %s bytes, found %s bytes" % (
namespace,
repository,
tag,
parsed_manifest.digest,
leaf_image_id,
estimated_file_size,
yielded_size,
)
raise FileEstimationException(message)
# If the yielded size is less than the estimated size (which is likely), fill the rest with
# zeros.
if yielded_size < estimated_file_size:
to_yield = estimated_file_size - yielded_size
while to_yield > 0:
yielded = min(to_yield, GZIP_BUFFER_SIZE)
yield b"\0" * yielded
to_yield -= yielded
# Yield any file padding to 512 bytes that is necessary.
yield self.tar_file_padding(estimated_file_size)
# Last two records are empty in tar spec.
yield b"\0" * 512
yield b"\0" * 512
@staticmethod
def _build_layer_json(manifest, synthetic_image_id):
updated_json = json.loads(manifest.leaf_layer.raw_v1_metadata)
updated_json["id"] = synthetic_image_id
if "parent" in updated_json:
del updated_json["parent"]
if "config" in updated_json and "Image" in updated_json["config"]:
updated_json["config"]["Image"] = synthetic_image_id
if "container_config" in updated_json and "Image" in updated_json["container_config"]:
updated_json["container_config"]["Image"] = synthetic_image_id
return updated_json

20
image/docker/test/test_schema1.py
View File

@ -37,10 +37,12 @@ MANIFEST_BYTES = json.dumps(
"tag": "latest",
"architecture": "amd64",
"fsLayers": [
{"blobSum": "sha256:cd8567d70002e957612902a8e985ea129d831ebe04057d88fb644857caa45d11"},
{"blobSum": "sha256:cc8567d70002e957612902a8e985ea129d831ebe04057d88fb644857caa45d11"},
{"blobSum": "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef"},
],
"history": [
{"v1Compatibility": '{"id":"sizedid", "parent": "someid", "Size": 1234}'},
{"v1Compatibility": '{"id":"someid", "parent": "anotherid"}'},
{"v1Compatibility": '{"id":"anotherid"}'},
],
@ -71,10 +73,12 @@ def test_valid_manifest():
assert manifest.namespace == ""
assert manifest.repo_name == "hello-world"
assert manifest.tag == "latest"
assert manifest.image_ids == {"someid", "anotherid"}
assert manifest.parent_image_ids == {"anotherid"}
assert manifest.image_ids == {"sizedid", "someid", "anotherid"}
assert manifest.parent_image_ids == {"someid", "anotherid"}
assert manifest.layers_compressed_size == 1234
assert manifest.config_media_type is None
assert len(manifest.layers) == 2
assert len(manifest.layers) == 3
assert manifest.layers[0].v1_metadata.image_id == "anotherid"
assert manifest.layers[0].v1_metadata.parent_image_id is None
@ -82,10 +86,14 @@ def test_valid_manifest():
assert manifest.layers[1].v1_metadata.image_id == "someid"
assert manifest.layers[1].v1_metadata.parent_image_id == "anotherid"
assert manifest.layers[2].v1_metadata.image_id == "sizedid"
assert manifest.layers[2].v1_metadata.parent_image_id == "someid"
assert manifest.layers[0].compressed_size is None
assert manifest.layers[1].compressed_size is None
assert manifest.layers[2].compressed_size == 1234
assert manifest.leaf_layer == manifest.layers[1]
assert manifest.leaf_layer == manifest.layers[2]
assert manifest.created_datetime is None
unsigned = manifest.unsigned()
@ -97,8 +105,8 @@ def test_valid_manifest():
assert unsigned.digest != manifest.digest
image_layers = list(manifest.get_layers(None))
assert len(image_layers) == 2
for index in range(0, 2):
assert len(image_layers) == 3
for index in range(0, 3):
assert image_layers[index].layer_id == manifest.layers[index].v1_metadata.image_id
assert image_layers[index].blob_digest == manifest.layers[index].digest
assert image_layers[index].command == manifest.layers[index].v1_metadata.command

22
image/oci/index.py
View File

@ -41,7 +41,7 @@ from jsonschema import validate as validate_schema, ValidationError
from digest import digest_tools
from image.shared import ManifestException
from image.shared.interfaces import ManifestInterface
from image.shared.interfaces import ManifestListInterface
from image.shared.schemautil import LazyManifestLoader
from image.oci import OCI_IMAGE_INDEX_CONTENT_TYPE, OCI_IMAGE_MANIFEST_CONTENT_TYPE
from image.oci.descriptor import get_descriptor_schema
@ -81,7 +81,7 @@ class MalformedIndex(ManifestException):
pass
class OCIIndex(ManifestInterface):
class OCIIndex(ManifestListInterface):
METASCHEMA = {
"type": "object",
"properties": {
@ -227,6 +227,10 @@ class OCIIndex(ManifestInterface):
def layers_compressed_size(self):
return None
@property
def config_media_type(self):
return None
@lru_cache(maxsize=1)
def manifests(self, content_retriever):
"""
@ -275,6 +279,20 @@ class OCIIndex(ManifestInterface):
def has_legacy_image(self):
return False
@property
def amd64_linux_manifest_digest(self):
""" Returns the digest of the AMD64+Linux manifest in this list, if any, or None
if none.
"""
for manifest_ref in self._parsed[INDEX_MANIFESTS_KEY]:
platform = manifest_ref[INDEX_PLATFORM_KEY]
architecture = platform.get(INDEX_ARCHITECTURE_KEY, None)
os = platform.get(INDEX_OS_KEY, None)
if architecture == "amd64" and os == "linux":
return manifest_ref[INDEX_DIGEST_KEY]
return None
def get_requires_empty_layer_blob(self, content_retriever):
return False

4
image/oci/manifest.py
View File

@ -197,6 +197,10 @@ class OCIManifest(ManifestInterface):
"""
return self.filesystem_layers[-1]
@property
def config_media_type(self):
return self._parsed[OCI_MANIFEST_CONFIG_KEY][OCI_MANIFEST_MEDIATYPE_KEY]
@property
def layers_compressed_size(self):
return sum(layer.compressed_size for layer in self.filesystem_layers)

45
image/oci/test/test_oci_index.py
View File

@ -34,6 +34,35 @@ SAMPLE_INDEX = """{
}"""
SAMPLE_INDEX_NO_AMD = """{
"schemaVersion": 2,
"manifests": [
{
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"size": 7143,
"digest": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f",
"platform": {
"architecture": "ppc64le",
"os": "linux"
}
},
{
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"size": 7682,
"digest": "sha256:5b0bcabd1ed22e9fb1310cf6c2dec7cdef19f0ad69efa1f392e94a4333501270",
"platform": {
"architecture": "intel386",
"os": "linux"
}
}
],
"annotations": {
"com.example.key1": "value1",
"com.example.key2": "value2"
}
}"""
def test_parse_basic_index():
index = OCIIndex(Bytes.for_string_or_unicode(SAMPLE_INDEX))
assert index.is_manifest_list
@ -43,6 +72,10 @@ def test_parse_basic_index():
"sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f",
"sha256:5b0bcabd1ed22e9fb1310cf6c2dec7cdef19f0ad69efa1f392e94a4333501270",
]
assert (
index.amd64_linux_manifest_digest
== "sha256:5b0bcabd1ed22e9fb1310cf6c2dec7cdef19f0ad69efa1f392e94a4333501270"
)
def test_config_missing_required():
@ -56,3 +89,15 @@ def test_config_missing_required():
def test_invalid_index():
with pytest.raises(MalformedIndex):
OCIIndex(Bytes.for_string_or_unicode("{}"))
def test_index_without_amd():
index = OCIIndex(Bytes.for_string_or_unicode(SAMPLE_INDEX_NO_AMD))
assert index.is_manifest_list
assert index.digest == "sha256:a0ed0f2b3949bc731063320667062307faf4245f6872dc5bc98ee6ea5443f169"
assert index.local_blob_digests == []
assert index.child_manifest_digests() == [
"sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f",
"sha256:5b0bcabd1ed22e9fb1310cf6c2dec7cdef19f0ad69efa1f392e94a4333501270",
]
assert index.amd64_linux_manifest_digest is None

19
image/shared/interfaces.py
View File

@ -56,6 +56,12 @@ class ManifestInterface(object):
Returns None if this cannot be computed locally.
"""
@abstractproperty
def config_media_type(self):
""" Returns the media type of the config of this manifest or None if
this manifest does not support a configuration type.
"""
@abstractmethod
def validate(self, content_retriever):
"""
@ -184,6 +190,19 @@ class ManifestInterface(object):
"""
@add_metaclass(ABCMeta)
class ManifestListInterface(object):
"""
Defines the interface for the various manifest list types supported.
"""
@abstractmethod
def amd64_linux_manifest_digest(self):
""" Returns the digest of the AMD64+Linux manifest in this list, if any, or None
if none.
"""
@add_metaclass(ABCMeta)
class ContentRetriever(object):
"""

3
initdb.py
View File

@ -174,6 +174,7 @@ def __create_manifest_and_tags(
config = {
"id": current_id,
"Size": len(content),
}
if parent_id:
config["parent"] = parent_id
@ -1239,6 +1240,8 @@ WHITELISTED_EMPTY_MODELS = [
"LogEntry",
"LogEntry2",
"ManifestSecurityStatus",
"ManifestLegacyImage",
"Image",
]

1
requirements-nover.txt
View File

@ -34,6 +34,7 @@ geoip2
gevent
gipc
gunicorn
hashids
hiredis
html5lib==0.9999999 # pinned due to xhtml2pdf
httmock

2
requirements.txt
View File

@ -68,9 +68,9 @@ futures==3.1.1
geoip2==3.0.0
gevent==1.4.0
gipc==1.0.1
gpg==1.10.0
greenlet==0.4.15
gunicorn==20.0.4
hashids==1.2.0
hiredis==1.0.1
html5lib==1.0.1
httmock==1.3.0

11
static/directives/repo-view/image-tag-tooltip.html
View File

@ -1,11 +0,0 @@
<div class="popover image-tag-tooltip" tabindex="-1">
<div class="image-tag-tooltip-header"
ng-style="::{'backgroundColor': trackEntryForImage[tag.image_id].color,
'color': constrastingColor( trackEntryForImage[tag.image_id].color)}">
Image {{ tag.image_id.substr(0, 12) }}
</div>
<ul class="image-tag-tooltip-tags">
<li ng-repeat="tag in imageMap[tag.image_id] | limitTo:5"><i class="fa fa-tag"></i>{{ tag.name }}</li>
</ul>
<div class="image-tag-tooltip-tags-more" ng-if="imageMap[tag.image_id].length > 5">and {{ imageMap[tag.image_id].length - 5 }} more tags</div>
</div>

11
static/directives/repo-view/manifest-tag-tooltip.html Normal file
View File

@ -0,0 +1,11 @@
<div class="popover image-tag-tooltip" tabindex="-1">
<div class="image-tag-tooltip-header"
ng-style="::{'backgroundColor': trackEntryForManifest[tag.manifest_digest].color,
'color': constrastingColor(trackEntryForManifest[tag.manifest_digest].color)}">
Manifest {{ tag.manifest_digest.substr(7, 12) }}
</div>
<ul class="image-tag-tooltip-tags">
<li ng-repeat="tag in manifestMap[tag.manifest_digest] | limitTo:5"><i class="fa fa-tag"></i>{{ tag.name }}</li>
</ul>
<div class="image-tag-tooltip-tags-more" ng-if="manifestMap[tag.manifest_digest].length > 5">and {{ manifestMap[tag.manifest_digest].length - 5 }} more tags</div>
</div>

70
static/directives/repo-view/repo-panel-tags.html
View File

@ -32,9 +32,9 @@
<i class="fa fa-git"></i>Commit SHAs
</div>
<div class="cor-checkable-menu-item" item-filter="imageIDFilter(it.image_id, item)"
ng-repeat="it in imageTrackEntries" ng-if="::it.visible">
<i class="fa fa-circle-o" ng-style="::{'color': it.color}"></i> {{ ::it.image_id.substr(0, 12) }}
<div class="cor-checkable-menu-item" item-filter="manifestDigestFilter(mt.manifest_digest, item)"
ng-repeat="mt in manifestTrackEntries" ng-if="::it.visible">
<i class="fa fa-circle-o" ng-style="::{'color': ,t.color}"></i>
</div>
</span>
@ -116,16 +116,16 @@
style="width: 140px;">
<a ng-click="orderBy('expiration_date')" data-title="When the tag expires" data-container="body" bs-tooltip>Expires</a>
</td>
<td class="hidden-xs hidden-sm" ng-if="imageTracks.length > maxTrackCount"
<td class="hidden-xs hidden-sm" ng-if="manifestTracks.length > maxTrackCount"
style="width: 20px; position: relative;">
</td>
<td class="hidden-xs hidden-sm"
ng-class="tablePredicateClass('image_id', options.predicate, options.reverse)"
ng-class="tablePredicateClass('manifest_digest', options.predicate, options.reverse)"
style="width: 140px;">
<a ng-click="orderBy('image_id')">Manifest</a>
<a ng-click="orderBy('manifest_digest')">Manifest</a>
</td>
<td class="hidden-xs hidden-sm hidden-md image-track" ng-repeat="it in imageTracks"
ng-if="imageTracks.length <= maxTrackCount"></td>
<td class="hidden-xs hidden-sm hidden-md image-track" ng-repeat="mt in manifestTracks"
ng-if="manifestTracks.length <= maxTrackCount"></td>
<td class="options-col"></td>
<td class="options-col"></td>
<td class="hidden-xs hidden-sm" style="width: 4px"></td>
@ -167,14 +167,6 @@
See Child Manifests
</span>
<!-- No Digest -->
<span class="nodigest" ng-if="::!tag.manifest_digest"
data-title="The tag does not have a V2 digest and so is unsupported for scan"
bs-tooltip>
<span class="donut-chart" width="22" data="[{'index': 0, 'value': 1, 'color': '#eee'}]"></span>
Unsupported
</span>
<!-- Manifest security view -->
<manifest-security-view repository="::repository" manifest-digest="::tag.manifest_digest"
ng-if="::(tag.manifest_digest && !tag.is_manifest_list)">
@ -198,11 +190,11 @@
<!-- Manifest link -->
<td class="hidden-xs hidden-sm hidden-md image-track"
ng-if="imageTracks.length > maxTrackCount">
ng-if="manifestTracks.length > maxTrackCount">
<span class="image-track-filled-dot"
ng-if="::trackEntryForImage[tag.image_id]"
ng-style="::{'backgroundColor': trackEntryForImage[tag.image_id].color}"
ng-click="::selectTrack(trackEntryForImage[tag.image_id])"
ng-if="::trackEntryForManifest[tag.manifest_digest]"
ng-style="::{'backgroundColor': trackEntryForManifest[tag.manifest_digest].color}"
ng-click="::selectTrack(trackEntryForManifest[tag.manifest_digest])"
data-template-url="/static/directives/repo-view/image-tag-tooltip.html"
data-placement="left"
data-trigger="hover"
@ -213,22 +205,22 @@
<td class="hidden-xs hidden-sm image-id-col">
<manifest-link repository="repository" image-id="tag.image_id" manifest-digest="tag.manifest_digest"></manifest-link>
</td>
<td class="hidden-xs hidden-sm hidden-md image-track" ng-repeat="it in imageTracks"
ng-if="imageTracks.length <= maxTrackCount">
<td class="hidden-xs hidden-sm hidden-md image-track" ng-repeat="mt in manifestTracks"
ng-if="manifestTracks.length <= maxTrackCount">
<span class="image-track-dot"
ng-if="::it.entryByImageId[tag.image_id]"
ng-style="::{'borderColor': trackEntryForImage[tag.image_id].color}"
ng-click="::selectTrack(trackEntryForImage[tag.image_id])"
data-template-url="/static/directives/repo-view/image-tag-tooltip.html"
ng-if="::mt.entryByManifestDigest[tag.manifest_digest]"
ng-style="::{'borderColor': trackEntryForManifest[tag.manifest_digest].color}"
ng-click="::selectTrack(trackEntryForManifest[tag.manifest_digest])"
data-template-url="/static/directives/repo-view/manifest-tag-tooltip.html"
data-placement="left"
data-trigger="hover"
data-animation="am-flip-x"
data-auto-close="1"
bs-popover></span>
<span class="image-track-line"
ng-if="::getTrackEntryForIndex(it, $parent.$parent.$index)"
ng-class="::trackLineClass(it, $parent.$parent.$parent.$index)"
ng-style="::{'borderColor': getTrackEntryForIndex(it, $parent.$parent.$parent.$index).color}"></span>
ng-if="::getTrackEntryForIndex(mt, $parent.$parent.$index)"
ng-class="::trackLineClass(mt, $parent.$parent.$parent.$index)"
ng-style="::{'borderColor': getTrackEntryForIndex(mt, $parent.$parent.$parent.$index).color}"></span>
</td>
<td class="options-col">
<i class="fa fa-download" data-title="Fetch Tag" bs-tooltip
@ -288,12 +280,12 @@
<manifest-link repository="repository" manifest-digest="manifest.digest"></manifest-link>
</td>
<td class="hidden-xs hidden-sm hidden-md image-track" ng-repeat="it in imageTracks"
ng-if="imageTracks.length <= maxTrackCount" bindonce>
<td class="hidden-xs hidden-sm hidden-md image-track" ng-repeat="mt in manifestTracks"
ng-if="manifestTracks.length <= maxTrackCount" bindonce>
<span class="image-track-line"
ng-if="::getTrackEntryForIndex(it, $parent.$parent.$parent.$parent.$index)"
ng-class="::trackLineExpandedClass(it, $parent.$parent.$parent.$parent.$parent.$index)"
ng-style="::{'borderColor': getTrackEntryForIndex(it, $parent.$parent.$parent.$parent.$parent.$index).color}"></span>
ng-if="::getTrackEntryForIndex(mt, $parent.$parent.$parent.$parent.$index)"
ng-class="::trackLineExpandedClass(mt, $parent.$parent.$parent.$parent.$parent.$index)"
ng-style="::{'borderColor': getTrackEntryForIndex(mt, $parent.$parent.$parent.$parent.$parent.$index).color}"></span>
</td>
<td class="options-col"></td>
@ -320,12 +312,12 @@
</div>
</td>
<td class="hidden-xs hidden-sm hidden-md image-track" ng-repeat="it in imageTracks"
ng-if="imageTracks.length <= maxTrackCount" bindonce>
<td class="hidden-xs hidden-sm hidden-md image-track" ng-repeat="mt in manifestTracks"
ng-if="manifestTracks.length <= maxTrackCount" bindonce>
<span class="image-track-line"
ng-if="::getTrackEntryForIndex(it, $parent.$parent.$index)"
ng-class="::trackLineExpandedClass(it, $parent.$parent.$parent.$index)"
ng-style="::{'borderColor': getTrackEntryForIndex(it, $parent.$parent.$parent.$index).color}"></span>
ng-if="::getTrackEntryForIndex(mt, $parent.$parent.$index)"
ng-class="::trackLineExpandedClass(mt, $parent.$parent.$parent.$index)"
ng-style="::{'borderColor': getTrackEntryForIndex(mt, $parent.$parent.$parent.$index).color}"></span>
</td>
<td></td>
<td class="options-col hidden-xs hidden-sm"><!-- Whitespace col --></td>

100
static/js/directives/repo-view/repo-panel-tags.js
View File

@ -89,78 +89,78 @@ angular.module('quay').directive('repoPanelTags', function () {
}
// Sort the tags by the predicate and the reverse, and map the information.
var imageIDs = [];
var ordered = TableService.buildOrderedItems(allTags, $scope.options,
['name'], ['last_modified_datetime', 'size']).entries;
['name', 'manifest_digest'], ['last_modified_datetime', 'size']).entries;
var checked = [];
var imageMap = {};
var imageIndexMap = {};
var manifestMap = {};
var manifestIndexMap = {};
var manifestDigests = [];
for (var i = 0; i < ordered.length; ++i) {
var tagInfo = ordered[i];
if (!tagInfo.image_id) {
if (!tagInfo.manifest_digest) {
continue;
}
if (!imageMap[tagInfo.image_id]) {
imageMap[tagInfo.image_id] = [];
imageIDs.push(tagInfo.image_id)
if (!manifestMap[tagInfo.manifest_digest]) {
manifestMap[tagInfo.manifest_digest] = [];
manifestDigests.push(tagInfo.manifest_digest)
}
imageMap[tagInfo.image_id].push(tagInfo);
manifestMap[tagInfo.manifest_digest].push(tagInfo);
if ($.inArray(tagInfo.name, $scope.selectedTags) >= 0) {
checked.push(tagInfo);
}
if (!imageIndexMap[tagInfo.image_id]) {
imageIndexMap[tagInfo.image_id] = {'start': i, 'end': i};
if (!manifestIndexMap[tagInfo.manifest_digest]) {
manifestIndexMap[tagInfo.manifest_digest] = {'start': i, 'end': i};
}
imageIndexMap[tagInfo.image_id]['end'] = i;
manifestIndexMap[tagInfo.manifest_digest]['end'] = i;
};
// Calculate the image tracks.
var colors = d3.scale.category10();
if (Object.keys(imageMap).length > 10) {
if (Object.keys(manifestMap).length > 10) {
colors = d3.scale.category20();
}
var imageTracks = [];
var imageTrackEntries = [];
var trackEntryForImage = {};
var manifestTracks = [];
var manifestTrackEntries = [];
var trackEntryForManifest = {};
var visibleStartIndex = ($scope.options.page * $scope.tagsPerPage);
var visibleEndIndex = (($scope.options.page + 1) * $scope.tagsPerPage);
imageIDs.sort().map(function(image_id) {
if (imageMap[image_id].length >= 2){
manifestDigests.sort().map(function(manifest_digest) {
if (manifestMap[manifest_digest].length >= 2){
// Create the track entry.
var imageIndexRange = imageIndexMap[image_id];
var colorIndex = imageTrackEntries.length;
var manifestIndexRange = manifestIndexMap[manifest_digest];
var colorIndex = manifestTrackEntries.length;
var trackEntry = {
'image_id': image_id,
'manifest_digest': manifest_digest,
'color': colors(colorIndex),
'count': imageMap[image_id].length,
'tags': imageMap[image_id],
'index_range': imageIndexRange,
'visible': visibleStartIndex <= imageIndexRange.end && imageIndexRange.start <= visibleEndIndex,
'count': manifestMap[manifest_digest].length,
'tags': manifestMap[manifest_digest],
'index_range': manifestIndexRange,
'visible': visibleStartIndex <= manifestIndexRange.end && manifestIndexRange.start <= visibleEndIndex,
};
trackEntryForImage[image_id] = trackEntry;
imageMap[image_id]['color'] = colors(colorIndex);
trackEntryForManifest[manifest_digest] = trackEntry;
manifestMap[manifest_digest]['color'] = colors(colorIndex);
// Find the track in which we can place the entry, if any.
var existingTrack = null;
for (var i = 0; i < imageTracks.length; ++i) {
for (var i = 0; i < manifestTracks.length; ++i) {
// For the current track, ensure that the start and end index
// for the current entry is outside of the range of the track's
// entries. If so, then we can add the entry to the track.
var currentTrack = imageTracks[i];
var currentTrack = manifestTracks[i];
var canAddToCurrentTrack = true;
for (var j = 0; j < currentTrack.entries.length; ++j) {
var currentTrackEntry = currentTrack.entries[j];
var entryInfo = imageIndexMap[currentTrackEntry.image_id];
if (Math.max(entryInfo.start, imageIndexRange.start) <= Math.min(entryInfo.end, imageIndexRange.end)) {
var entryInfo = manifestIndexMap[currentTrackEntry.image_id];
if (Math.max(entryInfo.start, manifestIndexRange.start) <= Math.min(entryInfo.end, manifestIndexRange.end)) {
canAddToCurrentTrack = false;
break;
}
@ -175,38 +175,38 @@ angular.module('quay').directive('repoPanelTags', function () {
// Add the entry to the track or create a new track if necessary.
if (existingTrack) {
existingTrack.entries.push(trackEntry)
existingTrack.entryByImageId[image_id] = trackEntry;
existingTrack.endIndex = Math.max(existingTrack.endIndex, imageIndexRange.end);
existingTrack.entryByManifestDigest[manifest_digest] = trackEntry;
existingTrack.endIndex = Math.max(existingTrack.endIndex, manifestIndexRange.end);
for (var j = imageIndexRange.start; j <= imageIndexRange.end; j++) {
for (var j = manifestIndexRange.start; j <= manifestIndexRange.end; j++) {
existingTrack.entryByIndex[j] = trackEntry;
}
} else {
var entryByImageId = {};
entryByImageId[image_id] = trackEntry;
var entryByManifestDigest = {};
entryByManifestDigest[manifest_digest] = trackEntry;
var entryByIndex = {};
for (var j = imageIndexRange.start; j <= imageIndexRange.end; j++) {
for (var j = manifestIndexRange.start; j <= manifestIndexRange.end; j++) {
entryByIndex[j] = trackEntry;
}
imageTracks.push({
manifestTracks.push({
'entries': [trackEntry],
'entryByImageId': entryByImageId,
'startIndex': imageIndexRange.start,
'endIndex': imageIndexRange.end,
'entryByManifestDigest': entryByManifestDigest,
'startIndex': manifestIndexRange.start,
'endIndex': manifestIndexRange.end,
'entryByIndex': entryByIndex,
});
}
imageTrackEntries.push(trackEntry);
manifestTrackEntries.push(trackEntry);
}
});
$scope.imageMap = imageMap;
$scope.imageTracks = imageTracks;
$scope.imageTrackEntries = imageTrackEntries;
$scope.trackEntryForImage = trackEntryForImage;
$scope.manifestMap = manifestMap;
$scope.manifestTracks = manifestTracks;
$scope.manifestTrackEntries = manifestTrackEntries;
$scope.trackEntryForManifest = trackEntryForManifest;
$scope.options.page = 0;
@ -241,7 +241,7 @@ angular.module('quay').directive('repoPanelTags', function () {
});
$scope.$watch('selectedTags', function(selectedTags) {
if (!selectedTags || !$scope.repository || !$scope.imageMap) { return; }
if (!selectedTags || !$scope.repository || !$scope.manifestMap) { return; }
$scope.checkedTags.setChecked(selectedTags.map(function(tag) {
return $scope.repositoryTags[tag];
@ -410,8 +410,8 @@ angular.module('quay').directive('repoPanelTags', function () {
return false;
};
$scope.imageIDFilter = function(image_id, tag) {
return tag.image_id == image_id;
$scope.manifestDigestFilter = function(manifest_digest, tag) {
return tag.manifest_digest == manifest_digest;
};
$scope.setTab = function(tab) {
@ -420,7 +420,7 @@ angular.module('quay').directive('repoPanelTags', function () {
$scope.selectTrack = function(it) {
$scope.checkedTags.checkByFilter(function(tag) {
return $scope.imageIDFilter(it.image_id, tag);
return $scope.manifestDigestFilter(it.manifest_digest, tag);
});
};

2
test/fixtures.py
View File

@ -26,7 +26,6 @@ from endpoints.appr import appr_bp
from endpoints.web import web
from endpoints.v1 import v1_bp
from endpoints.v2 import v2_bp
from endpoints.verbs import verbs as verbs_bp
from endpoints.webhooks import webhooks
from initdb import initialize_database, populate_database
@ -312,7 +311,6 @@ def app(appconfig, initialized_db):
app.register_blueprint(api_bp, url_prefix="/api")
app.register_blueprint(appr_bp, url_prefix="/cnr")
app.register_blueprint(web, url_prefix="/")
app.register_blueprint(verbs_bp, url_prefix="/c1")
app.register_blueprint(v1_bp, url_prefix="/v1")
app.register_blueprint(v2_bp, url_prefix="/v2")
app.register_blueprint(webhooks, url_prefix="/webhooks")

35
test/registry/fixtures.py
View File

@ -16,9 +16,8 @@ from app import storage
from data.database import (
close_db_filter,
configure,
DerivedStorageForImage,
QueueItem,
Image,
ImageStorage,
TagManifest,
TagManifestToManifest,
Manifest,
@ -30,6 +29,7 @@ from data.database import (
from data import model
from data.registry_model import registry_model
from endpoints.csrf import generate_csrf_token
from image.docker.schema2 import EMPTY_LAYER_BLOB_DIGEST
from util.log import logfile_path
from test.registry.liveserverfixture import LiveServerExecutor
@ -46,15 +46,22 @@ def registry_server_executor(app):
)
return "OK"
def delete_image(image_id):
image = Image.get(docker_image_id=image_id)
image.docker_image_id = "DELETED"
image.save()
return "OK"
def verify_replication_for(namespace, repo_name, tag_name):
repo_ref = registry_model.lookup_repository(namespace, repo_name)
assert repo_ref
tag = registry_model.get_repo_tag(repo_ref, tag_name)
assert tag
manifest = registry_model.get_manifest_for_tag(tag)
assert manifest
for layer in registry_model.list_manifest_layers(manifest, storage):
if layer.blob.digest != EMPTY_LAYER_BLOB_DIGEST:
QueueItem.select().where(
QueueItem.queue_name ** ("%" + layer.blob.uuid + "%")
).get()
def get_storage_replication_entry(image_id):
image = Image.get(docker_image_id=image_id)
QueueItem.select().where(QueueItem.queue_name ** ("%" + image.storage.uuid + "%")).get()
return "OK"
def set_feature(feature_name, value):
@ -81,10 +88,6 @@ def registry_server_executor(app):
return jsonify({"old_value": old_value})
def clear_derived_cache():
DerivedStorageForImage.delete().execute()
return "OK"
def clear_uncompressed_size(image_id):
image = model.image.get_image_by_id("devtable", "newrepo", image_id)
image.storage.uncompressed_size = None
@ -158,11 +161,9 @@ def registry_server_executor(app):
executor = LiveServerExecutor()
executor.register("generate_csrf", generate_csrf)
executor.register("set_supports_direct_download", set_supports_direct_download)
executor.register("delete_image", delete_image)
executor.register("get_storage_replication_entry", get_storage_replication_entry)
executor.register("verify_replication_for", verify_replication_for)
executor.register("set_feature", set_feature)
executor.register("set_config_key", set_config_key)
executor.register("clear_derived_cache", clear_derived_cache)
executor.register("clear_uncompressed_size", clear_uncompressed_size)
executor.register("add_token", add_token)
executor.register("break_database", break_database)

7
test/registry/protocol_v1.py
View File

@ -153,6 +153,9 @@ class V1Protocol(RegistryProtocol):
assert expected_failure == Failures.UNKNOWN_TAG
return None
if expected_failure == Failures.UNKNOWN_TAG:
return None
tag_image_id = image_ids[tag_name]
assert image_id_data.json() == tag_image_id
@ -331,7 +334,7 @@ class V1Protocol(RegistryProtocol):
namespace,
repo_name,
tag_name,
image,
image_id,
credentials=None,
expected_failure=None,
options=None,
@ -341,7 +344,7 @@ class V1Protocol(RegistryProtocol):
session,
"PUT",
"/v1/repositories/%s/tags/%s" % (self.repo_name(namespace, repo_name), tag_name),
data='"%s"' % image.id,
data='"%s"' % image_id,
auth=auth,
expected_status=(200, expected_failure, V1ProtocolSteps.PUT_TAG),
)

488
test/registry/registry_tests.py
View File

@ -835,10 +835,11 @@ def test_image_replication(
credentials=credentials,
)
# Ensure that entries were created for each image.
for image_id in list(result.image_ids.values()):
r = registry_server_executor.on(liveserver).get_storage_replication_entry(image_id)
assert r.text == "OK"
# Ensure that entries were created for each layer.
r = registry_server_executor.on(liveserver).verify_replication_for(
"devtable", "newrepo", "latest"
)
assert r.text == "OK"
def test_image_replication_empty_layers(
@ -872,10 +873,11 @@ def test_image_replication_empty_layers(
credentials=credentials,
)
# Ensure that entries were created for each image.
for image_id in list(result.image_ids.values()):
r = registry_server_executor.on(liveserver).get_storage_replication_entry(image_id)
assert r.text == "OK"
# Ensure that entries were created for each layer.
r = registry_server_executor.on(liveserver).verify_replication_for(
"devtable", "newrepo", "latest"
)
assert r.text == "OK"
@pytest.mark.parametrize(
@ -1615,333 +1617,6 @@ def test_tags_disabled_namespace(
)
def test_squashed_image_disabled_namespace(
pusher, sized_images, liveserver_session, liveserver, registry_server_executor, app_reloader
):
""" Test: Attempting to pull a squashed image from a disabled namespace. """
credentials = ("devtable", "password")
# Push an image to download.
pusher.push(
liveserver_session, "buynlarge", "newrepo", "latest", sized_images, credentials=credentials
)
# Disable the buynlarge namespace.
registry_server_executor.on(liveserver).disable_namespace("buynlarge")
# Attempt to pull the squashed version.
response = liveserver_session.get("/c1/squash/buynlarge/newrepo/latest", auth=credentials)
assert response.status_code == 400
def test_squashed_image_disabled_user(
pusher, sized_images, liveserver_session, liveserver, registry_server_executor, app_reloader
):
""" Test: Attempting to pull a squashed image via a disabled user. """
credentials = ("devtable", "password")
# Push an image to download.
pusher.push(
liveserver_session, "buynlarge", "newrepo", "latest", sized_images, credentials=credentials
)
# Disable the devtable namespace.
registry_server_executor.on(liveserver).disable_namespace("devtable")
# Attempt to pull the squashed version.
response = liveserver_session.get("/c1/squash/buynlarge/newrepo/latest", auth=credentials)
assert response.status_code == 403
@pytest.mark.parametrize("use_estimates", [False, True,])
def test_multilayer_squashed_images(
use_estimates,
pusher,
multi_layer_images,
liveserver_session,
liveserver,
registry_server_executor,
app_reloader,
):
""" Test: Pulling of multilayer, complex squashed images. """
credentials = ("devtable", "password")
# Push an image to download.
pusher.push(
liveserver_session,
"devtable",
"newrepo",
"latest",
multi_layer_images,
credentials=credentials,
)
if use_estimates:
# Clear the uncompressed size stored for the images, to ensure that we estimate instead.
for image in multi_layer_images:
registry_server_executor.on(liveserver).clear_uncompressed_size(image.id)
# Pull the squashed version.
response = liveserver_session.get("/c1/squash/devtable/newrepo/latest", auth=credentials)
assert response.status_code == 200
tar = tarfile.open(fileobj=BytesIO(response.content))
# Verify the squashed image.
expected_image_id = next(
(name for name in tar.getnames() if not "/" in name and name != "repositories")
)
expected_names = [
"repositories",
expected_image_id,
"%s/json" % expected_image_id,
"%s/VERSION" % expected_image_id,
"%s/layer.tar" % expected_image_id,
]
assert tar.getnames() == expected_names
# Verify the JSON image data.
json_data = tar.extractfile(tar.getmember("%s/json" % expected_image_id)).read()
# Ensure the JSON loads and parses.
result = json.loads(json_data)
assert result["id"] == expected_image_id
assert result["config"]["internal_id"] == "layer5"
# Ensure that squashed layer tar can be opened.
tar = tarfile.open(fileobj=tar.extractfile(tar.getmember("%s/layer.tar" % expected_image_id)))
assert set(tar.getnames()) == {"contents", "file1", "file2", "file3", "file4"}
# Check the contents of various files.
assert tar.extractfile("contents").read() == b"layer 5 contents"
assert tar.extractfile("file1").read() == b"from-layer-3"
assert tar.extractfile("file2").read() == b"from-layer-2"
assert tar.extractfile("file3").read() == b"from-layer-4"
assert tar.extractfile("file4").read() == b"from-layer-5"
@pytest.mark.parametrize("use_estimates", [False, True,])
@pytest.mark.parametrize("is_readonly", [False, True,])
def test_squashed_images(
use_estimates,
pusher,
sized_images,
liveserver_session,
is_readonly,
liveserver,
registry_server_executor,
app_reloader,
):
""" Test: Pulling of squashed images. """
credentials = ("devtable", "password")
# Push an image to download.
pusher.push(
liveserver_session, "devtable", "newrepo", "latest", sized_images, credentials=credentials
)
if use_estimates:
# Clear the uncompressed size stored for the images, to ensure that we estimate instead.
for image in sized_images:
registry_server_executor.on(liveserver).clear_uncompressed_size(image.id)
# Pull the squashed version.
with ConfigChange(
"REGISTRY_STATE",
"readonly" if is_readonly else "normal",
registry_server_executor.on(liveserver),
liveserver,
):
response = liveserver_session.get("/c1/squash/devtable/newrepo/latest", auth=credentials)
assert response.status_code == 200
tar = tarfile.open(fileobj=BytesIO(response.content))
# Verify the squashed image.
expected_image_id = next(
(name for name in tar.getnames() if not "/" in name and name != "repositories")
)
expected_names = [
"repositories",
expected_image_id,
"%s/json" % expected_image_id,
"%s/VERSION" % expected_image_id,
"%s/layer.tar" % expected_image_id,
]
assert tar.getnames() == expected_names
# Verify the JSON image data.
json_data = tar.extractfile(tar.getmember("%s/json" % expected_image_id)).read()
# Ensure the JSON loads and parses.
result = json.loads(json_data)
assert result["id"] == expected_image_id
assert result["config"]["foo"] == "childbar"
# Ensure that squashed layer tar can be opened.
tar = tarfile.open(
fileobj=tar.extractfile(tar.getmember("%s/layer.tar" % expected_image_id))
)
assert tar.getnames() == ["contents"]
# Check the contents.
assert tar.extractfile("contents").read() == b"some contents"
EXPECTED_ACI_MANIFEST = {
"acKind": "ImageManifest",
"app": {
"environment": [],
"mountPoints": [],
"group": "root",
"user": "root",
"workingDirectory": "/",
"exec": ["/bin/sh", "-c", '""hello""'],
"isolators": [],
"eventHandlers": [],
"ports": [],
"annotations": [
{"name": "created", "value": "2018-04-03T18:37:09.284840891Z"},
{"name": "homepage", "value": "http://localhost:5000/devtable/newrepo:latest"},
{"name": "quay.io/derived-image", "value": "DERIVED_IMAGE_ID"},
],
},
"labels": [
{"name": "version", "value": "latest"},
{"name": "arch", "value": "amd64"},
{"name": "os", "value": "linux"},
],
"acVersion": "0.6.1",
"name": "localhost/devtable/newrepo",
}
@pytest.mark.parametrize("is_readonly", [False, True,])
def test_aci_conversion(
pusher,
sized_images,
liveserver_session,
is_readonly,
liveserver,
registry_server_executor,
app_reloader,
):
""" Test: Pulling of ACI converted images. """
credentials = ("devtable", "password")
# Push an image to download.
pusher.push(
liveserver_session, "devtable", "newrepo", "latest", sized_images, credentials=credentials
)
# Pull the ACI version.
with ConfigChange(
"REGISTRY_STATE",
"readonly" if is_readonly else "normal",
registry_server_executor.on(liveserver),
liveserver,
):
response = liveserver_session.get(
"/c1/aci/server_name/devtable/newrepo/latest/aci/linux/amd64", auth=credentials
)
assert response.status_code == 200
tar = tarfile.open(fileobj=BytesIO(response.content))
assert set(tar.getnames()) == {"manifest", "rootfs", "rootfs/contents"}
assert tar.extractfile("rootfs/contents").read() == b"some contents"
loaded = json.loads(tar.extractfile("manifest").read())
for annotation in loaded["app"]["annotations"]:
if annotation["name"] == "quay.io/derived-image":
annotation["value"] = "DERIVED_IMAGE_ID"
assert loaded == EXPECTED_ACI_MANIFEST
if not is_readonly:
# Wait for the ACI signature to be written.
time.sleep(1)
# Pull the ACI signature.
response = liveserver_session.get(
"/c1/aci/server_name/devtable/newrepo/latest/aci.asc/linux/amd64", auth=credentials
)
assert response.status_code == 200
@pytest.mark.parametrize("schema_version", [1, 2,])
def test_aci_conversion_manifest_list(
v22_protocol,
sized_images,
different_images,
liveserver_session,
data_model,
liveserver,
registry_server_executor,
app_reloader,
schema_version,
):
""" Test: Pulling of ACI converted image from a manifest list. """
credentials = ("devtable", "password")
options = ProtocolOptions()
# Build the manifests that will go in the list.
blobs = {}
signed = v22_protocol.build_schema1(
"devtable", "newrepo", "latest", sized_images, blobs, options, arch="amd64"
)
first_manifest = signed.unsigned()
if schema_version == 2:
first_manifest = v22_protocol.build_schema2(sized_images, blobs, options)
second_manifest = v22_protocol.build_schema2(different_images, blobs, options)
# Create and push the manifest list.
builder = DockerSchema2ManifestListBuilder()
builder.add_manifest(first_manifest, "amd64", "linux")
builder.add_manifest(second_manifest, "arm", "linux")
manifestlist = builder.build()
v22_protocol.push_list(
liveserver_session,
"devtable",
"newrepo",
"latest",
manifestlist,
[first_manifest, second_manifest],
blobs,
credentials=credentials,
options=options,
)
# Pull the ACI version.
response = liveserver_session.get(
"/c1/aci/server_name/devtable/newrepo/latest/aci/linux/amd64", auth=credentials
)
assert response.status_code == 200
tar = tarfile.open(fileobj=BytesIO(response.content))
assert set(tar.getnames()) == {"manifest", "rootfs", "rootfs/contents"}
assert tar.extractfile("rootfs/contents").read() == b"some contents"
loaded = json.loads(tar.extractfile("manifest").read())
for annotation in loaded["app"]["annotations"]:
if annotation["name"] == "quay.io/derived-image":
annotation["value"] = "DERIVED_IMAGE_ID"
assert loaded == EXPECTED_ACI_MANIFEST
# Wait for the ACI signature to be written.
time.sleep(1)
# Pull the ACI signature.
response = liveserver_session.get(
"/c1/aci/server_name/devtable/newrepo/latest/aci.asc/linux/amd64", auth=credentials
)
assert response.status_code == 200
@pytest.mark.parametrize(
"push_user, push_namespace, push_repo, mount_repo_name, expected_failure",
[
@ -2323,10 +1998,8 @@ def test_push_pull_same_blobs(pusher, puller, liveserver_session, app_reloader):
)
def test_push_tag_existing_image(
v1_protocol, puller, basic_images, liveserver_session, app_reloader
):
""" Test: Push a new tag on an existing manifest/image. """
def test_push_tag_existing_image(v1_protocol, basic_images, liveserver_session, app_reloader):
""" Test: Push a new tag on an existing image. """
credentials = ("devtable", "password")
# Push a new repository.
@ -2334,18 +2007,24 @@ def test_push_tag_existing_image(
liveserver_session, "devtable", "newrepo", "latest", basic_images, credentials=credentials
)
# Push the same image/manifest to another tag in the repository.
# Pull the repository to verify.
pulled = v1_protocol.pull(
liveserver_session, "devtable", "newrepo", "latest", basic_images, credentials=credentials,
)
assert pulled.image_ids
# Push the same image to another tag in the repository.
v1_protocol.tag(
liveserver_session,
"devtable",
"newrepo",
"anothertag",
basic_images[-1],
pulled.image_ids["latest"],
credentials=credentials,
)
# Pull the repository to verify.
puller.pull(
v1_protocol.pull(
liveserver_session,
"devtable",
"newrepo",
@ -2655,131 +2334,6 @@ def test_push_pull_manifest_list_duplicate_manifest(
)
def test_squashed_images_empty_layer(
pusher,
images_with_empty_layer,
liveserver_session,
liveserver,
registry_server_executor,
app_reloader,
):
""" Test: Pulling of squashed images for a manifest with empty layers. """
credentials = ("devtable", "password")
# Push an image to download.
pusher.push(
liveserver_session,
"devtable",
"newrepo",
"latest",
images_with_empty_layer,
credentials=credentials,
)
# Pull the squashed version.
response = liveserver_session.get("/c1/squash/devtable/newrepo/latest", auth=credentials)
assert response.status_code == 200
tar = tarfile.open(fileobj=BytesIO(response.content))
# Verify the squashed image.
expected_image_id = next(
(name for name in tar.getnames() if not "/" in name and name != "repositories")
)
expected_names = [
"repositories",
expected_image_id,
"%s/json" % expected_image_id,
"%s/VERSION" % expected_image_id,
"%s/layer.tar" % expected_image_id,
]
assert tar.getnames() == expected_names
def test_squashed_image_unsupported(
v22_protocol, basic_images, liveserver_session, liveserver, app_reloader, data_model
):
""" Test: Attempting to pull a squashed image for a manifest list without an amd64+linux entry.
"""
credentials = ("devtable", "password")
options = ProtocolOptions()
# Build the manifest that will go in the list.
blobs = {}
manifest = v22_protocol.build_schema2(basic_images, blobs, options)
# Create and push the manifest list.
builder = DockerSchema2ManifestListBuilder()
builder.add_manifest(manifest, "foobar", "someos")
manifestlist = builder.build()
v22_protocol.push_list(
liveserver_session,
"devtable",
"newrepo",
"latest",
manifestlist,
[manifest],
blobs,
credentials=credentials,
options=options,
)
# Attempt to pull the squashed version.
response = liveserver_session.get("/c1/squash/devtable/newrepo/latest", auth=credentials)
assert response.status_code == 404
def test_squashed_image_manifest_list(
v22_protocol, basic_images, liveserver_session, liveserver, app_reloader, data_model
):
""" Test: Pull a squashed image for a manifest list with an amd64+linux entry.
"""
credentials = ("devtable", "password")
options = ProtocolOptions()
# Build the manifest that will go in the list.
blobs = {}
manifest = v22_protocol.build_schema2(basic_images, blobs, options)
# Create and push the manifest list.
builder = DockerSchema2ManifestListBuilder()
builder.add_manifest(manifest, "amd64", "linux")
manifestlist = builder.build()
v22_protocol.push_list(
liveserver_session,
"devtable",
"newrepo",
"latest",
manifestlist,
[manifest],
blobs,
credentials=credentials,
options=options,
)
# Pull the squashed version.
response = liveserver_session.get("/c1/squash/devtable/newrepo/latest", auth=credentials)
assert response.status_code == 200
# Verify the squashed image.
tar = tarfile.open(fileobj=BytesIO(response.content))
expected_image_id = next(
(name for name in tar.getnames() if not "/" in name and name != "repositories")
)
expected_names = [
"repositories",
expected_image_id,
"%s/json" % expected_image_id,
"%s/VERSION" % expected_image_id,
"%s/layer.tar" % expected_image_id,
]
assert tar.getnames() == expected_names
def test_verify_schema2(
v22_protocol, basic_images, liveserver_session, liveserver, app_reloader, data_model
):

15
test/test_api_usage.py
View File

@ -2444,7 +2444,6 @@ class TestDeleteRepository(ApiTestCase):
# Make sure the repository has some images and tags.
repo_ref = registry_model.lookup_repository(ADMIN_ACCESS_USER, "complex")
self.assertTrue(len(list(registry_model.get_legacy_images(repo_ref))) > 0)
self.assertTrue(len(list(registry_model.list_all_active_repository_tags(repo_ref))) > 0)
# Add some data for the repository, in addition to is already existing images and tags.
@ -2525,11 +2524,11 @@ class TestGetRepository(ApiTestCase):
self.login(ADMIN_ACCESS_USER)
# base + repo + is_starred + tags
with assert_query_count(BASE_LOGGEDIN_QUERY_COUNT + 4 + 1):
with assert_query_count(BASE_LOGGEDIN_QUERY_COUNT + 4):
self.getJsonResponse(Repository, params=dict(repository=ADMIN_ACCESS_USER + "/simple"))
# base + repo + is_starred + tags
with assert_query_count(BASE_LOGGEDIN_QUERY_COUNT + 4 + 1):
with assert_query_count(BASE_LOGGEDIN_QUERY_COUNT + 4):
json = self.getJsonResponse(
Repository, params=dict(repository=ADMIN_ACCESS_USER + "/gargantuan")
)
@ -3326,8 +3325,7 @@ class TestListAndDeleteTag(ApiTestCase):
params=dict(repository=ADMIN_ACCESS_USER + "/complex", tag="sometag"),
)
sometag_images = json["images"]
self.assertEqual(sometag_images, staging_images)
assert json["images"]
# Move the tag.
self.putResponse(
@ -3344,8 +3342,7 @@ class TestListAndDeleteTag(ApiTestCase):
)
sometag_new_images = json["images"]
self.assertEqual(1, len(sometag_new_images))
self.assertEqual(staging_images[-1], sometag_new_images[0])
assert sometag_new_images
def test_deletesubtag(self):
self.login(ADMIN_ACCESS_USER)
@ -3384,7 +3381,7 @@ class TestListAndDeleteTag(ApiTestCase):
self.login(ADMIN_ACCESS_USER)
repo_ref = registry_model.lookup_repository(ADMIN_ACCESS_USER, "simple")
latest_tag = registry_model.get_repo_tag(repo_ref, "latest", include_legacy_image=True)
latest_tag = registry_model.get_repo_tag(repo_ref, "latest")
# Create 8 tags in the simple repo.
remaining_tags = {"latest", "prod"}
@ -3392,7 +3389,7 @@ class TestListAndDeleteTag(ApiTestCase):
tag_name = "tag" + str(i)
remaining_tags.add(tag_name)
assert registry_model.retarget_tag(
repo_ref, tag_name, latest_tag.legacy_image, storage, docker_v2_signing_key
repo_ref, tag_name, latest_tag.manifest, storage, docker_v2_signing_key
)
# Make sure we can iterate over all of them.

809
test/test_secscan.py
View File

@ -2,44 +2,26 @@ import json
import time
import unittest
from app import app, storage, notification_queue, url_scheme_and_hostname
from app import app, storage, url_scheme_and_hostname
from data import model
from data.registry_model import registry_model
from data.database import Image, IMAGE_NOT_SCANNED_ENGINE_VERSION
from endpoints.v2 import v2_bp
from data.database import Image, ManifestLegacyImage
from initdb import setup_database_for_testing, finished_database_for_testing
from notifications.notificationevent import VulnerabilityFoundEvent
from util.secscan.secscan_util import get_blob_download_uri_getter
from util.morecollections import AttrDict
from util.secscan.api import SecurityScannerAPI, APIRequestFailure
from util.secscan.analyzer import LayerAnalyzer
from util.secscan.fake import fake_security_scanner
from util.secscan.notifier import SecurityNotificationHandler, ProcessNotificationPageResult
from util.security.instancekeys import InstanceKeys
from workers.security_notification_worker import SecurityNotificationWorker
ADMIN_ACCESS_USER = "devtable"
SIMPLE_REPO = "simple"
COMPLEX_REPO = "complex"
def process_notification_data(legacy_api, notification_data):
handler = SecurityNotificationHandler(legacy_api, 100)
result = handler.process_notification_page_data(notification_data)
handler.send_notifications()
return result == ProcessNotificationPageResult.FINISHED_PROCESSING
def _get_legacy_image(namespace, repo, tag, include_storage=True):
repo_ref = registry_model.lookup_repository(namespace, repo)
repo_tag = registry_model.get_repo_tag(repo_ref, tag, include_legacy_image=True)
return Image.get(id=repo_tag.legacy_image._db_id)
def _delete_tag(namespace, repo, tag):
repo_ref = registry_model.lookup_repository(namespace, repo)
registry_model.delete_tag(repo_ref, tag)
repo_tag = registry_model.get_repo_tag(repo_ref, tag)
manifest = registry_model.get_manifest_for_tag(repo_tag)
return ManifestLegacyImage.get(manifest_id=manifest._db_id).image
class TestSecurityScanner(unittest.TestCase):
@ -93,785 +75,24 @@ class TestSecurityScanner(unittest.TestCase):
"""
Test for basic retrieval of layers from the security scanner.
"""
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
repo_ref = registry_model.lookup_repository(ADMIN_ACCESS_USER, SIMPLE_REPO)
repo_tag = registry_model.get_repo_tag(repo_ref, "latest")
manifest = registry_model.get_manifest_for_tag(repo_tag)
registry_model.populate_legacy_images_for_testing(manifest, storage)
with fake_security_scanner() as security_scanner:
# Ensure the layer doesn't exist yet.
self.assertFalse(security_scanner.has_layer(security_scanner.layer_id(layer)))
self.assertIsNone(self.api.get_layer_data(layer))
self.assertFalse(security_scanner.has_layer(security_scanner.layer_id(manifest)))
self.assertIsNone(self.api.get_layer_data(manifest))
# Add the layer.
security_scanner.add_layer(security_scanner.layer_id(layer))
security_scanner.add_layer(security_scanner.layer_id(manifest))
# Retrieve the results.
result = self.api.get_layer_data(layer, include_vulnerabilities=True)
result = self.api.get_layer_data(manifest, include_vulnerabilities=True)
self.assertIsNotNone(result)
self.assertEqual(result["Layer"]["Name"], security_scanner.layer_id(layer))
def test_analyze_layer_nodirectdownload_success(self):
"""
Tests analyzing a layer when direct download is disabled.
"""
# Disable direct download in fake storage.
storage.put_content(["local_us"], "supports_direct_download", b"false")
try:
app.register_blueprint(v2_bp, url_prefix="/v2")
except:
# Already registered.
pass
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
self.assertFalse(layer.security_indexed)
self.assertEqual(-1, layer.security_indexed_engine)
# Ensure that the download is a registry+JWT download.
uri, auth_header = self.api._get_image_url_and_auth(layer)
self.assertIsNotNone(uri)
self.assertIsNotNone(auth_header)
# Ensure the download doesn't work without the header.
rv = self.app.head(uri)
self.assertEqual(rv.status_code, 401)
# Ensure the download works with the header. Note we use a HEAD here, as GET causes DB
# access which messes with the test runner's rollback.
rv = self.app.head(uri, headers=[("authorization", auth_header)])
self.assertEqual(rv.status_code, 200)
# Ensure the code works when called via analyze.
with fake_security_scanner() as security_scanner:
analyzer = LayerAnalyzer(app.config, self.api)
analyzer.analyze_recursively(layer)
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
self.assertAnalyzed(layer, security_scanner, True, 1)
def test_analyze_layer_success(self):
"""
Tests that analyzing a layer successfully marks it as analyzed.
"""
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
self.assertFalse(layer.security_indexed)
self.assertEqual(-1, layer.security_indexed_engine)
with fake_security_scanner() as security_scanner:
analyzer = LayerAnalyzer(app.config, self.api)
analyzer.analyze_recursively(layer)
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
self.assertAnalyzed(layer, security_scanner, True, 1)
def test_analyze_layer_failure(self):
"""
Tests that failing to analyze a layer (because it 422s) marks it as analyzed but failed.
"""
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
self.assertFalse(layer.security_indexed)
self.assertEqual(-1, layer.security_indexed_engine)
with fake_security_scanner() as security_scanner:
security_scanner.set_fail_layer_id(security_scanner.layer_id(layer))
analyzer = LayerAnalyzer(app.config, self.api)
analyzer.analyze_recursively(layer)
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
self.assertAnalyzed(layer, security_scanner, False, 1)
def test_analyze_layer_internal_error(self):
"""
Tests that failing to analyze a layer (because it 500s) marks it as not analyzed.
"""
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
self.assertFalse(layer.security_indexed)
self.assertEqual(-1, layer.security_indexed_engine)
with fake_security_scanner() as security_scanner:
security_scanner.set_internal_error_layer_id(security_scanner.layer_id(layer))
analyzer = LayerAnalyzer(app.config, self.api)
with self.assertRaises(APIRequestFailure):
analyzer.analyze_recursively(layer)
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
self.assertAnalyzed(layer, security_scanner, False, -1)
def test_analyze_layer_error(self):
"""
Tests that failing to analyze a layer (because it 400s) marks it as analyzed but failed.
"""
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
self.assertFalse(layer.security_indexed)
self.assertEqual(-1, layer.security_indexed_engine)
with fake_security_scanner() as security_scanner:
# Make is so trying to analyze the parent will fail with an error.
security_scanner.set_error_layer_id(security_scanner.layer_id(layer.parent))
# Try to the layer and its parents, but with one request causing an error.
analyzer = LayerAnalyzer(app.config, self.api)
analyzer.analyze_recursively(layer)
# Make sure it is marked as analyzed, but in a failed state.
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
self.assertAnalyzed(layer, security_scanner, False, 1)
def test_analyze_layer_unexpected_status(self):
"""
Tests that a response from a scanner with an unexpected status code fails correctly.
"""
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
self.assertFalse(layer.security_indexed)
self.assertEqual(-1, layer.security_indexed_engine)
with fake_security_scanner() as security_scanner:
# Make is so trying to analyze the parent will fail with an error.
security_scanner.set_unexpected_status_layer_id(security_scanner.layer_id(layer.parent))
# Try to the layer and its parents, but with one request causing an error.
analyzer = LayerAnalyzer(app.config, self.api)
with self.assertRaises(APIRequestFailure):
analyzer.analyze_recursively(layer)
# Make sure it isn't analyzed.
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
self.assertAnalyzed(layer, security_scanner, False, -1)
def test_analyze_layer_missing_parent_handled(self):
"""
Tests that a missing parent causes an automatic reanalysis, which succeeds.
"""
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
self.assertFalse(layer.security_indexed)
self.assertEqual(-1, layer.security_indexed_engine)
with fake_security_scanner() as security_scanner:
# Analyze the layer and its parents.
analyzer = LayerAnalyzer(app.config, self.api)
analyzer.analyze_recursively(layer)
# Make sure it was analyzed.
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
self.assertAnalyzed(layer, security_scanner, True, 1)
# Mark the layer as not yet scanned.
layer.security_indexed_engine = IMAGE_NOT_SCANNED_ENGINE_VERSION
layer.security_indexed = False
layer.save()
# Remove the layer's parent entirely from the security scanner.
security_scanner.remove_layer(security_scanner.layer_id(layer.parent))
# Analyze again, which should properly re-analyze the missing parent and this layer.
analyzer.analyze_recursively(layer)
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
self.assertAnalyzed(layer, security_scanner, True, 1)
def test_analyze_layer_invalid_parent(self):
"""
Tests that trying to reanalyze a parent that is invalid causes the layer to be marked as
analyzed, but failed.
"""
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
self.assertFalse(layer.security_indexed)
self.assertEqual(-1, layer.security_indexed_engine)
with fake_security_scanner() as security_scanner:
# Analyze the layer and its parents.
analyzer = LayerAnalyzer(app.config, self.api)
analyzer.analyze_recursively(layer)
# Make sure it was analyzed.
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
self.assertAnalyzed(layer, security_scanner, True, 1)
# Mark the layer as not yet scanned.
layer.security_indexed_engine = IMAGE_NOT_SCANNED_ENGINE_VERSION
layer.security_indexed = False
layer.save()
# Remove the layer's parent entirely from the security scanner.
security_scanner.remove_layer(security_scanner.layer_id(layer.parent))
# Make is so trying to analyze the parent will fail.
security_scanner.set_error_layer_id(security_scanner.layer_id(layer.parent))
# Try to analyze again, which should try to reindex the parent and fail.
analyzer.analyze_recursively(layer)
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
self.assertAnalyzed(layer, security_scanner, False, 1)
def test_analyze_layer_unsupported_parent(self):
"""
Tests that attempting to analyze a layer whose parent is unanalyzable, results in the layer
being marked as analyzed, but failed.
"""
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
self.assertFalse(layer.security_indexed)
self.assertEqual(-1, layer.security_indexed_engine)
with fake_security_scanner() as security_scanner:
# Make is so trying to analyze the parent will fail.
security_scanner.set_fail_layer_id(security_scanner.layer_id(layer.parent))
# Attempt to the layer and its parents. This should mark the layer itself as unanalyzable.
analyzer = LayerAnalyzer(app.config, self.api)
analyzer.analyze_recursively(layer)
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
self.assertAnalyzed(layer, security_scanner, False, 1)
def test_analyze_layer_missing_storage(self):
"""
Tests trying to analyze a layer with missing storage.
"""
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
self.assertFalse(layer.security_indexed)
self.assertEqual(-1, layer.security_indexed_engine)
# Delete the storage for the layer.
path = model.storage.get_layer_path(layer.storage)
locations = app.config["DISTRIBUTED_STORAGE_PREFERENCE"]
storage.remove(locations, path)
storage.remove(locations, "all_files_exist")
with fake_security_scanner() as security_scanner:
analyzer = LayerAnalyzer(app.config, self.api)
analyzer.analyze_recursively(layer)
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
self.assertAnalyzed(layer, security_scanner, False, 1)
def assert_analyze_layer_notify(
self, security_indexed_engine, security_indexed, expect_notification
):
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
self.assertFalse(layer.security_indexed)
self.assertEqual(-1, layer.security_indexed_engine)
# Ensure there are no existing events.
self.assertIsNone(notification_queue.get())
# Add a repo event for the layer.
repo = model.repository.get_repository(ADMIN_ACCESS_USER, SIMPLE_REPO)
model.notification.create_repo_notification(
repo, "vulnerability_found", "quay_notification", {}, {"level": 100}
)
# Update the layer's state before analyzing.
layer.security_indexed_engine = security_indexed_engine
layer.security_indexed = security_indexed
layer.save()
with fake_security_scanner() as security_scanner:
security_scanner.set_vulns(
security_scanner.layer_id(layer),
[
{
"Name": "CVE-2014-9471",
"Namespace": "debian:8",
"Description": "Some service",
"Link": "https://security-tracker.debian.org/tracker/CVE-2014-9471",
"Severity": "Low",
"FixedBy": "9.23-5",
},
{
"Name": "CVE-2016-7530",
"Namespace": "debian:8",
"Description": "Some other service",
"Link": "https://security-tracker.debian.org/tracker/CVE-2016-7530",
"Severity": "Unknown",
"FixedBy": "19.343-2",
},
],
)
analyzer = LayerAnalyzer(app.config, self.api)
analyzer.analyze_recursively(layer)
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
self.assertAnalyzed(layer, security_scanner, True, 1)
# Ensure an event was written for the tag (if necessary).
time.sleep(1)
queue_item = notification_queue.get()
if expect_notification:
self.assertIsNotNone(queue_item)
body = json.loads(queue_item.body)
self.assertEqual(set(["latest", "prod"]), set(body["event_data"]["tags"]))
self.assertEqual("CVE-2014-9471", body["event_data"]["vulnerability"]["id"])
self.assertEqual("Low", body["event_data"]["vulnerability"]["priority"])
self.assertTrue(body["event_data"]["vulnerability"]["has_fix"])
self.assertEqual("CVE-2014-9471", body["event_data"]["vulnerabilities"][0]["id"])
self.assertEqual(2, len(body["event_data"]["vulnerabilities"]))
# Ensure we get the correct event message out as well.
event = VulnerabilityFoundEvent()
msg = "1 Low and 1 more vulnerabilities were detected in repository devtable/simple in 2 tags"
self.assertEqual(msg, event.get_summary(body["event_data"], {}))
self.assertEqual("info", event.get_level(body["event_data"], {}))
else:
self.assertIsNone(queue_item)
# Ensure its security indexed engine was updated.
updated_layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
self.assertEquals(updated_layer.id, layer.id)
self.assertTrue(updated_layer.security_indexed_engine > 0)
def test_analyze_layer_success_events(self):
# Not previously indexed at all => Notification
self.assert_analyze_layer_notify(IMAGE_NOT_SCANNED_ENGINE_VERSION, False, True)
def test_analyze_layer_success_no_notification(self):
# Previously successfully indexed => No notification
self.assert_analyze_layer_notify(0, True, False)
def test_analyze_layer_failed_then_success_notification(self):
# Previously failed to index => Notification
self.assert_analyze_layer_notify(0, False, True)
def test_notification_new_layers_not_vulnerable(self):
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
layer_id = "%s.%s" % (layer.docker_image_id, layer.storage.uuid)
# Add a repo event for the layer.
repo = model.repository.get_repository(ADMIN_ACCESS_USER, SIMPLE_REPO)
model.notification.create_repo_notification(
repo, "vulnerability_found", "quay_notification", {}, {"level": 100}
)
# Ensure that there are no event queue items for the layer.
self.assertIsNone(notification_queue.get())
# Fire off the notification processing.
with fake_security_scanner() as security_scanner:
analyzer = LayerAnalyzer(app.config, self.api)
analyzer.analyze_recursively(layer)
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
self.assertAnalyzed(layer, security_scanner, True, 1)
# Add a notification for the layer.
notification_data = security_scanner.add_notification([layer_id], [], {}, {})
# Process the notification.
self.assertTrue(process_notification_data(self.api, notification_data))
# Ensure that there are no event queue items for the layer.
self.assertIsNone(notification_queue.get())
def test_notification_delete(self):
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
layer_id = "%s.%s" % (layer.docker_image_id, layer.storage.uuid)
# Add a repo event for the layer.
repo = model.repository.get_repository(ADMIN_ACCESS_USER, SIMPLE_REPO)
model.notification.create_repo_notification(
repo, "vulnerability_found", "quay_notification", {}, {"level": 100}
)
# Ensure that there are no event queue items for the layer.
self.assertIsNone(notification_queue.get())
# Fire off the notification processing.
with fake_security_scanner() as security_scanner:
analyzer = LayerAnalyzer(app.config, self.api)
analyzer.analyze_recursively(layer)
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
self.assertAnalyzed(layer, security_scanner, True, 1)
# Add a notification for the layer.
notification_data = security_scanner.add_notification([layer_id], None, {}, None)
# Process the notification.
self.assertTrue(process_notification_data(self.api, notification_data))
# Ensure that there are no event queue items for the layer.
self.assertIsNone(notification_queue.get())
def test_notification_new_layers(self):
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
layer_id = "%s.%s" % (layer.docker_image_id, layer.storage.uuid)
# Add a repo event for the layer.
repo = model.repository.get_repository(ADMIN_ACCESS_USER, SIMPLE_REPO)
model.notification.create_repo_notification(
repo, "vulnerability_found", "quay_notification", {}, {"level": 100}
)
# Ensure that there are no event queue items for the layer.
self.assertIsNone(notification_queue.get())
# Fire off the notification processing.
with fake_security_scanner() as security_scanner:
analyzer = LayerAnalyzer(app.config, self.api)
analyzer.analyze_recursively(layer)
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
self.assertAnalyzed(layer, security_scanner, True, 1)
vuln_info = {
"Name": "CVE-TEST",
"Namespace": "debian:8",
"Description": "Some service",
"Link": "https://security-tracker.debian.org/tracker/CVE-2014-9471",
"Severity": "Low",
"FixedIn": {"Version": "9.23-5"},
}
security_scanner.set_vulns(layer_id, [vuln_info])
# Add a notification for the layer.
notification_data = security_scanner.add_notification(
[], [layer_id], vuln_info, vuln_info
)
# Process the notification.
self.assertTrue(process_notification_data(self.api, notification_data))
# Ensure an event was written for the tag.
time.sleep(1)
queue_item = notification_queue.get()
self.assertIsNotNone(queue_item)
item_body = json.loads(queue_item.body)
self.assertEqual(sorted(["prod", "latest"]), sorted(item_body["event_data"]["tags"]))
self.assertEqual("CVE-TEST", item_body["event_data"]["vulnerability"]["id"])
self.assertEqual("Low", item_body["event_data"]["vulnerability"]["priority"])
self.assertTrue(item_body["event_data"]["vulnerability"]["has_fix"])
def test_notification_no_new_layers(self):
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
# Add a repo event for the layer.
repo = model.repository.get_repository(ADMIN_ACCESS_USER, SIMPLE_REPO)
model.notification.create_repo_notification(
repo, "vulnerability_found", "quay_notification", {}, {"level": 100}
)
# Ensure that there are no event queue items for the layer.
self.assertIsNone(notification_queue.get())
# Fire off the notification processing.
with fake_security_scanner() as security_scanner:
analyzer = LayerAnalyzer(app.config, self.api)
analyzer.analyze_recursively(layer)
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
self.assertAnalyzed(layer, security_scanner, True, 1)
# Add a notification for the layer.
notification_data = security_scanner.add_notification([], [], {}, {})
# Process the notification.
self.assertTrue(process_notification_data(self.api, notification_data))
# Ensure that there are no event queue items for the layer.
self.assertIsNone(notification_queue.get())
def notification_tuple(self, notification):
# TODO: Replace this with a method once we refactor the notification stuff into its
# own module.
return AttrDict(
{
"event_config_dict": json.loads(notification.event_config_json),
"method_config_dict": json.loads(notification.config_json),
}
)
def test_notification_no_new_layers_increased_severity(self):
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
layer_id = "%s.%s" % (layer.docker_image_id, layer.storage.uuid)
# Add a repo event for the layer.
repo = model.repository.get_repository(ADMIN_ACCESS_USER, SIMPLE_REPO)
notification = model.notification.create_repo_notification(
repo, "vulnerability_found", "quay_notification", {}, {"level": 100}
)
# Ensure that there are no event queue items for the layer.
self.assertIsNone(notification_queue.get())
# Fire off the notification processing.
with fake_security_scanner() as security_scanner:
analyzer = LayerAnalyzer(app.config, self.api)
analyzer.analyze_recursively(layer)
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
self.assertAnalyzed(layer, security_scanner, True, 1)
old_vuln_info = {
"Name": "CVE-TEST",
"Namespace": "debian:8",
"Description": "Some service",
"Link": "https://security-tracker.debian.org/tracker/CVE-2014-9471",
"Severity": "Low",
}
new_vuln_info = {
"Name": "CVE-TEST",
"Namespace": "debian:8",
"Description": "Some service",
"Link": "https://security-tracker.debian.org/tracker/CVE-2014-9471",
"Severity": "Critical",
"FixedIn": {"Version": "9.23-5"},
}
security_scanner.set_vulns(layer_id, [new_vuln_info])
# Add a notification for the layer.
notification_data = security_scanner.add_notification(
[layer_id], [layer_id], old_vuln_info, new_vuln_info
)
# Process the notification.
self.assertTrue(process_notification_data(self.api, notification_data))
# Ensure an event was written for the tag.
time.sleep(1)
queue_item = notification_queue.get()
self.assertIsNotNone(queue_item)
item_body = json.loads(queue_item.body)
self.assertEqual(sorted(["prod", "latest"]), sorted(item_body["event_data"]["tags"]))
self.assertEqual("CVE-TEST", item_body["event_data"]["vulnerability"]["id"])
self.assertEqual("Critical", item_body["event_data"]["vulnerability"]["priority"])
self.assertTrue(item_body["event_data"]["vulnerability"]["has_fix"])
# Verify that an event would be raised.
event_data = item_body["event_data"]
notification = self.notification_tuple(notification)
self.assertTrue(VulnerabilityFoundEvent().should_perform(event_data, notification))
# Create another notification with a matching level and verify it will be raised.
notification = model.notification.create_repo_notification(
repo, "vulnerability_found", "quay_notification", {}, {"level": 1}
)
notification = self.notification_tuple(notification)
self.assertTrue(VulnerabilityFoundEvent().should_perform(event_data, notification))
# Create another notification with a higher level and verify it won't be raised.
notification = model.notification.create_repo_notification(
repo, "vulnerability_found", "quay_notification", {}, {"level": 0}
)
notification = self.notification_tuple(notification)
self.assertFalse(VulnerabilityFoundEvent().should_perform(event_data, notification))
def test_select_images_to_scan(self):
# Set all images to have a security index of a version to that of the config.
expected_version = app.config["SECURITY_SCANNER_ENGINE_VERSION_TARGET"]
Image.update(security_indexed_engine=expected_version).execute()
# Ensure no images are available for scanning.
self.assertIsNone(model.image.get_min_id_for_sec_scan(expected_version))
self.assertTrue(len(model.image.get_images_eligible_for_scan(expected_version)) == 0)
# Check for a higher version.
self.assertIsNotNone(model.image.get_min_id_for_sec_scan(expected_version + 1))
self.assertTrue(len(model.image.get_images_eligible_for_scan(expected_version + 1)) > 0)
def test_notification_worker(self):
layer1 = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
layer2 = _get_legacy_image(ADMIN_ACCESS_USER, COMPLEX_REPO, "prod", include_storage=True)
# Add a repo events for the layers.
simple_repo = model.repository.get_repository(ADMIN_ACCESS_USER, SIMPLE_REPO)
complex_repo = model.repository.get_repository(ADMIN_ACCESS_USER, COMPLEX_REPO)
model.notification.create_repo_notification(
simple_repo, "vulnerability_found", "quay_notification", {}, {"level": 100}
)
model.notification.create_repo_notification(
complex_repo, "vulnerability_found", "quay_notification", {}, {"level": 100}
)
# Ensure that there are no event queue items for the layer.
self.assertIsNone(notification_queue.get())
with fake_security_scanner() as security_scanner:
# Test with an unknown notification.
worker = SecurityNotificationWorker(None)
self.assertFalse(worker.perform_notification_work({"Name": "unknownnotification"}))
# Add some analyzed layers.
analyzer = LayerAnalyzer(app.config, self.api)
analyzer.analyze_recursively(layer1)
analyzer.analyze_recursively(layer2)
# Add a notification with pages of data.
new_vuln_info = {
"Name": "CVE-TEST",
"Namespace": "debian:8",
"Description": "Some service",
"Link": "https://security-tracker.debian.org/tracker/CVE-2014-9471",
"Severity": "Critical",
"FixedIn": {"Version": "9.23-5"},
}
security_scanner.set_vulns(security_scanner.layer_id(layer1), [new_vuln_info])
security_scanner.set_vulns(security_scanner.layer_id(layer2), [new_vuln_info])
layer_ids = [security_scanner.layer_id(layer1), security_scanner.layer_id(layer2)]
notification_data = security_scanner.add_notification(
[], layer_ids, None, new_vuln_info
)
# Test with a known notification with pages.
data = {
"Name": notification_data["Name"],
}
worker = SecurityNotificationWorker(None)
self.assertTrue(worker.perform_notification_work(data, layer_limit=2))
# Make sure all pages were processed by ensuring we have two notifications.
time.sleep(1)
self.assertIsNotNone(notification_queue.get())
self.assertIsNotNone(notification_queue.get())
def test_notification_worker_offset_pages_not_indexed(self):
# Try without indexes.
self.assert_notification_worker_offset_pages(indexed=False)
def test_notification_worker_offset_pages_indexed(self):
# Try with indexes.
self.assert_notification_worker_offset_pages(indexed=True)
def assert_notification_worker_offset_pages(self, indexed=False):
layer1 = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
layer2 = _get_legacy_image(ADMIN_ACCESS_USER, COMPLEX_REPO, "prod", include_storage=True)
# Add a repo events for the layers.
simple_repo = model.repository.get_repository(ADMIN_ACCESS_USER, SIMPLE_REPO)
complex_repo = model.repository.get_repository(ADMIN_ACCESS_USER, COMPLEX_REPO)
model.notification.create_repo_notification(
simple_repo, "vulnerability_found", "quay_notification", {}, {"level": 100}
)
model.notification.create_repo_notification(
complex_repo, "vulnerability_found", "quay_notification", {}, {"level": 100}
)
# Ensure that there are no event queue items for the layer.
self.assertIsNone(notification_queue.get())
with fake_security_scanner() as security_scanner:
# Test with an unknown notification.
worker = SecurityNotificationWorker(None)
self.assertFalse(worker.perform_notification_work({"Name": "unknownnotification"}))
# Add some analyzed layers.
analyzer = LayerAnalyzer(app.config, self.api)
analyzer.analyze_recursively(layer1)
analyzer.analyze_recursively(layer2)
# Add a notification with pages of data.
new_vuln_info = {
"Name": "CVE-TEST",
"Namespace": "debian:8",
"Description": "Some service",
"Link": "https://security-tracker.debian.org/tracker/CVE-2014-9471",
"Severity": "Critical",
"FixedIn": {"Version": "9.23-5"},
}
security_scanner.set_vulns(security_scanner.layer_id(layer1), [new_vuln_info])
security_scanner.set_vulns(security_scanner.layer_id(layer2), [new_vuln_info])
# Define offsetting sets of layer IDs, to test cross-pagination support. In this test, we
# will only serve 2 layer IDs per page: the first page will serve both of the 'New' layer IDs,
# but since the first 2 'Old' layer IDs are "earlier" than the shared ID of
# `devtable/simple:latest`, they won't get served in the 'New' list until the *second* page.
# The notification handling system should correctly not notify for this layer, even though it
# is marked 'New' on page 1 and marked 'Old' on page 2. Clair will served these
# IDs sorted in the same manner.
idx_old_layer_ids = [
{"LayerName": "old1", "Index": 1},
{"LayerName": "old2", "Index": 2},
{"LayerName": security_scanner.layer_id(layer1), "Index": 3},
]
idx_new_layer_ids = [
{"LayerName": security_scanner.layer_id(layer1), "Index": 3},
{"LayerName": security_scanner.layer_id(layer2), "Index": 4},
]
old_layer_ids = [t["LayerName"] for t in idx_old_layer_ids]
new_layer_ids = [t["LayerName"] for t in idx_new_layer_ids]
if not indexed:
idx_old_layer_ids = None
idx_new_layer_ids = None
notification_data = security_scanner.add_notification(
old_layer_ids,
new_layer_ids,
None,
new_vuln_info,
max_per_page=2,
indexed_old_layer_ids=idx_old_layer_ids,
indexed_new_layer_ids=idx_new_layer_ids,
)
# Test with a known notification with pages.
data = {
"Name": notification_data["Name"],
}
worker = SecurityNotificationWorker(None)
self.assertTrue(worker.perform_notification_work(data, layer_limit=2))
# Make sure all pages were processed by ensuring we have only one notification. If the second
# page was not processed, then the `Old` entry for layer1 will not be found, and we'd get two
# notifications.
time.sleep(1)
self.assertIsNotNone(notification_queue.get())
self.assertIsNone(notification_queue.get())
def test_layer_gc(self):
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
# Delete the prod tag so that only the `latest` tag remains.
_delete_tag(ADMIN_ACCESS_USER, SIMPLE_REPO, "prod")
with fake_security_scanner() as security_scanner:
# Analyze the layer.
analyzer = LayerAnalyzer(app.config, self.api)
analyzer.analyze_recursively(layer)
layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
self.assertAnalyzed(layer, security_scanner, True, 1)
self.assertTrue(security_scanner.has_layer(security_scanner.layer_id(layer)))
namespace_user = model.user.get_user(ADMIN_ACCESS_USER)
model.user.change_user_tag_expiration(namespace_user, 0)
# Delete the tag in the repository and GC.
_delete_tag(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
time.sleep(1)
repo = model.repository.get_repository(ADMIN_ACCESS_USER, SIMPLE_REPO)
model.gc.garbage_collect_repo(repo)
# Ensure that the security scanner no longer has the image.
self.assertFalse(security_scanner.has_layer(security_scanner.layer_id(layer)))
self.assertEquals(result["Layer"]["Name"], security_scanner.layer_id(manifest))
if __name__ == "__main__":

1
test/testconfig.py
View File

@ -66,7 +66,6 @@ class TestConfig(DefaultConfig):
SECURITY_SCANNER_ENGINE_VERSION_TARGET = 1
SECURITY_SCANNER_API_TIMEOUT_SECONDS = 1
SECURITY_SCANNER_V4_ENDPOINT = "http://fakesecurityscanner/"
SECURITY_SCANNER_V4_NAMESPACE_WHITELIST = ["devtable"]
FEATURE_SIGNING = True

9
util/config/configutil.py
View File

@ -48,15 +48,6 @@ def add_enterprise_config_defaults(config_obj, current_secret_key):
config_obj["REPO_MIRROR_TLS_VERIFY"] = config_obj.get("REPO_MIRROR_TLS_VERIFY", True)
config_obj["REPO_MIRROR_SERVER_HOSTNAME"] = config_obj.get("REPO_MIRROR_SERVER_HOSTNAME", None)
# Default the signer config.
config_obj["GPG2_PRIVATE_KEY_FILENAME"] = config_obj.get(
"GPG2_PRIVATE_KEY_FILENAME", "signing-private.gpg"
)
config_obj["GPG2_PUBLIC_KEY_FILENAME"] = config_obj.get(
"GPG2_PUBLIC_KEY_FILENAME", "signing-public.gpg"
)
config_obj["SIGNING_ENGINE"] = config_obj.get("SIGNING_ENGINE", "gpg2")
# Default security scanner config.
config_obj["FEATURE_SECURITY_NOTIFICATIONS"] = config_obj.get(
"FEATURE_SECURITY_NOTIFICATIONS", True

3
util/config/schema.py
View File

@ -18,6 +18,7 @@ INTERNAL_ONLY_PROPERTIES = {
"FEATURE_REPOSITORY_ACTION_COUNTER",
"APP_REGISTRY_PACKAGE_LIST_CACHE_WHITELIST",
"APP_REGISTRY_SHOW_PACKAGE_CACHE_WHITELIST",
"FEATURE_MANIFEST_SIZE_BACKFILL",
"TESTING",
"SEND_FILE_MAX_AGE_DEFAULT",
"DISABLED_FOR_AUDIT_LOGS",
@ -29,7 +30,6 @@ INTERNAL_ONLY_PROPERTIES = {
"REPLICATION_QUEUE_NAME",
"DOCKERFILE_BUILD_QUEUE_NAME",
"CHUNK_CLEANUP_QUEUE_NAME",
"SECSCAN_NOTIFICATION_QUEUE_NAME",
"SECURITY_SCANNER_ISSUER_NAME",
"NOTIFICATION_QUEUE_NAME",
"REPOSITORY_GC_QUEUE_NAME",
@ -57,7 +57,6 @@ INTERNAL_ONLY_PROPERTIES = {
"JWTPROXY_AUDIENCE",
"JWTPROXY_SIGNER",
"SECURITY_SCANNER_INDEXING_MIN_ID",
"SECURITY_SCANNER_V4_NAMESPACE_WHITELIST",
"SECURITY_SCANNER_V4_REINDEX_THRESHOLD",
"STATIC_SITE_BUCKET",
"LABEL_KEY_RESERVED_PREFIXES",

2
util/config/validator.py
View File

@ -12,7 +12,6 @@ from util.config.validators.validate_ldap import LDAPValidator
from util.config.validators.validate_keystone import KeystoneValidator
from util.config.validators.validate_jwt import JWTAuthValidator
from util.config.validators.validate_secscan import SecurityScannerValidator
from util.config.validators.validate_signer import SignerValidator
from util.config.validators.validate_ssl import SSLValidator, SSL_FILENAMES
from util.config.validators.validate_google_login import GoogleLoginValidator
from util.config.validators.validate_bitbucket_trigger import BitbucketTriggerValidator
@ -62,7 +61,6 @@ VALIDATORS = {
LDAPValidator.name: LDAPValidator.validate,
JWTAuthValidator.name: JWTAuthValidator.validate,
KeystoneValidator.name: KeystoneValidator.validate,
SignerValidator.name: SignerValidator.validate,
SecurityScannerValidator.name: SecurityScannerValidator.validate,
OIDCLoginValidator.name: OIDCLoginValidator.validate,
TimeMachineValidator.name: TimeMachineValidator.validate,

24
util/config/validators/test/test_validate_signer.py
View File

@ -1,24 +0,0 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_signer import SignerValidator
from test.fixtures import *
@pytest.mark.parametrize(
"unvalidated_config,expected",
[
({}, None),
({"SIGNING_ENGINE": "foobar"}, ConfigValidationException),
({"SIGNING_ENGINE": "gpg2"}, Exception),
],
)
def test_validate_signer(unvalidated_config, expected, app):
validator = SignerValidator()
if expected is not None:
with pytest.raises(expected):
validator.validate(ValidatorContext(unvalidated_config))
else:
validator.validate(ValidatorContext(unvalidated_config))

Some files were not shown because too many files have changed in this diff Show More