Deprecate Image rows and move to in-memory synthesized legacy images [Python 3] (#442)

* Change verbs to use a DerivedStorageForManifest table instead of DerivedStorageForImage

This allows us to deprecate the DerivedStorageForImage table.

Fixes https://issues.redhat.com/browse/PROJQUAY-519

* Change uploaded blob tracking to use its own table and deprecate
RepositoryTag

* Start recording the compressed layers size and config media type on the
manifest row in the database

NOTE: This change includes a database migration which will *lock* the
manifest table

* Change tag API to return the layers size from the manifest

* Remove unused code

* Add new config_media_type field to OCI types

* Fix secscan V2 test for us no longer writing temp images

* Remove unused uploading field

* Switch registry model to use synthetic legacy images

Legacy images are now (with exception of the V2 security model) read from the *manifest* and sythensized in memory. The legacy image IDs are generated realtime based on the hashids library. This change also further deprecates a bunch of our Image APIs, reducing them to only returning the image IDs, and emptying out the remaining metadata (to avoid the requirement of us loading the information for the manifest from storage).

This has been tested with our full clients test suite with success.

* Add a backfill worker for manifest layers compressed sizes

* Change image tracks into manifest tracks now that we no longer have
manifest-less tags

* Add back in the missing method

* Add missing joins to reduce extra queries

* Remove unnecessary join when looking up legacy images

* Remove extra hidden filter on tag queries

* Further DB improvements

* Delete all Verbs, as they were deprecated

* Add back missing parameter in manifest data type

* Fix join to return None for the robot if not defined on mirror config

* switch to using secscan_v4_model for all indexing and remove most of secscan_v2_model code

* Add a missing join

* Remove files accidentally re-added due to rebase

* Add back hashids lib

* Rebase fixes

* Fix broken test

* Remove unused GPG signer now that ACI conversion is removed

* Remove duplicated repomirrorworker

* Remove unused notification code for secscan. We'll re-add it once Clair
V4 security notifications are ready to go

* Fix formatting

* Stop writing Image rows when creating manifests

* Stop writing empty layer blobs for manifests

As these blobs are shared, we don't need to write ManifestBlob rows
for them

* Remove further unused code

* Add doc comment to _build_blob_map

* Add unit test for synthetic V1 IDs

* Remove unused import

* Add an invalid value test to synthetic ID decode tests

* Add manifest backfill worker back in

Seems to have been removed at some point

* Add a test for cached active tags

* Rename test_shared to not conflict with another same-named test file

Pytest doesn't like having two test modules with the same name

* Have manifestbackfillworker also copy over the config_media_type if present

Co-authored-by: alecmerdler <alecmerdler@gmail.com>

app.py

@@ -66,7 +66,6 @@ from util.metrics.prometheus import PrometheusPlugin
 from util.repomirror.api import RepoMirrorAPI
 from util.tufmetadata.api import TUFMetadataAPI
 from util.security.instancekeys import InstanceKeys
-from util.security.signing import Signer
 from util.greenlet_tracing import enable_tracing
 
 
@@ -244,7 +243,6 @@ build_logs = BuildLogs(app)
 authentication = UserAuthentication(app, config_provider, OVERRIDE_CONFIG_DIRECTORY)
 userevents = UserEventsBuilderModule(app)
 superusers = SuperUserManager(app)
-signer = Signer(app, config_provider)
 instance_keys = InstanceKeys(app)
 label_validator = LabelValidator(app)
 build_canceller = BuildCanceller(app)
@@ -260,9 +258,6 @@ dockerfile_build_queue = WorkQueue(
     app.config["DOCKERFILE_BUILD_QUEUE_NAME"], tf, has_namespace=True
 )
 notification_queue = WorkQueue(app.config["NOTIFICATION_QUEUE_NAME"], tf, has_namespace=True)
-secscan_notification_queue = WorkQueue(
-    app.config["SECSCAN_NOTIFICATION_QUEUE_NAME"], tf, has_namespace=False
-)
 export_action_logs_queue = WorkQueue(
     app.config["EXPORT_ACTION_LOGS_QUEUE_NAME"], tf, has_namespace=True
 )
@@ -277,7 +272,6 @@ all_queues = [
     image_replication_queue,
     dockerfile_build_queue,
     notification_queue,
-    secscan_notification_queue,
     chunk_cleanup_queue,
     repository_gc_queue,
     namespace_gc_queue,
@@ -315,10 +309,13 @@ model.config.store = storage
 model.config.register_repo_cleanup_callback(tuf_metadata_api.delete_metadata)
 
 secscan_model.configure(app, instance_keys, storage)
-secscan_model.register_model_cleanup_callbacks(model.config)
 
 logs_model.configure(app.config)
 
+# NOTE: We re-use the page token key here as this is just to obfuscate IDs for V1, and
+# does not need to actually be secure.
+registry_model.set_id_hash_salt(app.config.get("PAGE_TOKEN_KEY"))
+
 
 @login_manager.user_loader
 def load_user(user_uuid):

application.py

@@ -13,7 +13,6 @@ from app import app as application
 
 # Bind all of the blueprints
 import web
-import verbs
 import registry
 import secscan
 

conf/init/supervisord_conf_create.py

@@ -29,14 +29,12 @@ def default_services():
         "notificationworker": {"autostart": "true"},
         "queuecleanupworker": {"autostart": "true"},
         "repositoryactioncounter": {"autostart": "true"},
-        "security_notification_worker": {"autostart": "true"},
         "securityworker": {"autostart": "true"},
         "storagereplication": {"autostart": "true"},
         "teamsyncworker": {"autostart": "true"},
         "dnsmasq": {"autostart": "true"},
         "gunicorn-registry": {"autostart": "true"},
         "gunicorn-secscan": {"autostart": "true"},
-        "gunicorn-verbs": {"autostart": "true"},
         "gunicorn-web": {"autostart": "true"},
         "ip-resolver-update-worker": {"autostart": "true"},
         "jwtproxy": {"autostart": "true"},
@@ -45,6 +43,7 @@ def default_services():
         "pushgateway": {"autostart": "true"},
         "servicekey": {"autostart": "true"},
         "repomirrorworker": {"autostart": "false"},
+        "backfillmanifestworker": {"autostart": "false"},
     }
 
 

conf/nginx/http-base.conf

@@ -49,9 +49,6 @@ upstream web_app_server {
 upstream jwtproxy_secscan {
     server unix:/tmp/jwtproxy_secscan.sock fail_timeout=0;
 }
-upstream verbs_app_server {
-    server unix:/tmp/gunicorn_verbs.sock fail_timeout=0;
-}
 upstream registry_app_server {
     server unix:/tmp/gunicorn_registry.sock fail_timeout=0;
 }

conf/nginx/server-base.conf.jnj

@@ -306,19 +306,6 @@ location = /v1/_ping {
     return 200 'true';
 }
 
-location /c1/ {
-    proxy_buffering off;
-
-    proxy_request_buffering off;
-
-    proxy_pass   http://verbs_app_server;
-    proxy_temp_path /tmp 1 2;
-
-    {% if enable_rate_limits %}
-    limit_req zone=staticauth burst=5 nodelay;
-    {% endif %}
-}
-
 location /static/ {
     # checks for static file, if not found proxy to app
     alias      {{static_dir}}/;

conf/supervisord.conf.jnj

@@ -138,14 +138,6 @@ autostart = {{ config['repositoryactioncounter']['autostart'] }}
 stdout_events_enabled = true
 stderr_events_enabled = true
 
-[program:security_notification_worker]
-environment=
-  PYTHONPATH=%(ENV_QUAYDIR)s
-command=python -m workers.security_notification_worker
-autostart = {{ config['security_notification_worker']['autostart'] }}
-stdout_events_enabled = true
-stderr_events_enabled = true
-
 [program:securityworker]
 environment=
   PYTHONPATH=%(ENV_QUAYDIR)s
@@ -194,14 +186,6 @@ autostart = {{ config['gunicorn-secscan']['autostart'] }}
 stdout_events_enabled = true
 stderr_events_enabled = true
 
-[program:gunicorn-verbs]
-environment=
-  PYTHONPATH=%(ENV_QUAYDIR)s
-command=nice -n 10 gunicorn -c %(ENV_QUAYCONF)s/gunicorn_verbs.py verbs:application
-autostart = {{ config['gunicorn-verbs']['autostart'] }}
-stdout_events_enabled = true
-stderr_events_enabled = true
-
 [program:gunicorn-web]
 environment=
   PYTHONPATH=%(ENV_QUAYDIR)s

config.py

@@ -259,7 +259,6 @@ class DefaultConfig(ImmutableConfig):
     NOTIFICATION_QUEUE_NAME = "notification"
     DOCKERFILE_BUILD_QUEUE_NAME = "dockerfilebuild"
     REPLICATION_QUEUE_NAME = "imagestoragereplication"
-    SECSCAN_NOTIFICATION_QUEUE_NAME = "security_notification"
     CHUNK_CLEANUP_QUEUE_NAME = "chunk_cleanup"
     NAMESPACE_GC_QUEUE_NAME = "namespacegc"
     REPOSITORY_GC_QUEUE_NAME = "repositorygc"
@@ -476,9 +475,6 @@ class DefaultConfig(ImmutableConfig):
     # The version of the API to use for the security scanner.
     SECURITY_SCANNER_API_VERSION = "v1"
 
-    # Namespace whitelist for security scanner.
-    SECURITY_SCANNER_V4_NAMESPACE_WHITELIST = []
-
     # Minimum number of seconds before re-indexing a manifest with the security scanner.
     SECURITY_SCANNER_V4_REINDEX_THRESHOLD = 300
 
@@ -739,3 +735,6 @@ class DefaultConfig(ImmutableConfig):
 
     # Feature Flag: Whether the repository action count worker is enabled.
     FEATURE_REPOSITORY_ACTION_COUNTER = True
+
+    # TEMP FEATURE: Backfill the sizes of manifests.
+    FEATURE_MANIFEST_SIZE_BACKFILL = True

config_app/js/core-config-setup/core-config-setup.js

@@ -74,10 +74,6 @@ angular.module("quay-config")
             return config.AUTHENTICATION_TYPE == 'AppToken';
           }},
 
-          {'id': 'signer', 'title': 'ACI Signing', 'condition': function(config) {
-            return config.FEATURE_ACI_CONVERSION;
-          }},
-
           {'id': 'github-login', 'title': 'Github (Enterprise) Authentication', 'condition': function(config) {
             return config.FEATURE_GITHUB_LOGIN;
           }},

data/database.py

@@ -685,6 +685,7 @@ class User(BaseModel):
                     NamespaceGeoRestriction,
                     ManifestSecurityStatus,
                     RepoMirrorConfig,
+                    UploadedBlob,
                 }
                 | appr_classes
                 | v22_classes
@@ -888,6 +889,7 @@ class Repository(BaseModel):
                 RepoMirrorRule,
                 DeletedRepository,
                 ManifestSecurityStatus,
+                UploadedBlob,
             }
             | appr_classes
             | v22_classes
@@ -1115,6 +1117,7 @@ class Image(BaseModel):
         return list(map(int, self.ancestors.split("/")[1:-1]))
 
 
+@deprecated_model
 class DerivedStorageForImage(BaseModel):
     source_image = ForeignKeyField(Image)
     derivative = ForeignKeyField(ImageStorage)
@@ -1127,6 +1130,7 @@ class DerivedStorageForImage(BaseModel):
         indexes = ((("source_image", "transformation", "uniqueness_hash"), True),)
 
 
+@deprecated_model
 class RepositoryTag(BaseModel):
     name = CharField()
     image = ForeignKeyField(Image)
@@ -1391,8 +1395,8 @@ class ExternalNotificationMethod(BaseModel):
 class RepositoryNotification(BaseModel):
     uuid = CharField(default=uuid_generator, index=True)
     repository = ForeignKeyField(Repository)
-    event = ForeignKeyField(ExternalNotificationEvent)
+    event = EnumField(ExternalNotificationEvent)
-    method = ForeignKeyField(ExternalNotificationMethod)
+    method = EnumField(ExternalNotificationMethod)
     title = CharField(null=True)
     config_json = TextField()
     event_config_json = TextField(default="{}")
@@ -1414,6 +1418,19 @@ class RepositoryAuthorizedEmail(BaseModel):
         )
 
 
+class UploadedBlob(BaseModel):
+    """
+    UploadedBlob tracks a recently uploaded blob and prevents it from being GCed
+    while within the expiration window.
+    """
+
+    id = BigAutoField()
+    repository = ForeignKeyField(Repository)
+    blob = ForeignKeyField(ImageStorage)
+    uploaded_at = DateTimeField(default=datetime.utcnow)
+    expires_at = DateTimeField(index=True)
+
+
 class BlobUpload(BaseModel):
     repository = ForeignKeyField(Repository)
     uuid = CharField(index=True, unique=True)
@@ -1699,12 +1716,16 @@ class Manifest(BaseModel):
     media_type = EnumField(MediaType)
     manifest_bytes = TextField()
 
+    config_media_type = CharField(null=True)
+    layers_compressed_size = BigIntegerField(null=True)
+
     class Meta:
         database = db
         read_only_config = read_only_config
         indexes = (
             (("repository", "digest"), True),
             (("repository", "media_type"), False),
+            (("repository", "config_media_type"), False),
         )
 
 

data/migrations/dba_operator/3383aad1e992-databasemigration.yaml

@@ -0,0 +1,51 @@
+
+---
+apiVersion: dbaoperator.app-sre.redhat.com/v1alpha1
+kind: DatabaseMigration
+metadata:
+  name: 3383aad1e992
+spec:
+  migrationContainerSpec:
+    command:
+    - /quay-registry/quay-entrypoint.sh
+    - migrate
+    - 3383aad1e992
+    image: quay.io/quay/quay
+    name: 3383aad1e992
+  previous: !!python/tuple
+  - 04b9d2191450
+  schemaHints:
+  - columns:
+    - name: id
+      nullable: false
+    - name: repository_id
+      nullable: false
+    - name: blob_id
+      nullable: false
+    - name: uploaded_at
+      nullable: false
+    - name: expires_at
+      nullable: false
+    operation: createTable
+    table: uploadedblob
+  - columns:
+    - name: blob_id
+      nullable: false
+    indexName: uploadedblob_blob_id
+    indexType: index
+    operation: createIndex
+    table: uploadedblob
+  - columns:
+    - name: expires_at
+      nullable: false
+    indexName: uploadedblob_expires_at
+    indexType: index
+    operation: createIndex
+    table: uploadedblob
+  - columns:
+    - name: repository_id
+      nullable: false
+    indexName: uploadedblob_repository_id
+    indexType: index
+    operation: createIndex
+    table: uploadedblob

data/migrations/dba_operator/88e64904d000-databasemigration.yaml

@@ -0,0 +1,36 @@
+
+---
+apiVersion: dbaoperator.app-sre.redhat.com/v1alpha1
+kind: DatabaseMigration
+metadata:
+  name: 88e64904d000
+spec:
+  migrationContainerSpec:
+    command:
+    - /quay-registry/quay-entrypoint.sh
+    - migrate
+    - 88e64904d000
+    image: quay.io/quay/quay
+    name: 88e64904d000
+  previous: !!python/tuple
+  - 3383aad1e992
+  schemaHints:
+  - columns:
+    - name: config_media_type
+      nullable: true
+    operation: addColumn
+    table: manifest
+  - columns:
+    - name: layers_compressed_size
+      nullable: true
+    operation: addColumn
+    table: manifest
+  - columns:
+    - name: repository_id
+      nullable: false
+    - name: config_media_type
+      nullable: true
+    indexName: manifest_repository_id_config_media_type
+    indexType: index
+    operation: createIndex
+    table: manifest

data/migrations/versions/3383aad1e992_add_uploadedblob_table.py

@@ -0,0 +1,57 @@
+"""Add UploadedBlob table
+
+Revision ID: 3383aad1e992
+Revises: 04b9d2191450
+Create Date: 2020-04-21 11:45:54.837077
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "3383aad1e992"
+down_revision = "04b9d2191450"
+
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+
+def upgrade(op, tables, tester):
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "uploadedblob",
+        sa.Column("id", sa.BigInteger(), nullable=False),
+        sa.Column("repository_id", sa.Integer(), nullable=False),
+        sa.Column("blob_id", sa.Integer(), nullable=False),
+        sa.Column("uploaded_at", sa.DateTime(), nullable=False),
+        sa.Column("expires_at", sa.DateTime(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["blob_id"], ["imagestorage.id"], name=op.f("fk_uploadedblob_blob_id_imagestorage")
+        ),
+        sa.ForeignKeyConstraint(
+            ["repository_id"],
+            ["repository.id"],
+            name=op.f("fk_uploadedblob_repository_id_repository"),
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_uploadedblob")),
+    )
+    op.create_index("uploadedblob_blob_id", "uploadedblob", ["blob_id"], unique=False)
+    op.create_index("uploadedblob_expires_at", "uploadedblob", ["expires_at"], unique=False)
+    op.create_index("uploadedblob_repository_id", "uploadedblob", ["repository_id"], unique=False)
+    # ### end Alembic commands ###
+
+    # ### population of test data ### #
+    tester.populate_table(
+        "uploadedblob",
+        [
+            ("repository_id", tester.TestDataType.Foreign("repository")),
+            ("blob_id", tester.TestDataType.Foreign("imagestorage")),
+            ("uploaded_at", tester.TestDataType.DateTime),
+            ("expires_at", tester.TestDataType.DateTime),
+        ],
+    )
+    # ### end population of test data ### #
+
+
+def downgrade(op, tables, tester):
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("uploadedblob")
+    # ### end Alembic commands ###

data/migrations/versions/88e64904d000_add_new_metadata_columns_to_manifest_.py

@@ -0,0 +1,39 @@
+"""Add new metadata columns to Manifest table
+
+Revision ID: 88e64904d000
+Revises: 3383aad1e992
+Create Date: 2020-04-21 14:00:50.376517
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "88e64904d000"
+down_revision = "3383aad1e992"
+
+import sqlalchemy as sa
+
+
+def upgrade(op, tables, tester):
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("manifest", sa.Column("config_media_type", sa.String(length=255), nullable=True))
+    op.add_column("manifest", sa.Column("layers_compressed_size", sa.BigInteger(), nullable=True))
+    op.create_index(
+        "manifest_repository_id_config_media_type",
+        "manifest",
+        ["repository_id", "config_media_type"],
+        unique=False,
+    )
+    # ### end Alembic commands ###
+
+    # ### population of test data ### #
+    tester.populate_column("manifest", "config_media_type", tester.TestDataType.String)
+    tester.populate_column("manifest", "layers_compressed_size", tester.TestDataType.Integer)
+    # ### end population of test data ### #
+
+
+def downgrade(op, tables, tester):
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index("manifest_repository_id_config_media_type", table_name="manifest")
+    op.drop_column("manifest", "layers_compressed_size")
+    op.drop_column("manifest", "config_media_type")
+    # ### end Alembic commands ###

data/model/blob.py

@@ -1,6 +1,6 @@
 import logging
 
-from datetime import datetime
+from datetime import datetime, timedelta
 from uuid import uuid4
 
 from data.model import (
@@ -14,11 +14,13 @@ from data.model import (
 )
 from data.database import (
     Repository,
+    RepositoryState,
     Namespace,
     ImageStorage,
     Image,
     ImageStoragePlacement,
     BlobUpload,
+    UploadedBlob,
     ImageStorageLocation,
     db_random_func,
 )
@@ -27,53 +29,6 @@ from data.database import (
 logger = logging.getLogger(__name__)
 
 
-def get_repository_blob_by_digest(repository, blob_digest):
-    """
-    Find the content-addressable blob linked to the specified repository.
-    """
-    assert blob_digest
-    try:
-        storage = (
-            ImageStorage.select(ImageStorage.uuid)
-            .join(Image)
-            .where(
-                Image.repository == repository,
-                ImageStorage.content_checksum == blob_digest,
-                ImageStorage.uploading == False,
-            )
-            .get()
-        )
-
-        return storage_model.get_storage_by_uuid(storage.uuid)
-    except (ImageStorage.DoesNotExist, InvalidImageException):
-        raise BlobDoesNotExist("Blob does not exist with digest: {0}".format(blob_digest))
-
-
-def get_repo_blob_by_digest(namespace, repo_name, blob_digest):
-    """
-    Find the content-addressable blob linked to the specified repository.
-    """
-    assert blob_digest
-    try:
-        storage = (
-            ImageStorage.select(ImageStorage.uuid)
-            .join(Image)
-            .join(Repository)
-            .join(Namespace, on=(Namespace.id == Repository.namespace_user))
-            .where(
-                Repository.name == repo_name,
-                Namespace.username == namespace,
-                ImageStorage.content_checksum == blob_digest,
-                ImageStorage.uploading == False,
-            )
-            .get()
-        )
-
-        return storage_model.get_storage_by_uuid(storage.uuid)
-    except (ImageStorage.DoesNotExist, InvalidImageException):
-        raise BlobDoesNotExist("Blob does not exist with digest: {0}".format(blob_digest))
-
-
 def store_blob_record_and_temp_link(
     namespace,
     repo_name,
@@ -157,16 +112,26 @@ def temp_link_blob(repository_id, blob_digest, link_expiration_s):
 
 def _temp_link_blob(repository_id, storage, link_expiration_s):
     """ Note: Should *always* be called by a parent under a transaction. """
-    random_image_name = str(uuid4())
+    try:
+        repository = Repository.get(id=repository_id)
+    except Repository.DoesNotExist:
+        return None
 
-    # Create a temporary link into the repository, to be replaced by the v1 metadata later
+    if repository.state == RepositoryState.MARKED_FOR_DELETION:
-    # and create a temporary tag to reference it
+        return None
-    image = Image.create(
+
-        storage=storage, docker_image_id=random_image_name, repository=repository_id
+    return UploadedBlob.create(
+        repository=repository_id,
+        blob=storage,
+        expires_at=datetime.utcnow() + timedelta(seconds=link_expiration_s),
+    )
+
+
+def lookup_expired_uploaded_blobs(repository):
+    """ Looks up all expired uploaded blobs in a repository. """
+    return UploadedBlob.select().where(
+        UploadedBlob.repository == repository, UploadedBlob.expires_at <= datetime.utcnow()
     )
-    temp_tag = tag.create_temporary_hidden_tag(repository_id, image, link_expiration_s)
-    if temp_tag is None:
-        image.delete_instance()
 
 
 def get_stale_blob_upload(stale_timespan):
@@ -192,7 +157,12 @@ def get_blob_upload_by_uuid(upload_uuid):
     Loads the upload with the given UUID, if any.
     """
     try:
-        return BlobUpload.select().where(BlobUpload.uuid == upload_uuid).get()
+        return (
+            BlobUpload.select(BlobUpload, ImageStorageLocation)
+            .join(ImageStorageLocation)
+            .where(BlobUpload.uuid == upload_uuid)
+            .get()
+        )
     except BlobUpload.DoesNotExist:
         return None
 

data/model/gc.py

@@ -1,8 +1,9 @@
 import logging
 
 from peewee import fn, IntegrityError
+from datetime import datetime
 
-from data.model import config, db_transaction, storage, _basequery, tag as pre_oci_tag
+from data.model import config, db_transaction, storage, _basequery, tag as pre_oci_tag, blob
 from data.model.oci import tag as oci_tag
 from data.database import Repository, db_for_update
 from data.database import ApprTag
@@ -28,8 +29,14 @@ from data.database import (
     RepoMirrorConfig,
     RepositoryPermission,
     RepositoryAuthorizedEmail,
+    UploadedBlob,
+)
+from data.database import (
+    RepositoryTag,
+    TagManifest,
+    Image,
+    DerivedStorageForImage,
 )
-from data.database import RepositoryTag, TagManifest, Image, DerivedStorageForImage
 from data.database import TagManifestToManifest, TagToRepositoryTag, TagManifestLabelMap
 
 logger = logging.getLogger(__name__)
@@ -98,6 +105,7 @@ def purge_repository(repo, force=False):
     assert RepositoryTag.select().where(RepositoryTag.repository == repo).count() == 0
     assert Manifest.select().where(Manifest.repository == repo).count() == 0
     assert ManifestBlob.select().where(ManifestBlob.repository == repo).count() == 0
+    assert UploadedBlob.select().where(UploadedBlob.repository == repo).count() == 0
     assert (
         ManifestSecurityStatus.select().where(ManifestSecurityStatus.repository == repo).count()
         == 0
@@ -194,7 +202,27 @@ def _purge_repository_contents(repo):
         if not found:
             break
 
-    # TODO: remove this once we're fully on the OCI data model.
+    # Purge any uploaded blobs that have expired.
+    while True:
+        found = False
+        for uploaded_blobs in _chunk_iterate_for_deletion(
+            UploadedBlob.select().where(UploadedBlob.repository == repo)
+        ):
+            logger.debug(
+                "Found %s uploaded blobs to GC under repository %s", len(uploaded_blobs), repo
+            )
+            found = True
+            context = _GarbageCollectorContext(repo)
+            for uploaded_blob in uploaded_blobs:
+                logger.debug("Deleting uploaded blob %s under repository %s", uploaded_blob, repo)
+                assert uploaded_blob.repository_id == repo.id
+                _purge_uploaded_blob(uploaded_blob, context, allow_non_expired=True)
+
+        if not found:
+            break
+
+    # TODO: remove this once we've removed the foreign key constraints from RepositoryTag
+    # and Image.
     while True:
         found = False
         repo_tag_query = RepositoryTag.select().where(RepositoryTag.repository == repo)
@@ -217,6 +245,7 @@ def _purge_repository_contents(repo):
     assert RepositoryTag.select().where(RepositoryTag.repository == repo).count() == 0
     assert Manifest.select().where(Manifest.repository == repo).count() == 0
     assert ManifestBlob.select().where(ManifestBlob.repository == repo).count() == 0
+    assert UploadedBlob.select().where(UploadedBlob.repository == repo).count() == 0
 
     # Add all remaining images to a new context. We do this here to minimize the number of images
     # we need to load.
@@ -259,6 +288,7 @@ def garbage_collect_repo(repo):
         _run_garbage_collection(context)
         had_changes = True
 
+    # TODO: Remove once we've removed the foreign key constraints from RepositoryTag and Image.
     for tags in _chunk_iterate_for_deletion(pre_oci_tag.lookup_unrecoverable_tags(repo)):
         logger.debug("Found %s tags to GC under repository %s", len(tags), repo)
         context = _GarbageCollectorContext(repo)
@@ -271,6 +301,18 @@ def garbage_collect_repo(repo):
         _run_garbage_collection(context)
         had_changes = True
 
+    # Purge expired uploaded blobs.
+    for uploaded_blobs in _chunk_iterate_for_deletion(blob.lookup_expired_uploaded_blobs(repo)):
+        logger.debug("Found %s uploaded blobs to GC under repository %s", len(uploaded_blobs), repo)
+        context = _GarbageCollectorContext(repo)
+        for uploaded_blob in uploaded_blobs:
+            logger.debug("Deleting uploaded blob %s under repository %s", uploaded_blob, repo)
+            assert uploaded_blob.repository_id == repo.id
+            _purge_uploaded_blob(uploaded_blob, context)
+
+        _run_garbage_collection(context)
+        had_changes = True
+
     return had_changes
 
 
@@ -376,6 +418,16 @@ def _purge_pre_oci_tag(tag, context, allow_non_expired=False):
         reloaded_tag.delete_instance()
 
 
+def _purge_uploaded_blob(uploaded_blob, context, allow_non_expired=False):
+    assert allow_non_expired or uploaded_blob.expires_at <= datetime.utcnow()
+
+    # Add the storage to be checked.
+    context.add_blob_id(uploaded_blob.blob_id)
+
+    # Delete the uploaded blob.
+    uploaded_blob.delete_instance()
+
+
 def _check_manifest_used(manifest_id):
     assert manifest_id is not None
 

data/model/image.py

@@ -23,13 +23,10 @@ from data.database import (
     ImageStorage,
     ImageStorageLocation,
     RepositoryPermission,
-    DerivedStorageForImage,
     ImageStorageTransformation,
     User,
 )
 
-from util.canonicaljson import canonicalize
-
 logger = logging.getLogger(__name__)
 
 
@@ -554,62 +551,3 @@ def set_secscan_status(image, indexed, version):
         .where((Image.security_indexed_engine != version) | (Image.security_indexed != indexed))
         .execute()
     ) != 0
-
-
-def _get_uniqueness_hash(varying_metadata):
-    if not varying_metadata:
-        return None
-
-    return hashlib.sha256(json.dumps(canonicalize(varying_metadata)).encode("utf-8")).hexdigest()
-
-
-def find_or_create_derived_storage(
-    source_image, transformation_name, preferred_location, varying_metadata=None
-):
-    existing = find_derived_storage_for_image(source_image, transformation_name, varying_metadata)
-    if existing is not None:
-        return existing
-
-    uniqueness_hash = _get_uniqueness_hash(varying_metadata)
-    trans = ImageStorageTransformation.get(name=transformation_name)
-    new_storage = storage.create_v1_storage(preferred_location)
-
-    try:
-        derived = DerivedStorageForImage.create(
-            source_image=source_image,
-            derivative=new_storage,
-            transformation=trans,
-            uniqueness_hash=uniqueness_hash,
-        )
-    except IntegrityError:
-        # Storage was created while this method executed. Just return the existing.
-        ImageStoragePlacement.delete().where(ImageStoragePlacement.storage == new_storage).execute()
-        new_storage.delete_instance()
-        return find_derived_storage_for_image(source_image, transformation_name, varying_metadata)
-
-    return derived
-
-
-def find_derived_storage_for_image(source_image, transformation_name, varying_metadata=None):
-    uniqueness_hash = _get_uniqueness_hash(varying_metadata)
-
-    try:
-        found = (
-            DerivedStorageForImage.select(ImageStorage, DerivedStorageForImage)
-            .join(ImageStorage)
-            .switch(DerivedStorageForImage)
-            .join(ImageStorageTransformation)
-            .where(
-                DerivedStorageForImage.source_image == source_image,
-                ImageStorageTransformation.name == transformation_name,
-                DerivedStorageForImage.uniqueness_hash == uniqueness_hash,
-            )
-            .get()
-        )
-        return found
-    except DerivedStorageForImage.DoesNotExist:
-        return None
-
-
-def delete_derived_storage(derived_storage):
-    derived_storage.derivative.delete_instance(recursive=True)

data/model/oauth.py

@@ -352,9 +352,14 @@ def lookup_access_token_by_uuid(token_uuid):
 
 def lookup_access_token_for_user(user_obj, token_uuid):
     try:
-        return OAuthAccessToken.get(
+        return (
+            OAuthAccessToken.select(OAuthAccessToken, User)
+            .join(User)
+            .where(
                 OAuthAccessToken.authorized_user == user_obj, OAuthAccessToken.uuid == token_uuid
             )
+            .get()
+        )
     except OAuthAccessToken.DoesNotExist:
         return None
 

data/model/oci/blob.py

@@ -1,7 +1,6 @@
-from data.database import ImageStorage, ManifestBlob
+from data.database import ImageStorage, ManifestBlob, UploadedBlob
 from data.model import BlobDoesNotExist
 from data.model.storage import get_storage_by_uuid, InvalidImageException
-from data.model.blob import get_repository_blob_by_digest as legacy_get
 
 
 def get_repository_blob_by_digest(repository, blob_digest):
@@ -9,8 +8,34 @@ def get_repository_blob_by_digest(repository, blob_digest):
     Find the content-addressable blob linked to the specified repository and returns it or None if
     none.
     """
+    # First try looking for a recently uploaded blob. If none found that is matching,
+    # check the repository itself.
+    storage = _lookup_blob_uploaded(repository, blob_digest)
+    if storage is None:
+        storage = _lookup_blob_in_repository(repository, blob_digest)
+
+    return get_storage_by_uuid(storage.uuid) if storage is not None else None
+
+
+def _lookup_blob_uploaded(repository, blob_digest):
     try:
-        storage = (
+        return (
+            ImageStorage.select(ImageStorage.uuid)
+            .join(UploadedBlob)
+            .where(
+                UploadedBlob.repository == repository,
+                ImageStorage.content_checksum == blob_digest,
+                ImageStorage.uploading == False,
+            )
+            .get()
+        )
+    except ImageStorage.DoesNotExist:
+        return None
+
+
+def _lookup_blob_in_repository(repository, blob_digest):
+    try:
+        return (
             ImageStorage.select(ImageStorage.uuid)
             .join(ManifestBlob)
             .where(
@@ -20,12 +45,5 @@ def get_repository_blob_by_digest(repository, blob_digest):
             )
             .get()
         )
-
+    except ImageStorage.DoesNotExist:
-        return get_storage_by_uuid(storage.uuid)
-    except (ImageStorage.DoesNotExist, InvalidImageException):
-        # TODO: Remove once we are no longer using the legacy tables.
-        # Try the legacy call.
-        try:
-            return legacy_get(repository, blob_digest)
-        except BlobDoesNotExist:
         return None

data/model/oci/manifest.py

@@ -1,4 +1,6 @@
+import json
 import logging
+import os
 
 from collections import namedtuple
 
@@ -10,6 +12,10 @@ from data.database import (
     ManifestBlob,
     ManifestLegacyImage,
     ManifestChild,
+    ImageStorage,
+    ImageStoragePlacement,
+    ImageStorageTransformation,
+    ImageStorageSignature,
     db_transaction,
 )
 from data.model import BlobDoesNotExist
@@ -17,11 +23,12 @@ from data.model.blob import get_or_create_shared_blob, get_shared_blob
 from data.model.oci.tag import filter_to_alive_tags, create_temporary_tag_if_necessary
 from data.model.oci.label import create_manifest_label
 from data.model.oci.retriever import RepositoryContentRetriever
-from data.model.storage import lookup_repo_storages_by_content_checksum
+from data.model.storage import lookup_repo_storages_by_content_checksum, create_v1_storage
 from data.model.image import lookup_repository_images, get_image, synthesize_v1_image
 from image.docker.schema2 import EMPTY_LAYER_BLOB_DIGEST, EMPTY_LAYER_BYTES
 from image.docker.schema1 import ManifestException
 from image.docker.schema2.list import MalformedSchema2ManifestList
+from util.canonicaljson import canonicalize
 from util.validation import is_json
 
 
@@ -206,90 +213,16 @@ def _create_manifest(
             child_manifest_rows[child_manifest_info.manifest.digest] = child_manifest_info.manifest
             child_manifest_label_dicts.append(labels)
 
-    # Ensure all the blobs in the manifest exist.
+    # Build the map from required blob digests to the blob objects.
-    digests = set(manifest_interface_instance.local_blob_digests)
+    blob_map = _build_blob_map(
-    blob_map = {}
-
-    # If the special empty layer is required, simply load it directly. This is much faster
-    # than trying to load it on a per repository basis, and that is unnecessary anyway since
-    # this layer is predefined.
-    if EMPTY_LAYER_BLOB_DIGEST in digests:
-        digests.remove(EMPTY_LAYER_BLOB_DIGEST)
-        blob_map[EMPTY_LAYER_BLOB_DIGEST] = get_shared_blob(EMPTY_LAYER_BLOB_DIGEST)
-        if not blob_map[EMPTY_LAYER_BLOB_DIGEST]:
-            if raise_on_error:
-                raise CreateManifestException("Unable to retrieve specialized empty blob")
-
-            logger.warning("Could not find the special empty blob in storage")
-            return None
-
-    if digests:
-        query = lookup_repo_storages_by_content_checksum(repository_id, digests)
-        blob_map.update({s.content_checksum: s for s in query})
-        for digest_str in digests:
-            if digest_str not in blob_map:
-                logger.warning(
-                    "Unknown blob `%s` under manifest `%s` for repository `%s`",
-                    digest_str,
-                    manifest_interface_instance.digest,
         repository_id,
+        manifest_interface_instance,
+        retriever,
+        storage,
+        raise_on_error,
+        require_empty_layer=False,
     )
-
+    if blob_map is None:
-                if raise_on_error:
-                    raise CreateManifestException("Unknown blob `%s`" % digest_str)
-
-                return None
-
-    # Special check: If the empty layer blob is needed for this manifest, add it to the
-    # blob map. This is necessary because Docker decided to elide sending of this special
-    # empty layer in schema version 2, but we need to have it referenced for GC and schema version 1.
-    if EMPTY_LAYER_BLOB_DIGEST not in blob_map:
-        try:
-            requires_empty_layer = manifest_interface_instance.get_requires_empty_layer_blob(
-                retriever
-            )
-        except ManifestException as ex:
-            if raise_on_error:
-                raise CreateManifestException(str(ex))
-
-            return None
-
-        if requires_empty_layer is None:
-            if raise_on_error:
-                raise CreateManifestException("Could not load configuration blob")
-
-            return None
-
-        if requires_empty_layer:
-            shared_blob = get_or_create_shared_blob(
-                EMPTY_LAYER_BLOB_DIGEST, EMPTY_LAYER_BYTES, storage
-            )
-            assert not shared_blob.uploading
-            assert shared_blob.content_checksum == EMPTY_LAYER_BLOB_DIGEST
-            blob_map[EMPTY_LAYER_BLOB_DIGEST] = shared_blob
-
-    # Determine and populate the legacy image if necessary. Manifest lists will not have a legacy
-    # image.
-    legacy_image = None
-    if manifest_interface_instance.has_legacy_image:
-        try:
-            legacy_image_id = _populate_legacy_image(
-                repository_id, manifest_interface_instance, blob_map, retriever, raise_on_error
-            )
-        except ManifestException as me:
-            logger.error("Got manifest error when populating legacy images: %s", me)
-            if raise_on_error:
-                raise CreateManifestException(
-                    "Attempt to create an invalid manifest: %s. Please report this issue." % me
-                )
-
-            return None
-
-        if legacy_image_id is None:
-            return None
-
-        legacy_image = get_image(repository_id, legacy_image_id)
-        if legacy_image is None:
         return None
 
     # Create the manifest and its blobs.
@@ -314,6 +247,8 @@ def _create_manifest(
                 digest=manifest_interface_instance.digest,
                 media_type=media_type,
                 manifest_bytes=manifest_interface_instance.bytes.as_encoded_str(),
+                config_media_type=manifest_interface_instance.config_media_type,
+                layers_compressed_size=manifest_interface_instance.layers_compressed_size,
             )
         except IntegrityError as ie:
             try:
@@ -339,12 +274,6 @@ def _create_manifest(
         if blobs_to_insert:
             ManifestBlob.insert_many(blobs_to_insert).execute()
 
-        # Set the legacy image (if applicable).
-        if legacy_image is not None:
-            ManifestLegacyImage.create(
-                repository=repository_id, image=legacy_image, manifest=manifest
-            )
-
         # Insert the manifest child rows (if applicable).
         if child_manifest_rows:
             children_to_insert = [
@@ -392,6 +321,131 @@ def _create_manifest(
     return CreatedManifest(manifest=manifest, newly_created=True, labels_to_apply=labels_to_apply)
 
 
+def _build_blob_map(
+    repository_id,
+    manifest_interface_instance,
+    retriever,
+    storage,
+    raise_on_error=False,
+    require_empty_layer=True,
+):
+    """ Builds a map containing the digest of each blob referenced by the given manifest,
+        to its associated Blob row in the database. This method also verifies that the blob
+        is accessible under the given repository. Returns None on error (unless raise_on_error
+        is specified). If require_empty_layer is set to True, the method will check if the manifest
+        references the special shared empty layer blob and, if so, add it to the map. Otherwise,
+        the empty layer blob is only returned if it was *explicitly* referenced in the manifest.
+        This is necessary because Docker V2_2/OCI manifests can implicitly reference an empty blob
+        layer for image layers that only change metadata.
+    """
+
+    # Ensure all the blobs in the manifest exist.
+    digests = set(manifest_interface_instance.local_blob_digests)
+    blob_map = {}
+
+    # If the special empty layer is required, simply load it directly. This is much faster
+    # than trying to load it on a per repository basis, and that is unnecessary anyway since
+    # this layer is predefined.
+    if EMPTY_LAYER_BLOB_DIGEST in digests:
+        digests.remove(EMPTY_LAYER_BLOB_DIGEST)
+        blob_map[EMPTY_LAYER_BLOB_DIGEST] = get_shared_blob(EMPTY_LAYER_BLOB_DIGEST)
+        if not blob_map[EMPTY_LAYER_BLOB_DIGEST]:
+            if raise_on_error:
+                raise CreateManifestException("Unable to retrieve specialized empty blob")
+
+            logger.warning("Could not find the special empty blob in storage")
+            return None
+
+    if digests:
+        query = lookup_repo_storages_by_content_checksum(repository_id, digests, with_uploads=True)
+        blob_map.update({s.content_checksum: s for s in query})
+        for digest_str in digests:
+            if digest_str not in blob_map:
+                logger.warning(
+                    "Unknown blob `%s` under manifest `%s` for repository `%s`",
+                    digest_str,
+                    manifest_interface_instance.digest,
+                    repository_id,
+                )
+
+                if raise_on_error:
+                    raise CreateManifestException("Unknown blob `%s`" % digest_str)
+
+                return None
+
+    # Special check: If the empty layer blob is needed for this manifest, add it to the
+    # blob map. This is necessary because Docker decided to elide sending of this special
+    # empty layer in schema version 2, but we need to have it referenced for schema version 1.
+    if require_empty_layer and EMPTY_LAYER_BLOB_DIGEST not in blob_map:
+        try:
+            requires_empty_layer = manifest_interface_instance.get_requires_empty_layer_blob(
+                retriever
+            )
+        except ManifestException as ex:
+            if raise_on_error:
+                raise CreateManifestException(str(ex))
+
+            return None
+
+        if requires_empty_layer is None:
+            if raise_on_error:
+                raise CreateManifestException("Could not load configuration blob")
+
+            return None
+
+        if requires_empty_layer:
+            shared_blob = get_or_create_shared_blob(
+                EMPTY_LAYER_BLOB_DIGEST, EMPTY_LAYER_BYTES, storage
+            )
+            assert not shared_blob.uploading
+            assert shared_blob.content_checksum == EMPTY_LAYER_BLOB_DIGEST
+            blob_map[EMPTY_LAYER_BLOB_DIGEST] = shared_blob
+
+    return blob_map
+
+
+def populate_legacy_images_for_testing(manifest, manifest_interface_instance, storage):
+    """ Populates the legacy image rows for the given manifest. """
+    # NOTE: This method is only kept around for use by legacy tests that still require
+    # legacy images. As a result, we make sure we're in testing mode before we run.
+    assert os.getenv("TEST") == "true"
+
+    repository_id = manifest.repository_id
+    retriever = RepositoryContentRetriever.for_repository(repository_id, storage)
+
+    blob_map = _build_blob_map(
+        repository_id, manifest_interface_instance, storage, True, require_empty_layer=True
+    )
+    if blob_map is None:
+        return None
+
+    # Determine and populate the legacy image if necessary. Manifest lists will not have a legacy
+    # image.
+    legacy_image = None
+    if manifest_interface_instance.has_legacy_image:
+        try:
+            legacy_image_id = _populate_legacy_image(
+                repository_id, manifest_interface_instance, blob_map, retriever, True
+            )
+        except ManifestException as me:
+            raise CreateManifestException(
+                "Attempt to create an invalid manifest: %s. Please report this issue." % me
+            )
+
+        if legacy_image_id is None:
+            return None
+
+        legacy_image = get_image(repository_id, legacy_image_id)
+        if legacy_image is None:
+            return None
+
+        # Set the legacy image (if applicable).
+        if legacy_image is not None:
+            ManifestLegacyImage.create(
+                repository=repository_id, image=legacy_image, manifest=manifest
+            )
+
+
 def _populate_legacy_image(
     repository_id, manifest_interface_instance, blob_map, retriever, raise_on_error=False
 ):

data/model/oci/tag.py

@@ -123,7 +123,14 @@ def list_repository_tag_history(
     Note that the returned Manifest will not contain the manifest contents.
     """
     query = (
-        Tag.select(Tag, Manifest.id, Manifest.digest, Manifest.media_type)
+        Tag.select(
+            Tag,
+            Manifest.id,
+            Manifest.digest,
+            Manifest.media_type,
+            Manifest.layers_compressed_size,
+            Manifest.config_media_type,
+        )
         .join(Manifest)
         .where(Tag.repository == repository_id)
         .order_by(Tag.lifetime_start_ms.desc(), Tag.name)
@@ -141,31 +148,14 @@ def list_repository_tag_history(
 
     if active_tags_only:
         query = filter_to_alive_tags(query)
-
+    else:
         query = filter_to_visible_tags(query)
+
     results = list(query)
 
     return results[0:page_size], len(results) > page_size
 
 
-def get_legacy_images_for_tags(tags):
-    """
-    Returns a map from tag ID to the legacy image for the tag.
-    """
-    if not tags:
-        return {}
-
-    query = (
-        ManifestLegacyImage.select(ManifestLegacyImage, Image, ImageStorage)
-        .join(Image)
-        .join(ImageStorage)
-        .where(ManifestLegacyImage.manifest << [tag.manifest_id for tag in tags])
-    )
-
-    by_manifest = {mli.manifest_id: mli.image for mli in query}
-    return {tag.id: by_manifest[tag.manifest_id] for tag in tags if tag.manifest_id in by_manifest}
-
-
 def find_matching_tag(repository_id, tag_names, tag_kinds=None):
     """
     Finds an alive tag in the specified repository with one of the specified tag names and returns
@@ -417,7 +407,6 @@ def delete_tags_for_manifest(manifest):
     """
     query = Tag.select().where(Tag.manifest == manifest)
     query = filter_to_alive_tags(query)
-    query = filter_to_visible_tags(query)
 
     tags = list(query)
     now_ms = get_epoch_timestamp_ms()
@@ -446,9 +435,8 @@ def filter_to_alive_tags(query, now_ms=None, model=Tag):
     if now_ms is None:
         now_ms = get_epoch_timestamp_ms()
 
-    return query.where((model.lifetime_end_ms >> None) | (model.lifetime_end_ms > now_ms)).where(
+    query = query.where((model.lifetime_end_ms >> None) | (model.lifetime_end_ms > now_ms))
-        model.hidden == False
+    return filter_to_visible_tags(query)
-    )
 
 
 def set_tag_expiration_sec_for_manifest(manifest_id, expiration_seconds):
@@ -578,70 +566,6 @@ def tags_containing_legacy_image(image):
     return filter_to_alive_tags(tags)
 
 
-def lookup_notifiable_tags_for_legacy_image(docker_image_id, storage_uuid, event_name):
-    """
-    Yields any alive Tags found in repositories with an event with the given name registered and
-    whose legacy Image has the given docker image ID and storage UUID.
-    """
-    event = ExternalNotificationEvent.get(name=event_name)
-    images = (
-        Image.select()
-        .join(ImageStorage)
-        .where(Image.docker_image_id == docker_image_id, ImageStorage.uuid == storage_uuid)
-    )
-
-    for image in list(images):
-        # Ensure the image is under a repository that supports the event.
-        try:
-            RepositoryNotification.get(repository=image.repository_id, event=event)
-        except RepositoryNotification.DoesNotExist:
-            continue
-
-        # If found in a repository with the valid event, yield the tag(s) that contains the image.
-        for tag in tags_containing_legacy_image(image):
-            yield tag
-
-
-def get_tags_for_legacy_image(image_id):
-    """ Returns the Tag's that have the associated legacy image. 
-    
-        NOTE: This is for legacy support in the old security notification worker and should
-        be removed once that code is no longer necessary.
-    """
-    return filter_to_alive_tags(
-        Tag.select()
-        .distinct()
-        .join(Manifest)
-        .join(ManifestLegacyImage)
-        .where(ManifestLegacyImage.image == image_id)
-    )
-
-
-def _filter_has_repository_event(query, event):
-    """ Filters the query by ensuring the repositories returned have the given event.
-    
-        NOTE: This is for legacy support in the old security notification worker and should
-        be removed once that code is no longer necessary.
-    """
-    return (
-        query.join(Repository)
-        .join(RepositoryNotification)
-        .where(RepositoryNotification.event == event)
-    )
-
-
-def filter_tags_have_repository_event(query, event):
-    """ Filters the query by ensuring the tags live in a repository that has the given
-        event. Also orders the results by lifetime_start_ms.
-    
-        NOTE: This is for legacy support in the old security notification worker and should
-        be removed once that code is no longer necessary.
-    """
-    query = _filter_has_repository_event(query, event)
-    query = query.switch(Tag).order_by(Tag.lifetime_start_ms.desc())
-    return query
-
-
 def find_repository_with_garbage(limit_to_gc_policy_s):
     """ Returns a repository that has garbage (defined as an expired Tag that is past
         the repo's namespace's expiration window) or None if none.
@@ -680,3 +604,20 @@ def find_repository_with_garbage(limit_to_gc_policy_s):
         return None
     except Repository.DoesNotExist:
         return None
+
+
+def get_legacy_images_for_tags(tags):
+    """
+    Returns a map from tag ID to the legacy image for the tag.
+    """
+    if not tags:
+        return {}
+
+    query = (
+        ManifestLegacyImage.select(ManifestLegacyImage, Image)
+        .join(Image)
+        .where(ManifestLegacyImage.manifest << [tag.manifest_id for tag in tags])
+    )
+
+    by_manifest = {mli.manifest_id: mli.image for mli in query}
+    return {tag.id: by_manifest[tag.manifest_id] for tag in tags if tag.manifest_id in by_manifest}

data/model/oci/test/test_oci_manifest.py

@@ -166,6 +166,8 @@ def test_get_or_create_manifest(schema_version, initialized_db):
         builder.add_layer(random_digest, len(random_data.encode("utf-8")))
         sample_manifest_instance = builder.build()
 
+    assert sample_manifest_instance.layers_compressed_size is not None
+
     # Create a new manifest.
     created_manifest = get_or_create_manifest(repository, sample_manifest_instance, storage)
     created = created_manifest.manifest
@@ -177,15 +179,18 @@ def test_get_or_create_manifest(schema_version, initialized_db):
     assert created.digest == sample_manifest_instance.digest
     assert created.manifest_bytes == sample_manifest_instance.bytes.as_encoded_str()
     assert created_manifest.labels_to_apply == expected_labels
+    assert created.config_media_type == sample_manifest_instance.config_media_type
+    assert created.layers_compressed_size == sample_manifest_instance.layers_compressed_size
+
+    # Lookup the manifest and verify.
+    found = lookup_manifest(repository, created.digest, allow_dead=True)
+    assert found.digest == created.digest
+    assert found.config_media_type == created.config_media_type
+    assert found.layers_compressed_size == created.layers_compressed_size
 
     # Verify it has a temporary tag pointing to it.
     assert Tag.get(manifest=created, hidden=True).lifetime_end_ms
 
-    # Verify the legacy image.
-    legacy_image = get_legacy_image_for_manifest(created)
-    assert legacy_image is not None
-    assert legacy_image.storage.content_checksum == random_digest
-
     # Verify the linked blobs.
     blob_digests = [
         mb.blob.content_checksum
@@ -295,6 +300,8 @@ def test_get_or_create_manifest_list(initialized_db):
     assert created_list
     assert created_list.media_type.name == manifest_list.media_type
     assert created_list.digest == manifest_list.digest
+    assert created_list.config_media_type == manifest_list.config_media_type
+    assert created_list.layers_compressed_size == manifest_list.layers_compressed_size
 
     # Ensure the child manifest links exist.
     child_manifests = {
@@ -423,6 +430,8 @@ def test_get_or_create_manifest_with_remote_layers(initialized_db):
     assert created_manifest
     assert created_manifest.media_type.name == manifest.media_type
     assert created_manifest.digest == manifest.digest
+    assert created_manifest.config_media_type == manifest.config_media_type
+    assert created_manifest.layers_compressed_size == manifest.layers_compressed_size
 
     # Verify the legacy image.
     legacy_image = get_legacy_image_for_manifest(created_manifest)

data/model/oci/test/test_oci_tag.py

@@ -18,7 +18,6 @@ from data.model.oci.tag import (
     get_most_recent_tag,
     get_most_recent_tag_lifetime_start,
     list_alive_tags,
-    get_legacy_images_for_tags,
     filter_to_alive_tags,
     filter_to_visible_tags,
     list_repository_tag_history,
@@ -92,13 +91,6 @@ def test_list_alive_tags(initialized_db):
     for tag in filter_to_visible_tags(filter_to_alive_tags(Tag.select())):
         tags = list_alive_tags(tag.repository)
         assert tag in tags
-
-        with assert_query_count(1):
-            legacy_images = get_legacy_images_for_tags(tags)
-
-        for tag in tags:
-            assert ManifestLegacyImage.get(manifest=tag.manifest).image == legacy_images[tag.id]
-
         found = True
 
     assert found
@@ -154,6 +146,11 @@ def test_list_repository_tag_history(namespace_name, repo_name, initialized_db):
     assert results
     assert not has_more
 
+    assert results[0].manifest.id is not None
+    assert results[0].manifest.digest is not None
+    assert results[0].manifest.media_type is not None
+    assert results[0].manifest.layers_compressed_size is not None
+
 
 def test_list_repository_tag_history_with_history(initialized_db):
     repo = get_repository("devtable", "history")

data/model/repo_mirror.py

@@ -2,7 +2,7 @@ import re
 
 from datetime import datetime, timedelta
 
-from peewee import IntegrityError, fn
+from peewee import IntegrityError, fn, JOIN
 from jsonschema import ValidationError
 
 from data.database import (
@@ -14,6 +14,7 @@ from data.database import (
     Repository,
     uuid_generator,
     db_transaction,
+    User,
 )
 from data.fields import DecryptedValue
 from data.model import DataModelException
@@ -362,7 +363,14 @@ def get_mirror(repository):
     Return the RepoMirrorConfig associated with the given Repository, or None if it doesn't exist.
     """
     try:
-        return RepoMirrorConfig.get(repository=repository)
+        return (
+            RepoMirrorConfig.select(RepoMirrorConfig, User, RepoMirrorRule)
+            .join(User, JOIN.LEFT_OUTER)
+            .switch(RepoMirrorConfig)
+            .join(RepoMirrorRule)
+            .where(RepoMirrorConfig.repository == repository)
+            .get()
+        )
     except RepoMirrorConfig.DoesNotExist:
         return None
 

data/model/repository.py

@@ -32,7 +32,6 @@ from data.database import (
     RepositoryActionCount,
     Role,
     RepositoryAuthorizedEmail,
-    DerivedStorageForImage,
     Label,
     db_for_update,
     get_epoch_timestamp,
@@ -500,6 +499,10 @@ def lookup_repository(repo_id):
         return None
 
 
+def repository_visibility_name(repository):
+    return "public" if is_repository_public(repository) else "private"
+
+
 def is_repository_public(repository):
     return repository.visibility_id == _basequery.get_public_repo_visibility().id
 

data/model/storage.py

@@ -25,6 +25,7 @@ from data.database import (
     ApprBlob,
     ensure_under_transaction,
     ManifestBlob,
+    UploadedBlob,
 )
 
 logger = logging.getLogger(__name__)
@@ -86,6 +87,12 @@ def _is_storage_orphaned(candidate_id):
         except Image.DoesNotExist:
             pass
 
+        try:
+            UploadedBlob.get(blob=candidate_id)
+            return False
+        except UploadedBlob.DoesNotExist:
+            pass
+
     return True
 
 
@@ -307,23 +314,46 @@ def get_layer_path_for_storage(storage_uuid, cas_path, content_checksum):
     return store.blob_path(content_checksum)
 
 
-def lookup_repo_storages_by_content_checksum(repo, checksums, by_manifest=False):
+def lookup_repo_storages_by_content_checksum(repo, checksums, with_uploads=False):
     """
     Looks up repository storages (without placements) matching the given repository and checksum.
     """
+    checksums = list(set(checksums))
     if not checksums:
         return []
 
+    # If the request is not with uploads, simply return the blobs found under the manifests
+    # for the repository.
+    if not with_uploads:
+        return _lookup_repo_storages_by_content_checksum(repo, checksums, ManifestBlob)
+
+    # Otherwise, first check the UploadedBlob table and, once done, then check the ManifestBlob
+    # table.
+    found_via_uploaded = list(
+        _lookup_repo_storages_by_content_checksum(repo, checksums, UploadedBlob)
+    )
+    if len(found_via_uploaded) == len(checksums):
+        return found_via_uploaded
+
+    checksums_remaining = set(checksums) - {
+        uploaded.content_checksum for uploaded in found_via_uploaded
+    }
+    found_via_manifest = list(
+        _lookup_repo_storages_by_content_checksum(repo, checksums_remaining, ManifestBlob)
+    )
+    return found_via_uploaded + found_via_manifest
+
+
+def _lookup_repo_storages_by_content_checksum(repo, checksums, model_class):
+    assert checksums
+
     # There may be many duplicates of the checksums, so for performance reasons we are going
     # to use a union to select just one storage with each checksum
     queries = []
 
-    for counter, checksum in enumerate(set(checksums)):
+    for counter, checksum in enumerate(checksums):
         query_alias = "q{0}".format(counter)
 
-        # TODO: Remove once we have a new-style model for tracking temp uploaded blobs and
-        # all legacy tables have been removed.
-        if by_manifest:
         candidate_subq = (
             ImageStorage.select(
                 ImageStorage.id,
@@ -334,30 +364,15 @@ def lookup_repo_storages_by_content_checksum(repo, checksums, by_manifest=False)
                 ImageStorage.uncompressed_size,
                 ImageStorage.uploading,
             )
-                .join(ManifestBlob)
+            .join(model_class)
-                .where(ManifestBlob.repository == repo, ImageStorage.content_checksum == checksum)
+            .where(model_class.repository == repo, ImageStorage.content_checksum == checksum)
-                .limit(1)
-                .alias(query_alias)
-            )
-        else:
-            candidate_subq = (
-                ImageStorage.select(
-                    ImageStorage.id,
-                    ImageStorage.content_checksum,
-                    ImageStorage.image_size,
-                    ImageStorage.uuid,
-                    ImageStorage.cas_path,
-                    ImageStorage.uncompressed_size,
-                    ImageStorage.uploading,
-                )
-                .join(Image)
-                .where(Image.repository == repo, ImageStorage.content_checksum == checksum)
             .limit(1)
             .alias(query_alias)
         )
 
         queries.append(ImageStorage.select(SQL("*")).from_(candidate_subq))
 
+    assert queries
     return _basequery.reduce_as_tree(queries)
 
 

data/model/tag.py

@@ -1,75 +1,9 @@
-import logging
-
-from calendar import timegm
-from datetime import datetime
-from uuid import uuid4
-
-from peewee import IntegrityError, JOIN, fn
-from data.model import (
-    image,
-    storage,
-    db_transaction,
-    DataModelException,
-    _basequery,
-    InvalidManifestException,
-    TagAlreadyCreatedException,
-    StaleTagException,
-    config,
-)
 from data.database import (
     RepositoryTag,
     Repository,
-    RepositoryState,
-    Image,
-    ImageStorage,
     Namespace,
-    TagManifest,
-    RepositoryNotification,
-    Label,
-    TagManifestLabel,
     get_epoch_timestamp,
-    db_for_update,
-    Manifest,
-    ManifestLabel,
-    ManifestBlob,
-    ManifestLegacyImage,
-    TagManifestToManifest,
-    TagManifestLabelMap,
-    TagToRepositoryTag,
-    Tag,
-    get_epoch_timestamp_ms,
 )
-from util.timedeltastring import convert_to_timedelta
-
-
-logger = logging.getLogger(__name__)
-
-
-def create_temporary_hidden_tag(repo, image_obj, expiration_s):
-    """
-    Create a tag with a defined timeline, that will not appear in the UI or CLI.
-
-    Returns the name of the temporary tag or None on error.
-    """
-    now_ts = get_epoch_timestamp()
-    expire_ts = now_ts + expiration_s
-    tag_name = str(uuid4())
-
-    # Ensure the repository is not marked for deletion.
-    with db_transaction():
-        current = Repository.get(id=repo)
-        if current.state == RepositoryState.MARKED_FOR_DELETION:
-            return None
-
-        RepositoryTag.create(
-            repository=repo,
-            image=image_obj,
-            name=tag_name,
-            lifetime_start_ts=now_ts,
-            lifetime_end_ts=expire_ts,
-            hidden=True,
-        )
-        return tag_name
 
 
 def lookup_unrecoverable_tags(repo):

data/model/test/test_gc.py

@@ -30,6 +30,7 @@ from data.database import (
     TagToRepositoryTag,
     ImageStorageLocation,
     RepositoryTag,
+    UploadedBlob,
 )
 from data.model.oci.test.test_oci_manifest import create_manifest_for_testing
 from digest.digest_tools import sha256_digest
@@ -61,11 +62,7 @@ def default_tag_policy(initialized_db):
 
 def _delete_temp_links(repo):
     """ Deletes any temp links to blobs. """
-    for hidden in list(
+    UploadedBlob.delete().where(UploadedBlob.repository == repo).execute()
-        RepositoryTag.select().where(RepositoryTag.hidden == True, RepositoryTag.repository == repo)
-    ):
-        hidden.delete_instance()
-        hidden.image.delete_instance()
 
 
 def _populate_blob(repo, content):
@@ -128,6 +125,10 @@ def move_tag(repository, tag, image_ids, expect_gc=True):
         repo_ref, manifest, tag, storage, raise_on_error=True
     )
 
+    tag_ref = registry_model.get_repo_tag(repo_ref, tag)
+    manifest_ref = registry_model.get_manifest_for_tag(tag_ref)
+    registry_model.populate_legacy_images_for_testing(manifest_ref, storage)
+
     if expect_gc:
         assert model.gc.garbage_collect_repo(repository) == expect_gc
 
@@ -156,10 +157,17 @@ def _get_dangling_storage_count():
     storage_ids = set([current.id for current in ImageStorage.select()])
     referenced_by_image = set([image.storage_id for image in Image.select()])
     referenced_by_manifest = set([blob.blob_id for blob in ManifestBlob.select()])
-    referenced_by_derived = set(
+    referenced_by_uploaded = set([upload.blob_id for upload in UploadedBlob.select()])
+    referenced_by_derived_image = set(
         [derived.derivative_id for derived in DerivedStorageForImage.select()]
     )
-    return len(storage_ids - referenced_by_image - referenced_by_derived - referenced_by_manifest)
+    return len(
+        storage_ids
+        - referenced_by_image
+        - referenced_by_derived_image
+        - referenced_by_manifest
+        - referenced_by_uploaded
+    )
 
 
 def _get_dangling_label_count():
@@ -199,7 +207,7 @@ def assert_gc_integrity(expect_storage_removed=True):
     for blob_row in ApprBlob.select():
         existing_digests.add(blob_row.digest)
 
-    # Store the number of dangling storages and labels.
+    # Store the number of dangling objects.
     existing_storage_count = _get_dangling_storage_count()
     existing_label_count = _get_dangling_label_count()
     existing_manifest_count = _get_dangling_manifest_count()
@@ -247,6 +255,13 @@ def assert_gc_integrity(expect_storage_removed=True):
                 .count()
             )
 
+        if shared == 0:
+            shared = (
+                UploadedBlob.select()
+                .where(UploadedBlob.blob == removed_image_and_storage.storage_id)
+                .count()
+            )
+
         if shared == 0:
             with pytest.raises(ImageStorage.DoesNotExist):
                 ImageStorage.get(id=removed_image_and_storage.storage_id)
@@ -672,6 +687,10 @@ def test_images_shared_cas(default_tag_policy, initialized_db):
             repo_ref, manifest, "first", storage, raise_on_error=True
         )
 
+        tag_ref = registry_model.get_repo_tag(repo_ref, "first")
+        manifest_ref = registry_model.get_manifest_for_tag(tag_ref)
+        registry_model.populate_legacy_images_for_testing(manifest_ref, storage)
+
         # Store another as `second`.
         builder = DockerSchema1ManifestBuilder(
             repository.namespace_user.username, repository.name, "second"
@@ -682,6 +701,10 @@ def test_images_shared_cas(default_tag_policy, initialized_db):
             repo_ref, manifest, "second", storage, raise_on_error=True
         )
 
+        tag_ref = registry_model.get_repo_tag(repo_ref, "second")
+        manifest_ref = registry_model.get_manifest_for_tag(tag_ref)
+        registry_model.populate_legacy_images_for_testing(manifest_ref, storage)
+
         # Manually retarget the second manifest's blob to the second row.
         try:
             second_blob = ManifestBlob.get(manifest=created._db_id, blob=is1)

data/model/test/test_image.py

@@ -1,109 +0,0 @@
-import pytest
-
-from collections import defaultdict
-from data.model import image, repository
-from playhouse.test_utils import assert_query_count
-
-from test.fixtures import *
-
-
-@pytest.fixture()
-def images(initialized_db):
-    images = image.get_repository_images("devtable", "simple")
-    assert len(images)
-    return images
-
-
-def test_get_image_with_storage(images, initialized_db):
-    for current in images:
-        storage_uuid = current.storage.uuid
-
-        with assert_query_count(1):
-            retrieved = image.get_image_with_storage(current.docker_image_id, storage_uuid)
-            assert retrieved.id == current.id
-            assert retrieved.storage.uuid == storage_uuid
-
-
-def test_get_parent_images(images, initialized_db):
-    for current in images:
-        if not len(current.ancestor_id_list()):
-            continue
-
-        with assert_query_count(1):
-            parent_images = list(image.get_parent_images("devtable", "simple", current))
-
-        assert len(parent_images) == len(current.ancestor_id_list())
-        assert set(current.ancestor_id_list()) == {i.id for i in parent_images}
-
-        for parent in parent_images:
-            with assert_query_count(0):
-                assert parent.storage.id
-
-
-def test_get_image(images, initialized_db):
-    for current in images:
-        repo = current.repository
-
-        with assert_query_count(1):
-            found = image.get_image(repo, current.docker_image_id)
-
-        assert found.id == current.id
-
-
-def test_placements(images, initialized_db):
-    with assert_query_count(1):
-        placements_map = image.get_placements_for_images(images)
-
-    for current in images:
-        assert current.storage.id in placements_map
-
-        with assert_query_count(2):
-            expected_image, expected_placements = image.get_image_and_placements(
-                "devtable", "simple", current.docker_image_id
-            )
-
-        assert expected_image.id == current.id
-        assert len(expected_placements) == len(placements_map.get(current.storage.id))
-        assert {p.id for p in expected_placements} == {
-            p.id for p in placements_map.get(current.storage.id)
-        }
-
-
-def test_get_repo_image(images, initialized_db):
-    for current in images:
-        with assert_query_count(1):
-            found = image.get_repo_image("devtable", "simple", current.docker_image_id)
-
-        assert found.id == current.id
-        with assert_query_count(1):
-            assert found.storage.id
-
-
-def test_get_repo_image_and_storage(images, initialized_db):
-    for current in images:
-        with assert_query_count(1):
-            found = image.get_repo_image_and_storage("devtable", "simple", current.docker_image_id)
-
-        assert found.id == current.id
-        with assert_query_count(0):
-            assert found.storage.id
-
-
-def test_get_repository_images_without_placements(images, initialized_db):
-    ancestors_map = defaultdict(list)
-    for img in images:
-        current = img.parent
-        while current is not None:
-            ancestors_map[current.id].append(img.id)
-            current = current.parent
-
-    for current in images:
-        repo = current.repository
-
-        with assert_query_count(1):
-            found = list(
-                image.get_repository_images_without_placements(repo, with_ancestor=current)
-            )
-
-        assert len(found) == len(ancestors_map[current.id]) + 1
-        assert {i.id for i in found} == set(ancestors_map[current.id] + [current.id])

data/model/test/test_tag.py

@@ -1,23 +0,0 @@
-import pytest
-
-from data.database import (
-    RepositoryState,
-    Image,
-)
-
-from test.fixtures import *
-
-
-def test_create_temp_tag(initialized_db):
-    repo = model.repository.get_repository("devtable", "simple")
-    image = Image.get(repository=repo)
-    assert model.tag.create_temporary_hidden_tag(repo, image, 10000000) is not None
-
-
-def test_create_temp_tag_deleted_repo(initialized_db):
-    repo = model.repository.get_repository("devtable", "simple")
-    repo.state = RepositoryState.MARKED_FOR_DELETION
-    repo.save()
-
-    image = Image.get(repository=repo)
-    assert model.tag.create_temporary_hidden_tag(repo, image, 10000000) is None

data/model/user.py

@@ -1326,7 +1326,11 @@ def get_region_locations(user):
     """
     Returns the locations defined as preferred storage for the given user.
     """
-    query = UserRegion.select().join(ImageStorageLocation).where(UserRegion.user == user)
+    query = (
+        UserRegion.select(UserRegion, ImageStorageLocation)
+        .join(ImageStorageLocation)
+        .where(UserRegion.user == user)
+    )
     return set([region.location.name for region in query])
 
 

data/registry_model/__init__.py

@@ -13,6 +13,9 @@ class RegistryModelProxy(object):
     def __getattr__(self, attr):
         return getattr(self._model, attr)
 
+    def set_id_hash_salt(self, hash_salt):
+        self._model.set_id_hash_salt(hash_salt)
+
 
 registry_model = RegistryModelProxy()
 logger.info("===============================")

data/registry_model/datatypes.py

@@ -1,4 +1,5 @@
 import hashlib
+import json
 
 from collections import namedtuple
 from enum import Enum, unique
@@ -172,8 +173,8 @@ class Label(datatype("Label", ["key", "value", "uuid", "source_type_name", "medi
             key=label.key,
             value=label.value,
             uuid=label.uuid,
-            media_type_name=label.media_type.name,
+            media_type_name=model.label.get_media_types()[label.media_type_id],
-            source_type_name=label.source_type.name,
+            source_type_name=model.label.get_label_source_types()[label.source_type_id],
         )
 
 
@@ -189,13 +190,6 @@ class ShallowTag(datatype("ShallowTag", ["name"])):
 
         return ShallowTag(db_id=tag.id, name=tag.name)
 
-    @classmethod
-    def for_repository_tag(cls, repository_tag):
-        if repository_tag is None:
-            return None
-
-        return ShallowTag(db_id=repository_tag.id, name=repository_tag.name)
-
     @property
     def id(self):
         """
@@ -223,7 +217,7 @@ class Tag(
     """
 
     @classmethod
-    def for_tag(cls, tag, legacy_image=None):
+    def for_tag(cls, tag, legacy_id_handler, manifest_row=None, legacy_image_row=None):
         if tag is None:
             return None
 
@@ -235,55 +229,34 @@ class Tag(
             lifetime_end_ms=tag.lifetime_end_ms,
             lifetime_start_ts=tag.lifetime_start_ms // 1000,
             lifetime_end_ts=tag.lifetime_end_ms // 1000 if tag.lifetime_end_ms else None,
-            manifest_digest=tag.manifest.digest,
+            manifest_digest=manifest_row.digest if manifest_row else tag.manifest.digest,
             inputs=dict(
-                legacy_image=legacy_image,
+                legacy_id_handler=legacy_id_handler,
-                manifest=tag.manifest,
+                legacy_image_row=legacy_image_row,
+                manifest_row=manifest_row or tag.manifest,
                 repository=RepositoryReference.for_id(tag.repository_id),
             ),
         )
 
-    @classmethod
+    @property
-    def for_repository_tag(cls, repository_tag, manifest_digest=None, legacy_image=None):
+    @requiresinput("manifest_row")
-        if repository_tag is None:
+    def _manifest_row(self, manifest_row):
-            return None
+        """
-
+        Returns the database Manifest object for this tag.
-        return Tag(
+        """
-            db_id=repository_tag.id,
+        return manifest_row
-            name=repository_tag.name,
-            reversion=repository_tag.reversion,
-            lifetime_start_ts=repository_tag.lifetime_start_ts,
-            lifetime_end_ts=repository_tag.lifetime_end_ts,
-            lifetime_start_ms=repository_tag.lifetime_start_ts * 1000,
-            lifetime_end_ms=(
-                repository_tag.lifetime_end_ts * 1000 if repository_tag.lifetime_end_ts else None
-            ),
-            manifest_digest=manifest_digest,
-            inputs=dict(
-                legacy_image=legacy_image,
-                repository=RepositoryReference.for_id(repository_tag.repository_id),
-            ),
-        )
 
     @property
-    @requiresinput("manifest")
+    @requiresinput("manifest_row")
-    def _manifest(self, manifest):
+    @requiresinput("legacy_id_handler")
+    @optionalinput("legacy_image_row")
+    def manifest(self, manifest_row, legacy_id_handler, legacy_image_row):
         """
         Returns the manifest for this tag.
-
-        Will only apply to new-style OCI tags.
         """
-        return manifest
+        return Manifest.for_manifest(
-
+            manifest_row, legacy_id_handler, legacy_image_row=legacy_image_row
-    @property
+        )
-    @optionalinput("manifest")
-    def manifest(self, manifest):
-        """
-        Returns the manifest for this tag or None if none.
-
-        Will only apply to new-style OCI tags.
-        """
-        return Manifest.for_manifest(manifest, self.legacy_image_if_present)
 
     @property
     @requiresinput("repository")
@@ -293,28 +266,6 @@ class Tag(
         """
         return repository
 
-    @property
-    @requiresinput("legacy_image")
-    def legacy_image(self, legacy_image):
-        """
-        Returns the legacy Docker V1-style image for this tag.
-
-        Note that this will be None for tags whose manifests point to other manifests instead of
-        images.
-        """
-        return legacy_image
-
-    @property
-    @optionalinput("legacy_image")
-    def legacy_image_if_present(self, legacy_image):
-        """
-        Returns the legacy Docker V1-style image for this tag.
-
-        Note that this will be None for tags whose manifests point to other manifests instead of
-        images.
-        """
-        return legacy_image
-
     @property
     def id(self):
         """
@@ -322,31 +273,32 @@ class Tag(
         """
         return self._db_id
 
+    @property
+    def manifest_layers_size(self):
+        """ Returns the compressed size of the layers of the manifest for the Tag or
+            None if none applicable or loaded.
+        """
+        return self.manifest.layers_compressed_size
 
-class Manifest(datatype("Manifest", ["digest", "media_type", "internal_manifest_bytes"])):
+
+class Manifest(
+    datatype(
+        "Manifest",
+        [
+            "digest",
+            "media_type",
+            "config_media_type",
+            "_layers_compressed_size",
+            "internal_manifest_bytes",
+        ],
+    )
+):
     """
     Manifest represents a manifest in a repository.
     """
 
     @classmethod
-    def for_tag_manifest(cls, tag_manifest, legacy_image=None):
+    def for_manifest(cls, manifest, legacy_id_handler, legacy_image_row=None):
-        if tag_manifest is None:
-            return None
-
-        return Manifest(
-            db_id=tag_manifest.id,
-            digest=tag_manifest.digest,
-            internal_manifest_bytes=Bytes.for_string_or_unicode(tag_manifest.json_data),
-            media_type=DOCKER_SCHEMA1_SIGNED_MANIFEST_CONTENT_TYPE,  # Always in legacy.
-            inputs=dict(
-                legacy_image=legacy_image,
-                tag_manifest=True,
-                repository=RepositoryReference.for_id(tag_manifest.repository_id),
-            ),
-        )
-
-    @classmethod
-    def for_manifest(cls, manifest, legacy_image):
         if manifest is None:
             return None
 
@@ -361,36 +313,15 @@ class Manifest(datatype("Manifest", ["digest", "media_type", "internal_manifest_
             digest=manifest.digest,
             internal_manifest_bytes=manifest_bytes,
             media_type=ManifestTable.media_type.get_name(manifest.media_type_id),
+            _layers_compressed_size=manifest.layers_compressed_size,
+            config_media_type=manifest.config_media_type,
             inputs=dict(
-                legacy_image=legacy_image,
+                legacy_id_handler=legacy_id_handler,
-                tag_manifest=False,
+                legacy_image_row=legacy_image_row,
                 repository=RepositoryReference.for_id(manifest.repository_id),
             ),
         )
 
-    @property
-    @requiresinput("tag_manifest")
-    def _is_tag_manifest(self, tag_manifest):
-        return tag_manifest
-
-    @property
-    @requiresinput("legacy_image")
-    def legacy_image(self, legacy_image):
-        """
-        Returns the legacy Docker V1-style image for this manifest.
-        """
-        return legacy_image
-
-    @property
-    @optionalinput("legacy_image")
-    def legacy_image_if_present(self, legacy_image):
-        """
-        Returns the legacy Docker V1-style image for this manifest.
-
-        Note that this will be None for manifests that point to other manifests instead of images.
-        """
-        return legacy_image
-
     def get_parsed_manifest(self, validate=True):
         """
         Returns the parsed manifest for this manifest.
@@ -400,17 +331,6 @@ class Manifest(datatype("Manifest", ["digest", "media_type", "internal_manifest_
             self.internal_manifest_bytes, self.media_type, validate=validate
         )
 
-    @property
-    def layers_compressed_size(self):
-        """
-        Returns the total compressed size of the layers in the manifest or None if this could not be
-        computed.
-        """
-        try:
-            return self.get_parsed_manifest().layers_compressed_size
-        except ManifestException:
-            return None
-
     @property
     def is_manifest_list(self):
         """
@@ -426,9 +346,67 @@ class Manifest(datatype("Manifest", ["digest", "media_type", "internal_manifest_
         """
         return repository
 
+    @optionalinput("legacy_image_row")
+    def _legacy_image_row(self, legacy_image_row):
+        return legacy_image_row
+
+    @property
+    def layers_compressed_size(self):
+        # TODO: Simplify once we've stopped writing Image rows and we've backfilled the
+        # sizes.
+
+        # First check the manifest itself, as all newly written manifests will have the
+        # size.
+        if self._layers_compressed_size is not None:
+            return self._layers_compressed_size
+
+        # Secondly, check for the size of the legacy Image row.
+        legacy_image_row = self._legacy_image_row
+        if legacy_image_row:
+            return legacy_image_row.aggregate_size
+
+        # Otherwise, return None.
+        return None
+
+    @property
+    @requiresinput("legacy_id_handler")
+    def legacy_image_root_id(self, legacy_id_handler):
+        """
+        Returns the legacy Docker V1-style image ID for this manifest. Note that an ID will
+        be returned even if the manifest does not support a legacy image.
+        """
+        return legacy_id_handler.encode(self._db_id)
+
+    def as_manifest(self):
+        """ Returns the manifest or legacy image as a manifest. """
+        return self
+
+    @property
+    @requiresinput("legacy_id_handler")
+    def _legacy_id_handler(self, legacy_id_handler):
+        return legacy_id_handler
+
+    def lookup_legacy_image(self, layer_index, retriever):
+        """ Looks up and returns the legacy image for index-th layer in this manifest
+            or None if none. The indexes here are from leaf to root, with index 0 being
+            the leaf.
+        """
+        # Retrieve the schema1 manifest. If none exists, legacy images are not supported.
+        parsed = self.get_parsed_manifest()
+        if parsed is None:
+            return None
+
+        schema1 = parsed.get_schema1_manifest("$namespace", "$repo", "$tag", retriever)
+        if schema1 is None:
+            return None
+
+        return LegacyImage.for_schema1_manifest_layer_index(
+            self, schema1, layer_index, self._legacy_id_handler
+        )
+
 
 class LegacyImage(
-    datatype(
+    namedtuple(
         "LegacyImage",
         [
             "docker_image_id",
@@ -437,8 +415,14 @@ class LegacyImage(
             "command",
             "image_size",
             "aggregate_size",
-            "uploading",
+            "blob",
+            "blob_digest",
             "v1_metadata_string",
+            # Internal fields.
+            "layer_index",
+            "manifest",
+            "parsed_manifest",
+            "id_handler",
         ],
     )
 ):
@@ -447,74 +431,80 @@ class LegacyImage(
     """
 
     @classmethod
-    def for_image(cls, image, images_map=None, tags_map=None, blob=None):
+    def for_schema1_manifest_layer_index(
-        if image is None:
+        cls, manifest, parsed_manifest, layer_index, id_handler, blob=None
+    ):
+        assert parsed_manifest.schema_version == 1
+        layers = parsed_manifest.layers
+        if layer_index >= len(layers):
+            return None
+
+        # NOTE: Schema1 keeps its layers in the order from base to leaf, so we have
+        # to reverse our lookup order.
+        leaf_to_base = list(reversed(layers))
+
+        aggregated_size = sum(
+            [
+                l.compressed_size
+                for index, l in enumerate(leaf_to_base)
+                if index >= layer_index and l.compressed_size is not None
+            ]
+        )
+
+        layer = leaf_to_base[layer_index]
+        synthetic_layer_id = id_handler.encode(manifest._db_id, layer_index)
+
+        # Replace the image ID and parent ID with our synethetic IDs.
+        try:
+            parsed = json.loads(layer.raw_v1_metadata)
+            parsed["id"] = synthetic_layer_id
+            if layer_index < len(leaf_to_base) - 1:
+                parsed["parent"] = id_handler.encode(manifest._db_id, layer_index + 1)
+        except (ValueError, TypeError):
             return None
 
         return LegacyImage(
-            db_id=image.id,
+            docker_image_id=synthetic_layer_id,
-            inputs=dict(
+            created=layer.v1_metadata.created,
-                images_map=images_map,
+            comment=layer.v1_metadata.comment,
-                tags_map=tags_map,
+            command=layer.v1_metadata.command,
-                ancestor_id_list=image.ancestor_id_list(),
+            image_size=layer.compressed_size,
+            aggregate_size=aggregated_size,
             blob=blob,
-            ),
+            blob_digest=layer.digest,
-            docker_image_id=image.docker_image_id,
+            v1_metadata_string=json.dumps(parsed),
-            created=image.created,
+            layer_index=layer_index,
-            comment=image.comment,
+            manifest=manifest,
-            command=image.command,
+            parsed_manifest=parsed_manifest,
-            v1_metadata_string=image.v1_json_metadata,
+            id_handler=id_handler,
-            image_size=image.storage.image_size,
-            aggregate_size=image.aggregate_size,
-            uploading=image.storage.uploading,
         )
 
-    @property
+    def with_blob(self, blob):
-    def id(self):
+        """ Sets the blob for the legacy image. """
-        """
+        return self._replace(blob=blob)
-        Returns the database ID of the legacy image.
-        """
-        return self._db_id
 
     @property
-    @requiresinput("images_map")
+    def parent_image_id(self):
-    @requiresinput("ancestor_id_list")
+        ancestor_ids = self.ancestor_ids
-    def parents(self, images_map, ancestor_id_list):
+        if not ancestor_ids:
-        """
+            return None
-        Returns the parent images for this image.
 
-        Raises an exception if the parents have not been loaded before this property is invoked.
+        return ancestor_ids[-1]
-        Parents are returned starting at the leaf image.
-        """
-        return [
-            LegacyImage.for_image(images_map[ancestor_id], images_map=images_map)
-            for ancestor_id in reversed(ancestor_id_list)
-            if images_map.get(ancestor_id)
-        ]
 
     @property
-    @requiresinput("blob")
+    def ancestor_ids(self):
-    def blob(self, blob):
+        ancestor_ids = []
-        """
+        for layer_index in range(self.layer_index + 1, len(self.parsed_manifest.layers)):
-        Returns the blob for this image.
+            ancestor_ids.append(self.id_handler.encode(self.manifest._db_id, layer_index))
-
+        return ancestor_ids
-        Raises an exception if the blob has not been loaded before this property is invoked.
-        """
-        return blob
 
     @property
-    @requiresinput("tags_map")
+    def full_image_id_chain(self):
-    def tags(self, tags_map):
+        return [self.docker_image_id] + self.ancestor_ids
-        """
-        Returns the tags pointing to this image.
 
-        Raises an exception if the tags have not been loaded before this property is invoked.
+    def as_manifest(self):
-        """
+        """ Returns the parent manifest for the legacy image. """
-        tags = tags_map.get(self._db_id)
+        return self.manifest
-        if not tags:
-            return []
-
-        return [Tag.for_tag(tag) for tag in tags]
 
 
 @unique
@@ -579,7 +569,6 @@ class Blob(
         """
         Returns the path of this blob in storage.
         """
-        # TODO: change this to take in the storage engine?
         return storage_path
 
     @property
@@ -591,27 +580,6 @@ class Blob(
         return placements
 
 
-class DerivedImage(datatype("DerivedImage", ["verb", "varying_metadata", "blob"])):
-    """
-    DerivedImage represents an image derived from a manifest via some form of verb.
-    """
-
-    @classmethod
-    def for_derived_storage(cls, derived, verb, varying_metadata, blob):
-        return DerivedImage(
-            db_id=derived.id, verb=verb, varying_metadata=varying_metadata, blob=blob
-        )
-
-    @property
-    def unique_id(self):
-        """
-        Returns a unique ID for this derived image.
-
-        This call will consistently produce the same unique ID across calls in the same code base.
-        """
-        return hashlib.sha256(("%s:%s" % (self.verb, self._db_id)).encode("utf-8")).hexdigest()
-
-
 class BlobUpload(
     datatype(
         "BlobUpload",
@@ -662,13 +630,6 @@ class LikelyVulnerableTag(datatype("LikelyVulnerableTag", ["layer_id", "name"]))
             db_id=tag.id, name=tag.name, layer_id=layer_id, inputs=dict(repository=repository)
         )
 
-    @classmethod
-    def for_repository_tag(cls, tag, repository):
-        tag_layer_id = "%s.%s" % (tag.image.docker_image_id, tag.image.storage.uuid)
-        return LikelyVulnerableTag(
-            db_id=tag.id, name=tag.name, layer_id=tag_layer_id, inputs=dict(repository=repository)
-        )
-
     @property
     @requiresinput("repository")
     def repository(self, repository):

data/registry_model/interface.py

@@ -14,16 +14,13 @@ class RegistryDataInterface(object):
     @abstractmethod
     def get_tag_legacy_image_id(self, repository_ref, tag_name, storage):
         """
-        Returns the legacy image ID for the tag with a legacy images in the repository.
+        Returns the legacy image ID for the tag in the repository or None if none.
-
-        Returns None if None.
         """
 
     @abstractmethod
     def get_legacy_tags_map(self, repository_ref, storage):
         """
-        Returns a map from tag name to its legacy image ID, for all tags with legacy images in the
+        Returns a map from tag name to its legacy image ID, for all tags in the repository.
-        repository.
 
         Note that this can be a *very* heavy operation.
         """
@@ -51,19 +48,14 @@ class RegistryDataInterface(object):
         """
 
     @abstractmethod
-    def get_manifest_for_tag(self, tag, backfill_if_necessary=False, include_legacy_image=False):
+    def get_manifest_for_tag(self, tag):
         """
         Returns the manifest associated with the given tag.
         """
 
     @abstractmethod
     def lookup_manifest_by_digest(
-        self,
+        self, repository_ref, manifest_digest, allow_dead=False, require_available=False,
-        repository_ref,
-        manifest_digest,
-        allow_dead=False,
-        include_legacy_image=False,
-        require_available=False,
     ):
         """
         Looks up the manifest with the given digest under the given repository and returns it or
@@ -92,15 +84,7 @@ class RegistryDataInterface(object):
         """
 
     @abstractmethod
-    def get_legacy_images(self, repository_ref):
+    def get_legacy_image(self, repository_ref, docker_image_id, storage, include_blob=False):
-        """
-        Returns an iterator of all the LegacyImage's defined in the matching repository.
-        """
-
-    @abstractmethod
-    def get_legacy_image(
-        self, repository_ref, docker_image_id, include_parents=False, include_blob=False
-    ):
         """
         Returns the matching LegacyImages under the matching repository, if any.
 
@@ -170,12 +154,12 @@ class RegistryDataInterface(object):
         """
 
     @abstractmethod
-    def list_all_active_repository_tags(self, repository_ref, include_legacy_images=False):
+    def list_all_active_repository_tags(self, repository_ref):
         """
         Returns a list of all the active tags in the repository.
 
         Note that this is a *HEAVY* operation on repositories with a lot of tags, and should only be
-        used for testing or where other more specific operations are not possible.
+        used for testing or legacy operations.
         """
 
     @abstractmethod
@@ -204,7 +188,7 @@ class RegistryDataInterface(object):
         """
 
     @abstractmethod
-    def get_repo_tag(self, repository_ref, tag_name, include_legacy_image=False):
+    def get_repo_tag(self, repository_ref, tag_name):
         """
         Returns the latest, *active* tag found in the repository, with the matching name or None if
         none.
@@ -259,12 +243,6 @@ class RegistryDataInterface(object):
         previous expiration timestamp in seconds (if any), and whether the operation succeeded.
         """
 
-    @abstractmethod
-    def get_legacy_images_owned_by_tag(self, tag):
-        """
-        Returns all legacy images *solely owned and used* by the given tag.
-        """
-
     @abstractmethod
     def get_security_status(self, manifest_or_legacy_image):
         """
@@ -319,57 +297,6 @@ class RegistryDataInterface(object):
         `image.docker.types.ManifestImageLayer`. Should not be called for a manifest list.
         """
 
-    @abstractmethod
-    def lookup_derived_image(
-        self, manifest, verb, storage, varying_metadata=None, include_placements=False
-    ):
-        """
-        Looks up the derived image for the given manifest, verb and optional varying metadata and
-        returns it or None if none.
-        """
-
-    @abstractmethod
-    def lookup_or_create_derived_image(
-        self,
-        manifest,
-        verb,
-        storage_location,
-        storage,
-        varying_metadata=None,
-        include_placements=False,
-    ):
-        """
-        Looks up the derived image for the given maniest, verb and optional varying metadata and
-        returns it.
-
-        If none exists, a new derived image is created.
-        """
-
-    @abstractmethod
-    def get_derived_image_signature(self, derived_image, signer_name):
-        """
-        Returns the signature associated with the derived image and a specific signer or None if
-        none.
-        """
-
-    @abstractmethod
-    def set_derived_image_signature(self, derived_image, signer_name, signature):
-        """
-        Sets the calculated signature for the given derived image and signer to that specified.
-        """
-
-    @abstractmethod
-    def delete_derived_image(self, derived_image):
-        """
-        Deletes a derived image and all of its storage.
-        """
-
-    @abstractmethod
-    def set_derived_image_size(self, derived_image, compressed_size):
-        """
-        Sets the compressed size on the given derived image.
-        """
-
     @abstractmethod
     def get_repo_blob_by_digest(self, repository_ref, blob_digest, include_placements=False):
         """
@@ -474,17 +401,14 @@ class RegistryDataInterface(object):
         If not possible, or an error occurs, returns None.
         """
 
-    @abstractmethod
-    def yield_tags_for_vulnerability_notification(self, layer_id_pairs):
-        """
-        Yields tags that contain one (or more) of the given layer ID pairs, in repositories which
-        have been registered for vulnerability_found notifications.
-
-        Returns an iterator of LikelyVulnerableTag instances.
-        """
-
     @abstractmethod
     def find_repository_with_garbage(self, limit_to_gc_policy_s):
         """ Returns a repository reference to a repository that contains garbage for collection
             or None if none.
         """
+
+    @abstractmethod
+    def populate_legacy_images_for_testing(self, manifest, storage):
+        """ Populates legacy images for the given manifest, for testing only. This call
+            will fail if called under non-testing code.
+        """

data/registry_model/manifestbuilder.py

@@ -85,8 +85,8 @@ class _ManifestBuilder(object):
         Returns the tags committed by this builder, if any.
         """
         return [
-            registry_model.get_repo_tag(self._repository_ref, tag_name, include_legacy_image=True)
+            registry_model.get_repo_tag(self._repository_ref, tag_name)
-            for tag_name in list(self._builder_state.tags.keys())
+            for tag_name in self._builder_state.tags.keys()
         ]
 
     def start_layer(

data/registry_model/registry_oci_model.py

@@ -25,13 +25,13 @@ from data.registry_model.datatypes import (
     SecurityScanStatus,
     Blob,
     BlobUpload,
-    DerivedImage,
     ShallowTag,
     LikelyVulnerableTag,
     RepositoryReference,
     ManifestLayer,
 )
 from data.registry_model.label_handlers import apply_label_to_manifest
+from data.registry_model.shared import SyntheticIDHandler
 from image.shared import ManifestException
 from image.docker.schema1 import (
     DOCKER_SCHEMA1_CONTENT_TYPES,
@@ -42,9 +42,6 @@ from image.docker.schema2 import EMPTY_LAYER_BLOB_DIGEST
 
 logger = logging.getLogger(__name__)
 
-# The maximum size for generated manifest after which we remove extra metadata.
-MAXIMUM_GENERATED_MANIFEST_SIZE = 3 * 1024 * 1024  # 3 MB
-
 
 class OCIModel(RegistryDataInterface):
     """
@@ -52,78 +49,71 @@ class OCIModel(RegistryDataInterface):
     changed to support the OCI specification.
     """
 
+    def __init__(self):
+        self._legacy_image_id_handler = SyntheticIDHandler()
+
+    def set_id_hash_salt(self, id_hash_salt):
+        self._legacy_image_id_handler = SyntheticIDHandler(id_hash_salt)
+
+    def _resolve_legacy_image_id_to_manifest_row(self, legacy_image_id):
+        decoded = self._legacy_image_id_handler.decode(legacy_image_id)
+        if len(decoded) == 0:
+            return (None, None)
+
+        manifest_id, layer_index = decoded
+        if manifest_id is None:
+            return (None, None)
+
+        try:
+            return database.Manifest.get(id=manifest_id), layer_index
+        except database.Manifest.DoesNotExist:
+            return (None, None)
+
+    def _resolve_legacy_image_id(self, legacy_image_id):
+        """ Decodes the given legacy image ID and returns the manifest to which it points,
+            as well as the layer index for the image. If invalid, or the manifest was not found,
+            returns (None, None).
+        """
+        manifest, layer_index = self._resolve_legacy_image_id_to_manifest_row(legacy_image_id)
+        if manifest is None:
+            return (None, None)
+
+        return Manifest.for_manifest(manifest, self._legacy_image_id_handler), layer_index
+
     def get_tag_legacy_image_id(self, repository_ref, tag_name, storage):
         """
-        Returns the legacy image ID for the tag with a legacy images in the repository.
+        Returns the legacy image ID for the tag in the repository. If there is no legacy image,
-
+        returns None.
-        Returns None if None.
         """
-        tag = self.get_repo_tag(repository_ref, tag_name, include_legacy_image=True)
+        tag = self.get_repo_tag(repository_ref, tag_name)
         if tag is None:
             return None
 
-        if tag.legacy_image_if_present is not None:
+        retriever = RepositoryContentRetriever(repository_ref.id, storage)
-            return tag.legacy_image_if_present.docker_image_id
+        legacy_image = tag.manifest.lookup_legacy_image(0, retriever)
-
+        if legacy_image is None:
-        if tag.manifest.is_manifest_list:
-            # See if we can lookup a schema1 legacy image.
-            v1_compatible = self.get_schema1_parsed_manifest(tag.manifest, "", "", "", storage)
-            if v1_compatible is not None:
-                return v1_compatible.leaf_layer_v1_image_id
-
             return None
 
+        return legacy_image.docker_image_id
+
     def get_legacy_tags_map(self, repository_ref, storage):
         """
-        Returns a map from tag name to its legacy image ID, for all tags with legacy images in the
+        Returns a map from tag name to its legacy image ID, for all tags in the
         repository.
 
         Note that this can be a *very* heavy operation.
         """
         tags = oci.tag.list_alive_tags(repository_ref._db_id)
-        legacy_images_map = oci.tag.get_legacy_images_for_tags(tags)
-
         tags_map = {}
         for tag in tags:
-            legacy_image = legacy_images_map.get(tag.id)
+            root_id = Manifest.for_manifest(
-            if legacy_image is not None:
+                tag.manifest, self._legacy_image_id_handler
-                tags_map[tag.name] = legacy_image.docker_image_id
+            ).legacy_image_root_id
-            else:
+            if root_id is not None:
-                manifest = Manifest.for_manifest(tag.manifest, None)
+                tags_map[tag.name] = root_id
-                if legacy_image is None and manifest.is_manifest_list:
-                    # See if we can lookup a schema1 legacy image.
-                    v1_compatible = self.get_schema1_parsed_manifest(manifest, "", "", "", storage)
-                    if v1_compatible is not None:
-                        v1_id = v1_compatible.leaf_layer_v1_image_id
-                        if v1_id is not None:
-                            tags_map[tag.name] = v1_id
 
         return tags_map
 
-    def _get_legacy_compatible_image_for_manifest(self, manifest, storage):
-        # Check for a legacy image directly on the manifest.
-        if not manifest.is_manifest_list:
-            return oci.shared.get_legacy_image_for_manifest(manifest._db_id)
-
-        # Otherwise, lookup a legacy image associated with the v1-compatible manifest
-        # in the list.
-        try:
-            manifest_obj = database.Manifest.get(id=manifest._db_id)
-        except database.Manifest.DoesNotExist:
-            logger.exception("Could not find manifest for manifest `%s`", manifest._db_id)
-            return None
-
-        # See if we can lookup a schema1 legacy image.
-        v1_compatible = self.get_schema1_parsed_manifest(manifest, "", "", "", storage)
-        if v1_compatible is None:
-            return None
-
-        v1_id = v1_compatible.leaf_layer_v1_image_id
-        if v1_id is None:
-            return None
-
-        return model.image.get_image(manifest_obj.repository_id, v1_id)
-
     def find_matching_tag(self, repository_ref, tag_names):
         """
         Finds an alive tag in the repository matching one of the given tag names and returns it or
@@ -131,7 +121,7 @@ class OCIModel(RegistryDataInterface):
         """
         found_tag = oci.tag.find_matching_tag(repository_ref._db_id, tag_names)
         assert found_tag is None or not found_tag.hidden
-        return Tag.for_tag(found_tag)
+        return Tag.for_tag(found_tag, self._legacy_image_id_handler)
 
     def get_most_recent_tag(self, repository_ref):
         """
@@ -141,27 +131,17 @@ class OCIModel(RegistryDataInterface):
         """
         found_tag = oci.tag.get_most_recent_tag(repository_ref._db_id)
         assert found_tag is None or not found_tag.hidden
-        return Tag.for_tag(found_tag)
+        return Tag.for_tag(found_tag, self._legacy_image_id_handler)
 
-    def get_manifest_for_tag(self, tag, backfill_if_necessary=False, include_legacy_image=False):
+    def get_manifest_for_tag(self, tag):
         """
         Returns the manifest associated with the given tag.
         """
         assert tag is not None
-
+        return tag.manifest
-        legacy_image = None
-        if include_legacy_image:
-            legacy_image = oci.shared.get_legacy_image_for_manifest(tag._manifest)
-
-        return Manifest.for_manifest(tag._manifest, LegacyImage.for_image(legacy_image))
 
     def lookup_manifest_by_digest(
-        self,
+        self, repository_ref, manifest_digest, allow_dead=False, require_available=False,
-        repository_ref,
-        manifest_digest,
-        allow_dead=False,
-        include_legacy_image=False,
-        require_available=False,
     ):
         """
         Looks up the manifest with the given digest under the given repository and returns it or
@@ -176,19 +156,7 @@ class OCIModel(RegistryDataInterface):
         if manifest is None:
             return None
 
-        legacy_image = None
+        return Manifest.for_manifest(manifest, self._legacy_image_id_handler)
-        if include_legacy_image:
-            try:
-                legacy_image_id = database.ManifestLegacyImage.get(
-                    manifest=manifest
-                ).image.docker_image_id
-                legacy_image = self.get_legacy_image(
-                    repository_ref, legacy_image_id, include_parents=True
-                )
-            except database.ManifestLegacyImage.DoesNotExist:
-                pass
-
-        return Manifest.for_manifest(manifest, legacy_image)
 
     def create_manifest_label(self, manifest, key, value, source_type_name, media_type_name=None):
         """
@@ -276,22 +244,15 @@ class OCIModel(RegistryDataInterface):
         tags = oci.tag.lookup_alive_tags_shallow(repository_ref._db_id, start_pagination_id, limit)
         return [ShallowTag.for_tag(tag) for tag in tags]
 
-    def list_all_active_repository_tags(self, repository_ref, include_legacy_images=False):
+    def list_all_active_repository_tags(self, repository_ref):
         """
         Returns a list of all the active tags in the repository.
 
         Note that this is a *HEAVY* operation on repositories with a lot of tags, and should only be
-        used for testing or where other more specific operations are not possible.
+        used for testing or legacy operations.
         """
         tags = list(oci.tag.list_alive_tags(repository_ref._db_id))
-        legacy_images_map = {}
+        return [Tag.for_tag(tag, self._legacy_image_id_handler) for tag in tags]
-        if include_legacy_images:
-            legacy_images_map = oci.tag.get_legacy_images_for_tags(tags)
-
-        return [
-            Tag.for_tag(tag, legacy_image=LegacyImage.for_image(legacy_images_map.get(tag.id)))
-            for tag in tags
-        ]
 
     def list_repository_tag_history(
         self,
@@ -312,11 +273,19 @@ class OCIModel(RegistryDataInterface):
             repository_ref._db_id, page, size, specific_tag_name, active_tags_only, since_time_ms
         )
 
-        # TODO: do we need legacy images here?
+        # TODO: Remove this once the layers compressed sizes have been fully backfilled.
-        legacy_images_map = oci.tag.get_legacy_images_for_tags(tags)
+        tags_missing_sizes = [tag for tag in tags if tag.manifest.layers_compressed_size is None]
+        legacy_images_map = {}
+        if tags_missing_sizes:
+            legacy_images_map = oci.tag.get_legacy_images_for_tags(tags_missing_sizes)
+
         return (
             [
-                Tag.for_tag(tag, LegacyImage.for_image(legacy_images_map.get(tag.id)))
+                Tag.for_tag(
+                    tag,
+                    self._legacy_image_id_handler,
+                    legacy_image_row=legacy_images_map.get(tag.id),
+                )
                 for tag in tags
             ],
             has_more,
@@ -342,7 +311,7 @@ class OCIModel(RegistryDataInterface):
 
         return {repo_id: toSeconds(ms) for repo_id, ms in list(last_modified.items())}
 
-    def get_repo_tag(self, repository_ref, tag_name, include_legacy_image=False):
+    def get_repo_tag(self, repository_ref, tag_name):
         """
         Returns the latest, *active* tag found in the repository, with the matching name or None if
         none.
@@ -353,12 +322,7 @@ class OCIModel(RegistryDataInterface):
         if tag is None:
             return None
 
-        legacy_image = None
+        return Tag.for_tag(tag, self._legacy_image_id_handler)
-        if include_legacy_image:
-            legacy_images = oci.tag.get_legacy_images_for_tags([tag])
-            legacy_image = legacy_images.get(tag.id)
-
-        return Tag.for_tag(tag, legacy_image=LegacyImage.for_image(legacy_image))
 
     def create_manifest_and_retarget_tag(
         self, repository_ref, manifest_interface_instance, tag_name, storage, raise_on_error=False
@@ -395,9 +359,9 @@ class OCIModel(RegistryDataInterface):
             if tag is None:
                 return (None, None)
 
-            legacy_image = oci.shared.get_legacy_image_for_manifest(created_manifest.manifest)
+            wrapped_manifest = Manifest.for_manifest(
-            li = LegacyImage.for_image(legacy_image)
+                created_manifest.manifest, self._legacy_image_id_handler
-            wrapped_manifest = Manifest.for_manifest(created_manifest.manifest, li)
+            )
 
             # Apply any labels that should modify the created tag.
             if created_manifest.labels_to_apply:
@@ -407,7 +371,12 @@ class OCIModel(RegistryDataInterface):
                 # Reload the tag in case any updates were applied.
                 tag = database.Tag.get(id=tag.id)
 
-            return (wrapped_manifest, Tag.for_tag(tag, li))
+            return (
+                wrapped_manifest,
+                Tag.for_tag(
+                    tag, self._legacy_image_id_handler, manifest_row=created_manifest.manifest
+                ),
+            )
 
     def retarget_tag(
         self,
@@ -427,62 +396,37 @@ class OCIModel(RegistryDataInterface):
         """
         with db_disallow_replica_use():
             assert legacy_manifest_key is not None
-            manifest_id = manifest_or_legacy_image._db_id
+            manifest = manifest_or_legacy_image.as_manifest()
-            if isinstance(manifest_or_legacy_image, LegacyImage):
+            manifest_id = manifest._db_id
-                # If a legacy image was required, build a new manifest for it and move the tag to that.
-                try:
-                    image_row = database.Image.get(id=manifest_or_legacy_image._db_id)
-                except database.Image.DoesNotExist:
-                    return None
 
-                manifest_instance = self._build_manifest_for_legacy_image(tag_name, image_row)
-                if manifest_instance is None:
-                    return None
-
-                created = oci.manifest.get_or_create_manifest(
-                    repository_ref._db_id, manifest_instance, storage
-                )
-                if created is None:
-                    return None
-
-                manifest_id = created.manifest.id
-            else:
             # If the manifest is a schema 1 manifest and its tag name does not match that
             # specified, then we need to create a new manifest, but with that tag name.
-                if manifest_or_legacy_image.media_type in DOCKER_SCHEMA1_CONTENT_TYPES:
+            if manifest.media_type in DOCKER_SCHEMA1_CONTENT_TYPES:
                 try:
-                        parsed = manifest_or_legacy_image.get_parsed_manifest()
+                    parsed = manifest.get_parsed_manifest()
                 except ManifestException:
                     logger.exception(
-                            "Could not parse manifest `%s` in retarget_tag",
+                        "Could not parse manifest `%s` in retarget_tag", manifest._db_id,
-                            manifest_or_legacy_image._db_id,
                     )
                     return None
 
                 if parsed.tag != tag_name:
                     logger.debug(
-                            "Rewriting manifest `%s` for tag named `%s`",
+                        "Rewriting manifest `%s` for tag named `%s`", manifest._db_id, tag_name,
-                            manifest_or_legacy_image._db_id,
-                            tag_name,
                     )
 
                     repository_id = repository_ref._db_id
                     updated = parsed.with_tag_name(tag_name, legacy_manifest_key)
                     assert updated.is_signed
 
-                        created = oci.manifest.get_or_create_manifest(
+                    created = oci.manifest.get_or_create_manifest(repository_id, updated, storage)
-                            repository_id, updated, storage
-                        )
                     if created is None:
                         return None
 
                     manifest_id = created.manifest.id
 
             tag = oci.tag.retarget_tag(tag_name, manifest_id, is_reversion=is_reversion)
-            legacy_image = LegacyImage.for_image(
+            return Tag.for_tag(tag, self._legacy_image_id_handler)
-                oci.shared.get_legacy_image_for_manifest(manifest_id)
-            )
-            return Tag.for_tag(tag, legacy_image)
 
     def delete_tag(self, repository_ref, tag_name):
         """
@@ -496,18 +440,18 @@ class OCIModel(RegistryDataInterface):
                 msg = "Invalid repository tag '%s' on repository" % tag_name
                 raise DataModelException(msg)
 
-            return Tag.for_tag(deleted_tag)
+            return Tag.for_tag(deleted_tag, self._legacy_image_id_handler)
 
     def delete_tags_for_manifest(self, manifest):
         """
         Deletes all tags pointing to the given manifest, making the manifest inaccessible for
         pulling.
 
-        Returns the tags deleted, if any. Returns None on error.
+        Returns the tags (ShallowTag) deleted. Returns None on error.
         """
         with db_disallow_replica_use():
             deleted_tags = oci.tag.delete_tags_for_manifest(manifest._db_id)
-            return [Tag.for_tag(tag) for tag in deleted_tags]
+            return [ShallowTag.for_tag(tag) for tag in deleted_tags]
 
     def change_repository_tag_expiration(self, tag, expiration_date):
         """
@@ -519,75 +463,15 @@ class OCIModel(RegistryDataInterface):
         with db_disallow_replica_use():
             return oci.tag.change_tag_expiration(tag._db_id, expiration_date)
 
-    def get_legacy_images_owned_by_tag(self, tag):
-        """
-        Returns all legacy images *solely owned and used* by the given tag.
-        """
-        tag_obj = oci.tag.get_tag_by_id(tag._db_id)
-        if tag_obj is None:
-            return None
-
-        tags = oci.tag.list_alive_tags(tag_obj.repository_id)
-        legacy_images = oci.tag.get_legacy_images_for_tags(tags)
-
-        tag_legacy_image = legacy_images.get(tag._db_id)
-        if tag_legacy_image is None:
-            return None
-
-        assert isinstance(tag_legacy_image, Image)
-
-        # Collect the IDs of all images that the tag uses.
-        tag_image_ids = set()
-        tag_image_ids.add(tag_legacy_image.id)
-        tag_image_ids.update(tag_legacy_image.ancestor_id_list())
-
-        # Remove any images shared by other tags.
-        for current in tags:
-            if current == tag_obj:
-                continue
-
-            current_image = legacy_images.get(current.id)
-            if current_image is None:
-                continue
-
-            tag_image_ids.discard(current_image.id)
-            tag_image_ids = tag_image_ids.difference(current_image.ancestor_id_list())
-            if not tag_image_ids:
-                return []
-
-        if not tag_image_ids:
-            return []
-
-        # Load the images we need to return.
-        images = database.Image.select().where(database.Image.id << list(tag_image_ids))
-        all_image_ids = set()
-        for image in images:
-            all_image_ids.add(image.id)
-            all_image_ids.update(image.ancestor_id_list())
-
-        # Build a map of all the images and their parents.
-        images_map = {}
-        all_images = database.Image.select().where(database.Image.id << list(all_image_ids))
-        for image in all_images:
-            images_map[image.id] = image
-
-        return [LegacyImage.for_image(image, images_map=images_map) for image in images]
-
     def get_security_status(self, manifest_or_legacy_image):
         """
         Returns the security status for the given manifest or legacy image or None if none.
         """
-        image = None
+        # TODO: change from using the Image row once we've moved all security info into MSS.
-
+        manifest_id = manifest_or_legacy_image.as_manifest()._db_id
-        if isinstance(manifest_or_legacy_image, Manifest):
+        image = oci.shared.get_legacy_image_for_manifest(manifest_id)
-            image = oci.shared.get_legacy_image_for_manifest(manifest_or_legacy_image._db_id)
         if image is None:
             return SecurityScanStatus.UNSUPPORTED
-        else:
-            try:
-                image = database.Image.get(id=manifest_or_legacy_image._db_id)
-            except database.Image.DoesNotExist:
-                return None
 
         if image.security_indexed_engine is not None and image.security_indexed_engine >= 0:
             return (
@@ -602,17 +486,11 @@ class OCIModel(RegistryDataInterface):
         re-indexed.
         """
         with db_disallow_replica_use():
-            image = None
+            # TODO: change from using the Image row once we've moved all security info into MSS.
-
+            manifest_id = manifest_or_legacy_image.as_manifest()._db_id
-            if isinstance(manifest_or_legacy_image, Manifest):
+            image = oci.shared.get_legacy_image_for_manifest(manifest_id)
-                image = oci.shared.get_legacy_image_for_manifest(manifest_or_legacy_image._db_id)
             if image is None:
                 return None
-            else:
-                try:
-                    image = database.Image.get(id=manifest_or_legacy_image._db_id)
-                except database.Image.DoesNotExist:
-                    return None
 
                 assert image
                 image.security_indexed = False
@@ -633,48 +511,9 @@ class OCIModel(RegistryDataInterface):
             return None
 
         return self._list_manifest_layers(
-            manifest_obj.repository_id, parsed, storage, include_placements, by_manifest=True
+            manifest_obj.repository_id, parsed, storage, include_placements
         )
 
-    def lookup_derived_image(
-        self, manifest, verb, storage, varying_metadata=None, include_placements=False
-    ):
-        """
-        Looks up the derived image for the given manifest, verb and optional varying metadata and
-        returns it or None if none.
-        """
-        legacy_image = self._get_legacy_compatible_image_for_manifest(manifest, storage)
-        if legacy_image is None:
-            return None
-
-        derived = model.image.find_derived_storage_for_image(legacy_image, verb, varying_metadata)
-        return self._build_derived(derived, verb, varying_metadata, include_placements)
-
-    def lookup_or_create_derived_image(
-        self,
-        manifest,
-        verb,
-        storage_location,
-        storage,
-        varying_metadata=None,
-        include_placements=False,
-    ):
-        """
-        Looks up the derived image for the given maniest, verb and optional varying metadata and
-        returns it.
-
-        If none exists, a new derived image is created.
-        """
-        with db_disallow_replica_use():
-            legacy_image = self._get_legacy_compatible_image_for_manifest(manifest, storage)
-            if legacy_image is None:
-                return None
-
-            derived = model.image.find_or_create_derived_storage(
-                legacy_image, verb, storage_location, varying_metadata
-            )
-            return self._build_derived(derived, verb, varying_metadata, include_placements)
-
     def set_tags_expiration_for_manifest(self, manifest, expiration_sec):
         """
         Sets the expiration on all tags that point to the given manifest to that specified.
@@ -737,9 +576,7 @@ class OCIModel(RegistryDataInterface):
             if created_manifest is None:
                 return None
 
-            legacy_image = oci.shared.get_legacy_image_for_manifest(created_manifest.manifest)
+            return Manifest.for_manifest(created_manifest.manifest, self._legacy_image_id_handler)
-            li = LegacyImage.for_image(legacy_image)
-            return Manifest.for_manifest(created_manifest.manifest, li)
 
     def get_repo_blob_by_digest(self, repository_ref, blob_digest, include_placements=False):
         """
@@ -777,11 +614,7 @@ class OCIModel(RegistryDataInterface):
         specified).
         """
         return self._list_manifest_layers(
-            repository_ref._db_id,
+            repository_ref._db_id, parsed_manifest, storage, include_placements=include_placements,
-            parsed_manifest,
-            storage,
-            include_placements=include_placements,
-            by_manifest=True,
         )
 
     def get_manifest_local_blobs(self, manifest, include_placements=False):
@@ -794,23 +627,7 @@ class OCIModel(RegistryDataInterface):
             return None
 
         return self._get_manifest_local_blobs(
-            manifest, manifest_row.repository_id, include_placements, by_manifest=True
+            manifest, manifest_row.repository_id, include_placements
-        )
-
-    def yield_tags_for_vulnerability_notification(self, layer_id_pairs):
-        """
-        Yields tags that contain one (or more) of the given layer ID pairs, in repositories which
-        have been registered for vulnerability_found notifications.
-
-        Returns an iterator of LikelyVulnerableTag instances.
-        """
-        for docker_image_id, storage_uuid in layer_id_pairs:
-            tags = oci.tag.lookup_notifiable_tags_for_legacy_image(
-                docker_image_id, storage_uuid, "vulnerability_found"
-            )
-            for tag in tags:
-                yield LikelyVulnerableTag.for_tag(
-                    tag, tag.repository, docker_image_id, storage_uuid
         )
 
     def find_repository_with_garbage(self, limit_to_gc_policy_s):
@@ -849,66 +666,6 @@ class OCIModel(RegistryDataInterface):
         namespace = model.user.get_namespace_user(namespace_name)
         return namespace is not None and namespace.enabled
 
-    def get_derived_image_signature(self, derived_image, signer_name):
-        """
-        Returns the signature associated with the derived image and a specific signer or None if
-        none.
-        """
-        try:
-            derived_storage = database.DerivedStorageForImage.get(id=derived_image._db_id)
-        except database.DerivedStorageForImage.DoesNotExist:
-            return None
-
-        storage = derived_storage.derivative
-        signature_entry = model.storage.lookup_storage_signature(storage, signer_name)
-        if signature_entry is None:
-            return None
-
-        return signature_entry.signature
-
-    def set_derived_image_signature(self, derived_image, signer_name, signature):
-        """
-        Sets the calculated signature for the given derived image and signer to that specified.
-        """
-        with db_disallow_replica_use():
-            try:
-                derived_storage = database.DerivedStorageForImage.get(id=derived_image._db_id)
-            except database.DerivedStorageForImage.DoesNotExist:
-                return None
-
-            storage = derived_storage.derivative
-            signature_entry = model.storage.find_or_create_storage_signature(storage, signer_name)
-            signature_entry.signature = signature
-            signature_entry.uploading = False
-            signature_entry.save()
-
-    def delete_derived_image(self, derived_image):
-        """
-        Deletes a derived image and all of its storage.
-        """
-        with db_disallow_replica_use():
-            try:
-                derived_storage = database.DerivedStorageForImage.get(id=derived_image._db_id)
-            except database.DerivedStorageForImage.DoesNotExist:
-                return None
-
-            model.image.delete_derived_storage(derived_storage)
-
-    def set_derived_image_size(self, derived_image, compressed_size):
-        """
-        Sets the compressed size on the given derived image.
-        """
-        with db_disallow_replica_use():
-            try:
-                derived_storage = database.DerivedStorageForImage.get(id=derived_image._db_id)
-            except database.DerivedStorageForImage.DoesNotExist:
-                return None
-
-            storage_entry = derived_storage.derivative
-            storage_entry.image_size = compressed_size
-            storage_entry.uploading = False
-            storage_entry.save()
-
     def lookup_cached_active_repository_tags(
         self, model_cache, repository_ref, start_pagination_id, limit
     ):
@@ -1098,68 +855,41 @@ class OCIModel(RegistryDataInterface):
             )
             return bool(storage)
 
-    def get_legacy_images(self, repository_ref):
+    def get_legacy_image(self, repository_ref, docker_image_id, storage, include_blob=False):
         """
-        Returns an iterator of all the LegacyImage's defined in the matching repository.
+        Returns the matching LegacyImage under the matching repository, if any.
-        """
-        repo = model.repository.lookup_repository(repository_ref._db_id)
-        if repo is None:
-            return None
-
-        all_images = model.image.get_repository_images_without_placements(repo)
-        all_images_map = {image.id: image for image in all_images}
-
-        all_tags = model.oci.tag.list_alive_tags(repo)
-        tags_by_image_id = defaultdict(list)
-        for tag in all_tags:
-            try:
-                mli = database.ManifestLegacyImage.get(manifest=tag.manifest_id)
-                tags_by_image_id[mli.image_id].append(tag)
-            except database.ManifestLegacyImage.DoesNotExist:
-                continue
-
-        return [
-            LegacyImage.for_image(image, images_map=all_images_map, tags_map=tags_by_image_id)
-            for image in all_images
-        ]
-
-    def get_legacy_image(
-        self, repository_ref, docker_image_id, include_parents=False, include_blob=False
-    ):
-        """
-        Returns the matching LegacyImages under the matching repository, if any.
 
         If none, returns None.
         """
-        repo = model.repository.lookup_repository(repository_ref._db_id)
+        retriever = RepositoryContentRetriever(repository_ref._db_id, storage)
-        if repo is None:
+
+        # Resolves the manifest and the layer index from the synthetic ID.
+        manifest, layer_index = self._resolve_legacy_image_id(docker_image_id)
+        if manifest is None:
             return None
 
-        image = model.image.get_image(repository_ref._db_id, docker_image_id)
+        # Lookup the legacy image for the index.
-        if image is None:
+        legacy_image = manifest.lookup_legacy_image(layer_index, retriever)
-            return None
+        if legacy_image is None or not include_blob:
+            return legacy_image
 
-        parent_images_map = None
+        # If a blob was requested, load it into the legacy image.
-        if include_parents:
+        return legacy_image.with_blob(
-            parent_images = model.image.get_parent_images(
+            self.get_repo_blob_by_digest(
-                repo.namespace_user.username, repo.name, image
+                repository_ref, legacy_image.blob_digest, include_placements=True
             )
-            parent_images_map = {image.id: image for image in parent_images}
-
-        blob = None
-        if include_blob:
-            placements = list(model.storage.get_storage_locations(image.storage.uuid))
-            blob = Blob.for_image_storage(
-                image.storage,
-                storage_path=model.storage.get_layer_path(image.storage),
-                placements=placements,
         )
 
-        return LegacyImage.for_image(image, images_map=parent_images_map, blob=blob)
+    def populate_legacy_images_for_testing(self, manifest, storage):
+        """ Populates legacy images for the given manifest, for testing only. This call
+            will fail if called under non-testing code.
+        """
+        manifest_row = database.Manifest.get(id=manifest._db_id)
+        oci.manifest.populate_legacy_images_for_testing(
+            manifest_row, manifest.get_parsed_manifest(), storage
+        )
 
-    def _get_manifest_local_blobs(
+    def _get_manifest_local_blobs(self, manifest, repo_id, include_placements=False):
-        self, manifest, repo_id, include_placements=False, by_manifest=False
-    ):
         parsed = manifest.get_parsed_manifest()
         if parsed is None:
             return None
@@ -1168,9 +898,7 @@ class OCIModel(RegistryDataInterface):
         if not len(local_blob_digests):
             return []
 
-        blob_query = self._lookup_repo_storages_by_content_checksum(
+        blob_query = self._lookup_repo_storages_by_content_checksum(repo_id, local_blob_digests)
-            repo_id, local_blob_digests, by_manifest=by_manifest
-        )
         blobs = []
         for image_storage in blob_query:
             placements = None
@@ -1186,9 +914,7 @@ class OCIModel(RegistryDataInterface):
 
         return blobs
 
-    def _list_manifest_layers(
+    def _list_manifest_layers(self, repo_id, parsed, storage, include_placements=False):
-        self, repo_id, parsed, storage, include_placements=False, by_manifest=False
-    ):
         """
         Returns an *ordered list* of the layers found in the manifest, starting at the base and
         working towards the leaf, including the associated Blob and its placements (if specified).
@@ -1206,9 +932,7 @@ class OCIModel(RegistryDataInterface):
             blob_digests.append(EMPTY_LAYER_BLOB_DIGEST)
 
         if blob_digests:
-            blob_query = self._lookup_repo_storages_by_content_checksum(
+            blob_query = self._lookup_repo_storages_by_content_checksum(repo_id, blob_digests)
-                repo_id, blob_digests, by_manifest=by_manifest
-            )
             storage_map = {blob.content_checksum: blob for blob in blob_query}
 
         layers = parsed.get_layers(retriever)
@@ -1246,84 +970,6 @@ class OCIModel(RegistryDataInterface):
 
         return manifest_layers
 
-    def _build_derived(self, derived, verb, varying_metadata, include_placements):
-        if derived is None:
-            return None
-
-        derived_storage = derived.derivative
-        placements = None
-        if include_placements:
-            placements = list(model.storage.get_storage_locations(derived_storage.uuid))
-
-        blob = Blob.for_image_storage(
-            derived_storage,
-            storage_path=model.storage.get_layer_path(derived_storage),
-            placements=placements,
-        )
-
-        return DerivedImage.for_derived_storage(derived, verb, varying_metadata, blob)
-
-    def _build_manifest_for_legacy_image(self, tag_name, legacy_image_row):
-        import features
-
-        from app import app, docker_v2_signing_key
-
-        repo = legacy_image_row.repository
-        namespace_name = repo.namespace_user.username
-        repo_name = repo.name
-
-        # Find the v1 metadata for this image and its parents.
-        try:
-            parents = model.image.get_parent_images(namespace_name, repo_name, legacy_image_row)
-        except model.DataModelException:
-            logger.exception(
-                "Could not load parent images for legacy image %s", legacy_image_row.id
-            )
-            return None
-
-        # If the manifest is being generated under the library namespace, then we make its namespace
-        # empty.
-        manifest_namespace = namespace_name
-        if features.LIBRARY_SUPPORT and namespace_name == app.config["LIBRARY_NAMESPACE"]:
-            manifest_namespace = ""
-
-        # Create and populate the manifest builder
-        builder = DockerSchema1ManifestBuilder(manifest_namespace, repo_name, tag_name)
-
-        # Add the leaf layer
-        builder.add_layer(
-            legacy_image_row.storage.content_checksum, legacy_image_row.v1_json_metadata
-        )
-        if legacy_image_row.storage.uploading:
-            logger.error("Cannot add an uploading storage row: %s", legacy_image_row.storage.id)
-            return None
-
-        for parent_image in parents:
-            if parent_image.storage.uploading:
-                logger.error("Cannot add an uploading storage row: %s", legacy_image_row.storage.id)
-                return None
-
-            builder.add_layer(parent_image.storage.content_checksum, parent_image.v1_json_metadata)
-
-        try:
-            built_manifest = builder.build(docker_v2_signing_key)
-
-            # If the generated manifest is greater than the maximum size, regenerate it with
-            # intermediate metadata layers stripped down to their bare essentials.
-            if len(built_manifest.bytes.as_encoded_str()) > MAXIMUM_GENERATED_MANIFEST_SIZE:
-                built_manifest = builder.with_metadata_removed().build(docker_v2_signing_key)
-
-            if len(built_manifest.bytes.as_encoded_str()) > MAXIMUM_GENERATED_MANIFEST_SIZE:
-                logger.error("Legacy image is too large to generate manifest")
-                return None
-
-            return built_manifest
-        except ManifestException as me:
-            logger.exception(
-                "Got exception when trying to build manifest for legacy image %s", legacy_image_row
-            )
-            return None
-
     def _get_shared_storage(self, blob_digest):
         """
         Returns an ImageStorage row for the blob digest if it is a globally shared storage.
@@ -1337,7 +983,7 @@ class OCIModel(RegistryDataInterface):
 
         return None
 
-    def _lookup_repo_storages_by_content_checksum(self, repo, checksums, by_manifest=False):
+    def _lookup_repo_storages_by_content_checksum(self, repo, checksums):
         checksums = set(checksums)
 
         # Load any shared storages first.
@@ -1350,11 +996,7 @@ class OCIModel(RegistryDataInterface):
 
         found = []
         if checksums:
-            found = list(
+            found = list(model.storage.lookup_repo_storages_by_content_checksum(repo, checksums))
-                model.storage.lookup_repo_storages_by_content_checksum(
-                    repo, checksums, by_manifest=by_manifest
-                )
-            )
         return found + extra_storages
 
 

data/registry_model/shared.py

@@ -0,0 +1,17 @@
+import uuid
+
+from hashids import Hashids
+
+
+class SyntheticIDHandler(object):
+    def __init__(self, hash_salt=None):
+        self.hash_salt = hash_salt or str(uuid.uuid4())
+        self.hashids = Hashids(alphabet="0123456789abcdef", min_length=64, salt=self.hash_salt)
+
+    def encode(self, manifest_id, layer_index=0):
+        encoded = self.hashids.encode(manifest_id, layer_index)
+        assert len(encoded) == 64
+        return encoded
+
+    def decode(self, synthetic_v1_id):
+        return self.hashids.decode(synthetic_v1_id)

data/registry_model/test/test_interface.py

@@ -23,7 +23,6 @@ from data.database import (
     ManifestLabel,
     TagManifest,
     TagManifestLabel,
-    DerivedStorageForImage,
     Tag,
     TagToRepositoryTag,
     ImageStorageLocation,
@@ -32,6 +31,7 @@ from data.cache.impl import InMemoryDataModelCache
 from data.registry_model.registry_oci_model import OCIModel
 from data.registry_model.datatypes import RepositoryReference
 from data.registry_model.blobuploader import upload_blob, BlobUploadSettings
+from data.model.oci.retriever import RepositoryContentRetriever
 from data.model.blob import store_blob_record_and_temp_link
 from image.shared.types import ManifestImageLayer
 from image.docker.schema1 import (
@@ -78,7 +78,6 @@ def test_find_matching_tag(names, expected, registry_model):
         assert found is None
     else:
         assert found.name in expected
-        assert found.repository.namespace_name == "devtable"
         assert found.repository.name == "simple"
 
 
@@ -120,13 +119,9 @@ def test_lookup_manifests(repo_namespace, repo_name, registry_model):
     repository_ref = RepositoryReference.for_repo_obj(repo)
     found_tag = registry_model.find_matching_tag(repository_ref, ["latest"])
     found_manifest = registry_model.get_manifest_for_tag(found_tag)
-    found = registry_model.lookup_manifest_by_digest(
+    found = registry_model.lookup_manifest_by_digest(repository_ref, found_manifest.digest)
-        repository_ref, found_manifest.digest, include_legacy_image=True
-    )
     assert found._db_id == found_manifest._db_id
     assert found.digest == found_manifest.digest
-    assert found.legacy_image
-    assert found.legacy_image.parents
 
     schema1_parsed = registry_model.get_schema1_parsed_manifest(found, "foo", "bar", "baz", storage)
     assert schema1_parsed is not None
@@ -211,25 +206,23 @@ def test_batch_labels(registry_model):
 )
 def test_repository_tags(repo_namespace, repo_name, registry_model):
     repository_ref = registry_model.lookup_repository(repo_namespace, repo_name)
-    tags = registry_model.list_all_active_repository_tags(
+    tags = registry_model.list_all_active_repository_tags(repository_ref)
-        repository_ref, include_legacy_images=True
-    )
     assert len(tags)
 
     tags_map = registry_model.get_legacy_tags_map(repository_ref, storage)
 
     for tag in tags:
-        found_tag = registry_model.get_repo_tag(repository_ref, tag.name, include_legacy_image=True)
+        found_tag = registry_model.get_repo_tag(repository_ref, tag.name)
         assert found_tag == tag
 
-        if found_tag.legacy_image is None:
+        retriever = RepositoryContentRetriever(repository_ref.id, storage)
-            continue
+        legacy_image = tag.manifest.lookup_legacy_image(0, retriever)
-
         found_image = registry_model.get_legacy_image(
-            repository_ref, found_tag.legacy_image.docker_image_id
+            repository_ref, found_tag.manifest.legacy_image_root_id, storage
         )
-        assert found_image == found_tag.legacy_image
+
-        assert tag.name in tags_map
+        if found_image is not None:
+            assert found_image.docker_image_id == legacy_image.docker_image_id
             assert tags_map[tag.name] == found_image.docker_image_id
 
 
@@ -242,12 +235,19 @@ def test_repository_tags(repo_namespace, repo_name, registry_model):
         ("public", "publicrepo", 1, False),
     ],
 )
-def test_repository_tag_history(namespace, name, expected_tag_count, has_expired, registry_model):
+@pytest.mark.parametrize("with_size_fallback", [False, True,])
+def test_repository_tag_history(
+    namespace, name, expected_tag_count, has_expired, registry_model, with_size_fallback
+):
     # Pre-cache media type loads to ensure consistent query count.
     Manifest.media_type.get_name(1)
 
+    # If size fallback is requested, delete the sizes on the manifest rows.
+    if with_size_fallback:
+        Manifest.update(layers_compressed_size=None).execute()
+
     repository_ref = registry_model.lookup_repository(namespace, name)
-    with assert_query_count(2):
+    with assert_query_count(2 if with_size_fallback else 1):
         history, has_more = registry_model.list_repository_tag_history(repository_ref)
         assert not has_more
         assert len(history) == expected_tag_count
@@ -323,9 +323,7 @@ def test_delete_tags(repo_namespace, repo_name, via_manifest, registry_model):
 
         # Make sure the tag is no longer found.
         with assert_query_count(1):
-            found_tag = registry_model.get_repo_tag(
+            found_tag = registry_model.get_repo_tag(repository_ref, tag.name)
-                repository_ref, tag.name, include_legacy_image=True
-            )
             assert found_tag is None
 
     # Ensure all tags have been deleted.
@@ -347,7 +345,9 @@ def test_retarget_tag_history(use_manifest, registry_model):
             repository_ref, history[0].manifest_digest, allow_dead=True
         )
     else:
-        manifest_or_legacy_image = history[0].legacy_image
+        manifest_or_legacy_image = registry_model.get_legacy_image(
+            repository_ref, history[0].manifest.legacy_image_root_id, storage
+        )
 
     # Retarget the tag.
     assert manifest_or_legacy_image
@@ -364,7 +364,7 @@ def test_retarget_tag_history(use_manifest, registry_model):
     if use_manifest:
         assert updated_tag.manifest_digest == manifest_or_legacy_image.digest
     else:
-        assert updated_tag.legacy_image == manifest_or_legacy_image
+        assert updated_tag.manifest.legacy_image_root_id == manifest_or_legacy_image.docker_image_id
 
     # Ensure history has been updated.
     new_history, _ = registry_model.list_repository_tag_history(repository_ref)
@@ -388,15 +388,17 @@ def test_change_repository_tag_expiration(registry_model):
 
 def test_get_security_status(registry_model):
     repository_ref = registry_model.lookup_repository("devtable", "simple")
-    tags = registry_model.list_all_active_repository_tags(
+    tags = registry_model.list_all_active_repository_tags(repository_ref)
-        repository_ref, include_legacy_images=True
-    )
     assert len(tags)
 
     for tag in tags:
-        assert registry_model.get_security_status(tag.legacy_image)
+        legacy_image = registry_model.get_legacy_image(
-        registry_model.reset_security_status(tag.legacy_image)
+            repository_ref, tag.manifest.legacy_image_root_id, storage
-        assert registry_model.get_security_status(tag.legacy_image)
+        )
+        assert legacy_image
+        assert registry_model.get_security_status(legacy_image)
+        registry_model.reset_security_status(legacy_image)
+        assert registry_model.get_security_status(legacy_image)
 
 
 @pytest.fixture()
@@ -504,145 +506,6 @@ def test_manifest_remote_layers(oci_model):
     assert layers[0].blob is None
 
 
-def test_derived_image(registry_model):
-    # Clear all existing derived storage.
-    DerivedStorageForImage.delete().execute()
-
-    repository_ref = registry_model.lookup_repository("devtable", "simple")
-    tag = registry_model.get_repo_tag(repository_ref, "latest")
-    manifest = registry_model.get_manifest_for_tag(tag)
-
-    # Ensure the squashed image doesn't exist.
-    assert registry_model.lookup_derived_image(manifest, "squash", storage, {}) is None
-
-    # Create a new one.
-    squashed = registry_model.lookup_or_create_derived_image(
-        manifest, "squash", "local_us", storage, {}
-    )
-    assert (
-        registry_model.lookup_or_create_derived_image(manifest, "squash", "local_us", storage, {})
-        == squashed
-    )
-    assert squashed.unique_id
-
-    # Check and set the size.
-    assert squashed.blob.compressed_size is None
-    registry_model.set_derived_image_size(squashed, 1234)
-
-    found = registry_model.lookup_derived_image(manifest, "squash", storage, {})
-    assert found.blob.compressed_size == 1234
-    assert found.unique_id == squashed.unique_id
-
-    # Ensure its returned now.
-    assert found == squashed
-
-    # Ensure different metadata results in a different derived image.
-    found = registry_model.lookup_derived_image(manifest, "squash", storage, {"foo": "bar"})
-    assert found is None
-
-    squashed_foo = registry_model.lookup_or_create_derived_image(
-        manifest, "squash", "local_us", storage, {"foo": "bar"}
-    )
-    assert squashed_foo != squashed
-
-    found = registry_model.lookup_derived_image(manifest, "squash", storage, {"foo": "bar"})
-    assert found == squashed_foo
-
-    assert squashed.unique_id != squashed_foo.unique_id
-
-    # Lookup with placements.
-    squashed = registry_model.lookup_or_create_derived_image(
-        manifest, "squash", "local_us", storage, {}, include_placements=True
-    )
-    assert squashed.blob.placements
-
-    # Delete the derived image.
-    registry_model.delete_derived_image(squashed)
-    assert registry_model.lookup_derived_image(manifest, "squash", storage, {}) is None
-
-
-def test_derived_image_signatures(registry_model):
-    repository_ref = registry_model.lookup_repository("devtable", "simple")
-    tag = registry_model.get_repo_tag(repository_ref, "latest")
-    manifest = registry_model.get_manifest_for_tag(tag)
-
-    derived = registry_model.lookup_or_create_derived_image(
-        manifest, "squash", "local_us", storage, {}
-    )
-    assert derived
-
-    registry_model.set_derived_image_signature(derived, "gpg2", "foo")
-    assert registry_model.get_derived_image_signature(derived, "gpg2") == "foo"
-
-
-@pytest.mark.parametrize(
-    "manifest_builder, list_builder",
-    [
-        (DockerSchema2ManifestBuilder, DockerSchema2ManifestListBuilder),
-        (OCIManifestBuilder, OCIIndexBuilder),
-    ],
-)
-def test_derived_image_for_manifest_list(manifest_builder, list_builder, oci_model):
-    # Clear all existing derived storage.
-    DerivedStorageForImage.delete().execute()
-
-    # Create a config blob for testing.
-    config_json = json.dumps(
-        {
-            "config": {},
-            "architecture": "amd64",
-            "os": "linux",
-            "rootfs": {"type": "layers", "diff_ids": []},
-            "history": [
-                {"created": "2018-04-03T18:37:09.284840891Z", "created_by": "do something",},
-            ],
-        }
-    )
-
-    app_config = {"TESTING": True}
-    repository_ref = oci_model.lookup_repository("devtable", "simple")
-    with upload_blob(repository_ref, storage, BlobUploadSettings(500, 500)) as upload:
-        upload.upload_chunk(app_config, BytesIO(config_json.encode("utf-8")))
-        blob = upload.commit_to_blob(app_config)
-
-    # Create the manifest in the repo.
-    builder = manifest_builder()
-    builder.set_config_digest(blob.digest, blob.compressed_size)
-    builder.add_layer(blob.digest, blob.compressed_size)
-    amd64_manifest = builder.build()
-
-    oci_model.create_manifest_and_retarget_tag(
-        repository_ref, amd64_manifest, "submanifest", storage, raise_on_error=True
-    )
-
-    # Create a manifest list, pointing to at least one amd64+linux manifest.
-    builder = list_builder()
-    builder.add_manifest(amd64_manifest, "amd64", "linux")
-    manifestlist = builder.build()
-
-    oci_model.create_manifest_and_retarget_tag(
-        repository_ref, manifestlist, "listtag", storage, raise_on_error=True
-    )
-
-    manifest = oci_model.get_manifest_for_tag(oci_model.get_repo_tag(repository_ref, "listtag"))
-    assert manifest
-    assert manifest.get_parsed_manifest().is_manifest_list
-
-    # Ensure the squashed image doesn't exist.
-    assert oci_model.lookup_derived_image(manifest, "squash", storage, {}) is None
-
-    # Create a new one.
-    squashed = oci_model.lookup_or_create_derived_image(manifest, "squash", "local_us", storage, {})
-    assert squashed.unique_id
-    assert (
-        oci_model.lookup_or_create_derived_image(manifest, "squash", "local_us", storage, {})
-        == squashed
-    )
-
-    # Perform lookup.
-    assert oci_model.lookup_derived_image(manifest, "squash", storage, {}) == squashed
-
-
 def test_blob_uploads(registry_model):
     repository_ref = registry_model.lookup_repository("devtable", "simple")
 
@@ -763,13 +626,11 @@ def test_get_cached_repo_blob(registry_model):
 
 def test_create_manifest_and_retarget_tag(registry_model):
     repository_ref = registry_model.lookup_repository("devtable", "simple")
-    latest_tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True)
+    latest_tag = registry_model.get_repo_tag(repository_ref, "latest")
     manifest = registry_model.get_manifest_for_tag(latest_tag).get_parsed_manifest()
 
     builder = DockerSchema1ManifestBuilder("devtable", "simple", "anothertag")
-    builder.add_layer(
+    builder.add_layer(manifest.blob_digests[0], '{"id": "%s"}' % "someid")
-        manifest.blob_digests[0], '{"id": "%s"}' % latest_tag.legacy_image.docker_image_id
-    )
     sample_manifest = builder.build(docker_v2_signing_key)
     assert sample_manifest is not None
 
@@ -785,14 +646,14 @@ def test_create_manifest_and_retarget_tag(registry_model):
 
 def test_get_schema1_parsed_manifest(registry_model):
     repository_ref = registry_model.lookup_repository("devtable", "simple")
-    latest_tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True)
+    latest_tag = registry_model.get_repo_tag(repository_ref, "latest")
     manifest = registry_model.get_manifest_for_tag(latest_tag)
     assert registry_model.get_schema1_parsed_manifest(manifest, "", "", "", storage)
 
 
 def test_convert_manifest(registry_model):
     repository_ref = registry_model.lookup_repository("devtable", "simple")
-    latest_tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True)
+    latest_tag = registry_model.get_repo_tag(repository_ref, "latest")
     manifest = registry_model.get_manifest_for_tag(latest_tag)
 
     mediatypes = DOCKER_SCHEMA1_CONTENT_TYPES
@@ -804,11 +665,11 @@ def test_convert_manifest(registry_model):
 
 def test_create_manifest_and_retarget_tag_with_labels(registry_model):
     repository_ref = registry_model.lookup_repository("devtable", "simple")
-    latest_tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True)
+    latest_tag = registry_model.get_repo_tag(repository_ref, "latest")
     manifest = registry_model.get_manifest_for_tag(latest_tag).get_parsed_manifest()
 
     json_metadata = {
-        "id": latest_tag.legacy_image.docker_image_id,
+        "id": "someid",
         "config": {"Labels": {"quay.expires-after": "2w",},},
     }
 
@@ -903,7 +764,8 @@ def test_unicode_emoji(registry_model):
     assert found.get_parsed_manifest().digest == manifest.digest
 
 
-def test_lookup_active_repository_tags(oci_model):
+@pytest.mark.parametrize("test_cached", [False, True,])
+def test_lookup_active_repository_tags(test_cached, oci_model):
     repository_ref = oci_model.lookup_repository("devtable", "simple")
     latest_tag = oci_model.get_repo_tag(repository_ref, "latest")
     manifest = oci_model.get_manifest_for_tag(latest_tag)
@@ -924,7 +786,14 @@ def test_lookup_active_repository_tags(oci_model):
     tags_found = set()
     tag_id = None
     while True:
+        if test_cached:
+            model_cache = InMemoryDataModelCache()
+            tags = oci_model.lookup_cached_active_repository_tags(
+                model_cache, repository_ref, tag_id, 11
+            )
+        else:
             tags = oci_model.lookup_active_repository_tags(repository_ref, tag_id, 11)
+
         assert len(tags) <= 11
         for tag in tags[0:10]:
             assert tag.name not in tags_found
@@ -942,49 +811,27 @@ def test_lookup_active_repository_tags(oci_model):
     assert not tags_expected
 
 
-def test_yield_tags_for_vulnerability_notification(registry_model):
+def test_create_manifest_with_temp_tag(initialized_db, registry_model):
-    repository_ref = registry_model.lookup_repository("devtable", "complex")
+    builder = DockerSchema1ManifestBuilder("devtable", "simple", "latest")
-
+    builder.add_layer(
-    # Check for all legacy images under the tags and ensure not raised because
+        "sha256:abcde", json.dumps({"id": "someid", "author": "some user",}, ensure_ascii=False)
-    # no notification is yet registered.
-    for tag in registry_model.list_all_active_repository_tags(
-        repository_ref, include_legacy_images=True
-    ):
-        image = registry_model.get_legacy_image(
-            repository_ref, tag.legacy_image.docker_image_id, include_blob=True
-        )
-        pairs = [(image.docker_image_id, image.blob.uuid)]
-        results = list(registry_model.yield_tags_for_vulnerability_notification(pairs))
-        assert not len(results)
-
-    # Register a notification.
-    model.notification.create_repo_notification(
-        repository_ref.id, "vulnerability_found", "email", {}, {}
     )
 
-    # Check again.
+    manifest = builder.build(ensure_ascii=False)
-    for tag in registry_model.list_all_active_repository_tags(
-        repository_ref, include_legacy_images=True
-    ):
-        image = registry_model.get_legacy_image(
-            repository_ref,
-            tag.legacy_image.docker_image_id,
-            include_blob=True,
-            include_parents=True,
-        )
 
-        # Check for every parent of the image.
+    for blob_digest in manifest.local_blob_digests:
-        for current in image.parents:
+        _populate_blob(blob_digest)
-            img = registry_model.get_legacy_image(
-                repository_ref, current.docker_image_id, include_blob=True
-            )
-            pairs = [(img.docker_image_id, img.blob.uuid)]
-            results = list(registry_model.yield_tags_for_vulnerability_notification(pairs))
-            assert len(results) > 0
-            assert tag.name in {t.name for t in results}
 
-        # Check for the image itself.
+    # Create the manifest in the database.
-        pairs = [(image.docker_image_id, image.blob.uuid)]
+    repository_ref = registry_model.lookup_repository("devtable", "simple")
-        results = list(registry_model.yield_tags_for_vulnerability_notification(pairs))
+    created = registry_model.create_manifest_with_temp_tag(repository_ref, manifest, 300, storage)
-        assert len(results) > 0
+    assert created.digest == manifest.digest
-        assert tag.name in {t.name for t in results}
+
+    # Ensure it cannot be found normally, since it is simply temp-tagged.
+    assert registry_model.lookup_manifest_by_digest(repository_ref, manifest.digest) is None
+
+    # Ensure it can be found, which means it is temp-tagged.
+    found = registry_model.lookup_manifest_by_digest(
+        repository_ref, manifest.digest, allow_dead=True
+    )
+    assert found is not None

data/registry_model/test/test_manifestbuilder.py

@@ -82,10 +82,9 @@ def test_build_manifest(layers, fake_session, registry_model):
     builder.done()
 
     # Verify the legacy image for the tag.
-    found = registry_model.get_repo_tag(repository_ref, "somenewtag", include_legacy_image=True)
+    found = registry_model.get_repo_tag(repository_ref, "somenewtag")
     assert found
     assert found.name == "somenewtag"
-    assert found.legacy_image.docker_image_id == layers[-1][0]
 
     # Verify the blob and manifest.
     manifest = registry_model.get_manifest_for_tag(found)

data/registry_model/test/test_model_shared.py

@@ -0,0 +1,19 @@
+import pytest
+
+from data.registry_model.shared import SyntheticIDHandler
+
+
+@pytest.mark.parametrize("manifest_id", [1, 1000, 10000, 60000])
+@pytest.mark.parametrize("hash_salt", [None, "", "testing1234", "foobarbaz",])
+def test_handler(manifest_id, hash_salt):
+    handler = SyntheticIDHandler(hash_salt)
+    for index in range(0, 10):
+        assert handler.decode(handler.encode(manifest_id, layer_index=index)) == (
+            manifest_id,
+            index,
+        )
+
+
+def test_invalid_value():
+    handler = SyntheticIDHandler("somehash")
+    assert handler.decode("invalidvalue") == ()

data/secscan_model/__init__.py

@@ -3,8 +3,13 @@ import logging
 from collections import namedtuple
 
 from data.secscan_model.secscan_v2_model import V2SecurityScanner, NoopV2SecurityScanner
-from data.secscan_model.secscan_v4_model import V4SecurityScanner, NoopV4SecurityScanner
+from data.secscan_model.secscan_v4_model import (
+    V4SecurityScanner,
+    NoopV4SecurityScanner,
+    ScanToken as V4ScanToken,
+)
 from data.secscan_model.interface import SecurityScannerInterface, InvalidConfigurationException
+from data.secscan_model.datatypes import SecurityInformationLookupResult, ScanLookupStatus
 from data.database import Manifest
 from data.registry_model.datatypes import Manifest as ManifestDataType
 
@@ -12,68 +17,52 @@ from data.registry_model.datatypes import Manifest as ManifestDataType
 logger = logging.getLogger(__name__)
 
 
-SplitScanToken = namedtuple("NextScanToken", ["version", "token"])
-
-
 class SecurityScannerModelProxy(SecurityScannerInterface):
     def configure(self, app, instance_keys, storage):
-        # TODO(alecmerdler): Just use `V4SecurityScanner` once Clair V2 is removed.
         try:
-            self._model = V2SecurityScanner(app, instance_keys, storage)
+            self._model = V4SecurityScanner(app, instance_keys, storage)
         except InvalidConfigurationException:
-            self._model = NoopV2SecurityScanner()
+            self._model = NoopV4SecurityScanner()
 
         try:
-            self._v4_model = V4SecurityScanner(app, instance_keys, storage)
+            self._legacy_model = V2SecurityScanner(app, instance_keys, storage)
         except InvalidConfigurationException:
-            self._v4_model = NoopV4SecurityScanner()
+            self._legacy_model = NoopV2SecurityScanner()
-
-        self._v4_namespace_whitelist = app.config.get("SECURITY_SCANNER_V4_NAMESPACE_WHITELIST", [])
 
         logger.info("===============================")
-        logger.info("Using split secscan model: `%s`", [self._model, self._v4_model])
+        logger.info("Using split secscan model: `%s`", [self._legacy_model, self._model])
-        logger.info("v4 whitelist `%s`", self._v4_namespace_whitelist)
         logger.info("===============================")
 
         return self
 
     def perform_indexing(self, next_token=None):
-        if next_token is None:
+        if next_token is not None:
-            return SplitScanToken("v4", self._v4_model.perform_indexing(None))
+            assert isinstance(next_token, V4ScanToken)
+            assert isinstance(next_token.min_id, int)
 
-        if next_token.version == "v4" and next_token.token is not None:
+        return self._model.perform_indexing(next_token)
-            return SplitScanToken("v4", self._v4_model.perform_indexing(next_token.token))
-
-        if next_token.version == "v4" and next_token.token is None:
-            return SplitScanToken("v2", self._model.perform_indexing(None))
-
-        if next_token.version == "v2" and next_token.token is not None:
-            return SplitScanToken("v2", self._model.perform_indexing(next_token.token))
-
-        if next_token.version == "v2" and next_token.token is None:
-            return None
 
     def load_security_information(self, manifest_or_legacy_image, include_vulnerabilities):
-        if isinstance(manifest_or_legacy_image, ManifestDataType):
+        manifest = manifest_or_legacy_image.as_manifest()
-            namespace = Manifest.get(
-                manifest_or_legacy_image._db_id
-            ).repository.namespace_user.username
 
-            if namespace in self._v4_namespace_whitelist:
+        info = self._model.load_security_information(manifest, include_vulnerabilities)
-                return self._v4_model.load_security_information(
+        if info.status != ScanLookupStatus.NOT_YET_INDEXED:
+            return info
+
+        legacy_info = self._legacy_model.load_security_information(
             manifest_or_legacy_image, include_vulnerabilities
         )
+        if legacy_info.status != ScanLookupStatus.UNSUPPORTED_FOR_INDEXING:
+            return legacy_info
 
-        return self._model.load_security_information(
+        return SecurityInformationLookupResult.with_status(ScanLookupStatus.NOT_YET_INDEXED)
-            manifest_or_legacy_image, include_vulnerabilities
-        )
 
     def register_model_cleanup_callbacks(self, data_model_config):
         return self._model.register_model_cleanup_callbacks(data_model_config)
 
     @property
     def legacy_api_handler(self):
-        return self._model.legacy_api_handler
+        return self._legacy_model.legacy_api_handler
 
 
 secscan_model = SecurityScannerModelProxy()

data/secscan_model/secscan_v2_model.py

@@ -1,13 +1,10 @@
 import logging
 
 from collections import namedtuple
-from math import log10
 
 from prometheus_client import Gauge
 from deprecated import deprecated
 
-from data.database import UseThenDisconnect
-
 from data.secscan_model.interface import SecurityScannerInterface, InvalidConfigurationException
 from data.secscan_model.datatypes import (
     ScanLookupStatus,
@@ -21,14 +18,6 @@ from data.secscan_model.datatypes import (
 from data.registry_model import registry_model
 from data.registry_model.datatypes import SecurityScanStatus
 
-from data.model.image import (
-    get_images_eligible_for_scan,
-    get_image_pk_field,
-    get_max_id_for_sec_scan,
-    get_min_id_for_sec_scan,
-)
-
-from util.migrate.allocator import yield_random_entries
 from util.config import URLSchemeAndHostname
 from util.secscan.api import V2SecurityConfigValidator, SecurityScannerAPI, APIRequestFailure
 from util.secscan.secscan_util import get_blob_download_uri_getter
@@ -111,12 +100,8 @@ class V2SecurityScanner(SecurityScannerInterface):
             instance_keys=instance_keys,
         )
 
-        # NOTE: This import is in here because otherwise this class would depend upon app.
+    def register_model_cleanup_callbacks(self, data_model_config):
-        # Its not great, but as this is intended to be legacy until its removed, its okay.
+        pass
-        from util.secscan.analyzer import LayerAnalyzer
-
-        self._target_version = app.config.get("SECURITY_SCANNER_ENGINE_VERSION_TARGET", 3)
-        self._analyzer = LayerAnalyzer(app.config, self._legacy_secscan_api)
 
     @property
     def legacy_api_handler(self):
@@ -125,12 +110,6 @@ class V2SecurityScanner(SecurityScannerInterface):
         """
         return self._legacy_secscan_api
 
-    def register_model_cleanup_callbacks(self, data_model_config):
-        if self._legacy_secscan_api is not None:
-            data_model_config.register_image_cleanup_callback(
-                self._legacy_secscan_api.cleanup_layers
-            )
-
     def load_security_information(self, manifest_or_legacy_image, include_vulnerabilities=False):
         status = registry_model.get_security_status(manifest_or_legacy_image)
         if status is None:
@@ -164,80 +143,13 @@ class V2SecurityScanner(SecurityScannerInterface):
             return SecurityInformationLookupResult.for_request_error(str(arf))
 
         if data is None:
-            # If no data was found but we reached this point, then it indicates we have incorrect security
-            # status for the manifest or legacy image. Mark the manifest or legacy image as unindexed
-            # so it automatically gets re-indexed.
-            if self.app.config.get("REGISTRY_STATE", "normal") == "normal":
-                registry_model.reset_security_status(manifest_or_legacy_image)
-
             return SecurityInformationLookupResult.with_status(ScanLookupStatus.NOT_YET_INDEXED)
 
         return SecurityInformationLookupResult.for_data(SecurityInformation.from_dict(data))
 
-    def _candidates_to_scan(self, start_token=None):
-        target_version = self._target_version
-
-        def batch_query():
-            return get_images_eligible_for_scan(target_version)
-
-        # Find the minimum ID.
-        min_id = None
-        if start_token is not None:
-            min_id = start_token.min_id
-        else:
-            min_id = self.app.config.get("SECURITY_SCANNER_INDEXING_MIN_ID")
-            if min_id is None:
-                min_id = get_min_id_for_sec_scan(target_version)
-
-        # Get the ID of the last image we can analyze. Will be None if there are no images in the
-        # database.
-        max_id = get_max_id_for_sec_scan()
-        if max_id is None:
-            return (None, None)
-
-        if min_id is None or min_id > max_id:
-            return (None, None)
-
-        # 4^log10(total) gives us a scalable batch size into the billions.
-        batch_size = int(4 ** log10(max(10, max_id - min_id)))
-
-        # TODO: Once we have a clean shared NamedTuple for Images, send that to the secscan analyzer
-        # rather than the database Image itself.
-        iterator = yield_random_entries(
-            batch_query, get_image_pk_field(), batch_size, max_id, min_id,
-        )
-
-        return (iterator, ScanToken(max_id + 1))
-
     def perform_indexing(self, start_token=None):
         """
         Performs indexing of the next set of unindexed manifests/images.
-
+        NOTE: Raises `NotImplementedError` because indexing for v2 is not supported.
-        If start_token is given, the indexing should resume from that point. Returns a new start
-        index for the next iteration of indexing. The tokens returned and given are assumed to be
-        opaque outside of this implementation and should not be relied upon by the caller to conform
-        to any particular format.
         """
-        # NOTE: This import is in here because otherwise this class would depend upon app.
+        raise NotImplementedError("Unsupported for this security scanner version")
-        # Its not great, but as this is intended to be legacy until its removed, its okay.
-        from util.secscan.analyzer import PreemptedException
-
-        iterator, next_token = self._candidates_to_scan(start_token)
-        if iterator is None:
-            logger.debug("Found no additional images to scan")
-            return None
-
-        with UseThenDisconnect(self.app.config):
-            for candidate, abt, num_remaining in iterator:
-                try:
-                    self._analyzer.analyze_recursively(candidate)
-                except PreemptedException:
-                    logger.debug("Another worker pre-empted us for layer: %s", candidate.id)
-                    abt.set()
-                except APIRequestFailure:
-                    logger.exception("Security scanner service unavailable")
-                    return
-
-                unscanned_images.set(num_remaining)
-
-        return next_token

data/secscan_model/secscan_v4_model.py

@@ -148,19 +148,11 @@ class V4SecurityScanner(SecurityScannerInterface):
         )
 
     def perform_indexing(self, start_token=None):
-        whitelisted_namespaces = self.app.config.get("SECURITY_SCANNER_V4_NAMESPACE_WHITELIST", [])
         try:
             indexer_state = self._secscan_api.state()
         except APIRequestFailure:
             return None
 
-        def eligible_manifests(base_query):
-            return (
-                base_query.join(Repository)
-                .join(User)
-                .where(User.username << whitelisted_namespaces)
-            )
-
         min_id = (
             start_token.min_id
             if start_token is not None
@@ -178,16 +170,14 @@ class V4SecurityScanner(SecurityScannerInterface):
         # TODO(alecmerdler): Filter out any `Manifests` that are still being uploaded
         def not_indexed_query():
             return (
-                eligible_manifests(Manifest.select())
+                Manifest.select()
-                .switch(Manifest)
                 .join(ManifestSecurityStatus, JOIN.LEFT_OUTER)
                 .where(ManifestSecurityStatus.id >> None)
             )
 
         def index_error_query():
             return (
-                eligible_manifests(Manifest.select())
+                Manifest.select()
-                .switch(Manifest)
                 .join(ManifestSecurityStatus)
                 .where(
                     ManifestSecurityStatus.index_status == IndexStatus.FAILED,
@@ -197,8 +187,7 @@ class V4SecurityScanner(SecurityScannerInterface):
 
         def needs_reindexing_query(indexer_hash):
             return (
-                eligible_manifests(Manifest.select())
+                Manifest.select()
-                .switch(Manifest)
                 .join(ManifestSecurityStatus)
                 .where(
                     ManifestSecurityStatus.indexer_hash != indexer_hash,
@@ -209,6 +198,7 @@ class V4SecurityScanner(SecurityScannerInterface):
         # 4^log10(total) gives us a scalable batch size into the billions.
         batch_size = int(4 ** log10(max(10, max_id - min_id)))
 
+        # TODO(alecmerdler): We want to index newer manifests first, while backfilling older manifests...
         iterator = itertools.chain(
             yield_random_entries(not_indexed_query, Manifest.id, batch_size, max_id, min_id,),
             yield_random_entries(index_error_query, Manifest.id, batch_size, max_id, min_id,),

data/secscan_model/test/test_secscan_interface.py

@@ -1,4 +1,5 @@
 import pytest
+
 from mock import patch, Mock
 
 from data.secscan_model.datatypes import ScanLookupStatus, SecurityInformationLookupResult
@@ -8,8 +9,10 @@ from data.secscan_model.secscan_v4_model import (
     IndexReportState,
     ScanToken as V4ScanToken,
 )
-from data.secscan_model import secscan_model, SplitScanToken
+from data.secscan_model import secscan_model
 from data.registry_model import registry_model
+from data.model.oci import shared
+from data.database import ManifestSecurityStatus, IndexerVersion, IndexStatus, ManifestLegacyImage
 
 from test.fixtures import *
 
@@ -17,84 +20,62 @@ from app import app, instance_keys, storage
 
 
 @pytest.mark.parametrize(
-    "repository, v4_whitelist",
+    "indexed_v2, indexed_v4, expected_status",
-    [(("devtable", "complex"), []), (("devtable", "complex"), ["devtable"]),],
+    [
+        (False, False, ScanLookupStatus.NOT_YET_INDEXED),
+        (False, True, ScanLookupStatus.UNSUPPORTED_FOR_INDEXING),
+        (True, False, ScanLookupStatus.FAILED_TO_INDEX),
+        (True, True, ScanLookupStatus.UNSUPPORTED_FOR_INDEXING),
+    ],
 )
-def test_load_security_information_v2_only(repository, v4_whitelist, initialized_db):
+def test_load_security_information(indexed_v2, indexed_v4, expected_status, initialized_db):
-    app.config["SECURITY_SCANNER_V4_NAMESPACE_WHITELIST"] = v4_whitelist
-
     secscan_model.configure(app, instance_keys, storage)
 
-    repo = registry_model.lookup_repository(*repository)
+    repository_ref = registry_model.lookup_repository("devtable", "simple")
-    for tag in registry_model.list_all_active_repository_tags(repo):
+    tag = registry_model.find_matching_tag(repository_ref, ["latest"])
     manifest = registry_model.get_manifest_for_tag(tag)
     assert manifest
 
-        result = secscan_model.load_security_information(manifest, True)
+    registry_model.populate_legacy_images_for_testing(manifest, storage)
-        assert isinstance(result, SecurityInformationLookupResult)
-        assert result.status == ScanLookupStatus.NOT_YET_INDEXED
 
+    image = shared.get_legacy_image_for_manifest(manifest._db_id)
 
-@pytest.mark.parametrize(
+    if indexed_v2:
-    "repository, v4_whitelist",
+        image.security_indexed = False
-    [
+        image.security_indexed_engine = 3
-        (("devtable", "complex"), []),
+        image.save()
-        (("devtable", "complex"), ["devtable"]),
+    else:
-        (("buynlarge", "orgrepo"), ["devtable"]),
+        ManifestLegacyImage.delete().where(
-        (("buynlarge", "orgrepo"), ["devtable", "buynlarge"]),
+            ManifestLegacyImage.manifest == manifest._db_id
-        (("buynlarge", "orgrepo"), ["devtable", "buynlarge", "sellnsmall"]),
+        ).execute()
-    ],
+
+    if indexed_v4:
+        ManifestSecurityStatus.create(
+            manifest=manifest._db_id,
+            repository=repository_ref._db_id,
+            error_json={},
+            index_status=IndexStatus.MANIFEST_UNSUPPORTED,
+            indexer_hash="abc",
+            indexer_version=IndexerVersion.V4,
+            metadata_json={},
         )
-def test_load_security_information(repository, v4_whitelist, initialized_db):
-    app.config["SECURITY_SCANNER_V4_NAMESPACE_WHITELIST"] = v4_whitelist
-    app.config["SECURITY_SCANNER_V4_ENDPOINT"] = "http://clairv4:6060"
-    secscan_api = Mock()
-
-    with patch("data.secscan_model.secscan_v4_model.ClairSecurityScannerAPI", secscan_api):
-        secscan_model.configure(app, instance_keys, storage)
-
-        repo = registry_model.lookup_repository(*repository)
-        for tag in registry_model.list_all_active_repository_tags(repo):
-            manifest = registry_model.get_manifest_for_tag(tag)
-            assert manifest
 
     result = secscan_model.load_security_information(manifest, True)
+
     assert isinstance(result, SecurityInformationLookupResult)
-            assert result.status == ScanLookupStatus.NOT_YET_INDEXED
+    assert result.status == expected_status
 
 
 @pytest.mark.parametrize(
-    "next_token, expected_next_token",
+    "next_token, expected_next_token, expected_error",
     [
-        (None, SplitScanToken("v4", None)),
+        (None, V4ScanToken(56), None),
-        (SplitScanToken("v4", V4ScanToken(1)), SplitScanToken("v4", None)),
+        (V4ScanToken(None), V4ScanToken(56), AssertionError),
-        (SplitScanToken("v4", None), SplitScanToken("v2", V2ScanToken(318))),
+        (V4ScanToken(1), V4ScanToken(56), None),
-        (SplitScanToken("v2", V2ScanToken(318)), SplitScanToken("v2", None)),
+        (V2ScanToken(158), V4ScanToken(56), AssertionError),
-        (SplitScanToken("v2", None), None),
     ],
 )
-def test_perform_indexing_v2_only(next_token, expected_next_token, initialized_db):
+def test_perform_indexing(next_token, expected_next_token, expected_error, initialized_db):
-    def layer_analyzer(*args, **kwargs):
-        return Mock()
-
-    with patch("util.secscan.analyzer.LayerAnalyzer", layer_analyzer):
-        secscan_model.configure(app, instance_keys, storage)
-
-        assert secscan_model.perform_indexing(next_token) == expected_next_token
-
-
-@pytest.mark.parametrize(
-    "next_token, expected_next_token",
-    [
-        (None, SplitScanToken("v4", V4ScanToken(56))),
-        (SplitScanToken("v4", V4ScanToken(1)), SplitScanToken("v4", V4ScanToken(56))),
-        (SplitScanToken("v4", None), SplitScanToken("v2", V2ScanToken(318))),
-        (SplitScanToken("v2", V2ScanToken(318)), SplitScanToken("v2", None)),
-        (SplitScanToken("v2", None), None),
-    ],
-)
-def test_perform_indexing(next_token, expected_next_token, initialized_db):
-    app.config["SECURITY_SCANNER_V4_NAMESPACE_WHITELIST"] = ["devtable"]
     app.config["SECURITY_SCANNER_V4_ENDPOINT"] = "http://clairv4:6060"
 
     def secscan_api(*args, **kwargs):
@@ -104,11 +85,11 @@ def test_perform_indexing(next_token, expected_next_token, initialized_db):
 
         return api
 
-    def layer_analyzer(*args, **kwargs):
-        return Mock()
-
     with patch("data.secscan_model.secscan_v4_model.ClairSecurityScannerAPI", secscan_api):
-        with patch("util.secscan.analyzer.LayerAnalyzer", layer_analyzer):
         secscan_model.configure(app, instance_keys, storage)
 
+        if expected_error is not None:
+            with pytest.raises(expected_error):
+                secscan_model.perform_indexing(next_token)
+        else:
             assert secscan_model.perform_indexing(next_token) == expected_next_token

data/secscan_model/test/test_secscan_v2_model.py

@@ -4,7 +4,7 @@ import pytest
 from data.secscan_model.datatypes import ScanLookupStatus, SecurityInformation
 from data.secscan_model.secscan_v2_model import V2SecurityScanner
 from data.registry_model import registry_model
-from data.database import Manifest, Image
+from data.database import Manifest, Image, ManifestSecurityStatus, IndexStatus, IndexerVersion
 from data.model.oci import shared
 from data.model.image import set_secscan_status
 
@@ -15,8 +15,10 @@ from app import app, instance_keys, storage
 
 def test_load_security_information_unknown_manifest(initialized_db):
     repository_ref = registry_model.lookup_repository("devtable", "simple")
-    tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True)
+    tag = registry_model.get_repo_tag(repository_ref, "latest")
-    manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True)
+    manifest = registry_model.get_manifest_for_tag(tag)
+
+    registry_model.populate_legacy_images_for_testing(manifest, storage)
 
     # Delete the manifest.
     Manifest.get(id=manifest._db_id).delete_instance(recursive=True)
@@ -30,8 +32,10 @@ def test_load_security_information_unknown_manifest(initialized_db):
 
 def test_load_security_information_failed_to_index(initialized_db):
     repository_ref = registry_model.lookup_repository("devtable", "simple")
-    tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True)
+    tag = registry_model.get_repo_tag(repository_ref, "latest")
-    manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True)
+    manifest = registry_model.get_manifest_for_tag(tag)
+
+    registry_model.populate_legacy_images_for_testing(manifest, storage)
 
     # Set the index status.
     image = shared.get_legacy_image_for_manifest(manifest._db_id)
@@ -45,8 +49,10 @@ def test_load_security_information_failed_to_index(initialized_db):
 
 def test_load_security_information_queued(initialized_db):
     repository_ref = registry_model.lookup_repository("devtable", "simple")
-    tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True)
+    tag = registry_model.get_repo_tag(repository_ref, "latest")
-    manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True)
+    manifest = registry_model.get_manifest_for_tag(tag)
+
+    registry_model.populate_legacy_images_for_testing(manifest, storage)
 
     secscan = V2SecurityScanner(app, instance_keys, storage)
     assert secscan.load_security_information(manifest).status == ScanLookupStatus.NOT_YET_INDEXED
@@ -87,11 +93,14 @@ def test_load_security_information_queued(initialized_db):
 )
 def test_load_security_information_api_responses(secscan_api_response, initialized_db):
     repository_ref = registry_model.lookup_repository("devtable", "simple")
-    tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True)
+    tag = registry_model.get_repo_tag(repository_ref, "latest")
-    manifest = registry_model.get_manifest_for_tag(
+    manifest = registry_model.get_manifest_for_tag(tag)
-        tag, backfill_if_necessary=True, include_legacy_image=True
+
-    )
+    registry_model.populate_legacy_images_for_testing(manifest, storage)
-    set_secscan_status(Image.get(id=manifest.legacy_image._db_id), True, 3)
+
+    legacy_image_row = shared.get_legacy_image_for_manifest(manifest._db_id)
+    assert legacy_image_row is not None
+    set_secscan_status(legacy_image_row, True, 3)
 
     secscan = V2SecurityScanner(app, instance_keys, storage)
     secscan._legacy_secscan_api = mock.Mock()
@@ -110,3 +119,10 @@ def test_load_security_information_api_responses(secscan_api_response, initializ
     assert len(security_information.Layer.Features) == len(
         secscan_api_response["Layer"].get("Features", [])
     )
+
+
+def test_perform_indexing(initialized_db):
+    secscan = V2SecurityScanner(app, instance_keys, storage)
+
+    with pytest.raises(NotImplementedError):
+        secscan.perform_indexing()

data/secscan_model/test/test_secscan_v4_model.py

@@ -33,8 +33,8 @@ def set_secscan_config():
 
 def test_load_security_information_queued(initialized_db, set_secscan_config):
     repository_ref = registry_model.lookup_repository("devtable", "simple")
-    tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True)
+    tag = registry_model.get_repo_tag(repository_ref, "latest")
-    manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True)
+    manifest = registry_model.get_manifest_for_tag(tag)
 
     secscan = V4SecurityScanner(app, instance_keys, storage)
     assert secscan.load_security_information(manifest).status == ScanLookupStatus.NOT_YET_INDEXED
@@ -42,8 +42,8 @@ def test_load_security_information_queued(initialized_db, set_secscan_config):
 
 def test_load_security_information_failed_to_index(initialized_db, set_secscan_config):
     repository_ref = registry_model.lookup_repository("devtable", "simple")
-    tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True)
+    tag = registry_model.get_repo_tag(repository_ref, "latest")
-    manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True)
+    manifest = registry_model.get_manifest_for_tag(tag)
 
     ManifestSecurityStatus.create(
         manifest=manifest._db_id,
@@ -61,8 +61,8 @@ def test_load_security_information_failed_to_index(initialized_db, set_secscan_c
 
 def test_load_security_information_api_returns_none(initialized_db, set_secscan_config):
     repository_ref = registry_model.lookup_repository("devtable", "simple")
-    tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True)
+    tag = registry_model.get_repo_tag(repository_ref, "latest")
-    manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True)
+    manifest = registry_model.get_manifest_for_tag(tag)
 
     ManifestSecurityStatus.create(
         manifest=manifest._db_id,
@@ -83,8 +83,8 @@ def test_load_security_information_api_returns_none(initialized_db, set_secscan_
 
 def test_load_security_information_api_request_failure(initialized_db, set_secscan_config):
     repository_ref = registry_model.lookup_repository("devtable", "simple")
-    tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True)
+    tag = registry_model.get_repo_tag(repository_ref, "latest")
-    manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True)
+    manifest = registry_model.get_manifest_for_tag(tag)
 
     mss = ManifestSecurityStatus.create(
         manifest=manifest._db_id,
@@ -106,8 +106,8 @@ def test_load_security_information_api_request_failure(initialized_db, set_secsc
 
 def test_load_security_information_success(initialized_db, set_secscan_config):
     repository_ref = registry_model.lookup_repository("devtable", "simple")
-    tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True)
+    tag = registry_model.get_repo_tag(repository_ref, "latest")
-    manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True)
+    manifest = registry_model.get_manifest_for_tag(tag)
 
     ManifestSecurityStatus.create(
         manifest=manifest._db_id,
@@ -140,11 +140,6 @@ def test_load_security_information_success(initialized_db, set_secscan_config):
 
 
 def test_perform_indexing_whitelist(initialized_db, set_secscan_config):
-    app.config["SECURITY_SCANNER_V4_NAMESPACE_WHITELIST"] = ["devtable"]
-    expected_manifests = (
-        Manifest.select().join(Repository).join(User).where(User.username == "devtable")
-    )
-
     secscan = V4SecurityScanner(app, instance_keys, storage)
     secscan._secscan_api = mock.Mock()
     secscan._secscan_api.state.return_value = {"state": "abc"}
@@ -155,38 +150,15 @@ def test_perform_indexing_whitelist(initialized_db, set_secscan_config):
 
     next_token = secscan.perform_indexing()
 
-    assert secscan._secscan_api.index.call_count == expected_manifests.count()
-    for mss in ManifestSecurityStatus.select():
-        assert mss.repository.namespace_user.username == "devtable"
-    assert ManifestSecurityStatus.select().count() == expected_manifests.count()
-    assert (
-        Manifest.get_by_id(next_token.min_id - 1).repository.namespace_user.username == "devtable"
-    )
-
-
-def test_perform_indexing_empty_whitelist(initialized_db, set_secscan_config):
-    app.config["SECURITY_SCANNER_V4_NAMESPACE_WHITELIST"] = []
-    secscan = V4SecurityScanner(app, instance_keys, storage)
-    secscan._secscan_api = mock.Mock()
-    secscan._secscan_api.state.return_value = {"state": "abc"}
-    secscan._secscan_api.index.return_value = (
-        {"err": None, "state": IndexReportState.Index_Finished},
-        "abc",
-    )
-
-    next_token = secscan.perform_indexing()
-
-    assert secscan._secscan_api.index.call_count == 0
-    assert ManifestSecurityStatus.select().count() == 0
     assert next_token.min_id == Manifest.select(fn.Max(Manifest.id)).scalar() + 1
 
+    assert secscan._secscan_api.index.call_count == Manifest.select().count()
+    assert ManifestSecurityStatus.select().count() == Manifest.select().count()
+    for mss in ManifestSecurityStatus.select():
+        assert mss.index_status == IndexStatus.COMPLETED
+
 
 def test_perform_indexing_failed(initialized_db, set_secscan_config):
-    app.config["SECURITY_SCANNER_V4_NAMESPACE_WHITELIST"] = ["devtable"]
-    expected_manifests = (
-        Manifest.select().join(Repository).join(User).where(User.username == "devtable")
-    )
-
     secscan = V4SecurityScanner(app, instance_keys, storage)
     secscan._secscan_api = mock.Mock()
     secscan._secscan_api.state.return_value = {"state": "abc"}
@@ -195,7 +167,7 @@ def test_perform_indexing_failed(initialized_db, set_secscan_config):
         "abc",
     )
 
-    for manifest in expected_manifests:
+    for manifest in Manifest.select():
         ManifestSecurityStatus.create(
             manifest=manifest,
             repository=manifest.repository,
@@ -210,16 +182,13 @@ def test_perform_indexing_failed(initialized_db, set_secscan_config):
 
     secscan.perform_indexing()
 
-    assert ManifestSecurityStatus.select().count() == expected_manifests.count()
+    assert ManifestSecurityStatus.select().count() == Manifest.select().count()
     for mss in ManifestSecurityStatus.select():
         assert mss.index_status == IndexStatus.COMPLETED
 
 
 def test_perform_indexing_failed_within_reindex_threshold(initialized_db, set_secscan_config):
     app.config["SECURITY_SCANNER_V4_REINDEX_THRESHOLD"] = 300
-    expected_manifests = (
-        Manifest.select().join(Repository).join(User).where(User.username == "devtable")
-    )
 
     secscan = V4SecurityScanner(app, instance_keys, storage)
     secscan._secscan_api = mock.Mock()
@@ -229,7 +198,7 @@ def test_perform_indexing_failed_within_reindex_threshold(initialized_db, set_se
         "abc",
     )
 
-    for manifest in expected_manifests:
+    for manifest in Manifest.select():
         ManifestSecurityStatus.create(
             manifest=manifest,
             repository=manifest.repository,
@@ -242,17 +211,12 @@ def test_perform_indexing_failed_within_reindex_threshold(initialized_db, set_se
 
     secscan.perform_indexing()
 
-    assert ManifestSecurityStatus.select().count() == expected_manifests.count()
+    assert ManifestSecurityStatus.select().count() == Manifest.select().count()
     for mss in ManifestSecurityStatus.select():
         assert mss.index_status == IndexStatus.FAILED
 
 
 def test_perform_indexing_needs_reindexing(initialized_db, set_secscan_config):
-    app.config["SECURITY_SCANNER_V4_NAMESPACE_WHITELIST"] = ["devtable"]
-    expected_manifests = (
-        Manifest.select().join(Repository).join(User).where(User.username == "devtable")
-    )
-
     secscan = V4SecurityScanner(app, instance_keys, storage)
     secscan._secscan_api = mock.Mock()
     secscan._secscan_api.state.return_value = {"state": "xyz"}
@@ -261,7 +225,7 @@ def test_perform_indexing_needs_reindexing(initialized_db, set_secscan_config):
         "xyz",
     )
 
-    for manifest in expected_manifests:
+    for manifest in Manifest.select():
         ManifestSecurityStatus.create(
             manifest=manifest,
             repository=manifest.repository,
@@ -276,7 +240,7 @@ def test_perform_indexing_needs_reindexing(initialized_db, set_secscan_config):
 
     secscan.perform_indexing()
 
-    assert ManifestSecurityStatus.select().count() == expected_manifests.count()
+    assert ManifestSecurityStatus.select().count() == Manifest.select().count()
     for mss in ManifestSecurityStatus.select():
         assert mss.indexer_hash == "xyz"
 
@@ -285,10 +249,6 @@ def test_perform_indexing_needs_reindexing_within_reindex_threshold(
     initialized_db, set_secscan_config
 ):
     app.config["SECURITY_SCANNER_V4_REINDEX_THRESHOLD"] = 300
-    app.config["SECURITY_SCANNER_V4_NAMESPACE_WHITELIST"] = ["devtable"]
-    expected_manifests = (
-        Manifest.select().join(Repository).join(User).where(User.username == "devtable")
-    )
 
     secscan = V4SecurityScanner(app, instance_keys, storage)
     secscan._secscan_api = mock.Mock()
@@ -298,7 +258,7 @@ def test_perform_indexing_needs_reindexing_within_reindex_threshold(
         "xyz",
     )
 
-    for manifest in expected_manifests:
+    for manifest in Manifest.select():
         ManifestSecurityStatus.create(
             manifest=manifest,
             repository=manifest.repository,
@@ -311,14 +271,12 @@ def test_perform_indexing_needs_reindexing_within_reindex_threshold(
 
     secscan.perform_indexing()
 
-    assert ManifestSecurityStatus.select().count() == expected_manifests.count()
+    assert ManifestSecurityStatus.select().count() == Manifest.select().count()
     for mss in ManifestSecurityStatus.select():
         assert mss.indexer_hash == "abc"
 
 
 def test_perform_indexing_api_request_failure_state(initialized_db, set_secscan_config):
-    app.config["SECURITY_SCANNER_V4_NAMESPACE_WHITELIST"] = ["devtable"]
-
     secscan = V4SecurityScanner(app, instance_keys, storage)
     secscan._secscan_api = mock.Mock()
     secscan._secscan_api.state.side_effect = APIRequestFailure()
@@ -330,14 +288,6 @@ def test_perform_indexing_api_request_failure_state(initialized_db, set_secscan_
 
 
 def test_perform_indexing_api_request_failure_index(initialized_db, set_secscan_config):
-    app.config["SECURITY_SCANNER_V4_NAMESPACE_WHITELIST"] = ["devtable"]
-    expected_manifests = (
-        Manifest.select(fn.Max(Manifest.id))
-        .join(Repository)
-        .join(User)
-        .where(User.username == "devtable")
-    )
-
     secscan = V4SecurityScanner(app, instance_keys, storage)
     secscan._secscan_api = mock.Mock()
     secscan._secscan_api.state.return_value = {"state": "abc"}
@@ -357,8 +307,8 @@ def test_perform_indexing_api_request_failure_index(initialized_db, set_secscan_
 
     next_token = secscan.perform_indexing()
 
-    assert next_token.min_id == expected_manifests.scalar() + 1
+    assert next_token.min_id == Manifest.select(fn.Max(Manifest.id)).scalar() + 1
-    assert ManifestSecurityStatus.select().count() == expected_manifests.count()
+    assert ManifestSecurityStatus.select().count() == Manifest.select(fn.Max(Manifest.id)).count()
 
 
 def test_features_for():

endpoints/api/image.py

@@ -3,6 +3,10 @@ List and lookup repository images.
 """
 import json
 
+from collections import defaultdict
+from datetime import datetime
+
+from app import storage
 from data.registry_model import registry_model
 from endpoints.api import (
     resource,
@@ -17,7 +21,7 @@ from endpoints.api import (
 from endpoints.exception import NotFound
 
 
-def image_dict(image, with_history=False, with_tags=False):
+def image_dict(image):
     parsed_command = None
     if image.command:
         try:
@@ -31,19 +35,11 @@ def image_dict(image, with_history=False, with_tags=False):
         "comment": image.comment,
         "command": parsed_command,
         "size": image.image_size,
-        "uploading": image.uploading,
+        "uploading": False,
-        "sort_index": len(image.parents),
+        "sort_index": 0,
     }
 
-    if with_tags:
+    image_data["ancestors"] = "/{0}/".format("/".join(image.ancestor_ids))
-        image_data["tags"] = [tag.name for tag in image.tags]
-
-    if with_history:
-        image_data["history"] = [image_dict(parent) for parent in image.parents]
-
-    # Calculate the ancestors string, with the DBID's replaced with the docker IDs.
-    parent_docker_ids = [parent_image.docker_image_id for parent_image in image.parents]
-    image_data["ancestors"] = "/{0}/".format("/".join(parent_docker_ids))
     return image_data
 
 
@@ -66,8 +62,35 @@ class RepositoryImageList(RepositoryParamResource):
         if repo_ref is None:
             raise NotFound()
 
-        images = registry_model.get_legacy_images(repo_ref)
+        tags = registry_model.list_all_active_repository_tags(repo_ref)
-        return {"images": [image_dict(image, with_tags=True) for image in images]}
+        images_with_tags = defaultdict(list)
+        for tag in tags:
+            legacy_image_id = tag.manifest.legacy_image_root_id
+            if legacy_image_id is not None:
+                images_with_tags[legacy_image_id].append(tag)
+
+        # NOTE: This is replicating our older response for this endpoint, but
+        # returns empty for the metadata fields. This is to ensure back-compat
+        # for callers still using the deprecated API, while not having to load
+        # all the manifests from storage.
+        return {
+            "images": [
+                {
+                    "id": image_id,
+                    "created": format_date(
+                        datetime.utcfromtimestamp((min([tag.lifetime_start_ts for tag in tags])))
+                    ),
+                    "comment": "",
+                    "command": "",
+                    "size": 0,
+                    "uploading": False,
+                    "sort_index": 0,
+                    "tags": [tag.name for tag in tags],
+                    "ancestors": "",
+                }
+                for image_id, tags in images_with_tags.items()
+            ]
+        }
 
 
 @resource("/v1/repository/<apirepopath:repository>/image/<image_id>")
@@ -90,8 +113,8 @@ class RepositoryImage(RepositoryParamResource):
         if repo_ref is None:
             raise NotFound()
 
-        image = registry_model.get_legacy_image(repo_ref, image_id, include_parents=True)
+        image = registry_model.get_legacy_image(repo_ref, image_id, storage)
         if image is None:
             raise NotFound()
 
-        return image_dict(image, with_history=True)
+        return image_dict(image)

endpoints/api/manifest.py

@@ -4,6 +4,7 @@ Manage the manifests of a repository.
 import json
 import logging
 
+from datetime import datetime
 from flask import request
 
 from app import label_validator, storage
@@ -74,10 +75,6 @@ def _layer_dict(manifest_layer, index):
 
 
 def _manifest_dict(manifest):
-    image = None
-    if manifest.legacy_image_if_present is not None:
-        image = image_dict(manifest.legacy_image, with_history=True)
-
     layers = None
     if not manifest.is_manifest_list:
         layers = registry_model.list_manifest_layers(manifest, storage)
@@ -85,14 +82,30 @@ def _manifest_dict(manifest):
             logger.debug("Missing layers for manifest `%s`", manifest.digest)
             abort(404)
 
+    image = None
+    if manifest.legacy_image_root_id:
+        # NOTE: This is replicating our older response for this endpoint, but
+        # returns empty for the metadata fields. This is to ensure back-compat
+        # for callers still using the deprecated API.
+        image = {
+            "id": manifest.legacy_image_root_id,
+            "created": format_date(datetime.utcnow()),
+            "comment": "",
+            "command": "",
+            "size": 0,
+            "uploading": False,
+            "sort_index": 0,
+            "ancestors": "",
+        }
+
     return {
         "digest": manifest.digest,
         "is_manifest_list": manifest.is_manifest_list,
         "manifest_data": manifest.internal_manifest_bytes.as_unicode(),
-        "image": image,
         "layers": (
             [_layer_dict(lyr.layer_info, idx) for idx, lyr in enumerate(layers)] if layers else None
         ),
+        "image": image,
     }
 
 
@@ -112,9 +125,7 @@ class RepositoryManifest(RepositoryParamResource):
         if repo_ref is None:
             raise NotFound()
 
-        manifest = registry_model.lookup_manifest_by_digest(
+        manifest = registry_model.lookup_manifest_by_digest(repo_ref, manifestref)
-            repo_ref, manifestref, include_legacy_image=True
-        )
         if manifest is None:
             raise NotFound()
 

endpoints/api/repository_models_pre_oci.py

@@ -161,7 +161,7 @@ class PreOCIModel(RepositoryDataInterface):
                     repo.namespace_user.username,
                     repo.name,
                     repo.rid in star_set,
-                    repo.visibility_id == model.repository.get_public_repo_visibility().id,
+                    model.repository.is_repository_public(repo),
                     repo_kind,
                     repo.description,
                     repo.namespace_user.organization,
@@ -257,8 +257,8 @@ class PreOCIModel(RepositoryDataInterface):
             tags = [
                 Tag(
                     tag.name,
-                    tag.legacy_image.docker_image_id if tag.legacy_image_if_present else None,
+                    tag.manifest.legacy_image_root_id,
-                    tag.legacy_image.aggregate_size if tag.legacy_image_if_present else None,
+                    tag.manifest_layers_size,
                     tag.lifetime_start_ts,
                     tag.manifest_digest,
                     tag.lifetime_end_ts,

endpoints/api/robot_models_pre_oci.py

@@ -25,7 +25,7 @@ class RobotPreOCIModel(RobotInterface):
         return [
             Permission(
                 permission.repository.name,
-                permission.repository.visibility.name,
+                model.repositoy.repository_visibility_name(permission.repository),
                 permission.role.name,
             )
             for permission in permissions

endpoints/api/secscan.py

@@ -7,6 +7,7 @@ import features
 
 from enum import Enum, unique
 
+from app import storage
 from auth.decorators import process_basic_auth_no_pass
 from data.registry_model import registry_model
 from data.secscan_model import secscan_model
@@ -101,7 +102,7 @@ class RepositoryImageSecurity(RepositoryParamResource):
         if repo_ref is None:
             raise NotFound()
 
-        legacy_image = registry_model.get_legacy_image(repo_ref, imageid)
+        legacy_image = registry_model.get_legacy_image(repo_ref, imageid, storage)
         if legacy_image is None:
             raise NotFound()
 

endpoints/api/tag.py

@@ -9,6 +9,7 @@ from auth.auth_context import get_authenticated_user
 from data.registry_model import registry_model
 from endpoints.api import (
     resource,
+    deprecated,
     nickname,
     require_repo_read,
     require_repo_write,
@@ -40,18 +41,11 @@ def _tag_dict(tag):
     if tag.lifetime_end_ts and tag.lifetime_end_ts > 0:
         tag_info["end_ts"] = tag.lifetime_end_ts
 
-    # TODO: Remove this once fully on OCI data model.
-    if tag.legacy_image_if_present:
-        tag_info["docker_image_id"] = tag.legacy_image.docker_image_id
-        tag_info["image_id"] = tag.legacy_image.docker_image_id
-        tag_info["size"] = tag.legacy_image.aggregate_size
-
-    # TODO: Remove this check once fully on OCI data model.
-    if tag.manifest_digest:
     tag_info["manifest_digest"] = tag.manifest_digest
-
-        if tag.manifest:
     tag_info["is_manifest_list"] = tag.manifest.is_manifest_list
+    tag_info["size"] = tag.manifest_layers_size
+    tag_info["docker_image_id"] = tag.manifest.legacy_image_root_id
+    tag_info["image_id"] = tag.manifest.legacy_image_root_id
 
     if tag.lifetime_start_ts and tag.lifetime_start_ts > 0:
         last_modified = format_date(datetime.utcfromtimestamp(tag.lifetime_start_ts))
@@ -188,7 +182,7 @@ class RepositoryTag(RepositoryParamResource):
                 raise InvalidRequest("Could not update tag expiration; Tag has probably changed")
 
         if "image" in request.get_json() or "manifest_digest" in request.get_json():
-            existing_tag = registry_model.get_repo_tag(repo_ref, tag, include_legacy_image=True)
+            existing_tag = registry_model.get_repo_tag(repo_ref, tag)
 
             manifest_or_image = None
             image_id = None
@@ -201,7 +195,7 @@ class RepositoryTag(RepositoryParamResource):
                 )
             else:
                 image_id = request.get_json()["image"]
-                manifest_or_image = registry_model.get_legacy_image(repo_ref, image_id)
+                manifest_or_image = registry_model.get_legacy_image(repo_ref, image_id, storage)
 
             if manifest_or_image is None:
                 raise NotFound()
@@ -272,6 +266,7 @@ class RepositoryTagImages(RepositoryParamResource):
     @nickname("listTagImages")
     @disallow_for_app_repositories
     @parse_args()
+    @deprecated()
     @query_param(
         "owned",
         "If specified, only images wholely owned by this tag are returned.",
@@ -286,30 +281,42 @@ class RepositoryTagImages(RepositoryParamResource):
         if repo_ref is None:
             raise NotFound()
 
-        tag_ref = registry_model.get_repo_tag(repo_ref, tag, include_legacy_image=True)
+        tag_ref = registry_model.get_repo_tag(repo_ref, tag)
         if tag_ref is None:
             raise NotFound()
 
-        if tag_ref.legacy_image_if_present is None:
+        if parsed_args["owned"]:
+            # NOTE: This is deprecated, so we just return empty now.
             return {"images": []}
 
-        image_id = tag_ref.legacy_image.docker_image_id
+        manifest = registry_model.get_manifest_for_tag(tag_ref)
-
+        if manifest is None:
-        all_images = None
-        if parsed_args["owned"]:
-            # TODO: Remove the `owned` image concept once we are fully on V2_2.
-            all_images = registry_model.get_legacy_images_owned_by_tag(tag_ref)
-        else:
-            image_with_parents = registry_model.get_legacy_image(
-                repo_ref, image_id, include_parents=True
-            )
-            if image_with_parents is None:
             raise NotFound()
 
-            all_images = [image_with_parents] + image_with_parents.parents
+        legacy_image = registry_model.get_legacy_image(
+            repo_ref, manifest.legacy_image_root_id, storage
+        )
+        if legacy_image is None:
+            raise NotFound()
 
+        # NOTE: This is replicating our older response for this endpoint, but
+        # returns empty for the metadata fields. This is to ensure back-compat
+        # for callers still using the deprecated API, while not having to load
+        # all the manifests from storage.
         return {
-            "images": [image_dict(image) for image in all_images],
+            "images": [
+                {
+                    "id": image_id,
+                    "created": format_date(datetime.utcfromtimestamp(tag_ref.lifetime_start_ts)),
+                    "comment": "",
+                    "command": "",
+                    "size": 0,
+                    "uploading": False,
+                    "sort_index": 0,
+                    "ancestors": "",
+                }
+                for image_id in legacy_image.full_image_id_chain
+            ]
         }
 
 
@@ -374,7 +381,7 @@ class RestoreTag(RepositoryParamResource):
                 repo_ref, manifest_digest, allow_dead=True, require_available=True
             )
         elif image_id is not None:
-            manifest_or_legacy_image = registry_model.get_legacy_image(repo_ref, image_id)
+            manifest_or_legacy_image = registry_model.get_legacy_image(repo_ref, image_id, storage)
 
         if manifest_or_legacy_image is None:
             raise NotFound()

endpoints/api/team.py

@@ -49,7 +49,7 @@ def permission_view(permission):
     return {
         "repository": {
             "name": permission.repository.name,
-            "is_public": permission.repository.visibility.name == "public",
+            "is_public": model.repository.is_repository_public(permission.repository),
         },
         "role": permission.role.name,
     }

endpoints/api/test/test_deprecated_route.py

@@ -11,16 +11,15 @@ from test.fixtures import *
 
 def test_deprecated_route(client):
     repository_ref = registry_model.lookup_repository("devtable", "simple")
-    tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True)
+    tag = registry_model.get_repo_tag(repository_ref, "latest")
-    manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True)
+    manifest = registry_model.get_manifest_for_tag(tag)
-    image = shared.get_legacy_image_for_manifest(manifest._db_id)
 
     with client_with_identity("devtable", client) as cl:
         resp = conduct_api_call(
             cl,
             RepositoryImageSecurity,
             "get",
-            {"repository": "devtable/simple", "imageid": image.docker_image_id},
+            {"repository": "devtable/simple", "imageid": manifest.legacy_image_root_id},
             expected_code=200,
         )
 

endpoints/api/test/test_secscan.py

@@ -13,12 +13,12 @@ from test.fixtures import *
 @pytest.mark.parametrize("endpoint", [RepositoryImageSecurity, RepositoryManifestSecurity,])
 def test_get_security_info_with_pull_secret(endpoint, client):
     repository_ref = registry_model.lookup_repository("devtable", "simple")
-    tag = registry_model.get_repo_tag(repository_ref, "latest", include_legacy_image=True)
+    tag = registry_model.get_repo_tag(repository_ref, "latest")
-    manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True)
+    manifest = registry_model.get_manifest_for_tag(tag)
 
     params = {
         "repository": "devtable/simple",
-        "imageid": tag.legacy_image.docker_image_id,
+        "imageid": tag.manifest.legacy_image_root_id,
         "manifestref": manifest.digest,
     }
 

endpoints/api/test/test_tag.py

@@ -69,10 +69,10 @@ def test_move_tag(image_exists, test_tag, expected_status, client, app):
         test_image = "unknown"
         if image_exists:
             repo_ref = registry_model.lookup_repository("devtable", "simple")
-            tag_ref = registry_model.get_repo_tag(repo_ref, "latest", include_legacy_image=True)
+            tag_ref = registry_model.get_repo_tag(repo_ref, "latest")
             assert tag_ref
 
-            test_image = tag_ref.legacy_image.docker_image_id
+            test_image = tag_ref.manifest.legacy_image_root_id
 
         params = {"repository": "devtable/simple", "tag": test_tag}
         request_body = {"image": test_image}
@@ -86,12 +86,12 @@ def test_move_tag(image_exists, test_tag, expected_status, client, app):
 @pytest.mark.parametrize(
     "repo_namespace, repo_name, query_count",
     [
-        ("devtable", "simple", 5),
+        ("devtable", "simple", 4),
-        ("devtable", "history", 5),
+        ("devtable", "history", 4),
-        ("devtable", "complex", 5),
+        ("devtable", "complex", 4),
-        ("devtable", "gargantuan", 5),
+        ("devtable", "gargantuan", 4),
-        ("buynlarge", "orgrepo", 7),  # +2 for permissions checks.
+        ("buynlarge", "orgrepo", 6),  # +2 for permissions checks.
-        ("buynlarge", "anotherorgrepo", 7),  # +2 for permissions checks.
+        ("buynlarge", "anotherorgrepo", 6),  # +2 for permissions checks.
     ],
 )
 def test_list_repo_tags(repo_namespace, repo_name, client, query_count, app):
@@ -109,18 +109,15 @@ def test_list_repo_tags(repo_namespace, repo_name, client, query_count, app):
 
 
 @pytest.mark.parametrize(
-    "repository, tag, owned, expect_images",
+    "repository, tag, expect_images",
     [
-        ("devtable/simple", "prod", False, True),
+        ("devtable/simple", "prod", True),
-        ("devtable/simple", "prod", True, False),
+        ("devtable/simple", "latest", True),
-        ("devtable/simple", "latest", False, True),
+        ("devtable/complex", "prod", True),
-        ("devtable/simple", "latest", True, False),
-        ("devtable/complex", "prod", False, True),
-        ("devtable/complex", "prod", True, True),
     ],
 )
-def test_list_tag_images(repository, tag, owned, expect_images, client, app):
+def test_list_tag_images(repository, tag, expect_images, client, app):
     with client_with_identity("devtable", client) as cl:
-        params = {"repository": repository, "tag": tag, "owned": owned}
+        params = {"repository": repository, "tag": tag}
         result = conduct_api_call(cl, RepositoryTagImages, "get", params, None, 200).json
         assert bool(result["images"]) == expect_images

endpoints/api/user.py

@@ -1087,7 +1087,7 @@ class StarredRepositoryList(ApiResource):
                 "namespace": repo_obj.namespace_user.username,
                 "name": repo_obj.name,
                 "description": repo_obj.description,
-                "is_public": repo_obj.visibility.name == "public",
+                "is_public": model.repository.is_repository_public(repo_obj),
             }
 
         return {"repositories": [repo_view(repo) for repo in repos]}, next_page_token

endpoints/appr/models_cnr.py

@@ -10,6 +10,7 @@ import data.model
 
 from app import app, storage, authentication, model_cache
 from data import appr_model
+from data import model as data_model
 from data.cache import cache_key
 from data.database import Repository, MediaType, db_transaction
 from data.appr_model.models import NEW_MODELS
@@ -173,7 +174,7 @@ class CNRAppModel(AppRegistryDataInterface):
             view = ApplicationSummaryView(
                 namespace=repo.namespace_user.username,
                 name=app_name,
-                visibility=repo.visibility.name,
+                visibility=data_model.repository.repository_visibility_name(repo),
                 default=available_releases[0],
                 channels=channels,
                 manifests=manifests,

endpoints/secscan.py

@@ -1,33 +1,12 @@
 import logging
-import json
 
-import features
+from flask import make_response, Blueprint
-
+from endpoints.decorators import anon_allowed
-from app import secscan_notification_queue
-from flask import request, make_response, Blueprint, abort
-from endpoints.decorators import route_show_if, anon_allowed
 
 logger = logging.getLogger(__name__)
 secscan = Blueprint("secscan", __name__)
 
 
-@route_show_if(features.SECURITY_SCANNER)
-@secscan.route("/notify", methods=["POST"])
-def secscan_notification():
-    data = request.get_json()
-    logger.debug("Got notification from Security Scanner: %s", data)
-    if "Notification" not in data:
-        abort(400)
-
-    notification = data["Notification"]
-    name = ["named", notification["Name"]]
-
-    if not secscan_notification_queue.alive(name):
-        secscan_notification_queue.put(name, json.dumps(notification))
-
-    return make_response("Okay")
-
-
 @secscan.route("/_internal_ping")
 @anon_allowed
 def internal_ping():

endpoints/test/test_anon_checked.py

@@ -3,10 +3,9 @@ import pytest
 from app import app
 from endpoints.v1 import v1_bp
 from endpoints.v2 import v2_bp
-from endpoints.verbs import verbs
 
 
-@pytest.mark.parametrize("blueprint", [v2_bp, v1_bp, verbs,])
+@pytest.mark.parametrize("blueprint", [v2_bp, v1_bp,])
 def test_verify_blueprint(blueprint):
     class Checker(object):
         def __init__(self):

endpoints/v1/registry.py

@@ -40,18 +40,7 @@ def require_completion(f):
 
     @wraps(f)
     def wrapper(namespace, repository, *args, **kwargs):
-        image_id = kwargs["image_id"]
+        # TODO: Remove this
-        repository_ref = registry_model.lookup_repository(namespace, repository)
-        if repository_ref is not None:
-            legacy_image = registry_model.get_legacy_image(repository_ref, image_id)
-            if legacy_image is not None and legacy_image.uploading:
-                abort(
-                    400,
-                    "Image %(image_id)s is being uploaded, retry later",
-                    issue="upload-in-progress",
-                    image_id=image_id,
-                )
-
         return f(namespace, repository, *args, **kwargs)
 
     return wrapper
@@ -102,7 +91,9 @@ def head_image_layer(namespace, repository, image_id, headers):
             abort(404)
 
         logger.debug("Looking up placement locations")
-        legacy_image = registry_model.get_legacy_image(repository_ref, image_id, include_blob=True)
+        legacy_image = registry_model.get_legacy_image(
+            repository_ref, image_id, store, include_blob=True
+        )
         if legacy_image is None:
             logger.debug("Could not find any blob placement locations")
             abort(404, "Image %(image_id)s not found", issue="unknown-image", image_id=image_id)
@@ -139,7 +130,9 @@ def get_image_layer(namespace, repository, image_id, headers):
         if repository_ref is None:
             abort(404)
 
-        legacy_image = registry_model.get_legacy_image(repository_ref, image_id, include_blob=True)
+        legacy_image = registry_model.get_legacy_image(
+            repository_ref, image_id, store, include_blob=True
+        )
         if legacy_image is None:
             abort(404, "Image %(image_id)s not found", issue="unknown-image", image_id=image_id)
 
@@ -351,7 +344,9 @@ def get_image_json(namespace, repository, image_id, headers):
         abort(403)
 
     logger.debug("Looking up repo image")
-    legacy_image = registry_model.get_legacy_image(repository_ref, image_id, include_blob=True)
+    legacy_image = registry_model.get_legacy_image(
+        repository_ref, image_id, store, include_blob=True
+    )
     if legacy_image is None:
         flask_abort(404)
 
@@ -381,15 +376,12 @@ def get_image_ancestry(namespace, repository, image_id, headers):
         abort(403)
 
     logger.debug("Looking up repo image")
-    legacy_image = registry_model.get_legacy_image(repository_ref, image_id, include_parents=True)
+    legacy_image = registry_model.get_legacy_image(repository_ref, image_id, store)
     if legacy_image is None:
         abort(404, "Image %(image_id)s not found", issue="unknown-image", image_id=image_id)
 
     # NOTE: We can not use jsonify here because we are returning a list not an object.
-    ancestor_ids = [legacy_image.docker_image_id] + [
+    response = make_response(json.dumps(legacy_image.full_image_id_chain), 200)
-        a.docker_image_id for a in legacy_image.parents
-    ]
-    response = make_response(json.dumps(ancestor_ids), 200)
     response.headers.extend(headers)
     return response
 

endpoints/v1/tag.py

@@ -98,7 +98,7 @@ def put_tag(namespace_name, repo_name, tag):
 
         # Check if there is an existing image we should use (for PUT calls outside of a normal push
         # operation).
-        legacy_image = registry_model.get_legacy_image(repository_ref, image_id)
+        legacy_image = registry_model.get_legacy_image(repository_ref, image_id, storage)
         if legacy_image is None:
             abort(400)
 

endpoints/v2/manifest.py

@@ -68,7 +68,7 @@ def fetch_manifest_by_tagname(namespace_name, repo_name, manifest_ref):
         image_pulls.labels("v2", "tag", 404).inc()
         raise ManifestUnknown()
 
-    manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True)
+    manifest = registry_model.get_manifest_for_tag(tag)
     if manifest is None:
         # Something went wrong.
         image_pulls.labels("v2", "tag", 400).inc()

endpoints/v2/test/test_blob.py

@@ -129,12 +129,13 @@ def test_blob_mounting(mount_digest, source_repo, username, expect_success, clie
         headers=headers,
     )
 
+    repository = model.repository.get_repository("devtable", "building")
+
     if expect_success:
         # Ensure the blob now exists under the repo.
-        model.blob.get_repo_blob_by_digest("devtable", "building", mount_digest)
+        assert model.oci.blob.get_repository_blob_by_digest(repository, mount_digest)
     else:
-        with pytest.raises(model.blob.BlobDoesNotExist):
+        assert model.oci.blob.get_repository_blob_by_digest(repository, mount_digest) is None
-            model.blob.get_repo_blob_by_digest("devtable", "building", mount_digest)
 
 
 def test_blob_upload_offset(client, app):

endpoints/v2/test/test_manifest_cornercases.py

@@ -31,6 +31,23 @@ def _perform_cleanup():
     model.gc.garbage_collect_repo(repo_object)
 
 
+def _get_legacy_image_row_id(tag):
+    return (
+        database.ManifestLegacyImage.select(database.ManifestLegacyImage, database.Image)
+        .join(database.Image)
+        .where(database.ManifestLegacyImage.manifest == tag.manifest._db_id)
+        .get()
+        .image.docker_image_id
+    )
+
+
+def _add_legacy_image(namespace, repo_name, tag_name):
+    repo_ref = registry_model.lookup_repository(namespace, repo_name)
+    tag_ref = registry_model.get_repo_tag(repo_ref, tag_name)
+    manifest_ref = registry_model.get_manifest_for_tag(tag_ref)
+    registry_model.populate_legacy_images_for_testing(manifest_ref, storage)
+
+
 def test_missing_link(initialized_db):
     """
     Tests for a corner case that could result in missing a link to a blob referenced by a manifest.
@@ -54,6 +71,8 @@ def test_missing_link(initialized_db):
                 that of `SECOND_ID`, leaving `THIRD_ID` unlinked and therefore, after a GC, missing
                 `FOURTH_BLOB`.
     """
+    # TODO: Remove this test once we stop writing legacy image rows.
+
     with set_tag_expiration_policy("devtable", 0):
         location_name = storage.preferred_locations[0]
         location = database.ImageStorageLocation.get(name=location_name)
@@ -72,21 +91,19 @@ def test_missing_link(initialized_db):
         )
 
         _write_manifest(ADMIN_ACCESS_USER, REPO, FIRST_TAG, first_manifest)
+        _add_legacy_image(ADMIN_ACCESS_USER, REPO, FIRST_TAG)
 
         # Delete all temp tags and perform GC.
         _perform_cleanup()
 
         # Ensure that the first blob still exists, along with the first tag.
-        assert (
+        repo = model.repository.get_repository(ADMIN_ACCESS_USER, REPO)
-            model.blob.get_repo_blob_by_digest(ADMIN_ACCESS_USER, REPO, first_blob_sha) is not None
+        assert model.oci.blob.get_repository_blob_by_digest(repo, first_blob_sha) is not None
-        )
 
         repository_ref = registry_model.lookup_repository(ADMIN_ACCESS_USER, REPO)
-        found_tag = registry_model.get_repo_tag(
+        found_tag = registry_model.get_repo_tag(repository_ref, FIRST_TAG)
-            repository_ref, FIRST_TAG, include_legacy_image=True
-        )
         assert found_tag is not None
-        assert found_tag.legacy_image.docker_image_id == "first"
+        assert _get_legacy_image_row_id(found_tag) == "first"
 
         # Create the second and third blobs.
         second_blob_sha = "sha256:" + hashlib.sha256(b"SECOND").hexdigest()
@@ -108,6 +125,7 @@ def test_missing_link(initialized_db):
         )
 
         _write_manifest(ADMIN_ACCESS_USER, REPO, SECOND_TAG, second_manifest)
+        _add_legacy_image(ADMIN_ACCESS_USER, REPO, SECOND_TAG)
 
         # Delete all temp tags and perform GC.
         _perform_cleanup()
@@ -117,18 +135,14 @@ def test_missing_link(initialized_db):
         assert registry_model.get_repo_blob_by_digest(repository_ref, second_blob_sha) is not None
         assert registry_model.get_repo_blob_by_digest(repository_ref, third_blob_sha) is not None
 
-        found_tag = registry_model.get_repo_tag(
+        found_tag = registry_model.get_repo_tag(repository_ref, FIRST_TAG)
-            repository_ref, FIRST_TAG, include_legacy_image=True
-        )
         assert found_tag is not None
-        assert found_tag.legacy_image.docker_image_id == "first"
+        assert _get_legacy_image_row_id(found_tag) == "first"
 
         # Ensure the IDs have changed.
-        found_tag = registry_model.get_repo_tag(
+        found_tag = registry_model.get_repo_tag(repository_ref, SECOND_TAG)
-            repository_ref, SECOND_TAG, include_legacy_image=True
-        )
         assert found_tag is not None
-        assert found_tag.legacy_image.docker_image_id != "second"
+        assert _get_legacy_image_row_id(found_tag) != "second"
 
         # Create the fourth blob.
         fourth_blob_sha = "sha256:" + hashlib.sha256(b"FOURTH").hexdigest()
@@ -147,6 +161,7 @@ def test_missing_link(initialized_db):
         )
 
         _write_manifest(ADMIN_ACCESS_USER, REPO, THIRD_TAG, third_manifest)
+        _add_legacy_image(ADMIN_ACCESS_USER, REPO, THIRD_TAG)
 
         # Delete all temp tags and perform GC.
         _perform_cleanup()
@@ -158,10 +173,6 @@ def test_missing_link(initialized_db):
         assert registry_model.get_repo_blob_by_digest(repository_ref, fourth_blob_sha) is not None
 
         # Ensure new synthesized IDs were created.
-        second_tag = registry_model.get_repo_tag(
+        second_tag = registry_model.get_repo_tag(repository_ref, SECOND_TAG)
-            repository_ref, SECOND_TAG, include_legacy_image=True
+        third_tag = registry_model.get_repo_tag(repository_ref, THIRD_TAG)
-        )
+        assert _get_legacy_image_row_id(second_tag) != _get_legacy_image_row_id(third_tag)
-        third_tag = registry_model.get_repo_tag(
-            repository_ref, THIRD_TAG, include_legacy_image=True
-        )
-        assert second_tag.legacy_image.docker_image_id != third_tag.legacy_image.docker_image_id

endpoints/verbs/__init__.py

@@ -1,535 +0,0 @@
-import hashlib
-import json
-import logging
-import uuid
-
-from functools import wraps
-
-from flask import redirect, Blueprint, abort, send_file, make_response, request
-from prometheus_client import Counter
-
-import features
-
-from app import app, signer, storage, config_provider, ip_resolver, instance_keys
-from auth.auth_context import get_authenticated_user
-from auth.decorators import process_auth
-from auth.permissions import ReadRepositoryPermission
-from data import database
-from data import model
-from data.registry_model import registry_model
-from endpoints.decorators import (
-    anon_protect,
-    anon_allowed,
-    route_show_if,
-    parse_repository_name,
-    check_region_blacklisted,
-)
-from endpoints.metrics import image_pulls, image_pulled_bytes
-from endpoints.v2.blob import BLOB_DIGEST_ROUTE
-from image.appc import AppCImageFormatter
-from image.shared import ManifestException
-from image.docker.squashed import SquashedDockerImageFormatter
-from storage import Storage
-from util.audit import track_and_log, wrap_repository
-from util.http import exact_abort
-from util.metrics.prometheus import timed_blueprint
-from util.registry.filelike import wrap_with_handler
-from util.registry.queuefile import QueueFile
-from util.registry.queueprocess import QueueProcess
-from util.registry.tarlayerformat import TarLayerFormatterReporter
-
-
-logger = logging.getLogger(__name__)
-verbs = timed_blueprint(Blueprint("verbs", __name__))
-
-
-verb_stream_passes = Counter(
-    "quay_verb_stream_passes_total",
-    "number of passes over a tar stream used by verb requests",
-    labelnames=["kind"],
-)
-
-
-LAYER_MIMETYPE = "binary/octet-stream"
-QUEUE_FILE_TIMEOUT = 15  # seconds
-
-
-class VerbReporter(TarLayerFormatterReporter):
-    def __init__(self, kind):
-        self.kind = kind
-
-    def report_pass(self, pass_count):
-        if pass_count:
-            verb_stream_passes.labels(self.kind).inc(pass_count)
-
-
-def _open_stream(formatter, tag, schema1_manifest, derived_image_id, handlers, reporter):
-    """
-    This method generates a stream of data which will be replicated and read from the queue files.
-
-    This method runs in a separate process.
-    """
-    # For performance reasons, we load the full image list here, cache it, then disconnect from
-    # the database.
-    with database.UseThenDisconnect(app.config):
-        layers = registry_model.list_parsed_manifest_layers(
-            tag.repository, schema1_manifest, storage, include_placements=True
-        )
-
-    def image_stream_getter(store, blob):
-        def get_stream_for_storage():
-            current_image_stream = store.stream_read_file(blob.placements, blob.storage_path)
-            logger.debug("Returning blob %s: %s", blob.digest, blob.storage_path)
-            return current_image_stream
-
-        return get_stream_for_storage
-
-    def tar_stream_getter_iterator():
-        # Re-Initialize the storage engine because some may not respond well to forking (e.g. S3)
-        store = Storage(app, config_provider=config_provider, ip_resolver=ip_resolver)
-
-        # Note: We reverse because we have to start at the leaf layer and move upward,
-        # as per the spec for the formatters.
-        for layer in reversed(layers):
-            yield image_stream_getter(store, layer.blob)
-
-    stream = formatter.build_stream(
-        tag,
-        schema1_manifest,
-        derived_image_id,
-        layers,
-        tar_stream_getter_iterator,
-        reporter=reporter,
-    )
-
-    for handler_fn in handlers:
-        stream = wrap_with_handler(stream, handler_fn)
-
-    return stream.read
-
-
-def _sign_derived_image(verb, derived_image, queue_file):
-    """
-    Read from the queue file and sign the contents which are generated.
-
-    This method runs in a separate process.
-    """
-    signature = None
-    try:
-        signature = signer.detached_sign(queue_file)
-    except Exception as e:
-        logger.exception(
-            "Exception when signing %s deriving image %s: $s", verb, derived_image, str(e)
-        )
-        return
-
-    # Setup the database (since this is a new process) and then disconnect immediately
-    # once the operation completes.
-    if not queue_file.raised_exception:
-        with database.UseThenDisconnect(app.config):
-            registry_model.set_derived_image_signature(derived_image, signer.name, signature)
-
-
-def _write_derived_image_to_storage(
-    verb, derived_image, queue_file, namespace, repository, tag_name
-):
-    """
-    Read from the generated stream and write it back to the storage engine.
-
-    This method runs in a separate process.
-    """
-
-    def handle_exception(ex):
-        logger.debug(
-            "Exception when building %s derived image %s (%s/%s:%s): %s",
-            verb,
-            derived_image,
-            namespace,
-            repository,
-            tag_name,
-            ex,
-        )
-
-        with database.UseThenDisconnect(app.config):
-            registry_model.delete_derived_image(derived_image)
-
-    queue_file.add_exception_handler(handle_exception)
-
-    # Re-Initialize the storage engine because some may not respond well to forking (e.g. S3)
-    store = Storage(app, config_provider=config_provider, ip_resolver=ip_resolver)
-
-    try:
-        store.stream_write(
-            derived_image.blob.placements, derived_image.blob.storage_path, queue_file
-        )
-    except IOError as ex:
-        logger.error(
-            "Exception when writing %s derived image %s (%s/%s:%s): %s",
-            verb,
-            derived_image,
-            namespace,
-            repository,
-            tag_name,
-            ex,
-        )
-
-        with database.UseThenDisconnect(app.config):
-            registry_model.delete_derived_image(derived_image)
-
-    queue_file.close()
-
-
-def _verify_repo_verb(_, namespace, repo_name, tag_name, verb, checker=None):
-    permission = ReadRepositoryPermission(namespace, repo_name)
-    repo = model.repository.get_repository(namespace, repo_name)
-    repo_is_public = repo is not None and model.repository.is_repository_public(repo)
-    if not permission.can() and not repo_is_public:
-        logger.debug(
-            "No permission to read repository %s/%s for user %s with verb %s",
-            namespace,
-            repo_name,
-            get_authenticated_user(),
-            verb,
-        )
-        abort(403)
-
-    if repo is not None and repo.kind.name != "image":
-        logger.debug(
-            "Repository %s/%s for user %s is not an image repo",
-            namespace,
-            repo_name,
-            get_authenticated_user(),
-        )
-        abort(405)
-
-    # Make sure the repo's namespace isn't disabled.
-    if not registry_model.is_namespace_enabled(namespace):
-        abort(400)
-
-    # Lookup the requested tag.
-    repo_ref = registry_model.lookup_repository(namespace, repo_name)
-    if repo_ref is None:
-        abort(404)
-
-    tag = registry_model.get_repo_tag(repo_ref, tag_name)
-    if tag is None:
-        logger.debug(
-            "Tag %s does not exist in repository %s/%s for user %s",
-            tag,
-            namespace,
-            repo_name,
-            get_authenticated_user(),
-        )
-        abort(404)
-
-    # Get its associated manifest.
-    manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True)
-    if manifest is None:
-        logger.debug("Could not get manifest on %s/%s:%s::%s", namespace, repo_name, tag.name, verb)
-        abort(404)
-
-    # Retrieve the schema1-compatible version of the manifest.
-    try:
-        schema1_manifest = registry_model.get_schema1_parsed_manifest(
-            manifest, namespace, repo_name, tag.name, storage
-        )
-    except ManifestException:
-        logger.exception(
-            "Could not get manifest on %s/%s:%s::%s", namespace, repo_name, tag.name, verb
-        )
-        abort(400)
-
-    if schema1_manifest is None:
-        abort(404)
-
-    # If there is a data checker, call it first.
-    if checker is not None:
-        if not checker(tag, schema1_manifest):
-            logger.debug(
-                "Check mismatch on %s/%s:%s, verb %s", namespace, repo_name, tag.name, verb
-            )
-            abort(404)
-
-    # Preload the tag's repository information, so it gets cached.
-    assert tag.repository.namespace_name
-    assert tag.repository.name
-
-    return tag, manifest, schema1_manifest
-
-
-def _repo_verb_signature(namespace, repository, tag_name, verb, checker=None, **kwargs):
-    # Verify that the tag exists and that we have access to it.
-    tag, manifest, _ = _verify_repo_verb(storage, namespace, repository, tag_name, verb, checker)
-
-    # Find the derived image storage for the verb.
-    derived_image = registry_model.lookup_derived_image(
-        manifest, verb, storage, varying_metadata={"tag": tag.name}
-    )
-
-    if derived_image is None or derived_image.blob.uploading:
-        return make_response("", 202)
-
-    # Check if we have a valid signer configured.
-    if not signer.name:
-        abort(404)
-
-    # Lookup the signature for the verb.
-    signature_value = registry_model.get_derived_image_signature(derived_image, signer.name)
-    if signature_value is None:
-        abort(404)
-
-    # Return the signature.
-    return make_response(signature_value)
-
-
-class SimpleHasher(object):
-    def __init__(self):
-        self._current_offset = 0
-
-    def update(self, buf):
-        self._current_offset += len(buf)
-
-    @property
-    def hashed_bytes(self):
-        return self._current_offset
-
-
-@check_region_blacklisted()
-def _repo_verb(
-    namespace, repository, tag_name, verb, formatter, sign=False, checker=None, **kwargs
-):
-    # Verify that the image exists and that we have access to it.
-    logger.debug(
-        "Verifying repo verb %s for repository %s/%s with user %s with mimetype %s",
-        verb,
-        namespace,
-        repository,
-        get_authenticated_user(),
-        request.accept_mimetypes.best,
-    )
-    tag, manifest, schema1_manifest = _verify_repo_verb(
-        storage, namespace, repository, tag_name, verb, checker
-    )
-
-    # Load the repository for later.
-    repo = model.repository.get_repository(namespace, repository)
-    if repo is None:
-        abort(404)
-
-    # Check for torrent, which is no longer supported.
-    if request.accept_mimetypes.best == "application/x-bittorrent":
-        abort(406)
-
-    # Log the action.
-    track_and_log("repo_verb", wrap_repository(repo), tag=tag.name, verb=verb, **kwargs)
-
-    is_readonly = app.config.get("REGISTRY_STATE", "normal") == "readonly"
-
-    # Lookup/create the derived image for the verb and repo image.
-    if is_readonly:
-        derived_image = registry_model.lookup_derived_image(
-            manifest, verb, storage, varying_metadata={"tag": tag.name}, include_placements=True
-        )
-    else:
-        derived_image = registry_model.lookup_or_create_derived_image(
-            manifest,
-            verb,
-            storage.preferred_locations[0],
-            storage,
-            varying_metadata={"tag": tag.name},
-            include_placements=True,
-        )
-        if derived_image is None:
-            logger.error("Could not create or lookup a derived image for manifest %s", manifest)
-            abort(400)
-
-    if derived_image is not None and not derived_image.blob.uploading:
-        logger.debug("Derived %s image %s exists in storage", verb, derived_image)
-        is_head_request = request.method == "HEAD"
-
-        if derived_image.blob.compressed_size:
-            image_pulled_bytes.labels("verbs").inc(derived_image.blob.compressed_size)
-
-        download_url = storage.get_direct_download_url(
-            derived_image.blob.placements, derived_image.blob.storage_path, head=is_head_request
-        )
-        if download_url:
-            logger.debug("Redirecting to download URL for derived %s image %s", verb, derived_image)
-            return redirect(download_url)
-
-        # Close the database handle here for this process before we send the long download.
-        database.close_db_filter(None)
-
-        logger.debug("Sending cached derived %s image %s", verb, derived_image)
-        return send_file(
-            storage.stream_read_file(
-                derived_image.blob.placements, derived_image.blob.storage_path
-            ),
-            mimetype=LAYER_MIMETYPE,
-        )
-
-    logger.debug("Building and returning derived %s image", verb)
-    hasher = SimpleHasher()
-
-    # Close the database connection before any process forking occurs. This is important because
-    # the Postgres driver does not react kindly to forking, so we need to make sure it is closed
-    # so that each process will get its own unique connection.
-    database.close_db_filter(None)
-
-    def _cleanup():
-        # Close any existing DB connection once the process has exited.
-        database.close_db_filter(None)
-
-    def _store_metadata_and_cleanup():
-        if is_readonly:
-            return
-
-        with database.UseThenDisconnect(app.config):
-            registry_model.set_derived_image_size(derived_image, hasher.hashed_bytes)
-
-    # Create a queue process to generate the data. The queue files will read from the process
-    # and send the results to the client and storage.
-    unique_id = (
-        derived_image.unique_id
-        if derived_image is not None
-        else hashlib.sha256(("%s:%s" % (verb, uuid.uuid4())).encode("utf-8")).hexdigest()
-    )
-    handlers = [hasher.update]
-    reporter = VerbReporter(verb)
-    args = (formatter, tag, schema1_manifest, unique_id, handlers, reporter)
-    queue_process = QueueProcess(
-        _open_stream,
-        8 * 1024,
-        10 * 1024 * 1024,  # 8K/10M chunk/max
-        args,
-        finished=_store_metadata_and_cleanup,
-    )
-
-    client_queue_file = QueueFile(
-        queue_process.create_queue(), "client", timeout=QUEUE_FILE_TIMEOUT
-    )
-
-    if not is_readonly:
-        storage_queue_file = QueueFile(
-            queue_process.create_queue(), "storage", timeout=QUEUE_FILE_TIMEOUT
-        )
-
-        # If signing is required, add a QueueFile for signing the image as we stream it out.
-        signing_queue_file = None
-        if sign and signer.name:
-            signing_queue_file = QueueFile(
-                queue_process.create_queue(), "signing", timeout=QUEUE_FILE_TIMEOUT
-            )
-
-    # Start building.
-    queue_process.run()
-
-    # Start the storage saving.
-    if not is_readonly:
-        storage_args = (verb, derived_image, storage_queue_file, namespace, repository, tag_name)
-        QueueProcess.run_process(_write_derived_image_to_storage, storage_args, finished=_cleanup)
-
-        if sign and signer.name:
-            signing_args = (verb, derived_image, signing_queue_file)
-            QueueProcess.run_process(_sign_derived_image, signing_args, finished=_cleanup)
-
-    # Close the database handle here for this process before we send the long download.
-    database.close_db_filter(None)
-
-    # Return the client's data.
-    return send_file(client_queue_file, mimetype=LAYER_MIMETYPE)
-
-
-def os_arch_checker(os, arch):
-    def checker(tag, manifest):
-        try:
-            image_json = json.loads(manifest.leaf_layer.raw_v1_metadata)
-        except ValueError:
-            logger.exception("Could not parse leaf layer JSON for manifest %s", manifest)
-            return False
-        except TypeError:
-            logger.exception("Could not parse leaf layer JSON for manifest %s", manifest)
-            return False
-
-        # Verify the architecture and os.
-        operating_system = image_json.get("os", "linux")
-        if operating_system != os:
-            return False
-
-        architecture = image_json.get("architecture", "amd64")
-
-        # Note: Some older Docker images have 'x86_64' rather than 'amd64'.
-        # We allow the conversion here.
-        if architecture == "x86_64" and operating_system == "linux":
-            architecture = "amd64"
-
-        if architecture != arch:
-            return False
-
-        return True
-
-    return checker
-
-
-def observe_route(protocol):
-    """
-    Decorates verb endpoints to record the image_pulls metric into Prometheus.
-    """
-
-    def decorator(func):
-        @wraps(func)
-        def wrapper(*args, **kwargs):
-            rv = func(*args, **kwargs)
-            image_pulls.labels(protocol, "tag", rv.status_code)
-            return rv
-
-        return wrapper
-
-    return decorator
-
-
-@route_show_if(features.ACI_CONVERSION)
-@anon_protect
-@verbs.route("/aci/<server>/<namespace>/<repository>/<tag>/sig/<os>/<arch>/", methods=["GET"])
-@verbs.route("/aci/<server>/<namespace>/<repository>/<tag>/aci.asc/<os>/<arch>/", methods=["GET"])
-@observe_route("aci")
-@process_auth
-def get_aci_signature(server, namespace, repository, tag, os, arch):
-    return _repo_verb_signature(
-        namespace, repository, tag, "aci", checker=os_arch_checker(os, arch), os=os, arch=arch
-    )
-
-
-@route_show_if(features.ACI_CONVERSION)
-@anon_protect
-@verbs.route(
-    "/aci/<server>/<namespace>/<repository>/<tag>/aci/<os>/<arch>/", methods=["GET", "HEAD"]
-)
-@observe_route("aci")
-@process_auth
-def get_aci_image(server, namespace, repository, tag, os, arch):
-    return _repo_verb(
-        namespace,
-        repository,
-        tag,
-        "aci",
-        AppCImageFormatter(),
-        sign=True,
-        checker=os_arch_checker(os, arch),
-        os=os,
-        arch=arch,
-    )
-
-
-@anon_protect
-@verbs.route("/squash/<namespace>/<repository>/<tag>", methods=["GET"])
-@observe_route("squash")
-@process_auth
-def get_squashed_tag(namespace, repository, tag):
-    return _repo_verb(namespace, repository, tag, "squash", SquashedDockerImageFormatter())
-
-
-@verbs.route("/_internal_ping")
-@anon_allowed
-def internal_ping():
-    return make_response("true", 200)

endpoints/verbs/test/test_security.py

@@ -1,97 +0,0 @@
-import pytest
-
-from flask import url_for
-from endpoints.test.shared import conduct_call, gen_basic_auth
-from test.fixtures import *
-
-NO_ACCESS_USER = "freshuser"
-READ_ACCESS_USER = "reader"
-ADMIN_ACCESS_USER = "devtable"
-CREATOR_ACCESS_USER = "creator"
-
-PUBLIC_REPO = "public/publicrepo"
-PRIVATE_REPO = "devtable/shared"
-ORG_REPO = "buynlarge/orgrepo"
-ANOTHER_ORG_REPO = "buynlarge/anotherorgrepo"
-
-ACI_ARGS = {
-    "server": "someserver",
-    "tag": "fake",
-    "os": "linux",
-    "arch": "x64",
-}
-
-
-@pytest.mark.parametrize(
-    "user",
-    [
-        (0, None),
-        (1, NO_ACCESS_USER),
-        (2, READ_ACCESS_USER),
-        (3, CREATOR_ACCESS_USER),
-        (4, ADMIN_ACCESS_USER),
-    ],
-)
-@pytest.mark.parametrize(
-    "endpoint,method,repository,single_repo_path,params,expected_statuses",
-    [
-        ("get_aci_signature", "GET", PUBLIC_REPO, False, ACI_ARGS, (404, 404, 404, 404, 404)),
-        ("get_aci_signature", "GET", PRIVATE_REPO, False, ACI_ARGS, (403, 403, 404, 403, 404)),
-        ("get_aci_signature", "GET", ORG_REPO, False, ACI_ARGS, (403, 403, 404, 403, 404)),
-        ("get_aci_signature", "GET", ANOTHER_ORG_REPO, False, ACI_ARGS, (403, 403, 403, 403, 404)),
-        # get_aci_image
-        ("get_aci_image", "GET", PUBLIC_REPO, False, ACI_ARGS, (404, 404, 404, 404, 404)),
-        ("get_aci_image", "GET", PRIVATE_REPO, False, ACI_ARGS, (403, 403, 404, 403, 404)),
-        ("get_aci_image", "GET", ORG_REPO, False, ACI_ARGS, (403, 403, 404, 403, 404)),
-        ("get_aci_image", "GET", ANOTHER_ORG_REPO, False, ACI_ARGS, (403, 403, 403, 403, 404)),
-        # get_squashed_tag
-        (
-            "get_squashed_tag",
-            "GET",
-            PUBLIC_REPO,
-            False,
-            dict(tag="fake"),
-            (404, 404, 404, 404, 404),
-        ),
-        (
-            "get_squashed_tag",
-            "GET",
-            PRIVATE_REPO,
-            False,
-            dict(tag="fake"),
-            (403, 403, 404, 403, 404),
-        ),
-        ("get_squashed_tag", "GET", ORG_REPO, False, dict(tag="fake"), (403, 403, 404, 403, 404)),
-        (
-            "get_squashed_tag",
-            "GET",
-            ANOTHER_ORG_REPO,
-            False,
-            dict(tag="fake"),
-            (403, 403, 403, 403, 404),
-        ),
-    ],
-)
-def test_verbs_security(
-    user, endpoint, method, repository, single_repo_path, params, expected_statuses, app, client
-):
-    headers = {}
-    if user[1] is not None:
-        headers["Authorization"] = gen_basic_auth(user[1], "password")
-
-    if single_repo_path:
-        params["repository"] = repository
-    else:
-        (namespace, repo_name) = repository.split("/")
-        params["namespace"] = namespace
-        params["repository"] = repo_name
-
-    conduct_call(
-        client,
-        "verbs." + endpoint,
-        url_for,
-        method,
-        params,
-        expected_code=expected_statuses[user[0]],
-        headers=headers,
-    )

endpoints/web.py

@@ -27,7 +27,6 @@ from app import (
     billing as stripe,
     build_logs,
     avatar,
-    signer,
     log_archive,
     config_provider,
     get_app_url,
@@ -144,17 +143,6 @@ def user_view(path):
     return index("")
 
 
-@route_show_if(features.ACI_CONVERSION)
-@web.route("/aci-signing-key")
-@no_cache
-@anon_protect
-def aci_signing_key():
-    if not signer.name:
-        abort(404)
-
-    return send_file(signer.open_public_key_file(), mimetype=PGP_KEY_MIMETYPE)
-
-
 @web.route("/plans/")
 @no_cache
 @route_show_if(features.BILLING)

health/services.py

@@ -178,7 +178,6 @@ def _check_disk_space(for_warning):
 _INSTANCE_SERVICES = {
     "registry_gunicorn": _check_gunicorn("v1/_internal_ping"),
     "web_gunicorn": _check_gunicorn("_internal_ping"),
-    "verbs_gunicorn": _check_gunicorn("c1/_internal_ping"),
     "service_key": _check_service_key,
     "disk_space": _check_disk_space(for_warning=False),
     "jwtproxy": _check_jwt_proxy,

image/appc/__init__.py

@@ -1,227 +0,0 @@
-import json
-import re
-import calendar
-
-from uuid import uuid4
-
-from app import app
-from util.registry.streamlayerformat import StreamLayerMerger
-from util.dict_wrappers import JSONPathDict
-from image.common import TarImageFormatter
-
-
-ACNAME_REGEX = re.compile(r"[^a-z-]+")
-
-
-class AppCImageFormatter(TarImageFormatter):
-    """
-    Image formatter which produces an tarball according to the AppC specification.
-    """
-
-    def stream_generator(
-        self,
-        tag,
-        parsed_manifest,
-        synthetic_image_id,
-        layer_iterator,
-        tar_stream_getter_iterator,
-        reporter=None,
-    ):
-        image_mtime = 0
-        created = parsed_manifest.created_datetime
-        if created is not None:
-            image_mtime = calendar.timegm(created.utctimetuple())
-
-        # ACI Format (.tar):
-        #   manifest - The JSON manifest
-        #   rootfs - The root file system
-
-        # Yield the manifest.
-        aci_manifest = json.dumps(
-            DockerV1ToACIManifestTranslator.build_manifest(tag, parsed_manifest, synthetic_image_id)
-        )
-        yield self.tar_file("manifest", aci_manifest.encode("utf-8"), mtime=image_mtime)
-
-        # Yield the merged layer dtaa.
-        yield self.tar_folder("rootfs", mtime=image_mtime)
-
-        layer_merger = StreamLayerMerger(
-            tar_stream_getter_iterator, path_prefix="rootfs/", reporter=reporter
-        )
-        for entry in layer_merger.get_generator():
-            yield entry
-
-
-class DockerV1ToACIManifestTranslator(object):
-    @staticmethod
-    def _build_isolators(docker_config):
-        """
-        Builds ACI isolator config from the docker config.
-        """
-
-        def _isolate_memory(memory):
-            return {"name": "memory/limit", "value": {"request": str(memory) + "B",}}
-
-        def _isolate_swap(memory):
-            return {"name": "memory/swap", "value": {"request": str(memory) + "B",}}
-
-        def _isolate_cpu(cpu):
-            return {"name": "cpu/shares", "value": {"request": str(cpu),}}
-
-        def _isolate_capabilities(capabilities_set_value):
-            capabilities_set = re.split(r"[\s,]", capabilities_set_value)
-            return {"name": "os/linux/capabilities-retain-set", "value": {"set": capabilities_set,}}
-
-        mappers = {
-            "Memory": _isolate_memory,
-            "MemorySwap": _isolate_swap,
-            "CpuShares": _isolate_cpu,
-            "Cpuset": _isolate_capabilities,
-        }
-
-        isolators = []
-
-        for config_key in mappers:
-            value = docker_config.get(config_key)
-            if value:
-                isolators.append(mappers[config_key](value))
-
-        return isolators
-
-    @staticmethod
-    def _build_ports(docker_config):
-        """
-        Builds the ports definitions for the ACI.
-
-        Formats:
-          port/tcp
-          port/udp
-          port
-        """
-        ports = []
-
-        exposed_ports = docker_config["ExposedPorts"]
-        if exposed_ports is not None:
-            port_list = list(exposed_ports.keys())
-        else:
-            port_list = docker_config["Ports"] or docker_config["ports"] or []
-
-        for docker_port in port_list:
-            protocol = "tcp"
-            port_number = -1
-
-            if "/" in docker_port:
-                (port_number, protocol) = docker_port.split("/")
-            else:
-                port_number = docker_port
-
-            try:
-                port_number = int(port_number)
-                ports.append(
-                    {"name": "port-%s" % port_number, "port": port_number, "protocol": protocol,}
-                )
-            except ValueError:
-                pass
-
-        return ports
-
-    @staticmethod
-    def _ac_name(value):
-        sanitized = ACNAME_REGEX.sub("-", value.lower()).strip("-")
-        if sanitized == "":
-            return str(uuid4())
-        return sanitized
-
-    @staticmethod
-    def _build_volumes(docker_config):
-        """
-        Builds the volumes definitions for the ACI.
-        """
-        volumes = []
-
-        def get_name(docker_volume_path):
-            volume_name = DockerV1ToACIManifestTranslator._ac_name(docker_volume_path)
-            return "volume-%s" % volume_name
-
-        volume_list = docker_config["Volumes"] or docker_config["volumes"] or {}
-        for docker_volume_path in volume_list.keys():
-            if not docker_volume_path:
-                continue
-
-            volumes.append(
-                {
-                    "name": get_name(docker_volume_path),
-                    "path": docker_volume_path,
-                    "readOnly": False,
-                }
-            )
-        return volumes
-
-    @staticmethod
-    def build_manifest(tag, manifest, synthetic_image_id):
-        """
-        Builds an ACI manifest of an existing repository image.
-        """
-        docker_layer_data = JSONPathDict(json.loads(manifest.leaf_layer.raw_v1_metadata))
-        config = docker_layer_data["config"] or JSONPathDict({})
-
-        namespace = tag.repository.namespace_name
-        repo_name = tag.repository.name
-        source_url = "%s://%s/%s/%s:%s" % (
-            app.config["PREFERRED_URL_SCHEME"],
-            app.config["SERVER_HOSTNAME"],
-            namespace,
-            repo_name,
-            tag.name,
-        )
-
-        # ACI requires that the execution command be absolutely referenced. Therefore, if we find
-        # a relative command, we give it as an argument to /bin/sh to resolve and execute for us.
-        entrypoint = config["Entrypoint"] or []
-        exec_path = entrypoint + (config["Cmd"] or [])
-        if exec_path and not exec_path[0].startswith("/"):
-            exec_path = ["/bin/sh", "-c", '""%s""' % " ".join(exec_path)]
-
-        # TODO: ACI doesn't support : in the name, so remove any ports.
-        hostname = app.config["SERVER_HOSTNAME"]
-        hostname = hostname.split(":", 1)[0]
-
-        # Calculate the environment variables.
-        docker_env_vars = config.get("Env") or []
-        env_vars = []
-        for var in docker_env_vars:
-            pieces = var.split("=")
-            if len(pieces) != 2:
-                continue
-
-            env_vars.append(pieces)
-
-        manifest = {
-            "acKind": "ImageManifest",
-            "acVersion": "0.6.1",
-            "name": "%s/%s/%s" % (hostname.lower(), namespace.lower(), repo_name.lower()),
-            "labels": [
-                {"name": "version", "value": tag.name,},
-                {"name": "arch", "value": docker_layer_data.get("architecture") or "amd64"},
-                {"name": "os", "value": docker_layer_data.get("os") or "linux"},
-            ],
-            "app": {
-                "exec": exec_path,
-                # Below, `or 'root'` is required to replace empty string from Dockerfiles.
-                "user": config.get("User") or "root",
-                "group": config.get("Group") or "root",
-                "eventHandlers": [],
-                "workingDirectory": config.get("WorkingDir") or "/",
-                "environment": [{"name": key, "value": value} for (key, value) in env_vars],
-                "isolators": DockerV1ToACIManifestTranslator._build_isolators(config),
-                "mountPoints": DockerV1ToACIManifestTranslator._build_volumes(config),
-                "ports": DockerV1ToACIManifestTranslator._build_ports(config),
-                "annotations": [
-                    {"name": "created", "value": docker_layer_data.get("created") or ""},
-                    {"name": "homepage", "value": source_url},
-                    {"name": "quay.io/derived-image", "value": synthetic_image_id},
-                ],
-            },
-        }
-
-        return manifest

image/appc/test/test_appc.py

@@ -1,74 +0,0 @@
-import pytest
-
-from image.appc import DockerV1ToACIManifestTranslator
-from util.dict_wrappers import JSONPathDict
-
-
-EXAMPLE_MANIFEST_OBJ = {
-    "architecture": "amd64",
-    "config": {
-        "Hostname": "1d811a9194c4",
-        "Domainname": "",
-        "User": "",
-        "AttachStdin": False,
-        "AttachStdout": False,
-        "AttachStderr": False,
-        "ExposedPorts": {"2379/tcp": {}, "2380/tcp": {}},
-        "Tty": False,
-        "OpenStdin": False,
-        "StdinOnce": False,
-        "Env": ["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],
-        "Cmd": ["/usr/local/bin/etcd"],
-        "ArgsEscaped": True,
-        "Image": "sha256:4c86d1f362d42420c137846fae31667ee85ce6f2cab406cdff26a8ff8a2c31c4",
-        "Volumes": None,
-        "WorkingDir": "",
-        "Entrypoint": None,
-        "OnBuild": [],
-        "Labels": {},
-    },
-    "container": "5a3565ce9b808a0eb0bcbc966dad624f76ad308ad24e11525b5da1201a1df135",
-    "container_config": {
-        "Hostname": "1d811a9194c4",
-        "Domainname": "",
-        "User": "",
-        "AttachStdin": False,
-        "AttachStdout": False,
-        "AttachStderr": False,
-        "ExposedPorts": {"2379/tcp": {}, "2380/tcp": {}},
-        "Tty": False,
-        "OpenStdin": False,
-        "StdinOnce": False,
-        "Env": ["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],
-        "Cmd": ["/bin/sh", "-c", '#(nop) CMD ["/usr/local/bin/etcd"]'],
-        "ArgsEscaped": True,
-        "Image": "sha256:4c86d1f362d42420c137846fae31667ee85ce6f2cab406cdff26a8ff8a2c31c4",
-        "Volumes": None,
-        "WorkingDir": "",
-        "Entrypoint": None,
-        "OnBuild": [],
-        "Labels": {},
-    },
-    "created": "2016-11-11T19:03:55.137387628Z",
-    "docker_version": "1.11.1",
-    "id": "3314a3781a526fe728e2e96cfcfb3cc0de901b5c102e6204e8b0155c8f7d5fd2",
-    "os": "linux",
-    "parent": "625342ec4d0f3d7a96fd3bb1ef0b4b0b6bc65ebb3d252fd33af0691f7984440e",
-    "throwaway": True,
-}
-
-
-@pytest.mark.parametrize(
-    "vcfg,expected",
-    [
-        ({"Volumes": None}, []),
-        ({"Volumes": {}}, []),
-        ({"Volumes": {"/bin": {}}}, [{"name": "volume-bin", "path": "/bin", "readOnly": False}]),
-        ({"volumes": None}, []),
-        ({"volumes": {}}, []),
-        ({"volumes": {"/bin": {}}}, [{"name": "volume-bin", "path": "/bin", "readOnly": False}]),
-    ],
-)
-def test_volume_version_easy(vcfg, expected):
-    output = DockerV1ToACIManifestTranslator._build_volumes(JSONPathDict(vcfg))
-    assert output == expected

image/common.py

@@ -1,89 +0,0 @@
-import tarfile
-from util.registry.gzipwrap import GzipWrap
-
-
-class TarImageFormatter(object):
-    """
-    Base class for classes which produce a tar containing image and layer data.
-    """
-
-    def build_stream(
-        self,
-        tag,
-        manifest,
-        synthetic_image_id,
-        layer_iterator,
-        tar_stream_getter_iterator,
-        reporter=None,
-    ):
-        """
-        Builds and streams a synthetic .tar.gz that represents the formatted tar created by this
-        class's implementation.
-        """
-        return GzipWrap(
-            self.stream_generator(
-                tag,
-                manifest,
-                synthetic_image_id,
-                layer_iterator,
-                tar_stream_getter_iterator,
-                reporter=reporter,
-            )
-        )
-
-    def stream_generator(
-        self,
-        tag,
-        manifest,
-        synthetic_image_id,
-        layer_iterator,
-        tar_stream_getter_iterator,
-        reporter=None,
-    ):
-        raise NotImplementedError
-
-    def tar_file(self, name, contents, mtime=None):
-        """
-        Returns the tar binary representation for a file with the given name and file contents.
-        """
-        assert isinstance(contents, bytes)
-        length = len(contents)
-        tar_data = self.tar_file_header(name, length, mtime=mtime)
-        tar_data += contents
-        tar_data += self.tar_file_padding(length)
-        return tar_data
-
-    def tar_file_padding(self, length):
-        """
-        Returns tar file padding for file data of the given length.
-        """
-        if length % 512 != 0:
-            return b"\0" * (512 - (length % 512))
-
-        return b""
-
-    def tar_file_header(self, name, file_size, mtime=None):
-        """
-        Returns tar file header data for a file with the given name and size.
-        """
-        info = tarfile.TarInfo(name=name)
-        info.type = tarfile.REGTYPE
-        info.size = file_size
-
-        if mtime is not None:
-            info.mtime = mtime
-        return info.tobuf()
-
-    def tar_folder(self, name, mtime=None):
-        """
-        Returns tar file header data for a folder with the given name.
-        """
-        info = tarfile.TarInfo(name=name)
-        info.type = tarfile.DIRTYPE
-
-        if mtime is not None:
-            info.mtime = mtime
-
-        # allow the directory to be readable by non-root users
-        info.mode = 0o755
-        return info.tobuf()

image/docker/schema1.py

@@ -220,7 +220,17 @@ class DockerSchema1Manifest(ManifestInterface):
 
         Raises a ManifestException on failure.
         """
-        # Already validated.
+        # Validate the parent image IDs.
+        encountered_ids = set()
+        for layer in self.layers:
+            if layer.v1_metadata.parent_image_id:
+                if layer.v1_metadata.parent_image_id not in encountered_ids:
+                    raise ManifestException(
+                        "Unknown parent image %s" % layer.v1_metadata.parent_image_id
+                    )
+
+            if layer.v1_metadata.image_id:
+                encountered_ids.add(layer.v1_metadata.image_id)
 
     @property
     def is_signed(self):
@@ -283,6 +293,10 @@ class DockerSchema1Manifest(ManifestInterface):
 
     @property
     def layers_compressed_size(self):
+        return sum(l.compressed_size for l in self.layers if l.compressed_size is not None)
+
+    @property
+    def config_media_type(self):
         return None
 
     @property

image/docker/schema2/list.py

@@ -6,7 +6,7 @@ from jsonschema import validate as validate_schema, ValidationError
 
 from digest import digest_tools
 from image.shared import ManifestException
-from image.shared.interfaces import ManifestInterface
+from image.shared.interfaces import ManifestListInterface
 from image.shared.schemautil import LazyManifestLoader
 from image.docker.schema1 import DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE
 from image.docker.schema1 import DockerSchema1Manifest
@@ -53,7 +53,7 @@ class MismatchManifestException(MalformedSchema2ManifestList):
     pass
 
 
-class DockerSchema2ManifestList(ManifestInterface):
+class DockerSchema2ManifestList(ManifestListInterface):
     METASCHEMA = {
         "type": "object",
         "properties": {
@@ -228,6 +228,10 @@ class DockerSchema2ManifestList(ManifestInterface):
     def layers_compressed_size(self):
         return None
 
+    @property
+    def config_media_type(self):
+        return None
+
     @lru_cache(maxsize=1)
     def manifests(self, content_retriever):
         """
@@ -249,6 +253,20 @@ class DockerSchema2ManifestList(ManifestInterface):
             for m in manifests
         ]
 
+    @property
+    def amd64_linux_manifest_digest(self):
+        """ Returns the digest of the AMD64+Linux manifest in this list, if any, or None
+            if none.
+        """
+        for manifest_ref in self._parsed[DOCKER_SCHEMA2_MANIFESTLIST_MANIFESTS_KEY]:
+            platform = manifest_ref[DOCKER_SCHEMA2_MANIFESTLIST_PLATFORM_KEY]
+            architecture = platform[DOCKER_SCHEMA2_MANIFESTLIST_ARCHITECTURE_KEY]
+            os = platform[DOCKER_SCHEMA2_MANIFESTLIST_OS_KEY]
+            if architecture == "amd64" and os == "linux":
+                return manifest_ref[DOCKER_SCHEMA2_MANIFESTLIST_DIGEST_KEY]
+
+        return None
+
     def validate(self, content_retriever):
         """
         Performs validation of required assertions about the manifest.

image/docker/schema2/manifest.py

@@ -172,7 +172,7 @@ class DockerSchema2Manifest(ManifestInterface):
 
         Raises a ManifestException on failure.
         """
-        # Nothing to validate.
+        self._get_built_config(content_retriever)
 
     @property
     def is_manifest_list(self):
@@ -222,6 +222,12 @@ class DockerSchema2Manifest(ManifestInterface):
     def layers_compressed_size(self):
         return sum(layer.compressed_size for layer in self.filesystem_layers)
 
+    @property
+    def config_media_type(self):
+        return self._parsed[DOCKER_SCHEMA2_MANIFEST_CONFIG_KEY][
+            DOCKER_SCHEMA2_MANIFEST_MEDIATYPE_KEY
+        ]
+
     @property
     def has_remote_layer(self):
         for layer in self.filesystem_layers:

image/docker/schema2/test/test_list.py

@@ -50,7 +50,7 @@ MANIFESTLIST_BYTES = json.dumps(
             },
             {
                 "mediaType": "application/vnd.docker.distribution.manifest.v1+json",
-                "size": 878,
+                "size": 1051,
                 "digest": "sha256:5b",
                 "platform": {"architecture": "amd64", "os": "linux", "features": ["sse4"]},
             },
@@ -84,6 +84,8 @@ def test_valid_manifestlist():
     assert manifestlist.bytes.as_encoded_str() == MANIFESTLIST_BYTES
     assert manifestlist.manifest_dict == json.loads(MANIFESTLIST_BYTES)
     assert manifestlist.get_layers(retriever) is None
+    assert manifestlist.config_media_type is None
+    assert manifestlist.layers_compressed_size is None
     assert not manifestlist.blob_digests
 
     for index, manifest in enumerate(manifestlist.manifests(retriever)):
@@ -114,6 +116,8 @@ def test_valid_manifestlist():
     # Ensure it validates.
     manifestlist.validate(retriever)
 
+    assert manifestlist.amd64_linux_manifest_digest == "sha256:5b"
+
 
 def test_get_schema1_manifest_no_matching_list():
     manifestlist = DockerSchema2ManifestList(Bytes.for_string_or_unicode(NO_AMD_MANIFESTLIST_BYTES))
@@ -121,6 +125,7 @@ def test_get_schema1_manifest_no_matching_list():
 
     assert manifestlist.media_type == "application/vnd.docker.distribution.manifest.list.v2+json"
     assert manifestlist.bytes.as_encoded_str() == NO_AMD_MANIFESTLIST_BYTES
+    assert manifestlist.amd64_linux_manifest_digest is None
 
     compatible_manifest = manifestlist.get_schema1_manifest("foo", "bar", "baz", retriever)
     assert compatible_manifest is None
@@ -130,10 +135,22 @@ def test_builder():
     existing = DockerSchema2ManifestList(Bytes.for_string_or_unicode(MANIFESTLIST_BYTES))
     builder = DockerSchema2ManifestListBuilder()
     for index, manifest in enumerate(existing.manifests(retriever)):
-        builder.add_manifest(manifest.manifest_obj, "amd64", "os")
+        builder.add_manifest(manifest.manifest_obj, "amd64", "linux")
 
     built = builder.build()
     assert len(built.manifests(retriever)) == 2
+    assert built.amd64_linux_manifest_digest is not None
+
+
+def test_builder_no_amd():
+    existing = DockerSchema2ManifestList(Bytes.for_string_or_unicode(MANIFESTLIST_BYTES))
+    builder = DockerSchema2ManifestListBuilder()
+    for index, manifest in enumerate(existing.manifests(retriever)):
+        builder.add_manifest(manifest.manifest_obj, "intel386", "os")
+
+    built = builder.build()
+    assert len(built.manifests(retriever)) == 2
+    assert built.amd64_linux_manifest_digest is None
 
 
 def test_invalid_manifestlist():

image/docker/schema2/test/test_manifest.py

@@ -119,6 +119,8 @@ def test_valid_manifest():
     assert manifest.media_type == "application/vnd.docker.distribution.manifest.v2+json"
     assert not manifest.has_remote_layer
     assert manifest.has_legacy_image
+    assert manifest.config_media_type == "application/vnd.docker.container.image.v1+json"
+    assert manifest.layers_compressed_size == 123721
 
     retriever = ContentRetrieverForTesting.for_config(
         {
@@ -171,6 +173,8 @@ def test_valid_remote_manifest():
     )
     assert manifest.media_type == "application/vnd.docker.distribution.manifest.v2+json"
     assert manifest.has_remote_layer
+    assert manifest.config_media_type == "application/vnd.docker.container.image.v1+json"
+    assert manifest.layers_compressed_size == 123721
 
     assert len(manifest.filesystem_layers) == 4
     assert manifest.filesystem_layers[0].compressed_size == 1234

image/docker/squashed.py

@@ -1,149 +0,0 @@
-import copy
-import json
-import math
-import calendar
-
-from app import app
-from image.common import TarImageFormatter
-from util.registry.gzipwrap import GZIP_BUFFER_SIZE
-from util.registry.streamlayerformat import StreamLayerMerger
-
-
-class FileEstimationException(Exception):
-    """
-    Exception raised by build_docker_load_stream if the estimated size of the layer tar was lower
-    than the actual size.
-
-    This means the sent tar header is wrong, and we have to fail.
-    """
-
-    pass
-
-
-class SquashedDockerImageFormatter(TarImageFormatter):
-    """
-    Image formatter which produces a squashed image compatible with the `docker load` command.
-    """
-
-    # Multiplier against the image size reported by Docker to account for the tar metadata.
-    # Note: This multiplier was not formally calculated in anyway and should be adjusted overtime
-    # if/when we encounter issues with it. Unfortunately, we cannot make it too large or the Docker
-    # daemon dies when trying to load the entire tar into memory.
-    SIZE_MULTIPLIER = 1.2
-
-    def stream_generator(
-        self,
-        tag,
-        parsed_manifest,
-        synthetic_image_id,
-        layer_iterator,
-        tar_stream_getter_iterator,
-        reporter=None,
-    ):
-        image_mtime = 0
-        created = parsed_manifest.created_datetime
-        if created is not None:
-            image_mtime = calendar.timegm(created.utctimetuple())
-
-        # Docker import V1 Format (.tar):
-        #  repositories - JSON file containing a repo -> tag -> image map
-        #  {image ID folder}:
-        #     json - The layer JSON
-        #     layer.tar - The tarballed contents of the layer
-        #     VERSION - The docker import version: '1.0'
-        layer_merger = StreamLayerMerger(tar_stream_getter_iterator, reporter=reporter)
-
-        # Yield the repositories file:
-        synthetic_layer_info = {}
-        synthetic_layer_info[tag.name + ".squash"] = synthetic_image_id
-
-        hostname = app.config["SERVER_HOSTNAME"]
-        repositories = {}
-        namespace = tag.repository.namespace_name
-        repository = tag.repository.name
-        repositories[hostname + "/" + namespace + "/" + repository] = synthetic_layer_info
-
-        yield self.tar_file(
-            "repositories", json.dumps(repositories).encode("utf-8"), mtime=image_mtime
-        )
-
-        # Yield the image ID folder.
-        yield self.tar_folder(synthetic_image_id, mtime=image_mtime)
-
-        # Yield the JSON layer data.
-        layer_json = SquashedDockerImageFormatter._build_layer_json(
-            parsed_manifest, synthetic_image_id
-        )
-        yield self.tar_file(
-            synthetic_image_id + "/json", json.dumps(layer_json).encode("utf-8"), mtime=image_mtime
-        )
-
-        # Yield the VERSION file.
-        yield self.tar_file(synthetic_image_id + "/VERSION", b"1.0", mtime=image_mtime)
-
-        # Yield the merged layer data's header.
-        estimated_file_size = 0
-        for layer in layer_iterator:
-            estimated_file_size += layer.estimated_size(
-                SquashedDockerImageFormatter.SIZE_MULTIPLIER
-            )
-
-        # Make sure the estimated file size is an integer number of bytes.
-        estimated_file_size = int(math.ceil(estimated_file_size))
-
-        yield self.tar_file_header(
-            synthetic_image_id + "/layer.tar", estimated_file_size, mtime=image_mtime
-        )
-
-        # Yield the contents of the merged layer.
-        yielded_size = 0
-        for entry in layer_merger.get_generator():
-            yield entry
-            yielded_size += len(entry)
-
-        # If the yielded size is more than the estimated size (which is unlikely but possible), then
-        # raise an exception since the tar header will be wrong.
-        if yielded_size > estimated_file_size:
-            leaf_image_id = parsed_manifest.leaf_layer_v1_image_id
-            message = "For %s/%s:%s (%s:%s): Expected %s bytes, found %s bytes" % (
-                namespace,
-                repository,
-                tag,
-                parsed_manifest.digest,
-                leaf_image_id,
-                estimated_file_size,
-                yielded_size,
-            )
-            raise FileEstimationException(message)
-
-        # If the yielded size is less than the estimated size (which is likely), fill the rest with
-        # zeros.
-        if yielded_size < estimated_file_size:
-            to_yield = estimated_file_size - yielded_size
-            while to_yield > 0:
-                yielded = min(to_yield, GZIP_BUFFER_SIZE)
-                yield b"\0" * yielded
-                to_yield -= yielded
-
-        # Yield any file padding to 512 bytes that is necessary.
-        yield self.tar_file_padding(estimated_file_size)
-
-        # Last two records are empty in tar spec.
-        yield b"\0" * 512
-        yield b"\0" * 512
-
-    @staticmethod
-    def _build_layer_json(manifest, synthetic_image_id):
-        updated_json = json.loads(manifest.leaf_layer.raw_v1_metadata)
-        updated_json["id"] = synthetic_image_id
-
-        if "parent" in updated_json:
-            del updated_json["parent"]
-
-        if "config" in updated_json and "Image" in updated_json["config"]:
-            updated_json["config"]["Image"] = synthetic_image_id
-
-        if "container_config" in updated_json and "Image" in updated_json["container_config"]:
-            updated_json["container_config"]["Image"] = synthetic_image_id
-
-        return updated_json

image/docker/test/test_schema1.py

@@ -37,10 +37,12 @@ MANIFEST_BYTES = json.dumps(
         "tag": "latest",
         "architecture": "amd64",
         "fsLayers": [
+            {"blobSum": "sha256:cd8567d70002e957612902a8e985ea129d831ebe04057d88fb644857caa45d11"},
             {"blobSum": "sha256:cc8567d70002e957612902a8e985ea129d831ebe04057d88fb644857caa45d11"},
             {"blobSum": "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef"},
         ],
         "history": [
+            {"v1Compatibility": '{"id":"sizedid", "parent": "someid", "Size": 1234}'},
             {"v1Compatibility": '{"id":"someid", "parent": "anotherid"}'},
             {"v1Compatibility": '{"id":"anotherid"}'},
         ],
@@ -71,10 +73,12 @@ def test_valid_manifest():
     assert manifest.namespace == ""
     assert manifest.repo_name == "hello-world"
     assert manifest.tag == "latest"
-    assert manifest.image_ids == {"someid", "anotherid"}
+    assert manifest.image_ids == {"sizedid", "someid", "anotherid"}
-    assert manifest.parent_image_ids == {"anotherid"}
+    assert manifest.parent_image_ids == {"someid", "anotherid"}
+    assert manifest.layers_compressed_size == 1234
+    assert manifest.config_media_type is None
 
-    assert len(manifest.layers) == 2
+    assert len(manifest.layers) == 3
 
     assert manifest.layers[0].v1_metadata.image_id == "anotherid"
     assert manifest.layers[0].v1_metadata.parent_image_id is None
@@ -82,10 +86,14 @@ def test_valid_manifest():
     assert manifest.layers[1].v1_metadata.image_id == "someid"
     assert manifest.layers[1].v1_metadata.parent_image_id == "anotherid"
 
+    assert manifest.layers[2].v1_metadata.image_id == "sizedid"
+    assert manifest.layers[2].v1_metadata.parent_image_id == "someid"
+
     assert manifest.layers[0].compressed_size is None
     assert manifest.layers[1].compressed_size is None
+    assert manifest.layers[2].compressed_size == 1234
 
-    assert manifest.leaf_layer == manifest.layers[1]
+    assert manifest.leaf_layer == manifest.layers[2]
     assert manifest.created_datetime is None
 
     unsigned = manifest.unsigned()
@@ -97,8 +105,8 @@ def test_valid_manifest():
     assert unsigned.digest != manifest.digest
 
     image_layers = list(manifest.get_layers(None))
-    assert len(image_layers) == 2
+    assert len(image_layers) == 3
-    for index in range(0, 2):
+    for index in range(0, 3):
         assert image_layers[index].layer_id == manifest.layers[index].v1_metadata.image_id
         assert image_layers[index].blob_digest == manifest.layers[index].digest
         assert image_layers[index].command == manifest.layers[index].v1_metadata.command

image/oci/index.py

@@ -41,7 +41,7 @@ from jsonschema import validate as validate_schema, ValidationError
 
 from digest import digest_tools
 from image.shared import ManifestException
-from image.shared.interfaces import ManifestInterface
+from image.shared.interfaces import ManifestListInterface
 from image.shared.schemautil import LazyManifestLoader
 from image.oci import OCI_IMAGE_INDEX_CONTENT_TYPE, OCI_IMAGE_MANIFEST_CONTENT_TYPE
 from image.oci.descriptor import get_descriptor_schema
@@ -81,7 +81,7 @@ class MalformedIndex(ManifestException):
     pass
 
 
-class OCIIndex(ManifestInterface):
+class OCIIndex(ManifestListInterface):
     METASCHEMA = {
         "type": "object",
         "properties": {
@@ -227,6 +227,10 @@ class OCIIndex(ManifestInterface):
     def layers_compressed_size(self):
         return None
 
+    @property
+    def config_media_type(self):
+        return None
+
     @lru_cache(maxsize=1)
     def manifests(self, content_retriever):
         """
@@ -275,6 +279,20 @@ class OCIIndex(ManifestInterface):
     def has_legacy_image(self):
         return False
 
+    @property
+    def amd64_linux_manifest_digest(self):
+        """ Returns the digest of the AMD64+Linux manifest in this list, if any, or None
+            if none.
+        """
+        for manifest_ref in self._parsed[INDEX_MANIFESTS_KEY]:
+            platform = manifest_ref[INDEX_PLATFORM_KEY]
+            architecture = platform.get(INDEX_ARCHITECTURE_KEY, None)
+            os = platform.get(INDEX_OS_KEY, None)
+            if architecture == "amd64" and os == "linux":
+                return manifest_ref[INDEX_DIGEST_KEY]
+
+        return None
+
     def get_requires_empty_layer_blob(self, content_retriever):
         return False
 

image/oci/manifest.py

@@ -197,6 +197,10 @@ class OCIManifest(ManifestInterface):
         """
         return self.filesystem_layers[-1]
 
+    @property
+    def config_media_type(self):
+        return self._parsed[OCI_MANIFEST_CONFIG_KEY][OCI_MANIFEST_MEDIATYPE_KEY]
+
     @property
     def layers_compressed_size(self):
         return sum(layer.compressed_size for layer in self.filesystem_layers)

image/oci/test/test_oci_index.py

@@ -34,6 +34,35 @@ SAMPLE_INDEX = """{
 }"""
 
 
+SAMPLE_INDEX_NO_AMD = """{
+  "schemaVersion": 2,
+  "manifests": [
+    {
+      "mediaType": "application/vnd.oci.image.manifest.v1+json",
+      "size": 7143,
+      "digest": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f",
+      "platform": {
+        "architecture": "ppc64le",
+        "os": "linux"
+      }
+    },
+    {
+      "mediaType": "application/vnd.oci.image.manifest.v1+json",
+      "size": 7682,
+      "digest": "sha256:5b0bcabd1ed22e9fb1310cf6c2dec7cdef19f0ad69efa1f392e94a4333501270",
+      "platform": {
+        "architecture": "intel386",
+        "os": "linux"
+      }
+    }
+  ],
+  "annotations": {
+    "com.example.key1": "value1",
+    "com.example.key2": "value2"
+  }
+}"""
+
+
 def test_parse_basic_index():
     index = OCIIndex(Bytes.for_string_or_unicode(SAMPLE_INDEX))
     assert index.is_manifest_list
@@ -43,6 +72,10 @@ def test_parse_basic_index():
         "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f",
         "sha256:5b0bcabd1ed22e9fb1310cf6c2dec7cdef19f0ad69efa1f392e94a4333501270",
     ]
+    assert (
+        index.amd64_linux_manifest_digest
+        == "sha256:5b0bcabd1ed22e9fb1310cf6c2dec7cdef19f0ad69efa1f392e94a4333501270"
+    )
 
 
 def test_config_missing_required():
@@ -56,3 +89,15 @@ def test_config_missing_required():
 def test_invalid_index():
     with pytest.raises(MalformedIndex):
         OCIIndex(Bytes.for_string_or_unicode("{}"))
+
+
+def test_index_without_amd():
+    index = OCIIndex(Bytes.for_string_or_unicode(SAMPLE_INDEX_NO_AMD))
+    assert index.is_manifest_list
+    assert index.digest == "sha256:a0ed0f2b3949bc731063320667062307faf4245f6872dc5bc98ee6ea5443f169"
+    assert index.local_blob_digests == []
+    assert index.child_manifest_digests() == [
+        "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f",
+        "sha256:5b0bcabd1ed22e9fb1310cf6c2dec7cdef19f0ad69efa1f392e94a4333501270",
+    ]
+    assert index.amd64_linux_manifest_digest is None

image/shared/interfaces.py

@@ -56,6 +56,12 @@ class ManifestInterface(object):
         Returns None if this cannot be computed locally.
         """
 
+    @abstractproperty
+    def config_media_type(self):
+        """ Returns the media type of the config of this manifest or None if
+            this manifest does not support a configuration type.
+        """
+
     @abstractmethod
     def validate(self, content_retriever):
         """
@@ -184,6 +190,19 @@ class ManifestInterface(object):
         """
 
 
+@add_metaclass(ABCMeta)
+class ManifestListInterface(object):
+    """
+    Defines the interface for the various manifest list types supported.
+    """
+
+    @abstractmethod
+    def amd64_linux_manifest_digest(self):
+        """ Returns the digest of the AMD64+Linux manifest in this list, if any, or None
+            if none.
+        """
+
+
 @add_metaclass(ABCMeta)
 class ContentRetriever(object):
     """

initdb.py

@@ -174,6 +174,7 @@ def __create_manifest_and_tags(
 
         config = {
             "id": current_id,
+            "Size": len(content),
         }
         if parent_id:
             config["parent"] = parent_id
@@ -1239,6 +1240,8 @@ WHITELISTED_EMPTY_MODELS = [
     "LogEntry",
     "LogEntry2",
     "ManifestSecurityStatus",
+    "ManifestLegacyImage",
+    "Image",
 ]
 
 

requirements-nover.txt

@@ -34,6 +34,7 @@ geoip2
 gevent
 gipc
 gunicorn
+hashids
 hiredis
 html5lib==0.9999999 # pinned due to xhtml2pdf
 httmock

requirements.txt

@@ -68,9 +68,9 @@ futures==3.1.1
 geoip2==3.0.0
 gevent==1.4.0
 gipc==1.0.1
-gpg==1.10.0
 greenlet==0.4.15
 gunicorn==20.0.4
+hashids==1.2.0
 hiredis==1.0.1
 html5lib==1.0.1
 httmock==1.3.0

static/directives/repo-view/image-tag-tooltip.html

@@ -1,11 +0,0 @@
-<div class="popover image-tag-tooltip" tabindex="-1">
-  <div class="image-tag-tooltip-header"
-       ng-style="::{'backgroundColor': trackEntryForImage[tag.image_id].color, 
-                    'color': constrastingColor( trackEntryForImage[tag.image_id].color)}">
-      Image {{ tag.image_id.substr(0, 12) }}
-   </div>
-   <ul class="image-tag-tooltip-tags">
-      <li ng-repeat="tag in imageMap[tag.image_id] | limitTo:5"><i class="fa fa-tag"></i>{{ tag.name }}</li>
-   </ul>
-   <div class="image-tag-tooltip-tags-more" ng-if="imageMap[tag.image_id].length > 5">and {{ imageMap[tag.image_id].length - 5 }} more tags</div>
-</div>

static/directives/repo-view/manifest-tag-tooltip.html

@@ -0,0 +1,11 @@
+<div class="popover image-tag-tooltip" tabindex="-1">
+  <div class="image-tag-tooltip-header"
+       ng-style="::{'backgroundColor': trackEntryForManifest[tag.manifest_digest].color, 
+                    'color': constrastingColor(trackEntryForManifest[tag.manifest_digest].color)}">
+      Manifest {{ tag.manifest_digest.substr(7, 12) }}
+   </div>
+   <ul class="image-tag-tooltip-tags">
+      <li ng-repeat="tag in manifestMap[tag.manifest_digest] | limitTo:5"><i class="fa fa-tag"></i>{{ tag.name }}</li>
+   </ul>
+   <div class="image-tag-tooltip-tags-more" ng-if="manifestMap[tag.manifest_digest].length > 5">and {{ manifestMap[tag.manifest_digest].length - 5 }} more tags</div>
+</div>

static/directives/repo-view/repo-panel-tags.html

@@ -32,9 +32,9 @@
       <i class="fa fa-git"></i>Commit SHAs
      </div>
 
-     <div class="cor-checkable-menu-item" item-filter="imageIDFilter(it.image_id, item)"
+     <div class="cor-checkable-menu-item" item-filter="manifestDigestFilter(mt.manifest_digest, item)"
-          ng-repeat="it in imageTrackEntries" ng-if="::it.visible">
+          ng-repeat="mt in manifestTrackEntries" ng-if="::it.visible">
-      <i class="fa fa-circle-o" ng-style="::{'color': it.color}"></i> {{ ::it.image_id.substr(0, 12) }}
+      <i class="fa fa-circle-o" ng-style="::{'color': ,t.color}"></i>
      </div>
    </span>
 
@@ -116,16 +116,16 @@
         style="width: 140px;">
       <a ng-click="orderBy('expiration_date')" data-title="When the tag expires" data-container="body" bs-tooltip>Expires</a>
     </td>
-    <td class="hidden-xs hidden-sm" ng-if="imageTracks.length > maxTrackCount"
+    <td class="hidden-xs hidden-sm" ng-if="manifestTracks.length > maxTrackCount"
         style="width: 20px; position: relative;">
     </td>
     <td class="hidden-xs hidden-sm"
-        ng-class="tablePredicateClass('image_id', options.predicate, options.reverse)"
+        ng-class="tablePredicateClass('manifest_digest', options.predicate, options.reverse)"
         style="width: 140px;">
-      <a ng-click="orderBy('image_id')">Manifest</a>
+      <a ng-click="orderBy('manifest_digest')">Manifest</a>
     </td>
-    <td class="hidden-xs hidden-sm hidden-md image-track" ng-repeat="it in imageTracks"
+    <td class="hidden-xs hidden-sm hidden-md image-track" ng-repeat="mt in manifestTracks"
-        ng-if="imageTracks.length <= maxTrackCount"></td>
+        ng-if="manifestTracks.length <= maxTrackCount"></td>
     <td class="options-col"></td>
     <td class="options-col"></td>
     <td class="hidden-xs hidden-sm" style="width: 4px"></td>
@@ -167,14 +167,6 @@
           See Child Manifests
         </span>
 
-        <!-- No Digest -->
-        <span class="nodigest" ng-if="::!tag.manifest_digest"
-              data-title="The tag does not have a V2 digest and so is unsupported for scan"
-              bs-tooltip>
-          <span class="donut-chart" width="22" data="[{'index': 0, 'value': 1, 'color': '#eee'}]"></span>
-          Unsupported
-        </span>
-
         <!-- Manifest security view -->
         <manifest-security-view repository="::repository" manifest-digest="::tag.manifest_digest"
                                 ng-if="::(tag.manifest_digest && !tag.is_manifest_list)">
@@ -198,11 +190,11 @@
 
       <!-- Manifest link -->
       <td class="hidden-xs hidden-sm hidden-md image-track"
-          ng-if="imageTracks.length > maxTrackCount">
+          ng-if="manifestTracks.length > maxTrackCount">
         <span class="image-track-filled-dot"
-              ng-if="::trackEntryForImage[tag.image_id]"
+              ng-if="::trackEntryForManifest[tag.manifest_digest]"
-              ng-style="::{'backgroundColor': trackEntryForImage[tag.image_id].color}"
+              ng-style="::{'backgroundColor': trackEntryForManifest[tag.manifest_digest].color}"
-              ng-click="::selectTrack(trackEntryForImage[tag.image_id])"
+              ng-click="::selectTrack(trackEntryForManifest[tag.manifest_digest])"
               data-template-url="/static/directives/repo-view/image-tag-tooltip.html"
               data-placement="left"
               data-trigger="hover"
@@ -213,22 +205,22 @@
       <td class="hidden-xs hidden-sm image-id-col">
         <manifest-link repository="repository" image-id="tag.image_id" manifest-digest="tag.manifest_digest"></manifest-link>
       </td>
-      <td class="hidden-xs hidden-sm hidden-md image-track" ng-repeat="it in imageTracks"
+      <td class="hidden-xs hidden-sm hidden-md image-track" ng-repeat="mt in manifestTracks"
-          ng-if="imageTracks.length <= maxTrackCount">
+          ng-if="manifestTracks.length <= maxTrackCount">
         <span class="image-track-dot"
-              ng-if="::it.entryByImageId[tag.image_id]"
+              ng-if="::mt.entryByManifestDigest[tag.manifest_digest]"
-              ng-style="::{'borderColor': trackEntryForImage[tag.image_id].color}"
+              ng-style="::{'borderColor': trackEntryForManifest[tag.manifest_digest].color}"
-              ng-click="::selectTrack(trackEntryForImage[tag.image_id])"
+              ng-click="::selectTrack(trackEntryForManifest[tag.manifest_digest])"
-              data-template-url="/static/directives/repo-view/image-tag-tooltip.html"
+              data-template-url="/static/directives/repo-view/manifest-tag-tooltip.html"
               data-placement="left"
               data-trigger="hover"
               data-animation="am-flip-x"
               data-auto-close="1"
               bs-popover></span>
         <span class="image-track-line"
-              ng-if="::getTrackEntryForIndex(it, $parent.$parent.$index)"
+              ng-if="::getTrackEntryForIndex(mt, $parent.$parent.$index)"
-              ng-class="::trackLineClass(it, $parent.$parent.$parent.$index)"
+              ng-class="::trackLineClass(mt, $parent.$parent.$parent.$index)"
-              ng-style="::{'borderColor': getTrackEntryForIndex(it, $parent.$parent.$parent.$index).color}"></span>
+              ng-style="::{'borderColor': getTrackEntryForIndex(mt, $parent.$parent.$parent.$index).color}"></span>
       </td>
       <td class="options-col">
         <i class="fa fa-download" data-title="Fetch Tag" bs-tooltip
@@ -288,12 +280,12 @@
           <manifest-link repository="repository" manifest-digest="manifest.digest"></manifest-link>
       </td>
 
-      <td class="hidden-xs hidden-sm hidden-md image-track" ng-repeat="it in imageTracks"
+      <td class="hidden-xs hidden-sm hidden-md image-track" ng-repeat="mt in manifestTracks"
-          ng-if="imageTracks.length <= maxTrackCount" bindonce>
+          ng-if="manifestTracks.length <= maxTrackCount" bindonce>
         <span class="image-track-line"
-          ng-if="::getTrackEntryForIndex(it, $parent.$parent.$parent.$parent.$index)"
+          ng-if="::getTrackEntryForIndex(mt, $parent.$parent.$parent.$parent.$index)"
-          ng-class="::trackLineExpandedClass(it, $parent.$parent.$parent.$parent.$parent.$index)"
+          ng-class="::trackLineExpandedClass(mt, $parent.$parent.$parent.$parent.$parent.$index)"
-          ng-style="::{'borderColor': getTrackEntryForIndex(it, $parent.$parent.$parent.$parent.$parent.$index).color}"></span>
+          ng-style="::{'borderColor': getTrackEntryForIndex(mt, $parent.$parent.$parent.$parent.$parent.$index).color}"></span>
       </td>
 
       <td class="options-col"></td>
@@ -320,12 +312,12 @@
         </div>
       </td>
 
-      <td class="hidden-xs hidden-sm hidden-md image-track" ng-repeat="it in imageTracks"
+      <td class="hidden-xs hidden-sm hidden-md image-track" ng-repeat="mt in manifestTracks"
-          ng-if="imageTracks.length <= maxTrackCount" bindonce>
+          ng-if="manifestTracks.length <= maxTrackCount" bindonce>
         <span class="image-track-line"
-          ng-if="::getTrackEntryForIndex(it, $parent.$parent.$index)"
+          ng-if="::getTrackEntryForIndex(mt, $parent.$parent.$index)"
-          ng-class="::trackLineExpandedClass(it, $parent.$parent.$parent.$index)"
+          ng-class="::trackLineExpandedClass(mt, $parent.$parent.$parent.$index)"
-          ng-style="::{'borderColor': getTrackEntryForIndex(it, $parent.$parent.$parent.$index).color}"></span>
+          ng-style="::{'borderColor': getTrackEntryForIndex(mt, $parent.$parent.$parent.$index).color}"></span>
       </td>
       <td></td>
       <td class="options-col hidden-xs hidden-sm"><!-- Whitespace col --></td>

static/js/directives/repo-view/repo-panel-tags.js

@@ -89,78 +89,78 @@ angular.module('quay').directive('repoPanelTags', function () {
         }
 
         // Sort the tags by the predicate and the reverse, and map the information.
-        var imageIDs = [];
         var ordered = TableService.buildOrderedItems(allTags, $scope.options,
-            ['name'], ['last_modified_datetime', 'size']).entries;
+            ['name', 'manifest_digest'], ['last_modified_datetime', 'size']).entries;
 
         var checked = [];
-        var imageMap = {};
+        var manifestMap = {};
-        var imageIndexMap = {};
+        var manifestIndexMap = {};
+        var manifestDigests = [];
         for (var i = 0; i < ordered.length; ++i) {
           var tagInfo = ordered[i];
-          if (!tagInfo.image_id) {
+          if (!tagInfo.manifest_digest) {
             continue;
           }
 
-          if (!imageMap[tagInfo.image_id]) {
+          if (!manifestMap[tagInfo.manifest_digest]) {
-            imageMap[tagInfo.image_id] = [];
+            manifestMap[tagInfo.manifest_digest] = [];
-            imageIDs.push(tagInfo.image_id)
+            manifestDigests.push(tagInfo.manifest_digest)
           }
 
-          imageMap[tagInfo.image_id].push(tagInfo);
+          manifestMap[tagInfo.manifest_digest].push(tagInfo);
           if ($.inArray(tagInfo.name, $scope.selectedTags) >= 0) {
             checked.push(tagInfo);
           }
 
-          if (!imageIndexMap[tagInfo.image_id]) {
+          if (!manifestIndexMap[tagInfo.manifest_digest]) {
-            imageIndexMap[tagInfo.image_id] = {'start': i, 'end': i};
+            manifestIndexMap[tagInfo.manifest_digest] = {'start': i, 'end': i};
           }
 
-          imageIndexMap[tagInfo.image_id]['end'] = i;
+          manifestIndexMap[tagInfo.manifest_digest]['end'] = i;
         };
 
         // Calculate the image tracks.
         var colors = d3.scale.category10();
-        if (Object.keys(imageMap).length > 10) {
+        if (Object.keys(manifestMap).length > 10) {
           colors = d3.scale.category20();
         }
 
-        var imageTracks = [];
+        var manifestTracks = [];
-        var imageTrackEntries = [];
+        var manifestTrackEntries = [];
-        var trackEntryForImage = {};
+        var trackEntryForManifest = {};
 
         var visibleStartIndex = ($scope.options.page * $scope.tagsPerPage);
         var visibleEndIndex = (($scope.options.page + 1) * $scope.tagsPerPage);
 
-        imageIDs.sort().map(function(image_id) {
+        manifestDigests.sort().map(function(manifest_digest) {
-          if (imageMap[image_id].length >= 2){
+          if (manifestMap[manifest_digest].length >= 2){
             // Create the track entry.
-            var imageIndexRange = imageIndexMap[image_id];
+            var manifestIndexRange = manifestIndexMap[manifest_digest];
-            var colorIndex = imageTrackEntries.length;
+            var colorIndex = manifestTrackEntries.length;
             var trackEntry = {
-              'image_id': image_id,
+              'manifest_digest': manifest_digest,
               'color': colors(colorIndex),
-              'count': imageMap[image_id].length,
+              'count': manifestMap[manifest_digest].length,
-              'tags': imageMap[image_id],
+              'tags': manifestMap[manifest_digest],
-              'index_range': imageIndexRange,
+              'index_range': manifestIndexRange,
-              'visible': visibleStartIndex <= imageIndexRange.end && imageIndexRange.start <= visibleEndIndex,
+              'visible': visibleStartIndex <= manifestIndexRange.end && manifestIndexRange.start <= visibleEndIndex,
             };
 
-            trackEntryForImage[image_id] = trackEntry;
+            trackEntryForManifest[manifest_digest] = trackEntry;
-            imageMap[image_id]['color'] = colors(colorIndex);
+            manifestMap[manifest_digest]['color'] = colors(colorIndex);
 
             // Find the track in which we can place the entry, if any.
             var existingTrack = null;
-            for (var i = 0; i < imageTracks.length; ++i) {
+            for (var i = 0; i < manifestTracks.length; ++i) {
               // For the current track, ensure that the start and end index
               // for the current entry is outside of the range of the track's
               // entries. If so, then we can add the entry to the track.
-              var currentTrack = imageTracks[i];
+              var currentTrack = manifestTracks[i];
               var canAddToCurrentTrack = true;
               for (var j = 0; j < currentTrack.entries.length; ++j) {
                 var currentTrackEntry = currentTrack.entries[j];
-                var entryInfo = imageIndexMap[currentTrackEntry.image_id];
+                var entryInfo = manifestIndexMap[currentTrackEntry.image_id];
-                if (Math.max(entryInfo.start, imageIndexRange.start) <= Math.min(entryInfo.end, imageIndexRange.end)) {
+                if (Math.max(entryInfo.start, manifestIndexRange.start) <= Math.min(entryInfo.end, manifestIndexRange.end)) {
                   canAddToCurrentTrack = false;
                   break;
                 }
@@ -175,38 +175,38 @@ angular.module('quay').directive('repoPanelTags', function () {
             // Add the entry to the track or create a new track if necessary.
             if (existingTrack) {
               existingTrack.entries.push(trackEntry)
-              existingTrack.entryByImageId[image_id] = trackEntry;
+              existingTrack.entryByManifestDigest[manifest_digest] = trackEntry;
-              existingTrack.endIndex = Math.max(existingTrack.endIndex, imageIndexRange.end);
+              existingTrack.endIndex = Math.max(existingTrack.endIndex, manifestIndexRange.end);
 
-              for (var j = imageIndexRange.start; j <= imageIndexRange.end; j++) {
+              for (var j = manifestIndexRange.start; j <= manifestIndexRange.end; j++) {
                 existingTrack.entryByIndex[j] = trackEntry;
               }
             } else {
-              var entryByImageId = {};
+              var entryByManifestDigest = {};
-              entryByImageId[image_id] = trackEntry;
+              entryByManifestDigest[manifest_digest] = trackEntry;
 
               var entryByIndex = {};
-              for (var j = imageIndexRange.start; j <= imageIndexRange.end; j++) {
+              for (var j = manifestIndexRange.start; j <= manifestIndexRange.end; j++) {
                 entryByIndex[j] = trackEntry;
               }
 
-              imageTracks.push({
+              manifestTracks.push({
                 'entries': [trackEntry],
-                'entryByImageId': entryByImageId,
+                'entryByManifestDigest': entryByManifestDigest,
-                'startIndex': imageIndexRange.start,
+                'startIndex': manifestIndexRange.start,
-                'endIndex': imageIndexRange.end,
+                'endIndex': manifestIndexRange.end,
                 'entryByIndex': entryByIndex,
               });
             }
 
-            imageTrackEntries.push(trackEntry);
+            manifestTrackEntries.push(trackEntry);
           }
         });
 
-        $scope.imageMap = imageMap;
+        $scope.manifestMap = manifestMap;
-        $scope.imageTracks = imageTracks;
+        $scope.manifestTracks = manifestTracks;
-        $scope.imageTrackEntries = imageTrackEntries;
+        $scope.manifestTrackEntries = manifestTrackEntries;
-        $scope.trackEntryForImage = trackEntryForImage;
+        $scope.trackEntryForManifest = trackEntryForManifest;
 
         $scope.options.page = 0;
 
@@ -241,7 +241,7 @@ angular.module('quay').directive('repoPanelTags', function () {
       });
 
       $scope.$watch('selectedTags', function(selectedTags) {
-        if (!selectedTags || !$scope.repository || !$scope.imageMap) { return; }
+        if (!selectedTags || !$scope.repository || !$scope.manifestMap) { return; }
 
         $scope.checkedTags.setChecked(selectedTags.map(function(tag) {
           return $scope.repositoryTags[tag];
@@ -410,8 +410,8 @@ angular.module('quay').directive('repoPanelTags', function () {
         return false;
       };
 
-      $scope.imageIDFilter = function(image_id, tag) {
+      $scope.manifestDigestFilter = function(manifest_digest, tag) {
-        return tag.image_id == image_id;
+        return tag.manifest_digest == manifest_digest;
       };
 
       $scope.setTab = function(tab) {
@@ -420,7 +420,7 @@ angular.module('quay').directive('repoPanelTags', function () {
 
       $scope.selectTrack = function(it) {
         $scope.checkedTags.checkByFilter(function(tag) {
-          return $scope.imageIDFilter(it.image_id, tag);
+          return $scope.manifestDigestFilter(it.manifest_digest, tag);
         });
       };
 

test/fixtures.py

@@ -26,7 +26,6 @@ from endpoints.appr import appr_bp
 from endpoints.web import web
 from endpoints.v1 import v1_bp
 from endpoints.v2 import v2_bp
-from endpoints.verbs import verbs as verbs_bp
 from endpoints.webhooks import webhooks
 
 from initdb import initialize_database, populate_database
@@ -312,7 +311,6 @@ def app(appconfig, initialized_db):
     app.register_blueprint(api_bp, url_prefix="/api")
     app.register_blueprint(appr_bp, url_prefix="/cnr")
     app.register_blueprint(web, url_prefix="/")
-    app.register_blueprint(verbs_bp, url_prefix="/c1")
     app.register_blueprint(v1_bp, url_prefix="/v1")
     app.register_blueprint(v2_bp, url_prefix="/v2")
     app.register_blueprint(webhooks, url_prefix="/webhooks")

test/registry/fixtures.py

@@ -16,9 +16,8 @@ from app import storage
 from data.database import (
     close_db_filter,
     configure,
-    DerivedStorageForImage,
     QueueItem,
-    Image,
+    ImageStorage,
     TagManifest,
     TagManifestToManifest,
     Manifest,
@@ -30,6 +29,7 @@ from data.database import (
 from data import model
 from data.registry_model import registry_model
 from endpoints.csrf import generate_csrf_token
+from image.docker.schema2 import EMPTY_LAYER_BLOB_DIGEST
 from util.log import logfile_path
 
 from test.registry.liveserverfixture import LiveServerExecutor
@@ -46,15 +46,22 @@ def registry_server_executor(app):
         )
         return "OK"
 
-    def delete_image(image_id):
+    def verify_replication_for(namespace, repo_name, tag_name):
-        image = Image.get(docker_image_id=image_id)
+        repo_ref = registry_model.lookup_repository(namespace, repo_name)
-        image.docker_image_id = "DELETED"
+        assert repo_ref
-        image.save()
+
-        return "OK"
+        tag = registry_model.get_repo_tag(repo_ref, tag_name)
+        assert tag
+
+        manifest = registry_model.get_manifest_for_tag(tag)
+        assert manifest
+
+        for layer in registry_model.list_manifest_layers(manifest, storage):
+            if layer.blob.digest != EMPTY_LAYER_BLOB_DIGEST:
+                QueueItem.select().where(
+                    QueueItem.queue_name ** ("%" + layer.blob.uuid + "%")
+                ).get()
 
-    def get_storage_replication_entry(image_id):
-        image = Image.get(docker_image_id=image_id)
-        QueueItem.select().where(QueueItem.queue_name ** ("%" + image.storage.uuid + "%")).get()
         return "OK"
 
     def set_feature(feature_name, value):
@@ -81,10 +88,6 @@ def registry_server_executor(app):
 
         return jsonify({"old_value": old_value})
 
-    def clear_derived_cache():
-        DerivedStorageForImage.delete().execute()
-        return "OK"
-
     def clear_uncompressed_size(image_id):
         image = model.image.get_image_by_id("devtable", "newrepo", image_id)
         image.storage.uncompressed_size = None
@@ -158,11 +161,9 @@ def registry_server_executor(app):
     executor = LiveServerExecutor()
     executor.register("generate_csrf", generate_csrf)
     executor.register("set_supports_direct_download", set_supports_direct_download)
-    executor.register("delete_image", delete_image)
+    executor.register("verify_replication_for", verify_replication_for)
-    executor.register("get_storage_replication_entry", get_storage_replication_entry)
     executor.register("set_feature", set_feature)
     executor.register("set_config_key", set_config_key)
-    executor.register("clear_derived_cache", clear_derived_cache)
     executor.register("clear_uncompressed_size", clear_uncompressed_size)
     executor.register("add_token", add_token)
     executor.register("break_database", break_database)

test/registry/protocol_v1.py

@@ -153,6 +153,9 @@ class V1Protocol(RegistryProtocol):
                 assert expected_failure == Failures.UNKNOWN_TAG
                 return None
 
+            if expected_failure == Failures.UNKNOWN_TAG:
+                return None
+
             tag_image_id = image_ids[tag_name]
             assert image_id_data.json() == tag_image_id
 
@@ -331,7 +334,7 @@ class V1Protocol(RegistryProtocol):
         namespace,
         repo_name,
         tag_name,
-        image,
+        image_id,
         credentials=None,
         expected_failure=None,
         options=None,
@@ -341,7 +344,7 @@ class V1Protocol(RegistryProtocol):
             session,
             "PUT",
             "/v1/repositories/%s/tags/%s" % (self.repo_name(namespace, repo_name), tag_name),
-            data='"%s"' % image.id,
+            data='"%s"' % image_id,
             auth=auth,
             expected_status=(200, expected_failure, V1ProtocolSteps.PUT_TAG),
         )

test/registry/registry_tests.py

@@ -835,9 +835,10 @@ def test_image_replication(
             credentials=credentials,
         )
 
-        # Ensure that entries were created for each image.
+        # Ensure that entries were created for each layer.
-        for image_id in list(result.image_ids.values()):
+        r = registry_server_executor.on(liveserver).verify_replication_for(
-            r = registry_server_executor.on(liveserver).get_storage_replication_entry(image_id)
+            "devtable", "newrepo", "latest"
+        )
         assert r.text == "OK"
 
 
@@ -872,9 +873,10 @@ def test_image_replication_empty_layers(
             credentials=credentials,
         )
 
-        # Ensure that entries were created for each image.
+        # Ensure that entries were created for each layer.
-        for image_id in list(result.image_ids.values()):
+        r = registry_server_executor.on(liveserver).verify_replication_for(
-            r = registry_server_executor.on(liveserver).get_storage_replication_entry(image_id)
+            "devtable", "newrepo", "latest"
+        )
         assert r.text == "OK"
 
 
@@ -1615,333 +1617,6 @@ def test_tags_disabled_namespace(
     )
 
 
-def test_squashed_image_disabled_namespace(
-    pusher, sized_images, liveserver_session, liveserver, registry_server_executor, app_reloader
-):
-    """ Test: Attempting to pull a squashed image from a disabled namespace. """
-    credentials = ("devtable", "password")
-
-    # Push an image to download.
-    pusher.push(
-        liveserver_session, "buynlarge", "newrepo", "latest", sized_images, credentials=credentials
-    )
-
-    # Disable the buynlarge namespace.
-    registry_server_executor.on(liveserver).disable_namespace("buynlarge")
-
-    # Attempt to pull the squashed version.
-    response = liveserver_session.get("/c1/squash/buynlarge/newrepo/latest", auth=credentials)
-    assert response.status_code == 400
-
-
-def test_squashed_image_disabled_user(
-    pusher, sized_images, liveserver_session, liveserver, registry_server_executor, app_reloader
-):
-    """ Test: Attempting to pull a squashed image via a disabled user. """
-    credentials = ("devtable", "password")
-
-    # Push an image to download.
-    pusher.push(
-        liveserver_session, "buynlarge", "newrepo", "latest", sized_images, credentials=credentials
-    )
-
-    # Disable the devtable namespace.
-    registry_server_executor.on(liveserver).disable_namespace("devtable")
-
-    # Attempt to pull the squashed version.
-    response = liveserver_session.get("/c1/squash/buynlarge/newrepo/latest", auth=credentials)
-    assert response.status_code == 403
-
-
-@pytest.mark.parametrize("use_estimates", [False, True,])
-def test_multilayer_squashed_images(
-    use_estimates,
-    pusher,
-    multi_layer_images,
-    liveserver_session,
-    liveserver,
-    registry_server_executor,
-    app_reloader,
-):
-    """ Test: Pulling of multilayer, complex squashed images. """
-    credentials = ("devtable", "password")
-
-    # Push an image to download.
-    pusher.push(
-        liveserver_session,
-        "devtable",
-        "newrepo",
-        "latest",
-        multi_layer_images,
-        credentials=credentials,
-    )
-
-    if use_estimates:
-        # Clear the uncompressed size stored for the images, to ensure that we estimate instead.
-        for image in multi_layer_images:
-            registry_server_executor.on(liveserver).clear_uncompressed_size(image.id)
-
-    # Pull the squashed version.
-    response = liveserver_session.get("/c1/squash/devtable/newrepo/latest", auth=credentials)
-    assert response.status_code == 200
-
-    tar = tarfile.open(fileobj=BytesIO(response.content))
-
-    # Verify the squashed image.
-    expected_image_id = next(
-        (name for name in tar.getnames() if not "/" in name and name != "repositories")
-    )
-    expected_names = [
-        "repositories",
-        expected_image_id,
-        "%s/json" % expected_image_id,
-        "%s/VERSION" % expected_image_id,
-        "%s/layer.tar" % expected_image_id,
-    ]
-
-    assert tar.getnames() == expected_names
-
-    # Verify the JSON image data.
-    json_data = tar.extractfile(tar.getmember("%s/json" % expected_image_id)).read()
-
-    # Ensure the JSON loads and parses.
-    result = json.loads(json_data)
-    assert result["id"] == expected_image_id
-    assert result["config"]["internal_id"] == "layer5"
-
-    # Ensure that squashed layer tar can be opened.
-    tar = tarfile.open(fileobj=tar.extractfile(tar.getmember("%s/layer.tar" % expected_image_id)))
-    assert set(tar.getnames()) == {"contents", "file1", "file2", "file3", "file4"}
-
-    # Check the contents of various files.
-    assert tar.extractfile("contents").read() == b"layer 5 contents"
-    assert tar.extractfile("file1").read() == b"from-layer-3"
-    assert tar.extractfile("file2").read() == b"from-layer-2"
-    assert tar.extractfile("file3").read() == b"from-layer-4"
-    assert tar.extractfile("file4").read() == b"from-layer-5"
-
-
-@pytest.mark.parametrize("use_estimates", [False, True,])
-@pytest.mark.parametrize("is_readonly", [False, True,])
-def test_squashed_images(
-    use_estimates,
-    pusher,
-    sized_images,
-    liveserver_session,
-    is_readonly,
-    liveserver,
-    registry_server_executor,
-    app_reloader,
-):
-    """ Test: Pulling of squashed images. """
-    credentials = ("devtable", "password")
-
-    # Push an image to download.
-    pusher.push(
-        liveserver_session, "devtable", "newrepo", "latest", sized_images, credentials=credentials
-    )
-
-    if use_estimates:
-        # Clear the uncompressed size stored for the images, to ensure that we estimate instead.
-        for image in sized_images:
-            registry_server_executor.on(liveserver).clear_uncompressed_size(image.id)
-
-    # Pull the squashed version.
-    with ConfigChange(
-        "REGISTRY_STATE",
-        "readonly" if is_readonly else "normal",
-        registry_server_executor.on(liveserver),
-        liveserver,
-    ):
-        response = liveserver_session.get("/c1/squash/devtable/newrepo/latest", auth=credentials)
-        assert response.status_code == 200
-
-        tar = tarfile.open(fileobj=BytesIO(response.content))
-
-        # Verify the squashed image.
-        expected_image_id = next(
-            (name for name in tar.getnames() if not "/" in name and name != "repositories")
-        )
-        expected_names = [
-            "repositories",
-            expected_image_id,
-            "%s/json" % expected_image_id,
-            "%s/VERSION" % expected_image_id,
-            "%s/layer.tar" % expected_image_id,
-        ]
-
-        assert tar.getnames() == expected_names
-
-        # Verify the JSON image data.
-        json_data = tar.extractfile(tar.getmember("%s/json" % expected_image_id)).read()
-
-        # Ensure the JSON loads and parses.
-        result = json.loads(json_data)
-        assert result["id"] == expected_image_id
-        assert result["config"]["foo"] == "childbar"
-
-        # Ensure that squashed layer tar can be opened.
-        tar = tarfile.open(
-            fileobj=tar.extractfile(tar.getmember("%s/layer.tar" % expected_image_id))
-        )
-        assert tar.getnames() == ["contents"]
-
-        # Check the contents.
-        assert tar.extractfile("contents").read() == b"some contents"
-
-
-EXPECTED_ACI_MANIFEST = {
-    "acKind": "ImageManifest",
-    "app": {
-        "environment": [],
-        "mountPoints": [],
-        "group": "root",
-        "user": "root",
-        "workingDirectory": "/",
-        "exec": ["/bin/sh", "-c", '""hello""'],
-        "isolators": [],
-        "eventHandlers": [],
-        "ports": [],
-        "annotations": [
-            {"name": "created", "value": "2018-04-03T18:37:09.284840891Z"},
-            {"name": "homepage", "value": "http://localhost:5000/devtable/newrepo:latest"},
-            {"name": "quay.io/derived-image", "value": "DERIVED_IMAGE_ID"},
-        ],
-    },
-    "labels": [
-        {"name": "version", "value": "latest"},
-        {"name": "arch", "value": "amd64"},
-        {"name": "os", "value": "linux"},
-    ],
-    "acVersion": "0.6.1",
-    "name": "localhost/devtable/newrepo",
-}
-
-
-@pytest.mark.parametrize("is_readonly", [False, True,])
-def test_aci_conversion(
-    pusher,
-    sized_images,
-    liveserver_session,
-    is_readonly,
-    liveserver,
-    registry_server_executor,
-    app_reloader,
-):
-    """ Test: Pulling of ACI converted images. """
-    credentials = ("devtable", "password")
-
-    # Push an image to download.
-    pusher.push(
-        liveserver_session, "devtable", "newrepo", "latest", sized_images, credentials=credentials
-    )
-
-    # Pull the ACI version.
-    with ConfigChange(
-        "REGISTRY_STATE",
-        "readonly" if is_readonly else "normal",
-        registry_server_executor.on(liveserver),
-        liveserver,
-    ):
-        response = liveserver_session.get(
-            "/c1/aci/server_name/devtable/newrepo/latest/aci/linux/amd64", auth=credentials
-        )
-        assert response.status_code == 200
-        tar = tarfile.open(fileobj=BytesIO(response.content))
-        assert set(tar.getnames()) == {"manifest", "rootfs", "rootfs/contents"}
-
-        assert tar.extractfile("rootfs/contents").read() == b"some contents"
-        loaded = json.loads(tar.extractfile("manifest").read())
-        for annotation in loaded["app"]["annotations"]:
-            if annotation["name"] == "quay.io/derived-image":
-                annotation["value"] = "DERIVED_IMAGE_ID"
-
-        assert loaded == EXPECTED_ACI_MANIFEST
-
-    if not is_readonly:
-        # Wait for the ACI signature to be written.
-        time.sleep(1)
-
-        # Pull the ACI signature.
-        response = liveserver_session.get(
-            "/c1/aci/server_name/devtable/newrepo/latest/aci.asc/linux/amd64", auth=credentials
-        )
-        assert response.status_code == 200
-
-
-@pytest.mark.parametrize("schema_version", [1, 2,])
-def test_aci_conversion_manifest_list(
-    v22_protocol,
-    sized_images,
-    different_images,
-    liveserver_session,
-    data_model,
-    liveserver,
-    registry_server_executor,
-    app_reloader,
-    schema_version,
-):
-    """ Test: Pulling of ACI converted image from a manifest list. """
-    credentials = ("devtable", "password")
-    options = ProtocolOptions()
-
-    # Build the manifests that will go in the list.
-    blobs = {}
-
-    signed = v22_protocol.build_schema1(
-        "devtable", "newrepo", "latest", sized_images, blobs, options, arch="amd64"
-    )
-    first_manifest = signed.unsigned()
-    if schema_version == 2:
-        first_manifest = v22_protocol.build_schema2(sized_images, blobs, options)
-
-    second_manifest = v22_protocol.build_schema2(different_images, blobs, options)
-
-    # Create and push the manifest list.
-    builder = DockerSchema2ManifestListBuilder()
-    builder.add_manifest(first_manifest, "amd64", "linux")
-    builder.add_manifest(second_manifest, "arm", "linux")
-    manifestlist = builder.build()
-
-    v22_protocol.push_list(
-        liveserver_session,
-        "devtable",
-        "newrepo",
-        "latest",
-        manifestlist,
-        [first_manifest, second_manifest],
-        blobs,
-        credentials=credentials,
-        options=options,
-    )
-
-    # Pull the ACI version.
-    response = liveserver_session.get(
-        "/c1/aci/server_name/devtable/newrepo/latest/aci/linux/amd64", auth=credentials
-    )
-    assert response.status_code == 200
-    tar = tarfile.open(fileobj=BytesIO(response.content))
-    assert set(tar.getnames()) == {"manifest", "rootfs", "rootfs/contents"}
-
-    assert tar.extractfile("rootfs/contents").read() == b"some contents"
-
-    loaded = json.loads(tar.extractfile("manifest").read())
-    for annotation in loaded["app"]["annotations"]:
-        if annotation["name"] == "quay.io/derived-image":
-            annotation["value"] = "DERIVED_IMAGE_ID"
-
-    assert loaded == EXPECTED_ACI_MANIFEST
-
-    # Wait for the ACI signature to be written.
-    time.sleep(1)
-
-    # Pull the ACI signature.
-    response = liveserver_session.get(
-        "/c1/aci/server_name/devtable/newrepo/latest/aci.asc/linux/amd64", auth=credentials
-    )
-    assert response.status_code == 200
-
-
 @pytest.mark.parametrize(
     "push_user, push_namespace, push_repo, mount_repo_name, expected_failure",
     [
@@ -2323,10 +1998,8 @@ def test_push_pull_same_blobs(pusher, puller, liveserver_session, app_reloader):
     )
 
 
-def test_push_tag_existing_image(
+def test_push_tag_existing_image(v1_protocol, basic_images, liveserver_session, app_reloader):
-    v1_protocol, puller, basic_images, liveserver_session, app_reloader
+    """ Test: Push a new tag on an existing image. """
-):
-    """ Test: Push a new tag on an existing manifest/image. """
     credentials = ("devtable", "password")
 
     # Push a new repository.
@@ -2334,18 +2007,24 @@ def test_push_tag_existing_image(
         liveserver_session, "devtable", "newrepo", "latest", basic_images, credentials=credentials
     )
 
-    # Push the same image/manifest to another tag in the repository.
+    # Pull the repository to verify.
+    pulled = v1_protocol.pull(
+        liveserver_session, "devtable", "newrepo", "latest", basic_images, credentials=credentials,
+    )
+    assert pulled.image_ids
+
+    # Push the same image to another tag in the repository.
     v1_protocol.tag(
         liveserver_session,
         "devtable",
         "newrepo",
         "anothertag",
-        basic_images[-1],
+        pulled.image_ids["latest"],
         credentials=credentials,
     )
 
     # Pull the repository to verify.
-    puller.pull(
+    v1_protocol.pull(
         liveserver_session,
         "devtable",
         "newrepo",
@@ -2655,131 +2334,6 @@ def test_push_pull_manifest_list_duplicate_manifest(
     )
 
 
-def test_squashed_images_empty_layer(
-    pusher,
-    images_with_empty_layer,
-    liveserver_session,
-    liveserver,
-    registry_server_executor,
-    app_reloader,
-):
-    """ Test: Pulling of squashed images for a manifest with empty layers. """
-    credentials = ("devtable", "password")
-
-    # Push an image to download.
-    pusher.push(
-        liveserver_session,
-        "devtable",
-        "newrepo",
-        "latest",
-        images_with_empty_layer,
-        credentials=credentials,
-    )
-
-    # Pull the squashed version.
-    response = liveserver_session.get("/c1/squash/devtable/newrepo/latest", auth=credentials)
-    assert response.status_code == 200
-
-    tar = tarfile.open(fileobj=BytesIO(response.content))
-
-    # Verify the squashed image.
-    expected_image_id = next(
-        (name for name in tar.getnames() if not "/" in name and name != "repositories")
-    )
-    expected_names = [
-        "repositories",
-        expected_image_id,
-        "%s/json" % expected_image_id,
-        "%s/VERSION" % expected_image_id,
-        "%s/layer.tar" % expected_image_id,
-    ]
-
-    assert tar.getnames() == expected_names
-
-
-def test_squashed_image_unsupported(
-    v22_protocol, basic_images, liveserver_session, liveserver, app_reloader, data_model
-):
-    """ Test: Attempting to pull a squashed image for a manifest list without an amd64+linux entry.
-  """
-    credentials = ("devtable", "password")
-    options = ProtocolOptions()
-
-    # Build the manifest that will go in the list.
-    blobs = {}
-    manifest = v22_protocol.build_schema2(basic_images, blobs, options)
-
-    # Create and push the manifest list.
-    builder = DockerSchema2ManifestListBuilder()
-    builder.add_manifest(manifest, "foobar", "someos")
-    manifestlist = builder.build()
-
-    v22_protocol.push_list(
-        liveserver_session,
-        "devtable",
-        "newrepo",
-        "latest",
-        manifestlist,
-        [manifest],
-        blobs,
-        credentials=credentials,
-        options=options,
-    )
-
-    # Attempt to pull the squashed version.
-    response = liveserver_session.get("/c1/squash/devtable/newrepo/latest", auth=credentials)
-    assert response.status_code == 404
-
-
-def test_squashed_image_manifest_list(
-    v22_protocol, basic_images, liveserver_session, liveserver, app_reloader, data_model
-):
-    """ Test: Pull a squashed image for a manifest list with an amd64+linux entry.
-  """
-    credentials = ("devtable", "password")
-    options = ProtocolOptions()
-
-    # Build the manifest that will go in the list.
-    blobs = {}
-    manifest = v22_protocol.build_schema2(basic_images, blobs, options)
-
-    # Create and push the manifest list.
-    builder = DockerSchema2ManifestListBuilder()
-    builder.add_manifest(manifest, "amd64", "linux")
-    manifestlist = builder.build()
-
-    v22_protocol.push_list(
-        liveserver_session,
-        "devtable",
-        "newrepo",
-        "latest",
-        manifestlist,
-        [manifest],
-        blobs,
-        credentials=credentials,
-        options=options,
-    )
-
-    # Pull the squashed version.
-    response = liveserver_session.get("/c1/squash/devtable/newrepo/latest", auth=credentials)
-    assert response.status_code == 200
-
-    # Verify the squashed image.
-    tar = tarfile.open(fileobj=BytesIO(response.content))
-    expected_image_id = next(
-        (name for name in tar.getnames() if not "/" in name and name != "repositories")
-    )
-    expected_names = [
-        "repositories",
-        expected_image_id,
-        "%s/json" % expected_image_id,
-        "%s/VERSION" % expected_image_id,
-        "%s/layer.tar" % expected_image_id,
-    ]
-
-    assert tar.getnames() == expected_names
-
-
 def test_verify_schema2(
     v22_protocol, basic_images, liveserver_session, liveserver, app_reloader, data_model
 ):

test/test_api_usage.py

@@ -2444,7 +2444,6 @@ class TestDeleteRepository(ApiTestCase):
 
         # Make sure the repository has some images and tags.
         repo_ref = registry_model.lookup_repository(ADMIN_ACCESS_USER, "complex")
-        self.assertTrue(len(list(registry_model.get_legacy_images(repo_ref))) > 0)
         self.assertTrue(len(list(registry_model.list_all_active_repository_tags(repo_ref))) > 0)
 
         # Add some data for the repository, in addition to is already existing images and tags.
@@ -2525,11 +2524,11 @@ class TestGetRepository(ApiTestCase):
         self.login(ADMIN_ACCESS_USER)
 
         # base + repo + is_starred + tags
-        with assert_query_count(BASE_LOGGEDIN_QUERY_COUNT + 4 + 1):
+        with assert_query_count(BASE_LOGGEDIN_QUERY_COUNT + 4):
             self.getJsonResponse(Repository, params=dict(repository=ADMIN_ACCESS_USER + "/simple"))
 
         # base + repo + is_starred + tags
-        with assert_query_count(BASE_LOGGEDIN_QUERY_COUNT + 4 + 1):
+        with assert_query_count(BASE_LOGGEDIN_QUERY_COUNT + 4):
             json = self.getJsonResponse(
                 Repository, params=dict(repository=ADMIN_ACCESS_USER + "/gargantuan")
             )
@@ -3326,8 +3325,7 @@ class TestListAndDeleteTag(ApiTestCase):
             params=dict(repository=ADMIN_ACCESS_USER + "/complex", tag="sometag"),
         )
 
-        sometag_images = json["images"]
+        assert json["images"]
-        self.assertEqual(sometag_images, staging_images)
 
         # Move the tag.
         self.putResponse(
@@ -3344,8 +3342,7 @@ class TestListAndDeleteTag(ApiTestCase):
         )
 
         sometag_new_images = json["images"]
-        self.assertEqual(1, len(sometag_new_images))
+        assert sometag_new_images
-        self.assertEqual(staging_images[-1], sometag_new_images[0])
 
     def test_deletesubtag(self):
         self.login(ADMIN_ACCESS_USER)
@@ -3384,7 +3381,7 @@ class TestListAndDeleteTag(ApiTestCase):
         self.login(ADMIN_ACCESS_USER)
 
         repo_ref = registry_model.lookup_repository(ADMIN_ACCESS_USER, "simple")
-        latest_tag = registry_model.get_repo_tag(repo_ref, "latest", include_legacy_image=True)
+        latest_tag = registry_model.get_repo_tag(repo_ref, "latest")
 
         # Create 8 tags in the simple repo.
         remaining_tags = {"latest", "prod"}
@@ -3392,7 +3389,7 @@ class TestListAndDeleteTag(ApiTestCase):
             tag_name = "tag" + str(i)
             remaining_tags.add(tag_name)
             assert registry_model.retarget_tag(
-                repo_ref, tag_name, latest_tag.legacy_image, storage, docker_v2_signing_key
+                repo_ref, tag_name, latest_tag.manifest, storage, docker_v2_signing_key
             )
 
         # Make sure we can iterate over all of them.

test/test_secscan.py

@@ -2,44 +2,26 @@ import json
 import time
 import unittest
 
-from app import app, storage, notification_queue, url_scheme_and_hostname
+from app import app, storage, url_scheme_and_hostname
 from data import model
 from data.registry_model import registry_model
-from data.database import Image, IMAGE_NOT_SCANNED_ENGINE_VERSION
+from data.database import Image, ManifestLegacyImage
-from endpoints.v2 import v2_bp
 from initdb import setup_database_for_testing, finished_database_for_testing
-from notifications.notificationevent import VulnerabilityFoundEvent
 from util.secscan.secscan_util import get_blob_download_uri_getter
-from util.morecollections import AttrDict
 from util.secscan.api import SecurityScannerAPI, APIRequestFailure
-from util.secscan.analyzer import LayerAnalyzer
 from util.secscan.fake import fake_security_scanner
-from util.secscan.notifier import SecurityNotificationHandler, ProcessNotificationPageResult
 from util.security.instancekeys import InstanceKeys
-from workers.security_notification_worker import SecurityNotificationWorker
 
 
 ADMIN_ACCESS_USER = "devtable"
 SIMPLE_REPO = "simple"
-COMPLEX_REPO = "complex"
-
-
-def process_notification_data(legacy_api, notification_data):
-    handler = SecurityNotificationHandler(legacy_api, 100)
-    result = handler.process_notification_page_data(notification_data)
-    handler.send_notifications()
-    return result == ProcessNotificationPageResult.FINISHED_PROCESSING
 
 
 def _get_legacy_image(namespace, repo, tag, include_storage=True):
     repo_ref = registry_model.lookup_repository(namespace, repo)
-    repo_tag = registry_model.get_repo_tag(repo_ref, tag, include_legacy_image=True)
+    repo_tag = registry_model.get_repo_tag(repo_ref, tag)
-    return Image.get(id=repo_tag.legacy_image._db_id)
+    manifest = registry_model.get_manifest_for_tag(repo_tag)
-
+    return ManifestLegacyImage.get(manifest_id=manifest._db_id).image
-
-def _delete_tag(namespace, repo, tag):
-    repo_ref = registry_model.lookup_repository(namespace, repo)
-    registry_model.delete_tag(repo_ref, tag)
 
 
 class TestSecurityScanner(unittest.TestCase):
@@ -93,785 +75,24 @@ class TestSecurityScanner(unittest.TestCase):
         """
         Test for basic retrieval of layers from the security scanner.
         """
-        layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
+
+        repo_ref = registry_model.lookup_repository(ADMIN_ACCESS_USER, SIMPLE_REPO)
+        repo_tag = registry_model.get_repo_tag(repo_ref, "latest")
+        manifest = registry_model.get_manifest_for_tag(repo_tag)
+        registry_model.populate_legacy_images_for_testing(manifest, storage)
 
         with fake_security_scanner() as security_scanner:
             # Ensure the layer doesn't exist yet.
-            self.assertFalse(security_scanner.has_layer(security_scanner.layer_id(layer)))
+            self.assertFalse(security_scanner.has_layer(security_scanner.layer_id(manifest)))
-            self.assertIsNone(self.api.get_layer_data(layer))
+            self.assertIsNone(self.api.get_layer_data(manifest))
 
             # Add the layer.
-            security_scanner.add_layer(security_scanner.layer_id(layer))
+            security_scanner.add_layer(security_scanner.layer_id(manifest))
 
             # Retrieve the results.
-            result = self.api.get_layer_data(layer, include_vulnerabilities=True)
+            result = self.api.get_layer_data(manifest, include_vulnerabilities=True)
             self.assertIsNotNone(result)
-            self.assertEqual(result["Layer"]["Name"], security_scanner.layer_id(layer))
+            self.assertEquals(result["Layer"]["Name"], security_scanner.layer_id(manifest))
-
-    def test_analyze_layer_nodirectdownload_success(self):
-        """
-        Tests analyzing a layer when direct download is disabled.
-        """
-
-        # Disable direct download in fake storage.
-        storage.put_content(["local_us"], "supports_direct_download", b"false")
-
-        try:
-            app.register_blueprint(v2_bp, url_prefix="/v2")
-        except:
-            # Already registered.
-            pass
-
-        layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
-        self.assertFalse(layer.security_indexed)
-        self.assertEqual(-1, layer.security_indexed_engine)
-
-        # Ensure that the download is a registry+JWT download.
-        uri, auth_header = self.api._get_image_url_and_auth(layer)
-        self.assertIsNotNone(uri)
-        self.assertIsNotNone(auth_header)
-
-        # Ensure the download doesn't work without the header.
-        rv = self.app.head(uri)
-        self.assertEqual(rv.status_code, 401)
-
-        # Ensure the download works with the header. Note we use a HEAD here, as GET causes DB
-        # access which messes with the test runner's rollback.
-        rv = self.app.head(uri, headers=[("authorization", auth_header)])
-        self.assertEqual(rv.status_code, 200)
-
-        # Ensure the code works when called via analyze.
-        with fake_security_scanner() as security_scanner:
-            analyzer = LayerAnalyzer(app.config, self.api)
-            analyzer.analyze_recursively(layer)
-
-            layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
-            self.assertAnalyzed(layer, security_scanner, True, 1)
-
-    def test_analyze_layer_success(self):
-        """
-        Tests that analyzing a layer successfully marks it as analyzed.
-        """
-
-        layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
-        self.assertFalse(layer.security_indexed)
-        self.assertEqual(-1, layer.security_indexed_engine)
-
-        with fake_security_scanner() as security_scanner:
-            analyzer = LayerAnalyzer(app.config, self.api)
-            analyzer.analyze_recursively(layer)
-
-            layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
-            self.assertAnalyzed(layer, security_scanner, True, 1)
-
-    def test_analyze_layer_failure(self):
-        """
-        Tests that failing to analyze a layer (because it 422s) marks it as analyzed but failed.
-        """
-
-        layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
-        self.assertFalse(layer.security_indexed)
-        self.assertEqual(-1, layer.security_indexed_engine)
-
-        with fake_security_scanner() as security_scanner:
-            security_scanner.set_fail_layer_id(security_scanner.layer_id(layer))
-
-            analyzer = LayerAnalyzer(app.config, self.api)
-            analyzer.analyze_recursively(layer)
-
-            layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
-            self.assertAnalyzed(layer, security_scanner, False, 1)
-
-    def test_analyze_layer_internal_error(self):
-        """
-        Tests that failing to analyze a layer (because it 500s) marks it as not analyzed.
-        """
-
-        layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
-        self.assertFalse(layer.security_indexed)
-        self.assertEqual(-1, layer.security_indexed_engine)
-
-        with fake_security_scanner() as security_scanner:
-            security_scanner.set_internal_error_layer_id(security_scanner.layer_id(layer))
-
-            analyzer = LayerAnalyzer(app.config, self.api)
-            with self.assertRaises(APIRequestFailure):
-                analyzer.analyze_recursively(layer)
-
-            layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
-            self.assertAnalyzed(layer, security_scanner, False, -1)
-
-    def test_analyze_layer_error(self):
-        """
-        Tests that failing to analyze a layer (because it 400s) marks it as analyzed but failed.
-        """
-
-        layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
-        self.assertFalse(layer.security_indexed)
-        self.assertEqual(-1, layer.security_indexed_engine)
-
-        with fake_security_scanner() as security_scanner:
-            # Make is so trying to analyze the parent will fail with an error.
-            security_scanner.set_error_layer_id(security_scanner.layer_id(layer.parent))
-
-            # Try to the layer and its parents, but with one request causing an error.
-            analyzer = LayerAnalyzer(app.config, self.api)
-            analyzer.analyze_recursively(layer)
-
-            # Make sure it is marked as analyzed, but in a failed state.
-            layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
-            self.assertAnalyzed(layer, security_scanner, False, 1)
-
-    def test_analyze_layer_unexpected_status(self):
-        """
-        Tests that a response from a scanner with an unexpected status code fails correctly.
-        """
-
-        layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
-        self.assertFalse(layer.security_indexed)
-        self.assertEqual(-1, layer.security_indexed_engine)
-
-        with fake_security_scanner() as security_scanner:
-            # Make is so trying to analyze the parent will fail with an error.
-            security_scanner.set_unexpected_status_layer_id(security_scanner.layer_id(layer.parent))
-
-            # Try to the layer and its parents, but with one request causing an error.
-            analyzer = LayerAnalyzer(app.config, self.api)
-            with self.assertRaises(APIRequestFailure):
-                analyzer.analyze_recursively(layer)
-
-            # Make sure it isn't analyzed.
-            layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
-            self.assertAnalyzed(layer, security_scanner, False, -1)
-
-    def test_analyze_layer_missing_parent_handled(self):
-        """
-        Tests that a missing parent causes an automatic reanalysis, which succeeds.
-        """
-
-        layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
-        self.assertFalse(layer.security_indexed)
-        self.assertEqual(-1, layer.security_indexed_engine)
-
-        with fake_security_scanner() as security_scanner:
-            # Analyze the layer and its parents.
-            analyzer = LayerAnalyzer(app.config, self.api)
-            analyzer.analyze_recursively(layer)
-
-            # Make sure it was analyzed.
-            layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
-            self.assertAnalyzed(layer, security_scanner, True, 1)
-
-            # Mark the layer as not yet scanned.
-            layer.security_indexed_engine = IMAGE_NOT_SCANNED_ENGINE_VERSION
-            layer.security_indexed = False
-            layer.save()
-
-            # Remove the layer's parent entirely from the security scanner.
-            security_scanner.remove_layer(security_scanner.layer_id(layer.parent))
-
-            # Analyze again, which should properly re-analyze the missing parent and this layer.
-            analyzer.analyze_recursively(layer)
-
-            layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
-            self.assertAnalyzed(layer, security_scanner, True, 1)
-
-    def test_analyze_layer_invalid_parent(self):
-        """
-        Tests that trying to reanalyze a parent that is invalid causes the layer to be marked as
-        analyzed, but failed.
-        """
-
-        layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
-        self.assertFalse(layer.security_indexed)
-        self.assertEqual(-1, layer.security_indexed_engine)
-
-        with fake_security_scanner() as security_scanner:
-            # Analyze the layer and its parents.
-            analyzer = LayerAnalyzer(app.config, self.api)
-            analyzer.analyze_recursively(layer)
-
-            # Make sure it was analyzed.
-            layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
-            self.assertAnalyzed(layer, security_scanner, True, 1)
-
-            # Mark the layer as not yet scanned.
-            layer.security_indexed_engine = IMAGE_NOT_SCANNED_ENGINE_VERSION
-            layer.security_indexed = False
-            layer.save()
-
-            # Remove the layer's parent entirely from the security scanner.
-            security_scanner.remove_layer(security_scanner.layer_id(layer.parent))
-
-            # Make is so trying to analyze the parent will fail.
-            security_scanner.set_error_layer_id(security_scanner.layer_id(layer.parent))
-
-            # Try to analyze again, which should try to reindex the parent and fail.
-            analyzer.analyze_recursively(layer)
-
-            layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
-            self.assertAnalyzed(layer, security_scanner, False, 1)
-
-    def test_analyze_layer_unsupported_parent(self):
-        """
-        Tests that attempting to analyze a layer whose parent is unanalyzable, results in the layer
-        being marked as analyzed, but failed.
-        """
-
-        layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
-        self.assertFalse(layer.security_indexed)
-        self.assertEqual(-1, layer.security_indexed_engine)
-
-        with fake_security_scanner() as security_scanner:
-            # Make is so trying to analyze the parent will fail.
-            security_scanner.set_fail_layer_id(security_scanner.layer_id(layer.parent))
-
-            # Attempt to the layer and its parents. This should mark the layer itself as unanalyzable.
-            analyzer = LayerAnalyzer(app.config, self.api)
-            analyzer.analyze_recursively(layer)
-
-            layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
-            self.assertAnalyzed(layer, security_scanner, False, 1)
-
-    def test_analyze_layer_missing_storage(self):
-        """
-        Tests trying to analyze a layer with missing storage.
-        """
-
-        layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
-        self.assertFalse(layer.security_indexed)
-        self.assertEqual(-1, layer.security_indexed_engine)
-
-        # Delete the storage for the layer.
-        path = model.storage.get_layer_path(layer.storage)
-        locations = app.config["DISTRIBUTED_STORAGE_PREFERENCE"]
-        storage.remove(locations, path)
-        storage.remove(locations, "all_files_exist")
-
-        with fake_security_scanner() as security_scanner:
-            analyzer = LayerAnalyzer(app.config, self.api)
-            analyzer.analyze_recursively(layer)
-
-            layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
-            self.assertAnalyzed(layer, security_scanner, False, 1)
-
-    def assert_analyze_layer_notify(
-        self, security_indexed_engine, security_indexed, expect_notification
-    ):
-        layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
-        self.assertFalse(layer.security_indexed)
-        self.assertEqual(-1, layer.security_indexed_engine)
-
-        # Ensure there are no existing events.
-        self.assertIsNone(notification_queue.get())
-
-        # Add a repo event for the layer.
-        repo = model.repository.get_repository(ADMIN_ACCESS_USER, SIMPLE_REPO)
-        model.notification.create_repo_notification(
-            repo, "vulnerability_found", "quay_notification", {}, {"level": 100}
-        )
-
-        # Update the layer's state before analyzing.
-        layer.security_indexed_engine = security_indexed_engine
-        layer.security_indexed = security_indexed
-        layer.save()
-
-        with fake_security_scanner() as security_scanner:
-            security_scanner.set_vulns(
-                security_scanner.layer_id(layer),
-                [
-                    {
-                        "Name": "CVE-2014-9471",
-                        "Namespace": "debian:8",
-                        "Description": "Some service",
-                        "Link": "https://security-tracker.debian.org/tracker/CVE-2014-9471",
-                        "Severity": "Low",
-                        "FixedBy": "9.23-5",
-                    },
-                    {
-                        "Name": "CVE-2016-7530",
-                        "Namespace": "debian:8",
-                        "Description": "Some other service",
-                        "Link": "https://security-tracker.debian.org/tracker/CVE-2016-7530",
-                        "Severity": "Unknown",
-                        "FixedBy": "19.343-2",
-                    },
-                ],
-            )
-
-            analyzer = LayerAnalyzer(app.config, self.api)
-            analyzer.analyze_recursively(layer)
-
-            layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
-            self.assertAnalyzed(layer, security_scanner, True, 1)
-
-        # Ensure an event was written for the tag (if necessary).
-        time.sleep(1)
-        queue_item = notification_queue.get()
-
-        if expect_notification:
-            self.assertIsNotNone(queue_item)
-
-            body = json.loads(queue_item.body)
-            self.assertEqual(set(["latest", "prod"]), set(body["event_data"]["tags"]))
-            self.assertEqual("CVE-2014-9471", body["event_data"]["vulnerability"]["id"])
-            self.assertEqual("Low", body["event_data"]["vulnerability"]["priority"])
-            self.assertTrue(body["event_data"]["vulnerability"]["has_fix"])
-
-            self.assertEqual("CVE-2014-9471", body["event_data"]["vulnerabilities"][0]["id"])
-            self.assertEqual(2, len(body["event_data"]["vulnerabilities"]))
-
-            # Ensure we get the correct event message out as well.
-            event = VulnerabilityFoundEvent()
-            msg = "1 Low and 1 more vulnerabilities were detected in repository devtable/simple in 2 tags"
-            self.assertEqual(msg, event.get_summary(body["event_data"], {}))
-            self.assertEqual("info", event.get_level(body["event_data"], {}))
-        else:
-            self.assertIsNone(queue_item)
-
-        # Ensure its security indexed engine was updated.
-        updated_layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
-        self.assertEquals(updated_layer.id, layer.id)
-        self.assertTrue(updated_layer.security_indexed_engine > 0)
-
-    def test_analyze_layer_success_events(self):
-        # Not previously indexed at all => Notification
-        self.assert_analyze_layer_notify(IMAGE_NOT_SCANNED_ENGINE_VERSION, False, True)
-
-    def test_analyze_layer_success_no_notification(self):
-        # Previously successfully indexed => No notification
-        self.assert_analyze_layer_notify(0, True, False)
-
-    def test_analyze_layer_failed_then_success_notification(self):
-        # Previously failed to index => Notification
-        self.assert_analyze_layer_notify(0, False, True)
-
-    def test_notification_new_layers_not_vulnerable(self):
-        layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
-        layer_id = "%s.%s" % (layer.docker_image_id, layer.storage.uuid)
-
-        # Add a repo event for the layer.
-        repo = model.repository.get_repository(ADMIN_ACCESS_USER, SIMPLE_REPO)
-        model.notification.create_repo_notification(
-            repo, "vulnerability_found", "quay_notification", {}, {"level": 100}
-        )
-
-        # Ensure that there are no event queue items for the layer.
-        self.assertIsNone(notification_queue.get())
-
-        # Fire off the notification processing.
-        with fake_security_scanner() as security_scanner:
-            analyzer = LayerAnalyzer(app.config, self.api)
-            analyzer.analyze_recursively(layer)
-
-            layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
-            self.assertAnalyzed(layer, security_scanner, True, 1)
-
-            # Add a notification for the layer.
-            notification_data = security_scanner.add_notification([layer_id], [], {}, {})
-
-            # Process the notification.
-            self.assertTrue(process_notification_data(self.api, notification_data))
-
-            # Ensure that there are no event queue items for the layer.
-            self.assertIsNone(notification_queue.get())
-
-    def test_notification_delete(self):
-        layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
-        layer_id = "%s.%s" % (layer.docker_image_id, layer.storage.uuid)
-
-        # Add a repo event for the layer.
-        repo = model.repository.get_repository(ADMIN_ACCESS_USER, SIMPLE_REPO)
-        model.notification.create_repo_notification(
-            repo, "vulnerability_found", "quay_notification", {}, {"level": 100}
-        )
-
-        # Ensure that there are no event queue items for the layer.
-        self.assertIsNone(notification_queue.get())
-
-        # Fire off the notification processing.
-        with fake_security_scanner() as security_scanner:
-            analyzer = LayerAnalyzer(app.config, self.api)
-            analyzer.analyze_recursively(layer)
-
-            layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
-            self.assertAnalyzed(layer, security_scanner, True, 1)
-
-            # Add a notification for the layer.
-            notification_data = security_scanner.add_notification([layer_id], None, {}, None)
-
-            # Process the notification.
-            self.assertTrue(process_notification_data(self.api, notification_data))
-
-            # Ensure that there are no event queue items for the layer.
-            self.assertIsNone(notification_queue.get())
-
-    def test_notification_new_layers(self):
-        layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
-        layer_id = "%s.%s" % (layer.docker_image_id, layer.storage.uuid)
-
-        # Add a repo event for the layer.
-        repo = model.repository.get_repository(ADMIN_ACCESS_USER, SIMPLE_REPO)
-        model.notification.create_repo_notification(
-            repo, "vulnerability_found", "quay_notification", {}, {"level": 100}
-        )
-
-        # Ensure that there are no event queue items for the layer.
-        self.assertIsNone(notification_queue.get())
-
-        # Fire off the notification processing.
-        with fake_security_scanner() as security_scanner:
-            analyzer = LayerAnalyzer(app.config, self.api)
-            analyzer.analyze_recursively(layer)
-
-            layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
-            self.assertAnalyzed(layer, security_scanner, True, 1)
-
-            vuln_info = {
-                "Name": "CVE-TEST",
-                "Namespace": "debian:8",
-                "Description": "Some service",
-                "Link": "https://security-tracker.debian.org/tracker/CVE-2014-9471",
-                "Severity": "Low",
-                "FixedIn": {"Version": "9.23-5"},
-            }
-            security_scanner.set_vulns(layer_id, [vuln_info])
-
-            # Add a notification for the layer.
-            notification_data = security_scanner.add_notification(
-                [], [layer_id], vuln_info, vuln_info
-            )
-
-            # Process the notification.
-            self.assertTrue(process_notification_data(self.api, notification_data))
-
-            # Ensure an event was written for the tag.
-            time.sleep(1)
-            queue_item = notification_queue.get()
-            self.assertIsNotNone(queue_item)
-
-            item_body = json.loads(queue_item.body)
-            self.assertEqual(sorted(["prod", "latest"]), sorted(item_body["event_data"]["tags"]))
-            self.assertEqual("CVE-TEST", item_body["event_data"]["vulnerability"]["id"])
-            self.assertEqual("Low", item_body["event_data"]["vulnerability"]["priority"])
-            self.assertTrue(item_body["event_data"]["vulnerability"]["has_fix"])
-
-    def test_notification_no_new_layers(self):
-        layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
-
-        # Add a repo event for the layer.
-        repo = model.repository.get_repository(ADMIN_ACCESS_USER, SIMPLE_REPO)
-        model.notification.create_repo_notification(
-            repo, "vulnerability_found", "quay_notification", {}, {"level": 100}
-        )
-
-        # Ensure that there are no event queue items for the layer.
-        self.assertIsNone(notification_queue.get())
-
-        # Fire off the notification processing.
-        with fake_security_scanner() as security_scanner:
-            analyzer = LayerAnalyzer(app.config, self.api)
-            analyzer.analyze_recursively(layer)
-
-            layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
-            self.assertAnalyzed(layer, security_scanner, True, 1)
-
-            # Add a notification for the layer.
-            notification_data = security_scanner.add_notification([], [], {}, {})
-
-            # Process the notification.
-            self.assertTrue(process_notification_data(self.api, notification_data))
-
-            # Ensure that there are no event queue items for the layer.
-            self.assertIsNone(notification_queue.get())
-
-    def notification_tuple(self, notification):
-        # TODO: Replace this with a method once we refactor the notification stuff into its
-        # own module.
-        return AttrDict(
-            {
-                "event_config_dict": json.loads(notification.event_config_json),
-                "method_config_dict": json.loads(notification.config_json),
-            }
-        )
-
-    def test_notification_no_new_layers_increased_severity(self):
-        layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
-        layer_id = "%s.%s" % (layer.docker_image_id, layer.storage.uuid)
-
-        # Add a repo event for the layer.
-        repo = model.repository.get_repository(ADMIN_ACCESS_USER, SIMPLE_REPO)
-        notification = model.notification.create_repo_notification(
-            repo, "vulnerability_found", "quay_notification", {}, {"level": 100}
-        )
-
-        # Ensure that there are no event queue items for the layer.
-        self.assertIsNone(notification_queue.get())
-
-        # Fire off the notification processing.
-        with fake_security_scanner() as security_scanner:
-            analyzer = LayerAnalyzer(app.config, self.api)
-            analyzer.analyze_recursively(layer)
-
-            layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
-            self.assertAnalyzed(layer, security_scanner, True, 1)
-
-            old_vuln_info = {
-                "Name": "CVE-TEST",
-                "Namespace": "debian:8",
-                "Description": "Some service",
-                "Link": "https://security-tracker.debian.org/tracker/CVE-2014-9471",
-                "Severity": "Low",
-            }
-
-            new_vuln_info = {
-                "Name": "CVE-TEST",
-                "Namespace": "debian:8",
-                "Description": "Some service",
-                "Link": "https://security-tracker.debian.org/tracker/CVE-2014-9471",
-                "Severity": "Critical",
-                "FixedIn": {"Version": "9.23-5"},
-            }
-
-            security_scanner.set_vulns(layer_id, [new_vuln_info])
-
-            # Add a notification for the layer.
-            notification_data = security_scanner.add_notification(
-                [layer_id], [layer_id], old_vuln_info, new_vuln_info
-            )
-
-            # Process the notification.
-            self.assertTrue(process_notification_data(self.api, notification_data))
-
-            # Ensure an event was written for the tag.
-            time.sleep(1)
-            queue_item = notification_queue.get()
-            self.assertIsNotNone(queue_item)
-
-            item_body = json.loads(queue_item.body)
-            self.assertEqual(sorted(["prod", "latest"]), sorted(item_body["event_data"]["tags"]))
-            self.assertEqual("CVE-TEST", item_body["event_data"]["vulnerability"]["id"])
-            self.assertEqual("Critical", item_body["event_data"]["vulnerability"]["priority"])
-            self.assertTrue(item_body["event_data"]["vulnerability"]["has_fix"])
-
-            # Verify that an event would be raised.
-            event_data = item_body["event_data"]
-            notification = self.notification_tuple(notification)
-            self.assertTrue(VulnerabilityFoundEvent().should_perform(event_data, notification))
-
-            # Create another notification with a matching level and verify it will be raised.
-            notification = model.notification.create_repo_notification(
-                repo, "vulnerability_found", "quay_notification", {}, {"level": 1}
-            )
-
-            notification = self.notification_tuple(notification)
-            self.assertTrue(VulnerabilityFoundEvent().should_perform(event_data, notification))
-
-            # Create another notification with a higher level and verify it won't be raised.
-            notification = model.notification.create_repo_notification(
-                repo, "vulnerability_found", "quay_notification", {}, {"level": 0}
-            )
-            notification = self.notification_tuple(notification)
-            self.assertFalse(VulnerabilityFoundEvent().should_perform(event_data, notification))
-
-    def test_select_images_to_scan(self):
-        # Set all images to have a security index of a version to that of the config.
-        expected_version = app.config["SECURITY_SCANNER_ENGINE_VERSION_TARGET"]
-        Image.update(security_indexed_engine=expected_version).execute()
-
-        # Ensure no images are available for scanning.
-        self.assertIsNone(model.image.get_min_id_for_sec_scan(expected_version))
-        self.assertTrue(len(model.image.get_images_eligible_for_scan(expected_version)) == 0)
-
-        # Check for a higher version.
-        self.assertIsNotNone(model.image.get_min_id_for_sec_scan(expected_version + 1))
-        self.assertTrue(len(model.image.get_images_eligible_for_scan(expected_version + 1)) > 0)
-
-    def test_notification_worker(self):
-        layer1 = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
-        layer2 = _get_legacy_image(ADMIN_ACCESS_USER, COMPLEX_REPO, "prod", include_storage=True)
-
-        # Add a repo events for the layers.
-        simple_repo = model.repository.get_repository(ADMIN_ACCESS_USER, SIMPLE_REPO)
-        complex_repo = model.repository.get_repository(ADMIN_ACCESS_USER, COMPLEX_REPO)
-
-        model.notification.create_repo_notification(
-            simple_repo, "vulnerability_found", "quay_notification", {}, {"level": 100}
-        )
-        model.notification.create_repo_notification(
-            complex_repo, "vulnerability_found", "quay_notification", {}, {"level": 100}
-        )
-
-        # Ensure that there are no event queue items for the layer.
-        self.assertIsNone(notification_queue.get())
-
-        with fake_security_scanner() as security_scanner:
-            # Test with an unknown notification.
-            worker = SecurityNotificationWorker(None)
-            self.assertFalse(worker.perform_notification_work({"Name": "unknownnotification"}))
-
-            # Add some analyzed layers.
-            analyzer = LayerAnalyzer(app.config, self.api)
-            analyzer.analyze_recursively(layer1)
-            analyzer.analyze_recursively(layer2)
-
-            # Add a notification with pages of data.
-            new_vuln_info = {
-                "Name": "CVE-TEST",
-                "Namespace": "debian:8",
-                "Description": "Some service",
-                "Link": "https://security-tracker.debian.org/tracker/CVE-2014-9471",
-                "Severity": "Critical",
-                "FixedIn": {"Version": "9.23-5"},
-            }
-
-            security_scanner.set_vulns(security_scanner.layer_id(layer1), [new_vuln_info])
-            security_scanner.set_vulns(security_scanner.layer_id(layer2), [new_vuln_info])
-
-            layer_ids = [security_scanner.layer_id(layer1), security_scanner.layer_id(layer2)]
-            notification_data = security_scanner.add_notification(
-                [], layer_ids, None, new_vuln_info
-            )
-
-            # Test with a known notification with pages.
-            data = {
-                "Name": notification_data["Name"],
-            }
-
-            worker = SecurityNotificationWorker(None)
-            self.assertTrue(worker.perform_notification_work(data, layer_limit=2))
-
-            # Make sure all pages were processed by ensuring we have two notifications.
-            time.sleep(1)
-            self.assertIsNotNone(notification_queue.get())
-            self.assertIsNotNone(notification_queue.get())
-
-    def test_notification_worker_offset_pages_not_indexed(self):
-        # Try without indexes.
-        self.assert_notification_worker_offset_pages(indexed=False)
-
-    def test_notification_worker_offset_pages_indexed(self):
-        # Try with indexes.
-        self.assert_notification_worker_offset_pages(indexed=True)
-
-    def assert_notification_worker_offset_pages(self, indexed=False):
-        layer1 = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
-        layer2 = _get_legacy_image(ADMIN_ACCESS_USER, COMPLEX_REPO, "prod", include_storage=True)
-
-        # Add a repo events for the layers.
-        simple_repo = model.repository.get_repository(ADMIN_ACCESS_USER, SIMPLE_REPO)
-        complex_repo = model.repository.get_repository(ADMIN_ACCESS_USER, COMPLEX_REPO)
-
-        model.notification.create_repo_notification(
-            simple_repo, "vulnerability_found", "quay_notification", {}, {"level": 100}
-        )
-        model.notification.create_repo_notification(
-            complex_repo, "vulnerability_found", "quay_notification", {}, {"level": 100}
-        )
-
-        # Ensure that there are no event queue items for the layer.
-        self.assertIsNone(notification_queue.get())
-
-        with fake_security_scanner() as security_scanner:
-            # Test with an unknown notification.
-            worker = SecurityNotificationWorker(None)
-            self.assertFalse(worker.perform_notification_work({"Name": "unknownnotification"}))
-
-            # Add some analyzed layers.
-            analyzer = LayerAnalyzer(app.config, self.api)
-            analyzer.analyze_recursively(layer1)
-            analyzer.analyze_recursively(layer2)
-
-            # Add a notification with pages of data.
-            new_vuln_info = {
-                "Name": "CVE-TEST",
-                "Namespace": "debian:8",
-                "Description": "Some service",
-                "Link": "https://security-tracker.debian.org/tracker/CVE-2014-9471",
-                "Severity": "Critical",
-                "FixedIn": {"Version": "9.23-5"},
-            }
-
-            security_scanner.set_vulns(security_scanner.layer_id(layer1), [new_vuln_info])
-            security_scanner.set_vulns(security_scanner.layer_id(layer2), [new_vuln_info])
-
-            # Define offsetting sets of layer IDs, to test cross-pagination support. In this test, we
-            # will only serve 2 layer IDs per page: the first page will serve both of the 'New' layer IDs,
-            # but since the first 2 'Old' layer IDs are "earlier" than the shared ID of
-            # `devtable/simple:latest`, they won't get served in the 'New' list until the *second* page.
-            # The notification handling system should correctly not notify for this layer, even though it
-            # is marked 'New' on page 1 and marked 'Old' on page 2. Clair will served these
-            # IDs sorted in the same manner.
-            idx_old_layer_ids = [
-                {"LayerName": "old1", "Index": 1},
-                {"LayerName": "old2", "Index": 2},
-                {"LayerName": security_scanner.layer_id(layer1), "Index": 3},
-            ]
-
-            idx_new_layer_ids = [
-                {"LayerName": security_scanner.layer_id(layer1), "Index": 3},
-                {"LayerName": security_scanner.layer_id(layer2), "Index": 4},
-            ]
-
-            old_layer_ids = [t["LayerName"] for t in idx_old_layer_ids]
-            new_layer_ids = [t["LayerName"] for t in idx_new_layer_ids]
-
-            if not indexed:
-                idx_old_layer_ids = None
-                idx_new_layer_ids = None
-
-            notification_data = security_scanner.add_notification(
-                old_layer_ids,
-                new_layer_ids,
-                None,
-                new_vuln_info,
-                max_per_page=2,
-                indexed_old_layer_ids=idx_old_layer_ids,
-                indexed_new_layer_ids=idx_new_layer_ids,
-            )
-
-            # Test with a known notification with pages.
-            data = {
-                "Name": notification_data["Name"],
-            }
-
-            worker = SecurityNotificationWorker(None)
-            self.assertTrue(worker.perform_notification_work(data, layer_limit=2))
-
-            # Make sure all pages were processed by ensuring we have only one notification. If the second
-            # page was not processed, then the `Old` entry for layer1 will not be found, and we'd get two
-            # notifications.
-            time.sleep(1)
-            self.assertIsNotNone(notification_queue.get())
-            self.assertIsNone(notification_queue.get())
-
-    def test_layer_gc(self):
-        layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest", include_storage=True)
-
-        # Delete the prod tag so that only the `latest` tag remains.
-        _delete_tag(ADMIN_ACCESS_USER, SIMPLE_REPO, "prod")
-
-        with fake_security_scanner() as security_scanner:
-            # Analyze the layer.
-            analyzer = LayerAnalyzer(app.config, self.api)
-            analyzer.analyze_recursively(layer)
-
-            layer = _get_legacy_image(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
-            self.assertAnalyzed(layer, security_scanner, True, 1)
-            self.assertTrue(security_scanner.has_layer(security_scanner.layer_id(layer)))
-
-            namespace_user = model.user.get_user(ADMIN_ACCESS_USER)
-            model.user.change_user_tag_expiration(namespace_user, 0)
-
-            # Delete the tag in the repository and GC.
-            _delete_tag(ADMIN_ACCESS_USER, SIMPLE_REPO, "latest")
-            time.sleep(1)
-
-            repo = model.repository.get_repository(ADMIN_ACCESS_USER, SIMPLE_REPO)
-            model.gc.garbage_collect_repo(repo)
-
-            # Ensure that the security scanner no longer has the image.
-            self.assertFalse(security_scanner.has_layer(security_scanner.layer_id(layer)))
 
 
 if __name__ == "__main__":

test/testconfig.py

@@ -66,7 +66,6 @@ class TestConfig(DefaultConfig):
     SECURITY_SCANNER_ENGINE_VERSION_TARGET = 1
     SECURITY_SCANNER_API_TIMEOUT_SECONDS = 1
     SECURITY_SCANNER_V4_ENDPOINT = "http://fakesecurityscanner/"
-    SECURITY_SCANNER_V4_NAMESPACE_WHITELIST = ["devtable"]
 
     FEATURE_SIGNING = True
 

util/config/configutil.py

@@ -48,15 +48,6 @@ def add_enterprise_config_defaults(config_obj, current_secret_key):
     config_obj["REPO_MIRROR_TLS_VERIFY"] = config_obj.get("REPO_MIRROR_TLS_VERIFY", True)
     config_obj["REPO_MIRROR_SERVER_HOSTNAME"] = config_obj.get("REPO_MIRROR_SERVER_HOSTNAME", None)
 
-    # Default the signer config.
-    config_obj["GPG2_PRIVATE_KEY_FILENAME"] = config_obj.get(
-        "GPG2_PRIVATE_KEY_FILENAME", "signing-private.gpg"
-    )
-    config_obj["GPG2_PUBLIC_KEY_FILENAME"] = config_obj.get(
-        "GPG2_PUBLIC_KEY_FILENAME", "signing-public.gpg"
-    )
-    config_obj["SIGNING_ENGINE"] = config_obj.get("SIGNING_ENGINE", "gpg2")
-
     # Default security scanner config.
     config_obj["FEATURE_SECURITY_NOTIFICATIONS"] = config_obj.get(
         "FEATURE_SECURITY_NOTIFICATIONS", True

util/config/schema.py

@@ -18,6 +18,7 @@ INTERNAL_ONLY_PROPERTIES = {
     "FEATURE_REPOSITORY_ACTION_COUNTER",
     "APP_REGISTRY_PACKAGE_LIST_CACHE_WHITELIST",
     "APP_REGISTRY_SHOW_PACKAGE_CACHE_WHITELIST",
+    "FEATURE_MANIFEST_SIZE_BACKFILL",
     "TESTING",
     "SEND_FILE_MAX_AGE_DEFAULT",
     "DISABLED_FOR_AUDIT_LOGS",
@@ -29,7 +30,6 @@ INTERNAL_ONLY_PROPERTIES = {
     "REPLICATION_QUEUE_NAME",
     "DOCKERFILE_BUILD_QUEUE_NAME",
     "CHUNK_CLEANUP_QUEUE_NAME",
-    "SECSCAN_NOTIFICATION_QUEUE_NAME",
     "SECURITY_SCANNER_ISSUER_NAME",
     "NOTIFICATION_QUEUE_NAME",
     "REPOSITORY_GC_QUEUE_NAME",
@@ -57,7 +57,6 @@ INTERNAL_ONLY_PROPERTIES = {
     "JWTPROXY_AUDIENCE",
     "JWTPROXY_SIGNER",
     "SECURITY_SCANNER_INDEXING_MIN_ID",
-    "SECURITY_SCANNER_V4_NAMESPACE_WHITELIST",
     "SECURITY_SCANNER_V4_REINDEX_THRESHOLD",
     "STATIC_SITE_BUCKET",
     "LABEL_KEY_RESERVED_PREFIXES",

util/config/validator.py

@@ -12,7 +12,6 @@ from util.config.validators.validate_ldap import LDAPValidator
 from util.config.validators.validate_keystone import KeystoneValidator
 from util.config.validators.validate_jwt import JWTAuthValidator
 from util.config.validators.validate_secscan import SecurityScannerValidator
-from util.config.validators.validate_signer import SignerValidator
 from util.config.validators.validate_ssl import SSLValidator, SSL_FILENAMES
 from util.config.validators.validate_google_login import GoogleLoginValidator
 from util.config.validators.validate_bitbucket_trigger import BitbucketTriggerValidator
@@ -62,7 +61,6 @@ VALIDATORS = {
     LDAPValidator.name: LDAPValidator.validate,
     JWTAuthValidator.name: JWTAuthValidator.validate,
     KeystoneValidator.name: KeystoneValidator.validate,
-    SignerValidator.name: SignerValidator.validate,
     SecurityScannerValidator.name: SecurityScannerValidator.validate,
     OIDCLoginValidator.name: OIDCLoginValidator.validate,
     TimeMachineValidator.name: TimeMachineValidator.validate,

util/config/validators/test/test_validate_signer.py

@@ -1,24 +0,0 @@
-import pytest
-
-from util.config.validator import ValidatorContext
-from util.config.validators import ConfigValidationException
-from util.config.validators.validate_signer import SignerValidator
-
-from test.fixtures import *
-
-
-@pytest.mark.parametrize(
-    "unvalidated_config,expected",
-    [
-        ({}, None),
-        ({"SIGNING_ENGINE": "foobar"}, ConfigValidationException),
-        ({"SIGNING_ENGINE": "gpg2"}, Exception),
-    ],
-)
-def test_validate_signer(unvalidated_config, expected, app):
-    validator = SignerValidator()
-    if expected is not None:
-        with pytest.raises(expected):
-            validator.validate(ValidatorContext(unvalidated_config))
-    else:
-        validator.validate(ValidatorContext(unvalidated_config))