diff --git a/app.py b/app.py index a793360ec..13832074c 100644 --- a/app.py +++ b/app.py @@ -41,6 +41,7 @@ from data.userfiles import Userfiles from data.users import UserAuthentication from data.registry_model import registry_model from data.secscan_model import secscan_model +from image.oci import register_artifact_type from path_converters import ( RegexConverter, RepositoryPathConverter, @@ -129,6 +130,13 @@ if app.config["PREFERRED_URL_SCHEME"] == "https" and not app.config.get( # Load features from config. features.import_features(app.config) +# Register additional experimental artifact types. +# TODO: extract this into a real, dynamic registration system. +if features.EXPERIMENTAL_HELM_OCI_SUPPORT: + HELM_CHART_CONFIG_TYPE = "application/vnd.cncf.helm.config.v1+json" + HELM_CHART_LAYER_TYPES = ["application/tar+gzip"] + register_artifact_type(HELM_CHART_CONFIG_TYPE, HELM_CHART_LAYER_TYPES) + CONFIG_DIGEST = hashlib.sha256(json.dumps(app.config, default=str)).hexdigest()[0:8] logger.debug("Loaded config", extra={"config": app.config}) diff --git a/config.py b/config.py index 8ad9eb93a..d915f6c5d 100644 --- a/config.py +++ b/config.py @@ -714,3 +714,10 @@ class DefaultConfig(ImmutableConfig): # Feature Flag: Whether to clear expired RepositoryActionCount entries. FEATURE_CLEAR_EXPIRED_RAC_ENTRIES = False + + # Feature Flag: Whether OCI manifest support should be enabled generally. + FEATURE_GENERAL_OCI_SUPPORT = False + + # Feature Flag: Whether to allow Helm OCI content types. + # See: https://helm.sh/docs/topics/registries/ + FEATURE_EXPERIMENTAL_HELM_OCI_SUPPORT = False diff --git a/data/migrations/dba_operator/04b9d2191450-databasemigration.yaml b/data/migrations/dba_operator/04b9d2191450-databasemigration.yaml new file mode 100644 index 000000000..4ea5d9998 --- /dev/null +++ b/data/migrations/dba_operator/04b9d2191450-databasemigration.yaml @@ -0,0 +1,16 @@ + +--- +apiVersion: dbaoperator.app-sre.redhat.com/v1alpha1 +kind: DatabaseMigration +metadata: + name: 04b9d2191450 +spec: + migrationContainerSpec: + command: + - /quay-registry/quay-entrypoint.sh + - migrate + - 04b9d2191450 + image: quay.io/quay/quay + name: 04b9d2191450 + previous: 8e6a363784bb + schemaHints: [] diff --git a/data/migrations/versions/04b9d2191450_add_oci_content_types.py b/data/migrations/versions/04b9d2191450_add_oci_content_types.py new file mode 100644 index 000000000..73d07445f --- /dev/null +++ b/data/migrations/versions/04b9d2191450_add_oci_content_types.py @@ -0,0 +1,30 @@ +from image.oci import OCI_CONTENT_TYPES + +"""Add OCI content types + +Revision ID: 04b9d2191450 +Revises: 8e6a363784bb +Create Date: 2020-03-23 16:03:39.789177 + +""" + +# revision identifiers, used by Alembic. +revision = "04b9d2191450" +down_revision = "8e6a363784bb" + +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + + +def upgrade(op, tables, tester): + for media_type in OCI_CONTENT_TYPES: + op.bulk_insert(tables.mediatype, [{"name": media_type},]) + + +def downgrade(op, tables, tester): + for media_type in OCI_CONTENT_TYPES: + op.execute( + tables.mediatype.delete().where( + tables.mediatype.c.name == op.inline_literal(media_type) + ) + ) diff --git a/data/model/oci/manifest.py b/data/model/oci/manifest.py index 534638f57..065c5db27 100644 --- a/data/model/oci/manifest.py +++ b/data/model/oci/manifest.py @@ -272,9 +272,19 @@ def _create_manifest( # image. legacy_image = None if manifest_interface_instance.has_legacy_image: - legacy_image_id = _populate_legacy_image( - repository_id, manifest_interface_instance, blob_map, retriever, raise_on_error - ) + try: + legacy_image_id = _populate_legacy_image( + repository_id, manifest_interface_instance, blob_map, retriever, raise_on_error + ) + except ManifestException as me: + logger.error("Got manifest error when populating legacy images: %s", me) + if raise_on_error: + raise CreateManifestException( + "Attempt to create an invalid manifest. Please report this issue." + ) + + return None + if legacy_image_id is None: return None diff --git a/data/registry_model/interface.py b/data/registry_model/interface.py index a851bb9c2..227ab2b76 100644 --- a/data/registry_model/interface.py +++ b/data/registry_model/interface.py @@ -11,12 +11,6 @@ class RegistryDataInterface(object): Manifests, Blobs, Images, and Labels. """ - @abstractmethod - def supports_schema2(self, namespace_name): - """ - Returns whether the implementation of the data interface supports schema 2 format manifests. - """ - @abstractmethod def get_tag_legacy_image_id(self, repository_ref, tag_name, storage): """ diff --git a/data/registry_model/registry_oci_model.py b/data/registry_model/registry_oci_model.py index ad4420acf..bd4642cc4 100644 --- a/data/registry_model/registry_oci_model.py +++ b/data/registry_model/registry_oci_model.py @@ -48,12 +48,6 @@ class OCIModel(RegistryDataInterface): changed to support the OCI specification. """ - def supports_schema2(self, namespace_name): - """ - Returns whether the implementation of the data interface supports schema 2 format manifests. - """ - return True - def get_tag_legacy_image_id(self, repository_ref, tag_name, storage): """ Returns the legacy image ID for the tag with a legacy images in the repository. diff --git a/endpoints/v2/manifest.py b/endpoints/v2/manifest.py index a94f7109a..d1d9b6b9f 100644 --- a/endpoints/v2/manifest.py +++ b/endpoints/v2/manifest.py @@ -25,7 +25,8 @@ from endpoints.v2.errors import ( from image.shared import ManifestException from image.shared.schemas import parse_manifest_from_bytes from image.docker.schema1 import DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE, DOCKER_SCHEMA1_CONTENT_TYPES -from image.docker.schema2 import DOCKER_SCHEMA2_CONTENT_TYPES, OCI_CONTENT_TYPES +from image.docker.schema2 import DOCKER_SCHEMA2_CONTENT_TYPES +from image.oci import OCI_CONTENT_TYPES from notifications import spawn_notification from util.audit import track_and_log from util.bytes import Bytes @@ -165,15 +166,23 @@ def _reject_manifest2_schema2(func): request.content_type and request.content_type != "application/json" and request.content_type - not in DOCKER_SCHEMA1_CONTENT_TYPES | DOCKER_SCHEMA2_CONTENT_TYPES + not in DOCKER_SCHEMA1_CONTENT_TYPES | DOCKER_SCHEMA2_CONTENT_TYPES | OCI_CONTENT_TYPES ): raise ManifestInvalid( detail={"message": "manifest schema version not supported"}, http_status_code=415 ) - if registry_model.supports_schema2(namespace_name) and namespace_name not in app.config.get( - "V22_NAMESPACE_BLACKLIST", [] - ): + if namespace_name not in app.config.get("V22_NAMESPACE_BLACKLIST", []): + if request.content_type in OCI_CONTENT_TYPES: + if ( + namespace_name not in app.config.get("OCI_NAMESPACE_WHITELIST", []) + and not features.GENERAL_OCI_SUPPORT + ): + raise ManifestInvalid( + detail={"message": "manifest schema version not supported"}, + http_status_code=415, + ) + return func(*args, **kwargs) if ( diff --git a/image/docker/schema1.py b/image/docker/schema1.py index 4b95bdb4d..156624f3f 100644 --- a/image/docker/schema1.py +++ b/image/docker/schema1.py @@ -437,6 +437,7 @@ class DockerSchema1Manifest(ManifestInterface): "Could not parse metadata string: %s" % metadata_string ) + v1_metadata = v1_metadata or {} container_config = v1_metadata.get("container_config") or {} command_list = container_config.get("Cmd", None) command = to_canonical_json(command_list) if command_list else None diff --git a/image/docker/schema2/__init__.py b/image/docker/schema2/__init__.py index 9e2170518..240c1d681 100644 --- a/image/docker/schema2/__init__.py +++ b/image/docker/schema2/__init__.py @@ -17,14 +17,10 @@ DOCKER_SCHEMA2_REMOTE_LAYER_CONTENT_TYPE = ( DOCKER_SCHEMA2_CONFIG_CONTENT_TYPE = "application/vnd.docker.container.image.v1+json" -OCI_MANIFEST_CONTENT_TYPE = "application/vnd.oci.image.manifest.v1+json" -OCI_MANIFESTLIST_CONTENT_TYPE = "application/vnd.oci.image.index.v1+json" - DOCKER_SCHEMA2_CONTENT_TYPES = { DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE, DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE, } -OCI_CONTENT_TYPES = {OCI_MANIFEST_CONTENT_TYPE, OCI_MANIFESTLIST_CONTENT_TYPE} # The magical digest to be used for "empty" layers. # https://github.com/docker/distribution/blob/749f6afb4572201e3c37325d0ffedb6f32be8950/manifest/schema1/config_builder.go#L22 diff --git a/image/docker/schema2/list.py b/image/docker/schema2/list.py index 9cba9d0ca..b2bfbe757 100644 --- a/image/docker/schema2/list.py +++ b/image/docker/schema2/list.py @@ -7,6 +7,7 @@ from jsonschema import validate as validate_schema, ValidationError from digest import digest_tools from image.shared import ManifestException from image.shared.interfaces import ManifestInterface +from image.shared.schemautil import LazyManifestLoader from image.docker.schema1 import DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE from image.docker.schema1 import DockerSchema1Manifest from image.docker.schema2 import ( @@ -52,48 +53,6 @@ class MismatchManifestException(MalformedSchema2ManifestList): pass -class LazyManifestLoader(object): - def __init__(self, manifest_data, content_retriever): - self._manifest_data = manifest_data - self._content_retriever = content_retriever - self._loaded_manifest = None - - @property - def manifest_obj(self): - if self._loaded_manifest is not None: - return self._loaded_manifest - - self._loaded_manifest = self._load_manifest() - return self._loaded_manifest - - def _load_manifest(self): - digest = self._manifest_data[DOCKER_SCHEMA2_MANIFESTLIST_DIGEST_KEY] - size = self._manifest_data[DOCKER_SCHEMA2_MANIFESTLIST_SIZE_KEY] - manifest_bytes = self._content_retriever.get_manifest_bytes_with_digest(digest) - if manifest_bytes is None: - raise MalformedSchema2ManifestList( - "Could not find child manifest with digest `%s`" % digest - ) - - if len(manifest_bytes) != size: - raise MalformedSchema2ManifestList( - "Size of manifest does not match that retrieved: %s vs %s", - len(manifest_bytes), - size, - ) - - content_type = self._manifest_data[DOCKER_SCHEMA2_MANIFESTLIST_MEDIATYPE_KEY] - if content_type == DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE: - return DockerSchema2Manifest(Bytes.for_string_or_unicode(manifest_bytes)) - - if content_type == DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE: - return DockerSchema1Manifest( - Bytes.for_string_or_unicode(manifest_bytes), validate=False - ) - - raise MalformedSchema2ManifestList("Unknown manifest content type") - - class DockerSchema2ManifestList(ManifestInterface): METASCHEMA = { "type": "object", @@ -275,7 +234,20 @@ class DockerSchema2ManifestList(ManifestInterface): Returns the manifests in the list. """ manifests = self._parsed[DOCKER_SCHEMA2_MANIFESTLIST_MANIFESTS_KEY] - return [LazyManifestLoader(m, content_retriever) for m in manifests] + supported_types = {} + supported_types[DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE] = DockerSchema1Manifest + supported_types[DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE] = DockerSchema2Manifest + return [ + LazyManifestLoader( + m, + content_retriever, + supported_types, + DOCKER_SCHEMA2_MANIFESTLIST_DIGEST_KEY, + DOCKER_SCHEMA2_MANIFESTLIST_SIZE_KEY, + DOCKER_SCHEMA2_MANIFESTLIST_MEDIATYPE_KEY, + ) + for m in manifests + ] def validate(self, content_retriever): """ diff --git a/image/docker/schema2/manifest.py b/image/docker/schema2/manifest.py index af1d2c7b4..361963c73 100644 --- a/image/docker/schema2/manifest.py +++ b/image/docker/schema2/manifest.py @@ -144,7 +144,7 @@ class DockerSchema2Manifest(ManifestInterface): ], } - def __init__(self, manifest_bytes): + def __init__(self, manifest_bytes, validate=False): assert isinstance(manifest_bytes, Bytes) self._payload = manifest_bytes diff --git a/image/oci/__init__.py b/image/oci/__init__.py new file mode 100644 index 000000000..7f0acd292 --- /dev/null +++ b/image/oci/__init__.py @@ -0,0 +1,37 @@ +OCI_IMAGE_MANIFEST_CONTENT_TYPE = "application/vnd.oci.image.manifest.v1+json" +OCI_IMAGE_INDEX_CONTENT_TYPE = "application/vnd.oci.image.index.v1+json" +OCI_IMAGE_CONFIG_CONTENT_TYPE = "application/vnd.oci.image.config.v1+json" + +OCI_IMAGE_TAR_LAYER_CONTENT_TYPE = "application/vnd.oci.image.layer.v1.tar" +OCI_IMAGE_TAR_GZIP_LAYER_CONTENT_TYPE = "application/vnd.oci.image.layer.v1.tar+gzip" + +OCI_IMAGE_DISTRIBUTABLE_LAYER_CONTENT_TYPES = [ + OCI_IMAGE_TAR_LAYER_CONTENT_TYPE, + OCI_IMAGE_TAR_GZIP_LAYER_CONTENT_TYPE, +] + +OCI_IMAGE_TAR_NON_DISTRIBUTABLE_LAYER_CONTENT_TYPE = ( + "application/vnd.oci.image.layer.nondistributable.v1.tar" +) +OCI_IMAGE_TAR_GZIP_NON_DISTRIBUTABLE_LAYER_CONTENT_TYPE = ( + "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip" +) + +OCI_IMAGE_NON_DISTRIBUTABLE_LAYER_CONTENT_TYPES = [ + OCI_IMAGE_TAR_NON_DISTRIBUTABLE_LAYER_CONTENT_TYPE, + OCI_IMAGE_TAR_GZIP_NON_DISTRIBUTABLE_LAYER_CONTENT_TYPE, +] + +OCI_IMAGE_LAYER_CONTENT_TYPES = ( + OCI_IMAGE_DISTRIBUTABLE_LAYER_CONTENT_TYPES + OCI_IMAGE_NON_DISTRIBUTABLE_LAYER_CONTENT_TYPES +) + +OCI_CONTENT_TYPES = {OCI_IMAGE_MANIFEST_CONTENT_TYPE, OCI_IMAGE_INDEX_CONTENT_TYPE} + +ALLOWED_ARTIFACT_TYPES = [OCI_IMAGE_CONFIG_CONTENT_TYPE] +ADDITIONAL_LAYER_CONTENT_TYPES = [] + + +def register_artifact_type(artifact_config_type, artifact_layer_types): + ALLOWED_ARTIFACT_TYPES.append(artifact_config_type) + ADDITIONAL_LAYER_CONTENT_TYPES.extend(artifact_layer_types) diff --git a/image/oci/config.py b/image/oci/config.py new file mode 100644 index 000000000..dc3f5bb03 --- /dev/null +++ b/image/oci/config.py @@ -0,0 +1,304 @@ +""" +Implements validation and conversion for the OCI config JSON. + +See: https://github.com/opencontainers/image-spec/blob/master/config.md + +Example: +{ + "created": "2015-10-31T22:22:56.015925234Z", + "author": "Alyssa P. Hacker ", + "architecture": "amd64", + "os": "linux", + "config": { + "User": "alice", + "ExposedPorts": { + "8080/tcp": {} + }, + "Env": [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "FOO=oci_is_a", + "BAR=well_written_spec" + ], + "Entrypoint": [ + "/bin/my-app-binary" + ], + "Cmd": [ + "--foreground", + "--config", + "/etc/my-app.d/default.cfg" + ], + "Volumes": { + "/var/job-result-data": {}, + "/var/log/my-app-logs": {} + }, + "WorkingDir": "/home/alice", + "Labels": { + "com.example.project.git.url": "https://example.com/project.git", + "com.example.project.git.commit": "45a939b2999782a3f005621a8d0f29aa387e1d6b" + } + }, + "rootfs": { + "diff_ids": [ + "sha256:c6f988f4874bb0add23a778f753c65efe992244e148a1d2ec2a8b664fb66bbd1", + "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef" + ], + "type": "layers" + }, + "history": [ + { + "created": "2015-10-31T22:22:54.690851953Z", + "created_by": "/bin/sh -c #(nop) ADD file:a3bc1e842b69636f9df5256c49c5374fb4eef1e281fe3f282c65fb853ee171c5 in /" + }, + { + "created": "2015-10-31T22:22:55.613815829Z", + "created_by": "/bin/sh -c #(nop) CMD [\"sh\"]", + "empty_layer": true + } + ] +} +""" + +import copy +import json +import hashlib + +from collections import namedtuple +from jsonschema import validate as validate_schema, ValidationError +from dateutil.parser import parse as parse_date + +from digest import digest_tools +from image.shared import ManifestException +from util.bytes import Bytes + + +CONFIG_HISTORY_KEY = "history" +CONFIG_ROOTFS_KEY = "rootfs" +CONFIG_CREATED_KEY = "created" +CONFIG_CREATED_BY_KEY = "created_by" +CONFIG_COMMENT_KEY = "comment" +CONFIG_AUTHOR_KEY = "author" +CONFIG_EMPTY_LAYER_KEY = "empty_layer" +CONFIG_TYPE_KEY = "type" +CONFIG_ARCHITECTURE_KEY = "architecture" +CONFIG_OS_KEY = "os" +CONFIG_CONFIG_KEY = "config" +CONFIG_DIFF_IDS_KEY = "diff_ids" + + +LayerHistory = namedtuple( + "LayerHistory", + ["created", "created_datetime", "command", "is_empty", "author", "comment", "raw_entry"], +) + + +class MalformedConfig(ManifestException): + """ + Raised when a config fails an assertion that should be true according to the + OCI Config Specification. + """ + + pass + + +class OCIConfig(object): + METASCHEMA = { + "type": "object", + "description": "The container configuration found in an OCI manifest", + "required": [CONFIG_ROOTFS_KEY, CONFIG_ARCHITECTURE_KEY, CONFIG_OS_KEY], + "properties": { + CONFIG_CREATED_KEY: { + "type": "string", + "description": "An combined date and time at which the image was created, formatted as defined by RFC 3339, section 5.6.", + }, + CONFIG_AUTHOR_KEY: { + "type": "string", + "description": "Gives the name and/or email address of the person or entity which created and is responsible for maintaining the image.", + }, + CONFIG_ARCHITECTURE_KEY: { + "type": "string", + "description": "The CPU architecture which the binaries in this image are built to run on. Configurations SHOULD use, and implementations SHOULD understand, values listed in the Go Language document for GOARCH.", + }, + CONFIG_OS_KEY: { + "type": "string", + "description": "The name of the operating system which the image is built to run on. Configurations SHOULD use, and implementations SHOULD understand, values listed in the Go Language document for GOOS.", + }, + CONFIG_CONFIG_KEY: { + "type": ["object", "null"], + "description": "The execution parameters which SHOULD be used as a base when running a container using the image", + "properties": { + "User": {"type": "string"}, + "ExposedPorts": {"type": "object"}, + "Env": {"type": "array"}, + "Entrypoint": {"type": "array"}, + "Cmd": {"type": "array"}, + "Volumes": {"type": "object"}, + "WorkingDir": {"type": "string"}, + "Labels": {"type": "object"}, + "StopSignal": {"type": "string"}, + }, + "additionalProperties": True, + }, + CONFIG_ROOTFS_KEY: { + "type": "object", + "description": "Describes the root filesystem for this image", + "properties": { + CONFIG_TYPE_KEY: { + "type": "string", + "description": "MUST be set to layers.", + "enum": ["layers"], + }, + CONFIG_DIFF_IDS_KEY: { + "type": "array", + "description": "An array of layer content hashes (DiffIDs), in order from first to last.", + "items": {"type": "string",}, + }, + }, + "required": [CONFIG_TYPE_KEY, CONFIG_DIFF_IDS_KEY], + "additionalProperties": True, + }, + CONFIG_HISTORY_KEY: { + "type": "array", + "description": "Describes the history of each layer. The array is ordered from first to last", + "items": { + "type": "object", + "properties": { + CONFIG_EMPTY_LAYER_KEY: { + "type": "boolean", + "description": "If present, this layer is empty", + }, + CONFIG_CREATED_KEY: { + "type": "string", + "description": "The date/time that the layer was created", + "format": "date-time", + "x-example": "2018-04-03T18:37:09.284840891Z", + }, + CONFIG_CREATED_BY_KEY: { + "type": "string", + "description": "The command used to create the layer", + "x-example": "\/bin\/sh -c #(nop) ADD file:somesha in /", + }, + CONFIG_COMMENT_KEY: { + "type": "string", + "description": "Comment describing the layer", + }, + CONFIG_AUTHOR_KEY: { + "type": "string", + "description": "The author of the layer", + }, + }, + "additionalProperties": True, + }, + }, + }, + "additionalProperties": True, + } + + def __init__(self, config_bytes): + assert isinstance(config_bytes, Bytes) + + self._config_bytes = config_bytes + + try: + self._parsed = json.loads(config_bytes.as_unicode()) + except ValueError as ve: + raise MalformedConfig("malformed config data: %s" % ve) + + try: + validate_schema(self._parsed, OCIConfig.METASCHEMA) + except ValidationError as ve: + raise MalformedConfig("config data does not match schema: %s" % ve) + + @property + def digest(self): + """ + Returns the digest of this config object. + """ + return digest_tools.sha256_digest(self._config_bytes.as_encoded_str()) + + @property + def size(self): + """ + Returns the size of this config object. + """ + return len(self._config_bytes.as_encoded_str()) + + @property + def bytes(self): + """ + Returns the bytes of this config object. + """ + return self._config_bytes + + @property + def labels(self): + """ + Returns a dictionary of all the labels defined in this configuration. + """ + return self._parsed.get("config", {}).get("Labels", {}) or {} + + @property + def has_empty_layer(self): + """ + Returns whether this config contains an empty layer. + """ + history = self._parsed.get(CONFIG_HISTORY_KEY) or [] + for history_entry in history: + if history_entry.get(CONFIG_EMPTY_LAYER_KEY, False): + return True + + return False + + @property + def history(self): + """ + Returns the history of the image, started at the base layer. + """ + history = self._parsed.get(CONFIG_HISTORY_KEY) or [] + for history_entry in history: + created_datetime_str = history_entry.get(CONFIG_CREATED_KEY) + created_datetime = parse_date(created_datetime_str) if created_datetime_str else None + yield LayerHistory( + created_datetime=created_datetime, + created=history_entry.get(CONFIG_CREATED_KEY), + command=history_entry.get(CONFIG_CREATED_BY_KEY), + author=history_entry.get(CONFIG_AUTHOR_KEY), + comment=history_entry.get(CONFIG_COMMENT_KEY), + is_empty=history_entry.get(CONFIG_EMPTY_LAYER_KEY, False), + raw_entry=history_entry, + ) + + def build_v1_compatibility(self, history, v1_id, v1_parent_id, is_leaf, compressed_size=None): + """ + Builds the V1 compatibility block for the given layer. + """ + # If the layer is the leaf, it gets the full config (minus 2 fields). Otherwise, it gets only + # IDs. + v1_compatibility = copy.deepcopy(self._parsed) if is_leaf else {} + v1_compatibility["id"] = v1_id + if v1_parent_id is not None: + v1_compatibility["parent"] = v1_parent_id + + if "created" not in v1_compatibility and history.created: + v1_compatibility["created"] = history.created + + if "author" not in v1_compatibility and history.author: + v1_compatibility["author"] = history.author + + if "comment" not in v1_compatibility and history.comment: + v1_compatibility["comment"] = history.comment + + if "throwaway" not in v1_compatibility and history.is_empty: + v1_compatibility["throwaway"] = True + + if "container_config" not in v1_compatibility: + v1_compatibility["container_config"] = { + "Cmd": [history.command], + } + + if compressed_size is not None: + v1_compatibility["Size"] = compressed_size + + # The history and rootfs keys are OCI-config specific. + v1_compatibility.pop(CONFIG_HISTORY_KEY, None) + v1_compatibility.pop(CONFIG_ROOTFS_KEY, None) + return v1_compatibility diff --git a/image/oci/descriptor.py b/image/oci/descriptor.py new file mode 100644 index 000000000..589e9f424 --- /dev/null +++ b/image/oci/descriptor.py @@ -0,0 +1,48 @@ +DESCRIPTOR_MEDIATYPE_KEY = "mediaType" +DESCRIPTOR_SIZE_KEY = "size" +DESCRIPTOR_DIGEST_KEY = "digest" +DESCRIPTOR_URLS_KEY = "urls" +DESCRIPTOR_ANNOTATIONS_KEY = "annotations" + + +def get_descriptor_schema( + allowed_media_types, additional_properties=None, additional_required=None +): + properties = { + DESCRIPTOR_MEDIATYPE_KEY: { + "type": "string", + "description": "The MIME type of the referenced manifest", + "enum": allowed_media_types, + }, + DESCRIPTOR_SIZE_KEY: { + "type": "number", + "description": "The size in bytes of the object. This field exists so that a " + + "client will have an expected size for the content before " + + "validating. If the length of the retrieved content does not " + + "match the specified length, the content should not be trusted.", + }, + DESCRIPTOR_DIGEST_KEY: { + "type": "string", + "description": "The content addressable digest of the manifest in the blob store", + }, + DESCRIPTOR_ANNOTATIONS_KEY: { + "type": "object", + "description": "The annotations, if any, on this descriptor", + "additionalProperties": True, + }, + DESCRIPTOR_URLS_KEY: { + "type": "array", + "description": "This OPTIONAL property specifies a list of URIs from which this object MAY be downloaded. Each entry MUST conform to RFC 3986. Entries SHOULD use the http and https schemes, as defined in RFC 7230.", + "items": {"type": "string",}, + }, + } + + if additional_properties: + properties.update(additional_properties) + + return { + "type": "object", + "properties": properties, + "required": [DESCRIPTOR_MEDIATYPE_KEY, DESCRIPTOR_SIZE_KEY, DESCRIPTOR_DIGEST_KEY,] + + (additional_required or []), + } diff --git a/image/oci/index.py b/image/oci/index.py new file mode 100644 index 000000000..dfabcd39c --- /dev/null +++ b/image/oci/index.py @@ -0,0 +1,330 @@ +""" +Implements validation and conversion for the OCI Index JSON. + +See: https://github.com/opencontainers/image-spec/blob/master/image-index.md + +Example: +{ + "schemaVersion": 2, + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": 7143, + "digest": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f", + "platform": { + "architecture": "ppc64le", + "os": "linux" + } + }, + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": 7682, + "digest": "sha256:5b0bcabd1ed22e9fb1310cf6c2dec7cdef19f0ad69efa1f392e94a4333501270", + "platform": { + "architecture": "amd64", + "os": "linux" + } + } + ], + "annotations": { + "com.example.key1": "value1", + "com.example.key2": "value2" + } +} +""" + +import logging +import json + +from cachetools.func import lru_cache +from jsonschema import validate as validate_schema, ValidationError + +from digest import digest_tools +from image.shared import ManifestException +from image.shared.interfaces import ManifestInterface +from image.shared.schemautil import LazyManifestLoader +from image.oci import OCI_IMAGE_INDEX_CONTENT_TYPE, OCI_IMAGE_MANIFEST_CONTENT_TYPE +from image.oci.descriptor import get_descriptor_schema +from util.bytes import Bytes + + +logger = logging.getLogger(__name__) + +# Keys. +INDEX_VERSION_KEY = "schemaVersion" +INDEX_MEDIATYPE_KEY = "mediaType" +INDEX_SIZE_KEY = "size" +INDEX_DIGEST_KEY = "digest" +INDEX_URLS_KEY = "urls" +INDEX_MANIFESTS_KEY = "manifests" +INDEX_PLATFORM_KEY = "platform" +INDEX_ARCHITECTURE_KEY = "architecture" +INDEX_OS_KEY = "os" +INDEX_OS_VERSION_KEY = "os.version" +INDEX_OS_FEATURES_KEY = "os.features" +INDEX_FEATURES_KEY = "features" +INDEX_VARIANT_KEY = "variant" +INDEX_ANNOTATIONS_KEY = "annotations" + + +class MalformedIndex(ManifestException): + """ + Raised when a index fails an assertion that should be true according to the OCI Index spec. + """ + + pass + + +class OCIIndex(ManifestInterface): + METASCHEMA = { + "type": "object", + "properties": { + INDEX_VERSION_KEY: { + "type": "number", + "description": "The version of the index. Must always be `2`.", + "minimum": 2, + "maximum": 2, + }, + INDEX_MEDIATYPE_KEY: { + "type": "string", + "description": "The media type of the index.", + "enum": [OCI_IMAGE_INDEX_CONTENT_TYPE], + }, + INDEX_MANIFESTS_KEY: { + "type": "array", + "description": "The manifests field contains a list of manifests for specific platforms", + "items": get_descriptor_schema( + allowed_media_types=[ + OCI_IMAGE_MANIFEST_CONTENT_TYPE, + OCI_IMAGE_INDEX_CONTENT_TYPE, + ], + additional_properties={ + INDEX_PLATFORM_KEY: { + "type": "object", + "description": "The platform object describes the platform which the image in " + + "the manifest runs on", + "properties": { + INDEX_ARCHITECTURE_KEY: { + "type": "string", + "description": "Specifies the CPU architecture, for example amd64 or ppc64le.", + }, + INDEX_OS_KEY: { + "type": "string", + "description": "Specifies the operating system, for example linux or windows", + }, + INDEX_OS_VERSION_KEY: { + "type": "string", + "description": "Specifies the operating system version, for example 10.0.10586", + }, + INDEX_OS_FEATURES_KEY: { + "type": "array", + "description": "specifies an array of strings, each listing a required OS " + + "feature (for example on Windows win32k)", + "items": {"type": "string",}, + }, + INDEX_VARIANT_KEY: { + "type": "string", + "description": "Specifies a variant of the CPU, for example armv6l to specify " + + "a particular CPU variant of the ARM CPU", + }, + INDEX_FEATURES_KEY: { + "type": "array", + "description": "specifies an array of strings, each listing a required CPU " + + "feature (for example sse4 or aes).", + "items": {"type": "string",}, + }, + }, + "required": [INDEX_ARCHITECTURE_KEY, INDEX_OS_KEY,], + }, + }, + additional_required=[INDEX_PLATFORM_KEY], + ), + }, + INDEX_ANNOTATIONS_KEY: { + "type": "object", + "description": "The annotations, if any, on this index", + "additionalProperties": True, + }, + }, + "required": [INDEX_VERSION_KEY, INDEX_MANIFESTS_KEY,], + } + + def __init__(self, manifest_bytes): + assert isinstance(manifest_bytes, Bytes) + + self._layers = None + self._manifest_bytes = manifest_bytes + + try: + self._parsed = json.loads(manifest_bytes.as_unicode()) + except ValueError as ve: + raise MalformedIndex("malformed manifest data: %s" % ve) + + try: + validate_schema(self._parsed, OCIIndex.METASCHEMA) + except ValidationError as ve: + raise MalformedIndex("manifest data does not match schema: %s" % ve) + + @property + def is_manifest_list(self): + """ + Returns whether this manifest is a list. + """ + return True + + @property + def schema_version(self): + return 2 + + @property + def digest(self): + """ + The digest of the manifest, including type prefix. + """ + return digest_tools.sha256_digest(self._manifest_bytes.as_encoded_str()) + + @property + def media_type(self): + """ + The media type of the schema. + """ + return OCI_IMAGE_INDEX_CONTENT_TYPE + + @property + def manifest_dict(self): + """ + Returns the manifest as a dictionary ready to be serialized to JSON. + """ + return self._parsed + + @property + def bytes(self): + return self._manifest_bytes + + def get_layers(self, content_retriever): + """ + Returns the layers of this manifest, from base to leaf or None if this kind of manifest does + not support layers. + """ + return None + + @property + def blob_digests(self): + # Manifest lists have no blob digests, since everything is stored as a manifest. + return [] + + @property + def local_blob_digests(self): + return self.blob_digests + + def get_blob_digests_for_translation(self): + return self.blob_digests + + @property + def layers_compressed_size(self): + return None + + @lru_cache(maxsize=1) + def manifests(self, content_retriever): + """ + Returns the manifests in the list. + """ + manifests = self._parsed[INDEX_MANIFESTS_KEY] + supported_types = {} + # supported_types[OCI_IMAGE_MANIFEST_CONTENT_TYPE] = OCIManifest + supported_types[OCI_IMAGE_INDEX_CONTENT_TYPE] = OCIIndex + return [ + LazyManifestLoader( + m, + content_retriever, + supported_types, + INDEX_DIGEST_KEY, + INDEX_SIZE_KEY, + INDEX_MEDIATYPE_KEY, + ) + for m in manifests + ] + + def validate(self, content_retriever): + """ + Performs validation of required assertions about the manifest. + + Raises a ManifestException on failure. + """ + # Nothing to validate. + + def child_manifests(self, content_retriever): + return self.manifests(content_retriever) + + def child_manifest_digests(self): + return [m[INDEX_DIGEST_KEY] for m in self._parsed[INDEX_MANIFESTS_KEY]] + + def get_manifest_labels(self, content_retriever): + return None + + def get_leaf_layer_v1_image_id(self, content_retriever): + return None + + def get_legacy_image_ids(self, content_retriever): + return None + + @property + def has_legacy_image(self): + return False + + def get_requires_empty_layer_blob(self, content_retriever): + return False + + def get_schema1_manifest(self, namespace_name, repo_name, tag_name, content_retriever): + """ + Returns the manifest that is compatible with V1, by virtue of being `amd64` and `linux`. + + If none, returns None. + """ + legacy_manifest = self._get_legacy_manifest(content_retriever) + if legacy_manifest is None: + return None + + return legacy_manifest.get_schema1_manifest( + namespace_name, repo_name, tag_name, content_retriever + ) + + def convert_manifest( + self, allowed_mediatypes, namespace_name, repo_name, tag_name, content_retriever + ): + if self.media_type in allowed_mediatypes: + return self + + legacy_manifest = self._get_legacy_manifest(content_retriever) + if legacy_manifest is None: + return None + + return legacy_manifest.convert_manifest( + allowed_mediatypes, namespace_name, repo_name, tag_name, content_retriever + ) + + def _get_legacy_manifest(self, content_retriever): + """ + Returns the manifest under this list with architecture amd64 and os linux, if any, or None + if none or error. + """ + for manifest_ref in self.manifests(content_retriever): + platform = manifest_ref._manifest_data[INDEX_PLATFORM_KEY] + architecture = platform[INDEX_ARCHITECTURE_KEY] + os = platform[INDEX_OS_KEY] + if architecture != "amd64" or os != "linux": + continue + + try: + return manifest_ref.manifest_obj + except (ManifestException, IOError): + logger.exception("Could not load child manifest") + return None + + return None + + def unsigned(self): + return self + + def generate_legacy_layers(self, images_map, content_retriever): + return None diff --git a/image/oci/manifest.py b/image/oci/manifest.py new file mode 100644 index 000000000..d1f2c2325 --- /dev/null +++ b/image/oci/manifest.py @@ -0,0 +1,529 @@ +""" +Implements validation and conversion for the OCI Manifest JSON. + +See: https://github.com/opencontainers/image-spec/blob/master/manifest.md + +Example: + +{ + "schemaVersion": 2, + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "size": 7023, + "digest": "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7" + }, + "layers": [ + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "size": 32654, + "digest": "sha256:9834876dcfb05cb167a5c24953eba58c4ac89b1adf57f28f2f9d09af107ee8f0" + }, + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "size": 16724, + "digest": "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b" + }, + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "size": 73109, + "digest": "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736" + } + ], + "annotations": { + "com.example.key1": "value1", + "com.example.key2": "value2" + } +} + +""" + +import json +import logging +import hashlib + +from collections import namedtuple +from jsonschema import validate as validate_schema, ValidationError + +from digest import digest_tools +from image.shared import ManifestException +from image.shared.interfaces import ManifestInterface +from image.shared.types import ManifestImageLayer +from image.docker.schema2 import EMPTY_LAYER_BLOB_DIGEST, EMPTY_LAYER_SIZE +from image.oci import ( + OCI_IMAGE_MANIFEST_CONTENT_TYPE, + OCI_IMAGE_CONFIG_CONTENT_TYPE, + OCI_IMAGE_LAYER_CONTENT_TYPES, + OCI_IMAGE_NON_DISTRIBUTABLE_LAYER_CONTENT_TYPES, + OCI_IMAGE_TAR_GZIP_LAYER_CONTENT_TYPE, + OCI_IMAGE_TAR_GZIP_NON_DISTRIBUTABLE_LAYER_CONTENT_TYPE, + ADDITIONAL_LAYER_CONTENT_TYPES, + ALLOWED_ARTIFACT_TYPES, +) +from image.oci.config import OCIConfig +from image.oci.descriptor import get_descriptor_schema +from image.docker.schema1 import DockerSchema1ManifestBuilder +from util.bytes import Bytes + +# Keys. +OCI_MANIFEST_VERSION_KEY = "schemaVersion" +OCI_MANIFEST_MEDIATYPE_KEY = "mediaType" +OCI_MANIFEST_CONFIG_KEY = "config" +OCI_MANIFEST_SIZE_KEY = "size" +OCI_MANIFEST_DIGEST_KEY = "digest" +OCI_MANIFEST_LAYERS_KEY = "layers" +OCI_MANIFEST_URLS_KEY = "urls" +OCI_MANIFEST_ANNOTATIONS_KEY = "annotations" + +# Named tuples. +OCIManifestConfig = namedtuple("OCIManifestConfig", ["size", "digest"]) +OCIManifestLayer = namedtuple( + "OCIManifestLayer", ["index", "digest", "is_remote", "urls", "compressed_size"] +) + +OCIManifestImageLayer = namedtuple( + "OCIManifestImageLayer", + ["history", "blob_layer", "v1_id", "v1_parent_id", "compressed_size", "blob_digest"], +) + +logger = logging.getLogger(__name__) + + +class MalformedOCIManifest(ManifestException): + """ + Raised when a manifest fails an assertion that should be true according to the OCI Manifest + spec. + """ + + pass + + +class OCIManifest(ManifestInterface): + METASCHEMA = { + "type": "object", + "properties": { + OCI_MANIFEST_VERSION_KEY: { + "type": "number", + "description": "The version of the schema. Must always be `2`.", + "minimum": 2, + "maximum": 2, + }, + OCI_MANIFEST_MEDIATYPE_KEY: { + "type": "string", + "description": "The media type of the schema.", + "enum": [OCI_IMAGE_MANIFEST_CONTENT_TYPE], + }, + OCI_MANIFEST_CONFIG_KEY: get_descriptor_schema(ALLOWED_ARTIFACT_TYPES), + OCI_MANIFEST_LAYERS_KEY: { + "type": "array", + "description": "The array MUST have the base layer at index 0. Subsequent layers MUST then follow in stack order (i.e. from layers[0] to layers[len(layers)-1])", + "items": get_descriptor_schema( + OCI_IMAGE_LAYER_CONTENT_TYPES + ADDITIONAL_LAYER_CONTENT_TYPES + ), + }, + }, + "required": [OCI_MANIFEST_VERSION_KEY, OCI_MANIFEST_CONFIG_KEY, OCI_MANIFEST_LAYERS_KEY,], + } + + def __init__(self, manifest_bytes, validate=False): + assert isinstance(manifest_bytes, Bytes) + + self._payload = manifest_bytes + + self._filesystem_layers = None + self._cached_built_config = None + + try: + self._parsed = json.loads(self._payload.as_unicode()) + except ValueError as ve: + raise MalformedOCIManifest("malformed manifest data: %s" % ve) + + try: + validate_schema(self._parsed, OCIManifest.METASCHEMA) + except ValidationError as ve: + raise MalformedOCIManifest("manifest data does not match schema: %s" % ve) + + for layer in self.filesystem_layers: + if layer.is_remote and not layer.urls: + raise MalformedOCIManifest("missing `urls` for remote layer") + + def validate(self, content_retriever): + """ + Performs validation of required assertions about the manifest. + + Raises a ManifestException on failure. + """ + # Nothing to validate. + + @property + def is_manifest_list(self): + return False + + @property + def schema_version(self): + return 2 + + @property + def manifest_dict(self): + return self._parsed + + @property + def media_type(self): + return OCI_IMAGE_MANIFEST_CONTENT_TYPE + + @property + def digest(self): + return digest_tools.sha256_digest(self._payload.as_encoded_str()) + + @property + def config(self): + config = self._parsed[OCI_MANIFEST_CONFIG_KEY] + return OCIManifestConfig( + size=config[OCI_MANIFEST_SIZE_KEY], digest=config[OCI_MANIFEST_DIGEST_KEY], + ) + + @property + def filesystem_layers(self): + """ + Returns the file system layers of this manifest, from base to leaf. + """ + if self._filesystem_layers is None: + self._filesystem_layers = list(self._generate_filesystem_layers()) + return self._filesystem_layers + + @property + def leaf_filesystem_layer(self): + """ + Returns the leaf file system layer for this manifest. + """ + return self.filesystem_layers[-1] + + @property + def layers_compressed_size(self): + return sum(layer.compressed_size for layer in self.filesystem_layers) + + @property + def has_remote_layer(self): + for layer in self.filesystem_layers: + if layer.is_remote: + return True + + return False + + @property + def is_image_manifest(self): + return self.manifest_dict["config"]["mediaType"] == OCI_IMAGE_CONFIG_CONTENT_TYPE + + @property + def blob_digests(self): + return [str(layer.digest) for layer in self.filesystem_layers] + [str(self.config.digest)] + + @property + def local_blob_digests(self): + return [str(layer.digest) for layer in self.filesystem_layers if not layer.is_remote] + [ + str(self.config.digest) + ] + + @property + def annotations(self): + """ Returns the annotations on the manifest itself. """ + return self._parsed.get(OCI_MANIFEST_ANNOTATIONS_KEY) or {} + + def get_blob_digests_for_translation(self): + return self.blob_digests + + def get_manifest_labels(self, content_retriever): + if not self.is_image_manifest: + return dict(self.annotations) + + built_config = self._get_built_config(content_retriever) + + labels = {} + labels.update(built_config.labels or {}) + labels.update(self.annotations) + return labels + + def get_layers(self, content_retriever): + """ + Returns the layers of this manifest, from base to leaf or None if this kind of manifest does + not support layers. + """ + if not self.is_image_manifest: + raise StopIteration() + + for image_layer in self._manifest_image_layers(content_retriever): + is_remote = image_layer.blob_layer.is_remote if image_layer.blob_layer else False + urls = image_layer.blob_layer.urls if image_layer.blob_layer else None + yield ManifestImageLayer( + layer_id=image_layer.v1_id, + compressed_size=image_layer.compressed_size, + is_remote=is_remote, + urls=urls, + command=image_layer.history.command, + blob_digest=image_layer.blob_digest, + created_datetime=image_layer.history.created_datetime, + author=image_layer.history.author, + comment=image_layer.history.comment, + internal_layer=image_layer, + ) + + @property + def bytes(self): + return self._payload + + def child_manifests(self, content_retriever): + return None + + def _manifest_image_layers(self, content_retriever): + assert self.is_image_manifest + + # Retrieve the configuration for the manifest. + config = self._get_built_config(content_retriever) + history = list(config.history) + if len(history) < len(self.filesystem_layers): + raise MalformedOCIManifest("Found less history than layer blobs") + + digest_history = hashlib.sha256() + v1_layer_parent_id = None + v1_layer_id = None + blob_index = 0 + + for history_index, history_entry in enumerate(history): + if not history_entry.is_empty and blob_index >= len(self.filesystem_layers): + raise MalformedOCIManifest("Missing history entry #%s" % blob_index) + + v1_layer_parent_id = v1_layer_id + blob_layer = None if history_entry.is_empty else self.filesystem_layers[blob_index] + blob_digest = EMPTY_LAYER_BLOB_DIGEST if blob_layer is None else str(blob_layer.digest) + compressed_size = EMPTY_LAYER_SIZE if blob_layer is None else blob_layer.compressed_size + + # Create a new synthesized V1 ID for the history layer by hashing its content and + # the blob associated with it. + digest_history.update(json.dumps(history_entry.raw_entry)) + digest_history.update("|") + digest_history.update(str(history_index)) + digest_history.update("|") + digest_history.update(blob_digest) + digest_history.update("||") + + v1_layer_id = digest_history.hexdigest() + yield OCIManifestImageLayer( + history=history_entry, + blob_layer=blob_layer, + blob_digest=blob_digest, + v1_id=v1_layer_id, + v1_parent_id=v1_layer_parent_id, + compressed_size=compressed_size, + ) + + if not history_entry.is_empty: + blob_index += 1 + + @property + def is_empty_manifest(self): + return len(self._parsed[OCI_MANIFEST_LAYERS_KEY]) == 0 + + @property + def has_legacy_image(self): + return self.is_image_manifest and not self.has_remote_layer and not self.is_empty_manifest + + def generate_legacy_layers(self, images_map, content_retriever): + assert not self.has_remote_layer + assert self.is_image_manifest + + # NOTE: We use the DockerSchema1ManifestBuilder here because it already contains + # the logic for generating the DockerV1Metadata. All of this will go away once we get + # rid of legacy images in the database, so this is a temporary solution. + v1_builder = DockerSchema1ManifestBuilder("", "", "") + self._populate_schema1_builder(v1_builder, content_retriever) + return v1_builder.build().generate_legacy_layers(images_map, content_retriever) + + def get_leaf_layer_v1_image_id(self, content_retriever): + # NOTE: If there exists a layer with remote content, then we consider this manifest + # to not support legacy images. + if self.has_remote_layer or not self.is_image_manifest: + return None + + return self.get_legacy_image_ids(content_retriever)[-1].v1_id + + def get_legacy_image_ids(self, content_retriever): + if self.has_remote_layer or not self.is_image_manifest: + return None + + return [l.v1_id for l in self._manifest_image_layers(content_retriever)] + + def convert_manifest( + self, allowed_mediatypes, namespace_name, repo_name, tag_name, content_retriever + ): + if self.media_type in allowed_mediatypes: + return self + + if not self.is_image_manifest: + return None + + # If this manifest is not on the allowed list, try to convert the schema 1 version (if any) + schema1 = self.get_schema1_manifest(namespace_name, repo_name, tag_name, content_retriever) + if schema1 is None: + return None + + return schema1.convert_manifest( + allowed_mediatypes, namespace_name, repo_name, tag_name, content_retriever + ) + + def get_schema1_manifest(self, namespace_name, repo_name, tag_name, content_retriever): + if self.has_remote_layer or not self.is_image_manifest: + return None + + v1_builder = DockerSchema1ManifestBuilder(namespace_name, repo_name, tag_name) + self._populate_schema1_builder(v1_builder, content_retriever) + return v1_builder.build() + + def unsigned(self): + return self + + def get_requires_empty_layer_blob(self, content_retriever): + if not self.is_image_manifest: + return False + + schema2_config = self._get_built_config(content_retriever) + if schema2_config is None: + return None + + return schema2_config.has_empty_layer + + def _populate_schema1_builder(self, v1_builder, content_retriever): + """ + Populates a DockerSchema1ManifestBuilder with the layers and config from this schema. + """ + assert not self.has_remote_layer + assert self.is_image_manifest + + schema2_config = self._get_built_config(content_retriever) + layers = list(self._manifest_image_layers(content_retriever)) + + for index, layer in enumerate(reversed(layers)): # Schema 1 layers are in reverse order + v1_compatibility = schema2_config.build_v1_compatibility( + layer.history, layer.v1_id, layer.v1_parent_id, index == 0, layer.compressed_size + ) + v1_builder.add_layer(str(layer.blob_digest), json.dumps(v1_compatibility)) + + return v1_builder + + def _get_built_config(self, content_retriever): + assert self.is_image_manifest + + if self._cached_built_config: + return self._cached_built_config + + config_bytes = content_retriever.get_blob_bytes_with_digest(self.config.digest) + if config_bytes is None: + raise MalformedOCIManifest("Could not load config blob for manifest") + + if len(config_bytes) != self.config.size: + msg = "Size of config does not match that retrieved: %s vs %s" % ( + len(config_bytes), + self.config.size, + ) + raise MalformedOCIManifest(msg) + + self._cached_built_config = OCIConfig(Bytes.for_string_or_unicode(config_bytes)) + return self._cached_built_config + + def _generate_filesystem_layers(self): + for index, layer in enumerate(self._parsed[OCI_MANIFEST_LAYERS_KEY]): + content_type = layer[OCI_MANIFEST_MEDIATYPE_KEY] + is_remote = content_type in OCI_IMAGE_NON_DISTRIBUTABLE_LAYER_CONTENT_TYPES + + try: + digest = digest_tools.Digest.parse_digest(layer[OCI_MANIFEST_DIGEST_KEY]) + except digest_tools.InvalidDigestException: + raise MalformedOCIManifest( + "could not parse manifest digest: %s" % layer[OCI_MANIFEST_DIGEST_KEY] + ) + + yield OCIManifestLayer( + index=index, + compressed_size=layer[OCI_MANIFEST_SIZE_KEY], + digest=digest, + is_remote=is_remote, + urls=layer.get(OCI_MANIFEST_URLS_KEY), + ) + + +class OCIManifestBuilder(object): + """ + A convenient abstraction around creating new OCIManifest. + """ + + def __init__(self): + self.config = None + self.filesystem_layers = [] + + def clone(self): + cloned = OCIManifestBuilder() + cloned.config = self.config + cloned.filesystem_layers = list(self.filesystem_layers) + return cloned + + def set_config(self, schema2_config): + """ + Sets the configuration for the manifest being built. + """ + self.set_config_digest(schema2_config.digest, schema2_config.size) + + def set_config_digest(self, config_digest, config_size): + """ + Sets the digest and size of the configuration layer. + """ + self.config = OCIManifestConfig(size=config_size, digest=config_digest) + + def add_layer(self, digest, size, urls=None): + """ + Adds a filesystem layer to the manifest. + """ + self.filesystem_layers.append( + OCIManifestLayer( + index=len(self.filesystem_layers), + digest=digest, + compressed_size=size, + urls=urls, + is_remote=bool(urls), + ) + ) + + def build(self, ensure_ascii=True): + """ + Builds and returns the OCIManifest. + """ + assert self.filesystem_layers + assert self.config + + def _build_layer(layer): + if layer.urls: + return { + OCI_MANIFEST_MEDIATYPE_KEY: OCI_IMAGE_TAR_GZIP_NON_DISTRIBUTABLE_LAYER_CONTENT_TYPE, + OCI_MANIFEST_SIZE_KEY: layer.compressed_size, + OCI_MANIFEST_DIGEST_KEY: str(layer.digest), + OCI_MANIFEST_URLS_KEY: layer.urls, + } + + return { + OCI_MANIFEST_MEDIATYPE_KEY: OCI_IMAGE_TAR_GZIP_LAYER_CONTENT_TYPE, + OCI_MANIFEST_SIZE_KEY: layer.compressed_size, + OCI_MANIFEST_DIGEST_KEY: str(layer.digest), + } + + manifest_dict = { + OCI_MANIFEST_VERSION_KEY: 2, + OCI_MANIFEST_MEDIATYPE_KEY: OCI_IMAGE_MANIFEST_CONTENT_TYPE, + # Config + OCI_MANIFEST_CONFIG_KEY: { + OCI_MANIFEST_MEDIATYPE_KEY: OCI_IMAGE_CONFIG_CONTENT_TYPE, + OCI_MANIFEST_SIZE_KEY: self.config.size, + OCI_MANIFEST_DIGEST_KEY: str(self.config.digest), + }, + # Layers + OCI_MANIFEST_LAYERS_KEY: [_build_layer(layer) for layer in self.filesystem_layers], + } + + json_str = json.dumps(manifest_dict, ensure_ascii=ensure_ascii, indent=3) + return OCIManifest(Bytes.for_string_or_unicode(json_str)) diff --git a/image/oci/test/test_oci_config.py b/image/oci/test/test_oci_config.py new file mode 100644 index 000000000..a5146cc85 --- /dev/null +++ b/image/oci/test/test_oci_config.py @@ -0,0 +1,116 @@ +import datetime +import json + +from dateutil.tz import tzutc + +import pytest + +from image.oci.config import OCIConfig, MalformedConfig, LayerHistory +from util.bytes import Bytes + +SAMPLE_CONFIG = """{ + "created": "2015-10-31T22:22:56.015925234Z", + "author": "Alyssa P. Hacker ", + "architecture": "amd64", + "os": "linux", + "config": { + "User": "alice", + "ExposedPorts": { + "8080/tcp": {} + }, + "Env": [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "FOO=oci_is_a", + "BAR=well_written_spec" + ], + "Entrypoint": [ + "/bin/my-app-binary" + ], + "Cmd": [ + "--foreground", + "--config", + "/etc/my-app.d/default.cfg" + ], + "Volumes": { + "/var/job-result-data": {}, + "/var/log/my-app-logs": {} + }, + "WorkingDir": "/home/alice", + "Labels": { + "com.example.project.git.url": "https://example.com/project.git", + "com.example.project.git.commit": "45a939b2999782a3f005621a8d0f29aa387e1d6b" + } + }, + "rootfs": { + "diff_ids": [ + "sha256:c6f988f4874bb0add23a778f753c65efe992244e148a1d2ec2a8b664fb66bbd1", + "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef" + ], + "type": "layers" + }, + "history": [ + { + "created": "2015-10-31T22:22:54.690851953Z", + "created_by": "/bin/sh -c #(nop) ADD file:a3bc1e842b69636f9df5256c49c5374fb4eef1e281fe3f282c65fb853ee171c5 in /" + }, + { + "created": "2015-10-31T22:22:55.613815829Z", + "created_by": "/bin/sh -c #(nop) CMD [\\"sh\\"]", + "empty_layer": true + } + ] +}""" + + +def test_parse_basic_config(): + config = OCIConfig(Bytes.for_string_or_unicode(SAMPLE_CONFIG)) + assert ( + config.digest == "sha256:b8410e43166c4e6b11cc0db4ede89539f206d5c9bb43d31d5b37f509b78d3f01" + ) + assert config.size == 1582 + + history = list(config.history) + assert config.has_empty_layer + assert len(history) == 2 + + expected = [ + LayerHistory( + created=u"2015-10-31T22:22:54.690851953Z", + created_datetime=datetime.datetime(2015, 10, 31, 22, 22, 54, 690851, tzinfo=tzutc()), + command=u"/bin/sh -c #(nop) ADD file:a3bc1e842b69636f9df5256c49c5374fb4eef1e281fe3f282c65fb853ee171c5 in /", + is_empty=False, + author=None, + comment=None, + raw_entry={ + u"created_by": u"/bin/sh -c #(nop) ADD file:a3bc1e842b69636f9df5256c49c5374fb4eef1e281fe3f282c65fb853ee171c5 in /", + u"created": u"2015-10-31T22:22:54.690851953Z", + }, + ), + LayerHistory( + created=u"2015-10-31T22:22:55.613815829Z", + created_datetime=datetime.datetime(2015, 10, 31, 22, 22, 55, 613815, tzinfo=tzutc()), + command=u'/bin/sh -c #(nop) CMD ["sh"]', + is_empty=True, + author=None, + comment=None, + raw_entry={ + u"empty_layer": True, + u"created_by": u'/bin/sh -c #(nop) CMD ["sh"]', + u"created": u"2015-10-31T22:22:55.613815829Z", + }, + ), + ] + assert history == expected + + +def test_config_missing_required(): + valid_config = json.loads(SAMPLE_CONFIG) + valid_config.pop("os") + + with pytest.raises(MalformedConfig): + OCIConfig(Bytes.for_string_or_unicode(json.dumps(valid_config))) + + +def test_invalid_config(): + with pytest.raises(MalformedConfig): + OCIConfig(Bytes.for_string_or_unicode("{}")) diff --git a/image/oci/test/test_oci_index.py b/image/oci/test/test_oci_index.py new file mode 100644 index 000000000..ffabffac3 --- /dev/null +++ b/image/oci/test/test_oci_index.py @@ -0,0 +1,58 @@ +import json + +import pytest + +from image.oci.index import OCIIndex, MalformedIndex +from util.bytes import Bytes + +SAMPLE_INDEX = """{ + "schemaVersion": 2, + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": 7143, + "digest": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f", + "platform": { + "architecture": "ppc64le", + "os": "linux" + } + }, + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": 7682, + "digest": "sha256:5b0bcabd1ed22e9fb1310cf6c2dec7cdef19f0ad69efa1f392e94a4333501270", + "platform": { + "architecture": "amd64", + "os": "linux" + } + } + ], + "annotations": { + "com.example.key1": "value1", + "com.example.key2": "value2" + } +}""" + + +def test_parse_basic_index(): + index = OCIIndex(Bytes.for_string_or_unicode(SAMPLE_INDEX)) + assert index.is_manifest_list + assert index.digest == "sha256:b1a216e8ed6a267bd3f0234d0d096c04658b28cb08b2b16bf812cf72694d7d04" + assert index.local_blob_digests == [] + assert index.child_manifest_digests() == [ + u"sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f", + u"sha256:5b0bcabd1ed22e9fb1310cf6c2dec7cdef19f0ad69efa1f392e94a4333501270", + ] + + +def test_config_missing_required(): + valid_index = json.loads(SAMPLE_INDEX) + valid_index.pop("schemaVersion") + + with pytest.raises(MalformedIndex): + OCIIndex(Bytes.for_string_or_unicode(json.dumps(valid_index))) + + +def test_invalid_index(): + with pytest.raises(MalformedIndex): + OCIIndex(Bytes.for_string_or_unicode("{}")) diff --git a/image/oci/test/test_oci_manifest.py b/image/oci/test/test_oci_manifest.py new file mode 100644 index 000000000..42deed786 --- /dev/null +++ b/image/oci/test/test_oci_manifest.py @@ -0,0 +1,189 @@ +import json + +import pytest + +from image.docker.schema1 import DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE +from image.oci.manifest import OCIManifest, MalformedOCIManifest +from image.shared.schemautil import ContentRetrieverForTesting +from util.bytes import Bytes + +SAMPLE_MANIFEST = """{ + "schemaVersion": 2, + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "size": 7023, + "digest": "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7" + }, + "layers": [ + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "size": 32654, + "digest": "sha256:9834876dcfb05cb167a5c24953eba58c4ac89b1adf57f28f2f9d09af107ee8f0" + }, + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "size": 16724, + "digest": "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b" + }, + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "size": 73109, + "digest": "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736" + } + ], + "annotations": { + "com.example.key1": "value1", + "com.example.key2": "value2" + } +}""" + + +SAMPLE_REMOTE_MANIFEST = """{ + "schemaVersion": 2, + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "size": 7023, + "digest": "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7" + }, + "layers": [ + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "size": 32654, + "digest": "sha256:9834876dcfb05cb167a5c24953eba58c4ac89b1adf57f28f2f9d09af107ee8f0" + }, + { + "mediaType": "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip", + "size": 16724, + "digest": "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b", + "urls": ["https://foo/bar"] + }, + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "size": 73109, + "digest": "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736" + } + ], + "annotations": { + "com.example.key1": "value1", + "com.example.key2": "value2" + } +}""" + + +def test_parse_basic_manifest(): + manifest = OCIManifest(Bytes.for_string_or_unicode(SAMPLE_MANIFEST)) + assert not manifest.is_manifest_list + assert ( + manifest.digest == "sha256:855b4e9ce4a4e5121dbc51a4f7ebfe7c2d6bcd16b159e754224f44573cfed5c2" + ) + + assert manifest.blob_digests == [ + "sha256:9834876dcfb05cb167a5c24953eba58c4ac89b1adf57f28f2f9d09af107ee8f0", + "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b", + "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736", + "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7", + ] + + assert manifest.local_blob_digests == manifest.blob_digests + + assert len(manifest.filesystem_layers) == 3 + assert ( + str(manifest.leaf_filesystem_layer.digest) + == "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736" + ) + + assert not manifest.has_remote_layer + assert manifest.has_legacy_image + assert manifest.annotations == {"com.example.key1": "value1", "com.example.key2": "value2"} + + +def test_parse_basic_remote_manifest(): + manifest = OCIManifest(Bytes.for_string_or_unicode(SAMPLE_REMOTE_MANIFEST)) + assert not manifest.is_manifest_list + assert ( + manifest.digest == "sha256:dd18ed87a00474aff683cee7160771e043f1f0eadd780232715bc0678a984a5e" + ) + + assert manifest.blob_digests == [ + "sha256:9834876dcfb05cb167a5c24953eba58c4ac89b1adf57f28f2f9d09af107ee8f0", + "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b", + "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736", + "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7", + ] + + assert manifest.local_blob_digests == [ + "sha256:9834876dcfb05cb167a5c24953eba58c4ac89b1adf57f28f2f9d09af107ee8f0", + "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736", + "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7", + ] + + assert len(manifest.filesystem_layers) == 3 + assert ( + str(manifest.leaf_filesystem_layer.digest) + == "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736" + ) + + assert manifest.has_remote_layer + + assert not manifest.has_legacy_image + assert not manifest.get_legacy_image_ids(None) + + +def test_get_schema1_manifest(): + retriever = ContentRetrieverForTesting.for_config( + { + "config": {"Labels": {"foo": "bar",},}, + "rootfs": {"type": "layers", "diff_ids": []}, + "history": [ + {"created": "2018-04-03T18:37:09.284840891Z", "created_by": "foo"}, + {"created": "2018-04-12T18:37:09.284840891Z", "created_by": "bar"}, + {"created": "2018-04-03T18:37:09.284840891Z", "created_by": "foo"}, + ], + "architecture": "amd64", + "os": "linux", + }, + "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7", + 7023, + ) + + manifest = OCIManifest(Bytes.for_string_or_unicode(SAMPLE_MANIFEST)) + assert manifest.get_manifest_labels(retriever) == { + "com.example.key1": "value1", + "com.example.key2": "value2", + "foo": "bar", + } + + schema1 = manifest.get_schema1_manifest("somenamespace", "somename", "sometag", retriever) + assert schema1 is not None + assert schema1.media_type == DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE + assert set(schema1.local_blob_digests) == ( + set(manifest.local_blob_digests) + - {"sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7"} + ) + assert len(schema1.layers) == 3 + + via_convert = manifest.convert_manifest( + [schema1.media_type], "somenamespace", "somename", "sometag", retriever + ) + assert via_convert.digest == schema1.digest + + +def test_validate_manifest_invalid_config_type(): + manifest_bytes = """{ + "schemaVersion": 2, + "config": { + "mediaType": "application/some.other.thing", + "digest": "sha256:6bd578ec7d1e7381f63184dfe5fbe7f2f15805ecc4bfd485e286b76b1e796524", + "size": 145 + }, + "layers": [ + { + "mediaType": "application/tar+gzip", + "digest": "sha256:ce879e86a8f71031c0f1ab149a26b000b3b5b8810d8d047f240ef69a6b2516ee", + "size": 2807 + } + ] + }""" + + with pytest.raises(MalformedOCIManifest): + OCIManifest(Bytes.for_string_or_unicode(manifest_bytes)) diff --git a/image/shared/schemas.py b/image/shared/schemas.py index 187165847..c3cc6a906 100644 --- a/image/shared/schemas.py +++ b/image/shared/schemas.py @@ -6,6 +6,9 @@ from image.docker.schema2 import ( ) from image.docker.schema2.manifest import DockerSchema2Manifest from image.docker.schema2.list import DockerSchema2ManifestList +from image.oci import OCI_IMAGE_INDEX_CONTENT_TYPE, OCI_IMAGE_MANIFEST_CONTENT_TYPE +from image.oci.index import OCIIndex +from image.oci.manifest import OCIManifest from util.bytes import Bytes @@ -23,6 +26,12 @@ def parse_manifest_from_bytes(manifest_bytes, media_type, validate=True): if media_type == DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE: return DockerSchema2ManifestList(manifest_bytes) + if media_type == OCI_IMAGE_MANIFEST_CONTENT_TYPE: + return OCIManifest(manifest_bytes) + + if media_type == OCI_IMAGE_INDEX_CONTENT_TYPE: + return OCIIndex(manifest_bytes) + if media_type in DOCKER_SCHEMA1_CONTENT_TYPES: return DockerSchema1Manifest(manifest_bytes, validate=validate) diff --git a/image/shared/schemautil.py b/image/shared/schemautil.py index 6e1d7f0ab..d578c849f 100644 --- a/image/shared/schemautil.py +++ b/image/shared/schemautil.py @@ -1,6 +1,8 @@ import json +from image.shared import ManifestException from image.shared.interfaces import ContentRetriever +from util.bytes import Bytes class ContentRetrieverForTesting(ContentRetriever): @@ -50,3 +52,56 @@ def to_canonical_json(value, ensure_ascii=True, indent=None): cls=_CustomEncoder, indent=indent, ) + + +class LazyManifestLoader(object): + """ Lazy loader for manifests referenced by another manifest list or index. """ + + def __init__( + self, + manifest_data, + content_retriever, + supported_types, + digest_key, + size_key, + media_type_key, + ): + self._manifest_data = manifest_data + self._content_retriever = content_retriever + self._loaded_manifest = None + self._digest_key = digest_key + self._size_key = size_key + self._media_type_key = media_type_key + self._supported_types = supported_types + + @property + def manifest_obj(self): + if self._loaded_manifest is not None: + return self._loaded_manifest + + self._loaded_manifest = self._load_manifest() + return self._loaded_manifest + + def _load_manifest(self): + digest = self._manifest_data[self._digest_key] + size = self._manifest_data[self._size_key] + manifest_bytes = self._content_retriever.get_manifest_bytes_with_digest(digest) + if manifest_bytes is None: + raise ManifestException("Could not find child manifest with digest `%s`" % digest) + + if len(manifest_bytes) != size: + raise ManifestException( + "Size of manifest does not match that retrieved: %s vs %s", + len(manifest_bytes), + size, + ) + + content_type = self._manifest_data[self._media_type_key] + if content_type not in self._supported_types: + raise ManifestException( + "Unknown or unsupported manifest media type `%s`" % content_type + ) + + return self._supported_types[content_type]( + Bytes.for_string_or_unicode(manifest_bytes), validate=False + ) diff --git a/initdb.py b/initdb.py index 8e905e01e..f0648c3ec 100644 --- a/initdb.py +++ b/initdb.py @@ -70,6 +70,7 @@ from image.docker.schema2 import DOCKER_SCHEMA2_CONTENT_TYPES from image.docker.schema1 import DockerSchema1ManifestBuilder from image.docker.schema2.manifest import DockerSchema2ManifestBuilder from image.docker.schema2.config import DockerSchema2Config +from image.oci import OCI_CONTENT_TYPES from workers import repositoryactioncounter @@ -489,6 +490,9 @@ def initialize_database(): for media_type in DOCKER_SCHEMA2_CONTENT_TYPES: MediaType.create(name=media_type) + for media_type in OCI_CONTENT_TYPES: + MediaType.create(name=media_type) + LabelSourceType.create(name="manifest") LabelSourceType.create(name="api", mutable=True) LabelSourceType.create(name="internal") diff --git a/test/registry/protocol_fixtures.py b/test/registry/protocol_fixtures.py index 828885ada..b929e9c63 100644 --- a/test/registry/protocol_fixtures.py +++ b/test/registry/protocol_fixtures.py @@ -218,9 +218,14 @@ def v2_protocol(request, jwk): return request.param(jwk) +@pytest.fixture() +def v21_protocol(request, jwk): + return V2Protocol(jwk, schema="schema1") + + @pytest.fixture() def v22_protocol(request, jwk): - return V2Protocol(jwk, schema2=True) + return V2Protocol(jwk, schema="schema2") @pytest.fixture(params=[V1Protocol]) @@ -228,18 +233,21 @@ def v1_protocol(request, jwk): return request.param(jwk) -@pytest.fixture(params=["schema1", "schema2"]) +@pytest.fixture(params=["schema1", "schema2", "oci"]) def manifest_protocol(request, data_model, jwk): - return V2Protocol(jwk, schema2=(request == "schema2" and data_model == "oci_model")) + return V2Protocol(jwk, schema=request.param) -@pytest.fixture(params=["v1", "v2_1", "v2_2"]) +@pytest.fixture(params=["v1", "v2_1", "v2_2", "oci"]) def pusher(request, data_model, jwk): if request.param == "v1": return V1Protocol(jwk) - if request.param == "v2_2" and data_model == "oci_model": - return V2Protocol(jwk, schema2=True) + if request.param == "v2_2": + return V2Protocol(jwk, schema="schema2") + + if request.param == "oci": + return V2Protocol(jwk, schema="oci") return V2Protocol(jwk) @@ -260,13 +268,16 @@ def legacy_pusher(request, data_model, jwk): return V2Protocol(jwk) -@pytest.fixture(params=["v1", "v2_1", "v2_2"]) +@pytest.fixture(params=["v1", "v2_1", "v2_2", "oci"]) def puller(request, data_model, jwk): if request.param == "v1": return V1Protocol(jwk) - if request.param == "v2_2" and data_model == "oci_model": - return V2Protocol(jwk, schema2=True) + if request.param == "v2_2": + return V2Protocol(jwk, schema="schema2") + + if request.param == "oci": + return V2Protocol(jwk, schema="oci") return V2Protocol(jwk) diff --git a/test/registry/protocol_v2.py b/test/registry/protocol_v2.py index e0cac77ac..c3a9c7f88 100644 --- a/test/registry/protocol_v2.py +++ b/test/registry/protocol_v2.py @@ -12,6 +12,9 @@ from image.docker.schema2 import DOCKER_SCHEMA2_CONTENT_TYPES from image.docker.schema2.manifest import DockerSchema2ManifestBuilder from image.docker.schema2.config import DockerSchema2Config from image.docker.schema2.list import DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE +from image.oci import OCI_CONTENT_TYPES +from image.oci.manifest import OCIManifestBuilder +from image.oci.config import OCIConfig from image.shared.schemas import parse_manifest_from_bytes from test.registry.protocols import ( RegistryProtocol, @@ -99,9 +102,9 @@ class V2Protocol(RegistryProtocol): }, } - def __init__(self, jwk, schema2=False): + def __init__(self, jwk, schema="schema1"): self.jwk = jwk - self.schema2 = schema2 + self.schema = schema def ping(self, session): result = session.get("/v2/") @@ -324,6 +327,53 @@ class V2Protocol(RegistryProtocol): return PushResult(manifests=None, headers=headers) + def build_oci(self, images, blobs, options): + builder = OCIManifestBuilder() + for image in images: + checksum = "sha256:" + hashlib.sha256(image.bytes).hexdigest() + + if image.urls is None: + blobs[checksum] = image.bytes + + # If invalid blob references were requested, just make it up. + if options.manifest_invalid_blob_references: + checksum = "sha256:" + hashlib.sha256("notarealthing").hexdigest() + + if not image.is_empty: + builder.add_layer(checksum, len(image.bytes), urls=image.urls) + + def history_for_image(image): + history = { + "created": "2018-04-03T18:37:09.284840891Z", + "created_by": ( + ("/bin/sh -c #(nop) ENTRYPOINT %s" % image.config["Entrypoint"]) + if image.config and image.config.get("Entrypoint") + else "/bin/sh -c #(nop) %s" % image.id + ), + } + + if image.is_empty: + history["empty_layer"] = True + + return history + + config = { + "os": "linux", + "architecture": "amd64", + "rootfs": {"type": "layers", "diff_ids": []}, + "history": [history_for_image(image) for image in images], + } + + if images[-1].config: + config["config"] = images[-1].config + + config_json = json.dumps(config, ensure_ascii=options.ensure_ascii) + oci_config = OCIConfig(Bytes.for_string_or_unicode(config_json)) + builder.set_config(oci_config) + + blobs[oci_config.digest] = oci_config.bytes.as_encoded_str() + return builder.build(ensure_ascii=options.ensure_ascii) + def build_schema2(self, images, blobs, options): builder = DockerSchema2ManifestBuilder() for image in images: @@ -456,12 +506,16 @@ class V2Protocol(RegistryProtocol): manifests = {} blobs = {} for tag_name in tag_names: - if self.schema2: + if self.schema == "oci": + manifests[tag_name] = self.build_oci(images, blobs, options) + elif self.schema == "schema2": manifests[tag_name] = self.build_schema2(images, blobs, options) - else: + elif self.schema == "schema1": manifests[tag_name] = self.build_schema1( namespace, repo_name, tag_name, images, blobs, options ) + else: + raise NotImplementedError(self.schema) # Push the blob data. if not self._push_blobs( @@ -562,6 +616,10 @@ class V2Protocol(RegistryProtocol): patch_headers.update(headers) contents_chunk = blob_bytes[start_byte:end_byte] + assert len(contents_chunk) == (end_byte - start_byte), "%s vs %s" % ( + len(contents_chunk), + end_byte - start_byte, + ) self.conduct( session, "PATCH", @@ -582,7 +640,10 @@ class V2Protocol(RegistryProtocol): session, "GET", status_url, expected_status=204, headers=headers ) assert response.headers["Docker-Upload-UUID"] == upload_uuid - assert response.headers["Range"] == "bytes=0-%s" % end_byte + assert response.headers["Range"] == "bytes=0-%s" % end_byte, "%s vs %s" % ( + response.headers["Range"], + "bytes=0-%s" % end_byte, + ) if options.cancel_blob_upload: self.conduct( @@ -716,7 +777,13 @@ class V2Protocol(RegistryProtocol): "Authorization": "Bearer " + token, } - if self.schema2: + if self.schema == "oci": + headers["Accept"] = ",".join( + options.accept_mimetypes + if options.accept_mimetypes is not None + else OCI_CONTENT_TYPES + ) + elif self.schema == "schema2": headers["Accept"] = ",".join( options.accept_mimetypes if options.accept_mimetypes is not None @@ -743,9 +810,19 @@ class V2Protocol(RegistryProtocol): # Ensure the manifest returned by us is valid. ct = response.headers["Content-Type"] - if not self.schema2: + if self.schema == "schema1": assert ct in DOCKER_SCHEMA1_CONTENT_TYPES + if options.require_matching_manifest_type: + if self.schema == "schema1": + assert ct in DOCKER_SCHEMA1_CONTENT_TYPES + + if self.schema == "schema2": + assert ct in DOCKER_SCHEMA2_CONTENT_TYPES + + if self.schema == "oci": + assert ct in OCI_CONTENT_TYPES + manifest = parse_manifest_from_bytes(Bytes.for_string_or_unicode(response.text), ct) manifests[tag_name] = manifest @@ -780,7 +857,7 @@ class V2Protocol(RegistryProtocol): assert (len(blob_digests) + empty_count) >= len( images - ) # Schema 2 has 1 extra for config + ) # OCI/Schema 2 has 1 extra for config return PullResult(manifests=manifests, image_ids=image_ids) diff --git a/test/registry/protocols.py b/test/registry/protocols.py index 603c4788d..8200f4720 100644 --- a/test/registry/protocols.py +++ b/test/registry/protocols.py @@ -95,6 +95,7 @@ class ProtocolOptions(object): self.ensure_ascii = True self.attempt_pull_without_token = False self.with_broken_manifest_config = False + self.require_matching_manifest_type = False @add_metaclass(ABCMeta) diff --git a/test/registry/registry_tests.py b/test/registry/registry_tests.py index 00473db8f..8719fd193 100644 --- a/test/registry/registry_tests.py +++ b/test/registry/registry_tests.py @@ -185,9 +185,18 @@ def test_basic_push_pull_by_manifest( ) # Pull the repository by digests to verify. + options = ProtocolOptions() + options.require_matching_manifest_type = True + digests = [str(manifest.digest) for manifest in result.manifests.values()] manifest_protocol.pull( - liveserver_session, "devtable", "newrepo", digests, basic_images, credentials=credentials + liveserver_session, + "devtable", + "newrepo", + digests, + basic_images, + credentials=credentials, + options=options, ) @@ -211,10 +220,44 @@ def test_basic_push_by_manifest_digest( options=options, ) + # If this is not schema 1, then verify we cannot pull. + expected_failure = None if manifest_protocol.schema == "schema1" else Failures.UNKNOWN_TAG + # Pull the repository by digests to verify. digests = [str(manifest.digest) for manifest in result.manifests.values()] manifest_protocol.pull( - liveserver_session, "devtable", "newrepo", digests, basic_images, credentials=credentials + liveserver_session, + "devtable", + "newrepo", + digests, + basic_images, + credentials=credentials, + expected_failure=expected_failure, + ) + + +def test_manifest_down_conversion( + manifest_protocol, v21_protocol, basic_images, liveserver_session, app_reloader +): + """ Test: Push using a new protocol and ensure down-conversion. """ + credentials = ("devtable", "password") + + # Push a new repository. + result = manifest_protocol.push( + liveserver_session, "devtable", "newrepo", "latest", basic_images, credentials=credentials + ) + + # Pull the repository with down conversion. + options = ProtocolOptions() + + v21_protocol.pull( + liveserver_session, + "devtable", + "newrepo", + "latest", + basic_images, + credentials=credentials, + options=options, ) @@ -1225,8 +1268,7 @@ def test_blob_caching( # Pull each blob, which should succeed due to caching. If caching is broken, this will # fail when it attempts to hit the database. - for layer in result.manifests["latest"].layers: - blob_id = str(layer.digest) + for blob_id in result.manifests["latest"].local_blob_digests: r = liveserver_session.get( "/v2/devtable/newrepo/blobs/%s" % blob_id, headers=result.headers ) @@ -1242,11 +1284,11 @@ def test_blob_caching( [(0, 10), (10, 20), (20, None)], [(0, 10), (10, 20), (20, 30), (30, 40), (40, 50), (50, None)], # Overlapping chunks. - [(0, 1024), (10, None)], + [(0, 90), (10, None)], ], ) def test_chunked_blob_uploading( - chunks, random_layer_data, manifest_protocol, puller, liveserver_session, app_reloader + chunks, random_layer_data, v21_protocol, puller, liveserver_session, app_reloader ): """ Test: Uploading of blobs as chunks. """ credentials = ("devtable", "password") @@ -1263,7 +1305,7 @@ def test_chunked_blob_uploading( options.chunks_for_upload = adjusted_chunks # Push the image, using the specified chunking. - manifest_protocol.push( + v21_protocol.push( liveserver_session, "devtable", "newrepo", @@ -1280,7 +1322,7 @@ def test_chunked_blob_uploading( def test_chunked_uploading_mismatched_chunks( - manifest_protocol, random_layer_data, liveserver_session, app_reloader + v21_protocol, random_layer_data, liveserver_session, app_reloader ): """ Test: Attempt to upload chunks with data missing. """ credentials = ("devtable", "password") @@ -1294,7 +1336,7 @@ def test_chunked_uploading_mismatched_chunks( options.chunks_for_upload = [(0, 100), (101, len(random_layer_data), 416)] # Attempt to push, with the chunked upload failing. - manifest_protocol.push( + v21_protocol.push( liveserver_session, "devtable", "newrepo", diff --git a/test/test_api_usage.py b/test/test_api_usage.py index 92a042fcd..3c49ae14b 100644 --- a/test/test_api_usage.py +++ b/test/test_api_usage.py @@ -2526,14 +2526,12 @@ class TestGetRepository(ApiTestCase): def test_get_largerepo(self): self.login(ADMIN_ACCESS_USER) - offset = 0 if registry_model.supports_schema2(ADMIN_ACCESS_USER) else 2 - # base + repo + is_starred + tags - with assert_query_count(BASE_LOGGEDIN_QUERY_COUNT + 4 + offset + 1): + with assert_query_count(BASE_LOGGEDIN_QUERY_COUNT + 4 + 1): self.getJsonResponse(Repository, params=dict(repository=ADMIN_ACCESS_USER + "/simple")) # base + repo + is_starred + tags - with assert_query_count(BASE_LOGGEDIN_QUERY_COUNT + 4 + offset + 1): + with assert_query_count(BASE_LOGGEDIN_QUERY_COUNT + 4 + 1): json = self.getJsonResponse( Repository, params=dict(repository=ADMIN_ACCESS_USER + "/gargantuan") ) diff --git a/test/testconfig.py b/test/testconfig.py index 3f308e437..9b5c7599b 100644 --- a/test/testconfig.py +++ b/test/testconfig.py @@ -112,3 +112,5 @@ class TestConfig(DefaultConfig): } FEATURE_REPO_MIRROR = True + FEATURE_GENERAL_OCI_SUPPORT = True + OCI_NAMESPACE_WHITELIST = [] diff --git a/util/config/schema.py b/util/config/schema.py index 711870e8e..1322605a2 100644 --- a/util/config/schema.py +++ b/util/config/schema.py @@ -10,6 +10,9 @@ INTERNAL_ONLY_PROPERTIES = { "DATABASE_SECRET_KEY", "V22_NAMESPACE_BLACKLIST", "MAXIMUM_CNR_LAYER_SIZE", + "OCI_NAMESPACE_WHITELIST", + "FEATURE_GENERAL_OCI_SUPPORT", + "FEATURE_EXPERIMENTAL_HELM_OCI_SUPPORT", "TESTING", "SEND_FILE_MAX_AGE_DEFAULT", "DISABLED_FOR_AUDIT_LOGS",