diff --git a/commit.go b/commit.go index 683990f97..7649b8b18 100644 --- a/commit.go +++ b/commit.go @@ -1,6 +1,7 @@ package buildah import ( + "bytes" "io" "github.com/Sirupsen/logrus" @@ -10,10 +11,23 @@ import ( "github.com/containers/image/transports" "github.com/containers/image/types" "github.com/containers/storage/pkg/archive" + "github.com/containers/storage/pkg/stringid" + "github.com/opencontainers/go-digest" "github.com/pkg/errors" "github.com/projectatomic/buildah/util" ) +var ( + // gzippedEmptyLayer is a gzip-compressed version of an empty tar file (just 1024 zero bytes). This + // comes from github.com/docker/distribution/manifest/schema1/config_builder.go by way of + // github.com/containers/image/image/docker_schema2.go; there is a non-zero embedded timestamp; we could + // zero that, but that would just waste storage space in registries, so let’s use the same values. + gzippedEmptyLayer = []byte{ + 31, 139, 8, 0, 0, 9, 110, 136, 0, 255, 98, 24, 5, 163, 96, 20, 140, 88, + 0, 8, 0, 0, 255, 255, 46, 175, 181, 239, 0, 4, 0, 0, + } +) + // CommitOptions can be used to alter how an image is committed. type CommitOptions struct { // PreferredManifestType is the preferred type of image manifest. The @@ -38,6 +52,159 @@ type CommitOptions struct { ReportWriter io.Writer } +// shallowCopy copies the most recent layer, the configuration, and the manifest from one image to another. +// For local storage, which doesn't care about histories and the manifest's contents, that's sufficient, but +// almost any other destination has higher expectations. +// We assume that "dest" is a reference to a local image (specifically, a containers/image/storage.storageReference), +// and will fail if it isn't. +func (b *Builder) shallowCopy(dest types.ImageReference, src types.ImageReference, systemContext *types.SystemContext) error { + // Read the target image name. + if dest.DockerReference() == nil { + return errors.New("can't write to an unnamed image") + } + names, err := util.ExpandTags([]string{dest.DockerReference().String()}) + if err != nil { + return err + } + // Make a temporary image reference. + tmpName := stringid.GenerateRandomID() + "-tmp-" + Package + "-commit" + tmpRef, err := storage.Transport.ParseStoreReference(b.store, tmpName) + if err != nil { + return err + } + defer func() { + if err2 := tmpRef.DeleteImage(systemContext); err2 != nil { + logrus.Debugf("error deleting temporary image %q: %v", tmpName, err2) + } + }() + // Open the source for reading and a temporary image for writing. + srcImage, err := src.NewImage(systemContext) + if err != nil { + return errors.Wrapf(err, "error reading configuration to write to image %q", transports.ImageName(dest)) + } + defer srcImage.Close() + tmpImage, err := tmpRef.NewImageDestination(systemContext) + if err != nil { + return errors.Wrapf(err, "error opening temporary copy of image %q for writing", transports.ImageName(dest)) + } + defer tmpImage.Close() + // Write an empty filesystem layer, because the image layer requires at least one. + _, err = tmpImage.PutBlob(bytes.NewReader(gzippedEmptyLayer), types.BlobInfo{Size: int64(len(gzippedEmptyLayer))}) + if err != nil { + return errors.Wrapf(err, "error writing dummy layer for image %q", transports.ImageName(dest)) + } + // Read the newly-generated configuration blob. + config, err := srcImage.ConfigBlob() + if err != nil { + return errors.Wrapf(err, "error reading new configuration for image %q", transports.ImageName(dest)) + } + if len(config) == 0 { + return errors.Errorf("error reading new configuration for image %q: it's empty", transports.ImageName(dest)) + } + logrus.Debugf("read configuration blob %q", string(config)) + // Write the configuration to the temporary image. + configBlobInfo := types.BlobInfo{ + Digest: digest.Canonical.FromBytes(config), + Size: int64(len(config)), + } + _, err = tmpImage.PutBlob(bytes.NewReader(config), configBlobInfo) + if err != nil && len(config) > 0 { + return errors.Wrapf(err, "error writing image configuration for temporary copy of %q", transports.ImageName(dest)) + } + // Read the newly-generated, mostly fake, manifest. + manifest, _, err := srcImage.Manifest() + if err != nil { + return errors.Wrapf(err, "error reading new manifest for image %q", transports.ImageName(dest)) + } + // Write the manifest to the temporary image. + err = tmpImage.PutManifest(manifest) + if err != nil { + return errors.Wrapf(err, "error writing new manifest to temporary copy of image %q", transports.ImageName(dest)) + } + // Save the temporary image. + err = tmpImage.Commit() + if err != nil { + return errors.Wrapf(err, "error committing new image %q", transports.ImageName(dest)) + } + // Locate the temporary image in the lower-level API. Read its item names. + tmpImg, err := storage.Transport.GetStoreImage(b.store, tmpRef) + if err != nil { + return errors.Wrapf(err, "error locating temporary image %q", transports.ImageName(dest)) + } + items, err := b.store.ListImageBigData(tmpImg.ID) + if err != nil { + return errors.Wrapf(err, "error reading list of named data for image %q", tmpImg.ID) + } + // Look up the container's read-write layer. + container, err := b.store.Container(b.ContainerID) + if err != nil { + return errors.Wrapf(err, "error reading information about working container %q", b.ContainerID) + } + parentLayer := "" + // Look up the container's source image's layer, if there is a source image. + if container.ImageID != "" { + img, err2 := b.store.Image(container.ImageID) + if err2 != nil { + return errors.Wrapf(err2, "error reading information about working container %q's source image", b.ContainerID) + } + parentLayer = img.TopLayer + } + // Extract the read-write layer's contents. + layerDiff, err := b.store.Diff(parentLayer, container.LayerID) + if err != nil { + return errors.Wrapf(err, "error reading layer from source image %q", transports.ImageName(src)) + } + defer layerDiff.Close() + // Write a copy of the layer for the new image to reference. + layer, _, err := b.store.PutLayer("", parentLayer, []string{}, "", false, layerDiff) + if err != nil { + return errors.Wrapf(err, "error creating new read-only layer from container %q", b.ContainerID) + } + // Create a low-level image record that uses the new layer. + image, err := b.store.CreateImage("", []string{}, layer.ID, "", nil) + if err != nil { + err2 := b.store.DeleteLayer(layer.ID) + if err2 != nil { + logrus.Debugf("error removing layer %q: %v", layer, err2) + } + return errors.Wrapf(err, "error creating new low-level image %q", transports.ImageName(dest)) + } + logrus.Debugf("created image ID %q", image.ID) + defer func() { + if err != nil { + _, err2 := b.store.DeleteImage(image.ID, true) + if err2 != nil { + logrus.Debugf("error removing image %q: %v", image.ID, err2) + } + } + }() + // Copy the configuration and manifest, which are big data items, along with whatever else is there. + for _, item := range items { + var data []byte + data, err = b.store.ImageBigData(tmpImg.ID, item) + if err != nil { + return errors.Wrapf(err, "error copying data item %q", item) + } + err = b.store.SetImageBigData(image.ID, item, data) + if err != nil { + return errors.Wrapf(err, "error copying data item %q", item) + } + logrus.Debugf("copied data item %q to %q", item, image.ID) + } + // Set low-level metadata in the new image so that the image library will accept it as a real image. + err = b.store.SetMetadata(image.ID, "{}") + if err != nil { + return errors.Wrapf(err, "error assigning metadata to new image %q", transports.ImageName(dest)) + } + // Move the target name(s) from the temporary image to the new image. + err = util.AddImageNames(b.store, image, names) + if err != nil { + return errors.Wrapf(err, "error assigning names %v to new image", names) + } + logrus.Debugf("assigned names %v to image %q", names, image.ID) + return nil +} + // Commit writes the contents of the container, along with its updated // configuration, to a new image in the specified location, and if we know how, // add any additional tags that were specified. @@ -50,13 +217,25 @@ func (b *Builder) Commit(dest types.ImageReference, options CommitOptions) error if err != nil { return err } - src, err := b.makeContainerImageRef(options.PreferredManifestType, options.Compression) + // Check if we're keeping everything in local storage. If so, we can take certain shortcuts. + _, destIsStorage := dest.Transport().(storage.StoreTransport) + exporting := !destIsStorage + src, err := b.makeContainerImageRef(options.PreferredManifestType, exporting, options.Compression) if err != nil { return errors.Wrapf(err, "error recomputing layer digests and building metadata") } - err = cp.Image(policyContext, dest, src, getCopyOptions(options.ReportWriter)) - if err != nil { - return errors.Wrapf(err, "error copying layers and metadata") + if exporting { + // Copy everything. + err = cp.Image(policyContext, dest, src, getCopyOptions(options.ReportWriter)) + if err != nil { + return errors.Wrapf(err, "error copying layers and metadata") + } + } else { + // Copy only the most recent layer, the configuration, and the manifest. + err = b.shallowCopy(dest, src, getSystemContext(options.SignaturePolicyPath)) + if err != nil { + return errors.Wrapf(err, "error copying layer and metadata") + } } if len(options.AdditionalTags) > 0 { switch dest.Transport().Name() { @@ -69,6 +248,7 @@ func (b *Builder) Commit(dest types.ImageReference, options CommitOptions) error if err != nil { return errors.Wrapf(err, "error setting image names to %v", append(img.Names, options.AdditionalTags...)) } + logrus.Debugf("assigned names %v to image %q", img.Names, img.ID) default: logrus.Warnf("don't know how to add tags to images stored in %q transport", dest.Transport().Name()) } diff --git a/image.go b/image.go index e9e13a131..2b34f3b47 100644 --- a/image.go +++ b/image.go @@ -46,6 +46,7 @@ type containerImageRef struct { createdBy string annotations map[string]string preferredManifestType string + exporting bool } type containerImageSource struct { @@ -58,6 +59,7 @@ type containerImageSource struct { configDigest digest.Digest manifest []byte manifestType string + exporting bool } func (i *containerImageRef) NewImage(sc *types.SystemContext) (types.Image, error) { @@ -175,6 +177,46 @@ func (i *containerImageRef) NewImageSource(sc *types.SystemContext, manifestType // Extract each layer and compute its digests, both compressed (if requested) and uncompressed. for _, layerID := range layers { + omediaType := v1.MediaTypeImageLayer + dmediaType := docker.V2S2MediaTypeUncompressedLayer + // Figure out which media type we want to call this. Assume no compression. + if i.compression != archive.Uncompressed { + switch i.compression { + case archive.Gzip: + omediaType = v1.MediaTypeImageLayerGzip + dmediaType = docker.V2S2MediaTypeLayer + logrus.Debugf("compressing layer %q with gzip", layerID) + case archive.Bzip2: + // Until the image specs define a media type for bzip2-compressed layers, even if we know + // how to decompress them, we can't try to compress layers with bzip2. + return nil, errors.New("media type for bzip2-compressed layers is not defined") + default: + logrus.Debugf("compressing layer %q with unknown compressor(?)", layerID) + } + } + // If we're not re-exporting the data, just fake up layer and diff IDs for the manifest. + if !i.exporting { + fakeLayerDigest := digest.NewDigestFromHex(digest.Canonical.String(), layerID) + // Add a note in the manifest about the layer. The blobs should be identified by their + // possibly-compressed blob digests, but just use the layer IDs here. + olayerDescriptor := v1.Descriptor{ + MediaType: omediaType, + Digest: fakeLayerDigest, + Size: -1, + } + omanifest.Layers = append(omanifest.Layers, olayerDescriptor) + dlayerDescriptor := docker.V2S2Descriptor{ + MediaType: dmediaType, + Digest: fakeLayerDigest, + Size: -1, + } + dmanifest.Layers = append(dmanifest.Layers, dlayerDescriptor) + // Add a note about the diffID, which should be uncompressed digest of the blob, but + // just use the layer ID here. + oimage.RootFS.DiffIDs = append(oimage.RootFS.DiffIDs, fakeLayerDigest.String()) + dimage.RootFS.DiffIDs = append(dimage.RootFS.DiffIDs, fakeLayerDigest) + continue + } // Start reading the layer. rc, err := i.store.Diff("", layerID) if err != nil { @@ -200,23 +242,7 @@ func (i *containerImageRef) NewImageSource(sc *types.SystemContext, manifestType destHasher := digest.Canonical.Digester() counter := ioutils.NewWriteCounter(layerFile) multiWriter := io.MultiWriter(counter, destHasher.Hash()) - // Figure out which media type we want to call this. Assume no compression. - omediaType := v1.MediaTypeImageLayer - dmediaType := docker.V2S2MediaTypeUncompressedLayer - if i.compression != archive.Uncompressed { - switch i.compression { - case archive.Gzip: - omediaType = v1.MediaTypeImageLayerGzip - dmediaType = docker.V2S2MediaTypeLayer - logrus.Debugf("compressing layer %q with gzip", layerID) - case archive.Bzip2: - // Until the image specs define a media type for bzip2-compressed layers, even if we know - // how to decompress them, we can't try to compress layers with bzip2. - return nil, errors.New("media type for bzip2-compressed layers is not defined") - default: - logrus.Debugf("compressing layer %q with unknown compressor(?)", layerID) - } - } + // Compress the layer, if we're compressing it. writer, err := archive.CompressStream(multiWriter, i.compression) if err != nil { return nil, errors.Wrapf(err, "error compressing layer %q", layerID) @@ -336,6 +362,7 @@ func (i *containerImageRef) NewImageSource(sc *types.SystemContext, manifestType manifestType: manifestType, config: config, configDigest: digest.Canonical.FromBytes(config), + exporting: i.exporting, } return src, nil } @@ -427,7 +454,7 @@ func (i *containerImageSource) GetBlob(blob types.BlobInfo) (reader io.ReadClose return ioutils.NewReadCloserWrapper(layerFile, closer), size, nil } -func (b *Builder) makeContainerImageRef(manifestType string, compress archive.Compression) (types.ImageReference, error) { +func (b *Builder) makeContainerImageRef(manifestType string, exporting bool, compress archive.Compression) (types.ImageReference, error) { var name reference.Named if manifestType == "" { manifestType = OCIv1ImageManifest @@ -461,6 +488,7 @@ func (b *Builder) makeContainerImageRef(manifestType string, compress archive.Co createdBy: b.CreatedBy(), annotations: b.Annotations(), preferredManifestType: manifestType, + exporting: exporting, } return ref, nil }