docker/buildah
1
0
Fork 0
mirror of https://github.com/containers/buildah.git synced 2025-08-07 23:02:58 +03:00
Code Activity

Take a shortcut when writing to local storage

Browse Source 
When writing to local storage, take a couple of shortcuts: instead of
recompressing layers to ensure that the values we store in the image
manifest will be correct for content-addressibility, just pretend that
the layer ID is a blob hash value, and that it's a valid layer diffID.

Local storage doesn't generally care if these values are correct, and we
already have to recompute these values when exporting an image, but this
saves us quite a bit of time.

The image library's Copy() routine actually cares about and
sanity-checks these things, so if we're going to take advantage of the
shortcuts, we need to use its higher-level APIs to write a layer, write
the configuration, and write the manifest, then move those items that it
writes to an image with the right set of layers.

Signed-off-by: Nalin Dahyabhai <nalin@redhat.com>

Closes: #141
Approved by: rhatdan
This commit is contained in:
Nalin Dahyabhai
2017-05-31 13:56:25 -04:00
committed by Atomic Bot
parent ebf3063cf2
commit b2baeb25f4
2 changed files with 230 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

188
commit.go
View File

@@ -1,6 +1,7 @@
package buildah
import (
"bytes"
"io"
"github.com/Sirupsen/logrus"
@@ -10,10 +11,23 @@ import (
"github.com/containers/image/transports"
"github.com/containers/image/types"
"github.com/containers/storage/pkg/archive"
"github.com/containers/storage/pkg/stringid"
"github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"github.com/projectatomic/buildah/util"
)
var (
// gzippedEmptyLayer is a gzip-compressed version of an empty tar file (just 1024 zero bytes). This
// comes from github.com/docker/distribution/manifest/schema1/config_builder.go by way of
// github.com/containers/image/image/docker_schema2.go; there is a non-zero embedded timestamp; we could
// zero that, but that would just waste storage space in registries, so let’s use the same values.
gzippedEmptyLayer = []byte{
31, 139, 8, 0, 0, 9, 110, 136, 0, 255, 98, 24, 5, 163, 96, 20, 140, 88,
0, 8, 0, 0, 255, 255, 46, 175, 181, 239, 0, 4, 0, 0,
}
)
// CommitOptions can be used to alter how an image is committed.
type CommitOptions struct {
// PreferredManifestType is the preferred type of image manifest. The
@@ -38,6 +52,159 @@ type CommitOptions struct {
ReportWriter io.Writer
}
// shallowCopy copies the most recent layer, the configuration, and the manifest from one image to another.
// For local storage, which doesn't care about histories and the manifest's contents, that's sufficient, but
// almost any other destination has higher expectations.
// We assume that "dest" is a reference to a local image (specifically, a containers/image/storage.storageReference),
// and will fail if it isn't.
func (b *Builder) shallowCopy(dest types.ImageReference, src types.ImageReference, systemContext *types.SystemContext) error {
// Read the target image name.
if dest.DockerReference() == nil {
return errors.New("can't write to an unnamed image")
}
names, err := util.ExpandTags([]string{dest.DockerReference().String()})
if err != nil {
return err
}
// Make a temporary image reference.
tmpName := stringid.GenerateRandomID() + "-tmp-" + Package + "-commit"
tmpRef, err := storage.Transport.ParseStoreReference(b.store, tmpName)
if err != nil {
return err
}
defer func() {
if err2 := tmpRef.DeleteImage(systemContext); err2 != nil {
logrus.Debugf("error deleting temporary image %q: %v", tmpName, err2)
}
}()
// Open the source for reading and a temporary image for writing.
srcImage, err := src.NewImage(systemContext)
if err != nil {
return errors.Wrapf(err, "error reading configuration to write to image %q", transports.ImageName(dest))
}
defer srcImage.Close()
tmpImage, err := tmpRef.NewImageDestination(systemContext)
if err != nil {
return errors.Wrapf(err, "error opening temporary copy of image %q for writing", transports.ImageName(dest))
}
defer tmpImage.Close()
// Write an empty filesystem layer, because the image layer requires at least one.
_, err = tmpImage.PutBlob(bytes.NewReader(gzippedEmptyLayer), types.BlobInfo{Size: int64(len(gzippedEmptyLayer))})
if err != nil {
return errors.Wrapf(err, "error writing dummy layer for image %q", transports.ImageName(dest))
}
// Read the newly-generated configuration blob.
config, err := srcImage.ConfigBlob()
if err != nil {
return errors.Wrapf(err, "error reading new configuration for image %q", transports.ImageName(dest))
}
if len(config) == 0 {
return errors.Errorf("error reading new configuration for image %q: it's empty", transports.ImageName(dest))
}
logrus.Debugf("read configuration blob %q", string(config))
// Write the configuration to the temporary image.
configBlobInfo := types.BlobInfo{
Digest: digest.Canonical.FromBytes(config),
Size: int64(len(config)),
}
_, err = tmpImage.PutBlob(bytes.NewReader(config), configBlobInfo)
if err != nil && len(config) > 0 {
return errors.Wrapf(err, "error writing image configuration for temporary copy of %q", transports.ImageName(dest))
}
// Read the newly-generated, mostly fake, manifest.
manifest, _, err := srcImage.Manifest()
if err != nil {
return errors.Wrapf(err, "error reading new manifest for image %q", transports.ImageName(dest))
}
// Write the manifest to the temporary image.
err = tmpImage.PutManifest(manifest)
if err != nil {
return errors.Wrapf(err, "error writing new manifest to temporary copy of image %q", transports.ImageName(dest))
}
// Save the temporary image.
err = tmpImage.Commit()
if err != nil {
return errors.Wrapf(err, "error committing new image %q", transports.ImageName(dest))
}
// Locate the temporary image in the lower-level API. Read its item names.
tmpImg, err := storage.Transport.GetStoreImage(b.store, tmpRef)
if err != nil {
return errors.Wrapf(err, "error locating temporary image %q", transports.ImageName(dest))
}
items, err := b.store.ListImageBigData(tmpImg.ID)
if err != nil {
return errors.Wrapf(err, "error reading list of named data for image %q", tmpImg.ID)
}
// Look up the container's read-write layer.
container, err := b.store.Container(b.ContainerID)
if err != nil {
return errors.Wrapf(err, "error reading information about working container %q", b.ContainerID)
}
parentLayer := ""
// Look up the container's source image's layer, if there is a source image.
if container.ImageID != "" {
img, err2 := b.store.Image(container.ImageID)
if err2 != nil {
return errors.Wrapf(err2, "error reading information about working container %q's source image", b.ContainerID)
}
parentLayer = img.TopLayer
}
// Extract the read-write layer's contents.
layerDiff, err := b.store.Diff(parentLayer, container.LayerID)
if err != nil {
return errors.Wrapf(err, "error reading layer from source image %q", transports.ImageName(src))
}
defer layerDiff.Close()
// Write a copy of the layer for the new image to reference.
layer, _, err := b.store.PutLayer("", parentLayer, []string{}, "", false, layerDiff)
if err != nil {
return errors.Wrapf(err, "error creating new read-only layer from container %q", b.ContainerID)
}
// Create a low-level image record that uses the new layer.
image, err := b.store.CreateImage("", []string{}, layer.ID, "", nil)
if err != nil {
err2 := b.store.DeleteLayer(layer.ID)
if err2 != nil {
logrus.Debugf("error removing layer %q: %v", layer, err2)
}
return errors.Wrapf(err, "error creating new low-level image %q", transports.ImageName(dest))
}
logrus.Debugf("created image ID %q", image.ID)
defer func() {
if err != nil {
_, err2 := b.store.DeleteImage(image.ID, true)
if err2 != nil {
logrus.Debugf("error removing image %q: %v", image.ID, err2)
}
}
}()
// Copy the configuration and manifest, which are big data items, along with whatever else is there.
for _, item := range items {
var data []byte
data, err = b.store.ImageBigData(tmpImg.ID, item)
if err != nil {
return errors.Wrapf(err, "error copying data item %q", item)
}
err = b.store.SetImageBigData(image.ID, item, data)
if err != nil {
return errors.Wrapf(err, "error copying data item %q", item)
}
logrus.Debugf("copied data item %q to %q", item, image.ID)
}
// Set low-level metadata in the new image so that the image library will accept it as a real image.
err = b.store.SetMetadata(image.ID, "{}")
if err != nil {
return errors.Wrapf(err, "error assigning metadata to new image %q", transports.ImageName(dest))
}
// Move the target name(s) from the temporary image to the new image.
err = util.AddImageNames(b.store, image, names)
if err != nil {
return errors.Wrapf(err, "error assigning names %v to new image", names)
}
logrus.Debugf("assigned names %v to image %q", names, image.ID)
return nil
}
// Commit writes the contents of the container, along with its updated
// configuration, to a new image in the specified location, and if we know how,
// add any additional tags that were specified.
@@ -50,13 +217,25 @@ func (b *Builder) Commit(dest types.ImageReference, options CommitOptions) error
if err != nil {
return err
}
src, err := b.makeContainerImageRef(options.PreferredManifestType, options.Compression)
// Check if we're keeping everything in local storage. If so, we can take certain shortcuts.
_, destIsStorage := dest.Transport().(storage.StoreTransport)
exporting := !destIsStorage
src, err := b.makeContainerImageRef(options.PreferredManifestType, exporting, options.Compression)
if err != nil {
return errors.Wrapf(err, "error recomputing layer digests and building metadata")
}
err = cp.Image(policyContext, dest, src, getCopyOptions(options.ReportWriter))
if err != nil {
return errors.Wrapf(err, "error copying layers and metadata")
if exporting {
// Copy everything.
err = cp.Image(policyContext, dest, src, getCopyOptions(options.ReportWriter))
if err != nil {
return errors.Wrapf(err, "error copying layers and metadata")
}
} else {
// Copy only the most recent layer, the configuration, and the manifest.
err = b.shallowCopy(dest, src, getSystemContext(options.SignaturePolicyPath))
if err != nil {
return errors.Wrapf(err, "error copying layer and metadata")
}
}
if len(options.AdditionalTags) > 0 {
switch dest.Transport().Name() {
@@ -69,6 +248,7 @@ func (b *Builder) Commit(dest types.ImageReference, options CommitOptions) error
if err != nil {
return errors.Wrapf(err, "error setting image names to %v", append(img.Names, options.AdditionalTags...))
}
logrus.Debugf("assigned names %v to image %q", img.Names, img.ID)
default:
logrus.Warnf("don't know how to add tags to images stored in %q transport", dest.Transport().Name())
}