Calculate hash based image IDs on pull

Generate a hash chain involving the image configuration, layer digests,
and parent image hashes. Use the digests to compute IDs for each image
in a manifest, instead of using the remotely specified IDs.

To avoid breaking users' caches, check for images already in the graph
under old IDs, and avoid repulling an image if the version on disk under
the legacy ID ends up with the same digest that was computed from the
manifest for that image.

When a calculated ID already exists in the graph but can't be verified,
continue trying SHA256(digest) until a suitable ID is found.

"save" and "load" are not changed to use a similar scheme. "load" will
preserve the IDs present in the tar file.

Signed-off-by: Aaron Lehmann <aaron.lehmann@docker.com>

integration-cli/docker_cli_pull_test.go

@@ -1,7 +1,11 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
 	"regexp"
 	"strings"
 	"time"
@@ -159,3 +163,254 @@ func (s *DockerHubPullSuite) TestPullClientDisconnect(c *check.C) {
 		time.Sleep(500 * time.Millisecond)
 	}
 }
+
+type idAndParent struct {
+	ID     string
+	Parent string
+}
+
+func inspectImage(c *check.C, imageRef string) idAndParent {
+	out, _ := dockerCmd(c, "inspect", imageRef)
+	var inspectOutput []idAndParent
+	err := json.Unmarshal([]byte(out), &inspectOutput)
+	if err != nil {
+		c.Fatal(err)
+	}
+
+	return inspectOutput[0]
+}
+
+func imageID(c *check.C, imageRef string) string {
+	return inspectImage(c, imageRef).ID
+}
+
+func imageParent(c *check.C, imageRef string) string {
+	return inspectImage(c, imageRef).Parent
+}
+
+// TestPullMigration verifies that pulling an image based on layers
+// that already exists locally will reuse those existing layers.
+func (s *DockerRegistrySuite) TestPullMigration(c *check.C) {
+	repoName := privateRegistryURL + "/dockercli/migration"
+
+	baseImage := repoName + ":base"
+	_, err := buildImage(baseImage, fmt.Sprintf(`
+	    FROM scratch
+	    ENV IMAGE base
+	    CMD echo %s
+	`, baseImage), true)
+	if err != nil {
+		c.Fatal(err)
+	}
+
+	baseIDBeforePush := imageID(c, baseImage)
+	baseParentBeforePush := imageParent(c, baseImage)
+
+	derivedImage := repoName + ":derived"
+	_, err = buildImage(derivedImage, fmt.Sprintf(`
+	    FROM %s
+	    CMD echo %s
+	`, baseImage, derivedImage), true)
+	if err != nil {
+		c.Fatal(err)
+	}
+
+	derivedIDBeforePush := imageID(c, derivedImage)
+
+	dockerCmd(c, "push", derivedImage)
+
+	// Remove derived image from the local store
+	dockerCmd(c, "rmi", derivedImage)
+
+	// Repull
+	dockerCmd(c, "pull", derivedImage)
+
+	// Check that the parent of this pulled image is the original base
+	// image
+	derivedIDAfterPull1 := imageID(c, derivedImage)
+	derivedParentAfterPull1 := imageParent(c, derivedImage)
+
+	if derivedIDAfterPull1 == derivedIDBeforePush {
+		c.Fatal("image's ID should have changed on after deleting and pulling")
+	}
+
+	if derivedParentAfterPull1 != baseIDBeforePush {
+		c.Fatalf("pulled image's parent ID (%s) does not match base image's ID (%s)", derivedParentAfterPull1, baseIDBeforePush)
+	}
+
+	// Confirm that repushing and repulling does not change the computed ID
+	dockerCmd(c, "push", derivedImage)
+	dockerCmd(c, "rmi", derivedImage)
+	dockerCmd(c, "pull", derivedImage)
+
+	derivedIDAfterPull2 := imageID(c, derivedImage)
+	derivedParentAfterPull2 := imageParent(c, derivedImage)
+
+	if derivedIDAfterPull2 != derivedIDAfterPull1 {
+		c.Fatal("image's ID unexpectedly changed after a repush/repull")
+	}
+
+	if derivedParentAfterPull2 != baseIDBeforePush {
+		c.Fatalf("pulled image's parent ID (%s) does not match base image's ID (%s)", derivedParentAfterPull2, baseIDBeforePush)
+	}
+
+	// Remove everything, repull, and make sure everything uses computed IDs
+	dockerCmd(c, "rmi", baseImage, derivedImage)
+	dockerCmd(c, "pull", derivedImage)
+
+	derivedIDAfterPull3 := imageID(c, derivedImage)
+	derivedParentAfterPull3 := imageParent(c, derivedImage)
+	derivedGrandparentAfterPull3 := imageParent(c, derivedParentAfterPull3)
+
+	if derivedIDAfterPull3 != derivedIDAfterPull1 {
+		c.Fatal("image's ID unexpectedly changed after a second repull")
+	}
+
+	if derivedParentAfterPull3 == baseIDBeforePush {
+		c.Fatalf("pulled image's parent ID (%s) should not match base image's original ID (%s)", derivedParentAfterPull3, derivedIDBeforePush)
+	}
+
+	if derivedGrandparentAfterPull3 == baseParentBeforePush {
+		c.Fatal("base image's parent ID should have been rewritten on pull")
+	}
+}
+
+// TestPullMigrationRun verifies that pulling an image based on layers
+// that already exists locally will result in an image that runs properly.
+func (s *DockerRegistrySuite) TestPullMigrationRun(c *check.C) {
+	type idAndParent struct {
+		ID     string
+		Parent string
+	}
+
+	derivedImage := privateRegistryURL + "/dockercli/migration-run"
+	baseImage := "busybox"
+
+	_, err := buildImage(derivedImage, fmt.Sprintf(`
+	    FROM %s
+	    RUN dd if=/dev/zero of=/file bs=1024 count=1024
+	    CMD echo %s
+	`, baseImage, derivedImage), true)
+	if err != nil {
+		c.Fatal(err)
+	}
+
+	baseIDBeforePush := imageID(c, baseImage)
+	derivedIDBeforePush := imageID(c, derivedImage)
+
+	dockerCmd(c, "push", derivedImage)
+
+	// Remove derived image from the local store
+	dockerCmd(c, "rmi", derivedImage)
+
+	// Repull
+	dockerCmd(c, "pull", derivedImage)
+
+	// Check that this pulled image is based on the original base image
+	derivedIDAfterPull1 := imageID(c, derivedImage)
+	derivedParentAfterPull1 := imageParent(c, imageParent(c, derivedImage))
+
+	if derivedIDAfterPull1 == derivedIDBeforePush {
+		c.Fatal("image's ID should have changed on after deleting and pulling")
+	}
+
+	if derivedParentAfterPull1 != baseIDBeforePush {
+		c.Fatalf("pulled image's parent ID (%s) does not match base image's ID (%s)", derivedParentAfterPull1, baseIDBeforePush)
+	}
+
+	// Make sure the image runs correctly
+	out, _ := dockerCmd(c, "run", "--rm", derivedImage)
+	if strings.TrimSpace(out) != derivedImage {
+		c.Fatalf("expected %s; got %s", derivedImage, out)
+	}
+
+	// Confirm that repushing and repulling does not change the computed ID
+	dockerCmd(c, "push", derivedImage)
+	dockerCmd(c, "rmi", derivedImage)
+	dockerCmd(c, "pull", derivedImage)
+
+	derivedIDAfterPull2 := imageID(c, derivedImage)
+	derivedParentAfterPull2 := imageParent(c, imageParent(c, derivedImage))
+
+	if derivedIDAfterPull2 != derivedIDAfterPull1 {
+		c.Fatal("image's ID unexpectedly changed after a repush/repull")
+	}
+
+	if derivedParentAfterPull2 != baseIDBeforePush {
+		c.Fatalf("pulled image's parent ID (%s) does not match base image's ID (%s)", derivedParentAfterPull2, baseIDBeforePush)
+	}
+
+	// Make sure the image still runs
+	out, _ = dockerCmd(c, "run", "--rm", derivedImage)
+	if strings.TrimSpace(out) != derivedImage {
+		c.Fatalf("expected %s; got %s", derivedImage, out)
+	}
+}
+
+// TestPullConflict provides coverage of the situation where a computed
+// strongID conflicts with some unverifiable data in the graph.
+func (s *DockerRegistrySuite) TestPullConflict(c *check.C) {
+	repoName := privateRegistryURL + "/dockercli/conflict"
+
+	_, err := buildImage(repoName, `
+	    FROM scratch
+	    ENV IMAGE conflict
+	    CMD echo conflict
+	`, true)
+	if err != nil {
+		c.Fatal(err)
+	}
+
+	dockerCmd(c, "push", repoName)
+
+	// Pull to make it content-addressable
+	dockerCmd(c, "rmi", repoName)
+	dockerCmd(c, "pull", repoName)
+
+	IDBeforeLoad := imageID(c, repoName)
+
+	// Load/save to turn this into an unverified image with the same ID
+	tmpDir, err := ioutil.TempDir("", "conflict-save-output")
+	if err != nil {
+		c.Errorf("failed to create temporary directory: %s", err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	tarFile := filepath.Join(tmpDir, "repo.tar")
+
+	dockerCmd(c, "save", "-o", tarFile, repoName)
+	dockerCmd(c, "rmi", repoName)
+	dockerCmd(c, "load", "-i", tarFile)
+
+	// Check that the the ID is the same after save/load.
+	IDAfterLoad := imageID(c, repoName)
+
+	if IDAfterLoad != IDBeforeLoad {
+		c.Fatal("image's ID should be the same after save/load")
+	}
+
+	// Repull
+	dockerCmd(c, "pull", repoName)
+
+	// Check that the ID is now different because of the conflict.
+	IDAfterPull1 := imageID(c, repoName)
+
+	// Expect the new ID to be SHA256(oldID)
+	expectedIDDigest, err := digest.FromBytes([]byte(IDBeforeLoad))
+	if err != nil {
+		c.Fatalf("digest error: %v", err)
+	}
+	expectedID := expectedIDDigest.Hex()
+	if IDAfterPull1 != expectedID {
+		c.Fatalf("image's ID should have changed on pull to %s (got %s)", expectedID, IDAfterPull1)
+	}
+
+	// A second pull should use the new ID again.
+	dockerCmd(c, "pull", repoName)
+
+	IDAfterPull2 := imageID(c, repoName)
+
+	if IDAfterPull2 != IDAfterPull1 {
+		c.Fatal("image's ID unexpectedly changed after a repull")
+	}
+}