Merge component 'engine' from git@github.com:moby/moby master

components/engine/daemon/graphdriver/lcow/lcow.go

@@ -56,11 +56,13 @@
 package lcow // import "github.com/docker/docker/daemon/graphdriver/lcow"
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -964,3 +966,87 @@ func hostToGuest(hostpath string) string {
 func unionMountName(disks []hcsshim.MappedVirtualDisk) string {
 	return fmt.Sprintf("%s-mount", disks[0].ContainerPath)
 }
+
+type nopCloser struct {
+	io.Reader
+}
+
+func (nopCloser) Close() error {
+	return nil
+}
+
+type fileGetCloserFromSVM struct {
+	id  string
+	svm *serviceVM
+	mvd *hcsshim.MappedVirtualDisk
+	d   *Driver
+}
+
+func (fgc *fileGetCloserFromSVM) Close() error {
+	if fgc.svm != nil {
+		if fgc.mvd != nil {
+			if err := fgc.svm.hotRemoveVHDs(*fgc.mvd); err != nil {
+				// We just log this as we're going to tear down the SVM imminently unless in global mode
+				logrus.Errorf("failed to remove mvd %s: %s", fgc.mvd.ContainerPath, err)
+			}
+		}
+	}
+	if fgc.d != nil && fgc.svm != nil && fgc.id != "" {
+		if err := fgc.d.terminateServiceVM(fgc.id, fmt.Sprintf("diffgetter %s", fgc.id), false); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (fgc *fileGetCloserFromSVM) Get(filename string) (io.ReadCloser, error) {
+	errOut := &bytes.Buffer{}
+	outOut := &bytes.Buffer{}
+	file := path.Join(fgc.mvd.ContainerPath, filename)
+	if err := fgc.svm.runProcess(fmt.Sprintf("cat %s", file), nil, outOut, errOut); err != nil {
+		logrus.Debugf("cat %s failed: %s", file, errOut.String())
+		return nil, err
+	}
+	return nopCloser{bytes.NewReader(outOut.Bytes())}, nil
+}
+
+// DiffGetter returns a FileGetCloser that can read files from the directory that
+// contains files for the layer differences. Used for direct access for tar-split.
+func (d *Driver) DiffGetter(id string) (graphdriver.FileGetCloser, error) {
+	title := fmt.Sprintf("lcowdriver: diffgetter: %s", id)
+	logrus.Debugf(title)
+
+	ld, err := getLayerDetails(d.dir(id))
+	if err != nil {
+		logrus.Debugf("%s: failed to get vhdx information of %s: %s", title, d.dir(id), err)
+		return nil, err
+	}
+
+	// Start the SVM with a mapped virtual disk. Note that if the SVM is
+	// already running and we are in global mode, this will be hot-added.
+	mvd := hcsshim.MappedVirtualDisk{
+		HostPath:          ld.filename,
+		ContainerPath:     hostToGuest(ld.filename),
+		CreateInUtilityVM: true,
+		ReadOnly:          true,
+	}
+
+	logrus.Debugf("%s: starting service VM", title)
+	svm, err := d.startServiceVMIfNotRunning(id, []hcsshim.MappedVirtualDisk{mvd}, fmt.Sprintf("diffgetter %s", id))
+	if err != nil {
+		return nil, err
+	}
+
+	logrus.Debugf("%s: waiting for svm to finish booting", title)
+	err = svm.getStartError()
+	if err != nil {
+		d.terminateServiceVM(id, fmt.Sprintf("diff %s", id), false)
+		return nil, fmt.Errorf("%s: svm failed to boot: %s", title, err)
+	}
+
+	return &fileGetCloserFromSVM{
+		id:  id,
+		svm: svm,
+		mvd: &mvd,
+		d:   d}, nil
+}

components/engine/image/tarexport/save.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
 	"runtime"
 	"time"
@@ -158,6 +159,9 @@ func (l *tarexporter) takeLayerReference(id image.ID, imgDescr *imageDescriptor)
 	if os == "" {
 		os = runtime.GOOS
 	}
+	if !system.IsOSSupported(os) {
+		return fmt.Errorf("os %q is not supported", os)
+	}
 	layer, err := l.lss[os].Get(topLayerID)
 	if err != nil {
 		return err
@@ -216,7 +220,11 @@ func (s *saveSession) save(outStream io.Writer) error {
 		}
 
 		for _, l := range imageDescr.layers {
-			layers = append(layers, filepath.Join(l, legacyLayerFileName))
+			// IMPORTANT: We use path, not filepath here to ensure the layers
+			// in the manifest use Unix-style forward-slashes. Otherwise, a
+			// Linux image saved from LCOW won't be able to be imported on
+			// LCOL.
+			layers = append(layers, path.Join(l, legacyLayerFileName))
 		}
 
 		manifest = append(manifest, manifestItem{
@@ -313,6 +321,7 @@ func (s *saveSession) saveImage(id image.ID) (map[layer.DiffID]distribution.Desc
 			v1Img.Parent = parent.Hex()
 		}
 
+		v1Img.OS = img.OS
 		src, err := s.saveLayer(rootFS.ChainID(), v1Img, img.Created)
 		if err != nil {
 			return nil, err

components/engine/volume/local/local_test.go

@@ -32,14 +32,13 @@ func TestGetAddress(t *testing.T) {
 
 func TestRemove(t *testing.T) {
 	skip.If(t, runtime.GOOS == "windows", "FIXME: investigate why this test fails on CI")
-	skip.If(t, os.Getuid() != 0, "skipping test that requires root")
 	rootDir, err := ioutil.TempDir("", "local-volume-test")
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer os.RemoveAll(rootDir)
 
-	r, err := New(rootDir, idtools.IDPair{UID: 0, GID: 0})
+	r, err := New(rootDir, idtools.IDPair{UID: os.Geteuid(), GID: os.Getegid()})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -75,14 +74,13 @@ func TestRemove(t *testing.T) {
 }
 
 func TestInitializeWithVolumes(t *testing.T) {
-	skip.If(t, os.Getuid() != 0, "skipping test that requires root")
 	rootDir, err := ioutil.TempDir("", "local-volume-test")
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer os.RemoveAll(rootDir)
 
-	r, err := New(rootDir, idtools.IDPair{UID: 0, GID: 0})
+	r, err := New(rootDir, idtools.IDPair{UID: os.Geteuid(), GID: os.Getegid()})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -92,7 +90,7 @@ func TestInitializeWithVolumes(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	r, err = New(rootDir, idtools.IDPair{UID: 0, GID: 0})
+	r, err = New(rootDir, idtools.IDPair{UID: os.Getuid(), GID: os.Getegid()})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -108,14 +106,13 @@ func TestInitializeWithVolumes(t *testing.T) {
 }
 
 func TestCreate(t *testing.T) {
-	skip.If(t, os.Getuid() != 0, "skipping test that requires root")
 	rootDir, err := ioutil.TempDir("", "local-volume-test")
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer os.RemoveAll(rootDir)
 
-	r, err := New(rootDir, idtools.IDPair{UID: 0, GID: 0})
+	r, err := New(rootDir, idtools.IDPair{UID: os.Getuid(), GID: os.Getegid()})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -152,7 +149,7 @@ func TestCreate(t *testing.T) {
 		}
 	}
 
-	r, err = New(rootDir, idtools.IDPair{UID: 0, GID: 0})
+	r, err = New(rootDir, idtools.IDPair{UID: os.Getuid(), GID: os.Getegid()})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -182,14 +179,14 @@ func TestValidateName(t *testing.T) {
 
 func TestCreateWithOpts(t *testing.T) {
 	skip.If(t, runtime.GOOS == "windows")
-	skip.If(t, os.Getuid() != 0, "skipping test that requires root")
+	skip.If(t, os.Getuid() != 0, "requires mounts")
 	rootDir, err := ioutil.TempDir("", "local-volume-test")
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer os.RemoveAll(rootDir)
 
-	r, err := New(rootDir, idtools.IDPair{UID: 0, GID: 0})
+	r, err := New(rootDir, idtools.IDPair{UID: os.Getuid(), GID: os.Getegid()})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -280,14 +277,13 @@ func TestCreateWithOpts(t *testing.T) {
 }
 
 func TestRelaodNoOpts(t *testing.T) {
-	skip.If(t, os.Getuid() != 0, "skipping test that requires root")
 	rootDir, err := ioutil.TempDir("", "volume-test-reload-no-opts")
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer os.RemoveAll(rootDir)
 
-	r, err := New(rootDir, idtools.IDPair{UID: 0, GID: 0})
+	r, err := New(rootDir, idtools.IDPair{UID: os.Getuid(), GID: os.Getegid()})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -315,7 +311,7 @@ func TestRelaodNoOpts(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	r, err = New(rootDir, idtools.IDPair{UID: 0, GID: 0})
+	r, err = New(rootDir, idtools.IDPair{UID: os.Getuid(), GID: os.Getegid()})
 	if err != nil {
 		t.Fatal(err)
 	}

components/engine/volume/store/store.go

@@ -80,7 +80,7 @@ func New(rootPath string, drivers *drivers.Store) (*VolumeStore, error) {
 	if rootPath != "" {
 		// initialize metadata store
 		volPath := filepath.Join(rootPath, volumeDataDir)
-		if err := os.MkdirAll(volPath, 750); err != nil {
+		if err := os.MkdirAll(volPath, 0750); err != nil {
 			return nil, err
 		}
 

components/engine/volume/store/store_test.go

@@ -15,11 +15,9 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/gotestyourself/gotestyourself/assert"
 	is "github.com/gotestyourself/gotestyourself/assert/cmp"
-	"github.com/gotestyourself/gotestyourself/skip"
 )
 
 func TestCreate(t *testing.T) {
-	skip.If(t, os.Getuid() != 0, "skipping test that requires root")
 	t.Parallel()
 
 	s, cleanup := setupTest(t)
@@ -49,7 +47,6 @@ func TestCreate(t *testing.T) {
 }
 
 func TestRemove(t *testing.T) {
-	skip.If(t, os.Getuid() != 0, "skipping test that requires root")
 	t.Parallel()
 
 	s, cleanup := setupTest(t)
@@ -128,7 +125,6 @@ func TestList(t *testing.T) {
 }
 
 func TestFilterByDriver(t *testing.T) {
-	skip.If(t, os.Getuid() != 0, "skipping test that requires root")
 	t.Parallel()
 	s, cleanup := setupTest(t)
 	defer cleanup()
@@ -156,7 +152,6 @@ func TestFilterByDriver(t *testing.T) {
 }
 
 func TestFilterByUsed(t *testing.T) {
-	skip.If(t, os.Getuid() != 0, "skipping test that requires root")
 	t.Parallel()
 	s, cleanup := setupTest(t)
 	defer cleanup()
@@ -194,7 +189,6 @@ func TestFilterByUsed(t *testing.T) {
 }
 
 func TestDerefMultipleOfSameRef(t *testing.T) {
-	skip.If(t, os.Getuid() != 0, "skipping test that requires root")
 	t.Parallel()
 	s, cleanup := setupTest(t)
 	defer cleanup()
@@ -216,7 +210,6 @@ func TestDerefMultipleOfSameRef(t *testing.T) {
 }
 
 func TestCreateKeepOptsLabelsWhenExistsRemotely(t *testing.T) {
-	skip.If(t, os.Getuid() != 0, "skipping test that requires root")
 	t.Parallel()
 	s, cleanup := setupTest(t)
 	defer cleanup()
@@ -245,7 +238,6 @@ func TestCreateKeepOptsLabelsWhenExistsRemotely(t *testing.T) {
 }
 
 func TestDefererencePluginOnCreateError(t *testing.T) {
-	skip.If(t, os.Getuid() != 0, "skipping test that requires root")
 	t.Parallel()
 
 	var (
@@ -292,7 +284,6 @@ func TestDefererencePluginOnCreateError(t *testing.T) {
 }
 
 func TestRefDerefRemove(t *testing.T) {
-	skip.If(t, os.Getuid() != 0, "skipping test that requires root")
 	t.Parallel()
 
 	driverName := "test-ref-deref-remove"
@@ -313,7 +304,6 @@ func TestRefDerefRemove(t *testing.T) {
 }
 
 func TestGet(t *testing.T) {
-	skip.If(t, os.Getuid() != 0, "skipping test that requires root")
 	t.Parallel()
 
 	driverName := "test-get"
@@ -340,7 +330,6 @@ func TestGet(t *testing.T) {
 }
 
 func TestGetWithRef(t *testing.T) {
-	skip.If(t, os.Getuid() != 0, "skipping test that requires root")
 	t.Parallel()
 
 	driverName := "test-get-with-ref"