From 47c7cacdf3865147f49d9a22d5c2a22033ff109e Mon Sep 17 00:00:00 2001 From: John Howard Date: Tue, 13 Mar 2018 09:46:21 -0700 Subject: [PATCH 1/5] LCOW: Check OS in takeLayerReference Signed-off-by: John Howard Upstream-commit: d8dbba14fd6a1be91e88082d4d4ba9aafd8c880a Component: engine --- components/engine/image/tarexport/save.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/engine/image/tarexport/save.go b/components/engine/image/tarexport/save.go index 1f32fadb5b..257d227880 100644 --- a/components/engine/image/tarexport/save.go +++ b/components/engine/image/tarexport/save.go @@ -158,6 +158,9 @@ func (l *tarexporter) takeLayerReference(id image.ID, imgDescr *imageDescriptor) if os == "" { os = runtime.GOOS } + if !system.IsOSSupported(os) { + return fmt.Errorf("os %q is not supported", os) + } layer, err := l.lss[os].Get(topLayerID) if err != nil { return err From d2852cf7acccc0d1638288e2e7ec6c03b0bc9b29 Mon Sep 17 00:00:00 2001 From: John Howard Date: Tue, 13 Mar 2018 10:12:56 -0700 Subject: [PATCH 2/5] LCOW: Make sure OS is copied for docker save Signed-off-by: John Howard Upstream-commit: d5c781323d5534a01e42cf8fd8cf773c6f1cdff2 Component: engine --- components/engine/image/tarexport/save.go | 1 + 1 file changed, 1 insertion(+) diff --git a/components/engine/image/tarexport/save.go b/components/engine/image/tarexport/save.go index 257d227880..9d7f92a452 100644 --- a/components/engine/image/tarexport/save.go +++ b/components/engine/image/tarexport/save.go @@ -316,6 +316,7 @@ func (s *saveSession) saveImage(id image.ID) (map[layer.DiffID]distribution.Desc v1Img.Parent = parent.Hex() } + v1Img.OS = img.OS src, err := s.saveLayer(rootFS.ChainID(), v1Img, img.Created) if err != nil { return nil, err From c113de75aefd25379a0593ac12fd8e9d3c4361d2 Mon Sep 17 00:00:00 2001 From: John Howard Date: Tue, 13 Mar 2018 15:55:17 -0700 Subject: [PATCH 3/5] LCOW: Graphdriver DiffGetter implementation Signed-off-by: John Howard Upstream-commit: 6b4f989bc2c19be58e10731b9b0504b5f94607bf Component: engine --- .../engine/daemon/graphdriver/lcow/lcow.go | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/components/engine/daemon/graphdriver/lcow/lcow.go b/components/engine/daemon/graphdriver/lcow/lcow.go index f5a2ee25fe..9cc06201e0 100644 --- a/components/engine/daemon/graphdriver/lcow/lcow.go +++ b/components/engine/daemon/graphdriver/lcow/lcow.go @@ -56,11 +56,13 @@ package lcow // import "github.com/docker/docker/daemon/graphdriver/lcow" import ( + "bytes" "encoding/json" "fmt" "io" "io/ioutil" "os" + "path" "path/filepath" "strconv" "strings" @@ -964,3 +966,87 @@ func hostToGuest(hostpath string) string { func unionMountName(disks []hcsshim.MappedVirtualDisk) string { return fmt.Sprintf("%s-mount", disks[0].ContainerPath) } + +type nopCloser struct { + io.Reader +} + +func (nopCloser) Close() error { + return nil +} + +type fileGetCloserFromSVM struct { + id string + svm *serviceVM + mvd *hcsshim.MappedVirtualDisk + d *Driver +} + +func (fgc *fileGetCloserFromSVM) Close() error { + if fgc.svm != nil { + if fgc.mvd != nil { + if err := fgc.svm.hotRemoveVHDs(*fgc.mvd); err != nil { + // We just log this as we're going to tear down the SVM imminently unless in global mode + logrus.Errorf("failed to remove mvd %s: %s", fgc.mvd.ContainerPath, err) + } + } + } + if fgc.d != nil && fgc.svm != nil && fgc.id != "" { + if err := fgc.d.terminateServiceVM(fgc.id, fmt.Sprintf("diffgetter %s", fgc.id), false); err != nil { + return err + } + } + return nil +} + +func (fgc *fileGetCloserFromSVM) Get(filename string) (io.ReadCloser, error) { + errOut := &bytes.Buffer{} + outOut := &bytes.Buffer{} + file := path.Join(fgc.mvd.ContainerPath, filename) + if err := fgc.svm.runProcess(fmt.Sprintf("cat %s", file), nil, outOut, errOut); err != nil { + logrus.Debugf("cat %s failed: %s", file, errOut.String()) + return nil, err + } + return nopCloser{bytes.NewReader(outOut.Bytes())}, nil +} + +// DiffGetter returns a FileGetCloser that can read files from the directory that +// contains files for the layer differences. Used for direct access for tar-split. +func (d *Driver) DiffGetter(id string) (graphdriver.FileGetCloser, error) { + title := fmt.Sprintf("lcowdriver: diffgetter: %s", id) + logrus.Debugf(title) + + ld, err := getLayerDetails(d.dir(id)) + if err != nil { + logrus.Debugf("%s: failed to get vhdx information of %s: %s", title, d.dir(id), err) + return nil, err + } + + // Start the SVM with a mapped virtual disk. Note that if the SVM is + // already running and we are in global mode, this will be hot-added. + mvd := hcsshim.MappedVirtualDisk{ + HostPath: ld.filename, + ContainerPath: hostToGuest(ld.filename), + CreateInUtilityVM: true, + ReadOnly: true, + } + + logrus.Debugf("%s: starting service VM", title) + svm, err := d.startServiceVMIfNotRunning(id, []hcsshim.MappedVirtualDisk{mvd}, fmt.Sprintf("diffgetter %s", id)) + if err != nil { + return nil, err + } + + logrus.Debugf("%s: waiting for svm to finish booting", title) + err = svm.getStartError() + if err != nil { + d.terminateServiceVM(id, fmt.Sprintf("diff %s", id), false) + return nil, fmt.Errorf("%s: svm failed to boot: %s", title, err) + } + + return &fileGetCloserFromSVM{ + id: id, + svm: svm, + mvd: &mvd, + d: d}, nil +} From a2e4c2ffca575eee4a4e0ed358a11df5878f9fca Mon Sep 17 00:00:00 2001 From: John Howard Date: Wed, 14 Mar 2018 10:00:15 -0700 Subject: [PATCH 4/5] LCOW: Write saved manifest.json in Unix paths Signed-off-by: John Howard Upstream-commit: 0a49de4eb551510cf4aafa7d212ec2bb8041f642 Component: engine --- components/engine/image/tarexport/save.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/components/engine/image/tarexport/save.go b/components/engine/image/tarexport/save.go index 9d7f92a452..4e734b3503 100644 --- a/components/engine/image/tarexport/save.go +++ b/components/engine/image/tarexport/save.go @@ -6,6 +6,7 @@ import ( "io" "io/ioutil" "os" + "path" "path/filepath" "runtime" "time" @@ -219,7 +220,11 @@ func (s *saveSession) save(outStream io.Writer) error { } for _, l := range imageDescr.layers { - layers = append(layers, filepath.Join(l, legacyLayerFileName)) + // IMPORTANT: We use path, not filepath here to ensure the layers + // in the manifest use Unix-style forward-slashes. Otherwise, a + // Linux image saved from LCOW won't be able to be imported on + // LCOL. + layers = append(layers, path.Join(l, legacyLayerFileName)) } manifest = append(manifest, manifestItem{ From 9edbbd0436ceeb775f69125dbcb95fa2ca2bd556 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Tue, 24 Apr 2018 10:26:10 -0400 Subject: [PATCH 5/5] Fix issues with running volume tests as non-root. - Volume store created dir with wrong permissions - Local volume driver hardcoded uid/gid 0 Signed-off-by: Brian Goff Upstream-commit: d15734ec3c10eda667b716f67e18d5d86e708e3e Component: engine --- components/engine/volume/local/local_test.go | 22 ++++++++------------ components/engine/volume/store/store.go | 2 +- components/engine/volume/store/store_test.go | 11 ---------- 3 files changed, 10 insertions(+), 25 deletions(-) diff --git a/components/engine/volume/local/local_test.go b/components/engine/volume/local/local_test.go index 70e435676a..541b8448aa 100644 --- a/components/engine/volume/local/local_test.go +++ b/components/engine/volume/local/local_test.go @@ -32,14 +32,13 @@ func TestGetAddress(t *testing.T) { func TestRemove(t *testing.T) { skip.If(t, runtime.GOOS == "windows", "FIXME: investigate why this test fails on CI") - skip.If(t, os.Getuid() != 0, "skipping test that requires root") rootDir, err := ioutil.TempDir("", "local-volume-test") if err != nil { t.Fatal(err) } defer os.RemoveAll(rootDir) - r, err := New(rootDir, idtools.IDPair{UID: 0, GID: 0}) + r, err := New(rootDir, idtools.IDPair{UID: os.Geteuid(), GID: os.Getegid()}) if err != nil { t.Fatal(err) } @@ -75,14 +74,13 @@ func TestRemove(t *testing.T) { } func TestInitializeWithVolumes(t *testing.T) { - skip.If(t, os.Getuid() != 0, "skipping test that requires root") rootDir, err := ioutil.TempDir("", "local-volume-test") if err != nil { t.Fatal(err) } defer os.RemoveAll(rootDir) - r, err := New(rootDir, idtools.IDPair{UID: 0, GID: 0}) + r, err := New(rootDir, idtools.IDPair{UID: os.Geteuid(), GID: os.Getegid()}) if err != nil { t.Fatal(err) } @@ -92,7 +90,7 @@ func TestInitializeWithVolumes(t *testing.T) { t.Fatal(err) } - r, err = New(rootDir, idtools.IDPair{UID: 0, GID: 0}) + r, err = New(rootDir, idtools.IDPair{UID: os.Getuid(), GID: os.Getegid()}) if err != nil { t.Fatal(err) } @@ -108,14 +106,13 @@ func TestInitializeWithVolumes(t *testing.T) { } func TestCreate(t *testing.T) { - skip.If(t, os.Getuid() != 0, "skipping test that requires root") rootDir, err := ioutil.TempDir("", "local-volume-test") if err != nil { t.Fatal(err) } defer os.RemoveAll(rootDir) - r, err := New(rootDir, idtools.IDPair{UID: 0, GID: 0}) + r, err := New(rootDir, idtools.IDPair{UID: os.Getuid(), GID: os.Getegid()}) if err != nil { t.Fatal(err) } @@ -152,7 +149,7 @@ func TestCreate(t *testing.T) { } } - r, err = New(rootDir, idtools.IDPair{UID: 0, GID: 0}) + r, err = New(rootDir, idtools.IDPair{UID: os.Getuid(), GID: os.Getegid()}) if err != nil { t.Fatal(err) } @@ -182,14 +179,14 @@ func TestValidateName(t *testing.T) { func TestCreateWithOpts(t *testing.T) { skip.If(t, runtime.GOOS == "windows") - skip.If(t, os.Getuid() != 0, "skipping test that requires root") + skip.If(t, os.Getuid() != 0, "requires mounts") rootDir, err := ioutil.TempDir("", "local-volume-test") if err != nil { t.Fatal(err) } defer os.RemoveAll(rootDir) - r, err := New(rootDir, idtools.IDPair{UID: 0, GID: 0}) + r, err := New(rootDir, idtools.IDPair{UID: os.Getuid(), GID: os.Getegid()}) if err != nil { t.Fatal(err) } @@ -280,14 +277,13 @@ func TestCreateWithOpts(t *testing.T) { } func TestRelaodNoOpts(t *testing.T) { - skip.If(t, os.Getuid() != 0, "skipping test that requires root") rootDir, err := ioutil.TempDir("", "volume-test-reload-no-opts") if err != nil { t.Fatal(err) } defer os.RemoveAll(rootDir) - r, err := New(rootDir, idtools.IDPair{UID: 0, GID: 0}) + r, err := New(rootDir, idtools.IDPair{UID: os.Getuid(), GID: os.Getegid()}) if err != nil { t.Fatal(err) } @@ -315,7 +311,7 @@ func TestRelaodNoOpts(t *testing.T) { t.Fatal(err) } - r, err = New(rootDir, idtools.IDPair{UID: 0, GID: 0}) + r, err = New(rootDir, idtools.IDPair{UID: os.Getuid(), GID: os.Getegid()}) if err != nil { t.Fatal(err) } diff --git a/components/engine/volume/store/store.go b/components/engine/volume/store/store.go index 67b4e7d7ef..990bc3077b 100644 --- a/components/engine/volume/store/store.go +++ b/components/engine/volume/store/store.go @@ -80,7 +80,7 @@ func New(rootPath string, drivers *drivers.Store) (*VolumeStore, error) { if rootPath != "" { // initialize metadata store volPath := filepath.Join(rootPath, volumeDataDir) - if err := os.MkdirAll(volPath, 750); err != nil { + if err := os.MkdirAll(volPath, 0750); err != nil { return nil, err } diff --git a/components/engine/volume/store/store_test.go b/components/engine/volume/store/store_test.go index 255480a75f..288a4ce824 100644 --- a/components/engine/volume/store/store_test.go +++ b/components/engine/volume/store/store_test.go @@ -15,11 +15,9 @@ import ( "github.com/google/go-cmp/cmp" "github.com/gotestyourself/gotestyourself/assert" is "github.com/gotestyourself/gotestyourself/assert/cmp" - "github.com/gotestyourself/gotestyourself/skip" ) func TestCreate(t *testing.T) { - skip.If(t, os.Getuid() != 0, "skipping test that requires root") t.Parallel() s, cleanup := setupTest(t) @@ -49,7 +47,6 @@ func TestCreate(t *testing.T) { } func TestRemove(t *testing.T) { - skip.If(t, os.Getuid() != 0, "skipping test that requires root") t.Parallel() s, cleanup := setupTest(t) @@ -128,7 +125,6 @@ func TestList(t *testing.T) { } func TestFilterByDriver(t *testing.T) { - skip.If(t, os.Getuid() != 0, "skipping test that requires root") t.Parallel() s, cleanup := setupTest(t) defer cleanup() @@ -156,7 +152,6 @@ func TestFilterByDriver(t *testing.T) { } func TestFilterByUsed(t *testing.T) { - skip.If(t, os.Getuid() != 0, "skipping test that requires root") t.Parallel() s, cleanup := setupTest(t) defer cleanup() @@ -194,7 +189,6 @@ func TestFilterByUsed(t *testing.T) { } func TestDerefMultipleOfSameRef(t *testing.T) { - skip.If(t, os.Getuid() != 0, "skipping test that requires root") t.Parallel() s, cleanup := setupTest(t) defer cleanup() @@ -216,7 +210,6 @@ func TestDerefMultipleOfSameRef(t *testing.T) { } func TestCreateKeepOptsLabelsWhenExistsRemotely(t *testing.T) { - skip.If(t, os.Getuid() != 0, "skipping test that requires root") t.Parallel() s, cleanup := setupTest(t) defer cleanup() @@ -245,7 +238,6 @@ func TestCreateKeepOptsLabelsWhenExistsRemotely(t *testing.T) { } func TestDefererencePluginOnCreateError(t *testing.T) { - skip.If(t, os.Getuid() != 0, "skipping test that requires root") t.Parallel() var ( @@ -292,7 +284,6 @@ func TestDefererencePluginOnCreateError(t *testing.T) { } func TestRefDerefRemove(t *testing.T) { - skip.If(t, os.Getuid() != 0, "skipping test that requires root") t.Parallel() driverName := "test-ref-deref-remove" @@ -313,7 +304,6 @@ func TestRefDerefRemove(t *testing.T) { } func TestGet(t *testing.T) { - skip.If(t, os.Getuid() != 0, "skipping test that requires root") t.Parallel() driverName := "test-get" @@ -340,7 +330,6 @@ func TestGet(t *testing.T) { } func TestGetWithRef(t *testing.T) { - skip.If(t, os.Getuid() != 0, "skipping test that requires root") t.Parallel() driverName := "test-get-with-ref"