From cd659bcc9c7c12377c11533f56516c2d1ec6eb94 Mon Sep 17 00:00:00 2001 From: Vivek Goyal Date: Tue, 6 Oct 2015 17:37:21 -0400 Subject: [PATCH] devmapper: Provide option to enabled deferred device deletion Provide a command line option dm.use_deferred_deletion to enable deferred device deletion feature. By default feature will be turned off. Not sure if there is much value in deferred deletion being turned on without deferred removal being turned on. So for now, this feature can be enabled only if deferred removal is on. Signed-off-by: Vivek Goyal Upstream-commit: 51e059e7e90f37848596a0b6ec83f7835e6e4131 Component: engine --- .../daemon/graphdriver/devmapper/deviceset.go | 35 ++++++++++++++-- .../daemon/graphdriver/devmapper/driver.go | 1 + .../docs/reference/commandline/daemon.md | 40 +++++++++++++++++++ components/engine/man/docker-daemon.8.md | 22 ++++++++++ 4 files changed, 95 insertions(+), 3 deletions(-) diff --git a/components/engine/daemon/graphdriver/devmapper/deviceset.go b/components/engine/daemon/graphdriver/devmapper/deviceset.go index 0a9aadbec6..8279d8200e 100644 --- a/components/engine/daemon/graphdriver/devmapper/deviceset.go +++ b/components/engine/daemon/graphdriver/devmapper/deviceset.go @@ -40,6 +40,7 @@ var ( logLevel = devicemapper.LogLevelFatal driverDeferredRemovalSupport = false enableDeferredRemoval = false + enableDeferredDeletion = false ) const deviceSetMetaFile string = "deviceset-metadata" @@ -106,6 +107,7 @@ type DeviceSet struct { transaction `json:"-"` overrideUdevSyncCheck bool deferredRemove bool // use deferred removal + deferredDelete bool // use deferred deletion BaseDeviceUUID string //save UUID of base device } @@ -141,6 +143,12 @@ type Status struct { UdevSyncSupported bool // DeferredRemoveEnabled is true then the device is not unmounted. DeferredRemoveEnabled bool + // True if deferred deletion is enabled. This is different from + // deferred removal. "removal" means that device mapper device is + // deactivated. Thin device is still in thin pool and can be activated + // again. But "deletion" means that thin device will be deleted from + // thin pool and it can't be activated again. + DeferredDeleteEnabled bool } // Structure used to export image/container metadata in docker inspect. @@ -1314,13 +1322,27 @@ func (devices *DeviceSet) initDevmapper(doInit bool) error { return graphdriver.ErrNotSupported } - // If user asked for deferred removal and both library and driver - // supports deferred removal use it. - if enableDeferredRemoval && driverDeferredRemovalSupport && devicemapper.LibraryDeferredRemovalSupport == true { + // If user asked for deferred removal then check both libdm library + // and kernel driver support deferred removal otherwise error out. + if enableDeferredRemoval { + if !driverDeferredRemovalSupport { + return fmt.Errorf("devmapper: Deferred removal can not be enabled as kernel does not support it") + } + if !devicemapper.LibraryDeferredRemovalSupport { + return fmt.Errorf("devmapper: Deferred removal can not be enabled as libdm does not support it") + } logrus.Debugf("devmapper: Deferred removal support enabled.") devices.deferredRemove = true } + if enableDeferredDeletion { + if !devices.deferredRemove { + return fmt.Errorf("devmapper: Deferred deletion can not be enabled as deferred removal is not enabled. Enable deferred removal using --storage-opt dm.use_deferred_removal=true parameter") + } + logrus.Debugf("devmapper: Deferred deletion support enabled.") + devices.deferredDelete = true + } + // https://github.com/docker/docker/issues/4036 if supported := devicemapper.UdevSetSyncSupport(true); !supported { logrus.Warn("Udev sync is not supported. This will lead to unexpected behavior, data loss and errors. For more information, see https://docs.docker.com/reference/commandline/daemon/#daemon-storage-driver-option") @@ -1996,6 +2018,7 @@ func (devices *DeviceSet) Status() *Status { status.MetadataLoopback = devices.metadataLoopFile status.UdevSyncSupported = devicemapper.UdevSyncSupported() status.DeferredRemoveEnabled = devices.deferredRemove + status.DeferredDeleteEnabled = devices.deferredDelete totalSizeInSectors, _, dataUsed, dataTotal, metadataUsed, metadataTotal, err := devices.poolStatus() if err == nil { @@ -2128,6 +2151,12 @@ func NewDeviceSet(root string, doInit bool, options []string) (*DeviceSet, error return nil, err } + case "dm.use_deferred_deletion": + enableDeferredDeletion, err = strconv.ParseBool(val) + if err != nil { + return nil, err + } + default: return nil, fmt.Errorf("Unknown option %s\n", key) } diff --git a/components/engine/daemon/graphdriver/devmapper/driver.go b/components/engine/daemon/graphdriver/devmapper/driver.go index dc2647d11d..f117cfb7c5 100644 --- a/components/engine/daemon/graphdriver/devmapper/driver.go +++ b/components/engine/daemon/graphdriver/devmapper/driver.go @@ -84,6 +84,7 @@ func (d *Driver) Status() [][2]string { {"Metadata Space Available", fmt.Sprintf("%s", units.HumanSize(float64(s.Metadata.Available)))}, {"Udev Sync Supported", fmt.Sprintf("%v", s.UdevSyncSupported)}, {"Deferred Removal Enabled", fmt.Sprintf("%v", s.DeferredRemoveEnabled)}, + {"Deferred Deletion Enabled", fmt.Sprintf("%v", s.DeferredDeleteEnabled)}, } if len(s.DataLoopback) > 0 { status = append(status, [2]string{"Data loop file", s.DataLoopback}) diff --git a/components/engine/docs/reference/commandline/daemon.md b/components/engine/docs/reference/commandline/daemon.md index e3ed62ef42..cef19ab038 100644 --- a/components/engine/docs/reference/commandline/daemon.md +++ b/components/engine/docs/reference/commandline/daemon.md @@ -368,6 +368,46 @@ options for `zfs` start with `zfs`. > Otherwise, set this flag for migrating existing Docker daemons to > a daemon with a supported environment. + * `dm.use_deferred_removal` + + Enables use of deferred device removal if `libdm` and the kernel driver + support the mechanism. + + Deferred device removal means that if device is busy when devices are + being removed/deactivated, then a deferred removal is scheduled on + device. And devices automatically go away when last user of the device + exits. + + For example, when a container exits, its associated thin device is removed. + If that device has leaked into some other mount namespace and can't be + removed, the container exit still succeeds and this option causes the + system to schedule the device for deferred removal. It does not wait in a + loop trying to remove a busy device. + + Example use: `docker daemon --storage-opt dm.use_deferred_removal=true` + + * `dm.use_deferred_deletion` + + Enables use of deferred device deletion for thin pool devices. By default, + thin pool device deletion is synchronous. Before a container is deleted, + the Docker daemon removes any associated devices. If the storage driver + can not remove a device, the container deletion fails and daemon returns. + + `Error deleting container: Error response from daemon: Cannot destroy container` + + To avoid this failure, enable both deferred device deletion and deferred + device removal on the daemon. + + `docker daemon --storage-opt dm.use_deferred_deletion=true --storage-opt dm.use_deferred_removal=true` + + With these two options enabled, if a device is busy when the driver is + deleting a container, the driver marks the device as deleted. Later, when + the device isn't in use, the driver deletes it. + + In general it should be safe to enable this option by default. It will help + when unintentional leaking of mount point happens across multiple mount + namespaces. + Currently supported options of `zfs`: * `zfs.fsname` diff --git a/components/engine/man/docker-daemon.8.md b/components/engine/man/docker-daemon.8.md index b37bd25e0d..59ba4d54f2 100644 --- a/components/engine/man/docker-daemon.8.md +++ b/components/engine/man/docker-daemon.8.md @@ -302,6 +302,28 @@ device. Example use: `docker daemon --storage-opt dm.use_deferred_removal=true` +#### dm.use_deferred_deletion + +Enables use of deferred device deletion for thin pool devices. By default, +thin pool device deletion is synchronous. Before a container is deleted, the +Docker daemon removes any associated devices. If the storage driver can not +remove a device, the container deletion fails and daemon returns. + +`Error deleting container: Error response from daemon: Cannot destroy container` + +To avoid this failure, enable both deferred device deletion and deferred +device removal on the daemon. + +`docker daemon --storage-opt dm.use_deferred_deletion=true --storage-opt dm.use_deferred_removal=true` + +With these two options enabled, if a device is busy when the driver is +deleting a container, the driver marks the device as deleted. Later, when the +device isn't in use, the driver deletes it. + +In general it should be safe to enable this option by default. It will help +when unintentional leaking of mount point happens across multiple mount +namespaces. + #### dm.loopdatasize **Note**: This option configures devicemapper loopback, which should not be used in production.