diff --git a/components/engine/daemon/graphdriver/devmapper/deviceset.go b/components/engine/daemon/graphdriver/devmapper/deviceset.go index 0a9aadbec6..8279d8200e 100644 --- a/components/engine/daemon/graphdriver/devmapper/deviceset.go +++ b/components/engine/daemon/graphdriver/devmapper/deviceset.go @@ -40,6 +40,7 @@ var ( logLevel = devicemapper.LogLevelFatal driverDeferredRemovalSupport = false enableDeferredRemoval = false + enableDeferredDeletion = false ) const deviceSetMetaFile string = "deviceset-metadata" @@ -106,6 +107,7 @@ type DeviceSet struct { transaction `json:"-"` overrideUdevSyncCheck bool deferredRemove bool // use deferred removal + deferredDelete bool // use deferred deletion BaseDeviceUUID string //save UUID of base device } @@ -141,6 +143,12 @@ type Status struct { UdevSyncSupported bool // DeferredRemoveEnabled is true then the device is not unmounted. DeferredRemoveEnabled bool + // True if deferred deletion is enabled. This is different from + // deferred removal. "removal" means that device mapper device is + // deactivated. Thin device is still in thin pool and can be activated + // again. But "deletion" means that thin device will be deleted from + // thin pool and it can't be activated again. + DeferredDeleteEnabled bool } // Structure used to export image/container metadata in docker inspect. @@ -1314,13 +1322,27 @@ func (devices *DeviceSet) initDevmapper(doInit bool) error { return graphdriver.ErrNotSupported } - // If user asked for deferred removal and both library and driver - // supports deferred removal use it. - if enableDeferredRemoval && driverDeferredRemovalSupport && devicemapper.LibraryDeferredRemovalSupport == true { + // If user asked for deferred removal then check both libdm library + // and kernel driver support deferred removal otherwise error out. + if enableDeferredRemoval { + if !driverDeferredRemovalSupport { + return fmt.Errorf("devmapper: Deferred removal can not be enabled as kernel does not support it") + } + if !devicemapper.LibraryDeferredRemovalSupport { + return fmt.Errorf("devmapper: Deferred removal can not be enabled as libdm does not support it") + } logrus.Debugf("devmapper: Deferred removal support enabled.") devices.deferredRemove = true } + if enableDeferredDeletion { + if !devices.deferredRemove { + return fmt.Errorf("devmapper: Deferred deletion can not be enabled as deferred removal is not enabled. Enable deferred removal using --storage-opt dm.use_deferred_removal=true parameter") + } + logrus.Debugf("devmapper: Deferred deletion support enabled.") + devices.deferredDelete = true + } + // https://github.com/docker/docker/issues/4036 if supported := devicemapper.UdevSetSyncSupport(true); !supported { logrus.Warn("Udev sync is not supported. This will lead to unexpected behavior, data loss and errors. For more information, see https://docs.docker.com/reference/commandline/daemon/#daemon-storage-driver-option") @@ -1996,6 +2018,7 @@ func (devices *DeviceSet) Status() *Status { status.MetadataLoopback = devices.metadataLoopFile status.UdevSyncSupported = devicemapper.UdevSyncSupported() status.DeferredRemoveEnabled = devices.deferredRemove + status.DeferredDeleteEnabled = devices.deferredDelete totalSizeInSectors, _, dataUsed, dataTotal, metadataUsed, metadataTotal, err := devices.poolStatus() if err == nil { @@ -2128,6 +2151,12 @@ func NewDeviceSet(root string, doInit bool, options []string) (*DeviceSet, error return nil, err } + case "dm.use_deferred_deletion": + enableDeferredDeletion, err = strconv.ParseBool(val) + if err != nil { + return nil, err + } + default: return nil, fmt.Errorf("Unknown option %s\n", key) } diff --git a/components/engine/daemon/graphdriver/devmapper/driver.go b/components/engine/daemon/graphdriver/devmapper/driver.go index dc2647d11d..f117cfb7c5 100644 --- a/components/engine/daemon/graphdriver/devmapper/driver.go +++ b/components/engine/daemon/graphdriver/devmapper/driver.go @@ -84,6 +84,7 @@ func (d *Driver) Status() [][2]string { {"Metadata Space Available", fmt.Sprintf("%s", units.HumanSize(float64(s.Metadata.Available)))}, {"Udev Sync Supported", fmt.Sprintf("%v", s.UdevSyncSupported)}, {"Deferred Removal Enabled", fmt.Sprintf("%v", s.DeferredRemoveEnabled)}, + {"Deferred Deletion Enabled", fmt.Sprintf("%v", s.DeferredDeleteEnabled)}, } if len(s.DataLoopback) > 0 { status = append(status, [2]string{"Data loop file", s.DataLoopback}) diff --git a/components/engine/docs/reference/commandline/daemon.md b/components/engine/docs/reference/commandline/daemon.md index e3ed62ef42..cef19ab038 100644 --- a/components/engine/docs/reference/commandline/daemon.md +++ b/components/engine/docs/reference/commandline/daemon.md @@ -368,6 +368,46 @@ options for `zfs` start with `zfs`. > Otherwise, set this flag for migrating existing Docker daemons to > a daemon with a supported environment. + * `dm.use_deferred_removal` + + Enables use of deferred device removal if `libdm` and the kernel driver + support the mechanism. + + Deferred device removal means that if device is busy when devices are + being removed/deactivated, then a deferred removal is scheduled on + device. And devices automatically go away when last user of the device + exits. + + For example, when a container exits, its associated thin device is removed. + If that device has leaked into some other mount namespace and can't be + removed, the container exit still succeeds and this option causes the + system to schedule the device for deferred removal. It does not wait in a + loop trying to remove a busy device. + + Example use: `docker daemon --storage-opt dm.use_deferred_removal=true` + + * `dm.use_deferred_deletion` + + Enables use of deferred device deletion for thin pool devices. By default, + thin pool device deletion is synchronous. Before a container is deleted, + the Docker daemon removes any associated devices. If the storage driver + can not remove a device, the container deletion fails and daemon returns. + + `Error deleting container: Error response from daemon: Cannot destroy container` + + To avoid this failure, enable both deferred device deletion and deferred + device removal on the daemon. + + `docker daemon --storage-opt dm.use_deferred_deletion=true --storage-opt dm.use_deferred_removal=true` + + With these two options enabled, if a device is busy when the driver is + deleting a container, the driver marks the device as deleted. Later, when + the device isn't in use, the driver deletes it. + + In general it should be safe to enable this option by default. It will help + when unintentional leaking of mount point happens across multiple mount + namespaces. + Currently supported options of `zfs`: * `zfs.fsname` diff --git a/components/engine/man/docker-daemon.8.md b/components/engine/man/docker-daemon.8.md index b37bd25e0d..59ba4d54f2 100644 --- a/components/engine/man/docker-daemon.8.md +++ b/components/engine/man/docker-daemon.8.md @@ -302,6 +302,28 @@ device. Example use: `docker daemon --storage-opt dm.use_deferred_removal=true` +#### dm.use_deferred_deletion + +Enables use of deferred device deletion for thin pool devices. By default, +thin pool device deletion is synchronous. Before a container is deleted, the +Docker daemon removes any associated devices. If the storage driver can not +remove a device, the container deletion fails and daemon returns. + +`Error deleting container: Error response from daemon: Cannot destroy container` + +To avoid this failure, enable both deferred device deletion and deferred +device removal on the daemon. + +`docker daemon --storage-opt dm.use_deferred_deletion=true --storage-opt dm.use_deferred_removal=true` + +With these two options enabled, if a device is busy when the driver is +deleting a container, the driver marks the device as deleted. Later, when the +device isn't in use, the driver deletes it. + +In general it should be safe to enable this option by default. It will help +when unintentional leaking of mount point happens across multiple mount +namespaces. + #### dm.loopdatasize **Note**: This option configures devicemapper loopback, which should not be used in production.