pull/from/commit/push: retry on most failures

If PullOptions/BuilderOptions/CommitOptions/PushOptions includes a
MaxRetries value other than 0, retry operations except for (currently)
connection-refused, authentication, and no-such-repository/no-such-tag
errors, at a default-but-configurable interval of 5 seconds.

Set the default for `buildah pull/from/commit/push` to 3 retries at 2
second intervals.

Signed-off-by: Nalin Dahyabhai <nalin@redhat.com>

buildah.go

@@ -408,6 +408,11 @@ type BuilderOptions struct {
 	Devices []configs.Device
 	//DefaultEnv for containers
 	DefaultEnv []string
+	// MaxPullRetries is the maximum number of attempts we'll make to pull
+	// any one image from the external registry if the first attempt fails.
+	MaxPullRetries int
+	// PullRetryDelay is how long to wait before retrying a pull attempt.
+	PullRetryDelay time.Duration
 }
 
 // ImportOptions are used to initialize a Builder from an existing container

cmd/buildah/bud.go

@@ -313,6 +313,7 @@ func budCmd(c *cobra.Command, inputArgs []string, iopts budOptions) error {
 		Isolation:               isolation,
 		Labels:                  iopts.Label,
 		Layers:                  layers,
+		MaxPullPushRetries:      maxPullPushRetries,
 		NamespaceOptions:        namespaceOptions,
 		NoCache:                 iopts.NoCache,
 		OS:                      os,
@@ -320,6 +321,7 @@ func budCmd(c *cobra.Command, inputArgs []string, iopts budOptions) error {
 		Output:                  output,
 		OutputFormat:            format,
 		PullPolicy:              pullPolicy,
+		PullPushRetryDelay:      pullPushRetryDelay,
 		Quiet:                   iopts.Quiet,
 		RemoveIntermediateCtrs:  iopts.Rm,
 		ReportWriter:            reporter,

cmd/buildah/common.go

@@ -25,6 +25,11 @@ var (
 	needToShutdownStore = false
 )
 
+const (
+	maxPullPushRetries = 3
+	pullPushRetryDelay = 2 * time.Second
+)
+
 func getStore(c *cobra.Command) (storage.Store, error) {
 	options, err := storage.DefaultStoreOptions(unshare.IsRootless(), unshare.GetRootlessUID())
 	if err != nil {

cmd/buildah/from.go

@@ -280,6 +280,9 @@ func fromCmd(c *cobra.Command, args []string, iopts fromReply) error {
 		Format:                format,
 		BlobDirectory:         iopts.BlobCache,
 		Devices:               devices,
+		DefaultEnv:            defaultContainerConfig.GetDefaultEnv(),
+		MaxPullRetries:        maxPullPushRetries,
+		PullRetryDelay:        pullPushRetryDelay,
 	}
 
 	if !iopts.quiet {

cmd/buildah/pull.go

@@ -109,6 +109,8 @@ func pullCmd(c *cobra.Command, args []string, iopts pullOptions) error {
 		AllTags:             iopts.allTags,
 		ReportWriter:        os.Stderr,
 		RemoveSignatures:    iopts.removeSignatures,
+		MaxRetries:          maxPullPushRetries,
+		RetryDelay:          pullPushRetryDelay,
 	}
 
 	if iopts.quiet {

cmd/buildah/push.go

@@ -173,6 +173,8 @@ func pushCmd(c *cobra.Command, args []string, iopts pushOptions) error {
 		BlobDirectory:       iopts.blobCache,
 		RemoveSignatures:    iopts.removeSignatures,
 		SignBy:              iopts.signBy,
+		MaxRetries:          maxPullPushRetries,
+		RetryDelay:          pullPushRetryDelay,
 	}
 	if !iopts.quiet {
 		options.ReportWriter = os.Stderr

commit.go

@@ -12,7 +12,6 @@ import (
 
 	"github.com/containers/buildah/pkg/blobcache"
 	"github.com/containers/buildah/util"
-	cp "github.com/containers/image/v5/copy"
 	"github.com/containers/image/v5/docker"
 	"github.com/containers/image/v5/docker/reference"
 	"github.com/containers/image/v5/manifest"
@@ -83,6 +82,12 @@ type CommitOptions struct {
 	OmitTimestamp bool
 	// SignBy is the fingerprint of a GPG key to use for signing the image.
 	SignBy string
+	// MaxRetries is the maximum number of attempts we'll make to commit
+	// the image to an external registry if the first attempt fails.
+	MaxRetries int
+	// RetryDelay is how long to wait before retrying a commit attempt to a
+	// registry.
+	RetryDelay time.Duration
 }
 
 // PushOptions can be used to alter how an image is copied somewhere.
@@ -122,6 +127,11 @@ type PushOptions struct {
 	// RemoveSignatures causes any existing signatures for the image to be
 	// discarded for the pushed copy.
 	RemoveSignatures bool
+	// MaxRetries is the maximum number of attempts we'll make to push any
+	// one image to the external registry if the first attempt fails.
+	MaxRetries int
+	// RetryDelay is how long to wait before retrying a push attempt.
+	RetryDelay time.Duration
 }
 
 var (
@@ -309,7 +319,7 @@ func (b *Builder) Commit(ctx context.Context, dest types.ImageReference, options
 	}
 
 	var manifestBytes []byte
-	if manifestBytes, err = cp.Image(ctx, policyContext, maybeCachedDest, maybeCachedSrc, getCopyOptions(b.store, options.ReportWriter, nil, systemContext, "", false, options.SignBy)); err != nil {
+	if manifestBytes, err = retryCopyImage(ctx, policyContext, maybeCachedDest, maybeCachedSrc, dest, "push", getCopyOptions(b.store, options.ReportWriter, nil, systemContext, "", false, options.SignBy), options.MaxRetries, options.RetryDelay); err != nil {
 		return imgID, nil, "", errors.Wrapf(err, "error copying layers and metadata for container %q", b.ContainerID)
 	}
 	// If we've got more names to attach, and we know how to do that for
@@ -441,7 +451,7 @@ func Push(ctx context.Context, image string, dest types.ImageReference, options
 		systemContext.DirForceCompress = true
 	}
 	var manifestBytes []byte
-	if manifestBytes, err = cp.Image(ctx, policyContext, dest, maybeCachedSrc, getCopyOptions(options.Store, options.ReportWriter, nil, systemContext, options.ManifestType, options.RemoveSignatures, options.SignBy)); err != nil {
+	if manifestBytes, err = retryCopyImage(ctx, policyContext, dest, maybeCachedSrc, dest, "push", getCopyOptions(options.Store, options.ReportWriter, nil, systemContext, options.ManifestType, options.RemoveSignatures, options.SignBy), options.MaxRetries, options.RetryDelay); err != nil {
 		return nil, "", errors.Wrapf(err, "error copying layers and metadata from %q to %q", transports.ImageName(maybeCachedSrc), transports.ImageName(dest))
 	}
 	if options.ReportWriter != nil {

common.go

@@ -1,14 +1,26 @@
 package buildah
 
 import (
+	"context"
 	"io"
+	"net"
+	"net/url"
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/containers/common/pkg/unshare"
 	cp "github.com/containers/image/v5/copy"
+	"github.com/containers/image/v5/docker"
+	"github.com/containers/image/v5/signature"
 	"github.com/containers/image/v5/types"
 	"github.com/containers/storage"
+	"github.com/docker/distribution/registry/api/errcode"
+	errcodev2 "github.com/docker/distribution/registry/api/v2"
+	multierror "github.com/hashicorp/go-multierror"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/sys/unix"
 )
 
 const (
@@ -59,3 +71,65 @@ func getSystemContext(store storage.Store, defaults *types.SystemContext, signat
 	}
 	return sc
 }
+
+func isRetryable(err error) bool {
+	err = errors.Cause(err)
+	type unwrapper interface {
+		Unwrap() error
+	}
+	if unwrapper, ok := err.(unwrapper); ok {
+		err = unwrapper.Unwrap()
+		return isRetryable(err)
+	}
+	if registryError, ok := err.(errcode.Error); ok {
+		switch registryError.Code {
+		case errcode.ErrorCodeUnauthorized, errcodev2.ErrorCodeNameUnknown, errcodev2.ErrorCodeManifestUnknown:
+			return false
+		}
+		return true
+	}
+	if op, ok := err.(*net.OpError); ok {
+		return isRetryable(op.Err)
+	}
+	if url, ok := err.(*url.Error); ok {
+		return isRetryable(url.Err)
+	}
+	if errno, ok := err.(unix.Errno); ok {
+		if errno == unix.ECONNREFUSED {
+			return false
+		}
+	}
+	if errs, ok := err.(errcode.Errors); ok {
+		// if this error is a group of errors, process them all in turn
+		for i := range errs {
+			if !isRetryable(errs[i]) {
+				return false
+			}
+		}
+	}
+	if errs, ok := err.(*multierror.Error); ok {
+		// if this error is a group of errors, process them all in turn
+		for i := range errs.Errors {
+			if !isRetryable(errs.Errors[i]) {
+				return false
+			}
+		}
+	}
+	return true
+}
+
+func retryCopyImage(ctx context.Context, policyContext *signature.PolicyContext, dest, src, registry types.ImageReference, action string, copyOptions *cp.Options, maxRetries int, retryDelay time.Duration) ([]byte, error) {
+	manifestBytes, err := cp.Image(ctx, policyContext, dest, src, copyOptions)
+	for retries := 0; err != nil && isRetryable(err) && registry != nil && registry.Transport().Name() == docker.Transport.Name() && retries < maxRetries; retries++ {
+		if retryDelay == 0 {
+			retryDelay = 5 * time.Second
+		}
+		logrus.Infof("Warning: %s failed, retrying in %s ... (%d/%d)", action, retryDelay, retries+1, maxRetries)
+		time.Sleep(retryDelay)
+		manifestBytes, err = cp.Image(ctx, policyContext, dest, src, copyOptions)
+		if err == nil {
+			break
+		}
+	}
+	return manifestBytes, err
+}

imagebuildah/build.go

@@ -11,6 +11,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/containers/buildah"
 	"github.com/containers/common/pkg/config"
@@ -166,6 +167,11 @@ type BuildOptions struct {
 	Architecture string
 	// OS is the specifies the operating system of the image to be built.
 	OS string
+	// MaxPullPushRetries is the maximum number of attempts we'll make to pull or push any one
+	// image from or to an external registry if the first attempt fails.
+	MaxPullPushRetries int
+	// PullPushRetryDelay is how long to wait before retrying a pull or push attempt.
+	PullPushRetryDelay time.Duration
 }
 
 // BuildDockerfiles parses a set of one or more Dockerfiles (which may be

imagebuildah/executor.go

@@ -9,6 +9,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/containers/buildah"
 	"github.com/containers/buildah/pkg/parse"
@@ -98,6 +99,8 @@ type Executor struct {
 	signBy                         string
 	architecture                   string
 	os                             string
+	maxPullPushRetries             int
+	retryPullPushDelay             time.Duration
 }
 
 // NewExecutor creates a new instance of the imagebuilder.Executor interface.
@@ -176,12 +179,14 @@ func NewExecutor(store storage.Store, options BuildOptions, mainNode *parser.Nod
 		rootfsMap:                      make(map[string]bool),
 		blobDirectory:                  options.BlobDirectory,
 		unusedArgs:                     make(map[string]struct{}),
-		buildArgs:                      options.Args,
+		buildArgs:                      copyStringStringMap(options.Args),
 		capabilities:                   capabilities,
 		devices:                        devices,
 		signBy:                         options.SignBy,
 		architecture:                   options.Architecture,
 		os:                             options.OS,
+		maxPullPushRetries:             options.MaxPullPushRetries,
+		retryPullPushDelay:             options.PullPushRetryDelay,
 	}
 	if exec.err == nil {
 		exec.err = os.Stderr

imagebuildah/stage_executor.go

@@ -613,6 +613,8 @@ func (s *StageExecutor) prepare(ctx context.Context, stage imagebuilder.Stage, f
 		Format:                s.executor.outputFormat,
 		Capabilities:          s.executor.capabilities,
 		Devices:               s.executor.devices,
+		MaxPullRetries:        s.executor.maxPullPushRetries,
+		PullRetryDelay:        s.executor.retryPullPushDelay,
 	}
 
 	// Check and see if the image is a pseudonym for the end result of a
@@ -1212,6 +1214,8 @@ func (s *StageExecutor) commit(ctx context.Context, ib *imagebuilder.Builder, cr
 		EmptyLayer:            emptyLayer,
 		BlobDirectory:         s.executor.blobDirectory,
 		SignBy:                s.executor.signBy,
+		MaxRetries:            s.executor.maxPullPushRetries,
+		RetryDelay:            s.executor.retryPullPushDelay,
 	}
 	imgID, _, manifestDigest, err := s.builder.Commit(ctx, imageRef, options)
 	if err != nil {

imagebuildah/util.go

@@ -165,3 +165,11 @@ func convertMounts(mounts []Mount) []specs.Mount {
 	}
 	return specmounts
 }
+
+func copyStringStringMap(m map[string]string) map[string]string {
+	n := map[string]string{}
+	for k, v := range m {
+		n[k] = v
+	}
+	return n
+}

new.go

@@ -34,6 +34,8 @@ func pullAndFindImage(ctx context.Context, store storage.Store, srcRef types.Ima
 		Store:         store,
 		SystemContext: options.SystemContext,
 		BlobDirectory: options.BlobDirectory,
+		MaxRetries:    options.MaxPullRetries,
+		RetryDelay:    options.PullRetryDelay,
 	}
 	ref, err := pullImage(ctx, store, srcRef, pullOptions, sc)
 	if err != nil {

pull.go

@@ -3,12 +3,11 @@ package buildah
 import (
 	"context"
 	"io"
-
 	"strings"
+	"time"
 
 	"github.com/containers/buildah/pkg/blobcache"
 	"github.com/containers/buildah/util"
-	cp "github.com/containers/image/v5/copy"
 	"github.com/containers/image/v5/directory"
 	"github.com/containers/image/v5/docker"
 	dockerarchive "github.com/containers/image/v5/docker/archive"
@@ -52,6 +51,11 @@ type PullOptions struct {
 	// RemoveSignatures causes any existing signatures for the image to be
 	// discarded when pulling it.
 	RemoveSignatures bool
+	// MaxRetries is the maximum number of attempts we'll make to pull any
+	// one image from the external registry if the first attempt fails.
+	MaxRetries int
+	// RetryDelay is how long to wait before retrying a pull attempt.
+	RetryDelay time.Duration
 }
 
 func localImageNameForReference(ctx context.Context, store storage.Store, srcRef types.ImageReference) (string, error) {
@@ -158,6 +162,8 @@ func Pull(ctx context.Context, imageName string, options PullOptions) (imageID s
 		SystemContext:       systemContext,
 		BlobDirectory:       options.BlobDirectory,
 		ReportWriter:        options.ReportWriter,
+		MaxPullRetries:      options.MaxRetries,
+		PullRetryDelay:      options.RetryDelay,
 	}
 
 	storageRef, transport, img, err := resolveImage(ctx, systemContext, options.Store, boptions)
@@ -264,7 +270,7 @@ func pullImage(ctx context.Context, store storage.Store, srcRef types.ImageRefer
 	}()
 
 	logrus.Debugf("copying %q to %q", transports.ImageName(srcRef), destName)
-	if _, err := cp.Image(ctx, policyContext, maybeCachedDestRef, srcRef, getCopyOptions(store, options.ReportWriter, sc, nil, "", options.RemoveSignatures, "")); err != nil {
+	if _, err := retryCopyImage(ctx, policyContext, maybeCachedDestRef, srcRef, srcRef, "pull", getCopyOptions(store, options.ReportWriter, sc, nil, "", options.RemoveSignatures, ""), options.MaxRetries, options.RetryDelay); err != nil {
 		logrus.Debugf("error copying src image [%q] to dest image [%q] err: %v", transports.ImageName(srcRef), destName, err)
 		return nil, err
 	}