diff --git a/buildah.go b/buildah.go index 809dcefea..2ece11acd 100644 --- a/buildah.go +++ b/buildah.go @@ -408,6 +408,11 @@ type BuilderOptions struct { Devices []configs.Device //DefaultEnv for containers DefaultEnv []string + // MaxPullRetries is the maximum number of attempts we'll make to pull + // any one image from the external registry if the first attempt fails. + MaxPullRetries int + // PullRetryDelay is how long to wait before retrying a pull attempt. + PullRetryDelay time.Duration } // ImportOptions are used to initialize a Builder from an existing container diff --git a/cmd/buildah/bud.go b/cmd/buildah/bud.go index eb3c8d9c5..26258a676 100644 --- a/cmd/buildah/bud.go +++ b/cmd/buildah/bud.go @@ -313,6 +313,7 @@ func budCmd(c *cobra.Command, inputArgs []string, iopts budOptions) error { Isolation: isolation, Labels: iopts.Label, Layers: layers, + MaxPullPushRetries: maxPullPushRetries, NamespaceOptions: namespaceOptions, NoCache: iopts.NoCache, OS: os, @@ -320,6 +321,7 @@ func budCmd(c *cobra.Command, inputArgs []string, iopts budOptions) error { Output: output, OutputFormat: format, PullPolicy: pullPolicy, + PullPushRetryDelay: pullPushRetryDelay, Quiet: iopts.Quiet, RemoveIntermediateCtrs: iopts.Rm, ReportWriter: reporter, diff --git a/cmd/buildah/common.go b/cmd/buildah/common.go index 7ffdc93bd..45b99f791 100644 --- a/cmd/buildah/common.go +++ b/cmd/buildah/common.go @@ -25,6 +25,11 @@ var ( needToShutdownStore = false ) +const ( + maxPullPushRetries = 3 + pullPushRetryDelay = 2 * time.Second +) + func getStore(c *cobra.Command) (storage.Store, error) { options, err := storage.DefaultStoreOptions(unshare.IsRootless(), unshare.GetRootlessUID()) if err != nil { diff --git a/cmd/buildah/from.go b/cmd/buildah/from.go index c090b1ff3..e7718388e 100644 --- a/cmd/buildah/from.go +++ b/cmd/buildah/from.go @@ -280,6 +280,9 @@ func fromCmd(c *cobra.Command, args []string, iopts fromReply) error { Format: format, BlobDirectory: iopts.BlobCache, Devices: devices, + DefaultEnv: defaultContainerConfig.GetDefaultEnv(), + MaxPullRetries: maxPullPushRetries, + PullRetryDelay: pullPushRetryDelay, } if !iopts.quiet { diff --git a/cmd/buildah/pull.go b/cmd/buildah/pull.go index c99b6803a..ce7593fe7 100644 --- a/cmd/buildah/pull.go +++ b/cmd/buildah/pull.go @@ -109,6 +109,8 @@ func pullCmd(c *cobra.Command, args []string, iopts pullOptions) error { AllTags: iopts.allTags, ReportWriter: os.Stderr, RemoveSignatures: iopts.removeSignatures, + MaxRetries: maxPullPushRetries, + RetryDelay: pullPushRetryDelay, } if iopts.quiet { diff --git a/cmd/buildah/push.go b/cmd/buildah/push.go index 93a60c8c8..38979a5c1 100644 --- a/cmd/buildah/push.go +++ b/cmd/buildah/push.go @@ -173,6 +173,8 @@ func pushCmd(c *cobra.Command, args []string, iopts pushOptions) error { BlobDirectory: iopts.blobCache, RemoveSignatures: iopts.removeSignatures, SignBy: iopts.signBy, + MaxRetries: maxPullPushRetries, + RetryDelay: pullPushRetryDelay, } if !iopts.quiet { options.ReportWriter = os.Stderr diff --git a/commit.go b/commit.go index 05b2437c8..d25ba110a 100644 --- a/commit.go +++ b/commit.go @@ -12,7 +12,6 @@ import ( "github.com/containers/buildah/pkg/blobcache" "github.com/containers/buildah/util" - cp "github.com/containers/image/v5/copy" "github.com/containers/image/v5/docker" "github.com/containers/image/v5/docker/reference" "github.com/containers/image/v5/manifest" @@ -83,6 +82,12 @@ type CommitOptions struct { OmitTimestamp bool // SignBy is the fingerprint of a GPG key to use for signing the image. SignBy string + // MaxRetries is the maximum number of attempts we'll make to commit + // the image to an external registry if the first attempt fails. + MaxRetries int + // RetryDelay is how long to wait before retrying a commit attempt to a + // registry. + RetryDelay time.Duration } // PushOptions can be used to alter how an image is copied somewhere. @@ -122,6 +127,11 @@ type PushOptions struct { // RemoveSignatures causes any existing signatures for the image to be // discarded for the pushed copy. RemoveSignatures bool + // MaxRetries is the maximum number of attempts we'll make to push any + // one image to the external registry if the first attempt fails. + MaxRetries int + // RetryDelay is how long to wait before retrying a push attempt. + RetryDelay time.Duration } var ( @@ -309,7 +319,7 @@ func (b *Builder) Commit(ctx context.Context, dest types.ImageReference, options } var manifestBytes []byte - if manifestBytes, err = cp.Image(ctx, policyContext, maybeCachedDest, maybeCachedSrc, getCopyOptions(b.store, options.ReportWriter, nil, systemContext, "", false, options.SignBy)); err != nil { + if manifestBytes, err = retryCopyImage(ctx, policyContext, maybeCachedDest, maybeCachedSrc, dest, "push", getCopyOptions(b.store, options.ReportWriter, nil, systemContext, "", false, options.SignBy), options.MaxRetries, options.RetryDelay); err != nil { return imgID, nil, "", errors.Wrapf(err, "error copying layers and metadata for container %q", b.ContainerID) } // If we've got more names to attach, and we know how to do that for @@ -441,7 +451,7 @@ func Push(ctx context.Context, image string, dest types.ImageReference, options systemContext.DirForceCompress = true } var manifestBytes []byte - if manifestBytes, err = cp.Image(ctx, policyContext, dest, maybeCachedSrc, getCopyOptions(options.Store, options.ReportWriter, nil, systemContext, options.ManifestType, options.RemoveSignatures, options.SignBy)); err != nil { + if manifestBytes, err = retryCopyImage(ctx, policyContext, dest, maybeCachedSrc, dest, "push", getCopyOptions(options.Store, options.ReportWriter, nil, systemContext, options.ManifestType, options.RemoveSignatures, options.SignBy), options.MaxRetries, options.RetryDelay); err != nil { return nil, "", errors.Wrapf(err, "error copying layers and metadata from %q to %q", transports.ImageName(maybeCachedSrc), transports.ImageName(dest)) } if options.ReportWriter != nil { diff --git a/common.go b/common.go index a3ef70ec5..4dadf3e76 100644 --- a/common.go +++ b/common.go @@ -1,14 +1,26 @@ package buildah import ( + "context" "io" + "net" + "net/url" "os" "path/filepath" + "time" "github.com/containers/common/pkg/unshare" cp "github.com/containers/image/v5/copy" + "github.com/containers/image/v5/docker" + "github.com/containers/image/v5/signature" "github.com/containers/image/v5/types" "github.com/containers/storage" + "github.com/docker/distribution/registry/api/errcode" + errcodev2 "github.com/docker/distribution/registry/api/v2" + multierror "github.com/hashicorp/go-multierror" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" ) const ( @@ -59,3 +71,65 @@ func getSystemContext(store storage.Store, defaults *types.SystemContext, signat } return sc } + +func isRetryable(err error) bool { + err = errors.Cause(err) + type unwrapper interface { + Unwrap() error + } + if unwrapper, ok := err.(unwrapper); ok { + err = unwrapper.Unwrap() + return isRetryable(err) + } + if registryError, ok := err.(errcode.Error); ok { + switch registryError.Code { + case errcode.ErrorCodeUnauthorized, errcodev2.ErrorCodeNameUnknown, errcodev2.ErrorCodeManifestUnknown: + return false + } + return true + } + if op, ok := err.(*net.OpError); ok { + return isRetryable(op.Err) + } + if url, ok := err.(*url.Error); ok { + return isRetryable(url.Err) + } + if errno, ok := err.(unix.Errno); ok { + if errno == unix.ECONNREFUSED { + return false + } + } + if errs, ok := err.(errcode.Errors); ok { + // if this error is a group of errors, process them all in turn + for i := range errs { + if !isRetryable(errs[i]) { + return false + } + } + } + if errs, ok := err.(*multierror.Error); ok { + // if this error is a group of errors, process them all in turn + for i := range errs.Errors { + if !isRetryable(errs.Errors[i]) { + return false + } + } + } + return true +} + +func retryCopyImage(ctx context.Context, policyContext *signature.PolicyContext, dest, src, registry types.ImageReference, action string, copyOptions *cp.Options, maxRetries int, retryDelay time.Duration) ([]byte, error) { + manifestBytes, err := cp.Image(ctx, policyContext, dest, src, copyOptions) + for retries := 0; err != nil && isRetryable(err) && registry != nil && registry.Transport().Name() == docker.Transport.Name() && retries < maxRetries; retries++ { + if retryDelay == 0 { + retryDelay = 5 * time.Second + } + logrus.Infof("Warning: %s failed, retrying in %s ... (%d/%d)", action, retryDelay, retries+1, maxRetries) + time.Sleep(retryDelay) + manifestBytes, err = cp.Image(ctx, policyContext, dest, src, copyOptions) + if err == nil { + break + } + } + return manifestBytes, err +} diff --git a/imagebuildah/build.go b/imagebuildah/build.go index 1033a92dd..e113871c7 100644 --- a/imagebuildah/build.go +++ b/imagebuildah/build.go @@ -11,6 +11,7 @@ import ( "os/exec" "path/filepath" "strings" + "time" "github.com/containers/buildah" "github.com/containers/common/pkg/config" @@ -166,6 +167,11 @@ type BuildOptions struct { Architecture string // OS is the specifies the operating system of the image to be built. OS string + // MaxPullPushRetries is the maximum number of attempts we'll make to pull or push any one + // image from or to an external registry if the first attempt fails. + MaxPullPushRetries int + // PullPushRetryDelay is how long to wait before retrying a pull or push attempt. + PullPushRetryDelay time.Duration } // BuildDockerfiles parses a set of one or more Dockerfiles (which may be diff --git a/imagebuildah/executor.go b/imagebuildah/executor.go index d43789ffb..29137ab70 100644 --- a/imagebuildah/executor.go +++ b/imagebuildah/executor.go @@ -9,6 +9,7 @@ import ( "sort" "strconv" "strings" + "time" "github.com/containers/buildah" "github.com/containers/buildah/pkg/parse" @@ -98,6 +99,8 @@ type Executor struct { signBy string architecture string os string + maxPullPushRetries int + retryPullPushDelay time.Duration } // NewExecutor creates a new instance of the imagebuilder.Executor interface. @@ -176,12 +179,14 @@ func NewExecutor(store storage.Store, options BuildOptions, mainNode *parser.Nod rootfsMap: make(map[string]bool), blobDirectory: options.BlobDirectory, unusedArgs: make(map[string]struct{}), - buildArgs: options.Args, + buildArgs: copyStringStringMap(options.Args), capabilities: capabilities, devices: devices, signBy: options.SignBy, architecture: options.Architecture, os: options.OS, + maxPullPushRetries: options.MaxPullPushRetries, + retryPullPushDelay: options.PullPushRetryDelay, } if exec.err == nil { exec.err = os.Stderr diff --git a/imagebuildah/stage_executor.go b/imagebuildah/stage_executor.go index 3c4b714ff..2dcf35689 100644 --- a/imagebuildah/stage_executor.go +++ b/imagebuildah/stage_executor.go @@ -613,6 +613,8 @@ func (s *StageExecutor) prepare(ctx context.Context, stage imagebuilder.Stage, f Format: s.executor.outputFormat, Capabilities: s.executor.capabilities, Devices: s.executor.devices, + MaxPullRetries: s.executor.maxPullPushRetries, + PullRetryDelay: s.executor.retryPullPushDelay, } // Check and see if the image is a pseudonym for the end result of a @@ -1212,6 +1214,8 @@ func (s *StageExecutor) commit(ctx context.Context, ib *imagebuilder.Builder, cr EmptyLayer: emptyLayer, BlobDirectory: s.executor.blobDirectory, SignBy: s.executor.signBy, + MaxRetries: s.executor.maxPullPushRetries, + RetryDelay: s.executor.retryPullPushDelay, } imgID, _, manifestDigest, err := s.builder.Commit(ctx, imageRef, options) if err != nil { diff --git a/imagebuildah/util.go b/imagebuildah/util.go index 520b92e3f..29ea60970 100644 --- a/imagebuildah/util.go +++ b/imagebuildah/util.go @@ -165,3 +165,11 @@ func convertMounts(mounts []Mount) []specs.Mount { } return specmounts } + +func copyStringStringMap(m map[string]string) map[string]string { + n := map[string]string{} + for k, v := range m { + n[k] = v + } + return n +} diff --git a/new.go b/new.go index b34ea598f..a6b6899e0 100644 --- a/new.go +++ b/new.go @@ -34,6 +34,8 @@ func pullAndFindImage(ctx context.Context, store storage.Store, srcRef types.Ima Store: store, SystemContext: options.SystemContext, BlobDirectory: options.BlobDirectory, + MaxRetries: options.MaxPullRetries, + RetryDelay: options.PullRetryDelay, } ref, err := pullImage(ctx, store, srcRef, pullOptions, sc) if err != nil { diff --git a/pull.go b/pull.go index 8605808b6..4a38abeab 100644 --- a/pull.go +++ b/pull.go @@ -3,12 +3,11 @@ package buildah import ( "context" "io" - "strings" + "time" "github.com/containers/buildah/pkg/blobcache" "github.com/containers/buildah/util" - cp "github.com/containers/image/v5/copy" "github.com/containers/image/v5/directory" "github.com/containers/image/v5/docker" dockerarchive "github.com/containers/image/v5/docker/archive" @@ -52,6 +51,11 @@ type PullOptions struct { // RemoveSignatures causes any existing signatures for the image to be // discarded when pulling it. RemoveSignatures bool + // MaxRetries is the maximum number of attempts we'll make to pull any + // one image from the external registry if the first attempt fails. + MaxRetries int + // RetryDelay is how long to wait before retrying a pull attempt. + RetryDelay time.Duration } func localImageNameForReference(ctx context.Context, store storage.Store, srcRef types.ImageReference) (string, error) { @@ -158,6 +162,8 @@ func Pull(ctx context.Context, imageName string, options PullOptions) (imageID s SystemContext: systemContext, BlobDirectory: options.BlobDirectory, ReportWriter: options.ReportWriter, + MaxPullRetries: options.MaxRetries, + PullRetryDelay: options.RetryDelay, } storageRef, transport, img, err := resolveImage(ctx, systemContext, options.Store, boptions) @@ -264,7 +270,7 @@ func pullImage(ctx context.Context, store storage.Store, srcRef types.ImageRefer }() logrus.Debugf("copying %q to %q", transports.ImageName(srcRef), destName) - if _, err := cp.Image(ctx, policyContext, maybeCachedDestRef, srcRef, getCopyOptions(store, options.ReportWriter, sc, nil, "", options.RemoveSignatures, "")); err != nil { + if _, err := retryCopyImage(ctx, policyContext, maybeCachedDestRef, srcRef, srcRef, "pull", getCopyOptions(store, options.ReportWriter, sc, nil, "", options.RemoveSignatures, ""), options.MaxRetries, options.RetryDelay); err != nil { logrus.Debugf("error copying src image [%q] to dest image [%q] err: %v", transports.ImageName(srcRef), destName, err) return nil, err }