diff --git a/Makefile b/Makefile index 7d806ebcd..8f4982d0a 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ LIBSECCOMP_COMMIT := release-2.3 EXTRA_LDFLAGS ?= BUILDAH_LDFLAGS := -ldflags '-X main.GitCommit=$(GIT_COMMIT) -X main.buildInfo=$(SOURCE_DATE_EPOCH) -X main.cniVersion=$(CNI_COMMIT) $(EXTRA_LDFLAGS)' -SOURCES=*.go imagebuildah/*.go bind/*.go chroot/*.go copier/*.go docker/*.go manifests/*.go pkg/blobcache/*.go pkg/chrootuser/*.go pkg/cli/*.go pkg/formats/*.go pkg/manifests/*.go pkg/overlay/*.go pkg/parse/*.go pkg/rusage/*.go util/*.go +SOURCES=*.go imagebuildah/*.go bind/*.go chroot/*.go copier/*.go docker/*.go manifests/*.go pkg/blobcache/*.go pkg/chrootuser/*.go pkg/cli/*.go pkg/completion/*.go pkg/formats/*.go pkg/overlay/*.go pkg/parse/*.go pkg/rusage/*.go pkg/sshagent/*.go pkg/umask/*.go util/*.go LINTFLAGS ?= diff --git a/cmd/buildah/bud.go b/cmd/buildah/bud.go index 5bbeac55d..759d18cc2 100644 --- a/cmd/buildah/bud.go +++ b/cmd/buildah/bud.go @@ -13,16 +13,10 @@ import ( buildahcli "github.com/containers/buildah/pkg/cli" "github.com/containers/buildah/pkg/parse" "github.com/containers/buildah/util" - "github.com/containers/common/libimage" - "github.com/containers/common/libimage/manifests" "github.com/containers/common/pkg/auth" - "github.com/containers/image/v5/manifest" - "github.com/containers/storage" - "github.com/hashicorp/go-multierror" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "golang.org/x/sync/semaphore" ) type budOptions struct { @@ -315,32 +309,6 @@ func budCmd(c *cobra.Command, inputArgs []string, iopts budOptions) error { return errors.Wrapf(err, "unable to obtain decrypt config") } - var ( - jobSemaphore *semaphore.Weighted - builds multierror.Group - ) - if iopts.Jobs > 0 { - jobSemaphore = semaphore.NewWeighted(int64(iopts.Jobs)) - iopts.Jobs = 0 - } - if iopts.Manifest != "" { - // Ensure that the list's ID is known before we spawn off any - // goroutines that'll want to modify it, so that they don't - // race and create two lists, one of which will rapidly become - // ignored. - rt, err := libimage.RuntimeFromStore(store, nil) - if err != nil { - return err - } - _, err = rt.LookupManifestList(iopts.Manifest) - if err != nil && errors.Cause(err) == storage.ErrImageUnknown { - list := manifests.Create() - _, err = list.SaveToImage(store, "", []string{iopts.Manifest}, manifest.DockerV2ListMediaType) - } - if err != nil { - return err - } - } var excludes []string if iopts.IgnoreFile != "" { if excludes, err = parseDockerignore(iopts.IgnoreFile); err != nil { @@ -352,82 +320,66 @@ func budCmd(c *cobra.Command, inputArgs []string, iopts budOptions) error { t := time.Unix(iopts.Timestamp, 0).UTC() timestamp = &t } - for _, platform := range platforms { - platform := platform - builds.Go(func() error { - platformContext := *systemContext - platformContext.OSChoice = platform.OS - platformContext.ArchitectureChoice = platform.Arch - platformContext.VariantChoice = platform.Variant - options := define.BuildOptions{ - AddCapabilities: iopts.CapAdd, - AdditionalTags: tags, - Annotations: iopts.Annotation, - Architecture: platform.Arch, - Args: args, - BlobDirectory: iopts.BlobCache, - CNIConfigDir: iopts.CNIConfigDir, - CNIPluginPath: iopts.CNIPlugInPath, - CommonBuildOpts: commonOpts, - Compression: compression, - ConfigureNetwork: networkPolicy, - ContextDirectory: contextDir, - DefaultMountsFilePath: globalFlagResults.DefaultMountsFile, - Devices: iopts.Devices, - DropCapabilities: iopts.CapDrop, - Err: stderr, - ForceRmIntermediateCtrs: iopts.ForceRm, - From: iopts.From, - IDMappingOptions: idmappingOptions, - IIDFile: iopts.Iidfile, - In: stdin, - Isolation: isolation, - Labels: iopts.Label, - Layers: layers, - LogRusage: iopts.LogRusage, - Manifest: iopts.Manifest, - MaxPullPushRetries: maxPullPushRetries, - NamespaceOptions: namespaceOptions, - NoCache: iopts.NoCache, - OS: platform.OS, - Out: stdout, - Output: output, - OutputFormat: format, - PullPolicy: pullPolicy, - PullPushRetryDelay: pullPushRetryDelay, - Quiet: iopts.Quiet, - RemoveIntermediateCtrs: iopts.Rm, - ReportWriter: reporter, - Runtime: iopts.Runtime, - RuntimeArgs: runtimeFlags, - RusageLogFile: iopts.RusageLogFile, - SignBy: iopts.SignBy, - SignaturePolicyPath: iopts.SignaturePolicy, - Squash: iopts.Squash, - SystemContext: &platformContext, - Target: iopts.Target, - TransientMounts: iopts.Volumes, - OciDecryptConfig: decConfig, - Jobs: &iopts.Jobs, - JobSemaphore: jobSemaphore, - Excludes: excludes, - Timestamp: timestamp, - } - if iopts.Quiet { - options.ReportWriter = ioutil.Discard - } + options := define.BuildOptions{ + AddCapabilities: iopts.CapAdd, + AdditionalTags: tags, + Annotations: iopts.Annotation, + Architecture: systemContext.ArchitectureChoice, + Args: args, + BlobDirectory: iopts.BlobCache, + CNIConfigDir: iopts.CNIConfigDir, + CNIPluginPath: iopts.CNIPlugInPath, + CommonBuildOpts: commonOpts, + Compression: compression, + ConfigureNetwork: networkPolicy, + ContextDirectory: contextDir, + DefaultMountsFilePath: globalFlagResults.DefaultMountsFile, + Devices: iopts.Devices, + DropCapabilities: iopts.CapDrop, + Err: stderr, + ForceRmIntermediateCtrs: iopts.ForceRm, + From: iopts.From, + IDMappingOptions: idmappingOptions, + IIDFile: iopts.Iidfile, + In: stdin, + Isolation: isolation, + Labels: iopts.Label, + Layers: layers, + LogRusage: iopts.LogRusage, + Manifest: iopts.Manifest, + MaxPullPushRetries: maxPullPushRetries, + NamespaceOptions: namespaceOptions, + NoCache: iopts.NoCache, + OS: systemContext.OSChoice, + Out: stdout, + Output: output, + OutputFormat: format, + PullPolicy: pullPolicy, + PullPushRetryDelay: pullPushRetryDelay, + Quiet: iopts.Quiet, + RemoveIntermediateCtrs: iopts.Rm, + ReportWriter: reporter, + Runtime: iopts.Runtime, + RuntimeArgs: runtimeFlags, + RusageLogFile: iopts.RusageLogFile, + SignBy: iopts.SignBy, + SignaturePolicyPath: iopts.SignaturePolicy, + Squash: iopts.Squash, + SystemContext: systemContext, + Target: iopts.Target, + TransientMounts: iopts.Volumes, + OciDecryptConfig: decConfig, + Jobs: &iopts.Jobs, + Excludes: excludes, + Timestamp: timestamp, + Platforms: platforms, + } + if iopts.Quiet { + options.ReportWriter = ioutil.Discard + } - _, _, err = imagebuildah.BuildDockerfiles(getContext(), store, options, dockerfiles...) - return err - }) - } - if merr := builds.Wait(); merr != nil { - if merr.Len() == 1 { - return merr.Errors[0] - } - return merr.ErrorOrNil() - } - return nil + _, _, err = imagebuildah.BuildDockerfiles(getContext(), store, options, dockerfiles...) + return err } // discoverContainerfile tries to find a Containerfile or a Dockerfile within the provided `path`. diff --git a/commit.go b/commit.go index c2afd84c7..bbf1727fb 100644 --- a/commit.go +++ b/commit.go @@ -197,7 +197,7 @@ func (b *Builder) addManifest(ctx context.Context, manifestName string, imageSpe names, err := util.ExpandNames([]string{manifestName}, systemContext, b.store) if err != nil { - return "", errors.Wrapf(err, "error encountered while expanding image name %q", manifestName) + return "", errors.Wrapf(err, "error encountered while expanding manifest list name %q", manifestName) } ref, err := util.VerifyTagName(imageSpec) diff --git a/define/build.go b/define/build.go index a6bb836e8..1fdad2e0e 100644 --- a/define/build.go +++ b/define/build.go @@ -230,4 +230,8 @@ type BuildOptions struct { // From is the image name to use to replace the value specified in the first // FROM instruction in the Containerfile From string + // Platforms is the list of parsed OS/Arch/Variant triples that we want + // to build the image for. If this slice has items in it, the OS and + // Architecture fields above are ignored. + Platforms []struct{ OS, Arch, Variant string } } diff --git a/docs/buildah-bud.md b/docs/buildah-bud.md index 9c202f897..2461a1d10 100644 --- a/docs/buildah-bud.md +++ b/docs/buildah-bud.md @@ -44,7 +44,7 @@ Note: this information is not present in Docker image formats, so it is discarde **--arch**="ARCH" -Set the ARCH of the image to be pulled to the provided value instead of using the architecture of the host. (Examples: arm, arm64, 386, amd64, ppc64le, s390x) +Set the ARCH of the image to be built, and that of the base image to be pulled, if the build uses one, to the provided value instead of using the architecture of the host. (Examples: arm, arm64, 386, amd64, ppc64le, s390x) **--authfile** *path* @@ -214,6 +214,7 @@ from inside a rootless container will fail. The **crun**(1) runtime offers a workaround for this by adding the option **--annotation run.oci.keep_original_groups=1**. **--disable-compression**, **-D** + Don't compress filesystem layers when building the image unless it is required by the location where the image is being written. This is the default setting, because image layers are compressed automatically when they are pushed to @@ -287,7 +288,8 @@ those. **--iidfile** *ImageIDfile* -Write the image ID to the file. +Write the built image's ID to the file. When `--platform` is specified more +than once, attempting to use this option will trigger an error. **--ignorefile** *file* @@ -322,7 +324,7 @@ BUILDAH\_ISOLATION environment variable. `export BUILDAH_ISOLATION=oci` Run up to N concurrent stages in parallel. If the number of jobs is greater than 1, stdin will be read from /dev/null. If 0 is specified, then there is -no limit in the number of jobs that run in parallel. +no limit on the number of jobs that run in parallel. **--label** *label* @@ -354,8 +356,9 @@ specified file instead of to standard output and standard error. **--manifest** "manifest" -Name of the manifest list to which the image will be added. Creates the manifest list -if it does not exist. This option is useful for building multi architecture images. +Name of the manifest list to which the built image will be added. Creates the +manifest list if it does not exist. This option is useful for building multi +architecture images. **--memory**, **-m**="" @@ -395,7 +398,7 @@ Do not use existing cached images for the container build. Build from the start **--os**="OS" -Set the OS of the image to be pulled instead of using the current operating system of the host. +Set the OS of the image to be built, and that of the base image to be pulled, if the build uses one, instead of using the current operating system of the host. **--pid** *how* @@ -479,6 +482,7 @@ Note: Do not pass the leading `--` to the flag. To pass the runc flag `--log-for to buildah bud, the option given would be `--runtime-flag log-format=json`. **--secret**=**id=id,src=path** + Pass secret information to be used in the Containerfile for building images in a safe way that will not end up stored in the final image, or be seen in other stages. The secret will be mounted in the container at the default location of `/run/secrets/id`. @@ -520,6 +524,7 @@ Squash all of the image's new layers into a single new layer; any preexisting la are not squashed. **--ssh**=**default**|*id[=socket>|[,]* + SSH agent socket or keys to expose to the build. The socket path can be left empty to use the value of `default=$SSH_AUTH_SOCK` diff --git a/imagebuildah/build.go b/imagebuildah/build.go index 954ef7f8a..f2e6f16c6 100644 --- a/imagebuildah/build.go +++ b/imagebuildah/build.go @@ -13,16 +13,22 @@ import ( "strings" "github.com/containers/buildah/define" + "github.com/containers/buildah/manifests" "github.com/containers/buildah/util" + "github.com/containers/common/libimage" "github.com/containers/common/pkg/config" "github.com/containers/image/v5/docker/reference" + "github.com/containers/image/v5/manifest" + "github.com/containers/image/v5/types" "github.com/containers/storage" "github.com/containers/storage/pkg/archive" + "github.com/hashicorp/go-multierror" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/openshift/imagebuilder" "github.com/openshift/imagebuilder/dockerfile/parser" "github.com/pkg/errors" "github.com/sirupsen/logrus" + "golang.org/x/sync/semaphore" ) const ( @@ -44,12 +50,19 @@ type Mount = specs.Mount type BuildOptions = define.BuildOptions // BuildDockerfiles parses a set of one or more Dockerfiles (which may be -// URLs), creates a new Executor, and then runs Prepare/Execute/Commit/Delete -// over the entire set of instructions. -func BuildDockerfiles(ctx context.Context, store storage.Store, options define.BuildOptions, paths ...string) (string, reference.Canonical, error) { +// URLs), creates one or more new Executors, and then runs +// Prepare/Execute/Commit/Delete over the entire set of instructions. +// If the Manifest option is set, returns the ID of the manifest list, else it +// returns the ID of the built image, and if a name was assigned to it, a +// canonical reference for that image. +func BuildDockerfiles(ctx context.Context, store storage.Store, options define.BuildOptions, paths ...string) (id string, ref reference.Canonical, err error) { if len(paths) == 0 { return "", nil, errors.Errorf("error building: no dockerfiles specified") } + if len(options.Platforms) > 1 && options.IIDFile != "" { + return "", nil, errors.Errorf("building multiple images, but iidfile %q can only be used to store one image ID", options.IIDFile) + } + logger := logrus.New() if options.Err != nil { logger.SetOutput(options.Err) @@ -73,6 +86,7 @@ func BuildDockerfiles(ctx context.Context, store storage.Store, options define.B return "", nil, errors.Wrapf(err, "tag %s", tag) } } + for _, dfile := range paths { var data io.ReadCloser @@ -143,21 +157,116 @@ func BuildDockerfiles(ctx context.Context, store storage.Store, options define.B dockerfiles = append(dockerfiles, data) } - mainNode, err := imagebuilder.ParseDockerfile(dockerfiles[0]) + var files [][]byte + for _, dockerfile := range dockerfiles { + var b bytes.Buffer + if _, err := b.ReadFrom(dockerfile); err != nil { + return "", nil, err + } + files = append(files, b.Bytes()) + } + + if options.Jobs != nil && *options.Jobs != 0 { + options.JobSemaphore = semaphore.NewWeighted(int64(*options.Jobs)) + } + + if options.Manifest != "" && len(options.Platforms) > 0 { + // Ensure that the list's ID is known before we spawn off any + // goroutines that'll want to modify it, so that they don't + // race and create two lists, one of which will rapidly become + // ignored. + names, err := util.ExpandNames([]string{options.Manifest}, options.SystemContext, store) + if err != nil { + return "", nil, errors.Wrapf(err, "while expanding manifest list name %q", options.Manifest) + } + rt, err := libimage.RuntimeFromStore(store, nil) + if err != nil { + return "", nil, err + } + _, err = rt.LookupManifestList(options.Manifest) + if err != nil && errors.Cause(err) == storage.ErrImageUnknown { + list := manifests.Create() + _, err = list.SaveToImage(store, "", names, manifest.DockerV2ListMediaType) + } + if err != nil { + return "", nil, err + } + } + + var builds multierror.Group + if options.SystemContext == nil { + options.SystemContext = &types.SystemContext{} + } + + if len(options.Platforms) == 0 { + options.Platforms = append(options.Platforms, struct{ OS, Arch, Variant string }{ + OS: options.SystemContext.OSChoice, + Arch: options.SystemContext.ArchitectureChoice, + }) + } + + systemContext := options.SystemContext + for _, platform := range options.Platforms { + platformContext := *systemContext + platformContext.OSChoice = platform.OS + platformContext.ArchitectureChoice = platform.Arch + platformContext.VariantChoice = platform.Variant + platformOptions := options + platformOptions.SystemContext = &platformContext + logPrefix := "" + if len(options.Platforms) > 1 { + logPrefix = "[" + platform.OS + "/" + platform.Arch + if platform.Variant != "" { + logPrefix += "/" + platform.Variant + } + logPrefix += "] " + } + builds.Go(func() error { + thisID, thisRef, err := buildDockerfilesOnce(ctx, store, logger, logPrefix, platformOptions, paths, files) + if err != nil { + return err + } + id, ref = thisID, thisRef + return nil + }) + } + + if merr := builds.Wait(); merr != nil { + if merr.Len() == 1 { + return "", nil, merr.Errors[0] + } + return "", nil, merr.ErrorOrNil() + } + if options.Manifest != "" { + rt, err := libimage.RuntimeFromStore(store, nil) + if err != nil { + return "", nil, err + } + list, err := rt.LookupManifestList(options.Manifest) + if err != nil { + return "", nil, err + } + id, ref = list.ID(), nil + } + return id, ref, nil +} + +func buildDockerfilesOnce(ctx context.Context, store storage.Store, logger *logrus.Logger, logPrefix string, options define.BuildOptions, dockerfiles []string, dockerfilecontents [][]byte) (string, reference.Canonical, error) { + mainNode, err := imagebuilder.ParseDockerfile(bytes.NewReader(dockerfilecontents[0])) if err != nil { return "", nil, errors.Wrapf(err, "error parsing main Dockerfile: %s", dockerfiles[0]) } warnOnUnsetBuildArgs(logger, mainNode, options.Args) - for _, d := range dockerfiles[1:] { - additionalNode, err := imagebuilder.ParseDockerfile(d) + for i, d := range dockerfilecontents[1:] { + additionalNode, err := imagebuilder.ParseDockerfile(bytes.NewReader(d)) if err != nil { - return "", nil, errors.Wrapf(err, "error parsing additional Dockerfile %s", d) + return "", nil, errors.Wrapf(err, "error parsing additional Dockerfile %s", dockerfiles[i]) } mainNode.Children = append(mainNode.Children, additionalNode.Children...) } - exec, err := NewExecutor(logger, store, options, mainNode) + exec, err := newExecutor(logger, logPrefix, store, options, mainNode) if err != nil { return "", nil, errors.Wrapf(err, "error creating build executor") } diff --git a/imagebuildah/executor.go b/imagebuildah/executor.go index 91ea26f97..9523dcd37 100644 --- a/imagebuildah/executor.go +++ b/imagebuildah/executor.go @@ -124,6 +124,7 @@ type Executor struct { manifest string secrets map[string]string sshsources map[string]*sshagent.Source + logPrefix string } type imageTypeAndHistoryAndDiffIDs struct { @@ -133,8 +134,8 @@ type imageTypeAndHistoryAndDiffIDs struct { err error } -// NewExecutor creates a new instance of the imagebuilder.Executor interface. -func NewExecutor(logger *logrus.Logger, store storage.Store, options define.BuildOptions, mainNode *parser.Node) (*Executor, error) { +// newExecutor creates a new instance of the imagebuilder.Executor interface. +func newExecutor(logger *logrus.Logger, logPrefix string, store storage.Store, options define.BuildOptions, mainNode *parser.Node) (*Executor, error) { defaultContainerConfig, err := config.Default() if err != nil { return nil, errors.Wrapf(err, "failed to get container config") @@ -266,6 +267,7 @@ func NewExecutor(logger *logrus.Logger, store storage.Store, options define.Buil manifest: options.Manifest, secrets: secrets, sshsources: sshsources, + logPrefix: logPrefix, } if exec.err == nil { exec.err = os.Stderr @@ -442,7 +444,7 @@ func (b *Executor) buildStage(ctx context.Context, cleanupStages map[int]*StageE if stageExecutor.log == nil { stepCounter := 0 stageExecutor.log = func(format string, args ...interface{}) { - prefix := "" + prefix := b.logPrefix if len(stages) > 1 { prefix += fmt.Sprintf("[%d/%d] ", stageIndex+1, len(stages)) }