docker/buildah
1
0
Fork 0
mirror of https://github.com/containers/buildah.git synced 2025-07-31 15:24:26 +03:00
Code Activity

imagebuildah: move multiple-platform building internal

Browse Source 
Move multiple-platform build juggling logic from the CLI wrapper
directly into the imagebuildah package, to make using it easier for
packages that consume us as a library.

This requires reading Dockerfiles into byte slices so that we can
re-parse them for each per-platform build, rather than parsing them
directly, as we used to, since building modifies the parsed tree.

When building for multiple platforms, prefix progress log messages with
the platform description.

Signed-off-by: Nalin Dahyabhai <nalin@redhat.com>
This commit is contained in:
Nalin Dahyabhai
2021-08-10 18:11:15 -04:00
parent ea7127cf69
commit 34d6ee13b8
7 changed files with 198 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -34,7 +34,7 @@ LIBSECCOMP_COMMIT := release-2.3
EXTRA_LDFLAGS ?=
BUILDAH_LDFLAGS := -ldflags '-X main.GitCommit=$(GIT_COMMIT) -X main.buildInfo=$(SOURCE_DATE_EPOCH) -X main.cniVersion=$(CNI_COMMIT) $(EXTRA_LDFLAGS)'
SOURCES=*.go imagebuildah/*.go bind/*.go chroot/*.go copier/*.go docker/*.go manifests/*.go pkg/blobcache/*.go pkg/chrootuser/*.go pkg/cli/*.go pkg/formats/*.go pkg/manifests/*.go pkg/overlay/*.go pkg/parse/*.go pkg/rusage/*.go util/*.go
SOURCES=*.go imagebuildah/*.go bind/*.go chroot/*.go copier/*.go docker/*.go manifests/*.go pkg/blobcache/*.go pkg/chrootuser/*.go pkg/cli/*.go pkg/completion/*.go pkg/formats/*.go pkg/overlay/*.go pkg/parse/*.go pkg/rusage/*.go pkg/sshagent/*.go pkg/umask/*.go util/*.go
LINTFLAGS ?=

56
cmd/buildah/bud.go
View File

@ -13,16 +13,10 @@ import (
buildahcli "github.com/containers/buildah/pkg/cli"
"github.com/containers/buildah/pkg/parse"
"github.com/containers/buildah/util"
"github.com/containers/common/libimage"
"github.com/containers/common/libimage/manifests"
"github.com/containers/common/pkg/auth"
"github.com/containers/image/v5/manifest"
"github.com/containers/storage"
"github.com/hashicorp/go-multierror"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"golang.org/x/sync/semaphore"
)
type budOptions struct {
@ -315,32 +309,6 @@ func budCmd(c *cobra.Command, inputArgs []string, iopts budOptions) error {
return errors.Wrapf(err, "unable to obtain decrypt config")
}
var (
jobSemaphore *semaphore.Weighted
builds multierror.Group
)
if iopts.Jobs > 0 {
jobSemaphore = semaphore.NewWeighted(int64(iopts.Jobs))
iopts.Jobs = 0
}
if iopts.Manifest != "" {
// Ensure that the list's ID is known before we spawn off any
// goroutines that'll want to modify it, so that they don't
// race and create two lists, one of which will rapidly become
// ignored.
rt, err := libimage.RuntimeFromStore(store, nil)
if err != nil {
return err
}
_, err = rt.LookupManifestList(iopts.Manifest)
if err != nil && errors.Cause(err) == storage.ErrImageUnknown {
list := manifests.Create()
_, err = list.SaveToImage(store, "", []string{iopts.Manifest}, manifest.DockerV2ListMediaType)
}
if err != nil {
return err
}
}
var excludes []string
if iopts.IgnoreFile != "" {
if excludes, err = parseDockerignore(iopts.IgnoreFile); err != nil {
@ -352,18 +320,11 @@ func budCmd(c *cobra.Command, inputArgs []string, iopts budOptions) error {
t := time.Unix(iopts.Timestamp, 0).UTC()
timestamp = &t
}
for _, platform := range platforms {
platform := platform
builds.Go(func() error {
platformContext := *systemContext
platformContext.OSChoice = platform.OS
platformContext.ArchitectureChoice = platform.Arch
platformContext.VariantChoice = platform.Variant
options := define.BuildOptions{
AddCapabilities: iopts.CapAdd,
AdditionalTags: tags,
Annotations: iopts.Annotation,
Architecture: platform.Arch,
Architecture: systemContext.ArchitectureChoice,
Args: args,
BlobDirectory: iopts.BlobCache,
CNIConfigDir: iopts.CNIConfigDir,
@ -389,7 +350,7 @@ func budCmd(c *cobra.Command, inputArgs []string, iopts budOptions) error {
MaxPullPushRetries: maxPullPushRetries,
NamespaceOptions: namespaceOptions,
NoCache: iopts.NoCache,
OS: platform.OS,
OS: systemContext.OSChoice,
Out: stdout,
Output: output,
OutputFormat: format,
@ -404,14 +365,14 @@ func budCmd(c *cobra.Command, inputArgs []string, iopts budOptions) error {
SignBy: iopts.SignBy,
SignaturePolicyPath: iopts.SignaturePolicy,
Squash: iopts.Squash,
SystemContext: &platformContext,
SystemContext: systemContext,
Target: iopts.Target,
TransientMounts: iopts.Volumes,
OciDecryptConfig: decConfig,
Jobs: &iopts.Jobs,
JobSemaphore: jobSemaphore,
Excludes: excludes,
Timestamp: timestamp,
Platforms: platforms,
}
if iopts.Quiet {
options.ReportWriter = ioutil.Discard
@ -419,15 +380,6 @@ func budCmd(c *cobra.Command, inputArgs []string, iopts budOptions) error {
_, _, err = imagebuildah.BuildDockerfiles(getContext(), store, options, dockerfiles...)
return err
})
}
if merr := builds.Wait(); merr != nil {
if merr.Len() == 1 {
return merr.Errors[0]
}
return merr.ErrorOrNil()
}
return nil
}
// discoverContainerfile tries to find a Containerfile or a Dockerfile within the provided `path`.

2
commit.go
View File

@ -197,7 +197,7 @@ func (b *Builder) addManifest(ctx context.Context, manifestName string, imageSpe
names, err := util.ExpandNames([]string{manifestName}, systemContext, b.store)
if err != nil {
return "", errors.Wrapf(err, "error encountered while expanding image name %q", manifestName)
return "", errors.Wrapf(err, "error encountered while expanding manifest list name %q", manifestName)
}
ref, err := util.VerifyTagName(imageSpec)

4
define/build.go
View File

@ -230,4 +230,8 @@ type BuildOptions struct {
// From is the image name to use to replace the value specified in the first
// FROM instruction in the Containerfile
From string
// Platforms is the list of parsed OS/Arch/Variant triples that we want
// to build the image for. If this slice has items in it, the OS and
// Architecture fields above are ignored.
Platforms []struct{ OS, Arch, Variant string }
}

17
docs/buildah-bud.md
View File

@ -44,7 +44,7 @@ Note: this information is not present in Docker image formats, so it is discarde
**--arch**="ARCH"
Set the ARCH of the image to be pulled to the provided value instead of using the architecture of the host. (Examples: arm, arm64, 386, amd64, ppc64le, s390x)
Set the ARCH of the image to be built, and that of the base image to be pulled, if the build uses one, to the provided value instead of using the architecture of the host. (Examples: arm, arm64, 386, amd64, ppc64le, s390x)
**--authfile** *path*
@ -214,6 +214,7 @@ from inside a rootless container will fail. The **crun**(1) runtime offers a
workaround for this by adding the option **--annotation run.oci.keep_original_groups=1**.
**--disable-compression**, **-D**
Don't compress filesystem layers when building the image unless it is required
by the location where the image is being written. This is the default setting,
because image layers are compressed automatically when they are pushed to
@ -287,7 +288,8 @@ those.
**--iidfile** *ImageIDfile*
Write the image ID to the file.
Write the built image's ID to the file. When `--platform` is specified more
than once, attempting to use this option will trigger an error.
**--ignorefile** *file*
@ -322,7 +324,7 @@ BUILDAH\_ISOLATION environment variable. `export BUILDAH_ISOLATION=oci`
Run up to N concurrent stages in parallel. If the number of jobs is greater than 1,
stdin will be read from /dev/null. If 0 is specified, then there is
no limit in the number of jobs that run in parallel.
no limit on the number of jobs that run in parallel.
**--label** *label*
@ -354,8 +356,9 @@ specified file instead of to standard output and standard error.
**--manifest** "manifest"
Name of the manifest list to which the image will be added. Creates the manifest list
if it does not exist. This option is useful for building multi architecture images.
Name of the manifest list to which the built image will be added. Creates the
manifest list if it does not exist. This option is useful for building multi
architecture images.
**--memory**, **-m**=""
@ -395,7 +398,7 @@ Do not use existing cached images for the container build. Build from the start
**--os**="OS"
Set the OS of the image to be pulled instead of using the current operating system of the host.
Set the OS of the image to be built, and that of the base image to be pulled, if the build uses one, instead of using the current operating system of the host.
**--pid** *how*
@ -479,6 +482,7 @@ Note: Do not pass the leading `--` to the flag. To pass the runc flag `--log-for
to buildah bud, the option given would be `--runtime-flag log-format=json`.
**--secret**=**id=id,src=path**
Pass secret information to be used in the Containerfile for building images
in a safe way that will not end up stored in the final image, or be seen in other stages.
The secret will be mounted in the container at the default location of `/run/secrets/id`.
@ -520,6 +524,7 @@ Squash all of the image's new layers into a single new layer; any preexisting la
are not squashed.
**--ssh**=**default**|*id[=socket>|<key>[,<key>]*
SSH agent socket or keys to expose to the build.
The socket path can be left empty to use the value of `default=$SSH_AUTH_SOCK`

125
imagebuildah/build.go
View File

@ -13,16 +13,22 @@ import (
"strings"
"github.com/containers/buildah/define"
"github.com/containers/buildah/manifests"
"github.com/containers/buildah/util"
"github.com/containers/common/libimage"
"github.com/containers/common/pkg/config"
"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/manifest"
"github.com/containers/image/v5/types"
"github.com/containers/storage"
"github.com/containers/storage/pkg/archive"
"github.com/hashicorp/go-multierror"
specs "github.com/opencontainers/runtime-spec/specs-go"
"github.com/openshift/imagebuilder"
"github.com/openshift/imagebuilder/dockerfile/parser"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"golang.org/x/sync/semaphore"
)
const (
@ -44,12 +50,19 @@ type Mount = specs.Mount
type BuildOptions = define.BuildOptions
// BuildDockerfiles parses a set of one or more Dockerfiles (which may be
// URLs), creates a new Executor, and then runs Prepare/Execute/Commit/Delete
// over the entire set of instructions.
func BuildDockerfiles(ctx context.Context, store storage.Store, options define.BuildOptions, paths ...string) (string, reference.Canonical, error) {
// URLs), creates one or more new Executors, and then runs
// Prepare/Execute/Commit/Delete over the entire set of instructions.
// If the Manifest option is set, returns the ID of the manifest list, else it
// returns the ID of the built image, and if a name was assigned to it, a
// canonical reference for that image.
func BuildDockerfiles(ctx context.Context, store storage.Store, options define.BuildOptions, paths ...string) (id string, ref reference.Canonical, err error) {
if len(paths) == 0 {
return "", nil, errors.Errorf("error building: no dockerfiles specified")
}
if len(options.Platforms) > 1 && options.IIDFile != "" {
return "", nil, errors.Errorf("building multiple images, but iidfile %q can only be used to store one image ID", options.IIDFile)
}
logger := logrus.New()
if options.Err != nil {
logger.SetOutput(options.Err)
@ -73,6 +86,7 @@ func BuildDockerfiles(ctx context.Context, store storage.Store, options define.B
return "", nil, errors.Wrapf(err, "tag %s", tag)
}
}
for _, dfile := range paths {
var data io.ReadCloser
@ -143,21 +157,116 @@ func BuildDockerfiles(ctx context.Context, store storage.Store, options define.B
dockerfiles = append(dockerfiles, data)
}
mainNode, err := imagebuilder.ParseDockerfile(dockerfiles[0])
var files [][]byte
for _, dockerfile := range dockerfiles {
var b bytes.Buffer
if _, err := b.ReadFrom(dockerfile); err != nil {
return "", nil, err
}
files = append(files, b.Bytes())
}
if options.Jobs != nil && *options.Jobs != 0 {
options.JobSemaphore = semaphore.NewWeighted(int64(*options.Jobs))
}
if options.Manifest != "" && len(options.Platforms) > 0 {
// Ensure that the list's ID is known before we spawn off any
// goroutines that'll want to modify it, so that they don't
// race and create two lists, one of which will rapidly become
// ignored.
names, err := util.ExpandNames([]string{options.Manifest}, options.SystemContext, store)
if err != nil {
return "", nil, errors.Wrapf(err, "while expanding manifest list name %q", options.Manifest)
}
rt, err := libimage.RuntimeFromStore(store, nil)
if err != nil {
return "", nil, err
}
_, err = rt.LookupManifestList(options.Manifest)
if err != nil && errors.Cause(err) == storage.ErrImageUnknown {
list := manifests.Create()
_, err = list.SaveToImage(store, "", names, manifest.DockerV2ListMediaType)
}
if err != nil {
return "", nil, err
}
}
var builds multierror.Group
if options.SystemContext == nil {
options.SystemContext = &types.SystemContext{}
}
if len(options.Platforms) == 0 {
options.Platforms = append(options.Platforms, struct{ OS, Arch, Variant string }{
OS: options.SystemContext.OSChoice,
Arch: options.SystemContext.ArchitectureChoice,
})
}
systemContext := options.SystemContext
for _, platform := range options.Platforms {
platformContext := *systemContext
platformContext.OSChoice = platform.OS
platformContext.ArchitectureChoice = platform.Arch
platformContext.VariantChoice = platform.Variant
platformOptions := options
platformOptions.SystemContext = &platformContext
logPrefix := ""
if len(options.Platforms) > 1 {
logPrefix = "[" + platform.OS + "/" + platform.Arch
if platform.Variant != "" {
logPrefix += "/" + platform.Variant
}
logPrefix += "] "
}
builds.Go(func() error {
thisID, thisRef, err := buildDockerfilesOnce(ctx, store, logger, logPrefix, platformOptions, paths, files)
if err != nil {
return err
}
id, ref = thisID, thisRef
return nil
})
}
if merr := builds.Wait(); merr != nil {
if merr.Len() == 1 {
return "", nil, merr.Errors[0]
}
return "", nil, merr.ErrorOrNil()
}
if options.Manifest != "" {
rt, err := libimage.RuntimeFromStore(store, nil)
if err != nil {
return "", nil, err
}
list, err := rt.LookupManifestList(options.Manifest)
if err != nil {
return "", nil, err
}
id, ref = list.ID(), nil
}
return id, ref, nil
}
func buildDockerfilesOnce(ctx context.Context, store storage.Store, logger *logrus.Logger, logPrefix string, options define.BuildOptions, dockerfiles []string, dockerfilecontents [][]byte) (string, reference.Canonical, error) {
mainNode, err := imagebuilder.ParseDockerfile(bytes.NewReader(dockerfilecontents[0]))
if err != nil {
return "", nil, errors.Wrapf(err, "error parsing main Dockerfile: %s", dockerfiles[0])
}
warnOnUnsetBuildArgs(logger, mainNode, options.Args)
for _, d := range dockerfiles[1:] {
additionalNode, err := imagebuilder.ParseDockerfile(d)
for i, d := range dockerfilecontents[1:] {
additionalNode, err := imagebuilder.ParseDockerfile(bytes.NewReader(d))
if err != nil {
return "", nil, errors.Wrapf(err, "error parsing additional Dockerfile %s", d)
return "", nil, errors.Wrapf(err, "error parsing additional Dockerfile %s", dockerfiles[i])
}
mainNode.Children = append(mainNode.Children, additionalNode.Children...)
}
exec, err := NewExecutor(logger, store, options, mainNode)
exec, err := newExecutor(logger, logPrefix, store, options, mainNode)
if err != nil {
return "", nil, errors.Wrapf(err, "error creating build executor")
}

8
imagebuildah/executor.go
View File

@ -124,6 +124,7 @@ type Executor struct {
manifest string
secrets map[string]string
sshsources map[string]*sshagent.Source
logPrefix string
}
type imageTypeAndHistoryAndDiffIDs struct {
@ -133,8 +134,8 @@ type imageTypeAndHistoryAndDiffIDs struct {
err error
}
// NewExecutor creates a new instance of the imagebuilder.Executor interface.
func NewExecutor(logger *logrus.Logger, store storage.Store, options define.BuildOptions, mainNode *parser.Node) (*Executor, error) {
// newExecutor creates a new instance of the imagebuilder.Executor interface.
func newExecutor(logger *logrus.Logger, logPrefix string, store storage.Store, options define.BuildOptions, mainNode *parser.Node) (*Executor, error) {
defaultContainerConfig, err := config.Default()
if err != nil {
return nil, errors.Wrapf(err, "failed to get container config")
@ -266,6 +267,7 @@ func NewExecutor(logger *logrus.Logger, store storage.Store, options define.Buil
manifest: options.Manifest,
secrets: secrets,
sshsources: sshsources,
logPrefix: logPrefix,
}
if exec.err == nil {
exec.err = os.Stderr
@ -442,7 +444,7 @@ func (b *Executor) buildStage(ctx context.Context, cleanupStages map[int]*StageE
if stageExecutor.log == nil {
stepCounter := 0
stageExecutor.log = func(format string, args ...interface{}) {
prefix := ""
prefix := b.logPrefix
if len(stages) > 1 {
prefix += fmt.Sprintf("[%d/%d] ", stageIndex+1, len(stages))
}