diff --git a/cmd/buildah/build.go b/cmd/buildah/build.go index 17324f31b..98d74c65a 100644 --- a/cmd/buildah/build.go +++ b/cmd/buildah/build.go @@ -328,6 +328,15 @@ func buildCmd(c *cobra.Command, inputArgs []string, iopts buildOptions) error { t := time.Unix(iopts.Timestamp, 0).UTC() timestamp = &t } + if c.Flag("output").Changed { + buildOption, err := parse.GetBuildOutput(iopts.BuildOutput) + if err != nil { + return err + } + if buildOption.IsStdout { + iopts.Quiet = true + } + } options := define.BuildOptions{ AddCapabilities: iopts.CapAdd, AdditionalTags: tags, @@ -363,6 +372,7 @@ func buildCmd(c *cobra.Command, inputArgs []string, iopts buildOptions) error { OS: systemContext.OSChoice, Out: stdout, Output: output, + BuildOutput: iopts.BuildOutput, OutputFormat: format, PullPolicy: pullPolicy, PullPushRetryDelay: pullPushRetryDelay, diff --git a/commit.go b/commit.go index 25c300716..1da8328f8 100644 --- a/commit.go +++ b/commit.go @@ -229,6 +229,7 @@ func (b *Builder) addManifest(ctx context.Context, manifestName string, imageSpe func (b *Builder) Commit(ctx context.Context, dest types.ImageReference, options CommitOptions) (string, reference.Canonical, digest.Digest, error) { var ( imgID string + src types.ImageReference ) // If we weren't given a name, build a destination reference using a @@ -300,7 +301,7 @@ func (b *Builder) Commit(ctx context.Context, dest types.ImageReference, options logrus.Debugf("committing image with reference %q is allowed by policy", transports.ImageName(dest)) // Build an image reference from which we can copy the finished image. - src, err := b.makeImageRef(options) + src, err = b.makeContainerImageRef(options) if err != nil { return imgID, nil, "", errors.Wrapf(err, "error computing layer digests and building metadata for container %q", b.ContainerID) } diff --git a/define/build.go b/define/build.go index 1d452d66d..2cb92e811 100644 --- a/define/build.go +++ b/define/build.go @@ -123,6 +123,10 @@ type BuildOptions struct { Args map[string]string // Name of the image to write to. Output string + // BuildOutput specifies if any custom build output is selected for following build. + // It allows end user to export recently built rootfs into a directory or tar. + // See the documentation of 'buildah build --output' for the details of the format. + BuildOutput string // Additional tags to add to the image that we write, if we know of a // way to add them. AdditionalTags []string diff --git a/define/types.go b/define/types.go index beedcd86e..e08e3cc9c 100644 --- a/define/types.go +++ b/define/types.go @@ -96,6 +96,13 @@ type Secret struct { SourceType string } +// BuildOutputOptions contains the the outcome of parsing the value of a build --output flag +type BuildOutputOption struct { + Path string // Only valid if !IsStdout + IsDir bool + IsStdout bool +} + // TempDirForURL checks if the passed-in string looks like a URL or -. If it is, // TempDirForURL creates a temporary directory, arranges for its contents to be // the contents of that URL, and returns the temporary directory's path, along diff --git a/docs/buildah-build.1.md b/docs/buildah-build.1.md index 1b0b512a4..eb80472cf 100644 --- a/docs/buildah-build.1.md +++ b/docs/buildah-build.1.md @@ -411,6 +411,25 @@ By default, Buildah manages _/etc/hosts_, adding the container's own IP address. Set the OS of the image to be built, and that of the base image to be pulled, if the build uses one, instead of using the current operating system of the host. +**--output**, **-o**="" + +Output destination (format: type=local,dest=path) + +The --output (or -o) option extends the default behavior of building a container image by allowing users to export the contents of the image as files on the local filesystem, which can be useful for generating local binaries, code generation, etc. + +The value for --output is a comma-separated sequence of key=value pairs, defining the output type and options. + +Supported _keys_ are: +- **dest**: Destination path for exported output. Valid value is absolute or relative path, `-` means the standard output. +- **type**: Defines the type of output to be used. Valid values is documented below. + +Valid _type_ values are: +- **local**: write the resulting build files to a directory on the client-side. +- **tar**: write the resulting files as a single tarball (.tar). + +If no type is specified, the value defaults to **local**. +Alternatively, instead of a comma-separated sequence, the value of **--output** can be just a destination (in the `**dest** format) (e.g. `--output some-path`, `--output -`) where `--output some-path` is treated as if **type=local** and `--output -` is treated as if **type=tar**. + **--pid** *how* Sets the configuration for PID namespaces when handling `RUN` instructions. @@ -824,6 +843,16 @@ buildah bud --platform linux/arm64 --platform linux/amd64 --manifest myimage /tm buildah bud --all-platforms --manifest myimage /tmp/mysrc +### Building an image using (--output) custom build output + +buildah build -o out . + +buildah build --output type=local,dest=out . + +buildah build --output type=tar,dest=out.tar . + +buildah build -o - . > out.tar + ### Building an image using a URL This will clone the specified GitHub repository from the URL and use it as context. The Containerfile or Dockerfile at the root of the repository is used as the context of the build. This only works if the GitHub repository is a dedicated repository. diff --git a/image.go b/image.go index e859de183..e3668bd0d 100644 --- a/image.go +++ b/image.go @@ -43,6 +43,15 @@ const ( Dockerv2ImageManifest = define.Dockerv2ImageManifest ) +// ExtractRootfsOptions is consumed by ExtractRootfs() which allows +// users to preserve nature of various modes like setuid, setgid and xattrs +// over the extracted file system objects. +type ExtractRootfsOptions struct { + StripSetuidBit bool // strip the setuid bit off of items being extracted. + StripSetgidBit bool // strip the setgid bit off of items being extracted. + StripXattrs bool // don't record extended attributes of items being extracted. +} + type containerImageRef struct { fromImageName string fromImageID string @@ -150,7 +159,10 @@ func computeLayerMIMEType(what string, layerCompression archive.Compression) (om } // Extract the container's whole filesystem as if it were a single layer. -func (i *containerImageRef) extractRootfs() (io.ReadCloser, chan error, error) { +// Takes ExtractRootfsOptions as argument which allows caller to configure +// preserve nature of setuid,setgid,sticky and extended attributes +// on extracted rootfs. +func (i *containerImageRef) extractRootfs(opts ExtractRootfsOptions) (io.ReadCloser, chan error, error) { var uidMap, gidMap []idtools.IDMap mountPoint, err := i.store.Mount(i.containerID, i.mountLabel) if err != nil { @@ -164,8 +176,11 @@ func (i *containerImageRef) extractRootfs() (io.ReadCloser, chan error, error) { uidMap, gidMap = convertRuntimeIDMaps(i.idMappingOptions.UIDMap, i.idMappingOptions.GIDMap) } copierOptions := copier.GetOptions{ - UIDMap: uidMap, - GIDMap: gidMap, + UIDMap: uidMap, + GIDMap: gidMap, + StripSetuidBit: opts.StripSetuidBit, + StripSetgidBit: opts.StripSetgidBit, + StripXattrs: opts.StripXattrs, } err = copier.Get(mountPoint, mountPoint, copierOptions, []string{"."}, pipeWriter) errChan <- err @@ -376,7 +391,7 @@ func (i *containerImageRef) NewImageSource(ctx context.Context, sc *types.System var errChan chan error if i.squash { // Extract the root filesystem as a single layer. - rc, errChan, err = i.extractRootfs() + rc, errChan, err = i.extractRootfs(ExtractRootfsOptions{}) if err != nil { return nil, err } @@ -738,7 +753,7 @@ func (i *containerImageSource) GetBlob(ctx context.Context, blob types.BlobInfo, return ioutils.NewReadCloserWrapper(layerReadCloser, closer), size, nil } -func (b *Builder) makeImageRef(options CommitOptions) (types.ImageReference, error) { +func (b *Builder) makeContainerImageRef(options CommitOptions) (*containerImageRef, error) { var name reference.Named container, err := b.store.Container(b.ContainerID) if err != nil { @@ -813,3 +828,12 @@ func (b *Builder) makeImageRef(options CommitOptions) (types.ImageReference, err } return ref, nil } + +// Extract the container's whole filesystem as if it were a single layer from current builder instance +func (b *Builder) ExtractRootfs(options CommitOptions, opts ExtractRootfsOptions) (io.ReadCloser, chan error, error) { + src, err := b.makeContainerImageRef(options) + if err != nil { + return nil, nil, errors.Wrapf(err, "error creating image reference for container %q to extract its contents", b.ContainerID) + } + return src.extractRootfs(opts) +} diff --git a/imagebuildah/executor.go b/imagebuildah/executor.go index bbc1def0c..ad98e51ee 100644 --- a/imagebuildah/executor.go +++ b/imagebuildah/executor.go @@ -133,6 +133,7 @@ type Executor struct { unsetEnvs []string processLabel string // Shares processLabel of first stage container with containers of other stages in same build mountLabel string // Shares mountLabel of first stage container with containers of other stages in same build + buildOutput string // Specifies instructions for any custom build output } type imageTypeAndHistoryAndDiffIDs struct { @@ -276,6 +277,7 @@ func newExecutor(logger *logrus.Logger, logPrefix string, store storage.Store, o sshsources: sshsources, logPrefix: logPrefix, unsetEnvs: options.UnsetEnvs, + buildOutput: options.BuildOutput, } if exec.err == nil { exec.err = os.Stderr diff --git a/imagebuildah/stage_executor.go b/imagebuildah/stage_executor.go index 4112a8187..3750edfc6 100644 --- a/imagebuildah/stage_executor.go +++ b/imagebuildah/stage_executor.go @@ -16,6 +16,7 @@ import ( "github.com/containers/buildah/define" buildahdocker "github.com/containers/buildah/docker" "github.com/containers/buildah/internal" + internalUtil "github.com/containers/buildah/internal/util" "github.com/containers/buildah/pkg/parse" "github.com/containers/buildah/pkg/rusage" "github.com/containers/buildah/util" @@ -28,6 +29,7 @@ import ( "github.com/containers/image/v5/types" "github.com/containers/storage" "github.com/containers/storage/pkg/chrootarchive" + "github.com/containers/storage/pkg/unshare" docker "github.com/fsouza/go-dockerclient" digest "github.com/opencontainers/go-digest" v1 "github.com/opencontainers/image-spec/specs-go/v1" @@ -1447,8 +1449,18 @@ func (s *StageExecutor) intermediateImageExists(ctx context.Context, currNode *p // commit writes the container's contents to an image, using a passed-in tag as // the name if there is one, generating a unique ID-based one otherwise. +// or commit via any custom exporter if specified. func (s *StageExecutor) commit(ctx context.Context, createdBy string, emptyLayer bool, output string) (string, reference.Canonical, error) { ib := s.stage.Builder + var buildOutputOption define.BuildOutputOption + if s.executor.buildOutput != "" { + var err error + logrus.Debugf("Generating custom build output with options %q", s.executor.buildOutput) + buildOutputOption, err = parse.GetBuildOutput(s.executor.buildOutput) + if err != nil { + return "", nil, errors.Wrapf(err, "failed to parse build output") + } + } var imageRef types.ImageReference if output != "" { imageRef2, err := s.executor.resolveNameToImageRef(output) @@ -1556,6 +1568,39 @@ func (s *StageExecutor) commit(ctx context.Context, createdBy string, emptyLayer Manifest: s.executor.manifest, UnsetEnvs: s.executor.unsetEnvs, } + // generate build output + if s.executor.buildOutput != "" { + extractRootfsOpts := buildah.ExtractRootfsOptions{} + if unshare.IsRootless() { + // In order to maintain as much parity as possible + // with buildkit's version of --output and to avoid + // unsafe invocation of exported executables it was + // decided to strip setuid,setgid and extended attributes. + // Since modes like setuid,setgid leaves room for executable + // to get invoked with different file-system permission its safer + // to strip them off for unpriviledged invocation. + // See: https://github.com/containers/buildah/pull/3823#discussion_r829376633 + extractRootfsOpts.StripSetuidBit = true + extractRootfsOpts.StripSetgidBit = true + extractRootfsOpts.StripXattrs = true + } + rc, errChan, err := s.builder.ExtractRootfs(options, extractRootfsOpts) + if err != nil { + return "", nil, errors.Wrapf(err, "failed to extract rootfs from given container image") + } + defer rc.Close() + err = internalUtil.ExportFromReader(rc, buildOutputOption) + if err != nil { + return "", nil, errors.Wrapf(err, "failed to export build output") + } + if errChan != nil { + err = <-errChan + if err != nil { + return "", nil, err + } + } + + } imgID, _, manifestDigest, err := s.builder.Commit(ctx, imageRef, options) if err != nil { return "", nil, err diff --git a/internal/util/util.go b/internal/util/util.go index cce508167..691d89d65 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -1,9 +1,18 @@ package util import ( + "io" + "os" + "path/filepath" + + "github.com/containers/buildah/define" "github.com/containers/common/libimage" "github.com/containers/image/v5/types" "github.com/containers/storage" + "github.com/containers/storage/pkg/archive" + "github.com/containers/storage/pkg/chrootarchive" + "github.com/containers/storage/pkg/unshare" + "github.com/pkg/errors" ) // LookupImage returns *Image to corresponding imagename or id @@ -22,3 +31,51 @@ func LookupImage(ctx *types.SystemContext, store storage.Store, image string) (* } return localImage, nil } + +// ExportFromReader reads bytes from given reader and exports to external tar, directory or stdout. +func ExportFromReader(input io.Reader, opts define.BuildOutputOption) error { + var err error + if !filepath.IsAbs(opts.Path) { + opts.Path, err = filepath.Abs(opts.Path) + if err != nil { + return err + } + } + if opts.IsDir { + // In order to keep this feature as close as possible to + // buildkit it was decided to preserve ownership when + // invoked as root since caller already has access to artifacts + // therefore we can preserve ownership as is, however for rootless users + // ownership has to be changed so exported artifacts can still + // be accessible by unpriviledged users. + // See: https://github.com/containers/buildah/pull/3823#discussion_r829376633 + noLChown := false + if unshare.IsRootless() { + noLChown = true + } + + err = os.MkdirAll(opts.Path, 0700) + if err != nil { + return errors.Wrapf(err, "failed while creating the destination path %q", opts.Path) + } + + err = chrootarchive.Untar(input, opts.Path, &archive.TarOptions{NoLchown: noLChown}) + if err != nil { + return errors.Wrapf(err, "failed while performing untar at %q", opts.Path) + } + } else { + outFile := os.Stdout + if !opts.IsStdout { + outFile, err = os.Create(opts.Path) + if err != nil { + return errors.Wrapf(err, "failed while creating destination tar at %q", opts.Path) + } + defer outFile.Close() + } + _, err = io.Copy(outFile, input) + if err != nil { + return errors.Wrapf(err, "failed while performing copy to %q", opts.Path) + } + } + return nil +} diff --git a/pkg/cli/common.go b/pkg/cli/common.go index ba0d7a13e..cc57b1e0f 100644 --- a/pkg/cli/common.go +++ b/pkg/cli/common.go @@ -85,6 +85,7 @@ type BudResults struct { Squash bool Stdin bool Tag []string + BuildOutput string Target string TLSVerify bool Jobs int @@ -242,6 +243,7 @@ func GetBudFlags(flags *BudResults) pflag.FlagSet { fs.StringArrayVar(&flags.SSH, "ssh", []string{}, "SSH agent socket or keys to expose to the build. (format: default|[=|[,]])") fs.BoolVar(&flags.Stdin, "stdin", false, "pass stdin into containers") fs.StringArrayVarP(&flags.Tag, "tag", "t", []string{}, "tagged `name` to apply to the built image") + fs.StringVarP(&flags.BuildOutput, "output", "o", "", "output destination (format: type=local,dest=path)") fs.StringVar(&flags.Target, "target", "", "set the target build stage to build") fs.Int64Var(&flags.Timestamp, "timestamp", 0, "set created timestamp to the specified epoch seconds to allow for deterministic builds, defaults to current time") fs.BoolVar(&flags.TLSVerify, "tls-verify", true, "require HTTPS and verify certificates when accessing the registry") @@ -281,6 +283,7 @@ func GetBudFlagsCompletions() commonComp.FlagCompletions { flagCompletion["timestamp"] = commonComp.AutocompleteNone flagCompletion["variant"] = commonComp.AutocompleteNone flagCompletion["unsetenv"] = commonComp.AutocompleteNone + flagCompletion["output"] = commonComp.AutocompleteNone return flagCompletion } diff --git a/pkg/parse/parse.go b/pkg/parse/parse.go index c5178636e..079863845 100644 --- a/pkg/parse/parse.go +++ b/pkg/parse/parse.go @@ -510,6 +510,73 @@ func AuthConfig(creds string) (*types.DockerAuthConfig, error) { }, nil } +// GetBuildOutput is responsible for parsing custom build output argument i.e `build --output` flag. +// Takes `buildOutput` as string and returns BuildOutputOption +func GetBuildOutput(buildOutput string) (define.BuildOutputOption, error) { + if len(buildOutput) == 1 && buildOutput == "-" { + // Feature parity with buildkit, output tar to stdout + // Read more here: https://docs.docker.com/engine/reference/commandline/build/#custom-build-outputs + return define.BuildOutputOption{Path: "", + IsDir: false, + IsStdout: true}, nil + } + if !strings.Contains(buildOutput, ",") { + // expect default --output + return define.BuildOutputOption{Path: buildOutput, + IsDir: true, + IsStdout: false}, nil + } + isDir := true + isStdout := false + typeSelected := false + pathSelected := false + path := "" + tokens := strings.Split(buildOutput, ",") + for _, option := range tokens { + arr := strings.SplitN(option, "=", 2) + if len(arr) != 2 { + return define.BuildOutputOption{}, fmt.Errorf("invalid build output options %q, expected format key=value", buildOutput) + } + switch arr[0] { + case "type": + if typeSelected { + return define.BuildOutputOption{}, fmt.Errorf("Duplicate %q not supported", arr[0]) + } + typeSelected = true + if arr[1] == "local" { + isDir = true + } else if arr[1] == "tar" { + isDir = false + } else { + return define.BuildOutputOption{}, fmt.Errorf("invalid type %q selected for build output options %q", arr[1], buildOutput) + } + case "dest": + if pathSelected { + return define.BuildOutputOption{}, fmt.Errorf("Duplicate %q not supported", arr[0]) + } + pathSelected = true + path = arr[1] + default: + return define.BuildOutputOption{}, fmt.Errorf("Unrecognized key %q in build output option: %q", arr[0], buildOutput) + } + } + + if !typeSelected || !pathSelected { + return define.BuildOutputOption{}, fmt.Errorf("invalid build output option %q, accepted keys are type and dest must be present", buildOutput) + } + + if path == "-" { + if isDir { + return define.BuildOutputOption{}, fmt.Errorf("invalid build output option %q, type=local and dest=- is not supported", buildOutput) + } + return define.BuildOutputOption{Path: "", + IsDir: false, + IsStdout: true}, nil + } + + return define.BuildOutputOption{Path: path, IsDir: isDir, IsStdout: isStdout}, nil +} + // IDMappingOptions parses the build options related to user namespaces and ID mapping. func IDMappingOptions(c *cobra.Command, isolation define.Isolation) (usernsOptions define.NamespaceOptions, idmapOptions *define.IDMappingOptions, err error) { return IDMappingOptionsFromFlagSet(c.Flags(), c.PersistentFlags(), c.Flag) diff --git a/tests/bud.bats b/tests/bud.bats index 6b79fd3c6..fcddc26f1 100644 --- a/tests/bud.bats +++ b/tests/bud.bats @@ -551,6 +551,67 @@ _EOF expect_output "[container=buildah date=tomorrow]" "No Path should be defined" } +@test "build with custom build output and output rootfs to directory" { + _prefetch alpine + mytmpdir=${TEST_SCRATCH_DIR}/my-dir + mkdir -p $mytmpdir + cat > $mytmpdir/Containerfile << _EOF +FROM alpine +RUN echo 'hello'> hello +_EOF + run_buildah build --output type=local,dest=$mytmpdir/rootfs $WITH_POLICY_JSON -t test-bud -f $mytmpdir/Containerfile . + ls $mytmpdir/rootfs + # exported rootfs must contain `hello` file which we created inside the image + expect_output --substring 'hello' +} + +@test "build with custom build output and output rootfs to tar" { + _prefetch alpine + mytmpdir=${TEST_SCRATCH_DIR}/my-dir + mkdir -p $mytmpdir + cat > $mytmpdir/Containerfile << _EOF +FROM alpine +RUN echo 'hello'> hello +_EOF + run_buildah build --output type=tar,dest=$mytmpdir/rootfs.tar $WITH_POLICY_JSON -t test-bud -f $mytmpdir/Containerfile . + # explode tar + mkdir $mytmpdir/rootfs + tar -C $mytmpdir/rootfs -xvf $mytmpdir/rootfs.tar + ls $mytmpdir/rootfs + # exported rootfs must contain `hello` file which we created inside the image + expect_output --substring 'hello' +} + +@test "build with custom build output and output rootfs to tar by pipe" { + _prefetch alpine + mytmpdir=${TEST_SCRATCH_DIR}/my-dir + mkdir -p $mytmpdir + cat > $mytmpdir/Containerfile << _EOF +FROM alpine +RUN echo 'hello'> hello +_EOF + # Using BUILDAH_BINARY since run_buildah adds unwanted chars to tar created by pipe. + ${BUILDAH_BINARY} build $WITH_POLICY_JSON -o - -t test-bud -f $mytmpdir/Containerfile . > $mytmpdir/rootfs.tar + # explode tar + mkdir $mytmpdir/rootfs + tar -C $mytmpdir/rootfs -xvf $mytmpdir/rootfs.tar + ls $mytmpdir/rootfs/hello +} + +@test "build with custom build output must fail for bad input" { + _prefetch alpine + mytmpdir=${TEST_SCRATCH_DIR}/my-dir + mkdir -p $mytmpdir + cat > $mytmpdir/Containerfile << _EOF +FROM alpine +RUN echo 'hello'> hello +_EOF + run_buildah 125 build --output type=tar, $WITH_POLICY_JSON -t test-bud -f $mytmpdir/Containerfile . + expect_output --substring 'invalid' + run_buildah 125 build --output type=wrong,dest=hello --signature-policy ${TESTSDIR}/policy.json -t test-bud -f $mytmpdir/Containerfile . + expect_output --substring 'invalid' +} + @test "bud-from-scratch-untagged" { run_buildah build --iidfile ${TEST_SCRATCH_DIR}/output.iid $WITH_POLICY_JSON $BUDFILES/from-scratch iid=$(cat ${TEST_SCRATCH_DIR}/output.iid)