1
0
mirror of https://github.com/moby/buildkit.git synced 2025-08-08 10:02:07 +03:00

replace resolveimageconfig with generic sourcemetaresolver

This is more versatile function that works for any source,
not just images.

It can be used together with a policy that switches
between input and output source as well as for adding
additional metadata for other sources in the future.

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
This commit is contained in:
Tonis Tiigi
2024-01-13 17:53:16 -08:00
parent effe19a9a1
commit 30c069cb03
36 changed files with 1712 additions and 497 deletions

View File

@@ -84,6 +84,11 @@ func (g *gatewayClientForBuild) ResolveImageConfig(ctx context.Context, in *gate
return g.gateway.ResolveImageConfig(ctx, in, opts...)
}
func (g *gatewayClientForBuild) ResolveSourceMeta(ctx context.Context, in *gatewayapi.ResolveSourceMetaRequest, opts ...grpc.CallOption) (*gatewayapi.ResolveSourceMetaResponse, error) {
ctx = buildid.AppendToOutgoingContext(ctx, g.buildID)
return g.gateway.ResolveSourceMeta(ctx, in, opts...)
}
func (g *gatewayClientForBuild) Solve(ctx context.Context, in *gatewayapi.SolveRequest, opts ...grpc.CallOption) (*gatewayapi.SolveResponse, error) {
ctx = buildid.AppendToOutgoingContext(ctx, g.buildID)
return g.gateway.Solve(ctx, in, opts...)

View File

@@ -41,6 +41,7 @@ import (
intoto "github.com/in-toto/in-toto-golang/in_toto"
controlapi "github.com/moby/buildkit/api/services/control"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/client/llb/sourceresolver"
"github.com/moby/buildkit/exporter/containerimage/exptypes"
gateway "github.com/moby/buildkit/frontend/gateway/client"
gatewaypb "github.com/moby/buildkit/frontend/gateway/pb"
@@ -3061,7 +3062,7 @@ func testSourceDateEpochClamp(t *testing.T, sb integration.Sandbox) {
var bboxConfig []byte
_, err = c.Build(sb.Context(), SolveOpt{}, "", func(ctx context.Context, c gateway.Client) (*gateway.Result, error) {
_, _, bboxConfig, err = c.ResolveImageConfig(ctx, "docker.io/library/busybox:latest", llb.ResolveImageConfigOpt{})
_, _, bboxConfig, err = c.ResolveImageConfig(ctx, "docker.io/library/busybox:latest", sourceresolver.Opt{})
if err != nil {
return nil, err
}
@@ -10059,7 +10060,7 @@ func testSourcePolicy(t *testing.T, sb integration.Sandbox) {
},
}
ref, dgst, _, err := c.ResolveImageConfig(ctx, origRef, llb.ResolveImageConfigOpt{
ref, dgst, _, err := c.ResolveImageConfig(ctx, origRef, sourceresolver.Opt{
SourcePolicies: pol,
})
if err != nil {

View File

@@ -9,6 +9,7 @@ import (
"github.com/containerd/containerd/remotes"
"github.com/containerd/containerd/remotes/docker"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/client/llb/sourceresolver"
"github.com/moby/buildkit/util/contentutil"
"github.com/moby/buildkit/util/imageutil"
"github.com/moby/buildkit/version"
@@ -70,32 +71,31 @@ type imageMetaResolver struct {
}
type resolveResult struct {
ref string
config []byte
dgst digest.Digest
}
func (imr *imageMetaResolver) ResolveImageConfig(ctx context.Context, ref string, opt llb.ResolveImageConfigOpt) (string, digest.Digest, []byte, error) {
func (imr *imageMetaResolver) ResolveImageConfig(ctx context.Context, ref string, opt sourceresolver.Opt) (string, digest.Digest, []byte, error) {
imr.locker.Lock(ref)
defer imr.locker.Unlock(ref)
platform := opt.Platform
if platform == nil {
platform = imr.platform
platform := imr.platform
if opt.Platform != nil {
platform = opt.Platform
}
k := imr.key(ref, platform)
if res, ok := imr.cache[k]; ok {
return res.ref, res.dgst, res.config, nil
return ref, res.dgst, res.config, nil
}
ref, dgst, config, err := imageutil.Config(ctx, ref, imr.resolver, imr.buffer, nil, platform, opt.SourcePolicies)
dgst, config, err := imageutil.Config(ctx, ref, imr.resolver, imr.buffer, nil, platform)
if err != nil {
return "", "", nil, err
}
imr.cache[k] = resolveResult{dgst: dgst, config: config, ref: ref}
imr.cache[k] = resolveResult{dgst: dgst, config: config}
return ref, dgst, config, nil
}

View File

@@ -1,11 +1,7 @@
package llb
import (
"context"
spb "github.com/moby/buildkit/sourcepolicy/pb"
digest "github.com/opencontainers/go-digest"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/moby/buildkit/client/llb/sourceresolver"
)
// WithMetaResolver adds a metadata resolver to an image
@@ -31,30 +27,4 @@ func WithLayerLimit(l int) ImageOption {
}
// ImageMetaResolver can resolve image config metadata from a reference
type ImageMetaResolver interface {
ResolveImageConfig(ctx context.Context, ref string, opt ResolveImageConfigOpt) (string, digest.Digest, []byte, error)
}
type ResolverType int
const (
ResolverTypeRegistry ResolverType = iota
ResolverTypeOCILayout
)
type ResolveImageConfigOpt struct {
ResolverType
Platform *ocispecs.Platform
ResolveMode string
LogName string
Store ResolveImageConfigOptStore
SourcePolicies []*spb.Policy
}
type ResolveImageConfigOptStore struct {
SessionID string
StoreID string
}
type ImageMetaResolver = sourceresolver.ImageMetaResolver

View File

@@ -6,6 +6,7 @@ import (
"testing"
"github.com/containerd/containerd/platforms"
"github.com/moby/buildkit/client/llb/sourceresolver"
"github.com/moby/buildkit/solver/pb"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
@@ -74,7 +75,7 @@ type testResolver struct {
platform string
}
func (r *testResolver) ResolveImageConfig(ctx context.Context, ref string, opt ResolveImageConfigOpt) (string, digest.Digest, []byte, error) {
func (r *testResolver) ResolveImageConfig(ctx context.Context, ref string, opt sourceresolver.Opt) (string, digest.Digest, []byte, error) {
var img struct {
Config struct {
Env []string `json:"Env,omitempty"`

View File

@@ -10,6 +10,7 @@ import (
"strings"
"github.com/distribution/reference"
"github.com/moby/buildkit/client/llb/sourceresolver"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/util/apicaps"
"github.com/moby/buildkit/util/gitutil"
@@ -136,10 +137,11 @@ func Image(ref string, opts ...ImageOption) State {
if p == nil {
p = c.Platform
}
_, _, dt, err := info.metaResolver.ResolveImageConfig(ctx, ref, ResolveImageConfigOpt{
_, _, dt, err := info.metaResolver.ResolveImageConfig(ctx, ref, sourceresolver.Opt{
Platform: p,
ImageOpt: &sourceresolver.ResolveImageOpt{
ResolveMode: info.resolveMode.String(),
ResolverType: ResolverTypeRegistry,
},
})
if err != nil {
return State{}, err
@@ -152,10 +154,11 @@ func Image(ref string, opts ...ImageOption) State {
if p == nil {
p = c.Platform
}
ref, dgst, dt, err := info.metaResolver.ResolveImageConfig(context.TODO(), ref, ResolveImageConfigOpt{
ref, dgst, dt, err := info.metaResolver.ResolveImageConfig(context.TODO(), ref, sourceresolver.Opt{
Platform: p,
ImageOpt: &sourceresolver.ResolveImageOpt{
ResolveMode: info.resolveMode.String(),
ResolverType: ResolverTypeRegistry,
},
})
if err != nil {
return State{}, err

View File

@@ -0,0 +1,59 @@
package sourceresolver
import (
"context"
"strings"
"github.com/distribution/reference"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/util/imageutil"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
)
type ImageMetaResolver interface {
ResolveImageConfig(ctx context.Context, ref string, opt Opt) (string, digest.Digest, []byte, error)
}
type imageMetaResolver struct {
mr MetaResolver
}
var _ ImageMetaResolver = &imageMetaResolver{}
func NewImageMetaResolver(mr MetaResolver) ImageMetaResolver {
return &imageMetaResolver{
mr: mr,
}
}
func (imr *imageMetaResolver) ResolveImageConfig(ctx context.Context, ref string, opt Opt) (string, digest.Digest, []byte, error) {
parsed, err := reference.ParseNormalizedNamed(ref)
if err != nil {
return "", "", nil, errors.Wrapf(err, "could not parse reference %q", ref)
}
ref = parsed.String()
op := &pb.SourceOp{
Identifier: "docker-image://" + ref,
}
if opt := opt.OCILayoutOpt; opt != nil {
op.Identifier = "oci-layout://" + ref
op.Attrs = map[string]string{}
if opt.Store.SessionID != "" {
op.Attrs[pb.AttrOCILayoutSessionID] = opt.Store.SessionID
}
if opt.Store.StoreID != "" {
op.Attrs[pb.AttrOCILayoutStoreID] = opt.Store.StoreID
}
}
res, err := imr.mr.ResolveSourceMetadata(ctx, op, opt)
if err != nil {
return "", "", nil, errors.Wrapf(err, "failed to resolve source metadata for %s", ref)
}
if res.Image == nil {
return "", "", nil, &imageutil.ResolveToNonImageError{Ref: ref, Updated: res.Op.Identifier}
}
ref = strings.TrimPrefix(res.Op.Identifier, "docker-image://")
ref = strings.TrimPrefix(ref, "oci-layout://")
return ref, res.Image.Digest, res.Image.Config, nil
}

View File

@@ -0,0 +1,54 @@
package sourceresolver
import (
"context"
"github.com/moby/buildkit/solver/pb"
spb "github.com/moby/buildkit/sourcepolicy/pb"
digest "github.com/opencontainers/go-digest"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
)
type ResolverType int
const (
ResolverTypeRegistry ResolverType = iota
ResolverTypeOCILayout
)
type MetaResolver interface {
ResolveSourceMetadata(ctx context.Context, op *pb.SourceOp, opt Opt) (*MetaResponse, error)
}
type Opt struct {
LogName string
SourcePolicies []*spb.Policy
Platform *ocispecs.Platform
ImageOpt *ResolveImageOpt
OCILayoutOpt *ResolveOCILayoutOpt
}
type MetaResponse struct {
Op *pb.SourceOp
Image *ResolveImageResponse
}
type ResolveImageOpt struct {
ResolveMode string
}
type ResolveImageResponse struct {
Digest digest.Digest
Config []byte
}
type ResolveOCILayoutOpt struct {
Store ResolveImageConfigOptStore
}
type ResolveImageConfigOptStore struct {
SessionID string
StoreID string
}

View File

@@ -96,6 +96,15 @@ func (gwf *GatewayForwarder) ResolveImageConfig(ctx context.Context, req *gwapi.
return fwd.ResolveImageConfig(ctx, req)
}
func (gwf *GatewayForwarder) ResolveSourceMeta(ctx context.Context, req *gwapi.ResolveSourceMetaRequest) (*gwapi.ResolveSourceMetaResponse, error) {
fwd, err := gwf.lookupForwarder(ctx)
if err != nil {
return nil, errors.Wrap(err, "forwarding ResolveSourceMeta")
}
return fwd.ResolveSourceMeta(ctx, req)
}
func (gwf *GatewayForwarder) Solve(ctx context.Context, req *gwapi.SolveRequest) (*gwapi.SolveResponse, error) {
fwd, err := gwf.lookupForwarder(ctx)
if err != nil {

View File

@@ -9,6 +9,7 @@ import (
intoto "github.com/in-toto/in-toto-golang/in_toto"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/client/llb/sourceresolver"
gatewaypb "github.com/moby/buildkit/frontend/gateway/pb"
"github.com/moby/buildkit/solver/result"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
@@ -33,12 +34,13 @@ const (
// attestation.
type Scanner func(ctx context.Context, name string, ref llb.State, extras map[string]llb.State, opts ...llb.ConstraintsOpt) (result.Attestation[*llb.State], error)
func CreateSBOMScanner(ctx context.Context, resolver llb.ImageMetaResolver, scanner string, resolveOpt llb.ResolveImageConfigOpt) (Scanner, error) {
func CreateSBOMScanner(ctx context.Context, resolver sourceresolver.MetaResolver, scanner string, resolveOpt sourceresolver.Opt) (Scanner, error) {
if scanner == "" {
return nil, nil
}
scanner, _, dt, err := resolver.ResolveImageConfig(ctx, scanner, resolveOpt)
imr := sourceresolver.NewImageMetaResolver(resolver)
scanner, _, dt, err := imr.ResolveImageConfig(ctx, scanner, resolveOpt)
if err != nil {
return nil, err
}

View File

@@ -7,6 +7,7 @@ import (
"github.com/containerd/containerd/platforms"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/client/llb/sourceresolver"
"github.com/moby/buildkit/exporter/containerimage/image"
"github.com/moby/buildkit/frontend"
"github.com/moby/buildkit/frontend/attestations/sbom"
@@ -101,8 +102,11 @@ func Build(ctx context.Context, c client.Client) (_ *client.Result, err error) {
var scanner sbom.Scanner
if bc.SBOM != nil {
scanner, err = sbom.CreateSBOMScanner(ctx, c, bc.SBOM.Generator, llb.ResolveImageConfigOpt{
// TODO: scanner should pass policy
scanner, err = sbom.CreateSBOMScanner(ctx, c, bc.SBOM.Generator, sourceresolver.Opt{
ImageOpt: &sourceresolver.ResolveImageOpt{
ResolveMode: opts["image-resolve-mode"],
},
})
if err != nil {
return nil, err

View File

@@ -20,6 +20,7 @@ import (
"github.com/docker/go-connections/nat"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/client/llb/imagemetaresolver"
"github.com/moby/buildkit/client/llb/sourceresolver"
"github.com/moby/buildkit/exporter/containerimage/image"
"github.com/moby/buildkit/frontend/dockerfile/instructions"
"github.com/moby/buildkit/frontend/dockerfile/parser"
@@ -423,12 +424,12 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS
prefix += platforms.Format(*platform) + " "
}
prefix += "internal]"
mutRef, dgst, dt, err := metaResolver.ResolveImageConfig(ctx, d.stage.BaseName, llb.ResolveImageConfigOpt{
Platform: platform,
ResolveMode: opt.ImageResolveMode.String(),
mutRef, dgst, dt, err := metaResolver.ResolveImageConfig(ctx, d.stage.BaseName, sourceresolver.Opt{
LogName: fmt.Sprintf("%s load metadata for %s", prefix, d.stage.BaseName),
ResolverType: llb.ResolverTypeRegistry,
SourcePolicies: nil,
Platform: platform,
ImageOpt: &sourceresolver.ResolveImageOpt{
ResolveMode: opt.ImageResolveMode.String(),
},
})
if err != nil {
return suggest.WrapError(errors.Wrap(err, origName), origName, append(allStageNames, commonImageNames()...), true)

View File

@@ -7046,7 +7046,7 @@ func testSourcePolicyWithNamedContext(t *testing.T, sb integration.Sandbox) {
FrontendAttrs: map[string]string{
"context:replace": "docker-image:docker.io/library/alpine:latest",
},
LocalDirs: map[string]string{
LocalMounts: map[string]fsutil.FS{
dockerui.DefaultLocalNameDockerfile: mainContext,
dockerui.DefaultLocalNameContext: mainContext,
"test": replaceContext,

View File

@@ -10,6 +10,7 @@ import (
"github.com/distribution/reference"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/client/llb/sourceresolver"
"github.com/moby/buildkit/exporter/containerimage/exptypes"
"github.com/moby/buildkit/exporter/containerimage/image"
"github.com/moby/buildkit/frontend/gateway/client"
@@ -72,11 +73,12 @@ func (bc *Client) namedContextRecursive(ctx context.Context, name string, nameWi
named = reference.TagNameOnly(named)
ref, dgst, data, err := bc.client.ResolveImageConfig(ctx, named.String(), llb.ResolveImageConfigOpt{
Platform: opt.Platform,
ResolveMode: opt.ResolveMode,
ref, dgst, data, err := bc.client.ResolveImageConfig(ctx, named.String(), sourceresolver.Opt{
LogName: fmt.Sprintf("[context %s] load metadata for %s", nameWithPlatform, ref),
ResolverType: llb.ResolverTypeRegistry,
Platform: opt.Platform,
ImageOpt: &sourceresolver.ResolveImageOpt{
ResolveMode: opt.ResolveMode,
},
})
if err != nil {
e := &imageutil.ResolveToNonImageError{}
@@ -146,16 +148,15 @@ func (bc *Client) namedContextRecursive(ctx context.Context, name string, nameWi
return nil, nil, errors.Wrapf(err, "could not wrap %q with digest", name)
}
// TODO: How should source policy be handled here with a dummy ref?
_, dgst, data, err := bc.client.ResolveImageConfig(ctx, dummyRef.String(), llb.ResolveImageConfigOpt{
Platform: opt.Platform,
ResolveMode: opt.ResolveMode,
_, dgst, data, err := bc.client.ResolveImageConfig(ctx, dummyRef.String(), sourceresolver.Opt{
LogName: fmt.Sprintf("[context %s] load metadata for %s", nameWithPlatform, dummyRef.String()),
ResolverType: llb.ResolverTypeOCILayout,
Store: llb.ResolveImageConfigOptStore{
Platform: opt.Platform,
OCILayoutOpt: &sourceresolver.ResolveOCILayoutOpt{
Store: sourceresolver.ResolveImageConfigOptStore{
SessionID: bc.bopts.SessionID,
StoreID: named.Name(),
},
},
})
if err != nil {
return nil, nil, err

View File

@@ -3,7 +3,7 @@ package frontend
import (
"context"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/client/llb/sourceresolver"
"github.com/moby/buildkit/executor"
gw "github.com/moby/buildkit/frontend/gateway/client"
"github.com/moby/buildkit/session"
@@ -22,8 +22,8 @@ type Frontend interface {
}
type FrontendLLBBridge interface {
sourceresolver.MetaResolver
Solve(ctx context.Context, req SolveRequest, sid string) (*Result, error)
ResolveImageConfig(ctx context.Context, ref string, opt llb.ResolveImageConfigOpt) (string, digest.Digest, []byte, error)
Warn(ctx context.Context, dgst digest.Digest, msg string, opts WarnOpts) error
}

View File

@@ -6,6 +6,7 @@ import (
"syscall"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/client/llb/sourceresolver"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/solver/result"
spb "github.com/moby/buildkit/sourcepolicy/pb"
@@ -26,8 +27,9 @@ func NewResult() *Result {
}
type Client interface {
sourceresolver.MetaResolver
Solve(ctx context.Context, req SolveRequest) (*Result, error)
ResolveImageConfig(ctx context.Context, ref string, opt llb.ResolveImageConfigOpt) (string, digest.Digest, []byte, error)
ResolveImageConfig(ctx context.Context, ref string, opt sourceresolver.Opt) (string, digest.Digest, []byte, error)
BuildOpts() BuildOpts
Inputs(ctx context.Context) (map[string]llb.State, error)
NewContainer(ctx context.Context, req NewContainerRequest) (Container, error)

View File

@@ -6,6 +6,7 @@ import (
cacheutil "github.com/moby/buildkit/cache/util"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/client/llb/sourceresolver"
"github.com/moby/buildkit/executor"
"github.com/moby/buildkit/frontend"
"github.com/moby/buildkit/frontend/gateway/client"
@@ -94,6 +95,12 @@ func (c *BridgeClient) Solve(ctx context.Context, req client.SolveRequest) (*cli
return cRes, nil
}
func (c *BridgeClient) ResolveImageConfig(ctx context.Context, ref string, opt sourceresolver.Opt) (string, digest.Digest, []byte, error) {
imr := sourceresolver.NewImageMetaResolver(c)
return imr.ResolveImageConfig(ctx, ref, opt)
}
func (c *BridgeClient) loadBuildOpts() client.BuildOpts {
wis := c.workers.WorkerInfos()
workers := make([]client.WorkerInfo, len(wis))

View File

@@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"io"
"log"
"net"
"os"
"path/filepath"
@@ -25,6 +26,7 @@ import (
cacheutil "github.com/moby/buildkit/cache/util"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/client/llb/sourceresolver"
"github.com/moby/buildkit/executor"
"github.com/moby/buildkit/exporter/containerimage/exptypes"
"github.com/moby/buildkit/exporter/containerimage/image"
@@ -164,7 +166,8 @@ func (gf *gatewayFrontend) Solve(ctx context.Context, llbBridge frontend.Fronten
return nil, err
}
ref, dgst, config, err := llbBridge.ResolveImageConfig(ctx, reference.TagNameOnly(sourceRef).String(), llb.ResolveImageConfigOpt{})
imr := sourceresolver.NewImageMetaResolver(llbBridge)
ref, dgst, config, err := imr.ResolveImageConfig(ctx, reference.TagNameOnly(sourceRef).String(), sourceresolver.Opt{})
if err != nil {
return nil, err
}
@@ -554,6 +557,49 @@ type llbBridgeForwarder struct {
ctrsMu sync.Mutex
}
func (lbf *llbBridgeForwarder) ResolveSourceMeta(ctx context.Context, req *pb.ResolveSourceMetaRequest) (*pb.ResolveSourceMetaResponse, error) {
if req.Source == nil {
return nil, status.Error(codes.InvalidArgument, "source is required")
}
log.Printf("bridge.ResolveSourceMeta: %v", req.Source)
ctx = tracing.ContextWithSpanFromContext(ctx, lbf.callCtx)
var platform *ocispecs.Platform
if p := req.Platform; p != nil {
platform = &ocispecs.Platform{
OS: p.OS,
Architecture: p.Architecture,
Variant: p.Variant,
OSVersion: p.OSVersion,
OSFeatures: p.OSFeatures,
}
}
resolveopt := sourceresolver.Opt{
LogName: req.LogName,
SourcePolicies: req.SourcePolicies,
Platform: platform,
}
resolveopt.ImageOpt = &sourceresolver.ResolveImageOpt{
ResolveMode: req.ResolveMode,
}
resp, err := lbf.llbBridge.ResolveSourceMetadata(ctx, req.Source, resolveopt)
if err != nil {
return nil, err
}
r := &pb.ResolveSourceMetaResponse{
Source: resp.Op,
}
if resp.Image != nil {
r.Image = &pb.ResolveSourceImageResponse{
Digest: resp.Image.Digest,
Config: resp.Image.Config,
}
}
return r, nil
}
func (lbf *llbBridgeForwarder) ResolveImageConfig(ctx context.Context, req *pb.ResolveImageConfigRequest) (*pb.ResolveImageConfigResponse, error) {
ctx = tracing.ContextWithSpanFromContext(ctx, lbf.callCtx)
var platform *ocispecs.Platform
@@ -566,17 +612,27 @@ func (lbf *llbBridgeForwarder) ResolveImageConfig(ctx context.Context, req *pb.R
OSFeatures: p.OSFeatures,
}
}
ref, dgst, dt, err := lbf.llbBridge.ResolveImageConfig(ctx, req.Ref, llb.ResolveImageConfigOpt{
ResolverType: llb.ResolverType(req.ResolverType),
Platform: platform,
ResolveMode: req.ResolveMode,
log.Printf("bridge.ResolveImageConfig: %v", req.Ref)
imr := sourceresolver.NewImageMetaResolver(lbf.llbBridge)
resolveopt := sourceresolver.Opt{
LogName: req.LogName,
Store: llb.ResolveImageConfigOptStore{
SourcePolicies: req.SourcePolicies,
Platform: platform,
}
if sourceresolver.ResolverType(req.ResolverType) == sourceresolver.ResolverTypeRegistry {
resolveopt.ImageOpt = &sourceresolver.ResolveImageOpt{
ResolveMode: req.ResolveMode,
}
} else if sourceresolver.ResolverType(req.ResolverType) == sourceresolver.ResolverTypeOCILayout {
resolveopt.OCILayoutOpt = &sourceresolver.ResolveOCILayoutOpt{
Store: sourceresolver.ResolveImageConfigOptStore{
SessionID: req.SessionID,
StoreID: req.StoreID,
},
SourcePolicies: req.SourcePolicies,
})
}
}
ref, dgst, dt, err := imr.ResolveImageConfig(ctx, req.Ref, resolveopt)
if err != nil {
return nil, err
}

View File

@@ -12,10 +12,12 @@ import (
"syscall"
"time"
distreference "github.com/distribution/reference"
"github.com/gogo/googleapis/google/rpc"
gogotypes "github.com/gogo/protobuf/types"
"github.com/golang/protobuf/ptypes/any"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/client/llb/sourceresolver"
"github.com/moby/buildkit/frontend/gateway/client"
pb "github.com/moby/buildkit/frontend/gateway/pb"
"github.com/moby/buildkit/identity"
@@ -23,6 +25,7 @@ import (
"github.com/moby/buildkit/util/apicaps"
"github.com/moby/buildkit/util/bklog"
"github.com/moby/buildkit/util/grpcerrors"
"github.com/moby/buildkit/util/imageutil"
"github.com/moby/sys/signal"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
@@ -479,7 +482,11 @@ func (c *grpcClient) Solve(ctx context.Context, creq client.SolveRequest) (res *
return res, nil
}
func (c *grpcClient) ResolveImageConfig(ctx context.Context, ref string, opt llb.ResolveImageConfigOpt) (string, digest.Digest, []byte, error) {
func (c *grpcClient) ResolveSourceMetadata(ctx context.Context, op *opspb.SourceOp, opt sourceresolver.Opt) (*sourceresolver.MetaResponse, error) {
if c.caps.Supports(pb.CapSourceMetaResolver) != nil {
return nil, errors.Errorf("fallback not implemented")
}
var p *opspb.Platform
if platform := opt.Platform; platform != nil {
p = &opspb.Platform{
@@ -491,16 +498,97 @@ func (c *grpcClient) ResolveImageConfig(ctx context.Context, ref string, opt llb
}
}
resp, err := c.client.ResolveImageConfig(ctx, &pb.ResolveImageConfigRequest{
ResolverType: int32(opt.ResolverType),
Ref: ref,
req := &pb.ResolveSourceMetaRequest{
Source: op,
Platform: p,
ResolveMode: opt.ResolveMode,
LogName: opt.LogName,
SessionID: opt.Store.SessionID,
StoreID: opt.Store.StoreID,
SourcePolicies: opt.SourcePolicies,
})
}
resp, err := c.client.ResolveSourceMeta(ctx, req)
if err != nil {
return nil, err
}
r := &sourceresolver.MetaResponse{
Op: resp.Source,
}
if resp.Image != nil {
r.Image = &sourceresolver.ResolveImageResponse{
Digest: resp.Image.Digest,
Config: resp.Image.Config,
}
}
return r, nil
}
func (c *grpcClient) resolveImageConfigViaSourceMetadata(ctx context.Context, ref string, opt sourceresolver.Opt, p *opspb.Platform) (string, digest.Digest, []byte, error) {
op := &opspb.SourceOp{
Identifier: "docker-image://" + ref,
}
if opt.OCILayoutOpt != nil {
named, err := distreference.ParseNormalizedNamed(ref)
if err != nil {
return "", "", nil, err
}
op.Identifier = "oci-layout://" + named.String()
op.Attrs = map[string]string{
opspb.AttrOCILayoutSessionID: opt.OCILayoutOpt.Store.SessionID,
opspb.AttrOCILayoutStoreID: opt.OCILayoutOpt.Store.StoreID,
}
}
req := &pb.ResolveSourceMetaRequest{
Source: op,
Platform: p,
LogName: opt.LogName,
SourcePolicies: opt.SourcePolicies,
}
resp, err := c.client.ResolveSourceMeta(ctx, req)
if err != nil {
return "", "", nil, err
}
if resp.Image == nil {
return "", "", nil, &imageutil.ResolveToNonImageError{Ref: ref, Updated: resp.Source.Identifier}
}
ref = strings.TrimPrefix(resp.Source.Identifier, "docker-image://")
ref = strings.TrimPrefix(ref, "oci-layout://")
return ref, resp.Image.Digest, resp.Image.Config, nil
}
func (c *grpcClient) ResolveImageConfig(ctx context.Context, ref string, opt sourceresolver.Opt) (string, digest.Digest, []byte, error) {
var p *opspb.Platform
if platform := opt.Platform; platform != nil {
p = &opspb.Platform{
OS: platform.OS,
Architecture: platform.Architecture,
Variant: platform.Variant,
OSVersion: platform.OSVersion,
OSFeatures: platform.OSFeatures,
}
}
if c.caps.Supports(pb.CapSourceMetaResolver) == nil {
return c.resolveImageConfigViaSourceMetadata(ctx, ref, opt, p)
}
req := &pb.ResolveImageConfigRequest{
Ref: ref,
LogName: opt.LogName,
SourcePolicies: opt.SourcePolicies,
Platform: p,
}
if iopt := opt.ImageOpt; iopt != nil {
req.ResolveMode = iopt.ResolveMode
req.ResolverType = int32(sourceresolver.ResolverTypeRegistry)
}
if iopt := opt.OCILayoutOpt; iopt != nil {
req.ResolverType = int32(sourceresolver.ResolverTypeOCILayout)
req.StoreID = iopt.Store.StoreID
req.SessionID = iopt.Store.SessionID
}
resp, err := c.client.ResolveImageConfig(ctx, req)
if err != nil {
return "", "", nil, err
}

View File

@@ -68,6 +68,10 @@ const (
// CapAttestations is the capability to indicate that attestation
// references will be attached to results
CapAttestations apicaps.CapID = "reference.attestations"
// CapSourceMetaResolver is the capability to indicates support for ResolveSourceMetadata
// function in gateway API
CapSourceMetaResolver apicaps.CapID = "source.metaresolver"
)
func init() {
@@ -231,4 +235,11 @@ func init() {
Enabled: true,
Status: apicaps.CapStatusExperimental,
})
Caps.Init(apicaps.Cap{
ID: CapSourceMetaResolver,
Name: "source meta resolver",
Enabled: true,
Status: apicaps.CapStatusExperimental,
})
}

File diff suppressed because it is too large Load Diff

View File

@@ -17,6 +17,8 @@ option (gogoproto.unmarshaler_all) = true;
service LLBBridge {
// apicaps:CapResolveImage
rpc ResolveImageConfig(ResolveImageConfigRequest) returns (ResolveImageConfigResponse);
// apicaps:CapSourceMetaResolver
rpc ResolveSourceMeta(ResolveSourceMetaRequest) returns (ResolveSourceMetaResponse);
// apicaps:CapSolveBase
rpc Solve(SolveRequest) returns (SolveResponse);
// apicaps:CapReadFile
@@ -132,6 +134,24 @@ message ResolveImageConfigResponse {
string Ref = 3;
}
message ResolveSourceMetaRequest {
pb.SourceOp Source = 1;
pb.Platform Platform = 2;
string LogName = 3;
string ResolveMode = 4;
repeated moby.buildkit.v1.sourcepolicy.Policy SourcePolicies = 8;
}
message ResolveSourceMetaResponse {
pb.SourceOp Source = 1;
ResolveSourceImageResponse Image = 2;
}
message ResolveSourceImageResponse {
string Digest = 1 [(gogoproto.customtype) = "github.com/opencontainers/go-digest.Digest", (gogoproto.nullable) = false];
bytes Config = 2;
}
message SolveRequest {
pb.Definition Definition = 1;
string Frontend = 2;

View File

@@ -10,7 +10,7 @@ import (
"github.com/mitchellh/hashstructure/v2"
"github.com/moby/buildkit/cache/remotecache"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/client/llb/sourceresolver"
"github.com/moby/buildkit/executor"
resourcestypes "github.com/moby/buildkit/executor/resources/types"
"github.com/moby/buildkit/frontend"
@@ -351,32 +351,44 @@ func (rp *resultProxy) Result(ctx context.Context) (res solver.CachedResult, err
})
}
func (b *llbBridge) ResolveImageConfig(ctx context.Context, ref string, opt llb.ResolveImageConfigOpt) (resolvedRef string, dgst digest.Digest, config []byte, err error) {
func (b *llbBridge) ResolveSourceMetadata(ctx context.Context, op *pb.SourceOp, opt sourceresolver.Opt) (resp *sourceresolver.MetaResponse, err error) {
w, err := b.resolveWorker()
if err != nil {
return "", "", nil, err
return nil, err
}
if opt.LogName == "" {
opt.LogName = fmt.Sprintf("resolve image config for %s", ref)
// TODO: better name
opt.LogName = fmt.Sprintf("resolve image config for %s", op.Identifier)
}
id := ref // make a deterministic ID for avoiding duplicates
if platform := opt.Platform; platform == nil {
id += platforms.Format(platforms.DefaultSpec())
id := op.Identifier
if opt.Platform != nil {
id += platforms.Format(*opt.Platform)
} else {
id += platforms.Format(*platform)
id += platforms.Format(platforms.DefaultSpec())
}
pol, err := loadSourcePolicy(b.builder)
if err != nil {
return "", "", nil, err
return nil, err
}
if pol != nil {
opt.SourcePolicies = append(opt.SourcePolicies, pol)
}
if _, err := sourcepolicy.NewEngine(opt.SourcePolicies).Evaluate(ctx, op); err != nil {
return nil, errors.Wrap(err, "could not resolve image due to policy")
}
// policy is evaluated, so we can remove it from the options
opt.SourcePolicies = nil
err = inBuilderContext(ctx, b.builder, opt.LogName, id, func(ctx context.Context, g session.Group) error {
resolvedRef, dgst, config, err = w.ResolveImageConfig(ctx, ref, opt, b.sm, g)
resp, err = w.ResolveSourceMetadata(ctx, op, opt, b.sm, g)
return err
})
return resolvedRef, dgst, config, err
if err != nil {
return nil, err
}
return resp, nil
}
type lazyCacheManager struct {

View File

@@ -4,6 +4,7 @@ import (
"context"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/client/llb/sourceresolver"
"github.com/moby/buildkit/executor/resources"
"github.com/moby/buildkit/exporter/containerimage/exptypes"
"github.com/moby/buildkit/frontend"
@@ -26,8 +27,10 @@ func SBOMProcessor(scannerRef string, useCache bool, resolveMode string) llbsolv
return nil, err
}
scanner, err := sbom.CreateSBOMScanner(ctx, s.Bridge(j), scannerRef, llb.ResolveImageConfigOpt{
scanner, err := sbom.CreateSBOMScanner(ctx, s.Bridge(j), scannerRef, sourceresolver.Opt{
ImageOpt: &sourceresolver.ResolveImageOpt{
ResolveMode: resolveMode,
},
})
if err != nil {
return nil, err

View File

@@ -3,6 +3,7 @@ package llbsolver
import (
"context"
"fmt"
"log"
"strconv"
"strings"
"sync"
@@ -11,7 +12,7 @@ import (
"github.com/containerd/containerd/platforms"
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/cache/config"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/client/llb/sourceresolver"
"github.com/moby/buildkit/executor/resources"
"github.com/moby/buildkit/exporter/containerimage"
"github.com/moby/buildkit/exporter/containerimage/exptypes"
@@ -133,21 +134,26 @@ func (b *provenanceBridge) findByResult(rp solver.ResultProxy) (*resultWithBridg
return nil, false
}
func (b *provenanceBridge) ResolveImageConfig(ctx context.Context, ref string, opt llb.ResolveImageConfigOpt) (resolvedRef string, dgst digest.Digest, config []byte, err error) {
ref, dgst, config, err = b.llbBridge.ResolveImageConfig(ctx, ref, opt)
func (b *provenanceBridge) ResolveSourceMetadata(ctx context.Context, op *pb.SourceOp, opt sourceresolver.Opt) (*sourceresolver.MetaResponse, error) {
log.Printf("prov.ResolveSourceMetadata: %#v %#v", op, opt)
resp, err := b.llbBridge.ResolveSourceMetadata(ctx, op, opt)
if err != nil {
return "", "", nil, err
return nil, err
}
if img := resp.Image; img != nil {
local := !strings.HasPrefix(resp.Op.Identifier, "docker-image://")
ref := strings.TrimPrefix(resp.Op.Identifier, "docker-image://")
ref = strings.TrimPrefix(ref, "oci-layout://")
b.mu.Lock()
b.images = append(b.images, provenance.ImageSource{
Ref: ref,
Platform: opt.Platform,
Digest: dgst,
Local: opt.ResolverType == llb.ResolverTypeOCILayout,
Digest: img.Digest,
Local: local,
})
b.mu.Unlock()
return ref, dgst, config, nil
}
return resp, nil
}
func (b *provenanceBridge) Solve(ctx context.Context, req frontend.SolveRequest, sid string) (res *frontend.Result, err error) {

View File

@@ -7,5 +7,5 @@ import (
)
type SourcePolicyEvaluator interface {
Evaluate(ctx context.Context, op *pb.Op) (bool, error)
Evaluate(ctx context.Context, op *pb.SourceOp) (bool, error)
}

View File

@@ -253,7 +253,7 @@ func loadLLB(ctx context.Context, def *pb.Definition, polEngine SourcePolicyEval
}
dgst := digest.FromBytes(dt)
if polEngine != nil {
mutated, err := polEngine.Evaluate(ctx, &op)
mutated, err := polEngine.Evaluate(ctx, op.GetSource())
if err != nil {
return solver.Edge{}, errors.Wrap(err, "error evaluating the source policy")
}

View File

@@ -8,7 +8,7 @@ import (
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/reference"
"github.com/containerd/containerd/remotes"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/client/llb/sourceresolver"
"github.com/moby/buildkit/session"
sessioncontent "github.com/moby/buildkit/session/content"
"github.com/moby/buildkit/util/imageutil"
@@ -21,7 +21,7 @@ const (
)
// getOCILayoutResolver gets a resolver to an OCI layout for a specified store from the client using the given session.
func getOCILayoutResolver(store llb.ResolveImageConfigOptStore, sm *session.Manager, g session.Group) *ociLayoutResolver {
func getOCILayoutResolver(store sourceresolver.ResolveImageConfigOptStore, sm *session.Manager, g session.Group) *ociLayoutResolver {
r := &ociLayoutResolver{
store: store,
sm: sm,
@@ -32,7 +32,7 @@ func getOCILayoutResolver(store llb.ResolveImageConfigOptStore, sm *session.Mana
type ociLayoutResolver struct {
remotes.Resolver
store llb.ResolveImageConfigOptStore
store sourceresolver.ResolveImageConfigOptStore
sm *session.Manager
g session.Group
}

View File

@@ -15,7 +15,7 @@ import (
"github.com/containerd/containerd/snapshots"
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/client/llb/sourceresolver"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/solver"
"github.com/moby/buildkit/solver/errdefs"
@@ -45,7 +45,7 @@ type puller struct {
layerLimit *int
vtx solver.Vertex
ResolverType
store llb.ResolveImageConfigOptStore
store sourceresolver.ResolveImageConfigOptStore
g flightcontrol.Group[struct{}]
cacheKeyErr error

View File

@@ -14,7 +14,7 @@ import (
"github.com/containerd/containerd/remotes/docker"
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/client/llb/sourceresolver"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/snapshot"
"github.com/moby/buildkit/solver"
@@ -89,7 +89,7 @@ func (is *Source) Resolve(ctx context.Context, id source.Identifier, sm *session
mode resolver.ResolveMode
recordType client.UsageRecordType
ref reference.Spec
store llb.ResolveImageConfigOptStore
store sourceresolver.ResolveImageConfigOptStore
layerLimit *int
)
switch is.ResolverType {
@@ -116,7 +116,7 @@ func (is *Source) Resolve(ctx context.Context, id source.Identifier, sm *session
platform = *ociIdentifier.Platform
}
mode = resolver.ResolveModeForcePull // with OCI layout, we always just "pull"
store = llb.ResolveImageConfigOptStore{
store = sourceresolver.ResolveImageConfigOptStore{
SessionID: ociIdentifier.SessionID,
StoreID: ociIdentifier.StoreID,
}
@@ -148,44 +148,51 @@ func (is *Source) Resolve(ctx context.Context, id source.Identifier, sm *session
return p, nil
}
func (is *Source) ResolveImageConfig(ctx context.Context, ref string, opt llb.ResolveImageConfigOpt, sm *session.Manager, g session.Group) (string, digest.Digest, []byte, error) {
func (is *Source) ResolveImageConfig(ctx context.Context, ref string, opt sourceresolver.Opt, sm *session.Manager, g session.Group) (digest.Digest, []byte, error) {
key := ref
if platform := opt.Platform; platform != nil {
key += platforms.Format(*platform)
}
var (
rm resolver.ResolveMode
rslvr remotes.Resolver
err error
)
if platform := opt.Platform; platform != nil {
key += platforms.Format(*platform)
}
switch is.ResolverType {
case ResolverTypeRegistry:
rm, err = resolver.ParseImageResolveMode(opt.ResolveMode)
iopt := opt.ImageOpt
if iopt == nil {
return "", nil, errors.Errorf("missing imageopt for resolve")
}
rm, err = resolver.ParseImageResolveMode(iopt.ResolveMode)
if err != nil {
return "", "", nil, err
return "", nil, err
}
rslvr = resolver.DefaultPool.GetResolver(is.RegistryHosts, ref, "pull", sm, g).WithImageStore(is.ImageStore, rm)
case ResolverTypeOCILayout:
iopt := opt.OCILayoutOpt
if iopt == nil {
return "", nil, errors.Errorf("missing ocilayoutopt for resolve")
}
rm = resolver.ResolveModeForcePull
rslvr = getOCILayoutResolver(opt.Store, sm, g)
rslvr = getOCILayoutResolver(iopt.Store, sm, g)
}
key += rm.String()
res, err := is.g.Do(ctx, key, func(ctx context.Context) (*resolveImageResult, error) {
newRef, dgst, dt, err := imageutil.Config(ctx, ref, rslvr, is.ContentStore, is.LeaseManager, opt.Platform, opt.SourcePolicies)
dgst, dt, err := imageutil.Config(ctx, ref, rslvr, is.ContentStore, is.LeaseManager, opt.Platform)
if err != nil {
return nil, err
}
return &resolveImageResult{dgst: dgst, dt: dt, ref: newRef}, nil
return &resolveImageResult{dgst: dgst, dt: dt}, nil
})
if err != nil {
return "", "", nil, err
return "", nil, err
}
return res.ref, res.dgst, res.dt, nil
return res.dgst, res.dt, nil
}
type resolveImageResult struct {
ref string
dgst digest.Digest
dt []byte
}

View File

@@ -61,8 +61,8 @@ func (e *Engine) selectorCache(src *spb.Selector) *selectorCache {
// This function may error out even if the op was mutated, in which case `true` will be returned along with the error.
//
// An error is returned when the source is denied by the policy.
func (e *Engine) Evaluate(ctx context.Context, op *pb.Op) (bool, error) {
if len(e.pol) == 0 {
func (e *Engine) Evaluate(ctx context.Context, op *pb.SourceOp) (bool, error) {
if len(e.pol) == 0 || op == nil {
return false, nil
}
@@ -74,15 +74,13 @@ func (e *Engine) Evaluate(ctx context.Context, op *pb.Op) (bool, error) {
return mutated, errors.Wrapf(ErrTooManyOps, "too many mutations on a single source")
}
srcOp := op.GetSource()
if srcOp == nil {
return false, nil
}
if i == 0 {
ctx = bklog.WithLogger(ctx, bklog.G(ctx).WithField("orig", *srcOp).WithField("updated", op.GetSource()))
ctx = bklog.WithLogger(ctx, bklog.G(ctx).WithField("orig", *op))
} else {
ctx = bklog.WithLogger(ctx, bklog.G(ctx).WithField("updated", *op))
}
mut, err := e.evaluatePolicies(ctx, srcOp)
mut, err := e.evaluatePolicies(ctx, op)
if mut {
mutated = true
}

View File

@@ -53,12 +53,8 @@ func testLastRuleWins(t *testing.T) {
}
e := NewEngine(pol)
mut, err := e.Evaluate(context.Background(), &pb.Op{
Op: &pb.Op_Source{
Source: &pb.SourceOp{
mut, err := e.Evaluate(context.Background(), &pb.SourceOp{
Identifier: "docker-image://docker.io/library/busybox:latest",
},
},
})
require.NoError(t, err)
require.False(t, mut)
@@ -89,12 +85,8 @@ func testMultiplePolicies(t *testing.T) {
}
e := NewEngine(pol)
mut, err := e.Evaluate(context.Background(), &pb.Op{
Op: &pb.Op_Source{
Source: &pb.SourceOp{
mut, err := e.Evaluate(context.Background(), &pb.SourceOp{
Identifier: "docker-image://docker.io/library/busybox:latest",
},
},
})
require.ErrorIs(t, err, ErrSourceDenied)
require.False(t, mut)
@@ -135,12 +127,8 @@ func testConvertMultiple(t *testing.T) {
},
}
op := &pb.Op{
Op: &pb.Op_Source{
Source: &pb.SourceOp{
op := &pb.SourceOp{
Identifier: "docker-image://docker.io/library/busybox:latest",
},
},
}
ctx := context.Background()
@@ -169,12 +157,8 @@ func testConvertWildcard(t *testing.T) {
},
}
op := &pb.Op{
Op: &pb.Op_Source{
Source: &pb.SourceOp{
op := &pb.SourceOp{
Identifier: "docker-image://docker.io/library/golang:1.19",
},
},
}
ctx := context.Background()
@@ -183,7 +167,7 @@ func testConvertWildcard(t *testing.T) {
mutated, err := e.Evaluate(ctx, op)
require.True(t, mutated)
require.NoError(t, err)
require.Equal(t, "docker-image://fakereg.io/library/golang:1.19", op.GetSource().Identifier)
require.Equal(t, "docker-image://fakereg.io/library/golang:1.19", op.Identifier)
}
func testConvertRegex(t *testing.T) {
@@ -202,12 +186,8 @@ func testConvertRegex(t *testing.T) {
},
}
op := &pb.Op{
Op: &pb.Op_Source{
Source: &pb.SourceOp{
op := &pb.SourceOp{
Identifier: "docker-image://docker.io/library/golang:1.19",
},
},
}
ctx := context.Background()
@@ -216,7 +196,7 @@ func testConvertRegex(t *testing.T) {
mutated, err := e.Evaluate(ctx, op)
require.True(t, mutated)
require.NoError(t, err)
require.Equal(t, "docker-image://fakereg.io/library/golang:1.19", op.GetSource().Identifier)
require.Equal(t, "docker-image://fakereg.io/library/golang:1.19", op.Identifier)
}
func testConvertHTTP(t *testing.T) {
@@ -234,12 +214,8 @@ func testConvertHTTP(t *testing.T) {
},
}
op := &pb.Op{
Op: &pb.Op_Source{
Source: &pb.SourceOp{
op := &pb.SourceOp{
Identifier: "https://example.com/foo",
},
},
}
ctx := context.Background()
@@ -248,7 +224,7 @@ func testConvertHTTP(t *testing.T) {
mutated, err := e.Evaluate(ctx, op)
require.True(t, mutated)
require.NoError(t, err)
require.Equal(t, "https://example.com/foo", op.GetSource().Identifier)
require.Equal(t, "https://example.com/foo", op.Identifier)
}
func testConvertLoop(t *testing.T) {
@@ -275,12 +251,8 @@ func testConvertLoop(t *testing.T) {
},
}
op := &pb.Op{
Op: &pb.Op_Source{
Source: &pb.SourceOp{
op := &pb.SourceOp{
Identifier: "docker-image://docker.io/library/busybox:latest",
},
},
}
ctx := context.Background()
@@ -324,12 +296,8 @@ func testAllowConvertDeny(t *testing.T) {
},
}
op := &pb.Op{
Op: &pb.Op_Source{
Source: &pb.SourceOp{
op := &pb.SourceOp{
Identifier: "docker-image://docker.io/library/busybox:latest",
},
},
}
ctx := context.Background()
@@ -338,7 +306,7 @@ func testAllowConvertDeny(t *testing.T) {
mutated, err := e.Evaluate(ctx, op)
require.True(t, mutated)
require.ErrorIs(t, err, ErrSourceDenied)
require.Equal(t, op.GetSource().Identifier, "docker-image://docker.io/library/alpine:latest")
require.Equal(t, op.Identifier, "docker-image://docker.io/library/alpine:latest")
}
func testConvertDeny(t *testing.T) {
@@ -362,12 +330,8 @@ func testConvertDeny(t *testing.T) {
},
}
op := &pb.Op{
Op: &pb.Op_Source{
Source: &pb.SourceOp{
op := &pb.SourceOp{
Identifier: "docker-image://docker.io/library/busybox:latest",
},
},
}
ctx := context.Background()
@@ -376,7 +340,7 @@ func testConvertDeny(t *testing.T) {
mutated, err := e.Evaluate(ctx, op)
require.True(t, mutated)
require.ErrorIs(t, err, ErrSourceDenied)
require.Equal(t, op.GetSource().Identifier, "docker-image://docker.io/library/alpine:latest")
require.Equal(t, op.Identifier, "docker-image://docker.io/library/alpine:latest")
}
func testConvert(t *testing.T) {
@@ -388,12 +352,8 @@ func testConvert(t *testing.T) {
for src, dst := range cases {
t.Run(src+"=>"+dst, func(t *testing.T) {
op := &pb.Op{
Op: &pb.Op_Source{
Source: &pb.SourceOp{
op := &pb.SourceOp{
Identifier: src,
},
},
}
pol := &spb.Policy{
@@ -416,18 +376,14 @@ func testConvert(t *testing.T) {
mutated, err := e.Evaluate(ctx, op)
require.True(t, mutated)
require.NoError(t, err)
require.Equal(t, dst, op.GetSource().Identifier)
require.Equal(t, dst, op.Identifier)
})
}
}
func testAllowDeny(t *testing.T) {
op := &pb.Op{
Op: &pb.Op_Source{
Source: &pb.SourceOp{
op := &pb.SourceOp{
Identifier: "docker-image://docker.io/library/alpine:latest",
},
},
}
pol := &spb.Policy{
Rules: []*spb.Rule{
@@ -453,12 +409,8 @@ func testAllowDeny(t *testing.T) {
require.False(t, mutated)
require.ErrorIs(t, err, ErrSourceDenied)
op = &pb.Op{
Op: &pb.Op_Source{
Source: &pb.SourceOp{
op = &pb.SourceOp{
Identifier: "docker-image://docker.io/library/busybox:latest",
},
},
}
mutated, err = e.Evaluate(ctx, op)
@@ -489,12 +441,8 @@ func testDenyAll(t *testing.T) {
e := NewEngine([]*spb.Policy{pol})
ctx := context.Background()
op := &pb.Op{
Op: &pb.Op_Source{
Source: &pb.SourceOp{
op := &pb.SourceOp{
Identifier: ref,
},
},
}
mutated, err := e.Evaluate(ctx, op)

View File

@@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"fmt"
"strings"
"sync"
"time"
@@ -16,10 +15,7 @@ import (
"github.com/containerd/containerd/remotes"
"github.com/containerd/containerd/remotes/docker"
intoto "github.com/in-toto/in-toto-golang/in_toto"
"github.com/moby/buildkit/solver/pb"
srctypes "github.com/moby/buildkit/source/types"
"github.com/moby/buildkit/sourcepolicy"
spb "github.com/moby/buildkit/sourcepolicy/pb"
"github.com/moby/buildkit/util/contentutil"
"github.com/moby/buildkit/util/leaseutil"
"github.com/moby/buildkit/util/resolver/limited"
@@ -63,8 +59,7 @@ func (e ResolveToNonImageError) Error() string {
return fmt.Sprintf("ref mutated by policy to non-image: %s://%s -> %s", srctypes.DockerImageScheme, e.Ref, e.Updated)
}
func Config(ctx context.Context, str string, resolver remotes.Resolver, cache ContentCache, leaseManager leases.Manager, p *ocispecs.Platform, spls []*spb.Policy) (string, digest.Digest, []byte, error) {
// TODO: fix buildkit to take interface instead of struct
func Config(ctx context.Context, str string, resolver remotes.Resolver, cache ContentCache, leaseManager leases.Manager, p *ocispecs.Platform) (digest.Digest, []byte, error) {
var platform platforms.MatchComparer
if p != nil {
platform = platforms.Only(*p)
@@ -73,46 +68,13 @@ func Config(ctx context.Context, str string, resolver remotes.Resolver, cache Co
}
ref, err := reference.Parse(str)
if err != nil {
return "", "", nil, errors.WithStack(err)
}
op := &pb.Op{
Op: &pb.Op_Source{
Source: &pb.SourceOp{
Identifier: srctypes.DockerImageScheme + "://" + ref.String(),
},
},
}
mut, err := sourcepolicy.NewEngine(spls).Evaluate(ctx, op)
if err != nil {
return "", "", nil, errors.Wrap(err, "could not resolve image due to policy")
}
if mut {
var (
t string
ok bool
)
sid := op.GetSource().GetIdentifier()
t, newRef, ok := strings.Cut(sid, "://")
if !ok {
return "", "", nil, errors.Errorf("could not parse ref: %s", sid)
}
if ok && t != srctypes.DockerImageScheme {
return "", "", nil, &ResolveToNonImageError{Ref: str, Updated: sid}
}
ref, err = reference.Parse(newRef)
if err != nil {
return "", "", nil, errors.WithStack(err)
}
return "", nil, errors.WithStack(err)
}
if leaseManager != nil {
ctx2, done, err := leaseutil.WithLease(ctx, leaseManager, leases.WithExpiration(5*time.Minute), leaseutil.MakeTemporary)
if err != nil {
return "", "", nil, errors.WithStack(err)
return "", nil, errors.WithStack(err)
}
ctx = ctx2
defer func() {
@@ -143,18 +105,18 @@ func Config(ctx context.Context, str string, resolver remotes.Resolver, cache Co
if desc.MediaType == "" {
_, desc, err = resolver.Resolve(ctx, ref.String())
if err != nil {
return "", "", nil, err
return "", nil, err
}
}
fetcher, err := resolver.Fetcher(ctx, ref.String())
if err != nil {
return "", "", nil, err
return "", nil, err
}
if desc.MediaType == images.MediaTypeDockerSchema1Manifest {
dgst, dt, err := readSchema1Config(ctx, ref.String(), desc, fetcher, cache)
return ref.String(), dgst, dt, err
return dgst, dt, err
}
children := childrenConfigHandler(cache, platform)
@@ -162,7 +124,7 @@ func Config(ctx context.Context, str string, resolver remotes.Resolver, cache Co
dslHandler, err := docker.AppendDistributionSourceLabel(cache, ref.String())
if err != nil {
return "", "", nil, err
return "", nil, err
}
handlers := []images.Handler{
@@ -171,19 +133,19 @@ func Config(ctx context.Context, str string, resolver remotes.Resolver, cache Co
children,
}
if err := images.Dispatch(ctx, images.Handlers(handlers...), nil, desc); err != nil {
return "", "", nil, err
return "", nil, err
}
config, err := images.Config(ctx, cache, desc, platform)
if err != nil {
return "", "", nil, err
return "", nil, err
}
dt, err := content.ReadBlob(ctx, cache, config)
if err != nil {
return "", "", nil, err
return "", nil, err
}
return ref.String(), desc.Digest, dt, nil
return desc.Digest, dt, nil
}
func childrenConfigHandler(provider content.Provider, platform platforms.MatchComparer) images.HandlerFunc {

View File

@@ -57,7 +57,7 @@ func TestConfigMultiplatform(t *testing.T) {
// Now we should be able to get the amd64 config without fetching anything from the remote
// If it tries to fetch from the remote this will error out.
const ref = "example.com/test:latest"
_, _, dt, err := Config(ctx, ref, r, cc, nil, &pAmd64, nil)
_, dt, err := Config(ctx, ref, r, cc, nil, &pAmd64)
require.NoError(t, err)
var cfg ocispecs.Image
@@ -67,7 +67,7 @@ func TestConfigMultiplatform(t *testing.T) {
// Make sure it doesn't select a non-matching platform
pArmv7 := platforms.MustParse("linux/arm/v7")
_, _, _, err = Config(ctx, ref, r, cc, nil, &pArmv7, nil)
_, _, err = Config(ctx, ref, r, cc, nil, &pArmv7)
require.ErrorIs(t, err, errdefs.ErrNotFound)
}

View File

@@ -18,7 +18,7 @@ import (
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/cache/metadata"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/client/llb/sourceresolver"
"github.com/moby/buildkit/executor"
"github.com/moby/buildkit/executor/resources"
"github.com/moby/buildkit/exporter"
@@ -365,16 +365,65 @@ func (w *Worker) PruneCacheMounts(ctx context.Context, ids []string) error {
return nil
}
func (w *Worker) ResolveImageConfig(ctx context.Context, ref string, opt llb.ResolveImageConfigOpt, sm *session.Manager, g session.Group) (string, digest.Digest, []byte, error) {
// is this an registry source? Or an OCI layout source?
switch opt.ResolverType {
case llb.ResolverTypeOCILayout:
return w.OCILayoutSource.ResolveImageConfig(ctx, ref, opt, sm, g)
// we probably should put an explicit case llb.ResolverTypeRegistry and default here,
// but then go complains that we do not have a return statement,
// so we just add it after
func (w *Worker) ResolveSourceMetadata(ctx context.Context, op *pb.SourceOp, opt sourceresolver.Opt, sm *session.Manager, g session.Group) (*sourceresolver.MetaResponse, error) {
if opt.SourcePolicies != nil {
return nil, errors.New("source policies can not be set for worker")
}
return w.ImageSource.ResolveImageConfig(ctx, ref, opt, sm, g)
var platform *pb.Platform
if p := opt.Platform; p != nil {
platform = &pb.Platform{
Architecture: p.Architecture,
OS: p.OS,
Variant: p.Variant,
OSVersion: p.OSVersion,
}
}
id, err := w.SourceManager.Identifier(&pb.Op_Source{Source: op}, platform)
if err != nil {
return nil, err
}
switch idt := id.(type) {
case *containerimage.ImageIdentifier:
if opt.ImageOpt == nil {
opt.ImageOpt = &sourceresolver.ResolveImageOpt{}
}
dgst, config, err := w.ImageSource.ResolveImageConfig(ctx, idt.Reference.String(), opt, sm, g)
if err != nil {
return nil, err
}
return &sourceresolver.MetaResponse{
Op: op,
Image: &sourceresolver.ResolveImageResponse{
Digest: dgst,
Config: config,
},
}, nil
case *containerimage.OCIIdentifier:
opt.OCILayoutOpt = &sourceresolver.ResolveOCILayoutOpt{
Store: sourceresolver.ResolveImageConfigOptStore{
StoreID: idt.StoreID,
SessionID: idt.SessionID,
},
}
dgst, config, err := w.OCILayoutSource.ResolveImageConfig(ctx, idt.Reference.String(), opt, sm, g)
if err != nil {
return nil, err
}
return &sourceresolver.MetaResponse{
Op: op,
Image: &sourceresolver.ResolveImageResponse{
Digest: dgst,
Config: config,
},
}, nil
}
return &sourceresolver.MetaResponse{
Op: op,
}, nil
}
func (w *Worker) DiskUsage(ctx context.Context, opt client.DiskUsageInfo) ([]*client.UsageInfo, error) {

View File

@@ -6,15 +6,15 @@ import (
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/client/llb/sourceresolver"
"github.com/moby/buildkit/executor"
"github.com/moby/buildkit/exporter"
"github.com/moby/buildkit/frontend"
"github.com/moby/buildkit/session"
containerdsnapshot "github.com/moby/buildkit/snapshot/containerd"
"github.com/moby/buildkit/solver"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/util/leaseutil"
digest "github.com/opencontainers/go-digest"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
)
@@ -30,7 +30,7 @@ type Worker interface {
LoadRef(ctx context.Context, id string, hidden bool) (cache.ImmutableRef, error)
// ResolveOp resolves Vertex.Sys() to Op implementation.
ResolveOp(v solver.Vertex, s frontend.FrontendLLBBridge, sm *session.Manager) (solver.Op, error)
ResolveImageConfig(ctx context.Context, ref string, opt llb.ResolveImageConfigOpt, sm *session.Manager, g session.Group) (string, digest.Digest, []byte, error)
ResolveSourceMetadata(ctx context.Context, op *pb.SourceOp, opt sourceresolver.Opt, sm *session.Manager, g session.Group) (*sourceresolver.MetaResponse, error)
DiskUsage(ctx context.Context, opt client.DiskUsageInfo) ([]*client.UsageInfo, error)
Exporter(name string, sm *session.Manager) (exporter.Exporter, error)
Prune(ctx context.Context, ch chan client.UsageInfo, opt ...client.PruneInfo) error