diff --git a/solver/llbsolver/ops/source.go b/solver/llbsolver/ops/source.go index fabd300d4..ccaded990 100644 --- a/solver/llbsolver/ops/source.go +++ b/solver/llbsolver/ops/source.go @@ -60,7 +60,7 @@ func (s *SourceOp) instance(ctx context.Context) (source.SourceInstance, error) if s.src != nil { return s.src, nil } - id, err := source.FromLLB(s.op, s.platform) + id, err := s.sm.Identifier(s.op, s.platform) if err != nil { return nil, err } diff --git a/solver/llbsolver/provenance.go b/solver/llbsolver/provenance.go index 9138d6d9f..a466296cb 100644 --- a/solver/llbsolver/provenance.go +++ b/solver/llbsolver/provenance.go @@ -20,7 +20,6 @@ import ( "github.com/moby/buildkit/solver/llbsolver/ops" "github.com/moby/buildkit/solver/llbsolver/provenance" "github.com/moby/buildkit/solver/pb" - "github.com/moby/buildkit/source" "github.com/moby/buildkit/worker" digest "github.com/opencontainers/go-digest" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" @@ -268,70 +267,9 @@ func captureProvenance(ctx context.Context, res solver.CachedResultWithProvenanc switch op := pp.(type) { case *ops.SourceOp: id, pin := op.Pin() - switch s := id.(type) { - case *source.ImageIdentifier: - dgst, err := digest.Parse(pin) - if err != nil { - return errors.Wrapf(err, "failed to parse image digest %s", pin) - } - c.AddImage(provenance.ImageSource{ - Ref: s.Reference.String(), - Platform: s.Platform, - Digest: dgst, - }) - case *source.LocalIdentifier: - c.AddLocal(provenance.LocalSource{ - Name: s.Name, - }) - case *source.GitIdentifier: - url := s.Remote - if s.Ref != "" { - url += "#" + s.Ref - } - c.AddGit(provenance.GitSource{ - URL: url, - Commit: pin, - }) - if s.AuthTokenSecret != "" { - c.AddSecret(provenance.Secret{ - ID: s.AuthTokenSecret, - Optional: true, - }) - } - if s.AuthHeaderSecret != "" { - c.AddSecret(provenance.Secret{ - ID: s.AuthHeaderSecret, - Optional: true, - }) - } - if s.MountSSHSock != "" { - c.AddSSH(provenance.SSH{ - ID: s.MountSSHSock, - Optional: true, - }) - } - case *source.HTTPIdentifier: - dgst, err := digest.Parse(pin) - if err != nil { - return errors.Wrapf(err, "failed to parse HTTP digest %s", pin) - } - c.AddHTTP(provenance.HTTPSource{ - URL: s.URL, - Digest: dgst, - }) - case *source.OCIIdentifier: - dgst, err := digest.Parse(pin) - if err != nil { - return errors.Wrapf(err, "failed to parse OCI digest %s", pin) - } - c.AddImage(provenance.ImageSource{ - Ref: s.Reference.String(), - Platform: s.Platform, - Digest: dgst, - Local: true, - }) - default: - return errors.Errorf("unknown source identifier %T", id) + err := id.Capture(c, pin) + if err != nil { + return err } case *ops.ExecOp: pr := op.Proto() diff --git a/solver/llbsolver/vertex.go b/solver/llbsolver/vertex.go index 41a31bb9b..154cc75dc 100644 --- a/solver/llbsolver/vertex.go +++ b/solver/llbsolver/vertex.go @@ -9,7 +9,6 @@ import ( "github.com/moby/buildkit/solver" "github.com/moby/buildkit/solver/llbsolver/ops/opsutils" "github.com/moby/buildkit/solver/pb" - "github.com/moby/buildkit/source" "github.com/moby/buildkit/util/entitlements" digest "github.com/opencontainers/go-digest" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" @@ -322,13 +321,6 @@ func loadLLB(ctx context.Context, def *pb.Definition, polEngine SourcePolicyEval func llbOpName(pbOp *pb.Op, load func(digest.Digest) (solver.Vertex, error)) (string, error) { switch op := pbOp.Op.(type) { case *pb.Op_Source: - if id, err := source.FromLLB(op, nil); err == nil { - if id, ok := id.(*source.LocalIdentifier); ok { - if len(id.IncludePatterns) == 1 { - return op.Source.Identifier + " (" + id.IncludePatterns[0] + ")", nil - } - } - } return op.Source.Identifier, nil case *pb.Op_Exec: return strings.Join(op.Exec.Meta.Args, " "), nil diff --git a/source/containerimage/identifier.go b/source/containerimage/identifier.go new file mode 100644 index 000000000..08db503a3 --- /dev/null +++ b/source/containerimage/identifier.go @@ -0,0 +1,92 @@ +package containerimage + +import ( + "github.com/containerd/containerd/reference" + "github.com/moby/buildkit/client" + "github.com/moby/buildkit/solver/llbsolver/provenance" + "github.com/moby/buildkit/source" + srctypes "github.com/moby/buildkit/source/types" + "github.com/moby/buildkit/util/resolver" + digest "github.com/opencontainers/go-digest" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" +) + +type ImageIdentifier struct { + Reference reference.Spec + Platform *ocispecs.Platform + ResolveMode resolver.ResolveMode + RecordType client.UsageRecordType + LayerLimit *int +} + +func NewImageIdentifier(str string) (*ImageIdentifier, error) { + ref, err := reference.Parse(str) + if err != nil { + return nil, errors.WithStack(err) + } + + if ref.Object == "" { + return nil, errors.WithStack(reference.ErrObjectRequired) + } + return &ImageIdentifier{Reference: ref}, nil +} + +var _ source.Identifier = (*ImageIdentifier)(nil) + +func (*ImageIdentifier) Scheme() string { + return srctypes.DockerImageScheme +} + +func (id *ImageIdentifier) Capture(c *provenance.Capture, pin string) error { + dgst, err := digest.Parse(pin) + if err != nil { + return errors.Wrapf(err, "failed to parse image digest %s", pin) + } + c.AddImage(provenance.ImageSource{ + Ref: id.Reference.String(), + Platform: id.Platform, + Digest: dgst, + }) + return nil +} + +type OCIIdentifier struct { + Reference reference.Spec + Platform *ocispecs.Platform + SessionID string + StoreID string + LayerLimit *int +} + +func NewOCIIdentifier(str string) (*OCIIdentifier, error) { + ref, err := reference.Parse(str) + if err != nil { + return nil, errors.WithStack(err) + } + + if ref.Object == "" { + return nil, errors.WithStack(reference.ErrObjectRequired) + } + return &OCIIdentifier{Reference: ref}, nil +} + +var _ source.Identifier = (*OCIIdentifier)(nil) + +func (*OCIIdentifier) Scheme() string { + return srctypes.OCIScheme +} + +func (id *OCIIdentifier) Capture(c *provenance.Capture, pin string) error { + dgst, err := digest.Parse(pin) + if err != nil { + return errors.Wrapf(err, "failed to parse OCI digest %s", pin) + } + c.AddImage(provenance.ImageSource{ + Ref: id.Reference.String(), + Platform: id.Platform, + Digest: dgst, + Local: true, + }) + return nil +} diff --git a/source/containerimage/pull.go b/source/containerimage/pull.go index 8792111aa..cdc6dae84 100644 --- a/source/containerimage/pull.go +++ b/source/containerimage/pull.go @@ -7,12 +7,9 @@ import ( "time" "github.com/containerd/containerd/content" - "github.com/containerd/containerd/diff" containerderrdefs "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/images" "github.com/containerd/containerd/leases" - "github.com/containerd/containerd/platforms" - "github.com/containerd/containerd/reference" "github.com/containerd/containerd/remotes" "github.com/containerd/containerd/remotes/docker" "github.com/containerd/containerd/snapshots" @@ -20,11 +17,8 @@ import ( "github.com/moby/buildkit/client" "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/session" - "github.com/moby/buildkit/snapshot" "github.com/moby/buildkit/solver" "github.com/moby/buildkit/solver/errdefs" - "github.com/moby/buildkit/source" - srctypes "github.com/moby/buildkit/source/types" "github.com/moby/buildkit/util/estargz" "github.com/moby/buildkit/util/flightcontrol" "github.com/moby/buildkit/util/imageutil" @@ -39,164 +33,12 @@ import ( "github.com/pkg/errors" ) -// TODO: break apart containerd specifics like contentstore so the resolver -// code can be used with any implementation - -type ResolverType int - -const ( - ResolverTypeRegistry ResolverType = iota - ResolverTypeOCILayout -) - -type SourceOpt struct { - Snapshotter snapshot.Snapshotter - ContentStore content.Store - Applier diff.Applier - CacheAccessor cache.Accessor - ImageStore images.Store // optional - RegistryHosts docker.RegistryHosts - ResolverType - LeaseManager leases.Manager -} - -type resolveImageResult struct { - ref string - dgst digest.Digest - dt []byte -} - -type Source struct { - SourceOpt - g flightcontrol.Group[*resolveImageResult] -} - -var _ source.Source = &Source{} - -func NewSource(opt SourceOpt) (*Source, error) { - is := &Source{ - SourceOpt: opt, - } - - return is, nil -} - -func (is *Source) ID() string { - if is.ResolverType == ResolverTypeOCILayout { - return srctypes.OCIScheme - } - return srctypes.DockerImageScheme -} - -func (is *Source) ResolveImageConfig(ctx context.Context, ref string, opt llb.ResolveImageConfigOpt, sm *session.Manager, g session.Group) (string, digest.Digest, []byte, error) { - key := ref - if platform := opt.Platform; platform != nil { - key += platforms.Format(*platform) - } - var ( - rm source.ResolveMode - rslvr remotes.Resolver - err error - ) - - switch is.ResolverType { - case ResolverTypeRegistry: - rm, err = source.ParseImageResolveMode(opt.ResolveMode) - if err != nil { - return "", "", nil, err - } - rslvr = resolver.DefaultPool.GetResolver(is.RegistryHosts, ref, "pull", sm, g).WithImageStore(is.ImageStore, rm) - case ResolverTypeOCILayout: - rm = source.ResolveModeForcePull - rslvr = getOCILayoutResolver(opt.Store, sm, g) - } - key += rm.String() - res, err := is.g.Do(ctx, key, func(ctx context.Context) (*resolveImageResult, error) { - newRef, dgst, dt, err := imageutil.Config(ctx, ref, rslvr, is.ContentStore, is.LeaseManager, opt.Platform, opt.SourcePolicies) - if err != nil { - return nil, err - } - return &resolveImageResult{dgst: dgst, dt: dt, ref: newRef}, nil - }) - if err != nil { - return "", "", nil, err - } - return res.ref, res.dgst, res.dt, nil -} - -func (is *Source) Resolve(ctx context.Context, id source.Identifier, sm *session.Manager, vtx solver.Vertex) (source.SourceInstance, error) { - var ( - p *puller - platform = platforms.DefaultSpec() - pullerUtil *pull.Puller - mode source.ResolveMode - recordType client.UsageRecordType - ref reference.Spec - store llb.ResolveImageConfigOptStore - layerLimit *int - ) - switch is.ResolverType { - case ResolverTypeRegistry: - imageIdentifier, ok := id.(*source.ImageIdentifier) - if !ok { - return nil, errors.Errorf("invalid image identifier %v", id) - } - - if imageIdentifier.Platform != nil { - platform = *imageIdentifier.Platform - } - mode = imageIdentifier.ResolveMode - recordType = imageIdentifier.RecordType - ref = imageIdentifier.Reference - layerLimit = imageIdentifier.LayerLimit - case ResolverTypeOCILayout: - ociIdentifier, ok := id.(*source.OCIIdentifier) - if !ok { - return nil, errors.Errorf("invalid OCI layout identifier %v", id) - } - - if ociIdentifier.Platform != nil { - platform = *ociIdentifier.Platform - } - mode = source.ResolveModeForcePull // with OCI layout, we always just "pull" - store = llb.ResolveImageConfigOptStore{ - SessionID: ociIdentifier.SessionID, - StoreID: ociIdentifier.StoreID, - } - ref = ociIdentifier.Reference - layerLimit = ociIdentifier.LayerLimit - default: - return nil, errors.Errorf("unknown resolver type: %v", is.ResolverType) - } - pullerUtil = &pull.Puller{ - ContentStore: is.ContentStore, - Platform: platform, - Src: ref, - } - p = &puller{ - CacheAccessor: is.CacheAccessor, - LeaseManager: is.LeaseManager, - Puller: pullerUtil, - RegistryHosts: is.RegistryHosts, - ResolverType: is.ResolverType, - ImageStore: is.ImageStore, - Mode: mode, - RecordType: recordType, - Ref: ref.String(), - SessionManager: sm, - vtx: vtx, - store: store, - layerLimit: layerLimit, - } - return p, nil -} - type puller struct { CacheAccessor cache.Accessor LeaseManager leases.Manager RegistryHosts docker.RegistryHosts ImageStore images.Store - Mode source.ResolveMode + Mode resolver.ResolveMode RecordType client.UsageRecordType Ref string SessionManager *session.Manager diff --git a/source/containerimage/source.go b/source/containerimage/source.go new file mode 100644 index 000000000..eec0be4f2 --- /dev/null +++ b/source/containerimage/source.go @@ -0,0 +1,286 @@ +package containerimage + +import ( + "context" + "strconv" + + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/diff" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/leases" + "github.com/containerd/containerd/platforms" + "github.com/containerd/containerd/reference" + "github.com/containerd/containerd/remotes" + "github.com/containerd/containerd/remotes/docker" + "github.com/moby/buildkit/cache" + "github.com/moby/buildkit/client" + "github.com/moby/buildkit/client/llb" + "github.com/moby/buildkit/session" + "github.com/moby/buildkit/snapshot" + "github.com/moby/buildkit/solver" + "github.com/moby/buildkit/solver/pb" + "github.com/moby/buildkit/source" + srctypes "github.com/moby/buildkit/source/types" + "github.com/moby/buildkit/util/flightcontrol" + "github.com/moby/buildkit/util/imageutil" + "github.com/moby/buildkit/util/pull" + "github.com/moby/buildkit/util/resolver" + digest "github.com/opencontainers/go-digest" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" +) + +// TODO: break apart containerd specifics like contentstore so the resolver +// code can be used with any implementation + +type ResolverType int + +const ( + ResolverTypeRegistry ResolverType = iota + ResolverTypeOCILayout +) + +type SourceOpt struct { + Snapshotter snapshot.Snapshotter + ContentStore content.Store + Applier diff.Applier + CacheAccessor cache.Accessor + ImageStore images.Store // optional + RegistryHosts docker.RegistryHosts + ResolverType + LeaseManager leases.Manager +} + +type Source struct { + SourceOpt + g flightcontrol.Group[*resolveImageResult] +} + +var _ source.Source = &Source{} + +func NewSource(opt SourceOpt) (*Source, error) { + is := &Source{ + SourceOpt: opt, + } + + return is, nil +} + +func (is *Source) Schemes() []string { + if is.ResolverType == ResolverTypeOCILayout { + return []string{srctypes.OCIScheme} + } + return []string{srctypes.DockerImageScheme} +} + +func (is *Source) Identifier(scheme, ref string, attrs map[string]string, platform *pb.Platform) (source.Identifier, error) { + if is.ResolverType == ResolverTypeOCILayout { + return is.ociIdentifier(ref, attrs, platform) + } + + return is.registryIdentifier(ref, attrs, platform) +} + +func (is *Source) Resolve(ctx context.Context, id source.Identifier, sm *session.Manager, vtx solver.Vertex) (source.SourceInstance, error) { + var ( + p *puller + platform = platforms.DefaultSpec() + pullerUtil *pull.Puller + mode resolver.ResolveMode + recordType client.UsageRecordType + ref reference.Spec + store llb.ResolveImageConfigOptStore + layerLimit *int + ) + switch is.ResolverType { + case ResolverTypeRegistry: + imageIdentifier, ok := id.(*ImageIdentifier) + if !ok { + return nil, errors.Errorf("invalid image identifier %v", id) + } + + if imageIdentifier.Platform != nil { + platform = *imageIdentifier.Platform + } + mode = imageIdentifier.ResolveMode + recordType = imageIdentifier.RecordType + ref = imageIdentifier.Reference + layerLimit = imageIdentifier.LayerLimit + case ResolverTypeOCILayout: + ociIdentifier, ok := id.(*OCIIdentifier) + if !ok { + return nil, errors.Errorf("invalid OCI layout identifier %v", id) + } + + if ociIdentifier.Platform != nil { + platform = *ociIdentifier.Platform + } + mode = resolver.ResolveModeForcePull // with OCI layout, we always just "pull" + store = llb.ResolveImageConfigOptStore{ + SessionID: ociIdentifier.SessionID, + StoreID: ociIdentifier.StoreID, + } + ref = ociIdentifier.Reference + layerLimit = ociIdentifier.LayerLimit + default: + return nil, errors.Errorf("unknown resolver type: %v", is.ResolverType) + } + pullerUtil = &pull.Puller{ + ContentStore: is.ContentStore, + Platform: platform, + Src: ref, + } + p = &puller{ + CacheAccessor: is.CacheAccessor, + LeaseManager: is.LeaseManager, + Puller: pullerUtil, + RegistryHosts: is.RegistryHosts, + ResolverType: is.ResolverType, + ImageStore: is.ImageStore, + Mode: mode, + RecordType: recordType, + Ref: ref.String(), + SessionManager: sm, + vtx: vtx, + store: store, + layerLimit: layerLimit, + } + return p, nil +} + +func (is *Source) ResolveImageConfig(ctx context.Context, ref string, opt llb.ResolveImageConfigOpt, sm *session.Manager, g session.Group) (string, digest.Digest, []byte, error) { + key := ref + if platform := opt.Platform; platform != nil { + key += platforms.Format(*platform) + } + var ( + rm resolver.ResolveMode + rslvr remotes.Resolver + err error + ) + + switch is.ResolverType { + case ResolverTypeRegistry: + rm, err = resolver.ParseImageResolveMode(opt.ResolveMode) + if err != nil { + return "", "", nil, err + } + rslvr = resolver.DefaultPool.GetResolver(is.RegistryHosts, ref, "pull", sm, g).WithImageStore(is.ImageStore, rm) + case ResolverTypeOCILayout: + rm = resolver.ResolveModeForcePull + rslvr = getOCILayoutResolver(opt.Store, sm, g) + } + key += rm.String() + res, err := is.g.Do(ctx, key, func(ctx context.Context) (*resolveImageResult, error) { + newRef, dgst, dt, err := imageutil.Config(ctx, ref, rslvr, is.ContentStore, is.LeaseManager, opt.Platform, opt.SourcePolicies) + if err != nil { + return nil, err + } + return &resolveImageResult{dgst: dgst, dt: dt, ref: newRef}, nil + }) + if err != nil { + return "", "", nil, err + } + return res.ref, res.dgst, res.dt, nil +} + +type resolveImageResult struct { + ref string + dgst digest.Digest + dt []byte +} + +func (is *Source) registryIdentifier(ref string, attrs map[string]string, platform *pb.Platform) (source.Identifier, error) { + id, err := NewImageIdentifier(ref) + if err != nil { + return nil, err + } + + if platform != nil { + id.Platform = &ocispecs.Platform{ + OS: platform.OS, + Architecture: platform.Architecture, + Variant: platform.Variant, + OSVersion: platform.OSVersion, + OSFeatures: platform.OSFeatures, + } + } + + for k, v := range attrs { + switch k { + case pb.AttrImageResolveMode: + rm, err := resolver.ParseImageResolveMode(v) + if err != nil { + return nil, err + } + id.ResolveMode = rm + case pb.AttrImageRecordType: + rt, err := parseImageRecordType(v) + if err != nil { + return nil, err + } + id.RecordType = rt + case pb.AttrImageLayerLimit: + l, err := strconv.Atoi(v) + if err != nil { + return nil, errors.Wrapf(err, "invalid layer limit %s", v) + } + if l <= 0 { + return nil, errors.Errorf("invalid layer limit %s", v) + } + id.LayerLimit = &l + } + } + + return id, nil +} + +func (is *Source) ociIdentifier(ref string, attrs map[string]string, platform *pb.Platform) (source.Identifier, error) { + id, err := NewOCIIdentifier(ref) + if err != nil { + return nil, err + } + + if platform != nil { + id.Platform = &ocispecs.Platform{ + OS: platform.OS, + Architecture: platform.Architecture, + Variant: platform.Variant, + OSVersion: platform.OSVersion, + OSFeatures: platform.OSFeatures, + } + } + + for k, v := range attrs { + switch k { + case pb.AttrOCILayoutSessionID: + id.SessionID = v + case pb.AttrOCILayoutStoreID: + id.StoreID = v + case pb.AttrOCILayoutLayerLimit: + l, err := strconv.Atoi(v) + if err != nil { + return nil, errors.Wrapf(err, "invalid layer limit %s", v) + } + if l <= 0 { + return nil, errors.Errorf("invalid layer limit %s", v) + } + id.LayerLimit = &l + } + } + + return id, nil +} + +func parseImageRecordType(v string) (client.UsageRecordType, error) { + switch client.UsageRecordType(v) { + case "", client.UsageRecordTypeRegular: + return client.UsageRecordTypeRegular, nil + case client.UsageRecordTypeInternal: + return client.UsageRecordTypeInternal, nil + case client.UsageRecordTypeFrontend: + return client.UsageRecordTypeFrontend, nil + default: + return "", errors.Errorf("invalid record type %s", v) + } +} diff --git a/source/gitidentifier.go b/source/git/identifier.go similarity index 70% rename from source/gitidentifier.go rename to source/git/identifier.go index 6055c94e2..ecfa35272 100644 --- a/source/gitidentifier.go +++ b/source/git/identifier.go @@ -1,10 +1,12 @@ -package source +package git import ( "net/url" "path" "strings" + "github.com/moby/buildkit/solver/llbsolver/provenance" + "github.com/moby/buildkit/source" srctypes "github.com/moby/buildkit/source/types" "github.com/moby/buildkit/util/sshutil" ) @@ -53,10 +55,42 @@ func NewGitIdentifier(remoteURL string) (*GitIdentifier, error) { return &repo, nil } -func (i *GitIdentifier) ID() string { +func (GitIdentifier) Scheme() string { return srctypes.GitScheme } +var _ source.Identifier = (*GitIdentifier)(nil) + +func (id *GitIdentifier) Capture(c *provenance.Capture, pin string) error { + url := id.Remote + if id.Ref != "" { + url += "#" + id.Ref + } + c.AddGit(provenance.GitSource{ + URL: url, + Commit: pin, + }) + if id.AuthTokenSecret != "" { + c.AddSecret(provenance.Secret{ + ID: id.AuthTokenSecret, + Optional: true, + }) + } + if id.AuthHeaderSecret != "" { + c.AddSecret(provenance.Secret{ + ID: id.AuthHeaderSecret, + Optional: true, + }) + } + if id.MountSSHSock != "" { + c.AddSSH(provenance.SSH{ + ID: id.MountSSHSock, + Optional: true, + }) + } + return nil +} + // isGitTransport returns true if the provided str is a git transport by inspecting // the prefix of the string for known protocols used in git. func isGitTransport(str string) bool { diff --git a/source/gitidentifier_test.go b/source/git/identifier_test.go similarity index 99% rename from source/gitidentifier_test.go rename to source/git/identifier_test.go index 0d7e8be9b..6f9bc7e55 100644 --- a/source/gitidentifier_test.go +++ b/source/git/identifier_test.go @@ -1,4 +1,4 @@ -package source +package git import ( "testing" diff --git a/source/git/gitsource.go b/source/git/source.go similarity index 96% rename from source/git/gitsource.go rename to source/git/source.go index fdc1b5002..c1f939d6b 100644 --- a/source/git/gitsource.go +++ b/source/git/source.go @@ -24,6 +24,7 @@ import ( "github.com/moby/buildkit/session/sshforward" "github.com/moby/buildkit/snapshot" "github.com/moby/buildkit/solver" + "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/source" srctypes "github.com/moby/buildkit/source/types" "github.com/moby/buildkit/util/bklog" @@ -63,8 +64,39 @@ func NewSource(opt Opt) (source.Source, error) { return gs, nil } -func (gs *gitSource) ID() string { - return srctypes.GitScheme +func (gs *gitSource) Schemes() []string { + return []string{srctypes.GitScheme} +} + +func (gs *gitSource) Identifier(scheme, ref string, attrs map[string]string, platform *pb.Platform) (source.Identifier, error) { + id, err := NewGitIdentifier(ref) + if err != nil { + return nil, err + } + + for k, v := range attrs { + switch k { + case pb.AttrKeepGitDir: + if v == "true" { + id.KeepGitDir = true + } + case pb.AttrFullRemoteURL: + if !isGitTransport(v) { + v = "https://" + v + } + id.Remote = v + case pb.AttrAuthHeaderSecret: + id.AuthHeaderSecret = v + case pb.AttrAuthTokenSecret: + id.AuthTokenSecret = v + case pb.AttrKnownSSHHosts: + id.KnownSSHHosts = v + case pb.AttrMountSSHSock: + id.MountSSHSock = v + } + } + + return id, nil } // needs to be called with repo lock @@ -151,7 +183,7 @@ func (gs *gitSource) mountRemote(ctx context.Context, remote string, auth []stri type gitSourceHandler struct { *gitSource - src source.GitIdentifier + src GitIdentifier cacheKey string sm *session.Manager auth []string @@ -169,7 +201,7 @@ func (gs *gitSourceHandler) shaToCacheKey(sha string) string { } func (gs *gitSource) Resolve(ctx context.Context, id source.Identifier, sm *session.Manager, _ solver.Vertex) (source.SourceInstance, error) { - gitIdentifier, ok := id.(*source.GitIdentifier) + gitIdentifier, ok := id.(*GitIdentifier) if !ok { return nil, errors.Errorf("invalid git identifier %v", id) } diff --git a/source/git/gitsource_test.go b/source/git/source_test.go similarity index 96% rename from source/git/gitsource_test.go rename to source/git/source_test.go index 28b2a54bb..a90cb7c88 100644 --- a/source/git/gitsource_test.go +++ b/source/git/source_test.go @@ -54,7 +54,7 @@ func testRepeatedFetch(t *testing.T, keepGitDir bool) { repo := setupGitRepo(t) - id := &source.GitIdentifier{Remote: repo.mainURL, KeepGitDir: keepGitDir} + id := &GitIdentifier{Remote: repo.mainURL, KeepGitDir: keepGitDir} g, err := gs.Resolve(ctx, id, nil, nil) require.NoError(t, err) @@ -95,7 +95,7 @@ func testRepeatedFetch(t *testing.T, keepGitDir bool) { require.ErrorAs(t, err, &os.ErrNotExist) // second fetch returns same dir - id = &source.GitIdentifier{Remote: repo.mainURL, Ref: "master", KeepGitDir: keepGitDir} + id = &GitIdentifier{Remote: repo.mainURL, Ref: "master", KeepGitDir: keepGitDir} g, err = gs.Resolve(ctx, id, nil, nil) require.NoError(t, err) @@ -112,7 +112,7 @@ func testRepeatedFetch(t *testing.T, keepGitDir bool) { require.Equal(t, ref1.ID(), ref2.ID()) - id = &source.GitIdentifier{Remote: repo.mainURL, Ref: "feature", KeepGitDir: keepGitDir} + id = &GitIdentifier{Remote: repo.mainURL, Ref: "feature", KeepGitDir: keepGitDir} g, err = gs.Resolve(ctx, id, nil, nil) require.NoError(t, err) @@ -174,7 +174,7 @@ func testFetchBySHA(t *testing.T, keepGitDir bool) { sha := strings.TrimSpace(string(out)) require.Equal(t, 40, len(sha)) - id := &source.GitIdentifier{Remote: repo.mainURL, Ref: sha, KeepGitDir: keepGitDir} + id := &GitIdentifier{Remote: repo.mainURL, Ref: sha, KeepGitDir: keepGitDir} g, err := gs.Resolve(ctx, id, nil, nil) require.NoError(t, err) @@ -243,7 +243,7 @@ func testFetchByTag(t *testing.T, tag, expectedCommitSubject string, isAnnotated repo := setupGitRepo(t) - id := &source.GitIdentifier{Remote: repo.mainURL, Ref: tag, KeepGitDir: keepGitDir} + id := &GitIdentifier{Remote: repo.mainURL, Ref: tag, KeepGitDir: keepGitDir} g, err := gs.Resolve(ctx, id, nil, nil) require.NoError(t, err) @@ -341,8 +341,8 @@ func testMultipleRepos(t *testing.T, keepGitDir bool) { ) repoURL2 := serveGitRepo(t, repodir2) - id := &source.GitIdentifier{Remote: repo.mainURL, KeepGitDir: keepGitDir} - id2 := &source.GitIdentifier{Remote: repoURL2, KeepGitDir: keepGitDir} + id := &GitIdentifier{Remote: repo.mainURL, KeepGitDir: keepGitDir} + id2 := &GitIdentifier{Remote: repoURL2, KeepGitDir: keepGitDir} g, err := gs.Resolve(ctx, id, nil, nil) require.NoError(t, err) @@ -415,7 +415,7 @@ func TestCredentialRedaction(t *testing.T) { gs := setupGitSource(t, t.TempDir()) url := "https://user:keepthissecret@non-existant-host/user/private-repo.git" - id := &source.GitIdentifier{Remote: url} + id := &GitIdentifier{Remote: url} g, err := gs.Resolve(ctx, id, nil, nil) require.NoError(t, err) @@ -457,7 +457,7 @@ func testSubdir(t *testing.T, keepGitDir bool) { ) repoURL := serveGitRepo(t, repodir) - id := &source.GitIdentifier{Remote: repoURL, KeepGitDir: keepGitDir, Subdir: "sub"} + id := &GitIdentifier{Remote: repoURL, KeepGitDir: keepGitDir, Subdir: "sub"} g, err := gs.Resolve(ctx, id, nil, nil) require.NoError(t, err) diff --git a/source/git/gitsource_unix.go b/source/git/source_unix.go similarity index 100% rename from source/git/gitsource_unix.go rename to source/git/source_unix.go diff --git a/source/git/gitsource_windows.go b/source/git/source_windows.go similarity index 100% rename from source/git/gitsource_windows.go rename to source/git/source_windows.go diff --git a/source/http/identifier.go b/source/http/identifier.go new file mode 100644 index 000000000..f56032104 --- /dev/null +++ b/source/http/identifier.go @@ -0,0 +1,48 @@ +package http + +import ( + "github.com/moby/buildkit/solver/llbsolver/provenance" + "github.com/moby/buildkit/source" + srctypes "github.com/moby/buildkit/source/types" + digest "github.com/opencontainers/go-digest" + "github.com/pkg/errors" +) + +func NewHTTPIdentifier(str string, tls bool) (*HTTPIdentifier, error) { + proto := "https://" + if !tls { + proto = "http://" + } + return &HTTPIdentifier{TLS: tls, URL: proto + str}, nil +} + +type HTTPIdentifier struct { + TLS bool + URL string + Checksum digest.Digest + Filename string + Perm int + UID int + GID int +} + +var _ source.Identifier = (*HTTPIdentifier)(nil) + +func (id *HTTPIdentifier) Scheme() string { + if id.TLS { + return srctypes.HTTPSScheme + } + return srctypes.HTTPScheme +} + +func (id *HTTPIdentifier) Capture(c *provenance.Capture, pin string) error { + dgst, err := digest.Parse(pin) + if err != nil { + return errors.Wrapf(err, "failed to parse HTTP digest %s", pin) + } + c.AddHTTP(provenance.HTTPSource{ + URL: id.URL, + Digest: dgst, + }) + return nil +} diff --git a/source/http/httpsource.go b/source/http/source.go similarity index 91% rename from source/http/httpsource.go rename to source/http/source.go index 9fde3cdee..f2db01c8d 100644 --- a/source/http/httpsource.go +++ b/source/http/source.go @@ -12,6 +12,7 @@ import ( "os" "path" "path/filepath" + "strconv" "strings" "time" @@ -20,6 +21,7 @@ import ( "github.com/moby/buildkit/session" "github.com/moby/buildkit/snapshot" "github.com/moby/buildkit/solver" + "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/source" srctypes "github.com/moby/buildkit/source/types" "github.com/moby/buildkit/util/tracing" @@ -52,20 +54,60 @@ func NewSource(opt Opt) (source.Source, error) { return hs, nil } -func (hs *httpSource) ID() string { - return srctypes.HTTPSScheme +func (hs *httpSource) Schemes() []string { + return []string{srctypes.HTTPScheme, srctypes.HTTPSScheme} +} + +func (hs *httpSource) Identifier(scheme, ref string, attrs map[string]string, platform *pb.Platform) (source.Identifier, error) { + id, err := NewHTTPIdentifier(ref, scheme == "https") + if err != nil { + return nil, err + } + + for k, v := range attrs { + switch k { + case pb.AttrHTTPChecksum: + dgst, err := digest.Parse(v) + if err != nil { + return nil, err + } + id.Checksum = dgst + case pb.AttrHTTPFilename: + id.Filename = v + case pb.AttrHTTPPerm: + i, err := strconv.ParseInt(v, 0, 64) + if err != nil { + return nil, err + } + id.Perm = int(i) + case pb.AttrHTTPUID: + i, err := strconv.ParseInt(v, 0, 64) + if err != nil { + return nil, err + } + id.UID = int(i) + case pb.AttrHTTPGID: + i, err := strconv.ParseInt(v, 0, 64) + if err != nil { + return nil, err + } + id.GID = int(i) + } + } + + return id, nil } type httpSourceHandler struct { *httpSource - src source.HTTPIdentifier + src HTTPIdentifier refID string cacheKey digest.Digest sm *session.Manager } func (hs *httpSource) Resolve(ctx context.Context, id source.Identifier, sm *session.Manager, _ solver.Vertex) (source.SourceInstance, error) { - httpIdentifier, ok := id.(*source.HTTPIdentifier) + httpIdentifier, ok := id.(*HTTPIdentifier) if !ok { return nil, errors.Errorf("invalid http identifier %v", id) } diff --git a/source/http/httpsource_test.go b/source/http/source_test.go similarity index 96% rename from source/http/httpsource_test.go rename to source/http/source_test.go index afe636257..8b8e429b6 100644 --- a/source/http/httpsource_test.go +++ b/source/http/source_test.go @@ -47,7 +47,7 @@ func TestHTTPSource(t *testing.T) { }) defer server.Close() - id := &source.HTTPIdentifier{URL: server.URL + "/foo"} + id := &HTTPIdentifier{URL: server.URL + "/foo"} h, err := hs.Resolve(ctx, id, nil, nil) require.NoError(t, err) @@ -166,7 +166,7 @@ func TestHTTPDefaultName(t *testing.T) { }) defer server.Close() - id := &source.HTTPIdentifier{URL: server.URL} + id := &HTTPIdentifier{URL: server.URL} h, err := hs.Resolve(ctx, id, nil, nil) require.NoError(t, err) @@ -206,7 +206,7 @@ func TestHTTPInvalidURL(t *testing.T) { server := httpserver.NewTestServer(map[string]httpserver.Response{}) defer server.Close() - id := &source.HTTPIdentifier{URL: server.URL + "/foo"} + id := &HTTPIdentifier{URL: server.URL + "/foo"} h, err := hs.Resolve(ctx, id, nil, nil) require.NoError(t, err) @@ -236,7 +236,7 @@ func TestHTTPChecksum(t *testing.T) { }) defer server.Close() - id := &source.HTTPIdentifier{URL: server.URL + "/foo", Checksum: digest.FromBytes([]byte("content-different"))} + id := &HTTPIdentifier{URL: server.URL + "/foo", Checksum: digest.FromBytes([]byte("content-different"))} h, err := hs.Resolve(ctx, id, nil, nil) require.NoError(t, err) @@ -262,7 +262,7 @@ func TestHTTPChecksum(t *testing.T) { require.Equal(t, server.Stats("/foo").AllRequests, 1) require.Equal(t, server.Stats("/foo").CachedRequests, 0) - id = &source.HTTPIdentifier{URL: server.URL + "/foo", Checksum: digest.FromBytes([]byte("content-correct"))} + id = &HTTPIdentifier{URL: server.URL + "/foo", Checksum: digest.FromBytes([]byte("content-correct"))} h, err = hs.Resolve(ctx, id, nil, nil) require.NoError(t, err) diff --git a/source/identifier.go b/source/identifier.go index aad9f226f..f591e9e34 100644 --- a/source/identifier.go +++ b/source/identifier.go @@ -1,18 +1,8 @@ package source import ( - "encoding/json" - "strconv" - "strings" - - "github.com/containerd/containerd/reference" - "github.com/moby/buildkit/client" - "github.com/moby/buildkit/solver/pb" - srctypes "github.com/moby/buildkit/source/types" - digest "github.com/opencontainers/go-digest" - ocispecs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/moby/buildkit/solver/llbsolver/provenance" "github.com/pkg/errors" - "github.com/tonistiigi/fsutil" ) var ( @@ -20,333 +10,11 @@ var ( errNotFound = errors.New("not found") ) -type ResolveMode int - -const ( - ResolveModeDefault ResolveMode = iota - ResolveModeForcePull - ResolveModePreferLocal -) - type Identifier interface { - ID() string // until sources are in process this string comparison could be avoided -} - -func FromString(s string) (Identifier, error) { - // TODO: improve this - parts := strings.SplitN(s, "://", 2) - if len(parts) != 2 { - return nil, errors.Wrapf(errInvalid, "failed to parse %s", s) - } - - switch parts[0] { - case srctypes.DockerImageScheme: - return NewImageIdentifier(parts[1]) - case srctypes.GitScheme: - return NewGitIdentifier(parts[1]) - case srctypes.LocalScheme: - return NewLocalIdentifier(parts[1]) - case srctypes.HTTPSScheme: - return NewHTTPIdentifier(parts[1], true) - case srctypes.HTTPScheme: - return NewHTTPIdentifier(parts[1], false) - case srctypes.OCIScheme: - return NewOCIIdentifier(parts[1]) - default: - return nil, errors.Wrapf(errNotFound, "unknown schema %s", parts[0]) - } -} - -func FromLLB(op *pb.Op_Source, platform *pb.Platform) (Identifier, error) { - id, err := FromString(op.Source.Identifier) - if err != nil { - return nil, err - } - - if id, ok := id.(*ImageIdentifier); ok { - if platform != nil { - id.Platform = &ocispecs.Platform{ - OS: platform.OS, - Architecture: platform.Architecture, - Variant: platform.Variant, - OSVersion: platform.OSVersion, - OSFeatures: platform.OSFeatures, - } - } - for k, v := range op.Source.Attrs { - switch k { - case pb.AttrImageResolveMode: - rm, err := ParseImageResolveMode(v) - if err != nil { - return nil, err - } - id.ResolveMode = rm - case pb.AttrImageRecordType: - rt, err := parseImageRecordType(v) - if err != nil { - return nil, err - } - id.RecordType = rt - case pb.AttrImageLayerLimit: - l, err := strconv.Atoi(v) - if err != nil { - return nil, errors.Wrapf(err, "invalid layer limit %s", v) - } - if l <= 0 { - return nil, errors.Errorf("invalid layer limit %s", v) - } - id.LayerLimit = &l - } - } - } - if id, ok := id.(*GitIdentifier); ok { - for k, v := range op.Source.Attrs { - switch k { - case pb.AttrKeepGitDir: - if v == "true" { - id.KeepGitDir = true - } - case pb.AttrFullRemoteURL: - if !isGitTransport(v) { - v = "https://" + v - } - id.Remote = v - case pb.AttrAuthHeaderSecret: - id.AuthHeaderSecret = v - case pb.AttrAuthTokenSecret: - id.AuthTokenSecret = v - case pb.AttrKnownSSHHosts: - id.KnownSSHHosts = v - case pb.AttrMountSSHSock: - id.MountSSHSock = v - } - } - } - if id, ok := id.(*LocalIdentifier); ok { - for k, v := range op.Source.Attrs { - switch k { - case pb.AttrLocalSessionID: - id.SessionID = v - if p := strings.SplitN(v, ":", 2); len(p) == 2 { - id.Name = p[0] + "-" + id.Name - id.SessionID = p[1] - } - case pb.AttrIncludePatterns: - var patterns []string - if err := json.Unmarshal([]byte(v), &patterns); err != nil { - return nil, err - } - id.IncludePatterns = patterns - case pb.AttrExcludePatterns: - var patterns []string - if err := json.Unmarshal([]byte(v), &patterns); err != nil { - return nil, err - } - id.ExcludePatterns = patterns - case pb.AttrFollowPaths: - var paths []string - if err := json.Unmarshal([]byte(v), &paths); err != nil { - return nil, err - } - id.FollowPaths = paths - case pb.AttrSharedKeyHint: - id.SharedKeyHint = v - case pb.AttrLocalDiffer: - switch v { - case pb.AttrLocalDifferMetadata, "": - id.Differ = fsutil.DiffMetadata - case pb.AttrLocalDifferNone: - id.Differ = fsutil.DiffNone - } - } - } - } - if id, ok := id.(*HTTPIdentifier); ok { - for k, v := range op.Source.Attrs { - switch k { - case pb.AttrHTTPChecksum: - dgst, err := digest.Parse(v) - if err != nil { - return nil, err - } - id.Checksum = dgst - case pb.AttrHTTPFilename: - id.Filename = v - case pb.AttrHTTPPerm: - i, err := strconv.ParseInt(v, 0, 64) - if err != nil { - return nil, err - } - id.Perm = int(i) - case pb.AttrHTTPUID: - i, err := strconv.ParseInt(v, 0, 64) - if err != nil { - return nil, err - } - id.UID = int(i) - case pb.AttrHTTPGID: - i, err := strconv.ParseInt(v, 0, 64) - if err != nil { - return nil, err - } - id.GID = int(i) - } - } - } - if id, ok := id.(*OCIIdentifier); ok { - if platform != nil { - id.Platform = &ocispecs.Platform{ - OS: platform.OS, - Architecture: platform.Architecture, - Variant: platform.Variant, - OSVersion: platform.OSVersion, - OSFeatures: platform.OSFeatures, - } - } - for k, v := range op.Source.Attrs { - switch k { - case pb.AttrOCILayoutSessionID: - id.SessionID = v - case pb.AttrOCILayoutStoreID: - id.StoreID = v - case pb.AttrOCILayoutLayerLimit: - l, err := strconv.Atoi(v) - if err != nil { - return nil, errors.Wrapf(err, "invalid layer limit %s", v) - } - if l <= 0 { - return nil, errors.Errorf("invalid layer limit %s", v) - } - id.LayerLimit = &l - } - } - } - return id, nil -} - -type ImageIdentifier struct { - Reference reference.Spec - Platform *ocispecs.Platform - ResolveMode ResolveMode - RecordType client.UsageRecordType - LayerLimit *int -} - -func NewImageIdentifier(str string) (*ImageIdentifier, error) { - ref, err := reference.Parse(str) - if err != nil { - return nil, errors.WithStack(err) - } - - if ref.Object == "" { - return nil, errors.WithStack(reference.ErrObjectRequired) - } - return &ImageIdentifier{Reference: ref}, nil -} - -func (*ImageIdentifier) ID() string { - return srctypes.DockerImageScheme -} - -type LocalIdentifier struct { - Name string - SessionID string - IncludePatterns []string - ExcludePatterns []string - FollowPaths []string - SharedKeyHint string - Differ fsutil.DiffType -} - -func NewLocalIdentifier(str string) (*LocalIdentifier, error) { - return &LocalIdentifier{Name: str}, nil -} - -func (*LocalIdentifier) ID() string { - return srctypes.LocalScheme -} - -func NewHTTPIdentifier(str string, tls bool) (*HTTPIdentifier, error) { - proto := "https://" - if !tls { - proto = "http://" - } - return &HTTPIdentifier{TLS: tls, URL: proto + str}, nil -} - -type HTTPIdentifier struct { - TLS bool - URL string - Checksum digest.Digest - Filename string - Perm int - UID int - GID int -} - -func (*HTTPIdentifier) ID() string { - return srctypes.HTTPSScheme -} - -type OCIIdentifier struct { - Reference reference.Spec - Platform *ocispecs.Platform - SessionID string - StoreID string - LayerLimit *int -} - -func NewOCIIdentifier(str string) (*OCIIdentifier, error) { - ref, err := reference.Parse(str) - if err != nil { - return nil, errors.WithStack(err) - } - - if ref.Object == "" { - return nil, errors.WithStack(reference.ErrObjectRequired) - } - return &OCIIdentifier{Reference: ref}, nil -} - -func (*OCIIdentifier) ID() string { - return srctypes.OCIScheme -} - -func (r ResolveMode) String() string { - switch r { - case ResolveModeDefault: - return pb.AttrImageResolveModeDefault - case ResolveModeForcePull: - return pb.AttrImageResolveModeForcePull - case ResolveModePreferLocal: - return pb.AttrImageResolveModePreferLocal - default: - return "" - } -} - -func ParseImageResolveMode(v string) (ResolveMode, error) { - switch v { - case pb.AttrImageResolveModeDefault, "": - return ResolveModeDefault, nil - case pb.AttrImageResolveModeForcePull: - return ResolveModeForcePull, nil - case pb.AttrImageResolveModePreferLocal: - return ResolveModePreferLocal, nil - default: - return 0, errors.Errorf("invalid resolvemode: %s", v) - } -} - -func parseImageRecordType(v string) (client.UsageRecordType, error) { - switch client.UsageRecordType(v) { - case "", client.UsageRecordTypeRegular: - return client.UsageRecordTypeRegular, nil - case client.UsageRecordTypeInternal: - return client.UsageRecordTypeInternal, nil - case client.UsageRecordTypeFrontend: - return client.UsageRecordTypeFrontend, nil - default: - return "", errors.Errorf("invalid record type %s", v) - } + // Scheme returns the scheme of the identifier so that it can be routed back + // to an appropriate Source. + Scheme() string + + // Capture records the provenance of the identifier. + Capture(dest *provenance.Capture, pin string) error } diff --git a/source/local/identifier.go b/source/local/identifier.go new file mode 100644 index 000000000..703d66890 --- /dev/null +++ b/source/local/identifier.go @@ -0,0 +1,35 @@ +package local + +import ( + "github.com/moby/buildkit/solver/llbsolver/provenance" + "github.com/moby/buildkit/source" + srctypes "github.com/moby/buildkit/source/types" + "github.com/tonistiigi/fsutil" +) + +type LocalIdentifier struct { + Name string + SessionID string + IncludePatterns []string + ExcludePatterns []string + FollowPaths []string + SharedKeyHint string + Differ fsutil.DiffType +} + +func NewLocalIdentifier(str string) (*LocalIdentifier, error) { + return &LocalIdentifier{Name: str}, nil +} + +func (*LocalIdentifier) Scheme() string { + return srctypes.LocalScheme +} + +var _ source.Identifier = (*LocalIdentifier)(nil) + +func (id *LocalIdentifier) Capture(c *provenance.Capture, pin string) error { + c.AddLocal(provenance.LocalSource{ + Name: id.Name, + }) + return nil +} diff --git a/source/local/local.go b/source/local/source.go similarity index 84% rename from source/local/local.go rename to source/local/source.go index 7e3f529f5..31f3dd7e2 100644 --- a/source/local/local.go +++ b/source/local/source.go @@ -4,8 +4,10 @@ import ( "context" "encoding/json" "fmt" + "strings" "time" + "github.com/moby/buildkit/solver/pb" srctypes "github.com/moby/buildkit/source/types" "github.com/moby/buildkit/util/bklog" @@ -41,12 +43,59 @@ type localSource struct { cm cache.Accessor } -func (ls *localSource) ID() string { - return srctypes.LocalScheme +func (ls *localSource) Schemes() []string { + return []string{srctypes.LocalScheme} +} + +func (ls *localSource) Identifier(scheme, ref string, attrs map[string]string, platform *pb.Platform) (source.Identifier, error) { + id, err := NewLocalIdentifier(ref) + if err != nil { + return nil, err + } + + for k, v := range attrs { + switch k { + case pb.AttrLocalSessionID: + id.SessionID = v + if p := strings.SplitN(v, ":", 2); len(p) == 2 { + id.Name = p[0] + "-" + id.Name + id.SessionID = p[1] + } + case pb.AttrIncludePatterns: + var patterns []string + if err := json.Unmarshal([]byte(v), &patterns); err != nil { + return nil, err + } + id.IncludePatterns = patterns + case pb.AttrExcludePatterns: + var patterns []string + if err := json.Unmarshal([]byte(v), &patterns); err != nil { + return nil, err + } + id.ExcludePatterns = patterns + case pb.AttrFollowPaths: + var paths []string + if err := json.Unmarshal([]byte(v), &paths); err != nil { + return nil, err + } + id.FollowPaths = paths + case pb.AttrSharedKeyHint: + id.SharedKeyHint = v + case pb.AttrLocalDiffer: + switch v { + case pb.AttrLocalDifferMetadata, "": + id.Differ = fsutil.DiffMetadata + case pb.AttrLocalDifferNone: + id.Differ = fsutil.DiffNone + } + } + } + + return id, nil } func (ls *localSource) Resolve(ctx context.Context, id source.Identifier, sm *session.Manager, _ solver.Vertex) (source.SourceInstance, error) { - localIdentifier, ok := id.(*source.LocalIdentifier) + localIdentifier, ok := id.(*LocalIdentifier) if !ok { return nil, errors.Errorf("invalid local identifier %v", id) } @@ -59,7 +108,7 @@ func (ls *localSource) Resolve(ctx context.Context, id source.Identifier, sm *se } type localSourceHandler struct { - src source.LocalIdentifier + src LocalIdentifier sm *session.Manager *localSource } diff --git a/source/manager.go b/source/manager.go index 6a9c831c9..9501362aa 100644 --- a/source/manager.go +++ b/source/manager.go @@ -2,48 +2,83 @@ package source import ( "context" + "strings" "sync" "github.com/moby/buildkit/cache" "github.com/moby/buildkit/session" "github.com/moby/buildkit/solver" + "github.com/moby/buildkit/solver/pb" "github.com/pkg/errors" ) +// Source implementations provide "root" vertices in the graph that can be +// constructed from a URI-like string and arbitrary attrs. type Source interface { - ID() string + // Schemes returns a list of SourceOp identifier schemes that this source + // should match. + Schemes() []string + + // Identifier constructs an Identifier from the given scheme, ref, and attrs, + // all of which come from a SourceOp. + Identifier(scheme, ref string, attrs map[string]string, platform *pb.Platform) (Identifier, error) + + // Resolve constructs an instance of the source from an Identifier. Resolve(ctx context.Context, id Identifier, sm *session.Manager, vtx solver.Vertex) (SourceInstance, error) } +// SourceInstance represents a cacheable vertex created by a Source. type SourceInstance interface { + // CacheKey returns the cache key for the instance. CacheKey(ctx context.Context, g session.Group, index int) (key, pin string, opts solver.CacheOpts, done bool, err error) + + // Snapshot creates a cache ref for the instance. Snapshot(ctx context.Context, g session.Group) (cache.ImmutableRef, error) } type Manager struct { mu sync.Mutex - sources map[string]Source + schemes map[string]Source } func NewManager() (*Manager, error) { return &Manager{ - sources: make(map[string]Source), + schemes: make(map[string]Source), }, nil } func (sm *Manager) Register(src Source) { sm.mu.Lock() - sm.sources[src.ID()] = src + for _, scheme := range src.Schemes() { + sm.schemes[scheme] = src + } sm.mu.Unlock() } +func (sm *Manager) Identifier(op *pb.Op_Source, platform *pb.Platform) (Identifier, error) { + scheme, ref, ok := strings.Cut(op.Source.Identifier, "://") + if !ok { + return nil, errors.Wrapf(errInvalid, "failed to parse %s", op.Source.Identifier) + } + + sm.mu.Lock() + source, found := sm.schemes[scheme] + sm.mu.Unlock() + + if !found { + return nil, errors.Wrapf(errNotFound, "unknown scheme %s", scheme) + } + + return source.Identifier(scheme, ref, op.Source.Attrs, platform) +} + func (sm *Manager) Resolve(ctx context.Context, id Identifier, sessM *session.Manager, vtx solver.Vertex) (SourceInstance, error) { sm.mu.Lock() - src, ok := sm.sources[id.ID()] + src, ok := sm.schemes[id.Scheme()] sm.mu.Unlock() if !ok { - return nil, errors.Errorf("no handler for %s", id.ID()) + return nil, errors.Errorf("no handler for %s", id.Scheme()) } return src.Resolve(ctx, id, sessM, vtx) diff --git a/util/resolver/pool.go b/util/resolver/pool.go index 7b6a2ef50..3b3575b80 100644 --- a/util/resolver/pool.go +++ b/util/resolver/pool.go @@ -14,9 +14,10 @@ import ( "github.com/containerd/containerd/remotes/docker" distreference "github.com/docker/distribution/reference" "github.com/moby/buildkit/session" - "github.com/moby/buildkit/source" + "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/version" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" ) // DefaultPool is the default shared resolver pool instance @@ -125,7 +126,7 @@ type Resolver struct { auth *dockerAuthorizer is images.Store - mode source.ResolveMode + mode ResolveMode } // HostsFunc implements registry configuration of this Resolver @@ -177,7 +178,7 @@ func (r *Resolver) WithSession(s session.Group) *Resolver { } // WithImageStore returns new resolver that can also resolve from local images store -func (r *Resolver) WithImageStore(is images.Store, mode source.ResolveMode) *Resolver { +func (r *Resolver) WithImageStore(is images.Store, mode ResolveMode) *Resolver { r2 := *r r2.Resolver = r.Resolver r2.is = is @@ -195,7 +196,7 @@ func (r *Resolver) Fetcher(ctx context.Context, ref string) (remotes.Fetcher, er // Resolve attempts to resolve the reference into a name and descriptor. func (r *Resolver) Resolve(ctx context.Context, ref string) (string, ocispecs.Descriptor, error) { - if r.mode == source.ResolveModePreferLocal && r.is != nil { + if r.mode == ResolveModePreferLocal && r.is != nil { if img, err := r.is.Get(ctx, ref); err == nil { return ref, img.Target, nil } @@ -207,7 +208,7 @@ func (r *Resolver) Resolve(ctx context.Context, ref string) (string, ocispecs.De return n, desc, nil } - if r.mode == source.ResolveModeDefault && r.is != nil { + if r.mode == ResolveModeDefault && r.is != nil { if img, err := r.is.Get(ctx, ref); err == nil { return ref, img.Target, nil } @@ -215,3 +216,37 @@ func (r *Resolver) Resolve(ctx context.Context, ref string) (string, ocispecs.De return "", ocispecs.Descriptor{}, err } + +type ResolveMode int + +const ( + ResolveModeDefault ResolveMode = iota + ResolveModeForcePull + ResolveModePreferLocal +) + +func (r ResolveMode) String() string { + switch r { + case ResolveModeDefault: + return pb.AttrImageResolveModeDefault + case ResolveModeForcePull: + return pb.AttrImageResolveModeForcePull + case ResolveModePreferLocal: + return pb.AttrImageResolveModePreferLocal + default: + return "" + } +} + +func ParseImageResolveMode(v string) (ResolveMode, error) { + switch v { + case pb.AttrImageResolveModeDefault, "": + return ResolveModeDefault, nil + case pb.AttrImageResolveModeForcePull: + return ResolveModeForcePull, nil + case pb.AttrImageResolveModePreferLocal: + return ResolveModePreferLocal, nil + default: + return 0, errors.Errorf("invalid resolvemode: %s", v) + } +} diff --git a/worker/tests/common.go b/worker/tests/common.go index de2c7ed08..458f41a6d 100644 --- a/worker/tests/common.go +++ b/worker/tests/common.go @@ -13,7 +13,7 @@ import ( "github.com/moby/buildkit/identity" "github.com/moby/buildkit/session" "github.com/moby/buildkit/snapshot" - "github.com/moby/buildkit/source" + "github.com/moby/buildkit/source/containerimage" "github.com/moby/buildkit/worker/base" "github.com/pkg/errors" "github.com/stretchr/testify/require" @@ -21,7 +21,7 @@ import ( ) func NewBusyboxSourceSnapshot(ctx context.Context, t *testing.T, w *base.Worker, sm *session.Manager) cache.ImmutableRef { - img, err := source.NewImageIdentifier("docker.io/library/busybox:latest") + img, err := containerimage.NewImageIdentifier("docker.io/library/busybox:latest") require.NoError(t, err) src, err := w.SourceManager.Resolve(ctx, img, sm, nil) require.NoError(t, err)