source: make sources pluggable

Sources are a pretty neat extension point, except there are a few code
paths that hard-code against each type. This moves code around and
adjusts interfaces so that Source implementations are self-contained and
merely need to be registered with the source.Manager.

Signed-off-by: Alex Suraci <suraci.alex@gmail.com>

solver/llbsolver/ops/source.go

@@ -60,7 +60,7 @@ func (s *SourceOp) instance(ctx context.Context) (source.SourceInstance, error)
 	if s.src != nil {
 		return s.src, nil
 	}
-	id, err := source.FromLLB(s.op, s.platform)
+	id, err := s.sm.Identifier(s.op, s.platform)
 	if err != nil {
 		return nil, err
 	}

solver/llbsolver/provenance.go

@@ -20,7 +20,6 @@ import (
 	"github.com/moby/buildkit/solver/llbsolver/ops"
 	"github.com/moby/buildkit/solver/llbsolver/provenance"
 	"github.com/moby/buildkit/solver/pb"
-	"github.com/moby/buildkit/source"
 	"github.com/moby/buildkit/worker"
 	digest "github.com/opencontainers/go-digest"
 	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
@@ -268,70 +267,9 @@ func captureProvenance(ctx context.Context, res solver.CachedResultWithProvenanc
 		switch op := pp.(type) {
 		case *ops.SourceOp:
 			id, pin := op.Pin()
-			switch s := id.(type) {
+			err := id.Capture(c, pin)
-			case *source.ImageIdentifier:
+			if err != nil {
-				dgst, err := digest.Parse(pin)
+				return err
-				if err != nil {
-					return errors.Wrapf(err, "failed to parse image digest %s", pin)
-				}
-				c.AddImage(provenance.ImageSource{
-					Ref:      s.Reference.String(),
-					Platform: s.Platform,
-					Digest:   dgst,
-				})
-			case *source.LocalIdentifier:
-				c.AddLocal(provenance.LocalSource{
-					Name: s.Name,
-				})
-			case *source.GitIdentifier:
-				url := s.Remote
-				if s.Ref != "" {
-					url += "#" + s.Ref
-				}
-				c.AddGit(provenance.GitSource{
-					URL:    url,
-					Commit: pin,
-				})
-				if s.AuthTokenSecret != "" {
-					c.AddSecret(provenance.Secret{
-						ID:       s.AuthTokenSecret,
-						Optional: true,
-					})
-				}
-				if s.AuthHeaderSecret != "" {
-					c.AddSecret(provenance.Secret{
-						ID:       s.AuthHeaderSecret,
-						Optional: true,
-					})
-				}
-				if s.MountSSHSock != "" {
-					c.AddSSH(provenance.SSH{
-						ID:       s.MountSSHSock,
-						Optional: true,
-					})
-				}
-			case *source.HTTPIdentifier:
-				dgst, err := digest.Parse(pin)
-				if err != nil {
-					return errors.Wrapf(err, "failed to parse HTTP digest %s", pin)
-				}
-				c.AddHTTP(provenance.HTTPSource{
-					URL:    s.URL,
-					Digest: dgst,
-				})
-			case *source.OCIIdentifier:
-				dgst, err := digest.Parse(pin)
-				if err != nil {
-					return errors.Wrapf(err, "failed to parse OCI digest %s", pin)
-				}
-				c.AddImage(provenance.ImageSource{
-					Ref:      s.Reference.String(),
-					Platform: s.Platform,
-					Digest:   dgst,
-					Local:    true,
-				})
-			default:
-				return errors.Errorf("unknown source identifier %T", id)
 			}
 		case *ops.ExecOp:
 			pr := op.Proto()

solver/llbsolver/vertex.go

@@ -9,7 +9,6 @@ import (
 	"github.com/moby/buildkit/solver"
 	"github.com/moby/buildkit/solver/llbsolver/ops/opsutils"
 	"github.com/moby/buildkit/solver/pb"
-	"github.com/moby/buildkit/source"
 	"github.com/moby/buildkit/util/entitlements"
 	digest "github.com/opencontainers/go-digest"
 	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
@@ -322,13 +321,6 @@ func loadLLB(ctx context.Context, def *pb.Definition, polEngine SourcePolicyEval
 func llbOpName(pbOp *pb.Op, load func(digest.Digest) (solver.Vertex, error)) (string, error) {
 	switch op := pbOp.Op.(type) {
 	case *pb.Op_Source:
-		if id, err := source.FromLLB(op, nil); err == nil {
-			if id, ok := id.(*source.LocalIdentifier); ok {
-				if len(id.IncludePatterns) == 1 {
-					return op.Source.Identifier + " (" + id.IncludePatterns[0] + ")", nil
-				}
-			}
-		}
 		return op.Source.Identifier, nil
 	case *pb.Op_Exec:
 		return strings.Join(op.Exec.Meta.Args, " "), nil

source/containerimage/identifier.go

@@ -0,0 +1,92 @@
+package containerimage
+
+import (
+	"github.com/containerd/containerd/reference"
+	"github.com/moby/buildkit/client"
+	"github.com/moby/buildkit/solver/llbsolver/provenance"
+	"github.com/moby/buildkit/source"
+	srctypes "github.com/moby/buildkit/source/types"
+	"github.com/moby/buildkit/util/resolver"
+	digest "github.com/opencontainers/go-digest"
+	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+)
+
+type ImageIdentifier struct {
+	Reference   reference.Spec
+	Platform    *ocispecs.Platform
+	ResolveMode resolver.ResolveMode
+	RecordType  client.UsageRecordType
+	LayerLimit  *int
+}
+
+func NewImageIdentifier(str string) (*ImageIdentifier, error) {
+	ref, err := reference.Parse(str)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	if ref.Object == "" {
+		return nil, errors.WithStack(reference.ErrObjectRequired)
+	}
+	return &ImageIdentifier{Reference: ref}, nil
+}
+
+var _ source.Identifier = (*ImageIdentifier)(nil)
+
+func (*ImageIdentifier) Scheme() string {
+	return srctypes.DockerImageScheme
+}
+
+func (id *ImageIdentifier) Capture(c *provenance.Capture, pin string) error {
+	dgst, err := digest.Parse(pin)
+	if err != nil {
+		return errors.Wrapf(err, "failed to parse image digest %s", pin)
+	}
+	c.AddImage(provenance.ImageSource{
+		Ref:      id.Reference.String(),
+		Platform: id.Platform,
+		Digest:   dgst,
+	})
+	return nil
+}
+
+type OCIIdentifier struct {
+	Reference  reference.Spec
+	Platform   *ocispecs.Platform
+	SessionID  string
+	StoreID    string
+	LayerLimit *int
+}
+
+func NewOCIIdentifier(str string) (*OCIIdentifier, error) {
+	ref, err := reference.Parse(str)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	if ref.Object == "" {
+		return nil, errors.WithStack(reference.ErrObjectRequired)
+	}
+	return &OCIIdentifier{Reference: ref}, nil
+}
+
+var _ source.Identifier = (*OCIIdentifier)(nil)
+
+func (*OCIIdentifier) Scheme() string {
+	return srctypes.OCIScheme
+}
+
+func (id *OCIIdentifier) Capture(c *provenance.Capture, pin string) error {
+	dgst, err := digest.Parse(pin)
+	if err != nil {
+		return errors.Wrapf(err, "failed to parse OCI digest %s", pin)
+	}
+	c.AddImage(provenance.ImageSource{
+		Ref:      id.Reference.String(),
+		Platform: id.Platform,
+		Digest:   dgst,
+		Local:    true,
+	})
+	return nil
+}

source/containerimage/pull.go

@@ -7,12 +7,9 @@ import (
 	"time"
 
 	"github.com/containerd/containerd/content"
-	"github.com/containerd/containerd/diff"
 	containerderrdefs "github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/images"
 	"github.com/containerd/containerd/leases"
-	"github.com/containerd/containerd/platforms"
-	"github.com/containerd/containerd/reference"
 	"github.com/containerd/containerd/remotes"
 	"github.com/containerd/containerd/remotes/docker"
 	"github.com/containerd/containerd/snapshots"
@@ -20,11 +17,8 @@ import (
 	"github.com/moby/buildkit/client"
 	"github.com/moby/buildkit/client/llb"
 	"github.com/moby/buildkit/session"
-	"github.com/moby/buildkit/snapshot"
 	"github.com/moby/buildkit/solver"
 	"github.com/moby/buildkit/solver/errdefs"
-	"github.com/moby/buildkit/source"
-	srctypes "github.com/moby/buildkit/source/types"
 	"github.com/moby/buildkit/util/estargz"
 	"github.com/moby/buildkit/util/flightcontrol"
 	"github.com/moby/buildkit/util/imageutil"
@@ -39,164 +33,12 @@ import (
 	"github.com/pkg/errors"
 )
 
-// TODO: break apart containerd specifics like contentstore so the resolver
-// code can be used with any implementation
-
-type ResolverType int
-
-const (
-	ResolverTypeRegistry ResolverType = iota
-	ResolverTypeOCILayout
-)
-
-type SourceOpt struct {
-	Snapshotter   snapshot.Snapshotter
-	ContentStore  content.Store
-	Applier       diff.Applier
-	CacheAccessor cache.Accessor
-	ImageStore    images.Store // optional
-	RegistryHosts docker.RegistryHosts
-	ResolverType
-	LeaseManager leases.Manager
-}
-
-type resolveImageResult struct {
-	ref  string
-	dgst digest.Digest
-	dt   []byte
-}
-
-type Source struct {
-	SourceOpt
-	g flightcontrol.Group[*resolveImageResult]
-}
-
-var _ source.Source = &Source{}
-
-func NewSource(opt SourceOpt) (*Source, error) {
-	is := &Source{
-		SourceOpt: opt,
-	}
-
-	return is, nil
-}
-
-func (is *Source) ID() string {
-	if is.ResolverType == ResolverTypeOCILayout {
-		return srctypes.OCIScheme
-	}
-	return srctypes.DockerImageScheme
-}
-
-func (is *Source) ResolveImageConfig(ctx context.Context, ref string, opt llb.ResolveImageConfigOpt, sm *session.Manager, g session.Group) (string, digest.Digest, []byte, error) {
-	key := ref
-	if platform := opt.Platform; platform != nil {
-		key += platforms.Format(*platform)
-	}
-	var (
-		rm    source.ResolveMode
-		rslvr remotes.Resolver
-		err   error
-	)
-
-	switch is.ResolverType {
-	case ResolverTypeRegistry:
-		rm, err = source.ParseImageResolveMode(opt.ResolveMode)
-		if err != nil {
-			return "", "", nil, err
-		}
-		rslvr = resolver.DefaultPool.GetResolver(is.RegistryHosts, ref, "pull", sm, g).WithImageStore(is.ImageStore, rm)
-	case ResolverTypeOCILayout:
-		rm = source.ResolveModeForcePull
-		rslvr = getOCILayoutResolver(opt.Store, sm, g)
-	}
-	key += rm.String()
-	res, err := is.g.Do(ctx, key, func(ctx context.Context) (*resolveImageResult, error) {
-		newRef, dgst, dt, err := imageutil.Config(ctx, ref, rslvr, is.ContentStore, is.LeaseManager, opt.Platform, opt.SourcePolicies)
-		if err != nil {
-			return nil, err
-		}
-		return &resolveImageResult{dgst: dgst, dt: dt, ref: newRef}, nil
-	})
-	if err != nil {
-		return "", "", nil, err
-	}
-	return res.ref, res.dgst, res.dt, nil
-}
-
-func (is *Source) Resolve(ctx context.Context, id source.Identifier, sm *session.Manager, vtx solver.Vertex) (source.SourceInstance, error) {
-	var (
-		p          *puller
-		platform   = platforms.DefaultSpec()
-		pullerUtil *pull.Puller
-		mode       source.ResolveMode
-		recordType client.UsageRecordType
-		ref        reference.Spec
-		store      llb.ResolveImageConfigOptStore
-		layerLimit *int
-	)
-	switch is.ResolverType {
-	case ResolverTypeRegistry:
-		imageIdentifier, ok := id.(*source.ImageIdentifier)
-		if !ok {
-			return nil, errors.Errorf("invalid image identifier %v", id)
-		}
-
-		if imageIdentifier.Platform != nil {
-			platform = *imageIdentifier.Platform
-		}
-		mode = imageIdentifier.ResolveMode
-		recordType = imageIdentifier.RecordType
-		ref = imageIdentifier.Reference
-		layerLimit = imageIdentifier.LayerLimit
-	case ResolverTypeOCILayout:
-		ociIdentifier, ok := id.(*source.OCIIdentifier)
-		if !ok {
-			return nil, errors.Errorf("invalid OCI layout identifier %v", id)
-		}
-
-		if ociIdentifier.Platform != nil {
-			platform = *ociIdentifier.Platform
-		}
-		mode = source.ResolveModeForcePull // with OCI layout, we always just "pull"
-		store = llb.ResolveImageConfigOptStore{
-			SessionID: ociIdentifier.SessionID,
-			StoreID:   ociIdentifier.StoreID,
-		}
-		ref = ociIdentifier.Reference
-		layerLimit = ociIdentifier.LayerLimit
-	default:
-		return nil, errors.Errorf("unknown resolver type: %v", is.ResolverType)
-	}
-	pullerUtil = &pull.Puller{
-		ContentStore: is.ContentStore,
-		Platform:     platform,
-		Src:          ref,
-	}
-	p = &puller{
-		CacheAccessor:  is.CacheAccessor,
-		LeaseManager:   is.LeaseManager,
-		Puller:         pullerUtil,
-		RegistryHosts:  is.RegistryHosts,
-		ResolverType:   is.ResolverType,
-		ImageStore:     is.ImageStore,
-		Mode:           mode,
-		RecordType:     recordType,
-		Ref:            ref.String(),
-		SessionManager: sm,
-		vtx:            vtx,
-		store:          store,
-		layerLimit:     layerLimit,
-	}
-	return p, nil
-}
-
 type puller struct {
 	CacheAccessor  cache.Accessor
 	LeaseManager   leases.Manager
 	RegistryHosts  docker.RegistryHosts
 	ImageStore     images.Store
-	Mode           source.ResolveMode
+	Mode           resolver.ResolveMode
 	RecordType     client.UsageRecordType
 	Ref            string
 	SessionManager *session.Manager

source/containerimage/source.go

@@ -0,0 +1,286 @@
+package containerimage
+
+import (
+	"context"
+	"strconv"
+
+	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/diff"
+	"github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/leases"
+	"github.com/containerd/containerd/platforms"
+	"github.com/containerd/containerd/reference"
+	"github.com/containerd/containerd/remotes"
+	"github.com/containerd/containerd/remotes/docker"
+	"github.com/moby/buildkit/cache"
+	"github.com/moby/buildkit/client"
+	"github.com/moby/buildkit/client/llb"
+	"github.com/moby/buildkit/session"
+	"github.com/moby/buildkit/snapshot"
+	"github.com/moby/buildkit/solver"
+	"github.com/moby/buildkit/solver/pb"
+	"github.com/moby/buildkit/source"
+	srctypes "github.com/moby/buildkit/source/types"
+	"github.com/moby/buildkit/util/flightcontrol"
+	"github.com/moby/buildkit/util/imageutil"
+	"github.com/moby/buildkit/util/pull"
+	"github.com/moby/buildkit/util/resolver"
+	digest "github.com/opencontainers/go-digest"
+	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+)
+
+// TODO: break apart containerd specifics like contentstore so the resolver
+// code can be used with any implementation
+
+type ResolverType int
+
+const (
+	ResolverTypeRegistry ResolverType = iota
+	ResolverTypeOCILayout
+)
+
+type SourceOpt struct {
+	Snapshotter   snapshot.Snapshotter
+	ContentStore  content.Store
+	Applier       diff.Applier
+	CacheAccessor cache.Accessor
+	ImageStore    images.Store // optional
+	RegistryHosts docker.RegistryHosts
+	ResolverType
+	LeaseManager leases.Manager
+}
+
+type Source struct {
+	SourceOpt
+	g flightcontrol.Group[*resolveImageResult]
+}
+
+var _ source.Source = &Source{}
+
+func NewSource(opt SourceOpt) (*Source, error) {
+	is := &Source{
+		SourceOpt: opt,
+	}
+
+	return is, nil
+}
+
+func (is *Source) Schemes() []string {
+	if is.ResolverType == ResolverTypeOCILayout {
+		return []string{srctypes.OCIScheme}
+	}
+	return []string{srctypes.DockerImageScheme}
+}
+
+func (is *Source) Identifier(scheme, ref string, attrs map[string]string, platform *pb.Platform) (source.Identifier, error) {
+	if is.ResolverType == ResolverTypeOCILayout {
+		return is.ociIdentifier(ref, attrs, platform)
+	}
+
+	return is.registryIdentifier(ref, attrs, platform)
+}
+
+func (is *Source) Resolve(ctx context.Context, id source.Identifier, sm *session.Manager, vtx solver.Vertex) (source.SourceInstance, error) {
+	var (
+		p          *puller
+		platform   = platforms.DefaultSpec()
+		pullerUtil *pull.Puller
+		mode       resolver.ResolveMode
+		recordType client.UsageRecordType
+		ref        reference.Spec
+		store      llb.ResolveImageConfigOptStore
+		layerLimit *int
+	)
+	switch is.ResolverType {
+	case ResolverTypeRegistry:
+		imageIdentifier, ok := id.(*ImageIdentifier)
+		if !ok {
+			return nil, errors.Errorf("invalid image identifier %v", id)
+		}
+
+		if imageIdentifier.Platform != nil {
+			platform = *imageIdentifier.Platform
+		}
+		mode = imageIdentifier.ResolveMode
+		recordType = imageIdentifier.RecordType
+		ref = imageIdentifier.Reference
+		layerLimit = imageIdentifier.LayerLimit
+	case ResolverTypeOCILayout:
+		ociIdentifier, ok := id.(*OCIIdentifier)
+		if !ok {
+			return nil, errors.Errorf("invalid OCI layout identifier %v", id)
+		}
+
+		if ociIdentifier.Platform != nil {
+			platform = *ociIdentifier.Platform
+		}
+		mode = resolver.ResolveModeForcePull // with OCI layout, we always just "pull"
+		store = llb.ResolveImageConfigOptStore{
+			SessionID: ociIdentifier.SessionID,
+			StoreID:   ociIdentifier.StoreID,
+		}
+		ref = ociIdentifier.Reference
+		layerLimit = ociIdentifier.LayerLimit
+	default:
+		return nil, errors.Errorf("unknown resolver type: %v", is.ResolverType)
+	}
+	pullerUtil = &pull.Puller{
+		ContentStore: is.ContentStore,
+		Platform:     platform,
+		Src:          ref,
+	}
+	p = &puller{
+		CacheAccessor:  is.CacheAccessor,
+		LeaseManager:   is.LeaseManager,
+		Puller:         pullerUtil,
+		RegistryHosts:  is.RegistryHosts,
+		ResolverType:   is.ResolverType,
+		ImageStore:     is.ImageStore,
+		Mode:           mode,
+		RecordType:     recordType,
+		Ref:            ref.String(),
+		SessionManager: sm,
+		vtx:            vtx,
+		store:          store,
+		layerLimit:     layerLimit,
+	}
+	return p, nil
+}
+
+func (is *Source) ResolveImageConfig(ctx context.Context, ref string, opt llb.ResolveImageConfigOpt, sm *session.Manager, g session.Group) (string, digest.Digest, []byte, error) {
+	key := ref
+	if platform := opt.Platform; platform != nil {
+		key += platforms.Format(*platform)
+	}
+	var (
+		rm    resolver.ResolveMode
+		rslvr remotes.Resolver
+		err   error
+	)
+
+	switch is.ResolverType {
+	case ResolverTypeRegistry:
+		rm, err = resolver.ParseImageResolveMode(opt.ResolveMode)
+		if err != nil {
+			return "", "", nil, err
+		}
+		rslvr = resolver.DefaultPool.GetResolver(is.RegistryHosts, ref, "pull", sm, g).WithImageStore(is.ImageStore, rm)
+	case ResolverTypeOCILayout:
+		rm = resolver.ResolveModeForcePull
+		rslvr = getOCILayoutResolver(opt.Store, sm, g)
+	}
+	key += rm.String()
+	res, err := is.g.Do(ctx, key, func(ctx context.Context) (*resolveImageResult, error) {
+		newRef, dgst, dt, err := imageutil.Config(ctx, ref, rslvr, is.ContentStore, is.LeaseManager, opt.Platform, opt.SourcePolicies)
+		if err != nil {
+			return nil, err
+		}
+		return &resolveImageResult{dgst: dgst, dt: dt, ref: newRef}, nil
+	})
+	if err != nil {
+		return "", "", nil, err
+	}
+	return res.ref, res.dgst, res.dt, nil
+}
+
+type resolveImageResult struct {
+	ref  string
+	dgst digest.Digest
+	dt   []byte
+}
+
+func (is *Source) registryIdentifier(ref string, attrs map[string]string, platform *pb.Platform) (source.Identifier, error) {
+	id, err := NewImageIdentifier(ref)
+	if err != nil {
+		return nil, err
+	}
+
+	if platform != nil {
+		id.Platform = &ocispecs.Platform{
+			OS:           platform.OS,
+			Architecture: platform.Architecture,
+			Variant:      platform.Variant,
+			OSVersion:    platform.OSVersion,
+			OSFeatures:   platform.OSFeatures,
+		}
+	}
+
+	for k, v := range attrs {
+		switch k {
+		case pb.AttrImageResolveMode:
+			rm, err := resolver.ParseImageResolveMode(v)
+			if err != nil {
+				return nil, err
+			}
+			id.ResolveMode = rm
+		case pb.AttrImageRecordType:
+			rt, err := parseImageRecordType(v)
+			if err != nil {
+				return nil, err
+			}
+			id.RecordType = rt
+		case pb.AttrImageLayerLimit:
+			l, err := strconv.Atoi(v)
+			if err != nil {
+				return nil, errors.Wrapf(err, "invalid layer limit %s", v)
+			}
+			if l <= 0 {
+				return nil, errors.Errorf("invalid layer limit %s", v)
+			}
+			id.LayerLimit = &l
+		}
+	}
+
+	return id, nil
+}
+
+func (is *Source) ociIdentifier(ref string, attrs map[string]string, platform *pb.Platform) (source.Identifier, error) {
+	id, err := NewOCIIdentifier(ref)
+	if err != nil {
+		return nil, err
+	}
+
+	if platform != nil {
+		id.Platform = &ocispecs.Platform{
+			OS:           platform.OS,
+			Architecture: platform.Architecture,
+			Variant:      platform.Variant,
+			OSVersion:    platform.OSVersion,
+			OSFeatures:   platform.OSFeatures,
+		}
+	}
+
+	for k, v := range attrs {
+		switch k {
+		case pb.AttrOCILayoutSessionID:
+			id.SessionID = v
+		case pb.AttrOCILayoutStoreID:
+			id.StoreID = v
+		case pb.AttrOCILayoutLayerLimit:
+			l, err := strconv.Atoi(v)
+			if err != nil {
+				return nil, errors.Wrapf(err, "invalid layer limit %s", v)
+			}
+			if l <= 0 {
+				return nil, errors.Errorf("invalid layer limit %s", v)
+			}
+			id.LayerLimit = &l
+		}
+	}
+
+	return id, nil
+}
+
+func parseImageRecordType(v string) (client.UsageRecordType, error) {
+	switch client.UsageRecordType(v) {
+	case "", client.UsageRecordTypeRegular:
+		return client.UsageRecordTypeRegular, nil
+	case client.UsageRecordTypeInternal:
+		return client.UsageRecordTypeInternal, nil
+	case client.UsageRecordTypeFrontend:
+		return client.UsageRecordTypeFrontend, nil
+	default:
+		return "", errors.Errorf("invalid record type %s", v)
+	}
+}

source/git/identifier.go

@@ -1,10 +1,12 @@
-package source
+package git
 
 import (
 	"net/url"
 	"path"
 	"strings"
 
+	"github.com/moby/buildkit/solver/llbsolver/provenance"
+	"github.com/moby/buildkit/source"
 	srctypes "github.com/moby/buildkit/source/types"
 	"github.com/moby/buildkit/util/sshutil"
 )
@@ -53,10 +55,42 @@ func NewGitIdentifier(remoteURL string) (*GitIdentifier, error) {
 	return &repo, nil
 }
 
-func (i *GitIdentifier) ID() string {
+func (GitIdentifier) Scheme() string {
 	return srctypes.GitScheme
 }
 
+var _ source.Identifier = (*GitIdentifier)(nil)
+
+func (id *GitIdentifier) Capture(c *provenance.Capture, pin string) error {
+	url := id.Remote
+	if id.Ref != "" {
+		url += "#" + id.Ref
+	}
+	c.AddGit(provenance.GitSource{
+		URL:    url,
+		Commit: pin,
+	})
+	if id.AuthTokenSecret != "" {
+		c.AddSecret(provenance.Secret{
+			ID:       id.AuthTokenSecret,
+			Optional: true,
+		})
+	}
+	if id.AuthHeaderSecret != "" {
+		c.AddSecret(provenance.Secret{
+			ID:       id.AuthHeaderSecret,
+			Optional: true,
+		})
+	}
+	if id.MountSSHSock != "" {
+		c.AddSSH(provenance.SSH{
+			ID:       id.MountSSHSock,
+			Optional: true,
+		})
+	}
+	return nil
+}
+
 // isGitTransport returns true if the provided str is a git transport by inspecting
 // the prefix of the string for known protocols used in git.
 func isGitTransport(str string) bool {

source/git/identifier_test.go

@@ -1,4 +1,4 @@
-package source
+package git
 
 import (
 	"testing"

source/git/source.go

@@ -24,6 +24,7 @@ import (
 	"github.com/moby/buildkit/session/sshforward"
 	"github.com/moby/buildkit/snapshot"
 	"github.com/moby/buildkit/solver"
+	"github.com/moby/buildkit/solver/pb"
 	"github.com/moby/buildkit/source"
 	srctypes "github.com/moby/buildkit/source/types"
 	"github.com/moby/buildkit/util/bklog"
@@ -63,8 +64,39 @@ func NewSource(opt Opt) (source.Source, error) {
 	return gs, nil
 }
 
-func (gs *gitSource) ID() string {
+func (gs *gitSource) Schemes() []string {
-	return srctypes.GitScheme
+	return []string{srctypes.GitScheme}
+}
+
+func (gs *gitSource) Identifier(scheme, ref string, attrs map[string]string, platform *pb.Platform) (source.Identifier, error) {
+	id, err := NewGitIdentifier(ref)
+	if err != nil {
+		return nil, err
+	}
+
+	for k, v := range attrs {
+		switch k {
+		case pb.AttrKeepGitDir:
+			if v == "true" {
+				id.KeepGitDir = true
+			}
+		case pb.AttrFullRemoteURL:
+			if !isGitTransport(v) {
+				v = "https://" + v
+			}
+			id.Remote = v
+		case pb.AttrAuthHeaderSecret:
+			id.AuthHeaderSecret = v
+		case pb.AttrAuthTokenSecret:
+			id.AuthTokenSecret = v
+		case pb.AttrKnownSSHHosts:
+			id.KnownSSHHosts = v
+		case pb.AttrMountSSHSock:
+			id.MountSSHSock = v
+		}
+	}
+
+	return id, nil
 }
 
 // needs to be called with repo lock
@@ -151,7 +183,7 @@ func (gs *gitSource) mountRemote(ctx context.Context, remote string, auth []stri
 
 type gitSourceHandler struct {
 	*gitSource
-	src      source.GitIdentifier
+	src      GitIdentifier
 	cacheKey string
 	sm       *session.Manager
 	auth     []string
@@ -169,7 +201,7 @@ func (gs *gitSourceHandler) shaToCacheKey(sha string) string {
 }
 
 func (gs *gitSource) Resolve(ctx context.Context, id source.Identifier, sm *session.Manager, _ solver.Vertex) (source.SourceInstance, error) {
-	gitIdentifier, ok := id.(*source.GitIdentifier)
+	gitIdentifier, ok := id.(*GitIdentifier)
 	if !ok {
 		return nil, errors.Errorf("invalid git identifier %v", id)
 	}

source/git/source_test.go

@@ -54,7 +54,7 @@ func testRepeatedFetch(t *testing.T, keepGitDir bool) {
 
 	repo := setupGitRepo(t)
 
-	id := &source.GitIdentifier{Remote: repo.mainURL, KeepGitDir: keepGitDir}
+	id := &GitIdentifier{Remote: repo.mainURL, KeepGitDir: keepGitDir}
 
 	g, err := gs.Resolve(ctx, id, nil, nil)
 	require.NoError(t, err)
@@ -95,7 +95,7 @@ func testRepeatedFetch(t *testing.T, keepGitDir bool) {
 	require.ErrorAs(t, err, &os.ErrNotExist)
 
 	// second fetch returns same dir
-	id = &source.GitIdentifier{Remote: repo.mainURL, Ref: "master", KeepGitDir: keepGitDir}
+	id = &GitIdentifier{Remote: repo.mainURL, Ref: "master", KeepGitDir: keepGitDir}
 
 	g, err = gs.Resolve(ctx, id, nil, nil)
 	require.NoError(t, err)
@@ -112,7 +112,7 @@ func testRepeatedFetch(t *testing.T, keepGitDir bool) {
 
 	require.Equal(t, ref1.ID(), ref2.ID())
 
-	id = &source.GitIdentifier{Remote: repo.mainURL, Ref: "feature", KeepGitDir: keepGitDir}
+	id = &GitIdentifier{Remote: repo.mainURL, Ref: "feature", KeepGitDir: keepGitDir}
 
 	g, err = gs.Resolve(ctx, id, nil, nil)
 	require.NoError(t, err)
@@ -174,7 +174,7 @@ func testFetchBySHA(t *testing.T, keepGitDir bool) {
 	sha := strings.TrimSpace(string(out))
 	require.Equal(t, 40, len(sha))
 
-	id := &source.GitIdentifier{Remote: repo.mainURL, Ref: sha, KeepGitDir: keepGitDir}
+	id := &GitIdentifier{Remote: repo.mainURL, Ref: sha, KeepGitDir: keepGitDir}
 
 	g, err := gs.Resolve(ctx, id, nil, nil)
 	require.NoError(t, err)
@@ -243,7 +243,7 @@ func testFetchByTag(t *testing.T, tag, expectedCommitSubject string, isAnnotated
 
 	repo := setupGitRepo(t)
 
-	id := &source.GitIdentifier{Remote: repo.mainURL, Ref: tag, KeepGitDir: keepGitDir}
+	id := &GitIdentifier{Remote: repo.mainURL, Ref: tag, KeepGitDir: keepGitDir}
 
 	g, err := gs.Resolve(ctx, id, nil, nil)
 	require.NoError(t, err)
@@ -341,8 +341,8 @@ func testMultipleRepos(t *testing.T, keepGitDir bool) {
 	)
 	repoURL2 := serveGitRepo(t, repodir2)
 
-	id := &source.GitIdentifier{Remote: repo.mainURL, KeepGitDir: keepGitDir}
+	id := &GitIdentifier{Remote: repo.mainURL, KeepGitDir: keepGitDir}
-	id2 := &source.GitIdentifier{Remote: repoURL2, KeepGitDir: keepGitDir}
+	id2 := &GitIdentifier{Remote: repoURL2, KeepGitDir: keepGitDir}
 
 	g, err := gs.Resolve(ctx, id, nil, nil)
 	require.NoError(t, err)
@@ -415,7 +415,7 @@ func TestCredentialRedaction(t *testing.T) {
 	gs := setupGitSource(t, t.TempDir())
 
 	url := "https://user:keepthissecret@non-existant-host/user/private-repo.git"
-	id := &source.GitIdentifier{Remote: url}
+	id := &GitIdentifier{Remote: url}
 
 	g, err := gs.Resolve(ctx, id, nil, nil)
 	require.NoError(t, err)
@@ -457,7 +457,7 @@ func testSubdir(t *testing.T, keepGitDir bool) {
 	)
 
 	repoURL := serveGitRepo(t, repodir)
-	id := &source.GitIdentifier{Remote: repoURL, KeepGitDir: keepGitDir, Subdir: "sub"}
+	id := &GitIdentifier{Remote: repoURL, KeepGitDir: keepGitDir, Subdir: "sub"}
 
 	g, err := gs.Resolve(ctx, id, nil, nil)
 	require.NoError(t, err)

source/http/identifier.go

@@ -0,0 +1,48 @@
+package http
+
+import (
+	"github.com/moby/buildkit/solver/llbsolver/provenance"
+	"github.com/moby/buildkit/source"
+	srctypes "github.com/moby/buildkit/source/types"
+	digest "github.com/opencontainers/go-digest"
+	"github.com/pkg/errors"
+)
+
+func NewHTTPIdentifier(str string, tls bool) (*HTTPIdentifier, error) {
+	proto := "https://"
+	if !tls {
+		proto = "http://"
+	}
+	return &HTTPIdentifier{TLS: tls, URL: proto + str}, nil
+}
+
+type HTTPIdentifier struct {
+	TLS      bool
+	URL      string
+	Checksum digest.Digest
+	Filename string
+	Perm     int
+	UID      int
+	GID      int
+}
+
+var _ source.Identifier = (*HTTPIdentifier)(nil)
+
+func (id *HTTPIdentifier) Scheme() string {
+	if id.TLS {
+		return srctypes.HTTPSScheme
+	}
+	return srctypes.HTTPScheme
+}
+
+func (id *HTTPIdentifier) Capture(c *provenance.Capture, pin string) error {
+	dgst, err := digest.Parse(pin)
+	if err != nil {
+		return errors.Wrapf(err, "failed to parse HTTP digest %s", pin)
+	}
+	c.AddHTTP(provenance.HTTPSource{
+		URL:    id.URL,
+		Digest: dgst,
+	})
+	return nil
+}

source/http/source.go

@@ -12,6 +12,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"time"
 
@@ -20,6 +21,7 @@ import (
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/snapshot"
 	"github.com/moby/buildkit/solver"
+	"github.com/moby/buildkit/solver/pb"
 	"github.com/moby/buildkit/source"
 	srctypes "github.com/moby/buildkit/source/types"
 	"github.com/moby/buildkit/util/tracing"
@@ -52,20 +54,60 @@ func NewSource(opt Opt) (source.Source, error) {
 	return hs, nil
 }
 
-func (hs *httpSource) ID() string {
+func (hs *httpSource) Schemes() []string {
-	return srctypes.HTTPSScheme
+	return []string{srctypes.HTTPScheme, srctypes.HTTPSScheme}
+}
+
+func (hs *httpSource) Identifier(scheme, ref string, attrs map[string]string, platform *pb.Platform) (source.Identifier, error) {
+	id, err := NewHTTPIdentifier(ref, scheme == "https")
+	if err != nil {
+		return nil, err
+	}
+
+	for k, v := range attrs {
+		switch k {
+		case pb.AttrHTTPChecksum:
+			dgst, err := digest.Parse(v)
+			if err != nil {
+				return nil, err
+			}
+			id.Checksum = dgst
+		case pb.AttrHTTPFilename:
+			id.Filename = v
+		case pb.AttrHTTPPerm:
+			i, err := strconv.ParseInt(v, 0, 64)
+			if err != nil {
+				return nil, err
+			}
+			id.Perm = int(i)
+		case pb.AttrHTTPUID:
+			i, err := strconv.ParseInt(v, 0, 64)
+			if err != nil {
+				return nil, err
+			}
+			id.UID = int(i)
+		case pb.AttrHTTPGID:
+			i, err := strconv.ParseInt(v, 0, 64)
+			if err != nil {
+				return nil, err
+			}
+			id.GID = int(i)
+		}
+	}
+
+	return id, nil
 }
 
 type httpSourceHandler struct {
 	*httpSource
-	src      source.HTTPIdentifier
+	src      HTTPIdentifier
 	refID    string
 	cacheKey digest.Digest
 	sm       *session.Manager
 }
 
 func (hs *httpSource) Resolve(ctx context.Context, id source.Identifier, sm *session.Manager, _ solver.Vertex) (source.SourceInstance, error) {
-	httpIdentifier, ok := id.(*source.HTTPIdentifier)
+	httpIdentifier, ok := id.(*HTTPIdentifier)
 	if !ok {
 		return nil, errors.Errorf("invalid http identifier %v", id)
 	}

source/http/source_test.go

@@ -47,7 +47,7 @@ func TestHTTPSource(t *testing.T) {
 	})
 	defer server.Close()
 
-	id := &source.HTTPIdentifier{URL: server.URL + "/foo"}
+	id := &HTTPIdentifier{URL: server.URL + "/foo"}
 
 	h, err := hs.Resolve(ctx, id, nil, nil)
 	require.NoError(t, err)
@@ -166,7 +166,7 @@ func TestHTTPDefaultName(t *testing.T) {
 	})
 	defer server.Close()
 
-	id := &source.HTTPIdentifier{URL: server.URL}
+	id := &HTTPIdentifier{URL: server.URL}
 
 	h, err := hs.Resolve(ctx, id, nil, nil)
 	require.NoError(t, err)
@@ -206,7 +206,7 @@ func TestHTTPInvalidURL(t *testing.T) {
 	server := httpserver.NewTestServer(map[string]httpserver.Response{})
 	defer server.Close()
 
-	id := &source.HTTPIdentifier{URL: server.URL + "/foo"}
+	id := &HTTPIdentifier{URL: server.URL + "/foo"}
 
 	h, err := hs.Resolve(ctx, id, nil, nil)
 	require.NoError(t, err)
@@ -236,7 +236,7 @@ func TestHTTPChecksum(t *testing.T) {
 	})
 	defer server.Close()
 
-	id := &source.HTTPIdentifier{URL: server.URL + "/foo", Checksum: digest.FromBytes([]byte("content-different"))}
+	id := &HTTPIdentifier{URL: server.URL + "/foo", Checksum: digest.FromBytes([]byte("content-different"))}
 
 	h, err := hs.Resolve(ctx, id, nil, nil)
 	require.NoError(t, err)
@@ -262,7 +262,7 @@ func TestHTTPChecksum(t *testing.T) {
 	require.Equal(t, server.Stats("/foo").AllRequests, 1)
 	require.Equal(t, server.Stats("/foo").CachedRequests, 0)
 
-	id = &source.HTTPIdentifier{URL: server.URL + "/foo", Checksum: digest.FromBytes([]byte("content-correct"))}
+	id = &HTTPIdentifier{URL: server.URL + "/foo", Checksum: digest.FromBytes([]byte("content-correct"))}
 
 	h, err = hs.Resolve(ctx, id, nil, nil)
 	require.NoError(t, err)

source/identifier.go

@@ -1,18 +1,8 @@
 package source
 
 import (
-	"encoding/json"
+	"github.com/moby/buildkit/solver/llbsolver/provenance"
-	"strconv"
-	"strings"
-
-	"github.com/containerd/containerd/reference"
-	"github.com/moby/buildkit/client"
-	"github.com/moby/buildkit/solver/pb"
-	srctypes "github.com/moby/buildkit/source/types"
-	digest "github.com/opencontainers/go-digest"
-	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
-	"github.com/tonistiigi/fsutil"
 )
 
 var (
@@ -20,333 +10,11 @@ var (
 	errNotFound = errors.New("not found")
 )
 
-type ResolveMode int
-
-const (
-	ResolveModeDefault ResolveMode = iota
-	ResolveModeForcePull
-	ResolveModePreferLocal
-)
-
 type Identifier interface {
-	ID() string // until sources are in process this string comparison could be avoided
+	// Scheme returns the scheme of the identifier so that it can be routed back
-}
+	// to an appropriate Source.
-
+	Scheme() string
-func FromString(s string) (Identifier, error) {
+
-	// TODO: improve this
+	// Capture records the provenance of the identifier.
-	parts := strings.SplitN(s, "://", 2)
+	Capture(dest *provenance.Capture, pin string) error
-	if len(parts) != 2 {
-		return nil, errors.Wrapf(errInvalid, "failed to parse %s", s)
-	}
-
-	switch parts[0] {
-	case srctypes.DockerImageScheme:
-		return NewImageIdentifier(parts[1])
-	case srctypes.GitScheme:
-		return NewGitIdentifier(parts[1])
-	case srctypes.LocalScheme:
-		return NewLocalIdentifier(parts[1])
-	case srctypes.HTTPSScheme:
-		return NewHTTPIdentifier(parts[1], true)
-	case srctypes.HTTPScheme:
-		return NewHTTPIdentifier(parts[1], false)
-	case srctypes.OCIScheme:
-		return NewOCIIdentifier(parts[1])
-	default:
-		return nil, errors.Wrapf(errNotFound, "unknown schema %s", parts[0])
-	}
-}
-
-func FromLLB(op *pb.Op_Source, platform *pb.Platform) (Identifier, error) {
-	id, err := FromString(op.Source.Identifier)
-	if err != nil {
-		return nil, err
-	}
-
-	if id, ok := id.(*ImageIdentifier); ok {
-		if platform != nil {
-			id.Platform = &ocispecs.Platform{
-				OS:           platform.OS,
-				Architecture: platform.Architecture,
-				Variant:      platform.Variant,
-				OSVersion:    platform.OSVersion,
-				OSFeatures:   platform.OSFeatures,
-			}
-		}
-		for k, v := range op.Source.Attrs {
-			switch k {
-			case pb.AttrImageResolveMode:
-				rm, err := ParseImageResolveMode(v)
-				if err != nil {
-					return nil, err
-				}
-				id.ResolveMode = rm
-			case pb.AttrImageRecordType:
-				rt, err := parseImageRecordType(v)
-				if err != nil {
-					return nil, err
-				}
-				id.RecordType = rt
-			case pb.AttrImageLayerLimit:
-				l, err := strconv.Atoi(v)
-				if err != nil {
-					return nil, errors.Wrapf(err, "invalid layer limit %s", v)
-				}
-				if l <= 0 {
-					return nil, errors.Errorf("invalid layer limit %s", v)
-				}
-				id.LayerLimit = &l
-			}
-		}
-	}
-	if id, ok := id.(*GitIdentifier); ok {
-		for k, v := range op.Source.Attrs {
-			switch k {
-			case pb.AttrKeepGitDir:
-				if v == "true" {
-					id.KeepGitDir = true
-				}
-			case pb.AttrFullRemoteURL:
-				if !isGitTransport(v) {
-					v = "https://" + v
-				}
-				id.Remote = v
-			case pb.AttrAuthHeaderSecret:
-				id.AuthHeaderSecret = v
-			case pb.AttrAuthTokenSecret:
-				id.AuthTokenSecret = v
-			case pb.AttrKnownSSHHosts:
-				id.KnownSSHHosts = v
-			case pb.AttrMountSSHSock:
-				id.MountSSHSock = v
-			}
-		}
-	}
-	if id, ok := id.(*LocalIdentifier); ok {
-		for k, v := range op.Source.Attrs {
-			switch k {
-			case pb.AttrLocalSessionID:
-				id.SessionID = v
-				if p := strings.SplitN(v, ":", 2); len(p) == 2 {
-					id.Name = p[0] + "-" + id.Name
-					id.SessionID = p[1]
-				}
-			case pb.AttrIncludePatterns:
-				var patterns []string
-				if err := json.Unmarshal([]byte(v), &patterns); err != nil {
-					return nil, err
-				}
-				id.IncludePatterns = patterns
-			case pb.AttrExcludePatterns:
-				var patterns []string
-				if err := json.Unmarshal([]byte(v), &patterns); err != nil {
-					return nil, err
-				}
-				id.ExcludePatterns = patterns
-			case pb.AttrFollowPaths:
-				var paths []string
-				if err := json.Unmarshal([]byte(v), &paths); err != nil {
-					return nil, err
-				}
-				id.FollowPaths = paths
-			case pb.AttrSharedKeyHint:
-				id.SharedKeyHint = v
-			case pb.AttrLocalDiffer:
-				switch v {
-				case pb.AttrLocalDifferMetadata, "":
-					id.Differ = fsutil.DiffMetadata
-				case pb.AttrLocalDifferNone:
-					id.Differ = fsutil.DiffNone
-				}
-			}
-		}
-	}
-	if id, ok := id.(*HTTPIdentifier); ok {
-		for k, v := range op.Source.Attrs {
-			switch k {
-			case pb.AttrHTTPChecksum:
-				dgst, err := digest.Parse(v)
-				if err != nil {
-					return nil, err
-				}
-				id.Checksum = dgst
-			case pb.AttrHTTPFilename:
-				id.Filename = v
-			case pb.AttrHTTPPerm:
-				i, err := strconv.ParseInt(v, 0, 64)
-				if err != nil {
-					return nil, err
-				}
-				id.Perm = int(i)
-			case pb.AttrHTTPUID:
-				i, err := strconv.ParseInt(v, 0, 64)
-				if err != nil {
-					return nil, err
-				}
-				id.UID = int(i)
-			case pb.AttrHTTPGID:
-				i, err := strconv.ParseInt(v, 0, 64)
-				if err != nil {
-					return nil, err
-				}
-				id.GID = int(i)
-			}
-		}
-	}
-	if id, ok := id.(*OCIIdentifier); ok {
-		if platform != nil {
-			id.Platform = &ocispecs.Platform{
-				OS:           platform.OS,
-				Architecture: platform.Architecture,
-				Variant:      platform.Variant,
-				OSVersion:    platform.OSVersion,
-				OSFeatures:   platform.OSFeatures,
-			}
-		}
-		for k, v := range op.Source.Attrs {
-			switch k {
-			case pb.AttrOCILayoutSessionID:
-				id.SessionID = v
-			case pb.AttrOCILayoutStoreID:
-				id.StoreID = v
-			case pb.AttrOCILayoutLayerLimit:
-				l, err := strconv.Atoi(v)
-				if err != nil {
-					return nil, errors.Wrapf(err, "invalid layer limit %s", v)
-				}
-				if l <= 0 {
-					return nil, errors.Errorf("invalid layer limit %s", v)
-				}
-				id.LayerLimit = &l
-			}
-		}
-	}
-	return id, nil
-}
-
-type ImageIdentifier struct {
-	Reference   reference.Spec
-	Platform    *ocispecs.Platform
-	ResolveMode ResolveMode
-	RecordType  client.UsageRecordType
-	LayerLimit  *int
-}
-
-func NewImageIdentifier(str string) (*ImageIdentifier, error) {
-	ref, err := reference.Parse(str)
-	if err != nil {
-		return nil, errors.WithStack(err)
-	}
-
-	if ref.Object == "" {
-		return nil, errors.WithStack(reference.ErrObjectRequired)
-	}
-	return &ImageIdentifier{Reference: ref}, nil
-}
-
-func (*ImageIdentifier) ID() string {
-	return srctypes.DockerImageScheme
-}
-
-type LocalIdentifier struct {
-	Name            string
-	SessionID       string
-	IncludePatterns []string
-	ExcludePatterns []string
-	FollowPaths     []string
-	SharedKeyHint   string
-	Differ          fsutil.DiffType
-}
-
-func NewLocalIdentifier(str string) (*LocalIdentifier, error) {
-	return &LocalIdentifier{Name: str}, nil
-}
-
-func (*LocalIdentifier) ID() string {
-	return srctypes.LocalScheme
-}
-
-func NewHTTPIdentifier(str string, tls bool) (*HTTPIdentifier, error) {
-	proto := "https://"
-	if !tls {
-		proto = "http://"
-	}
-	return &HTTPIdentifier{TLS: tls, URL: proto + str}, nil
-}
-
-type HTTPIdentifier struct {
-	TLS      bool
-	URL      string
-	Checksum digest.Digest
-	Filename string
-	Perm     int
-	UID      int
-	GID      int
-}
-
-func (*HTTPIdentifier) ID() string {
-	return srctypes.HTTPSScheme
-}
-
-type OCIIdentifier struct {
-	Reference  reference.Spec
-	Platform   *ocispecs.Platform
-	SessionID  string
-	StoreID    string
-	LayerLimit *int
-}
-
-func NewOCIIdentifier(str string) (*OCIIdentifier, error) {
-	ref, err := reference.Parse(str)
-	if err != nil {
-		return nil, errors.WithStack(err)
-	}
-
-	if ref.Object == "" {
-		return nil, errors.WithStack(reference.ErrObjectRequired)
-	}
-	return &OCIIdentifier{Reference: ref}, nil
-}
-
-func (*OCIIdentifier) ID() string {
-	return srctypes.OCIScheme
-}
-
-func (r ResolveMode) String() string {
-	switch r {
-	case ResolveModeDefault:
-		return pb.AttrImageResolveModeDefault
-	case ResolveModeForcePull:
-		return pb.AttrImageResolveModeForcePull
-	case ResolveModePreferLocal:
-		return pb.AttrImageResolveModePreferLocal
-	default:
-		return ""
-	}
-}
-
-func ParseImageResolveMode(v string) (ResolveMode, error) {
-	switch v {
-	case pb.AttrImageResolveModeDefault, "":
-		return ResolveModeDefault, nil
-	case pb.AttrImageResolveModeForcePull:
-		return ResolveModeForcePull, nil
-	case pb.AttrImageResolveModePreferLocal:
-		return ResolveModePreferLocal, nil
-	default:
-		return 0, errors.Errorf("invalid resolvemode: %s", v)
-	}
-}
-
-func parseImageRecordType(v string) (client.UsageRecordType, error) {
-	switch client.UsageRecordType(v) {
-	case "", client.UsageRecordTypeRegular:
-		return client.UsageRecordTypeRegular, nil
-	case client.UsageRecordTypeInternal:
-		return client.UsageRecordTypeInternal, nil
-	case client.UsageRecordTypeFrontend:
-		return client.UsageRecordTypeFrontend, nil
-	default:
-		return "", errors.Errorf("invalid record type %s", v)
-	}
 }

source/local/identifier.go

@@ -0,0 +1,35 @@
+package local
+
+import (
+	"github.com/moby/buildkit/solver/llbsolver/provenance"
+	"github.com/moby/buildkit/source"
+	srctypes "github.com/moby/buildkit/source/types"
+	"github.com/tonistiigi/fsutil"
+)
+
+type LocalIdentifier struct {
+	Name            string
+	SessionID       string
+	IncludePatterns []string
+	ExcludePatterns []string
+	FollowPaths     []string
+	SharedKeyHint   string
+	Differ          fsutil.DiffType
+}
+
+func NewLocalIdentifier(str string) (*LocalIdentifier, error) {
+	return &LocalIdentifier{Name: str}, nil
+}
+
+func (*LocalIdentifier) Scheme() string {
+	return srctypes.LocalScheme
+}
+
+var _ source.Identifier = (*LocalIdentifier)(nil)
+
+func (id *LocalIdentifier) Capture(c *provenance.Capture, pin string) error {
+	c.AddLocal(provenance.LocalSource{
+		Name: id.Name,
+	})
+	return nil
+}

source/local/source.go

@@ -4,8 +4,10 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"strings"
 	"time"
 
+	"github.com/moby/buildkit/solver/pb"
 	srctypes "github.com/moby/buildkit/source/types"
 	"github.com/moby/buildkit/util/bklog"
 
@@ -41,12 +43,59 @@ type localSource struct {
 	cm cache.Accessor
 }
 
-func (ls *localSource) ID() string {
+func (ls *localSource) Schemes() []string {
-	return srctypes.LocalScheme
+	return []string{srctypes.LocalScheme}
+}
+
+func (ls *localSource) Identifier(scheme, ref string, attrs map[string]string, platform *pb.Platform) (source.Identifier, error) {
+	id, err := NewLocalIdentifier(ref)
+	if err != nil {
+		return nil, err
+	}
+
+	for k, v := range attrs {
+		switch k {
+		case pb.AttrLocalSessionID:
+			id.SessionID = v
+			if p := strings.SplitN(v, ":", 2); len(p) == 2 {
+				id.Name = p[0] + "-" + id.Name
+				id.SessionID = p[1]
+			}
+		case pb.AttrIncludePatterns:
+			var patterns []string
+			if err := json.Unmarshal([]byte(v), &patterns); err != nil {
+				return nil, err
+			}
+			id.IncludePatterns = patterns
+		case pb.AttrExcludePatterns:
+			var patterns []string
+			if err := json.Unmarshal([]byte(v), &patterns); err != nil {
+				return nil, err
+			}
+			id.ExcludePatterns = patterns
+		case pb.AttrFollowPaths:
+			var paths []string
+			if err := json.Unmarshal([]byte(v), &paths); err != nil {
+				return nil, err
+			}
+			id.FollowPaths = paths
+		case pb.AttrSharedKeyHint:
+			id.SharedKeyHint = v
+		case pb.AttrLocalDiffer:
+			switch v {
+			case pb.AttrLocalDifferMetadata, "":
+				id.Differ = fsutil.DiffMetadata
+			case pb.AttrLocalDifferNone:
+				id.Differ = fsutil.DiffNone
+			}
+		}
+	}
+
+	return id, nil
 }
 
 func (ls *localSource) Resolve(ctx context.Context, id source.Identifier, sm *session.Manager, _ solver.Vertex) (source.SourceInstance, error) {
-	localIdentifier, ok := id.(*source.LocalIdentifier)
+	localIdentifier, ok := id.(*LocalIdentifier)
 	if !ok {
 		return nil, errors.Errorf("invalid local identifier %v", id)
 	}
@@ -59,7 +108,7 @@ func (ls *localSource) Resolve(ctx context.Context, id source.Identifier, sm *se
 }
 
 type localSourceHandler struct {
-	src source.LocalIdentifier
+	src LocalIdentifier
 	sm  *session.Manager
 	*localSource
 }

source/manager.go

@@ -2,48 +2,83 @@ package source
 
 import (
 	"context"
+	"strings"
 	"sync"
 
 	"github.com/moby/buildkit/cache"
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/solver"
+	"github.com/moby/buildkit/solver/pb"
 	"github.com/pkg/errors"
 )
 
+// Source implementations provide "root" vertices in the graph that can be
+// constructed from a URI-like string and arbitrary attrs.
 type Source interface {
-	ID() string
+	// Schemes returns a list of SourceOp identifier schemes that this source
+	// should match.
+	Schemes() []string
+
+	// Identifier constructs an Identifier from the given scheme, ref, and attrs,
+	// all of which come from a SourceOp.
+	Identifier(scheme, ref string, attrs map[string]string, platform *pb.Platform) (Identifier, error)
+
+	// Resolve constructs an instance of the source from an Identifier.
 	Resolve(ctx context.Context, id Identifier, sm *session.Manager, vtx solver.Vertex) (SourceInstance, error)
 }
 
+// SourceInstance represents a cacheable vertex created by a Source.
 type SourceInstance interface {
+	// CacheKey returns the cache key for the instance.
 	CacheKey(ctx context.Context, g session.Group, index int) (key, pin string, opts solver.CacheOpts, done bool, err error)
+
+	// Snapshot creates a cache ref for the instance.
 	Snapshot(ctx context.Context, g session.Group) (cache.ImmutableRef, error)
 }
 
 type Manager struct {
 	mu      sync.Mutex
-	sources map[string]Source
+	schemes map[string]Source
 }
 
 func NewManager() (*Manager, error) {
 	return &Manager{
-		sources: make(map[string]Source),
+		schemes: make(map[string]Source),
 	}, nil
 }
 
 func (sm *Manager) Register(src Source) {
 	sm.mu.Lock()
-	sm.sources[src.ID()] = src
+	for _, scheme := range src.Schemes() {
+		sm.schemes[scheme] = src
+	}
 	sm.mu.Unlock()
 }
 
+func (sm *Manager) Identifier(op *pb.Op_Source, platform *pb.Platform) (Identifier, error) {
+	scheme, ref, ok := strings.Cut(op.Source.Identifier, "://")
+	if !ok {
+		return nil, errors.Wrapf(errInvalid, "failed to parse %s", op.Source.Identifier)
+	}
+
+	sm.mu.Lock()
+	source, found := sm.schemes[scheme]
+	sm.mu.Unlock()
+
+	if !found {
+		return nil, errors.Wrapf(errNotFound, "unknown scheme %s", scheme)
+	}
+
+	return source.Identifier(scheme, ref, op.Source.Attrs, platform)
+}
+
 func (sm *Manager) Resolve(ctx context.Context, id Identifier, sessM *session.Manager, vtx solver.Vertex) (SourceInstance, error) {
 	sm.mu.Lock()
-	src, ok := sm.sources[id.ID()]
+	src, ok := sm.schemes[id.Scheme()]
 	sm.mu.Unlock()
 
 	if !ok {
-		return nil, errors.Errorf("no handler for %s", id.ID())
+		return nil, errors.Errorf("no handler for %s", id.Scheme())
 	}
 
 	return src.Resolve(ctx, id, sessM, vtx)

util/resolver/pool.go

@@ -14,9 +14,10 @@ import (
 	"github.com/containerd/containerd/remotes/docker"
 	distreference "github.com/docker/distribution/reference"
 	"github.com/moby/buildkit/session"
-	"github.com/moby/buildkit/source"
+	"github.com/moby/buildkit/solver/pb"
 	"github.com/moby/buildkit/version"
 	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
 )
 
 // DefaultPool is the default shared resolver pool instance
@@ -125,7 +126,7 @@ type Resolver struct {
 	auth    *dockerAuthorizer
 
 	is   images.Store
-	mode source.ResolveMode
+	mode ResolveMode
 }
 
 // HostsFunc implements registry configuration of this Resolver
@@ -177,7 +178,7 @@ func (r *Resolver) WithSession(s session.Group) *Resolver {
 }
 
 // WithImageStore returns new resolver that can also resolve from local images store
-func (r *Resolver) WithImageStore(is images.Store, mode source.ResolveMode) *Resolver {
+func (r *Resolver) WithImageStore(is images.Store, mode ResolveMode) *Resolver {
 	r2 := *r
 	r2.Resolver = r.Resolver
 	r2.is = is
@@ -195,7 +196,7 @@ func (r *Resolver) Fetcher(ctx context.Context, ref string) (remotes.Fetcher, er
 
 // Resolve attempts to resolve the reference into a name and descriptor.
 func (r *Resolver) Resolve(ctx context.Context, ref string) (string, ocispecs.Descriptor, error) {
-	if r.mode == source.ResolveModePreferLocal && r.is != nil {
+	if r.mode == ResolveModePreferLocal && r.is != nil {
 		if img, err := r.is.Get(ctx, ref); err == nil {
 			return ref, img.Target, nil
 		}
@@ -207,7 +208,7 @@ func (r *Resolver) Resolve(ctx context.Context, ref string) (string, ocispecs.De
 		return n, desc, nil
 	}
 
-	if r.mode == source.ResolveModeDefault && r.is != nil {
+	if r.mode == ResolveModeDefault && r.is != nil {
 		if img, err := r.is.Get(ctx, ref); err == nil {
 			return ref, img.Target, nil
 		}
@@ -215,3 +216,37 @@ func (r *Resolver) Resolve(ctx context.Context, ref string) (string, ocispecs.De
 
 	return "", ocispecs.Descriptor{}, err
 }
+
+type ResolveMode int
+
+const (
+	ResolveModeDefault ResolveMode = iota
+	ResolveModeForcePull
+	ResolveModePreferLocal
+)
+
+func (r ResolveMode) String() string {
+	switch r {
+	case ResolveModeDefault:
+		return pb.AttrImageResolveModeDefault
+	case ResolveModeForcePull:
+		return pb.AttrImageResolveModeForcePull
+	case ResolveModePreferLocal:
+		return pb.AttrImageResolveModePreferLocal
+	default:
+		return ""
+	}
+}
+
+func ParseImageResolveMode(v string) (ResolveMode, error) {
+	switch v {
+	case pb.AttrImageResolveModeDefault, "":
+		return ResolveModeDefault, nil
+	case pb.AttrImageResolveModeForcePull:
+		return ResolveModeForcePull, nil
+	case pb.AttrImageResolveModePreferLocal:
+		return ResolveModePreferLocal, nil
+	default:
+		return 0, errors.Errorf("invalid resolvemode: %s", v)
+	}
+}

worker/tests/common.go

@@ -13,7 +13,7 @@ import (
 	"github.com/moby/buildkit/identity"
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/snapshot"
-	"github.com/moby/buildkit/source"
+	"github.com/moby/buildkit/source/containerimage"
 	"github.com/moby/buildkit/worker/base"
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/require"
@@ -21,7 +21,7 @@ import (
 )
 
 func NewBusyboxSourceSnapshot(ctx context.Context, t *testing.T, w *base.Worker, sm *session.Manager) cache.ImmutableRef {
-	img, err := source.NewImageIdentifier("docker.io/library/busybox:latest")
+	img, err := containerimage.NewImageIdentifier("docker.io/library/busybox:latest")
 	require.NoError(t, err)
 	src, err := w.SourceManager.Resolve(ctx, img, sm, nil)
 	require.NoError(t, err)