mobi/buildkit
1
0
Fork 0
mirror of https://github.com/moby/buildkit.git synced 2025-04-18 18:04:03 +03:00
Code

Merge pull request #5897 from tonistiigi/local-metadata-transfer

Browse Source 
source: add metadata-only transfer option for local source
This commit is contained in:
CrazyMax 2025-04-11 17:50:29 +02:00 committed by GitHub
commit 43fc7e8585
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 444 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

161
client/client_test.go
View File

@ -9,6 +9,7 @@ import (
"crypto/rsa"
"crypto/x509"
"encoding/base64"
"encoding/binary"
"encoding/json"
"encoding/pem"
"fmt"
@ -73,6 +74,7 @@ import (
"github.com/spdx/tools-golang/spdx"
"github.com/stretchr/testify/require"
"github.com/tonistiigi/fsutil"
fsutiltypes "github.com/tonistiigi/fsutil/types"
"golang.org/x/crypto/ssh/agent"
"golang.org/x/mod/semver"
"golang.org/x/sync/errgroup"
@ -229,6 +231,7 @@ var allTests = []func(t *testing.T, sb integration.Sandbox){
testFrontendLintSkipVerifyPlatforms,
testRunValidExitCodes,
testFileOpSymlink,
testMetadataOnlyLocal,
}
func TestIntegration(t *testing.T) {
@ -8559,6 +8562,150 @@ func testParallelLocalBuilds(t *testing.T, sb integration.Sandbox) {
require.NoError(t, err)
}
func testMetadataOnlyLocal(t *testing.T, sb integration.Sandbox) {
ctx, cancel := context.WithCancelCause(sb.Context())
defer func() { cancel(errors.WithStack(context.Canceled)) }()
c, err := New(ctx, sb.Address())
require.NoError(t, err)
defer c.Close()
srcDir := integration.Tmpdir(
t,
fstest.CreateFile("data", []byte("contents"), 0600),
fstest.CreateDir("dir", 0700),
fstest.CreateFile("dir/file1", []byte("file1"), 0600),
fstest.CreateFile("dir/file2", []byte("file2"), 0600),
fstest.CreateDir("dir/subdir", 0700),
fstest.CreateDir("dir/subdir2", 0700),
fstest.CreateFile("dir/subdir/bar1", []byte("bar1"), 0600),
fstest.CreateFile("dir/subdir/bar2", []byte("bar2"), 0600),
fstest.CreateFile("dir/subdir2/bar3", []byte("bar3"), 0600),
fstest.CreateFile("foo", []byte("foo"), 0602),
)
def, err := llb.Local("source", llb.MetadataOnlyTransfer([]string{"dir/**/*1", "foo"})).Marshal(sb.Context())
require.NoError(t, err)
destDir := t.TempDir()
_, err = c.Solve(ctx, def, SolveOpt{
Exports: []ExportEntry{
{
Type: ExporterLocal,
OutputDir: destDir,
},
},
LocalMounts: map[string]fsutil.FS{
"source": srcDir,
},
}, nil)
require.NoError(t, err)
_, err = os.ReadFile(filepath.Join(destDir, "data"))
require.Error(t, err)
require.True(t, os.IsNotExist(err))
act, err := os.ReadFile(filepath.Join(destDir, "dir/file1"))
require.NoError(t, err)
require.Equal(t, "file1", string(act))
act, err = os.ReadFile(filepath.Join(destDir, "foo"))
require.NoError(t, err)
require.Equal(t, "foo", string(act))
_, err = os.ReadFile(filepath.Join(destDir, "dir/file2"))
require.Error(t, err)
require.True(t, os.IsNotExist(err))
act, err = os.ReadFile(filepath.Join(destDir, "dir/subdir/bar1"))
require.NoError(t, err)
require.Equal(t, "bar1", string(act))
_, err = os.Stat(filepath.Join(destDir, "dir/subdir2"))
require.Error(t, err)
require.True(t, os.IsNotExist(err))
_, err = os.ReadFile(filepath.Join(destDir, "dir/subdir/bar2"))
require.Error(t, err)
require.True(t, os.IsNotExist(err))
dt, err := os.ReadFile(filepath.Join(destDir, ".fsutil-metadata"))
require.NoError(t, err)
stats := parseFSMetadata(t, dt)
require.Equal(t, 10, len(stats))
require.Equal(t, "data", stats[0].Path)
require.Equal(t, "dir", stats[1].Path)
require.Equal(t, "dir/file1", stats[2].Path)
require.Equal(t, "dir/file2", stats[3].Path)
require.Equal(t, "dir/subdir", stats[4].Path)
require.Equal(t, "dir/subdir/bar1", stats[5].Path)
require.Equal(t, "dir/subdir/bar2", stats[6].Path)
require.Equal(t, "dir/subdir2", stats[7].Path)
require.Equal(t, "dir/subdir2/bar3", stats[8].Path)
require.Equal(t, "foo", stats[9].Path)
err = os.RemoveAll(filepath.Join(srcDir.Name, "dir/subdir"))
require.NoError(t, err)
err = os.WriteFile(filepath.Join(srcDir.Name, "dir/file1"), []byte("file1-updated"), 0600)
require.NoError(t, err)
err = os.WriteFile(filepath.Join(srcDir.Name, "dir/bar1"), []byte("bar1"), 0600)
require.NoError(t, err)
def, err = llb.Local("source", llb.MetadataOnlyTransfer([]string{"dir/**/*1", "foo"})).Marshal(sb.Context())
require.NoError(t, err)
destDir = t.TempDir()
_, err = c.Solve(ctx, def, SolveOpt{
Exports: []ExportEntry{
{
Type: ExporterLocal,
OutputDir: destDir,
},
},
LocalMounts: map[string]fsutil.FS{
"source": srcDir,
},
}, nil)
require.NoError(t, err)
_, err = os.ReadFile(filepath.Join(destDir, "data"))
require.Error(t, err)
require.True(t, os.IsNotExist(err))
act, err = os.ReadFile(filepath.Join(destDir, "dir/file1"))
require.NoError(t, err)
require.Equal(t, "file1-updated", string(act))
act, err = os.ReadFile(filepath.Join(destDir, "dir/bar1"))
require.NoError(t, err)
require.Equal(t, "bar1", string(act))
_, err = os.Stat(filepath.Join(destDir, "dir/subdir"))
require.Error(t, err)
require.True(t, os.IsNotExist(err))
dt, err = os.ReadFile(filepath.Join(destDir, ".fsutil-metadata"))
require.NoError(t, err)
stats = parseFSMetadata(t, dt)
require.Equal(t, 8, len(stats))
require.Equal(t, "data", stats[0].Path)
require.Equal(t, "dir", stats[1].Path)
require.Equal(t, "dir/bar1", stats[2].Path)
require.Equal(t, "dir/file1", stats[3].Path)
require.Equal(t, "dir/file2", stats[4].Path)
require.Equal(t, "dir/subdir2", stats[5].Path)
require.Equal(t, "dir/subdir2/bar3", stats[6].Path)
require.Equal(t, "foo", stats[7].Path)
}
// testRelativeMountpoint is a test that relative paths for mountpoints don't
// fail when runc is upgraded to at least rc95, which introduces an error when
// mountpoints are not absolute. Relative paths should be transformed to
@ -11769,3 +11916,17 @@ devices:
require.NotContains(t, strings.TrimSpace(string(dt)), `BAZ=injected`)
require.NotContains(t, strings.TrimSpace(string(dt)), `QUX=injected`)
}
func parseFSMetadata(t *testing.T, dt []byte) []fsutiltypes.Stat {
var m []fsutiltypes.Stat
for len(dt) > 0 {
var s fsutiltypes.Stat
n := binary.LittleEndian.Uint32(dt[:4])
dt = dt[4:]
err := s.Unmarshal(dt[:n])
require.NoError(t, err)
m = append(m, *s.CloneVT())
dt = dt[n:]
}
return m
}

33
client/llb/source.go
View File

@ -435,6 +435,13 @@ func Local(name string, opts ...LocalOption) State {
addCap(&gi.Constraints, pb.CapSourceLocalDiffer)
}
}
if gi.MetadataOnlyCollector {
attrs[pb.AttrMetadataTransfer] = "true"
if gi.MetadataOnlyExceptions != "" {
attrs[pb.AttrMetadataTransferExclude] = gi.MetadataOnlyExceptions
}
addCap(&gi.Constraints, pb.CapSourceMetadataTransfer)
}
addCap(&gi.Constraints, pb.CapSourceLocal)
@ -506,6 +513,18 @@ func Differ(t DiffType, required bool) LocalOption {
})
}
func MetadataOnlyTransfer(exceptions []string) LocalOption {
return localOptionFunc(func(li *LocalInfo) {
li.MetadataOnlyCollector = true
if len(exceptions) == 0 {
li.MetadataOnlyExceptions = ""
} else {
dt, _ := json.Marshal(exceptions) // empty on error
li.MetadataOnlyExceptions = string(dt)
}
})
}
func OCILayout(ref string, opts ...OCILayoutOption) State {
gi := &OCILayoutInfo{}
@ -578,12 +597,14 @@ type DifferInfo struct {
type LocalInfo struct {
constraintsWrapper
SessionID string
IncludePatterns string
ExcludePatterns string
FollowPaths string
SharedKeyHint string
Differ DifferInfo
SessionID string
IncludePatterns string
ExcludePatterns string
FollowPaths string
SharedKeyHint string
Differ DifferInfo
MetadataOnlyCollector bool
MetadataOnlyExceptions string
}
func HTTP(url string, opts ...HTTPOption) State {

2
go.mod
View File

@ -73,7 +73,7 @@ require (
github.com/spdx/tools-golang v0.5.3
github.com/stretchr/testify v1.10.0
github.com/tonistiigi/dchapes-mode v0.0.0-20250318174251-73d941a28323
github.com/tonistiigi/fsutil v0.0.0-20250318190121-d73a4b3b8a7e
github.com/tonistiigi/fsutil v0.0.0-20250410151801-5b74a7ad7583
github.com/tonistiigi/go-actions-cache v0.0.0-20250228231703-3e9a6642607f
github.com/tonistiigi/go-archvariant v1.0.0
github.com/tonistiigi/go-csvvalue v0.0.0-20240710180619-ddb21b71c0b4

4
go.sum
View File

@ -397,8 +397,8 @@ github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtse
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/tonistiigi/dchapes-mode v0.0.0-20250318174251-73d941a28323 h1:r0p7fK56l8WPequOaR3i9LBqfPtEdXIQbUTzT55iqT4=
github.com/tonistiigi/dchapes-mode v0.0.0-20250318174251-73d941a28323/go.mod h1:3Iuxbr0P7D3zUzBMAZB+ois3h/et0shEz0qApgHYGpY=
github.com/tonistiigi/fsutil v0.0.0-20250318190121-d73a4b3b8a7e h1:AiXT0JHwQA52AEOVMsxRytSI9mdJSie5gUp6OQ1R8fU=
github.com/tonistiigi/fsutil v0.0.0-20250318190121-d73a4b3b8a7e/go.mod h1:BKdcez7BiVtBvIcef90ZPc6ebqIWr4JWD7+EvLm6J98=
github.com/tonistiigi/fsutil v0.0.0-20250410151801-5b74a7ad7583 h1:mK+ZskNt7SG4dxfKIi27C7qHAQzyjAVt1iyTf0hmsNc=
github.com/tonistiigi/fsutil v0.0.0-20250410151801-5b74a7ad7583/go.mod h1:BKdcez7BiVtBvIcef90ZPc6ebqIWr4JWD7+EvLm6J98=
github.com/tonistiigi/go-actions-cache v0.0.0-20250228231703-3e9a6642607f h1:q/SWz3Bz0KtAsqaBo73CHVXjaz5O8PDnmD2JHVhgYnE=
github.com/tonistiigi/go-actions-cache v0.0.0-20250228231703-3e9a6642607f/go.mod h1:h0oRlVs3NoFIHysRQ4rU1+RG4QmU0M2JVSwTYrB4igk=
github.com/tonistiigi/go-archvariant v1.0.0 h1:5LC1eDWiBNflnTF1prCiX09yfNHIxDC/aukdhCdTyb0=

3
session/filesync/diffcopy.go
View File

@ -83,7 +83,7 @@ func (wc *streamWriterCloser) Close() error {
return nil
}
func recvDiffCopy(ds grpc.ClientStream, dest string, cu CacheUpdater, progress progressCb, differ fsutil.DiffType, filter func(string, *fstypes.Stat) bool) (err error) {
func recvDiffCopy(ds grpc.ClientStream, dest string, cu CacheUpdater, progress progressCb, differ fsutil.DiffType, filter, metadataOnlyFilter func(string, *fstypes.Stat) bool) (err error) {
st := time.Now()
defer func() {
bklog.G(ds.Context()).Debugf("diffcopy took: %v", time.Since(st))
@ -107,6 +107,7 @@ func recvDiffCopy(ds grpc.ClientStream, dest string, cu CacheUpdater, progress p
ProgressCb: progress,
Filter: fsutil.FilterFunc(filter),
Differ: differ,
MetadataOnly: metadataOnlyFilter,
}))
}

35
session/filesync/filesync.go
View File

@ -3,7 +3,7 @@ package filesync
import (
"context"
"fmt"
io "io"
"io"
"net/url"
"os"
"strconv"
@ -145,7 +145,7 @@ type progressCb func(int, bool)
type protocol struct {
name string
sendFn func(stream Stream, fs fsutil.FS, progress progressCb) error
recvFn func(stream grpc.ClientStream, destDir string, cu CacheUpdater, progress progressCb, differ fsutil.DiffType, mapFunc func(string, *fstypes.Stat) bool) error
recvFn func(stream grpc.ClientStream, destDir string, cu CacheUpdater, progress progressCb, differ fsutil.DiffType, mapFunc, metadataOnlyFilter func(string, *fstypes.Stat) bool) error
}
var supportedProtocols = []protocol{
@ -158,15 +158,17 @@ var supportedProtocols = []protocol{
// FSSendRequestOpt defines options for FSSend request
type FSSendRequestOpt struct {
Name string
IncludePatterns []string
ExcludePatterns []string
FollowPaths []string
DestDir string
CacheUpdater CacheUpdater
ProgressCb func(int, bool)
Filter func(string, *fstypes.Stat) bool
Differ fsutil.DiffType
Name string
IncludePatterns []string
ExcludePatterns []string
FollowPaths []string
DestDir string
CacheUpdater CacheUpdater
ProgressCb func(int, bool)
Filter func(string, *fstypes.Stat) bool
Differ fsutil.DiffType
MetadataOnly bool
MetadataOnlyFilter func(string, *fstypes.Stat) bool
}
// CacheUpdater is an object capable of sending notifications for the cache hash changes
@ -233,7 +235,16 @@ func FSSync(ctx context.Context, c session.Caller, opt FSSendRequestOpt) error {
panic(fmt.Sprintf("invalid protocol: %q", pr.name))
}
return pr.recvFn(stream, opt.DestDir, opt.CacheUpdater, opt.ProgressCb, opt.Differ, opt.Filter)
var metadataOnlyFilter func(string, *fstypes.Stat) bool
if opt.MetadataOnly {
if opt.MetadataOnlyFilter != nil {
metadataOnlyFilter = opt.MetadataOnlyFilter
} else {
metadataOnlyFilter = func(string, *fstypes.Stat) bool { return false }
}
}
return pr.recvFn(stream, opt.DestDir, opt.CacheUpdater, opt.ProgressCb, opt.Differ, opt.Filter, metadataOnlyFilter)
}
type FSSyncTarget interface {

2
solver/pb/attr.go
View File

@ -12,6 +12,8 @@ const AttrIncludePatterns = "local.includepattern"
const AttrFollowPaths = "local.followpaths"
const AttrExcludePatterns = "local.excludepatterns"
const AttrSharedKeyHint = "local.sharedkeyhint"
const AttrMetadataTransfer = "local.metadatatransfer"
const AttrMetadataTransferExclude = "local.metadatatransferexclude"
const AttrLLBDefinitionFilename = "llbbuild.filename"

7
solver/pb/caps.go
View File

@ -21,6 +21,7 @@ const (
CapSourceLocalExcludePatterns apicaps.CapID = "source.local.excludepatterns"
CapSourceLocalSharedKeyHint apicaps.CapID = "source.local.sharedkeyhint"
CapSourceLocalDiffer apicaps.CapID = "source.local.differ"
CapSourceMetadataTransfer apicaps.CapID = "source.local.metadatatransfer"
CapSourceGit apicaps.CapID = "source.git"
CapSourceGitKeepDir apicaps.CapID = "source.git.keepgitdir"
@ -173,6 +174,12 @@ func init() {
Status: apicaps.CapStatusExperimental,
})
Caps.Init(apicaps.Cap{
ID: CapSourceMetadataTransfer,
Enabled: true,
Status: apicaps.CapStatusExperimental,
})
Caps.Init(apicaps.Cap{
ID: CapSourceGit,
Enabled: true,

16
source/local/identifier.go
View File

@ -9,13 +9,15 @@ import (
)
type LocalIdentifier struct {
Name string
SessionID string
IncludePatterns []string
ExcludePatterns []string
FollowPaths []string
SharedKeyHint string
Differ fsutil.DiffType
Name string
SessionID string
IncludePatterns []string
ExcludePatterns []string
FollowPaths []string
SharedKeyHint string
Differ fsutil.DiffType
MetadataOnly bool
MetadataExceptions []string
}
func NewLocalIdentifier(str string) (*LocalIdentifier, error) {

51
source/local/source.go
View File

@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"strconv"
"strings"
"time"
@ -19,6 +20,7 @@ import (
srctypes "github.com/moby/buildkit/source/types"
"github.com/moby/buildkit/util/bklog"
"github.com/moby/buildkit/util/progress"
"github.com/moby/patternmatcher"
"github.com/moby/sys/user"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
@ -87,6 +89,18 @@ func (ls *localSource) Identifier(scheme, ref string, attrs map[string]string, p
case pb.AttrLocalDifferNone:
id.Differ = fsutil.DiffNone
}
case pb.AttrMetadataTransfer:
b, err := strconv.ParseBool(v)
if err != nil {
return nil, errors.Wrapf(err, "invalid value for local.metadatatransfer %q", v)
}
id.MetadataOnly = b
case pb.AttrMetadataTransferExclude:
var exceptions []string
if err := json.Unmarshal([]byte(v), &exceptions); err != nil {
return nil, err
}
id.MetadataExceptions = exceptions
}
}
@ -123,11 +137,20 @@ func (ls *localSourceHandler) CacheKey(ctx context.Context, g session.Group, ind
sessionID = id
}
dt, err := json.Marshal(struct {
SessionID string
IncludePatterns []string
ExcludePatterns []string
FollowPaths []string
}{SessionID: sessionID, IncludePatterns: ls.src.IncludePatterns, ExcludePatterns: ls.src.ExcludePatterns, FollowPaths: ls.src.FollowPaths})
SessionID string
IncludePatterns []string
ExcludePatterns []string
FollowPaths []string
MetadataTransfer bool `json:",omitempty"`
MetadataExceptions []string `json:",omitempty"`
}{
SessionID: sessionID,
IncludePatterns: ls.src.IncludePatterns,
ExcludePatterns: ls.src.ExcludePatterns,
FollowPaths: ls.src.FollowPaths,
MetadataTransfer: ls.src.MetadataOnly,
MetadataExceptions: ls.src.MetadataExceptions,
})
if err != nil {
return "", "", nil, false, err
}
@ -174,7 +197,11 @@ func (ls *localSourceHandler) snapshotWithAnySession(ctx context.Context, g sess
}
func (ls *localSourceHandler) snapshot(ctx context.Context, caller session.Caller) (out cache.ImmutableRef, retErr error) {
sharedKey := ls.src.Name + ":" + ls.src.SharedKeyHint + ":" + caller.SharedKey() // TODO: replace caller.SharedKey() with source based hint from client(absolute-path+nodeid)
metaSfx := ""
if ls.src.MetadataOnly {
metaSfx = ":metadata"
}
sharedKey := ls.src.Name + ":" + ls.src.SharedKeyHint + ":" + caller.SharedKey() + metaSfx // TODO: replace caller.SharedKey() with source based hint from client(absolute-path+nodeid)
var mutable cache.MutableRef
sis, err := searchSharedKey(ctx, ls.cm, sharedKey)
@ -243,6 +270,18 @@ func (ls *localSourceHandler) snapshot(ctx context.Context, caller session.Calle
CacheUpdater: &cacheUpdater{cc, mount.IdentityMapping()},
ProgressCb: newProgressHandler(ctx, "transferring "+ls.src.Name+":"),
Differ: ls.src.Differ,
MetadataOnly: ls.src.MetadataOnly,
}
if opt.MetadataOnly && len(ls.src.MetadataExceptions) > 0 {
matcher, err := patternmatcher.New(ls.src.MetadataExceptions)
if err != nil {
return nil, errors.WithStack(err)
}
opt.MetadataOnlyFilter = func(p string, _ *fstypes.Stat) bool {
v, err := matcher.MatchesOrParentMatches(p)
return err == nil && v
}
}
if idmap := mount.IdentityMapping(); idmap != nil {

44
vendor/github.com/tonistiigi/fsutil/buffer.go generated vendored Normal file
View File

@ -0,0 +1,44 @@
package fsutil
import (
"io"
)
const chunkSize = 32 * 1024
type buffer struct {
chunks [][]byte
}
func (b *buffer) alloc(n int) []byte {
if n > chunkSize {
buf := make([]byte, n)
b.chunks = append(b.chunks, buf)
return buf
}
if len(b.chunks) != 0 {
lastChunk := b.chunks[len(b.chunks)-1]
l := len(lastChunk)
if l+n <= cap(lastChunk) {
lastChunk = lastChunk[:l+n]
b.chunks[len(b.chunks)-1] = lastChunk
return lastChunk[l : l+n]
}
}
buf := make([]byte, n, chunkSize)
b.chunks = append(b.chunks, buf)
return buf
}
func (b *buffer) WriteTo(w io.Writer) (n int64, err error) {
for _, c := range b.chunks {
m, err := w.Write(c)
n += int64(m)
if err != nil {
return n, err
}
}
return n, nil
}

132
vendor/github.com/tonistiigi/fsutil/receive.go generated vendored
View File

@ -32,6 +32,7 @@ package fsutil
import (
"context"
"encoding/binary"
"io"
"os"
"path/filepath"
@ -51,6 +52,8 @@ const (
DiffContent
)
const metadataPath = ".fsutil-metadata"
type ReceiveOpt struct {
NotifyHashed ChangeFunc
ContentHasher ContentHasher
@ -58,6 +61,7 @@ type ReceiveOpt struct {
Merge bool
Filter FilterFunc
Differ DiffType
MetadataOnly FilterFunc
}
func Receive(ctx context.Context, conn Stream, dest string, opt ReceiveOpt) error {
@ -75,21 +79,23 @@ func Receive(ctx context.Context, conn Stream, dest string, opt ReceiveOpt) erro
merge: opt.Merge,
filter: opt.Filter,
differ: opt.Differ,
metadataOnly: opt.MetadataOnly,
}
return r.run(ctx)
}
type receiver struct {
dest string
conn Stream
files map[string]uint32
pipes map[uint32]io.WriteCloser
mu sync.RWMutex
muPipes sync.RWMutex
progressCb func(int, bool)
merge bool
filter FilterFunc
differ DiffType
dest string
conn Stream
files map[string]uint32
pipes map[uint32]io.WriteCloser
mu sync.RWMutex
muPipes sync.RWMutex
progressCb func(int, bool)
merge bool
filter FilterFunc
differ DiffType
metadataOnly FilterFunc
notifyHashed ChangeFunc
contentHasher ContentHasher
@ -164,6 +170,11 @@ func (r *receiver) run(ctx context.Context) error {
}
w := newDynamicWalker()
metadataTransfer := r.metadataOnly != nil
// buffer Stat metadata in framed proto
metadataBuffer := &buffer{}
// stack of parent paths that can be replayed if metadata filter matches
metadataParents := newStack[*currentPath]()
g.Go(func() (retErr error) {
defer func() {
@ -223,10 +234,26 @@ func (r *receiver) run(ctx context.Context) error {
// e.g. a linux path foo/bar\baz cannot be represented on windows
return errors.WithStack(&os.PathError{Path: p.Stat.Path, Err: syscall.EINVAL, Op: "unrepresentable path"})
}
var metaOnly bool
if metadataTransfer {
if path == metadataPath {
continue
}
n := p.Stat.SizeVT()
dt := metadataBuffer.alloc(n + 4)
binary.LittleEndian.PutUint32(dt[0:4], uint32(n))
_, err := p.Stat.MarshalToSizedBufferVT(dt[4:])
if err != nil {
return err
}
if !r.metadataOnly(path, p.Stat) {
metaOnly = true
}
}
p.Stat.Path = path
p.Stat.Linkname = filepath.FromSlash(p.Stat.Linkname)
if fileCanRequestData(os.FileMode(p.Stat.Mode)) {
if !metaOnly && fileCanRequestData(os.FileMode(p.Stat.Mode)) {
r.mu.Lock()
r.files[p.Stat.Path] = i
r.mu.Unlock()
@ -240,6 +267,31 @@ func (r *receiver) run(ctx context.Context) error {
if err := r.hlValidator.HandleChange(ChangeKindAdd, cp.path, &StatInfo{cp.stat}, nil); err != nil {
return err
}
if metadataTransfer {
parent := filepath.Dir(cp.path)
isDir := os.FileMode(p.Stat.Mode).IsDir()
for {
last, ok := metadataParents.peek()
if !ok || parent == last.path {
break
}
metadataParents.pop()
}
if isDir {
metadataParents.push(cp)
}
if metaOnly {
continue
} else {
for _, cp := range metadataParents.items {
if err := w.update(cp); err != nil {
return err
}
}
metadataParents.clear()
}
}
if err := w.update(cp); err != nil {
return err
}
@ -272,7 +324,27 @@ func (r *receiver) run(ctx context.Context) error {
}
}
})
return g.Wait()
if err := g.Wait(); err != nil {
return err
}
if !metadataTransfer {
return nil
}
// although we don't allow tranferring metadataPath, make sure there was no preexisting file/symlink
os.Remove(filepath.Join(r.dest, metadataPath))
f, err := os.OpenFile(filepath.Join(r.dest, metadataPath), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
return err
}
if _, err := metadataBuffer.WriteTo(f); err != nil {
f.Close()
return err
}
return f.Close()
}
func (r *receiver) asyncDataFunc(ctx context.Context, p string, wc io.WriteCloser) error {
@ -327,3 +399,39 @@ func (w *wrappedWriteCloser) Wait(ctx context.Context) error {
return w.err
}
}
type stack[T any] struct {
items []T
}
func newStack[T any]() *stack[T] {
return &stack[T]{
items: make([]T, 0, 8),
}
}
func (s *stack[T]) push(v T) {
s.items = append(s.items, v)
}
func (s *stack[T]) pop() (T, bool) {
if len(s.items) == 0 {
var zero T
return zero, false
}
v := s.items[len(s.items)-1]
s.items = s.items[:len(s.items)-1]
return v, true
}
func (s *stack[T]) peek() (T, bool) {
if len(s.items) == 0 {
var zero T
return zero, false
}
return s.items[len(s.items)-1], true
}
func (s *stack[T]) clear() {
s.items = s.items[:0]
}

2
vendor/modules.txt vendored
View File

@ -779,7 +779,7 @@ github.com/syndtr/gocapability/capability
# github.com/tonistiigi/dchapes-mode v0.0.0-20250318174251-73d941a28323
## explicit; go 1.21
github.com/tonistiigi/dchapes-mode
# github.com/tonistiigi/fsutil v0.0.0-20250318190121-d73a4b3b8a7e
# github.com/tonistiigi/fsutil v0.0.0-20250410151801-5b74a7ad7583
## explicit; go 1.21
github.com/tonistiigi/fsutil
github.com/tonistiigi/fsutil/copy