docker/regclient
1
0
Fork 0
mirror of https://github.com/regclient/regclient.git synced 2025-04-18 22:44:00 +03:00
Code

Feat: Support external referrers in regsync

Browse Source 
Signed-off-by: Brandon Mitchell <git@bmitch.net>
This commit is contained in:
Brandon Mitchell 2024-12-08 16:30:27 -05:00
parent 88ac519dc6
commit 851cc63093
No known key found for this signature in database
GPG Key ID: 6E0FF28C767A8BEE
4 changed files with 129 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
cmd/regsync/config.go
View File

@ -40,6 +40,8 @@ type ConfigDefaults struct {
DigestTags *bool `yaml:"digestTags" json:"digestTags"`
Referrers *bool `yaml:"referrers" json:"referrers"`
ReferrerFilters []ConfigReferrerFilter `yaml:"referrerFilters" json:"referrerFilters"`
ReferrerSrc string `yaml:"referrerSource" json:"referrerSource"`
ReferrerTgt string `yaml:"referrerTarget" json:"referrerTarget"`
FastCheck *bool `yaml:"fastCheck" json:"fastCheck"`
ForceRecursive *bool `yaml:"forceRecursive" json:"forceRecursive"`
IncludeExternal *bool `yaml:"includeExternal" json:"includeExternal"`
@ -69,6 +71,8 @@ type ConfigSync struct {
DigestTags *bool `yaml:"digestTags" json:"digestTags"`
Referrers *bool `yaml:"referrers" json:"referrers"`
ReferrerFilters []ConfigReferrerFilter `yaml:"referrerFilters" json:"referrerFilters"`
ReferrerSrc string `yaml:"referrerSource" json:"referrerSource"`
ReferrerTgt string `yaml:"referrerTarget" json:"referrerTarget"`
Platform string `yaml:"platform" json:"platform"`
Platforms []string `yaml:"platforms" json:"platforms"`
FastCheck *bool `yaml:"fastCheck" json:"fastCheck"`
@ -209,11 +213,24 @@ func configExpandTemplates(c *Config) error {
}
c.Sync[i].Source = val
dataSync.Sync.Source = val
val, err = template.String(c.Sync[i].ReferrerSrc, dataSync)
if err != nil {
return err
}
c.Sync[i].ReferrerSrc = val
dataSync.Sync.ReferrerSrc = val
val, err = template.String(c.Sync[i].Target, dataSync)
if err != nil {
return err
}
c.Sync[i].Target = val
val, err = template.String(c.Sync[i].ReferrerTgt, dataSync)
if err != nil {
return err
}
c.Sync[i].ReferrerTgt = val
dataSync.Sync.ReferrerTgt = val
// templates for Backup are expanded in each sync step
}
return nil
}
@ -255,6 +272,12 @@ func syncSetDefaults(s *ConfigSync, d ConfigDefaults) {
if s.ReferrerFilters == nil {
s.ReferrerFilters = d.ReferrerFilters
}
if s.ReferrerSrc == "" && d.ReferrerSrc != "" {
s.ReferrerSrc = d.ReferrerSrc
}
if s.ReferrerTgt == "" && d.ReferrerTgt != "" {
s.ReferrerTgt = d.ReferrerTgt
}
if s.FastCheck == nil {
b := (d.FastCheck != nil && *d.FastCheck)
s.FastCheck = &b

80
cmd/regsync/regsync_test.go
View File

@ -40,6 +40,10 @@ func TestProcess(t *testing.T) {
if err != nil {
t.Fatalf("failed to copy testrepo to tempdir: %v", err)
}
err = copyfs.Copy(tempDir+"/external", "../../testdata/external")
if err != nil {
t.Fatalf("failed to copy external to tempdir: %v", err)
}
regHandler := olareg.New(oConfig.Config{
Storage: oConfig.ConfigStorage{
StoreType: oConfig.StoreMem,
@ -115,6 +119,10 @@ defaults:
if err != nil {
t.Fatalf("failed to parse loop reference: %v", err)
}
rExt, err := ref.New(tsHost + "/external")
if err != nil {
t.Fatalf("failed to parse external reference: %v", err)
}
m1, err := rc.ManifestGet(ctx, r1)
if err != nil {
t.Fatalf("failed to get manifest v1: %v", err)
@ -150,6 +158,12 @@ defaults:
t.Fatalf("failed to get signature for v2: %v", err)
}
d2Sig := desc2Sig.Descriptors[0].Digest
desc2Ext, err := rc.ReferrerList(ctx, r2, scheme.WithReferrerMatchOpt(descriptor.MatchOpt{ArtifactType: "application/example.sbom"}), scheme.WithReferrerSource(rExt))
if err != nil || len(desc2Ext.Descriptors) < 2 {
t.Fatalf("failed to get external artifacts for v2: %v", err)
}
d2Ext1 := desc2Ext.Descriptors[0].Digest
d2Ext2 := desc2Ext.Descriptors[1].Digest
m3, err := rc.ManifestGet(ctx, r3)
if err != nil {
t.Fatalf("failed to get manifest v3: %v", err)
@ -480,6 +494,72 @@ defaults:
},
expErr: nil,
},
{
name: "ImageReferrersExtSrc",
sync: ConfigSync{
Source: tsHost + "/testrepo:v2",
Target: tsHost + "/test-referrer-ext1:v2",
Type: "image",
Referrers: &boolT,
ReferrerFilters: []ConfigReferrerFilter{
{
ArtifactType: "application/example.sbom",
},
},
ReferrerSrc: tsHost + "/external",
},
action: actionCopy,
expect: map[string]digest.Digest{
tsHost + "/test-referrer-ext1:v2": d2,
},
exists: []string{
tsHost + "/test-referrer-ext1@" + d2AMD.String(),
tsHost + "/test-referrer-ext1@" + d2Ext1.String(),
tsHost + "/test-referrer-ext1@" + d2Ext2.String(),
},
missing: []string{
tsHost + "/test-referrer-ext1@" + d2SBOM.String(),
tsHost + "/test-referrer-ext1@" + d2Sig.String(),
tsHost + "/test-referrer-ext1@" + d1.String(),
tsHost + "/test-referrer-ext1@" + d3.String(),
},
expErr: nil,
},
{
name: "ImageReferrersExtBoth",
sync: ConfigSync{
Source: tsHost + "/testrepo:v2",
Target: tsHost + "/test-referrer-ext2:v2",
Type: "image",
Referrers: &boolT,
ReferrerFilters: []ConfigReferrerFilter{
{
ArtifactType: "application/example.sbom",
},
},
ReferrerSrc: tsHost + "/external",
ReferrerTgt: tsHost + "/test-referrer-ext2-tgt",
},
action: actionCopy,
expect: map[string]digest.Digest{
tsHost + "/test-referrer-ext2:v2": d2,
},
exists: []string{
tsHost + "/test-referrer-ext2@" + d2AMD.String(),
tsHost + "/test-referrer-ext2-tgt@" + d2Ext1.String(),
tsHost + "/test-referrer-ext2-tgt@" + d2Ext2.String(),
},
missing: []string{
tsHost + "/test-referrer-ext2@" + d2SBOM.String(),
tsHost + "/test-referrer-ext2@" + d2Sig.String(),
tsHost + "/test-referrer-ext2@" + d1.String(),
tsHost + "/test-referrer-ext2@" + d3.String(),
tsHost + "/test-referrer-ext2@" + d2Ext1.String(),
tsHost + "/test-referrer-ext2@" + d2Ext2.String(),
},
expErr: nil,
},
{
name: "Backup",
sync: ConfigSync{

18
cmd/regsync/root.go
View File

@ -802,6 +802,24 @@ func (rootOpts *rootCmd) processRef(ctx context.Context, s ConfigSync, src, tgt
opts = append(opts, regclient.ImageWithReferrers(rOpts...))
}
}
if s.ReferrerSrc != "" {
referrerSrc, err := ref.New(s.ReferrerSrc)
if err != nil {
rootOpts.log.Error("failed to parse referrer source reference",
slog.String("referrerSource", s.ReferrerSrc),
slog.String("error", err.Error()))
}
opts = append(opts, regclient.ImageWithReferrerSrc(referrerSrc))
}
if s.ReferrerTgt != "" {
referrerTgt, err := ref.New(s.ReferrerTgt)
if err != nil {
rootOpts.log.Error("failed to parse referrer target reference",
slog.String("referrerTarget", s.ReferrerTgt),
slog.String("error", err.Error()))
}
opts = append(opts, regclient.ImageWithReferrerTgt(referrerTgt))
}
}
if s.FastCheck != nil && *s.FastCheck {
opts = append(opts, regclient.ImageWithFastCheck())

14
docs/regsync.md
View File

@ -43,7 +43,7 @@ The `version` command will show details about the git commit and tag if availabl
## Configuration File
The `regsync` configuration file is yaml formatted with the following layout:
The `regsync` configuration file is yaml formatted with the following example layout:
```yaml
x-sched-a: &sched-a "15 01 * * *"
@ -198,6 +198,8 @@ sync:
- `referrerFilters`: (array) list of filters for referrers to include, by default all referrers are included.
- `artifactType`: (string) artifact types to include.
- `annotations`: (map) mapping of annotations for referrers.
- `referrerSource`: (string) source repo for pulling referrers (defaults to sync source).
- `referrerTarget`: (string) target repo for pushing referrers (defaults to sync target).
- `fastCopy`: (bool) skip referrers and digest tag checks when image exists, overrides `forceRecursive`.
- `forceRecursive`: (bool) forces a copy of all manifests and blobs even when the target parent manifest already exists.
- `mediaTypes`:
@ -242,7 +244,7 @@ sync:
By default all platforms are copied along with the original upstream manifest list.
Note that looking up the platform from a multi-platform image counts against the Docker Hub rate limit, and that rate limits are not checked prior to resolving the platform.
When run with "server", the platform is only resolved once for each multi-platform digest seen.
- `backup`, `interval`, `schedule`, `ratelimit`, `digestTags`, `referrers`, `referrerFilters`, `fastCopy`, `forceRecursive`, and `mediaTypes`:
- `backup`, `interval`, `schedule`, `ratelimit`, `digestTags`, `referrers`, `referrerFilters`, `referrerSource`, `referrerTarget`, `fastCopy`, `forceRecursive`, and `mediaTypes`:
See description under `defaults`.
- `x-*`:
@ -251,11 +253,11 @@ sync:
## Templates
[Go templates](https://golang.org/pkg/text/template/) are used to expand values in `registry`, `user`, `pass`, `regcert`, `clientCert`, `clientKey`, `source`, `target`, and `backup`.
[Go templates](https://golang.org/pkg/text/template/) are used to expand values in `registry`, `user`, `pass`, `regcert`, `clientCert`, `clientKey`, `source`, `target`, `referrerSource`, `referrerTarget`, and `backup`.
The `source` and `target` templates support the following objects:
The `source`, `target`, `referrerSource`, `referrerTarget`, `backup` templates support the following objects:
- `.Sync`: Values from the current sync step
- `.Sync`: Values from the current sync step, including
- `.Sync.Source`: Source
- `.Sync.Target`: Target
- `.Sync.Type`: Type
@ -263,7 +265,7 @@ The `source` and `target` templates support the following objects:
- `.Sync.Interval`: Interval
- `.Sync.Schedule`: Schedule
Note that `source` is expanded before `target`, and both are expanded before `backup`.
Note that templates are expanded in the order `source`, `referrerSource`, `target`, `referrerTarget`, and then `backup`.
The `backup` template supports the following objects: