feat: image mod ability to add layers

Signed-off-by: Brandon Mitchell <git@bmitch.net>

cmd/regctl/image.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/regclient/regclient"
 	"github.com/regclient/regclient/internal/ascii"
+	"github.com/regclient/regclient/internal/strparse"
 	"github.com/regclient/regclient/internal/units"
 	"github.com/regclient/regclient/mod"
 	"github.com/regclient/regclient/pkg/archive"
@@ -260,6 +261,10 @@ regctl image mod registry.example.org/repo:v1 \
 # convert an image to the OCI media types, copying to local registry
 regctl image mod alpine:3.5 --to-oci --create registry.example.org/alpine:3.5
 
+# append a layer to only the linux/amd64 image using the file.tar contents
+regctl image mod registry.example.org/repo:v1 --create v1-extended \
+  --layer-add "tar=file.tar,platform=linux/amd64"
+
 # set the timestamp on the config and layers, ignoring the alpine base image layers
 regctl image mod registry.example.org/repo:v1 --create v1-mod \
   --time "set=2021-02-03T04:05:06Z,base-ref=alpine:3"
@@ -631,6 +636,45 @@ regctl image ratelimit alpine --format '{{.Remain}}'`,
 		},
 	}, "label-to-annotation", "", `set annotations from labels`)
 	flagLabelAnnot.NoOptDefVal = "true"
+	imageModCmd.Flags().VarP(&modFlagFunc{
+		t: "string",
+		f: func(val string) error {
+			kvSplit, err := strparse.SplitCSKV(val)
+			if err != nil {
+				return fmt.Errorf("failed to parse layer-add options %s", val)
+			}
+			var rdr io.Reader
+			var mt string
+			var platforms []platform.Platform
+			if filename, ok := kvSplit["tar"]; ok {
+				//#nosec G304 command is run by a user accessing their own files
+				fh, err := os.Open(filename)
+				if err != nil {
+					return fmt.Errorf("failed to open tar file %s: %v", filename, err)
+				}
+				rdr = fh
+				cobra.OnFinalize(func() {
+					_ = fh.Close()
+				})
+			}
+			if rdr == nil {
+				return fmt.Errorf("tar file input is required")
+			}
+			if mtArg, ok := kvSplit["mediaType"]; ok {
+				mt = mtArg
+			}
+			if pStr, ok := kvSplit["platform"]; ok {
+				p, err := platform.Parse(pStr)
+				if err != nil {
+					return fmt.Errorf("failed to parse platform %s: %v", pStr, err)
+				}
+				platforms = append(platforms, p)
+			}
+			imageOpts.modOpts = append(imageOpts.modOpts,
+				mod.WithLayerAddTar(rdr, mt, platforms))
+			return nil
+		},
+	}, "layer-add", "", `add a new layer (tar=file,platform=val)`)
 	imageModCmd.Flags().VarP(&modFlagFunc{
 		t: "string",
 		f: func(val string) error {

mod/dag.go

@@ -229,9 +229,9 @@ func dagPut(ctx context.Context, rc *regclient.RegClient, mc dagConfig, rSrc, rT
 				return fmt.Errorf("manifest does not have enough layers")
 			}
 			// keep config index aligned
-			for iConfig >= 0 && oc.History[iConfig].EmptyLayer {
+			for iConfig >= 0 && iConfig < len(oc.History) && oc.History[iConfig].EmptyLayer {
 				iConfig++
-				if iConfig >= len(oc.History) {
+				if iConfig >= len(oc.History) && layer.mod != added {
 					return fmt.Errorf("config history does not have enough entries")
 				}
 			}
@@ -283,7 +283,7 @@ func dagPut(ctx context.Context, rc *regclient.RegClient, mc dagConfig, rSrc, rT
 				}
 				if iConfig < 0 {
 					// noop
-				} else if len(oc.History) == iConfig {
+				} else if iConfig >= len(oc.History) {
 					oc.History = append(oc.History, newHistory)
 				} else {
 					oc.History = append(oc.History[:iConfig+1], oc.History[iConfig:]...)

mod/layer.go

@@ -16,11 +16,89 @@ import (
 	"github.com/regclient/regclient"
 	"github.com/regclient/regclient/pkg/archive"
 	"github.com/regclient/regclient/types/descriptor"
+	"github.com/regclient/regclient/types/errs"
 	"github.com/regclient/regclient/types/manifest"
 	"github.com/regclient/regclient/types/mediatype"
+	"github.com/regclient/regclient/types/platform"
 	"github.com/regclient/regclient/types/ref"
 )
 
+// WithLayerAddTar appends a new layer to the image based on a tar input stream.
+// If media type (mt) is not defined, it will default to Gzip and match Docker or OCI based on the manifest media type.
+// If the platform slice is empty, the layer is added to all platforms.
+func WithLayerAddTar(rdr io.Reader, mt string, platforms []platform.Platform) Opts {
+	return func(dc *dagConfig, dm *dagManifest) error {
+		if mt == "" {
+			switch dm.m.GetDescriptor().MediaType {
+			case mediatype.Docker2Manifest, mediatype.Docker2ManifestList:
+				mt = mediatype.Docker2LayerGzip
+			default:
+				mt = mediatype.OCI1LayerGzip
+			}
+		}
+		var comp archive.CompressType
+		switch mt {
+		case mediatype.OCI1Layer, mediatype.Docker2Layer:
+			comp = archive.CompressNone
+		case mediatype.OCI1LayerGzip, mediatype.Docker2LayerGzip:
+			comp = archive.CompressGzip
+		case mediatype.OCI1LayerZstd, mediatype.Docker2LayerZstd:
+			comp = archive.CompressZstd
+		default:
+			return fmt.Errorf("unsupported new layer media type %s%.0w", mt, errs.ErrUnsupportedMediaType)
+		}
+		var ucDig digest.Digest
+		var desc descriptor.Descriptor
+		dc.stepsManifest = append(dc.stepsManifest, func(ctx context.Context, rc *regclient.RegClient, rSrc, rTgt ref.Ref, dm *dagManifest) error {
+			// skip deleted, manifest lists, and platforms that aren't listed
+			if dm.mod == deleted || dm.m.IsList() {
+				return nil
+			}
+			if len(platforms) > 0 {
+				p := dm.config.oc.GetConfig().Platform
+				found := false
+				for _, pe := range platforms {
+					if platform.Match(p, pe) {
+						found = true
+						break
+					}
+				}
+				if !found {
+					return nil
+				}
+			}
+			// push the layer
+			if ucDig == "" {
+				digUC := digest.Canonical.Digester() // uncompressed digest
+				ucDigRdr := io.TeeReader(rdr, digUC.Hash())
+				cRdr, err := archive.Compress(ucDigRdr, comp)
+				if err != nil {
+					return fmt.Errorf("failed to compress layer with %s: %w", comp.String(), err)
+				}
+				descPut, err := rc.BlobPut(ctx, rTgt, descriptor.Descriptor{}, cRdr)
+				_ = cRdr.Close()
+				if err != nil {
+					return fmt.Errorf("failed to push layer to %s: %w", rTgt.CommonName(), err)
+				}
+				ucDig = digUC.Digest()
+				desc = descriptor.Descriptor{
+					MediaType: mt,
+					Digest:    descPut.Digest,
+					Size:      descPut.Size,
+				}
+			}
+			// add the layer to the dag
+			dm.layers = append(dm.layers, &dagLayer{
+				mod:      added,
+				desc:     desc,
+				ucDigest: ucDig,
+			})
+			return nil
+		})
+		return nil
+	}
+}
+
 // WithLayerCompression alters the media type and compression algorithm of the layers.
 func WithLayerCompression(algo archive.CompressType) Opts {
 	return func(dc *dagConfig, dm *dagManifest) error {

mod/mod_test.go

@@ -1,11 +1,13 @@
 package mod
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
 	"net/http/httptest"
 	"net/url"
+	"os"
 	"regexp"
 	"testing"
 	"time"
@@ -76,6 +78,10 @@ func TestMod(t *testing.T) {
 		tTgt.Close()
 		_ = regTgt.Close()
 	})
+	tarBytes, err := os.ReadFile("../testdata/layer.tar")
+	if err != nil {
+		t.Fatalf("failed to read testdata/layer.tar: %v", err)
+	}
 
 	// create regclient
 	rcHosts := []config.Host{
@@ -459,6 +465,13 @@ func TestMod(t *testing.T) {
 			ref:      "ocidir://testrepo:v1",
 			wantSame: true,
 		},
+		{
+			name: "Layer Add",
+			opts: []Opts{
+				WithLayerAddTar(bytes.NewReader(tarBytes), "", nil),
+			},
+			ref: "ocidir://testrepo:v1",
+		},
 		{
 			name: "Layer Uncompressed",
 			opts: []Opts{

mod/time.go

@@ -19,7 +19,7 @@ func timeNow() time.Time {
 	if err == nil {
 		return now
 	}
-	return time.Now()
+	return time.Now().UTC()
 }
 
 func timeEpocEnv() (time.Time, error) {