git source: add AttrGitChecksum

Not integrated to util/giturl, as PR 5974 is not merged yet.

Signed-off-by: Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp>

client/llb/source.go

@@ -322,6 +322,12 @@ func Git(url, ref string, opts ...GitOption) State {
 		addCap(&gi.Constraints, pb.CapSourceGitMountSSHSock)
 	}
 
+	checksum := gi.Checksum
+	if checksum != "" {
+		attrs[pb.AttrGitChecksum] = checksum
+		addCap(&gi.Constraints, pb.CapSourceGitChecksum)
+	}
+
 	addCap(&gi.Constraints, pb.CapSourceGit)
 
 	source := NewSource("git://"+id, attrs, gi.Constraints)
@@ -345,6 +351,7 @@ type GitInfo struct {
 	addAuthCap       bool
 	KnownSSHHosts    string
 	MountSSHSock     string
+	Checksum         string
 }
 
 func KeepGitDir() GitOption {
@@ -373,6 +380,12 @@ func MountSSHSock(sshID string) GitOption {
 	})
 }
 
+func GitChecksum(v string) GitOption {
+	return gitOptionFunc(func(gi *GitInfo) {
+		gi.Checksum = v
+	})
+}
+
 // AuthOption can be used with either HTTP or Git sources.
 type AuthOption interface {
 	GitOption

solver/pb/attr.go

@@ -6,6 +6,8 @@ const AttrAuthHeaderSecret = "git.authheadersecret"
 const AttrAuthTokenSecret = "git.authtokensecret"
 const AttrKnownSSHHosts = "git.knownsshhosts"
 const AttrMountSSHSock = "git.mountsshsock"
+const AttrGitChecksum = "git.checksum"
+
 const AttrLocalSessionID = "local.session"
 const AttrLocalUniqueID = "local.unique"
 const AttrIncludePatterns = "local.includepattern"

solver/pb/caps.go

@@ -30,6 +30,7 @@ const (
 	CapSourceGitKnownSSHHosts apicaps.CapID = "source.git.knownsshhosts"
 	CapSourceGitMountSSHSock  apicaps.CapID = "source.git.mountsshsock"
 	CapSourceGitSubdir        apicaps.CapID = "source.git.subdir"
+	CapSourceGitChecksum      apicaps.CapID = "source.git.checksum"
 
 	CapSourceHTTP         apicaps.CapID = "source.http"
 	CapSourceHTTPAuth     apicaps.CapID = "source.http.auth"
@@ -222,6 +223,12 @@ func init() {
 		Status:  apicaps.CapStatusExperimental,
 	})
 
+	Caps.Init(apicaps.Cap{
+		ID:      CapSourceGitChecksum,
+		Enabled: true,
+		Status:  apicaps.CapStatusExperimental,
+	})
+
 	Caps.Init(apicaps.Cap{
 		ID:      CapSourceHTTP,
 		Enabled: true,

source/git/identifier.go

@@ -13,6 +13,7 @@ import (
 type GitIdentifier struct {
 	Remote           string
 	Ref              string
+	Checksum         string
 	Subdir           string
 	KeepGitDir       bool
 	AuthTokenSecret  string

source/git/source.go

@@ -92,6 +92,8 @@ func (gs *gitSource) Identifier(scheme, ref string, attrs map[string]string, pla
 			id.KnownSSHHosts = v
 		case pb.AttrMountSSHSock:
 			id.MountSSHSock = v
+		case pb.AttrGitChecksum:
+			id.Checksum = v
 		}
 	}
 
@@ -349,10 +351,19 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context, g session.Group, index
 	gs.locker.Lock(remote)
 	defer gs.locker.Unlock(remote)
 
-	if ref := gs.src.Ref; ref != "" && gitutil.IsCommitSHA(ref) {
+	var refCommitFullHash, ref2 string
-		cacheKey := gs.shaToCacheKey(ref, "")
+	if gitutil.IsCommitSHA(gs.src.Checksum) && !gs.src.KeepGitDir {
+		refCommitFullHash = gs.src.Checksum
+		ref2 = gs.src.Ref
+	}
+	if refCommitFullHash == "" && gitutil.IsCommitSHA(gs.src.Ref) {
+		refCommitFullHash = gs.src.Ref
+	}
+	if refCommitFullHash != "" {
+		cacheKey := gs.shaToCacheKey(refCommitFullHash, ref2)
 		gs.cacheKey = cacheKey
-		return cacheKey, ref, nil, true, nil
+		// gs.src.Checksum is verified when checking out the commit
+		return cacheKey, refCommitFullHash, nil, true, nil
 	}
 
 	gs.getAuthToken(ctx, g)
@@ -415,7 +426,9 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context, g session.Group, index
 	if !gitutil.IsCommitSHA(sha) {
 		return "", "", nil, false, errors.Errorf("invalid commit sha %q", sha)
 	}
-
+	if gs.src.Checksum != "" && !strings.HasPrefix(sha, gs.src.Checksum) {
+		return "", "", nil, false, errors.Errorf("expected checksum to match %s, got %s", gs.src.Checksum, sha)
+	}
 	cacheKey := gs.shaToCacheKey(sha, usedRef)
 	gs.cacheKey = cacheKey
 	return cacheKey, sha, nil, true, nil
@@ -536,6 +549,7 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out
 		subdir = "."
 	}
 
+	checkedoutRef := "HEAD"
 	if gs.src.KeepGitDir && subdir == "." {
 		checkoutDirGit := filepath.Join(checkoutDir, ".git")
 		if err := os.MkdirAll(checkoutDir, 0711); err != nil {
@@ -605,6 +619,7 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to checkout remote %s", urlutil.RedactCredentials(gs.src.Remote))
 		}
+		checkedoutRef = ref // HEAD may not exist
 		if subdir != "." {
 			d, err := os.Open(filepath.Join(cd, subdir))
 			if err != nil {
@@ -635,6 +650,16 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out
 	}
 
 	git = git.New(gitutil.WithWorkTree(checkoutDir), gitutil.WithGitDir(gitDir))
+	if gs.src.Checksum != "" {
+		actualHashBuf, err := git.Run(ctx, "rev-parse", checkedoutRef)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to rev-parse %s for %s", checkedoutRef, urlutil.RedactCredentials(gs.src.Remote))
+		}
+		actualHash := strings.TrimSpace(string(actualHashBuf))
+		if !strings.HasPrefix(actualHash, gs.src.Checksum) {
+			return nil, errors.Errorf("expected checksum to match %s, got %s", gs.src.Checksum, actualHash)
+		}
+	}
 	_, err = git.Run(ctx, "submodule", "update", "--init", "--recursive", "--depth=1")
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to update submodules for %s", urlutil.RedactCredentials(gs.src.Remote))

source/git/source_test.go

@@ -148,6 +148,16 @@ func testRepeatedFetch(t *testing.T, keepGitDir bool) {
 	require.NoError(t, err)
 
 	require.Equal(t, "subcontents\n", string(dt))
+
+	// The key should not change regardless to the existence of Checksum
+	// https://github.com/moby/buildkit/pull/5975#discussion_r2092206059
+	id.Checksum = pin3
+	g, err = gs.Resolve(ctx, id, nil, nil)
+	require.NoError(t, err)
+	key4, pin4, _, _, err := g.CacheKey(ctx, nil, 0)
+	require.NoError(t, err)
+	require.Equal(t, key3, key4)
+	require.Equal(t, pin3, pin4)
 }
 
 func TestFetchBySHA(t *testing.T) {
@@ -304,54 +314,75 @@ func testFetchUnreferencedRefSha(t *testing.T, ref string, keepGitDir bool) {
 }
 
 func TestFetchByTag(t *testing.T) {
-	testFetchByTag(t, "lightweight-tag", "third", false, true, false)
+	testFetchByTag(t, "lightweight-tag", "third", false, true, false, testChecksumModeNone)
 }
 
 func TestFetchByTagKeepGitDir(t *testing.T) {
-	testFetchByTag(t, "lightweight-tag", "third", false, true, true)
+	testFetchByTag(t, "lightweight-tag", "third", false, true, true, testChecksumModeNone)
 }
 
 func TestFetchByTagFull(t *testing.T) {
-	testFetchByTag(t, "refs/tags/lightweight-tag", "third", false, true, true)
+	testFetchByTag(t, "refs/tags/lightweight-tag", "third", false, true, true, testChecksumModeNone)
 }
 
 func TestFetchByAnnotatedTag(t *testing.T) {
-	testFetchByTag(t, "v1.2.3", "second", true, false, false)
+	testFetchByTag(t, "v1.2.3", "second", true, false, false, testChecksumModeNone)
 }
 
 func TestFetchByAnnotatedTagKeepGitDir(t *testing.T) {
-	testFetchByTag(t, "v1.2.3", "second", true, false, true)
+	testFetchByTag(t, "v1.2.3", "second", true, false, true, testChecksumModeNone)
 }
 
 func TestFetchByAnnotatedTagFull(t *testing.T) {
-	testFetchByTag(t, "refs/tags/v1.2.3", "second", true, false, true)
+	testFetchByTag(t, "refs/tags/v1.2.3", "second", true, false, true, testChecksumModeNone)
 }
 
 func TestFetchByBranch(t *testing.T) {
-	testFetchByTag(t, "feature", "withsub", false, true, false)
+	testFetchByTag(t, "feature", "withsub", false, true, false, testChecksumModeNone)
 }
 
 func TestFetchByBranchKeepGitDir(t *testing.T) {
-	testFetchByTag(t, "feature", "withsub", false, true, true)
+	testFetchByTag(t, "feature", "withsub", false, true, true, testChecksumModeNone)
 }
 
 func TestFetchByBranchFull(t *testing.T) {
-	testFetchByTag(t, "refs/heads/feature", "withsub", false, true, true)
+	testFetchByTag(t, "refs/heads/feature", "withsub", false, true, true, testChecksumModeNone)
 }
 
 func TestFetchByRef(t *testing.T) {
-	testFetchByTag(t, "test", "feature", false, true, false)
+	testFetchByTag(t, "test", "feature", false, true, false, testChecksumModeNone)
 }
 
 func TestFetchByRefKeepGitDir(t *testing.T) {
-	testFetchByTag(t, "test", "feature", false, true, true)
+	testFetchByTag(t, "test", "feature", false, true, true, testChecksumModeNone)
 }
 
 func TestFetchByRefFull(t *testing.T) {
-	testFetchByTag(t, "refs/test", "feature", false, true, true)
+	testFetchByTag(t, "refs/test", "feature", false, true, true, testChecksumModeNone)
 }
 
-func testFetchByTag(t *testing.T, tag, expectedCommitSubject string, isAnnotatedTag, hasFoo13File, keepGitDir bool) {
+func TestFetchByTagWithChecksum(t *testing.T) {
+	testFetchByTag(t, "lightweight-tag", "third", false, true, false, testChecksumModeValid)
+}
+
+func TestFetchByTagWithChecksumPartial(t *testing.T) {
+	testFetchByTag(t, "lightweight-tag", "third", false, true, false, testChecksumModeValidPartial)
+}
+
+func TestFetchByTagWithChecksumInvalid(t *testing.T) {
+	testFetchByTag(t, "lightweight-tag", "third", false, true, false, testChecksumModeInvalid)
+}
+
+type testChecksumMode int
+
+const (
+	testChecksumModeNone testChecksumMode = iota
+	testChecksumModeValid
+	testChecksumModeValidPartial
+	testChecksumModeInvalid
+)
+
+func testFetchByTag(t *testing.T, tag, expectedCommitSubject string, isAnnotatedTag, hasFoo13File, keepGitDir bool, checksumMode testChecksumMode) {
 	if runtime.GOOS == "windows" {
 		t.Skip("Depends on unimplemented containerd bind-mount support on Windows")
 	}
@@ -366,6 +397,28 @@ func testFetchByTag(t *testing.T, tag, expectedCommitSubject string, isAnnotated
 
 	id := &GitIdentifier{Remote: repo.mainURL, Ref: tag, KeepGitDir: keepGitDir}
 
+	if checksumMode != testChecksumModeNone {
+		cmd := exec.Command("git", "rev-parse", tag)
+		cmd.Dir = repo.mainPath
+
+		out, err := cmd.Output()
+		require.NoError(t, err)
+
+		sha := strings.TrimSpace(string(out))
+		require.Equal(t, 40, len(sha))
+
+		switch checksumMode {
+		case testChecksumModeValid:
+			id.Checksum = sha
+		case testChecksumModeValidPartial:
+			id.Checksum = sha[:8]
+		case testChecksumModeInvalid:
+			id.Checksum = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"
+		default:
+			// NOTREACHED
+		}
+	}
+
 	g, err := gs.Resolve(ctx, id, nil, nil)
 	require.NoError(t, err)
 
@@ -383,6 +436,10 @@ func testFetchByTag(t *testing.T, tag, expectedCommitSubject string, isAnnotated
 	require.Equal(t, 40, len(pin1))
 
 	ref1, err := g.Snapshot(ctx, nil)
+	if checksumMode == testChecksumModeInvalid {
+		require.ErrorContains(t, err, "expected checksum to match "+id.Checksum)
+		return
+	}
 	require.NoError(t, err)
 	defer ref1.Release(context.TODO())