mobi/buildkit
1
0
Fork 0
mirror of https://github.com/moby/buildkit.git synced 2025-08-01 02:04:26 +03:00
Code Activity

git: centralize git cli operations

Browse Source 
Move all of the git command line logic into a single object, inspired by
the object already in buildx.

The basic implemenation allows for configuring a git cli for a specific
repository, along with various authorization settings and custom
binaries. Commands can be run for that repository, and a few helpers are
provided for accessing data on it - more to come in the future
hopefully.

Signed-off-by: Justin Chadwell <me@jedevc.com>
This commit is contained in:
Justin Chadwell
2023-08-04 12:47:40 +01:00
parent 4376f3861b
commit d34b2471cd
4 changed files with 419 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files

258
source/git/source.go
View File

@ -1,7 +1,6 @@
package git
import (
"bytes"
"context"
"encoding/base64"
"fmt"
@ -28,6 +27,7 @@ import (
"github.com/moby/buildkit/source"
srctypes "github.com/moby/buildkit/source/types"
"github.com/moby/buildkit/util/bklog"
"github.com/moby/buildkit/util/gitutil"
"github.com/moby/buildkit/util/progress/logs"
"github.com/moby/buildkit/util/urlutil"
"github.com/moby/locker"
@ -100,7 +100,7 @@ func (gs *gitSource) Identifier(scheme, ref string, attrs map[string]string, pla
}
// needs to be called with repo lock
func (gs *gitSource) mountRemote(ctx context.Context, remote string, auth []string, g session.Group) (target string, release func(), retErr error) {
func (gs *gitSource) mountRemote(ctx context.Context, remote string, authArgs []string, g session.Group) (target string, release func() error, retErr error) {
sis, err := searchGitRemote(ctx, gs.cache, remote)
if err != nil {
return "", nil, errors.Wrapf(err, "failed to search metadata for %s", urlutil.RedactCredentials(remote))
@ -129,8 +129,8 @@ func (gs *gitSource) mountRemote(ctx context.Context, remote string, auth []stri
initializeRepo = true
}
releaseRemoteRef := func() {
remoteRef.Release(context.TODO())
releaseRemoteRef := func() error {
return remoteRef.Release(context.TODO())
}
defer func() {
@ -156,16 +156,21 @@ func (gs *gitSource) mountRemote(ctx context.Context, remote string, auth []stri
}
}()
git := gitCLI(
gitutil.WithGitDir(dir),
gitutil.WithArgs(authArgs...),
)
if initializeRepo {
// Explicitly set the Git config 'init.defaultBranch' to the
// implied default to suppress "hint:" output about not having a
// default initial branch name set which otherwise spams unit
// test logs.
if _, err := gitWithinDir(ctx, dir, "", "", "", auth, "-c", "init.defaultBranch=master", "init", "--bare"); err != nil {
if _, err := git.Run(ctx, "-c", "init.defaultBranch=master", "init", "--bare"); err != nil {
return "", nil, errors.Wrapf(err, "failed to init repo at %s", dir)
}
if _, err := gitWithinDir(ctx, dir, "", "", "", auth, "remote", "add", "origin", remote); err != nil {
if _, err := git.Run(ctx, "remote", "add", "origin", remote); err != nil {
return "", nil, errors.Wrapf(err, "failed add origin repo at %s", dir)
}
@ -175,9 +180,12 @@ func (gs *gitSource) mountRemote(ctx context.Context, remote string, auth []stri
return "", nil, err
}
}
return dir, func() {
lm.Unmount()
releaseRemoteRef()
return dir, func() error {
err := lm.Unmount()
if err1 := releaseRemoteRef(); err == nil {
err = err1
}
return err
}, nil
}
@ -186,7 +194,7 @@ type gitSourceHandler struct {
src GitIdentifier
cacheKey string
sm *session.Manager
auth []string
authArgs []string
}
func (gs *gitSourceHandler) shaToCacheKey(sha string) string {
@ -239,7 +247,7 @@ func (gs *gitSourceHandler) authSecretNames() (sec []authSecret, _ error) {
}
func (gs *gitSourceHandler) getAuthToken(ctx context.Context, g session.Group) error {
if gs.auth != nil {
if gs.authArgs != nil {
return nil
}
sec, err := gs.authSecretNames()
@ -258,7 +266,7 @@ func (gs *gitSourceHandler) getAuthToken(ctx context.Context, g session.Group) e
if s.token {
dt = []byte("basic " + base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("x-access-token:%s", dt))))
}
gs.auth = []string{"-c", "http." + tokenScope(gs.src.Remote) + ".extraheader=Authorization: " + string(dt)}
gs.authArgs = []string{"-c", "http." + tokenScope(gs.src.Remote) + ".extraheader=Authorization: " + string(dt)}
break
}
return nil
@ -341,35 +349,15 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context, g session.Group, index
gs.getAuthToken(ctx, g)
gitDir, unmountGitDir, err := gs.mountRemote(ctx, remote, gs.auth, g)
git, cleanup, err := gs.gitCli(ctx, g)
if err != nil {
return "", "", nil, false, err
}
defer unmountGitDir()
var sock string
if gs.src.MountSSHSock != "" {
var unmountSock func() error
sock, unmountSock, err = gs.mountSSHAuthSock(ctx, gs.src.MountSSHSock, g)
if err != nil {
return "", "", nil, false, err
}
defer unmountSock()
}
var knownHosts string
if gs.src.KnownSSHHosts != "" {
var unmountKnownHosts func() error
knownHosts, unmountKnownHosts, err = gs.mountKnownHosts(ctx)
if err != nil {
return "", "", nil, false, err
}
defer unmountKnownHosts()
}
defer cleanup()
ref := gs.src.Ref
if ref == "" {
ref, err = getDefaultBranch(ctx, gitDir, "", sock, knownHosts, gs.auth, gs.src.Remote)
ref, err = getDefaultBranch(ctx, git, gs.src.Remote)
if err != nil {
return "", "", nil, false, err
}
@ -377,11 +365,11 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context, g session.Group, index
// TODO: should we assume that remote tag is immutable? add a timer?
buf, err := gitWithinDir(ctx, gitDir, "", sock, knownHosts, gs.auth, "ls-remote", "origin", ref)
buf, err := git.Run(ctx, "ls-remote", "origin", ref)
if err != nil {
return "", "", nil, false, errors.Wrapf(err, "failed to fetch remote %s", urlutil.RedactCredentials(remote))
}
out := buf.String()
out := string(buf)
idx := strings.Index(out, "\t")
if idx == -1 {
return "", "", nil, false, errors.Errorf("repository does not contain ref %s, output: %q", ref, string(out))
@ -422,35 +410,23 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out
gs.locker.Lock(gs.src.Remote)
defer gs.locker.Unlock(gs.src.Remote)
gitDir, unmountGitDir, err := gs.mountRemote(ctx, gs.src.Remote, gs.auth, g)
if err != nil {
return nil, err
}
defer unmountGitDir()
var sock string
if gs.src.MountSSHSock != "" {
var unmountSock func() error
sock, unmountSock, err = gs.mountSSHAuthSock(ctx, gs.src.MountSSHSock, g)
git, cleanup, err := gs.gitCli(ctx, g)
if err != nil {
return nil, err
}
defer unmountSock()
}
var knownHosts string
if gs.src.KnownSSHHosts != "" {
var unmountKnownHosts func() error
knownHosts, unmountKnownHosts, err = gs.mountKnownHosts(ctx)
if err != nil {
return nil, err
}
defer unmountKnownHosts()
defer cleanup()
gitDir, err := git.GitDir(ctx)
if err != nil {
return nil, err
}
ref := gs.src.Ref
if ref == "" {
ref, err = getDefaultBranch(ctx, gitDir, "", sock, knownHosts, gs.auth, gs.src.Remote)
ref, err = getDefaultBranch(ctx, git, gs.src.Remote)
if err != nil {
return nil, err
}
@ -459,7 +435,7 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out
doFetch := true
if isCommitSHA(ref) {
// skip fetch if commit already exists
if _, err := gitWithinDir(ctx, gitDir, "", sock, knownHosts, nil, "cat-file", "-e", ref+"^{commit}"); err == nil {
if _, err := git.Run(ctx, "cat-file", "-e", ref+"^{commit}"); err == nil {
doFetch = false
}
}
@ -483,10 +459,10 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out
// in case the ref is a branch and it now points to a different commit sha
// TODO: is there a better way to do this?
}
if _, err := gitWithinDir(ctx, gitDir, "", sock, knownHosts, gs.auth, args...); err != nil {
if _, err := git.Run(ctx, args...); err != nil {
return nil, errors.Wrapf(err, "failed to fetch remote %s", urlutil.RedactCredentials(gs.src.Remote))
}
_, err = gitWithinDir(ctx, gitDir, "", sock, knownHosts, nil, "reflog", "expire", "--all", "--expire=now")
_, err = git.Run(ctx, "reflog", "expire", "--all", "--expire=now")
if err != nil {
return nil, errors.Wrapf(err, "failed to expire reflog for remote %s", urlutil.RedactCredentials(gs.src.Remote))
}
@ -528,49 +504,50 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out
if err := os.MkdirAll(checkoutDir, 0711); err != nil {
return nil, err
}
_, err = gitWithinDir(ctx, checkoutDirGit, "", sock, knownHosts, nil, "-c", "init.defaultBranch=master", "init")
checkoutGit := git.New(gitutil.WithWorkTree(checkoutDir), gitutil.WithGitDir(checkoutDirGit))
_, err = checkoutGit.Run(ctx, "-c", "init.defaultBranch=master", "init")
if err != nil {
return nil, err
}
// Defense-in-depth: clone using the file protocol to disable local-clone
// optimizations which can be abused on some versions of Git to copy unintended
// host files into the build context.
_, err = gitWithinDir(ctx, checkoutDirGit, "", sock, knownHosts, nil, "remote", "add", "origin", "file://"+gitDir)
_, err = checkoutGit.Run(ctx, "remote", "add", "origin", "file://"+gitDir)
if err != nil {
return nil, err
}
gitCatFileBuf, err := gitWithinDir(ctx, gitDir, "", sock, knownHosts, gs.auth, "cat-file", "-t", ref)
gitCatFileBuf, err := git.Run(ctx, "cat-file", "-t", ref)
if err != nil {
return nil, err
}
isAnnotatedTag := strings.TrimSpace(gitCatFileBuf.String()) == "tag"
isAnnotatedTag := strings.TrimSpace(string(gitCatFileBuf)) == "tag"
pullref := ref
if isAnnotatedTag {
pullref += ":refs/tags/" + pullref
} else if isCommitSHA(ref) {
pullref = "refs/buildkit/" + identity.NewID()
_, err = gitWithinDir(ctx, gitDir, "", sock, knownHosts, gs.auth, "update-ref", pullref, ref)
_, err = git.Run(ctx, "update-ref", pullref, ref)
if err != nil {
return nil, err
}
} else {
pullref += ":" + pullref
}
_, err = gitWithinDir(ctx, checkoutDirGit, "", sock, knownHosts, gs.auth, "fetch", "-u", "--depth=1", "origin", pullref)
_, err = checkoutGit.Run(ctx, "fetch", "-u", "--depth=1", "origin", pullref)
if err != nil {
return nil, err
}
_, err = gitWithinDir(ctx, checkoutDirGit, checkoutDir, sock, knownHosts, nil, "checkout", "FETCH_HEAD")
_, err = checkoutGit.Run(ctx, "checkout", "FETCH_HEAD")
if err != nil {
return nil, errors.Wrapf(err, "failed to checkout remote %s", urlutil.RedactCredentials(gs.src.Remote))
}
_, err = gitWithinDir(ctx, checkoutDirGit, "", sock, knownHosts, nil, "remote", "set-url", "origin", urlutil.RedactCredentials(gs.src.Remote))
_, err = checkoutGit.Run(ctx, "remote", "set-url", "origin", urlutil.RedactCredentials(gs.src.Remote))
if err != nil {
return nil, errors.Wrapf(err, "failed to set remote origin to %s", urlutil.RedactCredentials(gs.src.Remote))
}
_, err = gitWithinDir(ctx, checkoutDirGit, "", sock, knownHosts, nil, "reflog", "expire", "--all", "--expire=now")
_, err = checkoutGit.Run(ctx, "reflog", "expire", "--all", "--expire=now")
if err != nil {
return nil, errors.Wrapf(err, "failed to expire reflog for remote %s", urlutil.RedactCredentials(gs.src.Remote))
}
@ -586,7 +563,8 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out
return nil, errors.Wrapf(err, "failed to create temporary checkout dir")
}
}
_, err = gitWithinDir(ctx, gitDir, cd, sock, knownHosts, nil, "checkout", ref, "--", ".")
checkoutGit := git.New(gitutil.WithWorkTree(cd), gitutil.WithGitDir(gitDir))
_, err = checkoutGit.Run(ctx, "checkout", ref, "--", ".")
if err != nil {
return nil, errors.Wrapf(err, "failed to checkout remote %s", urlutil.RedactCredentials(gs.src.Remote))
}
@ -619,7 +597,8 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out
}
}
_, err = gitWithinDir(ctx, gitDir, checkoutDir, sock, knownHosts, gs.auth, "submodule", "update", "--init", "--recursive", "--depth=1")
git = git.New(gitutil.WithWorkTree(checkoutDir), gitutil.WithGitDir(gitDir))
_, err = git.Run(ctx, "submodule", "update", "--init", "--recursive", "--depth=1")
if err != nil {
return nil, errors.Wrapf(err, "failed to update submodules for %s", urlutil.RedactCredentials(gs.src.Remote))
}
@ -656,83 +635,62 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out
return snap, nil
}
func (gs *gitSourceHandler) gitCli(ctx context.Context, g session.Group, opts ...gitutil.Option) (*gitutil.GitCLI, func() error, error) {
var cleanups []func() error
cleanup := func() error {
var err error
for _, c := range cleanups {
if err1 := c(); err == nil {
err = err1
}
}
cleanups = nil
return err
}
var err error
gitDir, unmountGitDir, err := gs.mountRemote(ctx, gs.src.Remote, gs.authArgs, g)
if err != nil {
cleanup()
return nil, nil, err
}
cleanups = append(cleanups, unmountGitDir)
var sock string
if gs.src.MountSSHSock != "" {
var unmountSock func() error
sock, unmountSock, err = gs.mountSSHAuthSock(ctx, gs.src.MountSSHSock, g)
if err != nil {
cleanup()
return nil, nil, err
}
cleanups = append(cleanups, unmountSock)
}
var knownHosts string
if gs.src.KnownSSHHosts != "" {
var unmountKnownHosts func() error
knownHosts, unmountKnownHosts, err = gs.mountKnownHosts(ctx)
if err != nil {
cleanup()
return nil, nil, err
}
cleanups = append(cleanups, unmountKnownHosts)
}
opts = append([]gitutil.Option{
gitutil.WithGitDir(gitDir),
gitutil.WithArgs(gs.authArgs...),
gitutil.WithSSHAuthSock(sock),
gitutil.WithSSHKnownHosts(knownHosts),
}, opts...)
return gitCLI(opts...), cleanup, err
}
func isCommitSHA(str string) bool {
return validHex.MatchString(str)
}
func gitWithinDir(ctx context.Context, gitDir, workDir, sshAuthSock, knownHosts string, auth []string, args ...string) (*bytes.Buffer, error) {
a := append([]string{"--git-dir", gitDir}, auth...)
if workDir != "" {
a = append(a, "--work-tree", workDir)
}
return git(ctx, workDir, sshAuthSock, knownHosts, append(a, args...)...)
}
func getGitSSHCommand(knownHosts string) string {
gitSSHCommand := "ssh -F /dev/null"
if knownHosts != "" {
gitSSHCommand += " -o UserKnownHostsFile=" + knownHosts
} else {
gitSSHCommand += " -o StrictHostKeyChecking=no"
}
return gitSSHCommand
}
func git(ctx context.Context, dir, sshAuthSock, knownHosts string, args ...string) (_ *bytes.Buffer, err error) {
for {
stdout, stderr, flush := logs.NewLogStreams(ctx, false)
defer stdout.Close()
defer stderr.Close()
defer func() {
if err != nil {
flush()
}
}()
args = append([]string{"-c", "protocol.file.allow=user"}, args...) // Block sneaky repositories from using repos from the filesystem as submodules.
cmd := exec.Command("git", args...)
cmd.Dir = dir // some commands like submodule require this
buf := bytes.NewBuffer(nil)
errbuf := bytes.NewBuffer(nil)
cmd.Stdin = nil
cmd.Stdout = io.MultiWriter(stdout, buf)
cmd.Stderr = io.MultiWriter(stderr, errbuf)
cmd.Env = []string{
"PATH=" + os.Getenv("PATH"),
"GIT_TERMINAL_PROMPT=0",
"GIT_SSH_COMMAND=" + getGitSSHCommand(knownHosts),
// "GIT_TRACE=1",
"GIT_CONFIG_NOSYSTEM=1", // Disable reading from system gitconfig.
"HOME=/dev/null", // Disable reading from user gitconfig.
"LC_ALL=C", // Ensure consistent output.
}
if sshAuthSock != "" {
cmd.Env = append(cmd.Env, "SSH_AUTH_SOCK="+sshAuthSock)
}
// remote git commands spawn helper processes that inherit FDs and don't
// handle parent death signal so exec.CommandContext can't be used
err := runWithStandardUmask(ctx, cmd)
if err != nil {
if strings.Contains(errbuf.String(), "--depth") || strings.Contains(errbuf.String(), "shallow") {
if newArgs := argsNoDepth(args); len(args) > len(newArgs) {
args = newArgs
continue
}
}
}
return buf, err
}
}
func argsNoDepth(args []string) []string {
out := make([]string, 0, len(args))
for _, a := range args {
if a != "--depth=1" {
out = append(out, a)
}
}
return out
}
func tokenScope(remote string) string {
// generally we can only use the token for fetching main remote but in case of github.com we do best effort
// to try reuse same token for all github.com remotes. This is the same behavior actions/checkout uses
@ -745,13 +703,13 @@ func tokenScope(remote string) string {
}
// getDefaultBranch gets the default branch of a repository using ls-remote
func getDefaultBranch(ctx context.Context, gitDir, workDir, sshAuthSock, knownHosts string, auth []string, remoteURL string) (string, error) {
buf, err := gitWithinDir(ctx, gitDir, workDir, sshAuthSock, knownHosts, auth, "ls-remote", "--symref", remoteURL, "HEAD")
func getDefaultBranch(ctx context.Context, git *gitutil.GitCLI, remoteURL string) (string, error) {
buf, err := git.Run(ctx, "ls-remote", "--symref", remoteURL, "HEAD")
if err != nil {
return "", errors.Wrapf(err, "error fetching default branch for repository %s", urlutil.RedactCredentials(remoteURL))
}
ss := defaultBranch.FindAllStringSubmatch(buf.String(), -1)
ss := defaultBranch.FindAllStringSubmatch(string(buf), -1)
if len(ss) == 0 || len(ss[0]) != 2 {
return "", errors.Errorf("could not find default branch for repository: %s", urlutil.RedactCredentials(remoteURL))
}
@ -794,3 +752,13 @@ func (md cacheRefMetadata) setGitSnapshot(key string) error {
func (md cacheRefMetadata) setGitRemote(key string) error {
return md.SetString(keyGitRemote, key, gitRemoteIndex+key)
}
func gitCLI(opts ...gitutil.Option) *gitutil.GitCLI {
opts = append([]gitutil.Option{
gitutil.WithExec(runWithStandardUmask),
gitutil.WithStreams(func(ctx context.Context) (stdout, stderr io.WriteCloser, flush func()) {
return logs.NewLogStreams(ctx, false)
}),
}, opts...)
return gitutil.NewGitCLI(opts...)
}

21
source/git/source_test.go
View File

@ -3,6 +3,7 @@ package git
import (
"bytes"
"context"
"io"
"net/http"
"net/http/cgi"
"net/http/httptest"
@ -27,8 +28,10 @@ import (
"github.com/moby/buildkit/snapshot"
containerdsnapshot "github.com/moby/buildkit/snapshot/containerd"
"github.com/moby/buildkit/source"
"github.com/moby/buildkit/util/gitutil"
"github.com/moby/buildkit/util/leaseutil"
"github.com/moby/buildkit/util/progress"
"github.com/moby/buildkit/util/progress/logs"
"github.com/moby/buildkit/util/winlayers"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -285,26 +288,34 @@ func testFetchByTag(t *testing.T, tag, expectedCommitSubject string, isAnnotated
}
if keepGitDir {
git := gitutil.NewGitCLI(
gitutil.WithExec(runWithStandardUmask),
gitutil.WithStreams(func(ctx context.Context) (stdout, stderr io.WriteCloser, flush func()) {
return logs.NewLogStreams(ctx, false)
}),
gitutil.WithWorkTree(dir),
)
if isAnnotatedTag {
// get commit sha that the annotated tag points to
annotatedTagCommit, err := git(ctx, dir, "", "", "rev-list", "-n", "1", tag)
annotatedTagCommit, err := git.Run(ctx, "rev-list", "-n", "1", tag)
require.NoError(t, err)
// get current commit sha
headCommit, err := git(ctx, dir, "", "", "rev-parse", "HEAD")
headCommit, err := git.Run(ctx, "rev-parse", "HEAD")
require.NoError(t, err)
// HEAD should match the actual commit sha (and not the sha of the annotated tag,
// since it's not possible to checkout a non-commit object)
require.Equal(t, annotatedTagCommit.String(), headCommit.String())
require.Equal(t, string(annotatedTagCommit), string(headCommit))
}
// test that we checked out the correct commit
// (in the case of an annotated tag, this message is of the commit the annotated tag points to
// and not the message of the tag)
gitLogOutput, err := git(ctx, dir, "", "", "log", "-n", "1", "--format=%s")
gitLogOutput, err := git.Run(ctx, "log", "-n", "1", "--format=%s")
require.NoError(t, err)
require.True(t, strings.Contains(strings.TrimSpace(gitLogOutput.String()), expectedCommitSubject))
require.Contains(t, strings.TrimSpace(string(gitLogOutput)), expectedCommitSubject)
}
}

243
util/gitutil/git_cli.go Normal file
View File

@ -0,0 +1,243 @@
package gitutil
import (
"bytes"
"context"
"io"
"os"
"os/exec"
"strings"
"github.com/pkg/errors"
)
// GitCLI carries config to pass to the git cli to make running multiple
// commands less repetitive.
type GitCLI struct {
git string
exec func(context.Context, *exec.Cmd) error
args []string
dir string
streams StreamFunc
workTree string
gitDir string
sshAuthSock string
sshKnownHosts string
}
// Option provides a variadic option for configuring the git client.
type Option func(b *GitCLI)
// WithGitBinary sets the git binary path.
func WithGitBinary(path string) Option {
return func(b *GitCLI) {
b.git = path
}
}
// WithExec sets the command exec function.
func WithExec(exec func(context.Context, *exec.Cmd) error) Option {
return func(b *GitCLI) {
b.exec = exec
}
}
// WithArgs sets extra args.
func WithArgs(args ...string) Option {
return func(b *GitCLI) {
b.args = append(b.args, args...)
}
}
// WithDir sets working directory.
//
// This should be a path to any directory within a standard git repository.
func WithDir(dir string) Option {
return func(b *GitCLI) {
b.dir = dir
}
}
// WithWorkTree sets the --work-tree arg.
//
// This should be the path to the top-level directory of the checkout. When
// setting this, you also likely need to set WithGitDir.
func WithWorkTree(workTree string) Option {
return func(b *GitCLI) {
b.workTree = workTree
}
}
// WithGitDir sets the --git-dir arg.
//
// This should be the path to the .git directory. When setting this, you may
// also need to set WithWorkTree, unless you are working with a bare
// repository.
func WithGitDir(gitDir string) Option {
return func(b *GitCLI) {
b.gitDir = gitDir
}
}
// WithSSHAuthSock sets the ssh auth sock.
func WithSSHAuthSock(sshAuthSock string) Option {
return func(b *GitCLI) {
b.sshAuthSock = sshAuthSock
}
}
// WithSSHKnownHosts sets the known hosts file.
func WithSSHKnownHosts(sshKnownHosts string) Option {
return func(b *GitCLI) {
b.sshKnownHosts = sshKnownHosts
}
}
type StreamFunc func(context.Context) (io.WriteCloser, io.WriteCloser, func())
// WithStreams configures a callback for getting the streams for a command. The
// stream callback will be called once for each command, and both writers will
// be closed after the command has finished.
func WithStreams(streams StreamFunc) Option {
return func(b *GitCLI) {
b.streams = streams
}
}
// New initializes a new git client
func NewGitCLI(opts ...Option) *GitCLI {
c := &GitCLI{}
for _, opt := range opts {
opt(c)
}
return c
}
// New returns a new git client with the same config as the current one, but
// with the given options applied on top.
func (cli *GitCLI) New(opts ...Option) *GitCLI {
c := &GitCLI{
git: cli.git,
dir: cli.dir,
workTree: cli.workTree,
gitDir: cli.gitDir,
args: append([]string{}, cli.args...),
streams: cli.streams,
sshAuthSock: cli.sshAuthSock,
sshKnownHosts: cli.sshKnownHosts,
}
for _, opt := range opts {
opt(c)
}
return c
}
// Run executes a git command with the given args.
func (cli *GitCLI) Run(ctx context.Context, args ...string) (_ []byte, err error) {
gitBinary := "git"
if cli.git != "" {
gitBinary = cli.git
}
for {
var cmd *exec.Cmd
if cli.exec == nil {
cmd = exec.CommandContext(ctx, gitBinary)
} else {
cmd = exec.Command(gitBinary)
}
cmd.Dir = cli.dir
if cmd.Dir == "" {
cmd.Dir = cli.workTree
}
// Block sneaky repositories from using repos from the filesystem as submodules.
cmd.Args = append(cmd.Args, "-c", "protocol.file.allow=user")
if cli.workTree != "" {
cmd.Args = append(cmd.Args, "--work-tree", cli.workTree)
}
if cli.gitDir != "" {
cmd.Args = append(cmd.Args, "--git-dir", cli.gitDir)
}
cmd.Args = append(cmd.Args, cli.args...)
cmd.Args = append(cmd.Args, args...)
buf := bytes.NewBuffer(nil)
errbuf := bytes.NewBuffer(nil)
cmd.Stdin = nil
cmd.Stdout = buf
cmd.Stderr = errbuf
if cli.streams != nil {
stdout, stderr, flush := cli.streams(ctx)
if stdout != nil {
cmd.Stdout = io.MultiWriter(stdout, cmd.Stdout)
}
if stderr != nil {
cmd.Stderr = io.MultiWriter(stderr, cmd.Stderr)
}
defer stdout.Close()
defer stderr.Close()
defer func() {
if err != nil {
flush()
}
}()
}
cmd.Env = []string{
"PATH=" + os.Getenv("PATH"),
"GIT_TERMINAL_PROMPT=0",
"GIT_SSH_COMMAND=" + getGitSSHCommand(cli.sshKnownHosts),
// "GIT_TRACE=1",
"GIT_CONFIG_NOSYSTEM=1", // Disable reading from system gitconfig.
"HOME=/dev/null", // Disable reading from user gitconfig.
"LC_ALL=C", // Ensure consistent output.
}
if cli.sshAuthSock != "" {
cmd.Env = append(cmd.Env, "SSH_AUTH_SOCK="+cli.sshAuthSock)
}
if cli.exec != nil {
// remote git commands spawn helper processes that inherit FDs and don't
// handle parent death signal so exec.CommandContext can't be used
err = cli.exec(ctx, cmd)
} else {
err = cmd.Run()
}
if err != nil {
if strings.Contains(errbuf.String(), "--depth") || strings.Contains(errbuf.String(), "shallow") {
if newArgs := argsNoDepth(args); len(args) > len(newArgs) {
args = newArgs
continue
}
}
return buf.Bytes(), errors.Errorf("git error: %s\nstderr:\n%s", err, errbuf.String())
}
return buf.Bytes(), nil
}
}
func getGitSSHCommand(knownHosts string) string {
gitSSHCommand := "ssh -F /dev/null"
if knownHosts != "" {
gitSSHCommand += " -o UserKnownHostsFile=" + knownHosts
} else {
gitSSHCommand += " -o StrictHostKeyChecking=no"
}
return gitSSHCommand
}
func argsNoDepth(args []string) []string {
out := make([]string, 0, len(args))
for _, a := range args {
if a != "--depth=1" {
out = append(out, a)
}
}
return out
}

44
util/gitutil/git_cli_helpers.go Normal file
View File

@ -0,0 +1,44 @@
package gitutil
import (
"context"
"path/filepath"
"strings"
"github.com/pkg/errors"
)
func (cli *GitCLI) Dir() string {
if cli.dir != "" {
return cli.dir
}
return cli.workTree
}
func (cli *GitCLI) WorkTree(ctx context.Context) (string, error) {
if cli.workTree != "" {
return cli.workTree, nil
}
return cli.clean(cli.Run(ctx, "rev-parse", "--show-toplevel"))
}
func (cli *GitCLI) GitDir(ctx context.Context) (string, error) {
if cli.gitDir != "" {
return cli.gitDir, nil
}
dir, err := cli.WorkTree(ctx)
if err != nil {
return "", err
}
return filepath.Join(dir, ".git"), nil
}
func (cli *GitCLI) clean(dt []byte, err error) (string, error) {
out := string(dt)
out = strings.ReplaceAll(strings.Split(out, "\n")[0], "'", "")
if err != nil {
err = errors.New(strings.TrimSuffix(err.Error(), "\n"))
}
return out, err
}