docker/quay
1
0
Fork 0
mirror of https://github.com/quay/quay.git synced 2026-01-26 06:21:37 +03:00
Code Activity

storage: Modify the STS S3 implementation of the storage backend to use Web Identity Tokens when available (PROJQUAY-8692) (#3715)

Browse Source 
Backport the Quay STS token file implementation from https://github.com/quay/quay/pull/3670

---------

Co-authored-by: Mathieu Bouchard <mathieu.bouchard@mcn.gouv.qc.ca>
Co-authored-by: Mathieu Bouchard <83231959+bouchardmathieu-qc@users.noreply.github.com>
This commit is contained in:
OpenShift Cherrypick Robot
2025-03-13 20:25:34 +01:00
committed by GitHub
parent 0a4e248ecb
commit 48691d648f
5 changed files with 107 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
build.sh
View File

@@ -18,7 +18,7 @@ git checkout $SHA
REPO=quay.io/quay/quay:$SHA
# Use buildah or podman or docker
# Use buildah, podman or docker
if [ -x /usr/bin/buildah ]; then
BUILDER="/usr/bin/buildah bud"
elif [ -x /usr/bin/podman ]; then
@@ -26,7 +26,11 @@ elif [ -x /usr/bin/podman ]; then
elif [ -x /usr/bin/docker ] ; then
BUILDER="/usr/bin/docker build"
fi
echo $BUILDER
if [[ -z "$BUILDER" ]]; then
echo 'Unable to find buildah, podman or docker' >&2
exit 1
fi
echo $BUILDER
$BUILDER -t $REPO .
echo $REPO

7
config-tool/pkg/lib/fieldgroups/distributedstorage/distributedstorage.go
View File

@@ -379,6 +379,13 @@ func NewDistributedStorageArgs(storageArgs map[string]interface{}) (*shared.Dist
}
}
if value, ok := storageArgs["sts_web_token_filen"]; ok {
newDistributedStorageArgs.STSWebIdentityTokenFile, ok = value.(string)
if !ok {
return newDistributedStorageArgs, errors.New("sts_web_token_file must be a string")
}
}
return newDistributedStorageArgs, nil
}

84
config-tool/pkg/lib/shared/storage_validators.go
View File

@@ -6,6 +6,7 @@ import (
"net/url"
"strconv"
"time"
"os"
"github.com/Azure/azure-storage-blob-go/azblob"
"github.com/aws/aws-sdk-go/aws"
@@ -162,30 +163,73 @@ func ValidateStorage(opts Options, storageName string, storageType string, args
}
roleArn := args.STSRoleArn
sess := session.Must(session.NewSession(&aws.Config{
Credentials: awscredentials.NewStaticCredentials(args.STSUserAccessKey, args.STSUserSecretKey, ""),
}))
svc := sts.New(sess)
if roleArn == "" {
roleArn = os.Getenv("AWS_ROLE_ARN")
}
roleToAssumeArn := roleArn
durationSeconds := int64(3600)
assumeRoleInput := &sts.AssumeRoleInput{
RoleArn: aws.String(roleToAssumeArn),
RoleSessionName: aws.String("quay"),
DurationSeconds: aws.Int64(durationSeconds),
}
assumeRoleOutput, err := svc.AssumeRole(assumeRoleInput)
if err != nil {
errors = append(errors, ValidationError{
Tags: []string{"DISTRIBUTED_STORAGE_CONFIG"},
FieldGroup: fgName,
Message: "Could not fetch credentials from STS. Error: " + err.Error(),
})
break
webIdentityTokenFile := args.STSWebIdentityTokenFile
// Only check the Web Identity Token File variable if no other credentials are present in the config
if args.STSUserAccessKey == "" && args.STSUserSecretKey == "" && args.STSWebIdentityTokenFile == "" {
webIdentityTokenFile = os.Getenv("AWS_WEB_IDENTITY_TOKEN_FILE")
}
accessKey := *assumeRoleOutput.Credentials.AccessKeyId
secretKey := *assumeRoleOutput.Credentials.SecretAccessKey
token = *assumeRoleOutput.Credentials.SessionToken
var credentials *sts.Credentials
// Prefer using web tokens to authenticate and fallback to access and secret keys
if webIdentityTokenFile != "" {
sess := session.Must(session.NewSession())
svc := sts.New(sess)
webIdentityToken, err := os.ReadFile(webIdentityTokenFile)
if err != nil {
errors = append(errors, ValidationError{
Tags: []string{"DISTRIBUTED_STORAGE_CONFIG"},
FieldGroup: fgName,
Message: "Could not read the STS Web Identity Token File, Error: " + err.Error(),
})
break
}
assumeRoleInput := &sts.AssumeRoleWithWebIdentityInput{
RoleArn: aws.String(roleToAssumeArn),
RoleSessionName: aws.String("quay"),
DurationSeconds: aws.Int64(durationSeconds),
WebIdentityToken: aws.String(string(webIdentityToken)),
}
assumeRoleOutput, err := svc.AssumeRoleWithWebIdentity(assumeRoleInput)
if err != nil {
errors = append(errors, ValidationError{
Tags: []string{"DISTRIBUTED_STORAGE_CONFIG"},
FieldGroup: fgName,
Message: "Could not fetch credentials from STS with Web Identity Token. Error: " + err.Error(),
})
break
}
credentials = assumeRoleOutput.Credentials
} else {
sess := session.Must(session.NewSession(&aws.Config{
Credentials: awscredentials.NewStaticCredentials(args.STSUserAccessKey, args.STSUserSecretKey, ""),
}))
svc := sts.New(sess)
assumeRoleInput := &sts.AssumeRoleInput{
RoleArn: aws.String(roleToAssumeArn),
RoleSessionName: aws.String("quay"),
DurationSeconds: aws.Int64(durationSeconds),
}
assumeRoleOutput, err := svc.AssumeRole(assumeRoleInput)
if err != nil {
errors = append(errors, ValidationError{
Tags: []string{"DISTRIBUTED_STORAGE_CONFIG"},
FieldGroup: fgName,
Message: "Could not fetch credentials from STS. Error: " + err.Error(),
})
break
}
credentials = assumeRoleOutput.Credentials
}
accessKey := *credentials.AccessKeyId
secretKey := *credentials.SecretAccessKey
token = *credentials.SessionToken
bucketName = args.S3Bucket
isSecure = true

7
config-tool/pkg/lib/shared/structs.go
View File

@@ -59,7 +59,8 @@ type DistributedStorageArgs struct {
Providers map[string]interface{} `default:"" validate:"" json:"providers,omitempty" yaml:"providers,omitempty"`
StorageConfig map[string]interface{} `default:"" validate:"" json:"storage_config,omitempty" yaml:"storage_config,omitempty"`
// Args for STSS3Storage
STSUserAccessKey string `default:"" validate:"" json:"sts_user_access_key,omitempty" yaml:"sts_user_access_key,omitempty"`
STSUserSecretKey string `default:"" validate:"" json:"sts_user_secret_key,omitempty" yaml:"sts_user_secret_key,omitempty"`
STSRoleArn string `default:"" validate:"" json:"sts_role_arn,omitempty" yaml:"sts_role_arn,omitempty"`
STSUserAccessKey string `default:"" validate:"" json:"sts_user_access_key,omitempty" yaml:"sts_user_access_key,omitempty"`
STSUserSecretKey string `default:"" validate:"" json:"sts_user_secret_key,omitempty" yaml:"sts_user_secret_key,omitempty"`
STSRoleArn string `default:"" validate:"" json:"sts_role_arn,omitempty" yaml:"sts_role_arn,omitempty"`
STSWebIdentityTokenFile string `default:"" validate:"" json:"sts_web_identity_token_file,omitempty" yaml:"sts_web_identity_token_file,omitempty"`
}

41
storage/cloud.py
View File

@@ -1245,28 +1245,39 @@ class STSS3Storage(S3Storage):
maximum_chunk_size_gb=None,
signature_version="s3v4",
):
sts_client = boto3.client(
"sts", aws_access_key_id=sts_user_access_key, aws_secret_access_key=sts_user_secret_key
)
assumed_role = sts_client.assume_role(RoleArn=sts_role_arn, RoleSessionName="quay")
credentials = assumed_role["Credentials"]
deferred_refreshable_credentials = DeferredRefreshableCredentials(
refresh_using=create_assume_role_refresher(
sts_client, {"RoleArn": sts_role_arn, "RoleSessionName": "quay"}
),
method="sts-assume-role",
)
if sts_user_access_key == "" or sts_user_secret_key == "":
sts_client = boto3.client("sts")
else:
sts_client = boto3.client(
"sts",
aws_access_key_id=sts_user_access_key,
aws_secret_access_key=sts_user_secret_key,
)
# !! NOTE !! connect_kwargs here initializes the S3Storage Class not the s3 connection (mis leading re-use of the name)
connect_kwargs = {
"s3_access_key": credentials["AccessKeyId"],
"s3_secret_key": credentials["SecretAccessKey"],
"aws_session_token": credentials["SessionToken"],
"s3_region": s3_region,
"endpoint_url": endpoint_url,
"maximum_chunk_size_gb": maximum_chunk_size_gb,
"deferred_refreshable_credentials": deferred_refreshable_credentials,
"signature_version": signature_version,
}
if sts_role_arn is not None:
assumed_role = sts_client.assume_role(RoleArn=sts_role_arn, RoleSessionName="quay")
credentials = assumed_role["Credentials"]
deferred_refreshable_credentials = DeferredRefreshableCredentials(
refresh_using=create_assume_role_refresher(
sts_client, {"RoleArn": sts_role_arn, "RoleSessionName": "quay"}
),
method="sts-assume-role",
)
connect_kwargs.update(
{
"s3_access_key": credentials["AccessKeyId"],
"s3_secret_key": credentials["SecretAccessKey"],
"aws_session_token": credentials["SessionToken"],
"deferred_refreshable_credentials": deferred_refreshable_credentials,
}
)
super().__init__(context, storage_path, s3_bucket, **connect_kwargs)