database/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2026-10-23 21:28:56 +03:00
Code Activity

Add web-id key type for GCS Workload Identity Federation.

Browse Source 
repo-gcs-key-type=web-id authenticates using an external account credential file (https://google.aip.dev/auth/4117) as generated by gcloud iam workload-identity-pools create-cred-config. The file is specified by repo-gcs-key and defines the audience, the STS token url, and the file that contains the web identity token. The web identity token is exchanged at the STS endpoint for an access token that authenticates the federated principal directly, without a service account key.

The web identity token file is reloaded on each exchange to pick up rotated tokens, e.g. Kubernetes projected service account tokens.

Service account impersonation (service_account_impersonation_url) is not supported. Only the file credential source is supported, and when credential_source.format is specified the type must be text.

Authentication token responses that are not JSON (e.g. an HTML error page) or that contain a nested error object are now reported with httpRequestError() rather than failing while parsing the response. This applies to all GCS key types that request a token.
This commit is contained in:
Adol Rodriguez
2026-10-06 10:06:32 +02:00
committed by GitHub
parent 08fbb97b85
commit 1d217e6263
9 changed files with 627 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
build/config.yaml
View File

@@ -2354,6 +2354,7 @@ option:
- service
- token
- auto
- web-id
command: repo-type
depend:
option: repo-type
@@ -2371,6 +2372,7 @@ option:
list:
- service
- token
- web-id
repo-gcs-user-project:
section: global

7
doc/xml/reference.xml
View File

@@ -737,7 +737,7 @@
<summary>GCS repository key.</summary>
<text>
<p>A token or service key file depending on the <br-option>repo-gcs-key-type</br-option> option.</p>
<p>A token, service key file, or external account credential file depending on the <br-option>repo-gcs-key-type</br-option> option.</p>
</text>
<example>/etc/pgbackrest/gcs-key.json</example>
@@ -751,9 +751,14 @@
<allow-item id="auto">Authorize using the instance service account.</allow-item>
<allow-item id="service">Service account from locally stored key.</allow-item>
<allow-item id="token">For local testing, e.g. <file>fakegcs</file>.</allow-item>
<allow-item id="web-id">Authorize using Workload Identity Federation.</allow-item>
</allow-list>
<p>When <br-option>repo-gcs-key-type=service</br-option> the credentials will be reloaded when the authentication token is renewed.</p>
<p>When <br-option>repo-gcs-key-type=web-id</br-option> the <br-option>repo-gcs-key</br-option> option must contain an external account credential file and the web identity token will be reloaded when the authentication token is renewed.</p>
<p><id>service_account_impersonation_url</id> is not supported. Only <id>credential_source.file</id> is supported and when <id>credential_source.format</id> is specified the type must be <id>text</id>.</p>
</text>
<example>auto</example>

13
doc/xml/release/2020s/2026/2.60.0.xml
View File

@@ -20,6 +20,19 @@
<p>Add <br-option>checksum-page-error</br-option> option to error on invalid page checksums.</p>
</release-item>
<release-item>
<github-issue id="2838"/>
<github-pull-request id="2839"/>
<release-item-contributor-list>
<release-item-contributor id="adol.rodriguez"/>
<release-item-reviewer id="douglas.j.hunley"/>
<release-item-reviewer id="david.steele"/>
</release-item-contributor-list>
<p>Support for <proper>GCS</proper> Workload Identity Federation.</p>
</release-item>
</release-feature-list>
<release-improvement-list>

3
src/config/config.auto.h
View File

@@ -314,6 +314,9 @@ Option value constants
#define CFGOPTVAL_REPO_GCS_KEY_TYPE_AUTO 2
#define CFGOPTVAL_REPO_GCS_KEY_TYPE_AUTO_STRID STRID5S("auto", 2, 0x7d2a16)
#define CFGOPTVAL_REPO_GCS_KEY_TYPE_AUTO_Z "auto"
#define CFGOPTVAL_REPO_GCS_KEY_TYPE_WEB_ID 3
#define CFGOPTVAL_REPO_GCS_KEY_TYPE_WEB_ID_STRID STRID5S("web-id", 3, 0x89d88b78)
#define CFGOPTVAL_REPO_GCS_KEY_TYPE_WEB_ID_Z "web-id"
#define CFGOPTVAL_REPO_HOST_TYPE_SSH STRID5("ssh", 0x22730)
#define CFGOPTVAL_REPO_HOST_TYPE_SSH_Z "ssh"

2
src/config/parse.auto.c.inc
View File

@@ -6383,6 +6383,7 @@ static const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] =
PARSE_RULE_VAL_OPT(RepoGcsKeyType), // opt/repo-gcs-key
PARSE_RULE_VAL_STRID(service), // opt/repo-gcs-key
PARSE_RULE_VAL_STRID(token), // opt/repo-gcs-key
PARSE_RULE_VAL_STRID(web_DS_id), // opt/repo-gcs-key
), // opt/repo-gcs-key
), // opt/repo-gcs-key
), // opt/repo-gcs-key
@@ -6468,6 +6469,7 @@ static const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] =
PARSE_RULE_VAL_STRID(service), // opt/repo-gcs-key-type
PARSE_RULE_VAL_STRID(token), // opt/repo-gcs-key-type
PARSE_RULE_VAL_STRID(auto), // opt/repo-gcs-key-type
PARSE_RULE_VAL_STRID(web_DS_id), // opt/repo-gcs-key-type
), // opt/repo-gcs-key-type
// opt/repo-gcs-key-type
PARSE_RULE_OPTIONAL_DEFAULT // opt/repo-gcs-key-type

214
src/storage/gcs/storage.c
View File

@@ -55,10 +55,14 @@ STRING_STATIC(GCS_QUERY_USER_PROJECT_STR, "userProject
JSON tokens
***********************************************************************************************************************************/
VARIANT_STRDEF_STATIC(GCS_JSON_ACCESS_TOKEN_VAR, "access_token");
VARIANT_STRDEF_STATIC(GCS_JSON_AUDIENCE_VAR, "audience");
VARIANT_STRDEF_STATIC(GCS_JSON_CLIENT_EMAIL_VAR, "client_email");
VARIANT_STRDEF_STATIC(GCS_JSON_CREDENTIAL_SOURCE_VAR, "credential_source");
VARIANT_STRDEF_STATIC(GCS_JSON_ERROR_VAR, "error");
VARIANT_STRDEF_STATIC(GCS_JSON_ERROR_DESCRIPTION_VAR, "error_description");
VARIANT_STRDEF_STATIC(GCS_JSON_EXPIRES_IN_VAR, "expires_in");
VARIANT_STRDEF_STATIC(GCS_JSON_FILE_VAR, "file");
VARIANT_STRDEF_STATIC(GCS_JSON_FORMAT_VAR, "format");
#define GCS_JSON_ITEMS "items"
VARIANT_STRDEF_STATIC(GCS_JSON_ITEMS_VAR, GCS_JSON_ITEMS);
VARIANT_STRDEF_EXTERN(GCS_JSON_GENERATION_VAR, GCS_JSON_GENERATION);
@@ -69,9 +73,12 @@ VARIANT_STRDEF_STATIC(GCS_JSON_NEXT_PAGE_TOKEN_VAR, GCS_JSON_NEX
#define GCS_JSON_PREFIXES "prefixes"
VARIANT_STRDEF_STATIC(GCS_JSON_PREFIXES_VAR, GCS_JSON_PREFIXES);
VARIANT_STRDEF_STATIC(GCS_JSON_PRIVATE_KEY_VAR, "private_key");
VARIANT_STRDEF_STATIC(GCS_JSON_SA_IMPERSONATION_URL_VAR, "service_account_impersonation_url");
VARIANT_STRDEF_EXTERN(GCS_JSON_SIZE_VAR, GCS_JSON_SIZE);
VARIANT_STRDEF_STATIC(GCS_JSON_TOKEN_TYPE_VAR, "token_type");
VARIANT_STRDEF_STATIC(GCS_JSON_TOKEN_URI_VAR, "token_uri");
VARIANT_STRDEF_STATIC(GCS_JSON_TOKEN_URL_VAR, "token_url");
VARIANT_STRDEF_STATIC(GCS_JSON_TYPE_VAR, "type");
#define GCS_JSON_UPDATED "updated"
VARIANT_STRDEF_STATIC(GCS_JSON_UPDATED_VAR, GCS_JSON_UPDATED);
@@ -103,8 +110,10 @@ struct StorageGcs
StorageGcsKeyType keyType; // Auth key type
const String *key; // Key (value depends on key type)
const String *webIdTokenFile; // Web identity token file
const String *webIdAudience; // Web identity provider audience
String *token; // Token
time_t tokenTimeExpire; // Token expiration time (if service auth)
time_t tokenTimeExpire; // Token expiration time (if renewable auth)
HttpUrl *authUrl; // URL for authentication server
HttpClient *authClient; // Client to service auth requests
};
@@ -122,6 +131,8 @@ typedef struct
time_t timeExpire;
} StorageGcsAuthTokenResult;
typedef StorageGcsAuthTokenResult (*StorageGcsAuthFunction)(StorageGcs *this, time_t timeBegin);
static StorageGcsAuthTokenResult
storageGcsAuthToken(HttpRequest *const request, const time_t timeBegin)
{
@@ -137,16 +148,37 @@ storageGcsAuthToken(HttpRequest *const request, const time_t timeBegin)
MEM_CONTEXT_TEMP_BEGIN()
{
// Get the response
const KeyValue *const kvResponse = varKv(jsonToVar(strNewBuf(httpResponseContent(httpRequestResponse(request, true)))));
HttpResponse *const response = httpRequestResponse(request, true);
const String *const contentType = httpHeaderGet(httpResponseHeader(response), HTTP_HEADER_CONTENT_TYPE_STR);
// Error when the response is not OK and is not JSON, e.g. an HTML error page returned by a misconfigured authentication
// server. Otherwise the JSON error fields checked below provide a more detailed report of the error.
if (!httpResponseCodeOk(response) && (contentType == NULL || !strBeginsWithZ(contentType, "application/json")))
httpRequestError(request, response);
const Variant *const responseVariant = jsonToVar(strNewBuf(httpResponseContent(response)));
CHECK(
FormatError, responseVariant != NULL && varType(responseVariant) == varTypeKeyValue,
"token response must be an object");
const KeyValue *const kvResponse = varKv(responseVariant);
// Check for an error
const String *const error = varStr(kvGet(kvResponse, GCS_JSON_ERROR_VAR));
const Variant *const error = kvGet(kvResponse, GCS_JSON_ERROR_VAR);
if (error != NULL)
{
THROW_FMT(
ProtocolError, "unable to get authentication token: [%s] %s", strZ(error),
strZNull(varStr(kvGet(kvResponse, GCS_JSON_ERROR_DESCRIPTION_VAR))));
// Report flat OAuth-style errors, e.g. {"error":"invalid_grant","error_description":"..."}
if (varType(error) == varTypeString)
{
const Variant *const description = kvGet(kvResponse, GCS_JSON_ERROR_DESCRIPTION_VAR);
THROW_FMT(
ProtocolError, "unable to get authentication token: [%s] %s", strZ(varStr(error)),
description != NULL && varType(description) == varTypeString ? strZ(varStr(description)) : "null");
}
// Some errors are nested objects, e.g. {"error":{"code":403,...}}, so report the entire response
httpRequestError(request, response);
}
MEM_CONTEXT_PRIOR_BEGIN()
@@ -294,6 +326,65 @@ storageGcsAuthService(StorageGcs *const this, const time_t timeBegin)
FUNCTION_TEST_RETURN_TYPE(StorageGcsAuthTokenResult, result);
}
/***********************************************************************************************************************************
Exchange a web identity token for an access token using the STS token exchange API
(https://docs.cloud.google.com/iam/docs/reference/sts/rest/v1/TopLevel/token). The resulting token authenticates the federated
principal directly, https://docs.cloud.google.com/iam/docs/workload-identity-federation#direct-resource-access
***********************************************************************************************************************************/
static StorageGcsAuthTokenResult
storageGcsAuthWebId(StorageGcs *const this, const time_t timeBegin)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STORAGE_GCS, this);
FUNCTION_TEST_PARAM(TIME, timeBegin);
FUNCTION_TEST_END();
FUNCTION_AUDIT_STRUCT();
ASSERT(this != NULL);
ASSERT(timeBegin > 0);
StorageGcsAuthTokenResult result = {0};
MEM_CONTEXT_TEMP_BEGIN()
{
// Reload on every exchange because the projected token may rotate
const String *const subjectToken =
strTrim(strNewBuf(storageGetP(storageNewReadP(storagePosixNewP(FSLASH_STR), this->webIdTokenFile))));
CHECK(FormatError, !strEmpty(subjectToken), "web identity token is empty");
// Build form content
HttpQuery *const contentQuery = httpQueryNewP();
httpQueryAdd(contentQuery, STRDEF("audience"), this->webIdAudience);
httpQueryAdd(contentQuery, STRDEF("grant_type"), STRDEF("urn:ietf:params:oauth:grant-type:token-exchange"));
httpQueryAdd(contentQuery, STRDEF("requested_token_type"), STRDEF("urn:ietf:params:oauth:token-type:access_token"));
httpQueryAdd(
contentQuery, STRDEF("scope"),
strNewFmt("https://www.googleapis.com/auth/devstorage.read%s", this->write ? "_write" : "_only"));
httpQueryAdd(contentQuery, STRDEF("subject_token"), subjectToken);
httpQueryAdd(contentQuery, STRDEF("subject_token_type"), STRDEF("urn:ietf:params:oauth:token-type:jwt"));
String *const content = httpQueryRenderP(contentQuery);
HttpHeader *const header = httpHeaderNew(NULL);
httpHeaderAdd(header, HTTP_HEADER_HOST_STR, httpUrlHost(this->authUrl));
httpHeaderAdd(header, HTTP_HEADER_CONTENT_TYPE_STR, HTTP_HEADER_CONTENT_TYPE_APP_FORM_URL_STR);
httpHeaderAdd(header, HTTP_HEADER_CONTENT_LENGTH_STR, strNewFmt("%zu", strSize(content)));
HttpRequest *const request = httpRequestNewP(
this->authClient, HTTP_VERB_POST_STR, httpUrlPath(this->authUrl), .header = header, .content = BUFSTR(content));
MEM_CONTEXT_PRIOR_BEGIN()
{
result = storageGcsAuthToken(request, timeBegin);
}
MEM_CONTEXT_PRIOR_END();
}
MEM_CONTEXT_TEMP_END();
FUNCTION_TEST_RETURN_TYPE(StorageGcsAuthTokenResult, result);
}
/***********************************************************************************************************************************
Get authentication token automatically for instances running in GCE.
@@ -352,19 +443,35 @@ storageGcsAuth(StorageGcs *const this, HttpHeader *const httpHeader)
MEM_CONTEXT_TEMP_BEGIN()
{
// Get the token if it was not supplied by the user
if (this->keyType != storageGcsKeyTypeToken)
{
ASSERT(this->keyType == storageGcsKeyTypeAuto || this->keyType == storageGcsKeyTypeService);
StorageGcsAuthFunction authFunction = NULL;
switch (this->keyType)
{
case storageGcsKeyTypeAuto:
authFunction = storageGcsAuthAuto;
break;
case storageGcsKeyTypeService:
authFunction = storageGcsAuthService;
break;
case storageGcsKeyTypeWebId:
authFunction = storageGcsAuthWebId;
break;
case storageGcsKeyTypeToken:
break;
}
// Get the token if it was not supplied by the user
if (authFunction != NULL)
{
const time_t timeBegin = time(NULL);
// If the current token has expired then request a new one
if (timeBegin >= this->tokenTimeExpire)
{
const StorageGcsAuthTokenResult tokenResult =
this->keyType == storageGcsKeyTypeAuto ?
storageGcsAuthAuto(this, timeBegin) : storageGcsAuthService(this, timeBegin);
const StorageGcsAuthTokenResult tokenResult = authFunction(this, timeBegin);
MEM_CONTEXT_OBJ_BEGIN(this)
{
@@ -1301,6 +1408,87 @@ storageGcsNew(
case storageGcsKeyTypeToken:
this->token = strDup(key);
break;
// Read data from an external account credential file, https://google.aip.dev/auth/4117. This is the file generated by
// gcloud iam workload-identity-pools create-cred-config and contains the audience, token url, and token file.
case storageGcsKeyTypeWebId:
{
ASSERT(key != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
const Variant *const keyVariant =
jsonToVar(strNewBuf(storageGetP(storageNewReadP(storagePosixNewP(FSLASH_STR), key))));
CHECK(
FormatError, keyVariant != NULL && varType(keyVariant) == varTypeKeyValue,
"not an external account credential file");
const KeyValue *const kvKey = varKv(keyVariant);
const Variant *const typeVariant = kvGet(kvKey, GCS_JSON_TYPE_VAR);
const String *const type =
typeVariant != NULL && varType(typeVariant) == varTypeString ? varStr(typeVariant) : NULL;
CHECK(FormatError, type != NULL && strEqZ(type, "external_account"), "not an external account credential file");
// Impersonation would require a second exchange so the federated identity must have direct access
CHECK(
FormatError, kvGet(kvKey, GCS_JSON_SA_IMPERSONATION_URL_VAR) == NULL,
"service account impersonation is not supported");
const Variant *const audienceVariant = kvGet(kvKey, GCS_JSON_AUDIENCE_VAR);
CHECK(FormatError, audienceVariant != NULL, "audience missing");
CHECK(FormatError, varType(audienceVariant) == varTypeString, "audience must be a string");
const String *const audience = varStr(audienceVariant);
const Variant *const tokenUrlVariant = kvGet(kvKey, GCS_JSON_TOKEN_URL_VAR);
CHECK(FormatError, tokenUrlVariant != NULL, "token url missing");
CHECK(FormatError, varType(tokenUrlVariant) == varTypeString, "token url must be a string");
const String *const tokenUrl = varStr(tokenUrlVariant);
const Variant *const credentialSourceVariant = kvGet(kvKey, GCS_JSON_CREDENTIAL_SOURCE_VAR);
CHECK(FormatError, credentialSourceVariant != NULL, "credential source missing");
CHECK(
FormatError, varType(credentialSourceVariant) == varTypeKeyValue,
"credential source must be an object");
const KeyValue *const credentialSource = varKv(credentialSourceVariant);
const Variant *const tokenFileVariant = kvGet(credentialSource, GCS_JSON_FILE_VAR);
CHECK(FormatError, tokenFileVariant != NULL, "only the file credential source is supported");
CHECK(FormatError, varType(tokenFileVariant) == varTypeString, "token file must be a string");
const String *const tokenFile = varStr(tokenFileVariant);
// Only the default text format is supported
const Variant *const format = kvGet(credentialSource, GCS_JSON_FORMAT_VAR);
if (format != NULL)
{
CHECK(
FormatError, varType(format) == varTypeKeyValue, "credential source format is not supported");
const Variant *const formatTypeVariant = kvGet(varKv(format), GCS_JSON_TYPE_VAR);
const String *const formatType =
formatTypeVariant != NULL && varType(formatTypeVariant) == varTypeString ?
varStr(formatTypeVariant) : NULL;
CHECK(
FormatError, formatType != NULL && strEqZ(formatType, "text"),
"credential source format is not supported");
}
MEM_CONTEXT_PRIOR_BEGIN()
{
this->webIdTokenFile = strDup(tokenFile);
this->webIdAudience = strDup(audience);
this->authUrl = httpUrlNewParseP(tokenUrl, .type = httpProtocolTypeHttps);
this->authClient = httpClientNew(
tlsClientNewP(
sckClientNew(httpUrlHost(this->authUrl), httpUrlPort(this->authUrl), timeout, timeout),
httpUrlHost(this->authUrl), timeout, timeout, verifyPeer, .caFile = caFile, .caPath = caPath),
timeout);
}
MEM_CONTEXT_PRIOR_END();
}
MEM_CONTEXT_TEMP_END();
break;
}
}
// Parse the endpoint to extract the host, port, and protocol

1
src/storage/gcs/storage.h
View File

@@ -19,6 +19,7 @@ typedef enum
storageGcsKeyTypeService,
storageGcsKeyTypeToken,
storageGcsKeyTypeAuto,
storageGcsKeyTypeWebId,
} StorageGcsKeyType;
/***********************************************************************************************************************************

2
test/define.yaml
View File

@@ -603,7 +603,7 @@ unit:
# --------------------------------------------------------------------------------------------------------------------------------
- name: storage/gcs
total: 4
total: 5
coverage:
- storage/gcs/helper

399
test/src/module/storage/gcsTest.c
View File

@@ -106,6 +106,7 @@ typedef struct TestRequestParam
const char *contentType;
const char *content;
const char *range;
const char *auth;
} TestRequestParam;
#define testRequestP(write, verb, ...) \
@@ -132,7 +133,7 @@ testRequest(IoWrite *write, const char *verb, TestRequestParam param)
// Add authorization string
if (!param.noAuth)
strCatZ(request, "authorization:X X\r\n");
strCatFmt(request, "authorization:%s\r\n", param.auth == NULL ? "X X" : param.auth);
// Add content-length
strCatFmt(request, "content-length:%zu\r\n", param.content == NULL ? 0 : strlen(param.content));
@@ -270,6 +271,120 @@ testRun(void)
TEST_RESULT_UINT(((StorageGcs *)storageDriver(storage))->chunkSize, 4 * 1024 * 1024, "check chunk size");
TEST_RESULT_STR(((StorageGcs *)storageDriver(storage))->token, TEST_TOKEN_STR, "check token");
TEST_RESULT_BOOL(storageFeature(storage, storageFeaturePath), false, "check path feature");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("web identity config and credential file validation");
hrnCfgEnvRemoveRaw(cfgOptRepoGcsKey);
argList = strLstNew();
hrnCfgArgRawZ(argList, cfgOptStanza, "test");
hrnCfgArgRawStrId(argList, cfgOptRepoType, STORAGE_GCS_TYPE);
hrnCfgArgRawZ(argList, cfgOptRepoPath, "/repo");
hrnCfgArgRawZ(argList, cfgOptRepoGcsBucket, TEST_BUCKET);
hrnCfgArgRawZ(argList, cfgOptRepoGcsKeyType, "web-id");
TEST_ERROR(
hrnCfgLoad(cfgCmdArchivePush, argList, (HrnCfgLoadParam){0}), OptionRequiredError,
"archive-push command requires option: repo1-gcs-key");
hrnCfgEnvRawZ(cfgOptRepoGcsKey, TEST_PATH "/credential.json");
HRN_CFG_LOAD(cfgCmdArchivePush, argList);
HRN_STORAGE_PUT_Z(storageTest, TEST_PATH "/credential.json", "{\"audience\":\"aud\"}");
TEST_ERROR(storageRepoGet(0, false), FormatError, "not an external account credential file");
HRN_STORAGE_PUT_Z(storageTest, TEST_PATH "/credential.json", "[]");
TEST_ERROR(storageRepoGet(0, false), FormatError, "not an external account credential file");
HRN_STORAGE_PUT_Z(storageTest, TEST_PATH "/credential.json", "null");
TEST_ERROR(storageRepoGet(0, false), FormatError, "not an external account credential file");
HRN_STORAGE_PUT_Z(storageTest, TEST_PATH "/credential.json", "{\"type\":{}}");
TEST_ERROR(storageRepoGet(0, false), FormatError, "not an external account credential file");
HRN_STORAGE_PUT_Z(storageTest, TEST_PATH "/credential.json", "{\"type\":\"service_account\"}");
TEST_ERROR(storageRepoGet(0, false), FormatError, "not an external account credential file");
HRN_STORAGE_PUT_Z(
storageTest, TEST_PATH "/credential.json",
"{\"type\":\"external_account\",\"service_account_impersonation_url\":\"https://x\"}");
TEST_ERROR(storageRepoGet(0, false), FormatError, "service account impersonation is not supported");
HRN_STORAGE_PUT_Z(storageTest, TEST_PATH "/credential.json", "{\"type\":\"external_account\"}");
TEST_ERROR(storageRepoGet(0, false), FormatError, "audience missing");
HRN_STORAGE_PUT_Z(
storageTest, TEST_PATH "/credential.json", "{\"type\":\"external_account\",\"audience\":{}}");
TEST_ERROR(storageRepoGet(0, false), FormatError, "audience must be a string");
HRN_STORAGE_PUT_Z(storageTest, TEST_PATH "/credential.json", "{\"type\":\"external_account\",\"audience\":\"aud\"}");
TEST_ERROR(storageRepoGet(0, false), FormatError, "token url missing");
HRN_STORAGE_PUT_Z(
storageTest, TEST_PATH "/credential.json",
"{\"type\":\"external_account\",\"audience\":\"aud\",\"token_url\":{}}");
TEST_ERROR(storageRepoGet(0, false), FormatError, "token url must be a string");
HRN_STORAGE_PUT_Z(
storageTest, TEST_PATH "/credential.json",
"{\"type\":\"external_account\",\"audience\":\"aud\",\"token_url\":\"https://localhost:12345/v1/token\"}");
TEST_ERROR(storageRepoGet(0, false), FormatError, "credential source missing");
HRN_STORAGE_PUT_Z(
storageTest, TEST_PATH "/credential.json",
"{\"type\":\"external_account\",\"audience\":\"aud\",\"token_url\":\"https://localhost:12345/v1/token\","
"\"credential_source\":\"file\"}");
TEST_ERROR(storageRepoGet(0, false), FormatError, "credential source must be an object");
HRN_STORAGE_PUT_Z(
storageTest, TEST_PATH "/credential.json",
"{\"type\":\"external_account\",\"audience\":\"aud\",\"token_url\":\"https://localhost:12345/v1/token\","
"\"credential_source\":{\"url\":\"http://169.254.169.254/token\"}}");
TEST_ERROR(storageRepoGet(0, false), FormatError, "only the file credential source is supported");
HRN_STORAGE_PUT_Z(
storageTest, TEST_PATH "/credential.json",
"{\"type\":\"external_account\",\"audience\":\"aud\",\"token_url\":\"https://localhost:12345/v1/token\","
"\"credential_source\":{\"file\":{}}}");
TEST_ERROR(storageRepoGet(0, false), FormatError, "token file must be a string");
HRN_STORAGE_PUT_Z(
storageTest, TEST_PATH "/credential.json",
"{\"type\":\"external_account\",\"audience\":\"aud\",\"token_url\":\"https://localhost:12345/v1/token\","
"\"credential_source\":{\"file\":\"/token\",\"format\":\"text\"}}");
TEST_ERROR(storageRepoGet(0, false), FormatError, "credential source format is not supported");
HRN_STORAGE_PUT_Z(
storageTest, TEST_PATH "/credential.json",
"{\"type\":\"external_account\",\"audience\":\"aud\",\"token_url\":\"https://localhost:12345/v1/token\","
"\"credential_source\":{\"file\":\"/token\",\"format\":{}}}");
TEST_ERROR(storageRepoGet(0, false), FormatError, "credential source format is not supported");
HRN_STORAGE_PUT_Z(
storageTest, TEST_PATH "/credential.json",
"{\"type\":\"external_account\",\"audience\":\"aud\",\"token_url\":\"https://localhost:12345/v1/token\","
"\"credential_source\":{\"file\":\"/token\",\"format\":{\"type\":\"json\"}}}");
TEST_ERROR(storageRepoGet(0, false), FormatError, "credential source format is not supported");
HRN_STORAGE_PUT_Z(
storageTest, TEST_PATH "/credential.json",
"{\"type\":\"external_account\",\"audience\":\"aud\",\"token_url\":\"https://localhost:12345/v1/token\","
"\"credential_source\":{\"file\":\"/token\",\"format\":{\"type\":123}}}");
TEST_ERROR(storageRepoGet(0, false), FormatError, "credential source format is not supported");
HRN_STORAGE_PUT_Z(
storageTest, TEST_PATH "/credential.json",
"{\"type\":\"external_account\",\"audience\":\"aud\",\"token_url\":\"https://localhost:12345/v1/token\","
"\"credential_source\":{\"file\":\"/token\"}}");
TEST_ASSIGN(storage, storageRepoGet(0, false), "valid credential file");
StorageGcs *const storageGcs = (StorageGcs *)storageDriver(storage);
TEST_RESULT_STR_Z(httpUrl(storageGcs->authUrl), "https://localhost:12345/v1/token", "check web identity STS url");
TEST_RESULT_UINT(httpUrlProtocolType(storageGcs->authUrl), httpProtocolTypeHttps, "check STS protocol");
TEST_RESULT_STR_Z(storageGcs->webIdTokenFile, "/token", "check web identity token file");
TEST_RESULT_STR_Z(storageGcs->webIdAudience, "aud", "check web identity audience");
hrnCfgEnvRemoveRaw(cfgOptRepoGcsKey);
}
// *****************************************************************************************************************************
@@ -326,6 +441,203 @@ testRun(void)
"jwt");
}
// *****************************************************************************************************************************
if (testBegin("storageGcsAuthWebId()"))
{
HRN_FORK_BEGIN()
{
const unsigned int testPort = hrnServerPortNext();
const unsigned int testPortSts = hrnServerPortNext();
HRN_FORK_CHILD_BEGIN(.prefix = "gcs web identity server", .timeout = 10000)
{
TEST_RESULT_VOID(hrnServerRunP(HRN_FORK_CHILD_READ(), hrnServerProtocolTls, testPort), "gcs server");
}
HRN_FORK_CHILD_END();
HRN_FORK_CHILD_BEGIN(.prefix = "gcs sts server", .timeout = 10000)
{
TEST_RESULT_VOID(hrnServerRunP(HRN_FORK_CHILD_READ(), hrnServerProtocolTls, testPortSts), "sts server");
}
HRN_FORK_CHILD_END();
HRN_FORK_PARENT_BEGIN()
{
IoWrite *const service = hrnServerScriptBegin(
ioFdWriteNewOpen(STRDEF("gcs web identity client write"), HRN_FORK_PARENT_WRITE_FD(0), 2000));
IoWrite *const sts = hrnServerScriptBegin(
ioFdWriteNewOpen(STRDEF("gcs sts client write"), HRN_FORK_PARENT_WRITE_FD(1), 2000));
HRN_STORAGE_PUT(storageTest, TEST_PATH "/web-id.jwt", BUFSTRDEF("header.payload+/=\n"));
HRN_STORAGE_PUT(
storageTest, TEST_PATH "/credential.json",
BUFSTR(
strNewFmt(
"{\"type\":\"external_account\","
"\"audience\":\"//iam.googleapis.com/projects/123/locations/global/workloadIdentityPools/pool/"
"providers/provider\","
"\"token_url\":\"https://%s:%u/v1/token\","
"\"credential_source\":{\"file\":\"" TEST_PATH "/web-id.jwt\",\"format\":{\"type\":\"text\"}}}",
strZ(hrnServerHost()), testPortSts)));
StringList *const argList = strLstNew();
hrnCfgArgRawZ(argList, cfgOptStanza, "test");
hrnCfgArgRawStrId(argList, cfgOptRepoType, STORAGE_GCS_TYPE);
hrnCfgArgRawZ(argList, cfgOptRepoPath, "/");
hrnCfgArgRawZ(argList, cfgOptRepoGcsBucket, TEST_BUCKET);
hrnCfgArgRawFmt(argList, cfgOptRepoGcsEndpoint, "https://%s:%u", strZ(hrnServerHost()), testPort);
hrnCfgArgRawZ(argList, cfgOptRepoGcsKeyType, "web-id");
hrnCfgEnvRawZ(cfgOptRepoGcsKey, TEST_PATH "/credential.json");
hrnCfgArgRawBool(argList, cfgOptRepoStorageVerifyTls, TEST_IN_CONTAINER);
HRN_CFG_LOAD(cfgCmdArchivePush, argList);
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("exchange web identity token");
Storage *const storage = storageRepoGet(0, false);
const char *const content =
"audience=%2F%2Fiam.googleapis.com%2Fprojects%2F123%2Flocations%2Fglobal%2FworkloadIdentityPools%2Fpool%2F"
"providers%2Fprovider&"
"grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Atoken-exchange&"
"requested_token_type=urn%3Aietf%3Aparams%3Aoauth%3Atoken-type%3Aaccess_token&"
"scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdevstorage.read_only&"
"subject_token=header.payload%2B%2F%3D&"
"subject_token_type=urn%3Aietf%3Aparams%3Aoauth%3Atoken-type%3Ajwt";
hrnServerScriptAccept(sts);
testRequestP(
sts, HTTP_VERB_POST, .path = "/v1/token", .noAuth = true, .contentType = "application/x-www-form-urlencoded",
.content = content);
testResponseP(sts, .content = "{\"access_token\":\"federated\",\"token_type\":\"Bearer\",\"expires_in\":1}");
hrnServerScriptClose(sts);
hrnServerScriptAccept(service);
testRequestP(service, HTTP_VERB_GET, .object = "missing", .query = "alt=media", .auth = "Bearer federated");
testResponseP(service, .code = 404);
TEST_RESULT_PTR(
storageGetP(storageNewReadP(storage, STRDEF("missing"), .ignoreMissing = true)), NULL,
"use exchanged bearer token");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("renew with rotated web identity token");
HRN_STORAGE_PUT(storageTest, TEST_PATH "/web-id.jwt", BUFSTRDEF("rotated jwt\n"));
const char *const rotatedContent =
"audience=%2F%2Fiam.googleapis.com%2Fprojects%2F123%2Flocations%2Fglobal%2FworkloadIdentityPools%2Fpool%2F"
"providers%2Fprovider&"
"grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Atoken-exchange&"
"requested_token_type=urn%3Aietf%3Aparams%3Aoauth%3Atoken-type%3Aaccess_token&"
"scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdevstorage.read_only&"
"subject_token=rotated%20jwt&"
"subject_token_type=urn%3Aietf%3Aparams%3Aoauth%3Atoken-type%3Ajwt";
hrnServerScriptAccept(sts);
testRequestP(
sts, HTTP_VERB_POST, .path = "/v1/token", .noAuth = true, .contentType = "application/x-www-form-urlencoded",
.content = rotatedContent);
testResponseP(sts, .content = "{\"access_token\":\"renewed\",\"token_type\":\"Bearer\",\"expires_in\":1}");
hrnServerScriptClose(sts);
testRequestP(service, HTTP_VERB_GET, .object = "missing2", .query = "alt=media", .auth = "Bearer renewed");
testResponseP(service, .code = 404);
TEST_RESULT_PTR(
storageGetP(storageNewReadP(storage, STRDEF("missing2"), .ignoreMissing = true)), NULL,
"renew with rotated web identity token");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("sts auth error");
hrnServerScriptAccept(sts);
testRequestP(
sts, HTTP_VERB_POST, .path = "/v1/token", .noAuth = true, .contentType = "application/x-www-form-urlencoded",
.content = rotatedContent);
testResponseP(
sts, .code = 400, .header = "content-type:application/json",
.content = "{\"error\":\"invalid_grant\",\"error_description\":\"token rotated rejected\"}");
hrnServerScriptClose(sts);
TEST_ERROR(
storageGetP(storageNewReadP(storage, STRDEF("missing3"), .ignoreMissing = true)), ProtocolError,
"unable to get authentication token: [invalid_grant] token rotated rejected");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("error on missing token response fields");
hrnServerScriptAccept(sts);
testRequestP(
sts, HTTP_VERB_POST, .path = "/v1/token", .noAuth = true, .contentType = "application/x-www-form-urlencoded",
.content = rotatedContent);
testResponseP(sts, .content = "{\"token_type\":\"Bearer\",\"expires_in\":3600}");
hrnServerScriptClose(sts);
TEST_ERROR(
storageGetP(storageNewReadP(storage, STRDEF("missing4"), .ignoreMissing = true)), FormatError,
"access token missing");
hrnServerScriptAccept(sts);
testRequestP(
sts, HTTP_VERB_POST, .path = "/v1/token", .noAuth = true, .contentType = "application/x-www-form-urlencoded",
.content = rotatedContent);
testResponseP(sts, .content = "{\"access_token\":\"token\",\"token_type\":\"Bearer\"}");
hrnServerScriptClose(sts);
TEST_ERROR(
storageGetP(storageNewReadP(storage, STRDEF("missing5"), .ignoreMissing = true)), FormatError,
"expiry missing");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("sts error response without content-type");
hrnServerScriptAccept(sts);
testRequestP(
sts, HTTP_VERB_POST, .path = "/v1/token", .noAuth = true, .contentType = "application/x-www-form-urlencoded",
.content = rotatedContent);
testResponseP(sts, .code = 400);
hrnServerScriptClose(sts);
TEST_ERROR_FMT(
storageGetP(storageNewReadP(storage, STRDEF("missing6"), .ignoreMissing = true)), ProtocolError,
"HTTP request failed with 400:\n"
"*** Path/Query ***:\n"
"POST /v1/token\n"
"*** Request Headers ***:\n"
"content-length: %zu\n"
"content-type: application/x-www-form-urlencoded\n"
"host: %s",
strlen(rotatedContent), strZ(hrnServerHost()));
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("write storage requests read_write scope");
const char *const writeContent =
"audience=%2F%2Fiam.googleapis.com%2Fprojects%2F123%2Flocations%2Fglobal%2FworkloadIdentityPools%2Fpool%2F"
"providers%2Fprovider&"
"grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Atoken-exchange&"
"requested_token_type=urn%3Aietf%3Aparams%3Aoauth%3Atoken-type%3Aaccess_token&"
"scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdevstorage.read_write&"
"subject_token=rotated%20jwt&"
"subject_token_type=urn%3Aietf%3Aparams%3Aoauth%3Atoken-type%3Ajwt";
hrnServerScriptAccept(sts);
testRequestP(
sts, HTTP_VERB_POST, .path = "/v1/token", .noAuth = true, .contentType = "application/x-www-form-urlencoded",
.content = writeContent);
testResponseP(sts, .content = "{\"access_token\":\"writer\",\"token_type\":\"Bearer\",\"expires_in\":3600}");
hrnServerScriptClose(sts);
hrnServerScriptAccept(service);
testRequestP(service, HTTP_VERB_GET, .object = "missing7", .query = "alt=media", .auth = "Bearer writer");
testResponseP(service, .code = 404);
TEST_RESULT_PTR(
storageGetP(storageNewReadP(storageRepoGet(0, true), STRDEF("missing7"), .ignoreMissing = true)), NULL,
"write-mode storage exchanges with read_write scope");
hrnServerScriptEnd(sts);
hrnServerScriptEnd(service);
}
HRN_FORK_PARENT_END();
}
HRN_FORK_END();
}
// *****************************************************************************************************************************
if (testBegin("StorageGcs, StorageReadGcs, and StorageWriteGcs with HTTPS"))
{
@@ -385,6 +697,7 @@ testRun(void)
// Generate the auth request
const String *const authRequest = testAuthRequest(storage);
const size_t authContentSize = strlen(strstr(strZ(authRequest), "\r\n\r\n") + 4);
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("create bucket");
@@ -419,6 +732,90 @@ testRun(void)
storageGetP(storageNewReadP(storage, STRDEF("fi&le.txt"), .ignoreMissing = true)), ProtocolError,
"unable to get authentication token: [error] description");
// Invalid or missing descriptions retain the standard flat-error format
hrnServerScriptAccept(auth);
hrnServerScriptExpect(auth, authRequest);
testResponseP(auth, .content = "{\"error\":\"error\",\"error_description\":{}}");
hrnServerScriptClose(auth);
TEST_ERROR(
storageGetP(storageNewReadP(storage, STRDEF("fi&le.txt"), .ignoreMissing = true)), ProtocolError,
"unable to get authentication token: [error] null");
hrnServerScriptAccept(auth);
hrnServerScriptExpect(auth, authRequest);
testResponseP(auth, .content = "{\"error\":\"error\"}");
hrnServerScriptClose(auth);
TEST_ERROR(
storageGetP(storageNewReadP(storage, STRDEF("fi&le.txt"), .ignoreMissing = true)), ProtocolError,
"unable to get authentication token: [error] null");
// Responses that are valid JSON but not an object are rejected before the fields are read
hrnServerScriptAccept(auth);
hrnServerScriptExpect(auth, authRequest);
testResponseP(auth, .content = "[]");
hrnServerScriptClose(auth);
TEST_ERROR(
storageGetP(storageNewReadP(storage, STRDEF("fi&le.txt"), .ignoreMissing = true)), FormatError,
"token response must be an object");
hrnServerScriptAccept(auth);
hrnServerScriptExpect(auth, authRequest);
testResponseP(auth, .content = "null");
hrnServerScriptClose(auth);
TEST_ERROR(
storageGetP(storageNewReadP(storage, STRDEF("fi&le.txt"), .ignoreMissing = true)), FormatError,
"token response must be an object");
// Nested JSON errors retain the full response for non-Web-ID authentication
hrnServerScriptAccept(auth);
hrnServerScriptExpect(auth, authRequest);
testResponseP(
auth, .code = 403, .header = "content-type:application/json",
.content = "{\"error\":{\"code\":403,\"message\":\"api disabled\"}}");
hrnServerScriptClose(auth);
TEST_ERROR_FMT(
storageGetP(storageNewReadP(storage, STRDEF("fi&le.txt"), .ignoreMissing = true)), ProtocolError,
"HTTP request failed with 403 (Forbidden):\n"
"*** Path/Query ***:\n"
"POST /token\n"
"*** Request Headers ***:\n"
"content-length: %zu\n"
"content-type: application/x-www-form-urlencoded\n"
"host: %s\n"
"*** Response Headers ***:\n"
"content-length: 47\n"
"content-type: application/json\n"
"*** Response Content ***:\n"
"{\"error\":{\"code\":403,\"message\":\"api disabled\"}}",
authContentSize, strZ(hrnServerHost()));
// Non-JSON errors also retain the full response for non-Web-ID authentication
hrnServerScriptAccept(auth);
hrnServerScriptExpect(auth, authRequest);
testResponseP(auth, .code = 400, .header = "content-type:text/html", .content = "<html>error</html>");
hrnServerScriptClose(auth);
TEST_ERROR_FMT(
storageGetP(storageNewReadP(storage, STRDEF("fi&le.txt"), .ignoreMissing = true)), ProtocolError,
"HTTP request failed with 400:\n"
"*** Path/Query ***:\n"
"POST /token\n"
"*** Request Headers ***:\n"
"content-length: %zu\n"
"content-type: application/x-www-form-urlencoded\n"
"host: %s\n"
"*** Response Headers ***:\n"
"content-length: 18\n"
"content-type: text/html\n"
"*** Response Content ***:\n"
"<html>error</html>",
authContentSize, strZ(hrnServerHost()));
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("ignore missing file");