database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

Fix for IAM/STS to have sts_region configurable and not tied to S3 bucket region.

Browse Source
This commit is contained in:
benthompson15
2020-10-01 18:03:44 -05:00
parent cd90372262
commit eb79e604e0
5 changed files with 36 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
oam/install_scripts/columnstore-post-install.in
View File

@ -255,6 +255,7 @@ if [ -z "$MCS_USE_S3_STORAGE" ]; then
MCS_S3_SECRET_ACCESS_KEY="$(find_env_var "MCS_S3_SECRET_ACCESS_KEY")"
MCS_S3_REGION="$(find_env_var "MCS_S3_REGION")"
MCS_S3_ROLE_NAME="$(find_env_var "MCS_S3_ROLE_NAME")"
MCS_S3_STS_REGION="$(find_env_var "MCS_S3_STS_REGION")"
MCS_S3_STS_ENDPOINT="$(find_env_var "MCS_S3_STS_ENDPOINT")"
fi
@ -274,28 +275,31 @@ if [ ! -z "$MCS_USE_S3_STORAGE" ] && [ $MCS_USE_S3_STORAGE -eq 1 ]; then
@ENGINE_BINDIR@/mcsSetConfig -d Installation DBRootStorageType "storagemanager"
@ENGINE_BINDIR@/mcsSetConfig -d StorageManager Enabled "Y"
@ENGINE_BINDIR@/mcsSetConfig -d SystemConfig DataFilePlugin "libcloudio.so"
sed -i "s|service =.*|service = S3|" /etc/columnstore/storagemanager.cnf
sed -i "s|^service =.*|service = S3|" /etc/columnstore/storagemanager.cnf
if [ ! -z "$MCS_S3_REGION" ]; then
sed -i "s|region =.*|region = $MCS_S3_REGION|" /etc/columnstore/storagemanager.cnf
sed -i "s|^region =.*|region = $MCS_S3_REGION|" /etc/columnstore/storagemanager.cnf
fi
if [ ! -z "$MCS_S3_ROLE_NAME" ]; then
sed -i "s|# iam_role_name =.*|iam_role_name = $MCS_S3_ROLE_NAME|" /etc/columnstore/storagemanager.cnf
sed -i "s|^# iam_role_name =.*|iam_role_name = $MCS_S3_ROLE_NAME|" /etc/columnstore/storagemanager.cnf
fi
if [ ! -z "$MCS_S3_STS_REGION" ]; then
sed -i "s|^# sts_region =.*|sts_region = $MCS_S3_STS_REGION|" /etc/columnstore/storagemanager.cnf
fi
if [ ! -z "$MCS_S3_STS_ENDPOINT" ]; then
sed -i "s|# sts_endpoint =.*|sts_endpoint = $MCS_S3_STS_ENDPOINT|" /etc/columnstore/storagemanager.cnf
sed -i "s|^# sts_endpoint =.*|sts_endpoint = $MCS_S3_STS_ENDPOINT|" /etc/columnstore/storagemanager.cnf
fi
sed -i "s|bucket =.*|bucket = $MCS_S3_BUCKET|" /etc/columnstore/storagemanager.cnf
sed -i "s|# endpoint =.*|endpoint = $MCS_S3_ENDPOINT|" /etc/columnstore/storagemanager.cnf
sed -i "s|# aws_access_key_id =.*|aws_access_key_id = $MCS_S3_ACCESS_KEY_ID|" /etc/columnstore/storagemanager.cnf
sed -i "s|# aws_secret_access_key =.*|aws_secret_access_key = $MCS_S3_SECRET_ACCESS_KEY|" /etc/columnstore/storagemanager.cnf
sed -i "s|^bucket =.*|bucket = $MCS_S3_BUCKET|" /etc/columnstore/storagemanager.cnf
sed -i "s|^# endpoint =.*|endpoint = $MCS_S3_ENDPOINT|" /etc/columnstore/storagemanager.cnf
sed -i "s|^# aws_access_key_id =.*|aws_access_key_id = $MCS_S3_ACCESS_KEY_ID|" /etc/columnstore/storagemanager.cnf
sed -i "s|^# aws_secret_access_key =.*|aws_secret_access_key = $MCS_S3_SECRET_ACCESS_KEY|" /etc/columnstore/storagemanager.cnf
testS3Connection
if [ $? -ne 0 ]; then
sed -i "s|iam_role_name =.*|# iam_role_name = |" /etc/columnstore/storagemanager.cnf
sed -i "s|sts_endpoint =.*|# sts_endpoint = |" /etc/columnstore/storagemanager.cnf
sed -i "s|^iam_role_name =.*|# iam_role_name = |" /etc/columnstore/storagemanager.cnf
sed -i "s|^sts_region =.*|# sts_region = |" /etc/columnstore/storagemanager.cnf
sed -i "s|^sts_endpoint =.*|# sts_endpoint = |" /etc/columnstore/storagemanager.cnf
sed -i "s|^endpoint =.*|# endpoint = |" /etc/columnstore/storagemanager.cnf
sed -i "s|aws_access_key_id =.*|# aws_access_key_id = |" /etc/columnstore/storagemanager.cnf
sed -i "s|aws_secret_access_key =.*|# aws_secret_access_key = |" /etc/columnstore/storagemanager.cnf
sed -i "s|^aws_access_key_id =.*|# aws_access_key_id = |" /etc/columnstore/storagemanager.cnf
sed -i "s|^aws_secret_access_key =.*|# aws_secret_access_key = |" /etc/columnstore/storagemanager.cnf
echo "There was an error validating the settings used to access S3."
echo "The specified user or role must have GET, PUT, HEAD, and DELETE permissions to the bucket."
echo "Verify the following environment variables are correct:"
@ -305,6 +309,7 @@ if [ ! -z "$MCS_USE_S3_STORAGE" ] && [ $MCS_USE_S3_STORAGE -eq 1 ]; then
echo "MCS_S3_SECRET_ACCESS_KEY"
echo "MCS_S3_REGION"
echo "MCS_S3_ROLE_NAME"
echo "MCS_S3_STS_REGION"
echo "MCS_S3_STS_ENDPOINT"
echo "After environment variables are fixed, run command: columnstore-post-install"
exit 1

9
storage-manager/src/S3Storage.cpp
View File

@ -104,6 +104,7 @@ S3Storage::S3Storage(bool skipRetry) : skipRetryableErrors(skipRetry)
secret = config->getValue("S3", "aws_secret_access_key");
IAMrole = config->getValue("S3", "iam_role_name");
STSendpoint = config->getValue("S3", "sts_endpoint");
STSregion = config->getValue("S3", "sts_region");
if (key.empty())
{
char *_key_id = getenv("AWS_ACCESS_KEY_ID");
@ -502,7 +503,6 @@ ms3_st * S3Storage::getConnection()
Connection &back = freeConns.back();
if (back.idleSince.tv_sec + maxIdleSecs <= now.tv_sec)
{
logger->log(LOG_ERR, "S3Storage::getConnection(): ms3_deinit");
ms3_deinit(back.conn);
//connMutexes.erase(back.conn);
back.conn = NULL;
@ -523,13 +523,14 @@ ms3_st * S3Storage::getConnection()
if(!IAMrole.empty())
{
res = ms3_init_assume_role(ret, (IAMrole.empty() ? NULL : IAMrole.c_str()),
(STSendpoint.empty() ? NULL : STSendpoint.c_str()));
(STSendpoint.empty() ? NULL : STSendpoint.c_str()),
(STSregion.empty() ? NULL : STSregion.c_str()));
if (res)
{
// Something is wrong with the assume role so abort as if the ms3_init failed
logger->log(LOG_ERR, "S3Storage::getConnection(): ERROR: ms3_init_assume_role. Verify iam_role_name = %s, aws_access_key_id, and aws_secret_access_key values.",IAMrole.c_str());
logger->log(LOG_ERR, "S3Storage::getConnection(): ERROR: ms3_init_assume_role. Verify iam_role_name = %s, aws_access_key_id, aws_secret_access_key values. Also check sts_region and sts_endpoint if configured.",IAMrole.c_str());
if (ms3_server_error(ret))
logger->log(LOG_ERR, "S3Storage::getConnection(): ms3_error: server says '%s'. role name = %s", ms3_server_error(ret), IAMrole.c_str());
logger->log(LOG_ERR, "S3Storage::getConnection(): ms3_error: server says '%s' role name = %s", ms3_server_error(ret), IAMrole.c_str());
ms3_deinit(ret);
ret = NULL;
}

1
storage-manager/src/S3Storage.h
View File

@ -57,6 +57,7 @@ class S3Storage : public CloudStorage
std::string endpoint;
std::string IAMrole;
std::string STSendpoint;
std::string STSregion;
struct Connection
{

17
storage-manager/storagemanager.cnf
View File

@ -110,14 +110,19 @@ bucket = some_bucket
# aws_access_key_id =
# aws_secret_access_key =
# The value for the IAM role that will provide the temporary credentials
# via AssumeRole. Assumes the role has been created and permissions are
# defined correctly prior to operation. This should contain only the name
# and omit all path information.
# If you want StorageManager to assume an IAM role to use for its S3
# accesses, specify the name of the role in iam_role_name. The name
# should be only the name, rather than the full path.
#
# The specified role must already exist and have permission to get, put,
# delete, and 'head' on the specified S3 bucket.
# iam_role_name =
# Specify the endpoint to connect to for AWS Security Token Service
# Default of sts.amazonaws.com used if iam_role_name set.
# If an IAM role is specified, sts_endpoint and sts_region are used to specify
# which STS server & region to use to assume the role. The default for
# sts_endpoint is 'sts.amazonaws.com', and the default for sts_region is
# 'us-east-1'.
# sts_region =
# sts_endpoint =
# The LocalStorage section configures the 'local storage' module

2
utils/libmarias3/libmarias3

Submodule utils/libmarias3/libmarias3 updated: 7fccf14d65...5e599c5f79