database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

Fix for IAM/STS to have sts_region configurable and not tied to S3 bucket region.

Browse Source
This commit is contained in:
benthompson15
2020-10-01 18:03:44 -05:00
parent cd90372262
commit eb79e604e0
5 changed files with 36 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
oam/install_scripts/columnstore-post-install.in
View File

@ -255,6 +255,7 @@ if [ -z "$MCS_USE_S3_STORAGE" ]; then
MCS_S3_SECRET_ACCESS_KEY="$(find_env_var "MCS_S3_SECRET_ACCESS_KEY")" MCS_S3_SECRET_ACCESS_KEY="$(find_env_var "MCS_S3_SECRET_ACCESS_KEY")"
MCS_S3_REGION="$(find_env_var "MCS_S3_REGION")" MCS_S3_REGION="$(find_env_var "MCS_S3_REGION")"
MCS_S3_ROLE_NAME="$(find_env_var "MCS_S3_ROLE_NAME")" MCS_S3_ROLE_NAME="$(find_env_var "MCS_S3_ROLE_NAME")"
MCS_S3_STS_REGION="$(find_env_var "MCS_S3_STS_REGION")"
MCS_S3_STS_ENDPOINT="$(find_env_var "MCS_S3_STS_ENDPOINT")" MCS_S3_STS_ENDPOINT="$(find_env_var "MCS_S3_STS_ENDPOINT")"
fi fi
@ -274,28 +275,31 @@ if [ ! -z "$MCS_USE_S3_STORAGE" ] && [ $MCS_USE_S3_STORAGE -eq 1 ]; then
@ENGINE_BINDIR@/mcsSetConfig -d Installation DBRootStorageType "storagemanager" @ENGINE_BINDIR@/mcsSetConfig -d Installation DBRootStorageType "storagemanager"
@ENGINE_BINDIR@/mcsSetConfig -d StorageManager Enabled "Y" @ENGINE_BINDIR@/mcsSetConfig -d StorageManager Enabled "Y"
@ENGINE_BINDIR@/mcsSetConfig -d SystemConfig DataFilePlugin "libcloudio.so" @ENGINE_BINDIR@/mcsSetConfig -d SystemConfig DataFilePlugin "libcloudio.so"
sed -i "s|service =.*|service = S3|" /etc/columnstore/storagemanager.cnf sed -i "s|^service =.*|service = S3|" /etc/columnstore/storagemanager.cnf
if [ ! -z "$MCS_S3_REGION" ]; then if [ ! -z "$MCS_S3_REGION" ]; then
sed -i "s|region =.*|region = $MCS_S3_REGION|" /etc/columnstore/storagemanager.cnf sed -i "s|^region =.*|region = $MCS_S3_REGION|" /etc/columnstore/storagemanager.cnf
fi fi
if [ ! -z "$MCS_S3_ROLE_NAME" ]; then if [ ! -z "$MCS_S3_ROLE_NAME" ]; then
sed -i "s|# iam_role_name =.*|iam_role_name = $MCS_S3_ROLE_NAME|" /etc/columnstore/storagemanager.cnf sed -i "s|^# iam_role_name =.*|iam_role_name = $MCS_S3_ROLE_NAME|" /etc/columnstore/storagemanager.cnf
fi
if [ ! -z "$MCS_S3_STS_REGION" ]; then
sed -i "s|^# sts_region =.*|sts_region = $MCS_S3_STS_REGION|" /etc/columnstore/storagemanager.cnf
fi fi
if [ ! -z "$MCS_S3_STS_ENDPOINT" ]; then if [ ! -z "$MCS_S3_STS_ENDPOINT" ]; then
sed -i "s|# sts_endpoint =.*|sts_endpoint = $MCS_S3_STS_ENDPOINT|" /etc/columnstore/storagemanager.cnf sed -i "s|^# sts_endpoint =.*|sts_endpoint = $MCS_S3_STS_ENDPOINT|" /etc/columnstore/storagemanager.cnf
fi fi
sed -i "s|bucket =.*|bucket = $MCS_S3_BUCKET|" /etc/columnstore/storagemanager.cnf sed -i "s|^bucket =.*|bucket = $MCS_S3_BUCKET|" /etc/columnstore/storagemanager.cnf
sed -i "s|# endpoint =.*|endpoint = $MCS_S3_ENDPOINT|" /etc/columnstore/storagemanager.cnf sed -i "s|^# endpoint =.*|endpoint = $MCS_S3_ENDPOINT|" /etc/columnstore/storagemanager.cnf
sed -i "s|# aws_access_key_id =.*|aws_access_key_id = $MCS_S3_ACCESS_KEY_ID|" /etc/columnstore/storagemanager.cnf sed -i "s|^# aws_access_key_id =.*|aws_access_key_id = $MCS_S3_ACCESS_KEY_ID|" /etc/columnstore/storagemanager.cnf
sed -i "s|# aws_secret_access_key =.*|aws_secret_access_key = $MCS_S3_SECRET_ACCESS_KEY|" /etc/columnstore/storagemanager.cnf sed -i "s|^# aws_secret_access_key =.*|aws_secret_access_key = $MCS_S3_SECRET_ACCESS_KEY|" /etc/columnstore/storagemanager.cnf
testS3Connection testS3Connection
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
sed -i "s|iam_role_name =.*|# iam_role_name = |" /etc/columnstore/storagemanager.cnf sed -i "s|^iam_role_name =.*|# iam_role_name = |" /etc/columnstore/storagemanager.cnf
sed -i "s|sts_endpoint =.*|# sts_endpoint = |" /etc/columnstore/storagemanager.cnf sed -i "s|^sts_region =.*|# sts_region = |" /etc/columnstore/storagemanager.cnf
sed -i "s|^sts_endpoint =.*|# sts_endpoint = |" /etc/columnstore/storagemanager.cnf
sed -i "s|^endpoint =.*|# endpoint = |" /etc/columnstore/storagemanager.cnf sed -i "s|^endpoint =.*|# endpoint = |" /etc/columnstore/storagemanager.cnf
sed -i "s|aws_access_key_id =.*|# aws_access_key_id = |" /etc/columnstore/storagemanager.cnf sed -i "s|^aws_access_key_id =.*|# aws_access_key_id = |" /etc/columnstore/storagemanager.cnf
sed -i "s|aws_secret_access_key =.*|# aws_secret_access_key = |" /etc/columnstore/storagemanager.cnf sed -i "s|^aws_secret_access_key =.*|# aws_secret_access_key = |" /etc/columnstore/storagemanager.cnf
echo "There was an error validating the settings used to access S3." echo "There was an error validating the settings used to access S3."
echo "The specified user or role must have GET, PUT, HEAD, and DELETE permissions to the bucket." echo "The specified user or role must have GET, PUT, HEAD, and DELETE permissions to the bucket."
echo "Verify the following environment variables are correct:" echo "Verify the following environment variables are correct:"
@ -305,6 +309,7 @@ if [ ! -z "$MCS_USE_S3_STORAGE" ] && [ $MCS_USE_S3_STORAGE -eq 1 ]; then
echo "MCS_S3_SECRET_ACCESS_KEY" echo "MCS_S3_SECRET_ACCESS_KEY"
echo "MCS_S3_REGION" echo "MCS_S3_REGION"
echo "MCS_S3_ROLE_NAME" echo "MCS_S3_ROLE_NAME"
echo "MCS_S3_STS_REGION"
echo "MCS_S3_STS_ENDPOINT" echo "MCS_S3_STS_ENDPOINT"
echo "After environment variables are fixed, run command: columnstore-post-install" echo "After environment variables are fixed, run command: columnstore-post-install"
exit 1 exit 1

9
storage-manager/src/S3Storage.cpp
View File

@ -104,6 +104,7 @@ S3Storage::S3Storage(bool skipRetry) : skipRetryableErrors(skipRetry)
secret = config->getValue("S3", "aws_secret_access_key"); secret = config->getValue("S3", "aws_secret_access_key");
IAMrole = config->getValue("S3", "iam_role_name"); IAMrole = config->getValue("S3", "iam_role_name");
STSendpoint = config->getValue("S3", "sts_endpoint"); STSendpoint = config->getValue("S3", "sts_endpoint");
STSregion = config->getValue("S3", "sts_region");
if (key.empty()) if (key.empty())
{ {
char *_key_id = getenv("AWS_ACCESS_KEY_ID"); char *_key_id = getenv("AWS_ACCESS_KEY_ID");
@ -502,7 +503,6 @@ ms3_st * S3Storage::getConnection()
Connection &back = freeConns.back(); Connection &back = freeConns.back();
if (back.idleSince.tv_sec + maxIdleSecs <= now.tv_sec) if (back.idleSince.tv_sec + maxIdleSecs <= now.tv_sec)
{ {
logger->log(LOG_ERR, "S3Storage::getConnection(): ms3_deinit");
ms3_deinit(back.conn); ms3_deinit(back.conn);
//connMutexes.erase(back.conn); //connMutexes.erase(back.conn);
back.conn = NULL; back.conn = NULL;
@ -523,13 +523,14 @@ ms3_st * S3Storage::getConnection()
if(!IAMrole.empty()) if(!IAMrole.empty())
{ {
res = ms3_init_assume_role(ret, (IAMrole.empty() ? NULL : IAMrole.c_str()), res = ms3_init_assume_role(ret, (IAMrole.empty() ? NULL : IAMrole.c_str()),
(STSendpoint.empty() ? NULL : STSendpoint.c_str())); (STSendpoint.empty() ? NULL : STSendpoint.c_str()),
(STSregion.empty() ? NULL : STSregion.c_str()));
if (res) if (res)
{ {
// Something is wrong with the assume role so abort as if the ms3_init failed // Something is wrong with the assume role so abort as if the ms3_init failed
logger->log(LOG_ERR, "S3Storage::getConnection(): ERROR: ms3_init_assume_role. Verify iam_role_name = %s, aws_access_key_id, and aws_secret_access_key values.",IAMrole.c_str()); logger->log(LOG_ERR, "S3Storage::getConnection(): ERROR: ms3_init_assume_role. Verify iam_role_name = %s, aws_access_key_id, aws_secret_access_key values. Also check sts_region and sts_endpoint if configured.",IAMrole.c_str());
if (ms3_server_error(ret)) if (ms3_server_error(ret))
logger->log(LOG_ERR, "S3Storage::getConnection(): ms3_error: server says '%s'. role name = %s", ms3_server_error(ret), IAMrole.c_str()); logger->log(LOG_ERR, "S3Storage::getConnection(): ms3_error: server says '%s' role name = %s", ms3_server_error(ret), IAMrole.c_str());
ms3_deinit(ret); ms3_deinit(ret);
ret = NULL; ret = NULL;
} }

1
storage-manager/src/S3Storage.h
View File

@ -57,6 +57,7 @@ class S3Storage : public CloudStorage
std::string endpoint; std::string endpoint;
std::string IAMrole; std::string IAMrole;
std::string STSendpoint; std::string STSendpoint;
std::string STSregion;
struct Connection struct Connection
{ {

17
storage-manager/storagemanager.cnf
View File

@ -110,14 +110,19 @@ bucket = some_bucket
# aws_access_key_id = # aws_access_key_id =
# aws_secret_access_key = # aws_secret_access_key =
# The value for the IAM role that will provide the temporary credentials # If you want StorageManager to assume an IAM role to use for its S3
# via AssumeRole. Assumes the role has been created and permissions are # accesses, specify the name of the role in iam_role_name. The name
# defined correctly prior to operation. This should contain only the name # should be only the name, rather than the full path.
# and omit all path information. #
# The specified role must already exist and have permission to get, put,
# delete, and 'head' on the specified S3 bucket.
# iam_role_name = # iam_role_name =
# Specify the endpoint to connect to for AWS Security Token Service # If an IAM role is specified, sts_endpoint and sts_region are used to specify
# Default of sts.amazonaws.com used if iam_role_name set. # which STS server & region to use to assume the role. The default for
# sts_endpoint is 'sts.amazonaws.com', and the default for sts_region is
# 'us-east-1'.
# sts_region =
# sts_endpoint = # sts_endpoint =
# The LocalStorage section configures the 'local storage' module # The LocalStorage section configures the 'local storage' module

2
utils/libmarias3/libmarias3

Submodule utils/libmarias3/libmarias3 updated: 7fccf14d65...5e599c5f79