diff --git a/oam/install_scripts/columnstore-post-install.in b/oam/install_scripts/columnstore-post-install.in index 0d6dbc18f..b90e9a19a 100755 --- a/oam/install_scripts/columnstore-post-install.in +++ b/oam/install_scripts/columnstore-post-install.in @@ -255,6 +255,7 @@ if [ -z "$MCS_USE_S3_STORAGE" ]; then MCS_S3_SECRET_ACCESS_KEY="$(find_env_var "MCS_S3_SECRET_ACCESS_KEY")" MCS_S3_REGION="$(find_env_var "MCS_S3_REGION")" MCS_S3_ROLE_NAME="$(find_env_var "MCS_S3_ROLE_NAME")" + MCS_S3_STS_REGION="$(find_env_var "MCS_S3_STS_REGION")" MCS_S3_STS_ENDPOINT="$(find_env_var "MCS_S3_STS_ENDPOINT")" fi @@ -274,28 +275,31 @@ if [ ! -z "$MCS_USE_S3_STORAGE" ] && [ $MCS_USE_S3_STORAGE -eq 1 ]; then @ENGINE_BINDIR@/mcsSetConfig -d Installation DBRootStorageType "storagemanager" @ENGINE_BINDIR@/mcsSetConfig -d StorageManager Enabled "Y" @ENGINE_BINDIR@/mcsSetConfig -d SystemConfig DataFilePlugin "libcloudio.so" - sed -i "s|service =.*|service = S3|" /etc/columnstore/storagemanager.cnf + sed -i "s|^service =.*|service = S3|" /etc/columnstore/storagemanager.cnf if [ ! -z "$MCS_S3_REGION" ]; then - sed -i "s|region =.*|region = $MCS_S3_REGION|" /etc/columnstore/storagemanager.cnf + sed -i "s|^region =.*|region = $MCS_S3_REGION|" /etc/columnstore/storagemanager.cnf fi if [ ! -z "$MCS_S3_ROLE_NAME" ]; then - sed -i "s|# iam_role_name =.*|iam_role_name = $MCS_S3_ROLE_NAME|" /etc/columnstore/storagemanager.cnf + sed -i "s|^# iam_role_name =.*|iam_role_name = $MCS_S3_ROLE_NAME|" /etc/columnstore/storagemanager.cnf + fi + if [ ! -z "$MCS_S3_STS_REGION" ]; then + sed -i "s|^# sts_region =.*|sts_region = $MCS_S3_STS_REGION|" /etc/columnstore/storagemanager.cnf fi if [ ! -z "$MCS_S3_STS_ENDPOINT" ]; then - sed -i "s|# sts_endpoint =.*|sts_endpoint = $MCS_S3_STS_ENDPOINT|" /etc/columnstore/storagemanager.cnf + sed -i "s|^# sts_endpoint =.*|sts_endpoint = $MCS_S3_STS_ENDPOINT|" /etc/columnstore/storagemanager.cnf fi - sed -i "s|bucket =.*|bucket = $MCS_S3_BUCKET|" /etc/columnstore/storagemanager.cnf - sed -i "s|# endpoint =.*|endpoint = $MCS_S3_ENDPOINT|" /etc/columnstore/storagemanager.cnf - sed -i "s|# aws_access_key_id =.*|aws_access_key_id = $MCS_S3_ACCESS_KEY_ID|" /etc/columnstore/storagemanager.cnf - sed -i "s|# aws_secret_access_key =.*|aws_secret_access_key = $MCS_S3_SECRET_ACCESS_KEY|" /etc/columnstore/storagemanager.cnf - + sed -i "s|^bucket =.*|bucket = $MCS_S3_BUCKET|" /etc/columnstore/storagemanager.cnf + sed -i "s|^# endpoint =.*|endpoint = $MCS_S3_ENDPOINT|" /etc/columnstore/storagemanager.cnf + sed -i "s|^# aws_access_key_id =.*|aws_access_key_id = $MCS_S3_ACCESS_KEY_ID|" /etc/columnstore/storagemanager.cnf + sed -i "s|^# aws_secret_access_key =.*|aws_secret_access_key = $MCS_S3_SECRET_ACCESS_KEY|" /etc/columnstore/storagemanager.cnf testS3Connection if [ $? -ne 0 ]; then - sed -i "s|iam_role_name =.*|# iam_role_name = |" /etc/columnstore/storagemanager.cnf - sed -i "s|sts_endpoint =.*|# sts_endpoint = |" /etc/columnstore/storagemanager.cnf + sed -i "s|^iam_role_name =.*|# iam_role_name = |" /etc/columnstore/storagemanager.cnf + sed -i "s|^sts_region =.*|# sts_region = |" /etc/columnstore/storagemanager.cnf + sed -i "s|^sts_endpoint =.*|# sts_endpoint = |" /etc/columnstore/storagemanager.cnf sed -i "s|^endpoint =.*|# endpoint = |" /etc/columnstore/storagemanager.cnf - sed -i "s|aws_access_key_id =.*|# aws_access_key_id = |" /etc/columnstore/storagemanager.cnf - sed -i "s|aws_secret_access_key =.*|# aws_secret_access_key = |" /etc/columnstore/storagemanager.cnf + sed -i "s|^aws_access_key_id =.*|# aws_access_key_id = |" /etc/columnstore/storagemanager.cnf + sed -i "s|^aws_secret_access_key =.*|# aws_secret_access_key = |" /etc/columnstore/storagemanager.cnf echo "There was an error validating the settings used to access S3." echo "The specified user or role must have GET, PUT, HEAD, and DELETE permissions to the bucket." echo "Verify the following environment variables are correct:" @@ -305,6 +309,7 @@ if [ ! -z "$MCS_USE_S3_STORAGE" ] && [ $MCS_USE_S3_STORAGE -eq 1 ]; then echo "MCS_S3_SECRET_ACCESS_KEY" echo "MCS_S3_REGION" echo "MCS_S3_ROLE_NAME" + echo "MCS_S3_STS_REGION" echo "MCS_S3_STS_ENDPOINT" echo "After environment variables are fixed, run command: columnstore-post-install" exit 1 diff --git a/storage-manager/src/S3Storage.cpp b/storage-manager/src/S3Storage.cpp index 6762502e3..700b515e8 100644 --- a/storage-manager/src/S3Storage.cpp +++ b/storage-manager/src/S3Storage.cpp @@ -104,6 +104,7 @@ S3Storage::S3Storage(bool skipRetry) : skipRetryableErrors(skipRetry) secret = config->getValue("S3", "aws_secret_access_key"); IAMrole = config->getValue("S3", "iam_role_name"); STSendpoint = config->getValue("S3", "sts_endpoint"); + STSregion = config->getValue("S3", "sts_region"); if (key.empty()) { char *_key_id = getenv("AWS_ACCESS_KEY_ID"); @@ -502,7 +503,6 @@ ms3_st * S3Storage::getConnection() Connection &back = freeConns.back(); if (back.idleSince.tv_sec + maxIdleSecs <= now.tv_sec) { - logger->log(LOG_ERR, "S3Storage::getConnection(): ms3_deinit"); ms3_deinit(back.conn); //connMutexes.erase(back.conn); back.conn = NULL; @@ -523,13 +523,14 @@ ms3_st * S3Storage::getConnection() if(!IAMrole.empty()) { res = ms3_init_assume_role(ret, (IAMrole.empty() ? NULL : IAMrole.c_str()), - (STSendpoint.empty() ? NULL : STSendpoint.c_str())); + (STSendpoint.empty() ? NULL : STSendpoint.c_str()), + (STSregion.empty() ? NULL : STSregion.c_str())); if (res) { // Something is wrong with the assume role so abort as if the ms3_init failed - logger->log(LOG_ERR, "S3Storage::getConnection(): ERROR: ms3_init_assume_role. Verify iam_role_name = %s, aws_access_key_id, and aws_secret_access_key values.",IAMrole.c_str()); + logger->log(LOG_ERR, "S3Storage::getConnection(): ERROR: ms3_init_assume_role. Verify iam_role_name = %s, aws_access_key_id, aws_secret_access_key values. Also check sts_region and sts_endpoint if configured.",IAMrole.c_str()); if (ms3_server_error(ret)) - logger->log(LOG_ERR, "S3Storage::getConnection(): ms3_error: server says '%s'. role name = %s", ms3_server_error(ret), IAMrole.c_str()); + logger->log(LOG_ERR, "S3Storage::getConnection(): ms3_error: server says '%s' role name = %s", ms3_server_error(ret), IAMrole.c_str()); ms3_deinit(ret); ret = NULL; } diff --git a/storage-manager/src/S3Storage.h b/storage-manager/src/S3Storage.h index e0653e8ba..64c8a7281 100644 --- a/storage-manager/src/S3Storage.h +++ b/storage-manager/src/S3Storage.h @@ -57,6 +57,7 @@ class S3Storage : public CloudStorage std::string endpoint; std::string IAMrole; std::string STSendpoint; + std::string STSregion; struct Connection { diff --git a/storage-manager/storagemanager.cnf b/storage-manager/storagemanager.cnf index 04b202ab0..4b0d93172 100644 --- a/storage-manager/storagemanager.cnf +++ b/storage-manager/storagemanager.cnf @@ -110,14 +110,19 @@ bucket = some_bucket # aws_access_key_id = # aws_secret_access_key = -# The value for the IAM role that will provide the temporary credentials -# via AssumeRole. Assumes the role has been created and permissions are -# defined correctly prior to operation. This should contain only the name -# and omit all path information. +# If you want StorageManager to assume an IAM role to use for its S3 +# accesses, specify the name of the role in iam_role_name. The name +# should be only the name, rather than the full path. +# +# The specified role must already exist and have permission to get, put, +# delete, and 'head' on the specified S3 bucket. # iam_role_name = -# Specify the endpoint to connect to for AWS Security Token Service -# Default of sts.amazonaws.com used if iam_role_name set. +# If an IAM role is specified, sts_endpoint and sts_region are used to specify +# which STS server & region to use to assume the role. The default for +# sts_endpoint is 'sts.amazonaws.com', and the default for sts_region is +# 'us-east-1'. +# sts_region = # sts_endpoint = # The LocalStorage section configures the 'local storage' module diff --git a/utils/libmarias3/libmarias3 b/utils/libmarias3/libmarias3 index 7fccf14d6..5e599c5f7 160000 --- a/utils/libmarias3/libmarias3 +++ b/utils/libmarias3/libmarias3 @@ -1 +1 @@ -Subproject commit 7fccf14d65412b001d3cc4a3a5f7b708b9699fc5 +Subproject commit 5e599c5f79fd3bc37a543f6953be78ec753488a3