database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

MCOL-5175 Increase the maximum effective length of S3 secret used as SHA256 key producing S3 signature (#2860)

Browse Source 
Co-authored-by: Roman Nozdrin <rnozdrin@mariadb.com>
This commit is contained in:
Roman Nozdrin
2023-06-07 13:24:42 +01:00
committed by GitHub
parent 727170dc66
commit 23a969dbe2
3 changed files with 55 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
storage-manager/src/S3Storage.cpp
View File

@ -28,14 +28,14 @@
#include <boost/uuid/random_generator.hpp> #include <boost/uuid/random_generator.hpp>
#define BOOST_SPIRIT_THREADSAFE #define BOOST_SPIRIT_THREADSAFE
#ifndef __clang__ #ifndef __clang__
#pragma GCC diagnostic push #pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" #pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#endif #endif
#include <boost/property_tree/ptree.hpp> #include <boost/property_tree/ptree.hpp>
#ifndef __clang__ #ifndef __clang__
#pragma GCC diagnostic pop #pragma GCC diagnostic pop
#endif #endif
#include <boost/property_tree/json_parser.hpp> #include <boost/property_tree/json_parser.hpp>
#include "Utilities.h" #include "Utilities.h"
@ -61,8 +61,7 @@ static size_t WriteCallback(void* contents, size_t size, size_t nmemb, void* use
inline bool retryable_error(uint8_t s3err) inline bool retryable_error(uint8_t s3err)
{ {
return (s3err == MS3_ERR_RESPONSE_PARSE || s3err == MS3_ERR_REQUEST_ERROR || s3err == MS3_ERR_OOM || return (s3err == MS3_ERR_RESPONSE_PARSE || s3err == MS3_ERR_REQUEST_ERROR || s3err == MS3_ERR_OOM ||
s3err == MS3_ERR_IMPOSSIBLE || s3err == MS3_ERR_SERVER || s3err == MS3_ERR_IMPOSSIBLE || s3err == MS3_ERR_SERVER || s3err == MS3_ERR_AUTH_ROLE);
s3err == MS3_ERR_AUTH_ROLE);
} }
// Best effort to map the errors returned by the ms3 API to linux errnos // Best effort to map the errors returned by the ms3 API to linux errnos
@ -76,7 +75,7 @@ const int s3err_to_errno[] = {
EBADMSG, // 4 MS3_ERR_RESPONSE_PARSE EBADMSG, // 4 MS3_ERR_RESPONSE_PARSE
ECOMM, // 5 MS3_ERR_REQUEST_ERROR ECOMM, // 5 MS3_ERR_REQUEST_ERROR
ENOMEM, // 6 MS3_ERR_OOM ENOMEM, // 6 MS3_ERR_OOM
EINVAL, // 7 MS3_ERR_IMPOSSIBLE. Will have to look through the code to find out what this is exactly. EINVAL, // 7 MS3_ERR_IMPOSSIBLE. Will have to look through the code to find out what this is exactly.
EKEYREJECTED, // 8 MS3_ERR_AUTH EKEYREJECTED, // 8 MS3_ERR_AUTH
ENOENT, // 9 MS3_ERR_NOT_FOUND ENOENT, // 9 MS3_ERR_NOT_FOUND
EPROTO, // 10 MS3_ERR_SERVER EPROTO, // 10 MS3_ERR_SERVER
@ -131,6 +130,7 @@ S3Storage::S3Storage(bool skipRetry) : skipRetryableErrors(skipRetry)
string use_http = tolower(config->getValue("S3", "use_http")); string use_http = tolower(config->getValue("S3", "use_http"));
string ssl_verify = tolower(config->getValue("S3", "ssl_verify")); string ssl_verify = tolower(config->getValue("S3", "ssl_verify"));
string port_number = config->getValue("S3", "port_number"); string port_number = config->getValue("S3", "port_number");
string libs3_debug = config->getValue("S3", "libs3_debug");
bool keyMissing = false; bool keyMissing = false;
isEC2Instance = false; isEC2Instance = false;
@ -212,7 +212,10 @@ S3Storage::S3Storage(bool skipRetry) : skipRetryableErrors(skipRetry)
endpoint = config->getValue("S3", "endpoint"); endpoint = config->getValue("S3", "endpoint");
ms3_library_init(); ms3_library_init();
// ms3_debug(); if (libs3_debug == "enabled")
{
ms3_debug();
}
testConnectivityAndPerms(); testConnectivityAndPerms();
} }
@ -308,7 +311,8 @@ void S3Storage::testConnectivityAndPerms()
err = exists(testObjKey, &_exists); err = exists(testObjKey, &_exists);
if (err) if (err)
{ {
logger->log(LOG_CRIT, "S3Storage::exists() failed on nonexistent object. Check 'ListBucket' permissions."); logger->log(LOG_CRIT,
"S3Storage::exists() failed on nonexistent object. Check 'ListBucket' permissions.");
FAIL(HEAD) FAIL(HEAD)
} }
logger->log(LOG_INFO, "S3Storage: S3 connectivity & permissions are OK"); logger->log(LOG_INFO, "S3Storage: S3 connectivity & permissions are OK");
@ -518,8 +522,10 @@ int S3Storage::putObject(const std::shared_ptr<uint8_t[]> data, size_t len, cons
s3err_msgs[s3err], bucket.c_str(), destKey.c_str()); s3err_msgs[s3err], bucket.c_str(), destKey.c_str());
errno = s3err_to_errno[s3err]; errno = s3err_to_errno[s3err];
if (s3err == MS3_ERR_ENDPOINT) if (s3err == MS3_ERR_ENDPOINT)
logger->log(LOG_ERR, "S3Storage::putObject(): Bucket location not match provided endpoint:, bucket = %s, endpoint = %s.", logger->log(
bucket.c_str(), endpoint.c_str()); LOG_ERR,
"S3Storage::putObject(): Bucket location not match provided endpoint:, bucket = %s, endpoint = %s.",
bucket.c_str(), endpoint.c_str());
return -1; return -1;
} }
return 0; return 0;

4
storage-manager/storagemanager.cnf.in
View File

@ -137,6 +137,10 @@ bucket = some_bucket
# Default is ssl_verify = enabled # Default is ssl_verify = enabled
# ssl_verify = disabled # ssl_verify = disabled
# libs3_debug setting controls S3 library debugging printouts
# Default is libs3_debug = disabled
# libs3_debug = disabled
# The LocalStorage section configures the 'local storage' module # The LocalStorage section configures the 'local storage' module
# if specified by ObjectStorage/service. # if specified by ObjectStorage/service.
[LocalStorage] [LocalStorage]

2
utils/libmarias3/libmarias3

Submodule utils/libmarias3/libmarias3 updated: d7028f429b...7d7079a080