database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

MCOL-3743: Add cmd-line config for S3 parameters to postConfig

Browse Source 
Checkpointing the first cut.  It builds.  Not tested yet.
This commit is contained in:
Patrick LeBlanc
2020-03-16 17:53:19 -04:00
parent 608e941229
commit 39c26f8c68
1 changed files with 126 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

174
oamapps/postConfigure/postConfigure.cpp
View File

@ -65,6 +65,7 @@
#include <readline.h>
#include <history.h>
#include <boost/property_tree/ini_parser.hpp>
#include <boost/filesystem/operations.hpp>
#include <boost/filesystem/path.hpp>
#include <boost/tokenizer.hpp>
@ -176,6 +177,8 @@ string hadoopInstalled = "n";
string mysqlPort = oam::UnassignedName;
string systemName;
bool noPrompting = false;
bool rootUser = true;
string USER = "root";
@ -207,6 +210,46 @@ typedef struct _thread_data_t
std::string command;
} thread_data_t;
bool configureS3 = false;
string s3Bucket, s3Region, s3Endpoint, s3Key, s3Secret, s3CacheSize, s3Location;
void modifySMConfigFile()
{
string smConfigFile = string(MCSSYSCONFDIR) + "/columnstore/storagemanager.cnf";
boost::property_tree::ptree smConfig;
boost::property_tree::ini_parser::read_ini(smConfigFile, smConfig);
smConfig.put<string>("ObjectStorage.service", "S3");
if (!s3Bucket.empty())
smConfig.put<string>("S3.bucket", s3Bucket);
if (!s3Region.empty())
smConfig.put<string>("S3.region", s3Region);
if (!s3Endpoint.empty())
smConfig.put<string>("S3.endpoint", s3Endpoint);
if (!s3Key.empty())
smConfig.put<string>("S3.aws_access_key_id", s3Key);
if (!s3Secret.empty())
smConfig.put<string>("S3.aws_secret_access_key", s3Secret);
if (!s3CacheSize.empty())
smConfig.put<string>("Cache.cache_size", s3CacheSize);
if (!s3Location.empty())
{
boost::filesystem::path tmp; // used to normalize the text of the path
tmp = s3Location + "/storagemanager/metadata";
smConfig.put<string>("ObjectStorage.metadata_path", tmp.normalize().string());
tmp = s3Location + "/storagemanager/journal";
smConfig.put<string>("ObjectStorage.journal_path", tmp.normalize().string());
tmp = s3Location + "/storagemanager/path";
smConfig.put<string>("LocalStorage.path", tmp.normalize().string());
tmp = s3Location + "/storagemanager/cache";
smConfig.put<string>("Cache.path", s3Location + tmp.normalize().string());
}
// save a copy of the original to preserve the in-line documentation
if (!boost::filesystem::exists(smConfigFile + ".original"))
boost::filesystem::copy_file(smConfigFile, smConfigFile + ".original");
boost::property_tree::ini_parser::write_ini(smConfigFile, smConfig);
}
int main(int argc, char* argv[])
{
@ -296,8 +339,88 @@ int main(int argc, char* argv[])
cout << " (percentages of the total memory need to be stated without suffix, explcit values with suffixes M or G)" << endl;
cout << " -totalUmMemory amount of physical memory to utilize for joins, intermediate results and set operations on the UM" << endl;
cout << " (percentages of the total memory need to be stated with suffix %, explcit values with suffixes M or G)" << endl;
cout << endl;
cout << " S3-compat storage quick-configure options:" << endl;
cout << " (See /etc/columnstore/storagemanager.cnf for more information on these)" << endl;
cout << " -sm-bucket bucket The bucket to use." << endl;
cout << " -sm-region region The region to use." << endl;
cout << " -sm-id id The ID to use; equivalent to AWS_ACCESS_KEY_ID in AWS" << endl;
cout << " -sm-secret secret The secret; equivalent to AWS_SECRET_ACCESS_KEY in AWS" << endl;
cout << " -sm-endpoint host The host to connect to" << endl;
cout << " -sm-cache size The amount of disk space to use as a local cache" << endl;
cout << " -sm-prefix prefix Where storagemanager will store its files" << endl;
exit (0);
}
else if (strcmp("-sm-bucket", argv[i]) == 0)
{
if (++i == argc)
{
cout << " ERROR: S3 bucket not provided" << endl;
exit(1);
}
s3Bucket = argv[i];
configureS3 = true;
}
else if (strcmp("-sm-region", argv[i]) == 0)
{
if (++i == argc)
{
cout << " ERROR: S3 region not provided" << endl;
exit(1);
}
s3Region = argv[i];
configureS3 = true;
}
else if (strcmp("-sm-id", argv[i]) == 0)
{
if (++i == argc)
{
cout << " ERROR: S3 ID not provided" << endl;
exit(1);
}
s3Key = argv[i];
configureS3 = true;
}
else if (strcmp("-sm-secret", argv[i]) == 0)
{
if (++i == argc)
{
cout << " ERROR: S3 secret not provided" << endl;
exit(1);
}
s3Secret = argv[i];
configureS3 = true;
}
else if (strcmp("-sm-endpoint", argv[i]) == 0)
{
if (++i == argc)
{
cout << " ERROR: S3 endpoint not provided" << endl;
exit(1);
}
s3Endpoint = argv[i];
configureS3 = true;
}
else if (strcmp("-sm-cache", argv[i]) == 0)
{
if (++i == argc)
{
cout << " ERROR: S3 cache size not provided" << endl;
exit(1);
}
s3CacheSize = argv[i];
configureS3 = true;
}
else if (strcmp("-sm-location", argv[i]) == 0)
{
if (++i == argc)
{
cout << " ERROR: S3 location not provided" << endl;
exit(1);
}
s3Location = argv[i];
configureS3 = true;
}
else if (string("-x") == argv[i])
{
doNotResolveHostNames = true;
@ -4889,6 +5012,9 @@ bool storageSetup(bool amazonInstall)
}
storageManagerInstalled = boost::filesystem::exists(storageManagerLocation);
if (configureS3)
modifySMConfigFile();
//
// get Backend Data storage type
//
@ -4928,54 +5054,6 @@ bool storageSetup(bool amazonInstall)
prompt = "Select the type of data storage (" + storageType + ") > ";
#if 0
// pre-storagemanager version
if (( glusterInstalled == "n" || (glusterInstalled == "y" && singleServerInstall == "1")) && hadoopInstalled == "n" )
{
cout << "There are 2 options when configuring the storage: internal or external" << endl << endl;
prompt = "Select the type of Data Storage [1=internal, 2=external] (" + storageType + ") > ";
}
if ( (glusterInstalled == "y" && singleServerInstall != "1") && hadoopInstalled == "n" )
{
cout << "There are 3 options when configuring the storage: internal, external, or DataRedundancy" << endl << endl;
prompt = "Select the type of Data Storage [1=internal, 2=external, 3=DataRedundancy] (" + storageType + ") > ";
}
if ( ( glusterInstalled == "n" || (glusterInstalled == "y" && singleServerInstall == "1")) && hadoopInstalled == "y" )
{
cout << "There are 3 options when configuring the storage: internal, external, or hdfs" << endl << endl;
prompt = "Select the type of Data Storage [1=internal, 2=external, 4=hdfs] (" + storageType + ") > ";
}
if ( (glusterInstalled == "y" && singleServerInstall != "1") && hadoopInstalled == "y" )
{
cout << "There are 5 options when configuring the storage: internal, external, DataRedundancy, or hdfs" << endl << endl;
prompt = "Select the type of Data Storage [1=internal, 2=external, 3=DataRedundancy, 4=hdfs] (" + storageType + ") > ";
}
cout << " 'internal' - This is specified when a local disk is used for the DBRoot storage." << endl;
cout << " High Availability Server Failover is not Supported in this mode" << endl << endl;
cout << " 'external' - This is specified when the DBRoot directories are mounted." << endl;
cout << " High Availability Server Failover is Supported in this mode." << endl << endl;
if ( glusterInstalled == "y" && singleServerInstall != "1")
{
cout << " 'DataRedundancy' - This is specified when gluster is installed and you want" << endl;
cout << " the DBRoot directories to be controlled by ColumnStore Data Redundancy." << endl;
cout << " High Availability Server Failover is Supported in this mode." << endl;
cout << " NOTE: glusterd service must be running and enabled on all PMs." << endl << endl;
}
if ( hadoopInstalled == "y" )
{
cout << " 'hdfs' - This is specified when hadoop is installed and you want the DBRoot" << endl;
cout << " directories to be controlled by the Hadoop Distributed File System (HDFS)." << endl;
cout << " High Availability Server Failover is Supported in this mode." << endl << endl;
}
#endif
while (true)
{
pcommand = callReadline(prompt.c_str());