database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

MCOL-3915. Cleanup postConfigure. PostConfigure and Installer no longer editing config file

Browse Source
This commit is contained in:
Jose
2020-04-15 17:15:34 +00:00
committed by root
parent 97b8e41ce3
commit ea39ac7f43
5 changed files with 31 additions and 2458 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
oam/etc/Columnstore.xml
View File

@ -1,15 +1,13 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- $Id: Columnstore.xml 3713 2013-08-16 20:04:47Z wweeks $ -->
<Columnstore Version="V1.0.0">
<!--
WARNING: Do not make changes to this file unless directed to do so by
MariaDB service engineers. Incorrect settings can render your system
unusable and will require a service call to correct.
-->
<Columnstore Version="V1.0.0">
<ExeMgr1>
<IPAddr>127.0.0.1</IPAddr>
<Port>8601</Port>
<Module>pm1</Module>
<Module>unassigned</Module>
</ExeMgr1>
<JobProc>
<IPAddr>0.0.0.0</IPAddr>
@ -83,7 +81,7 @@
</BatchInsert>
<PrimitiveServers>
<Count>1</Count>
<ConnectionsPerPrimProc>8</ConnectionsPerPrimProc>
<ConnectionsPerPrimProc>2</ConnectionsPerPrimProc>
<ProcessorThreshold>128</ProcessorThreshold>
<ProcessorQueueSize>10K</ProcessorQueueSize> <!-- minimum of extent size 8192 -->
<DebugLevel>0</DebugLevel>
@ -269,7 +267,6 @@
<hdfsRdwrScratch>/rdwrscratch</hdfsRdwrScratch> <!-- Do not set to an hdfs file path -->
<TempFileDir>/columnstore_tmp_files</TempFileDir>
<SystemTempFileDir>/tmp/columnstore_tmp_files</SystemTempFileDir>
<DataFilePlugin/>
</SystemConfig>
<SystemModuleConfig>
<ModuleType1>dm</ModuleType1>
@ -429,7 +426,7 @@
<!-- The percentage of RAM to use for the disk block cache. Defaults to 70% -->
<!-- Alternatively, this can be specified in absolute terms using
the suffixes 'm' or 'g' to denote size in megabytes or gigabytes.-->
<!-- <NumBlocksPct>70</NumBlocksPct> -->
<!-- <NumBlocksPct>95</NumBlocksPct> -->
<!-- <NumThreads>16</NumThreads> --> <!-- 1-256. Default is 16. -->
<NumCaches>1</NumCaches><!-- # of parallel caches to instantiate -->
<IOMTracing>0</IOMTracing>
@ -490,8 +487,8 @@
<MaxElems>128K</MaxElems> <!-- 128 buckets * 128K * 16 = 256 MB -->
<FifoSizeLargeSide>64</FifoSizeLargeSide>
<PmMaxMemorySmallSide>1G</PmMaxMemorySmallSide>
<TotalUmMemory>25%</TotalUmMemory>
<TotalPmUmMemory>10%</TotalPmUmMemory>
<TotalUmMemory>100%</TotalUmMemory>
<TotalPmUmMemory>100%</TotalPmUmMemory>
<CPUniqueLimit>100</CPUniqueLimit>
<AllowDiskBasedJoin>N</AllowDiskBasedJoin>
<!-- Be careful modifying TempFilePath! On start, ExeMgr deletes
@ -554,4 +551,7 @@
<MaxSockets>30</MaxSockets>
<Enabled>N</Enabled>
</StorageManager>
<DataRedundancyConfig>
<DBRoot1PMs/>
</DataRedundancyConfig>
</Columnstore>

59
oamapps/postConfigure/helpers.cpp
View File

@ -48,7 +48,7 @@ string prompt;
const char* pcommand = 0;
extern bool noPrompting;
bool noPrompting = false;
namespace installer
{
@ -644,10 +644,8 @@ void checkFilesPerPartion(int DBRootCount, Config* sysConfig)
return;
}
void checkMysqlPort( std::string& mysqlPort, Config* sysConfig )
void checkMysqlPort(std::string& mysqlPort)
{
Oam oam;
string tmpDir = startup::StartUp::tmpDir();
while (true)
@ -664,69 +662,18 @@ void checkMysqlPort( std::string& mysqlPort, Config* sysConfig )
int size = oldFile.tellg();
if (size != 0)
{
if ( noPrompting )
{
cout << endl << "The MariaDB ColumnStore port of '" + mysqlPort + "' is already in-use" << endl;
cout << "Either use the command line argument of 'port' to enter a different number" << endl;
cout << "or stop the process that is using port '" + mysqlPort + "'" << endl;
cout << "For No-prompt install, exiting" << endl;
cout << "Please stop the process that is using port '" + mysqlPort + "'" << endl;
exit(1);
}
cout << "The MariaDB ColumnStore port of '" + mysqlPort + "' is already in-use on local server" << endl;
cout << "Either enter a different port to use" << endl;
cout << "or stop the process that is using port '" + mysqlPort + "' and enter '" + mysqlPort + "' to continue" << endl;
while (true)
{
prompt = "Enter port number > ";
pcommand = callReadline(prompt.c_str());
if (pcommand)
{
if (strlen(pcommand) > 0) mysqlPort = pcommand;
callFree(pcommand);
pcommand = 0;
}
if ( atoi(mysqlPort.c_str()) < 1000 || atoi(mysqlPort.c_str()) > 9999)
{
cout << " ERROR: Invalid MariaDB ColumnStore Port ID supplied, must be between 1000-9999" << endl;
}
else
break;
}
}
else
{
cout << endl;
try
{
sysConfig->setConfig("Installation", "MySQLPort", mysqlPort);
}
catch (const std::exception& exc)
{
std::cerr << exc.what() << std::endl;
}
if ( !writeConfig(sysConfig) )
{
cout << "ERROR: Failed trying to update MariaDB Columnstore System Configuration file" << endl;
exit(1);
}
break;
}
}
else
break;
}
// set mysql password
oam.changeMyCnf( "port", mysqlPort );
}
void checkSystemMySQLPort(std::string& mysqlPort, Config* sysConfig, std::string USER, std::string password, ChildModuleList childmodulelist, int IserverTypeInstall, bool pmwithum)

2
oamapps/postConfigure/helpers.h
View File

@ -40,7 +40,7 @@ extern void mysqlSetup();
extern int sendMsgProcMon( std::string module, messageqcpp::ByteStream msg, int requestID, int timeout );
extern int sendReplicationRequest(int IserverTypeInstall, std::string password, bool pmwithum);
extern void checkFilesPerPartion(int DBRootCount, Config* sysConfig);
extern void checkMysqlPort( string& mysqlPort, Config* sysConfig);
extern void checkMysqlPort(string& mysqlPort);
extern bool writeConfig(Config* sysConfig);
extern void checkSystemMySQLPort(std::string& mysqlPort, Config* sysConfig, std::string USER, std::string password, ChildModuleList childmodulelist, int IserverTypeInstall, bool pmwithum);
extern const char* callReadline(string prompt);

291
oamapps/postConfigure/installer.cpp
View File

@ -78,7 +78,6 @@ bool uncommentCalpontXml( string entry);
extern string pwprompt;
bool noPrompting;
string mysqlpw = " ";
bool rootUser = true;
@ -96,8 +95,6 @@ int main(int argc, char* argv[])
ModuleList performancemodulelist;
Module childmodule;
struct sysinfo myinfo;
string prompt;
string nodeps = "-h";
string installer_debug = "0";
@ -216,291 +213,6 @@ int main(int argc, char* argv[])
IserverTypeInstall = atoi(serverTypeInstall.c_str());
if ( installType != "uninstall" )
{
//setup ProcessConfig.xml
switch ( IserverTypeInstall )
{
case (oam::INSTALL_COMBINE_DM_UM_PM): // combined #1 dm/um/pm on a single server
case (oam::INSTALL_COMBINE_DM_UM): // combined #2 dm/um on a same server
case (oam::INSTALL_COMBINE_PM_UM): // combined #3 um/pm on a same server
{
//module ProcessConfig.xml to setup all apps on the dm
if ( !updateProcessConfig(IserverTypeInstall) )
cout << "Update ProcessConfig.xml error" << endl;
break;
}
}
//set Resource Settings
switch ( IserverTypeInstall )
{
case (oam::INSTALL_COMBINE_DM_UM_PM): // combined #1 - dm/um/pm on a single server
{
if ( !writeConfig(sysConfig) )
{
cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl;
exit(1);
}
try
{
sysConfig->setConfig("PrimitiveServers", "RotatingDestination", "n");
}
catch (...)
{
cout << "ERROR: Problem setting RotatingDestination in the MariaDB ColumnStore System Configuration file" << endl;
exit(1);
}
// are we using settings from previous config file?
if ( reuseConfig == "n" )
{
try
{
DBRootStorageType = sysConfig->getConfig(InstallSection, "DBRootStorageType");
}
catch (...)
{
cout << "ERROR: Problem getting DB Storage Data from the MariaDB ColumnStore System Configuration file" << endl;
return false;
}
string numBlocksPct;
if (numBlocksPctParam.empty()) {
numBlocksPct = "50";
if (DBRootStorageType == "hdfs")
numBlocksPct = "25";
}
else {
numBlocksPct = numBlocksPctParam;
}
try
{
sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct);
if (*numBlocksPct.rbegin() == 'M' || *numBlocksPct.rbegin() == 'G') {
cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl;
}
else {
cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl;
}
}
catch (...)
{
cout << "ERROR: Problem setting NumBlocksPct in the MariaDB ColumnStore System Configuration file" << endl;
exit(1);
}
try
{
sysinfo(&myinfo);
}
catch (...) {}
//get memory stats
// long long total = myinfo.totalram / 1024 / 1000;
// adjust max memory, 25% of total memory
string percent;
if (totalUmMemoryParam.empty()) {
percent = "25%";
if (DBRootStorageType == "hdfs")
{
percent = "12%";
}
cout << " Setting 'TotalUmMemory' to " << percent << " of total memory. " << endl;
}
else {
percent = totalUmMemoryParam;
cout << " Setting 'TotalUmMemory' to " << percent << endl;
}
try
{
sysConfig->setConfig("HashJoin", "TotalUmMemory", percent);
}
catch (...)
{
cout << "ERROR: Problem setting TotalUmMemory in the MariaDB ColumnStore System Configuration file" << endl;
exit(1);
}
}
else
{
try
{
string numBlocksPct;
string totalUmMemory;
if (!numBlocksPctParam.empty()) {
numBlocksPct = numBlocksPctParam;
}
else {
numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct");
}
if (!totalUmMemoryParam.empty()) {
totalUmMemory = totalUmMemoryParam;
}
else {
totalUmMemory = sysConfig->getConfig("HashJoin", "TotalUmMemory");
}
if (numBlocksPct.empty() || numBlocksPct == "")
{
numBlocksPct = "50";
}
if (totalUmMemory.empty() || totalUmMemory == "") {
totalUmMemory = "25%";
}
try
{
sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct);
sysConfig->setConfig("HashJoin", "TotalUmMemory", totalUmMemory);
if (*numBlocksPct.rbegin() == 'M' || *numBlocksPct.rbegin() == 'G') {
cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl;
}
else {
cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl;
}
cout << " Setting 'TotalUmMemory' to " << totalUmMemory << endl;
}
catch (...)
{
cout << "ERROR: Problem setting NumBlocksPct/TotalUmMemory in the MariaDB ColumnStore System Configuration file" << endl;
exit(1);
}
}
catch (...)
{
cout << "ERROR: Problem reading NumBlocksPct/TotalUmMemory in the MariaDB ColumnStore System Configuration file" << endl;
exit(1);
}
}
if ( !writeConfig(sysConfig) )
{
cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl;
exit(1);
}
break;
}
default: // normal, seperate UM and PM
{
// are we using settings from previous config file?
if ( reuseConfig == "n" )
{
string numBlocksPct = "70";
string totalUmMemory = "50%";
if (!numBlocksPctParam.empty()) {
numBlocksPct = numBlocksPctParam;
cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl;
}
else {
cout << endl << "NOTE: Using the default setting for 'NumBlocksPct' at " << numBlocksPct << "%" << endl;
}
if (!totalUmMemoryParam.empty()) {
totalUmMemory = totalUmMemoryParam;
cout << endl << "Setting 'TotalUmMemory' to " << totalUmMemory << endl;
}
else {
cout << endl << "Setting 'TotalUmMemory' to " << totalUmMemory << " of total memory." << endl;
}
try
{
sysinfo(&myinfo);
}
catch (...) {}
try
{
sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct);
sysConfig->setConfig("HashJoin", "TotalUmMemory", totalUmMemory);
}
catch (...)
{
cout << "ERROR: Problem setting TotalUmMemory/NumBlocksPct in the MariaDB ColumnStore System Configuration file" << endl;
exit(1);
}
}
else
{
try
{
string numBlocksPct;
string totalUmMemory;
if (!numBlocksPctParam.empty()) {
numBlocksPct = numBlocksPctParam;
}
else {
numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct");
}
if (!totalUmMemoryParam.empty()) {
totalUmMemory = totalUmMemoryParam;
}
else {
totalUmMemory = sysConfig->getConfig("HashJoin", "TotalUmMemory");
}
if (numBlocksPct.empty() || numBlocksPct == "")
{
numBlocksPct = "70";
}
if (totalUmMemory.empty() || totalUmMemory == "") {
totalUmMemory = "50%";
}
try
{
sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct);
sysConfig->setConfig("HashJoin", "TotalUmMemory", totalUmMemory);
if (*numBlocksPct.rbegin() == 'M' || *numBlocksPct.rbegin() == 'G') {
cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl;
}
else {
cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl;
}
cout << " Setting 'TotalUmMemory' to " << totalUmMemory << endl;
}
catch (...)
{
cout << "ERROR: Problem setting NumBlocksPct/TotalUmMemory in the MariaDB ColumnStore System Configuration file" << endl;
exit(1);
}
}
catch (...)
{
cout << "ERROR: Problem reading NumBlocksPct/TotalUmMemory in the MariaDB ColumnStore System Configuration file" << endl;
exit(1);
}
}
if ( !writeConfig(sysConfig) )
{
cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl;
exit(1);
}
break;
}
}
}
if ( !writeConfig(sysConfig) )
{
cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl;
exit(1);
}
//get PPackage Type
try
{
@ -707,9 +419,6 @@ int main(int argc, char* argv[])
// call the mysql setup scripts
mysqlSetup();
sleep(5);
// cout << "(3) SKIP_OAM_INIT is set, so will not start ColumnStore or init the system catalog" << endl;
sysConfig->setConfig("Installation", "MySQLRep", "n");
sysConfig->write();
exit(0);
}

2111
oamapps/postConfigure/postConfigure.cpp
View File

File diff suppressed because it is too large Load Diff