database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-01 06:46:55 +03:00
Code Activity

MCOL-3606 Make ColumnStore use generic paths

Browse Source 
ColumnStore now uses standard bin/lib paths for pretty much everything.
Data path is now hard-coded to /var/lib/columnstore.

This patch also:

* Removes v1 decompression
* Removes a bunch of unneeded files
* Removes COLUMNSTORE_INSTALL_DIR / $INSTALLDIR
* Makes my.cnf.d work for all platforms (MCOL-3558)
* Changes configcpp to use recursive mutex (fixes possible config write deadlock)
* Fixes MCOL-3599 Fix regr functions, The library was installed in the wrong location
* Fixes a bunch of Ubuntu packaging issues
* Changes the binary names of several of the executables so as not to
clash with potential executables from other packages
This commit is contained in:
Andrew Hutchings
2019-11-04 11:06:33 +00:00
parent dea1dec507
commit 8ab9ebb0f4
178 changed files with 1310 additions and 18366 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
oamapps/columnstoreDB/columnstoreDB.cpp
View File

@ -110,7 +110,7 @@ int main(int argc, char** argv)
{
dbrm.setSystemSuspended(true);
sleep(5);
string cmd = startup::StartUp::installDir() + "/bin/save_brm > /var/log/mariadb/columnstore/save_brm.log1 2>&1";
string cmd = "save_brm > /var/log/mariadb/columnstore/save_brm.log1 2>&1";
int rtnCode = system(cmd.c_str());
if (rtnCode == 0)

2
oamapps/columnstoreSupport/CMakeLists.txt
View File

@ -14,7 +14,7 @@ target_link_libraries(columnstoreSupport ${ENGINE_LDFLAGS} readline ncurses ${M
install(TARGETS columnstoreSupport DESTINATION ${ENGINE_BINDIR} COMPONENT platform)
install(PROGRAMS alarmReport.sh bulklogReport.sh configReport.sh hadoopReport.sh
install(PROGRAMS alarmReport.sh bulklogReport.sh configReport.sh
hardwareReport.sh logReport.sh resourceReport.sh softwareReport.sh
DESTINATION ${ENGINE_BINDIR} COMPONENT platform)

2
oamapps/columnstoreSupport/alarmReport.sh
View File

@ -15,7 +15,7 @@ else
fi
#get temp directory
tmpDir=`$INSTALLDIR/bin/getConfig SystemConfig SystemTempFileDir`
tmpDir=`mcsGetConfig SystemConfig SystemTempFileDir`
rm -f ${tmpDir}/logReport.log

3
oamapps/columnstoreSupport/approximateRowCount.sh
View File

@ -9,7 +9,6 @@
#
if [ -z "$MYSQLCMD" ]; then
INSTALLDIR="/usr/local/mariadb/columnstore"
MYSQLCMD="mysql -u root"
fi
@ -50,7 +49,7 @@ colWidth=`$MYSQLCMD calpontsys --skip-column-names -e "$sql"`
#
# Use editem to count the extents.
#
extentCount=`/usr/local/mariadb/columnstore/bin/editem -o $objectid | wc -l`
extentCount=`editem -o $objectid | wc -l`
let extentCount-=2 # Take out the 2 extra rows for header and blank line at end.
let approximateRowCount=$extentCount*8192*1024;

14
oamapps/columnstoreSupport/bulklogReport.sh
View File

@ -8,26 +8,20 @@ else
MODULE="pm1"
fi
if [ $2 ] ; then
INSTALLDIR=$2
else
INSTALLDIR="/usr/local/mariadb/columnstore"
fi
#get temp directory
tmpDir=`$INSTALLDIR/bin/getConfig SystemConfig SystemTempFileDir`
tmpDir=`mcsGetConfig SystemConfig SystemTempFileDir`
rm -f ${tmpDir}/${MODULE}_bulklogReport.txt
{
if test -d $INSTALLDIR/data/bulk ; then
if test -d /var/lib/columnstore/data/bulk ; then
echo " "
echo "-- Check for Errors in Bulk Logs --"
echo " "
echo "################# egrep '(ERR|CRIT)' $INSTALLDIR/data/bulk/log/*.err #################"
echo "################# egrep '(ERR|CRIT)' /var/lib/columnstore/data/bulk/log/*.err #################"
echo " "
egrep '(ERR|CRIT)' $INSTALLDIR/data/bulk/log/*.err 2>/dev/null
egrep '(ERR|CRIT)' /var/lib/columnstore/data/bulk/log/*.err 2>/dev/null
fi
} > ${tmpDir}/${MODULE}_bulklogReport.txt

108
oamapps/columnstoreSupport/columnstoreSupport.cpp
View File

@ -93,8 +93,6 @@ void childReportThread(threadInfo_t& st)
string remoteModuleIP = (*list).moduleIP;
string remoteHostName = (*list).hostName;
string installDir(startup::StartUp::installDir());
pthread_mutex_lock( &mutex1 );
runningThreads++;
//cout << "++ " << runningThreads << endl;
@ -106,10 +104,6 @@ void childReportThread(threadInfo_t& st)
{
outputFile = remoteModuleName + "_" + reportType + "Report.tar.gz";
}
else if (reportType == "hadoop")
{
outputFile = "hadoopReport.txt";
}
else
{
outputFile = remoteModuleName + "_" + reportType + "Report.txt";
@ -124,16 +118,12 @@ void childReportThread(threadInfo_t& st)
system(cmd.c_str());
}
//run remote report script
if (reportType == "hadoop")
cout << "Get " + reportType + " report data" << endl;
else
cout << "Get " + reportType + " report data for " + remoteModuleName + " " << endl;
cout << "Get " + reportType + " report data for " + remoteModuleName + " " << endl;
cout.flush();
string cmd = installDir + "/bin/remote_command.sh " + remoteModuleIP + " " + rootPassword + " '. " + ProfileFile + ";" +
installDir + "/bin/" + reportType + "Report.sh " + remoteModuleName + " " + installDir +
string cmd = "remote_command.sh " + remoteModuleIP + " " + rootPassword + " '. " + ProfileFile + ";" +
reportType + "Report.sh " + remoteModuleName +
"' " + debug_flag + " - forcetty";
int rtnCode = system(cmd.c_str());
@ -143,7 +133,7 @@ void childReportThread(threadInfo_t& st)
cout << "Error with running remote_command.sh, exiting..." << endl;
}
cmd = installDir + "/bin/remote_scp_get.sh " + remoteModuleIP + " " + rootPassword + " " + tmpDir + "/" + outputFile + " > /dev/null 2>&1";
cmd = "remote_scp_get.sh " + remoteModuleIP + " " + rootPassword + " " + tmpDir + "/" + outputFile + " > /dev/null 2>&1";
rtnCode = system(cmd.c_str());
if (WEXITSTATUS(rtnCode) != 0)
@ -162,7 +152,6 @@ void reportThread(string reporttype)
{
string reportType = reporttype;
string installDir(startup::StartUp::installDir());
Oam oam;
pthread_mutex_lock( &mutex1 );
@ -199,7 +188,7 @@ void reportThread(string reporttype)
if (reportType == "log")
{
string cmd = installDir + "/bin/logReport.sh " + localModule + " " + installDir;
string cmd = "logReport.sh " + localModule;
system(cmd.c_str());
cmd = "mv -f " + tmpDir + "/" + localModule + "_logReport.tar.gz .";
@ -221,7 +210,7 @@ void reportThread(string reporttype)
system(cmd.c_str());
cmd = "echo ' ' >> " + outputFile;
system(cmd.c_str());
cmd = installDir + "/bin/mcsadmin getLogConfig >> " + outputFile;
cmd = "mcsadmin getLogConfig >> " + outputFile;
system(cmd.c_str());
}
else
@ -236,7 +225,7 @@ void reportThread(string reporttype)
cmd = "echo '=======================================================================' >> " + outputFile;
system(cmd.c_str());
cmd = installDir + "/bin/" + reportType + "Report.sh " + localModule + " " + installDir;
cmd = reportType + "Report.sh " + localModule;
system(cmd.c_str());
cmd = " mv -f " + tmpDir + "/" + localModule + "_" + reportType + "Report.txt .";
system(cmd.c_str());
@ -253,7 +242,7 @@ void reportThread(string reporttype)
system(cmd.c_str());
cmd = "echo ' ' >> " + outputFile;
system(cmd.c_str());
cmd = installDir + "/bin/mcsadmin getSystemNetworkConfig >> " + outputFile;
cmd = "mcsadmin getSystemNetworkConfig >> " + outputFile;
system(cmd.c_str());
cmd = "echo ' ' >> " + outputFile;
@ -266,7 +255,7 @@ void reportThread(string reporttype)
system(cmd.c_str());
cmd = "echo ' ' >> " + outputFile;
system(cmd.c_str());
cmd = installDir + "/bin/mcsadmin getModuleTypeConfig >> " + outputFile;
cmd = "mcsadmin getModuleTypeConfig >> " + outputFile;
system(cmd.c_str());
cmd = "echo ' ' >> " + outputFile;
@ -279,7 +268,7 @@ void reportThread(string reporttype)
system(cmd.c_str());
cmd = "echo ' ' >> " + outputFile;
system(cmd.c_str());
cmd = installDir + "/bin/mcsadmin getStorageConfig >> " + outputFile;
cmd = "mcsadmin getStorageConfig >> " + outputFile;
system(cmd.c_str());
cmd = "echo ' ' >> " + outputFile;
@ -292,7 +281,7 @@ void reportThread(string reporttype)
system(cmd.c_str());
cmd = "echo ' ' >> " + outputFile;
system(cmd.c_str());
cmd = installDir + "/bin/mcsadmin getStorageStatus >> " + outputFile;
cmd = "mcsadmin getStorageStatus >> " + outputFile;
system(cmd.c_str());
cmd = "echo ' ' >> " + outputFile;
@ -305,7 +294,7 @@ void reportThread(string reporttype)
system(cmd.c_str());
cmd = "echo ' ' >> " + outputFile;
system(cmd.c_str());
cmd = installDir + "/bin/mcsadmin getSystemInfo >> " + outputFile;
cmd = "mcsadmin getSystemInfo >> " + outputFile;
system(cmd.c_str());
cmd = "echo ' ' >> " + outputFile;
@ -318,7 +307,7 @@ void reportThread(string reporttype)
system(cmd.c_str());
cmd = "echo ' ' >> " + outputFile;
system(cmd.c_str());
cmd = installDir + "/bin/mcsadmin getSystemDirectories >> " + outputFile;
cmd = "mcsadmin getSystemDirectories >> " + outputFile;
system(cmd.c_str());
cmd = "echo ' ' >> " + outputFile;
@ -343,7 +332,7 @@ void reportThread(string reporttype)
system(cmd.c_str());
cmd = "echo ' ' >> " + outputFile;
system(cmd.c_str());
string cmd = installDir + "/bin/mcsadmin getModuleResourceUsage " + localModule + " >> " + outputFile;
string cmd = "mcsadmin getModuleResourceUsage " + localModule + " >> " + outputFile;
system(cmd.c_str());
}
else
@ -352,7 +341,7 @@ void reportThread(string reporttype)
system(cmd.c_str());
cmd = "echo ' ' >> " + outputFile;
system(cmd.c_str());
string cmd = installDir + "/bin/mcsadmin getSystemResourceUsage >> " + outputFile;
string cmd = "mcsadmin getSystemResourceUsage >> " + outputFile;
system(cmd.c_str());
}
}
@ -370,7 +359,6 @@ void reportThread(string reporttype)
int main(int argc, char* argv[])
{
Oam oam;
string installDir(startup::StartUp::installDir());
Config* sysConfig = Config::makeConfig();
string SystemSection = "SystemConfig";
@ -463,10 +451,6 @@ int main(int argc, char* argv[])
cout << endl;
cout << "Usage: columnstoreSupport [-h][-a][-hw][-s][-c][-db][-r][-l][-bl][-lc][-p 'root-password'][-de]";
// if hdfs set up print the hadoop option
if (!DataFilePlugin.empty())
cout << "[-hd]";
cout << endl;
cout << " -h help" << endl;
cout << " -a Output all Reports (excluding Bulk Logs Reports)" << endl;
@ -481,10 +465,6 @@ int main(int argc, char* argv[])
cout << " -p password (multi-server systems), root-password or 'ssh' to use 'ssh keys'" << endl;
cout << " -de Debug Flag" << endl;
// if hdfs set up print the hadoop option
if (!DataFilePlugin.empty())
cout << " -hd Output hadoop reports only" << endl;
exit (0);
}
else
@ -736,7 +716,6 @@ int main(int argc, char* argv[])
system("rm -f *_bulklogReport.txt");
system("rm -f *_resourceReport.txt");
system("rm -f *_softwareReport.txt");
system("rm -f hadoopReport.txt");
//
// Software
@ -888,7 +867,7 @@ int main(int argc, char* argv[])
if ( mysqlpw == " " )
{
//go check columnstore.cnf
string file = "/etc/my.cnf.d/columnstore.cnf";
string file = std::string(MCSMYCNFDIR) + "/columnstore.cnf";
ifstream oldFile (file.c_str());
vector <string> lines;
@ -987,11 +966,11 @@ int main(int argc, char* argv[])
system("echo ' ' >> columnstoreSupportReport.txt");
system("echo '******************** DBMS Columnstore System Catalog Data ********************' >> columnstoreSupportReport.txt");
system("echo ' ' >> columnstoreSupportReport.txt");
cmd = "echo '################# " + columnstoreMysql + " calpontsys < " + installDir + "/mysql/dumpcat_mysql.sql ################# ' >> columnstoreSupportReport.txt";
cmd = "echo '################# " + columnstoreMysql + " calpontsys < " + MCSSUPPORTDIR + "/dumpcat_mysql.sql ################# ' >> columnstoreSupportReport.txt";
system(cmd.c_str());
cmd = "echo ' ' >> columnstoreSupportReport.txt";
system(cmd.c_str());
cmd = columnstoreMysql + " calpontsys < " + installDir + "/mysql/dumpcat_mysql.sql >> columnstoreSupportReport.txt";
cmd = columnstoreMysql + " calpontsys < " + MCSSUPPORTDIR + "/dumpcat_mysql.sql >> columnstoreSupportReport.txt";
system(cmd.c_str());
system("echo ' ' >> columnstoreSupportReport.txt");
@ -1031,7 +1010,7 @@ int main(int argc, char* argv[])
system("echo '******************** Database Size Report ********************' >> columnstoreSupportReport.txt");
system("echo ' ' >> columnstoreSupportReport.txt");
string file = installDir + "/bin/databaseSizeReport";
string file = "databaseSizeReport";
ifstream File (file.c_str());
if (File)
@ -1041,18 +1020,18 @@ int main(int argc, char* argv[])
system(cmd.c_str());
cmd = "echo ' ' >> columnstoreSupportReport.txt";
system(cmd.c_str());
cmd = installDir + "/bin/databaseSizeReport >> columnstoreSupportReport.txt";
cmd = "databaseSizeReport >> columnstoreSupportReport.txt";
system(cmd.c_str());
}
system("echo ' ' >> columnstoreSupportReport.txt");
system("echo '******************** DBMS Columnstore config file ********************' >> columnstoreSupportReport.txt");
system("echo ' ' >> columnstoreSupportReport.txt");
string cmd = "echo '################# cat /etc/my.cnf.d/columnstore.cnf ################# ' >> columnstoreSupportReport.txt";
string cmd = "echo '################# cat " + std::string(MCSMYCNFDIR) + "/columnstore.cnf ################# ' >> columnstoreSupportReport.txt";
system(cmd.c_str());
cmd = "echo ' ' >> columnstoreSupportReport.txt";
system(cmd.c_str());
cmd = "cat /etc/my.cnf.d/columnstore.cnf 2>/dev/null >> columnstoreSupportReport.txt";
cmd = "cat " + std::string(MCSMYCNFDIR) + "/columnstore.cnf 2>/dev/null >> columnstoreSupportReport.txt";
system(cmd.c_str());
system("echo ' ' >> columnstoreSupportReport.txt");
@ -1062,54 +1041,13 @@ int main(int argc, char* argv[])
system(cmd.c_str());
cmd = "echo ' ' >> columnstoreSupportReport.txt";
system(cmd.c_str());
cmd = installDir + "/bin/mcsadmin getActiveSqlStatement >> columnstoreSupportReport.txt";
cmd = "mcsadmin getActiveSqlStatement >> columnstoreSupportReport.txt";
system(cmd.c_str());
cmd = "cat columnstoreSupportReport.txt > " + localModule + "_dbmsReport.txt";
system(cmd.c_str());
}
//
// HADOOP
//
if (HADOOP)
{
if (LOCAL || childmodulelist.empty())
{
cout << "Get hadoop report data" << endl;
string cmd = installDir + "/bin/hadoopReport.sh " + localModule + " " + installDir + "\n";
cmd += " mv -f " + tmpDir + "/hadoopReport.txt .";
FILE* pipe = popen(cmd.c_str(), "r");
if (!pipe)
{
cout << "Failed to get a pipe for hadoop health check commands" << endl;
exit(-1);
}
pclose(pipe);
}
else
{
// only get hadoop report from parentOAMModule, because it's consistant view.
parentmodulelist.push_back(parentOAMModule);
threadInfo_t* st = new threadInfo_t;
ChildModuleList::iterator iter = parentmodulelist.begin();
*st = boost::make_tuple(iter, "hadoop");
pthread_t hdthread;
int status = pthread_create (&hdthread, NULL, (void* (*)(void*)) &childReportThread, st);
if ( status != 0 )
{
cout << "ERROR: childreportthread: pthread_create failed, return status = " + oam.itoa(status) << endl;
}
}
}
//wait for all threads to complete
sleep(5);
int wait = 0;
while (true)

8
oamapps/columnstoreSupport/configReport.sh
View File

@ -8,14 +8,8 @@ else
MODULE="pm1"
fi
if [ $2 ] ; then
INSTALLDIR=$2
else
INSTALLDIR="/usr/local/mariadb/columnstore"
fi
#get temp directory
tmpDir=`$INSTALLDIR/bin/getConfig SystemConfig SystemTempFileDir`
tmpDir=`mcsGetConfig SystemConfig SystemTempFileDir`
rm -f ${tmpDir}/${MODULE}_configReport.txt

40
oamapps/columnstoreSupport/findStranded.sh
View File

@ -1,40 +0,0 @@
#!/bin/sh
#
# This script lists Columnstorelpont data files that do not have associated extent map entries.
#
# NOTES:
# 1) Only looks in $COLUMNSTORE_INSTALL_DIR/data* for the data files.
# 2) Only checks for an existing extent with a matching OID, doesn't validate that there is an
# existing extent for the exact segment.
#
# Close enough for hand grenades.
if [ -z "$COLUMNSTORE_INSTALL_DIR" ]; then
COLUMNSTORE_INSTALL_DIR=/usr/local/mariadb/columnstore
fi
export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR
if [ $COLUMNSTORE_INSTALL_DIR != "/usr/local/mariadb/columnstore" ]; then
export PATH=$COLUMNSTORE_INSTALL_DIR/bin:/bin:/usr/bin
export LD_LIBRARY_PATH=$COLUMNSTORE_INSTALL_DIR/lib
fi
cd $COLUMNSTORE_INSTALL_DIR
last=-1
existsInExtentMap=0
count=0
for i in $COLUMNSTORE_INSTALL_DIR/data*/*/*/*/*/*/FILE*cdf; do
let count++
oid=`$COLUMNSTORE_INSTALL_DIR/bin/file2oid.pl $i`
if [ $last -ne $oid ]; then
last=$oid
existsInExtentMap=`$COLUMNSTORE_INSTALL_DIR/bin/editem -o $oid | wc -l`
fi
if [ $existsInExtentMap -le 0 ]; then
echo "Missing oid $oid path $i"
fi
done

3
oamapps/columnstoreSupport/getMinMax.sh
View File

@ -9,7 +9,6 @@
#
if [ -z "$MYSQLCMD" ]; then
INSTALLDIR="/usr/local/mariadb/columnstore"
MYSQLCMD="mysql -u root"
fi
@ -76,7 +75,7 @@ fi
#
# Use the editem utility to get the min and max value.
#
/usr/local/mariadb/columnstore/bin/editem -o $objectid $parm | grep max | awk -v dataType=$dataType '
editem -o $objectid $parm | grep max | awk -v dataType=$dataType '
BEGIN {
allValid=1;
foundValidExtent=0;

66
oamapps/columnstoreSupport/hadoopReport.sh
View File

@ -1,66 +0,0 @@
#! /bin/sh
#
if [ $1 ] ; then
MODULE=$1
else
MODULE="pm1"
fi
if [ $2 ] ; then
INSTALLDIR=$2
else
INSTALLDIR="/usr/local/mariadb/columnstore"
fi
#get temp directory
tmpDir=`$INSTALLDIR/bin/getConfig SystemConfig SystemTempFileDir`
rm -f ${tmpDir}/hdfsReport.txt
{
echo
echo "****************************** HDFS REPORT ********************************"
echo
echo "-- Hadoop version --"
echo
echo "################# hadoop version #################"
echo
hadoop version
echo
echo "-- Data File Plugin --"
echo
echo "######### $INSTALLDIR/bin/getConfig SystemConfig DataFilePlugin ##########"
echo
$INSTALLDIR/bin/getConfig SystemConfig DataFilePlugin
echo
echo
echo "-- Hadoop Configuration File --"
echo
echo "################ core-site.xml ################"
echo
cat $HADOOP_CONF_DIR/core-site.xml
echo
echo "################ hdfs-site.xml ################"
echo
cat $HADOOP_CONF_DIR/hdfs-site.xml
echo
echo "-- Hadoop Health Check --"
echo
echo "################# hdfs dfsadmin -report #################"
echo
hadoop dfsadmin -report 2>/dev/null
echo
echo "-- HDFS check --"
echo
echo "################# hdfs fsck $INSTALLDIR #################"
echo
hadoop fsck $INSTALLDIR 2>/dev/null
} > ${tmpDir}/hadoopReport.txt
exit 0

14
oamapps/columnstoreSupport/hardwareReport.sh
View File

@ -8,14 +8,8 @@ else
MODULE="pm1"
fi
if [ $2 ] ; then
INSTALLDIR=$2
else
INSTALLDIR="/usr/local/mariadb/columnstore"
fi
#get temp directory
tmpDir=`$INSTALLDIR/bin/getConfig SystemConfig SystemTempFileDir`
tmpDir=`mcsGetConfig SystemConfig SystemTempFileDir`
rm -f ${tmpDir}/${MODULE}_hardwareReport.txt
@ -38,11 +32,11 @@ echo "################# cat /etc/issue #################"
echo " "
cat /etc/issue 2>/dev/null
echo " "
echo "run os_check.sh"
echo "run columnstore_os_check.sh"
echo " "
echo "################# /bin/os_check.sh #################"
echo "################# /bin/columnstore_os_check.sh #################"
echo " "
$INSTALLDIR/bin/os_check.sh 2>/dev/null
columnstore_os_check.sh 2>/dev/null
echo " "
echo "-- Server Uptime --"

8
oamapps/columnstoreSupport/logReport.sh
View File

@ -8,14 +8,8 @@ else
MODULE="pm1"
fi
if [ $2 ] ; then
INSTALLDIR=$2
else
INSTALLDIR="/usr/local/mariadb/columnstore"
fi
#get temp directory
tmpDir=`$INSTALLDIR/bin/getConfig SystemConfig SystemTempFileDir`
tmpDir=`mcsGetConfig SystemConfig SystemTempFileDir`
rm -f ${tmpDir}/${MODULE}_logReport.tar.gz

173
oamapps/columnstoreSupport/minMaxCheck.sh
View File

@ -1,173 +0,0 @@
#!/bin/bash
#
# $Id: minMaxCheck.sh 1479 2011-07-20 09:53:32Z wweeks $
#
#
# This script resets the EM Min/Max values for the coluns defined in the cols array.
#
# Usage:
# ./minMax.sh
# runs against the columns defined in the cols array below.
# ./minMaxCheck.sh all
# run for all CP columns in the database (will take a long time against a large database).
# ./minMaxCheck.sh schemaname
# run for CP columns in tables in the schema. If your schema is named all, you'll get all columns.
# ./minMaxCheck.sh schemaname tablename
# run for CP columns against the given table.
# ./minMaxCheck.sh schemaname tablename columnname
# run against the given column.
#
# The script does the following:
# 1) Runs editem for the column.
# 2) Clears the min/max for the column with editem -c.
# 3) Counts the column so that the min/max get set again.
# 4) Runs editem for the column again.
# 5) Diffs the two editem runs and reports / saves the sdiff log files for the column if any of extents had the min/max changed.
#
# Notes:
# 1) An info.log entry will be logged at the end of the script if none of the columns checked had a bad extent map entry.
# Example:
# Mar 11 14:11:29 srvqaperf8 oamcpp[9872]: 29.091980 |0|0|0| I 08 CAL0000: min-max-monitor: okay
# 2) Two warning.log entries will be logged at the end of the script if one or min/max EM entries were corrected.
# Example:
# Mar 11 14:16:36 srvqaperf8 oamcpp[16364]: 36.231731 |0|0|0| W 08 CAL0000: min-max-monitor: some values were reset for oids 3004 3005
# Mar 11 14:16:36 srvqaperf8 oamcpp[16365]: 36.263270 |0|0|0| W 08 CAL0000: min-max-monitor: log files are idb_mm_mon.sdiff.*.15190
# 3) The script outputs the results of the selects in #3 above as it's going through the columns and will echo a line when it finds a col that was corrected.
# Example:
# **** Extent map min/max changed on the scan. See results in idb_mm_mon.sdiff.3039.15190
# Define the cols array. Here's a sql statement that will list the date and datetime cols in the expected format.
# idbmysql calpontsys -e "select concat(objectid, ':', \`schema\`, '.', tablename, '.', columnname) from syscolumn where datatype in (8, 11) and tablename not like 'temp%';" > www.txt
#
# NOTE: The objectid will be looked up again when it's going through the columns in case the one in the array becomes stale.
#
if [ -z "$MYSQLCMD" ]; then
MYSQLCMD="mysql -u root"
fi
if [ -z "$INSTALLDIR" ]; then
INSTALLDIR="/usr/local/mariadb/columnstore"
fi
if [ -z "$PGMPATH" ]; then
PGMPATH=$INSTALLDIR/bin
fi
#get temp directory
tmpDir=`$INSTALLDIR/bin/getConfig SystemConfig SystemTempFileDir`
cols=(
1339664:tpch1.orders.o_orderdate
1339718:tpch1.lineitem.l_shipdate
1339719:tpch1.lineitem.l_commitdate
1339720:tpch1.lineitem.l_receiptdate
1339759:dml.orders.o_orderdate
1339813:dml.lineitem.l_shipdate
1339814:dml.lineitem.l_commitdate
1339815:dml.lineitem.l_receiptdate
)
#
# If called with "all", run the script against all of the column types that use CP.
#
if [ $# -eq 1 ] && [ "$1" == "all" ]; then
$MYSQLCMD --execute="select concat(objectid, ':', \`schema\`, '.', tablename, '.', columnname) from syscolumn where datatype not in (4, 10, 13) and not (datatype = 2 and columnlength > 8) and not (datatype = 12 and columnlength > 7);" calpontsys --skip-column-names > ${tmpDir}/idb_mm_mon.cols
cols=( $( cat${tmpDir}/idb_mm_mon.cols ) )
rm -f ${tmpDir}/idb_mm_mon.cols
#
# Else if one parm passed, run against the columns in the given schema.
#
elif [ $# -eq 1 ]; then
db=$1
$MYSQLCMD --execute="select concat(objectid, ':', \`schema\`, '.', tablename, '.', columnname) from syscolumn where datatype not in (4, 10, 13) and not (datatype = 2 and columnlength > 8) and not (datatype = 12 and columnlength > 7) and \`schema\` = '$db';" calpontsys --skip-column-names > ${tmpDir}/idb_mm_mon.cols
cols=( $( cat ${tmpDir}/idb_mm_mon.cols ) )
rm -f ${tmpDir}/idb_mm_mon.cols
#
# Else if two parms passed, run the script against all the columns that use CP for that table.
#
elif [ $# -eq 2 ]; then
db=$1
tbl=$2
$MYSQLCMD --execute="select concat(objectid, ':', \`schema\`, '.', tablename, '.', columnname) from syscolumn where datatype not in (4, 10, 13) and not (datatype = 2 and columnlength > 8) and not (datatype = 12 and columnlength > 7) and \`schema\` = '$db' and tablename = '$tbl';" calpontsys --skip-column-names > ${tmpDir}/idb_mm_mon.cols
cols=( $( cat ${tmpDir}/idb_mm_mon.cols ) )
rm -f ${tmpDir}/idb_mm_mon.cols
#
# Else if three parms passed, run the script against the column.
#
elif [ $# -eq 3 ]; then
db=$1
tbl=$2
col=$3
$MYSQLCMD --execute="select concat(objectid, ':', \`schema\`, '.', tablename, '.', columnname) from syscolumn where \`schema\` = '$db' and tablename = '$tbl' and columnname='$col';" calpontsys --skip-column-names > ${tmpDir}/idb_mm_mon.cols
cols=( $( cat ${tmpDir}/idb_mm_mon.cols ) )
rm -f ${tmpDir}/idb_mm_mon.cols
fi
i=0
j=0
if [ ${#cols[@]} -le 0 ]; then
$PGMPATH/cplogger -w 0 "min-max-monitor: no qualifying columns" "$badoidlist"
echo "min-max-monitor: no qualifying columns" "$badoidlist"
exit 1
fi
badoidlist=
while [ $i -lt ${#cols[@]} ]; do
let row=$i+1
echo ""
echo "Evaluating $row of ${#cols[@]} at `date`. Col is ${cols[$i]}."
eval $(echo ${cols[$i]} | awk -F: '{printf "oid=%d\ntcn=%s\n", $1, $2}')
eval $(echo $tcn | awk -F. '{printf "schema=%s\ntable=%s\ncolumn=%s\n", $1, $2, $3}')
#
# Look up the oid if the cols array is being used to keep from having to continually update the array if tables are dropped and recreated.
#
if [ $# -eq 0 ]; then
$MYSQLCMD --execute="select concat(objectid, ':', \`schema\`, '.', tablename, '.', columnname) from syscolumn where \`schema\` = '$schema' and tablename='$table' and columnname='$column';" calpontsys --skip-column-names > ${tmpDir}/idb_mm_mon.cols
results=`wc -l ${tmpDir}/idb_mm_mon.cols | awk '{print $1}'`
if [ $results -eq 0 ]; then
oid=0
else
oid=`cat ${tmpDir}/idb_mm_mon.cols`
fi
fi
$PGMPATH/editem -o$oid | awk '{print $1, $6, $8, $12}' >${tmpDir}/idb_mm_mon.$oid.1.$$
$PGMPATH/editem -c$oid
$MYSQLCMD --execute="select count($column) from $table" $schema -vvv
$PGMPATH/editem -o$oid | awk '{print $1, $6, $8, $12}' >${tmpDir}/idb_mm_mon.$oid.2.$$
sdiff ${tmpDir}/idb_mm_mon.$oid.1.$$ ${tmpDir}/idb_mm_mon.$oid.2.$$ --suppress-common-lines | grep -n -v invalid > ${tmpDir}/idb_mm_mon.sdiff.$oid.$$
count=`wc -l ${tmpDir}/idb_mm_mon.sdiff.$oid.$$ | awk '{print $1}'`
if [ $count -ne 0 ]; then
badoidlist="$badoidlist $oid"
((j++))
echo "**** Extent map min/max changed on the scan. See results in ${tmpDir}/idb_mm_mon.sdiff.$oid.$$"
else
rm -f ${tmpDir}/idb_mm_mon.sdiff.$oid.$$
fi
rm -f ${tmpDir}/idb_mm_mon.$oid.*.$$
((i++))
done
echo ""
if [ $j -eq 0 ]; then
$PGMPATH/cplogger -i 0 "min-max-monitor: okay"
echo "min-max-monitor: okay"
echo ""
exit 0
else
$PGMPATH/cplogger -w 0 "min-max-monitor: some values were reset for oids" "$badoidlist"
$PGMPATH/cplogger -w 0 "min-max-monitor: log files are ${tmpDir}/idb_mm_mon.sdiff.*.$$"
echo "min-max-monitor: some values were reset for oids" "$badoidlist"
echo "min-max-monitor: log files are ${tmpDir}/idb_mm_mon.sdiff.*.$$"
echo ""
exit 1
fi

24
oamapps/columnstoreSupport/resourceReport.sh
View File

@ -8,14 +8,8 @@ else
MODULE="pm1"
fi
if [ $2 ] ; then
INSTALLDIR=$2
else
INSTALLDIR="/usr/local/mariadb/columnstore"
fi
#get temp directory
tmpDir=`$INSTALLDIR/bin/getConfig SystemConfig SystemTempFileDir`
tmpDir=`mcsGetConfig SystemConfig SystemTempFileDir`
rm -f ${tmpDir}/${MODULE}_resourceReport.txt
@ -31,9 +25,9 @@ echo "################# ipcs -l #################"
echo " "
ipcs -l
echo "################# $INSTALLDIR/bin/clearShm -n #################"
echo "################# clearShm -n #################"
echo " "
$INSTALLDIR/bin/clearShm -n
clearShm -n
echo " "
echo "-- Disk Usage --"
@ -45,26 +39,26 @@ df -k
echo " "
echo "-- Disk BRM Data files --"
echo " "
ls -l $INSTALLDIR/data1/systemFiles/dbrm 2> /dev/null
ls -l $INSTALLDIR/dbrm 2> /dev/null
ls -l /var/lib/columnstore/data1/systemFiles/dbrm 2> /dev/null
ls -l /var/lib/columnstore/dbrm 2> /dev/null
echo "################# cat $INSTALLDIR/data1/systemFiles/dbrm/BRM_saves_current #################"
echo "################# cat /var/lib/columnstore/data1/systemFiles/dbrm/BRM_saves_current #################"
echo " "
cat $INSTALLDIR/data1/systemFiles/dbrm/BRM_saves_current 2> /dev/null
cat /var/lib/columnstore/data1/systemFiles/dbrm/BRM_saves_current 2> /dev/null
echo " "
echo "-- View Table Locks --"
echo " "
echo "################# cat bin/viewtablelock #################"
echo " "
$INSTALLDIR/bin/viewtablelock 2> /dev/null
viewtablelock 2> /dev/null
echo " "
echo "-- BRM Extent Map --"
echo " "
echo "################# bin/editem -i #################"
echo " "
$INSTALLDIR/bin/editem -i 2>/dev/null
editem -i 2>/dev/null
} > ${tmpDir}/${MODULE}_resourceReport.txt

12
oamapps/columnstoreSupport/softwareReport.sh
View File

@ -8,14 +8,8 @@ else
MODULE="pm1"
fi
if [ $2 ] ; then
INSTALLDIR=$2
else
INSTALLDIR="/usr/local/mariadb/columnstore"
fi
#get temp directory
tmpDir=`$INSTALLDIR/bin/getConfig SystemConfig SystemTempFileDir`
tmpDir=`mcsGetConfig SystemConfig SystemTempFileDir`
rm -f ${tmpDir}/${MODULE}_softwareReport.txt
@ -29,14 +23,14 @@ echo "-- Columnstore Package Details --"
echo " "
echo "################# mcsadmin getcolumnstoresoftwareinfo #################"
echo " "
$INSTALLDIR/bin/mcsadmin getsoftwareinfo
mcsadmin getsoftwareinfo
echo " "
echo "-- Columnstore Storage Configuration --"
echo " "
echo "################# mcsadmin getStorageConfig #################"
echo " "
$INSTALLDIR/bin/mcsadmin getStorageConfig
mcsadmin getStorageConfig
} > ${tmpDir}/${MODULE}_softwareReport.txt

228
oamapps/columnstoreSupport/sqlLogs.sh
View File

@ -1,228 +0,0 @@
#!/bin/bash
#
# Script that does analysis on SQL statements from an Columnstore debug log.
#
DB=idb_idb_sqllogs
TABLE=statements
if [ -z "$COLUMNSTORE_INSTALL_DIR" ]; then
COLUMNSTORE_INSTALL_DIR=/usr/local/mariadb/columnstore
fi
export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR
if [ $COLUMNSTORE_INSTALL_DIR != "/usr/local/mariadb/columnstore" ]; then
export PATH=$COLUMNSTORE_INSTALL_DIR/bin:/bin:/usr/bin
export LD_LIBRARY_PATH=$COLUMNSTORE_INSTALL_DIR/lib
fi
if [ -z "$MYSQLCMD" ]; then
MYSQLCMD="mysql -u root"
export MYSQLCMD
fi
#get temp directory
tmpDir=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig SystemConfig SystemTempFileDir`
main()
{
if [ "$option" == "usage" ]; then
usage
exit
elif [ "$option" == "create" ]; then
if [ -f $logFile ]; then
create
else
echo ""
usage
echo ""
echo "*** $logFile not found. Pleae specify a valid debug log file.***"
echo ""
exit
fi
elif [ "$option" == "list" ]; then
list
elif [ "$option" == "listAll" ]; then
listAll
elif [ "$option" == "active" ]; then
listActive
elif [ "$option" == "activeAll" ]; then
listActiveAll
else
echo ""
usage
echo "*** $option is not a valid option. ***"
echo ""
exit
fi
}
usage ()
{
echo "
This script can be used to analyze select statements from an Columnstore debug log.
Usage:
./sqlLogs.sh create [debug log file name - default:debug.log]
Creates an $DB.$TABLE table with the select statements from the debug log. Must be run before other options can be used.
./sqlLogs.sh list
Lists the SQL statements and run times showing the following:
id - statement id in sequential order based on the start time
starttime - start time of the statement
endtime - end time of the statement
runtime - total run time of the statement
sessionid - the MySQL session id for the statement
sessionstatementid - the sequence of the statement within the session
./sqlLogs.sh listAll
Same as list with the addition of the SQL statement.
./sqlLogs.sh active timestamp
Example: ./sqlLogs.sh active 00:35:50.291408
Lists the id, starttime, endtime, runtime, timeActive, sessionid, and sessionstatementid for the sql statements
that were active at the given timestamp.
./sqlLogs.sh activeAll timestamp
Same as listActive with the addition of the SQL statement.
"
}
create ()
{
echo ""
echo "Step 1 of 4. Building import file with Start SQL log entries."
grep "Start SQL" $logFile | grep -v syscolumn | grep -v systable | awk -F '|' '{print NR "|" substr($1,8,6) substr($1,length($1)-9,9) "|" $2 "|" $5}' |
sort -t '|' -n -k 3 -k 1 |
awk -F '|' '
{
if(NR == 1)
{
prevSession=$3;
val=1;
}
else if(prevSession == $3)
val++;
else
{
val=1;
prevSession=$3;
}
print $0 "|" val "|"
}' | sort -t '|' -n -k 1 > ${tmpDir}/idbtmp.tbl
echo "Step 2 of 4. Populating $DB.start table with Start SQL log entries."
sql="
create database if not exists $DB;
use $DB;
drop table if exists start;
CREATE TABLE start (
id int,
time char(20),
sessionid int,
statement varchar(8000),
sessionStatementId int
) ENGINE=MyISAM ;
create index start_idx on start (sessionid, sessionStatementId);
load data infile '${tmpDir}/idbtmp.tbl' into table start fields terminated by '|';
"
$MYSQLCMD -e "$sql"
echo "Step 3 of 4. Building import file with End SQL log entries."
grep "End SQL" $logFile | grep -v "2147483" | awk -F '|' '{print NR "|" substr($1,8,6) substr($1,length($1)-9,9) "|" $2}' |
sort -t '|' -n -k 3 -k 1 |
awk -F '|' '
{
if(NR == 1)
{
prevSession=$3;
val=1;
}
else if(prevSession == $3)
val++;
else
{
val=1;
prevSession=$3;
}
print $0 "|" val "|"
}' | sort -t '|' -n -k 1 > ${tmpDir}/idbtmp.tbl
echo "Step 4 of 4. Populating $DB.stop table with End SQL log entries."
sql="
drop table if exists stop;
CREATE TABLE stop (
id int,
time char(20),
sessionid int,
sessionStatementId int
) ENGINE=MyISAM ;
create index stop_idx on stop (sessionid, sessionStatementId);
load data infile '${tmpDir}/idbtmp.tbl' into table stop fields terminated by '|';
"
$MYSQLCMD $DB -e "$sql;"
echo "Step 5 of 5. Populating $DB.$TABLE table."
sql="
drop table if exists $TABLE;
create table $TABLE as
(select
a.id id, a.time starttime, b.time endtime, substr(timediff(b.time, a.time), 1, 30) runTime, a.sessionid sessionId, a.sessionstatementid sessionStatementId, a.statement statement
from start a left join stop b
on a.sessionid = b.sessionid and a.sessionstatementid = b.sessionstatementid);
"
$MYSQLCMD $DB -e "$sql"
echo "All done."
echo ""
}
list() {
sql="select id, starttime, endtime, runtime, sessionid, sessionstatementid from statements;"
$MYSQLCMD $DB -vvv -e "$sql"
}
listAll()
{
sql="select id, starttime, endtime, runtime, sessionid, sessionstatementid, trim(statement) statement from statements;"
$MYSQLCMD $DB -vvv -e "$sql"
}
listActive()
{
dtm=$parm2
sql="
select id, starttime, endtime, runtime, timediff('$dtm', starttime) timeActive, sessionid, sessionstatementid from statements
where starttime <= '$dtm' and (endtime is null or endtime > '$dtm');
"
$MYSQLCMD $DB -vvv -e "$sql"
}
listActiveAll()
{
dtm=$parm2
sql="
select id, starttime, endtime, runtime, timediff('$dtm', starttime) timeActive, sessionid, sessionstatementid, trim(statement) statement from statements
where starttime <= '$dtm' and (endtime is null or endtime > '$dtm');
"
$MYSQLCMD $DB -vvv -e "$sql"
}
if [ $# -lt 1 ]; then
usage
exit
else
option=$1
parm2=$2
logFile=debug.log
if [ $# -ge 2 ]; then
logFile=$2
fi
fi
main

45
oamapps/mcsadmin/mcsadmin.cpp
View File

@ -70,7 +70,6 @@ bool rootUser = true;
string HOME = "/root";
string SingleServerInstall;
string tmpDir;
string installDir;
bool repeatStop;
@ -200,7 +199,6 @@ int main(int argc, char* argv[])
string ccHistoryFile = HOME + "/.cc_history";
tmpDir = startup::StartUp::tmpDir();
installDir = startup::StartUp::installDir();
string cf = std::string(MCSSYSCONFDIR) + "/columnstore/" + ConsoleCmdsFile;
fConfig = Config::makeConfig(cf);
@ -2626,7 +2624,7 @@ int processCommand(string* arguments)
{
string logFile = tmpDir + "/cc-stop.pdsh";
cmd = "pdsh -a '/" + installDir + "/bin/columnstore stop' > " + logFile + " 2>&1";
cmd = "pdsh -a 'columnstore stop' > " + logFile + " 2>&1";
system(cmd.c_str());
if (oam.checkLogStatus(logFile, "exit") )
@ -2636,7 +2634,7 @@ int processCommand(string* arguments)
}
else
{
cmd = installDir + "/bin/columnstore stop > " + tmpDir + "/status.log";
cmd = "columnstore stop > " + tmpDir + "/status.log";
system(cmd.c_str());
}
}
@ -2646,7 +2644,7 @@ int processCommand(string* arguments)
if ( gracefulTemp == FORCEFUL )
{
cmd = installDir + "/bin/columnstore stop > " + tmpDir + "/status.log";
cmd = "columnstore stop > " + tmpDir + "/status.log";
system(cmd.c_str());
cout << endl << " Successful shutdown of System (stopped local columnstore service) " << endl << endl;
}
@ -2654,7 +2652,7 @@ int processCommand(string* arguments)
if (Failed.find("Connection refused") != string::npos)
{
cout << endl << "**** shutdownSystem Error : ProcessManager not Active, stopping columnstore service" << endl;
cmd = installDir + "/bin/columnstore stop > " + tmpDir + "/status.log";
cmd = "columnstore stop > " + tmpDir + "/status.log";
system(cmd.c_str());
cout << endl << " Successful stop of local columnstore service " << endl << endl;
}
@ -2676,7 +2674,7 @@ int processCommand(string* arguments)
if ( DBRootStorageType == "hdfs")
{
string logFile = tmpDir + "cc-stop.pdsh";
cmd = "pdsh -a '" + installDir + "/bin/columnstore stop' > " + logFile + " 2>&1";
cmd = "pdsh -a 'columnstore stop' > " + logFile + " 2>&1";
system(cmd.c_str());
if (oam.checkLogStatus(logFile, "exit") )
@ -2783,7 +2781,7 @@ int processCommand(string* arguments)
if ( DBRootStorageType == "hdfs")
{
string logFile = tmpDir + "/cc-restart.pdsh";
cmd = "pdsh -a '" + installDir + "/bin/columnstore restart' > " + logFile + " 2>&1";
cmd = "pdsh -a 'columnstore restart' > " + logFile + " 2>&1";
system(cmd.c_str());
if (oam.checkLogStatus(logFile, "exit") )
@ -2820,7 +2818,7 @@ int processCommand(string* arguments)
if ( modulename == localModule )
{
cmd = installDir + "/bin/columnstore restart > " + tmpDir + "/start.log 2>&1";
cmd = "columnstore restart > " + tmpDir + "/start.log 2>&1";
int rtnCode = system(cmd.c_str());
if (geteuid() == 0 && WEXITSTATUS(rtnCode) != 0)
@ -2838,7 +2836,7 @@ int processCommand(string* arguments)
for ( ; pt1 != (*pt).hostConfigList.end() ; pt1++)
{
//run remote command script
cmd = installDir + "/bin/remote_command.sh " + (*pt1).IPAddr + " " + password + " '" + installDir + "/bin/columnstore restart' 0";
cmd = "remote_command.sh " + (*pt1).IPAddr + " " + password + " 'columnstore restart' 0";
int rtnCode = system(cmd.c_str());
if (WEXITSTATUS(rtnCode) < 0)
@ -2847,7 +2845,7 @@ int processCommand(string* arguments)
cout << endl << "**** startSystem Failed" << endl;
// stop local columnstore service
cmd = installDir + "/bin/columnstore stop > " + tmpDir + "/stop.log 2>&1";
cmd = "columnstore stop > " + tmpDir + "/stop.log 2>&1";
system(cmd.c_str());
FAILED = true;
@ -2861,7 +2859,7 @@ int processCommand(string* arguments)
cout << endl << "**** startSystem Failed" << endl;
// stop local columnstore service
cmd = installDir + "/bin/columnstore stop > " + tmpDir + "/stop.log 2>&1";
cmd = "columnstore stop > " + tmpDir + "/stop.log 2>&1";
system(cmd.c_str());
FAILED = true;
@ -2887,7 +2885,7 @@ int processCommand(string* arguments)
//just kick off local server
cout << endl << " System being started, please wait...";
cout.flush();
cmd = installDir + "/bin/columnstore restart > " + tmpDir + "/start.log 2>&1";
cmd = "columnstore restart > " + tmpDir + "/start.log 2>&1";
int rtnCode = system(cmd.c_str());
if (geteuid() == 0 && WEXITSTATUS(rtnCode) != 0)
@ -3025,7 +3023,7 @@ int processCommand(string* arguments)
if ( DBRootStorageType == "hdfs")
{
string logFile = tmpDir + "/cc-restart.pdsh";
cmd = "pdsh -a '" + installDir + "/bin/columnstore restart' > " + logFile + " 2>&1";
cmd = "pdsh -a 'columnstore restart' > " + logFile + " 2>&1";
system(cmd.c_str());
if (oam.checkLogStatus(logFile, "exit") )
@ -3068,7 +3066,7 @@ int processCommand(string* arguments)
for ( ; pt1 != (*pt).hostConfigList.end() ; pt1++)
{
//run remote command script
cmd = installDir + "/bin/remote_command.sh " + (*pt1).IPAddr + " " + password + " '" + installDir + "/bin/columnstore restart' 0";
cmd = "remote_command.sh " + (*pt1).IPAddr + " " + password + " 'columnstore restart' 0";
int rtnCode = system(cmd.c_str());
@ -3078,7 +3076,7 @@ int processCommand(string* arguments)
cout << endl << "**** restartSystem Failed" << endl;
// stop local columnstore service
cmd = installDir + "/bin/columnstore stop > " + tmpDir + "/stop.log 2>&1";
cmd = "columnstore stop > " + tmpDir + "/stop.log 2>&1";
system(cmd.c_str());
FAILED = true;
@ -3093,7 +3091,7 @@ int processCommand(string* arguments)
FAILED = true;
// stop local columnstore service
cmd = installDir + "/bin/columnstore stop > " + tmpDir + "/stop.log 2>&1";
cmd = "columnstore stop > " + tmpDir + "/stop.log 2>&1";
system(cmd.c_str());
break;
@ -3109,7 +3107,7 @@ int processCommand(string* arguments)
break;
//RESTART LOCAL HOST
cmd = installDir + "/bin/columnstore restart > " + tmpDir + "/start.log 2>&1";
cmd = "columnstore restart > " + tmpDir + "/start.log 2>&1";
int rtnCode = system(cmd.c_str());
if (geteuid() == 0 && WEXITSTATUS(rtnCode) != 0)
@ -3129,7 +3127,7 @@ int processCommand(string* arguments)
//just kick off local server
cout << " System being restarted, please wait...";
cout.flush();
cmd = installDir + "/bin/columnstore restart > " + tmpDir + "/start.log 2>&1";
cmd = "columnstore restart > " + tmpDir + "/start.log 2>&1";
int rtnCode = system(cmd.c_str());
if (WEXITSTATUS(rtnCode) != 0)
@ -4936,7 +4934,6 @@ int processCommand(string* arguments)
{
cout << endl << "System Installation and Temporary File Directories" << endl << endl;
cout << "System Installation Directory = " << installDir << endl;
cout << "System Temporary File Directory = " << tmpDir << endl << endl;
}
break;
@ -6182,13 +6179,13 @@ int processCommand(string* arguments)
++it)
{
string deviceName = *it;
string entry = deviceName + " " + installDir + "/gluster/brick" + oam.itoa(brickID) + " " + deviceType + " defaults 1 2";
string entry = deviceName + " /var/lib/columnstore/gluster/brick" + oam.itoa(brickID) + " " + deviceType + " defaults 1 2";
//send update pm
oam.distributeFstabUpdates(entry, moduleName);
}
}
string command = installDir + "/bin/remote_command.sh " + (*hostConfigIter).IPAddr + " " + password + " 'mkdir -p " + installDir + "/gluster/brick" + oam.itoa(brickID) + "'";
string command = "remote_command.sh " + (*hostConfigIter).IPAddr + " " + password + " 'mkdir -p /var/lib/columnstore/gluster/brick" + oam.itoa(brickID) + "'";
system(command.c_str());
brickID++;
}
@ -8225,7 +8222,7 @@ int ProcessSupportCommand(int CommandID, std::string arguments[])
//run remote command script
HostConfigList::iterator pt1 = (*pt).hostConfigList.begin();
cmd = installDir + "/bin/remote_command.sh " + (*pt1).IPAddr + " " + password + " reboot " ;
cmd = "remote_command.sh " + (*pt1).IPAddr + " " + password + " reboot " ;
int rtnCode = system(cmd.c_str());
if (WEXITSTATUS(rtnCode) != 0)
@ -8344,7 +8341,7 @@ int ProcessSupportCommand(int CommandID, std::string arguments[])
HostConfigList::iterator pt1 = (*pt).hostConfigList.begin();
string ipAddr = (*pt1).IPAddr;
//run remote command script
cmd = installDir + "/bin/remote_command.sh " + ipAddr + " " + password + " reboot " ;
cmd = "remote_command.sh " + ipAddr + " " + password + " reboot " ;
int rtnCode = system(cmd.c_str());
if (WEXITSTATUS(rtnCode) != 0)

8
oamapps/postConfigure/CMakeLists.txt
View File

@ -19,13 +19,13 @@ install(TARGETS postConfigure DESTINATION ${ENGINE_BINDIR} COMPONENT platform)
set(installer_SRCS installer.cpp helpers.cpp)
add_executable(installer ${installer_SRCS})
add_executable(columnstore_installer ${installer_SRCS})
target_compile_options(installer PRIVATE -Wno-unused-result)
target_compile_options(columnstore_installer PRIVATE -Wno-unused-result)
target_link_libraries(installer ${ENGINE_LDFLAGS} readline ncurses ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS})
target_link_libraries(columnstore_installer ${ENGINE_LDFLAGS} readline ncurses ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS})
install(TARGETS installer DESTINATION ${ENGINE_BINDIR} COMPONENT platform)
install(TARGETS columnstore_installer DESTINATION ${ENGINE_BINDIR} COMPONENT platform)
########### next target ###############

111
oamapps/postConfigure/amazonConfig-1pm.xml
View File

@ -1,111 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- $Id: amazonConfig.xml 2830 2012-04-08 17:38:58Z dhill $ -->
<!-- Used with the amazonInstaller script-->
<Calpont Version="V1.0.0">
<SystemConfig>
<!-- System Name-->
<SystemName>calpont-1</SystemName>
<!-- System Type: combined (um and pm functionality on same instance)
separate (um and pm functionality on separate instances) -->
<SystemType>combined</SystemType>
<!--- VPC Configuration Setup -->
<!--- -->
<!--- subnet ID -->
<!--- The ID of the Amazon VPC subnet in which to launch the instance(s) -->
<SubNetID>unassigned</SubNetID>
<!-- VPC Starting Private IP Address -->
<!-- Requires the SubNetID to be setup -->
<!-- Use this if you want InfiniDB to assigned IP Addresses starting with this one -->
<!-- Set value to 'autoassign' if you want Amazon to auto-assign IP Addresses within the subnet range -->
<VPCStartPrivateIP>unassigned</VPCStartPrivateIP>
<!-- User Modules Private IP Address list, use when running on a Amazon VPC -->
<!-- Requires the SubNetID to be setup -->
<!-- This list would start with um1, list is seperated by commands. i.e 'ip1,ip2,ip3' -->
<!-- IGNORED when VPCStartPrivateIP is set -->
<UserModulePrivateIP>unassigned</UserModulePrivateIP>
<!-- Performance Modules Private IP Address list, use when running on a Amazon VPC -->
<!-- Requires the SubNetID to be setup -->
<!-- This list would start with pm2, list is seperated by commands. i.e 'ip1,ip2,ip3' -->
<!-- IGNORED when VPCStartPrivateIP is set -->
<PerformanceModulePrivateIP>unassigned</PerformanceModulePrivateIP>
<!-- Elastic IP to Module Assignment -->
<!--- Example usages: x.x.x.x:um1 -->
<!--- Example usages: x.x.x.x:um1,y.y.y.y:pm1 -->
<ElasticIPs>unassigned</ElasticIPs>
<!-- Number of User Modules: n/a for combination system type -->
<UserModuleCount>0</UserModuleCount>
<!-- User Modules Instances list, use existing instances instead of Launching new ones -->
<!-- This list would start with um1, list is seperated by commands. i.e 'id1,id2,id3' -->
<UserModuleInstances>unassigned</UserModuleInstances>
<!-- User Modules Instance Type, default to local Instance Type -->
<UserModuleInstanceType>unassigned</UserModuleInstanceType>
<!-- User Modules Security Group, default to local Security Group -->
<UserModuleSecurityGroup>unassigned</UserModuleSecurityGroup>
<!-- Use EBS Volumes for User Module data storage? -->
<!-- Applicable for system type = 'separate' only -->
<UseUMEBSVolumes>y</UseUMEBSVolumes>
<!-- Size of User Module EBS Volumes in gigibytes -->
<!-- Applicable for system type = 'separate' only -->
<UMEBSVolumeSize>10</UMEBSVolumeSize>
<!-- Number of Performance Modules -->
<PerformanceModuleCount>1</PerformanceModuleCount>
<!-- Performance Modules Instances list, use existing instances instead of Launching new ones -->
<!-- This list would start with pm2, list is seperated by commands. i.e 'id1,id2,id3' -->
<PerformanceModuleInstances>unassigned</PerformanceModuleInstances>
<!-- Number of DBRoots assigned to each Performance Module -->
<DBRootsPerPM>1</DBRootsPerPM>
<!-- Use EBS Volumes for (um/pm) data storage (system type = 'combined')? -->
<!-- Use EBS Volumes for pm data storage (system type = 'separate')? -->
<UsePMEBSVolumes>y</UsePMEBSVolumes>
<!-- Size of Back-End Data EBS Volumes in gigibytes -->
<PMEBSVolumeSize>100</PMEBSVolumeSize>
<!-- Enable Pm EBS Failvoer Support, meaning the EBS storage will be remounted -->
<!-- when a Non-OAM-Parent-PM fails -->
<PMEBSFailoverSupport>y</PMEBSFailoverSupport>
<!-- SNMP Trap Receiver IP Address: 0.0.0.0 means no snmptraps will be forwarded -->
<SNMPTrapIPAddr>0.0.0.0</SNMPTrapIPAddr>
<!-- Total UM Memory Size Percentage, default to postConfigure setting -->
<TotalUmMemory>unassigned</TotalUmMemory>
<!-- Blocks Cache Size Percentage, default to postConfigure setting -->
<NumBlocksPct>unassigned</NumBlocksPct>
<!-- Root Password or 'ssh' for ssh-key setup -->
<RootPassword>Calpont1</RootPassword>
<!-- Enables Automatic Amazon Instance/Volume tagging -->
<!-- When set 'y', Instance Name Tags will be set to 'system-name'-'module-name' -->
<!-- DBRoot Volume Name Tags will be set to 'system-name'-'dbrootID' -->
<!-- User Module Volume Name Tags will be set to 'system-name'-'module-name' -->
<AutoTagging>y</AutoTagging>
<!-- Amazon EC2 region your running in, run ec2-describe-regions to get valid region names -->
<Region>us-east-1</Region>
<!-- x.509 Certification File with directory location. i.e. /root/cert-xxxx -->
<x509CertificationFile>unassigned</x509CertificationFile>
<!-- x.509 Private Key File with directory location. i.e. /root/pk-xxxx -->
<x509PrivateFile>unassigned</x509PrivateFile>
</SystemConfig>
</Calpont>

111
oamapps/postConfigure/amazonConfig.xml
View File

@ -1,111 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- $Id: amazonConfig.xml 2830 2012-04-08 17:38:58Z dhill $ -->
<!-- Used with the amazonInstaller script-->
<Calpont Version="V1.0.0">
<SystemConfig>
<!-- System Name-->
<SystemName>calpont-1</SystemName>
<!-- System Type: combined (um and pm functionality on same instance)
separate (um and pm functionality on separate instances) -->
<SystemType>separate</SystemType>
<!--- VPC Configuration Setup -->
<!--- -->
<!--- subnet ID -->
<!--- The ID of the Amazon VPC subnet in which to launch the instance(s) -->
<SubNetID>unassigned</SubNetID>
<!-- VPC Starting Private IP Address -->
<!-- Requires the SubNetID to be setup -->
<!-- Use this if you want InfiniDB to assigned IP Addresses starting with this one -->
<!-- Set value to 'autoassign' if you want Amazon to auto-assign IP Addresses within the subnet range -->
<VPCStartPrivateIP>unassigned</VPCStartPrivateIP>
<!-- User Modules Private IP Address list, use when running on a Amazon VPC -->
<!-- Requires the SubNetID to be setup -->
<!-- This list would start with um1, list is seperated by commands. i.e 'ip1,ip2,ip3' -->
<!-- IGNORED when VPCStartPrivateIP is set -->
<UserModulePrivateIP>unassigned</UserModulePrivateIP>
<!-- Performance Modules Private IP Address list, use when running on a Amazon VPC -->
<!-- Requires the SubNetID to be setup -->
<!-- This list would start with pm2, list is seperated by commands. i.e 'ip1,ip2,ip3' -->
<!-- IGNORED when VPCStartPrivateIP is set -->
<PerformanceModulePrivateIP>unassigned</PerformanceModulePrivateIP>
<!-- Elastic IP to Module Assignment -->
<!--- Example usages: x.x.x.x:um1 -->
<!--- Example usages: x.x.x.x:um1,y.y.y.y:pm1 -->
<ElasticIPs>unassigned</ElasticIPs>
<!-- Number of User Modules: n/a for combination system type -->
<UserModuleCount>1</UserModuleCount>
<!-- User Modules Instances list, use existing instances instead of Launching new ones -->
<!-- This list would start with um1, list is seperated by commands. i.e 'id1,id2,id3' -->
<UserModuleInstances>unassigned</UserModuleInstances>
<!-- User Modules Instance Type, default to local Instance Type -->
<UserModuleInstanceType>unassigned</UserModuleInstanceType>
<!-- User Modules Security Group, default to local Security Group -->
<UserModuleSecurityGroup>unassigned</UserModuleSecurityGroup>
<!-- Use EBS Volumes for User Module data storage? -->
<!-- Applicable for system type = 'separate' only -->
<UseUMEBSVolumes>y</UseUMEBSVolumes>
<!-- Size of User Module EBS Volumes in gigibytes -->
<!-- Applicable for system type = 'separate' only -->
<UMEBSVolumeSize>10</UMEBSVolumeSize>
<!-- Number of Performance Modules -->
<PerformanceModuleCount>2</PerformanceModuleCount>
<!-- Performance Modules Instances list, use existing instances instead of Launching new ones -->
<!-- This list would start with pm2, list is seperated by commands. i.e 'id1,id2,id3' -->
<PerformanceModuleInstances>unassigned</PerformanceModuleInstances>
<!-- Number of DBRoots assigned to each Performance Module -->
<DBRootsPerPM>1</DBRootsPerPM>
<!-- Use EBS Volumes for (um/pm) data storage (system type = 'combined')? -->
<!-- Use EBS Volumes for pm data storage (system type = 'separate')? -->
<UsePMEBSVolumes>y</UsePMEBSVolumes>
<!-- Size of Back-End Data EBS Volumes in gigibytes -->
<PMEBSVolumeSize>100</PMEBSVolumeSize>
<!-- Enable Pm EBS Failvoer Support, meaning the EBS storage will be remounted -->
<!-- when a Non-OAM-Parent-PM fails -->
<PMEBSFailoverSupport>n</PMEBSFailoverSupport>
<!-- SNMP Trap Receiver IP Address: 0.0.0.0 means no snmptraps will be forwarded -->
<SNMPTrapIPAddr>0.0.0.0</SNMPTrapIPAddr>
<!-- Total UM Memory Size Percentage, default to postConfigure setting -->
<TotalUmMemory>unassigned</TotalUmMemory>
<!-- Blocks Cache Size Percentage, default to postConfigure setting -->
<NumBlocksPct>unassigned</NumBlocksPct>
<!-- Root Password or 'ssh' for ssh-key setup -->
<RootPassword>Calpont1</RootPassword>
<!-- Enables Automatic Amazon Instance/Volume tagging -->
<!-- When set 'y', Instance Name Tags will be set to 'system-name'-'module-name' -->
<!-- DBRoot Volume Name Tags will be set to 'system-name'-'dbrootID' -->
<!-- User Module Volume Name Tags will be set to 'system-name'-'module-name' -->
<AutoTagging>y</AutoTagging>
<!-- Amazon EC2 region your running in, run ec2-describe-regions to get valid region names -->
<Region>us-east-1</Region>
<!-- x.509 Certification File with directory location. i.e. /root/cert-xxxx -->
<x509CertificationFile>unassigned</x509CertificationFile>
<!-- x.509 Private Key File with directory location. i.e. /root/pk-xxxx -->
<x509PrivateFile>unassigned</x509PrivateFile>
</SystemConfig>
</Calpont>

3200
oamapps/postConfigure/amazonInstaller.cpp
View File

File diff suppressed because it is too large Load Diff

12
oamapps/postConfigure/helpers.cpp
View File

@ -48,8 +48,6 @@ string prompt;
const char* pcommand = 0;
extern string installDir;
extern bool noPrompting;
namespace installer
@ -242,7 +240,7 @@ void dbrmDirCheck()
}
}
cmd = "chmod 755 -R " + installDir + "/data1/systemFiles/dbrm > /dev/null 2>&1";
cmd = "chmod 755 -R /var/lib/columnstore/data1/systemFiles/dbrm > /dev/null 2>&1";
system(cmd.c_str());
return;
@ -261,7 +259,7 @@ void mysqlSetup()
if ( mysqlpw != oam::UnassignedName )
passwordOption = " --password=" + mysqlpw;
cmd = installDir + "/bin/post-mysqld-install --installdir=" + installDir + " " + passwordOption + " --tmpdir=" + tmpDir + " > " + tmpDir + "/post-mysqld-install.log 2>&1";
cmd = "post-mysqld-install " + passwordOption + " --tmpdir=" + tmpDir + " > " + tmpDir + "/post-mysqld-install.log 2>&1";
int rtnCode = system(cmd.c_str());
if (WEXITSTATUS(rtnCode) != 0)
@ -290,7 +288,7 @@ void mysqlSetup()
HOME = p;
}
cmd = installDir + "/bin/post-mysql-install --installdir=" + installDir + " --tmpdir=" + tmpDir + " > " + tmpDir + "/post-mysql-install.log";
cmd = "post-mysql-install --tmpdir=" + tmpDir + " > " + tmpDir + "/post-mysql-install.log";
rtnCode = system(cmd.c_str());
if (WEXITSTATUS(rtnCode) == 2)
@ -593,7 +591,7 @@ void checkFilesPerPartion(int DBRootCount, Config* sysConfig)
Oam oam;
string SystemSection = "SystemConfig";
string dbRoot = installDir + "/data1";
string dbRoot = "/var/lib/columnstore/data1";
try
{
@ -792,7 +790,7 @@ void checkSystemMySQLPort(std::string& mysqlPort, Config* sysConfig, std::string
(remoteModuleType == "pm" && pmwithum) )
{
string cmd = installDir + "/bin/remote_command_verify.sh " + remoteModuleIP + " " + " " + USER + " " + password + " '" + remotenetstat + "' tcp error 5 0 > /dev/null 2>&1";
string cmd = "remote_command_verify.sh " + remoteModuleIP + " " + " " + USER + " " + password + " '" + remotenetstat + "' tcp error 5 0 > /dev/null 2>&1";
int rtnCode = system(cmd.c_str());
if (WEXITSTATUS(rtnCode) == 0)

266
oamapps/postConfigure/installer.cpp
View File

@ -78,7 +78,6 @@ bool uncommentCalpontXml( string entry);
extern string pwprompt;
string installDir;
bool noPrompting;
string mysqlpw = " ";
@ -130,20 +129,16 @@ int main(int argc, char* argv[])
string numBlocksPctParam = "";
string totalUmMemoryParam = "";
if (argc >= 13) {
installDir = argv[12];
if (argc >= 15) {
if (string(argv[13]) != "-") {
if (argc >= 12) {
if (argc >= 14) {
if (string(argv[12]) != "-") {
numBlocksPctParam = argv[13];
}
if (string(argv[14]) != "-") {
if (string(argv[13]) != "-") {
totalUmMemoryParam = argv[14];
}
}
}
else {
installDir = "/usr/local/mariadb/columnstore";
}
ofstream file("/dev/null");
@ -164,7 +159,6 @@ int main(int argc, char* argv[])
cout << nodeps << endl;
cout << mysqlpw << endl;
cout << installer_debug << endl;
cout << installDir << endl;
if (!numBlocksPctParam.empty()) {
cout << numBlocksPctParam << endl;
}
@ -542,13 +536,13 @@ int main(int argc, char* argv[])
if ( installType == "initial" )
{
//cleanup/create /local/etc directories
cmd = "rm -rf " + installDir + "/local/etc > /dev/null 2>&1";
cmd = "rm -rf /var/lib/columnstore/local/etc > /dev/null 2>&1";
system(cmd.c_str());
cmd = "mkdir " + installDir + "/local/etc > /dev/null 2>&1";
cmd = "mkdir /var/lib/columnstore/local/etc > /dev/null 2>&1";
system(cmd.c_str());
//create associated /local/etc directory for parentOAMModuleName
cmd = "mkdir " + installDir + "/local/etc/" + parentOAMModuleName + " > /dev/null 2>&1";
cmd = "mkdir /var/lib/columnstore/local/etc/" + parentOAMModuleName + " > /dev/null 2>&1";
system(cmd.c_str());
}
@ -626,7 +620,7 @@ int main(int argc, char* argv[])
if ( installType == "initial" )
{
//create associated /local/etc directory for module
cmd = "mkdir " + installDir + "/local/etc/" + moduleName + " > /dev/null 2>&1";
cmd = "mkdir /var/lib/columnstore/local/etc/" + moduleName + " > /dev/null 2>&1";
system(cmd.c_str());
}
}
@ -644,7 +638,7 @@ int main(int argc, char* argv[])
for (; list1 != directormodulelist.end() ; list1++)
{
cmd = "mkdir " + installDir + "/local/etc/" + (*list1).moduleName + " > /dev/null 2>&1";
cmd = "mkdir /var/lib/columnstore/local/etc/" + (*list1).moduleName + " > /dev/null 2>&1";
system(cmd.c_str());
//make module file in /local/etc/"modulename"
@ -660,7 +654,7 @@ int main(int argc, char* argv[])
for (; list1 != usermodulelist.end() ; list1++)
{
cmd = "mkdir " + installDir + "/local/etc/" + (*list1).moduleName + " > /dev/null 2>&1";
cmd = "mkdir /var/lib/columnstore/local/etc/" + (*list1).moduleName + " > /dev/null 2>&1";
system(cmd.c_str());
//make module file in /local/etc/"modulename"
@ -676,7 +670,7 @@ int main(int argc, char* argv[])
for (; list1 != performancemodulelist.end() ; list1++)
{
cmd = "mkdir " + installDir + "/local/etc/" + (*list1).moduleName + " > /dev/null 2>&1";
cmd = "mkdir /var/lib/columnstore/local/etc/" + (*list1).moduleName + " > /dev/null 2>&1";
system(cmd.c_str());
//make module file in /local/etc/"modulename"
@ -689,99 +683,6 @@ int main(int argc, char* argv[])
}
}
//perform uninstall option
if ( installType == "uninstall" )
{
ModuleList::iterator list1 = childmodulelist.begin();
for (; list1 != childmodulelist.end() ; list1++)
{
string remoteModuleName = (*list1).moduleName;
string remoteModuleIP = (*list1).moduleIP;
string remoteHostName = (*list1).hostName;
string remoteModuleType = remoteModuleName.substr(0, MAX_MODULE_TYPE_SIZE);
if ( remoteModuleType == "um" ||
(remoteModuleType == "pm" && IserverTypeInstall == oam::INSTALL_COMBINE_DM_UM_PM) )
{
//run remote installer script
if ( packageType != "binary" )
{
string temppwprompt = pwprompt;
if ( pwprompt == " " )
temppwprompt = "none";
cout << endl << "----- Performing Uninstall on Module '" + remoteModuleName + "' -----" << endl << endl;
cmd = installDir + "/bin/package_installer.sh " + remoteModuleName + " " + remoteModuleIP + " " + password + " " + calpont_rpm1 + " " + calpont_rpm2 + " " + calpont_rpm3 + " " + mysql_rpm + " " + mysqld_rpm + " " + installType + " " + packageType + " " + nodeps + " " + temppwprompt + " " + installer_debug;
int rtnCode = system(cmd.c_str());
if (WEXITSTATUS(rtnCode) != 0)
{
cout << endl << "ERROR: returned from package_installer.sh" << endl;
exit(1);
}
}
else
{
// do a binary package install
cmd = installDir + "/bin/binary_installer.sh " + remoteModuleName + " " + remoteModuleIP + " " +
password + " " + calpont_rpm1 + " " + remoteModuleType + " " + installType + " " +
serverTypeInstall + " " + installer_debug + " " + installDir;
int rtnCode = system(cmd.c_str());
if (WEXITSTATUS(rtnCode) != 0)
{
cout << endl << "ERROR: returned from binary_installer.sh" << endl;
exit(1);
}
}
}
else
{
if (remoteModuleType == "pm")
{
if ( packageType != "binary" )
{
//run remote installer script
cout << endl << "----- Performing Uninstall on Module '" + remoteModuleName + "' -----" << endl << endl;
cmd = installDir + "/bin/package_installer.sh " + remoteModuleName + " " + remoteModuleIP + " " + password + " " + calpont_rpm1 + " " + calpont_rpm2 + " " + calpont_rpm3 + " " + mysql_rpm + " " + mysqld_rpm + " " + installType + " " + packageType + " " + nodeps + " " + installer_debug;
int rtnCode = system(cmd.c_str());
if (WEXITSTATUS(rtnCode) != 0)
{
cout << endl << "ERROR returned from package_installer.sh" << endl;
exit(1);
}
}
else
{
// do a binary package install
cmd = installDir + "/bin/binary_installer.sh " + remoteModuleName + " " + remoteModuleIP +
" " + password + " " + calpont_rpm1 + " " + remoteModuleType + " " + installType + " " +
serverTypeInstall + " " + installer_debug + " " + installDir;
int rtnCode = system(cmd.c_str());
if (WEXITSTATUS(rtnCode) != 0)
{
cout << endl << "ERROR returned from binary_installer.sh" << endl;
exit(1);
}
}
}
}
//unmount child module
cmd = "umount -fl /mnt/" + remoteModuleName + " > /dev/null 2>&1";
system(cmd.c_str());
cmd = "rm -fr /mnt/" + remoteModuleName;
system(cmd.c_str());
}
cout << "uninstall request successful" << endl;
return 0;
}
if ( installType == "initial" )
{
//setup local OS Files
@ -792,139 +693,12 @@ int main(int argc, char* argv[])
exit(1);
}
cmd = "chmod 755 -R " + installDir + "/data1/systemFiles/dbrm > /dev/null 2>&1";
cmd = "chmod 755 -R /var/lib/columnstore/data1/systemFiles/dbrm > /dev/null 2>&1";
system(cmd.c_str());
}
string idbstartcmd = installDir + "/bin/columnstore start";
string idbstartcmd = "columnstore start";
if ( IserverTypeInstall != oam::INSTALL_COMBINE_DM_UM_PM ||
performancemodulelist.size() > 1 )
{
//
// perform multi-server install
//
// perform remote install of servers in the system
ModuleList::iterator list1 = childmodulelist.begin();
for (; list1 != childmodulelist.end() ; list1++)
{
string remoteModuleName = (*list1).moduleName;
string remoteModuleIP = (*list1).moduleIP;
string remoteHostName = (*list1).hostName;
string remoteModuleType = remoteModuleName.substr(0, MAX_MODULE_TYPE_SIZE);
if ( remoteModuleType == "um" ||
(remoteModuleType == "pm" && IserverTypeInstall == oam::INSTALL_COMBINE_DM_UM_PM) )
{
//run remote installer script
if ( packageType != "binary" )
{
string temppwprompt = pwprompt;
if ( pwprompt == " " )
temppwprompt = "none";
cout << endl << "----- Performing Install on Module '" + remoteModuleName + "' -----" << endl << endl;
cmd = installDir + "/bin/package_installer.sh " + remoteModuleName + " " + remoteModuleIP + " " + password + " " + calpont_rpm1 + " " + calpont_rpm2 + " " + calpont_rpm3 + " " + mysql_rpm + " " + mysqld_rpm + " " + installType + " " + packageType + " " + nodeps + " " + temppwprompt + " " + installer_debug;
int rtnCode = system(cmd.c_str());
if (WEXITSTATUS(rtnCode) != 0)
{
cout << endl << "ERROR returned from package_installer.sh" << endl;
exit(1);
}
}
else
{
// do a binary package install
cmd = installDir + "/bin/binary_installer.sh " + remoteModuleName + " " + remoteModuleIP + " " +
password + " " + calpont_rpm1 + " " + remoteModuleType + " " + installType + " " +
serverTypeInstall + " " + installer_debug + " " + installDir;
int rtnCode = system(cmd.c_str());
if (WEXITSTATUS(rtnCode) != 0)
{
cout << endl << "ERROR returned from binary_installer.sh" << endl;
exit(1);
}
}
}
else
{
if (remoteModuleType == "pm" && IserverTypeInstall != oam::INSTALL_COMBINE_DM_UM_PM)
{
//run remote installer script
if ( packageType != "binary" )
{
cout << endl << "----- Performing Install on Module '" + remoteModuleName + "' -----" << endl << endl;
cmd = installDir + "/bin/package_installer.sh " + remoteModuleName + " " + remoteModuleIP + " " + password + " " + calpont_rpm1 + " " + calpont_rpm2 + " " + calpont_rpm3 + " " + mysql_rpm + " " + mysqld_rpm + " " + installType + " " + packageType + " " + nodeps + " " + installer_debug;
int rtnCode = system(cmd.c_str());
if (WEXITSTATUS(rtnCode) != 0)
{
cout << endl << "ERROR returned from package_installer.sh" << endl;
exit(1);
}
}
else
{
// do a binary package install
cmd = installDir + "/bin/binary_installer.sh " + remoteModuleName + " " + remoteModuleIP +
" " + password + " " + calpont_rpm1 + " " + remoteModuleType + " " + installType +
" " + serverTypeInstall + " " + installer_debug + " " + installDir;
int rtnCode = system(cmd.c_str());
if (WEXITSTATUS(rtnCode) != 0)
{
cout << endl << "ERROR returned from binary_installer.sh" << endl;
exit(1);
}
}
}
}
}
if ( IserverTypeInstall == oam::INSTALL_COMBINE_DM_UM_PM )
{
//run the mysql / mysqld setup scripts
cout << endl << "Running the MariaDB ColumnStore setup scripts" << endl << endl;
// call the mysql setup scripts
mysqlSetup();
sleep(5);
}
//
// perform start of MariaDB ColumnStore of other servers in the system
//
list1 = childmodulelist.begin();
for (; list1 != childmodulelist.end() ; list1++)
{
string remoteModuleName = (*list1).moduleName;
string remoteModuleIP = (*list1).moduleIP;
string remoteHostName = (*list1).hostName;
if ( remoteModuleName == parentOAMModuleName )
continue;
//run remote command script
cmd = installDir + "/bin/remote_command.sh " + remoteModuleIP + " " + password + " '" + idbstartcmd + "' " + installer_debug;
int rtnCode = system(cmd.c_str());
if (WEXITSTATUS(rtnCode) != 0)
cout << "error with running remote_command.sh" << endl;
}
//start on local module
int rtnCode = system(idbstartcmd.c_str());
if (rtnCode != 0)
cout << "Error starting MariaDB ColumnStore local module" << endl;
}
else
{
//
// perform single-server install from auto-installer
@ -962,7 +736,7 @@ int main(int argc, char* argv[])
//startup mysqld and infinidb processes
cout << endl;
cmd = installDir + "/bin/clearShm > /dev/null 2>&1";
cmd = "clearShm > /dev/null 2>&1";
system(cmd.c_str());
system(idbstartcmd.c_str());
}
@ -986,7 +760,7 @@ int main(int argc, char* argv[])
cout << " DONE" << endl;
string logFile = tmpDir + "/dbbuilder.log";
cmd = installDir + "/bin/dbbuilder 7 > " + logFile;
cmd = "dbbuilder 7 > " + logFile;
system(cmd.c_str());
if (oam.checkLogStatus(logFile, "System Catalog created") )
@ -1072,14 +846,14 @@ bool setOSFiles(string parentOAMModuleName, int serverTypeInstall)
cmd = "cp " + fileName + " " + fileName + ".columnstoreSave > /dev/null 2>&1";
system(cmd.c_str());
cmd = "cat " + installDir + "/local/etc/" + parentOAMModuleName + "/" + files[i] + ".calpont >> " + fileName;
cmd = "cat /var/lib/columnstore/local/etc/" + parentOAMModuleName + "/" + files[i] + ".calpont >> " + fileName;
if (geteuid() == 0) system(cmd.c_str());
cmd = "rm -f " + installDir + "/local/ " + files[i] + "*.calpont > /dev/null 2>&1";
cmd = "rm -f /var/lib/columnstore/local/ " + files[i] + "*.calpont > /dev/null 2>&1";
system(cmd.c_str());
cmd = "cp " + installDir + "/local/etc/" + parentOAMModuleName + "/" + files[i] + ".calpont " + installDir + "/local/. > /dev/null 2>&1";
cmd = "cp /var/lib/columnstore/local/etc/" + parentOAMModuleName + "/" + files[i] + ".calpont /var/lib/columnstore/local/. > /dev/null 2>&1";
system(cmd.c_str());
}
@ -1095,9 +869,9 @@ bool makeModuleFile(string moduleName, string parentOAMModuleName)
string fileName;
if ( moduleName == parentOAMModuleName )
fileName = installDir + "/local/module";
fileName = "/var/lib/columnstore/local/module";
else
fileName = installDir + "/local/etc/" + moduleName + "/module";
fileName = "/var/lib/columnstore/local/etc/" + moduleName + "/module";
unlink (fileName.c_str());
ofstream newFile (fileName.c_str());

11
oamapps/postConfigure/mycnfUpgrade.cpp
View File

@ -51,6 +51,7 @@
#include <boost/regex.hpp>
#include "liboamcpp.h"
#include "installdir.h"
#include "config.h"
using namespace std;
using namespace oam;
@ -69,8 +70,8 @@ string rtrim(const string &in) {
void mergeMycnfIncludeArgs()
{
string userArgsFilename = startup::StartUp::installDir() + "/bin/myCnf-include-args.text.rpmsave";
string packagedArgsFilename = startup::StartUp::installDir() + "/bin/myCnf-include-args.text";
string userArgsFilename = std::string(MCSSUPPORTDIR) + "/myCnf-include-args.text.rpmsave";
string packagedArgsFilename = std::string(MCSSUPPORTDIR) + "/myCnf-include-args.text";
ifstream userArgs(userArgsFilename.c_str());
fstream packagedArgs(packagedArgsFilename.c_str(), ios::in);
@ -125,7 +126,7 @@ int main(int argc, char* argv[])
}
//my.cnf file
string mycnfFile = "/etc/my.cnf.d/columnstore.cnf";
string mycnfFile = std::string(MCSMYCNFDIR) + "/columnstore.cnf";
ifstream mycnffile (mycnfFile.c_str());
if (!mycnffile)
@ -149,7 +150,7 @@ int main(int argc, char* argv[])
mergeMycnfIncludeArgs();
//include arguments file
string includeFile = startup::StartUp::installDir() + "/bin/myCnf-include-args.text";
string includeFile = std::string(MCSSUPPORTDIR) + "/myCnf-include-args.text";
ifstream includefile (includeFile.c_str());
if (!includefile)
@ -159,7 +160,7 @@ int main(int argc, char* argv[])
}
//exclude arguments file
string excludeFile = startup::StartUp::installDir() + "/bin/myCnf-exclude-args.text";
string excludeFile = std::string(MCSSUPPORTDIR) + "/myCnf-exclude-args.text";
ifstream excludefile (excludeFile.c_str());
if (!excludefile)

190
oamapps/postConfigure/patchInstaller.cpp
View File

@ -1,190 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
Copyright (C) 2016 MariaDB Corporation
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/******************************************************************************************
* $Id: patchInstaller.cpp 64 2006-10-12 22:21:51Z dhill $
*
*
* Installs Calpont Software Patches on Calpont System
*
******************************************************************************************/
/**
* @file
*/
#include <iterator>
#include <numeric>
#include <deque>
#include <iostream>
#include <ostream>
#include <fstream>
#include <cstdlib>
#include <string>
#include <limits.h>
#include <sstream>
#include <exception>
#include <stdexcept>
#include <vector>
#include "stdio.h"
#include "ctype.h"
#include <netdb.h>
#include "liboamcpp.h"
#include "configcpp.h"
#include "alarmmanager.h"
using namespace std;
using namespace oam;
using namespace config;
bool installParentOAM( string patchLocation, string installLocation, string softwareFile );
typedef std::vector<string> Devices;
typedef struct Child_Module_struct
{
std::string moduleName;
std::string moduleIP;
std::string hostName;
} ChildModule;
typedef std::vector<ChildModule> ChildModuleList;
int main(int argc, char* argv[])
{
Oam oam;
string parentOAMModuleIPAddr;
ChildModuleList childmodulelist;
ChildModule childmodule;
string prompt;
string installer_debug = "0";
Config* sysConfig = Config::makeConfig();
string SystemSection = "SystemConfig";
string patchLocation = argv[1];
string installLocation = argv[2];
string softwareFile = argv[3];
string password = argv[4];
installer_debug = argv[5];
//get Parent OAM Module Name
string parentOAMModuleName;
try
{
parentOAMModuleName = sysConfig->getConfig(SystemSection, "ParentOAMModuleName");
}
catch (...)
{
cout << "ERROR: Problem getting Parent OAM Module Name" << endl;
exit(-1);
}
//install patch on Parent OAM Module
cout << endl << "----- Performing Patch installation on Controller OAM Module -----" << endl;
if ( !installParentOAM( patchLocation, installLocation, softwareFile ) )
cout << "Install Patch on Parent OAM Module error" << endl;
else
cout << "!!!Patch Installation on Controller OAM Module Successfully Completed" << endl;
//Get list of configured system modules
SystemModuleTypeConfig sysModuleTypeConfig;
try
{
oam.getSystemConfig(sysModuleTypeConfig);
}
catch (...)
{
cout << "ERROR: Problem reading the Calpont System Configuration file" << endl;
exit(-1);
}
string ModuleSection = "SystemModuleConfig";
for ( unsigned int i = 0 ; i < sysModuleTypeConfig.moduletypeconfig.size(); i++)
{
int moduleCount = sysModuleTypeConfig.moduletypeconfig[i].ModuleCount;
if ( moduleCount == 0 )
//no modules equipped for this Module Type, skip
continue;
//get IP addresses and Host Names
for ( int k = 0 ; k < moduleCount ; k++ )
{
DeviceNetworkList::iterator listPT = sysModuleTypeConfig.moduletypeconfig[i].ModuleNetworkList.begin();
for ( ; listPT != sysModuleTypeConfig.moduletypeconfig[i].ModuleNetworkList.end() ; listPT++)
{
string moduleName = (*listPT).DeviceName;
HostConfigList::iterator pt1 = (*listPT).hostConfigList.begin();
string moduleIPAddr = (*pt1).IPAddr;
if ( (*pt1).IPAddr == oam::UnassignedIpAddr)
// skip, unassigned server
continue;
//Install Software Patch on non Parent OAM Modules
if ( moduleName != parentOAMModuleName )
{
//run remote patch installer script
cout << endl << "----- Performing Patch installation of Module '" + moduleName + "' -----" << endl << endl;
string cmd = "/usr/local/mariadb/columnstore/bin/patch_installer.sh " + moduleName + " " + moduleIPAddr + " " + password + " " + patchLocation + " " + installLocation + " " + softwareFile + " " + installer_debug;
int rtnCode = system(cmd.c_str());
if (rtnCode != 0)
cout << "Error with running patch_installer.sh" << endl;
}
}
} // end of k loop
} //end of i for loop
}
/*
* Install Software File patch on local Parent OAM Module
*/
bool installParentOAM( string patchLocation, string installLocation, string softwareFile )
{
// Rename current file
string cmd = "mv " + installLocation + softwareFile + " " + installLocation + softwareFile + ".patchSave";
int rtnCode = system(cmd.c_str());
if (rtnCode != 0)
{
cout << "Error save current file" << endl;
return false;
}
// Install patch file
cmd = "cp " + patchLocation + softwareFile + " " + installLocation + softwareFile;
rtnCode = system(cmd.c_str());
if (rtnCode != 0)
{
cout << "Error copying patch file" << endl;
return false;
}
return true;
}

360
oamapps/postConfigure/postConfigure.cpp
View File

@ -191,9 +191,6 @@ bool amazon_quick_install = false;
bool doNotResolveHostNames = false;
bool resolveHostNamesToReverseDNSNames = false;
string DataFileEnvFile;
string installDir;
string tmpDir;
string HOME = "/root";
string SUDO = "";
@ -237,37 +234,6 @@ int main(int argc, char* argv[])
// -f for force use nodeps on rpm install
// -o to prompt for process to start offline
//default
installDir = installDir + "";
//see if we can determine our own location
ostringstream oss;
oss << "/proc/" << getpid() << "/exe";
ssize_t rlrc;
const size_t psz = PATH_MAX;
char thisexepath[psz + 1];
memset(thisexepath, 0, psz + 1);
rlrc = readlink(oss.str().c_str(), thisexepath, psz);
if (rlrc > 0)
{
thisexepath[rlrc] = 0;
//should look something like '/usr/local/mariadb/columnstore/bin/postConfigure'
char* ptr;
ptr = strrchr(thisexepath, '/');
if (ptr)
{
*ptr = 0;
ptr = strrchr(thisexepath, '/');
if (ptr)
{
*ptr = 0;
installDir = thisexepath;
}
}
}
//check if root-user
int user;
user = getuid();
@ -320,7 +286,6 @@ int main(int argc, char* argv[])
cout << " -qs Quick Install - Single Server" << endl;
cout << " -qm Quick Install - Multi Server" << endl;
cout << " -port MariaDB ColumnStore Port Address" << endl;
cout << " -i Non-root Install directory, Only use for non-root installs" << endl;
cout << " -sn System Name" << endl;
cout << " -pm-ip-addrs Performance Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl;
cout << " -um-ip-addrs User Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl;
@ -411,17 +376,6 @@ int main(int argc, char* argv[])
exit (1);
}
}
else if( string("-i") == argv[i] )
{
i++;
if (i >= argc )
{
cout << " ERROR: Path not provided" << endl;
exit (1);
}
installDir = argv[i];
}
else if( string("-sn") == argv[i] )
{
i++;
@ -532,12 +486,6 @@ int main(int argc, char* argv[])
}
}
if (installDir[0] != '/')
{
cout << " ERROR: Install dir '" << installDir << "' is not absolute" << endl;
exit(1);
}
if ( oldFileName == oam::UnassignedName )
oldFileName = std::string(MCSSYSCONFDIR) + "/columnstore/Columnstore.xml.rpmsave";
@ -745,7 +693,7 @@ int main(int argc, char* argv[])
// run columnstore.cnf upgrade script
if ( reuseConfig == "y" )
{
cmd = installDir + "/bin/mycnfUpgrade > " + tmpDir + "/mycnfUpgrade.log 2>&1";
cmd = "mycnfUpgrade > " + tmpDir + "/mycnfUpgrade.log 2>&1";
int rtnCode = system(cmd.c_str());
if (WEXITSTATUS(rtnCode) != 0)
@ -970,7 +918,7 @@ int main(int argc, char* argv[])
totalUmMemoryParam = "-";
}
cmd = installDir + "/bin/installer dummy.rpm dummy.rpm dummy.rpm dummy.rpm dummy.rpm initial dummy " + reuseConfig + " --nodeps ' ' 1 " + installDir + " " + numBlocksPctParam + " " + totalUmMemoryParam;
cmd = "columnstore_installer dummy.rpm dummy.rpm dummy.rpm dummy.rpm dummy.rpm initial dummy " + reuseConfig + " --nodeps ' ' 1 " + numBlocksPctParam + " " + totalUmMemoryParam;
system(cmd.c_str());
exit(0);
}
@ -1105,9 +1053,9 @@ int main(int argc, char* argv[])
cout << endl;
//cleanup/create local/etc directory
cmd = "rm -rf " + installDir + "/local/etc > /dev/null 2>&1";
cmd = "rm -rf /var/lib/columnstore/local/etc > /dev/null 2>&1";
system(cmd.c_str());
cmd = "mkdir " + installDir + "/local/etc > /dev/null 2>&1";
cmd = "mkdir /var/lib/columnstore/local/etc > /dev/null 2>&1";
system(cmd.c_str());
cout << endl << "===== Setup System Module Type Configuration =====" << endl << endl;
@ -1471,7 +1419,7 @@ int main(int argc, char* argv[])
if ( amazonInstall )
{
string cmd = installDir + "/bin/MCSgetCredentials.sh >/dev/null 2>&1";
string cmd = "MCSgetCredentials.sh >/dev/null 2>&1";
int rtnCode = system(cmd.c_str());
if ( WEXITSTATUS(rtnCode) != 0 )
@ -1567,7 +1515,7 @@ int main(int argc, char* argv[])
}
//create associated local/etc directory for parentOAMModuleName
cmd = "mkdir " + installDir + "/local/etc/" + parentOAMModuleName + " > /dev/null 2>&1";
cmd = "mkdir /var/lib/columnstore/local/etc/" + parentOAMModuleName + " > /dev/null 2>&1";
system(cmd.c_str());
//setup local module file name
@ -2821,7 +2769,7 @@ int main(int argc, char* argv[])
}
//create associated local/etc directory for module
string cmd = "mkdir " + installDir + "/local/etc/" + newModuleName + " > /dev/null 2>&1";
string cmd = "mkdir /var/lib/columnstore/local/etc/" + newModuleName + " > /dev/null 2>&1";
system(cmd.c_str());
if ( newModuleName != parentOAMModuleName)
@ -3231,7 +3179,7 @@ int main(int argc, char* argv[])
}
string DBrootID = "DBRoot" + *it;
string pathID = installDir + "/data" + *it;
string pathID = "/var/lib/columnstore/data" + *it;
try
{
@ -3729,7 +3677,7 @@ int main(int argc, char* argv[])
//check if local MariaDB ColumnStore system logging is working
cout << endl << "===== Checking MariaDB ColumnStore System Logging Functionality =====" << endl << endl;
cmd = installDir + "/bin/syslogSetup.sh status > /dev/null 2>&1";
cmd = "columnstoreSyslogSetup.sh status > /dev/null 2>&1";
int ret = system(cmd.c_str());
@ -3762,7 +3710,7 @@ int main(int argc, char* argv[])
{
//start MariaDB ColumnStore on local server
cout << endl << "----- Starting MariaDB ColumnStore on local server -----" << endl << endl;
cmd = installDir + "/bin/columnstore restart > /dev/null 2>&1";
cmd = "columnstore restart > /dev/null 2>&1";
int rtnCode = system(cmd.c_str());
if (WEXITSTATUS(rtnCode) != 0)
@ -3784,7 +3732,7 @@ int main(int argc, char* argv[])
//start MariaDB ColumnStore on local server
cout << endl << "----- Starting MariaDB ColumnStore on local Server '" + parentOAMModuleName + "' -----" << endl << endl;
string cmd = installDir + "/bin/columnstore restart > /dev/null 2>&1";
string cmd = "columnstore restart > /dev/null 2>&1";
int rtnCode = system(cmd.c_str());
if (WEXITSTATUS(rtnCode) != 0)
@ -3832,7 +3780,7 @@ int main(int argc, char* argv[])
string ModuleDBRootID = "ModuleDBRootID" + oam.itoa(pm + 1) + "-" + oam.itoa(i + 1) + "-3";
string dbr = sysConfig->getConfig("SystemModuleConfig", ModuleDBRootID);
string command = "" + moduleIPAddr +
":/dbroot" + dbr + " " + installDir + "/data" + dbr +
":/dbroot" + dbr + " /var/lib/columnstore/data" + dbr +
" glusterfs defaults,direct-io-mode=enable 00";
string toPM = "pm" + oam.itoa(pm + 1);
oam.distributeFstabUpdates(command, toPM);
@ -3842,10 +3790,7 @@ int main(int argc, char* argv[])
string dbbuilderLog = tmpDir + "/dbbuilder.log";
if (hdfs)
cmd = "bash -c '. " + installDir + "/bin/" + DataFileEnvFile + ";" + installDir + "/bin/dbbuilder 7 > " + dbbuilderLog;
else
cmd = installDir + "/bin/dbbuilder 7 > " + dbbuilderLog;
cmd = "dbbuilder 7 > " + dbbuilderLog;
system(cmd.c_str());
@ -4076,7 +4021,7 @@ bool checkSaveConfigFile()
return false;
}
cmd = "cd " + std::string(MCSSYSCONFDIR) + "/columnstore;" + installDir + "/bin/autoConfigure " + extentMapCheckOnly;
cmd = "cd " + std::string(MCSSYSCONFDIR) + "/columnstore;autoConfigure " + extentMapCheckOnly;
rtnCode = system(cmd.c_str());
if (WEXITSTATUS(rtnCode) != 0)
@ -4174,17 +4119,17 @@ bool setOSFiles(string parentOAMModuleName, int serverTypeInstall)
system(cmd.c_str());
cmd = "cat " + installDir + "/local/etc/" + parentOAMModuleName + "/" + files[i] + ".calpont >> " + fileName;
cmd = "cat /var/lib/columnstore/local/etc/" + parentOAMModuleName + "/" + files[i] + ".calpont >> " + fileName;
int rtnCode = system(cmd.c_str());
if (WEXITSTATUS(rtnCode) != 0)
cout << "Error Updating " + files[i] << endl;
cmd = "rm -f " + installDir + "/local/ " + files[i] + "*.calpont > /dev/null 2>&1";
cmd = "rm -f /var/lib/columnstore/local/ " + files[i] + "*.calpont > /dev/null 2>&1";
system(cmd.c_str());
cmd = "cp " + installDir + "/local/etc/" + parentOAMModuleName + "/" + files[i] + ".calpont " + installDir + "/local/. > /dev/null 2>&1";
cmd = "cp /var/lib/columnstore/local/etc/" + parentOAMModuleName + "/" + files[i] + ".calpont /var/lib/columnstore/local/. > /dev/null 2>&1";
system(cmd.c_str());
}
@ -4195,7 +4140,7 @@ bool setOSFiles(string parentOAMModuleName, int serverTypeInstall)
if (!oldFile)
return allfound;
string cmd = "cp " + fileName + " " + installDir + "/local/etc/. > /dev/null 2>&1";
string cmd = "cp " + fileName + " /var/lib/columnstore/local/etc/. > /dev/null 2>&1";
system(cmd.c_str());
return allfound;
}
@ -4450,7 +4395,7 @@ bool createDbrootDirs(string DBRootStorageType, bool amazonInstall)
// mount data1 and create directories if configured with storage
if ( DBRootStorageType == "external" )
{
string cmd = "mount " + installDir + "/data1 > " + tmpDir + "/mount.log 2>&1";
string cmd = "mount /var/lib/columnstore/data1 > " + tmpDir + "/mount.log 2>&1";
system(cmd.c_str());
if ( !rootUser)
@ -4460,12 +4405,12 @@ bool createDbrootDirs(string DBRootStorageType, bool amazonInstall)
if ( amazonInstall )
SUDO = "sudo ";
cmd = SUDO + "chown -R " + USER + ":" + USER + " " + installDir + "/data1 > /dev/null";
cmd = SUDO + "chown -R " + USER + ":" + USER + " /var/lib/columnstore/data1 > /dev/null";
system(cmd.c_str());
}
// create system file directories
cmd = "mkdir -p " + installDir + "/data1/systemFiles/dbrm > /dev/null 2>&1";
cmd = "mkdir -p /var/lib/columnstore/data1/systemFiles/dbrm > /dev/null 2>&1";
rtnCode = system(cmd.c_str());
if (WEXITSTATUS(rtnCode) != 0)
@ -4475,7 +4420,7 @@ bool createDbrootDirs(string DBRootStorageType, bool amazonInstall)
}
}
cmd = "chmod 755 -R " + installDir + "/data1/systemFiles/dbrm > /dev/null 2>&1";
cmd = "chmod 755 -R /var/lib/columnstore/data1/systemFiles/dbrm > /dev/null 2>&1";
system(cmd.c_str());
return true;
@ -4578,105 +4523,7 @@ bool storageSetup(bool amazonInstall)
{
hdfs = false;
sysConfig->setConfig("StorageManager", "Enabled", "Y");
sysConfig->setConfig("SystemConfig", "DataFilePlugin", "$INSTALLDIR/lib/libcloudio.so");
sysConfig->setConfig("SystemConfig", "DataFileEnvFile", "");
}
else if (hdfs)
{
//default
DataFileEnvFile = "setenv-hdfs-20";
try
{
DataFileEnvFile = sysConfig->getConfig("SystemConfig", "DataFileEnvFile");
}
catch (...)
{
DataFileEnvFile = "setenv-hdfs-20";
}
string DataFilePlugin = installDir + "/lib/hdfs-20.so";
try
{
DataFilePlugin = sysConfig->getConfig("SystemConfig", "DataFilePlugin");
}
catch (...)
{
DataFilePlugin = installDir + "/lib/hdfs-20.so";
}
while (true)
{
cout << " Running HDFS Sanity Test (please wait): ";
cout.flush();
string logdir("/var/log/mariadb/columnstore");
if (access(logdir.c_str(), W_OK) != 0) logdir = tmpDir;
string hdfslog = logdir + "/hdfsCheck.log1";
string cmd = ". " + installDir + "/bin/" + DataFileEnvFile + ";" + installDir + "/bin/hdfsCheck " + DataFilePlugin + " > " + hdfslog + " 2>&1";
system(cmd.c_str());
if (oam.checkLogStatus(hdfslog, "All HDFS checks passed!"))
{
cout << " PASSED" << endl;
try
{
sysConfig->setConfig("SystemConfig", "DataFilePlugin", DataFilePlugin);
}
catch (...)
{
cout << "ERROR: Problem setting DataFilePlugin in the MariaDB ColumnStore System Configuration file" << endl;
return false;
}
try
{
sysConfig->setConfig("SystemConfig", "DataFileEnvFile", DataFileEnvFile);
}
catch (...)
{
cout << "ERROR: Problem setting DataFileEnvFile in the MariaDB ColumnStore System Configuration file" << endl;
return false;
}
if ( !writeConfig(sysConfig) )
{
cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl;
return false;
}
return true;
}
else
{
cout << " FAILED (Tested with Hadoop Datafile Plugin File (" + DataFilePlugin + "), please re-enter or enter 'exit' to Investigate)" << endl << endl;
if ( noPrompting )
exit(1);
prompt = "Enter the Hadoop Datafile Plugin File (" + DataFilePlugin + ") > ";
pcommand = callReadline(prompt.c_str());
if (pcommand)
{
if (strlen(pcommand) > 0) DataFilePlugin = pcommand;
callFree(pcommand);
}
if ( DataFilePlugin == "exit" )
exit (1);
if ( DataFilePlugin != installDir + "/lib/hdfs-20.so" )
DataFileEnvFile = "setenv-hdfs-12";
else
DataFileEnvFile = "setenv-hdfs-20";
}
}
sysConfig->setConfig("SystemConfig", "DataFilePlugin", "libcloudio.so");
}
return true;
@ -5416,136 +5263,11 @@ bool storageSetup(bool amazonInstall)
sysConfig->setConfig("PrimitiveServers", "DirectIO", "y");
}
// if hadoop / hdfs
#if 0
We are not supporting HDFS anymore. Leaving this here for future use.
if ( storageType == "4" )
{
hdfs = true;
string DataFilePlugin = installDir + "/lib/hdfs-20.so";
try
{
DataFilePlugin = sysConfig->getConfig("SystemConfig", "DataFilePlugin");
}
catch (...)
{
DataFilePlugin = installDir + "/lib/hdfs-20.so";
}
if (DataFilePlugin.empty() || DataFilePlugin == "")
DataFilePlugin = installDir + "/lib/hdfs-20.so";
DataFileEnvFile = "setenv-hdfs-20";
try
{
DataFileEnvFile = sysConfig->getConfig("SystemConfig", "DataFileEnvFile");
}
catch (...)
{
DataFileEnvFile = "setenv-hdfs-20";
}
if (DataFileEnvFile.empty() || DataFileEnvFile == "" || DataFileEnvFile == oam::UnassignedName)
DataFileEnvFile = "setenv-hdfs-20";
cout << endl;
while (true)
{
prompt = "Enter the Hadoop Datafile Plugin File (" + DataFilePlugin + ") > ";
pcommand = callReadline(prompt.c_str());
if (pcommand)
{
if (strlen(pcommand) > 0) DataFilePlugin = pcommand;
callFree(pcommand);
}
if ( DataFilePlugin == "exit" )
exit (1);
if ( DataFilePlugin != installDir + "/lib/hdfs-20.so" )
DataFileEnvFile = "setenv-hdfs-12";
ifstream File (DataFilePlugin.c_str());
if (!File)
cout << "Error: Hadoop Datafile Plugin File (" + DataFilePlugin + ") doesn't exist, please re-enter" << endl;
else
{
cout << endl << " Running HDFS Sanity Test (please wait): ";
cout.flush();
string logdir("/var/log/mariadb/columnstore");
if (access(logdir.c_str(), W_OK) != 0) logdir = tmpDir;
string hdfslog = logdir + "/hdfsCheck.log1";
string cmd = installDir + "/bin/hdfsCheck " + DataFilePlugin + " > " + hdfslog + " 2>&1";
system(cmd.c_str());
if (oam.checkLogStatus(hdfslog, "All HDFS checks passed!"))
{
cout << " PASSED" << endl;
break;
}
else
{
cout << " FAILED (Tested with Hadoop Datafile Plugin File (" + DataFilePlugin + "), please re-enter or enter 'exit' to Investigate)" << endl << endl;
}
}
}
try
{
sysConfig->setConfig("SystemConfig", "DataFilePlugin", DataFilePlugin);
}
catch (...)
{
cout << "ERROR: Problem setting DataFilePlugin in the MariaDB ColumnStore System Configuration file" << endl;
return false;
}
try
{
sysConfig->setConfig("SystemConfig", "DataFileEnvFile", DataFileEnvFile);
}
catch (...)
{
cout << "ERROR: Problem setting DataFileEnvFile in the MariaDB ColumnStore System Configuration file" << endl;
return false;
}
try
{
sysConfig->setConfig("SystemConfig", "DataFileLog", "OFF");
}
catch (...)
{
cout << "ERROR: Problem setting DataFileLog in the MariaDB ColumnStore System Configuration file" << endl;
return false;
}
try
{
sysConfig->setConfig("ExtentMap", "ExtentsPerSegmentFile", "1");
}
catch (...)
{
cout << "ERROR: Problem setting ExtentsPerSegmentFile in the MariaDB ColumnStore System Configuration file" << endl;
return false;
}
}
#endif
if (storageType == "4")
{
hdfs = false;
sysConfig->setConfig("StorageManager", "Enabled", "Y");
sysConfig->setConfig("SystemConfig", "DataFilePlugin", "$INSTALLDIR/lib/libcloudio.so");
sysConfig->setConfig("SystemConfig", "DataFileEnvFile", "");
sysConfig->setConfig("SystemConfig", "DataFilePlugin", "libcloudio.so");
}
else
{
@ -5561,16 +5283,6 @@ bool storageSetup(bool amazonInstall)
return false;
}
try
{
sysConfig->setConfig("SystemConfig", "DataFileEnvFile", "");
}
catch (...)
{
cout << "ERROR: Problem setting DataFileEnvFile in the MariaDB ColumnStore System Configuration file" << endl;
return false;
}
if ( !writeConfig(sysConfig) )
{
cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl;
@ -5645,7 +5357,7 @@ void setSystemName()
bool copyFstab(string moduleName)
{
string cmd;
cmd = "/bin/cp -f /etc/fstab " + installDir + "/local/etc/" + moduleName + "/. > /dev/null 2>&1";
cmd = "/bin/cp -f /etc/fstab /var/lib/columnstore/local/etc/" + moduleName + "/. > /dev/null 2>&1";
system(cmd.c_str());
@ -5661,9 +5373,9 @@ bool makeModuleFile(string moduleName, string parentOAMModuleName)
string fileName;
if ( moduleName == parentOAMModuleName)
fileName = installDir + "/local/module";
fileName = "/var/lib/columnstore/local/module";
else
fileName = installDir + "/local/etc/" + moduleName + "/module";
fileName = "/var/lib/columnstore/local/etc/" + moduleName + "/module";
unlink (fileName.c_str());
ofstream newFile (fileName.c_str());
@ -5685,16 +5397,6 @@ bool updateBash()
ifstream newFile (fileName.c_str());
if ( hdfs )
{
string cmd = "echo . " + installDir + "/bin/" + DataFileEnvFile + " >> " + fileName;
system(cmd.c_str());
cmd = "su - hdfs -c 'hadoop fs -mkdir -p " + installDir + ";hadoop fs -chown root:root " + installDir + "' >/dev/null 2>&1";
system(cmd.c_str());
}
newFile.close();
return true;
@ -5963,7 +5665,7 @@ bool singleServerDBrootSetup()
}
string DBrootID = "DBRoot" + *it;
string pathID = installDir + "/data" + *it;
string pathID = "/var/lib/columnstore/data" + *it;
try
{
@ -6146,7 +5848,7 @@ bool glusterSetup(string password, bool doNotResolveHostNames)
std::vector<int> dbrootPms[DBRootCount];
DataRedundancySetup DataRedundancyConfigs[pmNumber];
string command = "";
string remoteCommand = installDir + "/bin/remote_command.sh ";
string remoteCommand = "remote_command.sh ";
// how many copies?
if (pmNumber > 2)
@ -6584,7 +6286,7 @@ bool glusterSetup(string password, bool doNotResolveHostNames)
{
for ( int brick = 1; brick <= numberBricksPM; brick++)
{
command = remoteCommand + DataRedundancyConfigs[pm].pmIpAddr + " " + password + " 'mkdir -p " + installDir + "/gluster/brick" + oam.itoa(brick) + "'";
command = remoteCommand + DataRedundancyConfigs[pm].pmIpAddr + " " + password + " 'mkdir -p /var/lib/columnstore/gluster/brick" + oam.itoa(brick) + "'";
status = system(command.c_str());
if (WEXITSTATUS(status) != 0 )
@ -6712,7 +6414,7 @@ bool glusterSetup(string password, bool doNotResolveHostNames)
for (; dbrootPmIter < dbrootPms[db].end(); dbrootPmIter++ )
{
int pm = (*dbrootPmIter) - 1;
command += DataRedundancyConfigs[pm].pmIpAddr + ":" + installDir + "/gluster/brick" + oam.itoa(pmnextbrick[pm]) + " ";
command += DataRedundancyConfigs[pm].pmIpAddr + ":/var/lib/columnstore/gluster/brick" + oam.itoa(pmnextbrick[pm]) + " ";
pmnextbrick[pm]++;
}

36
oamapps/postConfigure/quick_installer_amazon.sh
View File

@ -54,32 +54,14 @@ else
fi
fi
if [[ $HOME = "/root" ]]; then
echo ""
echo "${bold}Run post-install script${normal}"
echo ""
/usr/local/mariadb/columnstore/bin/post-install
echo "${bold}Run postConfigure script${normal}"
echo ""
if [[ $umCount = "" ]]; then
/usr/local/mariadb/columnstore/bin/postConfigure -qa -pm-count $pmCount $systemName -d
else
/usr/local/mariadb/columnstore/bin/postConfigure -qa -pm-count $pmCount -um-count $umCount $systemName -d
fi
echo ""
echo "${bold}Run post-install script${normal}"
echo ""
columnstore-post-install
echo "${bold}Run postConfigure script${normal}"
echo ""
if [[ $umCount = "" ]]; then
postConfigure -qa -pm-count $pmCount $systemName -d
else
echo ""
echo "${bold}Run post-install script${normal}"
echo ""
$HOME/mariadb/columnstore/bin/post-install --installdir=$HOME/mariadb/columnstore
export COLUMNSTORE_INSTALL_DIR=$HOME/mariadb/columnstore
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HOME/mariadb/columnstore/lib
echo "${bold}Run postConfigure script${normal}"
echo ""
if [[ $umCount = "" ]]; then
$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qa -pm-count $pmCount $systemName -d
else
$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qa -pm-count $pmCount -um-count $umCount $systemName -d
fi
postConfigure -qa -pm-count $pmCount -um-count $umCount $systemName -d
fi

40
oamapps/postConfigure/quick_installer_multi_server.sh
View File

@ -6,7 +6,6 @@
pmIpAddrs=""
umIpAddrs=""
nonDistrubutedInstall="-n"
systemName=""
for arg in "$@"; do
@ -17,8 +16,6 @@ for arg in "$@"; do
elif [ `expr -- "$arg" : '--system-name='` -eq 14 ]; then
systemName="`echo $arg | awk -F= '{print $2}'`"
systemName="-sn "$systemName
elif [ `expr -- "$arg" : '--dist-install'` -eq 14 ]; then
nonDistrubutedInstall="-d"
elif [ `expr -- "$arg" : '--help'` -eq 6 ]; then
echo "Usage ./quick_installer_multi_server.sh [OPTION]"
echo ""
@ -34,7 +31,6 @@ for arg in "$@"; do
echo
echo "--pm-ip-addresses=xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx"
echo "--um-ip-addresses=xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx, optional"
echo "--dist-install Use Distributed Install, optional"
echo "--system-name=nnnn System Name, optional"
echo ""
exit 1
@ -60,32 +56,14 @@ else
fi
fi
if [[ $HOME = "/root" ]]; then
echo ""
echo "${bold}Run post-install script${normal}"
echo ""
/usr/local/mariadb/columnstore/bin/post-install
echo "${bold}Run postConfigure script${normal}"
echo ""
if [[ $umIpAddrs = "" ]]; then
/usr/local/mariadb/columnstore/bin/postConfigure -qm -pm-ip-addrs $pmIpAddrs $nonDistrubutedInstall $systemName
else
/usr/local/mariadb/columnstore/bin/postConfigure -qm -pm-ip-addrs $pmIpAddrs -um-ip-addrs $umIpAddrs $nonDistrubutedInstall $systemName
fi
echo ""
echo "${bold}Run post-install script${normal}"
echo ""
columnstore-post-install
echo "${bold}Run postConfigure script${normal}"
echo ""
if [[ $umIpAddrs = "" ]]; then
postConfigure -qm -pm-ip-addrs $pmIpAddrs $systemName
else
echo ""
echo "${bold}Run post-install script${normal}"
echo ""
$HOME/mariadb/columnstore/bin/post-install --installdir=$HOME/mariadb/columnstore
export COLUMNSTORE_INSTALL_DIR=$HOME/mariadb/columnstore
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HOME/mariadb/columnstore/lib
echo "${bold}Run postConfigure script${normal}"
echo ""
if [[ $umIpAddrs = "" ]]; then
$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qm -pm-ip-addrs $pmIpAddrs $nonDistrubutedInstall $systemName
else
$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qm -pm-ip-addrs $pmIpAddrs -um-ip-addrs $umIpAddrs $nonDistrubutedInstall $systemName
fi
postConfigure -qm -pm-ip-addrs $pmIpAddrs -um-ip-addrs $umIpAddrs $systemName
fi

28
oamapps/postConfigure/quick_installer_single_server.sh
View File

@ -17,24 +17,10 @@ for arg in "$@"; do
done
if [ $HOME == "/root" ]; then
echo ""
echo "Run post-install script"
echo ""
/usr/local/mariadb/columnstore/bin/post-install
echo "Run postConfigure script"
echo ""
/usr/local/mariadb/columnstore/bin/postConfigure -qs
else
echo ""
echo "Run post-install script"
echo ""
$HOME/mariadb/columnstore/bin/post-install --installdir=$HOME/mariadb/columnstore
export COLUMNSTORE_INSTALL_DIR=$HOME/mariadb/columnstore
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HOME/mariadb/columnstore/lib
echo "Run postConfigure script"
echo ""
$HOME/mariadb/columnstore/bin/postConfigure -i $HOME/mariadb/columnstore -qs
fi
echo ""
echo "Run post-install script"
echo ""
columnstore-post-install
echo "Run postConfigure script"
echo ""
postConfigure -qs

2
oamapps/serverMonitor/UMAutoSync.cpp
View File

@ -237,7 +237,7 @@ void UMAutoSync()
void rsync(std::string moduleName, std::string IPAddr, std::string rootPassword)
{
string cmd = startup::StartUp::installDir() + "/bin/rsync.sh " + IPAddr + " " + rootPassword + " 1 > /tmp/rsync_" + moduleName + ".log";
string cmd = "rsync.sh " + IPAddr + " " + rootPassword + " 1 > /tmp/rsync_" + moduleName + ".log";
int ret = system(cmd.c_str());
if ( WEXITSTATUS(ret) == 0 )

2
oamapps/serverMonitor/dbhealthMonitor.cpp
View File

@ -180,7 +180,7 @@ int ServerMonitor::healthCheck(bool action)
ACK_FLAG ackTemp = ACK_YES;
//run Health script
string cmd = startup::StartUp::installDir() + "/bin/dbhealth.sh > /var/log/mariadb/columnstore/dbfunctional.log1 2>&1";
string cmd = "dbhealth.sh > /var/log/mariadb/columnstore/dbfunctional.log1 2>&1";
system(cmd.c_str());
if (!oam.checkLogStatus("/var/log/mariadb/columnstore/dbfunctional.log1", "OK"))

2
oamapps/serverMonitor/diskMonitor.cpp
View File

@ -250,7 +250,7 @@ void diskMonitor()
// check local
if ( deviceName == "/")
{
fileName = deviceName + startup::StartUp::installDir();
fileName = deviceName + "/var/lib/columnstore/";
}
else
{

2
oamapps/serverMonitor/serverMonitor.cpp
View File

@ -323,7 +323,7 @@ bool ServerMonitor::sendResourceAlarm(string alarmItem, ALARMS alarmID, int acti
serverMonitor.checkSwapAlarm(alarmItem, alarmID);
// don't issue an alarm on thge dbroots is already issued by this or another server
if ( alarmItem.find(startup::StartUp::installDir() + "/data") == 0 )
if ( alarmItem.find("/var/lib/columnstore/data") == 0 )
{
// check if Alarm is already active from any module, don't resend
if ( !( oam.checkActiveAlarm(alarmID, "*", alarmItem)) )