database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-09-06 22:06:40 +03:00
Code Activity

chnages from 1.0.8

Browse Source
This commit is contained in:
David Hill
2017-03-27 16:39:35 -05:00
parent 61a7113eb0
commit 5000e79fdd
7 changed files with 165 additions and 234 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
oam/etc/ConsoleCmds.xml
View File

@@ -29,7 +29,7 @@
<Name>redistributeData</Name>
<Desc1>Redistribute table data accross all dbroots to balance disk usage</Desc1>
<Arg1>START to begin a redistribution</Arg1>
<Arg2>START REMOVE [dbroots] to redistribute to all but the enumerated dbroots, leaving those empty</Arg2>
<Arg2>START REMOVE n to being a redistribution where data is removed from dbroot 'n'</Arg2>
<Arg3>STOP to stop redistribution before completion</Arg3>
<Arg4>STATUS to to view statistics and progress</Arg4>
</Cmd4>

9
oam/install_scripts/module_installer.sh
View File

@@ -57,15 +57,22 @@ fi
cloud=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig Installation Cloud`
if [ $cloud = "amazon-ec2" ] || [ $cloud = "amazon-vpc" ]; then
cp $COLUMNSTORE_INSTALL_DIR/local/etc/*.pem $HOME/. > /dev/null 2>&1
cp $COLUMNSTORE_INSTALL_DIR/local/etc/credentials $HOME/.aws/. > /dev/null 2>&1
if [ $module = "pm" ]; then
if test -f $COLUMNSTORE_INSTALL_DIR/local/etc/pm1/fstab ; then
echo "Setup fstab on Module"
if [ $user = "root" ]; then
touch /etc/fstab
rm -f /etc/fstab.columnstoreSave
cp /etc/fstab /etc/fstab.columnstoreSave
cat $COLUMNSTORE_INSTALL_DIR/local/etc/pm1/fstab >> /etc/fstab
else
sudo touch /etc/fstab
sudo rm -f /etc/fstab.columnstoreSave
sudo cp /etc/fstab /etc/fstab.columnstoreSave
sudo cat $COLUMNSTORE_INSTALL_DIR/local/etc/pm1/fstab >> /etc/fstab
fi
fi
fi
fi

18
oam/install_scripts/performance_installer.sh
View File

@@ -63,7 +63,7 @@ if { $PKGTYPE == "rpm" } {
# check and see if remote server has ssh keys setup, set PASSWORD if so
send_user " "
send "ssh $USERNAME@$SERVER 'time'\n"
set timeout 60
set timeout 20
expect {
"Host key verification failed" { send_user "FAILED: Host key verification failed\n" ; exit 1 }
"service not known" { send_user "FAILED: Invalid Host\n" ; exit 1 }
@@ -82,15 +82,14 @@ expect {
"No route to host" { send_user "ERROR: No route to host\n" ; exit 1 }
timeout { send_user "ERROR: Timeout to host\n" ; exit 1 }
}
set timeout 30
set timeout 10
expect {
-re {[$#] } { }
"sys" { }
}
send_user "\n"
#BUG 5749 - SAS: didn't work on their system until I added the sleep 60
sleep 60
#sleep 60
if { $INSTALLTYPE == "initial" || $INSTALLTYPE == "uninstall" } {
#
@@ -107,7 +106,9 @@ if { $INSTALLTYPE == "initial" || $INSTALLTYPE == "uninstall" } {
}
set timeout 120
expect {
"package dummy" { send_user "DONE" }
"error: --purge needs at least one package" { send_user "DONE" }
"dummy is not installed" { send_user "DONE" }
"dummy which isn't installed" { send_user "DONE" }
"error: Failed dependencies" { send_user "ERROR: Failed dependencies\n" ; exit 1 }
"Permission denied, please try again" { send_user "ERROR: Invalid password\n" ; exit 1 }
"Connection refused" { send_user "ERROR: Connection refused\n" ; exit 1 }
@@ -154,7 +155,8 @@ if { $PASSWORD != "ssh" } {
}
set timeout 180
expect {
"package dummy" { send_user "DONE" }
"dummy is not installed" { send_user "DONE" }
"dummy which isn't installed" { send_user "DONE" }
"directory" { send_user "ERROR\n" ;
send_user "\n*** Installation ERROR\n" ;
exit 1 }
@@ -182,7 +184,8 @@ if { $INSTALLTYPE == "initial"} {
}
set timeout 180
expect {
"package dummy" { send_user "DONE" }
"dummy is not installed" { send_user "DONE" }
"dummy which isn't installed" { send_user "DONE" }
"error: Failed dependencies" { send_user "ERROR: Failed dependencies\n" ;
send_user "\n*** Installation ERROR\n" ;
exit 1 }
@@ -282,6 +285,7 @@ if { $INSTALLTYPE == "initial"} {
"FAILED" { send_user "ERROR: missing OS file\n" ; exit 1 }
"Connection closed" { send_user "ERROR: Connection closed\n" ; exit 1 }
"No such file" { send_user "ERROR: File Not Found\n" ; exit 1 }
"WARNING" { send_user "WARNING: SYSLOG setup failed\n" }
}
send_user "\n"
}

20
oam/install_scripts/post-install
View File

@@ -41,8 +41,8 @@ if [ $installdir != "/usr/local/mariadb/columnstore" ]; then
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib
else
# add library config file
$SUDO /bin/cp -f $installdir/bin/columnstore.conf /etc/ld.so.conf.d/.
$SUDO ldconfig
/bin/cp -f $installdir/bin/columnstore.conf /etc/ld.so.conf.d/.
ldconfig
fi
#check 64-bit OS compatiable
@@ -123,7 +123,11 @@ cd /
test -d /var/log/mariadb || $SUDO mkdir /var/log/mariadb >/dev/null 2>&1
test -d /var/log/mariadb/columnstore || $SUDO mkdir /var/log/mariadb/columnstore >/dev/null 2>&1
$SUDO chmod -R 777 /var/log/mariadb
if [ $user != "root" ]; then
$SUDO chmod -R 777 /var/log/mariadb >/dev/null 2>&1
$SUDO chown -R $user:$user /var/log/mariadb >/dev/null 2>&1
fi
test -d /var/log/mariadb/columnstore/archive || mkdir /var/log/mariadb/columnstore/archive >/dev/null 2>&1
test -d /var/log/mariadb/columnstore/corefiles || mkdir /var/log/mariadb/columnstore/corefiles >/dev/null 2>&1
@@ -147,9 +151,9 @@ mkdir -p $installdir/data/bulk/job >/dev/null 2>&1
mkdir -p $installdir/data/bulk/rollback >/dev/null 2>&1
mkdir -p $installdir/data/bulk/tmpjob >/dev/null 2>&1
rm -f $installdir/data/bulk/tmpjob/* >/dev/null 2>&1
chmod -R 755 $installdir/data/bulk >/dev/null 2>&1
#create columnstore temp file directory
$SUDO chmod 777 /tmp
mkdir -p /tmp/columnstore_tmp_files >/dev/null 2>&1
#setup core file directory and link
@@ -206,25 +210,21 @@ fi
#setup MariaDB Columnstore system logging
if [ $user = "root" ]; then
$installdir/bin/syslogSetup.sh install > /tmp/syslog_install.log 2>&1
chmod 777 -R /dev/shm
rm -f /etc/default/columnstore
else
$SUDO chmod 777 /tmp
$SUDO rm -fr /tmp/* > /dev/null 2>&1
$SUDO $installdir/bin/syslogSetup.sh --installdir=$installdir install > /tmp/syslog_install.log 2>&1
$installdir/bin/syslogSetup.sh --installdir=$installdir install > /tmp/syslog_install.log 2>&1
$SUDO chown $user:$user $installdir/etc/Columnstore.xml
$SUDO chmod -R 777 /dev/shm
$SUDO mkdir /var/lock/subsys > /dev/null 2>&1
$SUDO chmod 777 /var/lock/subsys > /dev/null 2>&1
$SUDO rm -f /var/lock/subsys/mysql-Columnstore
$SUDO chmod 777 /etc/fstab
$SUDO chmod 666 /etc/fstab
sed -i -e s@/usr/local/mariadb/columnstore@$installdir@g $installdir/bin/columnstore.def
$SUDO cp $installdir/bin/columnstore.def /etc/default/columnstore
$SUDO chown $user:$user /etc/default/columnstore
sed -i -e s@prefix=/usr/local@prefix=$HOME@g $installdir/bin/*
fi
#check if MariaDB Columnstore system logging was setup

17
oam/install_scripts/user_installer.sh
View File

@@ -67,7 +67,7 @@ if { $PKGTYPE == "rpm" } {
# check and see if remote server has ssh keys setup, set PASSWORD if so
send_user " "
send "ssh $USERNAME@$SERVER 'time'\n"
set timeout 60
set timeout 20
expect {
"Host key verification failed" { send_user "FAILED: Host key verification failed\n" ; exit 1 }
"service not known" { send_user "FAILED: Invalid Host\n" ; exit 1 }
@@ -86,14 +86,14 @@ expect {
"No route to host" { send_user "ERROR: No route to host\n" ; exit 1 }
timeout { send_user "ERROR: Timeout to host\n" ; exit 1 }
}
set timeout 30
set timeout 10
expect {
-re {[$#] } { }
"sys" { }
}
send_user "\n"
#BUG 5749 - SAS: didn't work on their system until I added the sleep 60
sleep 60
#sleep 60
if { $INSTALLTYPE == "initial" || $INSTALLTYPE == "uninstall" } {
#
@@ -110,7 +110,9 @@ if { $INSTALLTYPE == "initial" || $INSTALLTYPE == "uninstall" } {
}
set timeout 120
expect {
"package dummy" { send_user "DONE" }
"error: --purge needs at least one package" { send_user "DONE" }
"dummy is not installed" { send_user "DONE" }
"dummy which isn't installed" { send_user "DONE" }
"error: Failed dependencies" { send_user "ERROR: Failed dependencies\n" ; exit 1 }
"Permission denied, please try again" { send_user "ERROR: Invalid password\n" ; exit 1 }
"Connection closed" { send_user "ERROR: Connection closed\n" ; exit 1 }
@@ -156,7 +158,8 @@ if { $PASSWORD != "ssh" } {
}
set timeout 120
expect {
"package dummy" { send_user "DONE" }
"dummy is not installed" { send_user "DONE" }
"dummy which isn't installed" { send_user "DONE" }
"directory" { send_user "ERROR\n" ;
send_user "\n*** Installation ERROR\n" ;
exit 1 }
@@ -184,7 +187,8 @@ if { $INSTALLTYPE == "initial"} {
}
set timeout 180
expect {
"package dummy" { send_user "DONE" }
"dummy is not installed" { send_user "DONE" }
"dummy which isn't installed" { send_user "DONE" }
"error: Failed dependencies" { send_user "ERROR: Failed dependencies\n" ;
send_user "\n*** Installation ERROR\n" ;
exit 1 }
@@ -288,6 +292,7 @@ if { $INSTALLTYPE == "initial"} {
"FAILED" { send_user "ERROR: missing OS file\n" ; exit 1 }
"Connection closed" { send_user "ERROR: Connection closed\n" ; exit 1 }
"No such file" { send_user "ERROR: File Not Found\n" ; exit 1 }
"WARNING" { send_user "WARNING: SYSLOG setup failed\n" }
}
send_user "\n"
set timeout 30

47
oamapps/mcsadmin/mcsadmin.cpp
View File

@@ -719,11 +719,6 @@ int processCommand(string* arguments)
vector<uint32_t> srcDbroots; // all of the currently configured dbroots
vector<uint32_t> destDbroots; // srcDbroots - removeDbroots
set<int>::iterator dbiter;
if (!oam.checkSystemRunning())
{
cout << "Mariadb ColumnStore is not running" << endl;
break;
}
if (arguments[1] == "start")
{
// Get a list of all the configured dbroots in the xml file.
@@ -802,40 +797,6 @@ int processCommand(string* arguments)
cout << " " << *iter;
cout << endl << endl;
BRM::DBRM dbrm;
// Ready to start the redistribute. The system must SuspendDataBaseWrites for the duration.
if (dbrm.getSystemSuspended() == BRM::ERR_OK)
{
cout << "The system must be in read only mode for redistribeData to work" << endl;
cout << "You must run suspendDatabaseWrites before running redistributeData" << endl;
cout << "Be sure to run resumeDatabaseWrites when redistributeData status shows complete" << endl;
break;
}
#if 0
// This can be used when redistributeData doesn't return until complete.
if (dbrm.getSystemSuspended() == ERR_OK)
{
// System not in suspenddatabasewrites
// If there are bulkloads, ddl or dml happening, refuse the request
execplan::SessionManager sessionManager;
BRM::SIDTIDEntry blockingsid;
std::vector<BRM::TableLockInfo> tableLocks = dbrm.getAllTableLocks();
bool bActiveTransactions = false;
if (!tableLocks.empty())
{
cout << "There are active table locks. Redistribute won't run." << endl;
oam.DisplayLockedTables(tableLocks, &dbrm);
break;
}
else if (sessionManager.checkActiveTransaction(0, bIsDbrmUp, blockingsid))
{
cout << endl << "There are active transactions being processed. Redistribute won't run" << endl;
break;
}
}
// stop writes to MariaDB Columnstore Database
oam.SuspendWrites(gracefulTemp, ackTemp);
#endif
// Connect to PM for dbroot1
ByteStream bs;
// message WES ID, sequence #, action id
@@ -5046,12 +5007,12 @@ int processCommand(string* arguments)
cout << endl;
if ( rootUser)
{
system("rpm -qi mariadb-columnstore-platform > /tmp/columnstore.txt 2>&1");
if (oam.checkLogStatus("/tmp/columnstore.txt", "Name"))
int rtnCode = system("rpm -qi mariadb-columnstore-platform > /tmp/columnstore.txt 2>&1");
if (WEXITSTATUS(rtnCode) == 0)
system("cat /tmp/columnstore.txt");
else {
system("dpkg -s mariadb-columnstore > /tmp/columnstore.txt 2>&1");
if (oam.checkLogStatus("/tmp/columnstore.txt", "Status: install"))
rtnCode = system("dpkg -s mariadb-columnstore-platform > /tmp/columnstore.txt 2>&1");
if (WEXITSTATUS(rtnCode) == 0)
system("cat /tmp/columnstore.txt");
else {
SystemSoftware systemsoftware;

166
oamapps/postConfigure/postConfigure.cpp
View File

@@ -56,6 +56,7 @@
#include <sys/socket.h>
#include <ifaddrs.h>
#include <stdio.h>
#include <string.h> /* for strncpy */
#include <sys/types.h>
#include <sys/ioctl.h>
@@ -108,7 +109,7 @@ bool updateProcessConfig(int serverTypeInstall);
bool uncommentCalpontXml( string entry);
bool makeRClocal(string moduleType, string moduleName, int IserverTypeInstall);
bool createDbrootDirs(string DBRootStorageType);
bool pkgCheck();
bool pkgCheck(std::string columnstorePackage);
bool storageSetup(bool amazonInstall);
void setSystemName();
bool singleServerDBrootSetup();
@@ -125,7 +126,7 @@ typedef struct ModuleIP_struct
std::string launchInstance(ModuleIP moduleip);
string calpontPackage1;
string columnstorePackage;
//string calpontPackage2;
//string calpontPackage3;
//string mysqlPackage;
@@ -443,7 +444,7 @@ int main(int argc, char *argv[])
{
HostConfigList::iterator pt1 = moduleconfig.hostConfigList.begin();
string PM1ipAdd = (*pt1).IPAddr;
cout << PM1ipAdd << endl;
//cout << PM1ipAdd << endl;
if ( PM1ipAdd != "127.0.0.1" && PM1ipAdd != "0.0.0.0")
{
@@ -457,7 +458,7 @@ int main(int argc, char *argv[])
if (ifa->ifa_addr->sa_family==AF_INET) {
sa = (struct sockaddr_in *) ifa->ifa_addr;
addr = inet_ntoa(sa->sin_addr);
printf("Interface: %s\tAddress: %s\n", ifa->ifa_name, addr);
//printf("Interface: %s\tAddress: %s\n", ifa->ifa_name, addr);
if ( PM1ipAdd == addr )
{
@@ -487,7 +488,6 @@ int main(int argc, char *argv[])
{}
// run my.cnf upgrade script
if ( reuseConfig == "y" )
{
@@ -2676,7 +2676,42 @@ int main(int argc, char *argv[])
/* create a thread_data_t argument array */
thread_data_t thr_data[childmodulelist.size()];
// determine package type
string EEPackageType;
if (!rootUser)
EEPackageType = "binary";
else
{
int rtnCode = system("rpm -qi mariadb-columnstore-platform > /tmp/columnstore.txt 2>&1");
if (WEXITSTATUS(rtnCode) == 0)
EEPackageType = "rpm";
else {
rtnCode = system("dpkg -s mariadb-columnstore-platform > /tmp/columnstore.txt 2>&1");
if (WEXITSTATUS(rtnCode) == 0)
EEPackageType = "deb";
else
EEPackageType = "binary";
}
}
try {
sysConfig->setConfig(InstallSection, "EEPackageType", EEPackageType);
}
catch(...)
{
cout << "ERROR: Problem setting EEPackageType from the MariaDB ColumnStore System Configuration file" << endl;
exit(1);
}
if ( !writeConfig(sysConfig) ) {
cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl;
exit(1);
}
string install = "y";
if ( IserverTypeInstall != oam::INSTALL_COMBINE_DM_UM_PM ||
pmNumber > 1 ) {
//
@@ -2719,39 +2754,6 @@ int main(int argc, char *argv[])
cout << endl;
//Write out Updated System Configuration File
string EEPackageType;
if ( rootUser )
{
try {
EEPackageType = sysConfig->getConfig(InstallSection, "EEPackageType");
}
catch(...)
{
cout << "ERROR: Problem getting EEPackageType from the MariaDB ColumnStore System Configuration file" << endl;
exit(1);
}
}
else //nonroot, default to binary
EEPackageType = "binary";
while(true) {
prompt = "Enter the Package Type being installed to other servers [rpm,deb,binary] (" + EEPackageType + ") > ";
pcommand = callReadline(prompt);
if (pcommand) {
if (strlen(pcommand) > 0) EEPackageType = pcommand;
callFree(pcommand);
}
if ( EEPackageType == "rpm" || EEPackageType == "deb" || EEPackageType == "binary" ) {
break;
}
cout << "Invalid Package Type, please re-enter" << endl;
EEPackageType = "rpm";
if ( noPrompting )
exit(1);
}
if ( EEPackageType == "rpm" )
{
cout << "Performing an MariaDB ColumnStore System install using RPM packages" << endl;
@@ -2771,75 +2773,19 @@ int main(int argc, char *argv[])
}
}
//Write out Updated System Configuration File
try {
sysConfig->setConfig(InstallSection, "EEPackageType", EEPackageType);
}
catch(...)
{
cout << "ERROR: Problem setting EEPackageType from the MariaDB ColumnStore System Configuration file" << endl;
exit(1);
}
if ( !writeConfig(sysConfig) ) {
cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl;
exit(1);
}
//check if pkgs are located in $HOME directory
string version = systemsoftware.Version + "-" + systemsoftware.Release;
if ( EEPackageType != "binary") {
string separator = "-";
calpontPackage1 = "mariadb-columnstore-*" + separator + version;
//calpontPackage2 = "mariadb-columnstore-libs" + separator + version;
//calpontPackage3 = "mariadb-columnstore-enterprise" + separator + version;
//mysqlPackage = "mariadb-columnstore-storage-engine" + separator + version;
//mysqldPackage = "mariadb-columnstore-mysql" + separator + version;
if( !pkgCheck() ) {
exit(1);
}
if ( EEPackageType == "rpm")
columnstorePackage = HOME + "/" + "mariadb-columnstore-" + version + "*.rpm.tar.gz";
else
{
//mariadb
calpontPackage1 = "mariadb-columnstore-*" + separator + version;
calpontPackage1 = HOME + "/" + calpontPackage1 + "*." + EEPackageType;
//calpontPackage2 = HOME + "/" + calpontPackage2 + "*." + EEPackageType;
//calpontPackage3 = HOME + "/" + calpontPackage3 + "*." + EEPackageType;
//mysqlPackage = HOME + "/" + mysqlPackage + "*." + EEPackageType;
//mysqldPackage = HOME + "/" + mysqldPackage + "*." + EEPackageType;
}
}
if ( EEPackageType == "deb")
columnstorePackage = HOME + "/" + "mariadb-columnstore-" + version + "*.deb.tar.gz";
else
{
// binary
//string fileName = installDir + "/bin/healthcheck";
//ifstream file (fileName.c_str());
//if (!file) // CE
calpontPackage1 = "mariadb-columnstore-" + version;
//else // EE
//calpontPackage1 = "mariadb-columnstore-ent-" + version;
//calpontPackage2 = "dummy";
//calpontPackage3 = "dummy";
//mysqlPackage = calpontPackage1;
//mysqldPackage = calpontPackage1;
columnstorePackage = HOME + "/" + "mariadb-columnstore-" + version + "*.bin.tar.gz";
if( !pkgCheck() )
if( !pkgCheck(columnstorePackage) )
exit(1);
calpontPackage1 = HOME + "/" + calpontPackage1 + "*.bin.tar.gz";
//calpontPackage2 = "dummy";
//calpontPackage3 = "dummy";
}
//If ent pkg is not there, mark it as such
//{
// glob_t gt;
// memset(&gt, 0, sizeof(gt));
// if (glob(calpontPackage3.c_str(), 0, 0, &gt) != 0)
// calpontPackage3 = "dummy.rpm";
// globfree(&gt);
//}
if ( password.empty() )
{
@@ -3050,7 +2996,7 @@ int main(int argc, char *argv[])
// checkRemoteMysqlPort(remoteModuleIP, remoteModuleName, USER, password, mysqlPort, sysConfig);
cmd = installDir + "/bin/binary_installer.sh " + remoteModuleName + " " +
remoteModuleIP + " " + password + " " + calpontPackage1 + " " + remoteModuleType +
remoteModuleIP + " " + password + " " + columnstorePackage + " " + remoteModuleType +
" initial " + binservertype + " " + mysqlPort + " " + remote_installer_debug +
" " + installDir + " " + debug_logfile;
@@ -3120,7 +3066,7 @@ int main(int argc, char *argv[])
if ( pmwithum )
binservertype = "pmwithum";
cmd = installDir + "/bin/binary_installer.sh " + remoteModuleName + " " + remoteModuleIP +
" " + password + " " + calpontPackage1 + " " + remoteModuleType + " initial " +
" " + password + " " + columnstorePackage + " " + remoteModuleType + " initial " +
binservertype + " " + mysqlPort + " " + remote_installer_debug + " " + installDir + " " +
debug_logfile;
@@ -3737,6 +3683,14 @@ bool setOSFiles(string parentOAMModuleName, int serverTypeInstall)
system(cmd.c_str());
}
//check and do the amazon credentials file
string fileName = HOME + "/.aws/credentials";
ifstream oldFile (fileName.c_str());
if (!oldFile)
return allfound;
string cmd = "cp " + fileName + " " + installDir + "/local/etc/. > /dev/null 2>&1";
system(cmd.c_str());
return allfound;
}
@@ -4033,14 +3987,14 @@ bool createDbrootDirs(string DBRootStorageType)
/*
* pkgCheck
*/
bool pkgCheck()
bool pkgCheck(string columnstorePackage)
{
while(true)
{
string cmd = "ls " + HOME + " | grep " + calpontPackage1 + " > /tmp/calpontpkgs";
string cmd = "ls " + columnstorePackage + " > /tmp/calpontpkgs";
system(cmd.c_str());
string pkg = calpontPackage1;
string pkg = columnstorePackage;
string fileName = "/tmp/calpontpkgs";
ifstream oldFile (fileName.c_str());
if (oldFile) {