database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

MCOL-3606 Make ColumnStore use generic paths

Browse Source 
ColumnStore now uses standard bin/lib paths for pretty much everything.
Data path is now hard-coded to /var/lib/columnstore.

This patch also:

* Removes v1 decompression
* Removes a bunch of unneeded files
* Removes COLUMNSTORE_INSTALL_DIR / $INSTALLDIR
* Makes my.cnf.d work for all platforms (MCOL-3558)
* Changes configcpp to use recursive mutex (fixes possible config write deadlock)
* Fixes MCOL-3599 Fix regr functions, The library was installed in the wrong location
* Fixes a bunch of Ubuntu packaging issues
* Changes the binary names of several of the executables so as not to
clash with potential executables from other packages
This commit is contained in:
Andrew Hutchings
2019-11-04 11:06:33 +00:00
parent dea1dec507
commit 8ab9ebb0f4
178 changed files with 1310 additions and 18366 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
tools/configMgt/CMakeLists.txt
View File

@ -2,19 +2,6 @@
include_directories( ${ENGINE_COMMON_INCLUDES} )
########### next target ###############
set(autoInstaller_SRCS autoInstaller.cpp)
add_executable(autoInstaller ${autoInstaller_SRCS})
target_compile_options(autoInstaller PRIVATE -Wno-unused-result)
target_link_libraries(autoInstaller ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS} readline ncurses)
install(TARGETS autoInstaller DESTINATION ${ENGINE_BINDIR})
########### next target ###############
set(autoConfigure_SRCS autoConfigure.cpp)

47
tools/configMgt/autoConfigure.cpp
View File

@ -1634,7 +1634,7 @@ int main(int argc, char* argv[])
{
int id = *pt;
string DBrootID = "DBRoot" + oam.itoa(id);;
string pathID = "/usr/local/mariadb/columnstore/data" + oam.itoa(id);
string pathID = "/var/lib/columnstore/data" + oam.itoa(id);
try
{
@ -1731,7 +1731,7 @@ int main(int argc, char* argv[])
{
int id = *pt;
string DBrootID = "DBRoot" + oam.itoa(id);;
string pathID = "/usr/local/mariadb/columnstore/data" + oam.itoa(id);
string pathID = "/var/lib/columnstore/data" + oam.itoa(id);
try
{
@ -1810,7 +1810,7 @@ int main(int argc, char* argv[])
}
string DBrootID = "DBRoot" + oam.itoa(i);
string pathID = "/usr/local/mariadb/columnstore/data" + oam.itoa(i);
string pathID = "/var/lib/columnstore/data" + oam.itoa(i);
try
{
@ -1952,47 +1952,6 @@ int main(int argc, char* argv[])
exit(-1);
}
string DataFileEnvFile;
try
{
DataFileEnvFile = sysConfigOld->getConfig(SystemSection, "DataFileEnvFile");
if ( !DataFileEnvFile.empty() )
{
try
{
sysConfigNew->setConfig(SystemSection, "DataFileEnvFile", DataFileEnvFile);
}
catch (...)
{
cout << "ERROR: Problem setting DataFileEnvFile in the Calpont System Configuration file" << endl;
exit(-1);
}
}
else
{
string::size_type pos = DataFilePlugin.find("hdfs-20.so", 0);
if (pos != string::npos)
DataFileEnvFile = "setenv-hdfs-20";
else
DataFileEnvFile = "setenv-hdfs-12";
try
{
sysConfigNew->setConfig(SystemSection, "DataFileEnvFile", DataFileEnvFile);
}
catch (...)
{
cout << "ERROR: Problem setting DataFileEnvFile in the Calpont System Configuration file" << endl;
exit(-1);
}
}
}
catch (...)
{}
string ExtentsPerSegmentFile;
try

966
tools/configMgt/autoInstaller.cpp
View File

@ -1,966 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
Copyright (C) 2016 MariaDB Corporation
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/******************************************************************************************
* $Id: autoInstaller.cpp 64 2006-10-12 22:21:51Z dhill $
*
*
* List of files being updated by configure:
* Calpont/etc" + installLocation + ".xml
*
*
******************************************************************************************/
/**
* @file
*/
#include <iostream>
#include <cstdlib>
#include <cstdio>
#include <glob.h>
#include <string>
#include <readline/readline.h>
#include <readline/history.h>
#include "config.h"
#include "liboamcpp.h"
#include "configcpp.h"
using namespace std;
using namespace oam;
using namespace config;
int main(int argc, char* argv[])
{
Oam oam;
string systemName;
string systemPackage;
string systemUser = "root";
string calpontPackage;
string mysqlRPM;
string mysqldRPM;
string installParentModuleHostName;
string installParentModuleIPAddr;
string debug_flag = "0";
string uninstall_debug = " ";
string password;
string XMpassword = "dummypw";
string configFile = "NULL";
string release = "Latest";
string installDir = "/usr/local";
string MySQLport = oam::UnassignedName;
string installPackageType = "";
string company = "InfiniDB";
string calpontPackagename = "infinidb";
string installLocation = "/Calpont";
string adminCommand = "calpontConsole";
string product = "infinidb";
char* pcommand = 0;
string prompt;
bool noprompt = false;
string CE = "0";
int forceVer = -1;
Config* sysConfig = Config::makeConfig("./systems/CalpontSystems.xml");
//gethostname to determine where to get the packages
string SHARED = "//srvhill01/shared";
char hostname[128];
gethostname(hostname, sizeof hostname);
string hostName = hostname;
for ( int i = 1; i < argc; i++ )
{
if ( string("-h") == argv[i] )
{
cout << endl;
cout << "'autoInstaller' installs the RPMs located in" << endl;
cout << SHARED << "/packages on the specified" << endl;
cout << "system. It will either install the latest rpm located in" << endl;
cout << "in the /Latest directory or the rpm associated with the" << endl;
cout << "release-number entered." << endl;
cout << endl;
cout << "The list of systems are located in '" << sysConfig->configFile() << "'" << endl;
cout << endl;
cout << "The 'Columnstore.xml' used in the install will default to system" << endl;
cout << "configuration of the 'Columnstore.xml' located on the system being" << endl;
cout << "or can be passed as an argument into 'quickInstaller'." << endl;
cout << endl;
cout << "Usage: autoInstaller -s system [-h][-ce][-r release][-c configFile][-n][-d][-p package-type][-m mysql-password][-port mysql-port][-pr product-name]" << endl;
cout << " -s system-name" << endl;
cout << " -ce community-edition install" << endl;
cout << " -r release-number (optional, default to 'Latest')" << endl;
cout << " -c Config File located in system-name directory (optional, default to system configuration)" << endl;
cout << " -n No Prompt (Used for automated Installs)" << endl;
cout << " -port System MySQL Port, if set" << endl;
cout << " -d Debug Flag" << endl;
cout << " -p Install Package Type (rpm or binary), defaults to " << sysConfig->configFile() << " setting" << endl;
cout << " -3 Force a version 3 install, defaults to autodetect" << endl;
cout << " -pr Product installing (infinidb or columnstore), defaults to columnstore" << endl;
exit(0);
}
else if ( string("-s") == argv[i] )
{
i++;
if ( argc == i )
{
cout << "ERROR: missing system argument" << endl;
exit(1);
}
systemName = argv[i];
}
else if ( string("-r") == argv[i] )
{
i++;
if ( argc == i )
{
cout << "ERROR: missing release argument" << endl;
exit(1);
}
release = argv[i];
}
else if ( string("-c") == argv[i] )
{
i++;
if ( argc == i )
{
cout << "ERROR: missing config file argument" << endl;
exit(1);
}
configFile = argv[i];
string systemConfig = "systems/" + systemName + "/" + configFile;
ifstream oldFile (systemConfig.c_str());
if (!oldFile)
{
cout << "ERROR: Config file '" + systemConfig + "' not found" << endl;
exit(1);
}
}
else if ( string("-ce") == argv[i] )
{
CE = "1";
}
else if ( string("-n") == argv[i] )
{
noprompt = true;
}
else if ( string("-d") == argv[i] )
{
debug_flag = "1";
uninstall_debug = "-d";
}
else if ( string("-port") == argv[i] )
{
i++;
if ( argc == i )
{
cout << "ERROR: missing mysql port argument" << endl;
exit(1);
}
MySQLport = argv[i];
}
else if ( string("-p") == argv[i] )
{
i++;
if ( argc == i )
{
cout << "ERROR: package type argument" << endl;
exit(1);
}
installPackageType = argv[i];
}
else if ( string("-3") == argv[i] )
{
forceVer = 3;
}
else if ( string("-pr") == argv[i] )
{
i++;
if ( argc == i )
{
cout << "ERROR: product name type argument" << endl;
exit(1);
}
product = argv[i];
}
else
{
cout << "ERROR: Unknown option: " << argv[i] << endl;
exit(1);
}
}
if (systemName.empty() )
{
cout << endl;
cout << "Missing system name" << endl;
cout << "Usage: autoInstaller -s system [-h][-ce][-r release][-c configFile][-n][-d][-p package-type][-m mysql-password][-port mysql-port]" << endl;
exit(1);
}
if ( product == "columnstore" )
{
company = "MariaDB-Columnstore";
calpontPackagename = "mariadb-columnstore";
installLocation = "/mariadb/columnstore";
adminCommand = "mcsadmin";
}
else if ( product != "infinidb" )
{
cout << endl;
cout << "Invalid product name" << endl;
exit(1);
}
// get Parent OAM Module based on system name
int systemCount;
try
{
systemCount = strtol(sysConfig->getConfig("NetworkConfig", "SystemCount").c_str(), 0, 0);
if ( systemCount == 0 )
{
cout << "ERROR: SystemCount in " << sysConfig->configFile() << " equal to 0, exiting" << endl;
cerr << "ERROR: SystemCount in " << sysConfig->configFile() << " equal to 0, exiting" << endl;
exit(1);
}
}
catch (...)
{
cout << "ERROR: Problem getting SystemCount from " << sysConfig->configFile() << ", exiting" << endl;
cerr << "ERROR: Problem getting SystemCount from " << sysConfig->configFile() << ", exiting" << endl;
exit(1);
}
bool found = false;
for ( int i = 1 ; i <= systemCount ; i++)
{
string SystemName = "SystemName" + oam.itoa(i);
string oamParentModule = "OAMParentModule" + oam.itoa(i);
string SystemPackage = "SystemPackage" + oam.itoa(i);
string SystemPassword = "SystemPassword" + oam.itoa(i);
string SystemUser = "SystemUser" + oam.itoa(i);
string tempSystem;
try
{
tempSystem = sysConfig->getConfig("NetworkConfig", SystemName );
}
catch (...)
{
cout << "ERROR: Problem getting SystemName from " << sysConfig->configFile() << ", exiting" << endl;
cerr << "ERROR: Problem getting SystemName from " << sysConfig->configFile() << ", exiting" << endl;
exit(1);
}
if ( tempSystem == systemName )
{
try
{
installParentModuleHostName = sysConfig->getConfig("NetworkConfig", oamParentModule );
password = sysConfig->getConfig("NetworkConfig", SystemPassword );
systemPackage = sysConfig->getConfig("NetworkConfig", SystemPackage );
systemUser = sysConfig->getConfig("NetworkConfig", SystemUser );
found = true;
break;
}
catch (...)
{
cout << "ERROR: Problem getting SystemName from " << sysConfig->configFile() << ", exiting" << endl;
cerr << "ERROR: Problem getting SystemName from " << sysConfig->configFile() << ", exiting" << endl;
exit(1);
}
}
}
if ( !found )
{
cout << "ERROR: System Name '" + systemName + "' not in " << sysConfig->configFile() << ", exiting" << endl;
cerr << "ERROR: System Name '" + systemName + "' not in " << sysConfig->configFile() << ", exiting" << endl;
exit(1);
}
//check if package type option is used
if ( !installPackageType.empty() )
{
if ( installPackageType == "rpm" )
systemPackage = "*.x86_64.rpm";
else if ( installPackageType == "binary" )
systemPackage = "*x86_64.bin.tar.gz";
else
{
cout << "Invalid " + company + " Package Type Arugument entered, use 'rpm' or 'binary' exiting" << endl;
cerr << "Invalid " + company + " Package Type Arugument entered, use 'rpm' or 'binary' exiting" << endl;
exit(1);
}
}
cout << endl << "*** Make sure nobody is sitting in a mounted directory, like data1 ***" << endl;
if ( !noprompt )
{
while (true)
{
cout << endl;
if ( systemPackage == "*.x86_64.rpm" )
prompt = "Are you sure you want to install the '" + release + "' " + company + " RPM on '" + systemName + "' ? (y,n,exit) > ";
else
prompt = "Are you sure you want to install the '" + release + "/packages' " + company + " Binary Package on '" + systemName + "' ? (y,n,exit) > ";
pcommand = readline(prompt.c_str());
if (!pcommand)
exit(0);
string confirm = pcommand;
free(pcommand);
pcommand = 0;
if ( confirm == "exit" )
exit(0);
if ( confirm == "y" )
break;
else
{
if ( confirm == "n" )
{
prompt = "Enter system (" + systemName + ") > ";
pcommand = readline(prompt.c_str());
if (pcommand)
{
systemName = pcommand;
free(pcommand);
pcommand = 0;
}
else
{
prompt = "Enter release (" + release + ") > ";
pcommand = readline(prompt.c_str());
if (pcommand)
{
release = pcommand;
free(pcommand);
pcommand = 0;
}
}
}
}
}
}
//get if root or no-root user install
if ( systemUser != "root" )
installDir = "/home/" + systemUser;
//remove all calpont packages from local /root/ directory, can interfere with install
if (geteuid() == 0)
system("rm -f /root/calpont-*rpm /root/infinidb-*rpm /root/mariadb-*rpm ");
string systemDir = "systems/" + systemName + "/";
string cmd = "mkdir " + systemDir + " > /dev/null 2>&1";
system(cmd.c_str());
//delete any packages and xml files
cmd = "rm -f " + systemDir + "* > /dev/null 2>&1";
system(cmd.c_str());
int idbver = -1;
string currentPrefix;
string mysqlRPMname;
if ( systemPackage == "*.x86_64.rpm" )
{
//get rpm
string smbCmd = "cd " + systemDir + ";smbclient " + SHARED + " -WMARIADB -U root%Calpont1 -c 'cd packages/" + release + "/;prompt OFF;mget " + systemPackage + "' > /dev/null 2>&1";
system(smbCmd.c_str());
cout << smbCmd << endl;
if (forceVer > 0)
{
idbver = forceVer;
}
else
{
//try to guess the release, v4+ takes precedence
string sentinel = systemDir + "" + calpontPackagename + "-libs-*.rpm";
glob_t gt;
memset(&gt, 0, sizeof(gt));
idbver = 3;
if (glob(sentinel.c_str(), 0, 0, &gt) == 0)
idbver = 4;
globfree(&gt);
}
mysqlRPMname = calpontPackagename + "-storage-engine";
//check if package is there
cmd = "ls " + systemDir + mysqlRPMname + "* > /tmp/package.txt 2>&1";
int rtnCode = system(cmd.c_str());
if (rtnCode != 0)
{
{
cout << endl << "FAILED: " + company + " Package(s) not found in " << release << " , exiting" << endl;
cerr << endl << "FAILED: " + company + " Package(s) not found in " << release << " , exiting" << endl;
exit(1);
}
}
ifstream file ("/tmp/package.txt");
char line[400];
string buf;
while (file.getline(line, 400))
{
buf = line;
string::size_type pos;
string package = calpontPackagename + "-storage-engine-";
pos = buf.find(package, 0);
if (pos != string::npos)
{
currentPrefix = buf.substr(pos + 24, 1);
break;
}
}
file.close();
calpontPackage = calpontPackagename + "-" + currentPrefix + systemPackage;
cout << endl << "Using the " + company + " Packages '" + systemPackage + "' from " + SHARED + "/packages/" + release + "/" << endl;
}
else //binary package
{
//this will pickup more than we want...
string smbCmd = "cd " + systemDir + ";smbclient " + SHARED + " -WMARIADB -U root%Calpont1 -c 'cd packages/" + release + "/;prompt OFF;mget *.x86_64.bin.tar.gz' > /dev/null 2>&1";
system(smbCmd.c_str());
//TODO: consolodate this with above test...
if (forceVer > 0)
{
idbver = forceVer;
}
else
{
//try to guess the release, v4+ takes precedence
string sentinel = systemDir + "" + calpontPackagename + "-ent-*.tar.gz";
glob_t gt;
memset(&gt, 0, sizeof(gt));
idbver = 3;
if (glob(sentinel.c_str(), 0, 0, &gt) == 0)
idbver = 4;
globfree(&gt);
}
currentPrefix = oam.itoa(idbver);
mysqlRPMname = calpontPackagename + "-storage-engine";
calpontPackage = "" + calpontPackagename + "-ent-" + systemPackage;
//check if package is there
cmd = "ls " + systemDir + calpontPackage + " > /tmp/package.txt 2>&1";
int rtnCode = system(cmd.c_str());
if (rtnCode != 0)
{
{
cout << endl << "FAILED: " + company + " binary package not found in " << release << ", exiting" << endl;
cerr << endl << "FAILED: " + company + " binary package not found in " << release << ", exiting" << endl;
exit(1);
}
}
ifstream file ("/tmp/package.txt");
char line[400];
string buf;
while (file.getline(line, 400))
{
buf = line;
string::size_type pos;
string package = calpontPackagename + "-ent-";
pos = buf.find(package, 0);
if (pos != string::npos)
{
currentPrefix = buf.substr(pos + 13, 1);
break;
}
}
file.close();
cout << endl << "Using the " + company + " Package '" + systemPackage + "' from " + SHARED + "/Iterations/" + release + "/packages" << endl;
}
mysqlRPM = mysqlRPMname + "-" + systemPackage;
mysqldRPM = calpontPackagename + "-mysql-" + systemPackage;
//TODO: we go to all the effort of downloading the packages above only to delete them here...
cmd = "rm -f " + systemDir + systemPackage + " > /dev/null 2>&1";
system(cmd.c_str());
//get Network IP Address
installParentModuleIPAddr = oam.getIPAddress( installParentModuleHostName);
if ( installParentModuleIPAddr.empty() || installParentModuleIPAddr == "127.0.0.1" )
{
cout << "Invalid Host Name (No DNS IP found), exiting..." << endl;
cerr << "Invalid Host Name (No DNS IP found), exiting..." << endl;
exit(1);
}
//check to see if system is up
string cmdLine = "ping ";
string cmdOption = " -w 5 >> /dev/null";
cmd = cmdLine + installParentModuleIPAddr + cmdOption;
int rtnCode = system(cmd.c_str());
if ( rtnCode != 0 )
{
cout << " FAILED: failed ping test command installParentModuleIPAddr: " << installParentModuleIPAddr << ", exiting" << endl;
cerr << " FAILED: failed ping test command installParentModuleIPAddr: " << installParentModuleIPAddr << ", exiting" << endl;
exit(1);
}
cout << endl;
#if 0
cout << "idbver = " << idbver << endl;
cout << "calpontPackagename = " << calpontPackagename << endl;
cout << "mysqlRPMname = " << mysqlRPMname << endl;
cout << "calpontPackage = " << calpontPackage << endl;
cout << "systemPackage = " << systemPackage << endl;
exit(0);
#endif
//get release and system Columnstore.xml and generate a new Columnstore.xml file for installation
if ( configFile == "NULL" )
{
cout << "Get Release Columnstore.xml " << flush;
// get release Columnstore.xml
string cmd = "cd " + systemDir + ";smbclient " + SHARED + " -WMARIADB -U root%Calpont1 -c 'cd packages/" + release + "/;prompt OFF;mget Columnstore.xml' > /dev/null 2>&1";
rtnCode = system(cmd.c_str());
if (rtnCode != 0)
{
cout << "FAILED: no Release Columnstore.xml found for release '" + release + "', exiting" << endl;
cerr << "FAILED: no Release Columnstore.xml found for release '" + release + "', exiting" << endl;
exit(1);
}
else
{
cmd = "cd " + systemDir + ";mv -f Columnstore.xml Columnstore.xml.new > /dev/null 2>&1";
rtnCode = system(cmd.c_str());
if (rtnCode != 0)
{
cout << "FAILED: no Release Columnstore.xml found for release '" + release + "', exiting" << endl;
cerr << "FAILED: no Release Columnstore.xml found for release '" + release + "', exiting" << endl;
exit(1);
}
else
{
cout << "DONE" << endl;
//get system Columnstore.xml
cout << "Get System Columnstore.xml " << flush;
for ( int retry = 0 ; retry < 5 ; retry++ )
{
cmd = "./remote_scp_get.sh " + installParentModuleIPAddr + " " + password + " " + std::string(MCSSYSCONFDIR) + "/columnstore/Columnstore.xml " + systemUser + " " + debug_flag;
rtnCode = system(cmd.c_str());
sleep(2);
if (rtnCode == 0)
{
cmd = "mv Columnstore.xml " + systemDir + "/.";
rtnCode = system(cmd.c_str());
if ( rtnCode == 0 )
{
// Columnstore.xml found
//try to parse it
//Config* sysConfigOld;
ofstream file("/dev/null");
//save cout stream buffer
streambuf* strm_buffer = cerr.rdbuf();
try
{
// redirect cout to /dev/null
cerr.rdbuf(file.rdbuf());
//sysConfigOld = Config::makeConfig( systemDir + "/Columnstore.xml");
// restore cout stream buffer
cerr.rdbuf (strm_buffer);
//update release Columnstore.xml with System Configuration info from system Columnstore.xml
cout << "Run Columnstore.xml autoConfigure " << flush;
cmd = "cd " + systemDir + ";../../autoConfigure";
rtnCode = system(cmd.c_str());
if (rtnCode != 0)
{
cout << " FAILED, try Columnstore.xml.rpmsave" << endl;
goto RPMSAVE;
}
else
{
cout << "DONE" << endl;
configFile = "Columnstore.xml.new";
goto CONFIGDONE;
}
}
catch (...)
{
// cout << " FAILED to parse, try re-reading again" << endl;
// restore cout stream buffer
cerr.rdbuf (strm_buffer);
continue;
}
}
}
}
RPMSAVE:
//try Columnstore.xml.rpmsave
cout << "Get System Columnstore.xml.rpmsave " << flush;
cmd = "./remote_scp_get.sh " + installParentModuleIPAddr + " " + password + " " + std::string(MCSSYSCONFDIR) + "/columnstore/Columnstore.xml.rpmsave " + systemUser + " " + debug_flag;
rtnCode = system(cmd.c_str());
if (rtnCode == 0)
{
cmd = "mv Columnstore.xml.rpmsave " + systemDir + "Columnstore.xml";
rtnCode = system(cmd.c_str());
if ( rtnCode != 0 )
{
cout << "ERROR: No system Columnstore.xml or Columnstore.xml.rpmsave found, exiting" << endl;
cerr << "ERROR: No system Columnstore.xml or Columnstore.xml.rpmsave found, exiting" << endl;
exit(1);
}
}
//update release Columnstore.xml with System Configuration info from system Columnstore.xml.rpmsave
cout << "Run Columnstore.xml autoConfigure " << flush;
cmd = "cd " + systemDir + ";../../autoConfigure";
rtnCode = system(cmd.c_str());
if (rtnCode != 0)
{
cout << " FAILED, exiting..." << endl;
cerr << " FAILED, exiting..." << endl;
exit(1);
}
else
{
cout << "DONE" << endl;
configFile = "Columnstore.xml.new";
}
}
}
}
CONFIGDONE:
Config* sysConfigOld;
try
{
sysConfigOld = Config::makeConfig( systemDir + "/Columnstore.xml");
}
catch (...)
{
cout << "ERROR: Problem reading Columnstore.xml files, exiting" << endl;
cerr << "ERROR: Problem reading Columnstore.xml files, exiting" << endl;
exit(1);
}
//get Parent OAM Module Name of load being installed
string parentOAMModuleModuleName;
string parentOAMModuleIPAddr;
try
{
parentOAMModuleModuleName = sysConfigOld->getConfig("SystemConfig", "ParentOAMModuleName");
parentOAMModuleIPAddr = sysConfigOld->getConfig("ProcStatusControl", "IPAddr");
}
catch (...)
{
cout << "ERROR: Problem updating the " + company + " System Configuration file, exiting" << endl;
cerr << "ERROR: Problem updating the " + company + " System Configuration file, exiting" << endl;
exit(1);
}
if ( parentOAMModuleModuleName.empty() || parentOAMModuleIPAddr.empty() ||
parentOAMModuleIPAddr == "127.0.0.1" ||
parentOAMModuleIPAddr == "0.0.0.0" )
{
cout << " ERROR: ParentOAMModuleModuleName or ParentOAMModuleIPAddr are invalid in " << sysConfigOld->configFile() << ", exiting" << endl;
cerr << " ERROR: ParentOAMModuleModuleName or ParentOAMModuleIPAddr are invalid in " << sysConfigOld->configFile() << ", exiting" << endl;
exit(1);
}
int serverTypeInstall;
try
{
serverTypeInstall = atoi(sysConfigOld->getConfig("Installation", "ServerTypeInstall").c_str());
}
catch (...)
{
cout << "ERROR: Problem reading serverTypeInstall from the " + company + " System Configuration file, exiting" << endl;
cerr << "ERROR: Problem reading serverTypeInstall from the " + company + " System Configuration file, exiting" << endl;
exit(1);
}
bool HDFS = false;
string DBRootStorageType;
try
{
DBRootStorageType = sysConfigOld->getConfig("Installation", "DBRootStorageType");
}
catch (...)
{
cout << "ERROR: Problem reading DBRootStorageType from the " + company + " System Configuration file, exiting" << endl;
cerr << "ERROR: Problem reading DBRootStorageType from the " + company + " System Configuration file, exiting" << endl;
exit(1);
}
string DataFileEnvFile = "setenv-hdfs-20";
if ( DBRootStorageType == "hdfs" )
{
HDFS = true;
try
{
DataFileEnvFile = sysConfigOld->getConfig("Installation", "DataFileEnvFile");
}
catch (...)
{
cout << "ERROR: Problem reading DataFileEnvFile from the " + company + " System Configuration file, exiting" << endl;
cerr << "ERROR: Problem reading DataFileEnvFile from the " + company + " System Configuration file, exiting" << endl;
exit(1);
}
}
if ( serverTypeInstall != oam::INSTALL_COMBINE_DM_UM_PM && CE == "1")
{
cout << "Community-Edition only work on single-server systems, exiting" << endl;
cerr << "Community-Edition only work on single-server systems, exiting" << endl;
exit (0);
}
string::size_type pos = parentOAMModuleIPAddr.find("208", 0);
if ( pos != string::npos)
{
string newIP = parentOAMModuleIPAddr.substr(0, pos) + "8" + parentOAMModuleIPAddr.substr(pos + 3, 80);
parentOAMModuleIPAddr = newIP;
}
if ( parentOAMModuleIPAddr == "0.0.0.0" )
{
cout << "System Columnstore.xml is un-configured, exiting" << endl;
cerr << "System Columnstore.xml is un-configured, exiting" << endl;
exit (0);
}
//set Version Key Flags
Config* sysConfigNew;
try
{
sysConfigNew = Config::makeConfig( systemDir + "/" + configFile);
}
catch (...)
{
cout << "ERROR: Problem reading Columnstore.xml files, exiting" << endl;
cerr << "ERROR: Problem reading Columnstore.xml files, exiting" << endl;
exit(1);
}
try
{
sysConfigNew->setConfig("SystemConfig", "Flags", "75635259");
sysConfigNew->write();
}
catch (...)
{}
//set EE package Type to 'binary', if needed
if ( systemPackage == "*.x86_64.rpm" )
{
try
{
sysConfigNew->setConfig("Installation", "EEPackageType", "rpm");
sysConfigNew->write();
}
catch (...)
{}
}
else
{
try
{
sysConfigNew->setConfig("Installation", "EEPackageType", "binary");
sysConfigNew->write();
}
catch (...)
{}
}
cout << "Shutdown System " << flush;
cmd = "./remote_command.sh " + installParentModuleIPAddr + " " + systemUser + " " + password + " '" + installDir + "" + installLocation + "/bin/" + adminCommand + " shutdownsystem Force y' 'Successful shutdown' Error 60 " + debug_flag;
rtnCode = system(cmd.c_str());
if (rtnCode == 0)
cout << "DONE" << endl;
// Copy the Packages and Configuration files down to Parent OAM Module
string installer;
if ( systemPackage != "*.x86_64.rpm" )
{
//do binary install
installer = "parent_binary_installer.sh";
cmd = "cd " + systemDir + ";../../" + installer + " " + installParentModuleIPAddr + " " +
password + " " + systemPackage + " " + release + " " + configFile + " " + systemUser + " " +
installDir + " " + calpontPackagename + " " + installLocation + " " + debug_flag;
}
else
{
if ( serverTypeInstall == oam::INSTALL_COMBINE_DM_UM_PM )
{
installer = "dm_parent_installer.sh";
cmd = "cd " + systemDir + ";../../" + installer + " " + installParentModuleIPAddr + " " +
password + " " + systemPackage + " " + release + " " + configFile + " " + systemUser + " " + CE + " " + calpontPackagename + " " + installLocation + " " + debug_flag;
}
else
{
installer = "pm_parent_installer.sh";
cmd = "cd " + systemDir + ";../../" + installer + " " + installParentModuleIPAddr + " " +
password + " " + systemPackage + " " + release + " " + configFile + " " + currentPrefix + " " +
systemUser + " " + calpontPackagename + " " + installLocation + " " + debug_flag;
}
}
rtnCode = system(cmd.c_str());
if (rtnCode != 0)
{
cout << endl << "Installation Failed, exiting" << endl;
cerr << endl << "Installation Failed, exiting" << endl;
exit(1);
}
//
// Perform System Installing and launch
//
cout << "Install System " << flush;
if (HDFS)
{
if ( MySQLport == oam::UnassignedName )
{
cmd = "./remote_command.sh " + installParentModuleIPAddr + " " + systemUser + " " + password + " '. " + installDir + "" + installLocation + "/bin/" + DataFileEnvFile + ";" + installDir + "" + installLocation + "/bin/postConfigure -i " + installDir + "" + installLocation + " -u -p " + password + "' 'System is Active' Error 1200 " + debug_flag;
}
else
{
cmd = "./remote_command.sh " + installParentModuleIPAddr + " " + systemUser + " " + password + " '. " + installDir + "" + installLocation + "/bin/" + DataFileEnvFile + ";" + installDir + "" + installLocation + "/bin/postConfigure -i " + installDir + "" + installLocation + " -u -p " + password + " -port " + MySQLport + "' 'System is Active' Error 1200 " + debug_flag;
}
}
else
{
if ( MySQLport == oam::UnassignedName )
{
cmd = "./remote_command.sh " + installParentModuleIPAddr + " " + systemUser + " " + password + " '" + installDir + "" + installLocation + "/bin/postConfigure -i " + installDir + "" + installLocation + " -u -p " + password + "' 'System is Active' Error 1200 " + debug_flag;
}
else
{
cmd = "./remote_command.sh " + installParentModuleIPAddr + " " + systemUser + " " + password + " '" + installDir + "" + installLocation + "/bin/postConfigure -i " + installDir + "" + installLocation + " -u -p " + password + " -port " + MySQLport + "' 'System is Active' Error 1200 " + debug_flag;
string DataFileEnvFile = "setenv-hdfs-20";
}
}
rtnCode = system(cmd.c_str());
if (rtnCode != 0)
{
cout << "Installation Failed, exiting" << endl;
cerr << "Installation Failed, exiting" << endl;
exit(1);
}
else
cout << "DONE" << endl;
//needs to be cerr so buildTester will see it..
cout << endl << "Install Successfully completed" << endl;
cerr << endl << "Install Successfully completed" << endl;
}
// vim:ts=4 sw=4:

47
tools/configMgt/buildDatdupPackages.sh
View File

@ -1,47 +0,0 @@
#! /bin/sh
#
# $1 - release number or 'Latest' (default 'Latest')
if [ "$1" = "" ] ; then
DIR=Latest
REL="3.6-0"
elif [ "$1" = "Latest" ] ; then
DIR=Latest
REL="3.6-0"
elif [ "$2" = "" ] ; then
DIR=$1
REL=$1
else
DIR=$1
REL=$2
fi
#
if [[ $REL != *-* ]] ; then
REL=$REL"-0"
fi
#
{
cd /root/autoOAM/
rpm -e infinidb-libs infinidb-platform infinidb-enterprise --nodeps --allmatches
rpm -e infinidb-storage-engine infinidb-mysql --nodeps --allmatches
rpm -e infinidb-mysql --nodeps --allmatches
rpm -e infinidb-datdup --nodeps --allmatches
rm -rf /usr/local/Calpont
rm -rf infinidb*
rm -f *gz
#
smbclient //srvhill01/shared -WMARIADB -Uroot%Calpont1 -c "cd packages/$DIR/packages;prompt OFF;mget infinidb-datdup*.rpm"
#
test -f infinidb-datdup*.rpm || echo "infinidb-datdup rpm"
test -f infinidb-datdup*.rpm || exit -1
#
rpm -ivh infinidb-datdup*.x86_64.rpm --nodeps
cd /usr/local/
tar -zcvf infinidb-datdup-$REL.x86_64.bin.tar.gz Calpont
mv infinidb-datdup-$REL.x86_64.bin.tar.gz /root/autoOAM/
cd /root/autoOAM/
smbclient //srvhill01/shared -WMARIADB -Uroot%Calpont1 -c "cd packages/$DIR/packages;prompt OFF;del infinidb-datdup*gz;mput *gz"
} > /root/autoOAM/buildDatdupPackages-$DIR.log 2>&1
#
echo "DatDup Package Build Successfully Completed"
exit 0

2
tools/configMgt/configure.cpp
View File

@ -583,7 +583,7 @@ int main(int argc, char* argv[])
}
string DBrootID = "DBRoot" + oam.itoa(i);
string pathID = "/usr/local/mariadb/columnstore/data" + oam.itoa(i);
string pathID = "/var/lib/columnstore/data" + oam.itoa(i);
try
{

2
tools/dbbuilder/dbbuilder.cpp
View File

@ -287,7 +287,7 @@ int main(int argc, char* argv[])
cerr << cmd << endl;
#endif
cmd = startup::StartUp::installDir() + "/bin/save_brm";
cmd = "save_brm";
if (canWrite)
{

6
tools/getConfig/CMakeLists.txt
View File

@ -6,9 +6,9 @@ include_directories( ${ENGINE_COMMON_INCLUDES} )
set(getConfig_SRCS main.cpp)
add_executable(getConfig ${getConfig_SRCS})
add_executable(mcsGetConfig ${getConfig_SRCS})
target_link_libraries(getConfig ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS})
target_link_libraries(mcsGetConfig ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS})
install(TARGETS getConfig DESTINATION ${ENGINE_BINDIR} COMPONENT platform)
install(TARGETS mcsGetConfig DESTINATION ${ENGINE_BINDIR} COMPONENT platform)

6
tools/getConfig/main.cpp
View File

@ -95,7 +95,7 @@ int main(int argc, char** argv)
try
{
openlog("getConfig", LOG_CONS | LOG_PID | LOG_NDELAY, LOG_USER);
openlog("mcsGetConfig", LOG_CONS | LOG_PID | LOG_NDELAY, LOG_USER);
Config* cf;
if (configFile.length() > 0)
@ -154,12 +154,12 @@ int main(int argc, char** argv)
}
catch (exception& e)
{
syslog(LOG_ERR, "Exception in getConfig for %s %s : %s", argv[optind + 0], argv[optind + 1], e.what());
syslog(LOG_ERR, "Exception in mcsGetConfig for %s %s : %s", argv[optind + 0], argv[optind + 1], e.what());
closelog();
}
catch (...)
{
syslog(LOG_ERR, "Exception in getConfig for %s %s : Unknown exception", argv[optind + 0], argv[optind + 1]);
syslog(LOG_ERR, "Exception in mcsGetConfig for %s %s : Unknown exception", argv[optind + 0], argv[optind + 1]);
closelog();
}

832
tools/hdfsCheck/hdfsCheck.cpp
View File

@ -1,832 +0,0 @@
/*
* foptest.cc
*
* Created on: Jul 17, 2012
* Author: rtw
*/
#include <unistd.h>
#include <stdio.h>
#ifndef _MSC_VER
#include <stdio_ext.h>
#endif
#include <stdlib.h>
#include <iostream>
#include <iomanip>
#include <sstream>
#include <fcntl.h>
#include <sys/time.h>
#include <sys/types.h>
#include <syslog.h>
#include <boost/scoped_ptr.hpp>
#include <boost/thread.hpp>
#include "IDBDataFile.h"
#include "IDBFileSystem.h"
#include "IDBPolicy.h"
using namespace idbdatafile;
using namespace std;
#include "installdir.h"
string tmpDir;
size_t BLK_SIZE = 2048;
const size_t MAX_BLK_SIZE = 1048576;
size_t MAX_BLOCK = 1024;
int timeval_subtract (struct timeval* result, struct timeval* x, struct timeval* y)
{
/* Perform the carry for the later subtraction by updating y. */
if (x->tv_usec < y->tv_usec)
{
int nsec = (y->tv_usec - x->tv_usec) / 1000000 + 1;
y->tv_usec -= 1000000 * nsec;
y->tv_sec += nsec;
}
if (x->tv_usec - y->tv_usec > 1000000)
{
int nsec = (y->tv_usec - x->tv_usec) / 1000000;
y->tv_usec += 1000000 * nsec;
y->tv_sec -= nsec;
}
/* Compute the time remaining to wait.
tv_usec is certainly positive. */
result->tv_sec = x->tv_sec - y->tv_sec;
result->tv_usec = x->tv_usec - y->tv_usec;
/* Return 1 if result is negative. */
return x->tv_sec < y->tv_sec;
}
void usage()
{
cerr << "usage: hdfsCheck [-b:n:t:r:] <hdfs-plugin>" << endl;
cerr << " Required Arguments" << endl;
cerr << " <hdfs-plugin> - path to the HDFS plugin to be loaded" << endl;
cerr << " Options" << endl;
cerr << " -b <BLK_SIZE> block size in bytes" << endl;
cerr << " -n <num blocks> number of blocks to write/read" << endl;
cerr << " -t <timeout> how long to retry (in secs.) until we give up (default=30)" << endl;
cerr << " -r <dbroots> number of dbroot directories to check (default=0)" << endl;
}
struct foptest_opts
{
string pluginFile;
size_t blockSize;
int numBlocks;
int timeout;
int numDbRoots;
foptest_opts() :
pluginFile(""),
blockSize(BLK_SIZE),
numBlocks(64),
timeout(30),
numDbRoots(0)
{;}
};
class TestRunner
{
public:
TestRunner(int id, const foptest_opts& opts);
~TestRunner();
bool runTest( IDBDataFile::Types filetype, unsigned open_opts );
const foptest_opts& runOpts() const
{
return m_opts;
}
private:
bool fillDefault();
bool writeBlock(unsigned int blocknum, unsigned char* buf, unsigned char tag);
bool writeBlocks(unsigned int blocknum, unsigned char* buf, unsigned char tag, unsigned int count);
bool readBlock(unsigned int blocknum, unsigned char* buf, unsigned char tag);
bool readBlocks(unsigned int blocknum, unsigned char* buf, unsigned char tag, unsigned int count);
bool readTest( IDBDataFile::Types filetype );
bool openByModeStrTest();
bool writeTest( IDBDataFile::Types filetype );
bool rdwrTest( IDBDataFile::Types filetype );
bool hdfsRdwrExhaustTest();
bool openWF( bool reopen = false );
bool openRF();
void reset();
enum LogLevel
{
INFO,
ERROR
};
void logMsg( LogLevel level, const string& msg );
string m_fname;
foptest_opts m_opts;
unsigned char m_defbuf[MAX_BLK_SIZE];
IDBDataFile* m_file;
unsigned m_open_opts;
int m_id;
};
TestRunner::TestRunner(int id, const foptest_opts& opts) :
m_opts(opts),
m_file(NULL),
m_open_opts(0),
m_id(id)
{
for ( unsigned i = 0; i < BLK_SIZE; i++ )
{
m_defbuf[i] = 0xfe;
}
}
void TestRunner::reset()
{
delete m_file;
m_file = 0;
}
struct TestCleanup
{
TestCleanup( IDBDataFile::Types type, const string& dir, const string& file ) :
m_fs( IDBFileSystem::getFs( type ) ),
m_dir(dir),
mm_file(file) {}
~TestCleanup()
{
if ( m_fs.exists( mm_file.c_str() ) )
m_fs.remove(mm_file.c_str());
assert( !m_fs.exists( mm_file.c_str() ) );
if ( m_fs.exists( m_dir.c_str() ) )
m_fs.remove(m_dir.c_str());
assert( !m_fs.exists( m_dir.c_str() ) );
}
IDBFileSystem& m_fs;
string m_dir;
string mm_file;
};
bool TestRunner::runTest( IDBDataFile::Types filetype, unsigned open_opts )
{
m_open_opts = open_opts;
ostringstream infostr;
infostr << "Running test for file type "
<< ( filetype == IDBDataFile::UNBUFFERED ? "Unbuffered" :
( filetype == IDBDataFile::BUFFERED ? "Buffered" : "HDFS" ) );
logMsg( INFO, infostr.str() );
IDBFileSystem& fs = IDBFileSystem::getFs( filetype );
// build the file name we are going to use
ostringstream oss;
// embed pid so that this is a new directory path
oss << tmpDir << "/idbdf-dir-" << getpid() << "-" << m_id;
string dir = oss.str();
m_fname = dir + "/foobar";
// instantiate this here so that we always clean up files we created no matter
// where we exit the function from
boost::scoped_ptr<TestCleanup> cleanup( new TestCleanup( filetype, dir, m_fname ) );
logMsg( INFO, "Running writeTest" );
bool returnval = true;
returnval = writeTest( filetype );
if ( !returnval )
{
return false;
}
// going to check the size two different ways - once through the file and
// once through the file system.
unsigned fsize = m_file->size();
if ( fsize != m_opts.numBlocks * BLK_SIZE )
{
ostringstream errstr;
errstr << "bad file size from file " << fsize << " != " << m_opts.numBlocks* BLK_SIZE << "!";
logMsg( ERROR, errstr.str() );
return false;
}
// this deletes and closes the file - required to get accurate file size with
// buffered IO and in HDFS from the file system
reset();
fsize = fs.size(m_fname.c_str());
if ( fsize != m_opts.numBlocks * BLK_SIZE )
{
ostringstream errstr;
errstr << "bad file size from fs " << fsize << " != " << m_opts.numBlocks* BLK_SIZE << "!";
logMsg( ERROR, errstr.str() );
return false;
}
if ( returnval )
{
logMsg( INFO, "Running readTest" );
returnval = readTest( filetype );
}
if ( returnval )
{
logMsg( INFO, "Running rdwrTest" );
returnval = rdwrTest( filetype );
}
if ( returnval && filetype == IDBDataFile::HDFS )
{
logMsg( INFO, "Running hdfsRdwrExhaustTest" );
returnval = hdfsRdwrExhaustTest();
}
if ( m_opts.numDbRoots > 0 )
{
logMsg( INFO, "Checking dbroots" );
for ( int i = 0; i < m_opts.numDbRoots; ++i )
{
ostringstream dbroot;
dbroot << "/usr/local/mariadb/columnstore/data" << i + 1;
if ( !fs.exists(dbroot.str().c_str()) )
{
ostringstream msg;
msg << "Could not locate dbroot directory " << dbroot.str();
logMsg( ERROR, msg.str() );
returnval = false;
}
}
}
list<string> dircontents;
assert( fs.listDirectory( dir.c_str(), dircontents ) == 0 );
ostringstream ldstr;
ldstr << "Listed directory " << dir << ":";
list<string>::iterator iend = dircontents.end();
for ( list<string>::iterator i = dircontents.begin(); i != iend; ++i )
{
ldstr << (*i) << ",";
}
logMsg( INFO, ldstr.str() );
assert( dircontents.size() == 1 );
// now check a bogus path and make sure it returns -1
assert( fs.listDirectory( "/this-is-a-bogus-directory", dircontents ) == -1 );
assert( fs.remove( "/this-is-a-bogus-directory" ) == 0 );
assert( !fs.isDir( "/this-is-a-bogus-directory" ));
assert( !fs.isDir( m_fname.c_str() ));
if ( returnval )
{
logMsg( INFO, "All tests passed!" );
}
reset();
return returnval;
}
bool TestRunner::readTest( IDBDataFile::Types filetype )
{
reset();
m_file = IDBDataFile::open(filetype, m_fname.c_str(), "r", m_open_opts);
if ( !m_file )
{
ostringstream errstr;
errstr << "Unable to open " << m_fname << " for reading";
logMsg( ERROR, errstr.str() );
return false;
}
// check the mtime method.
time_t now = time(0);
time_t mtime = m_file->mtime();
assert( (now - mtime) <= 3 );
struct timeval starttime, endtime, timediff;
gettimeofday(&starttime, 0x0);
for ( int i = 0; i < m_opts.numBlocks; ++i)
{
if ( !readBlock(i, m_defbuf, i) )
return false;
}
gettimeofday(&endtime, 0x0);
timeval_subtract(&timediff, &endtime, &starttime);
float secs = timediff.tv_sec + (timediff.tv_usec * 0.000001);
ostringstream infostr;
infostr << "Read " << m_opts.numBlocks* BLK_SIZE << " bytes in " << secs << " secs, ";
infostr << "Throughput = " << setprecision(3) << ((m_opts.numBlocks * BLK_SIZE) / 1000000.0) / secs << "MB/sec";
logMsg( INFO, infostr.str() );
return true;
}
bool TestRunner::openByModeStrTest()
{
// in this test we want to check the alternate open modes available for buffered i/o
// this test is only run if we are doing buffered I/O and expects it is run after the
// write test to guarantee the file is there
reset();
m_file = IDBDataFile::open(IDBDataFile::BUFFERED, m_fname.c_str(), "r+b", m_open_opts);
if ( !m_file )
{
ostringstream errstr;
errstr << "Unable to open " << m_fname << " for read/write";
logMsg( ERROR, errstr.str() );
return false;
}
// keep this fairly simple - read a block then write a block
size_t readct = m_file->read(m_defbuf, BLK_SIZE);
if ( readct != BLK_SIZE )
{
ostringstream errstr;
errstr << "Only read " << readct << " bytes, expected 4";
logMsg( ERROR, errstr.str() );
return false;
}
if ( m_defbuf[0] != (unsigned char) 0 )
{
ostringstream errstr;
errstr << "Data error - expected " << 0 << ", read " << (int) m_defbuf[0];
logMsg( ERROR, errstr.str() );
return false;
}
// we should be at block 1
long filepos = m_file->tell();
if ( filepos != long(BLK_SIZE) )
{
ostringstream errstr;
errstr << "File position not at correct block, " << filepos << " != " << BLK_SIZE;
logMsg( ERROR, errstr.str() );
return false;
}
m_defbuf[0] = 1;
size_t bytes_written = m_file->write(m_defbuf, BLK_SIZE);
if ( bytes_written != BLK_SIZE )
{
ostringstream errstr;
errstr << "Only wrote " << bytes_written << " bytes, expected 4";
logMsg( ERROR, errstr.str() );
return false;
}
return true;
}
bool TestRunner::writeTest( IDBDataFile::Types filetype )
{
reset();
m_file = IDBDataFile::open(filetype, m_fname.c_str(), "w", m_open_opts);
if ( !m_file )
{
ostringstream errstr;
errstr << "Unable to open " << m_fname << " for writing";
logMsg( ERROR, errstr.str() );
return false;
}
struct timeval starttime, endtime, timediff;
gettimeofday(&starttime, 0x0);
for ( int i = 0; i < m_opts.numBlocks; ++i)
{
if ( !writeBlock(i, m_defbuf, i) )
return false;
}
gettimeofday(&endtime, 0x0);
timeval_subtract(&timediff, &endtime, &starttime);
float secs = timediff.tv_sec + (timediff.tv_usec * 0.000001);
ostringstream infostr;
infostr << "Wrote " << m_opts.numBlocks* BLK_SIZE << " bytes in " << secs << " secs, ";
infostr << "Throughput = " << setprecision(3) << ((m_opts.numBlocks * BLK_SIZE) / 1000000.0) / secs << "MB/sec";
logMsg( INFO, infostr.str() );
return true;
}
bool TestRunner::rdwrTest( IDBDataFile::Types filetype )
{
reset();
m_file = IDBDataFile::open(filetype, m_fname.c_str(), "r+", m_open_opts);
if ( !m_file )
{
ostringstream errstr;
errstr << "Unable to open " << m_fname << " for reading";
logMsg( ERROR, errstr.str() );
return false;
}
struct drand48_data d48data;
srand48_r(0xdeadbeef, &d48data);
// we will write to 5 random blocks and then come back and
// verify the contents and undo them
int blocks_to_touch = min( 5, m_opts.numBlocks);
vector<int> touched;
for ( int i = 0; i < blocks_to_touch; ++i )
{
long int blk_num;
// we need to make sure all the blocks we touch are unique or
// the pattern used by this test won't work
bool found = false;
while (!found)
{
lrand48_r( &d48data, &blk_num);
blk_num = blk_num % m_opts.numBlocks;
vector<int>::iterator pos = find( touched.begin(), touched.end(), blk_num );
if ( pos == touched.end())
found = true;
}
if ( m_file->seek(blk_num * BLK_SIZE, SEEK_SET) )
{
ostringstream errstr;
errstr << "failed to seek block " << blk_num;
logMsg( ERROR, errstr.str() );
return false;
}
unsigned char writeval = 0xb0;
size_t writect = m_file->write(&writeval, 1);
if ( writect != 1 )
{
ostringstream errstr;
errstr << "Only wrote " << writect << " bytes, expected 1";
logMsg( ERROR, errstr.str() );
return false;
}
touched.push_back(blk_num);
}
m_file->flush();
for ( int i = 0; i < (int) touched.size(); ++i )
{
unsigned char readbuf;
size_t readct = m_file->pread(&readbuf, touched[i] * BLK_SIZE, 1);
if ( readct != 1 || readbuf != (unsigned char) 0xb0 )
{
ostringstream errstr;
errstr << "Error reading expected value, readct=" << readct << " bytes, value" << (int) readbuf;
logMsg( ERROR, errstr.str() );
return false;
}
readbuf = touched[i];
if (m_file->seek(touched[i] * BLK_SIZE, SEEK_SET))
{
ostringstream errstr;
errstr << "failed to seek to " << touched[i] * BLK_SIZE;
logMsg( ERROR, errstr.str() );
return false;
}
size_t writect = m_file->write(&readbuf, 1);
if ( writect != 1 )
{
ostringstream errstr;
errstr << "Only wrote " << writect << " bytes, expected 1";
logMsg( ERROR, errstr.str() );
return false;
}
}
return true;
}
bool TestRunner::hdfsRdwrExhaustTest()
{
// this is going to be a self-contained test that attempts to test
// all logic inherent in HdfsRdwr
// choose a new filename that is specific to our thread
ostringstream oss;
// embed pid so that this is a new directory path
oss << tmpDir << "/hdfsrdwr-" << getpid() << "-" << m_id;
string newpath = oss.str();
// open a file with arbitrarily small buffer
IDBDataFile* file = IDBDataFile::open(IDBDataFile::HDFS, newpath.c_str(), "r+", 0, 8);
assert( file );
// check various empty file conditions
assert( file->size() == 0 );
assert( file->tell() == 0 );
assert( file->seek(-1, SEEK_CUR) == -1);
assert( file->seek(0, SEEK_SET) == 0);
unsigned char buf[4];
assert( file->read(buf, 4) == 0);
// write some data
buf[0] = 0xde;
buf[1] = 0xad;
buf[2] = 0xbe;
buf[3] = 0xef;
assert( file->write(buf, 4) == 4);
assert( file->size() == 4 );
assert( file->tell() == 4 );
assert( file->truncate(-1) == -1 );
// now make file empty again
assert( file->truncate(0) == 0 );
assert( file->size() == 0 );
assert( file->seek(0, SEEK_SET) == 0);
assert( file->tell() == 0 );
assert( file->read(buf, 4) == 0);
// write data again, this time exactly up to allocated size
assert( file->write(buf, 4) == 4);
assert( file->write(buf, 4) == 4);
assert( file->size() == 8 );
assert( file->tell() == 8 );
// truncate back to 4
assert( file->truncate(4) == 0 );
assert( file->size() == 4 );
assert( file->seek(4, SEEK_SET) == 0);
assert( file->tell() == 4 );
// now trigger a buffer reallocation
assert( file->write(buf, 4) == 4);
assert( file->write(buf, 4) == 4);
assert( file->size() == 12 );
// now delete and close.
delete file;
// check the file size through the file system
IDBFileSystem& fs = IDBFileSystem::getFs( IDBDataFile::HDFS );
assert( fs.size( newpath.c_str() ) == 12);
// open again - the file is bigger than the default buffer so it triggers alternate
// logic in the constructor
file = IDBDataFile::open(IDBDataFile::HDFS, newpath.c_str(), "r+", 0, 8);
assert( file );
assert( file->size() == 12);
unsigned char newbuf[4];
assert( file->pread(newbuf, 4, 4) == 4);
assert( newbuf[0] == 0xde && newbuf[1] == 0xad && newbuf[2] == 0xbe && newbuf[3] == 0xef);
delete file;
fs.remove(newpath.c_str());
return true;
}
TestRunner::~TestRunner()
{
}
bool TestRunner::fillDefault()
{
return true;
}
bool TestRunner::writeBlock(unsigned int blocknum, unsigned char* buf, unsigned char tag)
{
buf[0] = tag;
size_t rc = m_file->write(buf, BLK_SIZE);
if (rc != BLK_SIZE)
{
ostringstream errstr;
errstr << "writeBlock failed for block " << blocknum << ", wrote " << rc << " bytes, expecting " << BLK_SIZE;
logMsg( ERROR, errstr.str() );
return false;
}
return true;
}
bool TestRunner::writeBlocks(unsigned int blocknum, unsigned char* buf, unsigned char tag, unsigned int count)
{
for (unsigned i = 0; i < count; ++i)
{
writeBlock(blocknum + i, buf, tag + i);
}
// should actually aggregate return values but using messages for now
return true;
}
bool TestRunner::readBlock(unsigned int blocknum, unsigned char* buf, unsigned char tag)
{
size_t rc = m_file->pread(buf, blocknum * BLK_SIZE, BLK_SIZE);
//cout << "DEBUG: read " << rc << " bytes at offset " << blocknum * BLK_SIZE << endl;
if (rc != BLK_SIZE)
{
ostringstream errstr;
errstr << "readBlock failed for block " << blocknum << ", read " << rc << " bytes, expecting " << BLK_SIZE;
logMsg( ERROR, errstr.str() );
return false;
}
else if (tag != buf[0])
{
ostringstream errstr;
errstr << "read tag 0x" << setw(2) << hex << setfill('0') << (int) buf[0] << " at block " << dec
<< blocknum << ", expected 0x" << (int) tag;
logMsg( ERROR, errstr.str() );
return false;
}
return true;
}
bool TestRunner::readBlocks(unsigned int blocknum, unsigned char* buf, unsigned char tag, unsigned int count)
{
for (unsigned i = 0; i < count; ++i)
{
readBlock(blocknum + i, buf, (unsigned char)(tag + i));
}
// should actually aggregate return values but using messages for now
return true;
}
void TestRunner::logMsg( LogLevel level, const string& msg )
{
cout << "Test-" << m_id << ":" << ((level == INFO) ? "INFO:" : "ERROR:") << msg << endl;
}
void logSuccess()
{
::openlog("hdfsCheck", 0 | LOG_PID, LOG_LOCAL1);
::syslog(LOG_INFO, "All HDFS tests passed!");
::closelog();
}
void logFailure(const char* msg)
{
ostringstream oss;
oss << "hdfsCheck failed: " << msg;
::openlog("hdfsCheck", 0 | LOG_PID, LOG_LOCAL1);
::syslog(LOG_WARNING, oss.str().c_str());
::closelog();
}
int main(int argc, char** argv)
{
foptest_opts opts;
int c;
while ((c = getopt (argc, argv, "b:n:t:r:")) != -1)
switch (c)
{
case 'b':
opts.blockSize = atoi(optarg);
if ( opts.blockSize > MAX_BLK_SIZE )
{
opts.blockSize = MAX_BLK_SIZE;
cout << "WARNING: block size exceeds max, using " << opts.blockSize << endl;
}
else
{
cout << "INFO: using BLK_SIZE " << opts.blockSize << endl;
}
break;
case 'n':
opts.numBlocks = atoi(optarg);
cout << "INFO: will operate on " << opts.numBlocks << " blocks" << endl;
break;
case 't':
opts.timeout = atoi(optarg);
cout << "INFO: setting timeout to " << opts.timeout << " secs." << endl;
break;
case 'r':
opts.numDbRoots = atoi(optarg);
cout << "INFO: Will check " << opts.numDbRoots << " dbroot directories" << endl;
break;
default:
usage();
logFailure("bad command-line option");
return 1;
}
if ( optind != argc - 1 )
{
usage();
logFailure("missing hdfs-plugin argument");
return 1;
}
tmpDir = startup::StartUp::tmpDir();
opts.pluginFile = argv[optind];
try
{
// init the library with logging enabled
std::string hdfsRdwrScratch = tmpDir + "/rdwr_scratch";
IDBPolicy::init( true, true, hdfsRdwrScratch, 2147483648 );
if ( !IDBPolicy::installPlugin(opts.pluginFile) )
{
cerr << "ERROR: HDFS plugin not installed " << opts.pluginFile << endl;
return -1;
}
}
catch (...)
{
logFailure("failure to initialize IDBPolicy and/or load HDFS plugin");
return -1;
}
struct timeval starttime, endtime, timediff;
gettimeofday(&starttime, 0x0);
bool done = false;
while ( !done )
{
// check to see if timeout has expired
TestRunner trun(0, opts);
try
{
if ( trun.runTest( IDBDataFile::HDFS, 0 ) )
{
done = true;
cout << "INFO: All HDFS checks passed!" << endl;
}
}
catch (...)
{
; // treat an exception thrown as a failure
}
if ( !done )
{
gettimeofday(&endtime, 0x0);
timeval_subtract(&timediff, &endtime, &starttime);
float secs = timediff.tv_sec + (timediff.tv_usec * 0.000001);
if ( secs < opts.timeout )
{
// wait 5 seconds before we try again
sleep(5);
}
else
{
cerr << "ERROR: timeout out after unsuccessful HDFS tests" << endl;
logFailure("One or more HDFS tests failed");
return -1;
}
}
}
// only way we get here is if everything passes
logSuccess();
return 0;
}

6
tools/setConfig/CMakeLists.txt
View File

@ -7,11 +7,11 @@ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/configxml.sh.in" "${CMAKE_CURRENT_SO
set(setConfig_SRCS main.cpp)
add_executable(setConfig ${setConfig_SRCS})
add_executable(mcsSetConfig ${setConfig_SRCS})
target_link_libraries(setConfig ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS})
target_link_libraries(mcsSetConfig ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS})
install(TARGETS setConfig DESTINATION ${ENGINE_BINDIR} COMPONENT platform)
install(TARGETS mcsSetConfig DESTINATION ${ENGINE_BINDIR} COMPONENT platform)
install(PROGRAMS configxml.sh DESTINATION ${ENGINE_BINDIR} COMPONENT platform)

28
tools/setConfig/configxml.sh.in
View File

@ -4,22 +4,6 @@
#
#
if [ -z "$COLUMNSTORE_INSTALL_DIR" ]; then
test -f /etc/default/infinidb && . /etc/default/infinidb
fi
if [ -z "$COLUMNSTORE_INSTALL_DIR" ]; then
COLUMNSTORE_INSTALL_DIR=/usr/local/mariadb/columnstore
fi
export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR
InstallDir=$COLUMNSTORE_INSTALL_DIR
if [ $InstallDir != "/usr/local/mariadb/columnstore" ]; then
export PATH=$InstallDir/bin:/bin:/usr/bin
export LD_LIBRARY_PATH=$InstallDir/lib
fi
case "$1" in
setconfig)
@ -29,7 +13,7 @@ case "$1" in
exit 1
fi
oldvalue=$($InstallDir/bin/getConfig $2 $3)
oldvalue=$(mcsGetConfig $2 $3)
#if [ -z $oldvalue ]; then
# echo "$2 / $3 not found in Columnstore.xml"
@ -46,17 +30,17 @@ case "$1" in
echo "$calxml backed up to $calxml.$seconds"
echo
oldvalue=$($InstallDir/bin/getConfig $2 $3)
oldvalue=$(mcsGetConfig $2 $3)
echo "Old value of $2 / $3 is $oldvalue"
if ( [ $# -eq 4 ] && [ -z $4 ] ); then
$InstallDir/bin/setConfig $2 $3 ""
mcsSetConfig $2 $3 ""
else
$InstallDir/bin/setConfig $2 $3 $4
mcsSetConfig $2 $3 $4
fi
newvalue=$($InstallDir/bin/getConfig $2 $3)
newvalue=$(mcsGetConfig $2 $3)
echo "$2 / $3 now set to $newvalue"
;;
@ -67,7 +51,7 @@ case "$1" in
exit 1
fi
value=$($InstallDir/bin/getConfig $2 $3)
value=$(mcsGetConfig $2 $3)
if [ -z $value ]; then
echo "$2 / $3 not found in Columnstore.xml"

2
tools/setConfig/main.cpp
View File

@ -119,7 +119,7 @@ int main(int argc, char** argv)
if (!dflg && !parentOAMModuleFlag)
{
cerr << "Exiting, setConfig can only be run on the Active "
cerr << "Exiting, mcsSetConfig can only be run on the Active "
"OAM Parent Module '" << parentOAMModule << "'" << endl;
return 2;
}