database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

Merge branch 'develop-1.2' into MCOL-1822-c

Browse Source
This commit is contained in:
David Hall
2019-02-27 13:20:45 -06:00
parent c654e8621e e849af0ce6
commit cf056e42ac
10 changed files with 268 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
CMakeLists.txt
View File

@ -112,6 +112,17 @@ if (NOT SNAPPY_FOUND)
MESSAGE(FATAL_ERROR "Snappy not found please install snappy-devel for CentOS/RedHat or libsnappy-dev for Ubuntu/Debian")
endif()
# Jemalloc has issues with SLES 12, so disable for now
IF (EXISTS "/etc/SuSE-release")
SET(JEMALLOC_LIBRARIES "")
ELSE ()
INCLUDE (FindJeMalloc.cmake)
if (NOT JEMALLOC_FOUND)
message(FATAL_ERROR "jemalloc not found!")
SET(JEMALLOC_LIBRARIES "")
endif()
ENDIF ()
FIND_PROGRAM(AWK_EXECUTABLE awk DOC "path to the awk executable")
if(NOT AWK_EXECUTABLE)
message(FATAL_ERROR "awk not found!")
@ -163,7 +174,7 @@ SET (ENGINE_LOCALDIR "${INSTALL_ENGINE}/local")
SET (ENGINE_MYSQLDIR "${INSTALL_ENGINE}/mysql")
SET (ENGINE_TOOLSDIR "${INSTALL_ENGINE}/tools")
SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot ${Boost_LIBRARIES} xml2 pthread rt libmysql_client)
SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot ${Boost_LIBRARIES} xml2 pthread rt libmysql_client ${JEMALLOC_LIBRARIES})
SET (ENGINE_OAM_LIBS oamcpp alarmmanager)
SET (ENGINE_BRM_LIBS brm idbdatafile cacheutils rwlock ${ENGINE_OAM_LIBS} ${ENGINE_COMMON_LIBS})
SET (ENGINE_EXEC_LIBS joblist execplan windowfunction joiner rowgroup funcexp udfsdk regr dataconvert common compress querystats querytele thrift threadpool ${ENGINE_BRM_LIBS})

44
FindJeMalloc.cmake Normal file
View File

@ -0,0 +1,44 @@
# - Try to find jemalloc headers and libraries.
#
# Usage of this module as follows:
#
# find_package(JeMalloc)
#
# Variables used by this module, they can change the default behaviour and need
# to be set before calling find_package:
#
# JEMALLOC_ROOT_DIR Set this variable to the root installation of
# jemalloc if the module has problems finding
# the proper installation path.
#
# Variables defined by this module:
#
# JEMALLOC_FOUND System has jemalloc libs/headers
# JEMALLOC_LIBRARIES The jemalloc library/libraries
# JEMALLOC_INCLUDE_DIR The location of jemalloc headers
find_path(JEMALLOC_ROOT_DIR
NAMES include/jemalloc/jemalloc.h
)
find_library(JEMALLOC_LIBRARIES
NAMES jemalloc
HINTS ${JEMALLOC_ROOT_DIR}/lib
)
find_path(JEMALLOC_INCLUDE_DIR
NAMES jemalloc/jemalloc.h
HINTS ${JEMALLOC_ROOT_DIR}/include
)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(JeMalloc DEFAULT_MSG
JEMALLOC_LIBRARIES
JEMALLOC_INCLUDE_DIR
)
mark_as_advanced(
JEMALLOC_ROOT_DIR
JEMALLOC_LIBRARIES
JEMALLOC_INCLUDE_DIR
)

6
cpackEngineDEB.cmake
View File

@ -65,11 +65,11 @@ if (EXISTS "/etc/debian_version")
set(DEBIAN_VERSION_NUMBER "${CMAKE_MATCH_1}")
endif ()
if ("${DEBIAN_VERSION_NUMBER}" EQUAL "8")
SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libreadline-dev, rsync, net-tools, libboost-all-dev, mariadb-columnstore-libs, mariadb-columnstore-server, libsnappy1")
SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libreadline-dev, rsync, net-tools, libboost-all-dev, mariadb-columnstore-libs, mariadb-columnstore-server, libsnappy1, libjemalloc1")
elseif ("${DEBIAN_VERSION_NUMBER}" EQUAL "9")
SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libreadline-dev, rsync, net-tools, libboost-all-dev, mariadb-columnstore-libs, mariadb-columnstore-server, libsnappy1v5, libreadline5")
SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libreadline-dev, rsync, net-tools, libboost-all-dev, mariadb-columnstore-libs, mariadb-columnstore-server, libsnappy1v5, libreadline5, libjemalloc1")
else()
SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libboost-all-dev, libreadline-dev, rsync, libsnappy1v5, net-tools")
SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libboost-all-dev, libreadline-dev, rsync, libsnappy1v5, net-tools, libjemalloc1")
endif ()
SET(CPACK_DEBIAN_STORAGE-ENGINE_PACKAGE_DEPENDS "mariadb-columnstore-libs")

4
cpackEngineRPM.cmake
View File

@ -91,9 +91,9 @@ if (${REDHAT_VERSION_NUMBER} EQUAL 6)
# Disable auto require as this will also try to pull Boost via RPM
SET(CPACK_RPM_PACKAGE_AUTOREQPROV " no")
elseif (${SUSE_VERSION_NUMBER} EQUAL 12)
SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost-devel >= 1.54.0" "mariadb-columnstore-libs" "libsnappy1")
SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost-devel >= 1.54.0" "mariadb-columnstore-libs" "libsnappy1" "jemalloc")
else ()
SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost >= 1.53.0" "mariadb-columnstore-libs" "snappy")
SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost >= 1.53.0" "mariadb-columnstore-libs" "snappy" "jemalloc")
endif()
SETA(CPACK_RPM_storage-engine_PACKAGE_REQUIRES "mariadb-columnstore-libs")

56
dbcon/joblist/groupconcat.cpp
View File

@ -344,8 +344,6 @@ void GroupConcatAgUM::merge(const rowgroup::Row& inRow, int64_t i)
joblist::GroupConcatAgUM* gccAg = *((joblist::GroupConcatAgUM**)(data + inRow.getOffset(i)));
fConcator->merge(gccAg->concator().get());
// don't reset
// gccAg->orderBy().reset();
}
@ -393,12 +391,11 @@ GroupConcator::~GroupConcator()
void GroupConcator::initialize(const rowgroup::SP_GroupConcat& gcc)
{
// MCOL-901 This value comes from the Server and it is
// too high(3MB) to allocate it for every instance.
fGroupConcatLen = gcc->fSize;
fCurrentLength -= strlen(gcc->fSeparator.c_str());
fOutputString.reset(new uint8_t[fGroupConcatLen + 2]);
memset(fOutputString.get(), 0, fGroupConcatLen + 2);
fConstCols = gcc->fConstCols;
fConstantLen = strlen(gcc->fSeparator.c_str());
@ -477,7 +474,6 @@ void GroupConcator::outputRow(std::ostringstream& oss, const rowgroup::Row& row)
case CalpontSystemCatalog::TEXT:
{
oss << row.getStringField(*i).c_str();
//oss << row.getStringField(*i);
break;
}
@ -626,11 +622,6 @@ int64_t GroupConcator::lengthEstimate(const rowgroup::Row& row)
{
int64_t colWidth = row.getStringLength(*i);
fieldLen += colWidth; // getStringLength() does the same thing as below
//assert(!row.usesStringTable());
//int64_t colWidth = row.getColumnWidth(*i);
//uint8_t* pStr = row.getData() + row.getOffset(*i);
//while ((*pStr++ > 0) && (fieldLen < colWidth))
// fieldLen++;
break;
}
@ -759,7 +750,6 @@ void GroupConcatOrderBy::initialize(const rowgroup::SP_GroupConcat& gcc)
uint64_t GroupConcatOrderBy::getKeyLength() const
{
// only distinct the concatenated columns
//return (fRow0.getOffset(fConcatColumns.size()) - 2);
return fConcatColumns.size() - 1; // cols 0 to fConcatColumns will be conpared
}
@ -778,8 +768,6 @@ void GroupConcatOrderBy::processRow(const rowgroup::Row& row)
if (fCurrentLength < fGroupConcatLen)
{
copyRow(row, &fRow0);
//cout << "length < GB limit: " << fRow0.toString() << endl;
//memcpy(fRow0.getData(), row.getData(), row.getSize());
// the RID is no meaning here, use it to store the estimated length.
int16_t estLen = lengthEstimate(fRow0);
fRow0.setRid(estLen);
@ -869,7 +857,6 @@ void GroupConcatOrderBy::merge(GroupConcator* gc)
if (fDistinct)
fDistinctMap->insert(row.fData);
//fDistinctMap->insert(make_pair((row.fData+2), row.fData));
}
else if (fOrderByCond.size() > 0 && fRule.less(row.fData, fOrderByQueue.top().fData))
@ -883,8 +870,6 @@ void GroupConcatOrderBy::merge(GroupConcator* gc)
{
fDistinctMap->erase(swapRow.fData);
fDistinctMap->insert(row.fData);
//fDistinctMap->erase(fDistinctMap->find(swapRow.fData + 2));
//fDistinctMap->insert(make_pair((row.fData+2), row.fData));
}
row1.setData(row.fData);
@ -900,7 +885,6 @@ void GroupConcatOrderBy::merge(GroupConcator* gc)
void GroupConcatOrderBy::getResult(uint8_t* buff, const string& sep)
{
#if 1
ostringstream oss;
bool addSep = false;
@ -926,28 +910,12 @@ void GroupConcatOrderBy::getResult(uint8_t* buff, const string& sep)
rowStack.pop();
}
strncpy((char*) buff, oss.str().c_str(), fGroupConcatLen);
#else
ostringstream oss;
bool addSep = false;
priority_queue<OrderByRow>::reverse_iterator rit;
size_t resultSize = oss.str().size();
fOutputString.reset(new uint8_t[resultSize + 2]);
memset(fOutputString.get(), 0, resultSize + 2);
for (rit = fOrderByQueue.rbegin(); rit != fOrderByQueue.rend(); ++rit)
{
if (addSep)
oss << sep;
else
addSep = true;
const OrderByRow& topRow = *rit;
fRow0.setData(topRow.fData);
outputRow(oss, fRow0);
}
fOrderByQueue.clear();
strncpy((char*) buff, oss.str().c_str(), fGroupConcatLen);
#endif
strncpy((char*)fOutputString.get(),
oss.str().c_str(), resultSize);
}
uint8_t* GroupConcator::getResult(const string& sep)
@ -1017,7 +985,6 @@ void GroupConcatNoOrder::initialize(const rowgroup::SP_GroupConcat& gcc)
throw IDBExcept(fErrorCode);
}
//fData.reset(new uint8_t[fRowGroup.getDataSize(fRowsPerRG)]);
fData.reinit(fRowGroup, fRowsPerRG);
fRowGroup.setData(&fData);
fRowGroup.resetRowGroup(0);
@ -1032,7 +999,6 @@ void GroupConcatNoOrder::processRow(const rowgroup::Row& row)
if (fCurrentLength < fGroupConcatLen && concatColIsNull(row) == false)
{
copyRow(row, &fRow);
//memcpy(fRow.getData(), row.getData(), row.getSize());
// the RID is no meaning here, use it to store the estimated length.
int16_t estLen = lengthEstimate(fRow);
@ -1056,7 +1022,6 @@ void GroupConcatNoOrder::processRow(const rowgroup::Row& row)
fDataQueue.push(fData);
fData.reinit(fRowGroup, fRowsPerRG);
//fData.reset(new uint8_t[fRowGroup.getDataSize(fRowsPerRG)]);
fRowGroup.setData(&fData);
fRowGroup.resetRowGroup(0);
fRowGroup.getRow(0, &fRow);
@ -1107,7 +1072,12 @@ void GroupConcatNoOrder::getResult(uint8_t* buff, const string& sep)
fDataQueue.pop();
}
strncpy((char*) buff, oss.str().c_str(), fGroupConcatLen);
size_t resultSize = oss.str().size();
fOutputString.reset(new uint8_t[resultSize + 2]);
memset(fOutputString.get(), 0, resultSize + 2);
strncpy((char*)fOutputString.get(),
oss.str().c_str(), resultSize);
}

25
dbcon/joblist/limitedorderby.cpp
View File

@ -173,10 +173,10 @@ void LimitedOrderBy::finalize()
if (fOrderByQueue.size() > 0)
{
uint64_t newSize = fRowsPerRG * fRowGroup.getRowSize();
fMemSize += newSize;
uint64_t memSizeInc = fRowsPerRG * fRowGroup.getRowSize();
fMemSize += memSizeInc;
if (!fRm->getMemory(newSize, fSessionMemLimit))
if (!fRm->getMemory(memSizeInc, fSessionMemLimit))
{
cerr << IDBErrorInfo::instance()->errorMsg(fErrorCode)
<< " @" << __FILE__ << ":" << __LINE__;
@ -185,10 +185,12 @@ void LimitedOrderBy::finalize()
uint64_t offset = 0;
uint64_t i = 0;
// Reduce queue size by an offset value if it applicable.
uint64_t queueSizeWoOffset = fOrderByQueue.size() > fStart ?
fOrderByQueue.size() - fStart : 0;
list<RGData> tempRGDataList;
// Skip first LIMIT rows in the the RowGroup
if ( fCount <= fOrderByQueue.size() )
if ( fCount <= queueSizeWoOffset )
{
offset = fCount % fRowsPerRG;
if(!offset && fCount > 0)
@ -196,11 +198,11 @@ void LimitedOrderBy::finalize()
}
else
{
offset = fOrderByQueue.size() % fRowsPerRG;
if(!offset && fOrderByQueue.size() > 0)
offset = queueSizeWoOffset % fRowsPerRG;
if(!offset && queueSizeWoOffset > 0)
offset = fRowsPerRG;
}
list<RGData>::iterator tempListIter = tempRGDataList.begin();
i = 0;
@ -217,7 +219,6 @@ void LimitedOrderBy::finalize()
const OrderByRow& topRow = fOrderByQueue.top();
row1.setData(topRow.fData);
copyRow(row1, &fRow0);
//cerr << "LimitedOrderBy::finalize fRow0 " << fRow0.toString() << endl;
fRowGroup.incRowCount();
offset--;
fRow0.prevRow(rSize);
@ -226,9 +227,9 @@ void LimitedOrderBy::finalize()
if(offset == (uint64_t)-1)
{
tempRGDataList.push_front(fData);
fMemSize += newSize;
fMemSize += memSizeInc;
if (!fRm->getMemory(newSize, fSessionMemLimit))
if (!fRm->getMemory(memSizeInc, fSessionMemLimit))
{
cerr << IDBErrorInfo::instance()->errorMsg(fErrorCode)
<< " @" << __FILE__ << ":" << __LINE__;

25
oamapps/mcsadmin/mcsadmin.cpp
View File

@ -759,8 +759,11 @@ int processCommand(string* arguments)
vector<uint32_t> srcDbroots; // all of the currently configured dbroots
vector<uint32_t> destDbroots; // srcDbroots - removeDbroots
set<int>::iterator dbiter;
if (arguments[1] == "start")
#if _MSC_VER
if (_strnicmp(arguments[1].c_str(), "start", 5) == 0))
#else
if (strncasecmp(arguments[1].c_str(), "start", 5) == 0)
#endif
{
// Get a list of all the configured dbroots in the xml file.
DBRootConfigList dbRootConfigList;
@ -772,7 +775,11 @@ int processCommand(string* arguments)
// The user may choose to redistribute in such a way as to
// leave certain dbroots empty, presumably for later removal.
if (arguments[2] == "remove")
#if _MSC_VER
if (_strnicmp(arguments[1].c_str(), "remove", 6) == 0))
#else
if (strncasecmp(arguments[1].c_str(), "remove", 6) == 0)
#endif
{
int dbroot;
bool error = false;
@ -891,7 +898,11 @@ int processCommand(string* arguments)
SendToWES(oam, bs);
}
else if (arguments[1] == "stop")
#if _MSC_VER
if (_strnicmp(arguments[1].c_str(), "stop", 4) == 0))
#else
if (strncasecmp(arguments[1].c_str(), "stop", 4) == 0)
#endif
{
ByteStream bs;
// message WES ID, sequence #, action id
@ -901,7 +912,11 @@ int processCommand(string* arguments)
bs.append((const ByteStream::byte*) &header, sizeof(header));
SendToWES(oam, bs);
}
else if (arguments[1] == "status")
#if _MSC_VER
if (_strnicmp(arguments[1].c_str(), "status", 6) == 0))
#else
if (strncasecmp(arguments[1].c_str(), "status", 6) == 0)
#endif
{
ByteStream bs;
// message WES ID, sequence #, action id

234
oamapps/postConfigure/postConfigure.cpp
View File

@ -55,7 +55,6 @@
#include <arpa/inet.h>
#include <sys/socket.h>
#include <ifaddrs.h>
#include <stdio.h>
#include <string.h> /* for strncpy */
#include <sys/types.h>
@ -111,8 +110,6 @@ typedef std::vector<ModuleIP> ModuleIpList;
void offLineAppCheck();
bool setOSFiles(string parentOAMModuleName, int serverTypeInstall);
bool checkSaveConfigFile();
string getModuleName();
bool setModuleName(string moduleName);
bool updateBash();
bool makeModuleFile(string moduleName, string parentOAMModuleName);
bool updateProcessConfig();
@ -126,6 +123,7 @@ bool singleServerDBrootSetup();
bool copyFstab(string moduleName);
bool attachVolume(string instanceName, string volumeName, string deviceName, string dbrootPath);
void singleServerConfigSetup(Config* sysConfig);
std::string resolveHostNameToReverseDNSName(std::string hostname);
void remoteInstallThread(void*);
@ -193,6 +191,7 @@ bool single_server_quick_install = false;
bool multi_server_quick_install = false;
bool amazon_quick_install = false;
bool doNotResolveHostNames = false;
bool resolveHostNamesToReverseDNSNames = false;
string DataFileEnvFile;
@ -316,7 +315,7 @@ int main(int argc, char* argv[])
cout << " Enter one of the options within [], if available, or" << endl;
cout << " Enter a new value" << endl << endl;
cout << endl;
cout << "Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count][-numBlocksPct][-totalUmMemory]" << endl;
cout << "Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count][-x][-xr][-numBlocksPct][-totalUmMemory]" << endl;
cout << " -h Help" << endl;
cout << " -c Config File to use to extract configuration data, default is Columnstore.xml.rpmsave" << endl;
cout << " -u Upgrade, Install using the Config File from -c, default to Columnstore.xml.rpmsave" << endl;
@ -332,6 +331,7 @@ int main(int argc, char* argv[])
cout << " -pm-ip-addrs Performance Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl;
cout << " -um-ip-addrs User Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl;
cout << " -x Do not resolve IP Addresses from host names" << endl;
cout << " -xr Resolve host names into their reverse DNS host names. Only applied in combination with -x" << endl;
cout << " -numBlocksPct amount of physical memory to utilize for disk block caching" << endl;
cout << " (percentages of the total memory need to be stated without suffix, explcit values with suffixes M or G)" << endl;
cout << " -totalUmMemory amount of physical memory to utilize for joins, intermediate results and set operations on the UM" << endl;
@ -341,6 +341,10 @@ int main(int argc, char* argv[])
else if (string("-x") == argv[i])
{
doNotResolveHostNames = true;
}
else if (string("-xr") == argv[i])
{
resolveHostNamesToReverseDNSNames = true;
}
else if( string("-qs") == argv[i] )
{
@ -517,7 +521,7 @@ int main(int argc, char* argv[])
else
{
cout << " ERROR: Invalid Argument = " << argv[i] << endl;
cout << " Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count][-numBlocksPct][-totalUmMemory]" << endl;
cout << " Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count][-x][-xr][-numBlocksPct][-totalUmMemory]" << endl;
exit (1);
}
}
@ -658,9 +662,14 @@ int main(int argc, char* argv[])
if (moduleconfig.hostConfigList.size() > 0 )
{
HostConfigList::iterator pt1 = moduleconfig.hostConfigList.begin();
string PM1ipAdd = (*pt1).IPAddr;
// MCOL-1607. The 'am I pm1?' check below requires an ipaddr.
string PM1ipAdd = oam.getIPAddress((*pt1).IPAddr.c_str());
if (PM1ipAdd.empty())
PM1ipAdd = (*pt1).IPAddr; // this is what it was doing before
//cout << PM1ipAdd << endl;
if ( PM1ipAdd != "127.0.0.1" )
{
if ( PM1ipAdd != "0.0.0.0")
@ -2566,7 +2575,12 @@ int main(int argc, char* argv[])
//get IP Address
string IPAddress;
if (doNotResolveHostNames)
IPAddress = newModuleHostName;
if (resolveHostNamesToReverseDNSNames) {
IPAddress = resolveHostNameToReverseDNSName(newModuleHostName);
}
else {
IPAddress = newModuleHostName;
}
else
IPAddress = oam.getIPAddress( newModuleHostName);
@ -2593,6 +2607,13 @@ int main(int argc, char* argv[])
callFree(pcommand);
}
if (!doNotResolveHostNames)
{
string ugh = oam.getIPAddress(newModuleIPAddr);
if (ugh.length() > 0)
newModuleIPAddr = ugh;
}
if (newModuleIPAddr == "127.0.0.1" || newModuleIPAddr == "0.0.0.0" || newModuleIPAddr == "128.0.0.1")
{
@ -6550,7 +6571,12 @@ bool glusterSetup(string password, bool doNotResolveHostNames)
//get IP Address
string IPAddress;
if (doNotResolveHostNames)
IPAddress = moduleHostName;
if (resolveHostNamesToReverseDNSNames) {
IPAddress = resolveHostNameToReverseDNSName(moduleHostName);
}
else {
IPAddress = moduleHostName;
}
else
IPAddress = oam.getIPAddress( moduleHostName);
@ -6566,7 +6592,7 @@ bool glusterSetup(string password, bool doNotResolveHostNames)
//prompt for IP address
while (true)
{
prompt = "Enter PM #" + oam.itoa(DataRedundancyConfigs[pm].pmID) + " IP Address of " + moduleHostName + " (" + moduleIPAddr + ") > ";
prompt = "Enter PM #" + oam.itoa(DataRedundancyConfigs[pm].pmID) + " IP Address or hostname of " + moduleHostName + " (" + moduleIPAddr + ") > ";
pcommand = callReadline(prompt.c_str());
if (pcommand)
@ -6575,6 +6601,13 @@ bool glusterSetup(string password, bool doNotResolveHostNames)
callFree(pcommand);
}
if (!doNotResolveHostNames)
{
string ugh = oam.getIPAddress(moduleIPAddr);
if (ugh.length() > 0)
moduleIPAddr = ugh;
}
if (moduleIPAddr == "127.0.0.1" || moduleIPAddr == "0.0.0.0" || moduleIPAddr == "128.0.0.1")
{
@ -7011,29 +7044,29 @@ bool glusterSetup(string password, bool doNotResolveHostNames)
command = "sudo gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-uid " + oam.itoa(user) + " >> /tmp/glusterCommands.txt 2>&1";;
status = system(command.c_str());
if (WEXITSTATUS(status) != 0 )
{
cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl;
exit(1);
}
if (WEXITSTATUS(status) != 0)
{
cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl;
exit(1);
}
command = "sudo gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-gid " + oam.itoa(group) + " >> /tmp/glusterCommands.txt 2>&1";;
status = system(command.c_str());
command = "sudo gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-gid " + oam.itoa(group) + " >> /tmp/glusterCommands.txt 2>&1";;
status = system(command.c_str());
if (WEXITSTATUS(status) != 0 )
{
cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl;
exit(1);
}
if (WEXITSTATUS(status) != 0)
{
cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl;
exit(1);
}
command = "sudo gluster volume start dbroot" + oam.itoa(dbrootID) + " >> /tmp/glusterCommands.txt 2>&1";
status = system(command.c_str());
command = "sudo gluster volume start dbroot" + oam.itoa(dbrootID) + " >> /tmp/glusterCommands.txt 2>&1";
status = system(command.c_str());
if (WEXITSTATUS(status) != 0 )
{
cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl;
exit(1);
}
if (WEXITSTATUS(status) != 0)
{
cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl;
exit(1);
}
}
cout << "DONE" << endl;
@ -7047,70 +7080,89 @@ bool glusterSetup(string password, bool doNotResolveHostNames)
void singleServerConfigSetup(Config* sysConfig)
{
try
{
sysConfig->setConfig("ExeMgr1", "IPAddr", "127.0.0.1");
sysConfig->setConfig("ExeMgr1", "Module", "pm1");
sysConfig->setConfig("ProcMgr", "IPAddr", "127.0.0.1");
sysConfig->setConfig("ProcMgr_Alarm", "IPAddr", "127.0.0.1");
sysConfig->setConfig("ProcStatusControl", "IPAddr", "127.0.0.1");
sysConfig->setConfig("pm1_ProcessMonitor", "IPAddr", "127.0.0.1");
sysConfig->setConfig("pm1_ServerMonitor", "IPAddr", "127.0.0.1");
sysConfig->setConfig("pm1_WriteEngineServer", "IPAddr", "127.0.0.1");
sysConfig->setConfig("DDLProc", "IPAddr", "127.0.0.1");
sysConfig->setConfig("DMLProc", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS1", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS2", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS3", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS4", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS5", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS6", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS7", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS8", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS9", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS10", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS11", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS12", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS13", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS14", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS15", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS16", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS17", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS18", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS19", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS20", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS21", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS22", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS23", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS24", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS25", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS26", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS27", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS28", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS29", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS30", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS31", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS32", "IPAddr", "127.0.0.1");
sysConfig->setConfig("SystemModuleConfig", "ModuleCount2", "0");
sysConfig->setConfig("SystemModuleConfig", "ModuleIPAddr1-1-3", "127.0.0.1");
sysConfig->setConfig("SystemModuleConfig", "ModuleHostName1-1-3", "localhost");
sysConfig->setConfig("DBRM_Controller", "IPAddr", "127.0.0.1");
sysConfig->setConfig("DBRM_Worker1", "IPAddr", "127.0.0.1");
sysConfig->setConfig("DBRM_Worker1", "Module", "pm1");
sysConfig->setConfig("DBBC", "NumBlocksPct", "50");
sysConfig->setConfig("Installation", "InitialInstallFlag", "y");
sysConfig->setConfig("Installation", "SingleServerInstall", "y");
sysConfig->setConfig("HashJoin", "TotalUmMemory", "25%");
}
catch (...)
{
cout << "ERROR: Problem setting for Single Server in the MariaDB ColumnStore System Configuration file" << endl;
exit(1);
}
try
{
sysConfig->setConfig("ExeMgr1", "IPAddr", "127.0.0.1");
sysConfig->setConfig("ExeMgr1", "Module", "pm1");
sysConfig->setConfig("ProcMgr", "IPAddr", "127.0.0.1");
sysConfig->setConfig("ProcMgr_Alarm", "IPAddr", "127.0.0.1");
sysConfig->setConfig("ProcStatusControl", "IPAddr", "127.0.0.1");
sysConfig->setConfig("pm1_ProcessMonitor", "IPAddr", "127.0.0.1");
sysConfig->setConfig("pm1_ServerMonitor", "IPAddr", "127.0.0.1");
sysConfig->setConfig("pm1_WriteEngineServer", "IPAddr", "127.0.0.1");
sysConfig->setConfig("DDLProc", "IPAddr", "127.0.0.1");
sysConfig->setConfig("DMLProc", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS1", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS2", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS3", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS4", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS5", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS6", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS7", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS8", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS9", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS10", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS11", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS12", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS13", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS14", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS15", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS16", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS17", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS18", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS19", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS20", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS21", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS22", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS23", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS24", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS25", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS26", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS27", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS28", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS29", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS30", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS31", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS32", "IPAddr", "127.0.0.1");
sysConfig->setConfig("SystemModuleConfig", "ModuleCount2", "0");
sysConfig->setConfig("SystemModuleConfig", "ModuleIPAddr1-1-3", "127.0.0.1");
sysConfig->setConfig("SystemModuleConfig", "ModuleHostName1-1-3", "localhost");
sysConfig->setConfig("DBRM_Controller", "IPAddr", "127.0.0.1");
sysConfig->setConfig("DBRM_Worker1", "IPAddr", "127.0.0.1");
sysConfig->setConfig("DBRM_Worker1", "Module", "pm1");
sysConfig->setConfig("DBBC", "NumBlocksPct", "50");
sysConfig->setConfig("Installation", "InitialInstallFlag", "y");
sysConfig->setConfig("Installation", "SingleServerInstall", "y");
sysConfig->setConfig("HashJoin", "TotalUmMemory", "25%");
}
catch (...)
{
cout << "ERROR: Problem setting for Single Server in the MariaDB ColumnStore System Configuration file" << endl;
exit(1);
}
return;
return;
}
/**
Resolves the given hostname into its reverse DNS name.
@param hostname the hostname to resolve.
@return the reverse dns name of given hostname or an empty string in case the hostname could not be resolved.
*/
std::string resolveHostNameToReverseDNSName(std::string hostname) {
struct hostent *hp = gethostbyname(hostname.c_str());
if (hp == NULL) {
std::cout << "Error: Couldn't resolve hostname " << hostname << " to ip address" << std::endl;
return "";
}
struct hostent *rl = gethostbyaddr(hp->h_addr_list[0], sizeof hp->h_addr_list[0], AF_INET);
if (rl == NULL) {
std::cout << "Error: Couldn't resolve ip address of hostname " << hostname << " back to a hostname" << std::endl;
return "";
}
hostname = rl->h_name;
return hostname;
}
// vim:ts=4 sw=4:

13
utils/rowgroup/rowgroup.cpp
View File

@ -1207,12 +1207,13 @@ int64_t Row::getSignedNullValue(uint32_t colIndex) const
RowGroup::RowGroup() : columnCount(0), data(NULL), rgData(NULL), strings(NULL),
useStringTable(true), hasLongStringField(false), sTableThreshold(20)
{
oldOffsets.reserve(1024);
oids.reserve(1024);
keys.reserve(1024);
types.reserve(1024);
scale.reserve(1024);
precision.reserve(1024);
// 1024 is too generous to waste.
oldOffsets.reserve(10);
oids.reserve(10);
keys.reserve(10);
types.reserve(10);
scale.reserve(10);
precision.reserve(10);
}
RowGroup::RowGroup(uint32_t colCount,

17
writeengine/shared/we_convertor.cpp
View File

@ -194,15 +194,20 @@ long long Convertor::convertDecimalString(
// range check against int64
if (dval > LLONG_MAX || dval < LLONG_MIN)
if (dval > LLONG_MAX)
{
errno = ERANGE;
return LLONG_MAX;
}
if (dval < LLONG_MIN)
{
errno = ERANGE;
return LLONG_MIN;
}
errno = 0;
else
errno = 0;
ret = dval;
// get the fractional part of what's left & round ret up or down.
dval -= ret;
if (dval >= 0.5 && ret < LLONG_MAX)