database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-01 06:46:55 +03:00
Code Activity

Merge branch 'develop-1.2' into MCOL-1822-c

Browse Source
This commit is contained in:
David Hall
2019-02-27 13:20:45 -06:00
parent c654e8621e e849af0ce6
commit cf056e42ac
10 changed files with 268 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
CMakeLists.txt
View File

@ -112,6 +112,17 @@ if (NOT SNAPPY_FOUND)
MESSAGE(FATAL_ERROR "Snappy not found please install snappy-devel for CentOS/RedHat or libsnappy-dev for Ubuntu/Debian") MESSAGE(FATAL_ERROR "Snappy not found please install snappy-devel for CentOS/RedHat or libsnappy-dev for Ubuntu/Debian")
endif() endif()
# Jemalloc has issues with SLES 12, so disable for now
IF (EXISTS "/etc/SuSE-release")
SET(JEMALLOC_LIBRARIES "")
ELSE ()
INCLUDE (FindJeMalloc.cmake)
if (NOT JEMALLOC_FOUND)
message(FATAL_ERROR "jemalloc not found!")
SET(JEMALLOC_LIBRARIES "")
endif()
ENDIF ()
FIND_PROGRAM(AWK_EXECUTABLE awk DOC "path to the awk executable") FIND_PROGRAM(AWK_EXECUTABLE awk DOC "path to the awk executable")
if(NOT AWK_EXECUTABLE) if(NOT AWK_EXECUTABLE)
message(FATAL_ERROR "awk not found!") message(FATAL_ERROR "awk not found!")
@ -163,7 +174,7 @@ SET (ENGINE_LOCALDIR "${INSTALL_ENGINE}/local")
SET (ENGINE_MYSQLDIR "${INSTALL_ENGINE}/mysql") SET (ENGINE_MYSQLDIR "${INSTALL_ENGINE}/mysql")
SET (ENGINE_TOOLSDIR "${INSTALL_ENGINE}/tools") SET (ENGINE_TOOLSDIR "${INSTALL_ENGINE}/tools")
SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot ${Boost_LIBRARIES} xml2 pthread rt libmysql_client) SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot ${Boost_LIBRARIES} xml2 pthread rt libmysql_client ${JEMALLOC_LIBRARIES})
SET (ENGINE_OAM_LIBS oamcpp alarmmanager) SET (ENGINE_OAM_LIBS oamcpp alarmmanager)
SET (ENGINE_BRM_LIBS brm idbdatafile cacheutils rwlock ${ENGINE_OAM_LIBS} ${ENGINE_COMMON_LIBS}) SET (ENGINE_BRM_LIBS brm idbdatafile cacheutils rwlock ${ENGINE_OAM_LIBS} ${ENGINE_COMMON_LIBS})
SET (ENGINE_EXEC_LIBS joblist execplan windowfunction joiner rowgroup funcexp udfsdk regr dataconvert common compress querystats querytele thrift threadpool ${ENGINE_BRM_LIBS}) SET (ENGINE_EXEC_LIBS joblist execplan windowfunction joiner rowgroup funcexp udfsdk regr dataconvert common compress querystats querytele thrift threadpool ${ENGINE_BRM_LIBS})

44
FindJeMalloc.cmake Normal file
View File

@ -0,0 +1,44 @@
# - Try to find jemalloc headers and libraries.
#
# Usage of this module as follows:
#
# find_package(JeMalloc)
#
# Variables used by this module, they can change the default behaviour and need
# to be set before calling find_package:
#
# JEMALLOC_ROOT_DIR Set this variable to the root installation of
# jemalloc if the module has problems finding
# the proper installation path.
#
# Variables defined by this module:
#
# JEMALLOC_FOUND System has jemalloc libs/headers
# JEMALLOC_LIBRARIES The jemalloc library/libraries
# JEMALLOC_INCLUDE_DIR The location of jemalloc headers
find_path(JEMALLOC_ROOT_DIR
NAMES include/jemalloc/jemalloc.h
)
find_library(JEMALLOC_LIBRARIES
NAMES jemalloc
HINTS ${JEMALLOC_ROOT_DIR}/lib
)
find_path(JEMALLOC_INCLUDE_DIR
NAMES jemalloc/jemalloc.h
HINTS ${JEMALLOC_ROOT_DIR}/include
)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(JeMalloc DEFAULT_MSG
JEMALLOC_LIBRARIES
JEMALLOC_INCLUDE_DIR
)
mark_as_advanced(
JEMALLOC_ROOT_DIR
JEMALLOC_LIBRARIES
JEMALLOC_INCLUDE_DIR
)

6
cpackEngineDEB.cmake
View File

@ -65,11 +65,11 @@ if (EXISTS "/etc/debian_version")
set(DEBIAN_VERSION_NUMBER "${CMAKE_MATCH_1}") set(DEBIAN_VERSION_NUMBER "${CMAKE_MATCH_1}")
endif () endif ()
if ("${DEBIAN_VERSION_NUMBER}" EQUAL "8") if ("${DEBIAN_VERSION_NUMBER}" EQUAL "8")
SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libreadline-dev, rsync, net-tools, libboost-all-dev, mariadb-columnstore-libs, mariadb-columnstore-server, libsnappy1") SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libreadline-dev, rsync, net-tools, libboost-all-dev, mariadb-columnstore-libs, mariadb-columnstore-server, libsnappy1, libjemalloc1")
elseif ("${DEBIAN_VERSION_NUMBER}" EQUAL "9") elseif ("${DEBIAN_VERSION_NUMBER}" EQUAL "9")
SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libreadline-dev, rsync, net-tools, libboost-all-dev, mariadb-columnstore-libs, mariadb-columnstore-server, libsnappy1v5, libreadline5") SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libreadline-dev, rsync, net-tools, libboost-all-dev, mariadb-columnstore-libs, mariadb-columnstore-server, libsnappy1v5, libreadline5, libjemalloc1")
else() else()
SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libboost-all-dev, libreadline-dev, rsync, libsnappy1v5, net-tools") SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libboost-all-dev, libreadline-dev, rsync, libsnappy1v5, net-tools, libjemalloc1")
endif () endif ()
SET(CPACK_DEBIAN_STORAGE-ENGINE_PACKAGE_DEPENDS "mariadb-columnstore-libs") SET(CPACK_DEBIAN_STORAGE-ENGINE_PACKAGE_DEPENDS "mariadb-columnstore-libs")

4
cpackEngineRPM.cmake
View File

@ -91,9 +91,9 @@ if (${REDHAT_VERSION_NUMBER} EQUAL 6)
# Disable auto require as this will also try to pull Boost via RPM # Disable auto require as this will also try to pull Boost via RPM
SET(CPACK_RPM_PACKAGE_AUTOREQPROV " no") SET(CPACK_RPM_PACKAGE_AUTOREQPROV " no")
elseif (${SUSE_VERSION_NUMBER} EQUAL 12) elseif (${SUSE_VERSION_NUMBER} EQUAL 12)
SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost-devel >= 1.54.0" "mariadb-columnstore-libs" "libsnappy1") SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost-devel >= 1.54.0" "mariadb-columnstore-libs" "libsnappy1" "jemalloc")
else () else ()
SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost >= 1.53.0" "mariadb-columnstore-libs" "snappy") SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost >= 1.53.0" "mariadb-columnstore-libs" "snappy" "jemalloc")
endif() endif()
SETA(CPACK_RPM_storage-engine_PACKAGE_REQUIRES "mariadb-columnstore-libs") SETA(CPACK_RPM_storage-engine_PACKAGE_REQUIRES "mariadb-columnstore-libs")

56
dbcon/joblist/groupconcat.cpp
View File

@ -344,8 +344,6 @@ void GroupConcatAgUM::merge(const rowgroup::Row& inRow, int64_t i)
joblist::GroupConcatAgUM* gccAg = *((joblist::GroupConcatAgUM**)(data + inRow.getOffset(i))); joblist::GroupConcatAgUM* gccAg = *((joblist::GroupConcatAgUM**)(data + inRow.getOffset(i)));
fConcator->merge(gccAg->concator().get()); fConcator->merge(gccAg->concator().get());
// don't reset
// gccAg->orderBy().reset();
} }
@ -393,12 +391,11 @@ GroupConcator::~GroupConcator()
void GroupConcator::initialize(const rowgroup::SP_GroupConcat& gcc) void GroupConcator::initialize(const rowgroup::SP_GroupConcat& gcc)
{ {
// MCOL-901 This value comes from the Server and it is
// too high(3MB) to allocate it for every instance.
fGroupConcatLen = gcc->fSize; fGroupConcatLen = gcc->fSize;
fCurrentLength -= strlen(gcc->fSeparator.c_str()); fCurrentLength -= strlen(gcc->fSeparator.c_str());
fOutputString.reset(new uint8_t[fGroupConcatLen + 2]);
memset(fOutputString.get(), 0, fGroupConcatLen + 2);
fConstCols = gcc->fConstCols; fConstCols = gcc->fConstCols;
fConstantLen = strlen(gcc->fSeparator.c_str()); fConstantLen = strlen(gcc->fSeparator.c_str());
@ -477,7 +474,6 @@ void GroupConcator::outputRow(std::ostringstream& oss, const rowgroup::Row& row)
case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::TEXT:
{ {
oss << row.getStringField(*i).c_str(); oss << row.getStringField(*i).c_str();
//oss << row.getStringField(*i);
break; break;
} }
@ -626,11 +622,6 @@ int64_t GroupConcator::lengthEstimate(const rowgroup::Row& row)
{ {
int64_t colWidth = row.getStringLength(*i); int64_t colWidth = row.getStringLength(*i);
fieldLen += colWidth; // getStringLength() does the same thing as below fieldLen += colWidth; // getStringLength() does the same thing as below
//assert(!row.usesStringTable());
//int64_t colWidth = row.getColumnWidth(*i);
//uint8_t* pStr = row.getData() + row.getOffset(*i);
//while ((*pStr++ > 0) && (fieldLen < colWidth))
// fieldLen++;
break; break;
} }
@ -759,7 +750,6 @@ void GroupConcatOrderBy::initialize(const rowgroup::SP_GroupConcat& gcc)
uint64_t GroupConcatOrderBy::getKeyLength() const uint64_t GroupConcatOrderBy::getKeyLength() const
{ {
// only distinct the concatenated columns // only distinct the concatenated columns
//return (fRow0.getOffset(fConcatColumns.size()) - 2);
return fConcatColumns.size() - 1; // cols 0 to fConcatColumns will be conpared return fConcatColumns.size() - 1; // cols 0 to fConcatColumns will be conpared
} }
@ -778,8 +768,6 @@ void GroupConcatOrderBy::processRow(const rowgroup::Row& row)
if (fCurrentLength < fGroupConcatLen) if (fCurrentLength < fGroupConcatLen)
{ {
copyRow(row, &fRow0); copyRow(row, &fRow0);
//cout << "length < GB limit: " << fRow0.toString() << endl;
//memcpy(fRow0.getData(), row.getData(), row.getSize());
// the RID is no meaning here, use it to store the estimated length. // the RID is no meaning here, use it to store the estimated length.
int16_t estLen = lengthEstimate(fRow0); int16_t estLen = lengthEstimate(fRow0);
fRow0.setRid(estLen); fRow0.setRid(estLen);
@ -869,7 +857,6 @@ void GroupConcatOrderBy::merge(GroupConcator* gc)
if (fDistinct) if (fDistinct)
fDistinctMap->insert(row.fData); fDistinctMap->insert(row.fData);
//fDistinctMap->insert(make_pair((row.fData+2), row.fData));
} }
else if (fOrderByCond.size() > 0 && fRule.less(row.fData, fOrderByQueue.top().fData)) else if (fOrderByCond.size() > 0 && fRule.less(row.fData, fOrderByQueue.top().fData))
@ -883,8 +870,6 @@ void GroupConcatOrderBy::merge(GroupConcator* gc)
{ {
fDistinctMap->erase(swapRow.fData); fDistinctMap->erase(swapRow.fData);
fDistinctMap->insert(row.fData); fDistinctMap->insert(row.fData);
//fDistinctMap->erase(fDistinctMap->find(swapRow.fData + 2));
//fDistinctMap->insert(make_pair((row.fData+2), row.fData));
} }
row1.setData(row.fData); row1.setData(row.fData);
@ -900,7 +885,6 @@ void GroupConcatOrderBy::merge(GroupConcator* gc)
void GroupConcatOrderBy::getResult(uint8_t* buff, const string& sep) void GroupConcatOrderBy::getResult(uint8_t* buff, const string& sep)
{ {
#if 1
ostringstream oss; ostringstream oss;
bool addSep = false; bool addSep = false;
@ -926,28 +910,12 @@ void GroupConcatOrderBy::getResult(uint8_t* buff, const string& sep)
rowStack.pop(); rowStack.pop();
} }
strncpy((char*) buff, oss.str().c_str(), fGroupConcatLen); size_t resultSize = oss.str().size();
#else fOutputString.reset(new uint8_t[resultSize + 2]);
ostringstream oss; memset(fOutputString.get(), 0, resultSize + 2);
bool addSep = false;
priority_queue<OrderByRow>::reverse_iterator rit;
for (rit = fOrderByQueue.rbegin(); rit != fOrderByQueue.rend(); ++rit) strncpy((char*)fOutputString.get(),
{ oss.str().c_str(), resultSize);
if (addSep)
oss << sep;
else
addSep = true;
const OrderByRow& topRow = *rit;
fRow0.setData(topRow.fData);
outputRow(oss, fRow0);
}
fOrderByQueue.clear();
strncpy((char*) buff, oss.str().c_str(), fGroupConcatLen);
#endif
} }
uint8_t* GroupConcator::getResult(const string& sep) uint8_t* GroupConcator::getResult(const string& sep)
@ -1017,7 +985,6 @@ void GroupConcatNoOrder::initialize(const rowgroup::SP_GroupConcat& gcc)
throw IDBExcept(fErrorCode); throw IDBExcept(fErrorCode);
} }
//fData.reset(new uint8_t[fRowGroup.getDataSize(fRowsPerRG)]);
fData.reinit(fRowGroup, fRowsPerRG); fData.reinit(fRowGroup, fRowsPerRG);
fRowGroup.setData(&fData); fRowGroup.setData(&fData);
fRowGroup.resetRowGroup(0); fRowGroup.resetRowGroup(0);
@ -1032,7 +999,6 @@ void GroupConcatNoOrder::processRow(const rowgroup::Row& row)
if (fCurrentLength < fGroupConcatLen && concatColIsNull(row) == false) if (fCurrentLength < fGroupConcatLen && concatColIsNull(row) == false)
{ {
copyRow(row, &fRow); copyRow(row, &fRow);
//memcpy(fRow.getData(), row.getData(), row.getSize());
// the RID is no meaning here, use it to store the estimated length. // the RID is no meaning here, use it to store the estimated length.
int16_t estLen = lengthEstimate(fRow); int16_t estLen = lengthEstimate(fRow);
@ -1056,7 +1022,6 @@ void GroupConcatNoOrder::processRow(const rowgroup::Row& row)
fDataQueue.push(fData); fDataQueue.push(fData);
fData.reinit(fRowGroup, fRowsPerRG); fData.reinit(fRowGroup, fRowsPerRG);
//fData.reset(new uint8_t[fRowGroup.getDataSize(fRowsPerRG)]);
fRowGroup.setData(&fData); fRowGroup.setData(&fData);
fRowGroup.resetRowGroup(0); fRowGroup.resetRowGroup(0);
fRowGroup.getRow(0, &fRow); fRowGroup.getRow(0, &fRow);
@ -1107,7 +1072,12 @@ void GroupConcatNoOrder::getResult(uint8_t* buff, const string& sep)
fDataQueue.pop(); fDataQueue.pop();
} }
strncpy((char*) buff, oss.str().c_str(), fGroupConcatLen); size_t resultSize = oss.str().size();
fOutputString.reset(new uint8_t[resultSize + 2]);
memset(fOutputString.get(), 0, resultSize + 2);
strncpy((char*)fOutputString.get(),
oss.str().c_str(), resultSize);
} }

25
dbcon/joblist/limitedorderby.cpp
View File

@ -173,10 +173,10 @@ void LimitedOrderBy::finalize()
if (fOrderByQueue.size() > 0) if (fOrderByQueue.size() > 0)
{ {
uint64_t newSize = fRowsPerRG * fRowGroup.getRowSize(); uint64_t memSizeInc = fRowsPerRG * fRowGroup.getRowSize();
fMemSize += newSize; fMemSize += memSizeInc;
if (!fRm->getMemory(newSize, fSessionMemLimit)) if (!fRm->getMemory(memSizeInc, fSessionMemLimit))
{ {
cerr << IDBErrorInfo::instance()->errorMsg(fErrorCode) cerr << IDBErrorInfo::instance()->errorMsg(fErrorCode)
<< " @" << __FILE__ << ":" << __LINE__; << " @" << __FILE__ << ":" << __LINE__;
@ -185,10 +185,12 @@ void LimitedOrderBy::finalize()
uint64_t offset = 0; uint64_t offset = 0;
uint64_t i = 0; uint64_t i = 0;
// Reduce queue size by an offset value if it applicable.
uint64_t queueSizeWoOffset = fOrderByQueue.size() > fStart ?
fOrderByQueue.size() - fStart : 0;
list<RGData> tempRGDataList; list<RGData> tempRGDataList;
// Skip first LIMIT rows in the the RowGroup if ( fCount <= queueSizeWoOffset )
if ( fCount <= fOrderByQueue.size() )
{ {
offset = fCount % fRowsPerRG; offset = fCount % fRowsPerRG;
if(!offset && fCount > 0) if(!offset && fCount > 0)
@ -196,11 +198,11 @@ void LimitedOrderBy::finalize()
} }
else else
{ {
offset = fOrderByQueue.size() % fRowsPerRG; offset = queueSizeWoOffset % fRowsPerRG;
if(!offset && fOrderByQueue.size() > 0) if(!offset && queueSizeWoOffset > 0)
offset = fRowsPerRG; offset = fRowsPerRG;
} }
list<RGData>::iterator tempListIter = tempRGDataList.begin(); list<RGData>::iterator tempListIter = tempRGDataList.begin();
i = 0; i = 0;
@ -217,7 +219,6 @@ void LimitedOrderBy::finalize()
const OrderByRow& topRow = fOrderByQueue.top(); const OrderByRow& topRow = fOrderByQueue.top();
row1.setData(topRow.fData); row1.setData(topRow.fData);
copyRow(row1, &fRow0); copyRow(row1, &fRow0);
//cerr << "LimitedOrderBy::finalize fRow0 " << fRow0.toString() << endl;
fRowGroup.incRowCount(); fRowGroup.incRowCount();
offset--; offset--;
fRow0.prevRow(rSize); fRow0.prevRow(rSize);
@ -226,9 +227,9 @@ void LimitedOrderBy::finalize()
if(offset == (uint64_t)-1) if(offset == (uint64_t)-1)
{ {
tempRGDataList.push_front(fData); tempRGDataList.push_front(fData);
fMemSize += newSize; fMemSize += memSizeInc;
if (!fRm->getMemory(newSize, fSessionMemLimit)) if (!fRm->getMemory(memSizeInc, fSessionMemLimit))
{ {
cerr << IDBErrorInfo::instance()->errorMsg(fErrorCode) cerr << IDBErrorInfo::instance()->errorMsg(fErrorCode)
<< " @" << __FILE__ << ":" << __LINE__; << " @" << __FILE__ << ":" << __LINE__;

25
oamapps/mcsadmin/mcsadmin.cpp
View File

@ -759,8 +759,11 @@ int processCommand(string* arguments)
vector<uint32_t> srcDbroots; // all of the currently configured dbroots vector<uint32_t> srcDbroots; // all of the currently configured dbroots
vector<uint32_t> destDbroots; // srcDbroots - removeDbroots vector<uint32_t> destDbroots; // srcDbroots - removeDbroots
set<int>::iterator dbiter; set<int>::iterator dbiter;
#if _MSC_VER
if (arguments[1] == "start") if (_strnicmp(arguments[1].c_str(), "start", 5) == 0))
#else
if (strncasecmp(arguments[1].c_str(), "start", 5) == 0)
#endif
{ {
// Get a list of all the configured dbroots in the xml file. // Get a list of all the configured dbroots in the xml file.
DBRootConfigList dbRootConfigList; DBRootConfigList dbRootConfigList;
@ -772,7 +775,11 @@ int processCommand(string* arguments)
// The user may choose to redistribute in such a way as to // The user may choose to redistribute in such a way as to
// leave certain dbroots empty, presumably for later removal. // leave certain dbroots empty, presumably for later removal.
if (arguments[2] == "remove") #if _MSC_VER
if (_strnicmp(arguments[1].c_str(), "remove", 6) == 0))
#else
if (strncasecmp(arguments[1].c_str(), "remove", 6) == 0)
#endif
{ {
int dbroot; int dbroot;
bool error = false; bool error = false;
@ -891,7 +898,11 @@ int processCommand(string* arguments)
SendToWES(oam, bs); SendToWES(oam, bs);
} }
else if (arguments[1] == "stop") #if _MSC_VER
if (_strnicmp(arguments[1].c_str(), "stop", 4) == 0))
#else
if (strncasecmp(arguments[1].c_str(), "stop", 4) == 0)
#endif
{ {
ByteStream bs; ByteStream bs;
// message WES ID, sequence #, action id // message WES ID, sequence #, action id
@ -901,7 +912,11 @@ int processCommand(string* arguments)
bs.append((const ByteStream::byte*) &header, sizeof(header)); bs.append((const ByteStream::byte*) &header, sizeof(header));
SendToWES(oam, bs); SendToWES(oam, bs);
} }
else if (arguments[1] == "status") #if _MSC_VER
if (_strnicmp(arguments[1].c_str(), "status", 6) == 0))
#else
if (strncasecmp(arguments[1].c_str(), "status", 6) == 0)
#endif
{ {
ByteStream bs; ByteStream bs;
// message WES ID, sequence #, action id // message WES ID, sequence #, action id

234
oamapps/postConfigure/postConfigure.cpp
View File

@ -55,7 +55,6 @@
#include <arpa/inet.h> #include <arpa/inet.h>
#include <sys/socket.h> #include <sys/socket.h>
#include <ifaddrs.h> #include <ifaddrs.h>
#include <stdio.h>
#include <string.h> /* for strncpy */ #include <string.h> /* for strncpy */
#include <sys/types.h> #include <sys/types.h>
@ -111,8 +110,6 @@ typedef std::vector<ModuleIP> ModuleIpList;
void offLineAppCheck(); void offLineAppCheck();
bool setOSFiles(string parentOAMModuleName, int serverTypeInstall); bool setOSFiles(string parentOAMModuleName, int serverTypeInstall);
bool checkSaveConfigFile(); bool checkSaveConfigFile();
string getModuleName();
bool setModuleName(string moduleName);
bool updateBash(); bool updateBash();
bool makeModuleFile(string moduleName, string parentOAMModuleName); bool makeModuleFile(string moduleName, string parentOAMModuleName);
bool updateProcessConfig(); bool updateProcessConfig();
@ -126,6 +123,7 @@ bool singleServerDBrootSetup();
bool copyFstab(string moduleName); bool copyFstab(string moduleName);
bool attachVolume(string instanceName, string volumeName, string deviceName, string dbrootPath); bool attachVolume(string instanceName, string volumeName, string deviceName, string dbrootPath);
void singleServerConfigSetup(Config* sysConfig); void singleServerConfigSetup(Config* sysConfig);
std::string resolveHostNameToReverseDNSName(std::string hostname);
void remoteInstallThread(void*); void remoteInstallThread(void*);
@ -193,6 +191,7 @@ bool single_server_quick_install = false;
bool multi_server_quick_install = false; bool multi_server_quick_install = false;
bool amazon_quick_install = false; bool amazon_quick_install = false;
bool doNotResolveHostNames = false; bool doNotResolveHostNames = false;
bool resolveHostNamesToReverseDNSNames = false;
string DataFileEnvFile; string DataFileEnvFile;
@ -316,7 +315,7 @@ int main(int argc, char* argv[])
cout << " Enter one of the options within [], if available, or" << endl; cout << " Enter one of the options within [], if available, or" << endl;
cout << " Enter a new value" << endl << endl; cout << " Enter a new value" << endl << endl;
cout << endl; cout << endl;
cout << "Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count][-numBlocksPct][-totalUmMemory]" << endl; cout << "Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count][-x][-xr][-numBlocksPct][-totalUmMemory]" << endl;
cout << " -h Help" << endl; cout << " -h Help" << endl;
cout << " -c Config File to use to extract configuration data, default is Columnstore.xml.rpmsave" << endl; cout << " -c Config File to use to extract configuration data, default is Columnstore.xml.rpmsave" << endl;
cout << " -u Upgrade, Install using the Config File from -c, default to Columnstore.xml.rpmsave" << endl; cout << " -u Upgrade, Install using the Config File from -c, default to Columnstore.xml.rpmsave" << endl;
@ -332,6 +331,7 @@ int main(int argc, char* argv[])
cout << " -pm-ip-addrs Performance Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl; cout << " -pm-ip-addrs Performance Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl;
cout << " -um-ip-addrs User Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl; cout << " -um-ip-addrs User Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl;
cout << " -x Do not resolve IP Addresses from host names" << endl; cout << " -x Do not resolve IP Addresses from host names" << endl;
cout << " -xr Resolve host names into their reverse DNS host names. Only applied in combination with -x" << endl;
cout << " -numBlocksPct amount of physical memory to utilize for disk block caching" << endl; cout << " -numBlocksPct amount of physical memory to utilize for disk block caching" << endl;
cout << " (percentages of the total memory need to be stated without suffix, explcit values with suffixes M or G)" << endl; cout << " (percentages of the total memory need to be stated without suffix, explcit values with suffixes M or G)" << endl;
cout << " -totalUmMemory amount of physical memory to utilize for joins, intermediate results and set operations on the UM" << endl; cout << " -totalUmMemory amount of physical memory to utilize for joins, intermediate results and set operations on the UM" << endl;
@ -341,6 +341,10 @@ int main(int argc, char* argv[])
else if (string("-x") == argv[i]) else if (string("-x") == argv[i])
{ {
doNotResolveHostNames = true; doNotResolveHostNames = true;
}
else if (string("-xr") == argv[i])
{
resolveHostNamesToReverseDNSNames = true;
} }
else if( string("-qs") == argv[i] ) else if( string("-qs") == argv[i] )
{ {
@ -517,7 +521,7 @@ int main(int argc, char* argv[])
else else
{ {
cout << " ERROR: Invalid Argument = " << argv[i] << endl; cout << " ERROR: Invalid Argument = " << argv[i] << endl;
cout << " Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count][-numBlocksPct][-totalUmMemory]" << endl; cout << " Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count][-x][-xr][-numBlocksPct][-totalUmMemory]" << endl;
exit (1); exit (1);
} }
} }
@ -658,9 +662,14 @@ int main(int argc, char* argv[])
if (moduleconfig.hostConfigList.size() > 0 ) if (moduleconfig.hostConfigList.size() > 0 )
{ {
HostConfigList::iterator pt1 = moduleconfig.hostConfigList.begin(); HostConfigList::iterator pt1 = moduleconfig.hostConfigList.begin();
string PM1ipAdd = (*pt1).IPAddr;
// MCOL-1607. The 'am I pm1?' check below requires an ipaddr.
string PM1ipAdd = oam.getIPAddress((*pt1).IPAddr.c_str());
if (PM1ipAdd.empty())
PM1ipAdd = (*pt1).IPAddr; // this is what it was doing before
//cout << PM1ipAdd << endl; //cout << PM1ipAdd << endl;
if ( PM1ipAdd != "127.0.0.1" ) if ( PM1ipAdd != "127.0.0.1" )
{ {
if ( PM1ipAdd != "0.0.0.0") if ( PM1ipAdd != "0.0.0.0")
@ -2566,7 +2575,12 @@ int main(int argc, char* argv[])
//get IP Address //get IP Address
string IPAddress; string IPAddress;
if (doNotResolveHostNames) if (doNotResolveHostNames)
IPAddress = newModuleHostName; if (resolveHostNamesToReverseDNSNames) {
IPAddress = resolveHostNameToReverseDNSName(newModuleHostName);
}
else {
IPAddress = newModuleHostName;
}
else else
IPAddress = oam.getIPAddress( newModuleHostName); IPAddress = oam.getIPAddress( newModuleHostName);
@ -2593,6 +2607,13 @@ int main(int argc, char* argv[])
callFree(pcommand); callFree(pcommand);
} }
if (!doNotResolveHostNames)
{
string ugh = oam.getIPAddress(newModuleIPAddr);
if (ugh.length() > 0)
newModuleIPAddr = ugh;
}
if (newModuleIPAddr == "127.0.0.1" || newModuleIPAddr == "0.0.0.0" || newModuleIPAddr == "128.0.0.1") if (newModuleIPAddr == "127.0.0.1" || newModuleIPAddr == "0.0.0.0" || newModuleIPAddr == "128.0.0.1")
{ {
@ -6550,7 +6571,12 @@ bool glusterSetup(string password, bool doNotResolveHostNames)
//get IP Address //get IP Address
string IPAddress; string IPAddress;
if (doNotResolveHostNames) if (doNotResolveHostNames)
IPAddress = moduleHostName; if (resolveHostNamesToReverseDNSNames) {
IPAddress = resolveHostNameToReverseDNSName(moduleHostName);
}
else {
IPAddress = moduleHostName;
}
else else
IPAddress = oam.getIPAddress( moduleHostName); IPAddress = oam.getIPAddress( moduleHostName);
@ -6566,7 +6592,7 @@ bool glusterSetup(string password, bool doNotResolveHostNames)
//prompt for IP address //prompt for IP address
while (true) while (true)
{ {
prompt = "Enter PM #" + oam.itoa(DataRedundancyConfigs[pm].pmID) + " IP Address of " + moduleHostName + " (" + moduleIPAddr + ") > "; prompt = "Enter PM #" + oam.itoa(DataRedundancyConfigs[pm].pmID) + " IP Address or hostname of " + moduleHostName + " (" + moduleIPAddr + ") > ";
pcommand = callReadline(prompt.c_str()); pcommand = callReadline(prompt.c_str());
if (pcommand) if (pcommand)
@ -6575,6 +6601,13 @@ bool glusterSetup(string password, bool doNotResolveHostNames)
callFree(pcommand); callFree(pcommand);
} }
if (!doNotResolveHostNames)
{
string ugh = oam.getIPAddress(moduleIPAddr);
if (ugh.length() > 0)
moduleIPAddr = ugh;
}
if (moduleIPAddr == "127.0.0.1" || moduleIPAddr == "0.0.0.0" || moduleIPAddr == "128.0.0.1") if (moduleIPAddr == "127.0.0.1" || moduleIPAddr == "0.0.0.0" || moduleIPAddr == "128.0.0.1")
{ {
@ -7011,29 +7044,29 @@ bool glusterSetup(string password, bool doNotResolveHostNames)
command = "sudo gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-uid " + oam.itoa(user) + " >> /tmp/glusterCommands.txt 2>&1";; command = "sudo gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-uid " + oam.itoa(user) + " >> /tmp/glusterCommands.txt 2>&1";;
status = system(command.c_str()); status = system(command.c_str());
if (WEXITSTATUS(status) != 0 ) if (WEXITSTATUS(status) != 0)
{ {
cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl; cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl;
exit(1); exit(1);
} }
command = "sudo gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-gid " + oam.itoa(group) + " >> /tmp/glusterCommands.txt 2>&1";; command = "sudo gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-gid " + oam.itoa(group) + " >> /tmp/glusterCommands.txt 2>&1";;
status = system(command.c_str()); status = system(command.c_str());
if (WEXITSTATUS(status) != 0 ) if (WEXITSTATUS(status) != 0)
{ {
cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl; cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl;
exit(1); exit(1);
} }
command = "sudo gluster volume start dbroot" + oam.itoa(dbrootID) + " >> /tmp/glusterCommands.txt 2>&1"; command = "sudo gluster volume start dbroot" + oam.itoa(dbrootID) + " >> /tmp/glusterCommands.txt 2>&1";
status = system(command.c_str()); status = system(command.c_str());
if (WEXITSTATUS(status) != 0 ) if (WEXITSTATUS(status) != 0)
{ {
cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl; cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl;
exit(1); exit(1);
} }
} }
cout << "DONE" << endl; cout << "DONE" << endl;
@ -7047,70 +7080,89 @@ bool glusterSetup(string password, bool doNotResolveHostNames)
void singleServerConfigSetup(Config* sysConfig) void singleServerConfigSetup(Config* sysConfig)
{ {
try try
{ {
sysConfig->setConfig("ExeMgr1", "IPAddr", "127.0.0.1"); sysConfig->setConfig("ExeMgr1", "IPAddr", "127.0.0.1");
sysConfig->setConfig("ExeMgr1", "Module", "pm1"); sysConfig->setConfig("ExeMgr1", "Module", "pm1");
sysConfig->setConfig("ProcMgr", "IPAddr", "127.0.0.1"); sysConfig->setConfig("ProcMgr", "IPAddr", "127.0.0.1");
sysConfig->setConfig("ProcMgr_Alarm", "IPAddr", "127.0.0.1"); sysConfig->setConfig("ProcMgr_Alarm", "IPAddr", "127.0.0.1");
sysConfig->setConfig("ProcStatusControl", "IPAddr", "127.0.0.1"); sysConfig->setConfig("ProcStatusControl", "IPAddr", "127.0.0.1");
sysConfig->setConfig("pm1_ProcessMonitor", "IPAddr", "127.0.0.1"); sysConfig->setConfig("pm1_ProcessMonitor", "IPAddr", "127.0.0.1");
sysConfig->setConfig("pm1_ServerMonitor", "IPAddr", "127.0.0.1"); sysConfig->setConfig("pm1_ServerMonitor", "IPAddr", "127.0.0.1");
sysConfig->setConfig("pm1_WriteEngineServer", "IPAddr", "127.0.0.1"); sysConfig->setConfig("pm1_WriteEngineServer", "IPAddr", "127.0.0.1");
sysConfig->setConfig("DDLProc", "IPAddr", "127.0.0.1"); sysConfig->setConfig("DDLProc", "IPAddr", "127.0.0.1");
sysConfig->setConfig("DMLProc", "IPAddr", "127.0.0.1"); sysConfig->setConfig("DMLProc", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS1", "IPAddr", "127.0.0.1"); sysConfig->setConfig("PMS1", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS2", "IPAddr", "127.0.0.1"); sysConfig->setConfig("PMS2", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS3", "IPAddr", "127.0.0.1"); sysConfig->setConfig("PMS3", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS4", "IPAddr", "127.0.0.1"); sysConfig->setConfig("PMS4", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS5", "IPAddr", "127.0.0.1"); sysConfig->setConfig("PMS5", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS6", "IPAddr", "127.0.0.1"); sysConfig->setConfig("PMS6", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS7", "IPAddr", "127.0.0.1"); sysConfig->setConfig("PMS7", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS8", "IPAddr", "127.0.0.1"); sysConfig->setConfig("PMS8", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS9", "IPAddr", "127.0.0.1"); sysConfig->setConfig("PMS9", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS10", "IPAddr", "127.0.0.1"); sysConfig->setConfig("PMS10", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS11", "IPAddr", "127.0.0.1"); sysConfig->setConfig("PMS11", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS12", "IPAddr", "127.0.0.1"); sysConfig->setConfig("PMS12", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS13", "IPAddr", "127.0.0.1"); sysConfig->setConfig("PMS13", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS14", "IPAddr", "127.0.0.1"); sysConfig->setConfig("PMS14", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS15", "IPAddr", "127.0.0.1"); sysConfig->setConfig("PMS15", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS16", "IPAddr", "127.0.0.1"); sysConfig->setConfig("PMS16", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS17", "IPAddr", "127.0.0.1"); sysConfig->setConfig("PMS17", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS18", "IPAddr", "127.0.0.1"); sysConfig->setConfig("PMS18", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS19", "IPAddr", "127.0.0.1"); sysConfig->setConfig("PMS19", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS20", "IPAddr", "127.0.0.1"); sysConfig->setConfig("PMS20", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS21", "IPAddr", "127.0.0.1"); sysConfig->setConfig("PMS21", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS22", "IPAddr", "127.0.0.1"); sysConfig->setConfig("PMS22", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS23", "IPAddr", "127.0.0.1"); sysConfig->setConfig("PMS23", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS24", "IPAddr", "127.0.0.1"); sysConfig->setConfig("PMS24", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS25", "IPAddr", "127.0.0.1"); sysConfig->setConfig("PMS25", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS26", "IPAddr", "127.0.0.1"); sysConfig->setConfig("PMS26", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS27", "IPAddr", "127.0.0.1"); sysConfig->setConfig("PMS27", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS28", "IPAddr", "127.0.0.1"); sysConfig->setConfig("PMS28", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS29", "IPAddr", "127.0.0.1"); sysConfig->setConfig("PMS29", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS30", "IPAddr", "127.0.0.1"); sysConfig->setConfig("PMS30", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS31", "IPAddr", "127.0.0.1"); sysConfig->setConfig("PMS31", "IPAddr", "127.0.0.1");
sysConfig->setConfig("PMS32", "IPAddr", "127.0.0.1"); sysConfig->setConfig("PMS32", "IPAddr", "127.0.0.1");
sysConfig->setConfig("SystemModuleConfig", "ModuleCount2", "0"); sysConfig->setConfig("SystemModuleConfig", "ModuleCount2", "0");
sysConfig->setConfig("SystemModuleConfig", "ModuleIPAddr1-1-3", "127.0.0.1"); sysConfig->setConfig("SystemModuleConfig", "ModuleIPAddr1-1-3", "127.0.0.1");
sysConfig->setConfig("SystemModuleConfig", "ModuleHostName1-1-3", "localhost"); sysConfig->setConfig("SystemModuleConfig", "ModuleHostName1-1-3", "localhost");
sysConfig->setConfig("DBRM_Controller", "IPAddr", "127.0.0.1"); sysConfig->setConfig("DBRM_Controller", "IPAddr", "127.0.0.1");
sysConfig->setConfig("DBRM_Worker1", "IPAddr", "127.0.0.1"); sysConfig->setConfig("DBRM_Worker1", "IPAddr", "127.0.0.1");
sysConfig->setConfig("DBRM_Worker1", "Module", "pm1"); sysConfig->setConfig("DBRM_Worker1", "Module", "pm1");
sysConfig->setConfig("DBBC", "NumBlocksPct", "50"); sysConfig->setConfig("DBBC", "NumBlocksPct", "50");
sysConfig->setConfig("Installation", "InitialInstallFlag", "y"); sysConfig->setConfig("Installation", "InitialInstallFlag", "y");
sysConfig->setConfig("Installation", "SingleServerInstall", "y"); sysConfig->setConfig("Installation", "SingleServerInstall", "y");
sysConfig->setConfig("HashJoin", "TotalUmMemory", "25%"); sysConfig->setConfig("HashJoin", "TotalUmMemory", "25%");
} }
catch (...) catch (...)
{ {
cout << "ERROR: Problem setting for Single Server in the MariaDB ColumnStore System Configuration file" << endl; cout << "ERROR: Problem setting for Single Server in the MariaDB ColumnStore System Configuration file" << endl;
exit(1); exit(1);
} }
return; return;
} }
/**
Resolves the given hostname into its reverse DNS name.
@param hostname the hostname to resolve.
@return the reverse dns name of given hostname or an empty string in case the hostname could not be resolved.
*/
std::string resolveHostNameToReverseDNSName(std::string hostname) {
struct hostent *hp = gethostbyname(hostname.c_str());
if (hp == NULL) {
std::cout << "Error: Couldn't resolve hostname " << hostname << " to ip address" << std::endl;
return "";
}
struct hostent *rl = gethostbyaddr(hp->h_addr_list[0], sizeof hp->h_addr_list[0], AF_INET);
if (rl == NULL) {
std::cout << "Error: Couldn't resolve ip address of hostname " << hostname << " back to a hostname" << std::endl;
return "";
}
hostname = rl->h_name;
return hostname;
}
// vim:ts=4 sw=4: // vim:ts=4 sw=4:

13
utils/rowgroup/rowgroup.cpp
View File

@ -1207,12 +1207,13 @@ int64_t Row::getSignedNullValue(uint32_t colIndex) const
RowGroup::RowGroup() : columnCount(0), data(NULL), rgData(NULL), strings(NULL), RowGroup::RowGroup() : columnCount(0), data(NULL), rgData(NULL), strings(NULL),
useStringTable(true), hasLongStringField(false), sTableThreshold(20) useStringTable(true), hasLongStringField(false), sTableThreshold(20)
{ {
oldOffsets.reserve(1024); // 1024 is too generous to waste.
oids.reserve(1024); oldOffsets.reserve(10);
keys.reserve(1024); oids.reserve(10);
types.reserve(1024); keys.reserve(10);
scale.reserve(1024); types.reserve(10);
precision.reserve(1024); scale.reserve(10);
precision.reserve(10);
} }
RowGroup::RowGroup(uint32_t colCount, RowGroup::RowGroup(uint32_t colCount,

17
writeengine/shared/we_convertor.cpp
View File

@ -194,15 +194,20 @@ long long Convertor::convertDecimalString(
// range check against int64 // range check against int64
if (dval > LLONG_MAX || dval < LLONG_MIN) if (dval > LLONG_MAX)
{
errno = ERANGE; errno = ERANGE;
return LLONG_MAX;
}
if (dval < LLONG_MIN)
{
errno = ERANGE;
return LLONG_MIN;
}
errno = 0;
else
errno = 0;
ret = dval; ret = dval;
// get the fractional part of what's left & round ret up or down. // get the fractional part of what's left & round ret up or down.
dval -= ret; dval -= ret;
if (dval >= 0.5 && ret < LLONG_MAX) if (dval >= 0.5 && ret < LLONG_MAX)