database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-07 03:22:57 +03:00
Code Activity

Merge branch 'develop-1.1' into 1.1-merge-up-20180817

Browse Source
This commit is contained in:
Andrew Hutchings
2018-08-17 16:17:13 +01:00
parent e0a3f4278b fdb4ef7f79
commit 03da6df473
47 changed files with 1526 additions and 756 deletions
Download Patch File Download Diff File Expand all files Collapse all files

156
utils/clusterTester/columnstoreClusterTester.sh
View File

@@ -10,7 +10,7 @@ CHECK=true
REPORTPASS=true
LOGFILE=""
OS_LIST=("centos6" "centos7" "debian8" "debian9" "suse12" "ubuntu16")
OS_LIST=("centos6" "centos7" "debian8" "debian9" "suse12" "ubuntu16" "ubuntu18")
NODE_IPADDRESS=""
@@ -37,7 +37,7 @@ checkContinue() {
}
###
# Print Fucntions
# Print Functions
###
helpPrint () {
@@ -57,7 +57,7 @@ helpPrint () {
echo ""
echo "Additional information on Tool is documented at:"
echo ""
echo "https://mariadb.com/kb/en/mariadb/*****/"
echo "https://mariadb.com/kb/en/library/mariadb-columnstore-cluster-test-tool/"
echo ""
echo "Items that are checked:"
echo " Node Ping test"
@@ -65,6 +65,7 @@ helpPrint () {
echo " ColumnStore Port test"
echo " OS version"
echo " Locale settings"
echo " Umask settings"
echo " Firewall settings"
echo " Date/time settings"
echo " Dependent packages installed"
@@ -326,16 +327,18 @@ checkSSH()
rc="$?"
if [ $rc -eq 0 ] || ( [ $rc -eq 2 ] && [ $OS == "suse12" ] ) ; then
if [ $PASSWORD == "ssh" ] ; then
echo $ipadd " Node Passed SSH login test using ssh-keys"
echo $ipadd " Node Passed SSH login test using ssh-keys"
else
echo $ipadd " Node Passed SSH login test using user password"
echo $ipadd " Node Passed SSH login test using user password"
fi
else
if [ $PASSWORD == "ssh" ] ; then
echo $ipadd " Node ${bold}Failed${normal} SSH login test using ssh-keys"
echo $ipadd " Node ${bold}Failed${normal} SSH login test using ssh-keys"
else
echo $ipadd " Node ${bold}Failed${normal} SSH login test using user password"
echo $ipadd " Node ${bold}Failed${normal} SSH login test using user password"
fi
echo "Error - Fix the SSH login issue and rerun test"
exit 1
fi
done
@@ -489,12 +492,47 @@ checkLocale()
fi
}
checkSELINUX()
checkLocalUMASK()
{
# UMASK check
#
echo ""
echo "** Run Local UMASK check"
echo ""
pass=true
filename=UMASKtest
rm -f $filename
touch $filename
permission=$(stat -c "%A" "$filename")
result=${permission:4:1}
if [ ${result} == "r" ] ; then
result=${permission:7:1}
if [ ${result} == "r" ] ; then
echo "UMASK local setting test passed"
else
echo "${bold}Warning${normal}, UMASK test failed, check local UMASK setting. Requirement is set to 0022"
pass=false
fi
else
echo "${bold}Warning${normal}, UMASK test failed, check local UMASK setting. Requirement is set to 0022"
pass=false
fi
if ! $pass; then
checkContinue
fi
rm -f $filename
}
checkLocalSELINUX()
{
# SELINUX check
#
echo ""
echo "** Run SELINUX check"
echo "** Run Local SELINUX check"
echo ""
pass=true
@@ -511,21 +549,86 @@ checkSELINUX()
echo "Local Node SELINUX setting is Not Enabled"
fi
for ipadd in "${NODE_IPADDRESS[@]}"; do
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /etc/selinux/config > /tmp/remote_scp_get_check 2>&1`
if [ "$?" -ne 0 ]; then
echo "$ipadd Node SELINUX setting is Not Enabled"
else
`cat config | grep SELINUX | grep enforcing > /tmp/selinux_check 2>&1`
if [ "$?" -eq 0 ]; then
echo "${bold}Warning${normal}, $ipadd SELINUX setting is Enabled, check port test results"
pass=false
else
echo "$ipadd Node SELINUX setting is Not Enabled"
fi
`rm -f config`
fi
done
if ! $pass; then
checkContinue
fi
}
checkUMASK()
{
# UMASK check
#
echo ""
echo "** Run UMASK check"
echo ""
pass=true
for ipadd in "${NODE_IPADDRESS[@]}"; do
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'rm -f UMASKtest;touch UMASKtest;echo $(stat -c "%A" "UMASKtest") > test.log' > /tmp/remote_command_check 2>&1`
if [ "$?" -eq 0 ]; then
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd Calpont1 test.log >> /tmp/remote_scp_get 2>&1`
if [ "$?" -eq 0 ]; then
permission=`cat test.log`
result=${permission:4:1}
if [ ${result} == "r" ] ; then
result=${permission:7:1}
if [ ${result} == "r" ] ; then
echo "$ipadd Node UMASK setting test passed"
else
echo "${bold}Warning${normal}, $ipadd Node UMASK test failed, check UMASK setting. Requirement is set to 0022"
pass=false
fi
else
echo "${bold}Warning${normal}, $ipadd Node UMASK test failed, check UMASK setting. Requirement is set to 0022"
pass=false
fi
else
echo "${bold}Warning${normal}, $ipadd UMASK test failed, remote_scp_get.sh error, check /tmp/remote_scp_get"
pass=false
fi
else
echo "${bold}Warning${normal}, $ipadd UMASK test failed, remote_command.sh error, check /tmp/remote_command_check"
pass=false
fi
`rm -f test.log`
done
if ! $pass; then
checkContinue
fi
rm -f $filename
}
checkSELINUX()
{
# SELINUX check
#
echo ""
echo "** Run SELINUX check"
echo ""
pass=true
for ipadd in "${NODE_IPADDRESS[@]}"; do
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /etc/selinux/config > /tmp/remote_scp_get_check 2>&1`
if [ "$?" -ne 0 ]; then
echo "$ipadd Node SELINUX setting is Not Enabled"
else
`cat config | grep SELINUX | grep enforcing > /tmp/selinux_check 2>&1`
if [ "$?" -eq 0 ]; then
echo "${bold}Warning${normal}, $ipadd SELINUX setting is Enabled, check port test results"
pass=false
else
echo "$ipadd Node SELINUX setting is Not Enabled"
fi
`rm -f config`
fi
done
if ! $pass; then
checkContinue
fi
}
checkFirewalls()
@@ -949,7 +1052,7 @@ checkPackages()
declare -a UBUNTU_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline-dev" "rsync" "libsnappy1V5" "net-tools" "libnuma1" )
declare -a UBUNTU_PKG_NOT=("mariadb-server" "libmariadb18")
if [ "$OS" == "ubuntu16" ] ; then
if [ "$OS" == "ubuntu16" ] || [ "$OS" == "ubuntu18" ]; then
if [ ! `which dpkg 2>/dev/null` ] ; then
echo "${bold}Failed${normal}, Local Node ${bold}rpm${normal} package not installed"
pass=false
@@ -1307,12 +1410,15 @@ echo ""
checkLocalOS
checkLocalDir
checkLocalUMASK
checkLocalSELINUX
if [ "$IPADDRESSES" != "" ]; then
checkPing
checkSSH
checkRemoteDir
checkOS
checkLocale
checkUMASK
checkSELINUX
checkFirewalls
checkPorts

2
utils/clusterTester/os_detect.sh
View File

@@ -29,7 +29,7 @@ detectOS () {
echo Operating System name: $osPrettyName
echo Operating System tag: $osTag
case "$osTag" in
centos6|centos7|ubuntu16|debian8|suse12|debian9)
centos6|centos7|ubuntu16|debian8|suse12|debian9|ubuntu18)
;;
*)
echo OS not supported

70
utils/funcexp/func_case.cpp
View File

@@ -315,20 +315,13 @@ CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp,
CalpontSystemCatalog::ColType& resultType,
bool simpleCase)
{
FunctionParm::size_type n = fp.size();
uint64_t simple = simpleCase ? 1 : 0;
bool hasElse = (((fp.size()-simple) % 2) != 0); // if 1, then ELSE exist
if (simpleCase) // simple case has an expression
n -= 1; // remove expression from count of expression_i + result_i
bool hasElse = ((n % 2) != 0); // if 1, then ELSE exist
if (hasElse)
--n; // n now is an even number
uint64_t parmCount = hasElse ? (fp.size() - 2) : (fp.size() - 1);
uint64_t whereCount = hasElse ? (fp.size() - 2 + simpleCase) / 2 : (fp.size() - 1) / 2 + simpleCase;
idbassert((n % 2) == 0);
uint64_t whereCount = hasElse ? (fp.size() - 2 + simple) / 2 : (fp.size() - 1) / 2 + simple;
bool allStringO = true;
bool allStringR = true;
@@ -341,33 +334,24 @@ CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp,
for (uint64_t i = 0; i <= parmCount; i++)
{
// operation or result type
operation = ((i > 0) && (i <= whereCount));
// the result type of ELSE, if exists.
if (i == n)
{
if (!hasElse)
break;
if (simpleCase)
// for SimpleCase, we return the type of the case expression,
// which will always be in position 0.
if (i == 0 && simpleCase)
{
if (fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::CHAR &&
fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT &&
fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::VARCHAR)
{
// the case expression
if (fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::CHAR &&
fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT &&
fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::VARCHAR)
{
PredicateOperator op;
op.setOpType(oct, fp[i]->data()->resultType());
allStringO = false;
oct = op.operationType();
}
i += 1;
PredicateOperator op;
op.setOpType(oct, fp[i]->data()->resultType());
allStringO = false;
oct = op.operationType();
}
i += 1;
}
operation = false;
}
// operation or result type
operation = ((i > 0+simple) && (i <= whereCount));
if (fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::CHAR &&
fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT &&
@@ -378,10 +362,13 @@ CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp,
if (operation)
{
op.setOpType(oct, fp[i]->data()->resultType());
allStringO = false;
oct = op.operationType();
}
if (!simpleCase)
{
op.setOpType(oct, fp[i]->data()->resultType());
allStringO = false;
oct = op.operationType();
}
}
// If any parm is of string type, the result type should be string. (same as if)
else if (rct.colDataType != CalpontSystemCatalog::CHAR &&
@@ -457,6 +444,13 @@ bool Func_simple_case::getBoolVal(Row& row,
if (isNull)
return joblist::BIGINTNULL;
ParseTree* lop = parm[i]->left();
ParseTree* rop = parm[i]->right();
if (lop && rop)
{
return (reinterpret_cast<Operator*>(parm[i]->data()))->getBoolVal(row, isNull, lop, rop);
}
return parm[i]->data()->getBoolVal(row, isNull);
}

6
utils/joiner/tuplejoiner.cpp
View File

@@ -149,8 +149,7 @@ TupleJoiner::TupleJoiner(
for (uint32_t i = 0; i < smallKeyColumns.size(); i++)
{
discreteValues[i] = false;
if (isUnsigned(smallRG.getColType(i)))
if (isUnsigned(smallRG.getColTypes()[smallKeyColumns[i]]))
{
cpValues[i].push_back(static_cast<int64_t>(numeric_limits<uint64_t>::max()));
cpValues[i].push_back(0);
@@ -1033,8 +1032,7 @@ boost::shared_ptr<TupleJoiner> TupleJoiner::copyForDiskJoin()
for (uint32_t i = 0; i < smallKeyColumns.size(); i++)
{
ret->discreteValues[i] = false;
if (isUnsigned(smallRG.getColType(i)))
if (isUnsigned(smallRG.getColTypes()[smallKeyColumns[i]]))
{
ret->cpValues[i].push_back(static_cast<int64_t>(numeric_limits<uint64_t>::max()));
ret->cpValues[i].push_back(0);

65
utils/messageqcpp/messagequeue.cpp
View File

@@ -157,24 +157,42 @@ void MessageQueueClient::setup(bool syncProto)
{
string otherEndIPStr;
string otherEndPortStr;
uint16_t port;
struct addrinfo hints, *servinfo;
int rc = 0;
otherEndIPStr = fConfig->getConfig(fOtherEnd, "IPAddr");
otherEndPortStr = fConfig->getConfig(fOtherEnd, "Port");
if (otherEndIPStr.length() == 0) otherEndIPStr = "127.0.0.1";
if (otherEndPortStr.length() == 0 || (port = static_cast<uint16_t>(strtol(otherEndPortStr.c_str(), 0, 0))) == 0)
if (otherEndPortStr.length() == 0 || static_cast<uint16_t>(strtol(otherEndPortStr.c_str(), 0, 0)) == 0)
{
string msg = "MessageQueueClient::MessageQueueClient: config error: Invalid/Missing Port attribute";
string msg = "MessageQueueClient::setup(): config error: Invalid/Missing Port attribute";
throw runtime_error(msg);
}
memset(&fServ_addr, 0, sizeof(fServ_addr));
sockaddr_in* sinp = reinterpret_cast<sockaddr_in*>(&fServ_addr);
sinp->sin_family = AF_INET;
sinp->sin_port = htons(port);
sinp->sin_addr.s_addr = inet_addr(otherEndIPStr.c_str());
memset(&hints, 0, sizeof hints);
// ATM We support IPv4 only.
hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM;
if( !(rc = getaddrinfo(otherEndIPStr.c_str(), otherEndPortStr.c_str(), &hints, &servinfo)) )
{
memset(&fServ_addr, 0, sizeof(fServ_addr));
sockaddr_in* sinp = reinterpret_cast<sockaddr_in*>(&fServ_addr);
*sinp = *reinterpret_cast<sockaddr_in*>(servinfo->ai_addr);
freeaddrinfo(servinfo);
}
else
{
string msg = "MessageQueueClient::setup(): ";
msg.append(gai_strerror(rc));
logging::Message::Args args;
logging::LoggingID li(31);
args.add(msg);
fLogger.logMessage(logging::LOG_TYPE_ERROR, logging::M0000, args, li);
}
#ifdef SKIP_IDB_COMPRESSION
fClientSock.setSocketImpl(new InetStreamSocket());
@@ -200,15 +218,34 @@ MessageQueueClient::MessageQueueClient(const string& otherEnd, Config* config, b
setup(syncProto);
}
MessageQueueClient::MessageQueueClient(const string& ip, uint16_t port, bool syncProto) :
MessageQueueClient::MessageQueueClient(const string& dnOrIp, uint16_t port, bool syncProto) :
fLogger(31), fIsAvailable(true)
{
memset(&fServ_addr, 0, sizeof(fServ_addr));
sockaddr_in* sinp = reinterpret_cast<sockaddr_in*>(&fServ_addr);
sinp->sin_family = AF_INET;
sinp->sin_port = htons(port);
sinp->sin_addr.s_addr = inet_addr(ip.c_str());
struct addrinfo hints, *servinfo;
int rc = 0;
memset(&hints, 0, sizeof hints);
// ATM We support IPv4 only.
hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM;
if( !(rc = getaddrinfo(dnOrIp.c_str(), NULL, &hints, &servinfo)) )
{
memset(&fServ_addr, 0, sizeof(fServ_addr));
sockaddr_in* sinp = reinterpret_cast<sockaddr_in*>(&fServ_addr);
*sinp = *reinterpret_cast<sockaddr_in*>(servinfo->ai_addr);
sinp->sin_port = htons(port);
freeaddrinfo(servinfo);
}
else
{
string msg = "MessageQueueClient::MessageQueueClient(): ";
msg.append(gai_strerror(rc));
logging::Message::Args args;
logging::LoggingID li(31);
args.add(msg);
fLogger.logMessage(logging::LOG_TYPE_ERROR, logging::M0000, args, li);
}
#ifdef SKIP_IDB_COMPRESSION
fClientSock.setSocketImpl(new InetStreamSocket());
#else

3
utils/messageqcpp/messagequeue.h
View File

@@ -33,6 +33,7 @@
#include <stdio.h>
#else
#include <netinet/in.h>
#include <netdb.h>
#endif
#include "serversocket.h"
@@ -183,7 +184,7 @@ public:
*
* construct a queue from this process to otherEnd on the given IP and Port.
*/
EXPORT explicit MessageQueueClient(const std::string& ip, uint16_t port, bool syncProto = true);
EXPORT explicit MessageQueueClient(const std::string& dnOrIp, uint16_t port, bool syncProto=true);
/**

6
utils/messageqcpp/messagequeuepool.cpp
View File

@@ -37,12 +37,12 @@ static uint64_t TimeSpecToSeconds(struct timespec* ts)
return (uint64_t)ts->tv_sec + (uint64_t)ts->tv_nsec / 1000000000;
}
MessageQueueClient* MessageQueueClientPool::getInstance(const std::string& ip, uint64_t port)
MessageQueueClient *MessageQueueClientPool::getInstance(const std::string &dnOrIp, uint64_t port)
{
boost::mutex::scoped_lock lock(queueMutex);
std::ostringstream oss;
oss << ip << "_" << port;
oss << dnOrIp << "_" << port;
std::string searchString = oss.str();
MessageQueueClient* returnClient = MessageQueueClientPool::findInPool(searchString);
@@ -59,7 +59,7 @@ MessageQueueClient* MessageQueueClientPool::getInstance(const std::string& ip, u
clock_gettime(CLOCK_MONOTONIC, &now);
uint64_t nowSeconds = TimeSpecToSeconds(&now);
newClientObject->client = new MessageQueueClient(ip, port);
newClientObject->client = new MessageQueueClient(dnOrIp, port);
newClientObject->inUse = true;
newClientObject->lastUsed = nowSeconds;
clientMap.insert(std::pair<std::string, ClientObject*>(searchString, newClientObject));

2
utils/messageqcpp/messagequeuepool.h
View File

@@ -42,7 +42,7 @@ class MessageQueueClientPool
{
public:
static MessageQueueClient* getInstance(const std::string& module);
static MessageQueueClient* getInstance(const std::string& ip, uint64_t port);
static MessageQueueClient *getInstance(const std::string &dnOrIp, uint64_t port);
static void releaseInstance(MessageQueueClient* client);
static void deleteInstance(MessageQueueClient* client);
static MessageQueueClient* findInPool(const std::string& search);