database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-01 06:46:55 +03:00
Code Activity

Merge branch 'develop-1.1' into 1.1-merge-up

Browse Source
This commit is contained in:
Andrew Hutchings
2017-12-12 20:26:00 +00:00
parent b795e6b781 ca2c511f59
commit 59d0a45da3
8 changed files with 146 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
dbcon/execplan/simplecolumn.cpp
View File

@ -117,7 +117,7 @@ ParseTree* replaceRefCol(ParseTree*& n, CalpontSelectExecutionPlan::ReturnedColu
{
SimpleColumn* sc = dynamic_cast<SimpleColumn*>(rc);
if (sc)
if (sc && (sc->colPosition() > -1))
{
ReturnedColumn* tmp = derivedColList[sc->colPosition()]->clone();
delete sc;

4
oamapps/mcsadmin/mcsadmin.cpp
View File

@ -6836,6 +6836,8 @@ int processCommand(string* arguments)
cout << endl << "**** disableRep Failed : " << e.what() << endl;
}
cout << endl;
break;
}
@ -7196,8 +7198,6 @@ int processCommand(string* arguments)
printModuleMemory(systemmemory.modulememory[i]);
}
cout << endl;
}
catch (exception& e)
{

85
oamapps/postConfigure/postConfigure.cpp
View File

@ -669,13 +669,6 @@ int main(int argc, char* argv[])
nonDistribute = true;
}
// setup to start on reboot, for non-root amazon installs
if ( !rootUser )
{
system("sudo sed -i -e 's/#sudo runuser/sudo runuser/g' /etc/rc.d/rc.local >/dev/null 2>&1");
system("sudo chmod 666 /etc/fstab >/dev/null 2>&1");
}
cout << endl;
cout << "===== Setup System Server Type Configuration =====" << endl << endl;
@ -1192,6 +1185,13 @@ int main(int argc, char* argv[])
}
}
// setup to start on reboot, for non-root amazon installs
if ( !rootUser )
{
system("sudo sed -i -e 's/#sudo runuser/sudo runuser/g' /etc/rc.d/rc.local >/dev/null 2>&1");
system("sudo chmod 666 /etc/fstab >/dev/null 2>&1");
}
if ( !writeConfig(sysConfig) )
{
cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl;
@ -3522,6 +3522,69 @@ int main(int argc, char* argv[])
{
cout << endl;
//Ask for ssh password or certificate if not already set
bool passwordSetBefore = false;
if ( password.empty() )
{
cout << endl;
cout << "Next step is to enter the password to access the other Servers." << endl;
cout << "This is either your password or you can default to using a ssh key" << endl;
cout << "If using a password, the password needs to be the same on all Servers." << endl << endl;
}
while (true)
{
char* pass1, *pass2;
if ( noPrompting )
{
cout << "Enter password, hit 'enter' to default to using a ssh key, or 'exit' > " << endl;
if ( password.empty() )
password = "ssh";
break;
}
//check for command line option password
if ( !password.empty() )
{
passwordSetBefore = true;
break;
}
pass1 = getpass("Enter password, hit 'enter' to default to using a ssh key, or 'exit' > ");
if ( strcmp(pass1, "") == 0 )
{
password = "ssh";
break;
}
if ( pass1 == "exit")
exit(0);
string p1 = pass1;
pass2 = getpass("Confirm password > ");
string p2 = pass2;
if ( p1 == p2 )
{
password = p2;
break;
}
else
cout << "Password mismatch, please re-enter" << endl;
}
//add single quote for special characters
if ( password != "ssh" && !passwordSetBefore)
{
password = "'" + password + "'";
}
if ( reuseConfig != "y" )
{
cout << endl << "===== Configuring MariaDB ColumnStore Data Redundancy Functionality =====" << endl << endl;
@ -3580,8 +3643,8 @@ int main(int argc, char* argv[])
//
cout << endl << "===== MariaDB ColumnStore System Startup =====" << endl << endl;
cout << "System Installation is complete." << endl;
cout << "Performing System Start-up." << endl;
cout << "System Configuration is complete." << endl;
cout << "Performing System Installation." << endl;
if (hdfs && !nonDistribute )
{
@ -3616,8 +3679,8 @@ int main(int argc, char* argv[])
{
cout << endl << "===== MariaDB ColumnStore System Startup =====" << endl << endl;
cout << "System Installation is complete." << endl;
cout << "Performing System Start-up." << endl;
cout << "System Configuration is complete." << endl;
cout << "Performing System Installation." << endl;
//start MariaDB ColumnStore on local server
cout << endl << "----- Starting MariaDB ColumnStore on local Server '" + parentOAMModuleName + "' -----" << endl << endl;

63
utils/clusterTester/columnstoreClusterTester.sh
View File

@ -14,6 +14,10 @@ OS_LIST=("centos6" "centos7" "debian8" "debian9" "suse12" "ubuntu16")
NODE_IPADDRESS=""
if [ -z "$COLUMNSTORE_INSTALL_DIR" ]; then
COLUMNSTORE_INSTALL_DIR=/usr/local/mariadb/columnstore
fi
checkContinue() {
if [ "$CHECK" = false ]; then
@ -198,7 +202,7 @@ checkLocalOS()
echo ""
#get local OS
`./os_detect.sh > /tmp/os_detect 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/os_detect.sh > /tmp/os_detect 2>&1`
if [ "$?" -eq 0 ]; then
localOS=`cat /tmp/os_detect | grep "Operating System name" | cut -f2 -d '"'`
echo "Local Node OS System Name : $localOS"
@ -318,7 +322,7 @@ checkSSH()
echo ""
for ipadd in "${NODE_IPADDRESS[@]}"; do
`./remote_command.sh $ipadd $PASSWORD ls 1 > /dev/null 2>&1`;
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD ls 1 > /dev/null 2>&1`;
rc="$?"
if [ $rc -eq 0 ] || ( [ $rc -eq 2 ] && [ $OS == "suse12" ] ) ; then
if [ $PASSWORD == "ssh" ] ; then
@ -345,7 +349,7 @@ checkRemoteDir()
`sudo rm -f /tmp/*_check > /dev/null 2>&1`
for ipadd in "${NODE_IPADDRESS[@]}"; do
`./remote_command.sh $ipadd $PASSWORD 'sudo rm -f /tmp/*_check > /dev/null 2>&1' 1 > /tmp/remote_command_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'sudo rm -f /tmp/*_check > /dev/null 2>&1' 1 > /tmp/remote_command_check 2>&1`
done
if [ "$USER" != "root" ]; then
@ -356,7 +360,7 @@ checkRemoteDir()
echo ""
for ipadd in "${NODE_IPADDRESS[@]}"; do
`./remote_command.sh $ipadd $PASSWORD 'touch /tmp/cs_check' 1 > /tmp/remote_command_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'touch /tmp/cs_check' 1 > /tmp/remote_command_check 2>&1`
rc="$?"
if [ $rc -eq 0 ] || ( [ $rc -eq 2 ] && [ $OS == "suse12" ] ) ; then
`grep "Permission denied" /tmp/remote_command_check > /dev/null 2>&1`
@ -372,7 +376,7 @@ checkRemoteDir()
REPORTPASS=false
fi
`./remote_command.sh $ipadd $PASSWORD 'touch /dev/shm/cs_check' 1 > /tmp/remote_command_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'touch /dev/shm/cs_check' 1 > /tmp/remote_command_check 2>&1`
rc="$?"
if [ $rc -eq 0 ] || ( [ $rc -eq 2 ] && [ $OS == "suse12" ] ) ; then
`grep "Permission denied" /tmp/remote_command_check > /dev/null 2>&1`
@ -408,25 +412,26 @@ checkOS()
echo ""
pass=true
`/bin/cp -f $COLUMNSTORE_INSTALL_DIR/bin/os_detect.sh /tmp/.`
for ipadd in "${NODE_IPADDRESS[@]}"; do
`./remote_scp_put.sh $ipadd $PASSWORD os_detect.sh 1 > /tmp/remote_scp_put_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_put.sh $ipadd $PASSWORD /tmp/os_detect.sh 1 > /tmp/remote_scp_put_check 2>&1`
if [ "$?" -ne 0 ]; then
echo "Error running remote_scp_put.sh to $ipadd Node, check /tmp/remote_scp_put_check"
exit 1
else
`./remote_command.sh $ipadd $PASSWORD './os_detect.sh > /tmp/os_detect 2>&1' 1 > /tmp/remote_command_check`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD /tmp/os_detect.sh 1 > /tmp/remote_command_check`
rc="$?"
`./remote_scp_get.sh $ipadd $PASSWORD /tmp/os_detect > /tmp/remote_scp_get_check 2>&1`
if [ "$?" -ne 0 ]; then
echo "Error running remote_scp_get.sh to $ipadd Node, check /tmp/remote_scp_get_check"
echo "Error running remote_command.sh /tmp/os_detect.sh on $ipadd Node, check /tmp/remote_command_check"
exit 1
else
remoteOS=`cat os_detect | grep "Operating System name" | cut -f2 -d '"'`
remoteOS=`cat /tmp/remote_command_check | grep "Operating System name" | cut -f2 -d '"'`
echo "$ipadd Node OS Version : $remoteOS"
if [ $localOS != $remoteOS ]; then
echo "${bold}Failed${normal}, $ipadd has a different OS than local node"
pass=false
REPORTPASS=false
fi
rm -f os_detect
fi
fi
done
@ -454,12 +459,13 @@ checkLocale()
pass=true
for ipadd in "${NODE_IPADDRESS[@]}"; do
`./remote_command.sh $ipadd $PASSWORD 'locale | grep LANG= > /tmp/locale_check 2>&1' 1 > /tmp/remote_command_check`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'locale | grep LANG= > /tmp/locale_check 2>&1' 1 > /tmp/remote_command_check`
rc="$?"
if [ $rc -eq 0 ] || ( [ $rc -eq 2 ] && [ $OS == "suse12" ] ) ; then
`./remote_scp_get.sh $ipadd $PASSWORD /tmp/locale_check > /tmp/remote_scp_get_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /tmp/locale_check > /tmp/remote_scp_get_check 2>&1`
if [ "$?" -ne 0 ]; then
echo "Error running remote_scp_get.sh to $ipadd Node, check /tmp/remote_scp_get_check"
exit 1
else
echo "$ipadd Node Locale : `cat locale_check`"
`diff /tmp/locale_check locale_check > /dev/null 2>&1`
@ -472,6 +478,7 @@ checkLocale()
fi
else
echo "Error running remote_command.sh to $ipadd Node, check /tmp/remote_command_check"
exit 1
pass=false
REPORTPASS=false
fi
@ -506,7 +513,7 @@ checkSELINUX()
fi
for ipadd in "${NODE_IPADDRESS[@]}"; do
`./remote_scp_get.sh $ipadd $PASSWORD /etc/selinux/config > /tmp/remote_scp_get_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /etc/selinux/config > /tmp/remote_scp_get_check 2>&1`
if [ "$?" -ne 0 ]; then
echo "$ipadd Node SELINUX setting is Not Enabled"
else
@ -572,14 +579,14 @@ checkFirewalls()
# 'sysconfig not on remote node
for firewall in "${FIREWALL_LIST[@]}"; do
pass=true
`./remote_command.sh $ipadd $PASSWORD "service '$firewall' status > /tmp/firewall_check 2>&1" 1 > /tmp/remote_command_check`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "service '$firewall' status > /tmp/firewall_check 2>&1" 1 > /tmp/remote_command_check`
if [ "$?" -eq 0 ]; then
echo "${bold}Failed${normal}, $ipadd Node $firewall service is Active, please disable"
pass=false
fpass=false
REPORTPASS=false
else
`./remote_command.sh $ipadd $PASSWORD "systemctl status '$firewall' > /tmp/firewall_check 2>&1" 1 > /tmp/remote_command_check`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "systemctl status '$firewall' > /tmp/firewall_check 2>&1" 1 > /tmp/remote_command_check`
if [ "$?" -eq 0 ]; then
echo "${bold}Failed${normal}, $ipadd Node $firewall service is Active, please disable"
pass=false
@ -619,7 +626,7 @@ checkFirewalls()
fi
for ipadd in "${NODE_IPADDRESS[@]}"; do
`./remote_command.sh $ipadd $PASSWORD '/sbin/rcSuSEfirewall2 status > /tmp/rcSuSEfirewall2_check 2>&1' 1 > /tmp/remote_command_check`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD '/sbin/rcSuSEfirewall2 status > /tmp/rcSuSEfirewall2_check 2>&1' 1 > /tmp/remote_command_check`
rc="$?"
if [ $rc -eq 0 ] ; then
echo "${bold}Failed${normal}, $ipadd Node rcSuSEfirewall2 service is Enabled, please disable"
@ -674,14 +681,14 @@ checkTime()
#get local epoch time
localTime=`date +%s`
for ipadd in "${NODE_IPADDRESS[@]}"; do
`./remote_command.sh $ipadd $PASSWORD 'date +%s > /tmp/time_check' > /tmp/time_check`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'date +%s > /tmp/time_check' > /tmp/time_check`
rc="$?"
if [ $rc -ne 0 ] ; then
echo $ipadd " Node ${bold}Failed${normal} date/time check failed, check /tmp/time_check"
pass=false
REPORTPASS=false
else
`./remote_scp_get.sh $ipadd $PASSWORD /tmp/time_check > /tmp/remote_scp_get_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /tmp/time_check > /tmp/remote_scp_get_check 2>&1`
if [ "$?" -ne 0 ]; then
echo "Error running remote_scp_get.sh to $ipadd Node, check /tmp/remote_scp_get_check"
else
@ -756,14 +763,14 @@ checkPackages()
for ipadd in "${NODE_IPADDRESS[@]}"; do
for PKG in "${CENTOS_PKG[@]}"; do
if [ $OS == "centos6" ] && [ $PKG == "boost" ]; then
`./remote_command.sh $ipadd $PASSWORD 'ls /usr/lib/libboost_regex.so > /dev/null 2>&1' 1 > /tmp/remote_command_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'ls /usr/lib/libboost_regex.so > /dev/null 2>&1' 1 > /tmp/remote_command_check 2>&1`
if [ $? -ne 0 ] ; then
echo "${bold}Failed${normal}, $ipadd Node ${bold}boost libraries${normal} not installed"
pass=false
REPORTPASS=false
fi
else
`./remote_command.sh $ipadd $PASSWORD "yum list installed '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "yum list installed '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1`
rc="$?"
if [ $rc -eq 2 ] ; then
echo "${bold}Failed${normal}, $ipadd Node, 'yum' not installed"
@ -821,7 +828,7 @@ checkPackages()
if [ "$IPADDRESSES" != "" ]; then
for ipadd in "${NODE_IPADDRESS[@]}"; do
for PKG in "${SUSE_PKG[@]}"; do
`./remote_command.sh $ipadd $PASSWORD "rpm -qi '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "rpm -qi '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1`
rc="$?"
if [ $rc -ne 0 ] ; then
echo "${bold}Failed${normal}, $ipadd Node package ${bold}${PKG}${normal} is not installed, please install"
@ -873,8 +880,8 @@ checkPackages()
if [ "$IPADDRESSES" != "" ]; then
for ipadd in "${NODE_IPADDRESS[@]}"; do
for PKG in "${UBUNTU_PKG[@]}"; do
`./remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1`
`./remote_scp_get.sh $ipadd $PASSWORD /tmp/pkg_check > /tmp/remote_scp_get_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /tmp/pkg_check > /tmp/remote_scp_get_check 2>&1`
if [ "$?" -ne 0 ]; then
echo "Error running remote_scp_get.sh to $ipadd Node, check /tmp/remote_scp_get_check"
else
@ -938,8 +945,8 @@ checkPackages()
if [ "$IPADDRESSES" != "" ]; then
for ipadd in "${NODE_IPADDRESS[@]}"; do
for PKG in "${DEBIAN_PKG[@]}"; do
`./remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1`
`./remote_scp_get.sh $ipadd $PASSWORD /tmp/pkg_check > /tmp/remote_scp_get_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /tmp/pkg_check > /tmp/remote_scp_get_check 2>&1`
if [ "$?" -ne 0 ]; then
echo "Error running remote_scp_get.sh to $ipadd Node, check /tmp/remote_scp_get_check"
else
@ -1003,8 +1010,8 @@ checkPackages()
if [ "$IPADDRESSES" != "" ]; then
for ipadd in "${NODE_IPADDRESS[@]}"; do
for PKG in "${DEBIAN9_PKG[@]}"; do
`./remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1`
`./remote_scp_get.sh $ipadd $PASSWORD /tmp/pkg_check > /tmp/remote_scp_get_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1`
`$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /tmp/pkg_check > /tmp/remote_scp_get_check 2>&1`
if [ "$?" -ne 0 ]; then
echo "Error running remote_scp_get.sh to $ipadd Node, check /tmp/remote_scp_get_check"
else

25
utils/funcexp/utils_utf8.h
View File

@ -202,6 +202,31 @@ std::string wstring_to_utf8 (const std::wstring& str)
return std::string(outbuf, strmblen);
}
inline
uint8_t utf8_truncate_point(const char* input, size_t length)
{
// Find the beginning of a multibyte char to truncate at and return the
// number of bytes to truncate
if (length < 3)
{
return 0;
}
const unsigned char* b = (const unsigned char*)(input) + length - 3;
if (b[2] & 0x80)
{
// First byte in a new multi-byte sequence
if (b[2] & 0x40) return 1;
// 3 byte sequence
else if ((b[1] & 0xe0) == 0xe0) return 2;
// 4 byte sequence
else if ((b[0] & 0xf0) == 0xf0) return 3;
}
return 0;
}
} //namespace utf8
} //namespace funcexp

9
utils/rowgroup/rowgroup.cpp
View File

@ -701,6 +701,7 @@ void Row::initToNull()
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::VARCHAR:
case CalpontSystemCatalog::TEXT:
case CalpontSystemCatalog::STRINT:
{
if (inStringTable(i))
@ -744,6 +745,7 @@ void Row::initToNull()
}
case CalpontSystemCatalog::VARBINARY:
case CalpontSystemCatalog::BLOB:
*((uint16_t*) &data[offsets[i]]) = 0;
break;
@ -798,13 +800,6 @@ void Row::initToNull()
break;
}
case CalpontSystemCatalog::BLOB:
case CalpontSystemCatalog::TEXT:
{
memset(&data[offsets[i]], 0xFF, getColumnWidth(i));
break;
}
default:
ostringstream os;
os << "Row::initToNull(): got bad column type (" << types[i] <<

5
writeengine/bulk/we_bulkloadbuffer.cpp
View File

@ -41,6 +41,8 @@
#include "joblisttypes.h"
#include "utils_utf8.h"
using namespace std;
using namespace boost;
using namespace execplan;
@ -545,7 +547,8 @@ void BulkLoadBuffer::convert(char* field, int fieldLength,
// on disk (e.g. 5 for a varchar(5) instead of 8).
if (fieldLength > column.definedWidth)
{
memcpy( charTmpBuf, field, column.definedWidth );
uint8_t truncate_point = funcexp::utf8::utf8_truncate_point(field, column.definedWidth);
memcpy( charTmpBuf, field, column.definedWidth - truncate_point );
bufStats.satCount++;
}
else

4
writeengine/dictionary/we_dctnry.cpp
View File

@ -47,6 +47,7 @@ using namespace BRM;
#include "IDBPolicy.h"
#include "cacheutils.h"
using namespace idbdatafile;
#include "utils_utf8.h"
namespace
{
@ -858,7 +859,8 @@ int Dctnry::insertDctnry(const char* buf,
// @Bug 2565: Truncate any strings longer than schema's column width
if (curSig.size > m_colWidth)
{
curSig.size = m_colWidth;
uint8_t truncate_point = funcexp::utf8::utf8_truncate_point((const char*)curSig.signature, m_colWidth);
curSig.size = m_colWidth - truncate_point;
++truncCount;
}