database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-07 03:22:57 +03:00
Code Activity

Merge branch 'develop' into MCOL-1793

Browse Source
This commit is contained in:
David Hall
2018-11-23 12:43:17 -06:00
parent 176ef2f2c1 6d53e18a30
commit c3477fe243
9 changed files with 167 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
dbcon/mysql/ha_calpont.cpp
View File

@@ -1198,8 +1198,6 @@ void check_walk(const Item* item, void* arg)
}
}
#include <iostream>
/*@brief create_calpont_group_by_handler- Creates handler*/
/***********************************************************
* DESCRIPTION:
@@ -1221,19 +1219,19 @@ static group_by_handler*
create_calpont_group_by_handler(THD* thd, Query* query)
{
ha_calpont_group_by_handler* handler = NULL;
LEX* lex = thd->lex;
SELECT_LEX *select_lex = &lex->select_lex;
SELECT_LEX *select_lex = query->from->select_lex;
// Create a handler if query is valid. See comments for details.
if ( thd->infinidb_vtable.vtable_state == THD::INFINIDB_DISABLE_VTABLE
&& ( thd->variables.infinidb_vtable_mode == 0
|| thd->variables.infinidb_vtable_mode == 2 )
&& ( query->group_by || thd->lex->select_lex.with_sum_func ) )
&& ( query->group_by || select_lex->with_sum_func ) )
{
bool unsupported_feature = false;
// Impossible HAVING or WHERE
if ( ( select_lex->having && select_lex->having_value == Item::COND_FALSE )
|| ( select_lex->cond_value && select_lex->cond_value == Item::COND_FALSE ) )
if ( ( query->having && select_lex->having_value == Item::COND_FALSE )
|| ( select_lex->cond_count > 0
&& select_lex->cond_value == Item::COND_FALSE ) )
{
unsupported_feature = true;
}
@@ -1265,8 +1263,6 @@ create_calpont_group_by_handler(THD* thd, Query* query)
}
}
std::cerr << "create_calpont_group_by_handler handler " << handler << std::endl;
return handler;
}

6
dbcon/mysql/ha_calpont_execplan.cpp
View File

@@ -8248,13 +8248,11 @@ int cp_get_table_plan(THD* thd, SCSEP& csep, cal_table_info& ti)
int cp_get_group_plan(THD* thd, SCSEP& csep, cal_impl_if::cal_group_info& gi)
{
LEX* lex = thd->lex;
idbassert(lex != 0);
SELECT_LEX select_lex = lex->select_lex;
SELECT_LEX *select_lex = gi.groupByTables->select_lex;
gp_walk_info gwi;
gwi.thd = thd;
int status = getGroupPlan(gwi, select_lex, csep, gi);
int status = getGroupPlan(gwi, *select_lex, csep, gi);
#ifdef DEBUG_WALK_COND
cerr << "---------------- cp_get_group_plan EXECUTION PLAN ----------------" << endl;

4
oam/install_scripts/columnstoreAlias
View File

@@ -4,7 +4,7 @@ alias mcsmysql='/usr/local/mariadb/columnstore/mysql/bin/mysql --defaults-extra-
alias ma=/usr/local/mariadb/columnstore/bin/mcsadmin
alias mcsadmin=/usr/local/mariadb/columnstore/bin/mcsadmin
alias cpimport=/usr/local/mariadb/columnstore/bin/cpimport
alias home='cd /usr/local/mariadb/columnstore'
alias mcshome='cd /usr/local/mariadb/columnstore'
alias log='cd /var/log/mariadb/columnstore/'
alias core='cd /var/log/mariadb/columnstore/corefiles'
alias tmsg='tail -f /var/log/messages'
@@ -14,4 +14,4 @@ alias terror='tail -f /var/log/mariadb/columnstore/err.log'
alias twarning='tail -f /var/log/mariadb/columnstore/warning.log'
alias tcrit='tail -f /var/log/mariadb/columnstore/crit.log'
alias dbrm='cd /usr/local/mariadb/columnstore/data1/systemFiles/dbrm'
alias module='cat /usr/local/mariadb/columnstore/local/module'
alias mcsmodule='cat /usr/local/mariadb/columnstore/local/module'

8
oam/install_scripts/post-install
View File

@@ -315,14 +315,6 @@ if [ -z "aws" ]; then
$installdir/bin/MCSgetCredentials.sh >/dev/null 2>&1
fi
#log install message
test -f $installdir/post/functions && . $installdir/post/functions
if [ $user = "root" ]; then
$installdir/bin/cplogger -i 19 "***** MariaDB Columnstore Installed *****"
else
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib $installdir/bin/cplogger -i 19 "***** MariaDB Columnstore Installed *****"
fi
#setup hadoop
hadoop=`which hadoop 2>/dev/null`
if [ -z "$hadoop" ]; then

91
oam/install_scripts/syslogSetup.sh
View File

@@ -11,12 +11,26 @@ prefix=/usr/local
installdir=$prefix/mariadb/columnstore
syslog_conf=nofile
rsyslog7=0
non_root_user=no
user=`whoami 2>/dev/null`
#set default names
groupname=adm
username=syslog
# determine username/groupname
if [ -f /var/log/messages ]; then
username=`stat -c "%U %G" /var/log/messages | awk '{print $1}'`
groupname=`stat -c "%U %G" /var/log/messages | awk '{print $2}'`
fi
if [ -f /var/log/syslog ]; then
username=`stat -c "%U %G" /var/log/syslog | awk '{print $1}'`
groupname=`stat -c "%U %G" /var/log/syslog | awk '{print $2}'`
fi
for arg in "$@"; do
if [ `expr -- "$arg" : '--prefix='` -eq 9 ]; then
prefix="`echo $arg | awk -F= '{print $2}'`"
@@ -28,6 +42,7 @@ for arg in "$@"; do
user="`echo $arg | awk -F= '{print $2}'`"
groupname=$user
username=$user
non_root_user=yes
elif [ `expr -- "$arg" : '--..*'` -ge 3 ]; then
echo "ignoring unknown argument: $arg" 1>&2
elif [ `expr -- "$arg" : '--'` -eq 2 ]; then
@@ -157,12 +172,17 @@ fi
}
makeDir() {
test -d /var/log/mariadb/columnstore || mkdir -p /var/log/mariadb/columnstore >/dev/null 2>&1
test -d /var/log/mariadb/columnstore/archive || mkdir /var/log/mariadb/columnstore/archive >/dev/null 2>&1
test -d /var/log/mariadb/columnstore/corefiles || mkdir /var/log/mariadb/columnstore/corefiles >/dev/null 2>&1
test -d /var/log/mariadb/columnstore/trace || mkdir /var/log/mariadb/columnstore/trace >/dev/null 2>&1
chmod 777 -R /var/log/mariadb/columnstore
chown $user:$user -R /var/log/mariadb
if [ ! -d /var/log/mariadb/columnstore ]; then
mkdir -p /var/log/mariadb/columnstore >/dev/null 2>&1
test -d /var/log/mariadb/columnstore/archive || mkdir /var/log/mariadb/columnstore/archive >/dev/null 2>&1
test -d /var/log/mariadb/columnstore/corefiles || mkdir /var/log/mariadb/columnstore/corefiles >/dev/null 2>&1
test -d /var/log/mariadb/columnstore/trace || mkdir /var/log/mariadb/columnstore/trace >/dev/null 2>&1
chown $username:$groupname -R /var/log/mariadb
else
test -d /var/log/mariadb/columnstore/archive || mkdir /var/log/mariadb/columnstore/archive >/dev/null 2>&1
test -d /var/log/mariadb/columnstore/corefiles || mkdir /var/log/mariadb/columnstore/corefiles >/dev/null 2>&1
test -d /var/log/mariadb/columnstore/trace || mkdir /var/log/mariadb/columnstore/trace >/dev/null 2>&1
fi
}
install() {
@@ -170,15 +190,15 @@ makeDir
checkSyslog
if [ ! -z "$syslog_conf" ] ; then
$installdir/bin/setConfig -d Installation SystemLogConfigFile ${syslog_conf} >/dev/null 2>&1
if [ $user != "root" ]; then
chown $user:$user /home/$user/mariadb/columnstore/etc/*
fi
if [ $non_root_user == "yes" ]; then
chown $user:$user $installdir/etc/Columnstore.xml*
fi
rm -f ${syslog_conf}.columnstoreSave
if [ "$syslog_conf" == /etc/rsyslog.d/columnstore.conf ] ||
[ "$syslog_conf" == /etc/rsyslog.d/49-columnstore.conf ]; then
i=1
else
rm -f ${syslog_conf}.columnstoreSave
cp ${syslog_conf} ${syslog_conf}.columnstoreSave >/dev/null 2>&1
sed -i '/# MariaDB/,$d' ${syslog_conf}.columnstoreSave > /dev/null 2>&1
fi
@@ -188,21 +208,6 @@ if [ ! -z "$syslog_conf" ] ; then
#set the syslog for ColumnStore logging
# remove older version incase it was installed by previous build
rm -rf /etc/rsyslog.d/columnstore.conf
# determine username/groupname
if [ -f /var/log/messages ]; then
user=`stat -c "%U %G" /var/log/messages | awk '{print $1}'`
group=`stat -c "%U %G" /var/log/messages | awk '{print $2}'`
fi
if [ -f /var/log/syslog ]; then
user=`stat -c "%U %G" /var/log/syslog | awk '{print $1}'`
group=`stat -c "%U %G" /var/log/syslog | awk '{print $2}'`
fi
# set permissions
chown $user:$group -R /var/log/mariadb > /dev/null 2>&1
if [ $rsyslog7 == 1 ]; then
rm -f /etc/rsyslog.d/49-columnstore.conf
@@ -210,16 +215,41 @@ if [ ! -z "$syslog_conf" ] ; then
sed -i -e s/groupname/$groupname/g ${syslog_conf}
sed -i -e s/username/$username/g ${syslog_conf}
chmod 644 ${syslog_conf}
else
cp ${columnstoreSyslogFile} ${syslog_conf}
fi
fi
# install Columnstore Log Rotate File
cp $installdir/bin/columnstoreLogRotate /etc/logrotate.d/columnstore > /dev/null 2>&1
chmod 644 /etc/logrotate.d/columnstore
restartSyslog
# install Columnstore Log Rotate File
if [ -d /etc/logrotate.d ]; then
cp $installdir/bin/columnstoreLogRotate /etc/logrotate.d/columnstore > /dev/null 2>&1
chmod 644 /etc/logrotate.d/columnstore
#do the logrotate to start with a fresh log file during install
logrotate -f /etc/logrotate.d/columnstore > /dev/null 2>&1
fi
#log install message and find the least permission that allows logging to work
CHMOD_LIST=("750" "770" "775" "777")
for CHMOD in "${CHMOD_LIST[@]}"; do
chmod $CHMOD /var/log/mariadb
chmod $CHMOD /var/log/mariadb/columnstore
chmod $CHMOD /var/log/mariadb/columnstore/archive
chmod $CHMOD /var/log/mariadb/columnstore/corefiles
chmod $CHMOD /var/log/mariadb/columnstore/trace
test -f $installdir/post/functions && . $installdir/post/functions
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib $installdir/bin/cplogger -i 19 "***** MariaDB Columnstore Installed *****"
if [ -f /var/log/mariadb/columnstore/info.log ]; then
if [ -s /var/log/mariadb/columnstore/info.log ]; then
exit 0
fi
fi
done
fi
}
@@ -233,7 +263,7 @@ if [ ! -z "$syslog_conf" ] ; then
if [ $? -eq 0 ]; then
if [ -f ${syslog_conf}.columnstoreSave ] ; then
#uninstall the syslog for ColumnStore logging
v -f ${syslog_conf} ${syslog_conf}.ColumnStoreBackup
mv -f ${syslog_conf} ${syslog_conf}.ColumnStoreBackup
mv -f ${syslog_conf}.columnstoreSave ${syslog_conf} >/dev/null 2>&1
if [ ! -f ${syslog_conf} ] ; then
cp ${syslog_conf}.ColumnStoreBackup ${syslog_conf}
@@ -252,7 +282,6 @@ if [ ! -z "$syslog_conf" ] ; then
rm -f /etc/logrotate.d/columnstore
restartSyslog
fi
}

75
oamapps/postConfigure/mycnfUpgrade.cpp
View File

@@ -54,38 +54,39 @@
using namespace std;
using namespace oam;
/* MCOL-1844. On an upgrade, the user may have customized options in their old
/* MCOL-1844. On an upgrade, the user may have customized options in their old
* myCnf-include-args.text file. Merge it with the packaged version, and then process as we
* have before.
*/
string rtrim(const string &in) {
string::const_reverse_iterator rbegin = in.rbegin();
while (rbegin != in.rend() && isspace(*rbegin))
++rbegin;
return string(in.begin(), rbegin.base());
string::const_reverse_iterator rbegin = in.rbegin();
while (rbegin != in.rend() && isspace(*rbegin))
++rbegin;
return string(in.begin(), rbegin.base());
}
void mergeMycnfIncludeArgs()
void mergeMycnfIncludeArgs()
{
string userArgsFilename = startup::StartUp::installDir() + "/bin/myCnf-include-args.text.rpmsave";
string packagedArgsFilename = startup::StartUp::installDir() + "/bin/myCnf-include-args.text";
ifstream userArgs(userArgsFilename.c_str());
fstream packagedArgs(packagedArgsFilename.c_str(), ios::in);
string userArgsFilename = startup::StartUp::installDir() + "/bin/myCnf-include-args.text.rpmsave";
string packagedArgsFilename = startup::StartUp::installDir() + "/bin/myCnf-include-args.text";
ifstream userArgs(userArgsFilename.c_str());
fstream packagedArgs(packagedArgsFilename.c_str(), ios::in);
if (!userArgs || !packagedArgs)
return;
if (!userArgs || !packagedArgs)
return;
// de-dup the args and comments in both files
set<string> argMerger;
set<string> comments;
string line;
while (getline(packagedArgs, line)) {
line = rtrim(line);
if (line[0] == '#')
comments.insert(line);
else if (line.size() > 0)
argMerger.insert(line);
}
// de-dup the args and comments in both files
set<string> argMerger;
set<string> comments;
string line;
while (getline(packagedArgs, line)) {
line = rtrim(line);
if (line[0] == '#')
comments.insert(line);
else if (line.size() > 0)
argMerger.insert(line);
}
while (getline(userArgs, line)) {
line = rtrim(line);
if (line[0] == '#')
@@ -93,17 +94,17 @@ void mergeMycnfIncludeArgs()
else if (line.size() > 0)
argMerger.insert(line);
}
userArgs.close();
packagedArgs.close();
userArgs.close();
packagedArgs.close();
// write the merged version, comments first. They'll get ordered
// alphabetically but, meh.
packagedArgs.open(packagedArgsFilename.c_str(), ios::out | ios::trunc);
for (set<string>::iterator it = comments.begin(); it != comments.end(); it++)
packagedArgs << *it << endl;
for (set<string>::iterator it = argMerger.begin(); it != argMerger.end(); it++)
packagedArgs << *it << endl;
packagedArgs.close();
// write the merged version, comments first. They'll get ordered
// alphabetically but, meh.
packagedArgs.open(packagedArgsFilename.c_str(), ios::out | ios::trunc);
for (set<string>::iterator it = comments.begin(); it != comments.end(); it++)
packagedArgs << *it << endl;
for (set<string>::iterator it = argMerger.begin(); it != argMerger.end(); it++)
packagedArgs << *it << endl;
packagedArgs.close();
}
int main(int argc, char* argv[])
@@ -142,9 +143,9 @@ int main(int argc, char* argv[])
exit (1);
}
// MCOL-1844. The user may have added options to their myCnf-include-args file. Merge
// myCnf-include-args.text with myCnf-include-args.text.rpmsave, save in myCnf-include-args.text
mergeMycnfIncludeArgs();
// MCOL-1844. The user may have added options to their myCnf-include-args file. Merge
// myCnf-include-args.text with myCnf-include-args.text.rpmsave, save in myCnf-include-args.text
mergeMycnfIncludeArgs();
//include arguments file
string includeFile = startup::StartUp::installDir() + "/bin/myCnf-include-args.text";
@@ -218,7 +219,7 @@ int main(int argc, char* argv[])
ofstream newFile (mycnfFile.c_str());
//create new file
int fd = open(mycnfFile.c_str(), O_RDWR | O_CREAT, 0666);
int fd = open(mycnfFile.c_str(), O_RDWR|O_CREAT, 0644);
copy(lines.begin(), lines.end(), ostream_iterator<string>(newFile, "\n"));
newFile.close();

2
utils/funcexp/func_concat_ws.cpp
View File

@@ -97,7 +97,7 @@ string Func_concat_ws::getStrVal(Row& row,
string tmp;
for ( uint32_t i = 1 ; i < parm.size() ; i++)
{
string(stringValue(parm[i], row, isNull).c_str(), tmp);
stringValue(parm[i], row, isNull, tmp);
str += tmp;
if (isNull)

114
writeengine/bulk/we_bulkloadbuffer.cpp
View File

@@ -31,8 +31,6 @@
#include <ctype.h>
#include <cfloat>
#include <boost/algorithm/string/predicate.hpp>
#include "we_bulkload.h"
#include "we_bulkloadbuffer.h"
#include "we_brm.h"
@@ -370,18 +368,11 @@ void BulkLoadBuffer::convert(char* field, int fieldLength,
{
errno = 0;
if (iequals(field, "true"))
{
fVal = 1;
}
else
{
#ifdef _MSC_VER
fVal = (float)strtod( field, 0 );
fVal = (float)strtod( field, 0 );
#else
fVal = strtof( field, 0 );
fVal = strtof( field, 0 );
#endif
}
if (errno == ERANGE)
{
@@ -414,6 +405,11 @@ void BulkLoadBuffer::convert(char* field, int fieldLength,
fVal = minFltSat;
bufStats.satCount++;
}
if ( fVal == 0
&& isTrueWord(const_cast<const char*>(field), fieldLength) )
{
fVal = 1;
}
}
}
@@ -474,14 +470,7 @@ void BulkLoadBuffer::convert(char* field, int fieldLength,
{
errno = 0;
if (iequals(field, "true"))
{
dVal = 1;
}
else
{
dVal = strtod(field, 0);
}
dVal = strtod(field, 0);
if (errno == ERANGE)
{
@@ -514,6 +503,11 @@ void BulkLoadBuffer::convert(char* field, int fieldLength,
dVal = column.fMinDblSat;
bufStats.satCount++;
}
else if (dVal == 0
&& isTrueWord(const_cast<const char*>(field), fieldLength))
{
dVal = 1;
}
}
}
@@ -629,12 +623,6 @@ void BulkLoadBuffer::convert(char* field, int fieldLength,
}
else
{
if (iequals(field, "true"))
{
strcpy(field, "1");
fieldLength = 1;
}
if ( (column.dataType == CalpontSystemCatalog::DECIMAL ) ||
(column.dataType == CalpontSystemCatalog::UDECIMAL) )
{
@@ -664,6 +652,11 @@ void BulkLoadBuffer::convert(char* field, int fieldLength,
origVal = static_cast<int64_t>(column.fMaxIntSat);
bSatVal = true;
}
else if ( origVal == 0
&& isTrueWord(const_cast<const char*>(field), fieldLength) )
{
origVal = 1;
}
if (bSatVal)
bufStats.satCount++;
@@ -722,15 +715,8 @@ void BulkLoadBuffer::convert(char* field, int fieldLength,
{
errno = 0;
if (iequals(field, "true"))
{
origVal = 1;
}
else
{
origVal = strtoll(field, 0, 10);
}
origVal = strtoll(field, 0, 10);
if (errno == ERANGE)
bSatVal = true;
}
@@ -747,6 +733,11 @@ void BulkLoadBuffer::convert(char* field, int fieldLength,
origVal = static_cast<int64_t>(column.fMaxIntSat);
bSatVal = true;
}
else if ( origVal == 0
&& isTrueWord(const_cast<const char*>(field), fieldLength) )
{
origVal = 1;
}
if (bSatVal)
bufStats.satCount++;
@@ -805,12 +796,12 @@ void BulkLoadBuffer::convert(char* field, int fieldLength,
}
else
{
if (iequals(field, "true"))
if (isTrueWord(const_cast<const char*>(field), fieldLength))
{
strcpy(field, "1");
fieldLength = 1;
}
if ( (column.dataType == CalpontSystemCatalog::DECIMAL ) ||
(column.dataType == CalpontSystemCatalog::UDECIMAL))
{
@@ -898,14 +889,7 @@ void BulkLoadBuffer::convert(char* field, int fieldLength,
{
errno = 0;
if (iequals(field, "true"))
{
origVal = 1;
}
else
{
origVal = strtoll(field, 0, 10);
}
origVal = strtoll(field, 0, 10);
if (errno == ERANGE)
bSatVal = true;
@@ -923,6 +907,11 @@ void BulkLoadBuffer::convert(char* field, int fieldLength,
origVal = static_cast<int64_t>(column.fMaxIntSat);
bSatVal = true;
}
else if ( origVal == 0
&& isTrueWord(const_cast<const char*>(field), fieldLength) )
{
origVal = 1;
}
if (bSatVal)
bufStats.satCount++;
@@ -981,12 +970,12 @@ void BulkLoadBuffer::convert(char* field, int fieldLength,
}
else
{
if (iequals(field, "true"))
if (isTrueWord(const_cast<const char*>(field), fieldLength))
{
strcpy(field, "1");
fieldLength = 1;
}
if ( (column.dataType == CalpontSystemCatalog::DECIMAL) ||
(column.dataType == CalpontSystemCatalog::UDECIMAL))
{
@@ -1018,6 +1007,7 @@ void BulkLoadBuffer::convert(char* field, int fieldLength,
bSatVal = true;
}
if (bSatVal)
bufStats.satCount++;
@@ -1199,14 +1189,7 @@ void BulkLoadBuffer::convert(char* field, int fieldLength,
{
errno = 0;
if (iequals(field, "true"))
{
ullVal = 1;
}
else
{
ullVal = strtoull(field, 0, 10);
}
ullVal = strtoull(field, 0, 10);
if (errno == ERANGE)
bSatVal = true;
@@ -1220,6 +1203,11 @@ void BulkLoadBuffer::convert(char* field, int fieldLength,
ullVal = column.fMaxIntSat;
bSatVal = true;
}
else if ( ullVal == 0
&& isTrueWord(const_cast<const char*>(field), fieldLength) )
{
ullVal = 1;
}
if (bSatVal)
bufStats.satCount++;
@@ -1276,14 +1264,7 @@ void BulkLoadBuffer::convert(char* field, int fieldLength,
{
errno = 0;
if (iequals(field, "true"))
{
origVal = 1;
}
else
{
origVal = strtoll(field, 0, 10);
}
origVal = strtoll(field, 0, 10);
if (errno == ERANGE)
bSatVal = true;
@@ -1301,6 +1282,11 @@ void BulkLoadBuffer::convert(char* field, int fieldLength,
origVal = static_cast<int64_t>(column.fMaxIntSat);
bSatVal = true;
}
else if ( origVal == 0
&& isTrueWord(const_cast<const char*>(field), fieldLength) )
{
origVal = 1;
}
if (bSatVal)
bufStats.satCount++;
@@ -1361,12 +1347,12 @@ void BulkLoadBuffer::convert(char* field, int fieldLength,
}
else
{
if (iequals(field, "true"))
if (isTrueWord(const_cast<const char*>(field), fieldLength))
{
strcpy(field, "1");
fieldLength = 1;
}
if ( (column.dataType == CalpontSystemCatalog::DECIMAL) ||
(column.dataType == CalpontSystemCatalog::UDECIMAL))
{

9
writeengine/bulk/we_bulkloadbuffer.h
View File

@@ -383,5 +383,14 @@ public:
}
};
inline bool isTrueWord(const char *field, int fieldLength)
{
//return false;
return fieldLength == 4 && ( field[0] == 'T' || field[0] == 't' )
&& ( field[1] == 'R' || field[1] == 'r' )
&& ( field[2] == 'U' || field[2] == 'u' )
&& ( field[3] == 'E' || field[3] == 'e' );
}
}
#endif