database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

Merge branch 'develop' into join-abort-fix

Browse Source
This commit is contained in:
Patrick LeBlanc
2020-01-08 15:19:12 -06:00
committed by GitHub
parent 04807c3ef8 8e6fde3b71
commit a3409e1a94
24 changed files with 85 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
README.md
View File

@ -1,13 +1,9 @@
# MariaDB ColumnStore Storage/Execution engine 1.4
MariaDB ColumnStore 1.4 is a Beta version of MariaDB ColumnStore.
MariaDB ColumnStore 1.4 is a GA version of MariaDB ColumnStore.
It is built by porting InfiniDB 4.6.7 on MariaDB and adding entirely
new features not found anywhere else.
# MariaDB ColumnStore 1.4 is a Beta release.
- Do not use pre-releases on production systems.
# Building
This repository is not meant to be built independently outside of the server. This repository is integrated into http://mariadb-corporation/mariadb-columnstore-server (ie, the *server*) as a git submodule. As such, you can find complete build instructions on *the server* page.

2
VERSION
View File

@ -1,4 +1,4 @@
COLUMNSTORE_VERSION_MAJOR=1
COLUMNSTORE_VERSION_MINOR=4
COLUMNSTORE_VERSION_PATCH=2
COLUMNSTORE_VERSION_PATCH=3
COLUMNSTORE_VERSION_RELEASE=1

1
dbcon/joblist/tuplehashjoin.cpp
View File

@ -138,6 +138,7 @@ TupleHashJoinStep::~TupleHashJoinStep()
resourceManager->returnMemory(memUsedByEachJoin[i], sessionMemLimit);
}
//cout << "deallocated THJS, UM memory available: " << resourceManager.availableMemory() << endl;
}

1
dbcon/joblist/tupleunion.cpp
View File

@ -94,7 +94,6 @@ TupleUnion::TupleUnion(CalpontSystemCatalog::OID tableOID, const JobInfo& jobInf
outputIt(-1),
memUsage(0),
rm(jobInfo.rm),
allocator(64 * 1024 * 1024 + 1),
runnersDone(0),
distinctCount(0),
distinctDone(0),

1
dbcon/mysql/columnstore.cnf
View File

@ -16,6 +16,7 @@ plugin-load-add=ha_columnstore.so
# Required for Schema Sync
#server-id = 1
#log_bin
# Uncomment for this UM to apply slave DML on ColumnStore tables
#columnstore_replication_slave=ON

2
oam/etc/ConsoleCmds.xml
View File

@ -82,7 +82,7 @@
</Cmd10>
<Cmd11>
<Name>setAlarmConfig</Name>
<Desc1>Set a Alarm Configuration parameter</Desc1>
<Desc1>Set an Alarm Configuration parameter</Desc1>
<Arg1>Required: Alarm-ID is the Alarm to be updated</Arg1>
<Arg2>Required: Parameter-name is the parameter to be updated</Arg2>
<Arg3>Required: Parameter-value is the parameter value</Arg3>

21
oam/install_scripts/columnstore_run.sh
View File

@ -48,8 +48,27 @@ if [ $vflg -gt 0 ]; then
echo "starting $exename $args with sleep=$sopt and tries=$topt"
fi
which_jemalloc() {
LD_PRELOAD="$1" /bin/true >& /tmp/jemalloc_test
grep -i error /tmp/jemalloc_test >& /dev/null
if [ $? -ne 0 ] ; then
JEMALLOC="$1"
else
unset JEMALLOC
fi
rm /tmp/jemalloc_test
}
which_jemalloc libjemalloc.so
if [ -z "$JEMALLOC" ] ; then
which_jemalloc libjemalloc.so.1
fi
if [ -z "$JEMALLOC" ] ; then
which_jemalloc libjemalloc.so.2
fi
while [ $keep_going -ne 0 ]; do
LD_PRELOAD=libjemalloc.so $exename $args
LD_PRELOAD=$JEMALLOC $exename $args
if [ -e ${lopt}/StopColumnstore ]; then
exit 0
fi

2
oamapps/postConfigure/helpers.cpp
View File

@ -411,7 +411,7 @@ int sendReplicationRequest(int IserverTypeInstall, std::string password, bool pm
if ( returnStatus != API_SUCCESS)
{
cout << endl << "ERROR: Error return in running the MariaDB ColumnStore Master replication, check " + tmpDir + "master-rep*.logs on " << masterModule << endl;
cout << endl << "ERROR: Error return in running the MariaDB ColumnStore Master replication, check " + tmpDir + "/master-rep*.logs on " << masterModule << endl;
return returnStatus;
}

12
oamapps/postConfigure/postConfigure.cpp
View File

@ -3539,11 +3539,11 @@ int main(int argc, char* argv[])
{
cout << endl;
cout << "Next step is to enter the password to access the other Servers." << endl;
cout << "This is either user password or you can default to using a ssh key" << endl;
cout << "This is either user password or you can default to using an ssh key" << endl;
cout << "If using a user password, the password needs to be the same on all Servers." << endl << endl;
if ( noPrompting ) {
cout << "Enter password, hit 'enter' to default to using a ssh key, or 'exit' > " << endl;
cout << "Enter password, hit 'enter' to default to using an ssh key, or 'exit' > " << endl;
password = "ssh";
}
else
@ -3552,7 +3552,7 @@ int main(int argc, char* argv[])
{
char *pass1, *pass2;
pass1=getpass("Enter password, hit 'enter' to default to using a ssh key, or 'exit' > ");
pass1=getpass("Enter password, hit 'enter' to default to using an ssh key, or 'exit' > ");
if ( strcmp(pass1, "") == 0 ) {
password = "ssh";
break;
@ -3633,7 +3633,7 @@ int main(int argc, char* argv[])
{
cout << endl;
cout << "Next step is to enter the password to access the other Servers." << endl;
cout << "This is either your password or you can default to using a ssh key" << endl;
cout << "This is either your password or you can default to using an ssh key" << endl;
cout << "If using a password, the password needs to be the same on all Servers." << endl << endl;
}
@ -3643,7 +3643,7 @@ int main(int argc, char* argv[])
if ( noPrompting )
{
cout << "Enter password, hit 'enter' to default to using a ssh key, or 'exit' > " << endl;
cout << "Enter password, hit 'enter' to default to using an ssh key, or 'exit' > " << endl;
if ( password.empty() )
password = "ssh";
@ -3658,7 +3658,7 @@ int main(int argc, char* argv[])
break;
}
pass1 = getpass("Enter password, hit 'enter' to default to using a ssh key, or 'exit' > ");
pass1 = getpass("Enter password, hit 'enter' to default to using an ssh key, or 'exit' > ");
if ( strcmp(pass1, "") == 0 )
{

4
oamapps/postConfigure/quick_installer_amazon.sh
View File

@ -25,7 +25,7 @@ for arg in "$@"; do
echo "Performace Module (pm) number is required"
echo "User Module (um) number is option"
echo "When only pm counts provided, system is combined setup"
echo "When both pm/um counts provided, system is seperate setup"
echo "When both pm/um counts provided, system is separate setup"
echo
echo "--pm-count=x Number of pm instances to create"
echo "--um-count=x Number of um instances to create, optional"
@ -49,7 +49,7 @@ else
echo""
else
echo ""
echo "NOTE: Performing a Multi-Server Seperate install with um and pm running on seperate servers"
echo "NOTE: Performing a Multi-Server separate install with um and pm running on separate servers"
echo""
fi
fi

4
oamapps/postConfigure/quick_installer_multi_server.sh
View File

@ -27,7 +27,7 @@ for arg in "$@"; do
echo "Performace Module (pm) IP addresses are required"
echo "User Module (um) IP addresses are option"
echo "When only pm IP addresses provided, system is combined setup"
echo "When both pm/um IP addresses provided, system is seperate setup"
echo "When both pm/um IP addresses provided, system is separate setup"
echo
echo "--pm-ip-addresses=xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx"
echo "--um-ip-addresses=xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx, optional"
@ -51,7 +51,7 @@ else
echo""
else
echo ""
echo "NOTE: Performing a Multi-Server Seperate install with um and pm running on seperate servers"
echo "NOTE: Performing a Multi-Server separate install with um and pm running on separate servers"
echo""
fi
fi

12
procmon/processmonitor.cpp
View File

@ -4804,6 +4804,12 @@ int ProcessMonitor::changeMyCnf(std::string type)
}
}
pos = buf.find("log_bin", 0);
if ( pos != string::npos )
{
buf = "log_bin";
}
// set local query flag if on pm
if ( (PMwithUM == "y") && config.moduleType() == "pm" )
{
@ -4870,13 +4876,12 @@ int ProcessMonitor::changeMyCnf(std::string type)
system(cmd.c_str());
// restart mysql
/* try {
try {
oam.actionMysqlCalpont(MYSQL_RESTART);
sleep(5); // give after mysql restart
}
catch(...)
{}
*/
log.writeLog(__LINE__, "changeMyCnf function successfully completed", LOG_TYPE_DEBUG);
return oam::API_SUCCESS;
@ -5076,10 +5081,11 @@ int ProcessMonitor::runMasterRep(std::string& masterLogFile, std::string& master
while (file.getline(line, 200))
{
buf = line;
string::size_type pos = buf.find("mysql-bin", 0);
string::size_type pos = buf.find("000", 0);
if ( pos != string::npos )
{
pos = 0;
string::size_type pos1 = buf.find("\t", pos);
if ( pos1 != string::npos )

2
storage-manager/src/AppendTask.cpp
View File

@ -90,7 +90,7 @@ bool AppendTask::run()
}
catch (exception &e)
{
logger->log(LOG_DEBUG, "AppendTask: caught '%s'", e.what());
logger->log(LOG_ERR, "AppendTask: caught '%s'", e.what());
errno = EIO;
err = -1;
}

2
storage-manager/src/CopyTask.cpp
View File

@ -70,7 +70,7 @@ bool CopyTask::run()
}
catch (exception &e)
{
logger->log(LOG_DEBUG, "CopyTask: caught %s", e.what());
logger->log(LOG_ERR, "CopyTask: caught %s", e.what());
err = -1;
errno = EIO;
}

2
storage-manager/src/ListDirectoryTask.cpp
View File

@ -103,7 +103,7 @@ bool ListDirectoryTask::run()
}
catch (exception &e)
{
logger->log(LOG_DEBUG, "ListDirectoryTask: caught '%s'", e.what());
logger->log(LOG_ERR, "ListDirectoryTask: caught '%s'", e.what());
errno = EIO;
err = -1;
}

2
storage-manager/src/OpenTask.cpp
View File

@ -75,7 +75,7 @@ bool OpenTask::run()
}
catch (exception &e)
{
logger->log(LOG_DEBUG, "OpenTask: caught '%s'", e.what());
logger->log(LOG_ERR, "OpenTask: caught '%s'", e.what());
errno = EIO;
err = -1;
}

2
storage-manager/src/ReadTask.cpp
View File

@ -85,7 +85,7 @@ bool ReadTask::run()
}
catch (exception &e)
{
logger->log(LOG_DEBUG, "ReadTask: caught '%s'", e.what());
logger->log(LOG_ERR, "ReadTask: caught '%s'", e.what());
errno = EIO;
err = -1;
}

40
storage-manager/src/S3Storage.cpp
View File

@ -170,7 +170,7 @@ void S3Storage::testConnectivityAndPerms()
err = deleteObject(testObjKey);
if (err)
FAIL(DELETE)
logger->log(LOG_DEBUG, "S3Storage: S3 connectivity & permissions are OK");
logger->log(LOG_INFO, "S3Storage: S3 connectivity & permissions are OK");
}
int S3Storage::getObject(const string &sourceKey, const string &destFile, size_t *size)
@ -227,10 +227,10 @@ int S3Storage::getObject(const string &_sourceKey, boost::shared_array<uint8_t>
if (err && retryable_error(err))
{
if (ms3_server_error(creds))
logger->log(LOG_WARNING, "S3Storage::getObject(): failed to GET, server says '%s'. bucket = %s, key = %s."
logger->log(LOG_ERR, "S3Storage::getObject(): failed to GET, server says '%s'. bucket = %s, key = %s."
" Retrying...", ms3_server_error(creds), bucket.c_str(), sourceKey.c_str());
else
logger->log(LOG_WARNING, "S3Storage::getObject(): failed to GET, got '%s'. bucket = %s, key = %s. Retrying...",
logger->log(LOG_ERR, "S3Storage::getObject(): failed to GET, got '%s'. bucket = %s, key = %s. Retrying...",
s3err_msgs[err], bucket.c_str(), sourceKey.c_str());
sleep(5);
}
@ -238,10 +238,10 @@ int S3Storage::getObject(const string &_sourceKey, boost::shared_array<uint8_t>
if (err)
{
if (ms3_server_error(creds))
logger->log(LOG_WARNING, "S3Storage::getObject(): failed to GET, server says '%s'. bucket = %s, key = %s.",
logger->log(LOG_ERR, "S3Storage::getObject(): failed to GET, server says '%s'. bucket = %s, key = %s.",
ms3_server_error(creds), bucket.c_str(), sourceKey.c_str());
else
logger->log(LOG_WARNING, "S3Storage::getObject(): failed to GET, got '%s'. bucket = %s, key = %s.",
logger->log(LOG_ERR, "S3Storage::getObject(): failed to GET, got '%s'. bucket = %s, key = %s.",
s3err_msgs[err], bucket.c_str(), sourceKey.c_str());
data->reset();
errno = s3err_to_errno[err];
@ -317,10 +317,10 @@ int S3Storage::putObject(const boost::shared_array<uint8_t> data, size_t len, co
if (s3err && retryable_error(s3err))
{
if (ms3_server_error(creds))
logger->log(LOG_WARNING, "S3Storage::putObject(): failed to PUT, server says '%s'. bucket = %s, key = %s."
logger->log(LOG_ERR, "S3Storage::putObject(): failed to PUT, server says '%s'. bucket = %s, key = %s."
" Retrying...", ms3_server_error(creds), bucket.c_str(), destKey.c_str());
else
logger->log(LOG_WARNING, "S3Storage::putObject(): failed to PUT, got '%s'. bucket = %s, key = %s."
logger->log(LOG_ERR, "S3Storage::putObject(): failed to PUT, got '%s'. bucket = %s, key = %s."
" Retrying...", s3err_msgs[s3err], bucket.c_str(), destKey.c_str());
sleep(5);
}
@ -328,7 +328,7 @@ int S3Storage::putObject(const boost::shared_array<uint8_t> data, size_t len, co
if (s3err)
{
if (ms3_server_error(creds))
logger->log(LOG_WARNING, "S3Storage::putObject(): failed to PUT, server says '%s'. bucket = %s, key = %s.",
logger->log(LOG_ERR, "S3Storage::putObject(): failed to PUT, server says '%s'. bucket = %s, key = %s.",
ms3_server_error(creds), bucket.c_str(), destKey.c_str());
else
logger->log(LOG_ERR, "S3Storage::putObject(): failed to PUT, got '%s'. bucket = %s, key = %s.",
@ -351,10 +351,10 @@ int S3Storage::deleteObject(const string &_key)
if (s3err && s3err != MS3_ERR_NOT_FOUND && retryable_error(s3err))
{
if (ms3_server_error(creds))
logger->log(LOG_WARNING, "S3Storage::deleteObject(): failed to DELETE, server says '%s'. bucket = %s, key = %s."
logger->log(LOG_ERR, "S3Storage::deleteObject(): failed to DELETE, server says '%s'. bucket = %s, key = %s."
" Retrying...", ms3_server_error(creds), bucket.c_str(), key.c_str());
else
logger->log(LOG_CRIT, "S3Storage::deleteObject(): failed to DELETE, got '%s'. bucket = %s, key = %s. Retrying...",
logger->log(LOG_ERR, "S3Storage::deleteObject(): failed to DELETE, got '%s'. bucket = %s, key = %s. Retrying...",
s3err_msgs[s3err], bucket.c_str(), key.c_str());
sleep(5);
}
@ -363,10 +363,10 @@ int S3Storage::deleteObject(const string &_key)
if (s3err != 0 && s3err != MS3_ERR_NOT_FOUND)
{
if (ms3_server_error(creds))
logger->log(LOG_WARNING, "S3Storage::deleteObject(): failed to DELETE, server says '%s'. bucket = %s, key = %s.",
logger->log(LOG_ERR, "S3Storage::deleteObject(): failed to DELETE, server says '%s'. bucket = %s, key = %s.",
ms3_server_error(creds), bucket.c_str(), key.c_str());
else
logger->log(LOG_CRIT, "S3Storage::deleteObject(): failed to DELETE, got '%s'. bucket = %s, key = %s.",
logger->log(LOG_ERR, "S3Storage::deleteObject(): failed to DELETE, got '%s'. bucket = %s, key = %s.",
s3err_msgs[s3err], bucket.c_str(), key.c_str());
return -1;
}
@ -386,10 +386,10 @@ int S3Storage::copyObject(const string &_sourceKey, const string &_destKey)
if (s3err && retryable_error(s3err))
{
if (ms3_server_error(creds))
logger->log(LOG_WARNING, "S3Storage::copyObject(): failed to copy, server says '%s'. bucket = %s, srckey = %s, "
logger->log(LOG_ERR, "S3Storage::copyObject(): failed to copy, server says '%s'. bucket = %s, srckey = %s, "
"destkey = %s. Retrying...", ms3_server_error(creds), bucket.c_str(), sourceKey.c_str(), destKey.c_str());
else
logger->log(LOG_CRIT, "S3Storage::copyObject(): failed to copy, got '%s'. bucket = %s, srckey = %s, "
logger->log(LOG_ERR, "S3Storage::copyObject(): failed to copy, got '%s'. bucket = %s, srckey = %s, "
" destkey = %s. Retrying...", s3err_msgs[s3err], bucket.c_str(), sourceKey.c_str(), destKey.c_str());
sleep(5);
}
@ -399,10 +399,10 @@ int S3Storage::copyObject(const string &_sourceKey, const string &_destKey)
{
// added the add'l check MS3_ERR_NOT_FOUND to suppress error msgs for a legitimate case in IOC::copyFile()
if (ms3_server_error(creds) && s3err != MS3_ERR_NOT_FOUND)
logger->log(LOG_WARNING, "S3Storage::copyObject(): failed to copy, server says '%s'. bucket = %s, srckey = %s, "
logger->log(LOG_ERR, "S3Storage::copyObject(): failed to copy, server says '%s'. bucket = %s, srckey = %s, "
"destkey = %s.", ms3_server_error(creds), bucket.c_str(), sourceKey.c_str(), destKey.c_str());
else if (s3err != MS3_ERR_NOT_FOUND)
logger->log(LOG_CRIT, "S3Storage::copyObject(): failed to copy, got '%s'. bucket = %s, srckey = %s, "
logger->log(LOG_ERR, "S3Storage::copyObject(): failed to copy, got '%s'. bucket = %s, srckey = %s, "
"destkey = %s.", s3err_msgs[s3err], bucket.c_str(), sourceKey.c_str(), destKey.c_str());
errno = s3err_to_errno[s3err];
return -1;
@ -435,10 +435,10 @@ int S3Storage::exists(const string &_key, bool *out)
if (s3err && s3err != MS3_ERR_NOT_FOUND && retryable_error(s3err))
{
if (ms3_server_error(creds))
logger->log(LOG_WARNING, "S3Storage::exists(): failed to HEAD, server says '%s'. bucket = %s, key = %s."
logger->log(LOG_ERR, "S3Storage::exists(): failed to HEAD, server says '%s'. bucket = %s, key = %s."
" Retrying...", ms3_server_error(creds), bucket.c_str(), key.c_str());
else
logger->log(LOG_CRIT, "S3Storage::exists(): failed to HEAD, got '%s'. bucket = %s, key = %s. Retrying...",
logger->log(LOG_ERR, "S3Storage::exists(): failed to HEAD, got '%s'. bucket = %s, key = %s. Retrying...",
s3err_msgs[s3err], bucket.c_str(), key.c_str());
sleep(5);
}
@ -447,10 +447,10 @@ int S3Storage::exists(const string &_key, bool *out)
if (s3err != 0 && s3err != MS3_ERR_NOT_FOUND)
{
if (ms3_server_error(creds))
logger->log(LOG_WARNING, "S3Storage::exists(): failed to HEAD, server says '%s'. bucket = %s, key = %s.",
logger->log(LOG_ERR, "S3Storage::exists(): failed to HEAD, server says '%s'. bucket = %s, key = %s.",
ms3_server_error(creds), bucket.c_str(), key.c_str());
else
logger->log(LOG_CRIT, "S3Storage::exists(): failed to HEAD, got '%s'. bucket = %s, key = %s.",
logger->log(LOG_ERR, "S3Storage::exists(): failed to HEAD, got '%s'. bucket = %s, key = %s.",
s3err_msgs[s3err], bucket.c_str(), key.c_str());
errno = s3err_to_errno[s3err];
return -1;

2
storage-manager/src/SessionManager.cpp
View File

@ -475,7 +475,7 @@ void SessionManager::socketError(int socket)
void SessionManager::shutdownSM(int sig){
boost::mutex::scoped_lock s(ctrlMutex);
SMLogging* logger = SMLogging::get();
logger->log(LOG_INFO,"SessionManager Caught Signal %i",sig);
logger->log(LOG_DEBUG,"SessionManager Caught Signal %i",sig);
int err;
uint8_t ctrlCode = SHUTDOWN;
err = ::write(socketCtrl[1], &ctrlCode, 1);

2
storage-manager/src/StatTask.cpp
View File

@ -71,7 +71,7 @@ bool StatTask::run()
}
catch (exception &e)
{
logger->log(LOG_DEBUG, "StatTask: caught '%s'", e.what());
logger->log(LOG_ERR, "StatTask: caught '%s'", e.what());
errno = EIO;
err = -1;
}

2
storage-manager/src/TruncateTask.cpp
View File

@ -67,7 +67,7 @@ bool TruncateTask::run()
}
catch (exception &e)
{
logger->log(LOG_DEBUG, "TruncateTask: caught '%s'", e.what());
logger->log(LOG_ERR, "TruncateTask: caught '%s'", e.what());
errno = EIO;
err = -1;
}

2
storage-manager/src/UnlinkTask.cpp
View File

@ -68,7 +68,7 @@ bool UnlinkTask::run()
}
catch (exception &e)
{
logger->log(LOG_DEBUG, "UnlinkTask: caught '%s'", e.what());
logger->log(LOG_ERR, "UnlinkTask: caught '%s'", e.what());
errno = EIO;
err = -1;
}

2
storage-manager/src/WriteTask.cpp
View File

@ -90,7 +90,7 @@ bool WriteTask::run()
}
catch (exception &e)
{
logger->log(LOG_DEBUG, "WriteTask: caught '%s'", e.what());
logger->log(LOG_ERR, "WriteTask: caught '%s'", e.what());
errno = EIO;
err = -1;
}