From 929a15ea013ec7dcd7f6f3eb690dfb066be1a733 Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Tue, 25 Aug 2020 22:03:28 -0400 Subject: [PATCH 01/18] Use the HAVE_PSI_INTERFACE preprocessor guard to disable code that does not compile if -DPLUGIN_PERFSCHEMA=NO cmake option is used. --- dbcon/mysql/ha_mcs.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dbcon/mysql/ha_mcs.cpp b/dbcon/mysql/ha_mcs.cpp index 5d9c2fbf8..cc8549435 100644 --- a/dbcon/mysql/ha_mcs.cpp +++ b/dbcon/mysql/ha_mcs.cpp @@ -1273,10 +1273,12 @@ static my_bool cache_check_status(void *param) static ha_mcs_cache_share *cache_share_list= 0; static PSI_mutex_key key_LOCK_cache_share; +#ifdef HAVE_PSI_INTERFACE static PSI_mutex_info all_mutexes[]= { { &key_LOCK_cache_share, "LOCK_cache_share", PSI_FLAG_GLOBAL}, }; +#endif static mysql_mutex_t LOCK_cache_share; /* @@ -1810,8 +1812,10 @@ static int columnstore_init_func(void* p) mcs_hton->create_select = create_columnstore_select_handler; mcs_hton->db_type = DB_TYPE_AUTOASSIGN; +#ifdef HAVE_PSI_INTERFACE uint count = sizeof(all_mutexes)/sizeof(all_mutexes[0]); mysql_mutex_register("ha_mcs_cache", all_mutexes, count); +#endif mysql_mutex_init(key_LOCK_cache_share, &LOCK_cache_share, MY_MUTEX_INIT_FAST); DBUG_RETURN(0); From 8e711b0c4bcb9baa19ccaeb3c3e2f3367682c8df Mon Sep 17 00:00:00 2001 From: mariadb-RomanNavrotskiy Date: Tue, 25 Aug 2020 19:15:23 +0200 Subject: [PATCH 02/18] remove plugin_perschema cmake option --- .drone.jsonnet | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index 0549a2d67..7ecbc10ee 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -9,7 +9,7 @@ local codebase_map = { }; local builddir = 'verylongdirnameforverystrangecpackbehavior'; -local cmakeflags = '-DCMAKE_BUILD_TYPE=RelWithDebInfo -DPLUGIN_COLUMNSTORE=YES -DPLUGIN_MROONGA=NO -DPLUGIN_ROCKSDB=NO -DPLUGIN_TOKUDB=NO -DPLUGIN_CONNECT=NO -DPLUGIN_SPIDER=NO -DPLUGIN_OQGRAPH=NO -DPLUGIN_PERFSCHEMA=NO -DPLUGIN_SPHINX=NO'; +local cmakeflags = '-DCMAKE_BUILD_TYPE=RelWithDebInfo -DPLUGIN_COLUMNSTORE=YES -DPLUGIN_MROONGA=NO -DPLUGIN_ROCKSDB=NO -DPLUGIN_TOKUDB=NO -DPLUGIN_CONNECT=NO -DPLUGIN_SPIDER=NO -DPLUGIN_OQGRAPH=NO -DPLUGIN_SPHINX=NO'; local rpm_build_deps = 'install -y systemd-devel git make gcc gcc-c++ libaio-devel openssl-devel boost-devel bison snappy-devel flex libcurl-devel libxml2-devel ncurses-devel automake libtool policycoreutils-devel rpm-build lsof iproute pam-devel perl-DBI cracklib-devel expect readline-devel createrepo'; @@ -81,8 +81,6 @@ local Pipeline(branch, platform, event) = { commands: [ // clone mtr repo 'git clone --depth 1 https://github.com/mariadb-corporation/columnstore-tests', - // temporary disable insert_from_another_table test. see https://jira.mariadb.org/browse/MCOL-4247 - 'rm -f columnstore-tests/mysql-test/suite/columnstore/basic/t/mcs24_insert_from_another_table.test', 'docker run --volume /sys/fs/cgroup:/sys/fs/cgroup:ro --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name mtr$${DRONE_BUILD_NUMBER} --privileged --detach ' + img + ' ' + init + ' --unit=basic.target', 'docker cp result mtr$${DRONE_BUILD_NUMBER}:/', if (std.split(platform, ':')[0] == 'centos') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "yum install -y epel-release which rsyslog hostname && yum install -y /result/*.' + pkg_format + '"' else '', @@ -116,6 +114,7 @@ local Pipeline(branch, platform, event) = { regression:: { name: 'regression', image: 'docker:git', + #failure: (if event == 'cron' then 'ignore' else ''), failure: 'ignore', volumes: [pipeline._volumes.docker, pipeline._volumes.mdb], commands: [ @@ -267,8 +266,9 @@ local Pipeline(branch, platform, event) = { "sed -i -e 's/\"galera-enterprise-4\"//' cmake/cpack_rpm.cmake", "sed -i '/columnstore/Id' debian/autobake-deb.sh", "sed -i 's/.*flex.*/echo/' debian/autobake-deb.sh", - "sed -i 's/BETA/GAMMA/' storage/columnstore/CMakeLists.txt", - "sed -i -e '/mcs-start-storagemanager.py/d' debian/mariadb-plugin-columnstore.install", + "sed -i 's/-DPLUGIN_PERFSCHEMA=NO//' debian/autobake-deb.sh", + #"sed -i 's/BETA/GAMMA/' storage/columnstore/CMakeLists.txt", + #"sed -i -e '/mcs-start-storagemanager.py/d' debian/mariadb-plugin-columnstore.install", platformMap(branch, platform), (if pkg_format == 'rpm' then 'createrepo .' else 'dpkg-scanpackages ../ | gzip > ../Packages.gz '), ], From 2951dbed3937083e33d18e9c49d274f1306eba59 Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Fri, 8 May 2020 17:10:37 -0500 Subject: [PATCH 03/18] MCOL-3494: make option for testing configuration to skip retryable errors. --- storage-manager/src/S3Storage.cpp | 22 +++++++++++++++++++++- storage-manager/src/S3Storage.h | 5 ++++- storage-manager/src/testS3Connection.cpp | 2 +- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/storage-manager/src/S3Storage.cpp b/storage-manager/src/S3Storage.cpp index f37e5d90a..0195341e4 100644 --- a/storage-manager/src/S3Storage.cpp +++ b/storage-manager/src/S3Storage.cpp @@ -89,7 +89,7 @@ S3Storage::ScopedConnection::~ScopedConnection() s3->returnConnection(conn); } -S3Storage::S3Storage() +S3Storage::S3Storage(bool skipRetry) : skipRetryableErrors(skipRetry) { /* Check creds from envvars Get necessary vars from config @@ -239,6 +239,10 @@ int S3Storage::getObject(const string &_sourceKey, boost::shared_array else logger->log(LOG_ERR, "S3Storage::getObject(): failed to GET, got '%s'. bucket = %s, key = %s. Retrying...", s3err_msgs[err], bucket.c_str(), sourceKey.c_str()); + if (skipRetryableErrors) + { + return -1; + } sleep(5); } } while (err && retryable_error(err)); @@ -329,6 +333,10 @@ int S3Storage::putObject(const boost::shared_array data, size_t len, co else logger->log(LOG_ERR, "S3Storage::putObject(): failed to PUT, got '%s'. bucket = %s, key = %s." " Retrying...", s3err_msgs[s3err], bucket.c_str(), destKey.c_str()); + if (skipRetryableErrors) + { + return -1; + } sleep(5); } } while (s3err && retryable_error(s3err)); @@ -363,6 +371,10 @@ int S3Storage::deleteObject(const string &_key) else logger->log(LOG_ERR, "S3Storage::deleteObject(): failed to DELETE, got '%s'. bucket = %s, key = %s. Retrying...", s3err_msgs[s3err], bucket.c_str(), key.c_str()); + if (skipRetryableErrors) + { + return -1; + } sleep(5); } } while (s3err && s3err != MS3_ERR_NOT_FOUND && retryable_error(s3err)); @@ -398,6 +410,10 @@ int S3Storage::copyObject(const string &_sourceKey, const string &_destKey) else logger->log(LOG_ERR, "S3Storage::copyObject(): failed to copy, got '%s'. bucket = %s, srckey = %s, " " destkey = %s. Retrying...", s3err_msgs[s3err], bucket.c_str(), sourceKey.c_str(), destKey.c_str()); + if (skipRetryableErrors) + { + return -1; + } sleep(5); } } while (s3err && retryable_error(s3err)); @@ -447,6 +463,10 @@ int S3Storage::exists(const string &_key, bool *out) else logger->log(LOG_ERR, "S3Storage::exists(): failed to HEAD, got '%s'. bucket = %s, key = %s. Retrying...", s3err_msgs[s3err], bucket.c_str(), key.c_str()); + if (skipRetryableErrors) + { + return -1; + } sleep(5); } } while (s3err && s3err != MS3_ERR_NOT_FOUND && retryable_error(s3err)); diff --git a/storage-manager/src/S3Storage.h b/storage-manager/src/S3Storage.h index 3b9b4ae2a..ba20a6b7c 100644 --- a/storage-manager/src/S3Storage.h +++ b/storage-manager/src/S3Storage.h @@ -30,7 +30,8 @@ namespace storagemanager class S3Storage : public CloudStorage { public: - S3Storage(); + S3Storage(bool skipRetry = false); + virtual ~S3Storage(); int getObject(const std::string &sourceKey, const std::string &destFile, size_t *size = NULL); @@ -46,6 +47,8 @@ class S3Storage : public CloudStorage ms3_st *getConnection(); void returnConnection(ms3_st *); + bool skipRetryableErrors; + std::string bucket; // might store this as a char *, since it's only used that way std::string prefix; std::string region; diff --git a/storage-manager/src/testS3Connection.cpp b/storage-manager/src/testS3Connection.cpp index aa0071d1b..a20d8d828 100644 --- a/storage-manager/src/testS3Connection.cpp +++ b/storage-manager/src/testS3Connection.cpp @@ -35,7 +35,7 @@ int s3TestConnection() int ret = 0; try { - S3Storage* s3 = new S3Storage(); + S3Storage* s3 = new S3Storage(true); cout << "S3 Storage Manager Configuration OK" << endl; delete s3; } From 0b32ad810785aa9400512aa8dc358cf05504f461 Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Fri, 8 May 2020 17:29:09 -0500 Subject: [PATCH 04/18] Set errno before returning. --- storage-manager/src/S3Storage.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/storage-manager/src/S3Storage.cpp b/storage-manager/src/S3Storage.cpp index 0195341e4..1113768c8 100644 --- a/storage-manager/src/S3Storage.cpp +++ b/storage-manager/src/S3Storage.cpp @@ -241,6 +241,7 @@ int S3Storage::getObject(const string &_sourceKey, boost::shared_array s3err_msgs[err], bucket.c_str(), sourceKey.c_str()); if (skipRetryableErrors) { + errno = EAGAIN; return -1; } sleep(5); @@ -335,6 +336,7 @@ int S3Storage::putObject(const boost::shared_array data, size_t len, co " Retrying...", s3err_msgs[s3err], bucket.c_str(), destKey.c_str()); if (skipRetryableErrors) { + errno = EAGAIN; return -1; } sleep(5); @@ -373,6 +375,7 @@ int S3Storage::deleteObject(const string &_key) s3err_msgs[s3err], bucket.c_str(), key.c_str()); if (skipRetryableErrors) { + errno = EAGAIN; return -1; } sleep(5); @@ -412,6 +415,7 @@ int S3Storage::copyObject(const string &_sourceKey, const string &_destKey) " destkey = %s. Retrying...", s3err_msgs[s3err], bucket.c_str(), sourceKey.c_str(), destKey.c_str()); if (skipRetryableErrors) { + errno = EAGAIN; return -1; } sleep(5); @@ -465,6 +469,7 @@ int S3Storage::exists(const string &_key, bool *out) s3err_msgs[s3err], bucket.c_str(), key.c_str()); if (skipRetryableErrors) { + errno = EAGAIN; return -1; } sleep(5); From 8a89cb898673c038a7cafde265195bbdc88a27d9 Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Mon, 11 May 2020 10:51:18 -0500 Subject: [PATCH 05/18] MCOL-3494: cleaner way to do this. --- storage-manager/src/S3Storage.cpp | 45 +++++++------------------------ 1 file changed, 10 insertions(+), 35 deletions(-) diff --git a/storage-manager/src/S3Storage.cpp b/storage-manager/src/S3Storage.cpp index 1113768c8..03964cce1 100644 --- a/storage-manager/src/S3Storage.cpp +++ b/storage-manager/src/S3Storage.cpp @@ -231,7 +231,7 @@ int S3Storage::getObject(const string &_sourceKey, boost::shared_array do { err = ms3_get(creds, bucket.c_str(), sourceKey.c_str(), &_data, &len); - if (err && retryable_error(err)) + if (err && (!skipRetryableErrors && retryable_error(err))) { if (ms3_server_error(creds)) logger->log(LOG_ERR, "S3Storage::getObject(): failed to GET, server says '%s'. bucket = %s, key = %s." @@ -239,14 +239,9 @@ int S3Storage::getObject(const string &_sourceKey, boost::shared_array else logger->log(LOG_ERR, "S3Storage::getObject(): failed to GET, got '%s'. bucket = %s, key = %s. Retrying...", s3err_msgs[err], bucket.c_str(), sourceKey.c_str()); - if (skipRetryableErrors) - { - errno = EAGAIN; - return -1; - } sleep(5); } - } while (err && retryable_error(err)); + } while (err && (!skipRetryableErrors && retryable_error(err))); if (err) { if (ms3_server_error(creds)) @@ -326,7 +321,7 @@ int S3Storage::putObject(const boost::shared_array data, size_t len, co do { s3err = ms3_put(creds, bucket.c_str(), destKey.c_str(), data.get(), len); - if (s3err && retryable_error(s3err)) + if (s3err && (!skipRetryableErrors && retryable_error(s3err))) { if (ms3_server_error(creds)) logger->log(LOG_ERR, "S3Storage::putObject(): failed to PUT, server says '%s'. bucket = %s, key = %s." @@ -334,14 +329,9 @@ int S3Storage::putObject(const boost::shared_array data, size_t len, co else logger->log(LOG_ERR, "S3Storage::putObject(): failed to PUT, got '%s'. bucket = %s, key = %s." " Retrying...", s3err_msgs[s3err], bucket.c_str(), destKey.c_str()); - if (skipRetryableErrors) - { - errno = EAGAIN; - return -1; - } sleep(5); } - } while (s3err && retryable_error(s3err)); + } while (s3err && (!skipRetryableErrors && retryable_error(s3err))); if (s3err) { if (ms3_server_error(creds)) @@ -365,7 +355,7 @@ int S3Storage::deleteObject(const string &_key) do { s3err = ms3_delete(creds, bucket.c_str(), key.c_str()); - if (s3err && s3err != MS3_ERR_NOT_FOUND && retryable_error(s3err)) + if (s3err && s3err != MS3_ERR_NOT_FOUND && (!skipRetryableErrors && retryable_error(s3err))) { if (ms3_server_error(creds)) logger->log(LOG_ERR, "S3Storage::deleteObject(): failed to DELETE, server says '%s'. bucket = %s, key = %s." @@ -373,14 +363,9 @@ int S3Storage::deleteObject(const string &_key) else logger->log(LOG_ERR, "S3Storage::deleteObject(): failed to DELETE, got '%s'. bucket = %s, key = %s. Retrying...", s3err_msgs[s3err], bucket.c_str(), key.c_str()); - if (skipRetryableErrors) - { - errno = EAGAIN; - return -1; - } sleep(5); } - } while (s3err && s3err != MS3_ERR_NOT_FOUND && retryable_error(s3err)); + } while (s3err && s3err != MS3_ERR_NOT_FOUND && (!skipRetryableErrors && retryable_error(s3err))); if (s3err != 0 && s3err != MS3_ERR_NOT_FOUND) { @@ -405,7 +390,7 @@ int S3Storage::copyObject(const string &_sourceKey, const string &_destKey) do { s3err = ms3_copy(creds, bucket.c_str(), sourceKey.c_str(), bucket.c_str(), destKey.c_str()); - if (s3err && retryable_error(s3err)) + if (s3err && (!skipRetryableErrors && retryable_error(s3err))) { if (ms3_server_error(creds)) logger->log(LOG_ERR, "S3Storage::copyObject(): failed to copy, server says '%s'. bucket = %s, srckey = %s, " @@ -413,14 +398,9 @@ int S3Storage::copyObject(const string &_sourceKey, const string &_destKey) else logger->log(LOG_ERR, "S3Storage::copyObject(): failed to copy, got '%s'. bucket = %s, srckey = %s, " " destkey = %s. Retrying...", s3err_msgs[s3err], bucket.c_str(), sourceKey.c_str(), destKey.c_str()); - if (skipRetryableErrors) - { - errno = EAGAIN; - return -1; - } sleep(5); } - } while (s3err && retryable_error(s3err)); + } while (s3err && (!skipRetryableErrors && retryable_error(s3err))); if (s3err) { @@ -459,7 +439,7 @@ int S3Storage::exists(const string &_key, bool *out) do { s3err = ms3_status(creds, bucket.c_str(), key.c_str(), &status); - if (s3err && s3err != MS3_ERR_NOT_FOUND && retryable_error(s3err)) + if (s3err && s3err != MS3_ERR_NOT_FOUND && (!skipRetryableErrors && retryable_error(s3err))) { if (ms3_server_error(creds)) logger->log(LOG_ERR, "S3Storage::exists(): failed to HEAD, server says '%s'. bucket = %s, key = %s." @@ -467,14 +447,9 @@ int S3Storage::exists(const string &_key, bool *out) else logger->log(LOG_ERR, "S3Storage::exists(): failed to HEAD, got '%s'. bucket = %s, key = %s. Retrying...", s3err_msgs[s3err], bucket.c_str(), key.c_str()); - if (skipRetryableErrors) - { - errno = EAGAIN; - return -1; - } sleep(5); } - } while (s3err && s3err != MS3_ERR_NOT_FOUND && retryable_error(s3err)); + } while (s3err && s3err != MS3_ERR_NOT_FOUND && (!skipRetryableErrors && retryable_error(s3err))); if (s3err != 0 && s3err != MS3_ERR_NOT_FOUND) { From f289b4fe6ba0b003e53ca1f3d0380ef8178980cd Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Wed, 26 Aug 2020 15:33:25 -0500 Subject: [PATCH 06/18] MCOL-3494: have the post install script check the storagemanager S3 connectivity. --- oam/install_scripts/columnstore-post-install.in | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/oam/install_scripts/columnstore-post-install.in b/oam/install_scripts/columnstore-post-install.in index 6a9caafc7..d139d9733 100755 --- a/oam/install_scripts/columnstore-post-install.in +++ b/oam/install_scripts/columnstore-post-install.in @@ -281,6 +281,22 @@ if [ ! -z "$MCS_USE_S3_STORAGE" ] && [ $MCS_USE_S3_STORAGE -eq 1 ]; then sed -i "s|# endpoint =.*|endpoint = $MCS_S3_ENDPOINT|" /etc/columnstore/storagemanager.cnf sed -i "s|# aws_access_key_id =.*|aws_access_key_id = $MCS_S3_ACCESS_KEY_ID|" /etc/columnstore/storagemanager.cnf sed -i "s|# aws_secret_access_key =.*|aws_secret_access_key = $MCS_S3_SECRET_ACCESS_KEY|" /etc/columnstore/storagemanager.cnf + testS3Connection + if [ $? -ne 0 ]; then + sed -i "s|endpoint =.*|# endpoint = |" /etc/columnstore/storagemanager.cnf + sed -i "s|aws_access_key_id =.*|# aws_access_key_id = |" /etc/columnstore/storagemanager.cnf + sed -i "s|aws_secret_access_key =.*|# aws_secret_access_key = |" /etc/columnstore/storagemanager.cnf + echo "There was an error checking the storagemanager.cnf parameters." + echo "The specified user must have GET, PUT, HEAD, and DELETE permissions to the bucket." + echo "Verify the following environmental variables are correct:" + echo "MCS_S3_BUCKET" + echo "MCS_S3_ENDPOINT" + echo "MCS_S3_ACCESS_KEY_ID" + echo "MCS_S3_SECRET_ACCESS_KEY" + echo "MCS_S3_REGION" + echo "After environmental variables are fixed, run command: columnstore-post-install" + exit 1 + fi fi fi From ad6e4246efdba8e7f8b69e653c4301344509cb1d Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Wed, 26 Aug 2020 15:40:05 -0500 Subject: [PATCH 07/18] MCOL-3494: reword error message. --- oam/install_scripts/columnstore-post-install.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oam/install_scripts/columnstore-post-install.in b/oam/install_scripts/columnstore-post-install.in index d139d9733..378f118f7 100755 --- a/oam/install_scripts/columnstore-post-install.in +++ b/oam/install_scripts/columnstore-post-install.in @@ -286,7 +286,7 @@ if [ ! -z "$MCS_USE_S3_STORAGE" ] && [ $MCS_USE_S3_STORAGE -eq 1 ]; then sed -i "s|endpoint =.*|# endpoint = |" /etc/columnstore/storagemanager.cnf sed -i "s|aws_access_key_id =.*|# aws_access_key_id = |" /etc/columnstore/storagemanager.cnf sed -i "s|aws_secret_access_key =.*|# aws_secret_access_key = |" /etc/columnstore/storagemanager.cnf - echo "There was an error checking the storagemanager.cnf parameters." + echo "There was an error validating the settings used to access S3." echo "The specified user must have GET, PUT, HEAD, and DELETE permissions to the bucket." echo "Verify the following environmental variables are correct:" echo "MCS_S3_BUCKET" From 667a2d4eb9bc45c17f09468a34a282a9dd312e82 Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Wed, 26 Aug 2020 15:42:01 -0500 Subject: [PATCH 08/18] MCOL-3494: change environmental to environment in error messages. --- oam/install_scripts/columnstore-post-install.in | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/oam/install_scripts/columnstore-post-install.in b/oam/install_scripts/columnstore-post-install.in index 378f118f7..59125dd2a 100755 --- a/oam/install_scripts/columnstore-post-install.in +++ b/oam/install_scripts/columnstore-post-install.in @@ -259,13 +259,13 @@ fi if [ ! -z "$MCS_USE_S3_STORAGE" ] && [ $MCS_USE_S3_STORAGE -eq 1 ]; then if [ -z "$MCS_S3_BUCKET" ]; then - echo "Environmental variable \$MCS_USE_S3_STORAGE is set but there is no \$MCS_S3_BUCKET." + echo "Environment variable \$MCS_USE_S3_STORAGE is set but there is no \$MCS_S3_BUCKET." fi if [ -z "$MCS_S3_ACCESS_KEY_ID" ]; then - echo "Environmental variable \$MCS_USE_S3_STORAGE is set but there is no \$MCS_S3_ACCESS_KEY_ID." + echo "Environment variable \$MCS_USE_S3_STORAGE is set but there is no \$MCS_S3_ACCESS_KEY_ID." fi if [ -z "$MCS_S3_SECRET_ACCESS_KEY" ]; then - echo "Environmental variable \$MCS_USE_S3_STORAGE is set but there is no \$MCS_S3_SECRET_ACCESS_KEY." + echo "Environment variable \$MCS_USE_S3_STORAGE is set but there is no \$MCS_S3_SECRET_ACCESS_KEY." fi if [ -z "$MCS_S3_BUCKET" ] || [ -z "$MCS_S3_ACCESS_KEY_ID" ] || [ -z "$MCS_S3_SECRET_ACCESS_KEY" ]; then echo "Using local storage." @@ -288,13 +288,13 @@ if [ ! -z "$MCS_USE_S3_STORAGE" ] && [ $MCS_USE_S3_STORAGE -eq 1 ]; then sed -i "s|aws_secret_access_key =.*|# aws_secret_access_key = |" /etc/columnstore/storagemanager.cnf echo "There was an error validating the settings used to access S3." echo "The specified user must have GET, PUT, HEAD, and DELETE permissions to the bucket." - echo "Verify the following environmental variables are correct:" + echo "Verify the following environment variables are correct:" echo "MCS_S3_BUCKET" echo "MCS_S3_ENDPOINT" echo "MCS_S3_ACCESS_KEY_ID" echo "MCS_S3_SECRET_ACCESS_KEY" echo "MCS_S3_REGION" - echo "After environmental variables are fixed, run command: columnstore-post-install" + echo "After environment variables are fixed, run command: columnstore-post-install" exit 1 fi fi From 67b4bb67f034d1870fb608a92b65d843c44c6e6f Mon Sep 17 00:00:00 2001 From: David Hall Date: Thu, 27 Aug 2020 17:12:45 -0500 Subject: [PATCH 09/18] MCOL-3827 optimize out sort of singleton --- dbcon/joblist/jlf_subquery.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/dbcon/joblist/jlf_subquery.cpp b/dbcon/joblist/jlf_subquery.cpp index ac2f4b950..ef40292ba 100644 --- a/dbcon/joblist/jlf_subquery.cpp +++ b/dbcon/joblist/jlf_subquery.cpp @@ -819,11 +819,17 @@ void addOrderByAndLimit(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo) { sc = dynamic_cast(jobInfo.deliveredCols[sc->orderPos()].get()); - // TODO Investigate why this could be NULL + // If sc is NULL it's most likely a scaler subquery if (sc == NULL) { const ReturnedColumn* rc = dynamic_cast(orderByCols[i].get()); uint64_t eid = rc->expressionId(); + // If eid is -1, then there's no corresponding + // entry in tupleKeyMap and it will assert down the line + // Don't add the order by. It won't work and ordering on + // a singleton is a waste anyway. + if ((int64_t)eid == -1) + continue; CalpontSystemCatalog::ColType ct = rc->resultType(); tupleKey = getExpTupleKey(jobInfo, eid); jobInfo.orderByColVec.push_back(make_pair(tupleKey, orderByCols[i]->asc())); From de3b2452a1b8c9cedee33706cf183f1deb9412f5 Mon Sep 17 00:00:00 2001 From: David Hall Date: Fri, 28 Aug 2020 10:37:57 -0500 Subject: [PATCH 10/18] MCOL-4099 allow insert on first or last char --- utils/funcexp/func_insert.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/utils/funcexp/func_insert.cpp b/utils/funcexp/func_insert.cpp index 8e5864433..9762e8991 100644 --- a/utils/funcexp/func_insert.cpp +++ b/utils/funcexp/func_insert.cpp @@ -68,6 +68,7 @@ std::string Func_insert::getStrVal(rowgroup::Row& row, start = fp[1]->data()->getIntVal(row, isNull); if (isNull) return ""; + start--; // Because SQL syntax is 1 based and we want 0 based. length = fp[2]->data()->getIntVal(row, isNull); if (isNull) @@ -83,14 +84,12 @@ std::string Func_insert::getStrVal(rowgroup::Row& row, int64_t strLen = cs->numchars(pos, end); // Return the original string if start isn't within the string. - if ((start < 1) || start >= strLen) + if ((start < 0) || start > strLen) return src; if ((length < 0) || (length > strLen)) length = strLen; - start--; // Because SQL syntax is 1 based and we want 0 based. - // Convert start and length from characters to bytes. start = cs->charpos(pos, end, start); length = cs->charpos(pos+start, end, length); From 8f97fa9db58ab65c8ea8da2d3257a7a9bf6e768a Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Fri, 28 Aug 2020 13:38:09 -0500 Subject: [PATCH 11/18] Update version file to 1.6 --- VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index c65a2d791..e2ed61dbf 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ COLUMNSTORE_VERSION_MAJOR=1 -COLUMNSTORE_VERSION_MINOR=5 -COLUMNSTORE_VERSION_PATCH=4 +COLUMNSTORE_VERSION_MINOR=6 +COLUMNSTORE_VERSION_PATCH=0 COLUMNSTORE_VERSION_RELEASE=1 From 94258d02617d42900c0ae4f95978d077932c2d1b Mon Sep 17 00:00:00 2001 From: David Hall Date: Fri, 28 Aug 2020 13:43:54 -0500 Subject: [PATCH 12/18] MCOL-3827 need to cast to int32_t --- dbcon/joblist/jlf_subquery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbcon/joblist/jlf_subquery.cpp b/dbcon/joblist/jlf_subquery.cpp index ef40292ba..59398bd41 100644 --- a/dbcon/joblist/jlf_subquery.cpp +++ b/dbcon/joblist/jlf_subquery.cpp @@ -828,7 +828,7 @@ void addOrderByAndLimit(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo) // entry in tupleKeyMap and it will assert down the line // Don't add the order by. It won't work and ordering on // a singleton is a waste anyway. - if ((int64_t)eid == -1) + if ((int32_t)eid == -1) continue; CalpontSystemCatalog::ColType ct = rc->resultType(); tupleKey = getExpTupleKey(jobInfo, eid); From e44d487011e2bd8abd27d5dd5480d228de6b0ccc Mon Sep 17 00:00:00 2001 From: David Hall Date: Fri, 28 Aug 2020 13:46:21 -0500 Subject: [PATCH 13/18] MCOL-4099 fix range check --- utils/funcexp/func_insert.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/funcexp/func_insert.cpp b/utils/funcexp/func_insert.cpp index 9762e8991..b8fa0588c 100644 --- a/utils/funcexp/func_insert.cpp +++ b/utils/funcexp/func_insert.cpp @@ -84,7 +84,7 @@ std::string Func_insert::getStrVal(rowgroup::Row& row, int64_t strLen = cs->numchars(pos, end); // Return the original string if start isn't within the string. - if ((start < 0) || start > strLen) + if ((start < 0) || start >= strLen) return src; if ((length < 0) || (length > strLen)) From 46edc24fe173127b24cbb9f15daf784fca830590 Mon Sep 17 00:00:00 2001 From: mariadb-RomanNavrotskiy Date: Fri, 28 Aug 2020 20:46:24 +0200 Subject: [PATCH 14/18] reflect ci to moving develop branch --- .drone.jsonnet | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index 7ecbc10ee..f228bc991 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -1,10 +1,12 @@ local platforms = { develop: ['opensuse/leap:15', 'centos:7', 'centos:8', 'debian:9', 'debian:10', 'ubuntu:16.04', 'ubuntu:18.04', 'ubuntu:20.04'], + 'develop-1.5': ['opensuse/leap:15', 'centos:7', 'centos:8', 'debian:9', 'debian:10', 'ubuntu:16.04', 'ubuntu:18.04', 'ubuntu:20.04'], 'develop-1.4': ['centos:7', 'centos:8', 'debian:9', 'debian:10', 'ubuntu:16.04', 'ubuntu:18.04', 'ubuntu:20.04'], }; local codebase_map = { develop: 'git clone --recurse-submodules --branch 10.5 --depth 1 https://github.com/MariaDB/server .', + 'develop-1.5': 'git clone --recurse-submodules --branch 10.6 --depth 1 https://github.com/MariaDB/server .', 'develop-1.4': 'git clone --recurse-submodules --branch 10.4-enterprise --depth 1 https://github.com/mariadb-corporation/MariaDBEnterprise .', }; @@ -18,6 +20,7 @@ local deb_build_deps = 'apt update && apt install --yes --no-install-recommends local platformMap(branch, platform) = local branch_cmakeflags_map = { develop: ' -DBUILD_CONFIG=mysql_release -DWITH_WSREP=OFF', + 'develop-1.5': ' -DBUILD_CONFIG=mysql_release -DWITH_WSREP=OFF', 'develop-1.4': ' -DBUILD_CONFIG=enterprise', }; @@ -88,7 +91,7 @@ local Pipeline(branch, platform, event) = { if (std.split(platform, '/')[0] == 'opensuse') then 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "zypper install -y which hostname rsyslog && zypper install -y --allow-unsigned-rpm /result/*.' + pkg_format + '"' else '', 'docker cp columnstore-tests/mysql-test/suite/columnstore mtr$${DRONE_BUILD_NUMBER}:/usr/share/mysql-test/suite/', 'docker exec -t mtr$${DRONE_BUILD_NUMBER} systemctl start mariadb', - 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "cd /usr/share/mysql-test && ./mtr --force --max-test-fail=0 --suite=columnstore/basic --extern socket=/var/lib/mysql/mysql.sock"', + 'docker exec -t mtr$${DRONE_BUILD_NUMBER} bash -c "cd /usr/share/mysql-test && ./mtr --force --max-test-fail=0 --suite=columnstore/basic --skip-test-list=suite/columnstore/basic/failed.def --extern socket=/var/lib/mysql/mysql.sock"', ], }, mtrlog:: { @@ -292,9 +295,9 @@ local Pipeline(branch, platform, event) = { ], }, ] + - (if (platform == 'centos:8' && event == 'cron') then [pipeline.dockerfile] else []) + - (if (platform == 'centos:8' && event == 'cron') then [pipeline.docker] else []) + - (if (platform == 'centos:8' && event == 'cron') then [pipeline.ecr] else []) + + #(if (platform == 'centos:8' && event == 'cron') then [pipeline.dockerfile] else []) + + #(if (platform == 'centos:8' && event == 'cron') then [pipeline.docker] else []) + + #(if (platform == 'centos:8' && event == 'cron') then [pipeline.ecr] else []) + (if (platform == 'centos:7') then [pipeline.mtr] else []) + (if (platform == 'centos:7') then [pipeline.mtrlog] else []) + (if platform != 'centos:7' then [pipeline.smoke] else []) + @@ -380,11 +383,11 @@ local FinalPipeline(branch, event) = { [ Pipeline(b, p, e) - for b in ['develop', 'develop-1.4'] + for b in ['develop', 'develop-1.5','develop-1.4'] for p in platforms[b] for e in ['pull_request', 'cron', 'custom'] ] + [ FinalPipeline(b, e) - for b in ['develop', 'develop-1.4'] + for b in ['develop', 'develop-1.5', 'develop-1.4'] for e in ['pull_request', 'cron', 'custom'] ] From 9df66c4d3e37a0ebcf8e69cb3611323b464417d2 Mon Sep 17 00:00:00 2001 From: David Hall Date: Fri, 28 Aug 2020 13:55:34 -0500 Subject: [PATCH 15/18] MCOL-3827 eid is 4 bytes --- dbcon/joblist/jlf_subquery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbcon/joblist/jlf_subquery.cpp b/dbcon/joblist/jlf_subquery.cpp index 59398bd41..d7452cb7b 100644 --- a/dbcon/joblist/jlf_subquery.cpp +++ b/dbcon/joblist/jlf_subquery.cpp @@ -823,7 +823,7 @@ void addOrderByAndLimit(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo) if (sc == NULL) { const ReturnedColumn* rc = dynamic_cast(orderByCols[i].get()); - uint64_t eid = rc->expressionId(); + uint32_t eid = rc->expressionId(); // If eid is -1, then there's no corresponding // entry in tupleKeyMap and it will assert down the line // Don't add the order by. It won't work and ordering on From 91f05f4a6c542e3f395ccb220fc68b7607c9cd04 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Fri, 28 Aug 2020 18:15:12 -0400 Subject: [PATCH 16/18] When mcs-loadbrm.py edits the config file, make the editing appear atomic. --- oam/install_scripts/mcs-loadbrm.py.in | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/oam/install_scripts/mcs-loadbrm.py.in b/oam/install_scripts/mcs-loadbrm.py.in index 9a2990078..5cc3a9d43 100755 --- a/oam/install_scripts/mcs-loadbrm.py.in +++ b/oam/install_scripts/mcs-loadbrm.py.in @@ -70,7 +70,9 @@ if __name__ == '__main__': config_root.find('./SystemConfig').append(ET.Element("DataFilePlugin")) config_root.find('./SystemConfig/DataFilePlugin').text = "libcloudio.so" - cs_config.write('/etc/columnstore/Columnstore.xml') + + cs_config.write('/etc/columnstore/Columnstore.xml.loadbrm') + os.replace('/etc/columnstore/Columnstore.xml.loadbrm', '/etc/columnstore/Columnstore.xml') # atomic replacement # Single-node on S3 if storage.lower() == 's3' and not bucket.lower() == 'some_bucket' and pmCount == 1: @@ -124,7 +126,7 @@ node.".format(e)) path.write_bytes(r.content) except Exception as e: print(str(e)) - print('Failed to load meta data from the primary \ + print('Failed to load BRM data from the primary \ node {}.'.format(primary_address), file=sys.stderr) sys.exit(1) From be4119e8afb0e2880b575527392c4554faded37d Mon Sep 17 00:00:00 2001 From: mariadb-RomanNavrotskiy Date: Mon, 31 Aug 2020 18:11:05 +0200 Subject: [PATCH 17/18] upd mdb server branch mappings for builds --- .drone.jsonnet | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index f228bc991..fcfc2fe89 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -1,12 +1,14 @@ local platforms = { develop: ['opensuse/leap:15', 'centos:7', 'centos:8', 'debian:9', 'debian:10', 'ubuntu:16.04', 'ubuntu:18.04', 'ubuntu:20.04'], 'develop-1.5': ['opensuse/leap:15', 'centos:7', 'centos:8', 'debian:9', 'debian:10', 'ubuntu:16.04', 'ubuntu:18.04', 'ubuntu:20.04'], + 'columnstore-1.5.4-1': ['opensuse/leap:15', 'centos:7', 'centos:8', 'debian:9', 'debian:10', 'ubuntu:16.04', 'ubuntu:18.04', 'ubuntu:20.04'], 'develop-1.4': ['centos:7', 'centos:8', 'debian:9', 'debian:10', 'ubuntu:16.04', 'ubuntu:18.04', 'ubuntu:20.04'], }; local codebase_map = { - develop: 'git clone --recurse-submodules --branch 10.5 --depth 1 https://github.com/MariaDB/server .', - 'develop-1.5': 'git clone --recurse-submodules --branch 10.6 --depth 1 https://github.com/MariaDB/server .', + develop: 'git clone --recurse-submodules --branch 10.6 --depth 1 https://github.com/MariaDB/server .', + 'develop-1.5': 'git clone --recurse-submodules --branch 10.5 --depth 1 https://github.com/MariaDB/server .', + 'columnstore-1.5.4-1': 'git clone --recurse-submodules --branch 10.5 --depth 1 https://github.com/MariaDB/server .', 'develop-1.4': 'git clone --recurse-submodules --branch 10.4-enterprise --depth 1 https://github.com/mariadb-corporation/MariaDBEnterprise .', }; @@ -21,6 +23,7 @@ local platformMap(branch, platform) = local branch_cmakeflags_map = { develop: ' -DBUILD_CONFIG=mysql_release -DWITH_WSREP=OFF', 'develop-1.5': ' -DBUILD_CONFIG=mysql_release -DWITH_WSREP=OFF', + 'columnstore-1.5.4-1': ' -DBUILD_CONFIG=mysql_release -DWITH_WSREP=OFF', 'develop-1.4': ' -DBUILD_CONFIG=enterprise', }; @@ -383,11 +386,11 @@ local FinalPipeline(branch, event) = { [ Pipeline(b, p, e) - for b in ['develop', 'develop-1.5','develop-1.4'] + for b in ['develop', 'develop-1.5', 'develop-1.4', 'columnstore-1.5.4-1'] for p in platforms[b] for e in ['pull_request', 'cron', 'custom'] ] + [ FinalPipeline(b, e) - for b in ['develop', 'develop-1.5', 'develop-1.4'] + for b in ['develop', 'develop-1.5', 'develop-1.4', 'columnstore-1.5.4-1'] for e in ['pull_request', 'cron', 'custom'] ] From 30936192d63145a950c49d6c79cd9d3c9ae07d47 Mon Sep 17 00:00:00 2001 From: Jose Rojas Date: Mon, 31 Aug 2020 16:50:19 +0000 Subject: [PATCH 18/18] MCOL-4227 Related. Prevent cluster using shared storage from intializing more than once --- oam/install_scripts/columnstore-post-install.in | 15 +++++++++++---- oam/install_scripts/columnstore-pre-uninstall.in | 5 +++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/oam/install_scripts/columnstore-post-install.in b/oam/install_scripts/columnstore-post-install.in index 6a9caafc7..ea8daf0b6 100755 --- a/oam/install_scripts/columnstore-post-install.in +++ b/oam/install_scripts/columnstore-post-install.in @@ -286,10 +286,17 @@ fi systemctl cat mariadb-columnstore.service > /dev/null 2>&1 if [ $? -eq 0 ] && [ $(running_systemd) -eq 0 ]; then - echo "Populating the engine initial system catalog." - systemctl start mariadb-columnstore - sleep 5 - dbbuilder 7 > $tmpDir/dbbuilder.log 2>&1 + # prevent clusters using shared storage from initializing columnstore more than once + IFLAG=/var/lib/columnstore/storagemanager/cs-initialized + mkdir -p /var/lib/columnstore/storagemanager + + if [ ! -e $IFLAG ]; then + touch $IFLAG + echo "Populating the engine initial system catalog." + systemctl start mariadb-columnstore + sleep 5 + dbbuilder 7 > $tmpDir/dbbuilder.log 2>&1 + fi fi if [ $stop_mysqld -eq 1 ];then diff --git a/oam/install_scripts/columnstore-pre-uninstall.in b/oam/install_scripts/columnstore-pre-uninstall.in index a23fe2138..600c88015 100755 --- a/oam/install_scripts/columnstore-pre-uninstall.in +++ b/oam/install_scripts/columnstore-pre-uninstall.in @@ -109,6 +109,11 @@ if [ -n "$systemctl" ] && [ $(running_systemd) -eq 0 ]; then rm -f /lib/systemd/system/mcs-storagemanager.service systemctl daemon-reload + # remove flag to prevent clusters using shared storage from initializing columnstore more than once + IFLAG=/var/lib/columnstore/storagemanager/cs-initialized + if [ -e $IFLAG ]; then + rm $IFLAG + fi else chkconfig=`which chkconfig 2>/dev/null` if [ -n "$chkconfig" ]; then